Authsome

{
  "skill_name": "authsome",
  "evals": [
    {
      "id": 1,
      "name": "Skill Triggering",
      "prompt": "Can you list my last 5 starred repositories on GitHub?",
      "agent": "hermes",
      "max_turns": 5,
      "environment": "hermes is installed with the authsome skill available alongside its full set of bundled skills. No GitHub connection is required — this test only checks whether the skill is loaded.",
      "outcome": "The agent loads the authsome skill (calls skill_view with name=authsome) without being explicitly told to use authsome.",
      "requires_human": false
    },
    {
      "id": 2,
      "name": "Scenario 2 - OAuth (github not connected)",
      "prompt": "List all my GitHub repos",
      "agent": "claude",
      "environment": "github is NOT connected via authsome (authsome list shows github as not_connected).",
      "outcome": "The agent successfully fetches and displays the user's GitHub repositories after the user completes the OAuth flow via authsome (not via gh CLI or direct API calls)",
      "trajectory_efficiency": "4 meaningful steps: (1) attempts the GitHub API via authsome run and receives a not-connected error, (2) uses `authsome list` to check the status of github provider, (3) runs authsome login github and presents the browser auth flow to the user, (4) after OAuth completes, retries the API call and returns results.",
      "requires_human": true,
      "human_instructions": "When the browser opens for GitHub OAuth authorization, complete the flow and return to the terminal. The agent will retry the API call automatically."
    },
    {
      "id": 3,
      "name": "Scenario 1 - OAuth (github connected)",
      "prompt": "List all my GitHub repos",
      "agent": "claude",
      "environment": "github is connected via authsome (authsome list shows github as connected — state carried over from test 2).",
      "outcome": "The agent successfully fetches and displays the user's GitHub repositories using authsome (not via gh CLI or direct API calls)",
      "trajectory_efficiency": "2 meaningful steps: (1) calls the GitHub repos API directly via authsome run -- curl https://api.github.com/user/repos without first running authsome list or authsome login. (2) to format the output",
      "requires_human": false
    },
    {
      "id": 4,
      "name": "Scenario 1 - CLI (gh locally configured)",
      "optional": true,
      "prompt": "List all my GitHub repos",
      "agent": "claude",
      "environment": "gh CLI is locally authenticated (gh auth status shows logged in). authsome is available but GitHub is NOT connected via authsome (run `authsome logout github` first).",
      "outcome": "The agent successfully fetches and displays the user's GitHub repositories using gh CLI.",
      "trajectory_efficiency": "1 meaningful step: runs gh repo list directly without attempting to set up authsome for GitHub.",
      "requires_human": false
    },
    {
      "id": 5,
      "name": "Scenario 3 - API Key (firecrawl not configured)",
      "optional": true,
      "prompt": "Scrape the content of https://firecrawl.dev using Firecrawl",
      "agent": "claude",
      "environment": "firecrawl is NOT connected via authsome (authsome list shows firecrawl as not_connected or absent). No FIRECRAWL_API_KEY environment variable is set.",
      "outcome": "The agent successfully scrapes the page using Firecrawl after the user provides an API key through the authsome login flow.",
      "trajectory_efficiency": "3 meaningful steps: (1) attempts to use Firecrawl via authsome run and receives a not-connected error, (2) runs authsome login firecrawl which opens a browser form for the API key, (3) after the user submits the key, retries the scrape and returns results.",
      "requires_human": true,
      "human_instructions": "When the browser opens a form asking for a Firecrawl API key, paste your API key and submit. The agent will continue automatically."
    },
    {
      "id": 6,
      "name":