Agent Eval

{
  "_comment": "Test corpus for /agent-eval. Add entries freely. size: Small (<~150 files), Medium (~150-1500), Large (>~1500). 'question' is a representative architectural question that exercises cross-file understanding.",
  "TypeScript": [
    {
      "name": "ky",
      "repo": "https://github.com/sindresorhus/ky",
      "size": "Small",
      "files": "~25",
      "question": "How does ky implement request retries and timeouts?"
    },
    {
      "name": "excalidraw",
      "repo": "https://github.com/excalidraw/excalidraw",
      "size": "Medium",
      "files": "~600",
      "question": "How does Excalidraw render and update canvas elements?"
    },
    {
      "name": "vscode",
      "repo": "https://github.com/microsoft/vscode",
      "size": "Large",
      "files": "~10000",
      "question": "How does the extension host communicate with the main process?"
    }
  ],
  "JavaScript": [
    {
      "name": "express",
      "repo": "https://github.com/expressjs/express",
      "size": "Small",
      "files": "~50",
      "question": "How does Express route a request through its middleware stack?"
    }
  ],
  "Go": [
    {
      "name": "cobra",
      "repo": "https://github.com/spf13/cobra",
      "size": "Small",
      "files": "~50",
      "question": "How does cobra parse commands and flags?"
    },
    {
      "name": "gin",
      "repo": "https://github.com/gin-gonic/gin",
      "size": "Medium",
      "files": "~150",
      "question": "How does gin route requests through its middleware chain?"
    },
    {
      "name": "terraform",
      "repo": "https://github.com/hashicorp/terraform",
      "size": "Large",
      "files": "~4000",
      "question": "How does Terraform build and walk the resource dependency graph?"
    },
    {
      "name": "cosmos-sdk",
      "repo": "https://github.com/cosmos/cosmos-sdk",
      "size": "Large",
      "files": "~5000",
      "question": "How does a bank module MsgSend message reach the account balance update? Trace the cross-module call path from the bank keeper's Send handler through to the account/balance store update."
    }
  ],
  "Python": [
    {
      "name": "click",
      "repo": "https://github.com/pallets/click",
      "size": "Small",
      "files": "~60",
      "question": "How does click parse command-line arguments into commands?"
    },
    {
      "name": "flask",
      "repo": "https://github.com/pallets/flask",
      "size": "Medium",
      "files": "~90",
      "question": "How does Flask dispatch a request to a view function?"
    },
    {
      "name": "django",
      "repo": "https://github.com/django/django",
      "size": "Large",
      "files": "~2700",
      "question": "How does Django's ORM build and execute a query from a QuerySet?"
    }
  ],
  "Rust": [
    {
      "name": "clap",
      "repo": "https://github.com/clap-rs/clap",
      "size": "Medium",
      "files": "~200",
      "question": "How does clap parse arguments against a derived command definition?"
    },
    {
      "name": "tokio",
      "repo": "https://github.com/tokio-rs/tokio",
      "size": "Large",
      "files": "~700",
      "question": "How does tokio schedule and run async tasks on its runtime?"
    },
    {
      "name": "deno",
      "repo": "https://github.com/denoland/deno",
      "size": "Large",
      "files": "~1500",
      "question": "How does Deno load and execute a TypeScript module?"
    }
  ],
  "Java": [
    {
      "name": "gson",
      "repo": "https://github.com/google/gson",
      "size": "Medium",
      "files": "~200",
      "question": "How does Gson serialize an object to JSON?"
    },
    {
      "name": "okhttp",
      "repo": "https://github.com/square/okhttp",
      "size": "Medium",
      "files": "~640",
      "question": "How does OkHttp process a request through its interceptor chain?"
    },
    {
      "name": "guava",
      "repo": "https://github.com/google/guava",
      "size": "Large",
      "files": "~3000",
      "question": "How does Guava's CacheBuilder build and configure a cache?"
    }
  ],
  "Kotlin": [
    {
      "name": "koin",
      "repo": "https://github.com/InsertKoinIO/koin",
      "size": "Medium",
      "files": "~300",
      "question": "How does Koin resolve and inject dependencies?"
    },
    {
      "name": "leakcanary",
      "repo": "https://github.com/square/leakcanary",
      "size": "Medium",
      "files": "~250",
      "question": "How does LeakCanary detect and analyze a memory leak?"
    }
  ],
  "Swift": [
    {
      "name": "alamofire",
      "repo": "https://github.com/Alamofire/Alamofire",
      "size": "Small",
      "files": "~100",
      "question": "How does Alamofire build, send, and validate a request?"
    }
  ],
  "C#": [
    {
      "name": "serilog",
      "repo": "https://github.com/serilog/serilog",
      "size": "Medium",
      "files": "~250",
      "question": "How does Serilog route a log event to its sinks?"
    },
    {
      "name": "jellyfin",
      "repo": "https://github.com/jellyfin/jellyfin",
      "size": "Large",
      "files": "~2500",
      "question": "How does Jellyfin scan and identify items in a media library?"
    }
  ],
  "Ruby": [
    {
      "name": "sinatra",
      "repo": "https://github.com/sinatra/sinatra",
      "size": "Small",
      "files": "~60",
      "question": "How does Sinatra match a request to a route handler?"
    },
    {
      "name": "discourse",
      "repo": "https://github.com/discourse/discourse",
      "size": "Large",
      "files": "~3000",
      "question": "How does Discourse create and render a new post?"
    }
  ],
  "PHP": [
    {
      "name": "slim",
      "repo": "https://github.com/slimphp/Slim",
      "size": "Small",
      "files": "~80",
      "question": "How does Slim handle a request through its middleware?"
    },
    {
      "name": "laravel",
      "repo": "https://github.com/laravel/framework",
      "size": "Large",
      "files": "~3000",
      "question": "How does Laravel resolve and dispatch a route to a controller?"
    }
  ],
  "C": [
    {
      "name": "redis",
      "repo": "https://github.com/redis/redis",
      "size": "Large",
      "files": "~600",
      "question": "How does Redis parse and dispatch a client command?"
    }
  ],
  "C++": [
    {
      "name": "json",
      "repo": "https://github.com/nlohmann/json",
      "size": "Small",
      "files": "~100",
      "question": "How does nlohmann::json parse a JSON string into a value?"
    },
    {
      "name": "grpc",
      "repo": "https://github.com/grpc/grpc",
      "size": "Large",
      "files": "~3000",
      "question": "How does gRPC dispatch an incoming RPC to its handler?"
    }
  ],
  "Dart": [
    {
      "name": "flutter",
      "repo": "https://github.com/flutter/flutter",
      "size": "Large",
      "files": "~6000",
      "question": "How does Flutter build and lay out a widget tree?"
    }
  ],
  "Svelte": [
    {
      "name": "shadcn-svelte",
      "repo": "https://github.com/huntabyte/shadcn-svelte",
      "size": "Medium",
      "files": "~600",
      "question": "How do shadcn-svelte components compose and apply their styling?"
    }
  ],
  "Lua": [
    {
      "name": "lualine.nvim",
      "repo": "https://github.com/nvim-lualine/lualine.nvim",
      "size": "Small",
      "files": "~120",
      "question": "How does lualine assemble and render its statusline sections and components?"
    },
    {
      "name": "telescope.nvim",
      "repo": "https://github.com/nvim-telescope/telescope.nvim",
      "size": "Medium",
      "files": "~80",
      "question": "How does Telescope wire a picker to its finder, sorter, and previewer?"
    },
    {
      "name": "kong",
      "repo": "https://github.com/Kong/kong",
      "size": "Large",
      "files": "~1330",
      "question": "How does Kong execute plugins across a request's lifecycle phases?"
    }
  ],
  "Luau": [
    {
      "name": "Knit",
      "repo": "https://github.com/Sleitnick/Knit",
      "size": "Small",
      "files": "~10",
      "question": "How does Knit register services and expose them to clients?"
    },
    {
      "name": "vide",
      "repo": "https://github.com/centau/vide",
      "size": "Small",
      "files": "~40",
      "question": "How does vide track reactive sources and re-run effects when state changes?"
    },
    {
      "name": "Fusion",
      "repo": "https://github.com/dphfox/Fusion",
      "size": "Medium",
      "files": "~115",
      "question": "How does Fusion build and update its reactive UI graph from state objects?"
    }
  ],
  "Objective-C": [
    {
      "name": "Masonry",
      "repo": "https://github.com/SnapKit/Masonry",
      "size": "Small",
      "files": "~50",
      "question": "How does Masonry build and activate Auto Layout constraints from its block DSL?"
    },
    {
      "name": "FMDB",
      "repo": "https://github.com/ccgus/fmdb",
      "size": "Medium",
      "files": "~80",
      "question": "How does FMDB execute a prepared SQL statement and bind parameters?"
    },
    {
      "name": "SDWebImage",
      "repo": "https://github.com/SDWebImage/SDWebImage",
      "size": "Large",
      "files": "~400",
      "question": "How does SDWebImage download, cache, and decode an image for a UIImageView?"
    }
  ],
  "Mixed iOS (Swift+ObjC)": [
    {
      "name": "Charts",
      "repo": "https://github.com/danielgindi/Charts",
      "size": "Small",
      "files": "~270",
      "question": "How does the ChartsDemo ObjC demo controller drive the Swift Charts library to animate and notify a data update?"
    },
    {
      "name": "realm-swift",
      "repo": "https://github.com/realm/realm-swift",
      "size": "Medium",
      "files": "~370",
      "question": "How does a Swift `Realm.write { realm.add(obj) }` reach the Objective-C persistence layer?"
    },
    {
      "name": "wikipedia-ios",
      "repo": "https://github.com/wikimedia/wikipedia-ios",
      "size": "Large",
      "files": "~1700",
      "question": "How does tapping a search result reach the article-fetch network call across the Swift / ObjC boundary?"
    }
  ],
  "React Native (legacy bridge + TurboModule)": [
    {
      "name": "@react-native-async-storage",
      "repo": "https://github.com/react-native-async-storage/async-storage",
      "size": "Small",
      "files": "~60",
      "question": "How does `setItem` in JS reach the native `legacy_multiSet` implementation?"
    },
    {
      "name": "react-native-svg",
      "repo": "https://github.com/software-mansion/react-native-svg",
      "size": "Medium",
      "files": "~700",
      "question": "How does a JS `Svg.getTotalLength(...)` reach the iOS / Android native implementation via TurboModule?"
    },
    {
      "name": "react-native-firebase",
      "repo": "https://github.com/invertase/react-native-firebase",
      "size": "Large",
      "files": "~1100",
      "question": "How does a native iOS push notification reach the JS `messaging().onMessage(...)` listener?"
    }
  ],
  "Expo Modules": [
    {
      "name": "expo-haptics",
      "repo": "https://github.com/expo/expo/tree/main/packages/expo-haptics",
      "size": "Small",
      "files": "~15",
      "question": "How does `Haptics.notificationAsync(...)` in JS reach `UINotificationFeedbackGenerator` in the Swift Module?"
    },
    {
      "name": "expo-camera",
      "repo": "https://github.com/expo/expo/tree/main/packages/expo-camera",
      "size": "Medium",
      "files": "~70",
      "question": "How does a JS `CameraView.takePictureAsync(options)` reach the native AVCaptureSession / CameraDevice call?"
    }
  ],
  "React Native Fabric (view components)": [
    {
      "name": "react-native-segmented-control",
      "repo": "https://github.com/react-native-segmented-control/segmented-control",
      "size": "Small",
      "files": "~25",
      "question": "How does JSX `<SegmentedControl onChange={cb}/>` reach the native onChange handler on iOS/Android?"
    },
    {
      "name": "react-native-screens",
      "repo": "https://github.com/software-mansion/react-native-screens",
      "size": "Medium",
      "files": "~1200",
      "question": "How does JSX `<ScreenStack>` reach the native RNSScreenStackView component?"
    },
    {
      "name": "react-native-skia",
      "repo": "https://github.com/Shopify/react-native-skia",
      "size": "Large",
      "files": "~1000",
      "question": "How does a `<SkiaPictureView/>` JSX usage reach the iOS / Android native renderer?"
    }
  ],
  "R": [
    {
      "name": "AnomalyDetection",
      "repo": "https://github.com/twitter/AnomalyDetection",
      "size": "Small",
      "files": "~24",
      "question": "How does AnomalyDetectionTs go from the exported entry function to the underlying S-H-ESD statistical test? Name the functions on the path in order."
    },
    {
      "name": "dplyr",
      "repo": "https://github.com/tidyverse/dplyr",
      "size": "Medium",
      "files": "~450",
      "question": "When mutate() is called on a grouped data frame, which functions handle the grouping and expression evaluation, in order, from mutate() down?"
    },
    {
      "name": "ggplot2",
      "repo": "https://github.com/tidyverse/ggplot2",
      "size": "Large",
      "files": "~1150",
      "question": "When a ggplot object is printed, how does the plot actually get built and drawn \u2014 trace the path from print/plot to where geoms render. Name the key functions in order."
    }
  ]
}