{"name":"io.github.thejaredchapman/evalforge-lite","slug":"thejaredchapman-evalforge-lite","title":"EvalForge Lite","description":"Compare text LLMs across OpenRouter, Bedrock, Vertex AI and Foundry with automated grading.","url":"https://mcp.market/server/thejaredchapman-evalforge-lite","rating":null,"grade":"B","score":83,"certified":false,"status":"active","category":"other","tags":[],"presence":{"score":18,"stars":0,"forks":0,"downloads_week":null,"last_push_at":"2026-10-01T18:47:52.000Z","license":"MIT"},"uptime":null,"claimed":false,"transport":"pypi","callable_via_gateway":false,"default_price_micros":0,"repository":"https://github.com/thejaredchapman/evalforge-lite","website":"https://github.com/thejaredchapman/evalforge-lite/blob/main/docs/mcp-server.md","version":"1.0.1","remotes":[],"packages":[{"registryType":"pypi","identifier":"evalforge-lite","version":"1.0.1","transport":{"type":"stdio"},"environmentVariables":[{"description":"Optional. OpenRouter API key held by the server so tool calls need no credentials for OpenRouter. If unset, pass creds in each tool call.","isSecret":true,"name":"OPENROUTER_API_KEY"}]}],"tools":[{"name":"evaluate_prompt","description":"Get pre-run feedback on a prompt's clarity/specificity before running a comparison.","write_action":false,"price_micros":0,"input_schema":null},{"name":"get_report","description":"Get a PDF report (base64-encoded) for a run. Defaults to the most recent run.","write_action":false,"price_micros":0,"input_schema":null},{"name":"get_report_csv","description":"Get a CSV export for a run, one row per (test case x model) cell. Defaults to the most recent run.","write_action":false,"price_micros":0,"input_schema":null},{"name":"list_availability","description":"Return the current model-availability snapshot: live OpenRouter listing status (refreshed at most every 6 hours) plus curated Bedrock/Vertex/Foundry region coverage.","write_action":false,"price_micros":0,"input_schema":null},{"name":"list_models","description":"List every provider and model in the catalog, plus each provider's frontier (flagship) model.","write_action":false,"price_micros":0,"input_schema":null},{"name":"list_runs","description":"List metadata for the 5 most recent runs, newest first.","write_action":false,"price_micros":0,"input_schema":null},{"name":"run_comparison","description":"Run a set of test-case prompts against a set of models, scoring each response.","write_action":true,"price_micros":0,"input_schema":null},{"name":"set_policy","description":"Set the company policy text used to gate prompts before any model is called.","write_action":false,"price_micros":0,"input_schema":null},{"name":"suggest_models","description":"Suggest sibling models from the same family as the given model id.","write_action":false,"price_micros":0,"input_schema":null}],"scan":{"score":83,"grade":"B","scanned_at":"2026-10-01T23:10:43.156Z","report":{"scannerVersion":"0.1.10","scannedAt":"2026-10-01T23:10:43.070Z","components":{"code":{"score":25,"max":25,"notes":["28 source files scanned"]},"reliability":{"score":-1,"max":20,"notes":["no gateway calls yet and no remote to probe"]},"poisoning":{"score":-1,"max":15,"notes":["tools not inspected (local package is not executed); not counted"]},"auth":{"score":6,"max":15,"notes":["static API keys via environment variables"]},"maintenance":{"score":15,"max":15,"notes":["last push 0 days ago"]},"identity":{"score":8,"max":10,"notes":["registry namespace matches repository owner","GitHub account older than a year"]}},"findings":[],"inputs":{"packages":[{"registryType":"pypi","identifier":"evalforge-lite","version":"1.0.1","found":true,"license":"MIT","dependencyCount":8,"publishedAt":"2026-10-01T18:53:15.738777Z","repositoryUrl":"https://github.com/thejaredchapman/evalforge-lite"}],"repo":{"found":true,"owner":"thejaredchapman","repo":"evalforge-lite","archived":false,"pushedAt":"2026-10-01T18:47:52Z","stars":0,"forks":0,"openIssues":0,"ownerType":"User","ownerAvatarUrl":"https://avatars.githubusercontent.com/u/8992140?v=4","ownerCreatedAt":"2014-10-02T03:15:49Z","license":"MIT"},"icon":{"url":"https://avatars.githubusercontent.com/u/8992140?v=4&s=128","source":"github"},"presence":{"stars":0,"forks":0,"downloadsWeek":null,"license":"MIT","lastPushAt":"2026-10-01T18:47:52.000Z","score":18}}}},"grade_history":[],"reviews":[]}