{"name":"io.github.tjacquesson/llmtest-mcp","slug":"tjacquesson-llmtest-mcp","title":null,"description":"Benchmark AI models on real prompts. Find cheaper, faster alternatives across 340+ models.","url":"https://mcp.market/server/tjacquesson-llmtest-mcp","rating":null,"grade":"B","score":72,"certified":false,"status":"active","category":"other","tags":[],"presence":{"score":17,"stars":0,"forks":0,"downloads_week":40,"last_push_at":"2026-05-31T21:25:34.000Z","license":"MIT"},"uptime":null,"claimed":false,"transport":"npm","callable_via_gateway":false,"default_price_micros":0,"repository":"https://github.com/tjacquesson/llmtest-mcp","website":"https://llmtest.io","version":"0.6.1","remotes":[],"packages":[{"registryType":"npm","identifier":"llmtest-mcp","version":"0.6.1","transport":{"type":"stdio"},"environmentVariables":[{"description":"Your LLMTest API key from https://llmtest.io/dashboard","isRequired":true,"isSecret":true,"name":"LLMTEST_API_KEY"}]}],"tools":[{"name":"disable_autopilot","description":"Turn off autopilot. Already-accepted optimizations stay active; only future runs are paused. Use enable_autopilot to turn it back on.","write_action":false,"price_micros":0,"input_schema":null},{"name":"enable_autopilot","description":"Turn on autopilot. Once enabled, LLMTest will automatically optimize eligible flows every 7 days, auto-accept changes that pass all safety gates, and auto-revert if drift detection later sees regressions. Cost: each optimization run costs ~$1-2, billed to LLMTest credits. Runs are skipped silently if the account has no credits. Prerequisites: verified email, account age >= 14 days, and >=20 real c","write_action":false,"price_micros":0,"input_schema":null},{"name":"get_account","description":"Check your credit balance and usage stats.","write_action":false,"price_micros":0,"input_schema":null},{"name":"get_autopilot_status","description":"Show whether autopilot is enabled and whether the account meets the prerequisites. Autopilot automatically optimizes prompts and models for eligible flows every 7 days. Checks: optimization scheduler (weekly), drift detection (daily scan, per-optimization weekly). Prerequisites for a flow to be auto-optimized: - Autopilot enabled on the account - Email verified - Account age >= 14 days (trust ramp","write_action":false,"price_micros":0,"input_schema":null},{"name":"get_suggestions","description":"Get pending recommendations to switch to a cheaper or faster model without losing quality.","write_action":false,"price_micros":0,"input_schema":null},{"name":"list_active_optimizations","description":"List auto-accepted optimizations still within their 24h revert window. Each includes the flow, strategy, savings, and hours remaining to revert.","write_action":false,"price_micros":0,"input_schema":null},{"name":"list_flows","description":"Show all AI features in this project with their current model, call volume, latency, and cost.","write_action":false,"price_micros":0,"input_schema":null},{"name":"list_new_models","description":"Show new and trending models that are worth benchmarking soon.","write_action":false,"price_micros":0,"input_schema":null},{"name":"list_samples","description":"Show how many test samples are stored for each flow.","write_action":false,"price_micros":0,"input_schema":null},{"name":"optimize_prompt","description":"Analyze a flow's prompt and automatically generate an optimized version. Tests optimized prompt against baseline using real samples and a multi-judge AI (Claude Sonnet + GPT-4o, position-swapped, self-consistency). Also tests cheaper models. REQUIRES: at least 3 samples on the flow. If the flow has none (zero traffic), BEFORE calling this tool you should: (1) ask the user to describe the feature, ","write_action":false,"price_micros":0,"input_schema":null},{"name":"revert_optimization","description":"Revert an auto-accepted optimization. Only works within the 24h revert window. Restores the previous active prompt. Use list_active_optimizations to find IDs.","write_action":false,"price_micros":0,"input_schema":null},{"name":"run_benchmark","description":"Benchmark a flow against alternative models. Use optimize_for to focus the search: \"cost\" tests cheaper models, \"quality\" tests potentially better models, \"speed\" tests faster models, \"balanced\" (default) tests a diverse spread. Use challengers to test specific models (e.g. [\"anthropic/claude-sonnet-4\", \"google/gemini-2.5-flash-preview\"]). If omitted, challengers are selected automatically based o","write_action":true,"price_micros":0,"input_schema":null},{"name":"seed_samples","description":"Register test samples for a flow so it can be benchmarked without real traffic. Use this for pre-launch projects or when you want to benchmark a specific scenario. Each sample must be an array of messages in OpenAI chat format: [{ \"role\": \"system\", \"content\": \"...\" }, { \"role\": \"user\", \"content\": \"...\" }] To use: read the codebase to find the prompt for this flow, then generate 5-10 realistic and ","write_action":false,"price_micros":0,"input_schema":null},{"name":"status","description":"Show LLMTest proxy status and a summary of monitored AI activity.","write_action":false,"price_micros":0,"input_schema":null},{"name":"update_suggestion","description":"Accept or dismiss a model-switch suggestion by ID.","write_action":true,"price_micros":0,"input_schema":null}],"scan":{"score":72,"grade":"B","scanned_at":"2026-09-27T17:07:49.469Z","report":{"scannerVersion":"0.1.9","scannedAt":"2026-09-27T17:07:49.475Z","components":{"code":{"score":25,"max":25,"notes":["3 source files scanned"]},"reliability":{"score":-1,"max":20,"notes":["no gateway calls yet and no remote to probe"]},"poisoning":{"score":-1,"max":15,"notes":["tools not inspected (local package is not executed); not counted"]},"auth":{"score":6,"max":15,"notes":["static API keys via environment variables"]},"maintenance":{"score":8,"max":15,"notes":["last push 119 days ago"]},"identity":{"score":8,"max":10,"notes":["registry namespace matches repository owner","GitHub account older than a year"]}},"findings":[],"inputs":{"packages":[{"registryType":"npm","identifier":"llmtest-mcp","version":"0.6.1","found":true,"license":"MIT","hasInstallScripts":false,"dependencyCount":1,"publishedAt":"2026-05-22T09:05:38.607Z","repositoryUrl":"git+https://github.com/tjacquesson/llmtest-mcp.git","weeklyDownloads":40}],"repo":{"found":true,"owner":"tjacquesson","repo":"llmtest-mcp","archived":false,"pushedAt":"2026-05-31T21:25:34Z","stars":0,"forks":0,"openIssues":2,"ownerType":"User","ownerAvatarUrl":"https://avatars.githubusercontent.com/u/10845143?v=4","ownerCreatedAt":"2015-02-04T08:21:07Z","license":"MIT"},"icon":{"url":"data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 80 80'%3E%3Crect width='80' height='80' rx='16' fill='%23111'/%3E%3Crect x='26' y='50' width='28' height='8' rx='2' fill='%23555'/%3E%3Crect x='22' y='40' width='28' height='8' rx='2' fill='%23999'/%3E%3Crect x='18' y='30' width='28' height='8' rx='2' fill='%23fff'/%3E%3Ccircle cx='54' cy='34' r='4.5' fill='%23fff'/%3E%3Cpath d='M51.5 34 L53.5 36 L56.5 32' stroke='%23111' stroke-width='1.6' stroke-linecap='round' stroke-linejoin='round'/%3E%3C/svg%3E","source":"site"},"presence":{"stars":0,"forks":0,"downloadsWeek":40,"license":"MIT","lastPushAt":"2026-05-31T21:25:34.000Z","score":17}}}},"grade_history":[],"reviews":[]}