{"name":"io.github.manchittlab/thecrawler","slug":"manchittlab-thecrawler","title":"TheCrawler","description":"Universal web scraper with LLM-ready markdown, RAG chunking, PDF/DOCX support.","url":"https://mcp.market/server/manchittlab-thecrawler","rating":null,"grade":"A","score":86,"certified":false,"status":"active","category":"ai","tags":["ai"],"presence":{"score":22,"stars":2,"forks":0,"downloads_week":35,"last_push_at":"2026-07-02T17:48:41.000Z","license":"AGPL-3.0"},"uptime":null,"claimed":false,"transport":"npm","callable_via_gateway":false,"default_price_micros":0,"repository":"https://github.com/manchittlab/TheCrawler","website":null,"version":"0.5.8","remotes":[],"packages":[{"registryType":"npm","identifier":"thecrawler","version":"0.5.8","transport":{"type":"stdio"}}],"tools":[{"name":"crawl","description":"Scrape one or more URLs. Returns rich structured data: title, description, text, markdown, links, images, meta, OG/Twitter cards, JSON-LD, microdata, headings, tables, forms, redirect chain, hreflang, pagination links, commerce data (price/rating/sku from JSON-LD), analytics tracker detection (16 trackers), and optional email-like/phone-like public text fields when explicitly enabled. Auto-handles","write_action":false,"price_micros":0,"input_schema":null},{"name":"crawl_markdown","description":"Extract clean markdown for a single URL. Strips boilerplate (nav/header/footer/cookie banners). Returns markdown text only — no surrounding metadata. Ideal for feeding a single page to an LLM. For multi-URL or structured output, use `crawl`, `extract_structured`, or `extract_extraction_contract` instead.","write_action":false,"price_micros":0,"input_schema":null},{"name":"crawl_sitemap","description":"Fetch URL list from a sitemap.xml (supports sitemap-index files), then optionally scrape them. Use scrape=false to just get the URL list (cheap), scrape=true to also extract content from each.","write_action":false,"price_micros":0,"input_schema":null},{"name":"diagnose_extraction_contract","description":"Run a no-LLM readiness diagnostic for one extraction contract before spending LLM tokens. Returns per-URL verdicts, scores, blockers, branchable error types, missing readiness signals, and a workflow-level recommended next step. This does not claim a site will work; it tells the agent whether to extract, retry with browser/proxy, review manually, or stop.","write_action":true,"price_micros":0,"input_schema":null},{"name":"extract_extraction_contract","description":"Crawl URLs and extract against a built-in validated extraction contract with an OpenAI-compatible LLM. Returns parsed data plus required-field validation and missingRequiredFields evidence. Use diagnose_extraction_contract first when the source is unknown.","write_action":false,"price_micros":0,"input_schema":null},{"name":"extract_structured","description":"LLM-powered structured extraction. Crawls each URL → cleans to markdown → asks an OpenAI-compatible LLM (llama.cpp / vLLM / LM Studio / Ollama / OpenAI) to return ONLY a JSON object matching your jsonSchema or prompt. Returns parsed typed data, not raw markdown. LLM endpoint configurable per-call or via env vars (THECRAWLER_LLM_BASEURL, THECRAWLER_LLM_MODEL, THECRAWLER_LLM_API_KEY).","write_action":false,"price_micros":0,"input_schema":null},{"name":"list_extraction_contracts","description":"List built-in extraction contracts agents can use for validated schema extraction. Use this before diagnose_extraction_contract or extract_extraction_contract.","write_action":false,"price_micros":0,"input_schema":null},{"name":"search_and_crawl","description":"Search Google for a query, then scrape the top N results. Returns structured data per result page. Without a SerpAPI key, falls back to scraping Google's HTML — fragile, can break when Google updates their HTML. For production agent use, supply serpApiKey.","write_action":false,"price_micros":0,"input_schema":null}],"scan":{"score":86,"grade":"A","scanned_at":"2026-09-26T11:38:05.285Z","report":{"scannerVersion":"0.1.9","scannedAt":"2026-09-26T11:38:05.342Z","components":{"code":{"score":25,"max":25,"notes":["32 source files scanned"]},"reliability":{"score":-1,"max":20,"notes":["no gateway calls yet and no remote to probe"]},"poisoning":{"score":-1,"max":15,"notes":["tools not inspected (local package is not executed); not counted"]},"auth":{"score":12,"max":15,"notes":["local package, no credentials required"]},"maintenance":{"score":12,"max":15,"notes":["last push 86 days ago"]},"identity":{"score":7,"max":10,"notes":["registry namespace matches repository owner"]}},"findings":[],"inputs":{"packages":[{"registryType":"npm","identifier":"thecrawler","version":"0.5.8","found":true,"license":"AGPL-3.0-or-later","hasInstallScripts":false,"dependencyCount":11,"publishedAt":"2026-07-02T18:43:01.909Z","repositoryUrl":"git+https://github.com/manchittlab/TheCrawler.git","weeklyDownloads":35}],"repo":{"found":true,"owner":"manchittlab","repo":"TheCrawler","archived":false,"pushedAt":"2026-07-02T17:48:41Z","stars":2,"forks":0,"openIssues":1,"ownerType":"Organization","ownerAvatarUrl":"https://avatars.githubusercontent.com/u/275741058?v=4","ownerCreatedAt":"2026-04-13T14:44:17Z","license":"AGPL-3.0"},"icon":{"url":null,"source":"none"},"presence":{"stars":2,"forks":0,"downloadsWeek":35,"license":"AGPL-3.0","lastPushAt":"2026-07-02T17:48:41.000Z","score":22}}}},"grade_history":[],"reviews":[]}