{"name":"io.github.mark1russell7/polite-crawler","slug":"mark1russell7-polite-crawler","title":"polite-crawler","description":"A polite crawler for agents: obeys robots.txt, fetches each page once, finds and verifies quotes.","url":"https://mcp.market/server/mark1russell7-polite-crawler","rating":null,"grade":"A","score":92,"certified":false,"status":"active","category":"other","tags":[],"presence":{"score":18,"stars":0,"forks":0,"downloads_week":null,"last_push_at":"2026-10-13T12:56:06.000Z","license":"MIT"},"uptime":null,"claimed":false,"transport":"npm","callable_via_gateway":false,"default_price_micros":0,"repository":"https://github.com/mark1russell7/crawler","website":"https://mark1russell7.github.io/crawler/","version":"0.3.0","remotes":[],"packages":[{"registryType":"npm","registryBaseUrl":"https://registry.npmjs.org","identifier":"polite-crawler-mcp","version":"0.3.0","runtimeHint":"npx","transport":{"type":"stdio"},"environmentVariables":[{"description":"The folder of the store. The default is ~/.polite-crawler.","format":"filepath","name":"POLITE_CRAWLER_DIR"},{"description":"An email address or URL for the From header of each request.","name":"POLITE_CRAWLER_CONTACT"},{"description":"Set it to 1 to let the crawler fetch loopback, private and link-local addresses. The default refuses them.","format":"boolean","name":"POLITE_CRAWLER_ALLOW_PRIVATE"},{"description":"Set it to 1 to let user_provided read a page that refuses AI input when the client cannot ask the user to confirm.","format":"boolean","name":"POLITE_CRAWLER_ALLOW_USER_PROVIDED"},{"description":"The file of an Ed25519 private key (PKCS#8 PEM or JWK). The crawler signs each request with it (Web Bot Auth). It needs POLITE_CRAWLER_SIGNATURE_AGENT.","format":"filepath","name":"POLITE_CRAWLER_SIGN_KEY"},{"description":"Where a site finds the public key of POLITE_CRAWLER_SIGN_KEY: the https origin of the key directory, or the URL of a JSON Web Key Set.","name":"POLITE_CRAWLER_SIGNATURE_AGENT"}]}],"tools":[],"scan":null,"grade_history":[],"reviews":[]}