{"id":"4da80500-a92f-4ed2-be31-0d32e48855f4","url":"https://netintel.dev/robots-txt/analyze?domain=example.com","method":"GET","fingerprint":"b24da107b15902570615442ec56cbc203da9f6476e7bc7ecb0d0eacb1741546b","name":"NetIntel","description":null,"category":"other","docs_url":null,"advertised_price_usd":"0.010000","asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","scheme":"exact","networks":["eip155:8453","solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp"],"pay_to":"0xdaDc335482AD545296Fd7b28518A251fFCbEb9Df","input_schema":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["domain"],"properties":{"path":{"type":"string","description":"URL path to check permission for (e.g. /api/data)"},"domain":{"type":"string","description":"Domain name to fetch robots.txt from (e.g. example.com)"},"user_agent":{"type":"string","description":"Bot name to check rules for (default: *)"}}}},"additionalProperties":false},"output_schema":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object","properties":{"found":{"type":"boolean"},"grade":{"type":"string"},"rules":{"type":"array","items":{"type":"object"}},"score":{"type":"number"},"domain":{"type":"string"},"findings":{"type":"array","items":{"type":"object"}},"sitemaps":{"type":"array","items":{"type":"string"}},"path_check":{"type":"object"},"robots_url":{"type":"string"},"status_code":{"type":"number"},"raw_content_preview":{"type":"string"}}}}},"schema_completeness":0,"status":"live","first_seen_at":"2026-08-23 12:59:56.706788+00","last_seen_at":"2026-08-23 15:27:41.198+00","last_success_at":"2026-09-14 09:49:56.006+00","dead_since":null,"probe_backoff_tier":0,"next_probe_at":"2026-09-14 09:54:56.006+00","search_tsv":"'netintel':1","catalog_number":"2011","x402_version":2,"resource_url":"https://netintel.dev/robots-txt/analyze","resource_description":"Fetch and parse a domain's robots.txt file — returns all crawl rules by user-agent, sitemap URLs, crawl delay settings, and checks whether a specific path is allowed or blocked for any bot — so agents can respect crawl policies and locate sitemaps before scraping.","bazaar_extension":{"info":{"input":{"type":"http","method":"GET","queryParams":{"domain":"example.com"}},"output":{"type":"json","example":{"found":true,"grade":"B","rules":[{"allow":["/public/"],"disallow":["/private/","/admin/"],"user_agent":"*","crawl_delay":null}],"score":85,"domain":"example.com","findings":[],"sitemaps":["https://example.com/sitemap.xml"],"path_check":{"path":"/private/page","allowed":false,"rule_type":"disallow","user_agent":"*","matched_rule":"Disallow: /private/"},"robots_url":"https://example.com/robots.txt","status_code":200,"raw_content_preview":"User-agent: *\nDisallow: /private/\n..."}}},"schema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["domain"],"properties":{"path":{"type":"string","description":"URL path to check permission for (e.g. /api/data)"},"domain":{"type":"string","description":"Domain name to fetch robots.txt from (e.g. example.com)"},"user_agent":{"type":"string","description":"Bot name to check rules for (default: *)"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object","properties":{"found":{"type":"boolean"},"grade":{"type":"string"},"rules":{"type":"array","items":{"type":"object"}},"score":{"type":"number"},"domain":{"type":"string"},"findings":{"type":"array","items":{"type":"object"}},"sitemaps":{"type":"array","items":{"type":"string"}},"path_check":{"type":"object"},"robots_url":{"type":"string"},"status_code":{"type":"number"},"raw_content_preview":{"type":"string"}}}}}}}},"accepted_requirements":[{"raw":{"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","extra":{"name":"USD Coin","version":"2"},"payTo":"0xdaDc335482AD545296Fd7b28518A251fFCbEb9Df","amount":"10000","scheme":"exact","network":"eip155:8453","maxTimeoutSeconds":300},"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","payTo":"0xdaDc335482AD545296Fd7b28518A251fFCbEb9Df","scheme":"exact","network":"eip155:8453","amountAtomic":"10000"},{"raw":{"asset":"EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v","extra":{"feePayer":"GVJJ7rdGiXr5xaYbRwRbjfaJL7fmwRygFi1H6aGqDveb"},"payTo":"8FVeybWnB4dYrxTuPK7WzFZNAiEcFVaDLFnKpRwVGr5s","amount":"10000","scheme":"exact","network":"solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp","maxTimeoutSeconds":300},"asset":"EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v","payTo":"8FVeybWnB4dYrxTuPK7WzFZNAiEcFVaDLFnKpRwVGr5s","scheme":"exact","network":"solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp","amountAtomic":"10000"}],"bazaar_compatible":true,"bazaar_validated_at":"2026-09-14 09:49:56.006+00","bazaar_validation_error":null,"window":"24h","probes_n":23,"handshake_uptime":1,"settle_success_rate":0,"valid_result_rate":null,"latency_p50":477,"latency_p90":2068,"price_drift_pct":0,"score":0.7834,"computed_at":"2026-09-14 09:50:28.957+00","source_metadata":[{"raw":{"url":"https://netintel.dev/robots-txt/analyze?domain=example.com","name":"NetIntel","method":"GET","source":"bazaar:api.cdp.coinbase.com","fingerprint":"b24da107b15902570615442ec56cbc203da9f6476e7bc7ecb0d0eacb1741546b","advertisedPriceUsd":0.01},"source_id":1,"external_id":null}],"live_endpoint_metadata":{"x402_version":2,"resource_url":"https://netintel.dev/robots-txt/analyze","resource_description":"Fetch and parse a domain's robots.txt file — returns all crawl rules by user-agent, sitemap URLs, crawl delay settings, and checks whether a specific path is allowed or blocked for any bot — so agents can respect crawl policies and locate sitemaps before scraping.","bazaar_extension":{"info":{"input":{"type":"http","method":"GET","queryParams":{"domain":"example.com"}},"output":{"type":"json","example":{"found":true,"grade":"B","rules":[{"allow":["/public/"],"disallow":["/private/","/admin/"],"user_agent":"*","crawl_delay":null}],"score":85,"domain":"example.com","findings":[],"sitemaps":["https://example.com/sitemap.xml"],"path_check":{"path":"/private/page","allowed":false,"rule_type":"disallow","user_agent":"*","matched_rule":"Disallow: /private/"},"robots_url":"https://example.com/robots.txt","status_code":200,"raw_content_preview":"User-agent: *\nDisallow: /private/\n..."}}},"schema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["domain"],"properties":{"path":{"type":"string","description":"URL path to check permission for (e.g. /api/data)"},"domain":{"type":"string","description":"Domain name to fetch robots.txt from (e.g. example.com)"},"user_agent":{"type":"string","description":"Bot name to check rules for (default: *)"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object","properties":{"found":{"type":"boolean"},"grade":{"type":"string"},"rules":{"type":"array","items":{"type":"object"}},"score":{"type":"number"},"domain":{"type":"string"},"findings":{"type":"array","items":{"type":"object"}},"sitemaps":{"type":"array","items":{"type":"string"}},"path_check":{"type":"object"},"robots_url":{"type":"string"},"status_code":{"type":"number"},"raw_content_preview":{"type":"string"}}}}}}}},"accepted_requirements":[{"raw":{"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","extra":{"name":"USD Coin","version":"2"},"payTo":"0xdaDc335482AD545296Fd7b28518A251fFCbEb9Df","amount":"10000","scheme":"exact","network":"eip155:8453","maxTimeoutSeconds":300},"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","payTo":"0xdaDc335482AD545296Fd7b28518A251fFCbEb9Df","scheme":"exact","network":"eip155:8453","amountAtomic":"10000"},{"raw":{"asset":"EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v","extra":{"feePayer":"GVJJ7rdGiXr5xaYbRwRbjfaJL7fmwRygFi1H6aGqDveb"},"payTo":"8FVeybWnB4dYrxTuPK7WzFZNAiEcFVaDLFnKpRwVGr5s","amount":"10000","scheme":"exact","network":"solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp","maxTimeoutSeconds":300},"asset":"EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v","payTo":"8FVeybWnB4dYrxTuPK7WzFZNAiEcFVaDLFnKpRwVGr5s","scheme":"exact","network":"solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp","amountAtomic":"10000"}],"bazaar_compatible":true,"bazaar_validated_at":"2026-09-14 09:49:56.006+00","bazaar_validation_error":null},"sonar_metrics":{"score":0.7834,"uptime":1,"paid_success_rate":0,"valid_result_rate":null,"latency_p50":477,"latency_p90":2068,"price_drift_pct":0},"cdp_indexed":false,"confidence":"high"}