{"openapi":"3.1.0","info":{"title":"Robots.txt Parse and Crawler Policy Guard API","version":"1.0.0","description":"Parse caller-supplied robots.txt, select RFC 9309 crawler groups, evaluate one or many paths, diff policies, and apply explicit publication controls. The API never fetches a URL, crawls a site, bypasses access controls, or claims that robots.txt prevents indexing or protects private content."},"paths":{"/marketplace/rapid/robots/v1/parse":{"post":{"tags":["Robots.txt Parse"],"summary":"Parse Robots.txt","operationId":"parseRobotsText","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RobotsDocumentRequest"},"examples":{"example":{"summary":"Parse Robots.txt","value":{"content":"User-agent: *\nDisallow: /private/\nAllow: /private/public.html\nSitemap: https://example.com/sitemap.xml\n"}}}}}},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RobotsParseResult"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/marketplace/rapid/robots/v1/select":{"post":{"tags":["Crawler Rule Selection"],"summary":"Select Crawler Rules","operationId":"selectRobotsRules","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RobotsSelectRequest"},"examples":{"example":{"summary":"Select Crawler Rules","value":{"content":"User-agent: *\nDisallow: /private/\nAllow: /private/public.html\nSitemap: https://example.com/sitemap.xml\n","user_agent":"ExampleBot/1.0"}}}}}},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RobotsSelectResult"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/marketplace/rapid/robots/v1/can-fetch":{"post":{"tags":["Crawler Path Check"],"summary":"Check Crawler Path","operationId":"checkRobotsCanFetch","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RobotsCanFetchRequest"},"examples":{"example":{"summary":"Check Crawler Path","value":{"content":"User-agent: *\nDisallow: /private/\nAllow: /private/public.html\nSitemap: https://example.com/sitemap.xml\n","user_agent":"ExampleBot/1.0","url_or_path":"https://example.com/private/public.html"}}}}}},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RobotsCanFetchResult"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/marketplace/rapid/robots/v1/batch-check":{"post":{"tags":["Crawler Batch Check"],"summary":"Batch Check Crawler Paths","operationId":"batchCheckRobotsPaths","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RobotsBatchRequest"},"examples":{"example":{"summary":"Batch Check Crawler Paths","value":{"content":"User-agent: *\nDisallow: /private/\nAllow: /private/public.html\nSitemap: https://example.com/sitemap.xml\n","user_agent":"ExampleBot/1.0","paths":["/","/private/report.pdf","/private/public.html"]}}}}}},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RobotsBatchResult"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/marketplace/rapid/robots/v1/diff":{"post":{"tags":["Robots Policy Diff"],"summary":"Diff Robots Policies","operationId":"diffRobotsPolicies","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RobotsDiffRequest"},"examples":{"example":{"summary":"Diff Robots Policies","value":{"before":"User-agent: *\nDisallow: /private/\n","after":"User-agent: *\nDisallow: /private/\nAllow: /private/public.html\n","user_agent":"ExampleBot","sample_paths":["/private/public.html"]}}}}}},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RobotsDiffResult"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/marketplace/rapid/robots/v1/policy-check":{"post":{"tags":["Robots Policy"],"summary":"Check Robots Policy","operationId":"checkRobotsPolicy","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/RobotsPolicyRequest"},"examples":{"example":{"summary":"Check Robots Policy","value":{"content":"User-agent: *\nDisallow: /private/\nAllow: /private/public.html\nSitemap: https://example.com/sitemap.xml\n","require_wildcard_group":true,"require_https_sitemaps":true,"disallow_root_is_blocking":true}}}}}},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/RobotsPolicyResult"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}}},"components":{"schemas":{"HTTPValidationError":{"properties":{"detail":{"items":{"$ref":"#/components/schemas/ValidationError"},"type":"array","title":"Detail"}},"type":"object","title":"HTTPValidationError"},"RobotsBatchItem":{"properties":{"input":{"type":"string","title":"Input"},"path":{"type":"string","title":"Path"},"allowed":{"type":"boolean","title":"Allowed"},"reason":{"type":"string","enum":["ALLOW_RULE","DISALLOW_RULE","NO_MATCHING_RULE"],"title":"Reason"},"matched_rule":{"anyOf":[{"$ref":"#/components/schemas/RobotsRule"},{"type":"null"}]}},"additionalProperties":false,"type":"object","required":["input","path","allowed","reason","matched_rule"],"title":"RobotsBatchItem"},"RobotsBatchRequest":{"properties":{"content":{"type":"string","maxLength":512000,"title":"Content"},"user_agent":{"type":"string","maxLength":256,"minLength":1,"title":"User Agent"},"paths":{"items":{"type":"string"},"type":"array","maxItems":1000,"minItems":1,"title":"Paths"}},"additionalProperties":false,"type":"object","required":["content","user_agent","paths"],"title":"RobotsBatchRequest"},"RobotsBatchResult":{"properties":{"status":{"type":"string","const":"COMPLETED","title":"Status","default":"COMPLETED"},"decision":{"type":"string","enum":["CLEAR","REVIEW_REQUIRED","BLOCKED"],"title":"Decision"},"findings":{"items":{"$ref":"#/components/schemas/RobotsFinding"},"type":"array","title":"Findings"},"ruleset_version":{"type":"string","const":"RFC_9309_2022","title":"Ruleset Version","default":"RFC_9309_2022"},"scope":{"type":"string","const":"CALLER_SUPPLIED_ROBOTS_TEXT_ONLY","title":"Scope","default":"CALLER_SUPPLIED_ROBOTS_TEXT_ONLY"},"network_queried":{"type":"boolean","const":false,"title":"Network Queried","default":false},"url_fetched":{"type":"boolean","const":false,"title":"Url Fetched","default":false},"access_control_enforced":{"type":"boolean","const":false,"title":"Access Control Enforced","default":false},"indexing_outcome_verified":{"type":"boolean","const":false,"title":"Indexing Outcome Verified","default":false},"payload_stored":{"type":"boolean","const":false,"title":"Payload Stored","default":false},"next_action":{"type":"string","title":"Next Action"},"user_agent":{"type":"string","title":"User Agent"},"results":{"items":{"$ref":"#/components/schemas/RobotsBatchItem"},"type":"array","title":"Results"},"allowed_count":{"type":"integer","title":"Allowed Count"},"disallowed_count":{"type":"integer","title":"Disallowed Count"}},"additionalProperties":false,"type":"object","required":["decision","findings","next_action","user_agent","results","allowed_count","disallowed_count"],"title":"RobotsBatchResult"},"RobotsCanFetchRequest":{"properties":{"content":{"type":"string","maxLength":512000,"title":"Content"},"user_agent":{"type":"string","maxLength":256,"minLength":1,"title":"User Agent"},"url_or_path":{"type":"string","maxLength":4096,"minLength":1,"title":"Url Or Path"}},"additionalProperties":false,"type":"object","required":["content","user_agent","url_or_path"],"title":"RobotsCanFetchRequest"},"RobotsCanFetchResult":{"properties":{"status":{"type":"string","const":"COMPLETED","title":"Status","default":"COMPLETED"},"decision":{"type":"string","enum":["CLEAR","REVIEW_REQUIRED","BLOCKED"],"title":"Decision"},"findings":{"items":{"$ref":"#/components/schemas/RobotsFinding"},"type":"array","title":"Findings"},"ruleset_version":{"type":"string","const":"RFC_9309_2022","title":"Ruleset Version","default":"RFC_9309_2022"},"scope":{"type":"string","const":"CALLER_SUPPLIED_ROBOTS_TEXT_ONLY","title":"Scope","default":"CALLER_SUPPLIED_ROBOTS_TEXT_ONLY"},"network_queried":{"type":"boolean","const":false,"title":"Network Queried","default":false},"url_fetched":{"type":"boolean","const":false,"title":"Url Fetched","default":false},"access_control_enforced":{"type":"boolean","const":false,"title":"Access Control Enforced","default":false},"indexing_outcome_verified":{"type":"boolean","const":false,"title":"Indexing Outcome Verified","default":false},"payload_stored":{"type":"boolean","const":false,"title":"Payload Stored","default":false},"next_action":{"type":"string","title":"Next Action"},"user_agent":{"type":"string","title":"User Agent"},"path":{"type":"string","title":"Path"},"allowed":{"type":"boolean","title":"Allowed"},"matched_rule":{"anyOf":[{"$ref":"#/components/schemas/RobotsRule"},{"type":"null"}]},"matched_user_agents":{"items":{"type":"string"},"type":"array","title":"Matched User Agents"},"reason":{"type":"string","enum":["ALLOW_RULE","DISALLOW_RULE","NO_MATCHING_RULE"],"title":"Reason"}},"additionalProperties":false,"type":"object","required":["decision","findings","next_action","user_agent","path","allowed","matched_rule","matched_user_agents","reason"],"title":"RobotsCanFetchResult"},"RobotsDiffRequest":{"properties":{"before":{"type":"string","maxLength":512000,"title":"Before"},"after":{"type":"string","maxLength":512000,"title":"After"},"user_agent":{"type":"string","maxLength":256,"minLength":1,"title":"User Agent","default":"*"},"sample_paths":{"items":{"type":"string"},"type":"array","maxItems":256,"title":"Sample Paths"}},"additionalProperties":false,"type":"object","required":["before","after"],"title":"RobotsDiffRequest"},"RobotsDiffResult":{"properties":{"status":{"type":"string","const":"COMPLETED","title":"Status","default":"COMPLETED"},"decision":{"type":"string","enum":["CLEAR","REVIEW_REQUIRED","BLOCKED"],"title":"Decision"},"findings":{"items":{"$ref":"#/components/schemas/RobotsFinding"},"type":"array","title":"Findings"},"ruleset_version":{"type":"string","const":"RFC_9309_2022","title":"Ruleset Version","default":"RFC_9309_2022"},"scope":{"type":"string","const":"CALLER_SUPPLIED_ROBOTS_TEXT_ONLY","title":"Scope","default":"CALLER_SUPPLIED_ROBOTS_TEXT_ONLY"},"network_queried":{"type":"boolean","const":false,"title":"Network Queried","default":false},"url_fetched":{"type":"boolean","const":false,"title":"Url Fetched","default":false},"access_control_enforced":{"type":"boolean","const":false,"title":"Access Control Enforced","default":false},"indexing_outcome_verified":{"type":"boolean","const":false,"title":"Indexing Outcome Verified","default":false},"payload_stored":{"type":"boolean","const":false,"title":"Payload Stored","default":false},"next_action":{"type":"string","title":"Next Action"},"user_agent":{"type":"string","title":"User Agent"},"added_rules":{"items":{"$ref":"#/components/schemas/RobotsRule"},"type":"array","title":"Added Rules"},"removed_rules":{"items":{"$ref":"#/components/schemas/RobotsRule"},"type":"array","title":"Removed Rules"},"added_sitemaps":{"items":{"type":"string"},"type":"array","title":"Added Sitemaps"},"removed_sitemaps":{"items":{"type":"string"},"type":"array","title":"Removed Sitemaps"},"access_changes":{"items":{"additionalProperties":true,"type":"object"},"type":"array","title":"Access Changes"}},"additionalProperties":false,"type":"object","required":["decision","findings","next_action","user_agent","added_rules","removed_rules","added_sitemaps","removed_sitemaps","access_changes"],"title":"RobotsDiffResult"},"RobotsDocumentRequest":{"properties":{"content":{"type":"string","maxLength":512000,"title":"Content"}},"additionalProperties":false,"type":"object","required":["content"],"title":"RobotsDocumentRequest"},"RobotsFinding":{"properties":{"code":{"type":"string","title":"Code"},"severity":{"type":"string","enum":["LOW","MEDIUM","HIGH"],"title":"Severity"},"message":{"type":"string","title":"Message"},"evidence":{"additionalProperties":true,"type":"object","title":"Evidence"},"next_action":{"type":"string","title":"Next Action"}},"additionalProperties":false,"type":"object","required":["code","severity","message","next_action"],"title":"RobotsFinding"},"RobotsGroup":{"properties":{"user_agents":{"items":{"type":"string"},"type":"array","title":"User Agents"},"rules":{"items":{"$ref":"#/components/schemas/RobotsRule"},"type":"array","title":"Rules"},"start_line":{"type":"integer","title":"Start Line"}},"additionalProperties":false,"type":"object","required":["user_agents","rules","start_line"],"title":"RobotsGroup"},"RobotsParseResult":{"properties":{"status":{"type":"string","const":"COMPLETED","title":"Status","default":"COMPLETED"},"decision":{"type":"string","enum":["CLEAR","REVIEW_REQUIRED","BLOCKED"],"title":"Decision"},"findings":{"items":{"$ref":"#/components/schemas/RobotsFinding"},"type":"array","title":"Findings"},"ruleset_version":{"type":"string","const":"RFC_9309_2022","title":"Ruleset Version","default":"RFC_9309_2022"},"scope":{"type":"string","const":"CALLER_SUPPLIED_ROBOTS_TEXT_ONLY","title":"Scope","default":"CALLER_SUPPLIED_ROBOTS_TEXT_ONLY"},"network_queried":{"type":"boolean","const":false,"title":"Network Queried","default":false},"url_fetched":{"type":"boolean","const":false,"title":"Url Fetched","default":false},"access_control_enforced":{"type":"boolean","const":false,"title":"Access Control Enforced","default":false},"indexing_outcome_verified":{"type":"boolean","const":false,"title":"Indexing Outcome Verified","default":false},"payload_stored":{"type":"boolean","const":false,"title":"Payload Stored","default":false},"next_action":{"type":"string","title":"Next Action"},"groups":{"items":{"$ref":"#/components/schemas/RobotsGroup"},"type":"array","title":"Groups"},"sitemaps":{"items":{"type":"string"},"type":"array","title":"Sitemaps"},"extensions":{"items":{"additionalProperties":true,"type":"object"},"type":"array","title":"Extensions"},"comments":{"type":"integer","title":"Comments"},"blank_lines":{"type":"integer","title":"Blank Lines"}},"additionalProperties":false,"type":"object","required":["decision","findings","next_action","groups","sitemaps","extensions","comments","blank_lines"],"title":"RobotsParseResult"},"RobotsPolicyRequest":{"properties":{"content":{"type":"string","maxLength":512000,"title":"Content"},"require_wildcard_group":{"type":"boolean","title":"Require Wildcard Group","default":true},"require_https_sitemaps":{"type":"boolean","title":"Require Https Sitemaps","default":true},"require_absolute_sitemaps":{"type":"boolean","title":"Require Absolute Sitemaps","default":true},"disallow_root_is_blocking":{"type":"boolean","title":"Disallow Root Is Blocking","default":true},"allowed_user_agents":{"items":{"type":"string"},"type":"array","maxItems":128,"title":"Allowed User Agents"},"blocked_extensions":{"items":{"type":"string"},"type":"array","maxItems":32,"title":"Blocked Extensions"}},"additionalProperties":false,"type":"object","required":["content"],"title":"RobotsPolicyRequest"},"RobotsPolicyResult":{"properties":{"status":{"type":"string","const":"COMPLETED","title":"Status","default":"COMPLETED"},"decision":{"type":"string","enum":["CLEAR","REVIEW_REQUIRED","BLOCKED"],"title":"Decision"},"findings":{"items":{"$ref":"#/components/schemas/RobotsFinding"},"type":"array","title":"Findings"},"ruleset_version":{"type":"string","const":"RFC_9309_2022","title":"Ruleset Version","default":"RFC_9309_2022"},"scope":{"type":"string","const":"CALLER_SUPPLIED_ROBOTS_TEXT_ONLY","title":"Scope","default":"CALLER_SUPPLIED_ROBOTS_TEXT_ONLY"},"network_queried":{"type":"boolean","const":false,"title":"Network Queried","default":false},"url_fetched":{"type":"boolean","const":false,"title":"Url Fetched","default":false},"access_control_enforced":{"type":"boolean","const":false,"title":"Access Control Enforced","default":false},"indexing_outcome_verified":{"type":"boolean","const":false,"title":"Indexing Outcome Verified","default":false},"payload_stored":{"type":"boolean","const":false,"title":"Payload Stored","default":false},"next_action":{"type":"string","title":"Next Action"},"allowed":{"type":"boolean","title":"Allowed"},"group_count":{"type":"integer","title":"Group Count"},"wildcard_group_present":{"type":"boolean","title":"Wildcard Group Present"},"sitemap_count":{"type":"integer","title":"Sitemap Count"},"extension_names":{"items":{"type":"string"},"type":"array","title":"Extension Names"}},"additionalProperties":false,"type":"object","required":["decision","findings","next_action","allowed","group_count","wildcard_group_present","sitemap_count","extension_names"],"title":"RobotsPolicyResult"},"RobotsRule":{"properties":{"directive":{"type":"string","enum":["ALLOW","DISALLOW"],"title":"Directive"},"pattern":{"type":"string","title":"Pattern"},"line":{"type":"integer","title":"Line"},"specificity":{"type":"integer","title":"Specificity"}},"additionalProperties":false,"type":"object","required":["directive","pattern","line","specificity"],"title":"RobotsRule"},"RobotsSelectRequest":{"properties":{"content":{"type":"string","maxLength":512000,"title":"Content"},"user_agent":{"type":"string","maxLength":256,"minLength":1,"title":"User Agent"}},"additionalProperties":false,"type":"object","required":["content","user_agent"],"title":"RobotsSelectRequest"},"RobotsSelectResult":{"properties":{"status":{"type":"string","const":"COMPLETED","title":"Status","default":"COMPLETED"},"decision":{"type":"string","enum":["CLEAR","REVIEW_REQUIRED","BLOCKED"],"title":"Decision"},"findings":{"items":{"$ref":"#/components/schemas/RobotsFinding"},"type":"array","title":"Findings"},"ruleset_version":{"type":"string","const":"RFC_9309_2022","title":"Ruleset Version","default":"RFC_9309_2022"},"scope":{"type":"string","const":"CALLER_SUPPLIED_ROBOTS_TEXT_ONLY","title":"Scope","default":"CALLER_SUPPLIED_ROBOTS_TEXT_ONLY"},"network_queried":{"type":"boolean","const":false,"title":"Network Queried","default":false},"url_fetched":{"type":"boolean","const":false,"title":"Url Fetched","default":false},"access_control_enforced":{"type":"boolean","const":false,"title":"Access Control Enforced","default":false},"indexing_outcome_verified":{"type":"boolean","const":false,"title":"Indexing Outcome Verified","default":false},"payload_stored":{"type":"boolean","const":false,"title":"Payload Stored","default":false},"next_action":{"type":"string","title":"Next Action"},"user_agent":{"type":"string","title":"User Agent"},"matched_user_agents":{"items":{"type":"string"},"type":"array","title":"Matched User Agents"},"match_kind":{"type":"string","enum":["SPECIFIC","WILDCARD","NONE"],"title":"Match Kind"},"rules":{"items":{"$ref":"#/components/schemas/RobotsRule"},"type":"array","title":"Rules"}},"additionalProperties":false,"type":"object","required":["decision","findings","next_action","user_agent","matched_user_agents","match_kind","rules"],"title":"RobotsSelectResult"},"ValidationError":{"properties":{"loc":{"items":{"anyOf":[{"type":"string"},{"type":"integer"}]},"type":"array","title":"Location"},"msg":{"type":"string","title":"Message"},"type":{"type":"string","title":"Error Type"},"input":{"title":"Input"},"ctx":{"type":"object","title":"Context"}},"type":"object","required":["loc","msg","type"],"title":"ValidationError"}}},"tags":[{"name":"Robots.txt Parse","description":"A bounded, stateless RFC 9309 preflight for browsing agents, crawler operators, SEO tooling, site migrations, and publishing workflows."},{"name":"Crawler Rule Selection","description":"A bounded, stateless RFC 9309 preflight for browsing agents, crawler operators, SEO tooling, site migrations, and publishing workflows."},{"name":"Crawler Path Check","description":"A bounded, stateless RFC 9309 preflight for browsing agents, crawler operators, SEO tooling, site migrations, and publishing workflows."},{"name":"Crawler Batch Check","description":"A bounded, stateless RFC 9309 preflight for browsing agents, crawler operators, SEO tooling, site migrations, and publishing workflows."},{"name":"Robots Policy Diff","description":"A bounded, stateless RFC 9309 preflight for browsing agents, crawler operators, SEO tooling, site migrations, and publishing workflows."},{"name":"Robots Policy","description":"A bounded, stateless RFC 9309 preflight for browsing agents, crawler operators, SEO tooling, site migrations, and publishing workflows."}],"servers":[{"url":"https://api-production-9502.up.railway.app"}]}