{"sku":"crawler-policy-sentinel","label":"AI-crawler policy sentinel","amount_cents":17500,"cadence":"monthly","summary":"A site's robots.txt is free to read and impossible to look back at: publishers overwrite it in place, so the day a publisher started blocking a crawler is simply gone. We have held a dated daily copy across a 100,000-domain panel since 2026-06-09. Name the domains you care about and we tell you who started blocking which AI crawler, who stopped, and on which day — with the sealed snapshot behind every row.","delivers":["A monthly change feed for the domains you name, across the AI crawler product tokens we track","Who moved to blocked, who moved back, and the exact snapshot date the change first appeared in","The wildcard-block case counted separately — a site that blocks every crawler made no decision about AI in particular","Every row carries the two snapshot dates it was derived from","A domain we could not reach that day is reported unreachable, never as a policy we did not observe"],"checkout_armed":false,"request_url":"https://permits.ustechautomations.com/public/offers/crawler-policy-sentinel/request","contact_attribution":{"contract":"product-specific contact to L0 Opportunity","path_id":"permits-offers:crawler-policy-sentinel:contact","hypothesis_id":"h-crawler-policy-sentinel-priced-offer-2026-08","artifact_id":"permits-offer-crawler-policy-sentinel-v1","cutover_at":"2026-08-02T07:10:00Z","archetype":"diff-sentinel","data_type":"tabular-registry","geo_level":"global","cta_contract_id":"permits-offer:crawler-policy-sentinel:primary","proof_url":"https://permits.ustechautomations.com/public/offers/crawler-policy-sentinel","proof_occurred_at":"2026-08-02T07:10:00Z"},"sample":{"archive_dates_expected":60,"archive_dates_sealed":55,"archive_first_sealed":"2026-06-09","archive_latest_sealed":"2026-08-07","archive_totals":{"body_change_events":174288,"distinct_robots_bodies":204500,"distinct_verdict_signatures":4285,"domains_ever_body_changed":18627,"domains_ever_verdict_changed":2793,"token_transition_events":31194,"verdict_change_events":4346},"built_at":"2026-08-08T22:24:32Z","coverage_gaps":{"max_gap_days":3,"missing_dates":["2026-06-11","2026-06-12","2026-06-19","2026-07-04","2026-07-15"],"short_dates":[]},"method":"One hundred thousand domains are fetched on a fixed daily schedule and each response is sealed as fetched. A day's diff compares a domain against its own previous SEALED reading, so a missed collection day widens the span rather than inventing a daily rate, and the span is printed on every row. A verdict is the RFC 9309 grouping rule applied to an exact user-agent token match: it reports what the file says, never what a site intends. A domain we could not reach has no observed policy and is excluded from both sides of every figure — unreachable is unknown, not permission.","panel_domains":100000,"rows_withheld_reason":"No domain is named here and no robots.txt file is reproduced. Crawler policy is published per site, and thousands of those files carry contact addresses in their comments, so this feed exposes mechanical verdicts and aggregate movement only. A subscription reports the same movement against the domains you give us, which are yours to name.","sku":"crawler-policy-sentinel","universe_version":"tranco_L8QG4_top100000.txt:8778484edc1487aa","week":[{"bodies_changed":3027,"compared_against":"2026-07-31","domains_compared":61027,"snapshot_date":"2026-08-01","span_days":1,"verdicts_changed":52},{"bodies_changed":2619,"compared_against":"2026-08-01","domains_compared":61033,"snapshot_date":"2026-08-02","span_days":1,"verdicts_changed":41},{"bodies_changed":2841,"compared_against":"2026-08-02","domains_compared":61162,"snapshot_date":"2026-08-03","span_days":1,"verdicts_changed":49},{"bodies_changed":3207,"compared_against":"2026-08-03","domains_compared":61167,"snapshot_date":"2026-08-04","span_days":1,"verdicts_changed":83},{"bodies_changed":3193,"compared_against":"2026-08-04","domains_compared":61154,"snapshot_date":"2026-08-05","span_days":1,"verdicts_changed":61},{"bodies_changed":3161,"compared_against":"2026-08-05","domains_compared":60947,"snapshot_date":"2026-08-06","span_days":1,"verdicts_changed":68},{"bodies_changed":3206,"compared_against":"2026-08-06","domains_compared":61057,"snapshot_date":"2026-08-07","span_days":1,"verdicts_changed":106}],"week_of":["2026-08-01","2026-08-02","2026-08-03","2026-08-04","2026-08-05","2026-08-06","2026-08-07"],"week_verdict_moves":[{"agent_token":"meta-externalagent","compared_against":"2026-08-06","domains":34,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"blocked_full"},{"agent_token":"ccbot","compared_against":"2026-08-06","domains":34,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"blocked_full"},{"agent_token":"bytespider","compared_against":"2026-08-06","domains":33,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"blocked_full"},{"agent_token":"google-extended","compared_against":"2026-08-06","domains":31,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"blocked_full"},{"agent_token":"claudebot","compared_against":"2026-08-06","domains":31,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"blocked_full"},{"agent_token":"applebot-extended","compared_against":"2026-08-06","domains":31,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"blocked_full"},{"agent_token":"gptbot","compared_against":"2026-08-06","domains":30,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"blocked_full"},{"agent_token":"amazonbot","compared_against":"2026-08-06","domains":30,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"blocked_full"},{"agent_token":"gptbot","compared_against":"2026-08-06","domains":16,"from_verdict":"blocked_full","snapshot_date":"2026-08-07","to_verdict":"unmentioned"},{"agent_token":"google-extended","compared_against":"2026-08-06","domains":13,"from_verdict":"blocked_full","snapshot_date":"2026-08-07","to_verdict":"unmentioned"},{"agent_token":"ccbot","compared_against":"2026-08-06","domains":13,"from_verdict":"blocked_full","snapshot_date":"2026-08-07","to_verdict":"unmentioned"},{"agent_token":"perplexitybot","compared_against":"2026-08-06","domains":11,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"allowed_explicit"},{"agent_token":"gptbot","compared_against":"2026-08-06","domains":11,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"allowed_explicit"},{"agent_token":"claudebot","compared_against":"2026-08-06","domains":11,"from_verdict":"blocked_full","snapshot_date":"2026-08-07","to_verdict":"unmentioned"},{"agent_token":"amazonbot","compared_against":"2026-08-06","domains":11,"from_verdict":"blocked_full","snapshot_date":"2026-08-07","to_verdict":"unmentioned"},{"agent_token":"google-extended","compared_against":"2026-08-06","domains":10,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"allowed_explicit"},{"agent_token":"chatgpt-user","compared_against":"2026-08-06","domains":10,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"allowed_explicit"},{"agent_token":"meta-externalagent","compared_against":"2026-08-06","domains":9,"from_verdict":"blocked_full","snapshot_date":"2026-08-07","to_verdict":"unmentioned"},{"agent_token":"claudebot","compared_against":"2026-08-06","domains":9,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"allowed_explicit"},{"agent_token":"bytespider","compared_against":"2026-08-06","domains":9,"from_verdict":"blocked_full","snapshot_date":"2026-08-07","to_verdict":"unmentioned"},{"agent_token":"applebot-extended","compared_against":"2026-08-06","domains":9,"from_verdict":"blocked_full","snapshot_date":"2026-08-07","to_verdict":"unmentioned"},{"agent_token":"chatgpt-user","compared_against":"2026-08-06","domains":8,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"restricted"},{"agent_token":"perplexitybot","compared_against":"2026-08-06","domains":7,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"restricted"},{"agent_token":"perplexity-user","compared_against":"2026-08-06","domains":7,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"restricted"},{"agent_token":"oai-searchbot","compared_against":"2026-08-06","domains":7,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"restricted"},{"agent_token":"oai-searchbot","compared_against":"2026-08-06","domains":7,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"allowed_explicit"},{"agent_token":"cohere-ai","compared_against":"2026-08-06","domains":6,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"blocked_full"},{"agent_token":"claude-web","compared_against":"2026-08-06","domains":6,"from_verdict":"blocked_full","snapshot_date":"2026-08-07","to_verdict":"unmentioned"},{"agent_token":"perplexitybot","compared_against":"2026-08-06","domains":5,"from_verdict":"blocked_full","snapshot_date":"2026-08-07","to_verdict":"unmentioned"},{"agent_token":"perplexity-user","compared_against":"2026-08-06","domains":5,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"allowed_explicit"},{"agent_token":"google-extended","compared_against":"2026-08-06","domains":5,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"restricted"},{"agent_token":"diffbot","compared_against":"2026-08-06","domains":5,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"blocked_full"},{"agent_token":"anthropic-ai","compared_against":"2026-08-06","domains":5,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"blocked_full"},{"agent_token":"amazonbot","compared_against":"2026-08-06","domains":5,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"restricted"},{"agent_token":"omgilibot","compared_against":"2026-08-06","domains":4,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"blocked_full"},{"agent_token":"omgili","compared_against":"2026-08-06","domains":4,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"blocked_full"},{"agent_token":"oai-searchbot","compared_against":"2026-08-06","domains":4,"from_verdict":"restricted","snapshot_date":"2026-08-07","to_verdict":"unmentioned"},{"agent_token":"oai-searchbot","compared_against":"2026-08-06","domains":4,"from_verdict":"allowed_explicit","snapshot_date":"2026-08-07","to_verdict":"restricted"},{"agent_token":"meta-externalagent","compared_against":"2026-08-06","domains":4,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"allowed_explicit"},{"agent_token":"imagesiftbot","compared_against":"2026-08-06","domains":4,"from_verdict":"unmentioned","snapshot_date":"2026-08-07","to_verdict":"blocked_full"}],"week_verdict_moves_shown":40,"week_verdict_moves_total":400}}