{
  "schema": "usta.living-series.v1",
  "blogId": "D0422",
  "slug": "ai-crawler-blocking-daily-index",
  "title": "AI Crawler Blocking, Read Fresh Every Day",
  "seriesKey": "crawler",
  "dataThrough": "2026-08-23",
  "rowsSha256": "781d8cfb2d7c2a87328dbb5655befe10e8b7974d023332acc75471d39195d004",
  "canonicalUrl": "https://ustechautomations.com/resources/blog/ai-crawler-blocking-daily-index",
  "freshnessRule": "MAX() of the observation date column in the source data table; never a file timestamp or a collector run log",
  "rows": [
    {
      "crawler": "GPTBot",
      "operator": "OpenAI",
      "product_token": "gptbot",
      "sites_naming_it": 11078,
      "rule_blocks_whole_site": 8555,
      "rule_blocks_some_paths": 1209,
      "rule_names_it_and_allows": 1314,
      "change_vs_previous_day": -9,
      "sites_whose_verdict_changed": 25,
      "data_through": "2026-08-23"
    },
    {
      "crawler": "ClaudeBot",
      "operator": "Anthropic",
      "product_token": "claudebot",
      "sites_naming_it": 9842,
      "rule_blocks_whole_site": 7607,
      "rule_blocks_some_paths": 977,
      "rule_names_it_and_allows": 1258,
      "change_vs_previous_day": -3,
      "sites_whose_verdict_changed": 27,
      "data_through": "2026-08-23"
    },
    {
      "crawler": "CCBot",
      "operator": "Common Crawl",
      "product_token": "ccbot",
      "sites_naming_it": 9541,
      "rule_blocks_whole_site": 8362,
      "rule_blocks_some_paths": 585,
      "rule_names_it_and_allows": 594,
      "change_vs_previous_day": -3,
      "sites_whose_verdict_changed": 24,
      "data_through": "2026-08-23"
    },
    {
      "crawler": "Google-Extended",
      "operator": "Google",
      "product_token": "google-extended",
      "sites_naming_it": 8974,
      "rule_blocks_whole_site": 7040,
      "rule_blocks_some_paths": 849,
      "rule_names_it_and_allows": 1085,
      "change_vs_previous_day": -6,
      "sites_whose_verdict_changed": 23,
      "data_through": "2026-08-23"
    },
    {
      "crawler": "PerplexityBot",
      "operator": "Perplexity",
      "product_token": "perplexitybot",
      "sites_naming_it": 4613,
      "rule_blocks_whole_site": 2315,
      "rule_blocks_some_paths": 981,
      "rule_names_it_and_allows": 1317,
      "change_vs_previous_day": 0,
      "sites_whose_verdict_changed": 7,
      "data_through": "2026-08-23"
    },
    {
      "crawler": "Bytespider",
      "operator": "ByteDance",
      "product_token": "bytespider",
      "sites_naming_it": 8606,
      "rule_blocks_whole_site": 7911,
      "rule_blocks_some_paths": 412,
      "rule_names_it_and_allows": 283,
      "change_vs_previous_day": -1,
      "sites_whose_verdict_changed": 21,
      "data_through": "2026-08-23"
    },
    {
      "crawler": "Amazonbot",
      "operator": "Amazon",
      "product_token": "amazonbot",
      "sites_naming_it": 8040,
      "rule_blocks_whole_site": 6955,
      "rule_blocks_some_paths": 569,
      "rule_names_it_and_allows": 516,
      "change_vs_previous_day": -3,
      "sites_whose_verdict_changed": 17,
      "data_through": "2026-08-23"
    },
    {
      "crawler": "meta-externalagent",
      "operator": "Meta",
      "product_token": "meta-externalagent",
      "sites_naming_it": 7845,
      "rule_blocks_whole_site": 6835,
      "rule_blocks_some_paths": 572,
      "rule_names_it_and_allows": 438,
      "change_vs_previous_day": -6,
      "sites_whose_verdict_changed": 22,
      "data_through": "2026-08-23"
    }
  ]
}