{
 "blogId": "D0416",
 "slug": "closing-web-index-august-2026",
 "title": "19.3% of Top Sites Block AI Crawlers: USTA Index",
 "template": "R17_CLOSING_WEB_INDEX",
 "editionId": "2026-08",
 "snapshotSha256": "69544d238343330cf3a3403f61913938ad63a90779ba99e4bcc6c4bd4b82938e",
 "sealedAt": "2026-08-01",
 "source": "US Tech Automations Research",
 "methodology": "https://ustechautomations.com/resources/research",
 "canonicalUrl": "https://ustechautomations.com/resources/blog/closing-web-index-august-2026",
 "displays": [
  {
   "key": "edition.snapshotDate",
   "display": "August 1, 2026",
   "value": null,
   "kind": "window"
  },
  {
   "key": "edition.universe",
   "display": "Tranco top 100,000",
   "value": null,
   "kind": "window"
  },
  {
   "key": "edition.sitesChecked",
   "display": "100000",
   "value": 100000,
   "kind": "count"
  },
  {
   "key": "edition.sitesWithRobots",
   "display": "54730",
   "value": 54730,
   "kind": "count"
  },
  {
   "key": "edition.botsTracked",
   "display": "21",
   "value": 21,
   "kind": "count"
  },
  {
   "key": "edition.operatorsTracked",
   "display": "12",
   "value": 12,
   "kind": "count"
  },
  {
   "key": "agg.anyAIBlock.count",
   "display": "10573",
   "value": 10573,
   "kind": "count"
  },
  {
   "key": "agg.anyAIBlock.pct",
   "display": "19.3%",
   "value": 19.3,
   "kind": "rate"
  },
  {
   "key": "agg.starBlock.count",
   "display": "2175",
   "value": 2175,
   "kind": "count"
  },
  {
   "key": "agg.starBlock.pct",
   "display": "4%",
   "value": 4,
   "kind": "rate"
  },
  {
   "key": "agg.llmsTxt.count",
   "display": "18510",
   "value": 18510,
   "kind": "count"
  },
  {
   "key": "agg.llmsTxt.pct",
   "display": "24.2%",
   "value": 24.2,
   "kind": "rate"
  },
  {
   "key": "bot.GPTBot.blockCount",
   "display": "8507",
   "value": 8507,
   "kind": "count"
  },
  {
   "key": "bot.GPTBot.blockPct",
   "display": "15.5%",
   "value": 15.5,
   "kind": "rate"
  },
  {
   "key": "bot.ClaudeBot.blockCount",
   "display": "7541",
   "value": 7541,
   "kind": "count"
  },
  {
   "key": "bot.ClaudeBot.blockPct",
   "display": "13.8%",
   "value": 13.8,
   "kind": "rate"
  },
  {
   "key": "bot.Google-Extended.blockCount",
   "display": "6976",
   "value": 6976,
   "kind": "count"
  },
  {
   "key": "bot.Google-Extended.blockPct",
   "display": "12.7%",
   "value": 12.7,
   "kind": "rate"
  },
  {
   "key": "bot.CCBot.blockCount",
   "display": "8281",
   "value": 8281,
   "kind": "count"
  },
  {
   "key": "bot.CCBot.blockPct",
   "display": "15.1%",
   "value": 15.1,
   "kind": "rate"
  },
  {
   "key": "bot.PerplexityBot.blockCount",
   "display": "2387",
   "value": 2387,
   "kind": "count"
  },
  {
   "key": "bot.PerplexityBot.blockPct",
   "display": "4.4%",
   "value": 4.4,
   "kind": "rate"
  },
  {
   "key": "bot.Bytespider.blockCount",
   "display": "7802",
   "value": 7802,
   "kind": "count"
  },
  {
   "key": "bot.Bytespider.blockPct",
   "display": "14.3%",
   "value": 14.3,
   "kind": "rate"
  },
  {
   "key": "bot.Meta-ExternalAgent.blockCount",
   "display": "6643",
   "value": 6643,
   "kind": "count"
  },
  {
   "key": "bot.Meta-ExternalAgent.blockPct",
   "display": "12.1%",
   "value": 12.1,
   "kind": "rate"
  },
  {
   "key": "bot.Amazonbot.blockCount",
   "display": "6843",
   "value": 6843,
   "kind": "count"
  },
  {
   "key": "bot.Amazonbot.blockPct",
   "display": "12.5%",
   "value": 12.5,
   "kind": "rate"
  },
  {
   "key": "bot.Applebot-Extended.blockCount",
   "display": "6533",
   "value": 6533,
   "kind": "count"
  },
  {
   "key": "bot.Applebot-Extended.blockPct",
   "display": "11.9%",
   "value": 11.9,
   "kind": "rate"
  },
  {
   "key": "operator.OpenAI.blockCount",
   "display": "8601",
   "value": 8601,
   "kind": "count"
  },
  {
   "key": "operator.OpenAI.blockPct",
   "display": "15.7%",
   "value": 15.7,
   "kind": "rate"
  },
  {
   "key": "operator.Anthropic.blockCount",
   "display": "7888",
   "value": 7888,
   "kind": "count"
  },
  {
   "key": "operator.Anthropic.blockPct",
   "display": "14.4%",
   "value": 14.4,
   "kind": "rate"
  },
  {
   "key": "operator.Google.blockCount",
   "display": "6976",
   "value": 6976,
   "kind": "count"
  },
  {
   "key": "operator.Google.blockPct",
   "display": "12.7%",
   "value": 12.7,
   "kind": "rate"
  },
  {
   "key": "operator.Common Crawl.blockCount",
   "display": "8281",
   "value": 8281,
   "kind": "count"
  },
  {
   "key": "operator.Common Crawl.blockPct",
   "display": "15.1%",
   "value": 15.1,
   "kind": "rate"
  },
  {
   "key": "operator.Perplexity.blockCount",
   "display": "2400",
   "value": 2400,
   "kind": "count"
  },
  {
   "key": "operator.Perplexity.blockPct",
   "display": "4.4%",
   "value": 4.4,
   "kind": "rate"
  },
  {
   "key": "operator.ByteDance.blockCount",
   "display": "7802",
   "value": 7802,
   "kind": "count"
  },
  {
   "key": "operator.ByteDance.blockPct",
   "display": "14.3%",
   "value": 14.3,
   "kind": "rate"
  },
  {
   "key": "operator.Meta.blockCount",
   "display": "7196",
   "value": 7196,
   "kind": "count"
  },
  {
   "key": "operator.Meta.blockPct",
   "display": "13.1%",
   "value": 13.1,
   "kind": "rate"
  },
  {
   "key": "operator.Amazon.blockCount",
   "display": "6843",
   "value": 6843,
   "kind": "count"
  },
  {
   "key": "operator.Amazon.blockPct",
   "display": "12.5%",
   "value": 12.5,
   "kind": "rate"
  },
  {
   "key": "operator.Apple.blockCount",
   "display": "6533",
   "value": 6533,
   "kind": "count"
  },
  {
   "key": "operator.Apple.blockPct",
   "display": "11.9%",
   "value": 11.9,
   "kind": "rate"
  },
  {
   "key": "operator.Cohere.blockCount",
   "display": "2315",
   "value": 2315,
   "kind": "count"
  },
  {
   "key": "operator.Cohere.blockPct",
   "display": "4.2%",
   "value": 4.2,
   "kind": "rate"
  },
  {
   "key": "operator.Mistral.blockCount",
   "display": "609",
   "value": 609,
   "kind": "count"
  },
  {
   "key": "operator.Mistral.blockPct",
   "display": "1.1%",
   "value": 1.1,
   "kind": "rate"
  },
  {
   "key": "operator.Diffbot.blockCount",
   "display": "2152",
   "value": 2152,
   "kind": "count"
  },
  {
   "key": "operator.Diffbot.blockPct",
   "display": "3.9%",
   "value": 3.9,
   "kind": "rate"
  }
 ]
}