{
 "blogId": "D0389",
 "slug": "closing-web-index-july-2026",
 "title": "19.1% of Top Sites Block AI Crawlers: USTA Index",
 "template": "R17_CLOSING_WEB_INDEX",
 "editionId": "2026-07",
 "snapshotSha256": "c53b5e53a70a3fd1c11f7c824efbdfe244a846681fc61db377c8d833c6b6325d",
 "sealedAt": "2026-07-13",
 "source": "US Tech Automations Research",
 "methodology": "https://ustechautomations.com/resources/research",
 "canonicalUrl": "https://ustechautomations.com/resources/blog/closing-web-index-july-2026",
 "displays": [
  {
   "key": "edition.snapshotDate",
   "display": "July 13, 2026",
   "value": null,
   "kind": "window"
  },
  {
   "key": "edition.universe",
   "display": "Tranco top 100,000",
   "value": null,
   "kind": "window"
  },
  {
   "key": "edition.sitesChecked",
   "display": "100000",
   "value": 100000,
   "kind": "count"
  },
  {
   "key": "edition.sitesWithRobots",
   "display": "54771",
   "value": 54771,
   "kind": "count"
  },
  {
   "key": "edition.botsTracked",
   "display": "21",
   "value": 21,
   "kind": "count"
  },
  {
   "key": "edition.operatorsTracked",
   "display": "12",
   "value": 12,
   "kind": "count"
  },
  {
   "key": "agg.anyAIBlock.count",
   "display": "10458",
   "value": 10458,
   "kind": "count"
  },
  {
   "key": "agg.anyAIBlock.pct",
   "display": "19.1%",
   "value": 19.1,
   "kind": "rate"
  },
  {
   "key": "agg.starBlock.count",
   "display": "2120",
   "value": 2120,
   "kind": "count"
  },
  {
   "key": "agg.starBlock.pct",
   "display": "3.9%",
   "value": 3.9,
   "kind": "rate"
  },
  {
   "key": "agg.llmsTxt.count",
   "display": "18223",
   "value": 18223,
   "kind": "count"
  },
  {
   "key": "agg.llmsTxt.pct",
   "display": "23.6%",
   "value": 23.6,
   "kind": "rate"
  },
  {
   "key": "bot.GPTBot.blockCount",
   "display": "8432",
   "value": 8432,
   "kind": "count"
  },
  {
   "key": "bot.GPTBot.blockPct",
   "display": "15.4%",
   "value": 15.4,
   "kind": "rate"
  },
  {
   "key": "bot.ClaudeBot.blockCount",
   "display": "7442",
   "value": 7442,
   "kind": "count"
  },
  {
   "key": "bot.ClaudeBot.blockPct",
   "display": "13.6%",
   "value": 13.6,
   "kind": "rate"
  },
  {
   "key": "bot.Google-Extended.blockCount",
   "display": "6890",
   "value": 6890,
   "kind": "count"
  },
  {
   "key": "bot.Google-Extended.blockPct",
   "display": "12.6%",
   "value": 12.6,
   "kind": "rate"
  },
  {
   "key": "bot.CCBot.blockCount",
   "display": "8185",
   "value": 8185,
   "kind": "count"
  },
  {
   "key": "bot.CCBot.blockPct",
   "display": "14.9%",
   "value": 14.9,
   "kind": "rate"
  },
  {
   "key": "bot.PerplexityBot.blockCount",
   "display": "2423",
   "value": 2423,
   "kind": "count"
  },
  {
   "key": "bot.PerplexityBot.blockPct",
   "display": "4.4%",
   "value": 4.4,
   "kind": "rate"
  },
  {
   "key": "bot.Bytespider.blockCount",
   "display": "7617",
   "value": 7617,
   "kind": "count"
  },
  {
   "key": "bot.Bytespider.blockPct",
   "display": "13.9%",
   "value": 13.9,
   "kind": "rate"
  },
  {
   "key": "bot.Meta-ExternalAgent.blockCount",
   "display": "6462",
   "value": 6462,
   "kind": "count"
  },
  {
   "key": "bot.Meta-ExternalAgent.blockPct",
   "display": "11.8%",
   "value": 11.8,
   "kind": "rate"
  },
  {
   "key": "bot.Amazonbot.blockCount",
   "display": "6681",
   "value": 6681,
   "kind": "count"
  },
  {
   "key": "bot.Amazonbot.blockPct",
   "display": "12.2%",
   "value": 12.2,
   "kind": "rate"
  },
  {
   "key": "bot.Applebot-Extended.blockCount",
   "display": "6370",
   "value": 6370,
   "kind": "count"
  },
  {
   "key": "bot.Applebot-Extended.blockPct",
   "display": "11.6%",
   "value": 11.6,
   "kind": "rate"
  },
  {
   "key": "operator.OpenAI.blockCount",
   "display": "8525",
   "value": 8525,
   "kind": "count"
  },
  {
   "key": "operator.OpenAI.blockPct",
   "display": "15.6%",
   "value": 15.6,
   "kind": "rate"
  },
  {
   "key": "operator.Anthropic.blockCount",
   "display": "7800",
   "value": 7800,
   "kind": "count"
  },
  {
   "key": "operator.Anthropic.blockPct",
   "display": "14.2%",
   "value": 14.2,
   "kind": "rate"
  },
  {
   "key": "operator.Google.blockCount",
   "display": "6890",
   "value": 6890,
   "kind": "count"
  },
  {
   "key": "operator.Google.blockPct",
   "display": "12.6%",
   "value": 12.6,
   "kind": "rate"
  },
  {
   "key": "operator.Common Crawl.blockCount",
   "display": "8185",
   "value": 8185,
   "kind": "count"
  },
  {
   "key": "operator.Common Crawl.blockPct",
   "display": "14.9%",
   "value": 14.9,
   "kind": "rate"
  },
  {
   "key": "operator.Perplexity.blockCount",
   "display": "2435",
   "value": 2435,
   "kind": "count"
  },
  {
   "key": "operator.Perplexity.blockPct",
   "display": "4.4%",
   "value": 4.4,
   "kind": "rate"
  },
  {
   "key": "operator.ByteDance.blockCount",
   "display": "7617",
   "value": 7617,
   "kind": "count"
  },
  {
   "key": "operator.ByteDance.blockPct",
   "display": "13.9%",
   "value": 13.9,
   "kind": "rate"
  },
  {
   "key": "operator.Meta.blockCount",
   "display": "7044",
   "value": 7044,
   "kind": "count"
  },
  {
   "key": "operator.Meta.blockPct",
   "display": "12.9%",
   "value": 12.9,
   "kind": "rate"
  },
  {
   "key": "operator.Amazon.blockCount",
   "display": "6681",
   "value": 6681,
   "kind": "count"
  },
  {
   "key": "operator.Amazon.blockPct",
   "display": "12.2%",
   "value": 12.2,
   "kind": "rate"
  },
  {
   "key": "operator.Apple.blockCount",
   "display": "6370",
   "value": 6370,
   "kind": "count"
  },
  {
   "key": "operator.Apple.blockPct",
   "display": "11.6%",
   "value": 11.6,
   "kind": "rate"
  },
  {
   "key": "operator.Cohere.blockCount",
   "display": "2301",
   "value": 2301,
   "kind": "count"
  },
  {
   "key": "operator.Cohere.blockPct",
   "display": "4.2%",
   "value": 4.2,
   "kind": "rate"
  },
  {
   "key": "operator.Mistral.blockCount",
   "display": "569",
   "value": 569,
   "kind": "count"
  },
  {
   "key": "operator.Mistral.blockPct",
   "display": "1%",
   "value": 1,
   "kind": "rate"
  },
  {
   "key": "operator.Diffbot.blockCount",
   "display": "2157",
   "value": 2157,
   "kind": "count"
  },
  {
   "key": "operator.Diffbot.blockPct",
   "display": "3.9%",
   "value": 3.9,
   "kind": "rate"
  }
 ]
}