{
 "blogId": "D0417",
 "slug": "ai-crawler-blocking-trend-august-2026",
 "title": "AI-Crawler Blocking Climbed to 19.4% on Top Sites",
 "template": "R17T_CLOSING_WEB_TREND",
 "editionId": "2026-08",
 "snapshotSha256": "36ebdaf6b5adbd5670f896f476348e374d244f20fdde7cc2518d4969ac480e97",
 "sealedAt": "2026-08-07",
 "source": "US Tech Automations Research",
 "methodology": "https://ustechautomations.com/resources/research",
 "canonicalUrl": "https://ustechautomations.com/resources/blog/ai-crawler-blocking-trend-august-2026",
 "displays": [
  {
   "key": "edition.window",
   "display": "June 9, 2026 – August 7, 2026",
   "value": null,
   "kind": "window"
  },
  {
   "key": "edition.fromDate",
   "display": "June 9, 2026",
   "value": null,
   "kind": "window"
  },
  {
   "key": "edition.toDate",
   "display": "August 7, 2026",
   "value": null,
   "kind": "window"
  },
  {
   "key": "edition.spanDays",
   "display": "59",
   "value": 59,
   "kind": "count"
  },
  {
   "key": "edition.editionsCompared",
   "display": "6",
   "value": 6,
   "kind": "count"
  },
  {
   "key": "edition.universe",
   "display": "Tranco top 100,000",
   "value": null,
   "kind": "window"
  },
  {
   "key": "edition.fromSitesWithRobots",
   "display": "54904",
   "value": 54904,
   "kind": "count"
  },
  {
   "key": "edition.toSitesWithRobots",
   "display": "54927",
   "value": 54927,
   "kind": "count"
  },
  {
   "key": "trend.agg.anyAIBlock.from",
   "display": "18.5%",
   "value": 18.5,
   "kind": "rate"
  },
  {
   "key": "trend.agg.anyAIBlock.to",
   "display": "19.4%",
   "value": 19.4,
   "kind": "rate"
  },
  {
   "key": "trend.agg.anyAIBlock.deltaPp",
   "display": "+0.9",
   "value": 0.9,
   "kind": "delta"
  },
  {
   "key": "trend.agg.starBlock.from",
   "display": "4.2%",
   "value": 4.2,
   "kind": "rate"
  },
  {
   "key": "trend.agg.starBlock.to",
   "display": "4%",
   "value": 4,
   "kind": "rate"
  },
  {
   "key": "trend.agg.starBlock.deltaPp",
   "display": "-0.2",
   "value": -0.2,
   "kind": "delta"
  },
  {
   "key": "trend.agg.llmsTxt.from",
   "display": "22.6%",
   "value": 22.6,
   "kind": "rate"
  },
  {
   "key": "trend.agg.llmsTxt.to",
   "display": "24.5%",
   "value": 24.5,
   "kind": "rate"
  },
  {
   "key": "trend.agg.llmsTxt.deltaPp",
   "display": "+1.9",
   "value": 1.9,
   "kind": "delta"
  },
  {
   "key": "trend.bot.GPTBot.from",
   "display": "14.9%",
   "value": 14.9,
   "kind": "rate"
  },
  {
   "key": "trend.bot.GPTBot.to",
   "display": "15.6%",
   "value": 15.6,
   "kind": "rate"
  },
  {
   "key": "trend.bot.GPTBot.deltaPp",
   "display": "+0.7",
   "value": 0.7,
   "kind": "delta"
  },
  {
   "key": "trend.bot.ClaudeBot.from",
   "display": "12.9%",
   "value": 12.9,
   "kind": "rate"
  },
  {
   "key": "trend.bot.ClaudeBot.to",
   "display": "13.8%",
   "value": 13.8,
   "kind": "rate"
  },
  {
   "key": "trend.bot.ClaudeBot.deltaPp",
   "display": "+0.9",
   "value": 0.9,
   "kind": "delta"
  },
  {
   "key": "trend.bot.Google-Extended.from",
   "display": "12%",
   "value": 12,
   "kind": "rate"
  },
  {
   "key": "trend.bot.Google-Extended.to",
   "display": "12.9%",
   "value": 12.9,
   "kind": "rate"
  },
  {
   "key": "trend.bot.Google-Extended.deltaPp",
   "display": "+0.9",
   "value": 0.9,
   "kind": "delta"
  },
  {
   "key": "trend.bot.CCBot.from",
   "display": "14.4%",
   "value": 14.4,
   "kind": "rate"
  },
  {
   "key": "trend.bot.CCBot.to",
   "display": "15.2%",
   "value": 15.2,
   "kind": "rate"
  },
  {
   "key": "trend.bot.CCBot.deltaPp",
   "display": "+0.8",
   "value": 0.8,
   "kind": "delta"
  },
  {
   "key": "trend.bot.PerplexityBot.from",
   "display": "4.4%",
   "value": 4.4,
   "kind": "rate"
  },
  {
   "key": "trend.bot.PerplexityBot.to",
   "display": "4.3%",
   "value": 4.3,
   "kind": "rate"
  },
  {
   "key": "trend.bot.PerplexityBot.deltaPp",
   "display": "-0.1",
   "value": -0.1,
   "kind": "delta"
  },
  {
   "key": "trend.bot.Bytespider.from",
   "display": "13.2%",
   "value": 13.2,
   "kind": "rate"
  },
  {
   "key": "trend.bot.Bytespider.to",
   "display": "14.4%",
   "value": 14.4,
   "kind": "rate"
  },
  {
   "key": "trend.bot.Bytespider.deltaPp",
   "display": "+1.2",
   "value": 1.2,
   "kind": "delta"
  },
  {
   "key": "trend.bot.Meta-ExternalAgent.from",
   "display": "11.1%",
   "value": 11.1,
   "kind": "rate"
  },
  {
   "key": "trend.bot.Meta-ExternalAgent.to",
   "display": "12.3%",
   "value": 12.3,
   "kind": "rate"
  },
  {
   "key": "trend.bot.Meta-ExternalAgent.deltaPp",
   "display": "+1.2",
   "value": 1.2,
   "kind": "delta"
  },
  {
   "key": "trend.bot.Amazonbot.from",
   "display": "11.6%",
   "value": 11.6,
   "kind": "rate"
  },
  {
   "key": "trend.bot.Amazonbot.to",
   "display": "12.6%",
   "value": 12.6,
   "kind": "rate"
  },
  {
   "key": "trend.bot.Amazonbot.deltaPp",
   "display": "+1",
   "value": 1,
   "kind": "delta"
  },
  {
   "key": "trend.bot.Applebot-Extended.from",
   "display": "11%",
   "value": 11,
   "kind": "rate"
  },
  {
   "key": "trend.bot.Applebot-Extended.to",
   "display": "12.1%",
   "value": 12.1,
   "kind": "rate"
  },
  {
   "key": "trend.bot.Applebot-Extended.deltaPp",
   "display": "+1.1",
   "value": 1.1,
   "kind": "delta"
  },
  {
   "key": "trend.operator.OpenAI.from",
   "display": "15%",
   "value": 15,
   "kind": "rate"
  },
  {
   "key": "trend.operator.OpenAI.to",
   "display": "15.8%",
   "value": 15.8,
   "kind": "rate"
  },
  {
   "key": "trend.operator.OpenAI.deltaPp",
   "display": "+0.8",
   "value": 0.8,
   "kind": "delta"
  },
  {
   "key": "trend.operator.Anthropic.from",
   "display": "13.6%",
   "value": 13.6,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Anthropic.to",
   "display": "14.5%",
   "value": 14.5,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Anthropic.deltaPp",
   "display": "+0.9",
   "value": 0.9,
   "kind": "delta"
  },
  {
   "key": "trend.operator.Google.from",
   "display": "12%",
   "value": 12,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Google.to",
   "display": "12.9%",
   "value": 12.9,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Google.deltaPp",
   "display": "+0.9",
   "value": 0.9,
   "kind": "delta"
  },
  {
   "key": "trend.operator.Common Crawl.from",
   "display": "14.4%",
   "value": 14.4,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Common Crawl.to",
   "display": "15.2%",
   "value": 15.2,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Common Crawl.deltaPp",
   "display": "+0.8",
   "value": 0.8,
   "kind": "delta"
  },
  {
   "key": "trend.operator.Perplexity.from",
   "display": "4.4%",
   "value": 4.4,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Perplexity.to",
   "display": "4.3%",
   "value": 4.3,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Perplexity.deltaPp",
   "display": "-0.1",
   "value": -0.1,
   "kind": "delta"
  },
  {
   "key": "trend.operator.ByteDance.from",
   "display": "13.2%",
   "value": 13.2,
   "kind": "rate"
  },
  {
   "key": "trend.operator.ByteDance.to",
   "display": "14.4%",
   "value": 14.4,
   "kind": "rate"
  },
  {
   "key": "trend.operator.ByteDance.deltaPp",
   "display": "+1.2",
   "value": 1.2,
   "kind": "delta"
  },
  {
   "key": "trend.operator.Meta.from",
   "display": "12.1%",
   "value": 12.1,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Meta.to",
   "display": "13.3%",
   "value": 13.3,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Meta.deltaPp",
   "display": "+1.2",
   "value": 1.2,
   "kind": "delta"
  },
  {
   "key": "trend.operator.Amazon.from",
   "display": "11.6%",
   "value": 11.6,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Amazon.to",
   "display": "12.6%",
   "value": 12.6,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Amazon.deltaPp",
   "display": "+1",
   "value": 1,
   "kind": "delta"
  },
  {
   "key": "trend.operator.Apple.from",
   "display": "11%",
   "value": 11,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Apple.to",
   "display": "12.1%",
   "value": 12.1,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Apple.deltaPp",
   "display": "+1.1",
   "value": 1.1,
   "kind": "delta"
  },
  {
   "key": "trend.operator.Cohere.from",
   "display": "4.1%",
   "value": 4.1,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Cohere.to",
   "display": "4.2%",
   "value": 4.2,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Cohere.deltaPp",
   "display": "+0.1",
   "value": 0.1,
   "kind": "delta"
  },
  {
   "key": "trend.operator.Mistral.from",
   "display": "1%",
   "value": 1,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Mistral.to",
   "display": "1.1%",
   "value": 1.1,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Mistral.deltaPp",
   "display": "+0.1",
   "value": 0.1,
   "kind": "delta"
  },
  {
   "key": "trend.operator.Diffbot.from",
   "display": "3.7%",
   "value": 3.7,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Diffbot.to",
   "display": "3.9%",
   "value": 3.9,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Diffbot.deltaPp",
   "display": "+0.2",
   "value": 0.2,
   "kind": "delta"
  },
  {
   "key": "trend.blocking.crawlersTracked",
   "display": "21",
   "value": 21,
   "kind": "count"
  },
  {
   "key": "trend.blocking.crawlersBlockedMore",
   "display": "19",
   "value": 19,
   "kind": "count"
  },
  {
   "key": "trend.blocking.crawlersBlockedLess",
   "display": "2",
   "value": 2,
   "kind": "count"
  },
  {
   "key": "trend.blocking.crawlersUnchanged",
   "display": "0",
   "value": 0,
   "kind": "count"
  },
  {
   "key": "trend.series.2026-06-17.anyAIBlock",
   "display": "18.7%",
   "value": 18.7,
   "kind": "rate"
  },
  {
   "key": "trend.series.2026-06-30.anyAIBlock",
   "display": "18.9%",
   "value": 18.9,
   "kind": "rate"
  },
  {
   "key": "trend.series.2026-07-31.anyAIBlock",
   "display": "19.3%",
   "value": 19.3,
   "kind": "rate"
  },
  {
   "key": "trend.series.2026-08-01.anyAIBlock",
   "display": "19.3%",
   "value": 19.3,
   "kind": "rate"
  }
 ]
}