{
 "blogId": "D0377",
 "slug": "ai-crawler-blocking-trend-june-2026",
 "title": "AI-Crawler Blocking Climbed to 18.8% on Top Sites",
 "template": "R17T_CLOSING_WEB_TREND",
 "editionId": "2026-06",
 "snapshotSha256": "ae13dd3671d776567830b6d918c1bc246e622f4a61bb4a52e68f6ad01a20c3ef",
 "sealedAt": "2026-06-26",
 "source": "US Tech Automations Research",
 "methodology": "https://ustechautomations.com/resources/research",
 "canonicalUrl": "https://ustechautomations.com/resources/blog/ai-crawler-blocking-trend-june-2026",
 "displays": [
  {
   "key": "edition.window",
   "display": "June 9, 2026 – June 26, 2026",
   "value": null,
   "kind": "window"
  },
  {
   "key": "edition.fromDate",
   "display": "June 9, 2026",
   "value": null,
   "kind": "window"
  },
  {
   "key": "edition.toDate",
   "display": "June 26, 2026",
   "value": null,
   "kind": "window"
  },
  {
   "key": "edition.spanDays",
   "display": "17",
   "value": 17,
   "kind": "count"
  },
  {
   "key": "edition.editionsCompared",
   "display": "3",
   "value": 3,
   "kind": "count"
  },
  {
   "key": "edition.universe",
   "display": "Tranco top 100,000",
   "value": null,
   "kind": "window"
  },
  {
   "key": "edition.fromSitesWithRobots",
   "display": "54904",
   "value": 54904,
   "kind": "count"
  },
  {
   "key": "edition.toSitesWithRobots",
   "display": "55147",
   "value": 55147,
   "kind": "count"
  },
  {
   "key": "trend.agg.anyAIBlock.from",
   "display": "18.5%",
   "value": 18.5,
   "kind": "rate"
  },
  {
   "key": "trend.agg.anyAIBlock.to",
   "display": "18.8%",
   "value": 18.8,
   "kind": "rate"
  },
  {
   "key": "trend.agg.anyAIBlock.deltaPp",
   "display": "+0.3",
   "value": 0.3,
   "kind": "delta"
  },
  {
   "key": "trend.agg.starBlock.from",
   "display": "4.2%",
   "value": 4.2,
   "kind": "rate"
  },
  {
   "key": "trend.agg.starBlock.to",
   "display": "4.2%",
   "value": 4.2,
   "kind": "rate"
  },
  {
   "key": "trend.agg.starBlock.deltaPp",
   "display": "0",
   "value": 0,
   "kind": "delta"
  },
  {
   "key": "trend.agg.llmsTxt.from",
   "display": "22.6%",
   "value": 22.6,
   "kind": "rate"
  },
  {
   "key": "trend.agg.llmsTxt.to",
   "display": "23.2%",
   "value": 23.2,
   "kind": "rate"
  },
  {
   "key": "trend.agg.llmsTxt.deltaPp",
   "display": "+0.6",
   "value": 0.6,
   "kind": "delta"
  },
  {
   "key": "trend.bot.GPTBot.from",
   "display": "14.9%",
   "value": 14.9,
   "kind": "rate"
  },
  {
   "key": "trend.bot.GPTBot.to",
   "display": "15.2%",
   "value": 15.2,
   "kind": "rate"
  },
  {
   "key": "trend.bot.GPTBot.deltaPp",
   "display": "+0.3",
   "value": 0.3,
   "kind": "delta"
  },
  {
   "key": "trend.bot.ClaudeBot.from",
   "display": "12.9%",
   "value": 12.9,
   "kind": "rate"
  },
  {
   "key": "trend.bot.ClaudeBot.to",
   "display": "13.3%",
   "value": 13.3,
   "kind": "rate"
  },
  {
   "key": "trend.bot.ClaudeBot.deltaPp",
   "display": "+0.4",
   "value": 0.4,
   "kind": "delta"
  },
  {
   "key": "trend.bot.Google-Extended.from",
   "display": "12%",
   "value": 12,
   "kind": "rate"
  },
  {
   "key": "trend.bot.Google-Extended.to",
   "display": "12.4%",
   "value": 12.4,
   "kind": "rate"
  },
  {
   "key": "trend.bot.Google-Extended.deltaPp",
   "display": "+0.4",
   "value": 0.4,
   "kind": "delta"
  },
  {
   "key": "trend.bot.CCBot.from",
   "display": "14.4%",
   "value": 14.4,
   "kind": "rate"
  },
  {
   "key": "trend.bot.CCBot.to",
   "display": "14.7%",
   "value": 14.7,
   "kind": "rate"
  },
  {
   "key": "trend.bot.CCBot.deltaPp",
   "display": "+0.3",
   "value": 0.3,
   "kind": "delta"
  },
  {
   "key": "trend.bot.PerplexityBot.from",
   "display": "4.4%",
   "value": 4.4,
   "kind": "rate"
  },
  {
   "key": "trend.bot.PerplexityBot.to",
   "display": "4.4%",
   "value": 4.4,
   "kind": "rate"
  },
  {
   "key": "trend.bot.PerplexityBot.deltaPp",
   "display": "0",
   "value": 0,
   "kind": "delta"
  },
  {
   "key": "trend.bot.Bytespider.from",
   "display": "13.2%",
   "value": 13.2,
   "kind": "rate"
  },
  {
   "key": "trend.bot.Bytespider.to",
   "display": "13.6%",
   "value": 13.6,
   "kind": "rate"
  },
  {
   "key": "trend.bot.Bytespider.deltaPp",
   "display": "+0.4",
   "value": 0.4,
   "kind": "delta"
  },
  {
   "key": "trend.bot.Meta-ExternalAgent.from",
   "display": "11.1%",
   "value": 11.1,
   "kind": "rate"
  },
  {
   "key": "trend.bot.Meta-ExternalAgent.to",
   "display": "11.5%",
   "value": 11.5,
   "kind": "rate"
  },
  {
   "key": "trend.bot.Meta-ExternalAgent.deltaPp",
   "display": "+0.4",
   "value": 0.4,
   "kind": "delta"
  },
  {
   "key": "trend.bot.Amazonbot.from",
   "display": "11.6%",
   "value": 11.6,
   "kind": "rate"
  },
  {
   "key": "trend.bot.Amazonbot.to",
   "display": "11.9%",
   "value": 11.9,
   "kind": "rate"
  },
  {
   "key": "trend.bot.Amazonbot.deltaPp",
   "display": "+0.3",
   "value": 0.3,
   "kind": "delta"
  },
  {
   "key": "trend.bot.Applebot-Extended.from",
   "display": "11%",
   "value": 11,
   "kind": "rate"
  },
  {
   "key": "trend.bot.Applebot-Extended.to",
   "display": "11.4%",
   "value": 11.4,
   "kind": "rate"
  },
  {
   "key": "trend.bot.Applebot-Extended.deltaPp",
   "display": "+0.4",
   "value": 0.4,
   "kind": "delta"
  },
  {
   "key": "trend.operator.OpenAI.from",
   "display": "15%",
   "value": 15,
   "kind": "rate"
  },
  {
   "key": "trend.operator.OpenAI.to",
   "display": "15.3%",
   "value": 15.3,
   "kind": "rate"
  },
  {
   "key": "trend.operator.OpenAI.deltaPp",
   "display": "+0.3",
   "value": 0.3,
   "kind": "delta"
  },
  {
   "key": "trend.operator.Anthropic.from",
   "display": "13.6%",
   "value": 13.6,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Anthropic.to",
   "display": "14%",
   "value": 14,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Anthropic.deltaPp",
   "display": "+0.4",
   "value": 0.4,
   "kind": "delta"
  },
  {
   "key": "trend.operator.Google.from",
   "display": "12%",
   "value": 12,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Google.to",
   "display": "12.4%",
   "value": 12.4,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Google.deltaPp",
   "display": "+0.4",
   "value": 0.4,
   "kind": "delta"
  },
  {
   "key": "trend.operator.Common Crawl.from",
   "display": "14.4%",
   "value": 14.4,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Common Crawl.to",
   "display": "14.7%",
   "value": 14.7,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Common Crawl.deltaPp",
   "display": "+0.3",
   "value": 0.3,
   "kind": "delta"
  },
  {
   "key": "trend.operator.Perplexity.from",
   "display": "4.4%",
   "value": 4.4,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Perplexity.to",
   "display": "4.4%",
   "value": 4.4,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Perplexity.deltaPp",
   "display": "0",
   "value": 0,
   "kind": "delta"
  },
  {
   "key": "trend.operator.ByteDance.from",
   "display": "13.2%",
   "value": 13.2,
   "kind": "rate"
  },
  {
   "key": "trend.operator.ByteDance.to",
   "display": "13.6%",
   "value": 13.6,
   "kind": "rate"
  },
  {
   "key": "trend.operator.ByteDance.deltaPp",
   "display": "+0.4",
   "value": 0.4,
   "kind": "delta"
  },
  {
   "key": "trend.operator.Meta.from",
   "display": "12.1%",
   "value": 12.1,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Meta.to",
   "display": "12.5%",
   "value": 12.5,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Meta.deltaPp",
   "display": "+0.4",
   "value": 0.4,
   "kind": "delta"
  },
  {
   "key": "trend.operator.Amazon.from",
   "display": "11.6%",
   "value": 11.6,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Amazon.to",
   "display": "11.9%",
   "value": 11.9,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Amazon.deltaPp",
   "display": "+0.3",
   "value": 0.3,
   "kind": "delta"
  },
  {
   "key": "trend.operator.Apple.from",
   "display": "11%",
   "value": 11,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Apple.to",
   "display": "11.4%",
   "value": 11.4,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Apple.deltaPp",
   "display": "+0.4",
   "value": 0.4,
   "kind": "delta"
  },
  {
   "key": "trend.operator.Cohere.from",
   "display": "4.1%",
   "value": 4.1,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Cohere.to",
   "display": "4.1%",
   "value": 4.1,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Cohere.deltaPp",
   "display": "0",
   "value": 0,
   "kind": "delta"
  },
  {
   "key": "trend.operator.Mistral.from",
   "display": "1%",
   "value": 1,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Mistral.to",
   "display": "1%",
   "value": 1,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Mistral.deltaPp",
   "display": "0",
   "value": 0,
   "kind": "delta"
  },
  {
   "key": "trend.operator.Diffbot.from",
   "display": "3.7%",
   "value": 3.7,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Diffbot.to",
   "display": "3.8%",
   "value": 3.8,
   "kind": "rate"
  },
  {
   "key": "trend.operator.Diffbot.deltaPp",
   "display": "+0.1",
   "value": 0.1,
   "kind": "delta"
  },
  {
   "key": "trend.blocking.crawlersTracked",
   "display": "21",
   "value": 21,
   "kind": "count"
  },
  {
   "key": "trend.blocking.crawlersBlockedMore",
   "display": "17",
   "value": 17,
   "kind": "count"
  },
  {
   "key": "trend.blocking.crawlersBlockedLess",
   "display": "0",
   "value": 0,
   "kind": "count"
  },
  {
   "key": "trend.blocking.crawlersUnchanged",
   "display": "4",
   "value": 4,
   "kind": "count"
  },
  {
   "key": "trend.series.2026-06-17.anyAIBlock",
   "display": "18.7%",
   "value": 18.7,
   "kind": "rate"
  }
 ]
}