{
  "family": "crawler",
  "generated": "2026-10-14",
  "note": "Real rows out of dated copies we sealed ourselves. Nothing here is made up.",
  "where_these_rows_came_from": "Where these rows came from, and anything their publisher requires to be printed alongside them, is set out on the page this file came from: https://ustechautomations.com/feeds/crawler",
  "rows_published": 25,
  "columns": 7,
  "headers": [
    "domain",
    "crawler",
    "said_before",
    "says_now",
    "what_moved",
    "first_read",
    "second_read"
  ],
  "rows": [
    [
      "awardsplatform.com",
      "PetalBot",
      "blocked from the whole site",
      "named, with 1 allow line and no block",
      "whole-site block taken off",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "trivago.com",
      "ClaudeBot",
      "not named in the file",
      "blocked from the whole site, with 13 exceptions",
      "whole-site block added",
      "2026-10-09",
      "2026-10-10"
    ],
    [
      "travelandtourworld.com",
      "OAI-SearchBot",
      "named, with 1 allow line and no block",
      "not named in the file",
      "crawler name dropped from the file",
      "2026-10-09",
      "2026-10-10"
    ],
    [
      "itc.ua",
      "Amazonbot",
      "not named in the file",
      "blocked from 29 folders",
      "crawler named for the first time",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "bbc.com",
      "OAI-SearchBot",
      "blocked from the whole site",
      "blocked from the whole site, with 2 exceptions",
      "rules under the crawler rewritten",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "zoominfo.com",
      "Applebot-Extended",
      "blocked from the whole site",
      "blocked from 55 folders",
      "whole-site block taken off",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "afi.com",
      "Amazonbot",
      "not named in the file",
      "blocked from the whole site",
      "whole-site block added",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "confluent.io",
      "ClaudeBot",
      "named, with 1 allow line and no block",
      "not named in the file",
      "crawler name dropped from the file",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "larvalabs.com",
      "ChatGPT-User",
      "not named in the file",
      "blocked from 3 folders",
      "crawler named for the first time",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "figma.com",
      "CHATGPT-User",
      "blocked from the whole site, with 8938 exceptions",
      "blocked from the whole site, with 8940 exceptions",
      "rules under the crawler rewritten",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "apm.org.uk",
      "Bytespider",
      "not named in the file",
      "blocked from the whole site",
      "whole-site block added",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "cumhuriyet.com.tr",
      "ClaudeBot",
      "blocked from the whole site",
      "not named in the file",
      "crawler name dropped from the file",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "line6.com",
      "ChatGPT-User",
      "not named in the file",
      "blocked from 9 folders",
      "crawler named for the first time",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "fstoppers.com",
      "Amazonbot",
      "blocked from 4 folders",
      "blocked from 5 folders",
      "rules under the crawler rewritten",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "ausa.org",
      "Amazonbot",
      "not named in the file",
      "blocked from the whole site",
      "whole-site block added",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "euronews.com",
      "Google-Extended",
      "blocked from the whole site",
      "not named in the file",
      "crawler name dropped from the file",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "marc.info",
      "ChatGPT-User",
      "not named in the file",
      "blocked from 2 folders",
      "crawler named for the first time",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "glean.com",
      "Amazonbot",
      "named, with 1 allow line and no block",
      "blocked from 8 folders",
      "rules under the crawler rewritten",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "emol.com",
      "Amazonbot",
      "not named in the file",
      "blocked from the whole site",
      "whole-site block added",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "euronews.net",
      "Google-Extended",
      "blocked from the whole site",
      "not named in the file",
      "crawler name dropped from the file",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "port.ac.uk",
      "Amazonbot",
      "not named in the file",
      "blocked from 31 folders",
      "crawler named for the first time",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "ign.com",
      "ChatGPT-User",
      "blocked from the whole site, with 130 exceptions",
      "blocked from the whole site, with 131 exceptions",
      "rules under the crawler rewritten",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "fark.com",
      "Claude-SearchBot",
      "not named in the file",
      "blocked from the whole site, with 3 exceptions",
      "whole-site block added",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "join.chat",
      "Amazonbot",
      "blocked from the whole site",
      "not named in the file",
      "crawler name dropped from the file",
      "2026-10-08",
      "2026-10-09"
    ],
    [
      "securelist.com",
      "Applebot-Extended",
      "not named in the file",
      "named, with 1 allow line and no block",
      "crawler named for the first time",
      "2026-10-08",
      "2026-10-09"
    ]
  ]
}