{
  "family": "crawler",
  "generated": "2026-08-30",
  "note": "Real rows out of dated copies we sealed ourselves. Nothing here is made up.",
  "where_these_rows_came_from": "Where these rows came from, and anything their publisher requires to be printed alongside them, is set out on the page this file came from: https://ustechautomations.com/feeds/crawler",
  "rows_published": 12,
  "columns": 7,
  "headers": [
    "domain",
    "crawler",
    "said_before",
    "says_now",
    "what_moved",
    "first_read",
    "second_read"
  ],
  "rows": [
    [
      "dive.games",
      "Amazonbot",
      "blocked from the whole site",
      "named, with 1 allow line and no block",
      "whole-site block taken off",
      "2026-08-23",
      "2026-08-24"
    ],
    [
      "bild.de",
      "Amazonbot",
      "not named in the file",
      "blocked from the whole site",
      "whole-site block added",
      "2026-08-24",
      "2026-08-25"
    ],
    [
      "4life.com",
      "Bytespider",
      "not named in the file",
      "blocked from the whole site",
      "whole-site block added",
      "2026-08-23",
      "2026-08-24"
    ],
    [
      "imdb.com",
      "DuckAssistBot",
      "not named in the file",
      "blocked from 20 folders",
      "crawler named for the first time",
      "2026-08-24",
      "2026-08-25"
    ],
    [
      "delicious.com",
      "anthropic-ai",
      "named, with 1 allow line and no block",
      "blocked from 23 folders",
      "rules under the crawler rewritten",
      "2026-08-24",
      "2026-08-25"
    ],
    [
      "e-estekhdam.com",
      "ClaudeBot",
      "blocked from the whole site",
      "blocked from 10 folders",
      "whole-site block taken off",
      "2026-08-23",
      "2026-08-24"
    ],
    [
      "nytimes.com",
      "FirecrawlAgent",
      "not named in the file",
      "blocked from the whole site",
      "whole-site block added",
      "2026-08-24",
      "2026-08-25"
    ],
    [
      "butosklep.pl",
      "AI2Bot",
      "blocked from the whole site",
      "not named in the file",
      "crawler name dropped from the file",
      "2026-08-23",
      "2026-08-24"
    ],
    [
      "businessweekly.com.tw",
      "Amazonbot",
      "not named in the file",
      "blocked from the whole site",
      "whole-site block added",
      "2026-08-23",
      "2026-08-24"
    ],
    [
      "figma.com",
      "CHATGPT-User",
      "blocked from the whole site, with 8785 exceptions",
      "blocked from the whole site, with 8788 exceptions",
      "rules under the crawler rewritten",
      "2026-08-28",
      "2026-08-29"
    ],
    [
      "ocula.com",
      "AmazonBot",
      "blocked from the whole site",
      "blocked from 1 folder",
      "whole-site block taken off",
      "2026-08-23",
      "2026-08-24"
    ],
    [
      "anaplan.com",
      "Amazonbot",
      "not named in the file",
      "blocked from the whole site",
      "whole-site block added",
      "2026-08-23",
      "2026-08-24"
    ]
  ]
}