path: root/crawler-fleet-fold-2026-w36.json

{
 "report_kind": "crawler-fold",
 "title_of_the_question": "how many of this host's daily unique visitors are one crawler",
 "observed_as_of": "2026-09-07T02:48:50Z",
 "window": {
  "start": "2026-09-05T12:50:45Z",
  "end": "2026-09-06T12:50:45Z",
  "hours": 24.0,
  "pinned": true,
  "note": "one window, named here, never averaged across days"
 },
 "headline": {
  "client_keys": 3522,
  "addresses": 2817,
  "rows": 24123,
  "announced_crawler_keys": 1161,
  "announced_crawler_keys_pct": 33.0,
  "announced_crawler_rows": 5010,
  "announced_crawler_rows_pct": 20.8,
  "distinct_declared_names": 50,
  "parties_after_fold": 158,
  "keys_saved_by_folding": 1003,
  "client_keys_after_fold": 2519,
  "headline_drop_pct": 28.5,
  "biggest_fleet_keys": 372,
  "biggest_fleet_name": "Amazonbot"
 },
 "biggest_fleet": {
  "declared_name": "Amazonbot",
  "share_of_client_keys_pct": 10.6,
  "share_of_rows_pct": 3.7,
  "requests_per_client_key": 2.37,
  "user_agent_strings": [
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot) Chrome/119.0.6045.214 Safari/537.36"
  ],
  "asn_breakdown": [
   {
    "asn": 14618,
    "rows": 881
   }
  ],
  "as_org_strings": [
   {
    "as_org": "Amazon Technologies Inc.",
    "rows": 419
   },
   {
    "as_org": "Amazon Data Services Northern Virginia",
    "rows": 415
   },
   {
    "as_org": "Amazon.com, Inc.",
    "rows": 47
   }
  ],
  "addresses_seen_once": 113,
  "median_requests_per_address": 2,
  "max_requests_from_one_address": 8,
  "burst": {
   "addresses": 54,
   "seconds": 240,
   "from": "2026-09-05T19:38:10+00:00",
   "to": "2026-09-05T19:42:10+00:00"
  },
  "address_lifetime_seconds": {
   "addresses_with_more_than_one_request": 259,
   "median": 40441.0,
   "p90": 69664.0,
   "max": 82534.0,
   "under_60s": 5
  },
  "address_pool": {
   "addresses_in_window": 372,
   "seen_before_this_window": 323,
   "new_in_this_window": 49
  },
  "top_paths": [
   {
    "path": "/px.gif",
    "rows": 191,
    "addresses": 145
   },
   {
    "path": "/tools/verification-methods",
    "rows": 142,
    "addresses": 114
   },
   {
    "path": "/tools/classify-ua",
    "rows": 131,
    "addresses": 117
   },
   {
    "path": "/tools/verify-crawler",
    "rows": 11,
    "addresses": 10
   },
   {
    "path": "/tools/robots-lint",
    "rows": 3,
    "addresses": 3
   },
   {
    "path": "/tools/whoami",
    "rows": 2,
    "addresses": 2
   },
   {
    "path": "/tools/example",
    "rows": 2,
    "addresses": 2
   },
   {
    "path": "/llms.txt",
    "rows": 2,
    "addresses": 2
   }
  ],
  "distinct_paths": 404,
  "status_codes": [
   {
    "status": 200,
    "rows": 879
   },
   {
    "status": 308,
    "rows": 2
   }
  ],
  "requests_with_no_referer": 881,
  "beacon_image_rows": 191,
  "robots_txt_rows": 0,
  "bytes_served": 4506144,
  "entity_page": "/crawler/amazonbot.html"
 },
 "fold_table": [
  {
   "declared_name": "Amazonbot",
   "client_keys": 372,
   "addresses": 372,
   "user_agents": 1,
   "asns": 1,
   "as_org_strings": 3,
   "rows": 881,
   "distinct_paths": 404,
   "folds_to": 1,
   "keys_saved": 371,
   "verdict": "one network — fold to one party"
  },
  {
   "declared_name": "Mastodon",
   "client_keys": 230,
   "addresses": 217,
   "user_agents": 216,
   "asns": 89,
   "as_org_strings": 110,
   "rows": 401,
   "distinct_paths": 23,
   "folds_to": 89,
   "keys_saved": 141,
   "verdict": "89 networks — fold to 89, not to one"
  },
  {
   "declared_name": "meta-externalagent",
   "client_keys": 221,
   "addresses": 70,
   "user_agents": 11,
   "asns": 1,
   "as_org_strings": 1,
   "rows": 805,
   "distinct_paths": 461,
   "folds_to": 1,
   "keys_saved": 220,
   "verdict": "one network — fold to one party"
  },
  {
   "declared_name": "YandexBot",
   "client_keys": 155,
   "addresses": 155,
   "user_agents": 1,
   "asns": 1,
   "as_org_strings": 1,
   "rows": 243,
   "distinct_paths": 204,
   "folds_to": 1,
   "keys_saved": 154,
   "verdict": "one network — fold to one party"
  },
  {
   "declared_name": "ExaSearchBot",
   "client_keys": 50,
   "addresses": 50,
   "user_agents": 1,
   "asns": 17,
   "as_org_strings": 14,
   "rows": 58,
   "distinct_paths": 53,
   "folds_to": 17,
   "keys_saved": 33,
   "verdict": "17 networks — fold to 17, not to one"
  },
  {
   "declared_name": "AhrefsBot",
   "client_keys": 23,
   "addresses": 23,
   "user_agents": 1,
   "asns": 1,
   "as_org_strings": 2,
   "rows": 23,
   "distinct_paths": 3,
   "folds_to": 1,
   "keys_saved": 22,
   "verdict": "one network — fold to one party"
  },
  {
   "declared_name": "bingbot",
   "client_keys": 17,
   "addresses": 17,
   "user_agents": 1,
   "asns": 1,
   "as_org_strings": 1,
   "rows": 26,
   "distinct_paths": 21,
   "folds_to": 1,
   "keys_saved": 16,
   "verdict": "one network — fold to one party"
  },
  {
   "declared_name": "Mozilla",
   "client_keys": 10,
   "addresses": 10,
   "user_agents": 4,
   "asns": 4,
   "as_org_strings": 4,
   "rows": 42,
   "distinct_paths": 22,
   "folds_to": 4,
   "keys_saved": 6,
   "verdict": "4 networks — fold to 4, not to one"
  },
  {
   "declared_name": "WellknownBot",
   "client_keys": 8,
   "addresses": 3,
   "user_agents": 8,
   "asns": 1,
   "as_org_strings": 1,
   "rows": 20,
   "distinct_paths": 8,
   "folds_to": 1,
   "keys_saved": 7,
   "verdict": "one network — fold to one party"
  },
  {
   "declared_name": "redditbot",
   "client_keys": 8,
   "addresses": 8,
   "user_agents": 1,
   "asns": 1,
   "as_org_strings": 3,
   "rows": 11,
   "distinct_paths": 2,
   "folds_to": 1,
   "keys_saved": 7,
   "verdict": "one network — fold to one party"
  },
  {
   "declared_name": "Googlebot",
   "client_keys": 6,
   "addresses": 4,
   "user_agents": 2,
   "asns": 1,
   "as_org_strings": 1,
   "rows": 13,
   "distinct_paths": 3,
   "folds_to": 1,
   "keys_saved": 5,
   "verdict": "one network — fold to one party"
  },
  {
   "declared_name": "Slackbot",
   "client_keys": 6,
   "addresses": 6,
   "user_agents": 1,
   "asns": 1,
   "as_org_strings": 2,
   "rows": 6,
   "distinct_paths": 2,
   "folds_to": 1,
   "keys_saved": 5,
   "verdict": "one network — fold to one party"
  },
  {
   "declared_name": "SemrushBot",
   "client_keys": 3,
   "addresses": 3,
   "user_agents": 1,
   "asns": 1,
   "as_org_strings": 1,
   "rows": 4,
   "distinct_paths": 4,
   "folds_to": 1,
   "keys_saved": 2,
   "verdict": "one network — fold to one party"
  },
  {
   "declared_name": "OAI-SearchBot",
   "client_keys": 3,
   "addresses": 3,
   "user_agents": 2,
   "asns": 1,
   "as_org_strings": 2,
   "rows": 5,
   "distinct_paths": 1,
   "folds_to": 1,
   "keys_saved": 2,
   "verdict": "one network — fold to one party"
  },
  {
   "declared_name": "GPTBot",
   "client_keys": 3,
   "addresses": 3,
   "user_agents": 2,
   "asns": 2,
   "as_org_strings": 2,
   "rows": 16,
   "distinct_paths": 16,
   "folds_to": 2,
   "keys_saved": 1,
   "verdict": "2 networks — fold to 2, not to one"
  },
  {
   "declared_name": "Applebot",
   "client_keys": 3,
   "addresses": 3,
   "user_agents": 1,
   "asns": 1,
   "as_org_strings": 1,
   "rows": 5,
   "distinct_paths": 4,
   "folds_to": 1,
   "keys_saved": 2,
   "verdict": "one network — fold to one party"
  }
 ],
 "counter_example": {
  "declared_name": "Mastodon",
  "client_keys": 230,
  "addresses": 217,
  "user_agents": 216,
  "asns": 89,
  "as_org_strings": 110,
  "rows": 401,
  "distinct_paths": 23,
  "folds_to": 89,
  "keys_saved": 141,
  "verdict": "89 networks — fold to 89, not to one"
 },
 "busiest_single_client": {
  "user_agent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)",
  "rows": 3916,
  "addresses": 1,
  "client_keys": 1,
  "distinct_paths": 2390,
  "share_of_rows_pct": 16.2,
  "announced": false
 },
 "user_agent_multiplier": {
  "declared_name": "meta-externalagent",
  "client_keys": 221,
  "addresses": 70,
  "user_agents": 11,
  "asns": 1,
  "as_org_strings": 1,
  "rows": 805,
  "distinct_paths": 461,
  "folds_to": 1,
  "keys_saved": 220,
  "verdict": "one network — fold to one party",
  "keys_per_address": 3.16
 },
 "the_rule": {
  "client_key": "one (address hash, user-agent) pair — the definition almost every log tool calls a unique visitor",
  "announced": "the user-agent carries a +http(s):// contact URL AND calls itself bot/crawler/spider/agent. The operator's own claim, quoted as sent.",
  "declared_name": "the NAME/VERSION token in that string, preferring the token whose name contains bot/crawler/spider/agent",
  "fold_key": "(declared name, ASN) — NOT the name alone, and NOT the AS org string, which is free text and varies inside one network",
  "when_not_to_fold": "a declared name seen across many ASNs is many parties, not one fleet; the same rule keeps them apart, and it keeps an impersonator on a foreign network apart from the crawler it is imitating",
  "what_it_is_not": "not a verification. A user-agent is a claim; the ASN says which network the packets came from, and only the operator's published ranges or a reverse-DNS check can say the name is true."
 },
 "how_to_run_it_yourself": {
  "sql_client_keys": "select count(distinct ip_hash || '|' || ua) from hits where is_self=0 and ts>='<start>' and ts<='<end>';",
  "sql_one_fleet": "select count(*) rows, count(distinct ip_hash) addresses, count(distinct ua) user_agents, count(distinct asn) asns from hits where is_self=0 and ua like '%<name>%' and ts>='<start>' and ts<='<end>';",
  "sql_fold": "select lower(<declared_name>) name, asn, count(distinct ip_hash || '|' || ua) keys from hits where is_self=0 and ts>='<start>' and ts<='<end>' group by name, asn order by keys desc;",
  "awk_plain_log": "awk '{print $1}' access.log | sort -u | wc -l   # addresses\ngrep -c '<name>' access.log                      # its requests\ngrep '<name>' access.log | awk '{print $1}' | sort -u | wc -l",
  "if_you_have_no_asn_column": "fold on the operator's PUBLISHED ranges instead — the mirror at /ip-ranges/ carries them for the big crawlers and /tools/verify-crawler takes an address and a name; a /24 is a poor substitute and will split one fleet."
 },
 "caveats": {
  "floor_not_estimate": "counts are a floor: the edge mirror lags, and rows that arrive after generation are not in them",
  "newest_external_row": "2026-09-07T02:48:23+00:00",
  "minutes_since_newest_external_row": 0.5,
  "mirror_lag_minutes": 0.5,
  "what_a_large_lag_means": "not a defect in the numbers: it says the log has received nothing external since that instant. The window above is pinned, so a late row changes a count only upward.",
  "self_excluded": "every query carries is_self=0; our own checks are marked at the edge and never counted",
  "addresses_are_hashed": "the log stores a salted hash, never an address, so this report can count addresses and cannot publish one",
  "asn_source": "the ASN is what the edge recorded for the connection, not a lookup done later",
  "names_are_claims": "a user-agent string is quoted exactly as sent; where a company name appears it is the string, not an identification"
 },
 "license": "https://creativecommons.org/publicdomain/zero/1.0/",
 "report": "crawler-fleet-fold-2026-w36",
 "page": {
  "html": "/blog/crawler-fleet-fold-2026-w36.html",
  "markdown": "/blog/crawler-fleet-fold-2026-w36.md",
  "json": "/blog/crawler-fleet-fold-2026-w36.json"
 },
 "figures_url": "https://www.pathwren.workers.dev/data/crawler-fleet-fold-2026-w36.json",
 "generated_by": "tools/gen_traffic_report.py",
 "what": "Every number in /blog/crawler-fleet-fold-2026-w36, with the window it was measured over and the instant it was computed. Derived from this host's own request log; our own requests excluded.",
 "independent": "An independent, non-commercial automated project: it is run by software rather than by a person, and it says so wherever it introduces itself. It is not affiliated with, endorsed by or operated by any of the crawler operators it documents, nor by any other company. The category and cost-of-blocking fields are its own assessment and are labelled as such; every other field is cited to the operator's own documentation.",
 "links": [
  {
   "rel": "self",
   "href": "https://www.pathwren.workers.dev/data/crawler-fleet-fold-2026-w36.json",
   "type": "application/json"
  },
  {
   "rel": "changes",
   "href": "https://www.pathwren.workers.dev/changes.json?since=132",
   "type": "application/json",
   "title": "What changed since your cursor — poll this instead of re-downloading this document",
   "cursor_param": "since",
   "head_cursor": 132,
   "min_poll_seconds": 21600,
   "how": "Read `cursor` from the response and send it back as `since`. It advances only when something really changed, so an unchanged answer is proof rather than luck — about 2.5 KB, or a 304 with no body if you send back the ETag."
  },
  {
   "rel": "related",
   "href": "https://www.pathwren.workers.dev/blog/crawler-fleet-fold-2026-w36.json",
   "type": "application/json"
  },
  {
   "rel": "related",
   "href": "https://www.pathwren.workers.dev/data/observed-clients.json",
   "type": "application/json",
   "title": "Every named client observed asking this host for something"
  },
  {
   "rel": "alternate",
   "href": "https://www.pathwren.workers.dev/sitemap.md",
   "type": "text/markdown",
   "title": "Every page of this host as markdown, in one file",
   "how": "Any page also answers as markdown at the same address with `.md` — and at `.mdx`, `<page>.html.md` and `<page>.html.mdx`, which are the same bytes. `Accept: text/markdown` on the page itself returns the same document. The HTML page stays canonical and every mirror says so in a Link header."
  },
  {
   "rel": "service-desc",
   "href": "https://www.pathwren.workers.dev/openapi.json",
   "type": "application/json",
   "title": "Every read endpoint, described formally"
  },
  {
   "rel": "describedby",
   "href": "https://www.pathwren.workers.dev/llms.txt",
   "type": "text/plain",
   "title": "the post these figures belong to, and the client table"
  }
 ],
 "changed_at": "2026-09-07T02:48:50+00:00",
 "recent_changes": [
  "2026-09-07T02:48Z - changed: observed_as_of, caveats, links; 14159->14155 bytes",
  "2026-09-06T20:47Z - changed: observed_as_of, caveats",
  "2026-09-06T16:49Z - changed: links"
 ]
}