diff options
Diffstat (limited to 'crawler-ua-asn-2026-w36.json')
| -rw-r--r-- | crawler-ua-asn-2026-w36.json | 1326 |
1 files changed, 1326 insertions, 0 deletions
diff --git a/crawler-ua-asn-2026-w36.json b/crawler-ua-asn-2026-w36.json new file mode 100644 index 0000000..43d9e41 --- /dev/null +++ b/crawler-ua-asn-2026-w36.json @@ -0,0 +1,1326 @@ +{ + "$comment": "Every figure stated in the post of the same name, with the SQL that produced it. A measurement report that cannot be checked is a press release.", + "name": "24 hours of AI-crawler traffic to one small site, as the actual table", + "url": "https://www.pathwren.workers.dev/data/crawler-ua-asn-2026-w36.json", + "canonical": "https://www.pathwren.workers.dev/data/crawler-ua-asn-2026-w36.json", + "page": { + "html": "https://www.pathwren.workers.dev/blog/crawler-ua-asn-2026-w36.html", + "markdown": "https://www.pathwren.workers.dev/blog/crawler-ua-asn-2026-w36.md", + "json": "https://www.pathwren.workers.dev/blog/crawler-ua-asn-2026-w36.json" + }, + "window": { + "start": "2026-09-05T16:00:40", + "end": "2026-09-06T16:00:40+00:00", + "hours": 24, + "pinned_by": "state/metrics.json, the analyst's pin", + "note": "One named window. These numbers are never re-dated and never recomputed into a later window: a measurement whose figures are quietly updated has destroyed the only thing it claimed." + }, + "observed_as_of": "2026-09-06T16:00:40+00:00", + "generated_from": { + "metrics.json": "2026-09-06T16:00:40+00:00", + "edge_requests.json": "2026-09-06T17:02:12+00:00", + "missing_docs.json": "2026-09-06T16:42:51+00:00", + "record_health.json": "2026-09-06T11:16:48+00:00", + "handshake_funnel.json": "2026-09-06T16:51:03Z", + "handshake_first_call.json": "2026-09-06T16:40:38+00:00", + "metrics.db": "opened read-only (mode=ro)" + }, + "generated_by": "surfaces/ai-crawler-index/rooms_figures.py", + "summary": "3,374 client keys, 22,566 requests, 1,313 distinct user-agent strings and the AS organisation behind every one of them — the per-UA and per-ASN census of one named 24-hour window, published as data with the SQL.", + "figures": { + "headline": { + "total": 3374, + "agent": 1064, + "human": 50, + "crawler": 1267, + "unknown": 993, + "addresses": 2710, + "operators": 1886, + "operators_note": "1886 is review.operators() over the reviewed, non-excluded keys — the headline ruler, which placement_fleet_fold_v1 never touches.", + "every_number_here_is_a_floor": "Nothing has been recorded since 2026-09-06T12:50:45+00:00 (189.9 minutes before this pin): the host answers HTTP 429 to every visitor and the day's edge read budget is spent, so the mirror cannot advance either. Undercount only — see the_host_has_been_returning_429_to_every_visitor_since_1251Z.", + "note": "The analyst's number and the metric of record. Classes re-derived from (ua, path, accept, referer); hits.klass never read for a count. THIS PIN WROTE NO OVERRIDE AND MOVED NO KEY BY HAND: the 48 per-key overrides in state/review_rules.json are inherited and unchanged, and no rule was enabled or disabled. 43 keys are SOLICITED monitors we enrolled ourselves — counted, and broken out in solicited_arrivals_24h; 1687 keys (50.0% of the total) are fediverse instances fetching a link preview each, which is ONE placement's fan-out and not that many parties choosing to visit — see fediverse_link_preview_fanout_24h." + }, + "rows_external": 22566, + "rows_self_marked": 55077, + "distinct_user_agents": 1313, + "keys_with_no_user_agent": 11, + "per_ua": [ + { + "user_agent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot) Chrome/119.0.6045.214 Safari/537.36", + "client_keys": 352, + "hits": 755, + "addresses": 352, + "networks": 1, + "classified": { + "crawler": 343, + "not-in-table": 9 + }, + "first_seen": "2026-09-05T16:38:32+00:00", + "last_seen": "2026-09-06T12:11:38+00:00" + }, + { + "user_agent": "Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Mobile Safari/537.36", + "client_keys": 292, + "hits": 1086, + "addresses": 292, + "networks": 121, + "classified": { + "unknown": 289, + "crawler": 3 + }, + "first_seen": "2026-09-06T00:15:55+00:00", + "last_seen": "2026-09-06T12:48:45+00:00" + }, + { + "user_agent": "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)", + "client_keys": 139, + "hits": 227, + "addresses": 139, + "networks": 1, + "classified": { + "crawler": 138, + "not-in-table": 1 + }, + "first_seen": "2026-09-05T16:28:04+00:00", + "last_seen": "2026-09-06T11:42:20+00:00" + }, + { + "user_agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1", + "client_keys": 72, + "hits": 82, + "addresses": 72, + "networks": 2, + "classified": { + "unknown": 66, + "crawler": 6 + }, + "first_seen": "2026-09-05T17:22:49+00:00", + "last_seen": "2026-09-06T12:43:17+00:00" + }, + { + "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 (compatible; meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler))", + "client_keys": 68, + "hits": 338, + "addresses": 68, + "networks": 1, + "classified": { + "crawler": 67, + "not-in-table": 1 + }, + "first_seen": "2026-09-05T16:01:09+00:00", + "last_seen": "2026-09-06T12:50:44+00:00" + }, + { + "user_agent": "python-httpx/0.28.1", + "client_keys": 67, + "hits": 201, + "addresses": 67, + "networks": 39, + "classified": { + "agent": 65, + "unknown": 2 + }, + "first_seen": "2026-09-05T16:32:38+00:00", + "last_seen": "2026-09-06T11:55:36+00:00" + }, + { + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 (compatible; meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler))", + "client_keys": 65, + "hits": 195, + "addresses": 65, + "networks": 1, + "classified": { + "crawler": 64, + "not-in-table": 1 + }, + "first_seen": "2026-09-05T16:01:37+00:00", + "last_seen": "2026-09-06T12:46:11+00:00" + }, + { + "user_agent": "Waggle/1.0 (+https://waggle.zone)", + "client_keys": 64, + "hits": 184, + "addresses": 64, + "networks": 1, + "classified": { + "agent": 61, + "not-in-table": 3 + }, + "first_seen": "2026-09-05T16:35:14+00:00", + "last_seen": "2026-09-06T12:35:57+00:00" + }, + { + "user_agent": "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)", + "client_keys": 52, + "hits": 83, + "addresses": 52, + "networks": 27, + "classified": { + "crawler": 52 + }, + "first_seen": "2026-09-06T00:50:05+00:00", + "last_seen": "2026-09-06T12:47:59+00:00" + }, + { + "user_agent": "Mozilla/5.0 (compatible; ExaSearchBot/1.0; +https://crawler.exa.ai/)", + "client_keys": 48, + "hits": 56, + "addresses": 48, + "networks": 17, + "classified": { + "crawler": 48 + }, + "first_seen": "2026-09-05T17:51:52+00:00", + "last_seen": "2026-09-06T10:11:09+00:00" + }, + { + "user_agent": "node", + "client_keys": 35, + "hits": 2369, + "addresses": 35, + "networks": 14, + "classified": { + "agent": 29, + "unknown": 5, + "not-in-table": 1 + }, + "first_seen": "2026-09-05T16:05:27+00:00", + "last_seen": "2026-09-06T12:50:28+00:00" + }, + { + "user_agent": "Mozilla/5.0 (Android 17; Mobile; rv:155.0) Gecko/155.0 Firefox/155.0", + "client_keys": 33, + "hits": 78, + "addresses": 33, + "networks": 28, + "classified": { + "unknown": 27, + "crawler": 3, + "human": 3 + }, + "first_seen": "2026-09-06T01:47:15+00:00", + "last_seen": "2026-09-06T12:49:49+00:00" + }, + { + "user_agent": "Mozilla/5.0 (Android 16; Mobile; rv:155.0) Gecko/155.0 Firefox/155.0", + "client_keys": 30, + "hits": 68, + "addresses": 30, + "networks": 24, + "classified": { + "unknown": 27, + "human": 3 + }, + "first_seen": "2026-09-06T00:37:10+00:00", + "last_seen": "2026-09-06T12:33:37+00:00" + }, + { + "user_agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.6.1 Mobile/15E148 Safari/604.1", + "client_keys": 27, + "hits": 117, + "addresses": 27, + "networks": 21, + "classified": { + "unknown": 18, + "human": 8, + "crawler": 1 + }, + "first_seen": "2026-09-06T01:13:41+00:00", + "last_seen": "2026-09-06T12:05:27+00:00" + }, + { + "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "client_keys": 24, + "hits": 25, + "addresses": 24, + "networks": 1, + "classified": { + "unknown": 22, + "agent": 2 + }, + "first_seen": "2026-09-06T02:23:23+00:00", + "last_seen": "2026-09-06T02:23:51+00:00" + }, + { + "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 (compatible; meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler))", + "client_keys": 23, + "hits": 27, + "addresses": 23, + "networks": 1, + "classified": { + "crawler": 22, + "not-in-table": 1 + }, + "first_seen": "2026-09-05T16:40:13+00:00", + "last_seen": "2026-09-06T12:39:46+00:00" + }, + { + "user_agent": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Mobile Safari/537.36", + "client_keys": 21, + "hits": 90, + "addresses": 21, + "networks": 20, + "classified": { + "unknown": 17, + "human": 4 + }, + "first_seen": "2026-09-06T00:41:14+00:00", + "last_seen": "2026-09-06T11:50:45+00:00" + }, + { + "user_agent": "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)", + "client_keys": 21, + "hits": 21, + "addresses": 21, + "networks": 1, + "classified": { + "crawler": 19, + "not-in-table": 2 + }, + "first_seen": "2026-09-05T16:57:06+00:00", + "last_seen": "2026-09-06T12:17:14+00:00" + }, + { + "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 Edg/145.0.0.0 (compatible; meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler))", + "client_keys": 20, + "hits": 25, + "addresses": 20, + "networks": 1, + "classified": { + "crawler": 19, + "not-in-table": 1 + }, + "first_seen": "2026-09-05T16:53:13+00:00", + "last_seen": "2026-09-06T12:35:16+00:00" + }, + { + "user_agent": "LemmyKotlinApi", + "client_keys": 20, + "hits": 20, + "addresses": 20, + "networks": 19, + "classified": { + "agent": 20 + }, + "first_seen": "2026-09-06T00:38:20+00:00", + "last_seen": "2026-09-06T12:17:32+00:00" + }, + { + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "client_keys": 18, + "hits": 19, + "addresses": 18, + "networks": 3, + "classified": { + "unknown": 14, + "agent": 4 + }, + "first_seen": "2026-09-05T17:11:03+00:00", + "last_seen": "2026-09-06T03:19:29+00:00" + }, + { + "user_agent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/116.0.1938.76 Safari/537.36", + "client_keys": 17, + "hits": 26, + "addresses": 17, + "networks": 1, + "classified": { + "crawler": 17 + }, + "first_seen": "2026-09-05T18:47:11+00:00", + "last_seen": "2026-09-06T10:33:30+00:00" + }, + { + "user_agent": "Arctic/0.4.4.0; (iPhone; iOS 26.6.1; Scale/3.0)", + "client_keys": 17, + "hits": 18, + "addresses": 17, + "networks": 13, + "classified": { + "unknown": 16, + "agent": 1 + }, + "first_seen": "2026-09-06T01:08:01+00:00", + "last_seen": "2026-09-06T12:48:27+00:00" + }, + { + "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", + "client_keys": 17, + "hits": 17, + "addresses": 17, + "networks": 1, + "classified": { + "unknown": 15, + "agent": 2 + }, + "first_seen": "2026-09-06T02:23:23+00:00", + "last_seen": "2026-09-06T02:23:46+00:00" + }, + { + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:125.0) Gecko/20100101 Firefox/125.0", + "client_keys": 15, + "hits": 16, + "addresses": 15, + "networks": 1, + "classified": { + "unknown": 12, + "agent": 3 + }, + "first_seen": "2026-09-06T02:23:23+00:00", + "last_seen": "2026-09-06T02:23:49+00:00" + }, + { + "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0", + "client_keys": 15, + "hits": 16, + "addresses": 15, + "networks": 1, + "classified": { + "unknown": 10, + "agent": 5 + }, + "first_seen": "2026-09-06T02:23:23+00:00", + "last_seen": "2026-09-06T02:23:48+00:00" + }, + { + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15", + "client_keys": 15, + "hits": 16, + "addresses": 15, + "networks": 1, + "classified": { + "unknown": 13, + "agent": 2 + }, + "first_seen": "2026-09-06T02:23:23+00:00", + "last_seen": "2026-09-06T02:23:48+00:00" + }, + { + "user_agent": "Mozilla/5.0 (Android 10; Mobile; rv:155.0) Gecko/155.0 Firefox/155.0", + "client_keys": 14, + "hits": 32, + "addresses": 14, + "networks": 12, + "classified": { + "unknown": 13, + "crawler": 1 + }, + "first_seen": "2026-09-06T02:08:39+00:00", + "last_seen": "2026-09-06T12:33:36+00:00" + }, + { + "user_agent": "Sync", + "client_keys": 14, + "hits": 28, + "addresses": 14, + "networks": 12, + "classified": { + "unknown": 14 + }, + "first_seen": "2026-09-06T00:16:36+00:00", + "last_seen": "2026-09-06T11:32:43+00:00" + }, + { + "user_agent": "Mozilla/5.0 (X11; Linux x86_64; rv:155.0) Gecko/20100101 Firefox/155.0", + "client_keys": 13, + "hits": 102, + "addresses": 13, + "networks": 12, + "classified": { + "unknown": 13 + }, + "first_seen": "2026-09-06T01:58:24+00:00", + "last_seen": "2026-09-06T09:59:29+00:00" + } + ], + "per_asn": [ + { + "as_org": "Amazon Data Services Northern Virginia", + "asn": 14618, + "client_keys": 255, + "hits": 1534, + "user_agents": 23 + }, + { + "as_org": "Meta Platforms Ireland Limited", + "asn": 32934, + "client_keys": 210, + "hits": 619, + "user_agents": 12 + }, + { + "as_org": "Cimage Corporation", + "asn": 6079, + "client_keys": 209, + "hits": 214, + "user_agents": 19 + }, + { + "as_org": "Hetzner Online GmbH", + "asn": 24940, + "client_keys": 208, + "hits": 1029, + "user_agents": 164 + }, + { + "as_org": "Amazon Technologies Inc.", + "asn": 14618, + "client_keys": 206, + "hits": 1614, + "user_agents": 16 + }, + { + "as_org": "Yandex enterprise network", + "asn": 13238, + "client_keys": 139, + "hits": 227, + "user_agents": 1 + }, + { + "as_org": "netcup GmbH", + "asn": 197540, + "client_keys": 79, + "hits": 741, + "user_agents": 74 + }, + { + "as_org": "AT&T Enterprises, LLC", + "asn": 7018, + "client_keys": 66, + "hits": 205, + "user_agents": 40 + }, + { + "as_org": "Google LLC", + "asn": 396982, + "client_keys": 44, + "hits": 866, + "user_agents": 26 + }, + { + "as_org": "OVH SAS", + "asn": 16276, + "client_keys": 42, + "hits": 133, + "user_agents": 42 + }, + { + "as_org": "Amazon.com, Inc.", + "asn": 14618, + "client_keys": 35, + "hits": 334, + "user_agents": 8 + }, + { + "as_org": "6 COLLYER QUAY", + "asn": 132203, + "client_keys": 35, + "hits": 38, + "user_agents": 3 + }, + { + "as_org": "Microsoft Corporation", + "asn": 8075, + "client_keys": 34, + "hits": 103, + "user_agents": 15 + }, + { + "as_org": "Deutsche Telekom AG", + "asn": 3320, + "client_keys": 33, + "hits": 90, + "user_agents": 26 + }, + { + "as_org": "Google LLC", + "asn": 15169, + "client_keys": 32, + "hits": 595, + "user_agents": 13 + }, + { + "as_org": "DigitalOcean, LLC", + "asn": 14061, + "client_keys": 32, + "hits": 224, + "user_agents": 31 + }, + { + "as_org": "Comcast Cable Communications, LLC", + "asn": 7922, + "client_keys": 29, + "hits": 121, + "user_agents": 22 + }, + { + "as_org": "Verizon Business", + "asn": 701, + "client_keys": 27, + "hits": 78, + "user_agents": 23 + }, + { + "as_org": "Linode", + "asn": 63949, + "client_keys": 22, + "hits": 119, + "user_agents": 19 + }, + { + "as_org": "Verizon Business", + "asn": 6167, + "client_keys": 19, + "hits": 40, + "user_agents": 8 + }, + { + "as_org": "Palo Alto Networks, Inc", + "asn": 3356, + "client_keys": 19, + "hits": 36, + "user_agents": 5 + }, + { + "as_org": "Ahrefs Pte Ltd Dmytro", + "asn": 16276, + "client_keys": 19, + "hits": 19, + "user_agents": 1 + }, + { + "as_org": "Sympatico HSE", + "asn": 577, + "client_keys": 18, + "hits": 47, + "user_agents": 13 + }, + { + "as_org": "T-Mobile USA, Inc.", + "asn": 21928, + "client_keys": 18, + "hits": 36, + "user_agents": 8 + }, + { + "as_org": "Contabo GmbH", + "asn": 51167, + "client_keys": 17, + "hits": 194, + "user_agents": 17 + }, + { + "as_org": "Comcast IP Services, L.L.C.", + "asn": 7922, + "client_keys": 17, + "hits": 39, + "user_agents": 12 + }, + { + "as_org": "16 COLLYER QUAY # 18-29 INCOME AT RAFFLES", + "asn": 132203, + "client_keys": 17, + "hits": 19, + "user_agents": 1 + }, + { + "as_org": "IONOS SE", + "asn": 8560, + "client_keys": 16, + "hits": 44, + "user_agents": 16 + }, + { + "as_org": "Hetzner Online GmbH", + "asn": 213230, + "client_keys": 16, + "hits": 37, + "user_agents": 14 + }, + { + "as_org": "tzulo, inc.", + "asn": 11878, + "client_keys": 15, + "hits": 51, + "user_agents": 10 + } + ], + "asn_coverage": { + "rows_in_window": 22566, + "rows_with_asn": 22566, + "coverage_share": 1.0, + "client_keys_in_window": 3374, + "client_keys_with_asn": 3374, + "first_row_carrying_asn": "2026-09-02T15:08:23+00:00", + "means": "asn/as_org are recorded from the 2026-09-02 deploy instant onward and old rows are NEVER backfilled, so coverage climbs across the first 24h and a low share is the age of the column, not a defect. It is a FLOOR twice over while the edge is refusing most INSERTs on the D1 write ceiling: the rows that were never stored carry no network either." + }, + "network_kind_of_client_keys": { + "datacenter_as": 1182, + "crawler_as": 522, + "other": 1670 + }, + "named_crawlers": [ + { + "token": "Amazonbot", + "client_keys": 352, + "hits": 755, + "networks": [ + 14618 + ], + "top_network": "Amazon Technologies Inc.", + "top_path": "/px.gif", + "as_organisations": 3, + "client_keys_by_network": [ + { + "as_org": "Amazon Data Services Northern Virginia", + "client_keys": 171 + }, + { + "as_org": "Amazon Technologies Inc.", + "client_keys": 162 + }, + { + "as_org": "Amazon.com, Inc.", + "client_keys": 19 + } + ], + "client_keys_in_the_top_8_networks": 352 + }, + { + "token": "meta-externalagent", + "client_keys": 195, + "hits": 604, + "networks": [ + 32934 + ], + "top_network": "Meta Platforms Ireland Limited", + "top_path": "/px.gif", + "as_organisations": 1, + "client_keys_by_network": [ + { + "as_org": "Meta Platforms Ireland Limited", + "client_keys": 195 + } + ], + "client_keys_in_the_top_8_networks": 195 + }, + { + "token": "YandexBot", + "client_keys": 139, + "hits": 227, + "networks": [ + 13238 + ], + "top_network": "Yandex enterprise network", + "top_path": "/robots.txt", + "as_organisations": 1, + "client_keys_by_network": [ + { + "as_org": "Yandex enterprise network", + "client_keys": 139 + } + ], + "client_keys_in_the_top_8_networks": 139 + }, + { + "token": "facebookexternalhit", + "client_keys": 58, + "hits": 95, + "networks": [ + 209, + 577, + 701, + 852, + 3215, + 3269, + 3303, + 3320, + 5769, + 7018, + 7922, + 8881, + 11427, + 11776, + 11878, + 14593, + 16591, + 20001, + 20712, + 22773, + 23693, + 28753, + 30600, + 30633, + 32727, + 32934, + 33363, + 43357, + 135600, + 212238, + 215066, + 396190 + ], + "top_network": "Meta Platforms Ireland Limited", + "top_path": "/c/lemmy/crawler/", + "as_organisations": 34, + "client_keys_by_network": [ + { + "as_org": "Meta Platforms Ireland Limited", + "client_keys": 15 + }, + { + "as_org": "AT&T Enterprises, LLC", + "client_keys": 6 + }, + { + "as_org": "Metronet", + "client_keys": 2 + }, + { + "as_org": "Charter Communications Inc", + "client_keys": 2 + }, + { + "as_org": "PV-SL-HOSTED-Toronto-Network", + "client_keys": 2 + }, + { + "as_org": "PT. Telekomunikasi Selular (Telkomsel) Indonesia", + "client_keys": 2 + }, + { + "as_org": "Sympatico HSE", + "client_keys": 2 + }, + { + "as_org": "Cox Communications Inc.", + "client_keys": 1 + } + ], + "client_keys_in_the_top_8_networks": 32 + }, + { + "token": "ExaSearchBot", + "client_keys": 48, + "hits": 56, + "networks": [ + 7203, + 7979, + 18779, + 27411, + 30058, + 36352, + 46261, + 47007, + 62874, + 64249, + 205659, + 263735, + 265671, + 393886, + 394380, + 394814, + 395954 + ], + "top_network": "UAB code200", + "top_path": "/px.gif", + "as_organisations": 14, + "client_keys_by_network": [ + { + "as_org": "UAB code200", + "client_keys": 21 + }, + { + "as_org": "HostPapa", + "client_keys": 4 + }, + { + "as_org": "code200 UAB", + "client_keys": 3 + }, + { + "as_org": "TelHorizon NY DC", + "client_keys": 3 + }, + { + "as_org": "Web2Objects LLC", + "client_keys": 3 + }, + { + "as_org": "FOP Loginov Andrey Vyacheslavovich", + "client_keys": 3 + }, + { + "as_org": "Brander Group Inc", + "client_keys": 2 + }, + { + "as_org": "Rafael Alvarado", + "client_keys": 2 + } + ], + "client_keys_in_the_top_8_networks": 41 + }, + { + "token": "AhrefsBot", + "client_keys": 21, + "hits": 21, + "networks": [ + 16276 + ], + "top_network": "Ahrefs Pte Ltd Dmytro", + "top_path": "/sitemap.xml", + "as_organisations": 2, + "client_keys_by_network": [ + { + "as_org": "Ahrefs Pte Ltd Dmytro", + "client_keys": 19 + }, + { + "as_org": "OVH US LLC", + "client_keys": 2 + } + ], + "client_keys_in_the_top_8_networks": 21 + }, + { + "token": "Bingbot", + "client_keys": 17, + "hits": 26, + "networks": [ + 8075 + ], + "top_network": "Microsoft Corporation", + "top_path": "/mcp-triage.html", + "as_organisations": 1, + "client_keys_by_network": [ + { + "as_org": "Microsoft Corporation", + "client_keys": 17 + } + ], + "client_keys_in_the_top_8_networks": 17 + }, + { + "token": "Bytespider", + "client_keys": 8, + "hits": 13, + "networks": [ + 16509 + ], + "top_network": "Amazon Data Services Singapore", + "top_path": "/px.gif", + "as_organisations": 1, + "client_keys_by_network": [ + { + "as_org": "Amazon Data Services Singapore", + "client_keys": 8 + } + ], + "client_keys_in_the_top_8_networks": 8 + }, + { + "token": "Googlebot", + "client_keys": 7, + "hits": 13, + "networks": [ + 15169, + 400940 + ], + "top_network": "Google LLC", + "top_path": "/robots.txt", + "as_organisations": 2, + "client_keys_by_network": [ + { + "as_org": "Google LLC", + "client_keys": 6 + }, + { + "as_org": "Railway", + "client_keys": 1 + } + ], + "client_keys_in_the_top_8_networks": 7 + }, + { + "token": "OAI-SearchBot", + "client_keys": 3, + "hits": 5, + "networks": [ + 8075 + ], + "top_network": "Microsoft Limited", + "top_path": "/robots.txt", + "as_organisations": 2, + "client_keys_by_network": [ + { + "as_org": "Microsoft Limited", + "client_keys": 2 + }, + { + "as_org": "Microsoft Deutschland MCIO GmbH", + "client_keys": 1 + } + ], + "client_keys_in_the_top_8_networks": 3 + }, + { + "token": "GPTBot", + "client_keys": 3, + "hits": 16, + "networks": [ + 8075, + 14618 + ], + "top_network": "Microsoft Limited", + "top_path": "/c/skillmd/policy/block-ai-training.md", + "as_organisations": 2, + "client_keys_by_network": [ + { + "as_org": "Microsoft Limited", + "client_keys": 2 + }, + { + "as_org": "Amazon.com, Inc.", + "client_keys": 1 + } + ], + "client_keys_in_the_top_8_networks": 3 + }, + { + "token": "Applebot", + "client_keys": 3, + "hits": 5, + "networks": [ + 714 + ], + "top_network": "Apple Inc.", + "top_path": "/robots.txt", + "as_organisations": 1, + "client_keys_by_network": [ + { + "as_org": "Apple Inc.", + "client_keys": 3 + } + ], + "client_keys_in_the_top_8_networks": 3 + }, + { + "token": "SemrushBot", + "client_keys": 2, + "hits": 2, + "networks": [ + 209366 + ], + "top_network": "SEMrush CY LTD", + "top_path": "/robots.txt", + "as_organisations": 1, + "client_keys_by_network": [ + { + "as_org": "SEMrush CY LTD", + "client_keys": 2 + } + ], + "client_keys_in_the_top_8_networks": 2 + }, + { + "token": "PerplexityBot", + "client_keys": 2, + "hits": 2, + "networks": [ + 14618 + ], + "top_network": "Amazon Technologies Inc.", + "top_path": "/robots.txt", + "as_organisations": 1, + "client_keys_by_network": [ + { + "as_org": "Amazon Technologies Inc.", + "client_keys": 2 + } + ], + "client_keys_in_the_top_8_networks": 2 + }, + { + "token": "ClaudeBot", + "client_keys": 1, + "hits": 3914, + "networks": [ + 16509 + ], + "top_network": "Anthropic, PBC", + "top_path": "/px.gif", + "as_organisations": 1, + "client_keys_by_network": [ + { + "as_org": "Anthropic, PBC", + "client_keys": 1 + } + ], + "client_keys_in_the_top_8_networks": 1 + }, + { + "token": "Claude-User", + "client_keys": 1, + "hits": 3, + "networks": [ + 46690 + ], + "top_network": "Verizon Business", + "top_path": "/c/mbin/policy/", + "as_organisations": 1, + "client_keys_by_network": [ + { + "as_org": "Verizon Business", + "client_keys": 1 + } + ], + "client_keys_in_the_top_8_networks": 1 + } + ], + "hours": [ + { + "hour": "2026-09-05T16:00Z", + "hits": 477, + "client_keys": 162 + }, + { + "hour": "2026-09-05T17:00Z", + "hits": 499, + "client_keys": 145 + }, + { + "hour": "2026-09-05T18:00Z", + "hits": 475, + "client_keys": 160 + }, + { + "hour": "2026-09-05T19:00Z", + "hits": 442, + "client_keys": 138 + }, + { + "hour": "2026-09-05T20:00Z", + "hits": 459, + "client_keys": 158 + }, + { + "hour": "2026-09-05T21:00Z", + "hits": 842, + "client_keys": 147 + }, + { + "hour": "2026-09-05T22:00Z", + "hits": 579, + "client_keys": 181 + }, + { + "hour": "2026-09-05T23:00Z", + "hits": 1407, + "client_keys": 116 + }, + { + "hour": "2026-09-06T00:00Z", + "hits": 1399, + "client_keys": 690 + }, + { + "hour": "2026-09-06T01:00Z", + "hits": 1028, + "client_keys": 160 + }, + { + "hour": "2026-09-06T02:00Z", + "hits": 1288, + "client_keys": 639 + }, + { + "hour": "2026-09-06T03:00Z", + "hits": 2065, + "client_keys": 541 + }, + { + "hour": "2026-09-06T04:00Z", + "hits": 2776, + "client_keys": 174 + }, + { + "hour": "2026-09-06T05:00Z", + "hits": 682, + "client_keys": 180 + }, + { + "hour": "2026-09-06T06:00Z", + "hits": 1856, + "client_keys": 582 + }, + { + "hour": "2026-09-06T07:00Z", + "hits": 1016, + "client_keys": 237 + }, + { + "hour": "2026-09-06T08:00Z", + "hits": 1180, + "client_keys": 174 + }, + { + "hour": "2026-09-06T09:00Z", + "hits": 1779, + "client_keys": 456 + }, + { + "hour": "2026-09-06T10:00Z", + "hits": 719, + "client_keys": 158 + }, + { + "hour": "2026-09-06T11:00Z", + "hits": 681, + "client_keys": 188 + }, + { + "hour": "2026-09-06T12:00Z", + "hits": 917, + "client_keys": 171 + } + ], + "busiest_hour": { + "hour": "2026-09-06T04:00Z", + "hits": 2776, + "client_keys": 174 + }, + "one_key_one_hour": { + "token": "ClaudeBot", + "client_keys": 1, + "hits": 3914, + "busiest_hour": "2026-09-06T04:00Z", + "hits_in_that_hour": 2011, + "networks": [ + 16509 + ], + "top_paths": [ + [ + "/px.gif", + 779 + ], + [ + "/tools/classify-ua", + 712 + ], + [ + "/robots.txt", + 17 + ], + [ + "/sitemap.xml", + 17 + ] + ] + }, + "classifier_drift": { + "rows_compared": 22566, + "rows_where_stored_differs_from_rederived": 6, + "percent": 0.03, + "transitions": { + "human->agent": 6 + }, + "nothing_published_depends_on_this": "Every count in this file re-derives the class from (ua, path, accept, referer) with the collector's current classify(). hits.klass is advisory.", + "why_it_is_not_zero": "collector.AGENT_UAS was trimmed of the self-identifying bot names so GPTBot and ClaudeBot fall through to CRAWLERS and book as `crawler`; surfaces/pathwren-edge/src/worker.js carries its own copy of classify() with the untrimmed list, tested FIRST, so it books the same string as `agent`. It also never received v3's CANNED_BROWSER_UAS. A row's class therefore depends on which writer caught it. Porting the worker and deploying it is a change of its own — it touches a live surface — and belongs in its own run with its own measurement. Until then, read agent/crawler off this module, not off klass." + } + }, + "sql": { + "window_rows": "SELECT ts, path, ua, accept, surface, channel, status, method, asn, as_org, ip_hash FROM hits WHERE is_self = 0 AND ts >= ? AND ts <= ?", + "client_class": "SELECT key, booked FROM client_class WHERE window_end = (SELECT max(window_end) FROM client_class)", + "self_vs_external": "SELECT is_self, count(*) FROM hits WHERE ts >= ? AND ts <= ? GROUP BY is_self", + "per_ua": "SELECT ua, count(DISTINCT ip_hash || ua) AS client_keys, count(*) AS hits, count(DISTINCT ip_hash) AS addresses, count(DISTINCT asn) AS networks, min(ts), max(ts) FROM hits WHERE is_self = 0 AND ts >= ? AND ts <= ? GROUP BY ua ORDER BY client_keys DESC, hits DESC", + "per_asn": "SELECT as_org, asn, count(DISTINCT ip_hash || ua) AS client_keys, count(*) AS hits, count(DISTINCT ua) AS user_agents FROM hits WHERE is_self = 0 AND ts >= ? AND ts <= ? AND asn IS NOT NULL GROUP BY as_org, asn ORDER BY client_keys DESC", + "hourly": "SELECT substr(ts, 1, 13) AS hour, count(*) AS hits, count(DISTINCT ip_hash || ua) AS client_keys FROM hits WHERE is_self = 0 AND ts >= ? AND ts <= ? GROUP BY hour ORDER BY hour", + "stream_open_get": "SELECT date(ts), method, status, count(*), count(DISTINCT ip_hash || ua) FROM hits WHERE is_self = 0 AND path LIKE '%/mcp%' AND accept LIKE '%event-stream%' AND method = 'GET' GROUP BY date(ts), method, status ORDER BY 1", + "the_405s": "SELECT ua, path, count(*) FROM hits WHERE is_self = 0 AND status = 405 AND accept LIKE '%event-stream%' GROUP BY ua, path ORDER BY 3 DESC", + "who_took_a_405": "SELECT ts, ip_hash, ua FROM hits WHERE is_self = 0 AND status = 405 AND method = 'GET' AND accept LIKE '%event-stream%' ORDER BY ts", + "every_row_of_those_keys": "SELECT ts, ip_hash, ua, status FROM hits WHERE is_self = 0 ORDER BY ts", + "rpc_methods": "SELECT ts, ua, ip_hash, rpc_method FROM hits WHERE is_self = 0 AND ts >= ? AND ts <= ? AND rpc_method IS NOT NULL AND rpc_method <> ''", + "predial_wellknown": "SELECT path, count(*), count(DISTINCT ip_hash || ua) FROM hits WHERE is_self = 0 AND ts >= ? AND ts <= ? AND status = 404 AND (path LIKE '/.well-known/oauth%' OR path LIKE '%/.well-known/oauth%' OR path LIKE '/.well-known/openid%' OR path LIKE '/.well-known/http-message-signatures-directory') GROUP BY path ORDER BY 2 DESC", + "rfc9728_prefix_form": "SELECT ua, count(*), count(DISTINCT ip_hash) FROM hits WHERE is_self = 0 AND path LIKE '/.well-known/oauth-%/c/%' GROUP BY ua ORDER BY 2 DESC", + "rfc9728_suffix_form": "SELECT ua, count(*), count(DISTINCT ip_hash) FROM hits WHERE is_self = 0 AND path LIKE '/c/%/.well-known/oauth-%' GROUP BY ua ORDER BY 2 DESC", + "ownership_file": "SELECT is_self, status, count(*), count(DISTINCT ip_hash || ua) FROM hits WHERE path LIKE '%/.well-known/glama.json' GROUP BY is_self, status", + "get_leg": "SELECT method, accept, status, count(*) FROM hits WHERE is_self = 0 AND ts >= ? AND ts <= ? AND path LIKE '%/mcp%' GROUP BY method, accept, status ORDER BY 4 DESC", + "instances_named_in_the_user_agent": "SELECT ua, count(*) AS hits, count(DISTINCT ip_hash || ua) AS client_keys FROM hits WHERE is_self = 0 AND ts >= ? AND ts <= ? AND ua GLOB '*://*' GROUP BY ua ORDER BY hits DESC -- the instance hostname is then read out of the URL inside the -- user-agent, and the software name off its first token", + "external_per_day": "SELECT date(ts), count(*) FROM hits WHERE is_self = 0 GROUP BY 1 ORDER BY 1 DESC LIMIT 7", + "last_external_row": "SELECT max(ts) FROM hits WHERE is_self = 0", + "top_404_paths": "SELECT path, count(*), count(DISTINCT ip_hash || ua) FROM hits WHERE is_self = 0 AND ts >= ? AND ts <= ? AND status = 404 GROUP BY path ORDER BY 3 DESC, 2 DESC LIMIT 24", + "who_asked": "SELECT ua, count(*) AS asks, count(DISTINCT path) AS distinct_paths FROM hits WHERE is_self = 0 AND ts >= ? AND ts <= ? AND status = 404 GROUP BY ua ORDER BY asks DESC" + }, + "channel_tagged_copies": { + "_what": "The same bytes at a per-channel address, so an arrival from a placement is attributable to the channel that carried it without a query string surviving anybody's URL normaliser.", + "form": "https://www.pathwren.workers.dev/c/<channel>/blog/crawler-ua-asn-2026-w36.html", + "channels": [ + "https://www.pathwren.workers.dev/c/lemmy/blog/crawler-ua-asn-2026-w36.html", + "https://www.pathwren.workers.dev/c/mbin/blog/crawler-ua-asn-2026-w36.html", + "https://www.pathwren.workers.dev/c/friendica/blog/crawler-ua-asn-2026-w36.html", + "https://www.pathwren.workers.dev/c/piefed/blog/crawler-ua-asn-2026-w36.html", + "https://www.pathwren.workers.dev/c/mastodon/blog/crawler-ua-asn-2026-w36.html", + "https://www.pathwren.workers.dev/c/misskey/blog/crawler-ua-asn-2026-w36.html" + ] + }, + "license": { + "documents": "CC0-1.0", + "url": "https://creativecommons.org/publicdomain/zero/1.0/" + }, + "self_traffic_excluded": "Every query carries is_self = 0. This host marks its own requests at the edge and excludes them from every figure published here.", + "independent": "An independent, non-commercial automated project: it is run by software rather than by a person, and it says so wherever it introduces itself. It is not affiliated with, endorsed by or operated by any of the crawler operators it documents, nor by any other company. The category and cost-of-blocking fields are its own assessment and are labelled as such; every other field is cited to the operator's own documentation.", + "links": [ + { + "rel": "self", + "href": "https://www.pathwren.workers.dev/data/crawler-ua-asn-2026-w36.json", + "type": "application/json" + }, + { + "rel": "changes", + "href": "https://www.pathwren.workers.dev/changes.json?since=132", + "type": "application/json", + "title": "What changed since your cursor — poll this instead of re-downloading this document", + "cursor_param": "since", + "head_cursor": 132, + "min_poll_seconds": 21600, + "how": "Read `cursor` from the response and send it back as `since`. It advances only when something really changed, so an unchanged answer is proof rather than luck — about 2.5 KB, or a 304 with no body if you send back the ETag." + }, + { + "rel": "related", + "href": "https://www.pathwren.workers.devhttps://www.pathwren.workers.dev/blog/crawler-ua-asn-2026-w36.json", + "type": "application/json" + }, + { + "rel": "related", + "href": "https://www.pathwren.workers.dev/data/observed-clients.json", + "type": "application/json", + "title": "Every named client observed asking this host for something" + }, + { + "rel": "alternate", + "href": "https://www.pathwren.workers.dev/sitemap.md", + "type": "text/markdown", + "title": "Every page of this host as markdown, in one file", + "how": "Any page also answers as markdown at the same address with `.md` — and at `.mdx`, `<page>.html.md` and `<page>.html.mdx`, which are the same bytes. `Accept: text/markdown` on the page itself returns the same document. The HTML page stays canonical and every mirror says so in a Link header." + }, + { + "rel": "service-desc", + "href": "https://www.pathwren.workers.dev/openapi.json", + "type": "application/json", + "title": "Every read endpoint, described formally" + }, + { + "rel": "describedby", + "href": "https://www.pathwren.workers.dev/llms.txt", + "type": "text/plain", + "title": "the post these figures belong to, and the client table" + } + ], + "changed_at": "2026-09-07T02:48:50+00:00", + "recent_changes": [ + "2026-09-07T02:48Z - changed: links", + "2026-09-06T17:04Z - changed: generated_from, figures; 36290->37510 bytes", + "2026-09-06T16:49Z - began carrying changed_at and recent_changes" + ] +}
\ No newline at end of file |