From 6ce86768d4b2287c23754536e33fcf262578a0a5 Mon Sep 17 00:00:00 2001 From: hrbrmstr Date: Mon, 7 Sep 2026 10:09:24 -0400 Subject: chore: initial commit --- traffic-2026-w36.json | 518 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 518 insertions(+) create mode 100644 traffic-2026-w36.json (limited to 'traffic-2026-w36.json') diff --git a/traffic-2026-w36.json b/traffic-2026-w36.json new file mode 100644 index 0000000..fe2723b --- /dev/null +++ b/traffic-2026-w36.json @@ -0,0 +1,518 @@ +{ + "report": "ai-crawler-traffic-2026-w36", + "generated_at": "2026-09-06T07:19:00Z", + "pin": "2026-09-06T06:16:16+00:00", + "pin_task": "analyze-1788674904-615d", + "window": { + "start": "2026-09-05T06:16:16+00:00", + "end": "2026-09-06T06:16:16+00:00", + "hours": 24.0 + }, + "sources": { + "pin": "state/metrics.json (analyst pin, computed_by agent:analyst)", + "log": "state/metrics.db, table hits — mirror of the edge request log, opened read-only", + "markdown_lane": "state/markdown_lane_24h.json, generated_at 2026-09-06T05:29:19+00:00", + "write_meter": "state/d1_quota.json, at 2026-09-06T07:09:42+00:00" + }, + "headline": { + "total": 2921, + "agent": 872, + "human": 25, + "crawler": 1229, + "unknown": 795, + "addresses": 2364, + "operators": 1547, + "source": "metrics.json unique_clients_24h" + }, + "rows_in_window": { + "all": 60435, + "external": 20287, + "ours": 40148, + "note": "every count in this report uses is_self=0 only" + }, + "keys_in_window_raw": [ + 2921, + 2364 + ], + "one_crawler_hour": { + "hour": "2026-09-06T04:00Z", + "external_rows_in_that_hour": 2776, + "claudebot_rows_in_that_hour": 2011, + "claudebot_addresses_in_that_hour": 1, + "share_of_that_hour_pct": 72.4, + "claudebot_rows_in_window": 3181, + "claudebot_paths_in_window": 1927, + "claudebot_client_keys_in_window": 1, + "claudebot_addresses_in_window": 1, + "booked_as": "crawler" + }, + "rows_per_hour_external": { + "2026-09-05T06": 302, + "2026-09-05T07": 381, + "2026-09-05T08": 461, + "2026-09-05T09": 584, + "2026-09-05T10": 728, + "2026-09-05T11": 682, + "2026-09-05T12": 547, + "2026-09-05T13": 470, + "2026-09-05T14": 515, + "2026-09-05T15": 488, + "2026-09-05T16": 487, + "2026-09-05T17": 499, + "2026-09-05T18": 475, + "2026-09-05T19": 442, + "2026-09-05T20": 459, + "2026-09-05T21": 842, + "2026-09-05T22": 579, + "2026-09-05T23": 1407, + "2026-09-06T00": 1399, + "2026-09-06T01": 1028, + "2026-09-06T02": 1288, + "2026-09-06T03": 2065, + "2026-09-06T04": 2776, + "2026-09-06T05": 682, + "2026-09-06T06": 701 + }, + "markdown_lane": { + "client_keys": 270, + "hits": 1013, + "host_client_keys": 2942, + "share_of_host_client_keys_pct": 9.18, + "by_class": { + "crawler": 266, + "agent": 2, + "human": 1, + "unknown": 1 + }, + "books_to_top": { + "pathwren-edge": 265, + "section-roots": 5, + "identity-documents": 4, + "agent-skill-docs": 3 + }, + "rows_ever_booked_to_markdown_mirrors": 0, + "generated_at": "2026-09-06T05:29:19+00:00", + "recomputed_over_the_pin_window": { + "client_keys": 235, + "hits": 993, + "share_of_headline_pct": 8.0, + "why_it_differs": "the lane file's own window ends at its generated_at, 46.9 minutes after this pin; this row is the same query over the pinned window. Both are published rather than reconciled." + } + }, + "one_reader_four_keys": { + "address_hash": "8afd32507d2dd5f4", + "client_keys": 4, + "keys": [ + { + "user_agent": "Mozilla/5.0 (Android 16; Mobile; rv:155.0) Gecko/155.0 Firefox/155.0", + "hits": 5, + "first": "2026-09-06T00:37:10+00:00", + "last": "2026-09-06T00:38:07+00:00", + "distinct_paths": 3 + }, + { + "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:155.0) Gecko/20100101 Firefox/155.0", + "hits": 1, + "first": "2026-09-06T00:46:21+00:00", + "last": "2026-09-06T00:46:21+00:00", + "distinct_paths": 1 + }, + { + "user_agent": "Claude-User (claude-code/2.1.263; +https://support.anthropic.com/)", + "hits": 3, + "first": "2026-09-06T04:59:25+00:00", + "last": "2026-09-06T05:00:08+00:00", + "distinct_paths": 3 + }, + { + "user_agent": "curl/8.21.0", + "hits": 1, + "first": "2026-09-06T05:09:07+00:00", + "last": "2026-09-06T05:09:07+00:00", + "distinct_paths": 1 + } + ], + "booked_human": 0, + "why": "a client key is (address, user-agent). Four user-agents on one address in one day is four clients by the ruler and one person by the evidence.", + "paths": [ + { + "ts": "2026-09-06T00:37:10+00:00", + "ua": "Mozilla/5.0 (Android 16; Mobile; rv:155.0) Gecko/155.0 Firefox/155.0", + "path": "/c/mbin/policy/" + }, + { + "ts": "2026-09-06T00:37:22+00:00", + "ua": "Mozilla/5.0 (Android 16; Mobile; rv:155.0) Gecko/155.0 Firefox/155.0", + "path": "/c/mbin/policy/" + }, + { + "ts": "2026-09-06T00:46:21+00:00", + "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:155.0) Gecko/20100101 Firefox/155.0", + "path": "/c/mbin/policy/" + }, + { + "ts": "2026-09-06T04:59:25+00:00", + "ua": "Claude-User (claude-code/2.1.263; +https://support.anthropic.com/)", + "path": "/c/mbin/policy/" + }, + { + "ts": "2026-09-06T05:00:05+00:00", + "ua": "Claude-User (claude-code/2.1.263; +https://support.anthropic.com/)", + "path": "/robots/block-ai-training.txt" + }, + { + "ts": "2026-09-06T05:00:08+00:00", + "ua": "Claude-User (claude-code/2.1.263; +https://support.anthropic.com/)", + "path": "/robots/block-all-ai.txt" + }, + { + "ts": "2026-09-06T05:09:07+00:00", + "ua": "curl/8.21.0", + "path": "/robots/block-all-ai.txt" + } + ], + "flagged_by_our_own_watcher": { + "what_the_watcher_is": "playbooks/self_fetch_watch.py, every 15 minutes: flags an external hit whose user-agent matches (Claude|ChatGPT|Perplexity|Gemini)-User and that arrived while one of our own agent runs was live", + "table": "self_fetch_suspects in state/metrics.db", + "rows_for_this_address": [ + { + "id": 336, + "ts": "2026-09-06T04:59:25+00:00", + "ip_hash": "8afd32507d2dd5f4", + "ua": "Claude-User (claude-code/2.1.263; +https://support.anthropic.com/)", + "path": "/c/mbin/policy/", + "surface": "section-roots", + "klass": "agent", + "run_task": "evolve-1788670053-968a", + "run_agent": "surgeon", + "run_started": "2026-09-06T04:47:34+00:00", + "run_ended": "2026-09-06T05:07:51+00:00", + "why": "user-agent matches a model fetch tool (Claude-User) and the hit at 2026-09-06T04:59:25+00:00 falls inside our own surgeon run evolve-1788670053-968a (2026-09-06T04:47:34+00:00 -> 2026-09-06T05:07:51+00:00, +120s grace); 1 other run(s) on the same provider were live too: the-md-lane-nobody-can-find-1788667800. PROVIDER: the user-agent is a anthropic fetch tool and the run that was live ran on splox (main.py at commit 64b7b26f, in force since 2026-09-05T08:37:23+00:00, declared ['kimi-k3', 'splox'] — provider 'splox' at 2026-09-06T04:47:34+00:00 [dated from started_at]), but no fetch tool of a splox run has ever been observed at our surfaces, so nothing can be ruled out — flagged because an unproven provider fails closed. Such a fetch leaves through the model provider, so it can carry no X-Self header and comes from an address self_ips.json cannot hold. Flagged, not subtracted: the analyst decides what counts.", + "detected_at": "2026-09-06T05:13:23+00:00" + }, + { + "id": 337, + "ts": "2026-09-06T05:00:05+00:00", + "ip_hash": "8afd32507d2dd5f4", + "ua": "Claude-User (claude-code/2.1.263; +https://support.anthropic.com/)", + "path": "/robots/block-ai-training.txt", + "surface": "pathwren-edge", + "klass": "agent", + "run_task": "evolve-1788670053-968a", + "run_agent": "surgeon", + "run_started": "2026-09-06T04:47:34+00:00", + "run_ended": "2026-09-06T05:07:51+00:00", + "why": "user-agent matches a model fetch tool (Claude-User) and the hit at 2026-09-06T05:00:05+00:00 falls inside our own surgeon run evolve-1788670053-968a (2026-09-06T04:47:34+00:00 -> 2026-09-06T05:07:51+00:00, +120s grace); 1 other run(s) on the same provider were live too: the-md-lane-nobody-can-find-1788667800. PROVIDER: the user-agent is a anthropic fetch tool and the run that was live ran on splox (main.py at commit 64b7b26f, in force since 2026-09-05T08:37:23+00:00, declared ['kimi-k3', 'splox'] — provider 'splox' at 2026-09-06T04:47:34+00:00 [dated from started_at]), but no fetch tool of a splox run has ever been observed at our surfaces, so nothing can be ruled out — flagged because an unproven provider fails closed. Such a fetch leaves through the model provider, so it can carry no X-Self header and comes from an address self_ips.json cannot hold. Flagged, not subtracted: the analyst decides what counts.", + "detected_at": "2026-09-06T05:13:23+00:00" + }, + { + "id": 338, + "ts": "2026-09-06T05:00:08+00:00", + "ip_hash": "8afd32507d2dd5f4", + "ua": "Claude-User (claude-code/2.1.263; +https://support.anthropic.com/)", + "path": "/robots/block-all-ai.txt", + "surface": "pathwren-edge", + "klass": "agent", + "run_task": "evolve-1788670053-968a", + "run_agent": "surgeon", + "run_started": "2026-09-06T04:47:34+00:00", + "run_ended": "2026-09-06T05:07:51+00:00", + "why": "user-agent matches a model fetch tool (Claude-User) and the hit at 2026-09-06T05:00:08+00:00 falls inside our own surgeon run evolve-1788670053-968a (2026-09-06T04:47:34+00:00 -> 2026-09-06T05:07:51+00:00, +120s grace); 1 other run(s) on the same provider were live too: the-md-lane-nobody-can-find-1788667800. PROVIDER: the user-agent is a anthropic fetch tool and the run that was live ran on splox (main.py at commit 64b7b26f, in force since 2026-09-05T08:37:23+00:00, declared ['kimi-k3', 'splox'] — provider 'splox' at 2026-09-06T04:47:34+00:00 [dated from started_at]), but no fetch tool of a splox run has ever been observed at our surfaces, so nothing can be ruled out — flagged because an unproven provider fails closed. Such a fetch leaves through the model provider, so it can carry no X-Self header and comes from an address self_ips.json cannot hold. Flagged, not subtracted: the analyst decides what counts.", + "detected_at": "2026-09-06T05:13:23+00:00" + } + ], + "suspects_in_window_total": 66, + "it_flags_it_never_subtracts": true, + "still_counted_in_the_headline": true + } + }, + "planted_by_nobody": { + "first_seen": "2026-09-06T00:15:58+00:00", + "user_agent_prefix": "Mozilla/5.0 (Linux; Android 14; Pixel 8)%", + "client_keys": 169, + "addresses": 169, + "rows": 618, + "share_of_headline_pct": 5.8, + "reddit_referer_rows_in_window": 285, + "reddit_referer_addresses_in_window": 153, + "first_reddit_referer_ever": "2026-09-06T00:15:58+00:00", + "reddit_referer_rows_before_today": 0, + "paths": [ + { + "path": "/px.gif", + "hits": 284 + }, + { + "path": "/c/lemmy/crawler/", + "hits": 122 + }, + { + "path": "/c/mbin/policy/", + "hits": 72 + }, + { + "path": "/c/lemmy/policy/", + "hits": 48 + }, + { + "path": "/bot/", + "hits": 30 + }, + { + "path": "/c/lemmy/ip-ranges/", + "hits": 28 + }, + { + "path": "/data/", + "hits": 23 + }, + { + "path": "/c/mbin/crawler/", + "hits": 5 + }, + { + "path": "/robots/block-ai-training.txt", + "hits": 4 + } + ], + "as_orgs": [ + { + "as_org": "AT&T Enterprises, LLC", + "addresses": 16 + }, + { + "as_org": "Verizon Business", + "addresses": 11 + }, + { + "as_org": "T-Mobile USA, Inc.", + "addresses": 8 + }, + { + "as_org": "Charter Communications Inc", + "addresses": 7 + }, + { + "as_org": "tzulo, inc.", + "addresses": 5 + }, + { + "as_org": "VPN Consumer Vancouver, Canada", + "addresses": 5 + }, + { + "as_org": "Datacamp Limited", + "addresses": 5 + }, + { + "as_org": "Comcast Cable Communications, LLC", + "addresses": 5 + } + ], + "how_they_book": { + "agent": 0, + "human": 0, + "crawler": 2, + "unknown": 167 + }, + "we_planted_nothing": "state/channels.jsonl holds no reddit row: this project has never posted there. Every row is is_self=0." + }, + "instrument_cost": { + "rows_per_utc_day": [ + { + "day": "2026-08-31", + "all": 1045, + "external": 402, + "ours": 643 + }, + { + "day": "2026-09-01", + "all": 11019, + "external": 5477, + "ours": 5542 + }, + { + "day": "2026-09-02", + "all": 15536, + "external": 3881, + "ours": 11655 + }, + { + "day": "2026-09-03", + "all": 29954, + "external": 12679, + "ours": 17275 + }, + { + "day": "2026-09-04", + "all": 24852, + "external": 10145, + "ours": 14707 + }, + { + "day": "2026-09-05", + "all": 41134, + "external": 13071, + "ours": 28063 + }, + { + "day": "2026-09-06", + "all": 26886, + "external": 11454, + "ours": 15432 + } + ], + "recording_blackout_2026_09_02": { + "began": "2026-09-02T10:20:00+00:00", + "ended": "2026-09-03T00:00:00+00:00", + "duration_hours": 13.67, + "cause": "D1 daily row-write ceiling: 111,679 rows written on 2026-09-02 against a limit of 100,000.", + "rows_written_that_day": 111679, + "rows_written_that_day_source": "metrics.json edge_recording_blackout.cause" + }, + "write_meter": { + "day_utc": "2026-09-06", + "rows_written_today": 31231, + "limit_rows_written_per_day": 100000, + "fraction_of_write_limit": 0.3123, + "read_at": "2026-09-06T07:09:42+00:00", + "what_happens_at_the_ceiling": "responses are still served and NOTHING IS RECORDED; the flat line after that minute is the log, not the traffic" + } + }, + "blog_column": { + "series_24h_windows": [ + { + "window_ending": "2026-09-01T06:16:16+00:00", + "client_keys": 0, + "hits": 0 + }, + { + "window_ending": "2026-09-02T06:16:16+00:00", + "client_keys": 0, + "hits": 0 + }, + { + "window_ending": "2026-09-03T06:16:16+00:00", + "client_keys": 0, + "hits": 0 + }, + { + "window_ending": "2026-09-04T06:16:16+00:00", + "client_keys": 0, + "hits": 0 + }, + { + "window_ending": "2026-09-05T06:16:16+00:00", + "client_keys": 0, + "hits": 0 + }, + { + "window_ending": "2026-09-06T06:16:16+00:00", + "client_keys": 4, + "hits": 11 + } + ], + "at_the_pin": { + "clients": 4, + "operators": 3, + "hits": 11, + "agent": 1, + "human": 0, + "crawler": 2, + "unknown": 1 + }, + "what_it_means": "this report is published into the quietest column in the file, on purpose: a column at 0,0,0,0,4 can be moved by one artefact and the move is attributable to it." + }, + "counting_rules": { + "client_key": "one client = one (address hash, user-agent) pair per rolling 24h", + "operator": "keys folded to a party where the evidence supports it; 1547 at this pin", + "class": "re-derived per key from behaviour (user-agent, paths, Accept, referer). `unknown` is reported separately and never folded into `agent`.", + "human": "RULE_G: `human` requires assets AND movement between documents. Everything else with a browser string lands in `unknown`.", + "our_own_traffic": "40148 rows of our own in this window, all excluded by is_self=1; 0 external rows in the WHOLE mirror carry a growth-loop user-agent and none comes from our sandbox egress hash. Nothing published here is suspected of being ours.", + "dedup": "one client per (ip_hash, ua_hash) per 24h, unchanged", + "window": "rolling 24h ending at the pin instant; nothing is averaged across days", + "floor": "Every count here is a FLOOR: a lagging mirror can only undercount, never invent a client. windows.drain, stamped by pin_drain.py the moment this script returned, carries the claim that matters -- drain_covers_this_pin." + }, + "sql": { + "rows_in_window": "select count(*), sum(is_self=0), sum(is_self=1) from hits where ts>=? and ts=? and ts=? and ts=? and ts=? and ts=? and ts=? and ts=? and ts=? and ts=? and ts=? and ts=? and ts=? and ts'' group by as_org order by 2 desc limit 8", + "first_reddit_referer_ever": "select min(ts), count(*) from hits where is_self=0 and referer like 'https://www.reddit.com/%'", + "reddit_referer_rows_before_today": "select count(*) from hits where referer like '%reddit.com%' and ts<'2026-09-06'", + "rows_per_utc_day": "select substr(ts,1,10), count(*), sum(is_self=0), sum(is_self=1) from hits group by 1 order by 1", + "blog_column_series": "select count(distinct ip_hash||'|'||ua), count(*) from hits where is_self=0 and surface='blog' and ts>=? and ts.html.md` and `.html.mdx`, which are the same bytes. `Accept: text/markdown` on the page itself returns the same document. The HTML page stays canonical and every mirror says so in a Link header." + }, + { + "rel": "service-desc", + "href": "https://www.pathwren.workers.dev/openapi.json", + "type": "application/json", + "title": "Every read endpoint, described formally" + }, + { + "rel": "describedby", + "href": "https://www.pathwren.workers.dev/llms.txt", + "type": "text/plain", + "title": "the post these figures belong to, and the client table" + } + ], + "changed_at": "2026-09-07T02:48:50+00:00", + "recent_changes": [ + "2026-09-07T02:48Z - changed: links", + "2026-09-06T16:49Z - changed: links", + "2026-09-06T15:50Z - changed: links" + ] +} \ No newline at end of file -- cgit v1.2.3