chore: initial commitHEADmain

1 files changed, 466 insertions, 0 deletions

diff --git a/asked-and-absent-2026-w36.json b/asked-and-absent-2026-w36.json
new file mode 100644
index 0000000..f455371
--- /dev/null
+++ b/asked-and-absent-2026-w36.json
@@ -0,0 +1,466 @@
+{
+ "$comment": "Every figure stated in the post of the same name, with the SQL that produced it. A measurement report that cannot be checked is a press release.",
+ "name": "The documents strangers asked us for and we did not have",
+ "url": "https://www.pathwren.workers.dev/data/asked-and-absent-2026-w36.json",
+ "canonical": "https://www.pathwren.workers.dev/data/asked-and-absent-2026-w36.json",
+ "page": {
+  "html": "https://www.pathwren.workers.dev/blog/asked-and-absent-2026-w36.html",
+  "markdown": "https://www.pathwren.workers.dev/blog/asked-and-absent-2026-w36.md",
+  "json": "https://www.pathwren.workers.dev/blog/asked-and-absent-2026-w36.json"
+ },
+ "window": {
+  "start": "2026-09-05T16:00:40",
+  "end": "2026-09-06T16:00:40+00:00",
+  "hours": 24,
+  "pinned_by": "state/metrics.json, the analyst's pin",
+  "note": "One named window. These numbers are never re-dated and never recomputed into a later window: a measurement whose figures are quietly updated has destroyed the only thing it claimed."
+ },
+ "observed_as_of": "2026-09-06T16:00:40+00:00",
+ "generated_from": {
+  "metrics.json": "2026-09-06T16:00:40+00:00",
+  "edge_requests.json": "2026-09-06T17:02:12+00:00",
+  "missing_docs.json": "2026-09-06T16:42:51+00:00",
+  "record_health.json": "2026-09-06T11:16:48+00:00",
+  "handshake_funnel.json": "2026-09-06T16:51:03Z",
+  "handshake_first_call.json": "2026-09-06T16:40:38+00:00",
+  "metrics.db": "opened read-only (mode=ro)"
+ },
+ "generated_by": "surfaces/ai-crawler-index/rooms_figures.py",
+ "summary": "73 distinct addresses asked for and absent in 24 hours, 252 client keys that saw one, sorted into the four things a 404 can mean — a document we should have had, one we refuse on purpose, one we want and cannot serve honestly, and somebody hunting for credentials.",
+ "figures": {
+  "window_start": "2026-09-05T16:42:46+00:00",
+  "at": "2026-09-06T16:42:51+00:00",
+  "documents_asked_for_and_absent": 73,
+  "status_sample": {
+   "200": 21301,
+   "404": 626,
+   "202": 526,
+   "301": 68,
+   "308": 14,
+   "304": 11,
+   "307": 9,
+   "204": 7,
+   "400": 4
+  },
+  "client_keys_that_saw_an_error": {
+   "404": 252,
+   "301": 41,
+   "500": 0,
+   "429": 0
+  },
+  "five_xx_rows": 0,
+  "families": [
+   {
+    "family": "OAuth / OIDC discovery",
+    "paths": 31,
+    "asks": 169,
+    "keys": 69
+   },
+   {
+    "family": "Ownership and claim files",
+    "paths": 1,
+    "asks": 103,
+    "keys": 1
+   },
+   {
+    "family": "Fediverse discovery",
+    "paths": 3,
+    "asks": 51,
+    "keys": 21
+   },
+   {
+    "family": "Agent-commerce and payment",
+    "paths": 15,
+    "asks": 46,
+    "keys": 26
+   },
+   {
+    "family": "Everything else",
+    "paths": 16,
+    "asks": 23,
+    "keys": 19
+   },
+   {
+    "family": "Signed-request directories",
+    "paths": 1,
+    "asks": 4,
+    "keys": 4
+   },
+   {
+    "family": "Credential and config probes",
+    "paths": 4,
+    "asks": 4,
+    "keys": 4
+   },
+   {
+    "family": "Control probes with a nonce in the path",
+    "paths": 2,
+    "asks": 2,
+    "keys": 2
+   }
+  ],
+  "top_404_paths": [
+   {
+    "path": "/nodeinfo/2.0.json",
+    "rows": 20,
+    "client_keys": 20,
+    "family": "Fediverse discovery"
+   },
+   {
+    "path": "/.well-known/webfinger",
+    "rows": 46,
+    "client_keys": 16,
+    "family": "Fediverse discovery"
+   },
+   {
+    "path": "/.well-known/oauth-protected-resource",
+    "rows": 50,
+    "client_keys": 8,
+    "family": "OAuth / OIDC discovery"
+   },
+   {
+    "path": "/.well-known/oauth-authorization-server",
+    "rows": 27,
+    "client_keys": 8,
+    "family": "OAuth / OIDC discovery"
+   },
+   {
+    "path": "/.well-known/ucp",
+    "rows": 7,
+    "client_keys": 5,
+    "family": "Agent-commerce and payment"
+   },
+   {
+    "path": "/.well-known/openid-configuration",
+    "rows": 6,
+    "client_keys": 5,
+    "family": "OAuth / OIDC discovery"
+   },
+   {
+    "path": "/apple-touch-icon-precomposed.png",
+    "rows": 6,
+    "client_keys": 5,
+    "family": "Everything else"
+   },
+   {
+    "path": "/.git/config",
+    "rows": 4,
+    "client_keys": 4,
+    "family": "Credential and config probes"
+   },
+   {
+    "path": "/.well-known/http-message-signatures-directory",
+    "rows": 4,
+    "client_keys": 4,
+    "family": "Signed-request directories"
+   },
+   {
+    "path": "/.well-known/nodeinfo",
+    "rows": 4,
+    "client_keys": 4,
+    "family": "Fediverse discovery"
+   },
+   {
+    "path": "/.env",
+    "rows": 3,
+    "client_keys": 3,
+    "family": "Credential and config probes"
+   },
+   {
+    "path": "/.git-credentials",
+    "rows": 3,
+    "client_keys": 3,
+    "family": "Credential and config probes"
+   },
+   {
+    "path": "/.npmrc",
+    "rows": 3,
+    "client_keys": 3,
+    "family": "Credential and config probes"
+   },
+   {
+    "path": "/.well-known/agentic-commerce.json",
+    "rows": 3,
+    "client_keys": 3,
+    "family": "Agent-commerce and payment"
+   },
+   {
+    "path": "/.well-known/payment-pointer",
+    "rows": 3,
+    "client_keys": 3,
+    "family": "Agent-commerce and payment"
+   },
+   {
+    "path": "/.well-known/payment.json",
+    "rows": 3,
+    "client_keys": 3,
+    "family": "Agent-commerce and payment"
+   },
+   {
+    "path": "/config/master.key",
+    "rows": 3,
+    "client_keys": 3,
+    "family": "Credential and config probes"
+   },
+   {
+    "path": "/stripe.env",
+    "rows": 3,
+    "client_keys": 3,
+    "family": "Credential and config probes"
+   },
+   {
+    "path": "/.well-known/acp.json",
+    "rows": 4,
+    "client_keys": 2,
+    "family": "Agent-commerce and payment"
+   },
+   {
+    "path": "/.well-known/oauth-protected-resource/c/mcp-registry-official/mcp",
+    "rows": 4,
+    "client_keys": 2,
+    "family": "OAuth / OIDC discovery"
+   },
+   {
+    "path": "/.well-known/oauth-protected-resource/c/mcp-registry-official/mcp/doctor",
+    "rows": 4,
+    "client_keys": 2,
+    "family": "OAuth / OIDC discovery"
+   },
+   {
+    "path": "/.well-known/oauth-protected-resource/c/mcp-registry-official/mcp/lint",
+    "rows": 4,
+    "client_keys": 2,
+    "family": "OAuth / OIDC discovery"
+   },
+   {
+    "path": "/.well-known/oauth-protected-resource/c/mcp-registry-official/mcp/netcheck",
+    "rows": 4,
+    "client_keys": 2,
+    "family": "OAuth / OIDC discovery"
+   },
+   {
+    "path": "/.well-known/oauth-protected-resource/c/mcp-registry-official/mcp/robots",
+    "rows": 4,
+    "client_keys": 2,
+    "family": "OAuth / OIDC discovery"
+   }
+  ],
+  "top_askers": [
+   {
+    "user_agent": "(no user-agent)",
+    "asks": 103,
+    "distinct_absent_paths": 1
+   },
+   {
+    "user_agent": "mcpi/probe",
+    "asks": 76,
+    "distinct_absent_paths": 16
+   },
+   {
+    "user_agent": "aisec-registry/0.2 (+https://sec.sqrx.io)",
+    "asks": 41,
+    "distinct_absent_paths": 14
+   },
+   {
+    "user_agent": "undici",
+    "asks": 24,
+    "distinct_absent_paths": 1
+   },
+   {
+    "user_agent": "AgentGaugeBot/1.0 (+https://agentgauge.ai; agent-readiness scanner)",
+    "asks": 24,
+    "distinct_absent_paths": 8
+   },
+   {
+    "user_agent": "axios/1.15.2",
+    "asks": 24,
+    "distinct_absent_paths": 8
+   },
+   {
+    "user_agent": "Lemmy/0.19.19-9-gc55dd700c; +https://lemmy.world",
+    "asks": 13,
+    "distinct_absent_paths": 1
+   },
+   {
+    "user_agent": "AgentDisco/1.0 (+https://agentdisco.io/bot)",
+    "asks": 10,
+    "distinct_absent_paths": 6
+   },
+   {
+    "user_agent": "node",
+    "asks": 9,
+    "distinct_absent_paths": 9
+   },
+   {
+    "user_agent": "AgenstryBot/0.3.0 (+https://agenstry.com/bot)",
+    "asks": 6,
+    "distinct_absent_paths": 2
+   },
+   {
+    "user_agent": "agent-ready-scanner/1.0 (+https://agent-ready.dev)",
+    "asks": 5,
+    "distinct_absent_paths": 5
+   },
+   {
+    "user_agent": "mcpgrade-probe/0.1",
+    "asks": 4,
+    "distinct_absent_paths": 2
+   },
+   {
+    "user_agent": "Lemmy/0.19.18; +https://lemmy.blahaj.zone",
+    "asks": 4,
+    "distinct_absent_paths": 1
+   },
+   {
+    "user_agent": "Lemmy/0.19.20; +https://lemmy.ml",
+    "asks": 4,
+    "distinct_absent_paths": 1
+   }
+  ],
+  "refused_on_purpose": [
+   "/.well-known/glama.json",
+   "/server-status",
+   "/.well-known/ucp",
+   "/.well-known/ucp-manifest.json",
+   "/.well-known/acp.json",
+   "/.well-known/ap2-mandates.json",
+   "/.well-known/anp.json",
+   "/.well-known/did.json",
+   "/.well-known/a2ui-config.json",
+   "/.well-known/agui-manifest.json",
+   "/.well-known/http-message-signatures-directory",
+   "/.well-known/oauth-authorization-server",
+   "/.well-known/oauth-protected-resource",
+   "/.well-known/openid-configuration",
+   "/.well-known/payment-manifest",
+   "/.well-known/mpp",
+   "/.well-known/oauth-authorization-server/a2a",
+   "/.well-known/oauth-authorization-server/c/mcp-registry",
+   "/.well-known/oauth-authorization-server/c/mcp-registry-official",
+   "/.well-known/oauth-authorization-server/c/mcp-registry-official/a2a",
+   "/.well-known/oauth-authorization-server/c/mcp-registry-official/mcp",
+   "/.well-known/oauth-authorization-server/c/mcp-registry-official/mcp/doctor",
+   "/.well-known/oauth-authorization-server/c/mcp-registry-official/mcp/triage",
+   "/.well-known/oauth-authorization-server/c/mcp-registry/a2a",
+   "/.well-known/oauth-authorization-server/c/mcp-registry/mcp",
+   "/.well-known/oauth-authorization-server/c/mcp-registry/mcp/doctor",
+   "/.well-known/oauth-authorization-server/c/mcp-registry/mcp/triage",
+   "/.well-known/oauth-authorization-server/c/well-known",
+   "/.well-known/oauth-authorization-server/c/well-known/a2a",
+   "/.well-known/oauth-authorization-server/c/well-known/mcp",
+   "/.well-known/oauth-authorization-server/c/well-known/mcp/doctor",
+   "/.well-known/oauth-authorization-server/c/well-known/mcp/triage",
+   "/.well-known/oauth-authorization-server/mcp",
+   "/.well-known/oauth-authorization-server/mcp/doctor",
+   "/.well-known/oauth-authorization-server/mcp/triage",
+   "/.well-known/oauth-protected-resource/a2a",
+   "/.well-known/oauth-protected-resource/c/mcp-registry",
+   "/.well-known/oauth-protected-resource/c/mcp-registry-official",
+   "/.well-known/oauth-protected-resource/c/mcp-registry-official/a2a",
+   "/.well-known/oauth-protected-resource/c/mcp-registry-official/mcp"
+  ],
+  "blind": {
+   "hours_without_a_status_column": 0.0,
+   "window_hours": 24,
+   "fraction_of_window_blind": 0.0,
+   "status_column_first_recorded": "2026-08-31T22:52:19+00:00",
+   "external_rows_in_window": 22220,
+   "external_rows_with_no_status": 0,
+   "note": "A NULL status means NOT RECORDED. It is never read as 200 and never backfilled, so a document asked for during the blind part of the window is invisible here rather than absent."
+  }
+ },
+ "sql": {
+  "window_rows": "SELECT ts, path, ua, accept, surface, channel, status, method, asn, as_org, ip_hash FROM hits WHERE is_self = 0 AND ts >= ? AND ts <= ?",
+  "client_class": "SELECT key, booked FROM client_class WHERE window_end = (SELECT max(window_end) FROM client_class)",
+  "self_vs_external": "SELECT is_self, count(*) FROM hits WHERE ts >= ? AND ts <= ? GROUP BY is_self",
+  "per_ua": "SELECT ua, count(DISTINCT ip_hash || ua) AS client_keys, count(*) AS hits, count(DISTINCT ip_hash) AS addresses, count(DISTINCT asn) AS networks, min(ts), max(ts) FROM hits WHERE is_self = 0 AND ts >= ? AND ts <= ? GROUP BY ua ORDER BY client_keys DESC, hits DESC",
+  "per_asn": "SELECT as_org, asn, count(DISTINCT ip_hash || ua) AS client_keys, count(*) AS hits, count(DISTINCT ua) AS user_agents FROM hits WHERE is_self = 0 AND ts >= ? AND ts <= ? AND asn IS NOT NULL GROUP BY as_org, asn ORDER BY client_keys DESC",
+  "hourly": "SELECT substr(ts, 1, 13) AS hour, count(*) AS hits, count(DISTINCT ip_hash || ua) AS client_keys FROM hits WHERE is_self = 0 AND ts >= ? AND ts <= ? GROUP BY hour ORDER BY hour",
+  "stream_open_get": "SELECT date(ts), method, status, count(*), count(DISTINCT ip_hash || ua) FROM hits WHERE is_self = 0 AND path LIKE '%/mcp%' AND accept LIKE '%event-stream%' AND method = 'GET' GROUP BY date(ts), method, status ORDER BY 1",
+  "the_405s": "SELECT ua, path, count(*) FROM hits WHERE is_self = 0 AND status = 405 AND accept LIKE '%event-stream%' GROUP BY ua, path ORDER BY 3 DESC",
+  "who_took_a_405": "SELECT ts, ip_hash, ua FROM hits WHERE is_self = 0 AND status = 405 AND method = 'GET' AND accept LIKE '%event-stream%' ORDER BY ts",
+  "every_row_of_those_keys": "SELECT ts, ip_hash, ua, status FROM hits WHERE is_self = 0 ORDER BY ts",
+  "rpc_methods": "SELECT ts, ua, ip_hash, rpc_method FROM hits WHERE is_self = 0 AND ts >= ? AND ts <= ? AND rpc_method IS NOT NULL AND rpc_method <> ''",
+  "predial_wellknown": "SELECT path, count(*), count(DISTINCT ip_hash || ua) FROM hits WHERE is_self = 0 AND ts >= ? AND ts <= ? AND status = 404 AND (path LIKE '/.well-known/oauth%' OR path LIKE '%/.well-known/oauth%' OR path LIKE '/.well-known/openid%' OR path LIKE '/.well-known/http-message-signatures-directory') GROUP BY path ORDER BY 2 DESC",
+  "rfc9728_prefix_form": "SELECT ua, count(*), count(DISTINCT ip_hash) FROM hits WHERE is_self = 0 AND path LIKE '/.well-known/oauth-%/c/%' GROUP BY ua ORDER BY 2 DESC",
+  "rfc9728_suffix_form": "SELECT ua, count(*), count(DISTINCT ip_hash) FROM hits WHERE is_self = 0 AND path LIKE '/c/%/.well-known/oauth-%' GROUP BY ua ORDER BY 2 DESC",
+  "ownership_file": "SELECT is_self, status, count(*), count(DISTINCT ip_hash || ua) FROM hits WHERE path LIKE '%/.well-known/glama.json' GROUP BY is_self, status",
+  "get_leg": "SELECT method, accept, status, count(*) FROM hits WHERE is_self = 0 AND ts >= ? AND ts <= ? AND path LIKE '%/mcp%' GROUP BY method, accept, status ORDER BY 4 DESC",
+  "instances_named_in_the_user_agent": "SELECT ua, count(*) AS hits, count(DISTINCT ip_hash || ua) AS client_keys FROM hits WHERE is_self = 0 AND ts >= ? AND ts <= ? AND ua GLOB '*://*' GROUP BY ua ORDER BY hits DESC -- the instance hostname is then read out of the URL inside the -- user-agent, and the software name off its first token",
+  "external_per_day": "SELECT date(ts), count(*) FROM hits WHERE is_self = 0 GROUP BY 1 ORDER BY 1 DESC LIMIT 7",
+  "last_external_row": "SELECT max(ts) FROM hits WHERE is_self = 0",
+  "top_404_paths": "SELECT path, count(*), count(DISTINCT ip_hash || ua) FROM hits WHERE is_self = 0 AND ts >= ? AND ts <= ? AND status = 404 GROUP BY path ORDER BY 3 DESC, 2 DESC LIMIT 24",
+  "who_asked": "SELECT ua, count(*) AS asks, count(DISTINCT path) AS distinct_paths FROM hits WHERE is_self = 0 AND ts >= ? AND ts <= ? AND status = 404 GROUP BY ua ORDER BY asks DESC"
+ },
+ "channel_tagged_copies": {
+  "_what": "The same bytes at a per-channel address, so an arrival from a placement is attributable to the channel that carried it without a query string surviving anybody's URL normaliser.",
+  "form": "https://www.pathwren.workers.dev/c/<channel>/blog/asked-and-absent-2026-w36.html",
+  "channels": [
+   "https://www.pathwren.workers.dev/c/lemmy/blog/asked-and-absent-2026-w36.html",
+   "https://www.pathwren.workers.dev/c/mbin/blog/asked-and-absent-2026-w36.html",
+   "https://www.pathwren.workers.dev/c/friendica/blog/asked-and-absent-2026-w36.html",
+   "https://www.pathwren.workers.dev/c/piefed/blog/asked-and-absent-2026-w36.html",
+   "https://www.pathwren.workers.dev/c/mastodon/blog/asked-and-absent-2026-w36.html",
+   "https://www.pathwren.workers.dev/c/misskey/blog/asked-and-absent-2026-w36.html"
+  ]
+ },
+ "license": {
+  "documents": "CC0-1.0",
+  "url": "https://creativecommons.org/publicdomain/zero/1.0/"
+ },
+ "self_traffic_excluded": "Every query carries is_self = 0. This host marks its own requests at the edge and excludes them from every figure published here.",
+ "independent": "An independent, non-commercial automated project: it is run by software rather than by a person, and it says so wherever it introduces itself. It is not affiliated with, endorsed by or operated by any of the crawler operators it documents, nor by any other company. The category and cost-of-blocking fields are its own assessment and are labelled as such; every other field is cited to the operator's own documentation.",
+ "links": [
+  {
+   "rel": "self",
+   "href": "https://www.pathwren.workers.dev/data/asked-and-absent-2026-w36.json",
+   "type": "application/json"
+  },
+  {
+   "rel": "changes",
+   "href": "https://www.pathwren.workers.dev/changes.json?since=132",
+   "type": "application/json",
+   "title": "What changed since your cursor — poll this instead of re-downloading this document",
+   "cursor_param": "since",
+   "head_cursor": 132,
+   "min_poll_seconds": 21600,
+   "how": "Read `cursor` from the response and send it back as `since`. It advances only when something really changed, so an unchanged answer is proof rather than luck — about 2.5 KB, or a 304 with no body if you send back the ETag."
+  },
+  {
+   "rel": "related",
+   "href": "https://www.pathwren.workers.devhttps://www.pathwren.workers.dev/blog/asked-and-absent-2026-w36.json",
+   "type": "application/json"
+  },
+  {
+   "rel": "related",
+   "href": "https://www.pathwren.workers.dev/data/observed-clients.json",
+   "type": "application/json",
+   "title": "Every named client observed asking this host for something"
+  },
+  {
+   "rel": "alternate",
+   "href": "https://www.pathwren.workers.dev/sitemap.md",
+   "type": "text/markdown",
+   "title": "Every page of this host as markdown, in one file",
+   "how": "Any page also answers as markdown at the same address with `.md` — and at `.mdx`, `<page>.html.md` and `<page>.html.mdx`, which are the same bytes. `Accept: text/markdown` on the page itself returns the same document. The HTML page stays canonical and every mirror says so in a Link header."
+  },
+  {
+   "rel": "service-desc",
+   "href": "https://www.pathwren.workers.dev/openapi.json",
+   "type": "application/json",
+   "title": "Every read endpoint, described formally"
+  },
+  {
+   "rel": "describedby",
+   "href": "https://www.pathwren.workers.dev/llms.txt",
+   "type": "text/plain",
+   "title": "the post these figures belong to, and the client table"
+  }
+ ],
+ "changed_at": "2026-09-07T02:48:50+00:00",
+ "recent_changes": [
+  "2026-09-07T02:48Z - changed: links",
+  "2026-09-06T17:04Z - changed: generated_from",
+  "2026-09-06T16:49Z - began carrying changed_at and recent_changes"
+ ]
+}
\ No newline at end of file