diff options
Diffstat (limited to '.index.json')
| -rw-r--r-- | .index.json | 194 |
1 files changed, 194 insertions, 0 deletions
diff --git a/.index.json b/.index.json new file mode 100644 index 0000000..23030cf --- /dev/null +++ b/.index.json @@ -0,0 +1,194 @@ +{ + "name": "Bulk data", + "description": "Every endpoint this index serves as a file: no key, no signup, no rate limit, CORS open, and every one of them is the same file the HTML pages are generated from.", + "url": "https://www.pathwren.workers.dev/data", + "canonical": "https://www.pathwren.workers.dev/data/index.html", + "formats": { + "html": "https://www.pathwren.workers.dev/data/index.html", + "json": "https://www.pathwren.workers.dev/data/index.json", + "markdown": "https://www.pathwren.workers.dev/data/index.md" + }, + "curl": "curl -s https://www.pathwren.workers.dev/data/agents.json | jq '.crawlers[] | select(.category==\"ai-search\") | .robots_token'", + "count": 18, + "entries": [ + { + "path": "/data/agents.csv", + "url": "https://www.pathwren.workers.dev/data/agents.csv", + "bytes": 24812, + "content_type": "text/csv", + "what": "The same table, flat" + }, + { + "path": "/data/agents.json", + "url": "https://www.pathwren.workers.dev/data/agents.json", + "bytes": 201081, + "content_type": "application/json", + "what": "Every crawler record in one file" + }, + { + "path": "/data/asked-and-absent-2026-w36.json", + "url": "https://www.pathwren.workers.dev/data/asked-and-absent-2026-w36.json", + "bytes": 17892, + "content_type": "application/json", + "what": "Every address asked for and absent here in one named window, with who asked, how often, and which of the four kinds of 404 it is" + }, + { + "path": "/data/crawler-fleet-fold-2026-w36.json", + "url": "https://www.pathwren.workers.dev/data/crawler-fleet-fold-2026-w36.json", + "bytes": 14155, + "content_type": "application/json", + "what": "" + }, + { + "path": "/data/crawler-ua-asn-2026-w36.json", + "url": "https://www.pathwren.workers.dev/data/crawler-ua-asn-2026-w36.json", + "bytes": 37510, + "content_type": "application/json", + "what": "The per-user-agent and per-ASN census behind /blog/crawler-ua-asn-2026-w36.html — every client string, every AS organisation, one named 24-hour window, with the SQL" + }, + { + "path": "/data/fediverse-fanout-2026-w36.json", + "url": "https://www.pathwren.workers.dev/data/fediverse-fanout-2026-w36.json", + "bytes": 16787, + "content_type": "application/json", + "what": "The ActivityPub link-preview fan-out, per instance, per community and per network — how far one link posted to one community actually reaches" + }, + { + "path": "/data/ip-sources.json", + "url": "https://www.pathwren.workers.dev/data/ip-sources.json", + "bytes": 9176, + "content_type": "application/json", + "what": "Which operators publish IP ranges, and where" + }, + { + "path": "/data/markdown-lane.json", + "url": "https://www.pathwren.workers.dev/data/markdown-lane.json", + "bytes": 5860, + "content_type": "application/json", + "what": "" + }, + { + "path": "/data/mcp-conformance-2026-w36.json", + "url": "https://www.pathwren.workers.dev/data/mcp-conformance-2026-w36.json", + "bytes": 27672, + "content_type": "application/json", + "what": "What public MCP records get wrong, as data: the handshake funnel, the stream-open GET that answers 405, the discovery documents callers ask for before they dial" + }, + { + "path": "/data/mcp-endpoint-callers-2026-w36.json", + "url": "https://www.pathwren.workers.dev/data/mcp-endpoint-callers-2026-w36.json", + "bytes": 28870, + "content_type": "application/json", + "what": "Every figure in the dated MCP/A2A caller report, with the window it was measured over and the instant it was computed" + }, + { + "path": "/data/observed-clients.csv", + "url": "https://www.pathwren.workers.dev/data/observed-clients.csv", + "bytes": 294190, + "content_type": "text/csv", + "what": "" + }, + { + "path": "/data/observed-clients.json", + "url": "https://www.pathwren.workers.dev/data/observed-clients.json", + "bytes": 6382766, + "content_type": "application/json", + "what": "Every named client observed asking this host for something" + }, + { + "path": "/data/robots-tokens.txt", + "url": "https://www.pathwren.workers.dev/data/robots-tokens.txt", + "bytes": 2174, + "content_type": "text/plain", + "what": "" + }, + { + "path": "/data/traffic-2026-w36.json", + "url": "https://www.pathwren.workers.dev/data/traffic-2026-w36.json", + "bytes": 20577, + "content_type": "application/json", + "what": "Every figure in the dated traffic report for 2026-09-05/06, with the SQL that produced each one" + }, + { + "path": "/data/ua-regex.json", + "url": "https://www.pathwren.workers.dev/data/ua-regex.json", + "bytes": 13059, + "content_type": "application/json", + "what": "Compiled user-agent regexes for a log classifier or a WAF" + }, + { + "path": "/data/ua-regex.txt", + "url": "https://www.pathwren.workers.dev/data/ua-regex.txt", + "bytes": 2170, + "content_type": "text/plain", + "what": "" + }, + { + "path": "/data/user-agents.txt", + "url": "https://www.pathwren.workers.dev/data/user-agents.txt", + "bytes": 2145, + "content_type": "text/plain", + "what": "" + }, + { + "path": "/data/workers-plan-2026-w36.json", + "url": "https://www.pathwren.workers.dev/data/workers-plan-2026-w36.json", + "bytes": 15146, + "content_type": "application/json", + "what": "One UTC day of a 100,000-request/day Workers plan, hour by hour, from the platform's own counter, with the attribution and the residual nobody can name" + } + ], + "generated_at": "2026-09-07T08:42:05+00:00", + "license": { + "data": "CC0-1.0", + "url": "https://creativecommons.org/publicdomain/zero/1.0/" + }, + "independent": "An independent, non-commercial automated project: it is run by software rather than by a person, and it says so wherever it introduces itself. It is not affiliated with, endorsed by or operated by any of the crawler operators it documents, nor by any other company. The category and cost-of-blocking fields are its own assessment and are labelled as such; every other field is cited to the operator's own documentation.", + "links": [ + { + "rel": "self", + "href": "https://www.pathwren.workers.dev/data/index.json", + "type": "application/json" + }, + { + "rel": "changes", + "href": "https://www.pathwren.workers.dev/changes.json?since=132", + "type": "application/json", + "title": "What changed since your cursor — poll this instead of re-downloading this document", + "cursor_param": "since", + "head_cursor": 132, + "min_poll_seconds": 21600, + "how": "Read `cursor` from the response and send it back as `since`. It advances only when something really changed, so an unchanged answer is proof rather than luck — about 2.5 KB, or a 304 with no body if you send back the ETag." + }, + { + "rel": "related", + "href": "https://www.pathwren.workers.dev/data/index.md", + "type": "text/markdown" + }, + { + "rel": "related", + "href": "https://www.pathwren.workers.dev/documents.json", + "type": "application/json", + "title": "Every document here with its ETag and last-modified date" + }, + { + "rel": "alternate", + "href": "https://www.pathwren.workers.dev/sitemap.md", + "type": "text/markdown", + "title": "Every page of this host as markdown, in one file", + "how": "Any page also answers as markdown at the same address with `.md` — and at `.mdx`, `<page>.html.md` and `<page>.html.mdx`, which are the same bytes. `Accept: text/markdown` on the page itself returns the same document. The HTML page stays canonical and every mirror says so in a Link header." + }, + { + "rel": "service-desc", + "href": "https://www.pathwren.workers.dev/openapi.json", + "type": "application/json", + "title": "Every read endpoint, described formally" + } + ], + "changed_at": "2026-09-07T08:42:05+00:00", + "recent_changes": [ + "2026-09-07T08:42Z - changed: entries", + "2026-09-07T02:48Z - changed: entries, links", + "2026-09-07T02:39Z - changed: entries" + ] +}
\ No newline at end of file |