diff options
Diffstat (limited to 'markdown-lane.json')
| -rw-r--r-- | markdown-lane.json | 207 |
1 files changed, 207 insertions, 0 deletions
diff --git a/markdown-lane.json b/markdown-lane.json new file mode 100644 index 0000000..0fd8c62 --- /dev/null +++ b/markdown-lane.json @@ -0,0 +1,207 @@ +{ + "generated_at": "2026-09-07T08:15:50+00:00", + "published_at": "https://www.pathwren.workers.dev/data/markdown-lane.json", + "what": "external clients that fetched a markdown ADDRESS (.md/.mdx/.html.md/.html.mdx) or sent Accept: text/markdown to a page of this host, counted from its own request log", + "license": "CC0-1.0", + "self_excluded": "our own checks (X-Self: 1) never count toward any number here, and the self-fetch watch's flagged pairs are dropped on top of that", + "why_this_file_exists": "by_surface_24h.markdown-mirrors is declared_not_observed and can never be a reading: src/surface_table.js books every mirror to the column of the PAGE it renders, on purpose, because one document at two addresses is one reader. This slice reports the lane without re-attributing a single row.", + "window_24h": { + "from": "2026-09-06T08:15:50+00:00", + "to": "2026-09-07T08:15:50+00:00", + "hours": 24.0, + "hits": 1452, + "client_keys": 199, + "distinct_paths": 1166, + "share_of_host_client_keys": 0.081, + "host_client_keys": 2458, + "by_door": { + "address": 1445, + "accept_header": 7 + }, + "by_class": { + "crawler": 1426, + "unknown": 11, + "human": 14, + "agent": 1 + }, + "llms_txt_hits": 33, + "llms_txt_client_keys": 20 + }, + "window_7d": { + "from": "2026-08-31T08:15:50+00:00", + "to": "2026-09-07T08:15:50+00:00", + "hours": 168.0, + "hits": 4966, + "client_keys": 646, + "distinct_paths": 1749, + "share_of_host_client_keys": 0.0965, + "host_client_keys": 6694, + "by_door": { + "address": 4201, + "accept_header": 765 + }, + "by_class": { + "agent": 405, + "crawler": 4514, + "unknown": 20, + "human": 27 + }, + "llms_txt_hits": 215, + "llms_txt_client_keys": 90 + }, + "top_paths_24h": [ + { + "path": "/sitemap.md", + "hits": 11 + }, + { + "path": "/policy/index.md", + "hits": 5 + }, + { + "path": "/blog/ai-crawler-traffic-2026-w36.md", + "hits": 5 + }, + { + "path": "/crawler/index.md", + "hits": 4 + }, + { + "path": "/index.md", + "hits": 4 + }, + { + "path": "/bot/dataforseobot.md", + "hits": 4 + }, + { + "path": "/crawler/adsbot-google-mobile-apps.md", + "hits": 3 + }, + { + "path": "/about.md", + "hits": 3 + }, + { + "path": "/bot/siteguardian.md", + "hits": 3 + }, + { + "path": "/bot/apis-json-showcase-verifier.md", + "hits": 3 + }, + { + "path": "/bot/semrushbot.md", + "hits": 3 + }, + { + "path": "/bot/chainwitness-pr36.md", + "hits": 3 + } + ], + "top_agents_24h": [ + { + "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.4; +https://openai", + "hits": 908 + }, + { + "ua": "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)", + "hits": 196 + }, + { + "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@a", + "hits": 148 + }, + { + "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1; +https://dev", + "hits": 69 + }, + { + "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/14", + "hits": 48 + }, + { + "ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chr", + "hits": 22 + }, + { + "ua": "SofyaBot/1.0 (+https://sofya.co/bot)", + "hits": 17 + }, + { + "ua": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like", + "hits": 8 + }, + { + "ua": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Sa", + "hits": 5 + }, + { + "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/15", + "hits": 5 + } + ], + "how_to_read_this_host_as_markdown": { + "append_md": "any page, plus .md — e.g. https://www.pathwren.workers.dev/mcp-markdown.md", + "also_accepted": [ + ".mdx", + "<page>.html.md", + "<page>.html.mdx" + ], + "content_negotiation": "Accept: text/markdown on the .html address", + "index": "https://www.pathwren.workers.dev/sitemap.md", + "llms_txt": "https://www.pathwren.workers.dev/llms.txt" + }, + "check_your_own_site": { + "endpoint": "https://www.pathwren.workers.dev/mcp/markdown", + "tool": "check_markdown_lane", + "docs": "https://www.pathwren.workers.dev/mcp-markdown.html" + }, + "links": [ + { + "rel": "self", + "href": "https://www.pathwren.workers.dev/data/markdown-lane.json", + "type": "application/json" + }, + { + "rel": "changes", + "href": "https://www.pathwren.workers.dev/changes.json?since=132", + "type": "application/json", + "title": "What changed since your cursor — poll this instead of re-downloading this document", + "cursor_param": "since", + "head_cursor": 132, + "min_poll_seconds": 21600, + "how": "Read `cursor` from the response and send it back as `since`. It advances only when something really changed, so an unchanged answer is proof rather than luck — about 2.5 KB, or a 304 with no body if you send back the ETag." + }, + { + "rel": "related", + "href": "https://www.pathwren.workers.dev/sitemap.md", + "type": "text/markdown" + }, + { + "rel": "related", + "href": "https://www.pathwren.workers.dev/llms.txt", + "type": "text/plain", + "title": "The whole map in one text file" + }, + { + "rel": "alternate", + "href": "https://www.pathwren.workers.dev/sitemap.md", + "type": "text/markdown", + "title": "Every page of this host as markdown, in one file", + "how": "Any page also answers as markdown at the same address with `.md` — and at `.mdx`, `<page>.html.md` and `<page>.html.mdx`, which are the same bytes. `Accept: text/markdown` on the page itself returns the same document. The HTML page stays canonical and every mirror says so in a Link header." + }, + { + "rel": "service-desc", + "href": "https://www.pathwren.workers.dev/openapi.json", + "type": "application/json", + "title": "Every read endpoint, described formally" + } + ], + "changed_at": "2026-09-07T08:42:05+00:00", + "recent_changes": [ + "2026-09-07T08:42Z - changed: window_24h, window_7d, top_paths_24h, top_agents_24h; 5844->5860 bytes", + "2026-09-07T02:48Z - changed: links", + "2026-09-07T02:39Z - changed: window_24h, window_7d, top_paths_24h, top_agents_24h; 5765->5844 bytes" + ] +}
\ No newline at end of file |