From 6ce86768d4b2287c23754536e33fcf262578a0a5 Mon Sep 17 00:00:00 2001
From: hrbrmstr <bob@rud.is>
Date: Mon, 7 Sep 2026 10:09:24 -0400
Subject: chore: initial commit

---
 robots-tokens.txt | 151 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 151 insertions(+)
 create mode 100644 robots-tokens.txt

(limited to 'robots-tokens.txt')

diff --git a/robots-tokens.txt b/robots-tokens.txt
new file mode 100644
index 0000000..377e147
--- /dev/null
+++ b/robots-tokens.txt
@@ -0,0 +1,151 @@
+# AI Crawler Index — robots.txt user-agent tokens, one per line, 2026-09-07
+# https://www.pathwren.workers.dev/data/robots-tokens.txt
+AdsBot-Google
+AdsBot-Google-Mobile
+AdsBot-Google-Mobile-Apps
+AhrefsBot
+AhrefsSiteAudit
+AI2Bot
+Ai2Bot-Dolma
+aiHitBot
+AIWebIndex
+Amazonbot
+Andibot
+Anomura
+anthropic-ai
+APIs-Google
+Applebot
+Applebot-Extended
+archive.org_bot
+atlassian-bot
+AwarioRssBot
+AwarioSmartBot
+Baiduspider
+barkrowler
+bedrockbot
+bingbot
+Bytespider
+CCBot
+ChatGPT-User
+Claude-SearchBot
+Claude-User
+Claude-Web
+ClaudeBot
+Cloudflare-AutoRAG
+cohere-ai
+cohere-training-data-crawler
+Cotoyogi
+Crawl4AI
+Crawlspace
+DataForSeoBot
+Diffbot
+dotbot
+DuckAssistBot
+DuckDuckBot
+EchoboxBot
+ExaSearchBot
+FacebookBot
+facebookexternalhit
+Factset_spyderbot
+FeedFetcher-Google
+FirecrawlAgent
+Google-Agent
+Google-CloudVertexBot
+Google-CWS
+Google-Extended
+Google-GeminiNotebook
+Google-InspectionTool
+Google-Pinpoint
+Google-Read-Aloud
+Google-Safety
+Google-Site-Verification
+Googlebot
+Googlebot-Image
+Googlebot-News
+Googlebot-Video
+GoogleMessages
+GoogleOther
+GoogleOther-Image
+GoogleOther-Video
+GoogleProducer
+GPTBot
+ia_archiver
+ICC-Crawler
+ImagesiftBot
+img2dataset
+ISSCyberRiskCrawler
+Kagibot
+KlaviyoAIBot
+LAIONDownloader
+Lightpanda
+Linguee Bot
+Mediapartners-Google
+meta-externalagent
+meta-externalfetcher
+Meta-WebIndexer
+MistralAI-User
+MJ12bot
+MojeekBot
+OAI-SearchBot
+omgili
+omgilibot
+panscient.com
+Perplexity-User
+PerplexityBot
+PetalBot
+PhindBot
+Pinterestbot
+Poseidon Research Crawler
+QualifiedBot
+QuillBot
+Qwantbot
+Qwantbot-news
+Reflectionbot
+rogerbot
+SBIntuitionsBot
+Scrapy
+Screaming Frog SEO Spider
+SemrushBot
+SemrushBot-BA
+SemrushBot-ESI
+SemrushBot-FT
+SemrushBot-OCOB
+SemrushBot-SI
+SemrushBot-SWA
+SEOkicks
+serpstatbot
+SeznamBot
+ShapBot
+Sidetrade indexer bot
+SiteAuditBot
+Slackbot
+Slackbot-LinkExpanding
+SplitSignalBot
+Storebot-Google
+TerraCotta
+Thinkbot
+TikTokSpider
+Timpibot
+VelenPublicWebCrawler
+Webzio-Extended
+wpbot
+YaK
+YandexAdditional
+YandexAdditionalBot
+YandexBlogs
+YandexBot
+YandexCalendar
+YandexComBot
+YandexDirect
+YandexFavicons
+YandexImages
+YandexMarket
+YandexMedia
+YandexMetrika
+YandexMobileBot
+YandexRenderResourcesBot
+YandexScreenshotBot
+YandexVideo
+YandexWebmaster
+Yeti
+YouBot
-- 
cgit v1.2.3