path: root/agents.csv

slug,name,operator,category,robots_token,user_agent_substring,respects_robots_txt,verification_method,published_ip_ranges_url,ipv4_prefix_count,ipv6_prefix_count,operator_docs
gptbot,GPTBot,OpenAI,ai-training,GPTBot,GPTBot,documented,published-ranges,https://openai.com/gptbot.json,21,0,https://platform.openai.com/docs/bots
oai-searchbot,OAI-SearchBot,OpenAI,ai-search,OAI-SearchBot,OAI-SearchBot,documented,published-ranges,https://openai.com/searchbot.json,35,0,https://platform.openai.com/docs/bots
chatgpt-user,ChatGPT-User,OpenAI,user-fetch,ChatGPT-User,ChatGPT-User,documented,published-ranges,https://openai.com/chatgpt-user.json,207,0,https://platform.openai.com/docs/bots
claudebot,ClaudeBot,Anthropic,ai-training,ClaudeBot,ClaudeBot,documented,none,,0,0,https://support.anthropic.com/en/articles/8896518
claude-searchbot,Claude-SearchBot,Anthropic,ai-search,Claude-SearchBot,Claude-SearchBot,documented,none,,0,0,https://support.anthropic.com/en/articles/8896518
claude-user,Claude-User,Anthropic,user-fetch,Claude-User,Claude-User,documented,none,,0,0,https://support.anthropic.com/en/articles/8896518
anthropic-ai,anthropic-ai,Anthropic,ai-training,anthropic-ai,anthropic-ai,n-a,none,,0,0,https://support.anthropic.com/en/articles/8896518
claude-web,Claude-Web,Anthropic,ai-search,Claude-Web,Claude-Web,n-a,none,,0,0,https://support.anthropic.com/en/articles/8896518
google-extended,Google-Extended,Google,ai-training,Google-Extended,(control token only — no crawler),n-a,none,,0,0,https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers
googlebot,Googlebot,Google,search,Googlebot,Googlebot,documented,published-ranges,https://developers.google.com/static/search/apis/ipranges/googlebot.json,170,147,https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers
googleother,GoogleOther,Google,ai-training,GoogleOther,GoogleOther,documented,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers
google-cloudvertexbot,Google-CloudVertexBot,Google,ai-search,Google-CloudVertexBot,Google-CloudVertexBot,documented,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers
google-inspectiontool,Google-InspectionTool,Google,tool,Google-InspectionTool,Google-InspectionTool,documented,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers
googlebot-image,Googlebot-Image,Google,search,Googlebot-Image,Googlebot-Image,documented,published-ranges,https://developers.google.com/static/search/apis/ipranges/googlebot.json,170,147,https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers
googlebot-news,Googlebot-News,Google,search,Googlebot-News,Googlebot-News,documented,published-ranges,https://developers.google.com/static/search/apis/ipranges/googlebot.json,170,147,https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers
storebot-google,Storebot-Google,Google,search,Storebot-Google,Storebot-Google,documented,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers
bingbot,bingbot,Microsoft,search,bingbot,bingbot,documented,published-ranges,https://www.bing.com/toolbox/bingbot.json,28,0,https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0
applebot,Applebot,Apple,search,Applebot,Applebot,documented,published-ranges,https://search.developer.apple.com/applebot.json,33,0,https://support.apple.com/en-us/119829
applebot-extended,Applebot-Extended,Apple,ai-training,Applebot-Extended,(control token only — no crawler),n-a,none,,0,0,https://support.apple.com/en-us/119829
perplexitybot,PerplexityBot,Perplexity,ai-search,PerplexityBot,PerplexityBot,documented,published-ranges,https://www.perplexity.ai/perplexitybot.json,8,0,https://docs.perplexity.ai/guides/bots
perplexity-user,Perplexity-User,Perplexity,user-fetch,Perplexity-User,Perplexity-User,by-design-no,published-ranges,https://www.perplexity.ai/perplexity-user.json,4,0,https://docs.perplexity.ai/guides/bots
ccbot,CCBot,Common Crawl,dataset,CCBot,CCBot,documented,published-ranges,https://index.commoncrawl.org/ccbot.json,4,1,https://commoncrawl.org/ccbot
bytespider,Bytespider,ByteDance,ai-training,Bytespider,Bytespider,disputed,none,,0,0,https://www.bytespider.net/
tiktokspider,TikTokSpider,ByteDance,ai-training,TikTokSpider,TikTokSpider,disputed,none,,0,0,https://www.bytespider.net/
meta-externalagent,meta-externalagent,Meta,ai-training,meta-externalagent,meta-externalagent,documented,none,,0,0,https://developers.facebook.com/docs/sharing/webmasters/web-crawlers
meta-externalfetcher,meta-externalfetcher,Meta,user-fetch,meta-externalfetcher,meta-externalfetcher,documented,none,,0,0,https://developers.facebook.com/docs/sharing/webmasters/web-crawlers
facebookexternalhit,facebookexternalhit,Meta,preview,facebookexternalhit,facebookexternalhit,documented,none,,0,0,https://developers.facebook.com/docs/sharing/webmasters/web-crawlers
facebookbot,FacebookBot,Meta,ai-training,FacebookBot,FacebookBot,documented,none,,0,0,https://developers.facebook.com/docs/sharing/webmasters/web-crawlers
amazonbot,Amazonbot,Amazon,ai-search,Amazonbot,Amazonbot,documented,reverse-dns,,0,0,https://developer.amazon.com/amazonbot
duckassistbot,DuckAssistBot,DuckDuckGo,ai-search,DuckAssistBot,DuckAssistBot,documented,none,,0,0,https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/
duckduckbot,DuckDuckBot,DuckDuckGo,search,DuckDuckBot,DuckDuckBot,documented,published-ranges,https://duckduckgo.com/duckduckbot.json,486,0,https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/
ai2bot,AI2Bot,Allen Institute for AI,dataset,AI2Bot,AI2Bot,documented,none,,0,0,https://allenai.org/crawler
ai2bot-dolma,Ai2Bot-Dolma,Allen Institute for AI,dataset,Ai2Bot-Dolma,Ai2Bot-Dolma,documented,none,,0,0,https://allenai.org/crawler
cohere-ai,cohere-ai,Cohere,user-fetch,cohere-ai,cohere-ai,documented,none,,0,0,https://cohere.com/
cohere-training-data-crawler,cohere-training-data-crawler,Cohere,ai-training,cohere-training-data-crawler,cohere-training-data-crawler,documented,none,,0,0,https://cohere.com/
mistralai-user,MistralAI-User,Mistral AI,user-fetch,MistralAI-User,MistralAI-User,documented,none,,0,0,https://docs.mistral.ai/
youbot,YouBot,You.com,ai-search,YouBot,YouBot,documented,none,,0,0,https://about.you.com/youbot/
diffbot,Diffbot,Diffbot,dataset,Diffbot,Diffbot,documented,none,,0,0,https://docs.diffbot.com/
omgilibot,omgilibot,Webz.io,dataset,omgilibot,omgilibot,documented,none,,0,0,https://webz.io/blog/machine-learning/
omgili,omgili,Webz.io,dataset,omgili,omgili,documented,none,,0,0,https://webz.io/blog/machine-learning/
webzio-extended,Webzio-Extended,Webz.io,ai-training,Webzio-Extended,Webzio-Extended,documented,none,,0,0,https://webz.io/blog/machine-learning/
imagesiftbot,ImagesiftBot,Hive AI,dataset,ImagesiftBot,ImagesiftBot,documented,none,,0,0,https://imagesift.com/about
timpibot,Timpibot,Timpi,search,Timpibot,Timpibot,documented,none,,0,0,https://timpi.io/
semrushbot,SemrushBot,Semrush,seo,SemrushBot,SemrushBot,documented,reverse-dns,,0,0,https://www.semrush.com/bot/
semrushbot-ocob,SemrushBot-OCOB,Semrush,ai-training,SemrushBot-OCOB,SemrushBot-OCOB,documented,reverse-dns,,0,0,https://www.semrush.com/bot/
ahrefsbot,AhrefsBot,Ahrefs,seo,AhrefsBot,AhrefsBot,documented,published-ranges,https://api.ahrefs.com/v3/public/crawler-ip-ranges,81,0,https://ahrefs.com/robot
archive-org-bot,archive.org_bot,Internet Archive,archive,archive.org_bot,archive.org_bot,documented,none,,0,0,https://archive.org/details/archive.org_bot
ia-archiver,ia_archiver,Internet Archive,archive,ia_archiver,ia_archiver,documented,none,,0,0,https://archive.org/details/archive.org_bot
yandexbot,YandexBot,Yandex,search,YandexBot,YandexBot,documented,reverse-dns,,0,0,https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html
baiduspider,Baiduspider,Baidu,search,Baiduspider,Baiduspider,documented,reverse-dns,,0,0,https://help.baidu.com/question?prod_id=99&class=0&id=3001
seznambot,SeznamBot,Seznam,search,SeznamBot,SeznamBot,documented,reverse-dns,,0,0,https://napoveda.seznam.cz/en/seznamzbozi/subject-matter-crawler/
yeti,Yeti,Naver,search,Yeti,Yeti,documented,reverse-dns,,0,0,https://searchadvisor.naver.com/guide/seo-basic-crawl
petalbot,PetalBot,Huawei,search,PetalBot,PetalBot,documented,reverse-dns,,0,0,https://aspiegel.com/petalbot
firecrawlagent,FirecrawlAgent,Firecrawl,tool,FirecrawlAgent,FirecrawlAgent,documented,none,,0,0,https://docs.firecrawl.dev/
scrapy,Scrapy,Scrapy project,tool,Scrapy,Scrapy,documented,none,,0,0,https://scrapy.org/
img2dataset,img2dataset,LAION / img2dataset,dataset,img2dataset,img2dataset,documented,none,,0,0,https://github.com/rom1504/img2dataset
googlebot-video,Googlebot-Video,Google,search,Googlebot-Video,Googlebot-Video,documented,published-ranges,https://developers.google.com/static/search/apis/ipranges/googlebot.json,170,147,https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers
googleother-image,GoogleOther-Image,Google,ai-training,GoogleOther-Image,GoogleOther-Image,documented,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers
googleother-video,GoogleOther-Video,Google,ai-training,GoogleOther-Video,GoogleOther-Video,documented,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers
apis-google,APIs-Google,Google,tool,APIs-Google,APIs-Google,own-token-only,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers
adsbot-google,AdsBot-Google,Google,tool,AdsBot-Google,AdsBot-Google,own-token-only,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers
adsbot-google-mobile,AdsBot-Google-Mobile,Google,tool,AdsBot-Google-Mobile,AdsBot-Google-Mobile,own-token-only,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers
adsbot-google-mobile-apps,AdsBot-Google-Mobile-Apps,Google,tool,AdsBot-Google-Mobile-Apps,AdsBot-Google-Mobile-Apps,own-token-only,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers
mediapartners-google,Mediapartners-Google,Google,tool,Mediapartners-Google,Mediapartners-Google,own-token-only,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers
google-safety,Google-Safety,Google,tool,Google-Safety,Google-Safety,by-design-no,none,,0,0,https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers
feedfetcher-google,FeedFetcher-Google,Google,tool,FeedFetcher-Google,FeedFetcher-Google,by-design-no,published-ranges,https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json,529,529,https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers
google-read-aloud,Google-Read-Aloud,Google,user-fetch,Google-Read-Aloud,Google-Read-Aloud,by-design-no,published-ranges,https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json,529,529,https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers
google-site-verification,Google-Site-Verification,Google,tool,Google-Site-Verification,Google-Site-Verification,by-design-no,published-ranges,https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json,529,529,https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers
google-cws,Google-CWS,Google,tool,Google-CWS,Google-CWS,by-design-no,published-ranges,https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json,529,529,https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers
google-pinpoint,Google-Pinpoint,Google,user-fetch,Google-Pinpoint,Google-Pinpoint,by-design-no,published-ranges,https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json,529,529,https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers
googleproducer,GoogleProducer,Google,tool,GoogleProducer,GoogleProducer,by-design-no,published-ranges,https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json,529,529,https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers
googlemessages,GoogleMessages,Google,preview,GoogleMessages,GoogleMessages,by-design-no,published-ranges,https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json,529,529,https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers
google-gemininotebook,Google-GeminiNotebook,Google,user-fetch,Google-GeminiNotebook,Google-GeminiNotebook,by-design-no,published-ranges,https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json,529,529,https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers
google-agent,Google-Agent,Google,user-fetch,Google-Agent,Google-Agent,by-design-no,published-ranges,https://developers.google.com/static/search/apis/ipranges/user-triggered-agents.json,3,1,https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers
yandeximages,YandexImages,Yandex,search,YandexImages,YandexImages,documented,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots
yandexvideo,YandexVideo,Yandex,search,YandexVideo,YandexVideo,documented,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots
yandexmedia,YandexMedia,Yandex,search,YandexMedia,YandexMedia,documented,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots
yandexblogs,YandexBlogs,Yandex,search,YandexBlogs,YandexBlogs,documented,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots
yandexmarket,YandexMarket,Yandex,search,YandexMarket,YandexMarket,documented,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots
yandexwebmaster,YandexWebmaster,Yandex,tool,YandexWebmaster,YandexWebmaster,documented,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots
yandexmobilebot,YandexMobileBot,Yandex,search,YandexMobileBot,YandexMobileBot,own-token-only,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots
yandexfavicons,YandexFavicons,Yandex,search,YandexFavicons,YandexFavicons,own-token-only,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots
yandexcalendar,YandexCalendar,Yandex,user-fetch,YandexCalendar,YandexCalendar,own-token-only,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots
yandexdirect,YandexDirect,Yandex,tool,YandexDirect,YandexDirect,own-token-only,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots
yandexmetrika,YandexMetrika,Yandex,tool,YandexMetrika,YandexMetrika,by-design-no,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots
yandexrenderresourcesbot,YandexRenderResourcesBot,Yandex,search,YandexRenderResourcesBot,YandexRenderResourcesBot,own-token-only,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots
yandexscreenshotbot,YandexScreenshotBot,Yandex,tool,YandexScreenshotBot,YandexScreenshotBot,own-token-only,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots
yandexadditional,YandexAdditional,Yandex,ai-training,YandexAdditional,YandexAdditional,own-token-only,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots
yandexadditionalbot,YandexAdditionalBot,Yandex,ai-training,YandexAdditionalBot,YandexAdditionalBot,own-token-only,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots
yandexcombot,YandexComBot,Yandex,search,YandexComBot,YandexComBot,own-token-only,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots
siteauditbot,SiteAuditBot,Semrush,seo,SiteAuditBot,SiteAuditBot,documented,none,,0,0,https://www.semrush.com/bot/
semrushbot-ba,SemrushBot-BA,Semrush,seo,SemrushBot-BA,SemrushBot-BA,documented,none,,0,0,https://www.semrush.com/bot/
semrushbot-si,SemrushBot-SI,Semrush,seo,SemrushBot-SI,SemrushBot-SI,documented,none,,0,0,https://www.semrush.com/bot/
semrushbot-swa,SemrushBot-SWA,Semrush,seo,SemrushBot-SWA,SemrushBot-SWA,documented,none,,0,0,https://www.semrush.com/bot/
splitsignalbot,SplitSignalBot,Semrush,seo,SplitSignalBot,SplitSignalBot,documented,none,,0,0,https://www.semrush.com/bot/
semrushbot-ft,SemrushBot-FT,Semrush,seo,SemrushBot-FT,SemrushBot-FT,documented,none,,0,0,https://www.semrush.com/bot/
semrushbot-esi,SemrushBot-ESI,Semrush,seo,SemrushBot-ESI,SemrushBot-ESI,documented,none,,0,0,https://www.semrush.com/bot/
ahrefssiteaudit,AhrefsSiteAudit,Ahrefs,seo,AhrefsSiteAudit,AhrefsSiteAudit,documented,published-ranges,https://api.ahrefs.com/v3/public/crawler-ip-ranges,81,0,https://ahrefs.com/robot
mj12bot,MJ12bot,Majestic,seo,MJ12bot,MJ12bot,documented,none,,0,0,https://mj12bot.com/
dotbot,DotBot,Moz,seo,dotbot,dotbot,documented,none,,0,0,https://moz.com/help/moz-procedures/crawlers/dotbot
rogerbot,rogerbot,Moz,seo,rogerbot,rogerbot,documented,none,,0,0,https://moz.com/help/moz-procedures/crawlers/rogerbot
dataforseobot,DataForSeoBot,DataForSEO,seo,DataForSeoBot,DataForSeoBot,documented,none,,0,0,https://dataforseo.com/dataforseo-bot
serpstatbot,serpstatbot,Serpstat,seo,serpstatbot,serpstatbot,documented,none,,0,0,https://serpstatbot.com/
barkrowler,Barkrowler,Babbar,seo,barkrowler,barkrowler,documented,none,,0,0,https://babbar.tech/crawler
screaming-frog-seo-spider,Screaming Frog SEO Spider,Screaming Frog,tool,Screaming Frog SEO Spider,Screaming Frog SEO Spider,documented,none,,0,0,https://www.screamingfrog.co.uk/seo-spider/user-agent/
seokicks,SEOkicks,SEOkicks,seo,SEOkicks,SEOkicks,documented,none,,0,0,https://www.seokicks.de/robot.html
mojeekbot,MojeekBot,Mojeek,search,MojeekBot,MojeekBot,documented,none,,0,0,https://www.mojeek.com/bot.html
kagibot,Kagibot,Kagi,search,Kagibot,Kagibot,documented,none,,0,0,https://kagi.com/bot
qwantbot,Qwantbot,Qwant,search,Qwantbot,Qwantbot,documented,none,,0,0,https://help.qwant.com/bot/
qwantbot-news,Qwantbot-news,Qwant,search,Qwantbot-news,Qwantbot-news,documented,none,,0,0,https://help.qwant.com/bot/
slackbot-linkexpanding,Slackbot-LinkExpanding,Slack,preview,Slackbot-LinkExpanding,Slackbot-LinkExpanding,documented,none,,0,0,https://api.slack.com/robots
slackbot,Slackbot,Slack,preview,Slackbot,Slackbot,documented,none,,0,0,https://api.slack.com/robots
pinterestbot,Pinterestbot,Pinterest,search,Pinterestbot,Pinterestbot,documented,none,,0,0,https://help.pinterest.com/en/business/article/pinterest-crawler
bedrockbot,bedrockbot,Amazon,ai-search,bedrockbot,bedrockbot,documented,none,,0,0,https://docs.aws.amazon.com/bedrock/latest/userguide/webcrawl-data-source-connector.html
cloudflare-autorag,Cloudflare-AutoRAG,Cloudflare,ai-search,Cloudflare-AutoRAG,Cloudflare-AutoRAG,documented,none,,0,0,https://developers.cloudflare.com/ai-search/
exasearchbot,ExaSearchBot,Exa,ai-search,ExaSearchBot,ExaSearchBot,undocumented,none,,0,0,https://exa.ai
shapbot,ShapBot,Parallel,ai-search,ShapBot,ShapBot,documented,none,,0,0,https://docs.parallel.ai/features/crawler
terracotta,TerraCotta,Ceramic AI,ai-search,TerraCotta,TerraCotta,documented,none,,0,0,https://ceramic.ai/
crawlspace,Crawlspace,Crawlspace,tool,Crawlspace,Crawlspace,documented,none,,0,0,https://crawlspace.dev
panscient,Panscient,Panscient,dataset,panscient.com,panscient.com,documented,none,,0,0,https://panscient.com/faq.htm
sbintuitionsbot,SBIntuitionsBot,SB Intuitions,ai-training,SBIntuitionsBot,SBIntuitionsBot,documented,none,,0,0,https://www.sbintuitions.co.jp/en/bot/
icc-crawler,ICC-Crawler,NICT,ai-training,ICC-Crawler,ICC-Crawler,documented,none,,0,0,https://www.nict.go.jp/en/
cotoyogi,Cotoyogi,ROIS-DS,ai-training,Cotoyogi,Cotoyogi,documented,none,,0,0,https://ds.rois.ac.jp/en_center8/en_crawler/
isscyberriskcrawler,ISSCyberRiskCrawler,ISS Corporate Solutions,ai-training,ISSCyberRiskCrawler,ISSCyberRiskCrawler,disputed,none,,0,0,https://iss-cyber.com
sidetrade-indexer-bot,Sidetrade indexer bot,Sidetrade,ai-training,Sidetrade indexer bot,Sidetrade indexer bot,undocumented,none,,0,0,https://www.sidetrade.com
yak,YaK,Meltwater,dataset,YaK,YaK,undocumented,none,,0,0,https://www.meltwater.com/en/suite/consumer-intelligence
atlassian-bot,atlassian-bot,Atlassian,ai-search,atlassian-bot,atlassian-bot,documented,none,,0,0,https://support.atlassian.com/organization-administration/docs/connect-custom-website-to-rovo/
klaviyoaibot,KlaviyoAIBot,Klaviyo,ai-search,KlaviyoAIBot,KlaviyoAIBot,documented,none,,0,0,https://help.klaviyo.com/hc/en-us/articles/40496146232219
quillbot,QuillBot,QuillBot,ai-training,QuillBot,QuillBot,undocumented,none,,0,0,https://quillbot.com
phindbot,PhindBot,Phind,ai-search,PhindBot,PhindBot,undocumented,none,,0,0,https://www.phind.com/
andibot,Andibot,Andi,ai-search,Andibot,Andibot,undocumented,none,,0,0,https://andisearch.com/
anomura,Anomura,Direqt,ai-search,Anomura,Anomura,documented,none,,0,0,https://direqt.ai
aiwebindex,AIWebIndex,Lyrenth,ai-search,AIWebIndex,AIWebIndex,documented,none,,0,0,https://lyrenth.com/crawler-policy
factset-spyderbot,Factset_spyderbot,FactSet,ai-training,Factset_spyderbot,Factset_spyderbot,undocumented,none,,0,0,https://www.factset.com/ai
poseidon-research-crawler,Poseidon Research Crawler,Poseidon Research,ai-training,Poseidon Research Crawler,Poseidon Research Crawler,undocumented,none,,0,0,https://www.poseidonresearch.com
qualifiedbot,QualifiedBot,Qualified,ai-search,QualifiedBot,QualifiedBot,undocumented,none,,0,0,https://www.qualified.com
reflectionbot,Reflectionbot,Reflection AI,ai-training,Reflectionbot,Reflectionbot,undocumented,none,,0,0,https://reflection.ai/
thinkbot,Thinkbot,Thinkbot,dataset,Thinkbot,Thinkbot,disputed,none,,0,0,https://www.thinkbot.agency
aihitbot,aiHitBot,aiHit,dataset,aiHitBot,aiHitBot,documented,none,,0,0,https://www.aihitdata.com/about
linguee-bot,Linguee Bot,Linguee,ai-training,Linguee Bot,Linguee Bot,disputed,none,,0,0,https://www.linguee.com
lightpanda,Lightpanda,Lightpanda,tool,Lightpanda,Lightpanda,undocumented,none,,0,0,https://lightpanda.io/
laiondownloader,LAIONDownloader,LAION / img2dataset,dataset,LAIONDownloader,LAIONDownloader,by-design-no,none,,0,0,https://laion.ai/faq/
velenpublicwebcrawler,VelenPublicWebCrawler,Hunter (Velen),dataset,VelenPublicWebCrawler,VelenPublicWebCrawler,documented,none,,0,0,https://velen.io/
awariosmartbot,AwarioSmartBot,Awario,dataset,AwarioSmartBot,AwarioSmartBot,documented,none,,0,0,https://awario.com/bots.html
awariorssbot,AwarioRssBot,Awario,dataset,AwarioRssBot,AwarioRssBot,documented,none,,0,0,https://awario.com/bots.html
echoboxbot,EchoboxBot,Echobox,dataset,EchoboxBot,EchoboxBot,undocumented,none,,0,0,https://echobox.com
meta-webindexer,Meta-WebIndexer,Meta,ai-search,Meta-WebIndexer,Meta-WebIndexer,undocumented,none,,0,0,https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/
chatgpt-agent,ChatGPT Agent,OpenAI,user-fetch,ChatGPT-User,ChatGPT Agent,documented,published-ranges,https://openai.com/chatgpt-user.json,207,0,https://platform.openai.com/docs/bots
wpbot,wpbot,QuantumCloud,tool,wpbot,wpbot,undocumented,none,,0,0,https://www.quantumcloud.com
crawl4ai,Crawl4AI,Crawl4AI project,tool,Crawl4AI,Crawl4AI,undocumented,none,,0,0,https://github.com/unclecode/crawl4ai