diff options
Diffstat (limited to 'agents.csv')
| -rw-r--r-- | agents.csv | 151 |
1 files changed, 151 insertions, 0 deletions
diff --git a/agents.csv b/agents.csv new file mode 100644 index 0000000..711daa3 --- /dev/null +++ b/agents.csv @@ -0,0 +1,151 @@ +slug,name,operator,category,robots_token,user_agent_substring,respects_robots_txt,verification_method,published_ip_ranges_url,ipv4_prefix_count,ipv6_prefix_count,operator_docs +gptbot,GPTBot,OpenAI,ai-training,GPTBot,GPTBot,documented,published-ranges,https://openai.com/gptbot.json,21,0,https://platform.openai.com/docs/bots +oai-searchbot,OAI-SearchBot,OpenAI,ai-search,OAI-SearchBot,OAI-SearchBot,documented,published-ranges,https://openai.com/searchbot.json,35,0,https://platform.openai.com/docs/bots +chatgpt-user,ChatGPT-User,OpenAI,user-fetch,ChatGPT-User,ChatGPT-User,documented,published-ranges,https://openai.com/chatgpt-user.json,207,0,https://platform.openai.com/docs/bots +claudebot,ClaudeBot,Anthropic,ai-training,ClaudeBot,ClaudeBot,documented,none,,0,0,https://support.anthropic.com/en/articles/8896518 +claude-searchbot,Claude-SearchBot,Anthropic,ai-search,Claude-SearchBot,Claude-SearchBot,documented,none,,0,0,https://support.anthropic.com/en/articles/8896518 +claude-user,Claude-User,Anthropic,user-fetch,Claude-User,Claude-User,documented,none,,0,0,https://support.anthropic.com/en/articles/8896518 +anthropic-ai,anthropic-ai,Anthropic,ai-training,anthropic-ai,anthropic-ai,n-a,none,,0,0,https://support.anthropic.com/en/articles/8896518 +claude-web,Claude-Web,Anthropic,ai-search,Claude-Web,Claude-Web,n-a,none,,0,0,https://support.anthropic.com/en/articles/8896518 +google-extended,Google-Extended,Google,ai-training,Google-Extended,(control token only — no crawler),n-a,none,,0,0,https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers +googlebot,Googlebot,Google,search,Googlebot,Googlebot,documented,published-ranges,https://developers.google.com/static/search/apis/ipranges/googlebot.json,170,147,https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers +googleother,GoogleOther,Google,ai-training,GoogleOther,GoogleOther,documented,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers +google-cloudvertexbot,Google-CloudVertexBot,Google,ai-search,Google-CloudVertexBot,Google-CloudVertexBot,documented,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers +google-inspectiontool,Google-InspectionTool,Google,tool,Google-InspectionTool,Google-InspectionTool,documented,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers +googlebot-image,Googlebot-Image,Google,search,Googlebot-Image,Googlebot-Image,documented,published-ranges,https://developers.google.com/static/search/apis/ipranges/googlebot.json,170,147,https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers +googlebot-news,Googlebot-News,Google,search,Googlebot-News,Googlebot-News,documented,published-ranges,https://developers.google.com/static/search/apis/ipranges/googlebot.json,170,147,https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers +storebot-google,Storebot-Google,Google,search,Storebot-Google,Storebot-Google,documented,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers +bingbot,bingbot,Microsoft,search,bingbot,bingbot,documented,published-ranges,https://www.bing.com/toolbox/bingbot.json,28,0,https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0 +applebot,Applebot,Apple,search,Applebot,Applebot,documented,published-ranges,https://search.developer.apple.com/applebot.json,33,0,https://support.apple.com/en-us/119829 +applebot-extended,Applebot-Extended,Apple,ai-training,Applebot-Extended,(control token only — no crawler),n-a,none,,0,0,https://support.apple.com/en-us/119829 +perplexitybot,PerplexityBot,Perplexity,ai-search,PerplexityBot,PerplexityBot,documented,published-ranges,https://www.perplexity.ai/perplexitybot.json,8,0,https://docs.perplexity.ai/guides/bots +perplexity-user,Perplexity-User,Perplexity,user-fetch,Perplexity-User,Perplexity-User,by-design-no,published-ranges,https://www.perplexity.ai/perplexity-user.json,4,0,https://docs.perplexity.ai/guides/bots +ccbot,CCBot,Common Crawl,dataset,CCBot,CCBot,documented,published-ranges,https://index.commoncrawl.org/ccbot.json,4,1,https://commoncrawl.org/ccbot +bytespider,Bytespider,ByteDance,ai-training,Bytespider,Bytespider,disputed,none,,0,0,https://www.bytespider.net/ +tiktokspider,TikTokSpider,ByteDance,ai-training,TikTokSpider,TikTokSpider,disputed,none,,0,0,https://www.bytespider.net/ +meta-externalagent,meta-externalagent,Meta,ai-training,meta-externalagent,meta-externalagent,documented,none,,0,0,https://developers.facebook.com/docs/sharing/webmasters/web-crawlers +meta-externalfetcher,meta-externalfetcher,Meta,user-fetch,meta-externalfetcher,meta-externalfetcher,documented,none,,0,0,https://developers.facebook.com/docs/sharing/webmasters/web-crawlers +facebookexternalhit,facebookexternalhit,Meta,preview,facebookexternalhit,facebookexternalhit,documented,none,,0,0,https://developers.facebook.com/docs/sharing/webmasters/web-crawlers +facebookbot,FacebookBot,Meta,ai-training,FacebookBot,FacebookBot,documented,none,,0,0,https://developers.facebook.com/docs/sharing/webmasters/web-crawlers +amazonbot,Amazonbot,Amazon,ai-search,Amazonbot,Amazonbot,documented,reverse-dns,,0,0,https://developer.amazon.com/amazonbot +duckassistbot,DuckAssistBot,DuckDuckGo,ai-search,DuckAssistBot,DuckAssistBot,documented,none,,0,0,https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/ +duckduckbot,DuckDuckBot,DuckDuckGo,search,DuckDuckBot,DuckDuckBot,documented,published-ranges,https://duckduckgo.com/duckduckbot.json,486,0,https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/ +ai2bot,AI2Bot,Allen Institute for AI,dataset,AI2Bot,AI2Bot,documented,none,,0,0,https://allenai.org/crawler +ai2bot-dolma,Ai2Bot-Dolma,Allen Institute for AI,dataset,Ai2Bot-Dolma,Ai2Bot-Dolma,documented,none,,0,0,https://allenai.org/crawler +cohere-ai,cohere-ai,Cohere,user-fetch,cohere-ai,cohere-ai,documented,none,,0,0,https://cohere.com/ +cohere-training-data-crawler,cohere-training-data-crawler,Cohere,ai-training,cohere-training-data-crawler,cohere-training-data-crawler,documented,none,,0,0,https://cohere.com/ +mistralai-user,MistralAI-User,Mistral AI,user-fetch,MistralAI-User,MistralAI-User,documented,none,,0,0,https://docs.mistral.ai/ +youbot,YouBot,You.com,ai-search,YouBot,YouBot,documented,none,,0,0,https://about.you.com/youbot/ +diffbot,Diffbot,Diffbot,dataset,Diffbot,Diffbot,documented,none,,0,0,https://docs.diffbot.com/ +omgilibot,omgilibot,Webz.io,dataset,omgilibot,omgilibot,documented,none,,0,0,https://webz.io/blog/machine-learning/ +omgili,omgili,Webz.io,dataset,omgili,omgili,documented,none,,0,0,https://webz.io/blog/machine-learning/ +webzio-extended,Webzio-Extended,Webz.io,ai-training,Webzio-Extended,Webzio-Extended,documented,none,,0,0,https://webz.io/blog/machine-learning/ +imagesiftbot,ImagesiftBot,Hive AI,dataset,ImagesiftBot,ImagesiftBot,documented,none,,0,0,https://imagesift.com/about +timpibot,Timpibot,Timpi,search,Timpibot,Timpibot,documented,none,,0,0,https://timpi.io/ +semrushbot,SemrushBot,Semrush,seo,SemrushBot,SemrushBot,documented,reverse-dns,,0,0,https://www.semrush.com/bot/ +semrushbot-ocob,SemrushBot-OCOB,Semrush,ai-training,SemrushBot-OCOB,SemrushBot-OCOB,documented,reverse-dns,,0,0,https://www.semrush.com/bot/ +ahrefsbot,AhrefsBot,Ahrefs,seo,AhrefsBot,AhrefsBot,documented,published-ranges,https://api.ahrefs.com/v3/public/crawler-ip-ranges,81,0,https://ahrefs.com/robot +archive-org-bot,archive.org_bot,Internet Archive,archive,archive.org_bot,archive.org_bot,documented,none,,0,0,https://archive.org/details/archive.org_bot +ia-archiver,ia_archiver,Internet Archive,archive,ia_archiver,ia_archiver,documented,none,,0,0,https://archive.org/details/archive.org_bot +yandexbot,YandexBot,Yandex,search,YandexBot,YandexBot,documented,reverse-dns,,0,0,https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html +baiduspider,Baiduspider,Baidu,search,Baiduspider,Baiduspider,documented,reverse-dns,,0,0,https://help.baidu.com/question?prod_id=99&class=0&id=3001 +seznambot,SeznamBot,Seznam,search,SeznamBot,SeznamBot,documented,reverse-dns,,0,0,https://napoveda.seznam.cz/en/seznamzbozi/subject-matter-crawler/ +yeti,Yeti,Naver,search,Yeti,Yeti,documented,reverse-dns,,0,0,https://searchadvisor.naver.com/guide/seo-basic-crawl +petalbot,PetalBot,Huawei,search,PetalBot,PetalBot,documented,reverse-dns,,0,0,https://aspiegel.com/petalbot +firecrawlagent,FirecrawlAgent,Firecrawl,tool,FirecrawlAgent,FirecrawlAgent,documented,none,,0,0,https://docs.firecrawl.dev/ +scrapy,Scrapy,Scrapy project,tool,Scrapy,Scrapy,documented,none,,0,0,https://scrapy.org/ +img2dataset,img2dataset,LAION / img2dataset,dataset,img2dataset,img2dataset,documented,none,,0,0,https://github.com/rom1504/img2dataset +googlebot-video,Googlebot-Video,Google,search,Googlebot-Video,Googlebot-Video,documented,published-ranges,https://developers.google.com/static/search/apis/ipranges/googlebot.json,170,147,https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers +googleother-image,GoogleOther-Image,Google,ai-training,GoogleOther-Image,GoogleOther-Image,documented,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers +googleother-video,GoogleOther-Video,Google,ai-training,GoogleOther-Video,GoogleOther-Video,documented,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers +apis-google,APIs-Google,Google,tool,APIs-Google,APIs-Google,own-token-only,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers +adsbot-google,AdsBot-Google,Google,tool,AdsBot-Google,AdsBot-Google,own-token-only,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers +adsbot-google-mobile,AdsBot-Google-Mobile,Google,tool,AdsBot-Google-Mobile,AdsBot-Google-Mobile,own-token-only,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers +adsbot-google-mobile-apps,AdsBot-Google-Mobile-Apps,Google,tool,AdsBot-Google-Mobile-Apps,AdsBot-Google-Mobile-Apps,own-token-only,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers +mediapartners-google,Mediapartners-Google,Google,tool,Mediapartners-Google,Mediapartners-Google,own-token-only,published-ranges,https://developers.google.com/static/search/apis/ipranges/special-crawlers.json,136,136,https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers +google-safety,Google-Safety,Google,tool,Google-Safety,Google-Safety,by-design-no,none,,0,0,https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers +feedfetcher-google,FeedFetcher-Google,Google,tool,FeedFetcher-Google,FeedFetcher-Google,by-design-no,published-ranges,https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json,529,529,https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers +google-read-aloud,Google-Read-Aloud,Google,user-fetch,Google-Read-Aloud,Google-Read-Aloud,by-design-no,published-ranges,https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json,529,529,https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers +google-site-verification,Google-Site-Verification,Google,tool,Google-Site-Verification,Google-Site-Verification,by-design-no,published-ranges,https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json,529,529,https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers +google-cws,Google-CWS,Google,tool,Google-CWS,Google-CWS,by-design-no,published-ranges,https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json,529,529,https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers +google-pinpoint,Google-Pinpoint,Google,user-fetch,Google-Pinpoint,Google-Pinpoint,by-design-no,published-ranges,https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json,529,529,https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers +googleproducer,GoogleProducer,Google,tool,GoogleProducer,GoogleProducer,by-design-no,published-ranges,https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json,529,529,https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers +googlemessages,GoogleMessages,Google,preview,GoogleMessages,GoogleMessages,by-design-no,published-ranges,https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json,529,529,https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers +google-gemininotebook,Google-GeminiNotebook,Google,user-fetch,Google-GeminiNotebook,Google-GeminiNotebook,by-design-no,published-ranges,https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json,529,529,https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers +google-agent,Google-Agent,Google,user-fetch,Google-Agent,Google-Agent,by-design-no,published-ranges,https://developers.google.com/static/search/apis/ipranges/user-triggered-agents.json,3,1,https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers +yandeximages,YandexImages,Yandex,search,YandexImages,YandexImages,documented,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots +yandexvideo,YandexVideo,Yandex,search,YandexVideo,YandexVideo,documented,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots +yandexmedia,YandexMedia,Yandex,search,YandexMedia,YandexMedia,documented,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots +yandexblogs,YandexBlogs,Yandex,search,YandexBlogs,YandexBlogs,documented,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots +yandexmarket,YandexMarket,Yandex,search,YandexMarket,YandexMarket,documented,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots +yandexwebmaster,YandexWebmaster,Yandex,tool,YandexWebmaster,YandexWebmaster,documented,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots +yandexmobilebot,YandexMobileBot,Yandex,search,YandexMobileBot,YandexMobileBot,own-token-only,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots +yandexfavicons,YandexFavicons,Yandex,search,YandexFavicons,YandexFavicons,own-token-only,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots +yandexcalendar,YandexCalendar,Yandex,user-fetch,YandexCalendar,YandexCalendar,own-token-only,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots +yandexdirect,YandexDirect,Yandex,tool,YandexDirect,YandexDirect,own-token-only,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots +yandexmetrika,YandexMetrika,Yandex,tool,YandexMetrika,YandexMetrika,by-design-no,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots +yandexrenderresourcesbot,YandexRenderResourcesBot,Yandex,search,YandexRenderResourcesBot,YandexRenderResourcesBot,own-token-only,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots +yandexscreenshotbot,YandexScreenshotBot,Yandex,tool,YandexScreenshotBot,YandexScreenshotBot,own-token-only,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots +yandexadditional,YandexAdditional,Yandex,ai-training,YandexAdditional,YandexAdditional,own-token-only,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots +yandexadditionalbot,YandexAdditionalBot,Yandex,ai-training,YandexAdditionalBot,YandexAdditionalBot,own-token-only,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots +yandexcombot,YandexComBot,Yandex,search,YandexComBot,YandexComBot,own-token-only,reverse-dns,,0,0,https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots +siteauditbot,SiteAuditBot,Semrush,seo,SiteAuditBot,SiteAuditBot,documented,none,,0,0,https://www.semrush.com/bot/ +semrushbot-ba,SemrushBot-BA,Semrush,seo,SemrushBot-BA,SemrushBot-BA,documented,none,,0,0,https://www.semrush.com/bot/ +semrushbot-si,SemrushBot-SI,Semrush,seo,SemrushBot-SI,SemrushBot-SI,documented,none,,0,0,https://www.semrush.com/bot/ +semrushbot-swa,SemrushBot-SWA,Semrush,seo,SemrushBot-SWA,SemrushBot-SWA,documented,none,,0,0,https://www.semrush.com/bot/ +splitsignalbot,SplitSignalBot,Semrush,seo,SplitSignalBot,SplitSignalBot,documented,none,,0,0,https://www.semrush.com/bot/ +semrushbot-ft,SemrushBot-FT,Semrush,seo,SemrushBot-FT,SemrushBot-FT,documented,none,,0,0,https://www.semrush.com/bot/ +semrushbot-esi,SemrushBot-ESI,Semrush,seo,SemrushBot-ESI,SemrushBot-ESI,documented,none,,0,0,https://www.semrush.com/bot/ +ahrefssiteaudit,AhrefsSiteAudit,Ahrefs,seo,AhrefsSiteAudit,AhrefsSiteAudit,documented,published-ranges,https://api.ahrefs.com/v3/public/crawler-ip-ranges,81,0,https://ahrefs.com/robot +mj12bot,MJ12bot,Majestic,seo,MJ12bot,MJ12bot,documented,none,,0,0,https://mj12bot.com/ +dotbot,DotBot,Moz,seo,dotbot,dotbot,documented,none,,0,0,https://moz.com/help/moz-procedures/crawlers/dotbot +rogerbot,rogerbot,Moz,seo,rogerbot,rogerbot,documented,none,,0,0,https://moz.com/help/moz-procedures/crawlers/rogerbot +dataforseobot,DataForSeoBot,DataForSEO,seo,DataForSeoBot,DataForSeoBot,documented,none,,0,0,https://dataforseo.com/dataforseo-bot +serpstatbot,serpstatbot,Serpstat,seo,serpstatbot,serpstatbot,documented,none,,0,0,https://serpstatbot.com/ +barkrowler,Barkrowler,Babbar,seo,barkrowler,barkrowler,documented,none,,0,0,https://babbar.tech/crawler +screaming-frog-seo-spider,Screaming Frog SEO Spider,Screaming Frog,tool,Screaming Frog SEO Spider,Screaming Frog SEO Spider,documented,none,,0,0,https://www.screamingfrog.co.uk/seo-spider/user-agent/ +seokicks,SEOkicks,SEOkicks,seo,SEOkicks,SEOkicks,documented,none,,0,0,https://www.seokicks.de/robot.html +mojeekbot,MojeekBot,Mojeek,search,MojeekBot,MojeekBot,documented,none,,0,0,https://www.mojeek.com/bot.html +kagibot,Kagibot,Kagi,search,Kagibot,Kagibot,documented,none,,0,0,https://kagi.com/bot +qwantbot,Qwantbot,Qwant,search,Qwantbot,Qwantbot,documented,none,,0,0,https://help.qwant.com/bot/ +qwantbot-news,Qwantbot-news,Qwant,search,Qwantbot-news,Qwantbot-news,documented,none,,0,0,https://help.qwant.com/bot/ +slackbot-linkexpanding,Slackbot-LinkExpanding,Slack,preview,Slackbot-LinkExpanding,Slackbot-LinkExpanding,documented,none,,0,0,https://api.slack.com/robots +slackbot,Slackbot,Slack,preview,Slackbot,Slackbot,documented,none,,0,0,https://api.slack.com/robots +pinterestbot,Pinterestbot,Pinterest,search,Pinterestbot,Pinterestbot,documented,none,,0,0,https://help.pinterest.com/en/business/article/pinterest-crawler +bedrockbot,bedrockbot,Amazon,ai-search,bedrockbot,bedrockbot,documented,none,,0,0,https://docs.aws.amazon.com/bedrock/latest/userguide/webcrawl-data-source-connector.html +cloudflare-autorag,Cloudflare-AutoRAG,Cloudflare,ai-search,Cloudflare-AutoRAG,Cloudflare-AutoRAG,documented,none,,0,0,https://developers.cloudflare.com/ai-search/ +exasearchbot,ExaSearchBot,Exa,ai-search,ExaSearchBot,ExaSearchBot,undocumented,none,,0,0,https://exa.ai +shapbot,ShapBot,Parallel,ai-search,ShapBot,ShapBot,documented,none,,0,0,https://docs.parallel.ai/features/crawler +terracotta,TerraCotta,Ceramic AI,ai-search,TerraCotta,TerraCotta,documented,none,,0,0,https://ceramic.ai/ +crawlspace,Crawlspace,Crawlspace,tool,Crawlspace,Crawlspace,documented,none,,0,0,https://crawlspace.dev +panscient,Panscient,Panscient,dataset,panscient.com,panscient.com,documented,none,,0,0,https://panscient.com/faq.htm +sbintuitionsbot,SBIntuitionsBot,SB Intuitions,ai-training,SBIntuitionsBot,SBIntuitionsBot,documented,none,,0,0,https://www.sbintuitions.co.jp/en/bot/ +icc-crawler,ICC-Crawler,NICT,ai-training,ICC-Crawler,ICC-Crawler,documented,none,,0,0,https://www.nict.go.jp/en/ +cotoyogi,Cotoyogi,ROIS-DS,ai-training,Cotoyogi,Cotoyogi,documented,none,,0,0,https://ds.rois.ac.jp/en_center8/en_crawler/ +isscyberriskcrawler,ISSCyberRiskCrawler,ISS Corporate Solutions,ai-training,ISSCyberRiskCrawler,ISSCyberRiskCrawler,disputed,none,,0,0,https://iss-cyber.com +sidetrade-indexer-bot,Sidetrade indexer bot,Sidetrade,ai-training,Sidetrade indexer bot,Sidetrade indexer bot,undocumented,none,,0,0,https://www.sidetrade.com +yak,YaK,Meltwater,dataset,YaK,YaK,undocumented,none,,0,0,https://www.meltwater.com/en/suite/consumer-intelligence +atlassian-bot,atlassian-bot,Atlassian,ai-search,atlassian-bot,atlassian-bot,documented,none,,0,0,https://support.atlassian.com/organization-administration/docs/connect-custom-website-to-rovo/ +klaviyoaibot,KlaviyoAIBot,Klaviyo,ai-search,KlaviyoAIBot,KlaviyoAIBot,documented,none,,0,0,https://help.klaviyo.com/hc/en-us/articles/40496146232219 +quillbot,QuillBot,QuillBot,ai-training,QuillBot,QuillBot,undocumented,none,,0,0,https://quillbot.com +phindbot,PhindBot,Phind,ai-search,PhindBot,PhindBot,undocumented,none,,0,0,https://www.phind.com/ +andibot,Andibot,Andi,ai-search,Andibot,Andibot,undocumented,none,,0,0,https://andisearch.com/ +anomura,Anomura,Direqt,ai-search,Anomura,Anomura,documented,none,,0,0,https://direqt.ai +aiwebindex,AIWebIndex,Lyrenth,ai-search,AIWebIndex,AIWebIndex,documented,none,,0,0,https://lyrenth.com/crawler-policy +factset-spyderbot,Factset_spyderbot,FactSet,ai-training,Factset_spyderbot,Factset_spyderbot,undocumented,none,,0,0,https://www.factset.com/ai +poseidon-research-crawler,Poseidon Research Crawler,Poseidon Research,ai-training,Poseidon Research Crawler,Poseidon Research Crawler,undocumented,none,,0,0,https://www.poseidonresearch.com +qualifiedbot,QualifiedBot,Qualified,ai-search,QualifiedBot,QualifiedBot,undocumented,none,,0,0,https://www.qualified.com +reflectionbot,Reflectionbot,Reflection AI,ai-training,Reflectionbot,Reflectionbot,undocumented,none,,0,0,https://reflection.ai/ +thinkbot,Thinkbot,Thinkbot,dataset,Thinkbot,Thinkbot,disputed,none,,0,0,https://www.thinkbot.agency +aihitbot,aiHitBot,aiHit,dataset,aiHitBot,aiHitBot,documented,none,,0,0,https://www.aihitdata.com/about +linguee-bot,Linguee Bot,Linguee,ai-training,Linguee Bot,Linguee Bot,disputed,none,,0,0,https://www.linguee.com +lightpanda,Lightpanda,Lightpanda,tool,Lightpanda,Lightpanda,undocumented,none,,0,0,https://lightpanda.io/ +laiondownloader,LAIONDownloader,LAION / img2dataset,dataset,LAIONDownloader,LAIONDownloader,by-design-no,none,,0,0,https://laion.ai/faq/ +velenpublicwebcrawler,VelenPublicWebCrawler,Hunter (Velen),dataset,VelenPublicWebCrawler,VelenPublicWebCrawler,documented,none,,0,0,https://velen.io/ +awariosmartbot,AwarioSmartBot,Awario,dataset,AwarioSmartBot,AwarioSmartBot,documented,none,,0,0,https://awario.com/bots.html +awariorssbot,AwarioRssBot,Awario,dataset,AwarioRssBot,AwarioRssBot,documented,none,,0,0,https://awario.com/bots.html +echoboxbot,EchoboxBot,Echobox,dataset,EchoboxBot,EchoboxBot,undocumented,none,,0,0,https://echobox.com +meta-webindexer,Meta-WebIndexer,Meta,ai-search,Meta-WebIndexer,Meta-WebIndexer,undocumented,none,,0,0,https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/ +chatgpt-agent,ChatGPT Agent,OpenAI,user-fetch,ChatGPT-User,ChatGPT Agent,documented,published-ranges,https://openai.com/chatgpt-user.json,207,0,https://platform.openai.com/docs/bots +wpbot,wpbot,QuantumCloud,tool,wpbot,wpbot,undocumented,none,,0,0,https://www.quantumcloud.com +crawl4ai,Crawl4AI,Crawl4AI project,tool,Crawl4AI,Crawl4AI,undocumented,none,,0,0,https://github.com/unclecode/crawl4ai |