{"name":"optmz AI bot list","updated":"2026-11-05","source":"https://aeo-topaz.vercel.app/ai-bots","license":"CC BY 4.0: free to use with a link to the source","purposes":{"training":"AI training","search":"AI search","user":"User-requested fetches","search-engine":"Search engine"},"bots":[{"token":"GPTBot","operator":"OpenAI","purpose":"training","note":"Crawls content that may be used to train OpenAI models. Blocking it does not remove the site from ChatGPT search.","robotsToken":"GPTBot","uaExample":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.4; +https://openai.com/gptbot","docsUrl":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/gptbot.json","robots":"follows"},{"token":"OAI-SearchBot","operator":"OpenAI","purpose":"search","note":"Finds and indexes pages for ChatGPT search results and citations.","robotsToken":"OAI-SearchBot","uaExample":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36; compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot","docsUrl":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/searchbot.json","robots":"follows"},{"token":"ChatGPT-User","operator":"OpenAI","purpose":"user","note":"Fetches a page when a ChatGPT user asks about it. OpenAI says robots.txt rules may not apply to these user-initiated visits.","robotsToken":"ChatGPT-User","uaExample":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot","docsUrl":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/chatgpt-user.json","robots":"may-ignore"},{"token":"ClaudeBot","operator":"Anthropic","purpose":"training","note":"Crawls content that may be used to train Anthropic models.","robotsToken":"ClaudeBot","uaExample":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","docsUrl":"https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","ipRanges":"https://claude.com/crawling/bots.json","robots":"follows"},{"token":"Claude-SearchBot","operator":"Anthropic","purpose":"search","note":"Indexes pages to improve Claude's search results.","robotsToken":"Claude-SearchBot","uaExample":null,"docsUrl":"https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","ipRanges":"https://claude.com/crawling/bots.json","robots":"follows"},{"token":"Claude-User","operator":"Anthropic","purpose":"user","note":"Fetches a page when a Claude user asks about it.","robotsToken":"Claude-User","uaExample":null,"docsUrl":"https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","ipRanges":"https://claude.com/crawling/bots.json","robots":"follows"},{"token":"PerplexityBot","operator":"Perplexity","purpose":"search","note":"Indexes pages for Perplexity answers. Perplexity says it is not used to train foundation models.","robotsToken":"PerplexityBot","uaExample":"Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)","docsUrl":"https://docs.perplexity.ai/guides/bots","ipRanges":"https://www.perplexity.com/perplexitybot.json","robots":"follows"},{"token":"Perplexity-User","operator":"Perplexity","purpose":"user","note":"Fetches a page a Perplexity user asked about. Perplexity says it generally ignores robots.txt for these requests.","robotsToken":"Perplexity-User","uaExample":"Mozilla/5.0 (compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)","docsUrl":"https://docs.perplexity.ai/guides/bots","ipRanges":"https://www.perplexity.com/perplexity-user.json","robots":"may-ignore"},{"token":"Google-Extended","operator":"Google","purpose":"training","note":"A robots.txt token, not a crawler: it controls whether Gemini models may be trained on the site and use it to ground answers. Blocking it removes the site from Gemini app and Vertex AI grounding, but not from Google Search, AI Overviews or AI Mode, and doesn't affect ranking.","robotsToken":"Google-Extended","uaExample":null,"docsUrl":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","ipRanges":null,"robots":"token-only"},{"token":"Googlebot","operator":"Google","purpose":"search-engine","note":"Google Search's crawler. Pages must be crawlable by it to appear in Google Search, AI Overviews and AI Mode.","robotsToken":"Googlebot","uaExample":"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)","docsUrl":"https://developers.google.com/search/docs/crawling-indexing/googlebot","ipRanges":"https://developers.google.com/static/crawling/ipranges/common-crawlers.json","robots":"follows"},{"token":"Bingbot","operator":"Microsoft","purpose":"search-engine","note":"Bing's crawler. Pages must be crawlable by it to appear in Bing and in Microsoft Copilot answers that cite the web.","robotsToken":"Bingbot","uaExample":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/116.0.1938.76 Safari/537.36","docsUrl":"https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0","ipRanges":"https://www.bing.com/toolbox/bingbot.json","robots":"follows"},{"token":"Applebot","operator":"Apple","purpose":"search-engine","note":"Apple's crawler for Siri, Spotlight and Safari search suggestions. Apple says its data may also be used for Apple Intelligence unless Applebot-Extended is blocked.","robotsToken":"Applebot","uaExample":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)","docsUrl":"https://support.apple.com/en-us/119829","ipRanges":"https://search.developer.apple.com/applebot.json","robots":"follows"},{"token":"Applebot-Extended","operator":"Apple","purpose":"training","note":"A robots.txt token, not a crawler: blocking it stops Apple from using pages Applebot crawled to train its foundation models. It doesn't remove the site from Siri or Spotlight.","robotsToken":"Applebot-Extended","uaExample":null,"docsUrl":"https://support.apple.com/en-us/119829","ipRanges":null,"robots":"token-only"},{"token":"Meta-ExternalAgent","operator":"Meta","purpose":"training","note":"Crawls the web for use cases such as training AI models or improving products by indexing content directly.","robotsToken":"Meta-ExternalAgent","uaExample":"meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)","docsUrl":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/","ipRanges":null,"robots":"follows"},{"token":"Meta-ExternalFetcher","operator":"Meta","purpose":"user","note":"Fetches a page a person asked a Meta AI product about. Meta says that because a person requested the fetch, it may not follow robots.txt.","robotsToken":"Meta-ExternalFetcher","uaExample":"meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)","docsUrl":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/","ipRanges":null,"robots":"may-ignore"},{"token":"CCBot","operator":"Common Crawl","purpose":"training","note":"Builds Common Crawl's open web archive, a common source of AI training data.","robotsToken":"CCBot","uaExample":"CCBot/2.0 (https://commoncrawl.org/faq/)","docsUrl":"https://commoncrawl.org/ccbot","ipRanges":"https://index.commoncrawl.org/ccbot.json","robots":"follows"},{"token":"Amazonbot","operator":"Amazon","purpose":"training","note":"Crawls to improve Amazon's products and services; Amazon says the content may be used to train its AI models.","robotsToken":"Amazonbot","uaExample":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1) Chrome/W.X.Y.Z Safari/537.36","docsUrl":"https://developer.amazon.com/amazonbot","ipRanges":null,"robots":"follows"},{"token":"Amzn-SearchBot","operator":"Amazon","purpose":"search","note":"Indexes pages for search experiences in Amazon products such as Alexa and Rufus.","robotsToken":"Amzn-SearchBot","uaExample":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-SearchBot/0.1) Chrome/W.X.Y.Z Safari/537.36","docsUrl":"https://developer.amazon.com/amazonbot","ipRanges":null,"robots":"follows"},{"token":"Amzn-User","operator":"Amazon","purpose":"user","note":"Fetches a page live when an Amazon assistant user asks for current information.","robotsToken":"Amzn-User","uaExample":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-User/0.1) Chrome/W.X.Y.Z Safari/537.36","docsUrl":"https://developer.amazon.com/amazonbot","ipRanges":null,"robots":"follows"},{"token":"DuckAssistBot","operator":"DuckDuckGo","purpose":"search","note":"Fetches pages to source DuckDuckGo's AI-assisted answers (DuckAssist). DuckDuckGo says the data isn't used to train AI models.","robotsToken":"DuckAssistBot","uaExample":"DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)","docsUrl":"https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot","ipRanges":null,"robots":"follows"},{"token":"MistralAI-Index","operator":"Mistral AI","purpose":"search","note":"Indexes pages for Mistral's search, which helps answer user questions.","robotsToken":"MistralAI-Index","uaExample":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-Index/1.0; +https://docs.mistral.ai/robots)","docsUrl":"https://docs.mistral.ai/robots","ipRanges":null,"robots":"follows"},{"token":"MistralAI-User","operator":"Mistral AI","purpose":"user","note":"Fetches a page when a Mistral assistant user asks about it.","robotsToken":"MistralAI-User","uaExample":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)","docsUrl":"https://docs.mistral.ai/robots","ipRanges":null,"robots":"follows"}]}