{"source":"MatterMesh.ai · Reality Translation Engine (RTE)","canonical":"https://www.mattermesh.ai/api/public/agents","docs":"https://www.mattermesh.ai/ai-policy.txt","policy":"https://www.mattermesh.ai/ai-policy.txt","sitemap":"https://www.mattermesh.ai/sitemap.xml","count":20,"agents":[{"slug":"gptbot","name":"GPTBot","vendor":"OpenAI","user_agent_token":"GPTBot","detect_pattern":"gptbot","category":"llm","blurb":"OpenAI's primary training-data crawler for ChatGPT models.","docs_url":"https://platform.openai.com/docs/gptbot","ping_url":null,"enabled":true,"source":"static"},{"slug":"oai-searchbot","name":"OAI-SearchBot","vendor":"OpenAI","user_agent_token":"OAI-SearchBot","detect_pattern":"oai-?searchbot","category":"search","blurb":"OpenAI's search index crawler for ChatGPT browsing & citations.","docs_url":"https://platform.openai.com/docs/bots","ping_url":null,"enabled":true,"source":"static"},{"slug":"chatgpt-user","name":"ChatGPT-User","vendor":"OpenAI","user_agent_token":"ChatGPT-User","detect_pattern":"chatgpt","category":"llm","blurb":"On-demand fetcher when a ChatGPT user opens a link.","docs_url":null,"ping_url":null,"enabled":true,"source":"static"},{"slug":"claudebot","name":"ClaudeBot","vendor":"Anthropic","user_agent_token":"ClaudeBot","detect_pattern":"claude","category":"llm","blurb":"Anthropic's crawler for Claude training & retrieval.","docs_url":"https://support.anthropic.com/en/articles/8896518","ping_url":null,"enabled":true,"source":"static"},{"slug":"perplexitybot","name":"PerplexityBot","vendor":"Perplexity","user_agent_token":"PerplexityBot","detect_pattern":"perplexity","category":"search","blurb":"Perplexity's answer-engine crawler.","docs_url":"https://docs.perplexity.ai/guides/bots","ping_url":null,"enabled":true,"source":"static"},{"slug":"grok","name":"Grok","vendor":"xAI","user_agent_token":"Grok","detect_pattern":"grok","category":"llm","blurb":"xAI's Grok crawler.","docs_url":null,"ping_url":null,"enabled":true,"source":"static"},{"slug":"xai-bot","name":"xAI-Bot","vendor":"xAI","user_agent_token":"xAI-Bot","detect_pattern":"xai","category":"llm","blurb":"xAI's general-purpose retrieval bot.","docs_url":null,"ping_url":null,"enabled":true,"source":"static"},{"slug":"google-extended","name":"Google-Extended","vendor":"Google","user_agent_token":"Google-Extended","detect_pattern":"google-?extended","category":"llm","blurb":"Google's opt-in AI training fetcher (Gemini, Vertex).","docs_url":"https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers","ping_url":"https://www.mattermesh.ai/sitemap.xml","enabled":true,"source":"static"},{"slug":"googlebot","name":"Googlebot","vendor":"Google","user_agent_token":"Googlebot","detect_pattern":"googlebot","category":"search","blurb":"Google Search — accepts sitemap ping.","docs_url":null,"ping_url":"https://www.google.com/ping?sitemap=https%3A%2F%2Fwww.mattermesh.ai%2Fsitemap.xml","enabled":true,"source":"static"},{"slug":"bingbot","name":"Bingbot","vendor":"Microsoft","user_agent_token":"bingbot","detect_pattern":"bingbot","category":"search","blurb":"Bing Search / Copilot — accepts sitemap ping.","docs_url":null,"ping_url":"https://www.bing.com/ping?sitemap=https%3A%2F%2Fwww.mattermesh.ai%2Fsitemap.xml","enabled":true,"source":"static"},{"slug":"applebot-extended","name":"Applebot-Extended","vendor":"Apple","user_agent_token":"Applebot-Extended","detect_pattern":"applebot","category":"llm","blurb":"Apple Intelligence training-data crawler.","docs_url":null,"ping_url":null,"enabled":true,"source":"static"},{"slug":"meta-externalagent","name":"Meta-ExternalAgent","vendor":"Meta","user_agent_token":"Meta-ExternalAgent","detect_pattern":"meta-?externalagent","category":"llm","blurb":"Meta's Llama training crawler.","docs_url":null,"ping_url":null,"enabled":true,"source":"static"},{"slug":"bytespider","name":"Bytespider","vendor":"ByteDance","user_agent_token":"Bytespider","detect_pattern":"bytespider","category":"image-dataset","blurb":"ByteDance multimodal crawler (Doubao).","docs_url":null,"ping_url":null,"enabled":true,"source":"static"},{"slug":"imagesiftbot","name":"ImagesiftBot","vendor":"ImageSift","user_agent_token":"ImagesiftBot","detect_pattern":"imagesift","category":"image-dataset","blurb":"Image-dataset crawler for reverse-image AI.","docs_url":null,"ping_url":null,"enabled":true,"source":"static"},{"slug":"amazonbot","name":"Amazonbot","vendor":"Amazon","user_agent_token":"Amazonbot","detect_pattern":"amazonbot","category":"general","blurb":"Amazon's Alexa / retrieval crawler.","docs_url":null,"ping_url":null,"enabled":true,"source":"static"},{"slug":"cohere-ai","name":"cohere-ai","vendor":"Cohere","user_agent_token":"cohere-ai","detect_pattern":"cohere","category":"llm","blurb":"Cohere's training & retrieval crawler.","docs_url":null,"ping_url":null,"enabled":true,"source":"static"},{"slug":"diffbot","name":"Diffbot","vendor":"Diffbot","user_agent_token":"Diffbot","detect_pattern":"diffbot","category":"general","blurb":"Knowledge-graph extraction crawler.","docs_url":null,"ping_url":null,"enabled":true,"source":"static"},{"slug":"pangubot","name":"PanguBot","vendor":"Huawei","user_agent_token":"PanguBot","detect_pattern":"pangubot","category":"image-dataset","blurb":"Huawei Pangu multimodal crawler.","docs_url":null,"ping_url":null,"enabled":true,"source":"static"},{"slug":"timpibot","name":"Timpibot","vendor":"Timpi","user_agent_token":"Timpibot","detect_pattern":"timpibot","category":"image-dataset","blurb":"Timpi decentralized image-dataset crawler.","docs_url":null,"ping_url":null,"enabled":true,"source":"static"},{"slug":"ccbot","name":"CCBot","vendor":"Common Crawl","user_agent_token":"CCBot","detect_pattern":"ccbot","category":"general","blurb":"Common Crawl — feeds most open LLM training corpora.","docs_url":null,"ping_url":null,"enabled":true,"source":"static"}],"disabled":[]}