# MatterMesh.ai robots policy
# Public dataset endpoint for AI agents, LLMs, and crawlers.
# Landing surface + public scan library + JSON API: fully indexable.
# User-private surfaces (/scan, /library): off-limits.
#
# AI quick-fetch entry points (no auth, no search required):
#   /gallery                        HTML gallery, 60 recent scans
#   /api/public/random              random scan as JSON
#   /api/public/random?redirect=1   302 to a random public image
#   /openapi.yaml                   OpenAPI 3.1 spec (also at /.well-known/openapi.yaml)
#   /.well-known/ai-plugin.json     ChatGPT-style plugin manifest
#   /llms.txt                       full agent contract + fetch recipes

User-agent: *
Allow: /$
Allow: /auth
Allow: /what-is-mattermesh
Allow: /explore
Allow: /explore/
Allow: /assets
Allow: /assets/
Allow: /gallery
Allow: /view
Allow: /view/
Allow: /api/public/
Allow: /llms.txt
Allow: /ai-policy.txt
Allow: /openapi.yaml
Allow: /.well-known/openapi.yaml
Allow: /.well-known/ai-plugin.json
Disallow: /scan
Disallow: /library
Disallow: /library/

User-agent: Googlebot
Allow: /
Allow: /explore
Allow: /assets
Allow: /assets/
Allow: /api/public/
Disallow: /scan
Disallow: /library/

User-agent: GPTBot
Allow: /
Allow: /explore
Allow: /assets
Allow: /assets/
Allow: /api/public/
Allow: /llms.txt
Disallow: /scan
Disallow: /library/

User-agent: ClaudeBot
Allow: /
Allow: /explore
Allow: /assets
Allow: /assets/
Allow: /api/public/
Allow: /llms.txt
Disallow: /scan
Disallow: /library/

User-agent: PerplexityBot
Allow: /
Allow: /explore
Allow: /assets
Allow: /assets/
Allow: /api/public/
Allow: /llms.txt

User-agent: Grok
Allow: /
Allow: /explore
Allow: /assets
Allow: /assets/
Allow: /api/public/
Allow: /llms.txt
Disallow: /scan
Disallow: /library/

User-agent: xAI-Bot
Allow: /
Allow: /explore
Allow: /assets
Allow: /assets/
Allow: /api/public/
Allow: /llms.txt
Disallow: /scan
Disallow: /library/


User-agent: Google-Extended
Allow: /
Allow: /explore
Allow: /assets
Allow: /assets/
Allow: /api/public/
Allow: /llms.txt

User-agent: CCBot
Allow: /
Allow: /explore
Allow: /assets
Allow: /assets/
Allow: /api/public/
Disallow: /scan
Disallow: /library/

# Top image / multimodal dataset crawlers — explicitly invited.
# MatterMesh.ai is a public RTE dataset (JSON + SVG + splat + PNG +
# render). These agents are the dominant fetchers of image-bearing
# training corpora; allowlist them for /explore and /api/public/*.

User-agent: OAI-SearchBot
Allow: /
Allow: /explore
Allow: /assets
Allow: /assets/
Allow: /api/public/
Allow: /llms.txt
Disallow: /scan
Disallow: /library/

User-agent: Applebot-Extended
Allow: /
Allow: /explore
Allow: /assets
Allow: /assets/
Allow: /api/public/
Allow: /llms.txt

User-agent: Meta-ExternalAgent
Allow: /
Allow: /explore
Allow: /assets
Allow: /assets/
Allow: /api/public/
Allow: /llms.txt

User-agent: Bytespider
Allow: /
Allow: /explore
Allow: /assets
Allow: /assets/
Allow: /api/public/
Allow: /llms.txt
Disallow: /scan
Disallow: /library/

User-agent: ImagesiftBot
Allow: /
Allow: /explore
Allow: /assets
Allow: /assets/
Allow: /api/public/
Allow: /llms.txt

User-agent: Amazonbot
Allow: /
Allow: /explore
Allow: /assets
Allow: /assets/
Allow: /api/public/
Allow: /llms.txt
Disallow: /scan
Disallow: /library/

User-agent: cohere-ai
Allow: /
Allow: /explore
Allow: /assets
Allow: /assets/
Allow: /api/public/
Allow: /llms.txt

User-agent: Diffbot
Allow: /
Allow: /explore
Allow: /assets
Allow: /assets/
Allow: /api/public/
Allow: /llms.txt

User-agent: PanguBot
Allow: /
Allow: /explore
Allow: /assets
Allow: /assets/
Allow: /api/public/
Allow: /llms.txt

User-agent: Timpibot
Allow: /
Allow: /explore
Allow: /assets
Allow: /assets/
Allow: /api/public/
Allow: /llms.txt

Sitemap: https://www.mattermesh.ai/sitemap.xml
Sitemap: https://www.mattermesh.ai/sitemap-assets.xml
Sitemap: https://www.mattermesh.ai/sitemap-videos.xml