# robots.txt for My Tarot Today # https://mytarottoday.com User-agent: * Allow: / Allow: /index.html Allow: /pages/ Allow: /pages/gallery.html Allow: /pages/journey.html Allow: /pages/dictionary.html Allow: /assets/ Allow: /decks/ Allow: /images/ Allow: /data/ # Disallow documentation and internal files Disallow: /README.md Disallow: /archive/ Disallow: /docs/ Disallow: /scripts/ Disallow: /.git Disallow: /.gitignore Disallow: /CNAME # Crawl-delay for polite bots (1 second between requests) Crawl-delay: 1 # Sitemap location Sitemap: https://mytarottoday.com/sitemap.xml # Optional: Specific rules for different bots User-agent: Googlebot Allow: / Crawl-delay: 0 User-agent: Bingbot Allow: / Crawl-delay: 1 User-agent: Slurp Allow: / Crawl-delay: 1 # AI Search Engine Crawlers (for GEO - Generative Engine Optimization) User-agent: GPTBot Allow: / Crawl-delay: 1 User-agent: ChatGPT-User Allow: / Crawl-delay: 1 User-agent: PerplexityBot Allow: / Crawl-delay: 1 User-agent: Claude-Web Allow: / Crawl-delay: 1 User-agent: Google-Extended Allow: / Crawl-delay: 1 User-agent: Anthropic-AI Allow: / Crawl-delay: 1