# Universal Access - Welcome Everyone!
User-agent: *
Allow: /
Crawl-delay: 0

# Host
Host: https://www.kotipathi.lk


# OpenAI (ChatGPT, GPT-4)
User-agent: ChatGPT-User
User-agent: GPTBot
Allow: /

# Google AI (Bard, Gemini, Google-Extended)
User-agent: Google-Extended
User-agent: GoogleOther
Allow: /

# Anthropic (Claude)
User-agent: anthropic-ai
User-agent: Claude-Web
Allow: /

# Common Crawl (Research & Archive)
User-agent: CCBot
Allow: /

# Meta AI
User-agent: FacebookBot
User-agent: Meta-ExternalAgent
Allow: /

# Apple Intelligence
User-agent: Applebot
User-agent: Applebot-Extended
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# AI2 (Allen Institute)
User-agent: Ai2Bot
Allow: /

# Diffbot
User-agent: Diffbot
Allow: /

# ByteDance (TikTok)
User-agent: Bytespider
Allow: /

# Amazon AI
User-agent: Amazonbot
Allow: /

# Yandex AI
User-agent: YandexBot
Allow: /

# =====================================================
# Search Engine Bots - Priority Access
# =====================================================

User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Mobile
User-agent: Googlebot-Video
Allow: /

User-agent: Bingbot
User-agent: BingPreview
User-agent: msnbot
Allow: /

User-agent: Slurp
User-agent: yahoo-slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

# =====================================================
# Social Media & Content Discovery
# =====================================================

User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: Slackbot
User-agent: Discordbot
User-agent: WhatsApp
User-agent: TelegramBot
Allow: /

# =====================================================
# Archive & Research Bots
# =====================================================

User-agent: ia_archiver
User-agent: archive.org_bot
Allow: /

# =====================================================
# Performance & Monitoring
# =====================================================

# Cloudflare standards
Request-rate: 10/1s
Visit-time: 0000-2359

# Sitemap location (update with your actual sitemap URL)
Sitemap: https://www.kotipathi.lk/sitemap.xml
Sitemap: https://www.kotipathi.lk/sitemap_index.xml

# =====================================================
# Notes for Bot Operators:
# =====================================================
# - We support caching: Cache as needed
# - We support indexing: Index freely
# - We support AI training: Train responsibly
# - Rate limiting: Be reasonable, we won't throttle ethical crawlers
# - Contact: webmaster@kotipathi.lk
# =====================================================
# Last Updated: October 2025
# =====================================================