# Aaryavart Centre for Autism and Special Needs Foundation
# Robots.txt file for search engine optimization
# Last updated: December 2024

# ========================================
# ALLOW ALL SEARCH ENGINES
# ========================================
User-agent: *
Allow: /

# ========================================
# MAJOR SEARCH ENGINES
# ========================================
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Yandex
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: Slurp
Allow: /

# ========================================
# SOCIAL MEDIA CRAWLERS
# ========================================
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: Instagram
Allow: /

User-agent: Pinterest
Allow: /

# ========================================
# AI AND CHATGPT SEARCH CRAWLERS
# ========================================
User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: ChatGPT
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

# ========================================
# MOBILE-SPECIFIC CRAWLERS
# ========================================
User-agent: Googlebot-Mobile
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /

User-agent: Mediapartners-Google
Allow: /

# ========================================
# ANDROID AND IOS MOBILE BROWSERS
# ========================================
User-agent: Mozilla/5.0 (Linux; Android*
Allow: /

User-agent: Mozilla/5.0 (iPhone*
Allow: /

User-agent: Mozilla/5.0 (iPad*
Allow: /

# ========================================
# INDIAN MOBILE BROWSERS AND APPS
# ========================================
User-agent: UCWeb
Allow: /

User-agent: Opera Mini
Allow: /

User-agent: SamsungBrowser
Allow: /

User-agent: Chrome Mobile
Allow: /

User-agent: Safari Mobile
Allow: /

User-agent: JioBrowser
Allow: /

User-agent: Mi Browser
Allow: /

# ========================================
# INDIAN SEARCH ENGINES AND APPS
# ========================================
User-agent: JustDial
Allow: /

User-agent: Sulekha
Allow: /

User-agent: Practo
Allow: /

User-agent: HealthifyMe
Allow: /

User-agent: 1mg
Allow: /

User-agent: PharmEasy
Allow: /

User-agent: NetMeds
Allow: /

# ========================================
# HEALTHCARE AND MEDICAL CRAWLERS
# ========================================
User-agent: HealthBot
Allow: /

User-agent: MedicalBot
Allow: /

User-agent: TherapyBot
Allow: /

User-agent: AutismBot
Allow: /

# ========================================
# BLOCK SENSITIVE AREAS
# ========================================
Disallow: /admin/
Disallow: /private/
Disallow: /api/
Disallow: /_next/
Disallow: /node_modules/
Disallow: /.env
Disallow: /config/
Disallow: /temp/
Disallow: /cache/
Disallow: /logs/

# ========================================
# SITEMAP AND CRAWL SETTINGS
# ========================================
Sitemap: https://aaryavartcentre.com/sitemap.xml

# Crawl delay (helps with server load)
Crawl-delay: 1

# ========================================
# ADDITIONAL SEO DIRECTIVES
# ========================================
# Allow crawling of all content for better SEO
Allow: /assets/
Allow: /images/
Allow: /css/
Allow: /js/

# Block access to development files
Disallow: /.git/
Disallow: /.github/
Disallow: /package.json
Disallow: /package-lock.json
Disallow: /yarn.lock
Disallow: /README.md