# robots.txt for Local Emergency Locksmith
# https://local-emergencylocksmith.com
# Last updated: 2026-03-17

# ============================================
# Default rule for all crawlers
# ============================================
User-agent: *
Allow: /
Crawl-delay: 1

# Disallow unnecessary directories
Disallow: /api/

Disallow: /node_modules/
Disallow: /.git/

# ============================================
# Major Search Engine Bots (High Priority)
# ============================================

# Google Bot - Primary search engine
User-agent: Googlebot
Allow: /
Crawl-delay: 0

# Allow all image indexing
User-agent: Googlebot-Image
Allow: /

# Google Mobile Bot
User-agent: Googlebot-Mobile
Allow: /
Crawl-delay: 0

# Bing Bot - Microsoft search engine
User-agent: Bingbot
Allow: /
Crawl-delay: 0

# Yahoo Slurp
User-agent: Slurp
Allow: /
Crawl-delay: 0.5

# DuckDuckGo Bot
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 0.5

# ============================================
# International Search Engines
# ============================================

# Baidu (China) - Moderate crawl rate
User-agent: Baiduspider
Allow: /
Crawl-delay: 2

# Yandex (Russia)
User-agent: YandexBot
Allow: /
Crawl-delay: 2

# Sogou (China)
User-agent: Sogou web spider
Allow: /
Crawl-delay: 2

# Naver (Korea)
User-agent: Yeti
Allow: /
Crawl-delay: 1

# ============================================
# Social Media Bots (For Rich Previews)
# ============================================

# Twitter/X Bot
User-agent: Twitterbot
Allow: /
Crawl-delay: 0

# Facebook Bot
User-agent: facebookexternalhit
Allow: /
Crawl-delay: 0

# LinkedIn Bot
User-agent: LinkedInBot
Allow: /
Crawl-delay: 0

# Pinterest Bot
User-agent: Pinterestbot
Allow: /
Crawl-delay: 0.5

# Instagram Bot
User-agent: Instagrambot
Allow: /
Crawl-delay: 0.5

# WhatsApp Bot
User-agent: WhatsApp
Allow: /
Crawl-delay: 0

# ============================================
# AI/ML Training Bots
# ============================================

# OpenAI GPT Bot
User-agent: GPTBot
Allow: /
Crawl-delay: 1

# OpenAI ChatGPT User Agent
User-agent: ChatGPT-User
Allow: /
Crawl-delay: 1

# Common Crawl Bot
User-agent: CCBot
Allow: /
Crawl-delay: 2

# Anthropic AI (Claude)
User-agent: anthropic-ai
Allow: /
Crawl-delay: 1

User-agent: Claude-Web
Allow: /
Crawl-delay: 1

# Google AI (Bard/Gemini)
User-agent: Google-Extended
Allow: /
Crawl-delay: 1

# ============================================
# SEO & Analytics Tools
# ============================================

# Ahrefs Bot
User-agent: AhrefsBot
Allow: /
Crawl-delay: 1

# SEMrush Bot
User-agent: SemrushBot
Allow: /
Crawl-delay: 1

# Moz Bot
User-agent: rogerbot
Allow: /
Crawl-delay: 1

# Majestic SEO
User-agent: MJ12bot
Allow: /
Crawl-delay: 2

# ============================================
# Archive Bots
# ============================================

# Internet Archive Wayback Machine
User-agent: ia_archiver
Allow: /
Crawl-delay: 2

# ============================================
# Bad Bots (Blocked)
# ============================================

# Block email harvesters
User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: EmailWolf
Disallow: /

# Block content scrapers
User-agent: WebCopier
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: WebSauger
Disallow: /

# ============================================
# Sitemap Locations
# ============================================
Sitemap: https://local-emergencylocksmith.com/sitemap.xml

# ============================================
# Additional Directives
# ============================================

# Host directive (for search engines that support it)
Host: https://local-emergencylocksmith.com