# https://brokentopgaragedoors.com/robots.txt
# Brokentop Garage Doors — Bend, Oregon

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /sw.js
Disallow: /offline
Disallow: /thank-you
Disallow: /form-submit
Disallow: /performance

# Data files (not pages — should not be indexed)
Disallow: /service-area.geojson
Disallow: /service-area.kml
Disallow: /business.json
Disallow: /business-apple.json
Disallow: /business.csv
Disallow: /business-directory.json

# Old WordPress paths (site migrated from WP — these are gone)
Disallow: /author/
Disallow: /tag/
Disallow: /category/
Disallow: /page/
Disallow: /wp-json/
Disallow: /wp-admin/
Disallow: /wp-content/
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-cron.php
Disallow: /comments/
Disallow: /feed/
Disallow: /garage-door-designer
Disallow: /*?wordfence_lh=
Disallow: /*?et_core_page_resource=
Disallow: /*?et_blog
Disallow: /*?p=

# Google
User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /sw.js
Disallow: /offline
Disallow: /thank-you
Disallow: /form-submit
Disallow: /performance

# Old WordPress paths (site migrated from WP — these are gone)
Disallow: /author/
Disallow: /tag/
Disallow: /category/
Disallow: /page/
Disallow: /wp-json/
Disallow: /wp-admin/
Disallow: /wp-content/
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-cron.php
Disallow: /comments/
Disallow: /feed/
Disallow: /garage-door-designer
Disallow: /*?wordfence_lh=
Disallow: /*?et_core_page_resource=
Disallow: /*?et_blog
Disallow: /*?p=

# Google Shopping — full product access
User-agent: Googlebot-Shopping
Allow: /products/
Allow: /garage-door-products
Allow: /products-feed.xml
Disallow: /admin/
Disallow: /api/

# Google Ads landing page crawler
User-agent: AdsBot-Google
Allow: /
Disallow: /admin/
Disallow: /api/

# Bing
User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /sw.js
Disallow: /offline
Disallow: /thank-you
Disallow: /form-submit
Disallow: /performance
Disallow: /author/
Disallow: /tag/
Disallow: /category/
Disallow: /page/
Disallow: /wp-json/
Disallow: /wp-admin/
Disallow: /wp-content/
Disallow: /wp-includes/
Disallow: /comments/
Disallow: /feed/
Disallow: /garage-door-designer
Crawl-delay: 1

# AI assistants — welcome to crawl for search & answers
User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Claude-Web
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Amazonbot
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Applebot
Allow: /
Disallow: /admin/
Disallow: /api/

# Content scrapers — allow crawl with rate limit
User-agent: CCBot
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 5

User-agent: Bytespider
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 5

# Block aggressive scrapers
User-agent: SemrushBot
Crawl-delay: 10

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# Sitemaps
Sitemap: https://brokentopgaragedoors.com/sitemap.xml
Sitemap: https://brokentopgaragedoors.com/news-sitemap.xml