# ==============================================================
# robots.txt — noise-remover.com
# ==============================================================
# ---- Allow all well-behaved crawlers ----
User-agent: *
Allow: /
# ---- Block private / auth pages from indexing ----
Disallow: /dashboard
Disallow: /billing
Disallow: /verify-email
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /admin/
# ---- Block API backend routes ----
Disallow: /api/internal/
Disallow: /api/admin/
# ---- Block temp/upload paths ----
Disallow: /tmp/
Disallow: /uploads/
Disallow: /output/
# ---- Block development/test files ----
Disallow: /*.log$
Disallow: /*.sql$
Disallow: /*.env$
Disallow: /node_modules/
Disallow: /.git/
# ---- Sitemap reference ----
Sitemap: https://noise-remover.com/sitemap.xml
# ---- Crawl delay (be kind to the server) ----
Crawl-delay: 1
# ==============================================================
# Specific crawler rules
# ==============================================================
User-agent: Googlebot
Allow: /
Disallow: /dashboard
Disallow: /billing
Disallow: /verify-email
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /admin/

User-agent: Bingbot
Allow: /
Disallow: /dashboard
Disallow: /billing
Disallow: /admin/

User-agent: DuckDuckBot
Allow: /

User-agent: Yandex
Allow: /
Crawl-delay: 2

User-agent: ia_archiver
Allow: /

# AI training crawlers — disallow content scraping
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# SEO tools
User-agent: AhrefsBot
Allow: /
Crawl-delay: 5

User-agent: SemrushBot
Allow: /
Crawl-delay: 5

User-agent: MJ12bot
Disallow: /