# Appliance Forever — https://applianceforever.ca/ (Canada)
# Optimised robots.txt | Updated 2026-05-10 (D-040)
#
# Deploy: web root (same folder as wp-load.php / index.php).
# If SEOPress serves a virtual robots.txt, disable it so this file is used from the origin.

# -----------------------------------------------------------------------------
# AI / search assistants — ALLOW (citation-channel, low training-extraction risk)
# Decision D-040: opt in for AI-search visibility.
# -----------------------------------------------------------------------------

User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: meta-externalagent
Allow: /

# -----------------------------------------------------------------------------
# AI scrapers / aggregators — BLOCK (high training-extraction, low citation value)
# -----------------------------------------------------------------------------

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

# -----------------------------------------------------------------------------
# All crawlers — WordPress hygiene
# -----------------------------------------------------------------------------

User-agent: *
Allow: /

Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Backups & migration (do not index)
Disallow: /wp-content/ai1wm-backups/
Disallow: /ai1wm-backups/

# Security / noise URLs
Disallow: /xmlrpc.php
Disallow: /wp-login.php
Disallow: /readme.html
Disallow: /license.txt

# Fewer duplicate / thin URL variants (remove these lines if you need them crawled)
Disallow: /*?replytocom=
Disallow: /*?s=
Disallow: /search/
Disallow: /trackback/
# NOTE (D-143): /tag/ is intentionally NOT disallowed — the whole namespace now
# returns 410 Gone and Google must be able to crawl those URLs to see it and drop them.

# Optional: oEmbed duplicates
# Disallow: /*/embed/

# Human-readable site summary for AI tools: https://applianceforever.ca/llms.txt

Sitemap: https://applianceforever.ca/sitemaps.xml