# Each crawler obeys ONLY its most specific matching group — a rule added to
# `User-agent: *` alone is invisible to Googlebot and Bingbot below. Any
# Disallow added here must be mirrored into every group.
#
# /community/* is the one deliberately blocked path. It is not a page: it is a
# client-side bounce into FORUM, the members-only Circle install on
# citizens.pioneerlands.*. Because it redirects, a meta robots noindex on it is
# never evaluated — blocking the fetch is the only instrument that works. Left
# open it hands crawlers the door into ~600 gated thread/profile URLs that can
# only ever answer 404, or sit in "crawled - currently not indexed", for an
# anonymous Googlebot. The discussions stay private either way; this stops us
# advertising URLs that can never be indexed.

User-agent: Googlebot
Disallow: /community
Allow: /

User-agent: Bingbot
Disallow: /community
Allow: /

# Social card fetchers are not indexers — they need /community to render a
# preview when a member shares the link, and they crawl nothing onward.
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: *
Disallow: /community
Allow: /

# Sitemaps are served by Supabase edge functions (dynamic, DB-backed).
# Cross-host sitemaps ARE trusted by Google and Bing when referenced from
# this robots.txt (sitemaps.org cross-submission rules); GSC/Bing Webmaster
# submission is a belt-and-braces extra, not the only trust path. Lesser
# crawlers may still discard them until the sitemaps are proxied from the
# apex domain (planned via the edge worker).
#
# Note: private/utility routes (auth, checkout, admin, drafts, draupnir…)
# are deliberately NOT Disallowed here — they carry meta robots noindex,
# and crawlers must be able to fetch a page to see that directive. Add
# Disallow rules only after Search Console confirms they are deindexed.
Sitemap: https://pioneerlands.org/sitemap.xml
Sitemap: https://fetktsiynoevteojkajy.supabase.co/functions/v1/news-sitemap
Sitemap: https://fetktsiynoevteojkajy.supabase.co/functions/v1/sitemap