# JainSansar — crawler directives
#
# The platform is members-only: Core\Kernel redirects a logged-out visitor to
# login for everything except the handful of surfaces listed below. This file is
# therefore DEFAULT-DENY with explicit allows, mirroring that design — a new
# public route does not become crawlable by accident, the same reasoning as the
# /assets/ allowlist in .htaccess.
#
# What this file is and is not:
#   - It stops well-behaved crawlers from indexing member data. That matters:
#     nobody's family record, photo or relationship link should be reachable
#     from a search result.
#   - It does NOT stop a scraper. A crawler that ignores robots.txt is exactly
#     the one worth worrying about, and this file is advisory. Enforcement lives
#     in Core\Security\HarvestGuard, the rate limiter and the IP guard.
#   - It DOES establish intent. A party who crawled after being told not to
#     cannot claim the access was authorised — which is the point of
#     section 4 of the Terms of Service.

User-agent: *

# Deny everything by default.
Disallow: /

# --- Public, and meant to be found ------------------------------------------
# Sant profiles and Sangh fronts are shareable vanity URLs (also served at
# {name}.jainsansar.com) and the legal/policy pages must stay reachable.
Allow: /sants/$
Allow: /sangh/
Allow: /privacy
Allow: /terms
Allow: /community-guidelines
Allow: /cookie-policy
Allow: /team
Allow: /grievance$

# --- Explicitly denied, even though a guest can reach them -------------------
# These are open to logged-out visitors for a reason, and none of those reasons
# is search indexing.
#
#   /media/         opaque ids, no search value; only PUBLIC-scope bytes are
#                   served, but member photographs must never be indexed
#   /card/          greeting cards authorised by an unguessable share token
#   /event/invite/  event invitations authorised by an unguessable token
#   /timeline/post/ public post permalinks — shareable, not indexable
#   /ads/click/     click tracker, not content
#   /api/           health, telemetry and CSP report endpoints
#   /auth/          login, registration, OTP and password reset
Disallow: /media/
Disallow: /card/
Disallow: /event/invite/
Disallow: /timeline/post/
Disallow: /ads/click/
Disallow: /api/
Disallow: /auth/
Disallow: /grievance/status
Disallow: /parivar/
Disallow: /admin/
Disallow: /messages/
Disallow: /mail/

# --- Crawl rate --------------------------------------------------------------
# Honoured by Bing and Yandex; ignored by Google (set the rate in Search Console
# instead). Cheap to state, and it is one more piece of evidence that automated
# bulk access was never invited.
Crawl-delay: 10

# --- Archivers ---------------------------------------------------------------
# Snapshot services keep copies that outlive a member's deletion request, which
# cuts directly against the erasure duty under the DPDP framework.
User-agent: ia_archiver
Disallow: /

User-agent: archive.org_bot
Disallow: /