# https://gomask.ai/robots.txt
#
# One group for every crawler. Do not add per-bot groups: a crawler obeys the
# single most specific group that matches its user-agent and does NOT inherit
# from "*", so a bare "User-agent: Googlebot / Allow: /" silently cancels every
# rule below it. Cloudflare prepends its own managed block ahead of this file;
# same-token groups merge, so these rules still apply.
#
# Legacy URLs (old e-commerce, medical imaging, the TDM-era IA) are handled by
# 301s in next.config.ts and middleware.ts. They are deliberately NOT disallowed
# here: a blocked URL can never be crawled, so Google never sees the redirect
# and keeps the dead URL in the index.

User-agent: *
Allow: /

# Admin console and its API
Disallow: /admin/
Disallow: /admin-login
Disallow: /api/admin/

# Endpoints that return data, not pages
Disallow: /api/datasets/*/download
Disallow: /api/datasets/*/preview
Disallow: /api/datasets/*/schema
Disallow: /downloads/

# Faceted and transactional URLs with no standalone search value
Disallow: /marketplace/search?
Disallow: /unsubscribe

Sitemap: https://gomask.ai/sitemap.xml