# Crowment robots.txt

User-agent: *
Allow: /

# Private areas (also carry a noindex meta tag)
Disallow: /site-console
Disallow: /user-console
Disallow: /login