# Robots.txt for BankStatement.app
# This file guides legitimate web crawlers on how to access our site

User-agent: *
# Allow access to main pages
Allow: /
Allow: /blog/
Allow: /knowledge-base/
Allow: /privacy/
Allow: /terms/
Allow: /refund-policy/

# Explicitly allow static assets for proper page rendering
Allow: /static/
Allow: /media/uploads/featured/
Allow: /media/blog/

# Block sensitive areas
Disallow: /admin/
Disallow: /admin-*
Disallow: /*admin*/
Disallow: /api/
Disallow: /dashboard/
Disallow: /accounts/
Disallow: /media/statements/
Disallow: /logs/

# Block non-existent common paths to reduce 404s
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /phpmyadmin/
Disallow: /.env
Disallow: /config.php
Disallow: /backup.sql

# Special rules for AI bots
User-agent: GPTBot
Crawl-delay: 10
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /admin-*
Disallow: /*admin*/
Disallow: /accounts/
Allow: /blog/
Allow: /knowledge-base/
Allow: /static/
Allow: /fake-bank-statement-risks/
Allow: /bank-statement-analyzer-financial-compatibility/

User-agent: PerplexityBot
Crawl-delay: 10
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /admin-*
Disallow: /*admin*/
Disallow: /accounts/
Allow: /blog/
Allow: /knowledge-base/
Allow: /static/
Allow: /fake-bank-statement-risks/
Allow: /bank-statement-analyzer-financial-compatibility/

User-agent: ClaudeBot
Crawl-delay: 10
Disallow: /dashboard/
Disallow: /api/
Disallow: /admin/
Disallow: /admin-*
Disallow: /*admin*/
Disallow: /accounts/
Allow: /blog/
Allow: /knowledge-base/
Allow: /static/
Allow: /fake-bank-statement-risks/
Allow: /bank-statement-analyzer-financial-compatibility/

# Commercial crawlers
User-agent: AmazonBot
Crawl-delay: 10
Disallow: /dashboard/
Disallow: /api/
Allow: /static/

User-agent: DataForSeoBot
Crawl-delay: 10
Disallow: /dashboard/
Disallow: /api/
Allow: /static/

# Block known malicious bots
User-agent: sqlmap
Disallow: /

User-agent: nikto
Disallow: /

User-agent: nessus
Disallow: /

User-agent: masscan
Disallow: /

# Sitemap location
Sitemap: https://bankstatement.app/sitemap.xml

# Paths that are not publicly available
Disallow: /tools/bank-statement-generator/
Disallow: /tools/bank-statement-status/
Disallow: /tools/render-statement-template/
Disallow: /tools/render-print-statement/
Disallow: /automate-expense-categorization-bank-statement-generators/
Disallow: /bank-reconciliation-statement-generator-bankstatement-app-saves-hours/
Disallow: /bank-statement-generator/
Disallow: /business-bank-statement-generator-create-professional-statements-minutes/
Disallow: /business-bank-statement-generator-startups-why-you-need-one/
Disallow: /designing-perfect-bank-statement-sample-fintech-apps-demos/
Disallow: /freelancer-bank-statement-generator-analyzer-invoicing/
Disallow: /future-business-bank-statement-generators-expense-automation/
Disallow: /future-of-bank-statement-generation-ai-security-ux/
Disallow: /how-to-use-a-bank-statement-generator-to-create-custom-statements-for-your-clients/
Disallow: /interactive-bank-statement-generators-actionable-insights/
Disallow: /proof-of-income-bank-statement-generator/
Disallow: /what-is-a-bank-statement-generator-uses-risks-legal-considerations/
Disallow: /tools/pay-stub-generator/
Disallow: /tools/pay-stub-status/
Disallow: /tools/render-paystub-template/
Disallow: /tools/render-print-paystub/
Disallow: /tools/utility-bill-generator/
Disallow: /tools/render-utility-bill-template/
Disallow: /tools/render-print-utility-bill/