# PhishPin - Robots.txt for SEO Optimization
# https://www.phishpin.com/robots.txt

# Allow all crawlers by default
User-agent: *
Allow: /

# Block sensitive directories from crawling
Disallow: /feed_cache/
Disallow: /phishpin_cache/
Disallow: /nrd_cache/
Disallow: /blog_cache/
Disallow: /cgi-bin/
Disallow: /blog_images/*.json
Disallow: /.well-known/acme-challenge/

# Block sensitive files from indexing
Disallow: /admin-panel.php
Disallow: /admin-config.php
Disallow: /phishpin_admin.json
Disallow: /blocked_ips.json
Disallow: /traffic_data.json
Disallow: /blog_admin_key.txt
Disallow: /test_urlhaus.php
Disallow: /setup-admin.php

# Block private APIs
Disallow: /*?action=admin
Disallow: /*?action=API

# Allow blog indexing
Allow: /blogs.php

# Crawl delay (in seconds) - helps server performance
Crawl-delay: 1

# Sitemap location
Sitemap: https://www.phishpin.com/sitemap.php

# Allow Google bot special permissions
User-agent: Googlebot
Allow: /
Crawl-delay: 0.5

# Allow Bing bot
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Allow DuckDuckGo bot
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1