# ===========================================
# STORIBOT.AI ROBOTS.TXT
# Last updated: 2026-01-14
# Optimized for SEO, GEO, and AEO visibility
# ===========================================

# ----- SEARCH ENGINES (ALLOW ALL) -----
# These index your site for Google, Bing, etc.

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Yandex
Allow: /

User-agent: Baiduspider
Allow: /

# ----- SOCIAL MEDIA PREVIEW BOTS (ALLOW ALL) -----
# Critical for link previews on LinkedIn, Twitter, Slack, etc.

User-agent: LinkedInBot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: Facebot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Slackbot-LinkExpanding
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: Pinterestbot
Allow: /

User-agent: Embedly
Allow: /

# ----- SEO & ANALYTICS TOOLS (ALLOW WITH CRAWL DELAY) -----
# Needed for your own site analysis and agencies you work with

User-agent: Screaming Frog SEO Spider
Allow: /

User-agent: AhrefsBot
Allow: /
Crawl-delay: 10

User-agent: SemrushBot
Allow: /
Crawl-delay: 10

User-agent: rogerbot
Allow: /

User-agent: dotbot
Allow: /

User-agent: Moz
Allow: /

User-agent: MozBot
Allow: /

# ----- AI ANSWER ENGINES (ALLOW FOR AEO) -----
# Allow AI-powered answer engines for visibility in AI search results

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

# ----- AI TRAINING CRAWLERS (BLOCK) -----
# These scrape content to train AI models - block to protect IP

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

# ----- AGGRESSIVE/MALICIOUS BOTS (BLOCK) -----

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: wget
Disallow: /

User-agent: curl
Disallow: /

User-agent: ia_archiver
Disallow: /

# ----- DEFAULT RULE (ALLOW) -----
# Allow everything else by default for maximum visibility

User-agent: *
Allow: /

# ----- SITEMAP -----

Sitemap: https://storibot.ai/sitemap.xml
Sitemap: https://www.storibot.ai/sitemap.xml