# Allow Googlebot (search engine)
User-agent: Googlebot
Disallow:

# Allow Bingbot
User-agent: Bingbot
Disallow:

# Allow all AI Agents, LLM crawlers & web agents
User-agent: ChatGPT-User
Disallow:

User-agent: ChatGPT-User/1.0
Disallow:

User-agent: GPTBot
Disallow:

User-agent: GPTBot/1.0
Disallow:

User-agent: OpenAI-User
Disallow:

User-agent: ClaudeBot
Disallow:

User-agent: ChatLlama
Disallow:

User-agent: PerplexityBot
Disallow:

User-agent: Anthropic
Disallow:

User-agent: DeepMind
Disallow:

User-agent: GeminiBot
Disallow:

User-agent: Google-Extended
Disallow:

User-agent: CCBot
Disallow:

User-agent: YouBot
Disallow:

User-agent: Replika
Disallow:

User-agent: Quora-POE
Disallow:

User-agent: Diffbot
Disallow:

User-agent: Botify
Disallow:

User-agent: Screaming Frog SEO Spider
Disallow:

User-agent: Scrapy
Disallow:

User-agent: ai-crawler
Disallow:

User-agent: ai-agent
Disallow:

User-agent: AutoGPT
Disallow:

User-agent: BabyAGI
Disallow:

User-agent: langchain
Disallow:

User-agent: AgentGPT
Disallow:

User-agent: LLMAgent
Disallow:

User-agent: BrowserGPT
Disallow:

User-agent: Upstage-LLM
Disallow:

User-agent: airstack-maintainer
Disallow:

User-agent: ZenoAI
Disallow:

User-agent: SeekrBot
Disallow:

User-agent: GPTEngine
Disallow:

User-agent: crawlAi
Disallow:

User-agent: ai-search
Disallow:

User-agent: AI-UserAgent
Disallow:

User-agent: AI
Disallow:

# General bots - restrict admin/private areas
User-agent: *
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /private/

Sitemap: https://bizthon.com/sitemap.xml