# Northville Mobile Detailing
# Robots.txt - AI & Search Engine Optimization
# https://northvillemobiledetailing.com
# Updated: 2026-07-28
# Managed by AppWT Web & AI Solutions

# =====================================================
# xAI / GROK - LISTED FIRST BY POLICY
# Grok leads the allowlist on every property AppWT maintains.
# Group order does not change crawler behaviour, because each
# crawler selects its own most-specific group. This is a house
# standard ordering, not a functional gate.
# =====================================================

User-agent: Grok
Allow: /

User-agent: xAI-Bot
Allow: /

# =====================================================
# USA TIER 1: DOMINANT (OpenAI, Google, Microsoft)
# =====================================================

# OpenAI (ChatGPT, GPT-4o, GPT-5, SearchGPT, Operator)
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: ChatGPT-Agent
User-agent: OAI-SearchBot
User-agent: ChatGPT

# Google (Gemini, Bard, NotebookLM, Vertex AI, Deep Research)
User-agent: Google-Extended
User-agent: Googlebot-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: Google-CloudVertexBot
User-agent: Google-NotebookLM
User-agent: Google-Firebase
User-agent: Google-Agent
User-agent: GoogleAgent-Mariner
User-agent: Gemini-Deep-Research

# Microsoft (Bing, Copilot)
User-agent: Bingbot
User-agent: BingPreview
User-agent: AzureAI-SearchBot
Allow: /

# =====================================================
# USA TIER 2: MAJOR (Anthropic, Perplexity, Meta, xAI)
# =====================================================

# Anthropic (Claude)
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai

# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User

# Meta (Meta AI, Facebook, Instagram)
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: FacebookBot
User-agent: FacebookExternalHit
User-agent: facebookexternalhit

# xAI (Grok)
User-agent: xAI-Grok
User-agent: x.AI
Allow: /

# =====================================================
# USA TIER 3: APPLE, AMAZON, VOICE ASSISTANTS
# =====================================================

# Apple (Siri, Apple Intelligence)
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: SiriBot

# Amazon (Alexa, Q, Bedrock, BuyForMe)
User-agent: Amazonbot
User-agent: AlexaBot
User-agent: amazon-kendra
User-agent: AmazonBuyForMe
User-agent: Amzn-SearchBot
User-agent: Amzn-User
User-agent: bedrockbot

# Microsoft Voice
User-agent: CortanaBot
Allow: /

# =====================================================
# USA TIER 4: AI SEARCH ENGINES & ASSISTANTS
# =====================================================

# You.com
User-agent: YouBot

# Brave
User-agent: BraveBot

# DuckDuckGo
User-agent: DuckAssistBot
User-agent: DuckDuckBot

# Phind
User-agent: PhindBot

# iAsk
User-agent: iaskBot

# Exa
User-agent: ExaBot

# Pi / Inflection
User-agent: PiBot

# Andi
User-agent: Andibot

# Kagi
User-agent: KagiBot

# Arc / The Browser Company
User-agent: ArcBot
Allow: /

# =====================================================
# USA TIER 5: AI PLATFORMS, RESEARCH & DEV TOOLS
# =====================================================

# Diffbot
User-agent: Diffbot

# Common Crawl (feeds many LLMs)
User-agent: CCBot

# Hugging Face
User-agent: HuggingFaceBot

# GitHub Copilot
User-agent: GitHubCopilotBot

# Devin (Cognition AI agent)
User-agent: Devin

# Cloudflare
User-agent: Cloudflare-AutoRAG

# Firecrawl
User-agent: FirecrawlAgent

# Crawl4AI
User-agent: Crawl4AI

# Crawlspace
User-agent: Crawlspace

# Apify
User-agent: ApifyBot
User-agent: ApifyWebsiteContentCrawler

# Atlassian (Jira/Confluence AI)
User-agent: atlassian-bot

# OpenAI Agents (Nova Act, Operator)
User-agent: Nova-Act

# Anomura
User-agent: Anomura

# BuddyBot
User-agent: BuddyBot

# Brightbot
User-agent: Brightbot

# bigsur.ai
User-agent: bigsur.ai

# FriendlyCrawler
User-agent: FriendlyCrawler

# Cotoyogi
User-agent: Cotoyogi

# Datenbank Crawler
User-agent: Datenbank-Crawler
Allow: /

# =====================================================
# USA TIER 6: DATA, ANALYTICS & SEO AI CRAWLERS
# =====================================================

# Semrush
User-agent: SemrushBot
User-agent: SemrushBot-OCOB

# DataForSEO
User-agent: DataForSeoBot

# Moz
User-agent: DotBot

# Ahrefs
User-agent: AhrefsBot

# Majestic
User-agent: MJ12bot

# Meltwater
User-agent: MeltwaterBot

# Awario
User-agent: Awario
User-agent: AwarioBot
User-agent: AwarioSmartBot
User-agent: AwarioRssBot

# Echobox
User-agent: EchoboxBot

# Factset
User-agent: Factset_spyderbot

# Sentibot
User-agent: Sentibot

# Turnitin
User-agent: TurnitinBot

# Webz.io
User-agent: Omgilibot
User-agent: Omgili
User-agent: webzio-extended

# Veritone
User-agent: VeritoneBot

# ICC Crawler
User-agent: ICC-Crawler
Allow: /

# =====================================================
# USA TIER 7: AI IMAGE, VIDEO & CONTENT GENERATION
# =====================================================

# ImagesiftBot
User-agent: ImagesiftBot

# img2dataset (LAION)
User-agent: img2dataset

# Stability AI
User-agent: StabilityBot
Allow: /

# =====================================================
# USA TIER 8: SOCIAL & MESSAGING AI
# =====================================================

# LinkedIn
User-agent: LinkedInBot

# Slack
User-agent: Slackbot

# Snap
User-agent: SnapBot

# Reddit
User-agent: Redditbot

# Telegram
User-agent: TelegramBot

# WhatsApp
User-agent: WhatsApp

# Discord
User-agent: Discordbot

# Pinterest
User-agent: Pinterestbot
Allow: /

# =====================================================
# CANADA
# =====================================================

# Cohere
User-agent: cohere-ai
User-agent: CohereBot
User-agent: cohere-training-data-crawler

# Aranet (Canadian AI search)
User-agent: Aranet-SearchBot
Allow: /

# =====================================================
# EUROPE
# =====================================================

# Mistral AI (France)
User-agent: MistralBot

# Aleph Alpha (Germany)
User-agent: AlephAlphaBot

# Channel3 (EU)
User-agent: Channel3Bot

# Ecosia (Germany - AI search)
User-agent: EcosiaBot

# Qwant (France)
User-agent: QwantBot
Allow: /

# =====================================================
# CHINA
# =====================================================

# DeepSeek
User-agent: DeepSeekBot

# ByteDance (Doubao, TikTok)
User-agent: Bytespider
User-agent: ByteDance
User-agent: TikTokSpider

# Baidu (ERNIE)
User-agent: Baiduspider
User-agent: Baiduspider-render

# Alibaba (Qwen / Tongyi Qianwen)
User-agent: AlibabaBot
User-agent: QwenBot

# Huawei (PanGu)
User-agent: PanguBot

# Tencent
User-agent: TencentTraveler
User-agent: Sosospider

# Sogou
User-agent: Sogou
User-agent: SogouSpider

# 360 (Qihoo)
User-agent: 360Spider

# Yisou
User-agent: YisouSpider

# ChatGLM (Zhipu AI)
User-agent: ChatGLM-Spider

# Kimi (Moonshot AI)
User-agent: KimiBot
Allow: /

# =====================================================
# ASIA-PACIFIC (Korea, Japan, India, Southeast Asia)
# =====================================================

# Naver (South Korea - Clova X)
User-agent: Yeti
User-agent: NaverBot

# Samsung (South Korea)
User-agent: SamsungBot

# LINE (Japan)
User-agent: LINEBot

# Rakuten (Japan)
User-agent: RakutenBot

# Coccoc (Vietnam)
User-agent: CocCocBot

# Seekr (Asia-Pacific AI)
User-agent: SeekrBot
Allow: /

# =====================================================
# MIDDLE EAST & AFRICA
# =====================================================

# Falcon AI (UAE)
User-agent: FalconBot

# Jais (UAE - Inception/G42)
User-agent: JaisBot

# PetalBot (Huawei - serves Middle East & Africa)
User-agent: PetalBot
Allow: /

# =====================================================
# AI DATA COLLECTION, TRAINING & RESEARCH
# =====================================================

# Timpi (decentralized search)
User-agent: Timpibot

# VelenPublicWebCrawler
User-agent: VelenPublicWebCrawler

# Kangaroo LLM
User-agent: Kangaroo Bot

# AI2 (Allen Institute for AI)
User-agent: AI2Bot
User-agent: AI2Bot-Dolma
User-agent: Ai2Bot-DeepResearchEval

# ISSCyberRiskCrawler
User-agent: ISSCyberRiskCrawler

# aiHitBot
User-agent: aiHitBot

# Echobot
User-agent: Echobot

# NeevaBot (Neeva AI - acquired by Snowflake)
User-agent: NeevaBot

# Scrapy-based AI crawlers
User-agent: Scrapy

# OpenBot
User-agent: OpenBot

# Nicecrawler
User-agent: Nicecrawler

# Iframely
User-agent: Iframely

# Embedly
User-agent: Embedly

# PaperLiBot
User-agent: PaperLiBot

# Quora Link Preview
User-agent: QuoraBot

# Notion
User-agent: Notion
Allow: /

# =====================================================
# WILDCARD CATCH-ALL (Future Crawlers)
# =====================================================

User-agent: *
Allow: /
Disallow: /cgi-bin/
Crawl-delay: 2

# =====================================================
# SITEMAPS
# =====================================================

Sitemap: https://northvillemobiledetailing.com/sitemap.xml


# =====================================================
# AI DISCOVERY & TRANSPARENCY FILES
# =====================================================

llms.txt: https://northvillemobiledetailing.com/llms.txt
llms-full.txt: https://northvillemobiledetailing.com/llms-full.txt
business-facts: https://northvillemobiledetailing.com/business-facts/

# =====================================================
# EXPLICIT AI CRAWLER ALLOW (spec-clean, one rule per agent)
# Appended 2026-06-06 by AppWT — guarantees these agents resolve
# to Allow: / under strict RFC 9309 parsers (blank line = end of group).
# =====================================================

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Applebot-Extended
Allow: /

# --- AI engines added 2026-06-30 (Grok/xAI, Komo, DeepSeek) ---
User-agent: Grok
Allow: /

User-agent: GrokBot
Allow: /

User-agent: xAI
Allow: /

User-agent: Komo
Allow: /

User-agent: KomoBot
Allow: /

User-agent: DeepSeek
Allow: /

User-agent: DeepSeekBot
Allow: /