# umujapan.com — robots policy
# Normal search engines (Google / Bing etc.) are welcome.
# AI SEARCH bots (OAI-SearchBot / Claude-SearchBot / PerplexityBot / DuckAssistBot) and
# user-initiated AI fetchers (ChatGPT-User / Claude-User / Perplexity-User /
# Google-Agent / Amzn-User / MistralAI-User / meta-externalfetcher / Kimi-User etc.)
# are permitted so buyers can find us via AI search and AI assistants.
# AI TRAINING is permitted only for GPTBot (OpenAI) and Google-Extended (Google Gemini).
# All other AI training and bulk-corpus crawlers listed below are NOT permitted.
# Content (c) UMU LLC. Text-and-data-mining rights reserved, except for the
# crawlers permitted in this file.

# ---- AI: OpenAI ----
# GPTBot = 学習用。2026-10-28 から許可
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: OAI-AdsBot
Disallow: /

# ---- AI: Anthropic ----
User-agent: ClaudeBot
Disallow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: claude-web
Disallow: /

User-agent: anthropic-ai
Disallow: /

# ---- AI: Google (AI用途のみ。検索インデックスのGooglebotは無関係) ----
# Google-Extended = Gemini の学習・回答の根拠に使ってよいかの印。2026-10-28 から許可
User-agent: Google-Extended
Allow: /

User-agent: Google-CloudVertexBot
Disallow: /

User-agent: GoogleOther
Disallow: /

User-agent: GoogleOther-Image
Disallow: /

User-agent: GoogleOther-Video
Disallow: /

# ユーザーの指示で取りに来る取得用（Google の公式の一覧 user-triggered fetchers にある物だけ許可）
# Google-NotebookLM は旧名（2026-08 まで）。今の名前は Google-GeminiNotebook
User-agent: Google-NotebookLM
Allow: /

User-agent: Google-GeminiNotebook
Allow: /

User-agent: Google-Agent
Allow: /

# 次の 2 つは Google の公式の説明に無い名乗りなので、今までどおり拒否
User-agent: GoogleAgent-Mariner
Disallow: /

User-agent: Gemini-Deep-Research
Disallow: /

# ---- AI: Apple ----
User-agent: Applebot-Extended
Disallow: /

# ---- AI: Perplexity ----
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# ---- AI: xAI (Grok) ----
User-agent: GrokBot
Disallow: /

User-agent: xAI-Grok
Disallow: /

# Grok-DeepSearch は xAI の公式の説明が無いので、今までどおり拒否
User-agent: Grok-DeepSearch
Disallow: /

# ---- AI: Meta ----
User-agent: meta-externalagent
Disallow: /

# meta-externalfetcher = ユーザーの頼みで 1 件ずつ取りに来る取得用（Meta の公式の説明）
User-agent: meta-externalfetcher
Allow: /

User-agent: meta-webindexer
Disallow: /

User-agent: FacebookBot
Disallow: /

# ---- AI: ByteDance / TikTok ----
User-agent: Bytespider
Disallow: /

User-agent: TikTokSpider
Disallow: /

# ---- AI: Amazon ----
User-agent: Amazonbot
Disallow: /

User-agent: Amzn-SearchBot
Disallow: /

# Amzn-User = ユーザーの操作で取りに来る取得用（Amazon の公式の説明）
User-agent: Amzn-User
Allow: /

User-agent: BedrockBot
Disallow: /

# NovaAct は公式の説明が「作業を大量に自動化する道具」で、ユーザー操作の取得用と確かめられないので拒否
User-agent: NovaAct
Disallow: /

# ---- AI: 中国系LLM ----
User-agent: TongyiBot
Disallow: /

User-agent: DeepSeekBot
Disallow: /

User-agent: ChatGLM-Spider
Disallow: /

User-agent: YiyanBot
Disallow: /

# Kimi-User = ユーザーの操作で取りに来る取得用（Moonshot AI の公式の説明）
User-agent: Kimi-User
Allow: /

User-agent: PanguBot
Disallow: /

User-agent: PetalBot
Disallow: /

# ---- AI: 日本のLLM学習クローラー ----
User-agent: Cotoyogi
Disallow: /

User-agent: ICC-Crawler
Disallow: /

User-agent: SBIntuitionsBot
Disallow: /

# ---- AI: Common Crawl (学習コーパス供給源) ----
User-agent: CCBot
Disallow: /

# ---- AI: Cohere ----
User-agent: cohere-ai
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

# ---- AI: Mistral ----
# MistralAI-User = ユーザーの質問で取りに来る取得用（Mistral の公式の説明）
User-agent: MistralAI-User
Allow: /

# ---- AI: Allen Institute ----
User-agent: AI2Bot
Disallow: /

# ---- AI検索・AIリサーチ・スクレイピングSaaS ----
User-agent: Andibot
Disallow: /

User-agent: PhindBot
Disallow: /

User-agent: iAskBot
Disallow: /

User-agent: LinerBot
Disallow: /

User-agent: ExaBot
Disallow: /

User-agent: TavilyBot
Disallow: /

User-agent: LinkupBot
Disallow: /

User-agent: FirecrawlAgent
Disallow: /

User-agent: Crawl4AI
Disallow: /

User-agent: SemrushBot-OCOB
Disallow: /

User-agent: SemrushBot-SWA
Disallow: /

# DuckDuckGo の AI 回答用。学習用ではなく検索・引用用なので許可（上の OpenAI /
# Anthropic / Perplexity の検索系と同じ扱い）
User-agent: DuckAssistBot
Allow: /

User-agent: YouBot
Disallow: /

# ---- AI: その他 ----
User-agent: Diffbot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: webzio-extended
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Timpibot
Disallow: /

# ---- 通常のブラウザ・検索エンジンは歓迎 ----
User-agent: *
Allow: /

Sitemap: https://umujapan.com/sitemap-index.xml