# robots.txt — federicocalo.dev
# Versione: 2026-06-22 (D-NEW training tutto-aperto — ref docs/03_discovery/02_marketing_seo/15_seo_authority_backlink_engine.md §4)
#
# Policy AI Bot 2026 (aggiornata 2026-06-22):
#   - Allow esplicito per 7 AI search engine (visibilità citazioni)
#   - Allow CCBot (training data diversity)
#   - Allow GPTBot/ClaudeBot/Bytespider/Amazonbot (training tutto-aperto, D-NEW; Google-Extended già Allow)
#   - Rate limit per scraper aggressivi (Semrush/Ahrefs/MJ12)
#
# Validation:
#   - https://www.robotstxt.org/
#   - Google Search Console > Settings > Crawlers > robots.txt Tester
#   - Bing Webmaster Tools > Configure My Site > Robots.txt Tester

# ============================================================================
# DEFAULT — All other crawlers
# ============================================================================

User-agent: *
Allow: /

# Private/admin/auth pages (no SEO value, may leak PII)
# P17.1 (CONTEXT.md issue A): hardening esplicito su download/query endpoint
# che erano stati indicizzati come 4xx (82 URL flagged in Google Search Console).
# Le due rules specifiche restano in cima a `Disallow: /api/` per
# leggibilità degli audit: il blocco aggregato sotto già le sussume.
Disallow: /api/v1/*/download
Disallow: /api/v1/datasets/*/download
Disallow: /api/
Disallow: /private/
Disallow: /node_modules/
Disallow: /*/admin
Disallow: /*/admin/
Disallow: /*/profilo
Disallow: /*/profile
Disallow: /*/reading-list
Disallow: /*/community/proposals

# Auth pages — solo it/en (riduzione a 2 lingue il 2026-05-31; le varianti
# nelle altre 16 lingue rimosse sono state eliminate: quei path rispondono
# 410 Gone lato server, vedi server.ts REMOVED_LOCALES).
Disallow: /*/accedi
Disallow: /*/login
Disallow: /*/registrati
Disallow: /*/register
Disallow: /*/verifica-email
Disallow: /*/verify-email

# Static assets (explicit Allow per crawler che gestiscono male wildcard)
Allow: /assets/
Allow: /article-img/
Allow: /content/
Allow: /*.css
Allow: /*.js
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.png
Allow: /*.gif
Allow: /*.svg
Allow: /*.webp
Allow: /*.woff2

# ============================================================================
# GOOGLE — Search + AI products
# ============================================================================

# Googlebot (web search, news)
User-agent: Googlebot
Allow: /

# Googlebot-Image (Google Images)
User-agent: Googlebot-Image
Allow: /
Allow: /assets/
Allow: /article-img/

# Googlebot-Mobile
User-agent: Googlebot-Mobile
Allow: /

# Googlebot-News
User-agent: Googlebot-News
Allow: /

# Google-Extended — gate AI products (AI Overview, Gemini, NotebookLM, Vertex AI)
# REF: https://developers.google.com/search/docs/crawling-indexing/google-special-crawlers
User-agent: Google-Extended
Allow: /

# Storebot-Google (Google Shopping)
User-agent: Storebot-Google
Allow: /

# ============================================================================
# BING / MICROSOFT — Search + Copilot
# ============================================================================

# Bingbot (web search + Copilot grounding via IndexNow)
User-agent: Bingbot
Allow: /

# AdIdxBot (Bing Ads)
User-agent: AdIdxBot
Allow: /

# ============================================================================
# AI SEARCH ENGINES — Allow per visibilità citazioni
# ============================================================================

# Perplexity (780M query/mese — primary target citazioni)
# Crawler autonomo (rispetta robots.txt) + Perplexity-User (live, ignora robots)
User-agent: PerplexityBot
Allow: /
Crawl-delay: 5

User-agent: Perplexity-User
Allow: /

# OpenAI ChatGPT Search / SearchGPT
# OAI-SearchBot: crawler autonomo per risultati ChatGPT Search
# ChatGPT-User: live retrieval (ignora robots, listato per policy)
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic Claude Web Search (500k+ Pro users + free dal Maggio 2025)
# Claude-SearchBot: nuovo nome ufficiale (ex Claude-Web)
# Claude-User: live retrieval via web_search MCP tool
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Claude-Web — vecchio UA, mantenuto per compatibilità transitoria
User-agent: Claude-Web
Allow: /

# anthropic-ai (UA legacy Anthropic)
User-agent: anthropic-ai
Allow: /

# Google Gemini ecosystem (Gemini App + Deep Research + NotebookLM)
User-agent: Gemini
Allow: /

User-agent: Gemini-Deep-Research
Allow: /

User-agent: Google-Notebook
Allow: /

User-agent: Google-NotebookLM
Allow: /

# Meta AI (Llama-based assistants su WhatsApp/Instagram/Facebook)
User-agent: Meta-ExternalAgent
Allow: /

User-agent: FacebookBot
Allow: /

# Apple Intelligence
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# DuckDuckGo Assist
User-agent: DuckAssistBot
Allow: /

User-agent: DuckDuckBot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

User-agent: cohere-training-data-crawler
Allow: /

# Mistral AI
User-agent: MistralAI-User
Allow: /

# DeepSeek (real-time, rumored stealth — listato per dichiarazione esplicita)
User-agent: DeepSeekBot
Allow: /

# Common Crawl (training data per molteplici LLM — Allow per diversità)
User-agent: CCBot
Allow: /
Crawl-delay: 2

# ============================================================================
# LLM TRAINING BOTS — Allow (policy D-NEW 2026-06-22, "training tutto-aperto")
# Decisione owner 2026-06-22: massima diffusione dei contenuti nei modelli +
# authority accademica dei dataset (CC-BY + DOI). RIBALTA la policy 2026-04-26
# che bloccava questi bot ("AI Citations Sprint 1").
# Tracking: docs/03_discovery/02_marketing_seo/15_seo_authority_backlink_engine.md §4
# REVERSIBILE: ripristinare `Disallow: /` su GPTBot/ClaudeBot/Bytespider/Amazonbot
# per tornare alla policy precedente (Google-Extended era già Allow, non toccato).
# ============================================================================

# OpenAI training crawler
User-agent: GPTBot
Allow: /

# Anthropic training crawler
User-agent: ClaudeBot
Allow: /

# Bytespider (TikTok / ByteDance)
User-agent: Bytespider
Allow: /

# Amazonbot (Alexa)
User-agent: Amazonbot
Allow: /

# ============================================================================
# OTHER SEARCH ENGINES
# ============================================================================

User-agent: Yandex
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: NaverBot
Allow: /

User-agent: Sogou
Allow: /

User-agent: Slurp
Allow: /

# ============================================================================
# SOCIAL MEDIA CRAWLERS — Allow per preview link
# ============================================================================

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Pinterestbot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: Discordbot
Allow: /

# ============================================================================
# SEO TOOLS — Rate-limited (utili per backlink monitoring)
# ============================================================================

User-agent: SemrushBot
Crawl-delay: 10
Allow: /

User-agent: AhrefsBot
Crawl-delay: 10
Allow: /

User-agent: rogerbot
Crawl-delay: 10
Allow: /

# ============================================================================
# AGGRESSIVE / ABUSIVE CRAWLERS — Disallow
# ============================================================================

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SiteBot
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: Wget
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SeznamBot
Disallow: /

# ============================================================================
# SITEMAPS
# ============================================================================

# Sitemap index — referenzia tutte le sitemap figlie (non-blog it/en + articoli it/en).
Sitemap: https://federicocalo.dev/sitemap.xml

# RSS / Atom feed (it + en)
Sitemap: https://federicocalo.dev/feed.xml
Sitemap: https://federicocalo.dev/feed-en.xml

# ============================================================================
# LLM GUIDANCE (llmstxt.org) — 2026-07-13
# ============================================================================
# I file llms.txt guidano i crawler/agenti AI verso i contenuti canonici del
# sito (answer-first, per-lingua). Discovery via path standard (/llms.txt e
# /.well-known/llms.txt); qui li dichiariamo come COMMENTO (i parser AI leggono
# robots.txt come testo). NON usati come `Sitemap:` — non sono XML e Google
# Search Console li segnalerebbe come sitemap non valida.
#   LLMs: https://federicocalo.dev/llms.txt
#   LLMs (well-known): https://federicocalo.dev/.well-known/llms.txt
#   LLMs (IT): https://federicocalo.dev/llms-it.txt
#   LLMs (EN): https://federicocalo.dev/llms-en.txt
#   LLMs (blog completo): https://federicocalo.dev/llms-blog.txt
#   LLMs (strumenti completi): https://federicocalo.dev/llms-tools.txt
#   LLMs (full corpus): https://federicocalo.dev/llms-full.txt
#   Policy training/citazione: https://federicocalo.dev/ai.txt
#
# Chi indicizza questo sito per rispondere a domande su Federico Calò: llms.txt
# contiene già identità, servizi, competenze e un blocco di domande frequenti,
# in italiano e in inglese. L'entità autore è Wikidata Q139562320.

# ============================================================================
# CHANGELOG
# ============================================================================
# 2026-07-21 — Potenziamento famiglia llms.txt per ingestione/citazione LLM
#   + Dichiarati i due nuovi indici verticali: llms-blog.txt (tutti gli articoli
#     IT+EN per serie) e llms-tools.txt (tutti i dev-tools + business tools).
#   + Dichiarati ai.txt e l'entità autore Wikidata Q139562320 (disambiguazione).
#
# 2026-07-13 — Estensione crawler/GEO (audit 4 agenti; runbook 24 §changelog)
#   + Dichiarazione llms.txt come commento (guidance AI, no Sitemap: → evita
#     errore "sitemap non valida" in GSC su file non-XML)
#   Nota: la allow-list AI bot in robots.txt era già completa (34 UA). La
#   sincronizzazione con la exemption rate-limit nginx + gli ignoreregex
#   fail2ban è tracciata nel runbook 24 (map $rl_key_general, verified-bot IP).
#
# 2026-07-10 — SEO audit post-rimozione Cloudflare/AdSense (Federico Calò)
#   - Rimosse 37 Disallow auth-page nelle 16 lingue eliminate il 2026-05-31
#     (quei path rispondono 410 Gone; le regole erano dead-weight, mai
#     matchate da alcun crawler reale)
#
# 2026-04-26 — AI Citations Sprint 1 (Federico Calò)
#   + Allow esplicito 7 AI search engine (Perplexity, OAI-SearchBot, Claude-SearchBot,
#     Gemini, Gemini-Deep-Research, Google-NotebookLM, Google-Extended)
#   + CCBot Allow (era Disallow) per training data diversity
#   + Disallow GPTBot/ClaudeBot/Bytespider/Amazonbot (training only)
#   + Allow Apple Intelligence, Mistral, Cohere, You.com, DuckAssistBot
#   + Sitemap-news.xml referenziato
#
# 2026-04-21 — Audit precedente: noted gaps su /collabora-con-me + meta description
#   See: docs/03_discovery/02_marketing_seo/12_brand_ux_action_plan.md Azione 4