# robots.txt para www.42.netmeios.com  
# Última atualização: 2025-07-30  
# Política para motores de busca, LLMs e agentes automatizados

# Regras gerais para todos os agentes
User-agent: *
Allow: /
# Crawl-delay: 2   # removido para evitar aviso no Google


#  Diretórios e ficheiros sensíveis
Disallow: /admin/
Disallow: /includes/
Disallow: /*.bak
Disallow: /*.old
Disallow: /*.sql

#  Bots de IA e LLMs
User-agent: ChatGPT-User
Allow: /
Crawl-delay: 5

User-agent: GPTBot
Allow: /
Crawl-delay: 5

User-agent: Claude-Web
Allow: /
Crawl-delay: 5

User-agent: DeepSeek
Allow: /
Crawl-delay: 5

User-agent: Google-Extended
Allow: /
Crawl-delay: 5

User-agent: anthropic-ai
Allow: /
Crawl-delay: 5

User-agent: Meta-GPTBot
Allow: /
Crawl-delay: 5

User-agent: Firecrawler
Allow: /
Crawl-delay: 5

User-agent: OmigillBot
Allow: /
Crawl-delay: 5

User-agent: Cohere-ai
Allow: /
Crawl-delay: 5

#  Bots de redes sociais
User-agent: facebot
Allow: /
Crawl-delay: 4

User-agent: twitterbot
Allow: /
Crawl-delay: 4

User-agent: LinkedInBot
Allow: /
Crawl-delay: 4

User-agent: Applebot
Allow: /
Crawl-delay: 4

User-agent: TikTokBot
Allow: /
Crawl-delay: 4

User-agent: WhatsAppBot
Allow: /
Crawl-delay: 4

User-agent: TelegramBot
Allow: /
Crawl-delay: 4

User-agent: Discordbot
Allow: /
Crawl-delay: 4

User-agent: Slackbot
Allow: /
Crawl-delay: 4

#  Motores de busca clássicos
User-agent: Googlebot
Allow: /
# Crawl-delay: 3   # removido, Google não suporta

User-agent: Bingbot
Allow: /
Crawl-delay: 3

User-agent: YandexBot
Allow: /
Crawl-delay: 3

User-agent: Baiduspider
Allow: /
Crawl-delay: 3

# ️ Bots de arquivo e indexação histórica
User-agent: CCBot
Allow: /
Crawl-delay: 10

User-agent: ia_archiver
Allow: /
Crawl-delay: 10

# ️FICHEIROS DE POLÍTICAS E TRANSPARÊNCIA

# Documentos primários
Allow: /ai.txt
Allow: /ai.php               # Versão alternativa do ai.txt
Allow: /politica-ia.php
Allow: /politica-ia.md

# Configurações técnicas
Allow: /llms.txt
Allow: /llms.yaml
Allow: /llms.json

Allow: /data.json
Allow: /data.yaml 

# Ferramentas de transparência
Allow: /humans.txt
Allow: /humans.en.txt
Allow: /content-origin.md     
Allow: /content-origin.en.md

# Segurança e padrões web
Allow: /.well-known/security.txt
Allow: /.well-known/ai-policy.json

# Sistema de conteúdo
Allow: /sitemap-llms.xml
Allow: /remocao-ia.html
Allow: /processar-correcao.php
     

# ️ Sitemaps
Sitemap: https://www.42.netmeios.com/sitemap.xml
Sitemap: https://www.42.netmeios.com/sitemap-news.xml
Sitemap: https://www.42.netmeios.com/sitemap-images.xml
Sitemap: https://www.42.netmeios.com/sitemap-llms.xml

# Política de IA relacionada
# Policy-link: https://www.42.netmeios.com/ai.html