# ========================================
# FilesTool.com - Robots.txt
# ========================================
# Updated: 29 October 2025
# Managed by: ThE-DRiFTER


User-agent: Googlebot
Allow: /

# Google AdSense Bot (CRITICAL for ad serving)
User-agent: Mediapartners-Google
Allow: /

# Google AdsBot (CRITICAL for ad verification)
User-agent: AdsBot-Google
Allow: /

# Google Mobile AdSense
User-agent: AdsBot-Google-Mobile
Allow: /

# Google Images
User-agent: Googlebot-Image
Allow: /

# Google Mobile
User-agent: Googlebot-Mobile
Allow: /

# Google AI Extended (for AI Overviews)
User-agent: Google-Extended
Allow: /

# Bing Search
User-agent: Bingbot
Allow: /

# Yahoo Search
User-agent: Slurp
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Yandex (Russia)
User-agent: YandexBot
Allow: /

# Baidu (China)
User-agent: Baiduspider
Allow: /

# AI Training Bots - Blocked
# OpenAI GPT Crawler
User-agent: GPTBot
Disallow: /

# Common Crawl (used by AI companies)
User-agent: CCBot
Disallow: /

# Anthropic AI (Claude)
User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# Cohere AI
User-agent: cohere-ai
Disallow: /

# Omgili (Content scraper)
User-agent: omgili
Disallow: /

# PerplexityBot
User-agent: PerplexityBot
Disallow: /

# Meta AI (NOT regular FacebookBot for sharing)
User-agent: FacebookBot
Disallow: /

# Bytedance (TikTok)
User-agent: Bytespider
Disallow: /

# Additional AI scrapers
User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

# Amazon AI
User-agent: Amazonbot
Disallow: /

# AI2Bot (Allen Institute)
User-agent: AI2Bot
Disallow: /

# All other bots - ALLOWED (Default rule)
User-agent: *
Allow: /

# SITEMAP
Sitemap: https://www.filestool.com/sitemap.xml