# robots.txt for www.gloten.org
# Allow all search engines to crawl everything
# Updated for Bing crawl fix - 2024-12-03

# Bing (Priority Fix)
User-agent: bingbot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Microsoft Bing
User-agent: msnbot
Allow: /
Crawl-delay: 1

# Google
User-agent: Googlebot
Allow: /

# Google Images
User-agent: Googlebot-Image
Allow: /

# Google Mobile
User-agent: Googlebot-Mobile
Allow: /

# Yahoo (uses Bing)
User-agent: Slurp
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Yandex
User-agent: YandexBot
Allow: /

# Baidu
User-agent: Baiduspider
Allow: /

# Brave Search
User-agent: Brave-Indexer
Allow: /

# Naver
User-agent: Yeti
Allow: /

# Seznam
User-agent: SeznamBot
Allow: /

# Facebook
User-agent: facebookexternalhit
Allow: /

# Twitter
User-agent: Twitterbot
Allow: /

# LinkedIn
User-agent: LinkedInBot
Allow: /

# All other bots
User-agent: *
Allow: /
Disallow: /api/

# Sitemaps
Sitemap: https://www.gloten.org/sitemap.xml
Sitemap: https://www.gloten.org/bing-sitemap.xml