# BeginThings.com robots.txt
# Updated: 2026-09-08
# Public discovery is fully allowed. Gated paths remain access-controlled.
# AI reference: /llms.txt and /llms-full.txt
# AI policy: /ai.txt

# Googlebot search crawler
User-agent: Googlebot
Allow: /
Disallow: /_archive/
Disallow: /api/
Disallow: /cgi-bin/
Disallow: /workspace/
Disallow: /staging.html
Disallow: /hero-preview.html
Disallow: /home-preview.html
Disallow: /workspace-3d-preview.html
Disallow: /ui-kit.html
Disallow: /concept-designer.html

# Googlebot Image crawler
User-agent: Googlebot-Image
Allow: /images/
Allow: /assets/
Allow: /

# Google AdSense crawler
User-agent: Mediapartners-Google
Allow: /

# Bingbot
User-agent: Bingbot
Allow: /
Disallow: /_archive/
Disallow: /api/
Disallow: /cgi-bin/
Disallow: /workspace/
Disallow: /staging.html
Disallow: /hero-preview.html
Disallow: /home-preview.html
Disallow: /workspace-3d-preview.html

# AI search engines (Perplexity, SearchGPT, Claude, Gemini AI Overviews)
User-agent: GPTBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /
Allow: /llms.txt

User-agent: ClaudeBot
Allow: /
Allow: /llms.txt

User-agent: Google-Extended
Allow: /
Allow: /llms.txt

User-agent: Applebot
Allow: /

# Default fallback rule for all other compliant crawlers
User-agent: *
Allow: /
Disallow: /_archive/
Disallow: /api/
Disallow: /cgi-bin/
Disallow: /workspace/
Disallow: /staging.html
Disallow: /hero-preview.html
Disallow: /home-preview.html
Disallow: /workspace-3d-preview.html
Disallow: /ui-kit.html

# XML Sitemaps & Feeds
Sitemap: https://beginthings.com/sitemap.xml
Sitemap: https://beginthings.com/blog/rss.xml