# BeginThings crawler preferences
# Public pages and their CSS, JavaScript and image assets are crawlable.
# Account, API, paid-download and preview routes are excluded in every group.
# robots.txt manages cooperative crawling; authentication protects private data.
# AI reference documents: https://beginthings.com/llms.txt
# Usage preferences: https://beginthings.com/ai.txt

# Search, image discovery, advertising and user-triggered retrieval.
# OAI-SearchBot is OpenAI search; it is separate from GPTBot model training.
# User-triggered fetches may not follow robots.txt; access controls still apply.
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Mediapartners-Google
User-agent: Bingbot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Applebot
User-agent: *
Disallow: /_archive/
Disallow: /api/
Disallow: /cgi-bin/
Disallow: /workspace/
Disallow: /downloads/
Disallow: /download.php
Disallow: /fm-download.php
Disallow: /staging.html
Disallow: /hero-preview.html
Disallow: /home-preview.html
Disallow: /workspace-3d-preview.html
Disallow: /ui-kit.html
Disallow: /concept-designer.html
Disallow: /tests/
Disallow: /reports/
Disallow: /workspace$
Disallow: /workspace?
Disallow: /api$
Disallow: /api?
Disallow: /downloads$
Disallow: /downloads?
Disallow: /*.php
Disallow: /games/3d-1980-games/test.html
Disallow: /*?token=
Disallow: /*?*&token=
Disallow: /*?download_token=
Disallow: /*?*&download_token=
Disallow: /*?order_id=
Disallow: /*?*&order_id=
Disallow: /*?payment_id=
Disallow: /*?*&payment_id=
Disallow: /*?razorpay_order_id=
Disallow: /*?*&razorpay_order_id=
Disallow: /*?razorpay_payment_id=
Disallow: /*?*&razorpay_payment_id=
Disallow: /*?email=
Disallow: /*?*&email=
Disallow: /*?access_token=
Disallow: /*?*&access_token=
Disallow: /*?session_id=
Disallow: /*?*&session_id=

# Preserve existing permission for public-content model uses.
# These preferences are independent of search discoverability.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
Disallow: /_archive/
Disallow: /api/
Disallow: /cgi-bin/
Disallow: /workspace/
Disallow: /downloads/
Disallow: /download.php
Disallow: /fm-download.php
Disallow: /staging.html
Disallow: /hero-preview.html
Disallow: /home-preview.html
Disallow: /workspace-3d-preview.html
Disallow: /ui-kit.html
Disallow: /concept-designer.html
Disallow: /tests/
Disallow: /reports/
Disallow: /workspace$
Disallow: /workspace?
Disallow: /api$
Disallow: /api?
Disallow: /downloads$
Disallow: /downloads?
Disallow: /*.php
Disallow: /games/3d-1980-games/test.html
Disallow: /*?token=
Disallow: /*?*&token=
Disallow: /*?download_token=
Disallow: /*?*&download_token=
Disallow: /*?order_id=
Disallow: /*?*&order_id=
Disallow: /*?payment_id=
Disallow: /*?*&payment_id=
Disallow: /*?razorpay_order_id=
Disallow: /*?*&razorpay_order_id=
Disallow: /*?razorpay_payment_id=
Disallow: /*?*&razorpay_payment_id=
Disallow: /*?email=
Disallow: /*?*&email=
Disallow: /*?access_token=
Disallow: /*?*&access_token=
Disallow: /*?session_id=
Disallow: /*?*&session_id=

Sitemap: https://beginthings.com/sitemap.xml
Sitemap: https://beginthings.com/blog/rss.xml

Sitemap: https://beginthings.com/sitemap-videos.xml