# Allow all standard web crawlers and AI bots
User-agent: *
Allow: /

# Explicitly welcome major AI search and LLM crawlers
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: PerplexityBot
User-agent: Google-Extended
User-agent: OAI-SearchBot
Allow: /

# Block system and administration backend
Disallow: /adm/
Disallow: /includes/
Disallow: /cache/
Disallow: /store/
Disallow: /term/

# Block user actions and system operations
Disallow: /posting.php
Disallow: /mcp.php
Disallow: /ucp.php
Disallow: /report.php
Disallow: /cron.php

# Block search functions and dynamic sorting/filtering to prevent duplicate content loops
Disallow: /search.php
Disallow: /memberlist.php
Disallow: /viewonline.php
Disallow: /*&view=
Disallow: /*?view=
Disallow: /*&side=
Disallow: /*?side=
Disallow: /*&style=
Disallow: /*?style=

# Block session IDs from being indexed to protect security and bandwidth
Disallow: /*/*sid=
Disallow: /*?sid=
Disallow: /*&sid=

# Link to your XML Sitemap
Sitemap: https://newmitbbs.com