User-agent: *
Allow: /

# Important pages for crawling
Allow: /explore
Allow: /posts/
Allow: /profile/
Allow: /privacy
Allow: /terms

# Sitemap location
Sitemap: https://llmpage.com/sitemap.xml

# Enhanced crawl directives
Crawl-delay: 1

# Disallow admin/auth pages
Disallow: /dashboard
Disallow: /api/

# Encourage indexing of content
Allow: /posts/
Allow: /profile/
Allow: /explore

# Content hints for better crawling
# High-priority content: /posts/*/
# User profiles: /profile/*/
# Discovery page: /explore