# Causing server errors on some search requests (have to repeat disallows)
User-agent: SemrushBot
Disallow: /texis/search/
Disallow: /admin/
Disallow: /app/

User-agent: ClaudeBot
Disallow: /

User-agent: *
Disallow: /admin/
Disallow: /app/

# non-standard options to try and slow down crawlers
Crawl-delay: 10
Request-rate: 1/10s