# Robots.txt for Total Environment

User-agent: *
Sitemap: https://www.totalenvironmentindia.com/sitemap.xml

# Default allow all content
Allow: /

# Block sensitive or non-public areas
Disallow: /admin/
Disallow: /login/
Disallow: /staging/
Disallow: /temp/
Disallow: /drafts/
Disallow: /index.html/

# Block paths causing duplicate content or unnecessary crawling
Disallow: /?filter=
Disallow: /?sort=
Disallow: /tag/
Disallow: /category/
Disallow: /search/
Disallow: /author/

# Allow image crawling for better SEO
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.webp$

# Specific bot instructions
User-agent: Googlebot-Image
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$

# AI crawlers
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /