User-agent: *
Allow: /

# Sitemap
Sitemap: https://www.digilightindia.com/sitemap.xml

# Crawl-delay for respectful crawling
Crawl-delay: 1

# Specific rules for better SEO
User-agent: Googlebot
Allow: /
Crawl-delay: 0.5

User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Allow important resources
Allow: /css/
Allow: /js/
Allow: /images/
Allow: /fonts/

# Block admin and private areas
Disallow: /admin/
Disallow: /private/
Disallow: /seo-meta-tags.html
Disallow: /services-old.html
Disallow: /blog-old-backup.html

# Block dynamic blog URLs (use static URLs instead)
Disallow: /blog/post.html?*
Disallow: /blog/post.html

# Block template files
Disallow: /blog/blog-template.html

# Block markdown source files
Disallow: /blog/posts/*.md
Disallow: /blog/posts.json

# Allow good SEO bots with crawl delays
User-agent: AhrefsBot
Allow: /
Crawl-delay: 2

User-agent: SemrushBot
Allow: /
Crawl-delay: 2

User-agent: MJ12bot
Allow: /
Crawl-delay: 2

# Block spam/scraper bots
User-agent: MauiBot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: SiteBot
Disallow: /

# Google Image Bot
User-agent: Googlebot-Image
Allow: /