User-agent: *
Allow: /

# Sitemap location
Sitemap: https://warmseal-roofing.co.uk/sitemap.xml

# Block access to admin and config files
Disallow: /includes/
Disallow: /meta/
Disallow: /scss/
Disallow: /*.php$ 

# Allow important CSS and JS files for rendering
Allow: /css/
Allow: /js/
Allow: /images/
Allow: /fonts/
Allow: /fontawesome/

# Crawl delay to be respectful
Crawl-delay: 1

# Specific directives for major search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# Block unwanted bots
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /