User-agent: *
Allow: /

# Major Search Engines - Priority Crawling
User-agent: Googlebot
Allow: /
Crawl-delay: 1
# High priority pages for Google
Allow: /explore
Allow: /properties/
Allow: /location/
Allow: /property-type/
Allow: /nairobi/
Allow: /mombasa/
Allow: /kisumu/

User-agent: Googlebot-Image
Allow: /
Crawl-delay: 1

User-agent: Googlebot-Mobile
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1
Allow: /explore
Allow: /properties/
Allow: /location/

User-agent: msnbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 2
Allow: /properties/
Allow: /location/

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1
Allow: /explore
Allow: /properties/

User-agent: Baiduspider
Allow: /
Crawl-delay: 2

User-agent: YandexBot
Allow: /
Crawl-delay: 2

User-agent: Applebot
Allow: /
Crawl-delay: 1

User-agent: Sogou
Allow: /
Crawl-delay: 2

User-agent: Exabot
Allow: /
Crawl-delay: 3

# Social Media Bots
User-agent: facebookexternalhit
Allow: /

User-agent: Facebot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Pinterest
Allow: /

User-agent: Pinterestbot
Allow: /

User-agent: Discordbot
Allow: /

# Disallow admin and private areas (higher priority)
Disallow: /dashboard/
Disallow: /messages/
Disallow: /admin/
Disallow: /api/
Disallow: /auth/
Disallow: /*?token=*
Disallow: /*?session=*
Disallow: /settings/
Disallow: /account/

# Explicitly Allow important directories for all bots
Allow: /
Allow: /explore
Allow: /explore?*
Allow: /properties
Allow: /properties/
Allow: /properties/*
Allow: /location/
Allow: /location/*
Allow: /property-type/
Allow: /property-type/*
Allow: /about
Allow: /about-us
Allow: /contact
Allow: /faq
Allow: /create-listing
Allow: /nairobi/*
Allow: /mombasa/*
Allow: /kisumu/*
Allow: /nakuru/*
Allow: /kilimani/*
Allow: /westlands/*
Allow: /karen/*
Allow: /apartments*
Allow: /houses*
Allow: /land*
Allow: /commercial*
Allow: /villas*
Allow: /beachfront*
Allow: /short-stay*
Allow: /rental*
Allow: /for-sale*
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.gif
Allow: /*.webp
Allow: /*.svg

# Sitemap location
Sitemap: https://roamestate.com/sitemap.xml

# Host specification
Host: https://roamestate.com