65 lines
988 B
Plaintext
65 lines
988 B
Plaintext
# Karibeo - Sistema Integrado de Aplicaciones Turísticas
|
|
# Robots.txt optimizado para SEO
|
|
|
|
User-agent: *
|
|
Allow: /
|
|
Allow: /tourist
|
|
Allow: /politur
|
|
Allow: /taxi
|
|
Allow: /guides
|
|
Allow: /restaurants
|
|
Allow: /hotels
|
|
Allow: /destinations/
|
|
Allow: /explore
|
|
Allow: /about
|
|
Allow: /contact
|
|
Allow: /help
|
|
|
|
# Disallow admin and internal pages
|
|
Disallow: /admin/
|
|
Disallow: /api/
|
|
Disallow: /private/
|
|
Disallow: /_/
|
|
|
|
# Disallow search result pages to avoid duplicate content
|
|
Disallow: /search?*
|
|
Disallow: /*?page=*
|
|
Disallow: /*?filter=*
|
|
|
|
# Allow all major search engines
|
|
User-agent: Googlebot
|
|
Allow: /
|
|
|
|
User-agent: Bingbot
|
|
Allow: /
|
|
|
|
User-agent: Slurp
|
|
Allow: /
|
|
|
|
User-agent: DuckDuckBot
|
|
Allow: /
|
|
|
|
User-agent: Baiduspider
|
|
Allow: /
|
|
|
|
User-agent: YandexBot
|
|
Allow: /
|
|
|
|
User-agent: facebookexternalhit
|
|
Allow: /
|
|
|
|
User-agent: Twitterbot
|
|
Allow: /
|
|
|
|
User-agent: LinkedInBot
|
|
Allow: /
|
|
|
|
User-agent: WhatsApp
|
|
Allow: /
|
|
|
|
# Sitemap location
|
|
Sitemap: https://karibeo.com/sitemap.xml
|
|
|
|
# Crawl-delay for general bots
|
|
Crawl-delay: 1
|