# LiveData robots.txt
# Recommended root location:
# https://www.livedata.com/robots.txt
#
# Goal:
# - Allow major search engines and AI/LLM crawlers to crawl public website content.
# - Keep LiveData's product, solution, resource, FAQ, press, and company content discoverable.
# - Reduce crawling of duplicate tracking URLs, previews, internal search results,
#   and technical/system URLs.
#
# Notes:
# - robots.txt controls crawling, not guaranteed indexing.
# - For pages that should not appear in search results, use noindex where appropriate.
# - AI crawler access is explicitly allowed below to support AI search and discovery.

############################################
# Major search crawlers
############################################


User-agent: Googlebot
Allow: /
Disallow: /*?*utm_
Disallow: /*?*gclid=
Disallow: /*?*fbclid=
Disallow: /*?*msclkid=
Disallow: /*?*gad_source=
Disallow: /*?*hsa_
Disallow: /*?*preview=
Disallow: /*?*hs_preview=
Disallow: /*?*hsCacheBuster=
Disallow: /*?*hsCtaTracking=

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Bingbot
Allow: /
Disallow: /*?*utm_
Disallow: /*?*gclid=
Disallow: /*?*fbclid=
Disallow: /*?*msclkid=
Disallow: /*?*gad_source=
Disallow: /*?*hsa_
Disallow: /*?*preview=
Disallow: /*?*hs_preview=
Disallow: /*?*hsCacheBuster=
Disallow: /*?*hsCtaTracking=

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Slurp
Allow: /

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: DuckDuckBot
Allow: /

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Applebot
Allow: /

############################################
# AI / LLM crawlers
############################################

# OpenAI
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: GPTBot
Allow: /
Disallow: /*?*preview=
Disallow: /*?*hs_preview=

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: OAI-SearchBot
Allow: /
Disallow: /*?*preview=
Disallow: /*?*hs_preview=

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: ChatGPT-User
Allow: /
Disallow: /*?*preview=
Disallow: /*?*hs_preview=

# Anthropic / Claude
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: ClaudeBot
Allow: /
Disallow: /*?*preview=
Disallow: /*?*hs_preview=

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Claude-SearchBot
Allow: /
Disallow: /*?*preview=
Disallow: /*?*hs_preview=

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Claude-User
Allow: /
Disallow: /*?*preview=
Disallow: /*?*hs_preview=

# Perplexity
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: PerplexityBot
Allow: /
Disallow: /*?*preview=
Disallow: /*?*hs_preview=

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Perplexity-User
Allow: /
Disallow: /*?*preview=
Disallow: /*?*hs_preview=

# Google AI / Gemini-related access
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Google-Extended
Allow: /

# Apple AI
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Applebot-Extended
Allow: /

# Common Crawl
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: CCBot
Allow: /
Disallow: /*?*preview=
Disallow: /*?*hs_preview=

# Meta AI / external fetchers
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: meta-externalagent
Allow: /
Disallow: /*?*preview=
Disallow: /*?*hs_preview=

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /*?*preview=
Disallow: /*?*hs_preview=

# You.com / AI search
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: YouBot
Allow: /
Disallow: /*?*preview=
Disallow: /*?*hs_preview=

############################################
# Default rules
############################################

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: *
Allow: /

# HubSpot/internal search and preview URLs
Disallow: /_hcms/
Disallow: /hs-search-results
Disallow: /*?*preview=
Disallow: /*?*hs_preview=
Disallow: /*?*hsCacheBuster=
Disallow: /*?*hsCtaTracking=

# Tracking and paid-media query parameters
Disallow: /*?*utm_
Disallow: /*?*gclid=
Disallow: /*?*fbclid=
Disallow: /*?*msclkid=
Disallow: /*?*gad_source=
Disallow: /*?*hsa_

############################################
# Sitemap
############################################

Sitemap: https://www.livedata.com/sitemap.xml
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*