# ============================================================
# PlaneParts360.com - robots.txt
# ============================================================
#
# Purpose:
# - Allow normal search-engine crawling
# - Block selected internal and non-SEO pages
# - Reduce crawling of dynamic search/filter URLs
# - Reduce crawling of tracking-parameter URLs
# - Restrict selected AI training crawlers
# - Allow search and AI assistant crawlers
# - Allow OpenAI advertising crawler when required
# - Allow llms.txt access
# - Declare the XML sitemap
#
# ============================================================
# DEFAULT CRAWLING RULES
# ============================================================

User-agent: *
Allow: /

# ------------------------------------------------------------
# INTERNAL / NON-SEO PAGES
# ------------------------------------------------------------

Disallow: /cookie-policy/
Disallow: /quality/
Disallow: /services/
Disallow: /consignment-options/

# Internal rendering endpoints
Disallow: /partial/renderframedata
Disallow: /partial/rendercopyright/

# Account and transaction pages
Disallow: /login
Disallow: /register
Disallow: /cart
Disallow: /checkout
Disallow: /admin

# Internal site search
Disallow: /search

# ------------------------------------------------------------
# RFQ / INTERNAL SEARCH FUNCTIONS
# ------------------------------------------------------------

Disallow: /rfq/searchrfq

# ------------------------------------------------------------
# DYNAMIC PART SEARCH URLs
# ------------------------------------------------------------

Disallow: /partno-search?searchby=
Disallow: /mfg-search?searchby=
Disallow: /parttype-search?searchby=
Disallow: /nsnno-search?searchby=
Disallow: /niinno-search?searchby=
Disallow: /cagecodeno-search?searchby=

# ------------------------------------------------------------
# SEARCH / FILTER / SORT / PAGINATION PARAMETERS
# ------------------------------------------------------------

Disallow: /*?search=
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?page=

# Tracking parameters
Disallow: /*?utm_

# ============================================================
# LLMS.TXT
# ============================================================
#
# Public AI-readable website information.
#

Allow: /llms.txt

# ============================================================
# AI / TRAINING CRAWLERS
# ============================================================
#
# Restrict selected crawlers used for AI training/data collection.
#

User-agent: ClaudeBot
User-agent: GPTBot
User-agent: CCBot
User-agent: Bytespider
User-agent: meta-externalagent
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: cohere-ai
User-agent: DeepSeekBot
User-agent: Diffbot
User-agent: AI2Bot
User-agent: ImagesiftBot
User-agent: Timpibot
User-agent: omgili
User-agent: Webzio-Extended
User-agent: FirecrawlAgent
User-agent: TikTokSpider
User-agent: PanguBot

Disallow: /

# ============================================================
# SEARCH / AI SEARCH CRAWLERS
# ============================================================
#
# Permit search-oriented crawlers to access public content.
#

User-agent: Applebot
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: DuckDuckBot
User-agent: PetalBot
User-agent: YouBot
User-agent: AndiBot
User-agent: ExaBot
User-agent: PhindBot
User-agent: DuckAssistBot

Allow: /

# ============================================================
# OPENAI ADVERTISING CRAWLER
# ============================================================
#
# Allow if ChatGPT advertising/landing-page crawling is desired.
#

User-agent: OAI-AdsBot
Allow: /

# ============================================================
# OTHER CRAWLERS
# ============================================================

User-agent: Amazonbot
Disallow: /

User-agent: facebookexternalhit
Disallow: /

# ============================================================
# XML SITEMAP
# ============================================================

Sitemap: https://planeparts360.com/sitemap-index.xml