# ========================================================= # INSPRAY SYSTEMS — robots.txt # https://www.inspraysystems.com/ # Updated: 2026-09-28 # ========================================================= # Strategy: # - Canonical pages: index, follow (allowed to crawl) # - Duplicate/stale pages: handled via noindex meta + 301 # redirects in .htaccess. NOT blocked here so Googlebot # can process the noindex/301 and remove from index cleanly. # (Blocking duplicates in robots.txt causes the # "Indexed, though blocked by robots.txt" GSC error.) # ========================================================= # -------- GLOBAL RULES -------- User-agent: * Allow: / # Block only internal / non-page resources Disallow: /seo/ Disallow: /includes/ Disallow: /_fix_indexability.ps1 Disallow: /_seo_report.txt # Block PHP endpoints (not landing pages) Disallow: /send-email.php Disallow: /send-simple.php Disallow: /send-quote.php Disallow: /email-config.php # Block HTML fragments / components / templates / tests / debug pages # (These are not standalone pages — they are includes, templates or dev tests.) Disallow: /footer.html Disallow: /header.html Disallow: /header-complete.html Disallow: /header-test.html Disallow: /complete-header.html Disallow: /nav.html Disallow: /navbar.html Disallow: /quote-modal.html Disallow: /page-template.html Disallow: /category-template.html Disallow: /mobile-debug.html Disallow: /test-mobile.html Disallow: /printing-test.html Disallow: /printing-with-header-component.html Disallow: /simple-email-test.html Disallow: /php-email-test.html Disallow: /sheet-metal-fabrication-2.html Disallow: /sheet-metal-fabrication-backup.html Disallow: /sheet-metal-fabrication-copy.html Disallow: /spray-nozzles-new.html Disallow: /water-park-setup.html Disallow: /water-park-setups-new.html # Block tracking-parameter URL variants (canonicalisation help) Disallow: /*?s=* Disallow: /*?search=* Disallow: /*?utm_* Disallow: /*?fbclid=* Disallow: /*?gclid=* Disallow: /*?msclkid=* Disallow: /*?ref=* Disallow: /*?source=* # -------- CLEAN-URL / CANONICAL HELPERS -------- # (No Disallow entries for duplicate HTML pages. # Those pages carry # AND are 301-redirected via .htaccess to their canonical version. # Letting Googlebot crawl them ensures both signals are honoured.) # -------- GOOGLE FAMILY -------- User-agent: Googlebot Allow: / Crawl-delay: 0 User-agent: Googlebot-Image Allow: / User-agent: Googlebot-Mobile Allow: / User-agent: Googlebot-Video Allow: / User-agent: Googlebot-News Allow: / User-agent: AdsBot-Google Allow: / User-agent: AdsBot-Google-Mobile Allow: / User-agent: Mediapartners-Google Allow: / User-agent: Storebot-Google Allow: / User-agent: GoogleOther Allow: / User-agent: Google-InspectionTool Allow: / # -------- BING / MICROSOFT -------- User-agent: Bingbot Allow: / Crawl-delay: 0 User-agent: BingPreview Allow: / User-agent: MSNBot Allow: / User-agent: MSNBot-Media Allow: / User-agent: adidxbot Allow: / # -------- YAHOO / OTHERS -------- User-agent: Slurp Allow: / User-agent: DuckDuckBot Allow: / User-agent: DuckDuckGo-Favicons-Bot Allow: / User-agent: Baiduspider Allow: / User-agent: Baiduspider-image Allow: / User-agent: YandexBot Allow: / User-agent: Yandex Allow: / User-agent: YandexImages Allow: / User-agent: YandexMobileBot Allow: / User-agent: SeznamBot Allow: / User-agent: Applebot Allow: / User-agent: Applebot-Extended Allow: / # -------- SOCIAL PREVIEW BOTS -------- User-agent: facebot Allow: / User-agent: FacebookExternalHit Allow: / User-agent: LinkedInBot Allow: / User-agent: WhatsApp Allow: / User-agent: TelegramBot Allow: / User-agent: Twitterbot Allow: / User-agent: Pinterestbot Allow: / User-agent: Discordbot Allow: / User-agent: SkypeUriPreview Allow: / User-agent: SlackBot Allow: / # -------- ARCHIVES -------- User-agent: ia_archiver Allow: / User-agent: archive.org_bot Allow: / # -------- AI / LLM SEARCH ENGINES (opt-in for AI visibility) -------- User-agent: GPTBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: OAI-SearchBot Allow: / User-agent: ClaudeBot Allow: / User-agent: Claude-Web Allow: / User-agent: anthropic-ai Allow: / User-agent: PerplexityBot Allow: / User-agent: Perplexity-User Allow: / User-agent: Google-Extended Allow: / User-agent: cohere-ai Allow: / User-agent: Cohere Allow: / User-agent: Amazonbot Allow: / User-agent: CCBot Allow: / User-agent: Meta-ExternalAgent Allow: / User-agent: Meta-ExternalFetcher Allow: / User-agent: FacebookBot Allow: / User-agent: Bytespider Allow: / User-agent: DiffBot Allow: / User-agent: MistralAI-User Allow: / User-agent: DeepSeekBot Allow: / User-agent: YouBot Allow: / User-agent: Kagibot Allow: / User-agent: PhindBot Allow: / User-agent: NeevaBot Allow: / User-agent: Timpibot Allow: / User-agent: OmigiliBot Allow: / # -------- BLOCK SEO-SPYING / SCRAPER BOTS -------- User-agent: SemrushBot Disallow: / User-agent: SemrushBot-SA Disallow: / User-agent: SemrushBot-BA Disallow: / User-agent: SemrushBot-SI Disallow: / User-agent: AhrefsBot Disallow: / User-agent: AhrefsSiteAudit Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: rogerbot Disallow: / User-agent: PetalBot Disallow: / User-agent: BLEXBot Disallow: / User-agent: SEOkicks Disallow: / User-agent: SISTRIX Disallow: / User-agent: Sogou Disallow: / User-agent: LinkPadBot Disallow: / User-agent: SerpstatBot Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: barkrowler Disallow: / User-agent: Nimbostratus-Bot Disallow: / User-agent: MegaIndex Disallow: / User-agent: MauiBot Disallow: / # -------- SITEMAPS -------- Sitemap: https://www.inspraysystems.com/sitemap.xml Sitemap: https://www.inspraysystems.com/sitemap-images.xml Sitemap: https://www.inspraysystems.com/sitemap.html # -------- HOST (Yandex) -------- Host: https://www.inspraysystems.com