# ============================================================== # robots.txt — AIS Innovate | https://www.aisinnovate.com # Last updated : 2026-07-28 # Version : FINAL v1.0 # Maintained by: AIS Technolabs Marketing Team # ============================================================== # Deploy at: https://www.aisinnovate.com/robots.txt # ============================================================== # ══════════════════════════════════════════════════════════════ # SECTION 1 — DEFAULT RULES (all crawlers) # ══════════════════════════════════════════════════════════════ User-agent: * Allow: / # ── Backend & system paths ───────────────────────────────── Disallow: /admin/ Disallow: /admin Disallow: /login/ Disallow: /login Disallow: /logout/ Disallow: /register/ Disallow: /signup/ Disallow: /dashboard/ Disallow: /dashboard Disallow: /panel/ Disallow: /cpanel/ Disallow: /api/ Disallow: /api Disallow: /cdn-cgi/ Disallow: /tmp/ Disallow: /.git/ Disallow: /.env Disallow: /config/ Disallow: /private/ Disallow: /cgi-bin/ Disallow: /includes/ Disallow: /cache/ Disallow: /backup/ Disallow: /uploads/private/ Disallow: /wp-admin/ Disallow: /wp-login.php Disallow: /xmlrpc.php # ── Conversion & thank-you pages ────────────────────────── Disallow: /thank-you Disallow: /thank-you/ Disallow: /thankyou Disallow: /thankyou/ Disallow: /success Disallow: /success/ Disallow: /confirmation Disallow: /confirmation/ Disallow: /form-submitted Disallow: /form-submitted/ Disallow: /order-received/ Disallow: /payment-success/ Disallow: /download-complete/ # ── Duplicate URL prevention — tracking parameters ───────── Disallow: /*?utm_source= Disallow: /*?utm_medium= Disallow: /*?utm_campaign= Disallow: /*?utm_term= Disallow: /*?utm_content= Disallow: /*?utm_id= Disallow: /*?fbclid= Disallow: /*?gclid= Disallow: /*?gclsrc= Disallow: /*?dclid= Disallow: /*?msclkid= Disallow: /*?ttclid= Disallow: /*?li_fat_id= Disallow: /*?igshid= Disallow: /*?mc_cid= Disallow: /*?mc_eid= Disallow: /*?ref= Disallow: /*?source= Disallow: /*?_ga= Disallow: /*?_gl= Disallow: /*?sessionid= Disallow: /*?sid= Disallow: /*?hsa_acc= Disallow: /*?hsa_cam= Disallow: /*?hsa_grp= Disallow: /*?hsa_ad= Disallow: /*?hsa_src= Disallow: /*?hsa_tgt= Disallow: /*?hsa_kw= Disallow: /*?hsa_mt= Disallow: /*?hsa_net= Disallow: /*?hsa_ver= Disallow: /*?sort= Disallow: /*?filter= Disallow: /*?orderby= Disallow: /*?paged= # ── Search result pages ──────────────────────────────────── Disallow: /search/ Disallow: /search Disallow: /*?s= Disallow: /*?q= Disallow: /*?search= Disallow: /*?query= # ── Printer / embed / feed versions ─────────────────────── Disallow: /*?print= Disallow: /*?format=print Disallow: /feed/ Disallow: /*/feed/ Disallow: /*/feed$ # ══════════════════════════════════════════════════════════════ # SECTION 2 — AI CITATION BOTS (ALLOWED) # These bots power AI answers in ChatGPT, Claude, Perplexity, # Gemini, Meta AI, Copilot etc. Allowing them builds brand # visibility in AI-generated answers (AEO/GEO strategy). # ══════════════════════════════════════════════════════════════ User-agent: ClaudeBot Allow: / User-agent: GPTBot Allow: / User-agent: anthropic-ai Allow: / User-agent: PerplexityBot Allow: / User-agent: Google-Extended Allow: / User-agent: ChatGPT-User Allow: / User-agent: GPT-4o Allow: / User-agent: OAI-SearchBot Allow: / User-agent: cohere-ai Allow: / User-agent: Meta-ExternalAgent Allow: / User-agent: Meta-ExternalFetcher Allow: / User-agent: YouBot Allow: / User-agent: Applebot Allow: / User-agent: Amazonbot Allow: / User-agent: Bingbot Allow: / User-agent: Slurp Allow: / User-agent: DuckDuckBot Allow: / # ══════════════════════════════════════════════════════════════ # SECTION 3 — AI BULK TRAINING CRAWLERS (DISALLOWED) # These bots scrape content purely for model training without # providing citation/visibility value in return. # ══════════════════════════════════════════════════════════════ User-agent: Bytespider Disallow: / User-agent: CCBot Disallow: / User-agent: omgili Disallow: / User-agent: omgilibot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: PetalBot Disallow: / User-agent: Diffbot Disallow: / User-agent: Scrapy Disallow: / User-agent: python-requests Disallow: / User-agent: Go-http-client Disallow: / User-agent: ia_archiver Disallow: / User-agent: 008 Disallow: / User-agent: BLEXBot Disallow: / User-agent: magpie-crawler Disallow: / User-agent: Turnitin Disallow: / User-agent: NetcraftSurveyAgent Disallow: / User-agent: SiteCheck-sitecrawl Disallow: / User-agent: voltron Disallow: / User-agent: aiHitBot Disallow: / User-agent: ImagesiftBot Disallow: / # ══════════════════════════════════════════════════════════════ # SECTION 4 — SEO AUDIT TOOLS (RATE LIMITED) # Allowed to crawl for legitimate audits but rate-limited # to protect server performance. # ══════════════════════════════════════════════════════════════ User-agent: AhrefsBot Crawl-delay: 10 User-agent: SemrushBot Crawl-delay: 10 User-agent: SemrushBot-SA Crawl-delay: 10 User-agent: SemrushBot-SI Crawl-delay: 10 User-agent: DataForSeoBot Crawl-delay: 30 User-agent: Screaming Frog SEO Spider Crawl-delay: 5 User-agent: rogerbot Crawl-delay: 10 User-agent: MajesticBot Crawl-delay: 10 User-agent: serpstatbot Crawl-delay: 10 User-agent: linkdexbot Crawl-delay: 10 User-agent: SEOkicks Crawl-delay: 30 User-agent: spbot Crawl-delay: 30 # ══════════════════════════════════════════════════════════════ # SECTION 5 — SITEMAPS # ══════════════════════════════════════════════════════════════ # Primary sitemap index (master file — contains all sub-sitemaps) Sitemap: https://www.aisinnovate.com/sitemap.xml # AI / LLM content map — curated site guide for answer engines (https://llmstxt.org) # https://www.aisinnovate.com/llms.txt # Sub-sitemaps (also listed individually for faster discovery) Sitemap: https://www.aisinnovate.com/sitemap.pages.xml Sitemap: https://www.aisinnovate.com/sitemap.portfolio.xml Sitemap: https://www.aisinnovate.com/sitemap.blog.xml