# ══════════════════════════════════════════════════════════════
# KlipRok AI — robots.txt
# https://www.kliprok.com/robots.txt
# https://www.kliprok.in/robots.txt
#
# Last updated: 2026-05-21
# Maintained by: SEO Team — seo@kliprok.com
# ══════════════════════════════════════════════════════════════

# ── DEFAULT: All well-behaved bots ───────────────────────────
User-agent: *
Allow: /

# Public pages — explicitly allowed for clarity
Allow: /blog/
Allow: /vs/
Allow: /youtube-shorts-maker
Allow: /instagram-reels-maker
Allow: /tiktok-clip-maker
Allow: /podcast-clip-maker
Allow: /auto-captions
Allow: /silence-remover
Allow: /captions
Allow: /reframe
Allow: /podcast
Allow: /features
Allow: /pricing
Allow: /about
Allow: /contact
Allow: /signup
Allow: /login
Allow: /analytics
Allow: /social
Allow: /sitemap.xml
Allow: /sitemap-blog.xml
Allow: /sitemap-videos.xml

# Disallow private / backend / generated paths
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /internal/
Disallow: /cdn-cgi/
Disallow: /private/
Disallow: /tmp/
Disallow: /webhooks/
Disallow: /cron/
Disallow: /queue/
Disallow: /_next/
Disallow: /static/chunks/

# Disallow low-value / duplicate query parameters
Disallow: /*?ref=
Disallow: /*?utm_source=
Disallow: /*?session_id=
Disallow: /*?token=
Disallow: /*?preview=
Disallow: /*?sort=
Disallow: /*?page=

# Disallow JSON / config file direct access
Disallow: /*.json$
Disallow: /manifest.json
Disallow: /sw.js
Disallow: /workbox-*.js

# Crawl delay for default bots (polite crawling)
Crawl-delay: 2


# ── GOOGLEBOT — Full access, no delay ────────────────────────
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /internal/
Disallow: /cdn-cgi/
Disallow: /*?token=
Disallow: /*?session_id=
Crawl-delay: 0


# ── GOOGLE IMAGE BOT ─────────────────────────────────────────
User-agent: Googlebot-Image
Allow: /
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.webp$
Allow: /*.svg$
Allow: /*.gif$
Disallow: /api/
Crawl-delay: 0


# ── GOOGLE VIDEO BOT ─────────────────────────────────────────
User-agent: Googlebot-Video
Allow: /
Allow: /*.mp4$
Allow: /*.webm$
Allow: /*.mov$
Disallow: /api/
Crawl-delay: 0


# ── GOOGLE ADS BOT ───────────────────────────────────────────
User-agent: AdsBot-Google
Allow: /
Disallow: /api/
Disallow: /admin/


# ── BINGBOT ──────────────────────────────────────────────────
User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /internal/
Disallow: /cdn-cgi/
Disallow: /*?token=
Disallow: /*?session_id=
Crawl-delay: 1


# ── YAHOO / SLURP ────────────────────────────────────────────
User-agent: Slurp
Allow: /
Disallow: /api/
Disallow: /admin/
Crawl-delay: 2


# ── DUCKDUCKGO ───────────────────────────────────────────────
User-agent: DuckDuckBot
Allow: /
Disallow: /api/
Disallow: /admin/
Crawl-delay: 2


# ── YANDEX ───────────────────────────────────────────────────
User-agent: YandexBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Crawl-delay: 2


# ── BAIDU ────────────────────────────────────────────────────
User-agent: Baiduspider
Allow: /
Disallow: /api/
Disallow: /admin/
Crawl-delay: 3


# ── SOCIAL MEDIA CRAWLERS — Allow OG/meta scraping ──────────
User-agent: facebot
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: Facebookexternalhit
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: Twitterbot
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: LinkedInBot
Allow: /
Disallow: /api/
Disallow: /admin/

User-agent: WhatsApp
Allow: /
Disallow: /api/

User-agent: Slackbot
Allow: /
Disallow: /api/

User-agent: TelegramBot
Allow: /
Disallow: /api/

User-agent: Discordbot
Allow: /
Disallow: /api/

User-agent: Pinterest
Allow: /
Disallow: /api/


# ── AI TRAINING CRAWLERS — Block to protect content ─────────
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: PerplexityBot
Allow: /


# ── BAD BOTS / SCRAPERS — Block completely ───────────────────
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /


# ══════════════════════════════════════════════════════════════
# SITEMAPS
# ══════════════════════════════════════════════════════════════
Sitemap: https://www.kliprok.com/sitemap.xml
Sitemap: https://www.kliprok.in/sitemap.xml
Sitemap: https://www.kliprok.in/sitemap-india.xml
# Sitemap: https://www.kliprok.com/sitemap-blog.xml
# Sitemap: https://www.kliprok.com/sitemap-videos.xml