User-agent: *
Allow: /
# Tool AJAX/backend endpoints (JSON APIs, poll, logging, form handlers) — POST
# targets that return JSON or a 4xx to a bare crawl; never crawl or index them.
Disallow: /tools/*/api/
Disallow: /tools/poll.php
Disallow: /*/log.php
Disallow: /*/submit.php
Disallow: /*/submit_*.php
Disallow: /*/save.php
Disallow: /*/track.php
Disallow: /blog/wp-admin/
Allow: /blog/wp-content/uploads/
Disallow: /blog/wp-content/plugins/
Disallow: /blog/readme.html
Disallow: /blog/refer/
Disallow: /blog/feed/
Disallow: /blog/author/admin/
# NOTE: /blog/tag/ is deliberately NOT disallowed (2026-08-09). robots.txt cannot
# deindex anything — blocking the crawl stops Google reading the `noindex`, so the
# 551 tag archives sat in the index as "Indexed, though blocked by robots.txt" while
# also being submitted in post_tag-sitemap.xml. They now serve
# `<meta name="robots" content="noindex, follow">` (Yoast noindex-tax-post_tag) and
# are out of the sitemap. Google must be ALLOWED to crawl them to see that and drop
# them. Re-adding a Disallow here would freeze them in the index permanently.

Sitemap: https://seoultra.id/sitemap.xml
Sitemap: https://seoultra.id/tools/auto-sitemap.php
Sitemap: https://seoultra.id/blog/sitemap_index.xml
Sitemap: https://seoultra.id/blog/post-sitemap.xml
Sitemap: https://seoultra.id/blog/page-sitemap.xml
Sitemap: https://seoultra.id/blog/category-sitemap.xml
Sitemap: https://forum.seoultra.id/sitemap.xml


User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: Bytespider
User-agent: CCBot
User-agent: cohere-ai
User-agent: AI2Bot
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: Timpibot
User-agent: Webzio-Extended
User-agent: SemrushBot-OCOB
# Was `Disallow. /` (period, not colon) — an unrecognised line, which RFC 9309 says
# to ignore. That left this whole group with ZERO rules, so every AI training
# crawler above was effectively ALLOWED, the exact opposite of the intent. Fixed
# 2026-08-09.
Disallow: /

# --- Allowed: AI search + user-triggered fetches ---
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Google-CloudVertexBot
User-agent: Applebot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Meta-ExternalFetcher
User-agent: Amazonbot
User-agent: MistralAI-User
User-agent: DuckAssistBot
User-agent: YouBot
Allow: /