User-agent: *
Allow: /
Disallow: /sok
# Random discovery: three picks that change with the seed. Nothing to index.
Disallow: /overrask

# The 8.6 MB SQLite file is a rendering resource, not content: worker.js already
# serves every page's text as HTML, so a crawler that fetches this learns
# nothing it did not already have. Googlebot spent 3.2 of 3.26 GB on it in one
# week — roughly 40% of its requests to the site — while 51,310 pages sat in
# "Discovered, currently not indexed". Prefix matching covers the cache-busted
# form (/kulturperler.db/<version>) too.
Disallow: /kulturperler.db

# Backlink and SEO-tool crawlers. These index the link graph to sell reports;
# none of them sends a reader here. On 1 September 2026 they were 94% of all
# requests to the site — measured over two `wrangler tail` samples, 43 of 62
# requests from MJ12bot alone — against 3% from real browsers, and they walk the
# 44,500-URL sitemap so the edge cache never helps. Search engines that actually
# refer visitors (Googlebot, Bingbot, Applebot, DuckDuckBot) stay welcome above.
User-agent: MJ12bot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: serpstatbot
Disallow: /

Sitemap: https://kulturbase.no/sitemap.xml