# quartermaster.nl - Magento 1.9
# Goal: keep crawlers out of infinite/expensive URL spaces (search, layered
# navigation, sort/pagination permutations, session-id URLs).

User-agent: *

# Session IDs in URLs - each unique SID mints a new server-side session
Disallow: /*?SID=
Disallow: /*&SID=

# Internal search: unbounded URL space, inflates catalogsearch_query
Disallow: /catalogsearch/
Disallow: /*?q=
Disallow: /index.php/catalogsearch/

# Layered navigation / sorting / pagination permutations
Disallow: /*?dir=
Disallow: /*&dir=
Disallow: /*?order=
Disallow: /*&order=
Disallow: /*?price=
Disallow: /*&price=
Disallow: /*?limit=
Disallow: /*&limit=
Disallow: /*?mode=
Disallow: /*&mode=
Disallow: /*?p=
Disallow: /*&p=

# Customer / checkout / account areas - no crawl value, session-heavy
Disallow: /checkout/
Disallow: /customer/
Disallow: /customer_account/
Disallow: /wishlist/
Disallow: /sendfriend/
Disallow: /catalog/product_compare/
Disallow: /review/product/post/
Disallow: /index.php/checkout/
Disallow: /index.php/customer/
Disallow: /index.php/wishlist/

# Application internals
Disallow: /app/
Disallow: /downloader/
Disallow: /errors/
Disallow: /includes/
Disallow: /lib/
Disallow: /var/
Disallow: /shell/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /js/
Disallow: /cron.php

Crawl-delay: 10

# --- SEO / backlink crawlers: heavy, no commercial value to this shop ---
User-agent: AhrefsBot
Crawl-delay: 30

User-agent: SemrushBot
Crawl-delay: 30

User-agent: SERankingBacklinksBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Amazonbot
Crawl-delay: 30

Sitemap: https://www.quartermaster.nl/sitemap.xml