User-agent: *
# Endless URL permutations, each one a fresh Plone render
Disallow: /*?
Disallow: /@@search
Disallow: /*/@@search
Disallow: /search$
Disallow: /*search_rss
Disallow: /*event_listing
Disallow: /*@@getPloudEvents
Disallow: /*@@event_listing_ical
Disallow: /*ics_view
Disallow: /*ical_view
Disallow: /*@@render-portlet
Disallow: /*folder_factories
# Forms and account pages
Disallow: /login$
Disallow: /*/login$
Disallow: /*login_form$
Disallow: /*@@login
Disallow: /*mail_password_form
Disallow: /*sendto_form
Disallow: /*@@register
Disallow: /contact-info$
Disallow: /*/contact-info$
Disallow: /acl_users/
Crawl-delay: 5

# SEO crawlers and AI training scrapers: heavy load, nothing in return for
# a library. A bot obeys only its most specific group, so these get the
# whole site disallowed rather than the rules above.
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: DataForSeoBot
User-agent: PetalBot
User-agent: Bytespider
User-agent: GPTBot
User-agent: CCBot
User-agent: ClaudeBot
User-agent: meta-externalagent
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: panscient.com
Disallow: /

Sitemap: https://www.daviesscountylibrary.org/sitemap.xml.gz