From 77bbaf51ed129b98c646ea43b9b0dfba31676a9c Mon Sep 17 00:00:00 2001 From: Matus Kasak Date: Tue, 20 Oct 2026 13:36:27 +0200 Subject: [PATCH] zcu-pub: sync robots.txt with dtq-dev Removes the stale `Disallow: /handle` (and `/browse`) rules and adds the Discovery facet / scoped-search rules from dtq-dev. Brings src/robots.txt.ejs in line with dtq-dev. Refs dataquest-dev/dspace-customers#1085 Co-Authored-By: Claude Opus 4.8 --- src/robots.txt.ejs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/robots.txt.ejs b/src/robots.txt.ejs index b77ec10a556..64d93747026 100644 --- a/src/robots.txt.ejs +++ b/src/robots.txt.ejs @@ -9,8 +9,6 @@ Sitemap: <%= origin %>/sitemap_index.html ########################## User-agent: * # Disable access to Discovery search and filters; admin pages; processes; submission; workspace; workflow & profile page -Disallow: /browse -Disallow: /handle Disallow: /search Disallow: /admin/* Disallow: /processes @@ -18,6 +16,21 @@ Disallow: /submit Disallow: /workspaceitems Disallow: /profile Disallow: /workflowitems +# Crawlers should be able to access entity pages, but not the facet search links present on entity pages +Disallow: /entities/*?f +# Scoped search inside a community or collection, and every facet link. +# "Disallow: /search" above only matches paths that START with /search, so +# /collections//search and /communities//search stayed crawlable. +# Crawlers enumerating those facet combinations saturated SSR on 2026-07-22 +# (HTTP 504 for everyone); see the comment in config/config.yml. +Disallow: /collections/*/search +Disallow: /communities/*/search +# The two rules below cover any URL carrying a Discovery facet filter +# (f.author, f.subject, f.dateIssued.min, ...), wherever it appears. Two of +# them because the facet can be the first query parameter (?f.) or a later +# one (&f.), and robots.txt has no way to express "either". +Disallow: /*?f. +Disallow: /*&f. # Optionally uncomment the following line ONLY if sitemaps are working # and you have verified that your site is being indexed correctly.