From 4afba4e11d3ea104b0c0e8b675e8b72e5bdf5205 Mon Sep 17 00:00:00 2001 From: Chris Portscheller Date: Sun, 15 Nov 2026 20:20:03 -0600 Subject: [PATCH 1/2] Add automatic robots.txt from the bundled agent registry Site owners pick bot categories (AI training crawlers, SEO tools, ...) and the plugin appends one robots.txt group naming every agent in those categories that has a known robots.txt token, ending in Disallow: /. Uses the agent registry bundled with the plugin: no account, no network request, kept current with each plugin update. - WebDecoy_Robots_Txt: pure generation (registry order, case-insensitive dedupe, unsafe or missing tokens skipped), sanitizing, and a late robots_txt filter that appends after existing output and adds nothing when the site discourages search engines. - Setting robots_disallow_categories, empty by default, so no site's robots.txt changes on upgrade. - Settings: Automatic robots.txt section in Good Bots with per-category counts, a search-engine warning, a preview, a link to the live file, and notices for a physical robots.txt or a subfolder install. - wp webdecoy config accepts robots_disallow_categories. - Blocking behaviour is unchanged. --- admin/partials/settings-page.php | 83 +++++++ includes/class-webdecoy-activator.php | 1 + includes/class-webdecoy-cli.php | 26 ++- includes/class-webdecoy-robots-txt.php | 289 +++++++++++++++++++++++++ tests/RobotsTxtTest.php | 217 +++++++++++++++++++ webdecoy.php | 13 ++ 6 files changed, 628 insertions(+), 1 deletion(-) create mode 100644 includes/class-webdecoy-robots-txt.php create mode 100644 tests/RobotsTxtTest.php diff --git a/admin/partials/settings-page.php b/admin/partials/settings-page.php index 10f588d..8e298cb 100644 --- a/admin/partials/settings-page.php +++ b/admin/partials/settings-page.php @@ -520,6 +520,89 @@ + + +

+

+ +

+

+ +

+ + +
+

+ ' . esc_html($wd_robots_file) . '' + ); + ?> +

+
+ +
+

+
+ + +
+

+
+ + + + + + + + + + + +
+
+ + $wd_count) : ?> + + +
+

+
+ +

+ +
+

+ +

+ +

+
diff --git a/includes/class-webdecoy-activator.php b/includes/class-webdecoy-activator.php index 3070653..3007fbd 100644 --- a/includes/class-webdecoy-activator.php +++ b/includes/class-webdecoy-activator.php @@ -203,6 +203,7 @@ private static function set_default_options(): void 'allow_social_bots' => true, 'block_ai_crawlers' => false, 'custom_allowlist' => [], + 'robots_disallow_categories' => [], // Blocking Settings // Monitor mode on by default: detect and log everything, block nothing, diff --git a/includes/class-webdecoy-cli.php b/includes/class-webdecoy-cli.php index 194d170..a58a9d7 100644 --- a/includes/class-webdecoy-cli.php +++ b/includes/class-webdecoy-cli.php @@ -49,6 +49,7 @@ class WebDecoy_CLI_Command 'rate_limit_requests' => ['type' => 'int', 'min' => 1, 'max' => 100000], 'rate_limit_window' => ['type' => 'int', 'min' => 1, 'max' => 86400], 'behind_cloudflare' => ['type' => 'bool'], + 'robots_disallow_categories' => ['type' => 'categories'], ]; /** @@ -128,6 +129,8 @@ public function status($args, $assoc_args): void * * wp webdecoy config set mode monitor * wp webdecoy config set block_ai_crawlers true + * wp webdecoy config set robots_disallow_categories training_crawler,seo_crawler + * wp webdecoy config set robots_disallow_categories none * wp webdecoy config get sensitivity * * @when after_wp_load @@ -147,7 +150,7 @@ public function config($args, $assoc_args): void $rows[] = [ 'key' => $name, 'value' => self::display_value($name, $options), - 'type' => $spec['type'] === 'enum' ? implode('|', $spec['values']) : $spec['type'], + 'type' => $spec['type'] === 'enum' ? implode('|', $spec['values'] ?? []) : $spec['type'], ]; } \WP_CLI\Utils\format_items('table', $rows, ['key', 'value', 'type']); @@ -329,6 +332,17 @@ public static function parse_config_value(string $key, string $raw): ?array case 'enum': $lowered = strtolower(trim($raw)); return in_array($lowered, $spec['values'], true) ? [$key, $lowered] : null; + case 'categories': + // Comma-separated category ids; `none` or an empty value clears. + $lowered = strtolower(trim($raw)); + if ($lowered === '' || $lowered === 'none') { + return [$key, []]; + } + $requested = array_values(array_filter(array_map('trim', explode(',', $lowered)), 'strlen')); + $selected = WebDecoy_Robots_Txt::sanitize_categories($requested); + // Refuse the whole value when any id is unknown, rather than + // silently dropping a typo. + return count($selected) === count(array_unique($requested)) ? [$key, $selected] : null; default: return null; } @@ -348,6 +362,9 @@ public static function display_value(string $key, array $options): string if (is_bool($value)) { return $value ? 'true' : 'false'; } + if (is_array($value) || (self::CONFIG_KEYS[$key]['type'] ?? '') === 'categories') { + return (array) $value === [] ? 'none' : implode(',', array_map('strval', (array) $value)); + } return (string) ($value ?? ''); } @@ -366,6 +383,13 @@ public static function value_help(string $key, string $raw): string return sprintf("'%s' is out of range for %s (%d-%d).", $raw, $key, $spec['min'] ?? 0, $spec['max'] ?? 0); case 'enum': return sprintf("'%s' is not valid for %s. Use: %s.", $raw, $key, implode(', ', $spec['values'])); + case 'categories': + return sprintf( + "'%s' is not valid for %s. Use none, or a comma-separated list of: %s.", + $raw, + $key, + implode(', ', array_keys(WebDecoy_Robots_Txt::choices())) + ); default: return "'{$raw}' is not valid for {$key}."; } diff --git a/includes/class-webdecoy-robots-txt.php b/includes/class-webdecoy-robots-txt.php new file mode 100644 index 0000000..ddbef69 --- /dev/null +++ b/includes/class-webdecoy-robots-txt.php @@ -0,0 +1,289 @@ +>|null $agents Registry agents; null reads the bundled registry. + * @return array category id => agent count + */ + public static function choices(?array $agents = null): array + { + if ($agents === null) { + $agents = \WebDecoy\AgentRegistry::all(); + $order = array_values(array_diff(\WebDecoy\AgentRegistry::categories(), [self::UNCATEGORISED])); + } else { + $order = self::category_order($agents); + } + + $choices = []; + foreach ($order as $category) { + $count = count(self::agent_names([$category], $agents)); + if ($count > 0) { + $choices[$category] = $count; + } + } + return $choices; + } + + /** + * Reduce submitted input to known, choosable category ids, in registry + * order, without duplicates. Accepts an array (the settings form) or a + * comma-separated string (WP-CLI). Anything else is an empty selection. + * + * @param mixed $input + * @param list>|null $agents + * @return list + */ + public static function sanitize_categories($input, ?array $agents = null): array + { + if (is_string($input)) { + $input = explode(',', $input); + } + if (!is_array($input)) { + return []; + } + $wanted = []; + foreach ($input as $value) { + if (is_string($value)) { + $wanted[strtolower(trim($value))] = true; + } + } + $selected = []; + foreach (array_keys(self::choices($agents)) as $category) { + if (isset($wanted[$category])) { + $selected[] = $category; + } + } + return $selected; + } + + /** + * Distinct robots.txt names for the agents in these categories, in + * registry order. Duplicates are dropped case-insensitively (robots.txt + * matching ignores case), keeping the first spelling. Agents without a + * known name, unknown categories and unsafe names contribute nothing. + * + * @param list $categories + * @param list>|null $agents + * @return list + */ + public static function agent_names(array $categories, ?array $agents = null): array + { + $agents = $agents ?? \WebDecoy\AgentRegistry::all(); + $wanted = array_flip(array_diff($categories, [self::UNCATEGORISED])); + + $names = []; + $seen = []; + foreach ($agents as $agent) { + if (!isset($wanted[(string) ($agent['category'] ?? '')])) { + continue; + } + // The pattern also refuses an empty name (no known robots.txt token). + $name = trim((string) ($agent['robots_name'] ?? '')); + if (preg_match(self::SAFE_NAME, $name) !== 1) { + continue; + } + $key = strtolower($name); + if (isset($seen[$key])) { + continue; + } + $seen[$key] = true; + $names[] = $name; + } + return $names; + } + + /** + * The robots.txt block for these categories, or '' when it would name no + * agent. Ends with a newline. + * + * @param list $categories + * @param list>|null $agents + */ + public static function block(array $categories, string $version, ?array $agents = null): string + { + $names = self::agent_names($categories, $agents); + if ($names === []) { + return ''; + } + $count = count($names); + $header = self::BEGIN_MARKER . ' (' . $count . ($count === 1 ? ' agent' : ' agents'); + if ($version !== '') { + $header .= ', updated with plugin version ' . $version; + } + $header .= ')'; + + $lines = [$header]; + foreach ($names as $name) { + $lines[] = 'User-agent: ' . $name; + } + $lines[] = 'Disallow: /'; + $lines[] = self::END_MARKER; + return implode("\n", $lines) . "\n"; + } + + /** + * Append the block to robots.txt output from WordPress and other plugins. + * + * The existing output is kept as it is. Nothing is added when the site is + * set to discourage search engines (Settings, Reading), when the block + * would be empty, or when it is already there. + * + * @param mixed $public Whether the site is public. WordPress 5.7+ passes a + * bool; older versions passed the option string. + * @param list $categories + * @param list>|null $agents + */ + public static function append(string $output, $public, array $categories, string $version, ?array $agents = null): string + { + // false (WordPress 5.7+), '0' or 0 (older): "discourage search engines". + if (empty($public)) { + return $output; + } + if (strpos($output, self::BEGIN_MARKER) !== false) { + return $output; + } + $block = self::block($categories, $version, $agents); + if ($block === '') { + return $output; + } + if ($output === '') { + return $block; + } + $separator = substr($output, -1) === "\n" ? "\n" : "\n\n"; + return $output . $separator . $block; + } + + /** + * A physical robots.txt that the web server serves instead of WordPress's + * virtual one, or null when there is none. Checks the WordPress directory + * and, when WordPress lives in a subdirectory of the site, the site root. + * + * @param list $roots Directories to check; defaults to the WordPress ones. + */ + public static function physical_file(array $roots = []): ?string + { + if ($roots === []) { + $roots[] = ABSPATH; + if (function_exists('get_home_path')) { + $roots[] = get_home_path(); + } + } + foreach (array_unique($roots) as $root) { + $path = rtrim($root, '/\\') . '/robots.txt'; + if (is_file($path)) { + return $path; + } + } + return null; + } + + /** + * True when the site's home address sits below a path (example.com/blog). + * robots.txt is only read from the root of a host, and WordPress serves + * its virtual one only when the site lives at that root. + */ + public static function home_is_subdirectory(string $home_url): bool + { + $path = (string) wp_parse_url($home_url, PHP_URL_PATH); + return trim($path, '/') !== ''; + } + + /** + * Categories in the order the given agents first use them, without the + * uncategorised bucket. For agent lists other than the bundled registry, + * whose own category list is used otherwise. + * + * @param list> $agents + * @return list + */ + private static function category_order(array $agents): array + { + $order = []; + foreach ($agents as $agent) { + $category = (string) ($agent['category'] ?? ''); + if ($category !== '' && $category !== self::UNCATEGORISED && !in_array($category, $order, true)) { + $order[] = $category; + } + } + return $order; + } +} diff --git a/tests/RobotsTxtTest.php b/tests/RobotsTxtTest.php new file mode 100644 index 0000000..c635385 --- /dev/null +++ b/tests/RobotsTxtTest.php @@ -0,0 +1,217 @@ + $cb, 'priority' => $priority, 'args' => $args]; + return true; + } +} +require_once dirname(__DIR__) . '/sdk/src/AgentRegistry.php'; +require_once dirname(__DIR__) . '/includes/class-webdecoy-robots-txt.php'; +require_once dirname(__DIR__) . '/includes/class-webdecoy-runtime-config.php'; +require_once dirname(__DIR__) . '/includes/class-webdecoy-cli.php'; + +$t = ['TestRunner', 'test']; +$same = ['TestRunner', 'assertSame']; +$true = ['TestRunner', 'assertTrue']; + +/** A small registry in the generated file's shape. */ +function robots_fixture(): array +{ + $agent = function (string $id, string $category, string $robots) { + return ['id' => $id, 'name' => $id, 'category' => $category, 'robots_name' => $robots, 'patterns' => [strtolower($id)]]; + }; + return [ + $agent('gptbot', 'training_crawler', 'GPTBot'), + $agent('claudebot', 'training_crawler', 'ClaudeBot'), + $agent('nameless', 'training_crawler', ''), + $agent('gptbot-alt', 'training_crawler', 'gptbot'), + $agent('ahrefs', 'seo_crawler', 'AhrefsBot'), + $agent('perplexity', 'ai_search_crawler', 'PerplexityBot'), + $agent('dup-across', 'ai_search_crawler', 'ClaudeBot'), + $agent('injected', 'seo_crawler', "Evil\nDisallow: /"), + $agent('headless', 'headless_browser', ''), + $agent('other', 'none', 'OtherBot'), + ]; +} + +const ROBOTS_WP_DEFAULT = "User-agent: *\nDisallow: /wp-admin/\nAllow: /wp-admin/admin-ajax.php\n\nSitemap: https://example.com/wp-sitemap.xml\n"; + +echo "\nAutomatic robots.txt: generation\n"; + +$t('names each robots token once, in registry order, first spelling kept', function () use ($same) { + $same(['GPTBot', 'ClaudeBot'], WebDecoy_Robots_Txt::agent_names(['training_crawler'], robots_fixture())); +}); + +$t('dedupes across categories too, and order follows the registry not the selection', function () use ($same) { + $same( + ['GPTBot', 'ClaudeBot', 'AhrefsBot', 'PerplexityBot'], + WebDecoy_Robots_Txt::agent_names(['seo_crawler', 'ai_search_crawler', 'training_crawler'], robots_fixture()) + ); +}); + +$t('agents without a robots token, unknown categories and the none bucket add nothing', function () use ($same) { + $same([], WebDecoy_Robots_Txt::agent_names(['headless_browser', 'made_up', 'none'], robots_fixture())); +}); + +$t('a token that could break out of its line is never written', function () use ($same) { + $same(['AhrefsBot'], WebDecoy_Robots_Txt::agent_names(['seo_crawler'], robots_fixture())); +}); + +$t('the block is one group ending in Disallow: /, between markers', function () use ($same) { + $same( + "# WebDecoy: AI and bot rules (2 agents, updated with plugin version 9.9.9)\n" + . "User-agent: GPTBot\nUser-agent: ClaudeBot\nDisallow: /\n# End of WebDecoy rules\n", + WebDecoy_Robots_Txt::block(['training_crawler'], '9.9.9', robots_fixture()) + ); +}); + +$t('an empty selection, or one that names no agent, is no block at all', function () use ($same) { + $same('', WebDecoy_Robots_Txt::block([], '9.9.9', robots_fixture())); + $same('', WebDecoy_Robots_Txt::block(['headless_browser'], '9.9.9', robots_fixture())); +}); + +echo "\nAutomatic robots.txt: choices and sanitizing\n"; + +$t('choices are categories with at least one token, counted after dedupe, none excluded', function () use ($same) { + $same( + ['training_crawler' => 2, 'seo_crawler' => 1, 'ai_search_crawler' => 2], + WebDecoy_Robots_Txt::choices(robots_fixture()) + ); +}); + +$t('sanitizing keeps known choices only, in choice order, once each', function () use ($same) { + $same( + ['training_crawler', 'seo_crawler'], + WebDecoy_Robots_Txt::sanitize_categories(['seo_crawler', 'none', 'nope', 'Training_Crawler ', 'seo_crawler', 'headless_browser', 7], robots_fixture()) + ); +}); + +$t('a comma-separated string is accepted; anything else is an empty selection', function () use ($same) { + $same(['training_crawler', 'ai_search_crawler'], WebDecoy_Robots_Txt::sanitize_categories('ai_search_crawler, training_crawler', robots_fixture())); + $same([], WebDecoy_Robots_Txt::sanitize_categories(null, robots_fixture())); + $same([], WebDecoy_Robots_Txt::sanitize_categories(true, robots_fixture())); +}); + +$t('the bundled registry offers AI training crawlers and search engines, never none', function () use ($true) { + $choices = WebDecoy_Robots_Txt::choices(); + $true(($choices['training_crawler'] ?? 0) > 0, 'training_crawler must be offered'); + $true(isset($choices['search_crawler']), 'search engines are selectable'); + $true(!isset($choices['none']), 'none is never offered'); + $true(!isset($choices['headless_browser']), 'a category with no robots token has nothing to offer'); +}); + +$t('with the bundled registry, AI training crawlers include GPTBot and ClaudeBot', function () use ($true) { + $names = WebDecoy_Robots_Txt::agent_names(['training_crawler']); + $true(in_array('GPTBot', $names, true) && in_array('ClaudeBot', $names, true)); + $true(!in_array('Googlebot', $names, true), 'search engines are not training crawlers'); +}); + +echo "\nAutomatic robots.txt: joining WordPress's output\n"; + +$t('the block is appended after the existing output, which is kept byte for byte', function () use ($same, $true) { + $same(true, strpos(WebDecoy_Robots_Txt::append(ROBOTS_WP_DEFAULT, true, ['training_crawler'], '9.9.9', robots_fixture()), 'User-agent: GPTBot') !== false, 'a public site (bool true) gets the block'); + $out = WebDecoy_Robots_Txt::append(ROBOTS_WP_DEFAULT, '1', ['training_crawler'], '9.9.9', robots_fixture()); + $same(0, strpos($out, ROBOTS_WP_DEFAULT), 'existing rules and sitemap line come first, unchanged'); + $same( + ROBOTS_WP_DEFAULT . "\n" . WebDecoy_Robots_Txt::block(['training_crawler'], '9.9.9', robots_fixture()), + $out + ); +}); + +$t('output without a trailing newline still gets a blank line before the block', function () use ($true) { + $out = WebDecoy_Robots_Txt::append("User-agent: *\nDisallow:", '1', ['training_crawler'], '9.9.9', robots_fixture()); + $true(strpos($out, "Disallow:\n\n# WebDecoy: AI and bot rules") !== false, $out); +}); + +$t('nothing is added when the site discourages search engines', function () use ($same) { + // WordPress 5.7+ passes a bool; older versions passed the option string. + foreach ([false, '0', 0] as $public) { + $same(ROBOTS_WP_DEFAULT, WebDecoy_Robots_Txt::append(ROBOTS_WP_DEFAULT, $public, ['training_crawler'], '9.9.9', robots_fixture()), var_export($public, true)); + } +}); + +$t('nothing is added with no selection', function () use ($same) { + $same(ROBOTS_WP_DEFAULT, WebDecoy_Robots_Txt::append(ROBOTS_WP_DEFAULT, '1', [], '9.9.9', robots_fixture())); +}); + +$t('running the filter twice adds the block once', function () use ($same) { + $once = WebDecoy_Robots_Txt::append(ROBOTS_WP_DEFAULT, '1', ['training_crawler'], '9.9.9', robots_fixture()); + $same($once, WebDecoy_Robots_Txt::append($once, '1', ['training_crawler'], '9.9.9', robots_fixture())); +}); + +$t('register() hooks robots_txt late only when a category is chosen, and the hook appends', function () use ($same, $true) { + $GLOBALS['__wd_filters'] = []; + WebDecoy_Robots_Txt::register([]); + WebDecoy_Robots_Txt::register(['not_a_category']); + WebDecoy_Robots_Txt::register(null); + $same([], $GLOBALS['__wd_filters'], 'no selection must register nothing'); + + WebDecoy_Robots_Txt::register(['training_crawler']); + $hooks = $GLOBALS['__wd_filters']['robots_txt'] ?? []; + $same(1, count($hooks)); + $true($hooks[0]['priority'] > 10, 'runs after sitemaps and other plugins'); + $same(2, $hooks[0]['args']); + + $served = ($hooks[0]['cb'])(ROBOTS_WP_DEFAULT, '1'); + $same(0, strpos($served, ROBOTS_WP_DEFAULT)); + $true(strpos($served, "User-agent: GPTBot\n") !== false, $served); + $same(ROBOTS_WP_DEFAULT, ($hooks[0]['cb'])(ROBOTS_WP_DEFAULT, false)); + $GLOBALS['__wd_filters'] = []; +}); + +echo "\nAutomatic robots.txt: where it can apply\n"; + +$t('a physical robots.txt is found, and its absence is null', function () use ($same) { + $dir = sys_get_temp_dir() . '/wd-robots-' . bin2hex(random_bytes(4)); + mkdir($dir); + $same(null, WebDecoy_Robots_Txt::physical_file([$dir])); + file_put_contents($dir . '/robots.txt', "User-agent: *\n"); + $same($dir . '/robots.txt', WebDecoy_Robots_Txt::physical_file([$dir . '/'])); + unlink($dir . '/robots.txt'); + rmdir($dir); +}); + +$t('a home address below a path is a subdirectory install', function () use ($same) { + $same(false, WebDecoy_Robots_Txt::home_is_subdirectory('https://example.com/')); + $same(false, WebDecoy_Robots_Txt::home_is_subdirectory('https://example.com')); + $same(true, WebDecoy_Robots_Txt::home_is_subdirectory('https://example.com/blog/')); +}); + +echo "\nWP-CLI: robots_disallow_categories\n"; + +$t('a comma list of known categories sets them; none clears', function () use ($same) { + $same( + ['robots_disallow_categories', ['training_crawler', 'seo_crawler']], + WebDecoy_CLI_Command::parse_config_value('robots_disallow_categories', 'seo_crawler, training_crawler') + ); + $same(['robots_disallow_categories', []], WebDecoy_CLI_Command::parse_config_value('robots_disallow_categories', 'none')); +}); + +$t('an unknown category refuses the whole value', function () use ($same) { + $same(null, WebDecoy_CLI_Command::parse_config_value('robots_disallow_categories', 'training_crawler,trainng_crawler')); +}); + +$t('the stored list displays as a comma list, or none', function () use ($same) { + $same('training_crawler,seo_crawler', WebDecoy_CLI_Command::display_value('robots_disallow_categories', ['robots_disallow_categories' => ['training_crawler', 'seo_crawler']])); + $same('none', WebDecoy_CLI_Command::display_value('robots_disallow_categories', ['robots_disallow_categories' => []])); +}); diff --git a/webdecoy.php b/webdecoy.php index 5f4798e..94512d8 100644 --- a/webdecoy.php +++ b/webdecoy.php @@ -237,6 +237,11 @@ private function load_options(): void 'block_ai_crawlers' => false, 'custom_allowlist' => [], + // Automatic robots.txt: bot categories asked not to crawl, as a + // list of registry category ids. Empty (off) by default, so an + // upgrade never changes a site's robots.txt on its own. + 'robots_disallow_categories' => [], + // Blocking Settings // Monitor mode is the safe default: everything is detected, logged and // reported, and nothing is ever blocked, throttled or 403'd. Turn it off @@ -1070,6 +1075,7 @@ public function load_includes(): void require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-detection-sender.php'; require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-ai-referrals.php'; require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-crawler-visits.php'; + require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-robots-txt.php'; require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-request-counts.php'; require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-honeytoken.php'; require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-ip-enrichment.php'; @@ -1113,6 +1119,10 @@ public function load_includes(): void // One wordpress.org review ask, after a week with bots counted. Admin only. (new WebDecoy_Review_Ask())->register(); + // Automatic robots.txt: appends the chosen categories' agents to the + // virtual robots.txt. Adds nothing until the owner picks a category. + WebDecoy_Robots_Txt::register($this->options['robots_disallow_categories'] ?? []); + if (class_exists('WooCommerce')) { require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-woocommerce.php'; } @@ -2489,6 +2499,9 @@ public function sanitize_options(array $input): array $sanitized['block_ai_crawlers'] = !empty($input['block_ai_crawlers']); $sanitized['custom_allowlist'] = array_filter(array_map('sanitize_text_field', explode("\n", $input['custom_allowlist'] ?? ''))); + // Automatic robots.txt: only categories the bundled registry offers. + $sanitized['robots_disallow_categories'] = WebDecoy_Robots_Txt::sanitize_categories($input['robots_disallow_categories'] ?? []); + // Blocking Settings // Allowlist: validate as IPs/CIDRs (reuses the trusted-proxy validator), // stored as an array of valid entries. From 91d7c4174717bf4b1192b15a2d82f5bf7a4e8ca5 Mon Sep 17 00:00:00 2001 From: Chris Portscheller Date: Sun, 15 Nov 2026 20:23:20 -0600 Subject: [PATCH 2/2] Add robots.txt rules even when search engines are discouraged The owner chose these categories, and the noindex tag WordPress adds in that mode does not keep training crawlers away. The filter now takes only the output, so the public flag cannot decide it, and the settings notice that said rules were skipped is gone. --- admin/partials/settings-page.php | 6 ----- includes/class-webdecoy-robots-txt.php | 23 +++++++---------- tests/RobotsTxtTest.php | 34 ++++++++++++++------------ 3 files changed, 27 insertions(+), 36 deletions(-) diff --git a/admin/partials/settings-page.php b/admin/partials/settings-page.php index 8e298cb..6b7d7d8 100644 --- a/admin/partials/settings-page.php +++ b/admin/partials/settings-page.php @@ -527,7 +527,6 @@ $wd_robots_block = WebDecoy_Robots_Txt::block($wd_robots_selected, WEBDECOY_VERSION); $wd_robots_file = WebDecoy_Robots_Txt::physical_file(); $wd_robots_subdir = WebDecoy_Robots_Txt::home_is_subdirectory(home_url('/')); - $wd_robots_private = !get_option('blog_public'); ?>

@@ -554,11 +553,6 @@

- -
-

-
- diff --git a/includes/class-webdecoy-robots-txt.php b/includes/class-webdecoy-robots-txt.php index ddbef69..c0118f2 100644 --- a/includes/class-webdecoy-robots-txt.php +++ b/includes/class-webdecoy-robots-txt.php @@ -65,15 +65,17 @@ public static function register($categories): void if ($selected === []) { return; } - // Late, so sitemap lines and other plugins' rules come first. - add_filter('robots_txt', static function ($output, $public) use ($selected) { + // Late, so sitemap lines and other plugins' rules come first. Takes + // only the output: the site's "discourage search engines" flag does + // not decide this, because the owner chose these categories and the + // noindex tag that setting adds does not keep training crawlers away. + add_filter('robots_txt', static function ($output) use ($selected) { return self::append( (string) $output, - $public, $selected, defined('WEBDECOY_VERSION') ? (string) WEBDECOY_VERSION : '' ); - }, 100, 2); + }, 100, 1); } /** @@ -203,21 +205,14 @@ public static function block(array $categories, string $version, ?array $agents /** * Append the block to robots.txt output from WordPress and other plugins. * - * The existing output is kept as it is. Nothing is added when the site is - * set to discourage search engines (Settings, Reading), when the block - * would be empty, or when it is already there. + * The existing output is kept as it is. Nothing is added when the block + * would be empty or is already there. * - * @param mixed $public Whether the site is public. WordPress 5.7+ passes a - * bool; older versions passed the option string. * @param list $categories * @param list>|null $agents */ - public static function append(string $output, $public, array $categories, string $version, ?array $agents = null): string + public static function append(string $output, array $categories, string $version, ?array $agents = null): string { - // false (WordPress 5.7+), '0' or 0 (older): "discourage search engines". - if (empty($public)) { - return $output; - } if (strpos($output, self::BEGIN_MARKER) !== false) { return $output; } diff --git a/tests/RobotsTxtTest.php b/tests/RobotsTxtTest.php index c635385..7acc6ce 100644 --- a/tests/RobotsTxtTest.php +++ b/tests/RobotsTxtTest.php @@ -129,8 +129,7 @@ function robots_fixture(): array echo "\nAutomatic robots.txt: joining WordPress's output\n"; $t('the block is appended after the existing output, which is kept byte for byte', function () use ($same, $true) { - $same(true, strpos(WebDecoy_Robots_Txt::append(ROBOTS_WP_DEFAULT, true, ['training_crawler'], '9.9.9', robots_fixture()), 'User-agent: GPTBot') !== false, 'a public site (bool true) gets the block'); - $out = WebDecoy_Robots_Txt::append(ROBOTS_WP_DEFAULT, '1', ['training_crawler'], '9.9.9', robots_fixture()); + $out = WebDecoy_Robots_Txt::append(ROBOTS_WP_DEFAULT, ['training_crawler'], '9.9.9', robots_fixture()); $same(0, strpos($out, ROBOTS_WP_DEFAULT), 'existing rules and sitemap line come first, unchanged'); $same( ROBOTS_WP_DEFAULT . "\n" . WebDecoy_Robots_Txt::block(['training_crawler'], '9.9.9', robots_fixture()), @@ -139,24 +138,17 @@ function robots_fixture(): array }); $t('output without a trailing newline still gets a blank line before the block', function () use ($true) { - $out = WebDecoy_Robots_Txt::append("User-agent: *\nDisallow:", '1', ['training_crawler'], '9.9.9', robots_fixture()); + $out = WebDecoy_Robots_Txt::append("User-agent: *\nDisallow:", ['training_crawler'], '9.9.9', robots_fixture()); $true(strpos($out, "Disallow:\n\n# WebDecoy: AI and bot rules") !== false, $out); }); -$t('nothing is added when the site discourages search engines', function () use ($same) { - // WordPress 5.7+ passes a bool; older versions passed the option string. - foreach ([false, '0', 0] as $public) { - $same(ROBOTS_WP_DEFAULT, WebDecoy_Robots_Txt::append(ROBOTS_WP_DEFAULT, $public, ['training_crawler'], '9.9.9', robots_fixture()), var_export($public, true)); - } -}); - $t('nothing is added with no selection', function () use ($same) { - $same(ROBOTS_WP_DEFAULT, WebDecoy_Robots_Txt::append(ROBOTS_WP_DEFAULT, '1', [], '9.9.9', robots_fixture())); + $same(ROBOTS_WP_DEFAULT, WebDecoy_Robots_Txt::append(ROBOTS_WP_DEFAULT, [], '9.9.9', robots_fixture())); }); $t('running the filter twice adds the block once', function () use ($same) { - $once = WebDecoy_Robots_Txt::append(ROBOTS_WP_DEFAULT, '1', ['training_crawler'], '9.9.9', robots_fixture()); - $same($once, WebDecoy_Robots_Txt::append($once, '1', ['training_crawler'], '9.9.9', robots_fixture())); + $once = WebDecoy_Robots_Txt::append(ROBOTS_WP_DEFAULT, ['training_crawler'], '9.9.9', robots_fixture()); + $same($once, WebDecoy_Robots_Txt::append($once, ['training_crawler'], '9.9.9', robots_fixture())); }); $t('register() hooks robots_txt late only when a category is chosen, and the hook appends', function () use ($same, $true) { @@ -170,12 +162,22 @@ function robots_fixture(): array $hooks = $GLOBALS['__wd_filters']['robots_txt'] ?? []; $same(1, count($hooks)); $true($hooks[0]['priority'] > 10, 'runs after sitemaps and other plugins'); - $same(2, $hooks[0]['args']); + $same(1, $hooks[0]['args'], 'only the output is taken: the public flag must not decide this'); - $served = ($hooks[0]['cb'])(ROBOTS_WP_DEFAULT, '1'); + $served = ($hooks[0]['cb'])(ROBOTS_WP_DEFAULT); $same(0, strpos($served, ROBOTS_WP_DEFAULT)); $true(strpos($served, "User-agent: GPTBot\n") !== false, $served); - $same(ROBOTS_WP_DEFAULT, ($hooks[0]['cb'])(ROBOTS_WP_DEFAULT, false)); + $GLOBALS['__wd_filters'] = []; +}); + +$t('a site that discourages search engines still gets the chosen rules', function () use ($true) { + $GLOBALS['__wd_filters'] = []; + WebDecoy_Robots_Txt::register(['training_crawler']); + $cb = $GLOBALS['__wd_filters']['robots_txt'][0]['cb']; + // As if a caller passed WordPress's flag anyway, in every shape it has had. + foreach ([false, '0', 0] as $public) { + $true(strpos($cb(ROBOTS_WP_DEFAULT, $public), "User-agent: GPTBot\n") !== false, var_export($public, true)); + } $GLOBALS['__wd_filters'] = []; });