diff --git a/admin/partials/settings-page.php b/admin/partials/settings-page.php index 10f588d..6b7d7d8 100644 --- a/admin/partials/settings-page.php +++ b/admin/partials/settings-page.php @@ -520,6 +520,83 @@ + + +

+

+ +

+

+ +

+ + +
+

+ ' . esc_html($wd_robots_file) . '' + ); + ?> +

+
+ +
+

+
+ + + + + + + + + + + +
+
+ + $wd_count) : ?> + + +
+

+
+ +

+ +
+

+ +

+ +

+
diff --git a/includes/class-webdecoy-activator.php b/includes/class-webdecoy-activator.php index 3070653..3007fbd 100644 --- a/includes/class-webdecoy-activator.php +++ b/includes/class-webdecoy-activator.php @@ -203,6 +203,7 @@ private static function set_default_options(): void 'allow_social_bots' => true, 'block_ai_crawlers' => false, 'custom_allowlist' => [], + 'robots_disallow_categories' => [], // Blocking Settings // Monitor mode on by default: detect and log everything, block nothing, diff --git a/includes/class-webdecoy-cli.php b/includes/class-webdecoy-cli.php index 194d170..a58a9d7 100644 --- a/includes/class-webdecoy-cli.php +++ b/includes/class-webdecoy-cli.php @@ -49,6 +49,7 @@ class WebDecoy_CLI_Command 'rate_limit_requests' => ['type' => 'int', 'min' => 1, 'max' => 100000], 'rate_limit_window' => ['type' => 'int', 'min' => 1, 'max' => 86400], 'behind_cloudflare' => ['type' => 'bool'], + 'robots_disallow_categories' => ['type' => 'categories'], ]; /** @@ -128,6 +129,8 @@ public function status($args, $assoc_args): void * * wp webdecoy config set mode monitor * wp webdecoy config set block_ai_crawlers true + * wp webdecoy config set robots_disallow_categories training_crawler,seo_crawler + * wp webdecoy config set robots_disallow_categories none * wp webdecoy config get sensitivity * * @when after_wp_load @@ -147,7 +150,7 @@ public function config($args, $assoc_args): void $rows[] = [ 'key' => $name, 'value' => self::display_value($name, $options), - 'type' => $spec['type'] === 'enum' ? implode('|', $spec['values']) : $spec['type'], + 'type' => $spec['type'] === 'enum' ? implode('|', $spec['values'] ?? []) : $spec['type'], ]; } \WP_CLI\Utils\format_items('table', $rows, ['key', 'value', 'type']); @@ -329,6 +332,17 @@ public static function parse_config_value(string $key, string $raw): ?array case 'enum': $lowered = strtolower(trim($raw)); return in_array($lowered, $spec['values'], true) ? [$key, $lowered] : null; + case 'categories': + // Comma-separated category ids; `none` or an empty value clears. + $lowered = strtolower(trim($raw)); + if ($lowered === '' || $lowered === 'none') { + return [$key, []]; + } + $requested = array_values(array_filter(array_map('trim', explode(',', $lowered)), 'strlen')); + $selected = WebDecoy_Robots_Txt::sanitize_categories($requested); + // Refuse the whole value when any id is unknown, rather than + // silently dropping a typo. + return count($selected) === count(array_unique($requested)) ? [$key, $selected] : null; default: return null; } @@ -348,6 +362,9 @@ public static function display_value(string $key, array $options): string if (is_bool($value)) { return $value ? 'true' : 'false'; } + if (is_array($value) || (self::CONFIG_KEYS[$key]['type'] ?? '') === 'categories') { + return (array) $value === [] ? 'none' : implode(',', array_map('strval', (array) $value)); + } return (string) ($value ?? ''); } @@ -366,6 +383,13 @@ public static function value_help(string $key, string $raw): string return sprintf("'%s' is out of range for %s (%d-%d).", $raw, $key, $spec['min'] ?? 0, $spec['max'] ?? 0); case 'enum': return sprintf("'%s' is not valid for %s. Use: %s.", $raw, $key, implode(', ', $spec['values'])); + case 'categories': + return sprintf( + "'%s' is not valid for %s. Use none, or a comma-separated list of: %s.", + $raw, + $key, + implode(', ', array_keys(WebDecoy_Robots_Txt::choices())) + ); default: return "'{$raw}' is not valid for {$key}."; } diff --git a/includes/class-webdecoy-robots-txt.php b/includes/class-webdecoy-robots-txt.php new file mode 100644 index 0000000..c0118f2 --- /dev/null +++ b/includes/class-webdecoy-robots-txt.php @@ -0,0 +1,284 @@ +>|null $agents Registry agents; null reads the bundled registry. + * @return array category id => agent count + */ + public static function choices(?array $agents = null): array + { + if ($agents === null) { + $agents = \WebDecoy\AgentRegistry::all(); + $order = array_values(array_diff(\WebDecoy\AgentRegistry::categories(), [self::UNCATEGORISED])); + } else { + $order = self::category_order($agents); + } + + $choices = []; + foreach ($order as $category) { + $count = count(self::agent_names([$category], $agents)); + if ($count > 0) { + $choices[$category] = $count; + } + } + return $choices; + } + + /** + * Reduce submitted input to known, choosable category ids, in registry + * order, without duplicates. Accepts an array (the settings form) or a + * comma-separated string (WP-CLI). Anything else is an empty selection. + * + * @param mixed $input + * @param list>|null $agents + * @return list + */ + public static function sanitize_categories($input, ?array $agents = null): array + { + if (is_string($input)) { + $input = explode(',', $input); + } + if (!is_array($input)) { + return []; + } + $wanted = []; + foreach ($input as $value) { + if (is_string($value)) { + $wanted[strtolower(trim($value))] = true; + } + } + $selected = []; + foreach (array_keys(self::choices($agents)) as $category) { + if (isset($wanted[$category])) { + $selected[] = $category; + } + } + return $selected; + } + + /** + * Distinct robots.txt names for the agents in these categories, in + * registry order. Duplicates are dropped case-insensitively (robots.txt + * matching ignores case), keeping the first spelling. Agents without a + * known name, unknown categories and unsafe names contribute nothing. + * + * @param list $categories + * @param list>|null $agents + * @return list + */ + public static function agent_names(array $categories, ?array $agents = null): array + { + $agents = $agents ?? \WebDecoy\AgentRegistry::all(); + $wanted = array_flip(array_diff($categories, [self::UNCATEGORISED])); + + $names = []; + $seen = []; + foreach ($agents as $agent) { + if (!isset($wanted[(string) ($agent['category'] ?? '')])) { + continue; + } + // The pattern also refuses an empty name (no known robots.txt token). + $name = trim((string) ($agent['robots_name'] ?? '')); + if (preg_match(self::SAFE_NAME, $name) !== 1) { + continue; + } + $key = strtolower($name); + if (isset($seen[$key])) { + continue; + } + $seen[$key] = true; + $names[] = $name; + } + return $names; + } + + /** + * The robots.txt block for these categories, or '' when it would name no + * agent. Ends with a newline. + * + * @param list $categories + * @param list>|null $agents + */ + public static function block(array $categories, string $version, ?array $agents = null): string + { + $names = self::agent_names($categories, $agents); + if ($names === []) { + return ''; + } + $count = count($names); + $header = self::BEGIN_MARKER . ' (' . $count . ($count === 1 ? ' agent' : ' agents'); + if ($version !== '') { + $header .= ', updated with plugin version ' . $version; + } + $header .= ')'; + + $lines = [$header]; + foreach ($names as $name) { + $lines[] = 'User-agent: ' . $name; + } + $lines[] = 'Disallow: /'; + $lines[] = self::END_MARKER; + return implode("\n", $lines) . "\n"; + } + + /** + * Append the block to robots.txt output from WordPress and other plugins. + * + * The existing output is kept as it is. Nothing is added when the block + * would be empty or is already there. + * + * @param list $categories + * @param list>|null $agents + */ + public static function append(string $output, array $categories, string $version, ?array $agents = null): string + { + if (strpos($output, self::BEGIN_MARKER) !== false) { + return $output; + } + $block = self::block($categories, $version, $agents); + if ($block === '') { + return $output; + } + if ($output === '') { + return $block; + } + $separator = substr($output, -1) === "\n" ? "\n" : "\n\n"; + return $output . $separator . $block; + } + + /** + * A physical robots.txt that the web server serves instead of WordPress's + * virtual one, or null when there is none. Checks the WordPress directory + * and, when WordPress lives in a subdirectory of the site, the site root. + * + * @param list $roots Directories to check; defaults to the WordPress ones. + */ + public static function physical_file(array $roots = []): ?string + { + if ($roots === []) { + $roots[] = ABSPATH; + if (function_exists('get_home_path')) { + $roots[] = get_home_path(); + } + } + foreach (array_unique($roots) as $root) { + $path = rtrim($root, '/\\') . '/robots.txt'; + if (is_file($path)) { + return $path; + } + } + return null; + } + + /** + * True when the site's home address sits below a path (example.com/blog). + * robots.txt is only read from the root of a host, and WordPress serves + * its virtual one only when the site lives at that root. + */ + public static function home_is_subdirectory(string $home_url): bool + { + $path = (string) wp_parse_url($home_url, PHP_URL_PATH); + return trim($path, '/') !== ''; + } + + /** + * Categories in the order the given agents first use them, without the + * uncategorised bucket. For agent lists other than the bundled registry, + * whose own category list is used otherwise. + * + * @param list> $agents + * @return list + */ + private static function category_order(array $agents): array + { + $order = []; + foreach ($agents as $agent) { + $category = (string) ($agent['category'] ?? ''); + if ($category !== '' && $category !== self::UNCATEGORISED && !in_array($category, $order, true)) { + $order[] = $category; + } + } + return $order; + } +} diff --git a/tests/RobotsTxtTest.php b/tests/RobotsTxtTest.php new file mode 100644 index 0000000..7acc6ce --- /dev/null +++ b/tests/RobotsTxtTest.php @@ -0,0 +1,219 @@ + $cb, 'priority' => $priority, 'args' => $args]; + return true; + } +} +require_once dirname(__DIR__) . '/sdk/src/AgentRegistry.php'; +require_once dirname(__DIR__) . '/includes/class-webdecoy-robots-txt.php'; +require_once dirname(__DIR__) . '/includes/class-webdecoy-runtime-config.php'; +require_once dirname(__DIR__) . '/includes/class-webdecoy-cli.php'; + +$t = ['TestRunner', 'test']; +$same = ['TestRunner', 'assertSame']; +$true = ['TestRunner', 'assertTrue']; + +/** A small registry in the generated file's shape. */ +function robots_fixture(): array +{ + $agent = function (string $id, string $category, string $robots) { + return ['id' => $id, 'name' => $id, 'category' => $category, 'robots_name' => $robots, 'patterns' => [strtolower($id)]]; + }; + return [ + $agent('gptbot', 'training_crawler', 'GPTBot'), + $agent('claudebot', 'training_crawler', 'ClaudeBot'), + $agent('nameless', 'training_crawler', ''), + $agent('gptbot-alt', 'training_crawler', 'gptbot'), + $agent('ahrefs', 'seo_crawler', 'AhrefsBot'), + $agent('perplexity', 'ai_search_crawler', 'PerplexityBot'), + $agent('dup-across', 'ai_search_crawler', 'ClaudeBot'), + $agent('injected', 'seo_crawler', "Evil\nDisallow: /"), + $agent('headless', 'headless_browser', ''), + $agent('other', 'none', 'OtherBot'), + ]; +} + +const ROBOTS_WP_DEFAULT = "User-agent: *\nDisallow: /wp-admin/\nAllow: /wp-admin/admin-ajax.php\n\nSitemap: https://example.com/wp-sitemap.xml\n"; + +echo "\nAutomatic robots.txt: generation\n"; + +$t('names each robots token once, in registry order, first spelling kept', function () use ($same) { + $same(['GPTBot', 'ClaudeBot'], WebDecoy_Robots_Txt::agent_names(['training_crawler'], robots_fixture())); +}); + +$t('dedupes across categories too, and order follows the registry not the selection', function () use ($same) { + $same( + ['GPTBot', 'ClaudeBot', 'AhrefsBot', 'PerplexityBot'], + WebDecoy_Robots_Txt::agent_names(['seo_crawler', 'ai_search_crawler', 'training_crawler'], robots_fixture()) + ); +}); + +$t('agents without a robots token, unknown categories and the none bucket add nothing', function () use ($same) { + $same([], WebDecoy_Robots_Txt::agent_names(['headless_browser', 'made_up', 'none'], robots_fixture())); +}); + +$t('a token that could break out of its line is never written', function () use ($same) { + $same(['AhrefsBot'], WebDecoy_Robots_Txt::agent_names(['seo_crawler'], robots_fixture())); +}); + +$t('the block is one group ending in Disallow: /, between markers', function () use ($same) { + $same( + "# WebDecoy: AI and bot rules (2 agents, updated with plugin version 9.9.9)\n" + . "User-agent: GPTBot\nUser-agent: ClaudeBot\nDisallow: /\n# End of WebDecoy rules\n", + WebDecoy_Robots_Txt::block(['training_crawler'], '9.9.9', robots_fixture()) + ); +}); + +$t('an empty selection, or one that names no agent, is no block at all', function () use ($same) { + $same('', WebDecoy_Robots_Txt::block([], '9.9.9', robots_fixture())); + $same('', WebDecoy_Robots_Txt::block(['headless_browser'], '9.9.9', robots_fixture())); +}); + +echo "\nAutomatic robots.txt: choices and sanitizing\n"; + +$t('choices are categories with at least one token, counted after dedupe, none excluded', function () use ($same) { + $same( + ['training_crawler' => 2, 'seo_crawler' => 1, 'ai_search_crawler' => 2], + WebDecoy_Robots_Txt::choices(robots_fixture()) + ); +}); + +$t('sanitizing keeps known choices only, in choice order, once each', function () use ($same) { + $same( + ['training_crawler', 'seo_crawler'], + WebDecoy_Robots_Txt::sanitize_categories(['seo_crawler', 'none', 'nope', 'Training_Crawler ', 'seo_crawler', 'headless_browser', 7], robots_fixture()) + ); +}); + +$t('a comma-separated string is accepted; anything else is an empty selection', function () use ($same) { + $same(['training_crawler', 'ai_search_crawler'], WebDecoy_Robots_Txt::sanitize_categories('ai_search_crawler, training_crawler', robots_fixture())); + $same([], WebDecoy_Robots_Txt::sanitize_categories(null, robots_fixture())); + $same([], WebDecoy_Robots_Txt::sanitize_categories(true, robots_fixture())); +}); + +$t('the bundled registry offers AI training crawlers and search engines, never none', function () use ($true) { + $choices = WebDecoy_Robots_Txt::choices(); + $true(($choices['training_crawler'] ?? 0) > 0, 'training_crawler must be offered'); + $true(isset($choices['search_crawler']), 'search engines are selectable'); + $true(!isset($choices['none']), 'none is never offered'); + $true(!isset($choices['headless_browser']), 'a category with no robots token has nothing to offer'); +}); + +$t('with the bundled registry, AI training crawlers include GPTBot and ClaudeBot', function () use ($true) { + $names = WebDecoy_Robots_Txt::agent_names(['training_crawler']); + $true(in_array('GPTBot', $names, true) && in_array('ClaudeBot', $names, true)); + $true(!in_array('Googlebot', $names, true), 'search engines are not training crawlers'); +}); + +echo "\nAutomatic robots.txt: joining WordPress's output\n"; + +$t('the block is appended after the existing output, which is kept byte for byte', function () use ($same, $true) { + $out = WebDecoy_Robots_Txt::append(ROBOTS_WP_DEFAULT, ['training_crawler'], '9.9.9', robots_fixture()); + $same(0, strpos($out, ROBOTS_WP_DEFAULT), 'existing rules and sitemap line come first, unchanged'); + $same( + ROBOTS_WP_DEFAULT . "\n" . WebDecoy_Robots_Txt::block(['training_crawler'], '9.9.9', robots_fixture()), + $out + ); +}); + +$t('output without a trailing newline still gets a blank line before the block', function () use ($true) { + $out = WebDecoy_Robots_Txt::append("User-agent: *\nDisallow:", ['training_crawler'], '9.9.9', robots_fixture()); + $true(strpos($out, "Disallow:\n\n# WebDecoy: AI and bot rules") !== false, $out); +}); + +$t('nothing is added with no selection', function () use ($same) { + $same(ROBOTS_WP_DEFAULT, WebDecoy_Robots_Txt::append(ROBOTS_WP_DEFAULT, [], '9.9.9', robots_fixture())); +}); + +$t('running the filter twice adds the block once', function () use ($same) { + $once = WebDecoy_Robots_Txt::append(ROBOTS_WP_DEFAULT, ['training_crawler'], '9.9.9', robots_fixture()); + $same($once, WebDecoy_Robots_Txt::append($once, ['training_crawler'], '9.9.9', robots_fixture())); +}); + +$t('register() hooks robots_txt late only when a category is chosen, and the hook appends', function () use ($same, $true) { + $GLOBALS['__wd_filters'] = []; + WebDecoy_Robots_Txt::register([]); + WebDecoy_Robots_Txt::register(['not_a_category']); + WebDecoy_Robots_Txt::register(null); + $same([], $GLOBALS['__wd_filters'], 'no selection must register nothing'); + + WebDecoy_Robots_Txt::register(['training_crawler']); + $hooks = $GLOBALS['__wd_filters']['robots_txt'] ?? []; + $same(1, count($hooks)); + $true($hooks[0]['priority'] > 10, 'runs after sitemaps and other plugins'); + $same(1, $hooks[0]['args'], 'only the output is taken: the public flag must not decide this'); + + $served = ($hooks[0]['cb'])(ROBOTS_WP_DEFAULT); + $same(0, strpos($served, ROBOTS_WP_DEFAULT)); + $true(strpos($served, "User-agent: GPTBot\n") !== false, $served); + $GLOBALS['__wd_filters'] = []; +}); + +$t('a site that discourages search engines still gets the chosen rules', function () use ($true) { + $GLOBALS['__wd_filters'] = []; + WebDecoy_Robots_Txt::register(['training_crawler']); + $cb = $GLOBALS['__wd_filters']['robots_txt'][0]['cb']; + // As if a caller passed WordPress's flag anyway, in every shape it has had. + foreach ([false, '0', 0] as $public) { + $true(strpos($cb(ROBOTS_WP_DEFAULT, $public), "User-agent: GPTBot\n") !== false, var_export($public, true)); + } + $GLOBALS['__wd_filters'] = []; +}); + +echo "\nAutomatic robots.txt: where it can apply\n"; + +$t('a physical robots.txt is found, and its absence is null', function () use ($same) { + $dir = sys_get_temp_dir() . '/wd-robots-' . bin2hex(random_bytes(4)); + mkdir($dir); + $same(null, WebDecoy_Robots_Txt::physical_file([$dir])); + file_put_contents($dir . '/robots.txt', "User-agent: *\n"); + $same($dir . '/robots.txt', WebDecoy_Robots_Txt::physical_file([$dir . '/'])); + unlink($dir . '/robots.txt'); + rmdir($dir); +}); + +$t('a home address below a path is a subdirectory install', function () use ($same) { + $same(false, WebDecoy_Robots_Txt::home_is_subdirectory('https://example.com/')); + $same(false, WebDecoy_Robots_Txt::home_is_subdirectory('https://example.com')); + $same(true, WebDecoy_Robots_Txt::home_is_subdirectory('https://example.com/blog/')); +}); + +echo "\nWP-CLI: robots_disallow_categories\n"; + +$t('a comma list of known categories sets them; none clears', function () use ($same) { + $same( + ['robots_disallow_categories', ['training_crawler', 'seo_crawler']], + WebDecoy_CLI_Command::parse_config_value('robots_disallow_categories', 'seo_crawler, training_crawler') + ); + $same(['robots_disallow_categories', []], WebDecoy_CLI_Command::parse_config_value('robots_disallow_categories', 'none')); +}); + +$t('an unknown category refuses the whole value', function () use ($same) { + $same(null, WebDecoy_CLI_Command::parse_config_value('robots_disallow_categories', 'training_crawler,trainng_crawler')); +}); + +$t('the stored list displays as a comma list, or none', function () use ($same) { + $same('training_crawler,seo_crawler', WebDecoy_CLI_Command::display_value('robots_disallow_categories', ['robots_disallow_categories' => ['training_crawler', 'seo_crawler']])); + $same('none', WebDecoy_CLI_Command::display_value('robots_disallow_categories', ['robots_disallow_categories' => []])); +}); diff --git a/webdecoy.php b/webdecoy.php index 5f4798e..94512d8 100644 --- a/webdecoy.php +++ b/webdecoy.php @@ -237,6 +237,11 @@ private function load_options(): void 'block_ai_crawlers' => false, 'custom_allowlist' => [], + // Automatic robots.txt: bot categories asked not to crawl, as a + // list of registry category ids. Empty (off) by default, so an + // upgrade never changes a site's robots.txt on its own. + 'robots_disallow_categories' => [], + // Blocking Settings // Monitor mode is the safe default: everything is detected, logged and // reported, and nothing is ever blocked, throttled or 403'd. Turn it off @@ -1070,6 +1075,7 @@ public function load_includes(): void require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-detection-sender.php'; require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-ai-referrals.php'; require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-crawler-visits.php'; + require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-robots-txt.php'; require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-request-counts.php'; require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-honeytoken.php'; require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-ip-enrichment.php'; @@ -1113,6 +1119,10 @@ public function load_includes(): void // One wordpress.org review ask, after a week with bots counted. Admin only. (new WebDecoy_Review_Ask())->register(); + // Automatic robots.txt: appends the chosen categories' agents to the + // virtual robots.txt. Adds nothing until the owner picks a category. + WebDecoy_Robots_Txt::register($this->options['robots_disallow_categories'] ?? []); + if (class_exists('WooCommerce')) { require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-woocommerce.php'; } @@ -2489,6 +2499,9 @@ public function sanitize_options(array $input): array $sanitized['block_ai_crawlers'] = !empty($input['block_ai_crawlers']); $sanitized['custom_allowlist'] = array_filter(array_map('sanitize_text_field', explode("\n", $input['custom_allowlist'] ?? ''))); + // Automatic robots.txt: only categories the bundled registry offers. + $sanitized['robots_disallow_categories'] = WebDecoy_Robots_Txt::sanitize_categories($input['robots_disallow_categories'] ?? []); + // Blocking Settings // Allowlist: validate as IPs/CIDRs (reuses the trusted-proxy validator), // stored as an array of valid entries.