diff --git a/admin/partials/settings-page.php b/admin/partials/settings-page.php
index 10f588d..6b7d7d8 100644
--- a/admin/partials/settings-page.php
+++ b/admin/partials/settings-page.php
@@ -520,6 +520,83 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ' . esc_html($wd_robots_file) . ''
+ );
+ ?>
+
+
+
+
+
+
+
diff --git a/includes/class-webdecoy-activator.php b/includes/class-webdecoy-activator.php
index 3070653..3007fbd 100644
--- a/includes/class-webdecoy-activator.php
+++ b/includes/class-webdecoy-activator.php
@@ -203,6 +203,7 @@ private static function set_default_options(): void
'allow_social_bots' => true,
'block_ai_crawlers' => false,
'custom_allowlist' => [],
+ 'robots_disallow_categories' => [],
// Blocking Settings
// Monitor mode on by default: detect and log everything, block nothing,
diff --git a/includes/class-webdecoy-cli.php b/includes/class-webdecoy-cli.php
index 194d170..a58a9d7 100644
--- a/includes/class-webdecoy-cli.php
+++ b/includes/class-webdecoy-cli.php
@@ -49,6 +49,7 @@ class WebDecoy_CLI_Command
'rate_limit_requests' => ['type' => 'int', 'min' => 1, 'max' => 100000],
'rate_limit_window' => ['type' => 'int', 'min' => 1, 'max' => 86400],
'behind_cloudflare' => ['type' => 'bool'],
+ 'robots_disallow_categories' => ['type' => 'categories'],
];
/**
@@ -128,6 +129,8 @@ public function status($args, $assoc_args): void
*
* wp webdecoy config set mode monitor
* wp webdecoy config set block_ai_crawlers true
+ * wp webdecoy config set robots_disallow_categories training_crawler,seo_crawler
+ * wp webdecoy config set robots_disallow_categories none
* wp webdecoy config get sensitivity
*
* @when after_wp_load
@@ -147,7 +150,7 @@ public function config($args, $assoc_args): void
$rows[] = [
'key' => $name,
'value' => self::display_value($name, $options),
- 'type' => $spec['type'] === 'enum' ? implode('|', $spec['values']) : $spec['type'],
+ 'type' => $spec['type'] === 'enum' ? implode('|', $spec['values'] ?? []) : $spec['type'],
];
}
\WP_CLI\Utils\format_items('table', $rows, ['key', 'value', 'type']);
@@ -329,6 +332,17 @@ public static function parse_config_value(string $key, string $raw): ?array
case 'enum':
$lowered = strtolower(trim($raw));
return in_array($lowered, $spec['values'], true) ? [$key, $lowered] : null;
+ case 'categories':
+ // Comma-separated category ids; `none` or an empty value clears.
+ $lowered = strtolower(trim($raw));
+ if ($lowered === '' || $lowered === 'none') {
+ return [$key, []];
+ }
+ $requested = array_values(array_filter(array_map('trim', explode(',', $lowered)), 'strlen'));
+ $selected = WebDecoy_Robots_Txt::sanitize_categories($requested);
+ // Refuse the whole value when any id is unknown, rather than
+ // silently dropping a typo.
+ return count($selected) === count(array_unique($requested)) ? [$key, $selected] : null;
default:
return null;
}
@@ -348,6 +362,9 @@ public static function display_value(string $key, array $options): string
if (is_bool($value)) {
return $value ? 'true' : 'false';
}
+ if (is_array($value) || (self::CONFIG_KEYS[$key]['type'] ?? '') === 'categories') {
+ return (array) $value === [] ? 'none' : implode(',', array_map('strval', (array) $value));
+ }
return (string) ($value ?? '');
}
@@ -366,6 +383,13 @@ public static function value_help(string $key, string $raw): string
return sprintf("'%s' is out of range for %s (%d-%d).", $raw, $key, $spec['min'] ?? 0, $spec['max'] ?? 0);
case 'enum':
return sprintf("'%s' is not valid for %s. Use: %s.", $raw, $key, implode(', ', $spec['values']));
+ case 'categories':
+ return sprintf(
+ "'%s' is not valid for %s. Use none, or a comma-separated list of: %s.",
+ $raw,
+ $key,
+ implode(', ', array_keys(WebDecoy_Robots_Txt::choices()))
+ );
default:
return "'{$raw}' is not valid for {$key}.";
}
diff --git a/includes/class-webdecoy-robots-txt.php b/includes/class-webdecoy-robots-txt.php
new file mode 100644
index 0000000..c0118f2
--- /dev/null
+++ b/includes/class-webdecoy-robots-txt.php
@@ -0,0 +1,284 @@
+>|null $agents Registry agents; null reads the bundled registry.
+ * @return array category id => agent count
+ */
+ public static function choices(?array $agents = null): array
+ {
+ if ($agents === null) {
+ $agents = \WebDecoy\AgentRegistry::all();
+ $order = array_values(array_diff(\WebDecoy\AgentRegistry::categories(), [self::UNCATEGORISED]));
+ } else {
+ $order = self::category_order($agents);
+ }
+
+ $choices = [];
+ foreach ($order as $category) {
+ $count = count(self::agent_names([$category], $agents));
+ if ($count > 0) {
+ $choices[$category] = $count;
+ }
+ }
+ return $choices;
+ }
+
+ /**
+ * Reduce submitted input to known, choosable category ids, in registry
+ * order, without duplicates. Accepts an array (the settings form) or a
+ * comma-separated string (WP-CLI). Anything else is an empty selection.
+ *
+ * @param mixed $input
+ * @param list>|null $agents
+ * @return list
+ */
+ public static function sanitize_categories($input, ?array $agents = null): array
+ {
+ if (is_string($input)) {
+ $input = explode(',', $input);
+ }
+ if (!is_array($input)) {
+ return [];
+ }
+ $wanted = [];
+ foreach ($input as $value) {
+ if (is_string($value)) {
+ $wanted[strtolower(trim($value))] = true;
+ }
+ }
+ $selected = [];
+ foreach (array_keys(self::choices($agents)) as $category) {
+ if (isset($wanted[$category])) {
+ $selected[] = $category;
+ }
+ }
+ return $selected;
+ }
+
+ /**
+ * Distinct robots.txt names for the agents in these categories, in
+ * registry order. Duplicates are dropped case-insensitively (robots.txt
+ * matching ignores case), keeping the first spelling. Agents without a
+ * known name, unknown categories and unsafe names contribute nothing.
+ *
+ * @param list $categories
+ * @param list>|null $agents
+ * @return list
+ */
+ public static function agent_names(array $categories, ?array $agents = null): array
+ {
+ $agents = $agents ?? \WebDecoy\AgentRegistry::all();
+ $wanted = array_flip(array_diff($categories, [self::UNCATEGORISED]));
+
+ $names = [];
+ $seen = [];
+ foreach ($agents as $agent) {
+ if (!isset($wanted[(string) ($agent['category'] ?? '')])) {
+ continue;
+ }
+ // The pattern also refuses an empty name (no known robots.txt token).
+ $name = trim((string) ($agent['robots_name'] ?? ''));
+ if (preg_match(self::SAFE_NAME, $name) !== 1) {
+ continue;
+ }
+ $key = strtolower($name);
+ if (isset($seen[$key])) {
+ continue;
+ }
+ $seen[$key] = true;
+ $names[] = $name;
+ }
+ return $names;
+ }
+
+ /**
+ * The robots.txt block for these categories, or '' when it would name no
+ * agent. Ends with a newline.
+ *
+ * @param list $categories
+ * @param list>|null $agents
+ */
+ public static function block(array $categories, string $version, ?array $agents = null): string
+ {
+ $names = self::agent_names($categories, $agents);
+ if ($names === []) {
+ return '';
+ }
+ $count = count($names);
+ $header = self::BEGIN_MARKER . ' (' . $count . ($count === 1 ? ' agent' : ' agents');
+ if ($version !== '') {
+ $header .= ', updated with plugin version ' . $version;
+ }
+ $header .= ')';
+
+ $lines = [$header];
+ foreach ($names as $name) {
+ $lines[] = 'User-agent: ' . $name;
+ }
+ $lines[] = 'Disallow: /';
+ $lines[] = self::END_MARKER;
+ return implode("\n", $lines) . "\n";
+ }
+
+ /**
+ * Append the block to robots.txt output from WordPress and other plugins.
+ *
+ * The existing output is kept as it is. Nothing is added when the block
+ * would be empty or is already there.
+ *
+ * @param list $categories
+ * @param list>|null $agents
+ */
+ public static function append(string $output, array $categories, string $version, ?array $agents = null): string
+ {
+ if (strpos($output, self::BEGIN_MARKER) !== false) {
+ return $output;
+ }
+ $block = self::block($categories, $version, $agents);
+ if ($block === '') {
+ return $output;
+ }
+ if ($output === '') {
+ return $block;
+ }
+ $separator = substr($output, -1) === "\n" ? "\n" : "\n\n";
+ return $output . $separator . $block;
+ }
+
+ /**
+ * A physical robots.txt that the web server serves instead of WordPress's
+ * virtual one, or null when there is none. Checks the WordPress directory
+ * and, when WordPress lives in a subdirectory of the site, the site root.
+ *
+ * @param list $roots Directories to check; defaults to the WordPress ones.
+ */
+ public static function physical_file(array $roots = []): ?string
+ {
+ if ($roots === []) {
+ $roots[] = ABSPATH;
+ if (function_exists('get_home_path')) {
+ $roots[] = get_home_path();
+ }
+ }
+ foreach (array_unique($roots) as $root) {
+ $path = rtrim($root, '/\\') . '/robots.txt';
+ if (is_file($path)) {
+ return $path;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * True when the site's home address sits below a path (example.com/blog).
+ * robots.txt is only read from the root of a host, and WordPress serves
+ * its virtual one only when the site lives at that root.
+ */
+ public static function home_is_subdirectory(string $home_url): bool
+ {
+ $path = (string) wp_parse_url($home_url, PHP_URL_PATH);
+ return trim($path, '/') !== '';
+ }
+
+ /**
+ * Categories in the order the given agents first use them, without the
+ * uncategorised bucket. For agent lists other than the bundled registry,
+ * whose own category list is used otherwise.
+ *
+ * @param list> $agents
+ * @return list
+ */
+ private static function category_order(array $agents): array
+ {
+ $order = [];
+ foreach ($agents as $agent) {
+ $category = (string) ($agent['category'] ?? '');
+ if ($category !== '' && $category !== self::UNCATEGORISED && !in_array($category, $order, true)) {
+ $order[] = $category;
+ }
+ }
+ return $order;
+ }
+}
diff --git a/tests/RobotsTxtTest.php b/tests/RobotsTxtTest.php
new file mode 100644
index 0000000..7acc6ce
--- /dev/null
+++ b/tests/RobotsTxtTest.php
@@ -0,0 +1,219 @@
+ $cb, 'priority' => $priority, 'args' => $args];
+ return true;
+ }
+}
+require_once dirname(__DIR__) . '/sdk/src/AgentRegistry.php';
+require_once dirname(__DIR__) . '/includes/class-webdecoy-robots-txt.php';
+require_once dirname(__DIR__) . '/includes/class-webdecoy-runtime-config.php';
+require_once dirname(__DIR__) . '/includes/class-webdecoy-cli.php';
+
+$t = ['TestRunner', 'test'];
+$same = ['TestRunner', 'assertSame'];
+$true = ['TestRunner', 'assertTrue'];
+
+/** A small registry in the generated file's shape. */
+function robots_fixture(): array
+{
+ $agent = function (string $id, string $category, string $robots) {
+ return ['id' => $id, 'name' => $id, 'category' => $category, 'robots_name' => $robots, 'patterns' => [strtolower($id)]];
+ };
+ return [
+ $agent('gptbot', 'training_crawler', 'GPTBot'),
+ $agent('claudebot', 'training_crawler', 'ClaudeBot'),
+ $agent('nameless', 'training_crawler', ''),
+ $agent('gptbot-alt', 'training_crawler', 'gptbot'),
+ $agent('ahrefs', 'seo_crawler', 'AhrefsBot'),
+ $agent('perplexity', 'ai_search_crawler', 'PerplexityBot'),
+ $agent('dup-across', 'ai_search_crawler', 'ClaudeBot'),
+ $agent('injected', 'seo_crawler', "Evil\nDisallow: /"),
+ $agent('headless', 'headless_browser', ''),
+ $agent('other', 'none', 'OtherBot'),
+ ];
+}
+
+const ROBOTS_WP_DEFAULT = "User-agent: *\nDisallow: /wp-admin/\nAllow: /wp-admin/admin-ajax.php\n\nSitemap: https://example.com/wp-sitemap.xml\n";
+
+echo "\nAutomatic robots.txt: generation\n";
+
+$t('names each robots token once, in registry order, first spelling kept', function () use ($same) {
+ $same(['GPTBot', 'ClaudeBot'], WebDecoy_Robots_Txt::agent_names(['training_crawler'], robots_fixture()));
+});
+
+$t('dedupes across categories too, and order follows the registry not the selection', function () use ($same) {
+ $same(
+ ['GPTBot', 'ClaudeBot', 'AhrefsBot', 'PerplexityBot'],
+ WebDecoy_Robots_Txt::agent_names(['seo_crawler', 'ai_search_crawler', 'training_crawler'], robots_fixture())
+ );
+});
+
+$t('agents without a robots token, unknown categories and the none bucket add nothing', function () use ($same) {
+ $same([], WebDecoy_Robots_Txt::agent_names(['headless_browser', 'made_up', 'none'], robots_fixture()));
+});
+
+$t('a token that could break out of its line is never written', function () use ($same) {
+ $same(['AhrefsBot'], WebDecoy_Robots_Txt::agent_names(['seo_crawler'], robots_fixture()));
+});
+
+$t('the block is one group ending in Disallow: /, between markers', function () use ($same) {
+ $same(
+ "# WebDecoy: AI and bot rules (2 agents, updated with plugin version 9.9.9)\n"
+ . "User-agent: GPTBot\nUser-agent: ClaudeBot\nDisallow: /\n# End of WebDecoy rules\n",
+ WebDecoy_Robots_Txt::block(['training_crawler'], '9.9.9', robots_fixture())
+ );
+});
+
+$t('an empty selection, or one that names no agent, is no block at all', function () use ($same) {
+ $same('', WebDecoy_Robots_Txt::block([], '9.9.9', robots_fixture()));
+ $same('', WebDecoy_Robots_Txt::block(['headless_browser'], '9.9.9', robots_fixture()));
+});
+
+echo "\nAutomatic robots.txt: choices and sanitizing\n";
+
+$t('choices are categories with at least one token, counted after dedupe, none excluded', function () use ($same) {
+ $same(
+ ['training_crawler' => 2, 'seo_crawler' => 1, 'ai_search_crawler' => 2],
+ WebDecoy_Robots_Txt::choices(robots_fixture())
+ );
+});
+
+$t('sanitizing keeps known choices only, in choice order, once each', function () use ($same) {
+ $same(
+ ['training_crawler', 'seo_crawler'],
+ WebDecoy_Robots_Txt::sanitize_categories(['seo_crawler', 'none', 'nope', 'Training_Crawler ', 'seo_crawler', 'headless_browser', 7], robots_fixture())
+ );
+});
+
+$t('a comma-separated string is accepted; anything else is an empty selection', function () use ($same) {
+ $same(['training_crawler', 'ai_search_crawler'], WebDecoy_Robots_Txt::sanitize_categories('ai_search_crawler, training_crawler', robots_fixture()));
+ $same([], WebDecoy_Robots_Txt::sanitize_categories(null, robots_fixture()));
+ $same([], WebDecoy_Robots_Txt::sanitize_categories(true, robots_fixture()));
+});
+
+$t('the bundled registry offers AI training crawlers and search engines, never none', function () use ($true) {
+ $choices = WebDecoy_Robots_Txt::choices();
+ $true(($choices['training_crawler'] ?? 0) > 0, 'training_crawler must be offered');
+ $true(isset($choices['search_crawler']), 'search engines are selectable');
+ $true(!isset($choices['none']), 'none is never offered');
+ $true(!isset($choices['headless_browser']), 'a category with no robots token has nothing to offer');
+});
+
+$t('with the bundled registry, AI training crawlers include GPTBot and ClaudeBot', function () use ($true) {
+ $names = WebDecoy_Robots_Txt::agent_names(['training_crawler']);
+ $true(in_array('GPTBot', $names, true) && in_array('ClaudeBot', $names, true));
+ $true(!in_array('Googlebot', $names, true), 'search engines are not training crawlers');
+});
+
+echo "\nAutomatic robots.txt: joining WordPress's output\n";
+
+$t('the block is appended after the existing output, which is kept byte for byte', function () use ($same, $true) {
+ $out = WebDecoy_Robots_Txt::append(ROBOTS_WP_DEFAULT, ['training_crawler'], '9.9.9', robots_fixture());
+ $same(0, strpos($out, ROBOTS_WP_DEFAULT), 'existing rules and sitemap line come first, unchanged');
+ $same(
+ ROBOTS_WP_DEFAULT . "\n" . WebDecoy_Robots_Txt::block(['training_crawler'], '9.9.9', robots_fixture()),
+ $out
+ );
+});
+
+$t('output without a trailing newline still gets a blank line before the block', function () use ($true) {
+ $out = WebDecoy_Robots_Txt::append("User-agent: *\nDisallow:", ['training_crawler'], '9.9.9', robots_fixture());
+ $true(strpos($out, "Disallow:\n\n# WebDecoy: AI and bot rules") !== false, $out);
+});
+
+$t('nothing is added with no selection', function () use ($same) {
+ $same(ROBOTS_WP_DEFAULT, WebDecoy_Robots_Txt::append(ROBOTS_WP_DEFAULT, [], '9.9.9', robots_fixture()));
+});
+
+$t('running the filter twice adds the block once', function () use ($same) {
+ $once = WebDecoy_Robots_Txt::append(ROBOTS_WP_DEFAULT, ['training_crawler'], '9.9.9', robots_fixture());
+ $same($once, WebDecoy_Robots_Txt::append($once, ['training_crawler'], '9.9.9', robots_fixture()));
+});
+
+$t('register() hooks robots_txt late only when a category is chosen, and the hook appends', function () use ($same, $true) {
+ $GLOBALS['__wd_filters'] = [];
+ WebDecoy_Robots_Txt::register([]);
+ WebDecoy_Robots_Txt::register(['not_a_category']);
+ WebDecoy_Robots_Txt::register(null);
+ $same([], $GLOBALS['__wd_filters'], 'no selection must register nothing');
+
+ WebDecoy_Robots_Txt::register(['training_crawler']);
+ $hooks = $GLOBALS['__wd_filters']['robots_txt'] ?? [];
+ $same(1, count($hooks));
+ $true($hooks[0]['priority'] > 10, 'runs after sitemaps and other plugins');
+ $same(1, $hooks[0]['args'], 'only the output is taken: the public flag must not decide this');
+
+ $served = ($hooks[0]['cb'])(ROBOTS_WP_DEFAULT);
+ $same(0, strpos($served, ROBOTS_WP_DEFAULT));
+ $true(strpos($served, "User-agent: GPTBot\n") !== false, $served);
+ $GLOBALS['__wd_filters'] = [];
+});
+
+$t('a site that discourages search engines still gets the chosen rules', function () use ($true) {
+ $GLOBALS['__wd_filters'] = [];
+ WebDecoy_Robots_Txt::register(['training_crawler']);
+ $cb = $GLOBALS['__wd_filters']['robots_txt'][0]['cb'];
+ // As if a caller passed WordPress's flag anyway, in every shape it has had.
+ foreach ([false, '0', 0] as $public) {
+ $true(strpos($cb(ROBOTS_WP_DEFAULT, $public), "User-agent: GPTBot\n") !== false, var_export($public, true));
+ }
+ $GLOBALS['__wd_filters'] = [];
+});
+
+echo "\nAutomatic robots.txt: where it can apply\n";
+
+$t('a physical robots.txt is found, and its absence is null', function () use ($same) {
+ $dir = sys_get_temp_dir() . '/wd-robots-' . bin2hex(random_bytes(4));
+ mkdir($dir);
+ $same(null, WebDecoy_Robots_Txt::physical_file([$dir]));
+ file_put_contents($dir . '/robots.txt', "User-agent: *\n");
+ $same($dir . '/robots.txt', WebDecoy_Robots_Txt::physical_file([$dir . '/']));
+ unlink($dir . '/robots.txt');
+ rmdir($dir);
+});
+
+$t('a home address below a path is a subdirectory install', function () use ($same) {
+ $same(false, WebDecoy_Robots_Txt::home_is_subdirectory('https://example.com/'));
+ $same(false, WebDecoy_Robots_Txt::home_is_subdirectory('https://example.com'));
+ $same(true, WebDecoy_Robots_Txt::home_is_subdirectory('https://example.com/blog/'));
+});
+
+echo "\nWP-CLI: robots_disallow_categories\n";
+
+$t('a comma list of known categories sets them; none clears', function () use ($same) {
+ $same(
+ ['robots_disallow_categories', ['training_crawler', 'seo_crawler']],
+ WebDecoy_CLI_Command::parse_config_value('robots_disallow_categories', 'seo_crawler, training_crawler')
+ );
+ $same(['robots_disallow_categories', []], WebDecoy_CLI_Command::parse_config_value('robots_disallow_categories', 'none'));
+});
+
+$t('an unknown category refuses the whole value', function () use ($same) {
+ $same(null, WebDecoy_CLI_Command::parse_config_value('robots_disallow_categories', 'training_crawler,trainng_crawler'));
+});
+
+$t('the stored list displays as a comma list, or none', function () use ($same) {
+ $same('training_crawler,seo_crawler', WebDecoy_CLI_Command::display_value('robots_disallow_categories', ['robots_disallow_categories' => ['training_crawler', 'seo_crawler']]));
+ $same('none', WebDecoy_CLI_Command::display_value('robots_disallow_categories', ['robots_disallow_categories' => []]));
+});
diff --git a/webdecoy.php b/webdecoy.php
index 5f4798e..94512d8 100644
--- a/webdecoy.php
+++ b/webdecoy.php
@@ -237,6 +237,11 @@ private function load_options(): void
'block_ai_crawlers' => false,
'custom_allowlist' => [],
+ // Automatic robots.txt: bot categories asked not to crawl, as a
+ // list of registry category ids. Empty (off) by default, so an
+ // upgrade never changes a site's robots.txt on its own.
+ 'robots_disallow_categories' => [],
+
// Blocking Settings
// Monitor mode is the safe default: everything is detected, logged and
// reported, and nothing is ever blocked, throttled or 403'd. Turn it off
@@ -1070,6 +1075,7 @@ public function load_includes(): void
require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-detection-sender.php';
require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-ai-referrals.php';
require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-crawler-visits.php';
+ require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-robots-txt.php';
require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-request-counts.php';
require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-honeytoken.php';
require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-ip-enrichment.php';
@@ -1113,6 +1119,10 @@ public function load_includes(): void
// One wordpress.org review ask, after a week with bots counted. Admin only.
(new WebDecoy_Review_Ask())->register();
+ // Automatic robots.txt: appends the chosen categories' agents to the
+ // virtual robots.txt. Adds nothing until the owner picks a category.
+ WebDecoy_Robots_Txt::register($this->options['robots_disallow_categories'] ?? []);
+
if (class_exists('WooCommerce')) {
require_once WEBDECOY_PLUGIN_DIR . 'includes/class-webdecoy-woocommerce.php';
}
@@ -2489,6 +2499,9 @@ public function sanitize_options(array $input): array
$sanitized['block_ai_crawlers'] = !empty($input['block_ai_crawlers']);
$sanitized['custom_allowlist'] = array_filter(array_map('sanitize_text_field', explode("\n", $input['custom_allowlist'] ?? '')));
+ // Automatic robots.txt: only categories the bundled registry offers.
+ $sanitized['robots_disallow_categories'] = WebDecoy_Robots_Txt::sanitize_categories($input['robots_disallow_categories'] ?? []);
+
// Blocking Settings
// Allowlist: validate as IPs/CIDRs (reuses the trusted-proxy validator),
// stored as an array of valid entries.