Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 77 additions & 0 deletions admin/partials/settings-page.php
Original file line number Diff line number Diff line change
Expand Up @@ -520,6 +520,83 @@
</td>
</tr>
</table>

<?php
$wd_robots_choices = WebDecoy_Robots_Txt::choices();
$wd_robots_selected = WebDecoy_Robots_Txt::sanitize_categories($options['robots_disallow_categories'] ?? []);
$wd_robots_block = WebDecoy_Robots_Txt::block($wd_robots_selected, WEBDECOY_VERSION);
$wd_robots_file = WebDecoy_Robots_Txt::physical_file();
$wd_robots_subdir = WebDecoy_Robots_Txt::home_is_subdirectory(home_url('/'));
?>
<h3 id="webdecoy-robots-txt"><?php esc_html_e('Automatic robots.txt', 'webdecoy'); ?></h3>
<p class="description">
<?php esc_html_e('Ask whole categories of bots not to crawl this site. WebDecoy adds a rule to your robots.txt naming every known agent in the categories you pick. The list is kept current with each plugin update. No account needed.', 'webdecoy'); ?>
</p>
<p class="description">
<?php esc_html_e('robots.txt is a request: well-behaved crawlers follow it, and bots that ignore it are not stopped by it. To refuse them, use the AI Crawlers option above and the Blocking tab.', 'webdecoy'); ?>
</p>

<?php if ($wd_robots_file !== null) : ?>
<div class="notice notice-warning inline">
<p>
<?php
printf(
/* translators: %s: path to the robots.txt file on the server */
esc_html__('A robots.txt file exists on the server (%s). Your web server sends that file instead of the one WordPress builds, so these rules cannot be added automatically. To use them, delete or rename that file, or copy the rules from the preview below into it. WebDecoy never changes that file.', 'webdecoy'),
'<code>' . esc_html($wd_robots_file) . '</code>'
);
?>
</p>
</div>
<?php elseif ($wd_robots_subdir) : ?>
<div class="notice notice-warning inline">
<p><?php esc_html_e('This site lives in a subfolder of its domain. Crawlers only read robots.txt at the root of the domain, which WordPress does not serve here, so copy the rules from the preview below into the robots.txt at your domain root.', 'webdecoy'); ?></p>
</div>
<?php endif; ?>

<table class="form-table">
<tr>
<th scope="row"><?php esc_html_e('Ask these bots not to crawl', 'webdecoy'); ?></th>
<td>
<fieldset>
<legend class="screen-reader-text"><?php esc_html_e('Bot categories to disallow in robots.txt', 'webdecoy'); ?></legend>
<?php foreach ($wd_robots_choices as $wd_category => $wd_count) : ?>
<label style="display:block;margin-bottom:6px;">
<input type="checkbox" name="webdecoy_options[robots_disallow_categories][]"
value="<?php echo esc_attr($wd_category); ?>"
<?php checked(in_array($wd_category, $wd_robots_selected, true)); ?> />
<?php
printf(
/* translators: 1: bot category name, 2: number of agents in it */
esc_html(_n('%1$s (%2$d agent)', '%1$s (%2$d agents)', $wd_count, 'webdecoy')),
esc_html(WebDecoy_Crawler_Visits::category_label($wd_category)),
(int) $wd_count
);
?>
<?php if ($wd_category === 'search_crawler') : ?>
<br /><span class="description" style="margin-left:24px;"><?php esc_html_e('Warning: this asks Google, Bing and other search engines to stop crawling, which removes the site from search results.', 'webdecoy'); ?></span>
<?php endif; ?>
</label>
<?php endforeach; ?>
</fieldset>
<p class="description"><?php esc_html_e('Nothing is selected by default. Your existing robots.txt rules, including sitemap lines and other plugins\' rules, are kept. Bots in the custom allowlist above are still named here if their category is selected.', 'webdecoy'); ?></p>
</td>
</tr>
<tr>
<th scope="row"><?php esc_html_e('Rules added', 'webdecoy'); ?></th>
<td>
<?php if ($wd_robots_block === '') : ?>
<p class="description"><?php esc_html_e('None. Pick a category and save to add rules.', 'webdecoy'); ?></p>
<?php else : ?>
<pre class="code" style="max-height:240px;overflow:auto;background:#f6f7f7;padding:8px;"><?php echo esc_html($wd_robots_block); ?></pre>
<p class="description"><?php esc_html_e('Shows the saved selection. Save changes to update it.', 'webdecoy'); ?></p>
<?php endif; ?>
<p>
<a href="<?php echo esc_url(home_url('/robots.txt')); ?>" target="_blank" rel="noopener noreferrer"><?php esc_html_e('View your live robots.txt', 'webdecoy'); ?></a>
</p>
</td>
</tr>
</table>
</div>

<!-- Blocking Tab -->
Expand Down
1 change: 1 addition & 0 deletions includes/class-webdecoy-activator.php
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,7 @@ private static function set_default_options(): void
'allow_social_bots' => true,
'block_ai_crawlers' => false,
'custom_allowlist' => [],
'robots_disallow_categories' => [],

// Blocking Settings
// Monitor mode on by default: detect and log everything, block nothing,
Expand Down
26 changes: 25 additions & 1 deletion includes/class-webdecoy-cli.php
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ class WebDecoy_CLI_Command
'rate_limit_requests' => ['type' => 'int', 'min' => 1, 'max' => 100000],
'rate_limit_window' => ['type' => 'int', 'min' => 1, 'max' => 86400],
'behind_cloudflare' => ['type' => 'bool'],
'robots_disallow_categories' => ['type' => 'categories'],
];

/**
Expand Down Expand Up @@ -128,6 +129,8 @@ public function status($args, $assoc_args): void
*
* wp webdecoy config set mode monitor
* wp webdecoy config set block_ai_crawlers true
* wp webdecoy config set robots_disallow_categories training_crawler,seo_crawler
* wp webdecoy config set robots_disallow_categories none
* wp webdecoy config get sensitivity
*
* @when after_wp_load
Expand All @@ -147,7 +150,7 @@ public function config($args, $assoc_args): void
$rows[] = [
'key' => $name,
'value' => self::display_value($name, $options),
'type' => $spec['type'] === 'enum' ? implode('|', $spec['values']) : $spec['type'],
'type' => $spec['type'] === 'enum' ? implode('|', $spec['values'] ?? []) : $spec['type'],
];
}
\WP_CLI\Utils\format_items('table', $rows, ['key', 'value', 'type']);
Expand Down Expand Up @@ -329,6 +332,17 @@ public static function parse_config_value(string $key, string $raw): ?array
case 'enum':
$lowered = strtolower(trim($raw));
return in_array($lowered, $spec['values'], true) ? [$key, $lowered] : null;
case 'categories':
// Comma-separated category ids; `none` or an empty value clears.
$lowered = strtolower(trim($raw));
if ($lowered === '' || $lowered === 'none') {
return [$key, []];
}
$requested = array_values(array_filter(array_map('trim', explode(',', $lowered)), 'strlen'));
$selected = WebDecoy_Robots_Txt::sanitize_categories($requested);
// Refuse the whole value when any id is unknown, rather than
// silently dropping a typo.
return count($selected) === count(array_unique($requested)) ? [$key, $selected] : null;
default:
return null;
}
Expand All @@ -348,6 +362,9 @@ public static function display_value(string $key, array $options): string
if (is_bool($value)) {
return $value ? 'true' : 'false';
}
if (is_array($value) || (self::CONFIG_KEYS[$key]['type'] ?? '') === 'categories') {
return (array) $value === [] ? 'none' : implode(',', array_map('strval', (array) $value));
}
return (string) ($value ?? '');
}

Expand All @@ -366,6 +383,13 @@ public static function value_help(string $key, string $raw): string
return sprintf("'%s' is out of range for %s (%d-%d).", $raw, $key, $spec['min'] ?? 0, $spec['max'] ?? 0);
case 'enum':
return sprintf("'%s' is not valid for %s. Use: %s.", $raw, $key, implode(', ', $spec['values']));
case 'categories':
return sprintf(
"'%s' is not valid for %s. Use none, or a comma-separated list of: %s.",
$raw,
$key,
implode(', ', array_keys(WebDecoy_Robots_Txt::choices()))
);
default:
return "'{$raw}' is not valid for {$key}.";
}
Expand Down
Loading
Loading