From ce266f7ffdf3662e1f2b1f42f1ff8eaafb78ce61 Mon Sep 17 00:00:00 2001 From: Brian Greunke Date: Fri, 18 Sep 2026 09:34:11 -0500 Subject: [PATCH 1/3] fix: pin web security fallback tools (ENG-8602) --- capabilities/web-security/capability.yaml | 2 +- .../web-security/scripts/install_tools.sh | 54 ++++++++++++------- .../tests/test_install_tools_offline.py | 41 ++++++++++++-- 3 files changed, 74 insertions(+), 23 deletions(-) diff --git a/capabilities/web-security/capability.yaml b/capabilities/web-security/capability.yaml index f32d38b..db92478 100644 --- a/capabilities/web-security/capability.yaml +++ b/capabilities/web-security/capability.yaml @@ -1,6 +1,6 @@ schema: 1 name: web-security -version: "1.15.0" +version: "1.15.1" description: > Web application penetration testing with 83 attack technique playbooks covering HTTP desync/request smuggling, cache poisoning, SSRF, SSTI, DOM diff --git a/capabilities/web-security/scripts/install_tools.sh b/capabilities/web-security/scripts/install_tools.sh index 4a01d31..097b22c 100755 --- a/capabilities/web-security/scripts/install_tools.sh +++ b/capabilities/web-security/scripts/install_tools.sh @@ -58,13 +58,29 @@ py_install() { fi } -GO_TOOL_VERSIONS_pdtm="v0.1.5" GO_TOOL_VERSIONS_protoscope="v0.0.0-20221109213918-8e7a6aafa2c9" GO_TOOL_VERSIONS_interactsh="v1.3.1" GO_TOOL_VERSIONS_2fa="v1.2.0" GO_TOOL_VERSIONS_surf="v0.0.5" +GO_VERSION="1.26.6" +KATANA_VERSION="1.7.0" +KITERUNNER_VERSION="v1.0.2" + +have_pd_tool() { + if [ "$1" = "httpx" ]; then + command -v httpx >/dev/null 2>&1 && httpx -version >/dev/null 2>&1 \ + || [ -x "$HOME/.pdtm/go/bin/httpx" ] && "$HOME/.pdtm/go/bin/httpx" -version >/dev/null 2>&1 + else + have "$1" + fi +} -PD_TOOLS="nuclei httpx subfinder naabu dnsx uncover alterx tlsx asnmap" +install_pd_tool() { + local tool="$1" package="$2" version="$3" + have_pd_tool "$tool" && return + mkdir -p "$HOME/.pdtm/go/bin" + GOBIN="$HOME/.pdtm/go/bin" go install "${package}@${version}" +} # What is actually missing, before anything is fetched. missing_go_tools="" @@ -72,10 +88,9 @@ for tool in protoscope interactsh-client 2fa surf; do have "$tool" || missing_go_tools="$missing_go_tools $tool" done missing_pd_tools="" -for tool in $PD_TOOLS; do - have "$tool" || missing_pd_tools="$missing_pd_tools,$tool" +for tool in nuclei httpx subfinder naabu dnsx uncover alterx tlsx asnmap; do + have_pd_tool "$tool" || missing_pd_tools="$missing_pd_tools $tool" done -missing_pd_tools="${missing_pd_tools#,}" # -- Go toolchain (only when something still has to be built) -------------- # Deliberately last in the decision order: the toolchain is a ~150 MB download @@ -83,9 +98,8 @@ missing_pd_tools="${missing_pd_tools#,}" # never needed, so it is never requested. need_go=false [ -n "$missing_go_tools" ] && need_go=true -[ -n "$missing_pd_tools" ] && ! have pdtm && need_go=true +[ -n "$missing_pd_tools" ] && need_go=true if [ "$need_go" = true ] && ! command -v go &>/dev/null; then - GO_VERSION="1.24.3" case "$ARCH" in aarch64|arm64) GOARCH="arm64" ;; *) GOARCH="amd64" ;; @@ -94,18 +108,21 @@ if [ "$need_go" = true ] && ! command -v go &>/dev/null; then export PATH="/usr/local/go/bin:$PATH" fi -# -- PDTM + ProjectDiscovery tools ---------------------------------------- +# -- ProjectDiscovery tools ------------------------------------------------ if [ -n "$missing_pd_tools" ]; then - if ! have pdtm; then - go install "github.com/projectdiscovery/pdtm/cmd/pdtm@${GO_TOOL_VERSIONS_pdtm}" - fi - PDTM_BIN="$(command -v pdtm || echo "$(go env GOPATH)/bin/pdtm")" - "$PDTM_BIN" -install "$missing_pd_tools" + install_pd_tool nuclei github.com/projectdiscovery/nuclei/v3/cmd/nuclei v3.11.1 + install_pd_tool httpx github.com/projectdiscovery/httpx/cmd/httpx v1.12.0 + install_pd_tool subfinder github.com/projectdiscovery/subfinder/v2/cmd/subfinder v2.16.0 + install_pd_tool naabu github.com/projectdiscovery/naabu/v2/cmd/naabu v2.6.1 + install_pd_tool dnsx github.com/projectdiscovery/dnsx/cmd/dnsx v1.3.1 + install_pd_tool uncover github.com/projectdiscovery/uncover/cmd/uncover v1.2.1 + install_pd_tool alterx github.com/projectdiscovery/alterx/cmd/alterx v0.1.0 + install_pd_tool tlsx github.com/projectdiscovery/tlsx/cmd/tlsx v1.4.0 + install_pd_tool asnmap github.com/projectdiscovery/asnmap/cmd/asnmap v1.1.1 fi # -- katana (pre-built binary, go-tree-sitter build issue) ----------------- if ! have katana; then - KATANA_VERSION="1.5.0" DEB_ARCH="$(dpkg --print-architecture 2>/dev/null || echo amd64)" mkdir -p "$HOME/.pdtm/go/bin" curl -fsSL "https://github.com/projectdiscovery/katana/releases/download/v${KATANA_VERSION}/katana_${KATANA_VERSION}_linux_${DEB_ARCH}.zip" \ @@ -132,7 +149,7 @@ have surf || go install "github.com/assetnote/surf/cmd/surf@${GO_TOOL_VERSIONS_s # -- kiterunner (API content discovery) ------------------------------------ if ! have kr; then - if git clone --depth 1 https://github.com/assetnote/kiterunner /tmp/kiterunner; then + if git clone --depth 1 --branch "$KITERUNNER_VERSION" https://github.com/assetnote/kiterunner /tmp/kiterunner; then ( cd /tmp/kiterunner && make build ) \ && as_root mv /tmp/kiterunner/dist/kr /usr/local/bin/kr rm -rf /tmp/kiterunner @@ -239,8 +256,9 @@ if ! command -v exiftool &>/dev/null; then fi # -- Node.js + agent-browser ----------------------------------------------- -if ! command -v node &>/dev/null; then - curl -fsSL https://deb.nodesource.com/setup_22.x | as_root bash - \ +NODE_MAJOR="$(node -p 'process.versions.node.split(".")[0]' 2>/dev/null || echo 0)" +if [ "$NODE_MAJOR" -lt 24 ]; then + curl -fsSL https://deb.nodesource.com/setup_24.x | as_root bash - \ && as_root apt-get install -y --no-install-recommends nodejs \ || echo "WARN: Node.js install failed, skipping" fi @@ -256,7 +274,7 @@ fi # non-fatal because a disconnected deployment that cannot fetch a browser # should still get the rest of this capability's tooling. AGENT_BROWSER_CACHE="${AGENT_BROWSER_CACHE_DIR:-$HOME/.cache/agent-browser}" -if [ ! -d "$AGENT_BROWSER_CACHE" ]; then +if [ "${DREADNODE_CAPABILITY_INSTALL:-}" != "sealed" ] && [ ! -d "$AGENT_BROWSER_CACHE" ]; then agent-browser install || echo "WARN: agent-browser browser download failed, skipping" fi diff --git a/capabilities/web-security/tests/test_install_tools_offline.py b/capabilities/web-security/tests/test_install_tools_offline.py index 64a856e..30d2cd0 100644 --- a/capabilities/web-security/tests/test_install_tools_offline.py +++ b/capabilities/web-security/tests/test_install_tools_offline.py @@ -43,6 +43,32 @@ def test_no_unpinned_go_installs(self) -> None: ] assert not unpinned, f"unpinned installs: {unpinned}" + def test_projectdiscovery_tools_use_explicit_versions(self) -> None: + pins = { + "nuclei": "v3.11.1", + "httpx": "v1.12.0", + "subfinder": "v2.16.0", + "naabu": "v2.6.1", + "dnsx": "v1.3.1", + "uncover": "v1.2.1", + "alterx": "v0.1.0", + "tlsx": "v1.4.0", + "asnmap": "v1.1.1", + } + for tool, version in pins.items(): + assert re.search( + rf"install_pd_tool {tool} \S+ {re.escape(version)}$", + INSTALL_SCRIPT, + re.MULTILINE, + ), f"missing {tool} pin {version}" + + assert "pdtm -install" not in INSTALL_SCRIPT + + def test_toolchain_and_kiterunner_versions_are_pinned(self) -> None: + assert 'GO_VERSION="1.26.6"' in INSTALL_SCRIPT + assert 'KITERUNNER_VERSION="v1.0.2"' in INSTALL_SCRIPT + assert 'git clone --depth 1 --branch "$KITERUNNER_VERSION"' in INSTALL_SCRIPT + class TestFetchesAreGuarded: def test_every_go_install_is_guarded(self) -> None: @@ -94,11 +120,12 @@ def test_py_install_calls_are_guarded(self) -> None: unguarded.append(stripped) assert not unguarded, f"unguarded py_install: {unguarded}" - def test_pdtm_only_installs_missing_tools(self) -> None: - # `pdtm -install ` re-fetches every tool in the list. The - # set has to be narrowed to what is actually absent first. + def test_only_missing_projectdiscovery_tools_are_installed(self) -> None: assert "$missing_pd_tools" in INSTALL_SCRIPT - assert "-install nuclei,httpx" not in INSTALL_SCRIPT + assert 'have_pd_tool "$tool" || missing_pd_tools=' in INSTALL_SCRIPT + + def test_httpx_guard_rejects_the_python_cli(self) -> None: + assert "httpx -version >/dev/null 2>&1" in INSTALL_SCRIPT def test_katana_download_is_guarded(self) -> None: idx = next( @@ -162,6 +189,12 @@ def test_go_cache_cleanup_only_runs_when_go_was_used(self) -> None: idx = next(i for i, line in enumerate(LINES) if "go clean -cache" in line) assert "need_go" in _preceding_context(idx, span=3) + def test_node_24_floor_and_sealed_browser_guard(self) -> None: + assert "setup_24.x" in INSTALL_SCRIPT + assert "setup_22.x" not in INSTALL_SCRIPT + assert "${DREADNODE_CAPABILITY_INSTALL:-}" in INSTALL_SCRIPT + assert '!= "sealed"' in INSTALL_SCRIPT + class TestRootEscalation: """Writes to root-owned paths (/usr/local/bin, /opt) must use as_root.""" From b9380ae809e7e38e3399abba96bf664d5aa0df71 Mon Sep 17 00:00:00 2001 From: Brian Greunke Date: Fri, 25 Sep 2026 07:57:48 -0500 Subject: [PATCH 2/3] style: format web-security installer tests (ENG-8602) Applies the repository ruff-format hook to lines merged from main so the changed-file pre-commit hooks pass. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/test_install_tools_offline.py | 60 +++++++++++++++---- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/capabilities/web-security/tests/test_install_tools_offline.py b/capabilities/web-security/tests/test_install_tools_offline.py index 219ce97..23f7e92 100644 --- a/capabilities/web-security/tests/test_install_tools_offline.py +++ b/capabilities/web-security/tests/test_install_tools_offline.py @@ -36,7 +36,11 @@ def test_no_unpinned_go_installs(self) -> None: # run, so it reaches the network even when the binary is already # present — and produces a different tool set on different days, which # no SBOM can describe. - unpinned = [line.strip() for line in LINES if "@latest" in line and not line.strip().startswith("#")] + unpinned = [ + line.strip() + for line in LINES + if "@latest" in line and not line.strip().startswith("#") + ] assert not unpinned, f"unpinned installs: {unpinned}" def test_projectdiscovery_tools_use_explicit_versions(self) -> None: @@ -79,7 +83,9 @@ def test_every_go_install_is_guarded(self) -> None: def test_global_npm_install_is_guarded(self) -> None: for i, line in enumerate(LINES): if re.search(r"^\s*(as_root\s+)?npm install -g", line): - assert "have " in _preceding_context(i), f"unguarded global npm install at line {i + 1}: {line.strip()}" + assert "have " in _preceding_context( + i + ), f"unguarded global npm install at line {i + 1}: {line.strip()}" def test_npm_installs_are_version_pinned(self) -> None: # Same SBOM argument as the go pins: an unpinned `npm install -g` @@ -104,7 +110,11 @@ def test_py_install_calls_are_guarded(self) -> None: continue # Skip the py_install function definition and requirement file # installs (guarded by their parent clone check). - if stripped.startswith(("if", "elif", "def", "#")) or "-r " in stripped or "py_install()" in stripped: + if ( + stripped.startswith(("if", "elif", "def", "#")) + or "-r " in stripped + or "py_install()" in stripped + ): continue if "have " not in _preceding_context(i): unguarded.append(stripped) @@ -118,15 +128,21 @@ def test_httpx_guard_rejects_the_python_cli(self) -> None: assert "httpx -version >/dev/null 2>&1" in INSTALL_SCRIPT def test_katana_download_is_guarded(self) -> None: - idx = next(i for i, line in enumerate(LINES) if "katana_${KATANA_VERSION}" in line) + idx = next( + i for i, line in enumerate(LINES) if "katana_${KATANA_VERSION}" in line + ) assert "have katana" in _preceding_context(idx, span=8) def test_caido_cli_download_is_guarded(self) -> None: - idx = next(i for i, line in enumerate(LINES) if "caido.download/releases" in line) + idx = next( + i for i, line in enumerate(LINES) if "caido.download/releases" in line + ) assert "command -v caido-cli" in _preceding_context(idx, span=10) def test_caido_mcp_server_download_is_guarded(self) -> None: - idx = next(i for i, line in enumerate(LINES) if "caido-mcp-server-linux" in line) + idx = next( + i for i, line in enumerate(LINES) if "caido-mcp-server-linux" in line + ) assert "command -v caido-mcp-server" in _preceding_context(idx, span=15) def test_kiterunner_build_is_guarded(self) -> None: @@ -137,9 +153,13 @@ def test_wrangler_install_is_guarded_and_pinned(self) -> None: # wrangler is fetched from npm, so the guard-and-pin discipline applies # exactly as it does to the go installs: present binary -> no registry # request; absent binary -> the pinned version, not @latest. - idx = next(i for i, line in enumerate(LINES) if "wrangler@${WRANGLER_VERSION}" in line) + idx = next( + i for i, line in enumerate(LINES) if "wrangler@${WRANGLER_VERSION}" in line + ) assert "have wrangler" in _preceding_context(idx, span=6) - pin = next(i for i, line in enumerate(LINES) if line.startswith("WRANGLER_VERSION=")) + pin = next( + i for i, line in enumerate(LINES) if line.startswith("WRANGLER_VERSION=") + ) assert re.fullmatch( r"WRANGLER_VERSION=\"[0-9]+\.[0-9]+\.[0-9]+\"", LINES[pin].strip(), @@ -181,17 +201,25 @@ class TestRootEscalation: def test_caido_cli_tar_uses_as_root(self) -> None: idx = next( - i for i, line in enumerate(LINES) if "tar" in line and "caido-cli" in line and "/usr/local/bin" in line + i + for i, line in enumerate(LINES) + if "tar" in line and "caido-cli" in line and "/usr/local/bin" in line ) assert "as_root" in LINES[idx] def test_caido_mcp_server_install_uses_as_root(self) -> None: - idx = next(i for i, line in enumerate(LINES) if "install -m" in line and "caido-mcp-server" in line) + idx = next( + i + for i, line in enumerate(LINES) + if "install -m" in line and "caido-mcp-server" in line + ) assert "as_root" in LINES[idx] def test_kiterunner_mv_uses_as_root(self) -> None: idx = next( - i for i, line in enumerate(LINES) if "/usr/local/bin/kr" in line and ("mv " in line or "install " in line) + i + for i, line in enumerate(LINES) + if "/usr/local/bin/kr" in line and ("mv " in line or "install " in line) ) assert "as_root" in LINES[idx] @@ -200,11 +228,17 @@ def test_burp_suite_uses_as_root(self) -> None: assert "as_root" in LINES[idx] def test_exiftool_apt_uses_as_root(self) -> None: - idx = next(i for i, line in enumerate(LINES) if "apt-get" in line and "exiftool" in line) + idx = next( + i + for i, line in enumerate(LINES) + if "apt-get" in line and "exiftool" in line + ) assert "as_root" in LINES[idx] def test_nodejs_apt_uses_as_root(self) -> None: - idx = next(i for i, line in enumerate(LINES) if "apt-get" in line and "nodejs" in line) + idx = next( + i for i, line in enumerate(LINES) if "apt-get" in line and "nodejs" in line + ) assert "as_root" in LINES[idx] From 187259319969fbc8a8affb9bf740045ef22b98c4 Mon Sep 17 00:00:00 2001 From: Brian Greunke Date: Fri, 25 Sep 2026 08:03:57 -0500 Subject: [PATCH 3/3] fix: detect ProjectDiscovery httpx outside pdtm (ENG-8602) The httpx guard mixed && and || without grouping, so a working ProjectDiscovery httpx on PATH or in ~/go/bin was reported missing and reinstalled, which fails offline. Group each location's check, and replace the substring test with tests that run the guard. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../web-security/scripts/install_tools.sh | 7 ++- .../tests/test_install_tools_offline.py | 47 ++++++++++++++++++- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/capabilities/web-security/scripts/install_tools.sh b/capabilities/web-security/scripts/install_tools.sh index 9ef5213..119ce1b 100755 --- a/capabilities/web-security/scripts/install_tools.sh +++ b/capabilities/web-security/scripts/install_tools.sh @@ -67,8 +67,11 @@ KITERUNNER_VERSION="v1.0.2" have_pd_tool() { if [ "$1" = "httpx" ]; then - command -v httpx >/dev/null 2>&1 && httpx -version >/dev/null 2>&1 \ - || [ -x "$HOME/.pdtm/go/bin/httpx" ] && "$HOME/.pdtm/go/bin/httpx" -version >/dev/null 2>&1 + # Python's httpx package also installs an `httpx` CLI; only + # ProjectDiscovery's answers `-version`. + { command -v httpx >/dev/null 2>&1 && httpx -version >/dev/null 2>&1; } \ + || { [ -x "$HOME/.pdtm/go/bin/httpx" ] && "$HOME/.pdtm/go/bin/httpx" -version >/dev/null 2>&1; } \ + || { [ -x "$HOME/go/bin/httpx" ] && "$HOME/go/bin/httpx" -version >/dev/null 2>&1; } else have "$1" fi diff --git a/capabilities/web-security/tests/test_install_tools_offline.py b/capabilities/web-security/tests/test_install_tools_offline.py index 23f7e92..74ad28b 100644 --- a/capabilities/web-security/tests/test_install_tools_offline.py +++ b/capabilities/web-security/tests/test_install_tools_offline.py @@ -12,7 +12,9 @@ from __future__ import annotations +import os import re +import subprocess from pathlib import Path ROOT = Path(__file__).resolve().parents[1] @@ -30,6 +32,33 @@ def _surrounding_context(index: int, span: int = 4) -> str: return "\n".join(LINES[max(0, index - span) : min(len(LINES), index + span + 1)]) +def _shell_function(name: str) -> str: + match = re.search(rf"^{name}\(\) \{{\n.*?^\}}\n", INSTALL_SCRIPT, re.M | re.S) + assert match, f"{name}() not found in install_tools.sh" + return match.group(0) + + +def _stub(path: Path, exit_code: int) -> None: + """An executable whose `-version` succeeds only for ProjectDiscovery httpx.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"#!/bin/sh\nexit {exit_code}\n", encoding="utf-8") + path.chmod(0o755) + + +def _have_pd_httpx(tmp_path: Path) -> bool: + (tmp_path / "home").mkdir(exist_ok=True) + script = ( + _shell_function("have") + + _shell_function("have_pd_tool") + + "have_pd_tool httpx\n" + ) + env = { + "HOME": str(tmp_path / "home"), + "PATH": f"{tmp_path / 'bin'}:{os.environ['PATH']}", + } + return subprocess.run(["bash", "-c", script], env=env, check=False).returncode == 0 + + class TestVersionsArePinned: def test_no_unpinned_go_installs(self) -> None: # `go install ...@latest` re-resolves against the module proxy every @@ -124,8 +153,22 @@ def test_only_missing_projectdiscovery_tools_are_installed(self) -> None: assert "$missing_pd_tools" in INSTALL_SCRIPT assert 'have_pd_tool "$tool" || missing_pd_tools=' in INSTALL_SCRIPT - def test_httpx_guard_rejects_the_python_cli(self) -> None: - assert "httpx -version >/dev/null 2>&1" in INSTALL_SCRIPT + def test_httpx_guard_accepts_projectdiscovery_httpx_on_path( + self, tmp_path: Path + ) -> None: + _stub(tmp_path / "bin" / "httpx", exit_code=0) + assert _have_pd_httpx(tmp_path) + + def test_httpx_guard_rejects_the_python_cli(self, tmp_path: Path) -> None: + _stub(tmp_path / "bin" / "httpx", exit_code=2) + assert not _have_pd_httpx(tmp_path) + + def test_httpx_guard_accepts_pdtm_httpx_behind_the_python_cli( + self, tmp_path: Path + ) -> None: + _stub(tmp_path / "bin" / "httpx", exit_code=2) + _stub(tmp_path / "home" / ".pdtm" / "go" / "bin" / "httpx", exit_code=0) + assert _have_pd_httpx(tmp_path) def test_katana_download_is_guarded(self) -> None: idx = next(