fix: pin web security fallback tools (ENG-8602)

capabilities/web-security/capability.yaml

@@ -1,6 +1,6 @@
 schema: 1
 name: web-security
-version: "2.0.0"
+version: "2.0.1"
 description: >
   Web application penetration testing with 83 attack technique playbooks
   covering HTTP desync/request smuggling, cache poisoning, SSRF, SSTI, DOM

capabilities/web-security/scripts/install_tools.sh

@@ -58,33 +58,50 @@ py_install() {
   fi
 }
 
-GO_TOOL_VERSIONS_pdtm="v0.1.5"
 GO_TOOL_VERSIONS_protoscope="v0.0.0-20221109213918-8e7a6aafa2c9"
 GO_TOOL_VERSIONS_interactsh="v1.3.1"
 GO_TOOL_VERSIONS_2fa="v1.2.0"
+GO_VERSION="1.26.6"
+KATANA_VERSION="1.7.0"
+KITERUNNER_VERSION="v1.0.2"
 
-PD_TOOLS="nuclei httpx subfinder naabu dnsx uncover alterx tlsx asnmap"
+have_pd_tool() {
+  if [ "$1" = "httpx" ]; then
+    # Python's httpx package also installs an `httpx` CLI; only
+    # ProjectDiscovery's answers `-version`.
+    { command -v httpx >/dev/null 2>&1 && httpx -version >/dev/null 2>&1; } \
+      || { [ -x "$HOME/.pdtm/go/bin/httpx" ] && "$HOME/.pdtm/go/bin/httpx" -version >/dev/null 2>&1; } \
+      || { [ -x "$HOME/go/bin/httpx" ] && "$HOME/go/bin/httpx" -version >/dev/null 2>&1; }
+  else
+    have "$1"
+  fi
+}
+
+install_pd_tool() {
+  local tool="$1" package="$2" version="$3"
+  have_pd_tool "$tool" && return
+  mkdir -p "$HOME/.pdtm/go/bin"
+  GOBIN="$HOME/.pdtm/go/bin" go install "${package}@${version}"
+}
 
 # What is actually missing, before anything is fetched.
 missing_go_tools=""
 for tool in protoscope interactsh-client 2fa; do
   have "$tool" || missing_go_tools="$missing_go_tools $tool"
 done
 missing_pd_tools=""
-for tool in $PD_TOOLS; do
-  have "$tool" || missing_pd_tools="$missing_pd_tools,$tool"
+for tool in nuclei httpx subfinder naabu dnsx uncover alterx tlsx asnmap; do
+  have_pd_tool "$tool" || missing_pd_tools="$missing_pd_tools $tool"
 done
-missing_pd_tools="${missing_pd_tools#,}"
 
 # -- Go toolchain (only when something still has to be built) --------------
 # Deliberately last in the decision order: the toolchain is a ~150 MB download
 # whose only purpose is building the tools above. If they are all present it is
 # never needed, so it is never requested.
 need_go=false
 [ -n "$missing_go_tools" ] && need_go=true
-[ -n "$missing_pd_tools" ] && ! have pdtm && need_go=true
+[ -n "$missing_pd_tools" ] && need_go=true
 if [ "$need_go" = true ] && ! command -v go &>/dev/null; then
-  GO_VERSION="1.24.3"
   case "$ARCH" in
     aarch64|arm64) GOARCH="arm64" ;;
     *)             GOARCH="amd64" ;;
@@ -93,18 +110,21 @@ if [ "$need_go" = true ] && ! command -v go &>/dev/null; then
   export PATH="/usr/local/go/bin:$PATH"
 fi
 
-# -- PDTM + ProjectDiscovery tools ----------------------------------------
+# -- ProjectDiscovery tools ------------------------------------------------
 if [ -n "$missing_pd_tools" ]; then
-  if ! have pdtm; then
-    go install "github.com/projectdiscovery/pdtm/cmd/pdtm@${GO_TOOL_VERSIONS_pdtm}"
-  fi
-  PDTM_BIN="$(command -v pdtm || echo "$(go env GOPATH)/bin/pdtm")"
-  "$PDTM_BIN" -install "$missing_pd_tools"
+  install_pd_tool nuclei github.com/projectdiscovery/nuclei/v3/cmd/nuclei v3.11.1
+  install_pd_tool httpx github.com/projectdiscovery/httpx/cmd/httpx v1.12.0
+  install_pd_tool subfinder github.com/projectdiscovery/subfinder/v2/cmd/subfinder v2.16.0
+  install_pd_tool naabu github.com/projectdiscovery/naabu/v2/cmd/naabu v2.6.1
+  install_pd_tool dnsx github.com/projectdiscovery/dnsx/cmd/dnsx v1.3.1
+  install_pd_tool uncover github.com/projectdiscovery/uncover/cmd/uncover v1.2.1
+  install_pd_tool alterx github.com/projectdiscovery/alterx/cmd/alterx v0.1.0
+  install_pd_tool tlsx github.com/projectdiscovery/tlsx/cmd/tlsx v1.4.0
+  install_pd_tool asnmap github.com/projectdiscovery/asnmap/cmd/asnmap v1.1.1
 fi
 
 # -- katana (pre-built binary, go-tree-sitter build issue) -----------------
 if ! have katana; then
-  KATANA_VERSION="1.5.0"
   DEB_ARCH="$(dpkg --print-architecture 2>/dev/null || echo amd64)"
   mkdir -p "$HOME/.pdtm/go/bin"
   curl -fsSL "https://github.com/projectdiscovery/katana/releases/download/v${KATANA_VERSION}/katana_${KATANA_VERSION}_linux_${DEB_ARCH}.zip" \
@@ -131,7 +151,7 @@ have 2fa || go install "rsc.io/2fa@${GO_TOOL_VERSIONS_2fa}"
 
 # -- kiterunner (API content discovery) ------------------------------------
 if ! have kr; then
-  if git clone --depth 1 https://github.com/assetnote/kiterunner /tmp/kiterunner; then
+  if git clone --depth 1 --branch "$KITERUNNER_VERSION" https://github.com/assetnote/kiterunner /tmp/kiterunner; then
     ( cd /tmp/kiterunner && make build ) \
       && as_root mv /tmp/kiterunner/dist/kr /usr/local/bin/kr
     rm -rf /tmp/kiterunner
@@ -226,8 +246,9 @@ if ! command -v exiftool &>/dev/null; then
 fi
 
 # -- Node.js + agent-browser -----------------------------------------------
-if ! command -v node &>/dev/null; then
-  curl -fsSL https://deb.nodesource.com/setup_22.x | as_root bash - \
+NODE_MAJOR="$(node -p 'process.versions.node.split(".")[0]' 2>/dev/null || echo 0)"
+if [ "$NODE_MAJOR" -lt 24 ]; then
+  curl -fsSL https://deb.nodesource.com/setup_24.x | as_root bash - \
     && as_root apt-get install -y --no-install-recommends nodejs \
     || echo "WARN: Node.js install failed, skipping"
 fi
@@ -243,7 +264,7 @@ fi
 # non-fatal because a disconnected deployment that cannot fetch a browser
 # should still get the rest of this capability's tooling.
 AGENT_BROWSER_CACHE="${AGENT_BROWSER_CACHE_DIR:-$HOME/.cache/agent-browser}"
-if [ ! -d "$AGENT_BROWSER_CACHE" ]; then
+if [ "${DREADNODE_CAPABILITY_INSTALL:-}" != "sealed" ] && [ ! -d "$AGENT_BROWSER_CACHE" ]; then
   agent-browser install || echo "WARN: agent-browser browser download failed, skipping"
 fi
 

capabilities/web-security/tests/test_install_tools_offline.py

@@ -12,7 +12,9 @@
 
 from __future__ import annotations
 
+import os
 import re
+import subprocess
 from pathlib import Path
 
 ROOT = Path(__file__).resolve().parents[1]
@@ -30,15 +32,72 @@ def _surrounding_context(index: int, span: int = 4) -> str:
     return "\n".join(LINES[max(0, index - span) : min(len(LINES), index + span + 1)])
 
 
+def _shell_function(name: str) -> str:
+    match = re.search(rf"^{name}\(\) \{{\n.*?^\}}\n", INSTALL_SCRIPT, re.M | re.S)
+    assert match, f"{name}() not found in install_tools.sh"
+    return match.group(0)
+
+
+def _stub(path: Path, exit_code: int) -> None:
+    """An executable whose `-version` succeeds only for ProjectDiscovery httpx."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(f"#!/bin/sh\nexit {exit_code}\n", encoding="utf-8")
+    path.chmod(0o755)
+
+
+def _have_pd_httpx(tmp_path: Path) -> bool:
+    (tmp_path / "home").mkdir(exist_ok=True)
+    script = (
+        _shell_function("have")
+        + _shell_function("have_pd_tool")
+        + "have_pd_tool httpx\n"
+    )
+    env = {
+        "HOME": str(tmp_path / "home"),
+        "PATH": f"{tmp_path / 'bin'}:{os.environ['PATH']}",
+    }
+    return subprocess.run(["bash", "-c", script], env=env, check=False).returncode == 0
+
+
 class TestVersionsArePinned:
     def test_no_unpinned_go_installs(self) -> None:
         # `go install ...@latest` re-resolves against the module proxy every
         # run, so it reaches the network even when the binary is already
         # present — and produces a different tool set on different days, which
         # no SBOM can describe.
-        unpinned = [line.strip() for line in LINES if "@latest" in line and not line.strip().startswith("#")]
+        unpinned = [
+            line.strip()
+            for line in LINES
+            if "@latest" in line and not line.strip().startswith("#")
+        ]
         assert not unpinned, f"unpinned installs: {unpinned}"
 
+    def test_projectdiscovery_tools_use_explicit_versions(self) -> None:
+        pins = {
+            "nuclei": "v3.11.1",
+            "httpx": "v1.12.0",
+            "subfinder": "v2.16.0",
+            "naabu": "v2.6.1",
+            "dnsx": "v1.3.1",
+            "uncover": "v1.2.1",
+            "alterx": "v0.1.0",
+            "tlsx": "v1.4.0",
+            "asnmap": "v1.1.1",
+        }
+        for tool, version in pins.items():
+            assert re.search(
+                rf"install_pd_tool {tool} \S+ {re.escape(version)}$",
+                INSTALL_SCRIPT,
+                re.MULTILINE,
+            ), f"missing {tool} pin {version}"
+
+        assert "pdtm -install" not in INSTALL_SCRIPT
+
+    def test_toolchain_and_kiterunner_versions_are_pinned(self) -> None:
+        assert 'GO_VERSION="1.26.6"' in INSTALL_SCRIPT
+        assert 'KITERUNNER_VERSION="v1.0.2"' in INSTALL_SCRIPT
+        assert 'git clone --depth 1 --branch "$KITERUNNER_VERSION"' in INSTALL_SCRIPT
+
 
 class TestFetchesAreGuarded:
     def test_every_go_install_is_guarded(self) -> None:
@@ -53,7 +112,9 @@ def test_every_go_install_is_guarded(self) -> None:
     def test_global_npm_install_is_guarded(self) -> None:
         for i, line in enumerate(LINES):
             if re.search(r"^\s*(as_root\s+)?npm install -g", line):
-                assert "have " in _preceding_context(i), f"unguarded global npm install at line {i + 1}: {line.strip()}"
+                assert "have " in _preceding_context(
+                    i
+                ), f"unguarded global npm install at line {i + 1}: {line.strip()}"
 
     def test_npm_installs_are_version_pinned(self) -> None:
         # Same SBOM argument as the go pins: an unpinned `npm install -g`
@@ -78,28 +139,53 @@ def test_py_install_calls_are_guarded(self) -> None:
                 continue
             # Skip the py_install function definition and requirement file
             # installs (guarded by their parent clone check).
-            if stripped.startswith(("if", "elif", "def", "#")) or "-r " in stripped or "py_install()" in stripped:
+            if (
+                stripped.startswith(("if", "elif", "def", "#"))
+                or "-r " in stripped
+                or "py_install()" in stripped
+            ):
                 continue
             if "have " not in _preceding_context(i):
                 unguarded.append(stripped)
         assert not unguarded, f"unguarded py_install: {unguarded}"
 
-    def test_pdtm_only_installs_missing_tools(self) -> None:
-        # `pdtm -install <full list>` re-fetches every tool in the list. The
-        # set has to be narrowed to what is actually absent first.
+    def test_only_missing_projectdiscovery_tools_are_installed(self) -> None:
         assert "$missing_pd_tools" in INSTALL_SCRIPT
-        assert "-install nuclei,httpx" not in INSTALL_SCRIPT
+        assert 'have_pd_tool "$tool" || missing_pd_tools=' in INSTALL_SCRIPT
+
+    def test_httpx_guard_accepts_projectdiscovery_httpx_on_path(
+        self, tmp_path: Path
+    ) -> None:
+        _stub(tmp_path / "bin" / "httpx", exit_code=0)
+        assert _have_pd_httpx(tmp_path)
+
+    def test_httpx_guard_rejects_the_python_cli(self, tmp_path: Path) -> None:
+        _stub(tmp_path / "bin" / "httpx", exit_code=2)
+        assert not _have_pd_httpx(tmp_path)
+
+    def test_httpx_guard_accepts_pdtm_httpx_behind_the_python_cli(
+        self, tmp_path: Path
+    ) -> None:
+        _stub(tmp_path / "bin" / "httpx", exit_code=2)
+        _stub(tmp_path / "home" / ".pdtm" / "go" / "bin" / "httpx", exit_code=0)
+        assert _have_pd_httpx(tmp_path)
 
     def test_katana_download_is_guarded(self) -> None:
-        idx = next(i for i, line in enumerate(LINES) if "katana_${KATANA_VERSION}" in line)
+        idx = next(
+            i for i, line in enumerate(LINES) if "katana_${KATANA_VERSION}" in line
+        )
         assert "have katana" in _preceding_context(idx, span=8)
 
     def test_caido_cli_download_is_guarded(self) -> None:
-        idx = next(i for i, line in enumerate(LINES) if "caido.download/releases" in line)
+        idx = next(
+            i for i, line in enumerate(LINES) if "caido.download/releases" in line
+        )
         assert "command -v caido-cli" in _preceding_context(idx, span=10)
 
     def test_caido_mcp_server_download_is_guarded(self) -> None:
-        idx = next(i for i, line in enumerate(LINES) if "caido-mcp-server-linux" in line)
+        idx = next(
+            i for i, line in enumerate(LINES) if "caido-mcp-server-linux" in line
+        )
         assert "command -v caido-mcp-server" in _preceding_context(idx, span=15)
 
     def test_kiterunner_build_is_guarded(self) -> None:
@@ -110,9 +196,13 @@ def test_wrangler_install_is_guarded_and_pinned(self) -> None:
         # wrangler is fetched from npm, so the guard-and-pin discipline applies
         # exactly as it does to the go installs: present binary -> no registry
         # request; absent binary -> the pinned version, not @latest.
-        idx = next(i for i, line in enumerate(LINES) if "wrangler@${WRANGLER_VERSION}" in line)
+        idx = next(
+            i for i, line in enumerate(LINES) if "wrangler@${WRANGLER_VERSION}" in line
+        )
         assert "have wrangler" in _preceding_context(idx, span=6)
-        pin = next(i for i, line in enumerate(LINES) if line.startswith("WRANGLER_VERSION="))
+        pin = next(
+            i for i, line in enumerate(LINES) if line.startswith("WRANGLER_VERSION=")
+        )
         assert re.fullmatch(
             r"WRANGLER_VERSION=\"[0-9]+\.[0-9]+\.[0-9]+\"",
             LINES[pin].strip(),
@@ -142,23 +232,37 @@ def test_go_cache_cleanup_only_runs_when_go_was_used(self) -> None:
         idx = next(i for i, line in enumerate(LINES) if "go clean -cache" in line)
         assert "need_go" in _preceding_context(idx, span=3)
 
+    def test_node_24_floor_and_sealed_browser_guard(self) -> None:
+        assert "setup_24.x" in INSTALL_SCRIPT
+        assert "setup_22.x" not in INSTALL_SCRIPT
+        assert "${DREADNODE_CAPABILITY_INSTALL:-}" in INSTALL_SCRIPT
+        assert '!= "sealed"' in INSTALL_SCRIPT
+
 
 class TestRootEscalation:
     """Writes to root-owned paths (/usr/local/bin, /opt) must use as_root."""
 
     def test_caido_cli_tar_uses_as_root(self) -> None:
         idx = next(
-            i for i, line in enumerate(LINES) if "tar" in line and "caido-cli" in line and "/usr/local/bin" in line
+            i
+            for i, line in enumerate(LINES)
+            if "tar" in line and "caido-cli" in line and "/usr/local/bin" in line
         )
         assert "as_root" in LINES[idx]
 
     def test_caido_mcp_server_install_uses_as_root(self) -> None:
-        idx = next(i for i, line in enumerate(LINES) if "install -m" in line and "caido-mcp-server" in line)
+        idx = next(
+            i
+            for i, line in enumerate(LINES)
+            if "install -m" in line and "caido-mcp-server" in line
+        )
         assert "as_root" in LINES[idx]
 
     def test_kiterunner_mv_uses_as_root(self) -> None:
         idx = next(
-            i for i, line in enumerate(LINES) if "/usr/local/bin/kr" in line and ("mv " in line or "install " in line)
+            i
+            for i, line in enumerate(LINES)
+            if "/usr/local/bin/kr" in line and ("mv " in line or "install " in line)
         )
         assert "as_root" in LINES[idx]
 
@@ -167,11 +271,17 @@ def test_burp_suite_uses_as_root(self) -> None:
         assert "as_root" in LINES[idx]
 
     def test_exiftool_apt_uses_as_root(self) -> None:
-        idx = next(i for i, line in enumerate(LINES) if "apt-get" in line and "exiftool" in line)
+        idx = next(
+            i
+            for i, line in enumerate(LINES)
+            if "apt-get" in line and "exiftool" in line
+        )
         assert "as_root" in LINES[idx]
 
     def test_nodejs_apt_uses_as_root(self) -> None:
-        idx = next(i for i, line in enumerate(LINES) if "apt-get" in line and "nodejs" in line)
+        idx = next(
+            i for i, line in enumerate(LINES) if "apt-get" in line and "nodejs" in line
+        )
         assert "as_root" in LINES[idx]