feat(nodejs): prepare a monorepo's shared project root once

aws_lambda_builders/builder.py

@@ -171,5 +171,30 @@ def build(
 
         return workflow.run()
 
+    def prepare_shared_project(self, project_root, experimental_flags=None):
+        """
+        Do once, at the given project root, the work every function under it shares - for the nodejs
+        workflows, installing the dependency tree npm hoists there. Run it once, then call build()
+        per function with download_dependencies=False.
+
+        :type project_root: str
+        :param project_root:
+            Path to the project root shared by the functions about to be built
+
+        :type experimental_flags: list
+        :param experimental_flags:
+            The same flags the per-function build() calls get; a workflow may gate this on one
+
+        :raises aws_lambda_builders.exceptions.SharedProjectNotSupportedError:
+            when the selected workflow has no shared-project concept
+        :raises aws_lambda_builders.exceptions.SharedProjectPreparationError:
+            when the preparation fails; both are LambdaBuilderError, so a caller can catch that and
+            fall back to building each function on its own
+        """
+        LOG.debug(
+            "Preparing shared project root '%s' with workflow '%s'", project_root, self.selected_workflow_cls.NAME
+        )
+        self.selected_workflow_cls.prepare_shared_project(project_root, experimental_flags=experimental_flags)
+
     def _clear_workflows(self):
         DEFAULT_REGISTRY.clear()

aws_lambda_builders/exceptions.py

@@ -72,3 +72,24 @@ class WorkflowUnknownError(LambdaBuilderError):
     """
 
     MESSAGE = "{workflow_name}:{action_name} - {reason}"
+
+
+class SharedProjectNotSupportedError(LambdaBuilderError):
+    """
+    Raised when the selected workflow has no notion of work shared by every
+    package under one project root
+    """
+
+    MESSAGE = (
+        "Workflow for capability '{capabilities}' does not support preparing a project root "
+        "shared by several functions"
+    )
+
+
+class SharedProjectPreparationError(LambdaBuilderError):
+    """
+    Raised when preparing the shared project root itself failed; the caller can
+    fall back to building each function on its own
+    """
+
+    MESSAGE = "Preparing the shared project root {project_root} failed: {reason}"

aws_lambda_builders/workflow.py

@@ -15,6 +15,7 @@
 from aws_lambda_builders.exceptions import (
     MisMatchRuntimeError,
     RuntimeValidatorError,
+    SharedProjectNotSupportedError,
     WorkflowFailedError,
     WorkflowUnknownError,
 )
@@ -183,6 +184,20 @@ class BaseWorkflow(object, metaclass=_WorkflowMetaClass):
     # The directory where the workflow builds/installs by default, each workflow should define this.
     DEFAULT_BUILD_DIR = None
 
+    @classmethod
+    def prepare_shared_project(cls, project_root, experimental_flags=None):
+        """
+        Do once, at the given project root, the work every function under it shares.
+
+        Part of the workflow contract so that LambdaBuilder can delegate to it unconditionally; a
+        workflow with no shared-project concept inherits this refusal. Override it where several
+        functions built from one project root can share a step - for the nodejs workflows, the
+        dependency tree npm hoists to a workspace root.
+
+        :raises aws_lambda_builders.exceptions.SharedProjectNotSupportedError: always, by default
+        """
+        raise SharedProjectNotSupportedError(capabilities=cls.CAPABILITY)
+
     def __init__(
         self,
         source_dir,

aws_lambda_builders/workflows/nodejs_npm/actions.py

@@ -10,7 +10,7 @@
 from aws_lambda_builders import utils
 from aws_lambda_builders.actions import ActionFailedError, BaseAction, Purpose
 from aws_lambda_builders.utils import extract_tarfile
-from aws_lambda_builders.workflows.nodejs_npm.lockfile_closure import production_closure
+from aws_lambda_builders.workflows.nodejs_npm.lockfile_closure import dev_packages, production_closure
 from aws_lambda_builders.workflows.nodejs_npm.npm import NpmExecutionError, SubprocessNpm
 
 LOG = logging.getLogger(__name__)
@@ -85,19 +85,24 @@ class NodejsNpmInstallOrUpdateBaseAction(BaseAction):
 
     PURPOSE = Purpose.RESOLVE_DEPENDENCIES
 
-    def __init__(self, install_dir: str, subprocess_npm: SubprocessNpm):
+    def __init__(self, install_dir: str, subprocess_npm: SubprocessNpm, omit_dev: bool = True):
         """
         Parameters
         ----------
         install_dir : str
             Dependencies will be installed in this directory.
         subprocess_npm : SubprocessNpm
             An instance of the NPM process wrapper
+        omit_dev : bool
+            Passes --omit=dev if True, by default True. False is for installing a whole workspace root,
+            whose own dev dependencies (build tools like esbuild) must survive the install; the artifacts
+            leave them out at the link step instead.
         """
 
         super().__init__()
         self.install_dir = install_dir
         self.subprocess_npm = subprocess_npm
+        self.omit_dev = omit_dev
 
 
 class NodejsNpmInstallAction(NodejsNpmInstallOrUpdateBaseAction):
@@ -108,20 +113,24 @@ class NodejsNpmInstallAction(NodejsNpmInstallOrUpdateBaseAction):
     NAME = "NpmInstall"
     DESCRIPTION = "Installing dependencies from NPM"
 
-    def __init__(self, install_dir: str, subprocess_npm: SubprocessNpm, install_links: Optional[bool] = False):
+    def __init__(
+        self,
+        install_dir: str,
+        subprocess_npm: SubprocessNpm,
+        install_links: Optional[bool] = False,
+        omit_dev: bool = True,
+    ):
         """
         Parameters
         ----------
-        install_dir : str
-            Dependencies will be installed in this directory.
-        subprocess_npm : SubprocessNpm
-            An instance of the NPM process wrapper
         install_links : Optional[bool]
             Uses the --install-links npm option if True, by default False. Required when installing into the
             source directory, so that local file dependencies are installed as regular dependencies.
+
+        See NodejsNpmInstallOrUpdateBaseAction for the rest.
         """
 
-        super().__init__(install_dir=install_dir, subprocess_npm=subprocess_npm)
+        super().__init__(install_dir=install_dir, subprocess_npm=subprocess_npm, omit_dev=omit_dev)
         self.install_links = install_links
 
     def execute(self):
@@ -131,9 +140,11 @@ def execute(self):
         :raises lambda_builders.actions.ActionFailedError: when NPM execution fails
         """
         try:
-            LOG.debug("NODEJS installing production dependencies in: %s", self.install_dir)
+            LOG.debug("NODEJS installing dependencies in: %s", self.install_dir)
 
-            command = ["install", "-q", "--no-audit", "--no-save", "--omit=dev"]
+            command = ["install", "-q", "--no-audit", "--no-save"]
+            if self.omit_dev:
+                command.append("--omit=dev")
             if self.install_links:
                 command.append("--install-links")
             self.subprocess_npm.run(command, cwd=self.install_dir)
@@ -163,13 +174,12 @@ def execute(self):
         :raises lambda_builders.actions.ActionFailedError: when NPM execution fails
         """
         try:
-            LOG.debug("NODEJS updating production dependencies in: %s", self.install_dir)
+            LOG.debug("NODEJS updating dependencies in: %s", self.install_dir)
 
-            command = [
-                "update",
-                "--no-audit",
-                "--no-save",
-                "--omit=dev",
+            command = ["update", "--no-audit", "--no-save"]
+            if self.omit_dev:
+                command.append("--omit=dev")
+            command += [
                 "--no-package-lock",
                 "--install-links",
             ]
@@ -441,7 +451,13 @@ def _link_every_installed_package(self, destination):
 
         Per-entry links, not one link for the whole tree: only the overlay is a superset of what the
         function resolves, and it cannot dangle. See aws/aws-lambda-builders#935.
+
+        A superset of the production closure, but not of the dev tree: the shared install at a
+        workspace root keeps dev dependencies on purpose, and shipping those would put the root's
+        build tools in every function's artifacts, which the per-function install never does.
         """
+        dev_dirs, dev_names = dev_packages(self._project_root, self._install_dir)
+
         # Own directory last, and resolved into one mapping before linking: create_symlink_or_copy keeps
         # the first link at a destination, so linking as we go would give the hoisted copy precedence.
         chosen = {}
@@ -451,7 +467,11 @@ def _link_every_installed_package(self, destination):
                 LOG.debug("NODEJS no dependencies installed in %s, nothing to link from there", tree)
                 continue
             LOG.debug("NODEJS linking every package installed in %s into the artifacts", tree)
-            chosen.update(dict(self._installed_packages(tree)))
+            for name, package_dir in self._installed_packages(tree):
+                if name in dev_names or os.path.realpath(package_dir) in dev_dirs:
+                    LOG.debug("NODEJS %s is installed only for development; leaving it out", name)
+                    continue
+                chosen[name] = package_dir
 
         if not chosen:
             LOG.warning(

aws_lambda_builders/workflows/nodejs_npm/lockfile_closure.py

@@ -11,7 +11,7 @@
 import json
 import logging
 import os
-from typing import Dict, List, Optional, Set
+from typing import Dict, List, Optional, Set, Tuple
 
 LOG = logging.getLogger(__name__)
 
@@ -70,6 +70,62 @@ def _entry_dir(project_root: str, key: str, entry: Dict) -> str:
     return os.path.join(project_root, *key.split("/")) if key else project_root
 
 
+def _lockfile_packages(project_root: str) -> Optional[Dict]:
+    """
+    The packages map of whichever lockfile under `project_root` parses, hidden one first.
+
+    No staleness gate, unlike `production_closure`: this is read for `dev` markers only, and which
+    packages a project declares as dev survives a tree the hidden lockfile no longer describes.
+    """
+    candidates = [hidden_lockfile_path(project_root)] + [
+        os.path.join(project_root, name) for name in ("npm-shrinkwrap.json", "package-lock.json")
+    ]
+    for path in candidates:
+        try:
+            with open(path) as handle:
+                lock = json.load(handle)
+        except (OSError, ValueError):
+            continue
+        if lock.get("lockfileVersion", 1) >= MINIMUM_LOCKFILE_VERSION and "packages" in lock:
+            return lock["packages"]
+    return None
+
+
+def _declared_dev_dependencies(directory: str) -> Set[str]:
+    """The names in one manifest's `devDependencies`, empty when it cannot be read."""
+    try:
+        with open(os.path.join(directory, "package.json")) as handle:
+            return set(json.load(handle).get("devDependencies") or {})
+    except (OSError, ValueError, AttributeError):
+        return set()
+
+
+def dev_packages(project_root: str, install_dir: str) -> Tuple[Set[str], Set[str]]:
+    """
+    What is installed only for development, as `(real directories, package names)`.
+
+    For the shared install at a workspace root, which keeps dev dependencies on purpose - the root's
+    own build tools have to survive it - so that the artifacts can still leave them out.
+
+    A lockfile answers by directory, and answers properly: it marks an entry `dev` when every path to
+    it runs through a devDependency, so transitive ones are covered too. Without a usable lockfile
+    only the manifests' own declarations are known, and those answer by NAME: npm hoists a workspace
+    member's dev dependency to the root, so the directory it ends up in is not one this can predict.
+    """
+    packages = _lockfile_packages(project_root)
+    if packages is not None:
+        dev_dirs = {
+            os.path.realpath(_entry_dir(project_root, key, entry))
+            for key, entry in packages.items()
+            if key and entry.get("dev")
+        }
+        return dev_dirs, set()
+
+    LOG.debug("NODEJS no usable lockfile under %s; excluding only the manifests' own dev dependencies", project_root)
+    dev_names = _declared_dev_dependencies(project_root) | _declared_dev_dependencies(install_dir)
+    return set(), dev_names
+
+
 def production_closure(project_root: str, install_dir: str) -> Optional[List[str]]:
     """
     The production dependency closure of `install_dir`, as absolute directories.