diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ce99fd5..e1b9533 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,10 +35,11 @@ jobs: if: always() run: | pixi run --environment dev lint --diff - # - name: Mypy - # if: always() - # run: | - # pixi run --environment dev type-check + + - name: Mypy + if: always() + run: | + pixi run --environment dev type-check - name: Collect QC run: echo "All quality control checks passed" diff --git a/snakemake_interface_executor_plugins/executors/base.py b/snakemake_interface_executor_plugins/executors/base.py index 1f16f36..255e16d 100644 --- a/snakemake_interface_executor_plugins/executors/base.py +++ b/snakemake_interface_executor_plugins/executors/base.py @@ -21,13 +21,18 @@ class SubmittedJobInfo: class AbstractExecutor(ABC): + workflow: WorkflowExecutorInterface + logger: LoggerExecutorInterface + def __init__( self, workflow: WorkflowExecutorInterface, logger: LoggerExecutorInterface, ): self.workflow = workflow - self.dag = workflow.dag + self.dag = ( + workflow.dag + ) # we cant type annotate this because dag type is in snakemake self.logger = logger def get_resource_declarations_dict(self, job: JobExecutorInterface): diff --git a/snakemake_interface_executor_plugins/executors/real.py b/snakemake_interface_executor_plugins/executors/real.py index bfc072d..2a4f0ee 100644 --- a/snakemake_interface_executor_plugins/executors/real.py +++ b/snakemake_interface_executor_plugins/executors/real.py @@ -4,7 +4,7 @@ __license__ = "MIT" from abc import abstractmethod -from typing import Dict +from typing import Mapping from snakemake_interface_common import at_least_snakemake_version from snakemake_interface_executor_plugins.executors.base import ( @@ -12,7 +12,7 @@ SubmittedJobInfo, ) from snakemake_interface_executor_plugins.logging import LoggerExecutorInterface -from snakemake_interface_executor_plugins.settings import ExecMode +from snakemake_interface_executor_plugins.settings import ExecMode, ExecutorSettingsBase from snakemake_interface_executor_plugins.utils import ( encode_target_jobs_cli_args, format_cli_arg, @@ -23,6 +23,12 @@ class RealExecutor(AbstractExecutor): + # Class attributes with type annotations + workflow: WorkflowExecutorInterface + logger: LoggerExecutorInterface + executor_settings: ExecutorSettingsBase + snakefile: str + def __init__( self, workflow: WorkflowExecutorInterface, @@ -179,5 +185,5 @@ def format_job_exec(self, job: JobExecutorInterface) -> str: ) return args - def envvars(self) -> Dict[str, str]: + def envvars(self) -> Mapping[str, str]: return self.workflow.spawned_job_args_factory.envvars() diff --git a/snakemake_interface_executor_plugins/executors/remote.py b/snakemake_interface_executor_plugins/executors/remote.py index 74ef734..4614182 100644 --- a/snakemake_interface_executor_plugins/executors/remote.py +++ b/snakemake_interface_executor_plugins/executors/remote.py @@ -12,7 +12,7 @@ import sys import tempfile import threading -from typing import Generator, List +from typing import Generator, List, Optional from snakemake_interface_common.exceptions import WorkflowError from snakemake_interface_executor_plugins.executors.base import SubmittedJobInfo from snakemake_interface_executor_plugins.executors.real import RealExecutor @@ -36,7 +36,20 @@ class RemoteExecutor(RealExecutor, ABC): also for the cloud. """ - default_jobscript = "jobscript.sh" + # Class attributes with type annotations + default_jobscript: str = "jobscript.sh" + _next_seconds_between_status_checks: Optional[int] + max_status_checks_per_second: float + jobname: str + snakefile: str + is_default_jobscript: bool + jobscript: str + _tmpdir: Optional[str] + active_jobs: List[SubmittedJobInfo] + lock: threading.Lock + wait: bool + wait_thread: threading.Thread + status_rate_limiter: Throttler def __init__( self, @@ -118,7 +131,7 @@ def cancel_jobs(self, active_jobs: List[SubmittedJobInfo]): ... def get_exec_mode(self) -> ExecMode: - return ExecMode.REMOTE + return ExecMode.REMOTE # type: ignore def get_python_executable(self): return ( @@ -128,7 +141,7 @@ def get_python_executable(self): else "python" ) - def get_job_args(self, job: JobExecutorInterface): + def get_job_args(self, job: JobExecutorInterface, **kwargs): waitfiles_parameter = "" if SharedFSUsage.INPUT_OUTPUT in self.workflow.storage_settings.shared_fs_usage: wait_for_files = [] @@ -148,7 +161,7 @@ def get_job_args(self, job: JobExecutorInterface): else: waitfiles_parameter = format_cli_arg("--wait-for-files", wait_for_files) - return f"{super().get_job_args(job)} {waitfiles_parameter}" + return f"{super().get_job_args(job, **kwargs)} {waitfiles_parameter}" def report_job_submission( self, job_info: SubmittedJobInfo, register_job: bool = True diff --git a/snakemake_interface_executor_plugins/jobs.py b/snakemake_interface_executor_plugins/jobs.py index 3809d34..1e8814d 100644 --- a/snakemake_interface_executor_plugins/jobs.py +++ b/snakemake_interface_executor_plugins/jobs.py @@ -5,9 +5,10 @@ from abc import ABC, abstractmethod import sys -from typing import Any, Iterable, Mapping, Optional, Sequence, Union +from typing import Any, Iterable, Mapping, Optional, Sequence, Union, List from snakemake_interface_common.rules import RuleInterface +from snakemake_interface_executor_plugins.utils import TargetSpec class JobExecutorInterface(ABC): @@ -58,7 +59,7 @@ def output(self) -> Iterable[str]: ... def register(self, external_jobid: Optional[str] = None) -> None: ... @abstractmethod - def get_target_spec(self) -> str: ... + def get_target_spec(self) -> List[TargetSpec]: ... @abstractmethod def rules(self) -> Iterable[RuleInterface]: ... diff --git a/snakemake_interface_executor_plugins/scheduler.py b/snakemake_interface_executor_plugins/scheduler.py index b7a3843..a5ac324 100644 --- a/snakemake_interface_executor_plugins/scheduler.py +++ b/snakemake_interface_executor_plugins/scheduler.py @@ -4,8 +4,14 @@ __license__ = "MIT" from abc import ABC, abstractmethod +from typing import Callable +from snakemake_interface_executor_plugins.jobs import JobExecutorInterface class JobSchedulerExecutorInterface(ABC): + submit_callback: Callable[[JobExecutorInterface], None] + finish_callback: Callable[[JobExecutorInterface], None] + error_callback: Callable[[JobExecutorInterface], None] + @abstractmethod def executor_error_callback(self, exception: Exception) -> None: ... diff --git a/snakemake_interface_executor_plugins/settings.py b/snakemake_interface_executor_plugins/settings.py index 56ecb38..d3bf1af 100644 --- a/snakemake_interface_executor_plugins/settings.py +++ b/snakemake_interface_executor_plugins/settings.py @@ -1,8 +1,12 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import List, Sequence, Set +from typing import Callable, Iterable, List, Sequence, Set -from snakemake_interface_common.settings import SettingsEnumBase, TSettingsEnumBase +from snakemake_interface_common.settings import ( + SettingsEnumBase, + TContainer, + TSettingsEnumBase, +) import snakemake_interface_common.plugin_registry.plugin @@ -149,15 +153,18 @@ def choices(cls) -> List[str]: return super().choices() + ["none"] @classmethod - def _parse_choices_into(cls, choices: str, container) -> List[TSettingsEnumBase]: + def _parse_choices_into( + cls, + choices: List[str], + container: Callable[[Iterable[TSettingsEnumBase]], TContainer], + ) -> TContainer: if "none" in choices: if len(choices) > 1: raise ValueError( "Cannot specify 'none' together with other shared filesystem usages." ) return container([]) - else: - return container(cls.parse_choice(choice) for choice in choices) + return super()._parse_choices_into(choices, container) class StorageSettingsExecutorInterface(ABC): diff --git a/snakemake_interface_executor_plugins/workflow.py b/snakemake_interface_executor_plugins/workflow.py index 868d75d..9e7e65d 100644 --- a/snakemake_interface_executor_plugins/workflow.py +++ b/snakemake_interface_executor_plugins/workflow.py @@ -21,6 +21,7 @@ GroupSettingsExecutorInterface, RemoteExecutionSettingsExecutorInterface, StorageSettingsExecutorInterface, + ExecutorSettingsBase, ) @@ -53,6 +54,10 @@ def group_settings(self) -> GroupSettingsExecutorInterface: ... @abstractmethod def executor_plugin(self) -> Optional[Plugin]: ... + @property + @abstractmethod + def executor_settings(self) -> ExecutorSettingsBase: ... + @property @abstractmethod def resource_scopes(self): ... @@ -72,3 +77,7 @@ def workdir_init(self): ... @property @abstractmethod def scheduler(self) -> JobSchedulerExecutorInterface: ... + + @property + @abstractmethod + def dag(self): ... diff --git a/tests/tests.py b/tests/tests.py index 3c3453b..90c8186 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,9 +1,15 @@ from typing import List +from types import SimpleNamespace +from unittest.mock import Mock, patch +import pytest +from snakemake_interface_executor_plugins.executors.real import RealExecutor +from snakemake_interface_executor_plugins.executors.remote import RemoteExecutor from snakemake_interface_executor_plugins.registry import ExecutorPluginRegistry from snakemake_interface_common.plugin_registry.tests import TestRegistryBase from snakemake_interface_common.plugin_registry.plugin import PluginBase, SettingsBase from snakemake_interface_common.plugin_registry import PluginRegistryBase from snakemake_interface_executor_plugins.utils import format_cli_arg +from snakemake_interface_executor_plugins.settings import SharedFSUsage class TestRegistry(TestRegistryBase): @@ -54,3 +60,39 @@ def test_format_cli_arg_expr(): def test_format_cli_arg_list(): fmt = format_cli_arg("--config", ["foo={'bar': 1}"]) assert fmt == "--config \"foo={'bar': 1}\"" + + +def test_shared_fs_usage_parses_none_and_regular_choices(): + assert SharedFSUsage.parse_choices_list(["none"]) == [] + assert SharedFSUsage.parse_choices_set(["none"]) == set() + assert SharedFSUsage.parse_choices_list(["input-output"]) == [ + SharedFSUsage.INPUT_OUTPUT + ] + assert SharedFSUsage.parse_choices_set(["input-output"]) == { + SharedFSUsage.INPUT_OUTPUT + } + with pytest.raises(ValueError, match="Cannot specify 'none'"): + SharedFSUsage.parse_choices_set(["none", "input-output"]) + + +def test_remote_job_args_forward_cores_to_real_executor(): + class StubRemote(RemoteExecutor): + def run_job(self, job): + pass + + def cancel_jobs(self, active_jobs): + pass + + async def check_active_jobs(self, active_jobs): + pass + + remote = object.__new__(StubRemote) + remote.workflow = SimpleNamespace( + storage_settings=SimpleNamespace(shared_fs_usage=set()) + ) + job = Mock() + with patch.object(RealExecutor, "get_job_args", return_value="--cores 2") as parent: + result = remote.get_job_args(job, cores=2) + + parent.assert_called_once_with(job, cores=2) + assert "--cores 2" in result