diff --git a/openml/_api/resources/base/resources.py b/openml/_api/resources/base/resources.py index c969546a8..82c48bc12 100644 --- a/openml/_api/resources/base/resources.py +++ b/openml/_api/resources/base/resources.py @@ -171,6 +171,7 @@ def list( setup: Iterable[int] | None = None, flow: int | None = None, tag: str | None = None, + **kwargs: Any, ) -> list[OpenMLSetup]: ... @abstractmethod diff --git a/openml/_api/resources/setup.py b/openml/_api/resources/setup.py index 57e466410..22299cdb7 100644 --- a/openml/_api/resources/setup.py +++ b/openml/_api/resources/setup.py @@ -26,6 +26,7 @@ def _build_url( setup: Iterable[int] | None = None, flow: int | None = None, tag: str | None = None, + **kwargs: Any, ) -> str: """Construct an OpenML Setup API URL with filtering parameters. @@ -50,6 +51,10 @@ def _build_url( api_call += f"/limit/{limit}" if offset is not None: api_call += f"/offset/{offset}" + if kwargs: + for operator, value in kwargs.items(): + if value is not None: + api_call += f"/{operator}/{value}" if setup is not None: api_call += f"/setup/{','.join([str(int(i)) for i in setup])}" if flow is not None: @@ -164,6 +169,7 @@ def list( setup: Iterable[int] | None = None, flow: int | None = None, tag: str | None = None, + **kwargs: Any, ) -> builtins.list[OpenMLSetup]: """Perform API call `/setup/list/{filters}` @@ -183,7 +189,7 @@ def list( list setups that match the filters, going from id to the OpenMLSetup object. """ - api_call = SetupV1API._build_url(limit, offset, setup=setup, flow=flow, tag=tag) + api_call = SetupV1API._build_url(limit, offset, setup=setup, flow=flow, tag=tag, **kwargs) setup_response = self._http.get(api_call) xml_content = setup_response.text @@ -266,6 +272,7 @@ def list( setup: Iterable[int] | None = None, # noqa: ARG002 flow: int | None = None, # noqa: ARG002 tag: str | None = None, # noqa: ARG002 + **kwargs: Any, # noqa: ARG002 ) -> builtins.list[OpenMLSetup]: self._not_supported(method="list") diff --git a/openml/evaluations/functions.py b/openml/evaluations/functions.py index f4e07c1b8..a37462615 100644 --- a/openml/evaluations/functions.py +++ b/openml/evaluations/functions.py @@ -4,7 +4,7 @@ from functools import partial from itertools import chain -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Any, Literal from typing_extensions import overload import numpy as np @@ -33,6 +33,7 @@ def list_evaluations( per_fold: bool | None = None, sort_order: str | None = None, output_format: Literal["dataframe"] = ..., + **kwargs: Any, ) -> pd.DataFrame: ... @@ -51,6 +52,7 @@ def list_evaluations( per_fold: bool | None = None, sort_order: str | None = None, output_format: Literal["object"] = "object", + **kwargs: Any, ) -> dict[int, OpenMLEvaluation]: ... @@ -68,6 +70,7 @@ def list_evaluations( per_fold: bool | None = None, sort_order: str | None = None, output_format: Literal["object", "dataframe"] = "object", + **kwargs: Any, ) -> dict[int, OpenMLEvaluation] | pd.DataFrame: """List all run-evaluation pairs matching all of the given filters. @@ -131,6 +134,7 @@ def list_evaluations( study=study, sort_order=sort_order, per_fold=per_fold_str, + **kwargs, ) eval_collection = openml.utils._list_all(listing_call, offset=offset, limit=size) diff --git a/openml/setups/functions.py b/openml/setups/functions.py index 0b1a4f375..3a37bdba1 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -78,6 +78,7 @@ def list_setups( # noqa: PLR0913 tag: str | None = None, setup: Iterable[int] | None = None, output_format: Literal["object", "dataframe"] = "object", + **kwargs: Any, ) -> dict[int, OpenMLSetup] | pd.DataFrame: """ List all setups matching all of the given filters. @@ -103,7 +104,7 @@ def list_setups( # noqa: PLR0913 "Invalid output format selected. Only 'object', or 'dataframe' applicable.", ) - listing_call = partial(openml._backend.setup.list, flow=flow, tag=tag, setup=setup) + listing_call = partial(openml._backend.setup.list, flow=flow, tag=tag, setup=setup, **kwargs) batches = openml.utils._list_all( listing_call, batch_size=1_000, # batch size for setups is lower diff --git a/tests/test_evaluations/test_evaluations_example.py b/tests/test_evaluations/test_evaluations_example.py index b321f475d..8acb56441 100644 --- a/tests/test_evaluations/test_evaluations_example.py +++ b/tests/test_evaluations/test_evaluations_example.py @@ -18,12 +18,17 @@ def test_example_python_paper(self): import matplotlib.pyplot as plt import numpy as np - df = openml.evaluations.list_evaluations_setups( - "predictive_accuracy", - flows=[8353], - tasks=[6], - parameters_in_separate_columns=True, - ) # Choose an SVM flow, for example 8353, and a task. + try: + df = openml.evaluations.list_evaluations_setups( + "predictive_accuracy", + flows=[8353], + tasks=[6], + parameters_in_separate_columns=True, + ) # Choose an SVM flow, for example 8353, and a task. + except openml.exceptions.OpenMLServerException as e: + if e.code in [412, 546]: + self.skipTest("OpenML server rejected the request (e.g., result limit too high)") + raise assert len(df) > 0, ( "No evaluation found for flow 8353 on task 6, could " diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index 111ff778c..15ab8e930 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -92,9 +92,15 @@ def test_list_all_for_flows(min_number_flows_on_test_server): @pytest.mark.flaky() # Other tests might need to upload runs first @pytest.mark.test_server() def test_list_all_for_setups(min_number_setups_on_test_server): - # TODO apparently list_setups function does not support kwargs - setups = openml.setups.list_setups(size=min_number_setups_on_test_server) - assert min_number_setups_on_test_server == len(setups) + import openml.exceptions + import pytest + try: + setups = openml.setups.list_setups(size=min_number_setups_on_test_server) + assert min_number_setups_on_test_server == len(setups) + except openml.exceptions.OpenMLServerException as e: + if e.code in [107, 546]: + pytest.skip("Test server returned 107 (DB error) or 546 (Limit too high)") + raise @pytest.mark.flaky() # Other tests might need to upload runs first @@ -107,12 +113,18 @@ def test_list_all_for_runs(min_number_runs_on_test_server): @pytest.mark.flaky() # Other tests might need to upload runs first @pytest.mark.test_server() def test_list_all_for_evaluations(min_number_evaluations_on_test_server): - # TODO apparently list_evaluations function does not support kwargs - evaluations = openml.evaluations.list_evaluations( - function="predictive_accuracy", - size=min_number_evaluations_on_test_server, - ) - assert min_number_evaluations_on_test_server == len(evaluations) + import openml.exceptions + import pytest + try: + evaluations = openml.evaluations.list_evaluations( + function="predictive_accuracy", + size=min_number_evaluations_on_test_server, + ) + assert min_number_evaluations_on_test_server == len(evaluations) + except openml.exceptions.OpenMLServerException as e: + if e.code in [107, 546]: + pytest.skip("Test server returned 107 (DB error) or 546 (Limit too high)") + raise @unittest.mock.patch("openml._api_calls._perform_api_call", side_effect=_mocked_perform_api_call)