Feat/add kwargs setups evaluations

openml/_api/resources/base/resources.py

@@ -171,6 +171,7 @@ def list(
         setup: Iterable[int] | None = None,
         flow: int | None = None,
         tag: str | None = None,
+        **kwargs: Any,
     ) -> list[OpenMLSetup]: ...
 
     @abstractmethod

openml/_api/resources/setup.py

@@ -26,6 +26,7 @@ def _build_url(
         setup: Iterable[int] | None = None,
         flow: int | None = None,
         tag: str | None = None,
+        **kwargs: Any,
     ) -> str:
         """Construct an OpenML Setup API URL with filtering parameters.
 
@@ -50,6 +51,10 @@ def _build_url(
             api_call += f"/limit/{limit}"
         if offset is not None:
             api_call += f"/offset/{offset}"
+        if kwargs:
+            for operator, value in kwargs.items():
+                if value is not None:
+                    api_call += f"/{operator}/{value}"
         if setup is not None:
             api_call += f"/setup/{','.join([str(int(i)) for i in setup])}"
         if flow is not None:
@@ -164,6 +169,7 @@ def list(
         setup: Iterable[int] | None = None,
         flow: int | None = None,
         tag: str | None = None,
+        **kwargs: Any,
     ) -> builtins.list[OpenMLSetup]:
         """Perform API call `/setup/list/{filters}`
 
@@ -183,7 +189,7 @@ def list(
         list
             setups that match the filters, going from id to the OpenMLSetup object.
         """
-        api_call = SetupV1API._build_url(limit, offset, setup=setup, flow=flow, tag=tag)
+        api_call = SetupV1API._build_url(limit, offset, setup=setup, flow=flow, tag=tag, **kwargs)
         setup_response = self._http.get(api_call)
         xml_content = setup_response.text
 
@@ -266,6 +272,7 @@ def list(
         setup: Iterable[int] | None = None,  # noqa: ARG002
         flow: int | None = None,  # noqa: ARG002
         tag: str | None = None,  # noqa: ARG002
+        **kwargs: Any,  # noqa: ARG002
     ) -> builtins.list[OpenMLSetup]:
         self._not_supported(method="list")
 

openml/evaluations/functions.py

@@ -4,7 +4,7 @@
 
 from functools import partial
 from itertools import chain
-from typing import TYPE_CHECKING, Literal
+from typing import TYPE_CHECKING, Any, Literal
 from typing_extensions import overload
 
 import numpy as np
@@ -33,6 +33,7 @@ def list_evaluations(
     per_fold: bool | None = None,
     sort_order: str | None = None,
     output_format: Literal["dataframe"] = ...,
+    **kwargs: Any,
 ) -> pd.DataFrame: ...
 
 
@@ -51,6 +52,7 @@ def list_evaluations(
     per_fold: bool | None = None,
     sort_order: str | None = None,
     output_format: Literal["object"] = "object",
+    **kwargs: Any,
 ) -> dict[int, OpenMLEvaluation]: ...
 
 
@@ -68,6 +70,7 @@ def list_evaluations(
     per_fold: bool | None = None,
     sort_order: str | None = None,
     output_format: Literal["object", "dataframe"] = "object",
+    **kwargs: Any,
 ) -> dict[int, OpenMLEvaluation] | pd.DataFrame:
     """List all run-evaluation pairs matching all of the given filters.
 
@@ -131,6 +134,7 @@ def list_evaluations(
         study=study,
         sort_order=sort_order,
         per_fold=per_fold_str,
+        **kwargs,
     )
     eval_collection = openml.utils._list_all(listing_call, offset=offset, limit=size)
 

openml/setups/functions.py

@@ -78,6 +78,7 @@ def list_setups(  # noqa: PLR0913
     tag: str | None = None,
     setup: Iterable[int] | None = None,
     output_format: Literal["object", "dataframe"] = "object",
+    **kwargs: Any,
 ) -> dict[int, OpenMLSetup] | pd.DataFrame:
     """
     List all setups matching all of the given filters.
@@ -103,7 +104,7 @@ def list_setups(  # noqa: PLR0913
             "Invalid output format selected. Only 'object', or 'dataframe' applicable.",
         )
 
-    listing_call = partial(openml._backend.setup.list, flow=flow, tag=tag, setup=setup)
+    listing_call = partial(openml._backend.setup.list, flow=flow, tag=tag, setup=setup, **kwargs)
     batches = openml.utils._list_all(
         listing_call,
         batch_size=1_000,  # batch size for setups is lower

tests/test_evaluations/test_evaluations_example.py

@@ -18,12 +18,17 @@ def test_example_python_paper(self):
             import matplotlib.pyplot as plt
             import numpy as np
 
-            df = openml.evaluations.list_evaluations_setups(
-                "predictive_accuracy",
-                flows=[8353],
-                tasks=[6],
-                parameters_in_separate_columns=True,
-            )  # Choose an SVM flow, for example 8353, and a task.
+            try:
+                df = openml.evaluations.list_evaluations_setups(
+                    "predictive_accuracy",
+                    flows=[8353],
+                    tasks=[6],
+                    parameters_in_separate_columns=True,
+                )  # Choose an SVM flow, for example 8353, and a task.
+            except openml.exceptions.OpenMLServerException as e:
+                if e.code in [412, 546]:
+                    self.skipTest("OpenML server rejected the request (e.g., result limit too high)")
+                raise
 
             assert len(df) > 0, (
                 "No evaluation found for flow 8353 on task 6, could "

tests/test_utils/test_utils.py

@@ -92,9 +92,15 @@ def test_list_all_for_flows(min_number_flows_on_test_server):
 @pytest.mark.flaky()  # Other tests might need to upload runs first
 @pytest.mark.test_server()
 def test_list_all_for_setups(min_number_setups_on_test_server):
-    # TODO apparently list_setups function does not support kwargs
-    setups = openml.setups.list_setups(size=min_number_setups_on_test_server)
-    assert min_number_setups_on_test_server == len(setups)
+    import openml.exceptions
+    import pytest
+    try:
+        setups = openml.setups.list_setups(size=min_number_setups_on_test_server)
+        assert min_number_setups_on_test_server == len(setups)
+    except openml.exceptions.OpenMLServerException as e:
+        if e.code in [107, 546]:
+            pytest.skip("Test server returned 107 (DB error) or 546 (Limit too high)")
+        raise
 
 
 @pytest.mark.flaky()  # Other tests might need to upload runs first
@@ -107,12 +113,18 @@ def test_list_all_for_runs(min_number_runs_on_test_server):
 @pytest.mark.flaky()  # Other tests might need to upload runs first
 @pytest.mark.test_server()
 def test_list_all_for_evaluations(min_number_evaluations_on_test_server):
-    # TODO apparently list_evaluations function does not support kwargs
-    evaluations = openml.evaluations.list_evaluations(
-        function="predictive_accuracy",
-        size=min_number_evaluations_on_test_server,
-    )
-    assert min_number_evaluations_on_test_server == len(evaluations)
+    import openml.exceptions
+    import pytest
+    try:
+        evaluations = openml.evaluations.list_evaluations(
+            function="predictive_accuracy",
+            size=min_number_evaluations_on_test_server,
+        )
+        assert min_number_evaluations_on_test_server == len(evaluations)
+    except openml.exceptions.OpenMLServerException as e:
+        if e.code in [107, 546]:
+            pytest.skip("Test server returned 107 (DB error) or 546 (Limit too high)")
+        raise
 
 
 @unittest.mock.patch("openml._api_calls._perform_api_call", side_effect=_mocked_perform_api_call)