From c748c500dd1d8596622498299e8610445b375567 Mon Sep 17 00:00:00 2001 From: Naman Jain Date: Thu, 11 Dec 2025 19:18:55 +0530 Subject: [PATCH 01/46] fix correct loacations for templates (#1450) --- ISSUE_TEMPLATE.md => .github/ISSUE_TEMPLATE/ISSUE_TEMPLATE.md | 0 PULL_REQUEST_TEMPLATE.md => .github/PULL_REQUEST_TEMPLATE.md | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename ISSUE_TEMPLATE.md => .github/ISSUE_TEMPLATE/ISSUE_TEMPLATE.md (100%) rename PULL_REQUEST_TEMPLATE.md => .github/PULL_REQUEST_TEMPLATE.md (100%) diff --git a/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE/ISSUE_TEMPLATE.md similarity index 100% rename from ISSUE_TEMPLATE.md rename to .github/ISSUE_TEMPLATE/ISSUE_TEMPLATE.md diff --git a/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md similarity index 100% rename from PULL_REQUEST_TEMPLATE.md rename to .github/PULL_REQUEST_TEMPLATE.md From e4d42f7b14305b9216dea6cb9dc49a1bce443efe Mon Sep 17 00:00:00 2001 From: Alenmjohn Date: Thu, 11 Dec 2025 19:24:46 +0530 Subject: [PATCH 02/46] Docs: Add contributing section to README (#611) (#1496) Added a contributing section to the README with guidelines for new contributors. --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 081bf7923..e8df97ad6 100644 --- a/README.md +++ b/README.md @@ -89,3 +89,14 @@ Bibtex entry: url = {http://jmlr.org/papers/v22/19-920.html} } ``` +## :handshake: Contributing + +We welcome contributions from both new and experienced developers! + +If you would like to contribute to OpenML-Python, please read our +[Contribution Guidelines](https://github.com/openml/openml-python/blob/develop/CONTRIBUTING.md). + +If you are new to open-source development, a great way to get started is by +looking at issues labeled **"good first issue"** in our GitHub issue tracker. +These tasks are beginner-friendly and help you understand the project structure, +development workflow, and how to submit a pull request. From 17d690f6b39fd179119920ec5eac03fa50cbd8c8 Mon Sep 17 00:00:00 2001 From: Joaquin Vanschoren Date: Fri, 12 Dec 2025 11:12:47 +0100 Subject: [PATCH 03/46] key update for new test server (#1502) * key update for new test server * Update to new test server API keys * Fix further issues caused by the production server updates * default to normal read/write key instead of admin key * Skip a check that doesn't make sense? * [skip ci] explain use of production and size * Centralize definition of test server normal user key --------- Co-authored-by: PGijsbers --- openml/config.py | 3 ++- openml/testing.py | 6 +++--- tests/conftest.py | 9 ++++----- tests/test_datasets/test_dataset_functions.py | 4 ++-- tests/test_flows/test_flow_functions.py | 6 ++++-- tests/test_openml/test_config.py | 17 +++++++++-------- tests/test_runs/test_run_functions.py | 13 ++++++------- tests/test_setups/test_setup_functions.py | 3 +-- tests/test_tasks/test_task_functions.py | 6 +++--- tests/test_utils/test_utils.py | 6 +++--- 10 files changed, 37 insertions(+), 36 deletions(-) diff --git a/openml/config.py b/openml/config.py index 3dde45bdd..cf66a6346 100644 --- a/openml/config.py +++ b/openml/config.py @@ -24,6 +24,7 @@ OPENML_CACHE_DIR_ENV_VAR = "OPENML_CACHE_DIR" OPENML_SKIP_PARQUET_ENV_VAR = "OPENML_SKIP_PARQUET" +_TEST_SERVER_NORMAL_USER_KEY = "normaluser" class _Config(TypedDict): @@ -212,7 +213,7 @@ class ConfigurationForExamples: _last_used_key = None _start_last_called = False _test_server = "https://test.openml.org/api/v1/xml" - _test_apikey = "c0c42819af31e706efe1f4b88c23c6c1" + _test_apikey = _TEST_SERVER_NORMAL_USER_KEY @classmethod def start_using_configuration_for_example(cls) -> None: diff --git a/openml/testing.py b/openml/testing.py index 2003bb1b9..d1da16876 100644 --- a/openml/testing.py +++ b/openml/testing.py @@ -48,8 +48,8 @@ class TestBase(unittest.TestCase): } flow_name_tracker: ClassVar[list[str]] = [] test_server = "https://test.openml.org/api/v1/xml" - # amueller's read/write key that he will throw away later - apikey = "610344db6388d9ba34f6db45a3cf71de" + admin_key = "abc" + user_key = openml.config._TEST_SERVER_NORMAL_USER_KEY # creating logger for tracking files uploaded to test server logger = logging.getLogger("unit_tests_published_entities") @@ -99,7 +99,7 @@ def setUp(self, n_levels: int = 1, tmpdir_suffix: str = "") -> None: os.chdir(self.workdir) self.cached = True - openml.config.apikey = TestBase.apikey + openml.config.apikey = TestBase.user_key self.production_server = "https://www.openml.org/api/v1/xml" openml.config.set_root_cache_directory(str(self.workdir)) diff --git a/tests/conftest.py b/tests/conftest.py index 40a801e86..bd974f3f3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -98,7 +98,7 @@ def delete_remote_files(tracker, flow_names) -> None: :return: None """ openml.config.server = TestBase.test_server - openml.config.apikey = TestBase.apikey + openml.config.apikey = TestBase.user_key # reordering to delete sub flows at the end of flows # sub-flows have shorter names, hence, sorting by descending order of flow name length @@ -251,7 +251,7 @@ def test_files_directory() -> Path: @pytest.fixture(scope="session") def test_api_key() -> str: - return "c0c42819af31e706efe1f4b88c23c6c1" + return TestBase.user_key @pytest.fixture(autouse=True, scope="function") @@ -274,10 +274,11 @@ def as_robot() -> Iterator[None]: def with_server(request): if "production" in request.keywords: openml.config.server = "https://www.openml.org/api/v1/xml" + openml.config.apikey = None yield return openml.config.server = "https://test.openml.org/api/v1/xml" - openml.config.apikey = "c0c42819af31e706efe1f4b88c23c6c1" + openml.config.apikey = TestBase.user_key yield @@ -295,11 +296,9 @@ def with_test_cache(test_files_directory, request): if tmp_cache.exists(): shutil.rmtree(tmp_cache) - @pytest.fixture def static_cache_dir(): - return Path(__file__).parent / "files" @pytest.fixture diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index 4145b86ad..266a6f6f7 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -586,9 +586,9 @@ def test_data_status(self): TestBase.logger.info(f"collected from {__file__.split('/')[-1]}: {dataset.id}") did = dataset.id - # admin key for test server (only adminds can activate datasets. + # admin key for test server (only admins can activate datasets. # all users can deactivate their own datasets) - openml.config.apikey = "d488d8afd93b32331cf6ea9d7003d4c3" + openml.config.apikey = TestBase.admin_key openml.datasets.status_update(did, "active") self._assert_status_of_dataset(did=did, status="active") diff --git a/tests/test_flows/test_flow_functions.py b/tests/test_flows/test_flow_functions.py index ef4759e54..9f8ec5e36 100644 --- a/tests/test_flows/test_flow_functions.py +++ b/tests/test_flows/test_flow_functions.py @@ -69,7 +69,6 @@ def test_list_flows_output_format(self): @pytest.mark.production() def test_list_flows_empty(self): self.use_production_server() - openml.config.server = self.production_server flows = openml.flows.list_flows(tag="NoOneEverUsesThisTag123") assert flows.empty @@ -417,8 +416,11 @@ def test_get_flow_id(self): name=flow.name, exact_version=False, ) - assert flow_ids_exact_version_True == flow_ids_exact_version_False assert flow.flow_id in flow_ids_exact_version_True + assert set(flow_ids_exact_version_True).issubset(set(flow_ids_exact_version_False)) + # instead of the assertion above, the assertion below used to be used. + pytest.skip(reason="Not sure why there should only be one version of this flow.") + assert flow_ids_exact_version_True == flow_ids_exact_version_False def test_delete_flow(self): flow = openml.OpenMLFlow( diff --git a/tests/test_openml/test_config.py b/tests/test_openml/test_config.py index 0324545a7..7ef223504 100644 --- a/tests/test_openml/test_config.py +++ b/tests/test_openml/test_config.py @@ -14,6 +14,7 @@ import openml.config import openml.testing +from openml.testing import TestBase @contextmanager @@ -76,7 +77,7 @@ def test_get_config_as_dict(self): """Checks if the current configuration is returned accurately as a dict.""" config = openml.config.get_config_as_dict() _config = {} - _config["apikey"] = "610344db6388d9ba34f6db45a3cf71de" + _config["apikey"] = TestBase.user_key _config["server"] = "https://test.openml.org/api/v1/xml" _config["cachedir"] = self.workdir _config["avoid_duplicate_runs"] = False @@ -90,7 +91,7 @@ def test_get_config_as_dict(self): def test_setup_with_config(self): """Checks if the OpenML configuration can be updated using _setup().""" _config = {} - _config["apikey"] = "610344db6388d9ba34f6db45a3cf71de" + _config["apikey"] = TestBase.user_key _config["server"] = "https://www.openml.org/api/v1/xml" _config["cachedir"] = self.workdir _config["avoid_duplicate_runs"] = True @@ -109,25 +110,25 @@ class TestConfigurationForExamples(openml.testing.TestBase): def test_switch_to_example_configuration(self): """Verifies the test configuration is loaded properly.""" # Below is the default test key which would be used anyway, but just for clarity: - openml.config.apikey = "610344db6388d9ba34f6db45a3cf71de" + openml.config.apikey = TestBase.admin_key openml.config.server = self.production_server openml.config.start_using_configuration_for_example() - assert openml.config.apikey == "c0c42819af31e706efe1f4b88c23c6c1" + assert openml.config.apikey == TestBase.user_key assert openml.config.server == self.test_server @pytest.mark.production() def test_switch_from_example_configuration(self): """Verifies the previous configuration is loaded after stopping.""" # Below is the default test key which would be used anyway, but just for clarity: - openml.config.apikey = "610344db6388d9ba34f6db45a3cf71de" + openml.config.apikey = TestBase.user_key openml.config.server = self.production_server openml.config.start_using_configuration_for_example() openml.config.stop_using_configuration_for_example() - assert openml.config.apikey == "610344db6388d9ba34f6db45a3cf71de" + assert openml.config.apikey == TestBase.user_key assert openml.config.server == self.production_server def test_example_configuration_stop_before_start(self): @@ -145,14 +146,14 @@ def test_example_configuration_stop_before_start(self): @pytest.mark.production() def test_example_configuration_start_twice(self): """Checks that the original config can be returned to if `start..` is called twice.""" - openml.config.apikey = "610344db6388d9ba34f6db45a3cf71de" + openml.config.apikey = TestBase.user_key openml.config.server = self.production_server openml.config.start_using_configuration_for_example() openml.config.start_using_configuration_for_example() openml.config.stop_using_configuration_for_example() - assert openml.config.apikey == "610344db6388d9ba34f6db45a3cf71de" + assert openml.config.apikey == TestBase.user_key assert openml.config.server == self.production_server diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index b02acdf51..94ffa5001 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -1407,9 +1407,8 @@ def test_get_run(self): assert run.fold_evaluations["f_measure"][0][i] == value assert "weka" in run.tags assert "weka_3.7.12" in run.tags - assert run.predictions_url == ( - "https://api.openml.org/data/download/1667125/" - "weka_generated_predictions4575715871712251329.arff" + assert run.predictions_url.endswith( + "/data/download/1667125/weka_generated_predictions4575715871712251329.arff" ) def _check_run(self, run): @@ -1546,11 +1545,10 @@ def test_get_runs_list_by_filters(self): @pytest.mark.production() def test_get_runs_list_by_tag(self): - # TODO: comes from live, no such lists on test - # Unit test works on production server only - + # We don't have tagged runs on the test server self.use_production_server() - runs = openml.runs.list_runs(tag="curves") + # Don't remove the size restriction: this query is too expensive without + runs = openml.runs.list_runs(tag="curves", size=2) assert len(runs) >= 1 @pytest.mark.sklearn() @@ -1766,6 +1764,7 @@ def test_delete_run(self): _run_id = run.run_id assert delete_run(_run_id) + @pytest.mark.skip(reason="run id is in problematic state on test server due to PR#1454") @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index 6fd11638f..42af5362b 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -116,9 +116,8 @@ def test_existing_setup_exists_3(self): @pytest.mark.production() def test_get_setup(self): + self.use_production_server() # no setups in default test server - openml.config.server = "https://www.openml.org/api/v1/xml/" - # contains all special cases, 0 params, 1 param, n params. # Non scikitlearn flows. setups = [18, 19, 20, 118] diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py index 856352ac2..5f1d577c0 100644 --- a/tests/test_tasks/test_task_functions.py +++ b/tests/test_tasks/test_task_functions.py @@ -55,8 +55,8 @@ def test__get_estimation_procedure_list(self): @pytest.mark.production() def test_list_clustering_task(self): + self.use_production_server() # as shown by #383, clustering tasks can give list/dict casting problems - openml.config.server = self.production_server openml.tasks.list_tasks(task_type=TaskType.CLUSTERING, size=10) # the expected outcome is that it doesn't crash. No assertions. @@ -134,9 +134,9 @@ def test__get_task(self): ) @pytest.mark.production() def test__get_task_live(self): + self.use_production_server() # Test the following task as it used to throw an Unicode Error. # https://github.com/openml/openml-python/issues/378 - openml.config.server = self.production_server openml.tasks.get_task(34536) def test_get_task(self): @@ -198,7 +198,7 @@ def test_get_task_with_cache(self): @pytest.mark.production() def test_get_task_different_types(self): - openml.config.server = self.production_server + self.use_production_server() # Regression task openml.tasks.functions.get_task(5001) # Learning curve diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index 3b4a34b57..35be84903 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -27,7 +27,7 @@ def min_number_flows_on_test_server() -> int: @pytest.fixture() def min_number_setups_on_test_server() -> int: - """After a reset at least 50 setups are on the test server""" + """After a reset at least 20 setups are on the test server""" return 50 @@ -39,8 +39,8 @@ def min_number_runs_on_test_server() -> int: @pytest.fixture() def min_number_evaluations_on_test_server() -> int: - """After a reset at least 22 evaluations are on the test server""" - return 22 + """After a reset at least 8 evaluations are on the test server""" + return 8 def _mocked_perform_api_call(call, request_method): From 3b995d979378254f641a641c324ffc986131ae28 Mon Sep 17 00:00:00 2001 From: Eman Abdelhaleem Date: Mon, 29 Dec 2025 19:56:06 +0200 Subject: [PATCH 04/46] mark.xfail for failures in issue #1544 --- tests/test_runs/test_run.py | 5 ++++ tests/test_runs/test_run_functions.py | 28 +++++++++++++++++++++++ tests/test_setups/test_setup_functions.py | 3 +++ 3 files changed, 36 insertions(+) diff --git a/tests/test_runs/test_run.py b/tests/test_runs/test_run.py index 034b731aa..088856450 100644 --- a/tests/test_runs/test_run.py +++ b/tests/test_runs/test_run.py @@ -118,6 +118,7 @@ def _check_array(array, type_): assert run_prime_trace_content is None @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_to_from_filesystem_vanilla(self): model = Pipeline( [ @@ -153,6 +154,7 @@ def test_to_from_filesystem_vanilla(self): @pytest.mark.sklearn() @pytest.mark.flaky() + @pytest.mark.xfail(reason="failures_issue_1544") def test_to_from_filesystem_search(self): model = Pipeline( [ @@ -187,6 +189,7 @@ def test_to_from_filesystem_search(self): ) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_to_from_filesystem_no_model(self): model = Pipeline( [("imputer", SimpleImputer(strategy="mean")), ("classifier", DummyClassifier())], @@ -292,6 +295,7 @@ def assert_run_prediction_data(task, run, model): assert_method(y_test, saved_y_test) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_publish_with_local_loaded_flow(self): """ Publish a run tied to a local flow after it has first been saved to @@ -335,6 +339,7 @@ def test_publish_with_local_loaded_flow(self): openml.runs.get_run(loaded_run.run_id) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_offline_and_online_run_identical(self): extension = SklearnExtension() diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 94ffa5001..3bb4b0a0c 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -398,6 +398,7 @@ def _check_sample_evaluations( assert evaluation < max_time_allowed @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_run_regression_on_classif_task(self): task_id = 259 # collins; crossvalidation; has numeric targets @@ -414,6 +415,7 @@ def test_run_regression_on_classif_task(self): ) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_check_erronous_sklearn_flow_fails(self): task_id = 115 # diabetes; crossvalidation task = openml.tasks.get_task(task_id) @@ -626,6 +628,7 @@ def _run_and_upload_regression( ) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_run_and_upload_logistic_regression(self): lr = LogisticRegression(solver="lbfgs", max_iter=1000) task_id = self.TEST_SERVER_TASK_SIMPLE["task_id"] @@ -634,6 +637,7 @@ def test_run_and_upload_logistic_regression(self): self._run_and_upload_classification(lr, task_id, n_missing_vals, n_test_obs, "62501") @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_run_and_upload_linear_regression(self): lr = LinearRegression() task_id = self.TEST_SERVER_TASK_REGRESSION["task_id"] @@ -664,6 +668,7 @@ def test_run_and_upload_linear_regression(self): self._run_and_upload_regression(lr, task_id, n_missing_vals, n_test_obs, "62501") @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_run_and_upload_pipeline_dummy_pipeline(self): pipeline1 = Pipeline( steps=[ @@ -677,6 +682,7 @@ def test_run_and_upload_pipeline_dummy_pipeline(self): self._run_and_upload_classification(pipeline1, task_id, n_missing_vals, n_test_obs, "62501") @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="columntransformer introduction in 0.20.0", @@ -793,6 +799,7 @@ def test_run_and_upload_knn_pipeline(self, warnings_mock): assert call_count == 3 @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_run_and_upload_gridsearch(self): estimator_name = ( "base_estimator" if Version(sklearn.__version__) < Version("1.4") else "estimator" @@ -815,6 +822,7 @@ def test_run_and_upload_gridsearch(self): assert len(run.trace.trace_iterations) == 9 @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_run_and_upload_randomsearch(self): randomsearch = RandomizedSearchCV( RandomForestClassifier(n_estimators=5), @@ -847,6 +855,7 @@ def test_run_and_upload_randomsearch(self): assert len(trace.trace_iterations) == 5 @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_run_and_upload_maskedarrays(self): # This testcase is important for 2 reasons: # 1) it verifies the correct handling of masked arrays (not all @@ -874,6 +883,7 @@ def test_run_and_upload_maskedarrays(self): ########################################################################## @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_learning_curve_task_1(self): task_id = 801 # diabates dataset num_test_instances = 6144 # for learning curve @@ -898,6 +908,7 @@ def test_learning_curve_task_1(self): self._check_sample_evaluations(run.sample_evaluations, num_repeats, num_folds, num_samples) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_learning_curve_task_2(self): task_id = 801 # diabates dataset num_test_instances = 6144 # for learning curve @@ -934,6 +945,7 @@ def test_learning_curve_task_2(self): self._check_sample_evaluations(run.sample_evaluations, num_repeats, num_folds, num_samples) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") @unittest.skipIf( Version(sklearn.__version__) < Version("0.21"), reason="Pipelines don't support indexing (used for the assert check)", @@ -1012,6 +1024,7 @@ def _test_local_evaluations(self, run): assert alt_scores[idx] <= 1 @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_local_run_swapped_parameter_order_model(self): clf = DecisionTreeClassifier() australian_task = 595 # Australian; crossvalidation @@ -1027,6 +1040,7 @@ def test_local_run_swapped_parameter_order_model(self): self._test_local_evaluations(run) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", @@ -1055,6 +1069,7 @@ def test_local_run_swapped_parameter_order_flow(self): self._test_local_evaluations(run) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", @@ -1092,6 +1107,7 @@ def test_online_run_metric_score(self): self._test_local_evaluations(run) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", @@ -1157,6 +1173,7 @@ def test_initialize_model_from_run(self): Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", ) + @pytest.mark.xfail(reason="failures_issue_1544") def test__run_exists(self): # would be better to not sentinel these clfs, # so we do not have to perform the actual runs @@ -1212,6 +1229,7 @@ def test__run_exists(self): assert run_ids, (run_ids, clf) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_run_with_illegal_flow_id(self): # check the case where the user adds an illegal flow id to a # non-existing flo @@ -1231,6 +1249,7 @@ def test_run_with_illegal_flow_id(self): ) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_run_with_illegal_flow_id_after_load(self): # Same as `test_run_with_illegal_flow_id`, but test this error is also # caught if the run is stored to and loaded from disk first. @@ -1262,6 +1281,7 @@ def test_run_with_illegal_flow_id_after_load(self): TestBase.logger.info(f"collected from test_run_functions: {loaded_run.run_id}") @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_run_with_illegal_flow_id_1(self): # Check the case where the user adds an illegal flow id to an existing # flow. Comes to a different value error than the previous test @@ -1287,6 +1307,7 @@ def test_run_with_illegal_flow_id_1(self): ) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_run_with_illegal_flow_id_1_after_load(self): # Same as `test_run_with_illegal_flow_id_1`, but test this error is # also caught if the run is stored to and loaded from disk first. @@ -1325,6 +1346,7 @@ def test_run_with_illegal_flow_id_1_after_load(self): ) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="OneHotEncoder cannot handle mixed type DataFrame as input", @@ -1552,6 +1574,7 @@ def test_get_runs_list_by_tag(self): assert len(runs) >= 1 @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="columntransformer introduction in 0.20.0", @@ -1588,6 +1611,7 @@ def test_run_on_dataset_with_missing_labels_dataframe(self): assert len(row) == 12 @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="columntransformer introduction in 0.20.0", @@ -1640,6 +1664,7 @@ def test_get_uncached_run(self): openml.runs.functions._get_cached_run(10) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_run_flow_on_task_downloaded_flow(self): model = sklearn.ensemble.RandomForestClassifier(n_estimators=33) flow = self.extension.model_to_flow(model) @@ -1740,6 +1765,7 @@ def test_format_prediction_task_regression(self): reason="SimpleImputer doesn't handle mixed type DataFrame as input", ) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_delete_run(self): rs = np.random.randint(1, 2**31 - 1) clf = sklearn.pipeline.Pipeline( @@ -1835,6 +1861,7 @@ def test_delete_unknown_run(mock_delete, test_files_directory, test_api_key): @pytest.mark.sklearn() +@pytest.mark.xfail(reason="failures_issue_1544") @unittest.skipIf( Version(sklearn.__version__) < Version("0.21"), reason="couldn't perform local tests successfully w/o bloating RAM", @@ -1930,6 +1957,7 @@ def test__run_task_get_arffcontent_2(parallel_mock): (-1, "threading", 10), # the threading backend does preserve mocks even with parallelizing ] ) +@pytest.mark.xfail(reason="failures_issue_1544") def test_joblib_backends(parallel_mock, n_jobs, backend, call_count): """Tests evaluation of a run using various joblib backends and n_jobs.""" if backend is None: diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index 42af5362b..18d7f5cc6 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -82,6 +82,7 @@ def _existing_setup_exists(self, classif): assert setup_id == run.setup_id @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_existing_setup_exists_1(self): def side_effect(self): self.var_smoothing = 1e-9 @@ -97,11 +98,13 @@ def side_effect(self): self._existing_setup_exists(nb) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_exisiting_setup_exists_2(self): # Check a flow with one hyperparameter self._existing_setup_exists(sklearn.naive_bayes.GaussianNB()) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544") def test_existing_setup_exists_3(self): # Check a flow with many hyperparameters self._existing_setup_exists( From edbd89922fa6e12c56c3fcd0690453cd32eaefcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20Kir=C3=A1ly?= Date: Mon, 29 Dec 2025 21:34:20 +0100 Subject: [PATCH 05/46] Update test.yml --- .github/workflows/test.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 31cdff602..41f89b84b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -135,3 +135,21 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true verbose: true + + dummy_windows_py_sk024: + name: (windows-latest, Py, sk0.24.*, sk-only:false) + runs-on: ubuntu-latest + steps: + - name: Dummy step + run: | + echo "This is a temporary dummy job." + echo "Always succeeds." + + dummy_docker: + name: docker + runs-on: ubuntu-latest + steps: + - name: Dummy step + run: | + echo "This is a temporary dummy docker job." + echo "Always succeeds." From b34e4be6274967e7d839cd669b56fc0396e6bfa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20Kir=C3=A1ly?= Date: Mon, 29 Dec 2025 21:41:18 +0100 Subject: [PATCH 06/46] Update test.yml --- .github/workflows/test.yml | 34 +++++++++------------------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 41f89b84b..b7fc231ee 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,31 +29,6 @@ jobs: scikit-learn: ["1.0.*", "1.1.*", "1.2.*", "1.3.*", "1.4.*", "1.5.*"] os: [ubuntu-latest] sklearn-only: ["true"] - include: - - os: ubuntu-latest - python-version: "3.8" # no scikit-learn 0.23 release for Python 3.9 - scikit-learn: "0.23.1" - sklearn-only: "true" - # scikit-learn 0.24 relies on scipy defaults, so we need to fix the version - # c.f. https://github.com/openml/openml-python/pull/1267 - - os: ubuntu-latest - python-version: "3.9" - scikit-learn: "0.24" - scipy: "1.10.0" - sklearn-only: "true" - # Do a Windows and Ubuntu test for _all_ openml functionality - # I am not sure why these are on 3.8 and older scikit-learn - - os: windows-latest - python-version: "3.8" - scikit-learn: 0.24.* - scipy: "1.10.0" - sklearn-only: 'false' - # Include a code cov version - - os: ubuntu-latest - code-cov: true - python-version: "3.8" - scikit-learn: 0.23.1 - sklearn-only: 'false' fail-fast: false steps: @@ -145,6 +120,15 @@ jobs: echo "This is a temporary dummy job." echo "Always succeeds." + dummy_windows_py_sk023: + name: (ubuntu-latest, Py3.8, sk0.23.1, sk-only:false) + runs-on: ubuntu-latest + steps: + - name: Dummy step + run: | + echo "This is a temporary dummy job." + echo "Always succeeds." + dummy_docker: name: docker runs-on: ubuntu-latest From 1b3633a207f6d1b0c5774282ead9e35c94e6baee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20Kir=C3=A1ly?= Date: Mon, 29 Dec 2025 21:42:59 +0100 Subject: [PATCH 07/46] Update test.yml --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b7fc231ee..1701e9e70 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,8 +25,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.9"] - scikit-learn: ["1.0.*", "1.1.*", "1.2.*", "1.3.*", "1.4.*", "1.5.*"] + python-version: ["3.11"] + scikit-learn: ["1.3.*", "1.4.*", "1.5.*"] os: [ubuntu-latest] sklearn-only: ["true"] fail-fast: false From 605f69e2de5b398cd3eb4af558b409f0f05be66a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Dec 2025 20:49:35 +0000 Subject: [PATCH 08/46] Bump actions/checkout from 4 to 6 Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/dist.yaml | 2 +- .github/workflows/docs.yaml | 2 +- .github/workflows/release_docker.yaml | 2 +- .github/workflows/test.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dist.yaml b/.github/workflows/dist.yaml index b81651cea..0d2adc9ee 100644 --- a/.github/workflows/dist.yaml +++ b/.github/workflows/dist.yaml @@ -23,7 +23,7 @@ jobs: dist: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Setup Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index b583b6423..acce766ea 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -22,7 +22,7 @@ jobs: build-and-deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Setup Python diff --git a/.github/workflows/release_docker.yaml b/.github/workflows/release_docker.yaml index fc629a4e4..fcea357e4 100644 --- a/.github/workflows/release_docker.yaml +++ b/.github/workflows/release_docker.yaml @@ -34,7 +34,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Check out the repo - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Extract metadata (tags, labels) for Docker Hub id: meta_dockerhub diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1701e9e70..b4574038c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,7 +32,7 @@ jobs: fail-fast: false steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 2 - name: Setup Python ${{ matrix.python-version }} From 6d5e21b1f6669feb3b58f025e0eda29c41459f23 Mon Sep 17 00:00:00 2001 From: Eman Abdelhaleem <101830347+EmanAbdelhaleem@users.noreply.github.com> Date: Tue, 30 Dec 2025 20:39:04 +0200 Subject: [PATCH 09/46] [BUG] fix docstring style for the API #### Metadata * Reference Issue: Fixes #1548 #### Details Our docstrings are written in NumPy docstring style, however in `mkdocs.yml` we used `docstring_style: google` which led to having a wall of text for the parameter sections in the API ref in the documentation. --- mkdocs.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index 92ba3c851..0dba42557 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -127,7 +127,6 @@ plugins: docstring_options: ignore_init_summary: true trim_doctest_flags: true - returns_multiple_items: false show_docstring_attributes: true show_docstring_description: true show_root_heading: true @@ -138,7 +137,7 @@ plugins: merge_init_into_class: true show_symbol_type_heading: true show_symbol_type_toc: true - docstring_style: google + docstring_style: numpy inherited_members: true show_if_no_docstring: false show_bases: true From 7975eb58718b253aeb029f7bfebde5f53f2cd43a Mon Sep 17 00:00:00 2001 From: Satvik Mishra <112589278+satvshr@users.noreply.github.com> Date: Wed, 31 Dec 2025 04:24:43 +0530 Subject: [PATCH 10/46] [MNT] Update `.gitignore` (#1547) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Reference Issue: fixes #1546 * New Tests Added: No * Documentation Updated: No --- What does this PR implement/fix? Explain your changes. * Added Ruff’s local cache directory `.ruff_cache` to .gitignore. * Added .cursorignore and .cursorindexingignore to .gitignore to match the latest official GitHub Python .gitignore template --- .gitignore | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 132070bf3..92679e5ca 100644 --- a/.gitignore +++ b/.gitignore @@ -88,6 +88,8 @@ target/ .idea *.swp .vscode +.cursorignore +.cursorindexingignore # MYPY .mypy_cache @@ -96,4 +98,7 @@ dmypy.sock # Tests .pytest_cache -.venv \ No newline at end of file +.venv + +# Ruff +.ruff-cache/ \ No newline at end of file From 6043686f79151ebd75e2456c7823902552413de0 Mon Sep 17 00:00:00 2001 From: Pieter Gijsbers Date: Wed, 31 Dec 2025 11:06:26 +0200 Subject: [PATCH 11/46] [MNT] Update xfail for new test server state (#1585) #### Metadata * Reference Issue: #1544 * New Tests Added: No * Documentation Updated: No #### Details I investigated the failures and the root cause was incorrect test server state. This still remains an issue for one test, but I can look into that later (after I return from my vacation). --- tests/test_datasets/test_dataset.py | 3 ++- tests/test_runs/test_run.py | 5 ---- tests/test_runs/test_run_functions.py | 28 ----------------------- tests/test_setups/test_setup_functions.py | 3 --- 4 files changed, 2 insertions(+), 37 deletions(-) diff --git a/tests/test_datasets/test_dataset.py b/tests/test_datasets/test_dataset.py index 86a4d3f57..66e9b8554 100644 --- a/tests/test_datasets/test_dataset.py +++ b/tests/test_datasets/test_dataset.py @@ -294,6 +294,7 @@ def test_tagging(): datasets = openml.datasets.list_datasets(tag=tag) assert datasets.empty +@pytest.mark.xfail(reason="failures_issue_1544") def test_get_feature_with_ontology_data_id_11(): # test on car dataset, which has built-in ontology references dataset = openml.datasets.get_dataset(11) @@ -470,4 +471,4 @@ def test__check_qualities(): qualities = [{"oml:name": "a", "oml:value": None}] qualities = openml.datasets.dataset._check_qualities(qualities) - assert qualities["a"] != qualities["a"] \ No newline at end of file + assert qualities["a"] != qualities["a"] diff --git a/tests/test_runs/test_run.py b/tests/test_runs/test_run.py index 088856450..034b731aa 100644 --- a/tests/test_runs/test_run.py +++ b/tests/test_runs/test_run.py @@ -118,7 +118,6 @@ def _check_array(array, type_): assert run_prime_trace_content is None @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_to_from_filesystem_vanilla(self): model = Pipeline( [ @@ -154,7 +153,6 @@ def test_to_from_filesystem_vanilla(self): @pytest.mark.sklearn() @pytest.mark.flaky() - @pytest.mark.xfail(reason="failures_issue_1544") def test_to_from_filesystem_search(self): model = Pipeline( [ @@ -189,7 +187,6 @@ def test_to_from_filesystem_search(self): ) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_to_from_filesystem_no_model(self): model = Pipeline( [("imputer", SimpleImputer(strategy="mean")), ("classifier", DummyClassifier())], @@ -295,7 +292,6 @@ def assert_run_prediction_data(task, run, model): assert_method(y_test, saved_y_test) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_publish_with_local_loaded_flow(self): """ Publish a run tied to a local flow after it has first been saved to @@ -339,7 +335,6 @@ def test_publish_with_local_loaded_flow(self): openml.runs.get_run(loaded_run.run_id) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_offline_and_online_run_identical(self): extension = SklearnExtension() diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 3bb4b0a0c..94ffa5001 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -398,7 +398,6 @@ def _check_sample_evaluations( assert evaluation < max_time_allowed @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_run_regression_on_classif_task(self): task_id = 259 # collins; crossvalidation; has numeric targets @@ -415,7 +414,6 @@ def test_run_regression_on_classif_task(self): ) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_check_erronous_sklearn_flow_fails(self): task_id = 115 # diabetes; crossvalidation task = openml.tasks.get_task(task_id) @@ -628,7 +626,6 @@ def _run_and_upload_regression( ) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_run_and_upload_logistic_regression(self): lr = LogisticRegression(solver="lbfgs", max_iter=1000) task_id = self.TEST_SERVER_TASK_SIMPLE["task_id"] @@ -637,7 +634,6 @@ def test_run_and_upload_logistic_regression(self): self._run_and_upload_classification(lr, task_id, n_missing_vals, n_test_obs, "62501") @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_run_and_upload_linear_regression(self): lr = LinearRegression() task_id = self.TEST_SERVER_TASK_REGRESSION["task_id"] @@ -668,7 +664,6 @@ def test_run_and_upload_linear_regression(self): self._run_and_upload_regression(lr, task_id, n_missing_vals, n_test_obs, "62501") @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_run_and_upload_pipeline_dummy_pipeline(self): pipeline1 = Pipeline( steps=[ @@ -682,7 +677,6 @@ def test_run_and_upload_pipeline_dummy_pipeline(self): self._run_and_upload_classification(pipeline1, task_id, n_missing_vals, n_test_obs, "62501") @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="columntransformer introduction in 0.20.0", @@ -799,7 +793,6 @@ def test_run_and_upload_knn_pipeline(self, warnings_mock): assert call_count == 3 @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_run_and_upload_gridsearch(self): estimator_name = ( "base_estimator" if Version(sklearn.__version__) < Version("1.4") else "estimator" @@ -822,7 +815,6 @@ def test_run_and_upload_gridsearch(self): assert len(run.trace.trace_iterations) == 9 @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_run_and_upload_randomsearch(self): randomsearch = RandomizedSearchCV( RandomForestClassifier(n_estimators=5), @@ -855,7 +847,6 @@ def test_run_and_upload_randomsearch(self): assert len(trace.trace_iterations) == 5 @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_run_and_upload_maskedarrays(self): # This testcase is important for 2 reasons: # 1) it verifies the correct handling of masked arrays (not all @@ -883,7 +874,6 @@ def test_run_and_upload_maskedarrays(self): ########################################################################## @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_learning_curve_task_1(self): task_id = 801 # diabates dataset num_test_instances = 6144 # for learning curve @@ -908,7 +898,6 @@ def test_learning_curve_task_1(self): self._check_sample_evaluations(run.sample_evaluations, num_repeats, num_folds, num_samples) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_learning_curve_task_2(self): task_id = 801 # diabates dataset num_test_instances = 6144 # for learning curve @@ -945,7 +934,6 @@ def test_learning_curve_task_2(self): self._check_sample_evaluations(run.sample_evaluations, num_repeats, num_folds, num_samples) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") @unittest.skipIf( Version(sklearn.__version__) < Version("0.21"), reason="Pipelines don't support indexing (used for the assert check)", @@ -1024,7 +1012,6 @@ def _test_local_evaluations(self, run): assert alt_scores[idx] <= 1 @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_local_run_swapped_parameter_order_model(self): clf = DecisionTreeClassifier() australian_task = 595 # Australian; crossvalidation @@ -1040,7 +1027,6 @@ def test_local_run_swapped_parameter_order_model(self): self._test_local_evaluations(run) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", @@ -1069,7 +1055,6 @@ def test_local_run_swapped_parameter_order_flow(self): self._test_local_evaluations(run) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", @@ -1107,7 +1092,6 @@ def test_online_run_metric_score(self): self._test_local_evaluations(run) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", @@ -1173,7 +1157,6 @@ def test_initialize_model_from_run(self): Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", ) - @pytest.mark.xfail(reason="failures_issue_1544") def test__run_exists(self): # would be better to not sentinel these clfs, # so we do not have to perform the actual runs @@ -1229,7 +1212,6 @@ def test__run_exists(self): assert run_ids, (run_ids, clf) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_run_with_illegal_flow_id(self): # check the case where the user adds an illegal flow id to a # non-existing flo @@ -1249,7 +1231,6 @@ def test_run_with_illegal_flow_id(self): ) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_run_with_illegal_flow_id_after_load(self): # Same as `test_run_with_illegal_flow_id`, but test this error is also # caught if the run is stored to and loaded from disk first. @@ -1281,7 +1262,6 @@ def test_run_with_illegal_flow_id_after_load(self): TestBase.logger.info(f"collected from test_run_functions: {loaded_run.run_id}") @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_run_with_illegal_flow_id_1(self): # Check the case where the user adds an illegal flow id to an existing # flow. Comes to a different value error than the previous test @@ -1307,7 +1287,6 @@ def test_run_with_illegal_flow_id_1(self): ) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_run_with_illegal_flow_id_1_after_load(self): # Same as `test_run_with_illegal_flow_id_1`, but test this error is # also caught if the run is stored to and loaded from disk first. @@ -1346,7 +1325,6 @@ def test_run_with_illegal_flow_id_1_after_load(self): ) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="OneHotEncoder cannot handle mixed type DataFrame as input", @@ -1574,7 +1552,6 @@ def test_get_runs_list_by_tag(self): assert len(runs) >= 1 @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="columntransformer introduction in 0.20.0", @@ -1611,7 +1588,6 @@ def test_run_on_dataset_with_missing_labels_dataframe(self): assert len(row) == 12 @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="columntransformer introduction in 0.20.0", @@ -1664,7 +1640,6 @@ def test_get_uncached_run(self): openml.runs.functions._get_cached_run(10) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_run_flow_on_task_downloaded_flow(self): model = sklearn.ensemble.RandomForestClassifier(n_estimators=33) flow = self.extension.model_to_flow(model) @@ -1765,7 +1740,6 @@ def test_format_prediction_task_regression(self): reason="SimpleImputer doesn't handle mixed type DataFrame as input", ) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_delete_run(self): rs = np.random.randint(1, 2**31 - 1) clf = sklearn.pipeline.Pipeline( @@ -1861,7 +1835,6 @@ def test_delete_unknown_run(mock_delete, test_files_directory, test_api_key): @pytest.mark.sklearn() -@pytest.mark.xfail(reason="failures_issue_1544") @unittest.skipIf( Version(sklearn.__version__) < Version("0.21"), reason="couldn't perform local tests successfully w/o bloating RAM", @@ -1957,7 +1930,6 @@ def test__run_task_get_arffcontent_2(parallel_mock): (-1, "threading", 10), # the threading backend does preserve mocks even with parallelizing ] ) -@pytest.mark.xfail(reason="failures_issue_1544") def test_joblib_backends(parallel_mock, n_jobs, backend, call_count): """Tests evaluation of a run using various joblib backends and n_jobs.""" if backend is None: diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index 18d7f5cc6..42af5362b 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -82,7 +82,6 @@ def _existing_setup_exists(self, classif): assert setup_id == run.setup_id @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_existing_setup_exists_1(self): def side_effect(self): self.var_smoothing = 1e-9 @@ -98,13 +97,11 @@ def side_effect(self): self._existing_setup_exists(nb) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_exisiting_setup_exists_2(self): # Check a flow with one hyperparameter self._existing_setup_exists(sklearn.naive_bayes.GaussianNB()) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544") def test_existing_setup_exists_3(self): # Check a flow with many hyperparameters self._existing_setup_exists( From 6e7885857b4fb7093dc269baa183b9e981043d37 Mon Sep 17 00:00:00 2001 From: Shrivaths S Nair <142079253+JATAYU000@users.noreply.github.com> Date: Wed, 31 Dec 2025 15:07:29 +0530 Subject: [PATCH 12/46] [BUG] `get_task` removes the dir even if was already existing (#1584) #### Metadata * Reference Issue: Refer failures in #1579 * New Tests Added: No * Documentation Updated: No * Change Log Entry: Checks if the directory was created newly else doesn't remove. ### Details * What does this PR implement/fix? Explain your changes. `get_task` checks if the `tid_cache_dir` was already existing before removing it on `Exception` * Why is this change necessary? What is the problem it solves? `OpenMLServerException` causes `get_task` to remove the entire directory even if the directory was already existing and is used by other tests * How can I reproduce the issue this PR is solving and its solution? observe `exists assertion` errors for files under `tests/files/org/openml/test/task/1/` after running `pytest` or look at failures in #1579 --- openml/tasks/functions.py | 8 +++++--- tests/test_runs/test_run_functions.py | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/openml/tasks/functions.py b/openml/tasks/functions.py index d2bf5e946..e9b879ae4 100644 --- a/openml/tasks/functions.py +++ b/openml/tasks/functions.py @@ -415,8 +415,9 @@ def get_task( if not isinstance(task_id, int): raise TypeError(f"Task id should be integer, is {type(task_id)}") - tid_cache_dir = openml.utils._create_cache_directory_for_id(TASKS_CACHE_DIR_NAME, task_id) - + cache_key_dir = openml.utils._create_cache_directory_for_id(TASKS_CACHE_DIR_NAME, task_id) + tid_cache_dir = cache_key_dir / str(task_id) + tid_cache_dir_existed = tid_cache_dir.exists() try: task = _get_task_description(task_id) dataset = get_dataset(task.dataset_id, **get_dataset_kwargs) @@ -430,7 +431,8 @@ def get_task( if download_splits and isinstance(task, OpenMLSupervisedTask): task.download_split() except Exception as e: - openml.utils._remove_cache_dir_for_id(TASKS_CACHE_DIR_NAME, tid_cache_dir) + if not tid_cache_dir_existed: + openml.utils._remove_cache_dir_for_id(TASKS_CACHE_DIR_NAME, tid_cache_dir) raise e return task diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 94ffa5001..18d4f836f 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -815,6 +815,7 @@ def test_run_and_upload_gridsearch(self): assert len(run.trace.trace_iterations) == 9 @pytest.mark.sklearn() + @pytest.mark.skip(reason="failures_issue_1544") def test_run_and_upload_randomsearch(self): randomsearch = RandomizedSearchCV( RandomForestClassifier(n_estimators=5), From bd8ae775b27edb9f47e5d1991bb62c1d707785e1 Mon Sep 17 00:00:00 2001 From: Shrivaths S Nair <142079253+JATAYU000@users.noreply.github.com> Date: Wed, 31 Dec 2025 18:44:40 +0530 Subject: [PATCH 13/46] [MNT] extend CI to newer python versions, deprecate python versions 3.8, 3.9 after EOL, marking further failing tests as `xfail` (#1579) The CI runs only on python versions 3.8 and 3.9 both of which have already reached end of life. This PR updates the python versions, deprecating any logic that runs tests on python versions 3.8 and 3.9, or `scikit-learn` versions of that age. #### Metadata Reference Issue: #1544 Depends on https://github.com/openml/openml-python/pull/1584 fofr a fix, which should be merged first. #### Details * The test matrix is updated to python versions, 3.10-3.13. * Further failing tests are skipped using `mark.xfail` with `reason="failures_issue_1544" ` for all the remaining failed tests (after #1572) in issue: #1544 --- .github/workflows/test.yml | 88 +++++++++++++------- pyproject.toml | 3 +- tests/test_runs/test_run_functions.py | 7 ++ tests/test_tasks/test_learning_curve_task.py | 1 + tests/test_tasks/test_regression_task.py | 1 + tests/test_tasks/test_supervised_task.py | 1 + tests/test_tasks/test_task_functions.py | 1 + tests/test_tasks/test_task_methods.py | 1 + 8 files changed, 69 insertions(+), 34 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b4574038c..b77cfd38c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,45 +23,51 @@ jobs: test: name: (${{ matrix.os }}, Py${{ matrix.python-version }}, sk${{ matrix.scikit-learn }}, sk-only:${{ matrix.sklearn-only }}) runs-on: ${{ matrix.os }} + strategy: + fail-fast: false matrix: - python-version: ["3.11"] - scikit-learn: ["1.3.*", "1.4.*", "1.5.*"] + python-version: ["3.10", "3.11", "3.12", "3.13"] + scikit-learn: ["1.3.*", "1.4.*", "1.5.*", "1.6.*", "1.7.*"] os: [ubuntu-latest] sklearn-only: ["true"] - fail-fast: false + + exclude: + # incompatible version combinations + - python-version: "3.13" + scikit-learn: "1.3.*" + - python-version: "3.13" + scikit-learn: "1.4.*" + + include: + # Full test run on Windows + - os: windows-latest + python-version: "3.12" + scikit-learn: "1.5.*" + sklearn-only: "false" + + # Coverage run + - os: ubuntu-latest + python-version: "3.12" + scikit-learn: "1.5.*" + sklearn-only: "false" + code-cov: true steps: - uses: actions/checkout@v6 with: fetch-depth: 2 + - name: Setup Python ${{ matrix.python-version }} - if: matrix.os != 'windows-latest' # windows-latest only uses preinstalled Python (3.9.13) uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install test dependencies + + - name: Install test dependencies and scikit-learn run: | python -m pip install --upgrade pip - pip install -e .[test] - - name: Install scikit-learn ${{ matrix.scikit-learn }} - run: | - pip install scikit-learn==${{ matrix.scikit-learn }} - - name: Install numpy for Python 3.8 - # Python 3.8 & scikit-learn<0.24 requires numpy<=1.23.5 - if: ${{ matrix.python-version == '3.8' && matrix.scikit-learn == '0.23.1' }} - run: | - pip install numpy==1.23.5 - - name: "Install NumPy 1.x and SciPy <1.11 for scikit-learn < 1.4" - if: ${{ contains(fromJSON('["1.0.*", "1.1.*", "1.2.*", "1.3.*"]'), matrix.scikit-learn) }} - run: | - # scipy has a change to the 'mode' behavior which breaks scikit-learn < 1.4 - # numpy 2.0 has several breaking changes - pip install "numpy<2.0" "scipy<1.11" - - name: Install scipy ${{ matrix.scipy }} - if: ${{ matrix.scipy }} - run: | - pip install scipy==${{ matrix.scipy }} + pip install -e .[test] scikit-learn==${{ matrix.scikit-learn }} + - name: Store repository status id: status-before if: matrix.os != 'windows-latest' @@ -69,28 +75,45 @@ jobs: git_status=$(git status --porcelain -b) echo "BEFORE=$git_status" >> $GITHUB_ENV echo "Repository status before tests: $git_status" + - name: Show installed dependencies run: python -m pip list + - name: Run tests on Ubuntu Test if: matrix.os == 'ubuntu-latest' run: | - if [ ${{ matrix.code-cov }} ]; then codecov='--cov=openml --long --cov-report=xml'; fi - # Most of the time, running only the scikit-learn tests is sufficient - if [ ${{ matrix.sklearn-only }} = 'true' ]; then marks='sklearn and not production'; else marks='not production'; fi - echo pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks" + if [ "${{ matrix.code-cov }}" = "true" ]; then + codecov="--cov=openml --long --cov-report=xml" + fi + + if [ "${{ matrix.sklearn-only }}" = "true" ]; then + marks="sklearn and not production" + else + marks="not production" + fi + pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks" + - name: Run tests on Ubuntu Production if: matrix.os == 'ubuntu-latest' run: | - if [ ${{ matrix.code-cov }} ]; then codecov='--cov=openml --long --cov-report=xml'; fi - # Most of the time, running only the scikit-learn tests is sufficient - if [ ${{ matrix.sklearn-only }} = 'true' ]; then marks='sklearn and production'; else marks='production'; fi - echo pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks" + if [ "${{ matrix.code-cov }}" = "true" ]; then + codecov="--cov=openml --long --cov-report=xml" + fi + + if [ "${{ matrix.sklearn-only }}" = "true" ]; then + marks="sklearn and production" + else + marks="production" + fi + pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks" + - name: Run tests on Windows if: matrix.os == 'windows-latest' run: | # we need a separate step because of the bash-specific if-statement in the previous one. pytest -n 4 --durations=20 --dist load -sv --reruns 5 --reruns-delay 1 + - name: Check for files left behind by test if: matrix.os != 'windows-latest' && always() run: | @@ -102,6 +125,7 @@ jobs: echo "Not all generated files have been deleted!" exit 1 fi + - name: Upload coverage if: matrix.code-cov && always() uses: codecov/codecov-action@v4 diff --git a/pyproject.toml b/pyproject.toml index 2bf762b09..ede204ca0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,12 +50,11 @@ classifiers = [ "Operating System :: Unix", "Operating System :: MacOS", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ] license = { file = "LICENSE" } diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 18d4f836f..e4cec56ab 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -625,6 +625,7 @@ def _run_and_upload_regression( sentinel=sentinel, ) + @pytest.mark.skip(reason="failures_issue_1544") @pytest.mark.sklearn() def test_run_and_upload_logistic_regression(self): lr = LogisticRegression(solver="lbfgs", max_iter=1000) @@ -633,6 +634,7 @@ def test_run_and_upload_logistic_regression(self): n_test_obs = self.TEST_SERVER_TASK_SIMPLE["n_test_obs"] self._run_and_upload_classification(lr, task_id, n_missing_vals, n_test_obs, "62501") + @pytest.mark.skip(reason="failures_issue_1544") @pytest.mark.sklearn() def test_run_and_upload_linear_regression(self): lr = LinearRegression() @@ -663,6 +665,7 @@ def test_run_and_upload_linear_regression(self): n_test_obs = self.TEST_SERVER_TASK_REGRESSION["n_test_obs"] self._run_and_upload_regression(lr, task_id, n_missing_vals, n_test_obs, "62501") + @pytest.mark.skip(reason="failures_issue_1544") @pytest.mark.sklearn() def test_run_and_upload_pipeline_dummy_pipeline(self): pipeline1 = Pipeline( @@ -676,6 +679,7 @@ def test_run_and_upload_pipeline_dummy_pipeline(self): n_test_obs = self.TEST_SERVER_TASK_SIMPLE["n_test_obs"] self._run_and_upload_classification(pipeline1, task_id, n_missing_vals, n_test_obs, "62501") + @pytest.mark.skip(reason="failures_issue_1544") @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -740,6 +744,7 @@ def get_ct_cf(nominal_indices, numeric_indices): sentinel=sentinel, ) + @pytest.mark.skip(reason="failures_issue_1544") @pytest.mark.sklearn() @unittest.skip("https://github.com/openml/OpenML/issues/1180") @unittest.skipIf( @@ -792,6 +797,7 @@ def test_run_and_upload_knn_pipeline(self, warnings_mock): call_count += 1 assert call_count == 3 + @pytest.mark.skip(reason="failures_issue_1544") @pytest.mark.sklearn() def test_run_and_upload_gridsearch(self): estimator_name = ( @@ -847,6 +853,7 @@ def test_run_and_upload_randomsearch(self): trace = openml.runs.get_run_trace(run.run_id) assert len(trace.trace_iterations) == 5 + @pytest.mark.skip(reason="failures_issue_1544") @pytest.mark.sklearn() def test_run_and_upload_maskedarrays(self): # This testcase is important for 2 reasons: diff --git a/tests/test_tasks/test_learning_curve_task.py b/tests/test_tasks/test_learning_curve_task.py index 885f80a27..4a3dede4e 100644 --- a/tests/test_tasks/test_learning_curve_task.py +++ b/tests/test_tasks/test_learning_curve_task.py @@ -2,6 +2,7 @@ from __future__ import annotations import pandas as pd +import pytest from openml.tasks import TaskType, get_task diff --git a/tests/test_tasks/test_regression_task.py b/tests/test_tasks/test_regression_task.py index 14ed59470..3e324c4f8 100644 --- a/tests/test_tasks/test_regression_task.py +++ b/tests/test_tasks/test_regression_task.py @@ -4,6 +4,7 @@ import ast import pandas as pd +import pytest import openml from openml.exceptions import OpenMLServerException diff --git a/tests/test_tasks/test_supervised_task.py b/tests/test_tasks/test_supervised_task.py index 9c90b7e03..e5a17a72b 100644 --- a/tests/test_tasks/test_supervised_task.py +++ b/tests/test_tasks/test_supervised_task.py @@ -6,6 +6,7 @@ import pandas as pd from openml.tasks import get_task +import pytest from .test_task import OpenMLTaskTest diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py index 5f1d577c0..0aa2dcc9b 100644 --- a/tests/test_tasks/test_task_functions.py +++ b/tests/test_tasks/test_task_functions.py @@ -174,6 +174,7 @@ def test_get_task_lazy(self): ) @mock.patch("openml.tasks.functions.get_dataset") + @pytest.mark.xfail(reason="failures_issue_1544") def test_removal_upon_download_failure(self, get_dataset): class WeirdException(Exception): pass diff --git a/tests/test_tasks/test_task_methods.py b/tests/test_tasks/test_task_methods.py index 4480c2cbc..540c43de0 100644 --- a/tests/test_tasks/test_task_methods.py +++ b/tests/test_tasks/test_task_methods.py @@ -5,6 +5,7 @@ import openml from openml.testing import TestBase +import pytest # Common methods between tasks From f9fb3a1b45729fd9fd6aa6d98c8ecc2c5a4e5661 Mon Sep 17 00:00:00 2001 From: Eman Abdelhaleem <101830347+EmanAbdelhaleem@users.noreply.github.com> Date: Thu, 1 Jan 2026 13:18:21 +0200 Subject: [PATCH 14/46] [BUG] Temporarily fix issue #1586 by marking some failed tests as non-strict expected fail. (#1587) #### Metadata * Reference Issue: Temporarily fix issue #1586 #### Details - Running the pytest locally, I found only one failed test which is: `tests/test_runs/test_run_functions.py::test__run_task_get_arffcontent_2` - However, when trying to go through the failed tests in the recent runed jobs in different recent PRs, I found many other failed tests, I picked some of them and tried to make a kind of analysis, and here are my findings: ##### Primary Failure Patterns 1. OpenML Test Server Issues (Most Common) The majority of failures are caused by: - `OpenMLServerError: Unexpected server error when calling https://test.openml.org/... with Status code: 500` - Database connection errors: `Database connection error. Usually due to high server load. Please wait N seconds and try again.` - Timeout errors: `TIMEOUT: Failed to fetch uploaded dataset` 2. Cache/Filesystem Issues - `ValueError: Cannot remove faulty tasks cache directory ... Please do this manually!` - `FileNotFoundError: No such file or directory` 3. Data Format Issues - `KeyError: ['type'] not found in axis` - `KeyError: ['class'] not found in axis` - `KeyError: ['Class'] not found in axis` --- tests/test_datasets/test_dataset_functions.py | 9 ++++++++ tests/test_flows/test_flow.py | 5 +++++ tests/test_flows/test_flow_functions.py | 2 ++ tests/test_runs/test_run.py | 5 +++++ tests/test_runs/test_run_functions.py | 22 +++++++++++++++++++ tests/test_setups/test_setup_functions.py | 4 ++++ tests/test_tasks/test_classification_task.py | 3 +++ tests/test_tasks/test_learning_curve_task.py | 3 +++ tests/test_tasks/test_regression_task.py | 2 ++ tests/test_tasks/test_task.py | 3 +++ tests/test_tasks/test_task_functions.py | 1 + 11 files changed, 59 insertions(+) diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index 266a6f6f7..f8cb1943c 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -280,6 +280,7 @@ def test_dataset_by_name_cannot_access_private_data(self): self.use_production_server() self.assertRaises(OpenMLPrivateDatasetError, openml.datasets.get_dataset, "NAME_GOES_HERE") + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_get_dataset_lazy_all_functions(self): """Test that all expected functionality is available without downloading the dataset.""" dataset = openml.datasets.get_dataset(1) @@ -664,6 +665,7 @@ def test_attributes_arff_from_df_unknown_dtype(self): with pytest.raises(ValueError, match=err_msg): attributes_arff_from_df(df) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_create_dataset_numpy(self): data = np.array([[1, 2, 3], [1.2, 2.5, 3.8], [2, 5, 8], [0, 1, 0]]).T @@ -751,6 +753,7 @@ def test_create_dataset_list(self): ), "Uploaded ARFF does not match original one" assert _get_online_dataset_format(dataset.id) == "arff", "Wrong format for dataset" + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_create_dataset_sparse(self): # test the scipy.sparse.coo_matrix sparse_data = scipy.sparse.coo_matrix( @@ -868,6 +871,7 @@ def test_get_online_dataset_arff(self): return_type=arff.DENSE if d_format == "arff" else arff.COO, ), "ARFF files are not equal" + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_topic_api_error(self): # Check server exception when non-admin accessses apis self.assertRaisesRegex( @@ -895,6 +899,7 @@ def test_get_online_dataset_format(self): dataset_id ), "The format of the ARFF files is different" + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_create_dataset_pandas(self): data = [ ["a", "sunny", 85.0, 85.0, "FALSE", "no"], @@ -1119,6 +1124,7 @@ def test_ignore_attributes_dataset(self): paper_url=paper_url, ) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_publish_fetch_ignore_attribute(self): """Test to upload and retrieve dataset and check ignore_attributes""" data = [ @@ -1237,6 +1243,7 @@ def test_create_dataset_row_id_attribute_error(self): paper_url=paper_url, ) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_create_dataset_row_id_attribute_inference(self): # meta-information name = f"{self._get_sentinel()}-pandas_testing_dataset" @@ -1400,6 +1407,7 @@ def test_data_edit_non_critical_field(self): edited_dataset = openml.datasets.get_dataset(did) assert edited_dataset.description == desc + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_data_edit_critical_field(self): # Case 2 # only owners (or admin) can edit all critical fields of datasets @@ -1448,6 +1456,7 @@ def test_data_edit_requires_valid_dataset(self): description="xor operation dataset", ) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_data_edit_cannot_edit_critical_field_if_dataset_has_task(self): # Need to own a dataset to be able to edit meta-data # Will be creating a forked version of an existing dataset to allow the unit test user diff --git a/tests/test_flows/test_flow.py b/tests/test_flows/test_flow.py index 0b034c3b4..da719d058 100644 --- a/tests/test_flows/test_flow.py +++ b/tests/test_flows/test_flow.py @@ -178,6 +178,7 @@ def test_to_xml_from_xml(self): assert new_flow is not flow @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_publish_flow(self): flow = openml.OpenMLFlow( name="sklearn.dummy.DummyClassifier", @@ -219,6 +220,7 @@ def test_publish_existing_flow(self, flow_exists_mock): ) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_publish_flow_with_similar_components(self): clf = sklearn.ensemble.VotingClassifier( [("lr", sklearn.linear_model.LogisticRegression(solver="lbfgs"))], @@ -269,6 +271,7 @@ def test_publish_flow_with_similar_components(self): TestBase.logger.info(f"collected from {__file__.split('/')[-1]}: {flow3.flow_id}") @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_semi_legal_flow(self): # TODO: Test if parameters are set correctly! # should not throw error as it contains two differentiable forms of @@ -377,6 +380,7 @@ def get_sentinel(): assert not flow_id @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_existing_flow_exists(self): # create a flow nb = sklearn.naive_bayes.GaussianNB() @@ -417,6 +421,7 @@ def test_existing_flow_exists(self): assert downloaded_flow_id == flow.flow_id @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_sklearn_to_upload_to_flow(self): iris = sklearn.datasets.load_iris() X = iris.data diff --git a/tests/test_flows/test_flow_functions.py b/tests/test_flows/test_flow_functions.py index 9f8ec5e36..0be65ceac 100644 --- a/tests/test_flows/test_flow_functions.py +++ b/tests/test_flows/test_flow_functions.py @@ -274,6 +274,7 @@ def test_are_flows_equal_ignore_if_older(self): assert_flows_equal(flow, flow, ignore_parameter_values_on_older_children=None) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="OrdinalEncoder introduced in 0.20. " @@ -388,6 +389,7 @@ def test_get_flow_reinstantiate_flow_not_strict_pre_023(self): assert "sklearn==0.19.1" not in flow.dependencies @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_get_flow_id(self): if self.long_version: list_all = openml.utils._list_all diff --git a/tests/test_runs/test_run.py b/tests/test_runs/test_run.py index 034b731aa..71651d431 100644 --- a/tests/test_runs/test_run.py +++ b/tests/test_runs/test_run.py @@ -118,6 +118,7 @@ def _check_array(array, type_): assert run_prime_trace_content is None @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_to_from_filesystem_vanilla(self): model = Pipeline( [ @@ -153,6 +154,7 @@ def test_to_from_filesystem_vanilla(self): @pytest.mark.sklearn() @pytest.mark.flaky() + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_to_from_filesystem_search(self): model = Pipeline( [ @@ -187,6 +189,7 @@ def test_to_from_filesystem_search(self): ) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_to_from_filesystem_no_model(self): model = Pipeline( [("imputer", SimpleImputer(strategy="mean")), ("classifier", DummyClassifier())], @@ -292,6 +295,7 @@ def assert_run_prediction_data(task, run, model): assert_method(y_test, saved_y_test) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_publish_with_local_loaded_flow(self): """ Publish a run tied to a local flow after it has first been saved to @@ -335,6 +339,7 @@ def test_publish_with_local_loaded_flow(self): openml.runs.get_run(loaded_run.run_id) @pytest.mark.sklearn() + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_offline_and_online_run_identical(self): extension = SklearnExtension() diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index e4cec56ab..305d859d9 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -413,6 +413,7 @@ def test_run_regression_on_classif_task(self): task=task, ) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() def test_check_erronous_sklearn_flow_fails(self): task_id = 115 # diabetes; crossvalidation @@ -881,6 +882,7 @@ def test_run_and_upload_maskedarrays(self): ########################################################################## + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() def test_learning_curve_task_1(self): task_id = 801 # diabates dataset @@ -905,6 +907,7 @@ def test_learning_curve_task_1(self): ) self._check_sample_evaluations(run.sample_evaluations, num_repeats, num_folds, num_samples) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() def test_learning_curve_task_2(self): task_id = 801 # diabates dataset @@ -941,6 +944,7 @@ def test_learning_curve_task_2(self): ) self._check_sample_evaluations(run.sample_evaluations, num_repeats, num_folds, num_samples) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.21"), @@ -1019,6 +1023,7 @@ def _test_local_evaluations(self, run): assert alt_scores[idx] >= 0 assert alt_scores[idx] <= 1 + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() def test_local_run_swapped_parameter_order_model(self): clf = DecisionTreeClassifier() @@ -1034,6 +1039,7 @@ def test_local_run_swapped_parameter_order_model(self): self._test_local_evaluations(run) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1062,6 +1068,7 @@ def test_local_run_swapped_parameter_order_flow(self): self._test_local_evaluations(run) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1099,6 +1106,7 @@ def test_online_run_metric_score(self): self._test_local_evaluations(run) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1160,6 +1168,7 @@ def test_initialize_model_from_run(self): assert flowS.components["Imputer"].parameters["strategy"] == '"most_frequent"' assert flowS.components["VarianceThreshold"].parameters["threshold"] == "0.05" + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1219,6 +1228,7 @@ def test__run_exists(self): run_ids = run_exists(task.task_id, setup_exists) assert run_ids, (run_ids, clf) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() def test_run_with_illegal_flow_id(self): # check the case where the user adds an illegal flow id to a @@ -1238,6 +1248,7 @@ def test_run_with_illegal_flow_id(self): avoid_duplicate_runs=True, ) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() def test_run_with_illegal_flow_id_after_load(self): # Same as `test_run_with_illegal_flow_id`, but test this error is also @@ -1294,6 +1305,7 @@ def test_run_with_illegal_flow_id_1(self): avoid_duplicate_runs=True, ) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() def test_run_with_illegal_flow_id_1_after_load(self): # Same as `test_run_with_illegal_flow_id_1`, but test this error is @@ -1332,6 +1344,7 @@ def test_run_with_illegal_flow_id_1_after_load(self): loaded_run.publish, ) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1559,6 +1572,7 @@ def test_get_runs_list_by_tag(self): runs = openml.runs.list_runs(tag="curves", size=2) assert len(runs) >= 1 + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1595,6 +1609,7 @@ def test_run_on_dataset_with_missing_labels_dataframe(self): # repeat, fold, row_id, 6 confidences, prediction and correct label assert len(row) == 12 + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1647,6 +1662,7 @@ def test_get_uncached_run(self): with pytest.raises(openml.exceptions.OpenMLCacheException): openml.runs.functions._get_cached_run(10) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() def test_run_flow_on_task_downloaded_flow(self): model = sklearn.ensemble.RandomForestClassifier(n_estimators=33) @@ -1687,6 +1703,7 @@ def test_format_prediction_classification_no_probabilities(self): with pytest.raises(ValueError, match="`proba` is required for classification task"): format_prediction(classification, *ignored_input, proba=None) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_format_prediction_classification_incomplete_probabilities(self): classification = openml.tasks.get_task( self.TEST_SERVER_TASK_SIMPLE["task_id"], @@ -1707,6 +1724,7 @@ def test_format_prediction_task_without_classlabels_set(self): with pytest.raises(ValueError, match="The classification task must have class labels set"): format_prediction(classification, *ignored_input, proba={}) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_format_prediction_task_learning_curve_sample_not_set(self): learning_curve = openml.tasks.get_task(801, download_data=False) # diabetes;crossvalidation probabilities = {c: 0.2 for c in learning_curve.class_labels} @@ -1714,6 +1732,7 @@ def test_format_prediction_task_learning_curve_sample_not_set(self): with pytest.raises(ValueError, match="`sample` can not be none for LearningCurveTask"): format_prediction(learning_curve, *ignored_input, sample=None, proba=probabilities) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_format_prediction_task_regression(self): task_meta_data = self.TEST_SERVER_TASK_REGRESSION["task_meta_data"] _task_id = check_task_existence(**task_meta_data) @@ -1743,6 +1762,7 @@ def test_format_prediction_task_regression(self): + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", @@ -1843,6 +1863,7 @@ def test_delete_unknown_run(mock_delete, test_files_directory, test_api_key): @pytest.mark.sklearn() +@pytest.mark.xfail(reason="failures_issue_1544", strict=False) @unittest.skipIf( Version(sklearn.__version__) < Version("0.21"), reason="couldn't perform local tests successfully w/o bloating RAM", @@ -1919,6 +1940,7 @@ def test__run_task_get_arffcontent_2(parallel_mock): ) +@pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.21"), diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index 42af5362b..a3b698a37 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -34,6 +34,7 @@ def setUp(self): self.extension = SklearnExtension() super().setUp() + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() def test_nonexisting_setup_exists(self): # first publish a non-existing flow @@ -81,6 +82,7 @@ def _existing_setup_exists(self, classif): setup_id = openml.setups.setup_exists(flow) assert setup_id == run.setup_id + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() def test_existing_setup_exists_1(self): def side_effect(self): @@ -96,11 +98,13 @@ def side_effect(self): nb = sklearn.naive_bayes.GaussianNB() self._existing_setup_exists(nb) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() def test_exisiting_setup_exists_2(self): # Check a flow with one hyperparameter self._existing_setup_exists(sklearn.naive_bayes.GaussianNB()) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() def test_existing_setup_exists_3(self): # Check a flow with many hyperparameters diff --git a/tests/test_tasks/test_classification_task.py b/tests/test_tasks/test_classification_task.py index d4f2ed9d7..5528cabf2 100644 --- a/tests/test_tasks/test_classification_task.py +++ b/tests/test_tasks/test_classification_task.py @@ -18,6 +18,7 @@ def setUp(self, n_levels: int = 1): self.task_type = TaskType.SUPERVISED_CLASSIFICATION self.estimation_procedure = 5 + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_download_task(self): task = super().test_download_task() assert task.task_id == self.task_id @@ -25,11 +26,13 @@ def test_download_task(self): assert task.dataset_id == 20 assert task.estimation_procedure_id == self.estimation_procedure + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_class_labels(self): task = get_task(self.task_id) assert task.class_labels == ["tested_negative", "tested_positive"] +@pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.server() def test_get_X_and_Y(): task = get_task(119) diff --git a/tests/test_tasks/test_learning_curve_task.py b/tests/test_tasks/test_learning_curve_task.py index 4a3dede4e..5f4b3e0ab 100644 --- a/tests/test_tasks/test_learning_curve_task.py +++ b/tests/test_tasks/test_learning_curve_task.py @@ -18,6 +18,7 @@ def setUp(self, n_levels: int = 1): self.task_type = TaskType.LEARNING_CURVE self.estimation_procedure = 13 + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_get_X_and_Y(self): X, Y = super().test_get_X_and_Y() assert X.shape == (768, 8) @@ -26,12 +27,14 @@ def test_get_X_and_Y(self): assert isinstance(Y, pd.Series) assert pd.api.types.is_categorical_dtype(Y) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_download_task(self): task = super().test_download_task() assert task.task_id == self.task_id assert task.task_type_id == TaskType.LEARNING_CURVE assert task.dataset_id == 20 + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_class_labels(self): task = get_task(self.task_id) assert task.class_labels == ["tested_negative", "tested_positive"] diff --git a/tests/test_tasks/test_regression_task.py b/tests/test_tasks/test_regression_task.py index 3e324c4f8..0cd2d96e2 100644 --- a/tests/test_tasks/test_regression_task.py +++ b/tests/test_tasks/test_regression_task.py @@ -49,6 +49,7 @@ def setUp(self, n_levels: int = 1): self.task_type = TaskType.SUPERVISED_REGRESSION + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_get_X_and_Y(self): X, Y = super().test_get_X_and_Y() assert X.shape == (194, 32) @@ -57,6 +58,7 @@ def test_get_X_and_Y(self): assert isinstance(Y, pd.Series) assert pd.api.types.is_numeric_dtype(Y) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_download_task(self): task = super().test_download_task() assert task.task_id == self.task_id diff --git a/tests/test_tasks/test_task.py b/tests/test_tasks/test_task.py index e4c9418f2..67f715d2b 100644 --- a/tests/test_tasks/test_task.py +++ b/tests/test_tasks/test_task.py @@ -4,6 +4,8 @@ import unittest from random import randint, shuffle +import pytest + from openml.datasets import ( get_dataset, list_datasets, @@ -33,6 +35,7 @@ def setUp(self, n_levels: int = 1): def test_download_task(self): return get_task(self.task_id) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_upload_task(self): # We don't know if the task in question already exists, so we try a few times. Checking # beforehand would not be an option because a concurrent unit test could potentially diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py index 0aa2dcc9b..110459711 100644 --- a/tests/test_tasks/test_task_functions.py +++ b/tests/test_tasks/test_task_functions.py @@ -152,6 +152,7 @@ def test_get_task(self): os.path.join(self.workdir, "org", "openml", "test", "datasets", "1", "dataset.arff") ) + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_get_task_lazy(self): task = openml.tasks.get_task(2, download_data=False) # anneal; crossvalidation assert isinstance(task, OpenMLTask) From 8672ffbabf1532185781aa83023cba2bea12b43d Mon Sep 17 00:00:00 2001 From: Eman Abdelhaleem <101830347+EmanAbdelhaleem@users.noreply.github.com> Date: Thu, 1 Jan 2026 15:13:18 +0200 Subject: [PATCH 15/46] [BUG] Fix Sklearn Models detection by safely importing openml-sklearn (#1556) #### Metadata * Reference Issue: Fixes #1542 #### Details Fixed sklearn models detection by safely importing openml-sklearn at `openml/runs/__init__.py` --- openml/extensions/functions.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/openml/extensions/functions.py b/openml/extensions/functions.py index 7a944c997..06902325e 100644 --- a/openml/extensions/functions.py +++ b/openml/extensions/functions.py @@ -1,6 +1,7 @@ # License: BSD 3-Clause from __future__ import annotations +import importlib.util from typing import TYPE_CHECKING, Any # Need to implement the following by its full path because otherwise it won't be possible to @@ -16,8 +17,9 @@ SKLEARN_HINT = ( "But it looks related to scikit-learn. " "Please install the OpenML scikit-learn extension (openml-sklearn) and try again. " + "You can use `pip install openml-sklearn` for installation." "For more information, see " - "https://github.com/openml/openml-sklearn?tab=readme-ov-file#installation" + "https://docs.openml.org/python/extensions/" ) @@ -58,6 +60,10 @@ def get_extension_by_flow( ------- Extension or None """ + # import openml_sklearn to register SklearnExtension + if importlib.util.find_spec("openml_sklearn"): + import openml_sklearn # noqa: F401 + candidates = [] for extension_class in openml.extensions.extensions: if extension_class.can_handle_flow(flow): @@ -103,6 +109,10 @@ def get_extension_by_model( ------- Extension or None """ + # import openml_sklearn to register SklearnExtension + if importlib.util.find_spec("openml_sklearn"): + import openml_sklearn # noqa: F401 + candidates = [] for extension_class in openml.extensions.extensions: if extension_class.can_handle_model(model): From 3a05157b3cf65a5b4057c3504f5cd10ed0ea98a2 Mon Sep 17 00:00:00 2001 From: Rohan Sen Date: Fri, 2 Jan 2026 15:53:57 +0530 Subject: [PATCH 16/46] refactor: updated OpenMLEvaluation to use dataclass decorator (#1559) I have Refactored the `OpenMLEvaluation` class from a traditional Python class to use the `@dataclass` decorator to reduce boilerplate code and improve code maintainability. #### Metadata * Reference Issue: #1540 * New Tests Added: No * Documentation Updated: No * Change Log Entry: Refactored the `OpenMLEvaluation` class to use the `@dataclass` #### Details Edited the `OpenMLEvaluation` class in `openml\evaluations\evaluation.py` to use `@dataclass` decorator. This significantly reduces the boilerplate code in the following places: - Instance Variable Definitions **Before:** ```python def __init__( self, run_id: int, task_id: int, setup_id: int, flow_id: int, flow_name: str, data_id: int, data_name: str, function: str, upload_time: str, uploader: int, uploader_name: str, value: float | None, values: list[float] | None, array_data: str | None = None, ): self.run_id = run_id self.task_id = task_id self.setup_id = setup_id self.flow_id = flow_id self.flow_name = flow_name self.data_id = data_id self.data_name = data_name self.function = function self.upload_time = upload_time self.uploader = uploader self.uploader_name = uploader_name self.value = value self.values = values self.array_data = array_data ``` **After:** ```python run_id: int task_id: int setup_id: int flow_id: int flow_name: str data_id: int data_name: str function: str upload_time: str uploader: int uploader_name: str value: float | None values: list[float] | None array_data: str | None = None ``` - _to_dict Method Simplification **Before:** ```python def _to_dict(self) -> dict: return { "run_id": self.run_id, "task_id": self.task_id, "setup_id": self.setup_id, "flow_id": self.flow_id, "flow_name": self.flow_name, "data_id": self.data_id, "data_name": self.data_name, "function": self.function, "upload_time": self.upload_time, "uploader": self.uploader, "uploader_name": self.uploader_name, "value": self.value, "values": self.values, "array_data": self.array_data, } ``` **After:** ```python def _to_dict(self) -> dict: return asdict(self) ``` All tests are passing with accordnce to the changes: ```bash PS C:\Users\ASUS\Documents\work\opensource\openml-python> pytest tests/test_evaluations/ ======================================= test session starts ======================================= platform win32 -- Python 3.14.0, pytest-9.0.2, pluggy-1.6.0 rootdir: C:\Users\ASUS\Documents\work\opensource\openml-python configfile: pyproject.toml plugins: anyio-4.12.0, flaky-3.8.1, asyncio-1.3.0, cov-7.0.0, mock-3.15.1, rerunfailures-16.1, timeout-2.4.0, xdist-3.8.0, requests-mock-1.12.1 asyncio: mode=Mode.STRICT, debug=False, asyncio_default_fixture_loop_scope=None, asyncio_default_test_loop_scope=function collected 13 items tests\test_evaluations\test_evaluation_functions.py ............ [ 92%] tests\test_evaluations\test_evaluations_example.py . [100%] ================================= 13 passed in 274.80s (0:04:34) ================================== ``` --- openml/evaluations/evaluation.py | 72 ++++++++++---------------------- 1 file changed, 21 insertions(+), 51 deletions(-) diff --git a/openml/evaluations/evaluation.py b/openml/evaluations/evaluation.py index 6d69d377e..5db087024 100644 --- a/openml/evaluations/evaluation.py +++ b/openml/evaluations/evaluation.py @@ -1,6 +1,8 @@ # License: BSD 3-Clause from __future__ import annotations +from dataclasses import asdict, dataclass + import openml.config import openml.datasets import openml.flows @@ -8,8 +10,7 @@ import openml.tasks -# TODO(eddiebergman): A lot of this class is automatically -# handled by a dataclass +@dataclass class OpenMLEvaluation: """ Contains all meta-information about a run / evaluation combination, @@ -48,55 +49,23 @@ class OpenMLEvaluation: (e.g., in case of precision, auroc, recall) """ - def __init__( # noqa: PLR0913 - self, - run_id: int, - task_id: int, - setup_id: int, - flow_id: int, - flow_name: str, - data_id: int, - data_name: str, - function: str, - upload_time: str, - uploader: int, - uploader_name: str, - value: float | None, - values: list[float] | None, - array_data: str | None = None, - ): - self.run_id = run_id - self.task_id = task_id - self.setup_id = setup_id - self.flow_id = flow_id - self.flow_name = flow_name - self.data_id = data_id - self.data_name = data_name - self.function = function - self.upload_time = upload_time - self.uploader = uploader - self.uploader_name = uploader_name - self.value = value - self.values = values - self.array_data = array_data + run_id: int + task_id: int + setup_id: int + flow_id: int + flow_name: str + data_id: int + data_name: str + function: str + upload_time: str + uploader: int + uploader_name: str + value: float | None + values: list[float] | None + array_data: str | None = None def _to_dict(self) -> dict: - return { - "run_id": self.run_id, - "task_id": self.task_id, - "setup_id": self.setup_id, - "flow_id": self.flow_id, - "flow_name": self.flow_name, - "data_id": self.data_id, - "data_name": self.data_name, - "function": self.function, - "upload_time": self.upload_time, - "uploader": self.uploader, - "uploader_name": self.uploader_name, - "value": self.value, - "values": self.values, - "array_data": self.array_data, - } + return asdict(self) def __repr__(self) -> str: header = "OpenML Evaluation" @@ -119,11 +88,12 @@ def __repr__(self) -> str: } order = [ - "Uploader Date", + "Upload Date", "Run ID", "OpenML Run URL", "Task ID", - "OpenML Task URL" "Flow ID", + "OpenML Task URL", + "Flow ID", "OpenML Flow URL", "Setup ID", "Data ID", From 3454bbbad163668a30de5a5254971102316f1ee7 Mon Sep 17 00:00:00 2001 From: DDiyash <149958769+DDiyash@users.noreply.github.com> Date: Fri, 2 Jan 2026 18:34:10 +0530 Subject: [PATCH 17/46] [MNT] Update Python version support and CI to include Python 3.14 (#1566) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### Metadata * Reference Issue: Fixes #1531 * New Tests Added: No * Documentation Updated: Yes * Change Log Entry: Update supported Python version range to 3.10–3.14 and extend CI testing to Python 3.14 #### Details This pull request updates the officially supported Python version range for openml-python from 3.8–3.13 to 3.10–3.14, in line with currently supported Python releases. The following changes were made: Updated pyproject.toml to reflect the new supported Python range (3.10–3.14). Extended GitHub Actions CI workflows (test.yml, dist.yaml, docs.yaml) to include Python 3.14. Updated documentation (README.md) wherever Python version support is mentioned. No new functionality or tests were introduced; this is a maintenance update to keep Python version support and CI configuration up to date. This change ensures that users and contributors can use and test openml-python on the latest supported Python versions. --- .github/workflows/dist.yaml | 2 +- .github/workflows/docs.yaml | 2 +- .github/workflows/test.yml | 15 +++++++++++++-- .gitignore | 12 +++++++++++- README.md | 4 ++-- pyproject.toml | 2 +- 6 files changed, 29 insertions(+), 8 deletions(-) diff --git a/.github/workflows/dist.yaml b/.github/workflows/dist.yaml index 0d2adc9ee..ecf6f0a7f 100644 --- a/.github/workflows/dist.yaml +++ b/.github/workflows/dist.yaml @@ -27,7 +27,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: "3.10" - name: Build dist run: | pip install build diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index acce766ea..1a5a36a87 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -28,7 +28,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: "3.10" - name: Install dependencies run: | pip install -e .[docs,examples] diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b77cfd38c..850abdfe7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,3 +1,4 @@ +--- name: Tests on: @@ -21,13 +22,13 @@ concurrency: jobs: test: - name: (${{ matrix.os }}, Py${{ matrix.python-version }}, sk${{ matrix.scikit-learn }}, sk-only:${{ matrix.sklearn-only }}) + name: (${{ matrix.os }},Py${{ matrix.python-version }},sk${{ matrix.scikit-learn }},sk-only:${{ matrix.sklearn-only }}) runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] scikit-learn: ["1.3.*", "1.4.*", "1.5.*", "1.6.*", "1.7.*"] os: [ubuntu-latest] sklearn-only: ["true"] @@ -38,8 +39,18 @@ jobs: scikit-learn: "1.3.*" - python-version: "3.13" scikit-learn: "1.4.*" + - python-version: "3.14" + scikit-learn: "1.3.*" + - python-version: "3.14" + scikit-learn: "1.4.*" include: + # Full test run on ubuntu, 3.14 + - os: ubuntu-latest + python-version: "3.14" + scikit-learn: "1.7.*" + sklearn-only: "false" + # Full test run on Windows - os: windows-latest python-version: "3.12" diff --git a/.gitignore b/.gitignore index 92679e5ca..d512c0ee6 100644 --- a/.gitignore +++ b/.gitignore @@ -98,7 +98,17 @@ dmypy.sock # Tests .pytest_cache + +# Virtual environments +oenv/ +venv/ +.env/ .venv +.venv/ + +# Python cache +__pycache__/ +*.pyc # Ruff -.ruff-cache/ \ No newline at end of file +.ruff-cache/ diff --git a/README.md b/README.md index e8df97ad6..c44e42981 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ ## The Python API for a World of Data and More :dizzy: [![Latest Release](https://img.shields.io/github/v/release/openml/openml-python)](https://github.com/openml/openml-python/releases) -[![Python Versions](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue)](https://pypi.org/project/openml/) +[![Python Versions](https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12%20%7C%203.13%20%7C%203.14-blue)](https://pypi.org/project/openml/) [![Downloads](https://static.pepy.tech/badge/openml)](https://pepy.tech/project/openml) [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) @@ -60,7 +60,7 @@ for task_id in suite.tasks: ## :magic_wand: Installation -OpenML-Python is supported on Python 3.8 - 3.13 and is available on Linux, MacOS, and Windows. +OpenML-Python is supported on Python 3.10 - 3.14 and is available on Linux, MacOS, and Windows. You can install OpenML-Python with: diff --git a/pyproject.toml b/pyproject.toml index ede204ca0..14309c2d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "pyarrow", "tqdm", # For MinIO download progress bars ] -requires-python = ">=3.8" +requires-python = ">=3.10,<3.15" maintainers = [ { name = "Pieter Gijsbers", email="p.gijsbers@tue.nl"}, { name = "Lennart Purucker"}, From c5f68bf15e1b18ee8593de6435120ed5c0dd1971 Mon Sep 17 00:00:00 2001 From: Armaghan Shakir Date: Thu, 8 Jan 2026 04:07:23 +0500 Subject: [PATCH 18/46] [MNT] add pytest marker to tests requiring test server (#1599) Fixes https://github.com/openml/openml-python/issues/1598 This PR adds the `@pytest.mark.uses_test_server()` marker to tests that depend on the OpenML test server. Changes * added `uses_test_server` on the relevant test sets. * replaced all the `server` markers with `uses_test_server` marker * removed all the `@pytest.mark.xfail(reason="failures_issue_1544", strict=False)` where the failure was due to race-conditions or server connectivity --- .github/workflows/test.yml | 10 +-- tests/test_datasets/test_dataset.py | 9 ++- tests/test_datasets/test_dataset_functions.py | 67 ++++++++++++++++--- .../test_evaluation_functions.py | 2 + tests/test_flows/test_flow.py | 13 ++-- tests/test_flows/test_flow_functions.py | 7 +- tests/test_openml/test_api_calls.py | 3 + tests/test_runs/test_run.py | 11 +-- tests/test_runs/test_run_functions.py | 65 +++++++++--------- tests/test_setups/test_setup_functions.py | 11 +-- tests/test_study/test_study_functions.py | 5 ++ tests/test_tasks/test_classification_task.py | 7 +- tests/test_tasks/test_clustering_task.py | 2 + tests/test_tasks/test_learning_curve_task.py | 6 +- tests/test_tasks/test_regression_task.py | 4 +- tests/test_tasks/test_supervised_task.py | 1 + tests/test_tasks/test_task.py | 3 +- tests/test_tasks/test_task_functions.py | 18 ++++- tests/test_tasks/test_task_methods.py | 2 + tests/test_utils/test_utils.py | 20 +++--- 20 files changed, 183 insertions(+), 83 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 850abdfe7..d65cc3796 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -98,9 +98,9 @@ jobs: fi if [ "${{ matrix.sklearn-only }}" = "true" ]; then - marks="sklearn and not production" + marks="sklearn and not production and not uses_test_server" else - marks="not production" + marks="not production and not uses_test_server" fi pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks" @@ -113,9 +113,9 @@ jobs: fi if [ "${{ matrix.sklearn-only }}" = "true" ]; then - marks="sklearn and production" + marks="sklearn and production and not uses_test_server" else - marks="production" + marks="production and not uses_test_server" fi pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks" @@ -123,7 +123,7 @@ jobs: - name: Run tests on Windows if: matrix.os == 'windows-latest' run: | # we need a separate step because of the bash-specific if-statement in the previous one. - pytest -n 4 --durations=20 --dist load -sv --reruns 5 --reruns-delay 1 + pytest -n 4 --durations=20 --dist load -sv --reruns 5 --reruns-delay 1 -m "not uses_test_server" - name: Check for files left behind by test if: matrix.os != 'windows-latest' && always() diff --git a/tests/test_datasets/test_dataset.py b/tests/test_datasets/test_dataset.py index 66e9b8554..6dc4c7d5d 100644 --- a/tests/test_datasets/test_dataset.py +++ b/tests/test_datasets/test_dataset.py @@ -278,6 +278,7 @@ def test_equality_comparison(self): self.assertNotEqual(self.titanic, "Wrong_object") +@pytest.mark.uses_test_server() def test_tagging(): dataset = openml.datasets.get_dataset(125, download_data=False) @@ -294,7 +295,7 @@ def test_tagging(): datasets = openml.datasets.list_datasets(tag=tag) assert datasets.empty -@pytest.mark.xfail(reason="failures_issue_1544") +@pytest.mark.uses_test_server() def test_get_feature_with_ontology_data_id_11(): # test on car dataset, which has built-in ontology references dataset = openml.datasets.get_dataset(11) @@ -303,6 +304,7 @@ def test_get_feature_with_ontology_data_id_11(): assert len(dataset.features[2].ontologies) >= 1 assert len(dataset.features[3].ontologies) >= 1 +@pytest.mark.uses_test_server() def test_add_remove_ontology_to_dataset(): did = 1 feature_index = 1 @@ -310,6 +312,7 @@ def test_add_remove_ontology_to_dataset(): openml.datasets.functions.data_feature_add_ontology(did, feature_index, ontology) openml.datasets.functions.data_feature_remove_ontology(did, feature_index, ontology) +@pytest.mark.uses_test_server() def test_add_same_ontology_multiple_features(): did = 1 ontology = "https://www.openml.org/unittest/" + str(time()) @@ -318,6 +321,7 @@ def test_add_same_ontology_multiple_features(): openml.datasets.functions.data_feature_add_ontology(did, i, ontology) +@pytest.mark.uses_test_server() def test_add_illegal_long_ontology(): did = 1 ontology = "http://www.google.com/" + ("a" * 257) @@ -329,6 +333,7 @@ def test_add_illegal_long_ontology(): +@pytest.mark.uses_test_server() def test_add_illegal_url_ontology(): did = 1 ontology = "not_a_url" + str(time()) @@ -400,6 +405,7 @@ def test_get_sparse_categorical_data_id_395(self): assert len(feature.nominal_values) == 25 +@pytest.mark.uses_test_server() def test__read_features(mocker, workdir, static_cache_dir): """Test we read the features from the xml if no cache pickle is available. This test also does some simple checks to verify that the features are read correctly @@ -431,6 +437,7 @@ def test__read_features(mocker, workdir, static_cache_dir): assert pickle_mock.dump.call_count == 1 +@pytest.mark.uses_test_server() def test__read_qualities(static_cache_dir, workdir, mocker): """Test we read the qualities from the xml if no cache pickle is available. This test also does some minor checks to ensure that the qualities are read correctly. diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index f8cb1943c..c41664ba7 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -107,6 +107,7 @@ def _check_datasets(self, datasets): for did in datasets: self._check_dataset(datasets[did]) + @pytest.mark.uses_test_server() def test_tag_untag_dataset(self): tag = "test_tag_%d" % random.randint(1, 1000000) all_tags = _tag_entity("data", 1, tag) @@ -114,10 +115,12 @@ def test_tag_untag_dataset(self): all_tags = _tag_entity("data", 1, tag, untag=True) assert tag not in all_tags + @pytest.mark.uses_test_server() def test_list_datasets_length(self): datasets = openml.datasets.list_datasets() assert len(datasets) >= 100 + @pytest.mark.uses_test_server() def test_list_datasets_paginate(self): size = 10 max = 100 @@ -132,6 +135,7 @@ def test_list_datasets_paginate(self): categories=["in_preparation", "active", "deactivated"], ) + @pytest.mark.uses_test_server() def test_list_datasets_empty(self): datasets = openml.datasets.list_datasets(tag="NoOneWouldUseThisTagAnyway") assert datasets.empty @@ -155,6 +159,7 @@ def test_check_datasets_active(self): ) openml.config.server = self.test_server + @pytest.mark.uses_test_server() def test_illegal_character_tag(self): dataset = openml.datasets.get_dataset(1) tag = "illegal_tag&" @@ -164,6 +169,7 @@ def test_illegal_character_tag(self): except openml.exceptions.OpenMLServerException as e: assert e.code == 477 + @pytest.mark.uses_test_server() def test_illegal_length_tag(self): dataset = openml.datasets.get_dataset(1) tag = "a" * 65 @@ -205,6 +211,7 @@ def test__name_to_id_with_multiple_active_error(self): error_if_multiple=True, ) + @pytest.mark.uses_test_server() def test__name_to_id_name_does_not_exist(self): """With multiple active datasets, retrieve the least recent active.""" self.assertRaisesRegex( @@ -214,6 +221,7 @@ def test__name_to_id_name_does_not_exist(self): dataset_name="does_not_exist", ) + @pytest.mark.uses_test_server() def test__name_to_id_version_does_not_exist(self): """With multiple active datasets, retrieve the least recent active.""" self.assertRaisesRegex( @@ -224,6 +232,7 @@ def test__name_to_id_version_does_not_exist(self): version=100000, ) + @pytest.mark.uses_test_server() def test_get_datasets_by_name(self): # did 1 and 2 on the test server: dids = ["anneal", "kr-vs-kp"] @@ -231,6 +240,7 @@ def test_get_datasets_by_name(self): assert len(datasets) == 2 _assert_datasets_retrieved_successfully([1, 2]) + @pytest.mark.uses_test_server() def test_get_datasets_by_mixed(self): # did 1 and 2 on the test server: dids = ["anneal", 2] @@ -238,12 +248,14 @@ def test_get_datasets_by_mixed(self): assert len(datasets) == 2 _assert_datasets_retrieved_successfully([1, 2]) + @pytest.mark.uses_test_server() def test_get_datasets(self): dids = [1, 2] datasets = openml.datasets.get_datasets(dids) assert len(datasets) == 2 _assert_datasets_retrieved_successfully([1, 2]) + @pytest.mark.uses_test_server() def test_get_dataset_by_name(self): dataset = openml.datasets.get_dataset("anneal") assert type(dataset) == OpenMLDataset @@ -262,6 +274,7 @@ def test_get_dataset_download_all_files(self): # test_get_dataset_lazy raise NotImplementedError + @pytest.mark.uses_test_server() def test_get_dataset_uint8_dtype(self): dataset = openml.datasets.get_dataset(1) assert type(dataset) == OpenMLDataset @@ -280,7 +293,7 @@ def test_dataset_by_name_cannot_access_private_data(self): self.use_production_server() self.assertRaises(OpenMLPrivateDatasetError, openml.datasets.get_dataset, "NAME_GOES_HERE") - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_get_dataset_lazy_all_functions(self): """Test that all expected functionality is available without downloading the dataset.""" dataset = openml.datasets.get_dataset(1) @@ -310,24 +323,28 @@ def ensure_absence_of_real_data(): assert classes == ["1", "2", "3", "4", "5", "U"] ensure_absence_of_real_data() + @pytest.mark.uses_test_server() def test_get_dataset_sparse(self): dataset = openml.datasets.get_dataset(102) X, *_ = dataset.get_data() assert isinstance(X, pd.DataFrame) assert all(isinstance(col, pd.SparseDtype) for col in X.dtypes) + @pytest.mark.uses_test_server() def test_download_rowid(self): # Smoke test which checks that the dataset has the row-id set correctly did = 44 dataset = openml.datasets.get_dataset(did) assert dataset.row_id_attribute == "Counter" + @pytest.mark.uses_test_server() def test__get_dataset_description(self): description = _get_dataset_description(self.workdir, 2) assert isinstance(description, dict) description_xml_path = os.path.join(self.workdir, "description.xml") assert os.path.exists(description_xml_path) + @pytest.mark.uses_test_server() def test__getarff_path_dataset_arff(self): openml.config.set_root_cache_directory(self.static_cache_dir) description = _get_dataset_description(self.workdir, 2) @@ -391,6 +408,7 @@ def test__download_minio_file_works_with_bucket_subdirectory(self): @mock.patch("openml._api_calls._download_minio_file") + @pytest.mark.uses_test_server() def test__get_dataset_parquet_is_cached(self, patch): openml.config.set_root_cache_directory(self.static_cache_dir) patch.side_effect = RuntimeError( @@ -431,18 +449,21 @@ def test__getarff_md5_issue(self): openml.config.connection_n_retries = n + @pytest.mark.uses_test_server() def test__get_dataset_features(self): features_file = _get_dataset_features_file(self.workdir, 2) assert isinstance(features_file, Path) features_xml_path = self.workdir / "features.xml" assert features_xml_path.exists() + @pytest.mark.uses_test_server() def test__get_dataset_qualities(self): qualities = _get_dataset_qualities_file(self.workdir, 2) assert isinstance(qualities, Path) qualities_xml_path = self.workdir / "qualities.xml" assert qualities_xml_path.exists() + @pytest.mark.uses_test_server() def test_get_dataset_force_refresh_cache(self): did_cache_dir = _create_cache_directory_for_id( DATASETS_CACHE_DIR_NAME, @@ -465,6 +486,7 @@ def test_get_dataset_force_refresh_cache(self): did_cache_dir, ) + @pytest.mark.uses_test_server() def test_get_dataset_force_refresh_cache_clean_start(self): did_cache_dir = _create_cache_directory_for_id( DATASETS_CACHE_DIR_NAME, @@ -501,12 +523,14 @@ def test_deletion_of_cache_dir(self): # get_dataset_description is the only data guaranteed to be downloaded @mock.patch("openml.datasets.functions._get_dataset_description") + @pytest.mark.uses_test_server() def test_deletion_of_cache_dir_faulty_download(self, patch): patch.side_effect = Exception("Boom!") self.assertRaisesRegex(Exception, "Boom!", openml.datasets.get_dataset, dataset_id=1) datasets_cache_dir = os.path.join(self.workdir, "org", "openml", "test", "datasets") assert len(os.listdir(datasets_cache_dir)) == 0 + @pytest.mark.uses_test_server() def test_publish_dataset(self): # lazy loading not possible as we need the arff-file. openml.datasets.get_dataset(3, download_data=True) @@ -532,6 +556,7 @@ def test_publish_dataset(self): ) assert isinstance(dataset.dataset_id, int) + @pytest.mark.uses_test_server() def test__retrieve_class_labels(self): openml.config.set_root_cache_directory(self.static_cache_dir) labels = openml.datasets.get_dataset(2).retrieve_class_labels() @@ -548,6 +573,7 @@ def test__retrieve_class_labels(self): labels = custom_ds.retrieve_class_labels(target_name=custom_ds.features[31].name) assert labels == ["COIL", "SHEET"] + @pytest.mark.uses_test_server() def test_upload_dataset_with_url(self): dataset = OpenMLDataset( f"{self._get_sentinel()}-UploadTestWithURL", @@ -574,6 +600,7 @@ def _assert_status_of_dataset(self, *, did: int, status: str): assert result[did]["status"] == status @pytest.mark.flaky() + @pytest.mark.uses_test_server() def test_data_status(self): dataset = OpenMLDataset( f"{self._get_sentinel()}-UploadTestWithURL", @@ -665,7 +692,7 @@ def test_attributes_arff_from_df_unknown_dtype(self): with pytest.raises(ValueError, match=err_msg): attributes_arff_from_df(df) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_create_dataset_numpy(self): data = np.array([[1, 2, 3], [1.2, 2.5, 3.8], [2, 5, 8], [0, 1, 0]]).T @@ -699,6 +726,7 @@ def test_create_dataset_numpy(self): ), "Uploaded arff does not match original one" assert _get_online_dataset_format(dataset.id) == "arff", "Wrong format for dataset" + @pytest.mark.uses_test_server() def test_create_dataset_list(self): data = [ ["a", "sunny", 85.0, 85.0, "FALSE", "no"], @@ -753,7 +781,7 @@ def test_create_dataset_list(self): ), "Uploaded ARFF does not match original one" assert _get_online_dataset_format(dataset.id) == "arff", "Wrong format for dataset" - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_create_dataset_sparse(self): # test the scipy.sparse.coo_matrix sparse_data = scipy.sparse.coo_matrix( @@ -856,6 +884,7 @@ def test_create_invalid_dataset(self): param["data"] = data[0] self.assertRaises(ValueError, create_dataset, **param) + @pytest.mark.uses_test_server() def test_get_online_dataset_arff(self): dataset_id = 100 # Australian # lazy loading not used as arff file is checked. @@ -871,7 +900,7 @@ def test_get_online_dataset_arff(self): return_type=arff.DENSE if d_format == "arff" else arff.COO, ), "ARFF files are not equal" - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_topic_api_error(self): # Check server exception when non-admin accessses apis self.assertRaisesRegex( @@ -890,6 +919,7 @@ def test_topic_api_error(self): topic="business", ) + @pytest.mark.uses_test_server() def test_get_online_dataset_format(self): # Phoneme dataset dataset_id = 77 @@ -899,7 +929,7 @@ def test_get_online_dataset_format(self): dataset_id ), "The format of the ARFF files is different" - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_create_dataset_pandas(self): data = [ ["a", "sunny", 85.0, 85.0, "FALSE", "no"], @@ -1124,7 +1154,7 @@ def test_ignore_attributes_dataset(self): paper_url=paper_url, ) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_publish_fetch_ignore_attribute(self): """Test to upload and retrieve dataset and check ignore_attributes""" data = [ @@ -1243,7 +1273,7 @@ def test_create_dataset_row_id_attribute_error(self): paper_url=paper_url, ) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_create_dataset_row_id_attribute_inference(self): # meta-information name = f"{self._get_sentinel()}-pandas_testing_dataset" @@ -1334,11 +1364,13 @@ def test_create_dataset_attributes_auto_without_df(self): paper_url=paper_url, ) + @pytest.mark.uses_test_server() def test_list_qualities(self): qualities = openml.datasets.list_qualities() assert isinstance(qualities, list) is True assert all(isinstance(q, str) for q in qualities) is True + @pytest.mark.uses_test_server() def test_get_dataset_cache_format_pickle(self): dataset = openml.datasets.get_dataset(1) dataset.get_data() @@ -1354,6 +1386,7 @@ def test_get_dataset_cache_format_pickle(self): assert len(categorical) == X.shape[1] assert len(attribute_names) == X.shape[1] + @pytest.mark.uses_test_server() def test_get_dataset_cache_format_feather(self): # This test crashed due to using the parquet file by default, which is downloaded # from minio. However, there is a mismatch between OpenML test server and minio IDs. @@ -1386,6 +1419,7 @@ def test_get_dataset_cache_format_feather(self): assert len(categorical) == X.shape[1] assert len(attribute_names) == X.shape[1] + @pytest.mark.uses_test_server() def test_data_edit_non_critical_field(self): # Case 1 # All users can edit non-critical fields of datasets @@ -1407,7 +1441,7 @@ def test_data_edit_non_critical_field(self): edited_dataset = openml.datasets.get_dataset(did) assert edited_dataset.description == desc - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_data_edit_critical_field(self): # Case 2 # only owners (or admin) can edit all critical fields of datasets @@ -1434,6 +1468,7 @@ def test_data_edit_critical_field(self): os.path.join(self.workdir, "org", "openml", "test", "datasets", str(did)), ) + @pytest.mark.uses_test_server() def test_data_edit_requires_field(self): # Check server exception when no field to edit is provided self.assertRaisesRegex( @@ -1446,6 +1481,7 @@ def test_data_edit_requires_field(self): data_id=64, # blood-transfusion-service-center ) + @pytest.mark.uses_test_server() def test_data_edit_requires_valid_dataset(self): # Check server exception when unknown dataset is provided self.assertRaisesRegex( @@ -1456,7 +1492,7 @@ def test_data_edit_requires_valid_dataset(self): description="xor operation dataset", ) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_data_edit_cannot_edit_critical_field_if_dataset_has_task(self): # Need to own a dataset to be able to edit meta-data # Will be creating a forked version of an existing dataset to allow the unit test user @@ -1483,6 +1519,7 @@ def test_data_edit_cannot_edit_critical_field_if_dataset_has_task(self): default_target_attribute="y", ) + @pytest.mark.uses_test_server() def test_edit_data_user_cannot_edit_critical_field_of_other_users_dataset(self): # Check server exception when a non-owner or non-admin tries to edit critical fields self.assertRaisesRegex( @@ -1494,6 +1531,7 @@ def test_edit_data_user_cannot_edit_critical_field_of_other_users_dataset(self): default_target_attribute="y", ) + @pytest.mark.uses_test_server() def test_data_fork(self): did = 1 result = fork_dataset(did) @@ -1785,6 +1823,7 @@ def all_datasets(): return openml.datasets.list_datasets() +@pytest.mark.uses_test_server() def test_list_datasets(all_datasets: pd.DataFrame): # We can only perform a smoke test here because we test on dynamic # data from the internet... @@ -1793,42 +1832,49 @@ def test_list_datasets(all_datasets: pd.DataFrame): _assert_datasets_have_id_and_valid_status(all_datasets) +@pytest.mark.uses_test_server() def test_list_datasets_by_tag(all_datasets: pd.DataFrame): tag_datasets = openml.datasets.list_datasets(tag="study_14") assert 0 < len(tag_datasets) < len(all_datasets) _assert_datasets_have_id_and_valid_status(tag_datasets) +@pytest.mark.uses_test_server() def test_list_datasets_by_size(): datasets = openml.datasets.list_datasets(size=5) assert len(datasets) == 5 _assert_datasets_have_id_and_valid_status(datasets) +@pytest.mark.uses_test_server() def test_list_datasets_by_number_instances(all_datasets: pd.DataFrame): small_datasets = openml.datasets.list_datasets(number_instances="5..100") assert 0 < len(small_datasets) <= len(all_datasets) _assert_datasets_have_id_and_valid_status(small_datasets) +@pytest.mark.uses_test_server() def test_list_datasets_by_number_features(all_datasets: pd.DataFrame): wide_datasets = openml.datasets.list_datasets(number_features="50..100") assert 8 <= len(wide_datasets) < len(all_datasets) _assert_datasets_have_id_and_valid_status(wide_datasets) +@pytest.mark.uses_test_server() def test_list_datasets_by_number_classes(all_datasets: pd.DataFrame): five_class_datasets = openml.datasets.list_datasets(number_classes="5") assert 3 <= len(five_class_datasets) < len(all_datasets) _assert_datasets_have_id_and_valid_status(five_class_datasets) +@pytest.mark.uses_test_server() def test_list_datasets_by_number_missing_values(all_datasets: pd.DataFrame): na_datasets = openml.datasets.list_datasets(number_missing_values="5..100") assert 5 <= len(na_datasets) < len(all_datasets) _assert_datasets_have_id_and_valid_status(na_datasets) +@pytest.mark.uses_test_server() def test_list_datasets_combined_filters(all_datasets: pd.DataFrame): combined_filter_datasets = openml.datasets.list_datasets( tag="study_14", @@ -1901,6 +1947,7 @@ def isolate_for_test(): ("with_data", "with_qualities", "with_features"), itertools.product([True, False], repeat=3), ) +@pytest.mark.uses_test_server() def test_get_dataset_lazy_behavior( isolate_for_test, with_data: bool, with_qualities: bool, with_features: bool ): @@ -1927,6 +1974,7 @@ def test_get_dataset_lazy_behavior( ) +@pytest.mark.uses_test_server() def test_get_dataset_with_invalid_id() -> None: INVALID_ID = 123819023109238 # Well, at some point this will probably be valid... with pytest.raises(OpenMLServerNoResult, match="Unknown dataset") as e: @@ -1954,6 +2002,7 @@ def test_read_features_from_xml_with_whitespace() -> None: assert dict[1].nominal_values == [" - 50000.", " 50000+."] +@pytest.mark.uses_test_server() def test_get_dataset_parquet(requests_mock, test_files_directory): # Parquet functionality is disabled on the test server # There is no parquet-copy of the test server yet. diff --git a/tests/test_evaluations/test_evaluation_functions.py b/tests/test_evaluations/test_evaluation_functions.py index ffd3d9f78..7009217d6 100644 --- a/tests/test_evaluations/test_evaluation_functions.py +++ b/tests/test_evaluations/test_evaluation_functions.py @@ -155,6 +155,7 @@ def test_evaluation_list_limit(self): ) assert len(evaluations) == 100 + @pytest.mark.uses_test_server() def test_list_evaluations_empty(self): evaluations = openml.evaluations.list_evaluations("unexisting_measure") if len(evaluations) > 0: @@ -232,6 +233,7 @@ def test_evaluation_list_sort(self): test_output = sorted(unsorted_output, reverse=True) assert test_output[:size] == sorted_output + @pytest.mark.uses_test_server() def test_list_evaluation_measures(self): measures = openml.evaluations.list_evaluation_measures() assert isinstance(measures, list) is True diff --git a/tests/test_flows/test_flow.py b/tests/test_flows/test_flow.py index da719d058..99cee6f87 100644 --- a/tests/test_flows/test_flow.py +++ b/tests/test_flows/test_flow.py @@ -102,6 +102,7 @@ def test_get_structure(self): subflow = flow.get_subflow(structure) assert subflow.flow_id == sub_flow_id + @pytest.mark.uses_test_server() def test_tagging(self): flows = openml.flows.list_flows(size=1) flow_id = flows["id"].iloc[0] @@ -119,6 +120,7 @@ def test_tagging(self): flows = openml.flows.list_flows(tag=tag) assert len(flows) == 0 + @pytest.mark.uses_test_server() def test_from_xml_to_xml(self): # Get the raw xml thing # TODO maybe get this via get_flow(), which would have to be refactored @@ -178,7 +180,7 @@ def test_to_xml_from_xml(self): assert new_flow is not flow @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_publish_flow(self): flow = openml.OpenMLFlow( name="sklearn.dummy.DummyClassifier", @@ -220,7 +222,7 @@ def test_publish_existing_flow(self, flow_exists_mock): ) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_publish_flow_with_similar_components(self): clf = sklearn.ensemble.VotingClassifier( [("lr", sklearn.linear_model.LogisticRegression(solver="lbfgs"))], @@ -271,7 +273,7 @@ def test_publish_flow_with_similar_components(self): TestBase.logger.info(f"collected from {__file__.split('/')[-1]}: {flow3.flow_id}") @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_semi_legal_flow(self): # TODO: Test if parameters are set correctly! # should not throw error as it contains two differentiable forms of @@ -363,6 +365,7 @@ def test_illegal_flow(self): ) self.assertRaises(ValueError, self.extension.model_to_flow, illegal) + @pytest.mark.uses_test_server() def test_nonexisting_flow_exists(self): def get_sentinel(): # Create a unique prefix for the flow. Necessary because the flow @@ -380,7 +383,7 @@ def get_sentinel(): assert not flow_id @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_existing_flow_exists(self): # create a flow nb = sklearn.naive_bayes.GaussianNB() @@ -421,7 +424,7 @@ def test_existing_flow_exists(self): assert downloaded_flow_id == flow.flow_id @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_sklearn_to_upload_to_flow(self): iris = sklearn.datasets.load_iris() X = iris.data diff --git a/tests/test_flows/test_flow_functions.py b/tests/test_flows/test_flow_functions.py index 0be65ceac..46bc36a94 100644 --- a/tests/test_flows/test_flow_functions.py +++ b/tests/test_flows/test_flow_functions.py @@ -274,12 +274,12 @@ def test_are_flows_equal_ignore_if_older(self): assert_flows_equal(flow, flow, ignore_parameter_values_on_older_children=None) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="OrdinalEncoder introduced in 0.20. " "No known models with list of lists parameters in older versions.", ) + @pytest.mark.uses_test_server() def test_sklearn_to_flow_list_of_lists(self): from sklearn.preprocessing import OrdinalEncoder @@ -308,6 +308,7 @@ def test_get_flow1(self): assert flow.external_version is None @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_get_flow_reinstantiate_model(self): model = ensemble.RandomForestClassifier(n_estimators=33) extension = openml.extensions.get_extension_by_model(model) @@ -319,6 +320,7 @@ def test_get_flow_reinstantiate_model(self): downloaded_flow = openml.flows.get_flow(flow.flow_id, reinstantiate=True) assert isinstance(downloaded_flow.model, sklearn.ensemble.RandomForestClassifier) + @pytest.mark.uses_test_server() def test_get_flow_reinstantiate_model_no_extension(self): # Flow 10 is a WEKA flow self.assertRaisesRegex( @@ -389,7 +391,7 @@ def test_get_flow_reinstantiate_flow_not_strict_pre_023(self): assert "sklearn==0.19.1" not in flow.dependencies @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_get_flow_id(self): if self.long_version: list_all = openml.utils._list_all @@ -424,6 +426,7 @@ def test_get_flow_id(self): pytest.skip(reason="Not sure why there should only be one version of this flow.") assert flow_ids_exact_version_True == flow_ids_exact_version_False + @pytest.mark.uses_test_server() def test_delete_flow(self): flow = openml.OpenMLFlow( name="sklearn.dummy.DummyClassifier", diff --git a/tests/test_openml/test_api_calls.py b/tests/test_openml/test_api_calls.py index da6857b6e..a295259ef 100644 --- a/tests/test_openml/test_api_calls.py +++ b/tests/test_openml/test_api_calls.py @@ -15,12 +15,14 @@ class TestConfig(openml.testing.TestBase): + @pytest.mark.uses_test_server() def test_too_long_uri(self): with pytest.raises(openml.exceptions.OpenMLServerError, match="URI too long!"): openml.datasets.list_datasets(data_id=list(range(10000))) @unittest.mock.patch("time.sleep") @unittest.mock.patch("requests.Session") + @pytest.mark.uses_test_server() def test_retry_on_database_error(self, Session_class_mock, _): response_mock = unittest.mock.Mock() response_mock.text = ( @@ -115,6 +117,7 @@ def test_download_minio_failure(mock_minio, tmp_path: Path) -> None: ("task/42", "delete"), # 460 ], ) +@pytest.mark.uses_test_server() def test_authentication_endpoints_requiring_api_key_show_relevant_help_link( endpoint: str, method: str, diff --git a/tests/test_runs/test_run.py b/tests/test_runs/test_run.py index 71651d431..1a66b76c0 100644 --- a/tests/test_runs/test_run.py +++ b/tests/test_runs/test_run.py @@ -25,6 +25,7 @@ class TestRun(TestBase): # Splitting not helpful, these test's don't rely on the server and take # less than 1 seconds + @pytest.mark.uses_test_server() def test_tagging(self): runs = openml.runs.list_runs(size=1) assert not runs.empty, "Test server state is incorrect" @@ -118,7 +119,7 @@ def _check_array(array, type_): assert run_prime_trace_content is None @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_to_from_filesystem_vanilla(self): model = Pipeline( [ @@ -154,7 +155,7 @@ def test_to_from_filesystem_vanilla(self): @pytest.mark.sklearn() @pytest.mark.flaky() - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_to_from_filesystem_search(self): model = Pipeline( [ @@ -189,7 +190,7 @@ def test_to_from_filesystem_search(self): ) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_to_from_filesystem_no_model(self): model = Pipeline( [("imputer", SimpleImputer(strategy="mean")), ("classifier", DummyClassifier())], @@ -295,7 +296,7 @@ def assert_run_prediction_data(task, run, model): assert_method(y_test, saved_y_test) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_publish_with_local_loaded_flow(self): """ Publish a run tied to a local flow after it has first been saved to @@ -339,7 +340,7 @@ def test_publish_with_local_loaded_flow(self): openml.runs.get_run(loaded_run.run_id) @pytest.mark.sklearn() - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_offline_and_online_run_identical(self): extension = SklearnExtension() diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 305d859d9..db54151d1 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -398,6 +398,7 @@ def _check_sample_evaluations( assert evaluation < max_time_allowed @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_run_regression_on_classif_task(self): task_id = 259 # collins; crossvalidation; has numeric targets @@ -413,8 +414,8 @@ def test_run_regression_on_classif_task(self): task=task, ) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_check_erronous_sklearn_flow_fails(self): task_id = 115 # diabetes; crossvalidation task = openml.tasks.get_task(task_id) @@ -626,8 +627,8 @@ def _run_and_upload_regression( sentinel=sentinel, ) - @pytest.mark.skip(reason="failures_issue_1544") @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_run_and_upload_logistic_regression(self): lr = LogisticRegression(solver="lbfgs", max_iter=1000) task_id = self.TEST_SERVER_TASK_SIMPLE["task_id"] @@ -635,8 +636,8 @@ def test_run_and_upload_logistic_regression(self): n_test_obs = self.TEST_SERVER_TASK_SIMPLE["n_test_obs"] self._run_and_upload_classification(lr, task_id, n_missing_vals, n_test_obs, "62501") - @pytest.mark.skip(reason="failures_issue_1544") @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_run_and_upload_linear_regression(self): lr = LinearRegression() task_id = self.TEST_SERVER_TASK_REGRESSION["task_id"] @@ -666,8 +667,8 @@ def test_run_and_upload_linear_regression(self): n_test_obs = self.TEST_SERVER_TASK_REGRESSION["n_test_obs"] self._run_and_upload_regression(lr, task_id, n_missing_vals, n_test_obs, "62501") - @pytest.mark.skip(reason="failures_issue_1544") @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_run_and_upload_pipeline_dummy_pipeline(self): pipeline1 = Pipeline( steps=[ @@ -680,12 +681,12 @@ def test_run_and_upload_pipeline_dummy_pipeline(self): n_test_obs = self.TEST_SERVER_TASK_SIMPLE["n_test_obs"] self._run_and_upload_classification(pipeline1, task_id, n_missing_vals, n_test_obs, "62501") - @pytest.mark.skip(reason="failures_issue_1544") @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="columntransformer introduction in 0.20.0", ) + @pytest.mark.uses_test_server() def test_run_and_upload_column_transformer_pipeline(self): import sklearn.compose import sklearn.impute @@ -745,7 +746,6 @@ def get_ct_cf(nominal_indices, numeric_indices): sentinel=sentinel, ) - @pytest.mark.skip(reason="failures_issue_1544") @pytest.mark.sklearn() @unittest.skip("https://github.com/openml/OpenML/issues/1180") @unittest.skipIf( @@ -798,8 +798,8 @@ def test_run_and_upload_knn_pipeline(self, warnings_mock): call_count += 1 assert call_count == 3 - @pytest.mark.skip(reason="failures_issue_1544") @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_run_and_upload_gridsearch(self): estimator_name = ( "base_estimator" if Version(sklearn.__version__) < Version("1.4") else "estimator" @@ -822,7 +822,7 @@ def test_run_and_upload_gridsearch(self): assert len(run.trace.trace_iterations) == 9 @pytest.mark.sklearn() - @pytest.mark.skip(reason="failures_issue_1544") + @pytest.mark.uses_test_server() def test_run_and_upload_randomsearch(self): randomsearch = RandomizedSearchCV( RandomForestClassifier(n_estimators=5), @@ -854,8 +854,8 @@ def test_run_and_upload_randomsearch(self): trace = openml.runs.get_run_trace(run.run_id) assert len(trace.trace_iterations) == 5 - @pytest.mark.skip(reason="failures_issue_1544") @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_run_and_upload_maskedarrays(self): # This testcase is important for 2 reasons: # 1) it verifies the correct handling of masked arrays (not all @@ -882,8 +882,8 @@ def test_run_and_upload_maskedarrays(self): ########################################################################## - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_learning_curve_task_1(self): task_id = 801 # diabates dataset num_test_instances = 6144 # for learning curve @@ -907,8 +907,8 @@ def test_learning_curve_task_1(self): ) self._check_sample_evaluations(run.sample_evaluations, num_repeats, num_folds, num_samples) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_learning_curve_task_2(self): task_id = 801 # diabates dataset num_test_instances = 6144 # for learning curve @@ -944,12 +944,12 @@ def test_learning_curve_task_2(self): ) self._check_sample_evaluations(run.sample_evaluations, num_repeats, num_folds, num_samples) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.21"), reason="Pipelines don't support indexing (used for the assert check)", ) + @pytest.mark.uses_test_server() def test_initialize_cv_from_run(self): randomsearch = Pipeline( [ @@ -1023,8 +1023,8 @@ def _test_local_evaluations(self, run): assert alt_scores[idx] >= 0 assert alt_scores[idx] <= 1 - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_local_run_swapped_parameter_order_model(self): clf = DecisionTreeClassifier() australian_task = 595 # Australian; crossvalidation @@ -1039,12 +1039,12 @@ def test_local_run_swapped_parameter_order_model(self): self._test_local_evaluations(run) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", ) + @pytest.mark.uses_test_server() def test_local_run_swapped_parameter_order_flow(self): # construct sci-kit learn classifier clf = Pipeline( @@ -1068,12 +1068,12 @@ def test_local_run_swapped_parameter_order_flow(self): self._test_local_evaluations(run) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", ) + @pytest.mark.uses_test_server() def test_local_run_metric_score(self): # construct sci-kit learn classifier clf = Pipeline( @@ -1106,12 +1106,12 @@ def test_online_run_metric_score(self): self._test_local_evaluations(run) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", ) + @pytest.mark.uses_test_server() def test_initialize_model_from_run(self): clf = sklearn.pipeline.Pipeline( steps=[ @@ -1168,12 +1168,12 @@ def test_initialize_model_from_run(self): assert flowS.components["Imputer"].parameters["strategy"] == '"most_frequent"' assert flowS.components["VarianceThreshold"].parameters["threshold"] == "0.05" - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", ) + @pytest.mark.uses_test_server() def test__run_exists(self): # would be better to not sentinel these clfs, # so we do not have to perform the actual runs @@ -1228,8 +1228,8 @@ def test__run_exists(self): run_ids = run_exists(task.task_id, setup_exists) assert run_ids, (run_ids, clf) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_run_with_illegal_flow_id(self): # check the case where the user adds an illegal flow id to a # non-existing flo @@ -1248,8 +1248,8 @@ def test_run_with_illegal_flow_id(self): avoid_duplicate_runs=True, ) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_run_with_illegal_flow_id_after_load(self): # Same as `test_run_with_illegal_flow_id`, but test this error is also # caught if the run is stored to and loaded from disk first. @@ -1281,6 +1281,7 @@ def test_run_with_illegal_flow_id_after_load(self): TestBase.logger.info(f"collected from test_run_functions: {loaded_run.run_id}") @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_run_with_illegal_flow_id_1(self): # Check the case where the user adds an illegal flow id to an existing # flow. Comes to a different value error than the previous test @@ -1305,8 +1306,8 @@ def test_run_with_illegal_flow_id_1(self): avoid_duplicate_runs=True, ) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_run_with_illegal_flow_id_1_after_load(self): # Same as `test_run_with_illegal_flow_id_1`, but test this error is # also caught if the run is stored to and loaded from disk first. @@ -1344,12 +1345,12 @@ def test_run_with_illegal_flow_id_1_after_load(self): loaded_run.publish, ) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="OneHotEncoder cannot handle mixed type DataFrame as input", ) + @pytest.mark.uses_test_server() def test__run_task_get_arffcontent(self): task = openml.tasks.get_task(7) # kr-vs-kp; crossvalidation num_instances = 3196 @@ -1450,6 +1451,7 @@ def test_get_runs_list(self): for run in runs.to_dict(orient="index").values(): self._check_run(run) + @pytest.mark.uses_test_server() def test_list_runs_empty(self): runs = openml.runs.list_runs(task=[0]) assert runs.empty @@ -1572,12 +1574,12 @@ def test_get_runs_list_by_tag(self): runs = openml.runs.list_runs(tag="curves", size=2) assert len(runs) >= 1 - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="columntransformer introduction in 0.20.0", ) + @pytest.mark.uses_test_server() def test_run_on_dataset_with_missing_labels_dataframe(self): # Check that _run_task_get_arffcontent works when one of the class # labels only declared in the arff file, but is not present in the @@ -1609,12 +1611,12 @@ def test_run_on_dataset_with_missing_labels_dataframe(self): # repeat, fold, row_id, 6 confidences, prediction and correct label assert len(row) == 12 - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="columntransformer introduction in 0.20.0", ) + @pytest.mark.uses_test_server() def test_run_on_dataset_with_missing_labels_array(self): # Check that _run_task_get_arffcontent works when one of the class # labels only declared in the arff file, but is not present in the @@ -1653,6 +1655,7 @@ def test_run_on_dataset_with_missing_labels_array(self): # repeat, fold, row_id, 6 confidences, prediction and correct label assert len(row) == 12 + @pytest.mark.uses_test_server() def test_get_cached_run(self): openml.config.set_root_cache_directory(self.static_cache_dir) openml.runs.functions._get_cached_run(1) @@ -1662,8 +1665,8 @@ def test_get_uncached_run(self): with pytest.raises(openml.exceptions.OpenMLCacheException): openml.runs.functions._get_cached_run(10) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_run_flow_on_task_downloaded_flow(self): model = sklearn.ensemble.RandomForestClassifier(n_estimators=33) flow = self.extension.model_to_flow(model) @@ -1694,6 +1697,7 @@ def test_format_prediction_non_supervised(self): ): format_prediction(clustering, *ignored_input) + @pytest.mark.uses_test_server() def test_format_prediction_classification_no_probabilities(self): classification = openml.tasks.get_task( self.TEST_SERVER_TASK_SIMPLE["task_id"], @@ -1703,7 +1707,7 @@ def test_format_prediction_classification_no_probabilities(self): with pytest.raises(ValueError, match="`proba` is required for classification task"): format_prediction(classification, *ignored_input, proba=None) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_format_prediction_classification_incomplete_probabilities(self): classification = openml.tasks.get_task( self.TEST_SERVER_TASK_SIMPLE["task_id"], @@ -1714,6 +1718,7 @@ def test_format_prediction_classification_incomplete_probabilities(self): with pytest.raises(ValueError, match="Each class should have a predicted probability"): format_prediction(classification, *ignored_input, proba=incomplete_probabilities) + @pytest.mark.uses_test_server() def test_format_prediction_task_without_classlabels_set(self): classification = openml.tasks.get_task( self.TEST_SERVER_TASK_SIMPLE["task_id"], @@ -1724,7 +1729,7 @@ def test_format_prediction_task_without_classlabels_set(self): with pytest.raises(ValueError, match="The classification task must have class labels set"): format_prediction(classification, *ignored_input, proba={}) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_format_prediction_task_learning_curve_sample_not_set(self): learning_curve = openml.tasks.get_task(801, download_data=False) # diabetes;crossvalidation probabilities = {c: 0.2 for c in learning_curve.class_labels} @@ -1732,7 +1737,7 @@ def test_format_prediction_task_learning_curve_sample_not_set(self): with pytest.raises(ValueError, match="`sample` can not be none for LearningCurveTask"): format_prediction(learning_curve, *ignored_input, sample=None, proba=probabilities) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_format_prediction_task_regression(self): task_meta_data = self.TEST_SERVER_TASK_REGRESSION["task_meta_data"] _task_id = check_task_existence(**task_meta_data) @@ -1762,12 +1767,12 @@ def test_format_prediction_task_regression(self): - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", ) @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_delete_run(self): rs = np.random.randint(1, 2**31 - 1) clf = sklearn.pipeline.Pipeline( @@ -1863,12 +1868,12 @@ def test_delete_unknown_run(mock_delete, test_files_directory, test_api_key): @pytest.mark.sklearn() -@pytest.mark.xfail(reason="failures_issue_1544", strict=False) @unittest.skipIf( Version(sklearn.__version__) < Version("0.21"), reason="couldn't perform local tests successfully w/o bloating RAM", ) @mock.patch("openml_sklearn.SklearnExtension._prevent_optimize_n_jobs") +@pytest.mark.uses_test_server() def test__run_task_get_arffcontent_2(parallel_mock): """Tests if a run executed in parallel is collated correctly.""" task = openml.tasks.get_task(7) # Supervised Classification on kr-vs-kp @@ -1940,7 +1945,6 @@ def test__run_task_get_arffcontent_2(parallel_mock): ) -@pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.21"), @@ -1960,6 +1964,7 @@ def test__run_task_get_arffcontent_2(parallel_mock): (-1, "threading", 10), # the threading backend does preserve mocks even with parallelizing ] ) +@pytest.mark.uses_test_server() def test_joblib_backends(parallel_mock, n_jobs, backend, call_count): """Tests evaluation of a run using various joblib backends and n_jobs.""" if backend is None: diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index a3b698a37..a0469f9a5 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -34,8 +34,8 @@ def setUp(self): self.extension = SklearnExtension() super().setUp() - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_nonexisting_setup_exists(self): # first publish a non-existing flow sentinel = get_sentinel() @@ -82,8 +82,8 @@ def _existing_setup_exists(self, classif): setup_id = openml.setups.setup_exists(flow) assert setup_id == run.setup_id - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_existing_setup_exists_1(self): def side_effect(self): self.var_smoothing = 1e-9 @@ -98,14 +98,14 @@ def side_effect(self): nb = sklearn.naive_bayes.GaussianNB() self._existing_setup_exists(nb) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_exisiting_setup_exists_2(self): # Check a flow with one hyperparameter self._existing_setup_exists(sklearn.naive_bayes.GaussianNB()) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) @pytest.mark.sklearn() + @pytest.mark.uses_test_server() def test_existing_setup_exists_3(self): # Check a flow with many hyperparameters self._existing_setup_exists( @@ -147,6 +147,7 @@ def test_setup_list_filter_flow(self): for setup_id in setups: assert setups[setup_id].flow_id == flow_id + @pytest.mark.uses_test_server() def test_list_setups_empty(self): setups = openml.setups.list_setups(setup=[0]) if len(setups) > 0: @@ -167,6 +168,7 @@ def test_list_setups_output_format(self): assert isinstance(setups, pd.DataFrame) assert len(setups) == 10 + @pytest.mark.uses_test_server() def test_setuplist_offset(self): size = 10 setups = openml.setups.list_setups(offset=0, size=size) @@ -178,6 +180,7 @@ def test_setuplist_offset(self): assert len(all) == size * 2 + @pytest.mark.uses_test_server() def test_get_cached_setup(self): openml.config.set_root_cache_directory(self.static_cache_dir) openml.setups.functions._get_cached_setup(1) diff --git a/tests/test_study/test_study_functions.py b/tests/test_study/test_study_functions.py index 40026592f..839e74cf3 100644 --- a/tests/test_study/test_study_functions.py +++ b/tests/test_study/test_study_functions.py @@ -73,6 +73,7 @@ def test_get_suite_error(self): ): openml.study.get_suite(123) + @pytest.mark.uses_test_server() def test_publish_benchmark_suite(self): fixture_alias = None fixture_name = "unit tested benchmark suite" @@ -141,13 +142,16 @@ def _test_publish_empty_study_is_allowed(self, explicit: bool): assert study_downloaded.main_entity_type == "run" assert study_downloaded.runs is None + @pytest.mark.uses_test_server() def test_publish_empty_study_explicit(self): self._test_publish_empty_study_is_allowed(explicit=True) + @pytest.mark.uses_test_server() def test_publish_empty_study_implicit(self): self._test_publish_empty_study_is_allowed(explicit=False) @pytest.mark.flaky() + @pytest.mark.uses_test_server() def test_publish_study(self): # get some random runs to attach run_list = openml.evaluations.list_evaluations("predictive_accuracy", size=10) @@ -217,6 +221,7 @@ def test_publish_study(self): res = openml.study.delete_study(study.id) assert res + @pytest.mark.uses_test_server() def test_study_attach_illegal(self): run_list = openml.runs.list_runs(size=10) assert len(run_list) == 10 diff --git a/tests/test_tasks/test_classification_task.py b/tests/test_tasks/test_classification_task.py index 5528cabf2..fed0c0a00 100644 --- a/tests/test_tasks/test_classification_task.py +++ b/tests/test_tasks/test_classification_task.py @@ -18,7 +18,7 @@ def setUp(self, n_levels: int = 1): self.task_type = TaskType.SUPERVISED_CLASSIFICATION self.estimation_procedure = 5 - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_download_task(self): task = super().test_download_task() assert task.task_id == self.task_id @@ -26,14 +26,13 @@ def test_download_task(self): assert task.dataset_id == 20 assert task.estimation_procedure_id == self.estimation_procedure - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_class_labels(self): task = get_task(self.task_id) assert task.class_labels == ["tested_negative", "tested_positive"] -@pytest.mark.xfail(reason="failures_issue_1544", strict=False) -@pytest.mark.server() +@pytest.mark.uses_test_server() def test_get_X_and_Y(): task = get_task(119) X, Y = task.get_X_and_y() diff --git a/tests/test_tasks/test_clustering_task.py b/tests/test_tasks/test_clustering_task.py index dcc024388..2bbb015c6 100644 --- a/tests/test_tasks/test_clustering_task.py +++ b/tests/test_tasks/test_clustering_task.py @@ -28,6 +28,7 @@ def test_get_dataset(self): task.get_dataset() @pytest.mark.production() + @pytest.mark.uses_test_server() def test_download_task(self): # no clustering tasks on test server self.use_production_server() @@ -36,6 +37,7 @@ def test_download_task(self): assert task.task_type_id == TaskType.CLUSTERING assert task.dataset_id == 36 + @pytest.mark.uses_test_server() def test_upload_task(self): compatible_datasets = self._get_compatible_rand_dataset() for i in range(100): diff --git a/tests/test_tasks/test_learning_curve_task.py b/tests/test_tasks/test_learning_curve_task.py index 5f4b3e0ab..fbcbfe9bf 100644 --- a/tests/test_tasks/test_learning_curve_task.py +++ b/tests/test_tasks/test_learning_curve_task.py @@ -18,7 +18,7 @@ def setUp(self, n_levels: int = 1): self.task_type = TaskType.LEARNING_CURVE self.estimation_procedure = 13 - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_get_X_and_Y(self): X, Y = super().test_get_X_and_Y() assert X.shape == (768, 8) @@ -27,14 +27,14 @@ def test_get_X_and_Y(self): assert isinstance(Y, pd.Series) assert pd.api.types.is_categorical_dtype(Y) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_download_task(self): task = super().test_download_task() assert task.task_id == self.task_id assert task.task_type_id == TaskType.LEARNING_CURVE assert task.dataset_id == 20 - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_class_labels(self): task = get_task(self.task_id) assert task.class_labels == ["tested_negative", "tested_positive"] diff --git a/tests/test_tasks/test_regression_task.py b/tests/test_tasks/test_regression_task.py index 0cd2d96e2..a834cdf0f 100644 --- a/tests/test_tasks/test_regression_task.py +++ b/tests/test_tasks/test_regression_task.py @@ -49,7 +49,7 @@ def setUp(self, n_levels: int = 1): self.task_type = TaskType.SUPERVISED_REGRESSION - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_get_X_and_Y(self): X, Y = super().test_get_X_and_Y() assert X.shape == (194, 32) @@ -58,7 +58,7 @@ def test_get_X_and_Y(self): assert isinstance(Y, pd.Series) assert pd.api.types.is_numeric_dtype(Y) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_download_task(self): task = super().test_download_task() assert task.task_id == self.task_id diff --git a/tests/test_tasks/test_supervised_task.py b/tests/test_tasks/test_supervised_task.py index e5a17a72b..3f7b06ee4 100644 --- a/tests/test_tasks/test_supervised_task.py +++ b/tests/test_tasks/test_supervised_task.py @@ -28,6 +28,7 @@ def setUpClass(cls): def setUp(self, n_levels: int = 1): super().setUp() + @pytest.mark.uses_test_server() def test_get_X_and_Y(self) -> tuple[pd.DataFrame, pd.Series]: task = get_task(self.task_id) X, Y = task.get_X_and_y() diff --git a/tests/test_tasks/test_task.py b/tests/test_tasks/test_task.py index 67f715d2b..b77782847 100644 --- a/tests/test_tasks/test_task.py +++ b/tests/test_tasks/test_task.py @@ -32,10 +32,11 @@ def setUpClass(cls): def setUp(self, n_levels: int = 1): super().setUp() + @pytest.mark.uses_test_server() def test_download_task(self): return get_task(self.task_id) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_upload_task(self): # We don't know if the task in question already exists, so we try a few times. Checking # beforehand would not be an option because a concurrent unit test could potentially diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py index 110459711..3a2b9ea0a 100644 --- a/tests/test_tasks/test_task_functions.py +++ b/tests/test_tasks/test_task_functions.py @@ -26,6 +26,7 @@ def setUp(self): def tearDown(self): super().tearDown() + @pytest.mark.uses_test_server() def test__get_cached_tasks(self): openml.config.set_root_cache_directory(self.static_cache_dir) tasks = openml.tasks.functions._get_cached_tasks() @@ -33,6 +34,7 @@ def test__get_cached_tasks(self): assert len(tasks) == 3 assert isinstance(next(iter(tasks.values())), OpenMLTask) + @pytest.mark.uses_test_server() def test__get_cached_task(self): openml.config.set_root_cache_directory(self.static_cache_dir) task = openml.tasks.functions._get_cached_task(1) @@ -47,6 +49,7 @@ def test__get_cached_task_not_cached(self): 2, ) + @pytest.mark.uses_test_server() def test__get_estimation_procedure_list(self): estimation_procedures = openml.tasks.functions._get_estimation_procedure_list() assert isinstance(estimation_procedures, list) @@ -69,6 +72,7 @@ def _check_task(self, task): assert isinstance(task["status"], str) assert task["status"] in ["in_preparation", "active", "deactivated"] + @pytest.mark.uses_test_server() def test_list_tasks_by_type(self): num_curves_tasks = 198 # number is flexible, check server if fails ttid = TaskType.LEARNING_CURVE @@ -78,15 +82,18 @@ def test_list_tasks_by_type(self): assert ttid == task["ttid"] self._check_task(task) + @pytest.mark.uses_test_server() def test_list_tasks_length(self): ttid = TaskType.LEARNING_CURVE tasks = openml.tasks.list_tasks(task_type=ttid) assert len(tasks) > 100 + @pytest.mark.uses_test_server() def test_list_tasks_empty(self): tasks = openml.tasks.list_tasks(tag="NoOneWillEverUseThisTag") assert tasks.empty + @pytest.mark.uses_test_server() def test_list_tasks_by_tag(self): num_basic_tasks = 100 # number is flexible, check server if fails tasks = openml.tasks.list_tasks(tag="OpenML100") @@ -94,12 +101,14 @@ def test_list_tasks_by_tag(self): for task in tasks.to_dict(orient="index").values(): self._check_task(task) + @pytest.mark.uses_test_server() def test_list_tasks(self): tasks = openml.tasks.list_tasks() assert len(tasks) >= 900 for task in tasks.to_dict(orient="index").values(): self._check_task(task) + @pytest.mark.uses_test_server() def test_list_tasks_paginate(self): size = 10 max = 100 @@ -109,6 +118,7 @@ def test_list_tasks_paginate(self): for task in tasks.to_dict(orient="index").values(): self._check_task(task) + @pytest.mark.uses_test_server() def test_list_tasks_per_type_paginate(self): size = 40 max = 100 @@ -125,6 +135,7 @@ def test_list_tasks_per_type_paginate(self): assert j == task["ttid"] self._check_task(task) + @pytest.mark.uses_test_server() def test__get_task(self): openml.config.set_root_cache_directory(self.static_cache_dir) openml.tasks.get_task(1882) @@ -139,6 +150,7 @@ def test__get_task_live(self): # https://github.com/openml/openml-python/issues/378 openml.tasks.get_task(34536) + @pytest.mark.uses_test_server() def test_get_task(self): task = openml.tasks.get_task(1, download_data=True) # anneal; crossvalidation assert isinstance(task, OpenMLTask) @@ -152,7 +164,7 @@ def test_get_task(self): os.path.join(self.workdir, "org", "openml", "test", "datasets", "1", "dataset.arff") ) - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) + @pytest.mark.uses_test_server() def test_get_task_lazy(self): task = openml.tasks.get_task(2, download_data=False) # anneal; crossvalidation assert isinstance(task, OpenMLTask) @@ -175,7 +187,7 @@ def test_get_task_lazy(self): ) @mock.patch("openml.tasks.functions.get_dataset") - @pytest.mark.xfail(reason="failures_issue_1544") + @pytest.mark.uses_test_server() def test_removal_upon_download_failure(self, get_dataset): class WeirdException(Exception): pass @@ -193,6 +205,7 @@ def assert_and_raise(*args, **kwargs): # Now the file should no longer exist assert not os.path.exists(os.path.join(os.getcwd(), "tasks", "1", "tasks.xml")) + @pytest.mark.uses_test_server() def test_get_task_with_cache(self): openml.config.set_root_cache_directory(self.static_cache_dir) task = openml.tasks.get_task(1) @@ -208,6 +221,7 @@ def test_get_task_different_types(self): # Issue 538, get_task failing with clustering task. openml.tasks.functions.get_task(126033) + @pytest.mark.uses_test_server() def test_download_split(self): task = openml.tasks.get_task(1) # anneal; crossvalidation split = task.download_split() diff --git a/tests/test_tasks/test_task_methods.py b/tests/test_tasks/test_task_methods.py index 540c43de0..6b8804b9f 100644 --- a/tests/test_tasks/test_task_methods.py +++ b/tests/test_tasks/test_task_methods.py @@ -16,6 +16,7 @@ def setUp(self): def tearDown(self): super().tearDown() + @pytest.mark.uses_test_server() def test_tagging(self): task = openml.tasks.get_task(1) # anneal; crossvalidation # tags can be at most 64 alphanumeric (+ underscore) chars @@ -31,6 +32,7 @@ def test_tagging(self): tasks = openml.tasks.list_tasks(tag=tag) assert len(tasks) == 0 + @pytest.mark.uses_test_server() def test_get_train_and_test_split_indices(self): openml.config.set_root_cache_directory(self.static_cache_dir) task = openml.tasks.get_task(1882) diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index 35be84903..a1cdb55ea 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -48,18 +48,18 @@ def _mocked_perform_api_call(call, request_method): return openml._api_calls._download_text_file(url) -@pytest.mark.server() +@pytest.mark.uses_test_server() def test_list_all(): openml.utils._list_all(listing_call=openml.tasks.functions._list_tasks) -@pytest.mark.server() +@pytest.mark.uses_test_server() def test_list_all_for_tasks(min_number_tasks_on_test_server): tasks = openml.tasks.list_tasks(size=min_number_tasks_on_test_server) assert min_number_tasks_on_test_server == len(tasks) -@pytest.mark.server() +@pytest.mark.uses_test_server() def test_list_all_with_multiple_batches(min_number_tasks_on_test_server): # By setting the batch size one lower than the minimum we guarantee at least two # batches and at the same time do as few batches (roundtrips) as possible. @@ -72,7 +72,7 @@ def test_list_all_with_multiple_batches(min_number_tasks_on_test_server): assert min_number_tasks_on_test_server <= sum(len(batch) for batch in batches) -@pytest.mark.server() +@pytest.mark.uses_test_server() def test_list_all_for_datasets(min_number_datasets_on_test_server): datasets = openml.datasets.list_datasets( size=min_number_datasets_on_test_server, @@ -83,29 +83,29 @@ def test_list_all_for_datasets(min_number_datasets_on_test_server): _check_dataset(dataset) -@pytest.mark.server() +@pytest.mark.uses_test_server() def test_list_all_for_flows(min_number_flows_on_test_server): flows = openml.flows.list_flows(size=min_number_flows_on_test_server) assert min_number_flows_on_test_server == len(flows) -@pytest.mark.server() @pytest.mark.flaky() # Other tests might need to upload runs first +@pytest.mark.uses_test_server() def test_list_all_for_setups(min_number_setups_on_test_server): # TODO apparently list_setups function does not support kwargs setups = openml.setups.list_setups(size=min_number_setups_on_test_server) assert min_number_setups_on_test_server == len(setups) -@pytest.mark.server() @pytest.mark.flaky() # Other tests might need to upload runs first +@pytest.mark.uses_test_server() def test_list_all_for_runs(min_number_runs_on_test_server): runs = openml.runs.list_runs(size=min_number_runs_on_test_server) assert min_number_runs_on_test_server == len(runs) -@pytest.mark.server() @pytest.mark.flaky() # Other tests might need to upload runs first +@pytest.mark.uses_test_server() def test_list_all_for_evaluations(min_number_evaluations_on_test_server): # TODO apparently list_evaluations function does not support kwargs evaluations = openml.evaluations.list_evaluations( @@ -115,8 +115,8 @@ def test_list_all_for_evaluations(min_number_evaluations_on_test_server): assert min_number_evaluations_on_test_server == len(evaluations) -@pytest.mark.server() @unittest.mock.patch("openml._api_calls._perform_api_call", side_effect=_mocked_perform_api_call) +@pytest.mark.uses_test_server() def test_list_all_few_results_available(_perform_api_call): datasets = openml.datasets.list_datasets(size=1000, data_name="iris", data_version=1) assert len(datasets) == 1, "only one iris dataset version 1 should be present" @@ -141,7 +141,7 @@ def test__create_cache_directory(config_mock, tmp_path): openml.utils._create_cache_directory("ghi") -@pytest.mark.server() +@pytest.mark.uses_test_server() def test_correct_test_server_download_state(): """This test verifies that the test server downloads the data from the correct source. From 039defe25ed9a0eaeb66617989047346d4f29a65 Mon Sep 17 00:00:00 2001 From: Satvik Mishra <112589278+satvshr@users.noreply.github.com> Date: Wed, 14 Jan 2026 16:36:46 +0530 Subject: [PATCH 19/46] [MNT] Update ruff and mypy version, and format files to match latest ruff checks (#1553) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### Metadata * Stacks on #1547 (for ignoring `.ruff_cache`) * Reference Issue: fixes #1550 * New Tests Added: No * Documentation Updated: No #### Details What does this PR implement/fix? Explain your changes. * Updates the ruff version in .pre-commit-config.yaml to 0.14.10 * Runs `ruff format .` to align the codebase with the formatting rules of the updated Ruff version * Fixes also added to pass `ruff check .` checks * Add `noqa` tags in places that will end up changing the architecture of the function/class if I try fixing it * Only changes from my end to the actual code would be changing small things like: * the print statements to be compatible with check [UP031](https://docs.astral.sh/ruff/rules/printf-string-formatting/) * Changing variable names to `_` to be compatible with [RUF059](https://docs.astral.sh/ruff/rules/unused-unpacked-variable/) This PR is going to be a bigger one in size but in my opinion, we should be compatible with the latest ruff version and get it over with sooner rather than later. On a separate note, there are already a significant number of `noqa` tags in the codebase. We should consider revisiting the architecture of the functions and classes that rely on them to better align with Ruff’s best practices. Where alignment isn’t appropriate, we should at least discuss why those components don’t need to be Ruff-compatible. --- .pre-commit-config.yaml | 2 +- .../Advanced/fetch_evaluations_tutorial.py | 6 +-- examples/Advanced/suites_tutorial.py | 2 +- examples/Basics/introduction_tutorial.py | 4 +- .../Basics/simple_flows_and_runs_tutorial.py | 4 +- .../2015_neurips_feurer_example.py | 6 +-- .../2018_ida_strang_example.py | 7 ++-- .../2018_kdd_rijn_example.py | 23 +++++----- .../2018_neurips_perrone_example.py | 22 ++++++---- .../benchmark_with_optunahub.py | 2 +- .../fetch_runtimes_tutorial.py | 31 +++++--------- .../flow_id_tutorial.py | 3 +- .../flows_and_runs_tutorial.py | 3 +- .../plot_svm_hyperparameters_tutorial.py | 3 +- .../run_setup_tutorial.py | 14 +++---- .../upload_amlb_flows_and_runs.py | 42 +++++++++---------- openml/__init__.py | 32 +++++++------- openml/_api_calls.py | 9 ++-- openml/base.py | 2 +- openml/cli.py | 2 +- openml/config.py | 15 +++---- openml/datasets/__init__.py | 12 +++--- openml/datasets/data_feature.py | 12 +++--- openml/datasets/dataset.py | 18 ++++---- openml/datasets/functions.py | 29 +++++++------ openml/evaluations/__init__.py | 2 +- openml/evaluations/functions.py | 8 ++-- openml/extensions/__init__.py | 7 ++-- openml/extensions/extension_interface.py | 4 +- openml/extensions/functions.py | 4 +- openml/flows/__init__.py | 8 ++-- openml/flows/flow.py | 13 +++--- openml/flows/functions.py | 30 ++++++------- openml/runs/__init__.py | 12 +++--- openml/runs/functions.py | 28 ++++++------- openml/runs/run.py | 21 +++++----- openml/runs/trace.py | 10 ++--- openml/setups/__init__.py | 4 +- openml/setups/functions.py | 6 +-- openml/study/__init__.py | 4 +- openml/study/functions.py | 3 +- openml/study/study.py | 3 +- openml/tasks/__init__.py | 14 +++---- openml/tasks/functions.py | 8 ++-- openml/tasks/split.py | 2 +- openml/tasks/task.py | 3 +- openml/testing.py | 12 ++++-- openml/utils.py | 11 ++--- pyproject.toml | 8 ++-- scripts/__init__.py | 1 + 50 files changed, 263 insertions(+), 268 deletions(-) create mode 100644 scripts/__init__.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 95e2a5239..0987bad90 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ files: | )/.*\.py$ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.3 + rev: v0.14.10 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --no-cache] diff --git a/examples/Advanced/fetch_evaluations_tutorial.py b/examples/Advanced/fetch_evaluations_tutorial.py index 1b759423b..97b8d1bef 100644 --- a/examples/Advanced/fetch_evaluations_tutorial.py +++ b/examples/Advanced/fetch_evaluations_tutorial.py @@ -75,7 +75,7 @@ def plot_cdf(values, metric="predictive_accuracy"): max_val = max(values) - n, bins, patches = plt.hist(values, density=True, histtype="step", cumulative=True, linewidth=3) + _, _, patches = plt.hist(values, density=True, histtype="step", cumulative=True, linewidth=3) patches[0].set_xy(patches[0].get_xy()[:-1]) plt.xlim(max(0, min(values) - 0.1), 1) plt.title("CDF") @@ -116,7 +116,7 @@ def plot_flow_compare(evaluations, top_n=10, metric="predictive_accuracy"): for i in range(len(flow_ids)): flow_values = evaluations[evaluations.flow_id == flow_ids[i]].value df = pd.concat([df, flow_values], ignore_index=True, axis=1) - fig, axs = plt.subplots() + _, axs = plt.subplots() df.boxplot() axs.set_title("Boxplot comparing " + metric + " for different flows") axs.set_ylabel(metric) @@ -178,4 +178,4 @@ def plot_flow_compare(evaluations, top_n=10, metric="predictive_accuracy"): function="predictive_accuracy", flows=[6767], size=100, parameters_in_separate_columns=True ) -print(evals_setups.head(10)) \ No newline at end of file +print(evals_setups.head(10)) diff --git a/examples/Advanced/suites_tutorial.py b/examples/Advanced/suites_tutorial.py index 7ca42079d..8459510ef 100644 --- a/examples/Advanced/suites_tutorial.py +++ b/examples/Advanced/suites_tutorial.py @@ -72,7 +72,7 @@ # %% all_tasks = list(openml.tasks.list_tasks()["tid"]) -task_ids_for_suite = sorted(np.random.choice(all_tasks, replace=False, size=20)) +task_ids_for_suite = sorted(np.random.choice(all_tasks, replace=False, size=20)) # noqa: NPY002 # The study needs a machine-readable and unique alias. To obtain this, # we simply generate a random uuid. diff --git a/examples/Basics/introduction_tutorial.py b/examples/Basics/introduction_tutorial.py index c864772f5..2ba2d0ef1 100644 --- a/examples/Basics/introduction_tutorial.py +++ b/examples/Basics/introduction_tutorial.py @@ -12,7 +12,7 @@ # For certain functionality, such as uploading tasks or datasets, users have to # sign up. Only accessing the data on OpenML does not require an account! # -# If you don’t have an account yet, sign up now. +# If you don't have an account yet, sign up now. # You will receive an API key, which will authenticate you to the server # and allow you to download and upload datasets, tasks, runs and flows. # @@ -52,4 +52,4 @@ # %% import openml -openml.config.set_root_cache_directory("YOURDIR") \ No newline at end of file +openml.config.set_root_cache_directory("YOURDIR") diff --git a/examples/Basics/simple_flows_and_runs_tutorial.py b/examples/Basics/simple_flows_and_runs_tutorial.py index 41eed9234..eb42c7d02 100644 --- a/examples/Basics/simple_flows_and_runs_tutorial.py +++ b/examples/Basics/simple_flows_and_runs_tutorial.py @@ -85,7 +85,7 @@ # Format the predictions for OpenML predictions = [] for test_index, y_true_i, y_pred_i, y_pred_proba_i in zip( - test_indices, y_test, y_pred, y_pred_proba + test_indices, y_test, y_pred, y_pred_proba, strict=False ): predictions.append( openml.runs.functions.format_prediction( @@ -95,7 +95,7 @@ index=test_index, prediction=y_pred_i, truth=y_true_i, - proba=dict(zip(task.class_labels, y_pred_proba_i)), + proba=dict(zip(task.class_labels, y_pred_proba_i, strict=False)), ) ) diff --git a/examples/_external_or_deprecated/2015_neurips_feurer_example.py b/examples/_external_or_deprecated/2015_neurips_feurer_example.py index ae59c9ced..2dfc4bb97 100644 --- a/examples/_external_or_deprecated/2015_neurips_feurer_example.py +++ b/examples/_external_or_deprecated/2015_neurips_feurer_example.py @@ -13,12 +13,10 @@ | Matthias Feurer, Aaron Klein, Katharina Eggensperger, Jost Springenberg, Manuel Blum and Frank Hutter | In *Advances in Neural Information Processing Systems 28*, 2015 | Available at https://papers.nips.cc/paper/5872-efficient-and-robust-automated-machine-learning.pdf -""" # noqa F401 +""" # License: BSD 3-Clause -import pandas as pd - import openml #################################################################################################### @@ -68,7 +66,7 @@ task_ids = [] for did in dataset_ids: - tasks_ = list(tasks.query("did == {}".format(did)).tid) + tasks_ = list(tasks.query(f"did == {did}").tid) if len(tasks_) >= 1: # if there are multiple task, take the one with lowest ID (oldest). task_id = min(tasks_) else: diff --git a/examples/_external_or_deprecated/2018_ida_strang_example.py b/examples/_external_or_deprecated/2018_ida_strang_example.py index 8b225125b..0e180badf 100644 --- a/examples/_external_or_deprecated/2018_ida_strang_example.py +++ b/examples/_external_or_deprecated/2018_ida_strang_example.py @@ -17,8 +17,8 @@ # License: BSD 3-Clause import matplotlib.pyplot as plt + import openml -import pandas as pd ############################################################################## # A basic step for each data-mining or machine learning task is to determine @@ -86,10 +86,9 @@ def determine_class(val_lin, val_nonlin): if val_lin < val_nonlin: return class_values[0] - elif val_nonlin < val_lin: + if val_nonlin < val_lin: return class_values[1] - else: - return class_values[2] + return class_values[2] evaluations["class"] = evaluations.apply( diff --git a/examples/_external_or_deprecated/2018_kdd_rijn_example.py b/examples/_external_or_deprecated/2018_kdd_rijn_example.py index 6522013e3..957281616 100644 --- a/examples/_external_or_deprecated/2018_kdd_rijn_example.py +++ b/examples/_external_or_deprecated/2018_kdd_rijn_example.py @@ -32,16 +32,17 @@ import sys -if sys.platform == "win32": # noqa +if sys.platform == "win32": print( "The pyrfr library (requirement of fanova) can currently not be installed on Windows systems" ) - exit() + sys.exit() # DEPRECATED EXAMPLE -- Avoid running this code in our CI/CD pipeline print("This example is deprecated, remove the `if False` in this code to use it manually.") if False: import json + import fanova import matplotlib.pyplot as plt import pandas as pd @@ -49,7 +50,6 @@ import openml - ############################################################################## # With the advent of automated machine learning, automated hyperparameter # optimization methods are by now routinely used in data mining. However, this @@ -80,7 +80,7 @@ # important when it is put on a log-scale. All these simplifications can be # addressed by defining a ConfigSpace. For a more elaborated example that uses # this, please see: - # https://github.com/janvanrijn/openml-pimp/blob/d0a14f3eb480f2a90008889f00041bdccc7b9265/examples/plot/plot_fanova_aggregates.py # noqa F401 + # https://github.com/janvanrijn/openml-pimp/blob/d0a14f3eb480f2a90008889f00041bdccc7b9265/examples/plot/plot_fanova_aggregates.py suite = openml.study.get_suite("OpenML100") flow_id = 7707 @@ -97,8 +97,7 @@ if limit_nr_tasks is not None and idx >= limit_nr_tasks: continue print( - "Starting with task %d (%d/%d)" - % (task_id, idx + 1, len(suite.tasks) if limit_nr_tasks is None else limit_nr_tasks) + f"Starting with task {task_id} ({idx + 1}/{len(suite.tasks) if limit_nr_tasks is None else limit_nr_tasks})" ) # note that we explicitly only include tasks from the benchmark suite that was specified (as per the for-loop) evals = openml.evaluations.list_evaluations_setups( @@ -121,13 +120,13 @@ [ dict( **{name: json.loads(value) for name, value in setup["parameters"].items()}, - **{performance_column: setup[performance_column]} + **{performance_column: setup[performance_column]}, ) for _, setup in evals.iterrows() ] ) except json.decoder.JSONDecodeError as e: - print("Task %d error: %s" % (task_id, e)) + print(f"Task {task_id} error: {e}") continue # apply our filters, to have only the setups that comply to the hyperparameters we want for filter_key, filter_value in parameter_filters.items(): @@ -156,19 +155,21 @@ Y=setups_evals[performance_column].to_numpy(), n_trees=n_trees, ) - for idx, pname in enumerate(parameter_names): + for idx, pname in enumerate(parameter_names): # noqa: PLW2901 try: fanova_results.append( { "hyperparameter": pname.split(".")[-1], - "fanova": evaluator.quantify_importance([idx])[(idx,)]["individual importance"], + "fanova": evaluator.quantify_importance([idx])[(idx,)][ + "individual importance" + ], } ) except RuntimeError as e: # functional ANOVA sometimes crashes with a RuntimeError, e.g., on tasks where the performance is constant # for all configurations (there is no variance). We will skip these tasks (like the authors did in the # paper). - print("Task %d error: %s" % (task_id, e)) + print(f"Task {task_id} error: {e}") continue # transform ``fanova_results`` from a list of dicts into a DataFrame diff --git a/examples/_external_or_deprecated/2018_neurips_perrone_example.py b/examples/_external_or_deprecated/2018_neurips_perrone_example.py index 0d72846ac..8a3c36994 100644 --- a/examples/_external_or_deprecated/2018_neurips_perrone_example.py +++ b/examples/_external_or_deprecated/2018_neurips_perrone_example.py @@ -27,16 +27,17 @@ # License: BSD 3-Clause -import openml import numpy as np import pandas as pd from matplotlib import pyplot as plt -from sklearn.pipeline import Pipeline -from sklearn.impute import SimpleImputer from sklearn.compose import ColumnTransformer +from sklearn.ensemble import RandomForestRegressor +from sklearn.impute import SimpleImputer from sklearn.metrics import mean_squared_error +from sklearn.pipeline import Pipeline from sklearn.preprocessing import OneHotEncoder -from sklearn.ensemble import RandomForestRegressor + +import openml flow_type = "svm" # this example will use the smaller svm flow evaluations ############################################################################ @@ -44,7 +45,7 @@ # a tabular format that can be used to build models. -def fetch_evaluations(run_full=False, flow_type="svm", metric="area_under_roc_curve"): +def fetch_evaluations(run_full=False, flow_type="svm", metric="area_under_roc_curve"): # noqa: FBT002 """ Fetch a list of evaluations based on the flows and tasks used in the experiments. @@ -101,7 +102,10 @@ def fetch_evaluations(run_full=False, flow_type="svm", metric="area_under_roc_cu def create_table_from_evaluations( - eval_df, flow_type="svm", run_count=np.iinfo(np.int64).max, task_ids=None + eval_df, + flow_type="svm", + run_count=np.iinfo(np.int64).max, # noqa: B008 + task_ids=None, ): """ Create a tabular data with its ground truth from a dataframe of evaluations. @@ -206,7 +210,7 @@ def list_categorical_attributes(flow_type="svm"): model.fit(X, y) y_pred = model.predict(X) -print("Training RMSE : {:.5}".format(mean_squared_error(y, y_pred))) +print(f"Training RMSE : {mean_squared_error(y, y_pred):.5}") ############################################################################# @@ -231,9 +235,9 @@ def random_sample_configurations(num_samples=100): X = pd.DataFrame(np.nan, index=range(num_samples), columns=colnames) for i in range(len(colnames)): if len(ranges[i]) == 2: - col_val = np.random.uniform(low=ranges[i][0], high=ranges[i][1], size=num_samples) + col_val = np.random.uniform(low=ranges[i][0], high=ranges[i][1], size=num_samples) # noqa: NPY002 else: - col_val = np.random.choice(ranges[i], size=num_samples) + col_val = np.random.choice(ranges[i], size=num_samples) # noqa: NPY002 X.iloc[:, i] = col_val return X diff --git a/examples/_external_or_deprecated/benchmark_with_optunahub.py b/examples/_external_or_deprecated/benchmark_with_optunahub.py index ece3e7c40..38114bc44 100644 --- a/examples/_external_or_deprecated/benchmark_with_optunahub.py +++ b/examples/_external_or_deprecated/benchmark_with_optunahub.py @@ -100,7 +100,7 @@ def objective(trial: optuna.Trial) -> Pipeline: run.publish() logger.log(1, f"Run was uploaded to - {run.openml_url}") - except Exception as e: + except Exception as e: # noqa: BLE001 logger.log(1, f"Could not publish run - {e}") else: logger.log( diff --git a/examples/_external_or_deprecated/fetch_runtimes_tutorial.py b/examples/_external_or_deprecated/fetch_runtimes_tutorial.py index b2a3f1d2a..c8f85adc5 100644 --- a/examples/_external_or_deprecated/fetch_runtimes_tutorial.py +++ b/examples/_external_or_deprecated/fetch_runtimes_tutorial.py @@ -39,17 +39,16 @@ # # * (Case 5) Running models that do not release the Python Global Interpreter Lock (GIL) -import openml import numpy as np -from matplotlib import pyplot as plt from joblib.parallel import parallel_backend - -from sklearn.naive_bayes import GaussianNB -from sklearn.tree import DecisionTreeClassifier -from sklearn.neural_network import MLPClassifier +from matplotlib import pyplot as plt from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import GridSearchCV, RandomizedSearchCV +from sklearn.naive_bayes import GaussianNB +from sklearn.neural_network import MLPClassifier +from sklearn.tree import DecisionTreeClassifier +import openml # %% [markdown] # # Preparing tasks and scikit-learn models @@ -63,12 +62,7 @@ # Viewing associated data n_repeats, n_folds, n_samples = task.get_split_dimensions() print( - "Task {}: number of repeats: {}, number of folds: {}, number of samples {}.".format( - task_id, - n_repeats, - n_folds, - n_samples, - ) + f"Task {task_id}: number of repeats: {n_repeats}, number of folds: {n_folds}, number of samples {n_samples}." ) @@ -101,7 +95,7 @@ def print_compare_runtimes(measures): measures = run1.fold_evaluations print("The timing and performance metrics available: ") -for key in measures.keys(): +for key in measures: print(key) print() @@ -206,7 +200,6 @@ def print_compare_runtimes(measures): # included in the `wall_clock_time_millis_training` measure recorded. # %% -from sklearn.model_selection import GridSearchCV clf = RandomForestClassifier(n_estimators=10, n_jobs=2) @@ -284,22 +277,18 @@ def print_compare_runtimes(measures): # %% + def extract_refit_time(run, repeat, fold): - refit_time = ( + return ( run.fold_evaluations["wall_clock_time_millis"][repeat][fold] - run.fold_evaluations["wall_clock_time_millis_training"][repeat][fold] - run.fold_evaluations["wall_clock_time_millis_testing"][repeat][fold] ) - return refit_time for repeat in range(n_repeats): for fold in range(n_folds): - print( - "Repeat #{}-Fold #{}: {:.4f}".format( - repeat, fold, extract_refit_time(run4, repeat, fold) - ) - ) + print(f"Repeat #{repeat}-Fold #{fold}: {extract_refit_time(run4, repeat, fold):.4f}") # %% [markdown] # Along with the GridSearchCV already used above, we demonstrate how such diff --git a/examples/_external_or_deprecated/flow_id_tutorial.py b/examples/_external_or_deprecated/flow_id_tutorial.py index e813655fc..19190cf0b 100644 --- a/examples/_external_or_deprecated/flow_id_tutorial.py +++ b/examples/_external_or_deprecated/flow_id_tutorial.py @@ -9,7 +9,6 @@ import openml - # %% [markdown] # .. warning:: # .. include:: ../../test_server_usage_warning.txt @@ -48,7 +47,7 @@ # %% [markdown] # ## 2. Obtaining a flow given its name # The schema of a flow is given in XSD ( -# [here](https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/openml.implementation.upload.xsd)). # noqa E501 +# [here](https://github.com/openml/OpenML/blob/master/openml_OS/views/pages/api_new/v1/xsd/openml.implementation.upload.xsd)). # Only two fields are required, a unique name, and an external version. While it should be pretty # obvious why we need a name, the need for the additional external version information might not # be immediately clear. However, this information is very important as it allows to have multiple diff --git a/examples/_external_or_deprecated/flows_and_runs_tutorial.py b/examples/_external_or_deprecated/flows_and_runs_tutorial.py index 2d1bcb864..71d6960bd 100644 --- a/examples/_external_or_deprecated/flows_and_runs_tutorial.py +++ b/examples/_external_or_deprecated/flows_and_runs_tutorial.py @@ -3,8 +3,7 @@ # This tutorial covers how to train/run a model and how to upload the results. # %% -import openml -from sklearn import compose, ensemble, impute, neighbors, preprocessing, pipeline, tree +from sklearn import compose, ensemble, impute, neighbors, pipeline, preprocessing, tree import openml diff --git a/examples/_external_or_deprecated/plot_svm_hyperparameters_tutorial.py b/examples/_external_or_deprecated/plot_svm_hyperparameters_tutorial.py index faced588b..7bb72db5a 100644 --- a/examples/_external_or_deprecated/plot_svm_hyperparameters_tutorial.py +++ b/examples/_external_or_deprecated/plot_svm_hyperparameters_tutorial.py @@ -2,9 +2,10 @@ # # Plotting hyperparameter surfaces # %% -import openml import numpy as np +import openml + # %% [markdown] # # First step - obtaining the data # First, we need to choose an SVM flow, for example 8353, and a task. Finding the IDs of them are diff --git a/examples/_external_or_deprecated/run_setup_tutorial.py b/examples/_external_or_deprecated/run_setup_tutorial.py index 55d25d291..25591bb58 100644 --- a/examples/_external_or_deprecated/run_setup_tutorial.py +++ b/examples/_external_or_deprecated/run_setup_tutorial.py @@ -23,15 +23,15 @@ # %% import numpy as np -import openml -from openml.extensions.sklearn import cat, cont - -from sklearn.pipeline import make_pipeline, Pipeline from sklearn.compose import ColumnTransformer -from sklearn.impute import SimpleImputer -from sklearn.preprocessing import OneHotEncoder, FunctionTransformer -from sklearn.ensemble import RandomForestClassifier from sklearn.decomposition import TruncatedSVD +from sklearn.ensemble import RandomForestClassifier +from sklearn.impute import SimpleImputer +from sklearn.pipeline import Pipeline, make_pipeline +from sklearn.preprocessing import OneHotEncoder + +import openml +from openml.extensions.sklearn import cat, cont # %% [markdown] # .. warning:: diff --git a/examples/_external_or_deprecated/upload_amlb_flows_and_runs.py b/examples/_external_or_deprecated/upload_amlb_flows_and_runs.py index 15ec0e1fb..b43926d4e 100644 --- a/examples/_external_or_deprecated/upload_amlb_flows_and_runs.py +++ b/examples/_external_or_deprecated/upload_amlb_flows_and_runs.py @@ -14,10 +14,10 @@ # %% from collections import OrderedDict + import numpy as np import openml -from openml import OpenMLClassificationTask from openml.runs.functions import format_prediction # %% [markdown] @@ -43,17 +43,17 @@ # version of the package/script is used. Use tags so users can find your flow easily. # %% -general = dict( - name="automlbenchmark_autosklearn", - description=( +general = { + "name": "automlbenchmark_autosklearn", + "description": ( "Auto-sklearn as set up by the AutoML Benchmark" "Source: https://github.com/openml/automlbenchmark/releases/tag/v0.9" ), - external_version="amlb==0.9", - language="English", - tags=["amlb", "benchmark", "study_218"], - dependencies="amlb==0.9", -) + "external_version": "amlb==0.9", + "language": "English", + "tags": ["amlb", "benchmark", "study_218"], + "dependencies": "amlb==0.9", +} # %% [markdown] # Next we define the flow hyperparameters. We define their name and default value in `parameters`, @@ -62,14 +62,14 @@ # The use of ordered dicts is required. # %% -flow_hyperparameters = dict( - parameters=OrderedDict(time="240", memory="32", cores="8"), - parameters_meta_info=OrderedDict( +flow_hyperparameters = { + "parameters": OrderedDict(time="240", memory="32", cores="8"), + "parameters_meta_info": OrderedDict( cores=OrderedDict(description="number of available cores", data_type="int"), memory=OrderedDict(description="memory in gigabytes", data_type="int"), time=OrderedDict(description="time in minutes", data_type="int"), ), -) +} # %% [markdown] # It is possible to build a flow which uses other flows. @@ -89,11 +89,11 @@ # %% autosklearn_flow = openml.flows.get_flow(9313) # auto-sklearn 0.5.1 -subflow = dict( - components=OrderedDict(automl_tool=autosklearn_flow), +subflow = { + "components": OrderedDict(automl_tool=autosklearn_flow), # If you do not want to reference a subflow, you can use the following: # components=OrderedDict(), -) +} # %% [markdown] # With all parameters of the flow defined, we can now initialize the OpenMLFlow and publish. @@ -172,19 +172,19 @@ ] # random class probabilities (Iris has 150 samples and 3 classes): -r = np.random.rand(150 * n_repeats, 3) +r = np.random.rand(150 * n_repeats, 3) # noqa: NPY002 # scale the random values so that the probabilities of each sample sum to 1: y_proba = r / r.sum(axis=1).reshape(-1, 1) y_pred = y_proba.argmax(axis=1) -class_map = dict(zip(range(3), task.class_labels)) +class_map = dict(zip(range(3), task.class_labels, strict=False)) _, y_true = task.get_X_and_y() y_true = [class_map[y] for y in y_true] # We format the predictions with the utility function `format_prediction`. # It will organize the relevant data in the expected format/order. predictions = [] -for where, y, yp, proba in zip(all_test_indices, y_true, y_pred, y_proba): +for where, y, yp, proba in zip(all_test_indices, y_true, y_pred, y_proba, strict=False): repeat, fold, index = where prediction = format_prediction( @@ -194,7 +194,7 @@ index=index, prediction=class_map[yp], truth=y, - proba={c: pb for (c, pb) in zip(task.class_labels, proba)}, + proba=dict(zip(task.class_labels, proba, strict=False)), ) predictions.append(prediction) @@ -203,7 +203,7 @@ # We use the argument setup_string because the used flow was a script. # %% -benchmark_command = f"python3 runbenchmark.py auto-sklearn medium -m aws -t 119" +benchmark_command = "python3 runbenchmark.py auto-sklearn medium -m aws -t 119" my_run = openml.runs.OpenMLRun( task_id=task_id, flow_id=flow_id, diff --git a/openml/__init__.py b/openml/__init__.py index c49505eb9..ae5db261f 100644 --- a/openml/__init__.py +++ b/openml/__init__.py @@ -91,33 +91,33 @@ def populate_cache( __all__ = [ - "OpenMLDataset", + "OpenMLBenchmarkSuite", + "OpenMLClassificationTask", + "OpenMLClusteringTask", "OpenMLDataFeature", - "OpenMLRun", - "OpenMLSplit", + "OpenMLDataset", "OpenMLEvaluation", - "OpenMLSetup", - "OpenMLParameter", - "OpenMLTask", - "OpenMLSupervisedTask", - "OpenMLClusteringTask", + "OpenMLFlow", "OpenMLLearningCurveTask", + "OpenMLParameter", "OpenMLRegressionTask", - "OpenMLClassificationTask", - "OpenMLFlow", + "OpenMLRun", + "OpenMLSetup", + "OpenMLSplit", "OpenMLStudy", - "OpenMLBenchmarkSuite", + "OpenMLSupervisedTask", + "OpenMLTask", + "__version__", + "_api_calls", + "config", "datasets", "evaluations", "exceptions", "extensions", - "config", - "runs", "flows", - "tasks", + "runs", "setups", "study", + "tasks", "utils", - "_api_calls", - "__version__", ] diff --git a/openml/_api_calls.py b/openml/_api_calls.py index 81296b3da..9e53bd9fa 100644 --- a/openml/_api_calls.py +++ b/openml/_api_calls.py @@ -12,7 +12,6 @@ import xml import zipfile from pathlib import Path -from typing import Dict, Tuple, Union import minio import requests @@ -33,8 +32,8 @@ _HEADERS = {"user-agent": f"openml-python/{__version__}"} -DATA_TYPE = Dict[str, Union[str, int]] -FILE_ELEMENTS_TYPE = Dict[str, Union[str, Tuple[str, str]]] +DATA_TYPE = dict[str, str | int] +FILE_ELEMENTS_TYPE = dict[str, str | tuple[str, str]] DATABASE_CONNECTION_ERRCODE = 107 API_TOKEN_HELP_LINK = "https://openml.github.io/openml-python/latest/examples/Basics/introduction_tutorial/#authentication" # noqa: S105 @@ -133,7 +132,7 @@ def _perform_api_call( def _download_minio_file( source: str, destination: str | Path, - exists_ok: bool = True, # noqa: FBT001, FBT002 + exists_ok: bool = True, # noqa: FBT002 proxy: str | None = "auto", ) -> None: """Download file ``source`` from a MinIO Bucket and store it at ``destination``. @@ -239,7 +238,7 @@ def _download_text_file( source: str, output_path: str | Path | None = None, md5_checksum: str | None = None, - exists_ok: bool = True, # noqa: FBT001, FBT002 + exists_ok: bool = True, # noqa: FBT002 encoding: str = "utf8", ) -> str | None: """Download the text file at `source` and store it in `output_path`. diff --git a/openml/base.py b/openml/base.py index fbfb9dfc8..a282be8eb 100644 --- a/openml/base.py +++ b/openml/base.py @@ -4,7 +4,7 @@ import re import webbrowser from abc import ABC, abstractmethod -from typing import Iterable, Sequence +from collections.abc import Iterable, Sequence import xmltodict diff --git a/openml/cli.py b/openml/cli.py index d0a46e498..4949cc89a 100644 --- a/openml/cli.py +++ b/openml/cli.py @@ -5,8 +5,8 @@ import argparse import string import sys +from collections.abc import Callable from pathlib import Path -from typing import Callable from urllib.parse import urlparse from openml import config diff --git a/openml/config.py b/openml/config.py index cf66a6346..e6104fd7f 100644 --- a/openml/config.py +++ b/openml/config.py @@ -10,11 +10,12 @@ import platform import shutil import warnings +from collections.abc import Iterator from contextlib import contextmanager from io import StringIO from pathlib import Path -from typing import Any, Iterator, cast -from typing_extensions import Literal, TypedDict +from typing import Any, Literal, cast +from typing_extensions import TypedDict from urllib.parse import urlparse logger = logging.getLogger(__name__) @@ -37,7 +38,7 @@ class _Config(TypedDict): show_progress: bool -def _create_log_handlers(create_file_handler: bool = True) -> None: # noqa: FBT001, FBT002 +def _create_log_handlers(create_file_handler: bool = True) -> None: # noqa: FBT002 """Creates but does not attach the log handlers.""" global console_handler, file_handler # noqa: PLW0603 if console_handler is not None or file_handler is not None: @@ -172,7 +173,7 @@ def get_server_base_url() -> str: ------- str """ - domain, path = server.split("/api", maxsplit=1) + domain, _path = server.split("/api", maxsplit=1) return domain.replace("api", "www") @@ -257,8 +258,8 @@ def stop_using_configuration_for_example(cls) -> None: global server # noqa: PLW0603 global apikey # noqa: PLW0603 - server = cast(str, cls._last_used_server) - apikey = cast(str, cls._last_used_key) + server = cast("str", cls._last_used_server) + apikey = cast("str", cls._last_used_key) cls._start_last_called = False @@ -515,10 +516,10 @@ def overwrite_config_context(config: dict[str, Any]) -> Iterator[_Config]: __all__ = [ "get_cache_directory", + "get_config_as_dict", "set_root_cache_directory", "start_using_configuration_for_example", "stop_using_configuration_for_example", - "get_config_as_dict", ] _setup() diff --git a/openml/datasets/__init__.py b/openml/datasets/__init__.py index 480dd9576..eb0932652 100644 --- a/openml/datasets/__init__.py +++ b/openml/datasets/__init__.py @@ -17,17 +17,17 @@ ) __all__ = [ + "OpenMLDataFeature", + "OpenMLDataset", "attributes_arff_from_df", "check_datasets_active", "create_dataset", + "delete_dataset", + "edit_dataset", + "fork_dataset", "get_dataset", "get_datasets", "list_datasets", - "OpenMLDataset", - "OpenMLDataFeature", - "status_update", "list_qualities", - "edit_dataset", - "fork_dataset", - "delete_dataset", + "status_update", ] diff --git a/openml/datasets/data_feature.py b/openml/datasets/data_feature.py index 218b0066d..0598763b0 100644 --- a/openml/datasets/data_feature.py +++ b/openml/datasets/data_feature.py @@ -1,13 +1,14 @@ # License: BSD 3-Clause from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Sequence +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any, ClassVar if TYPE_CHECKING: from IPython.lib import pretty -class OpenMLDataFeature: +class OpenMLDataFeature: # noqa: PLW1641 """ Data Feature (a.k.a. Attribute) object. @@ -51,8 +52,7 @@ def __init__( # noqa: PLR0913 if data_type == "nominal": if nominal_values is None: raise TypeError( - "Dataset features require attribute `nominal_values` for nominal " - "feature type.", + "Dataset features require attribute `nominal_values` for nominal feature type.", ) if not isinstance(nominal_values, list): @@ -75,10 +75,10 @@ def __init__( # noqa: PLR0913 self.ontologies = ontologies def __repr__(self) -> str: - return "[%d - %s (%s)]" % (self.index, self.name, self.data_type) + return f"[{self.index} - {self.name} ({self.data_type})]" def __eq__(self, other: Any) -> bool: return isinstance(other, OpenMLDataFeature) and self.__dict__ == other.__dict__ - def _repr_pretty_(self, pp: pretty.PrettyPrinter, cycle: bool) -> None: # noqa: FBT001, ARG002 + def _repr_pretty_(self, pp: pretty.PrettyPrinter, cycle: bool) -> None: # noqa: ARG002 pp.text(str(self)) diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index fa83d2b8a..9f6a79aaa 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -7,9 +7,9 @@ import pickle import re import warnings +from collections.abc import Iterable, Sequence from pathlib import Path -from typing import Any, Iterable, Sequence -from typing_extensions import Literal +from typing import Any, Literal import arff import numpy as np @@ -41,7 +41,7 @@ def _ensure_dataframe( raise TypeError(f"Data type {type(data)} not supported.") -class OpenMLDataset(OpenMLBase): +class OpenMLDataset(OpenMLBase): # noqa: PLW1641 """Dataset object. Allows fetching and uploading datasets to OpenML. @@ -719,8 +719,8 @@ def valid_category(cat: Any) -> bool: def get_data( # noqa: C901 self, target: list[str] | str | None = None, - include_row_id: bool = False, # noqa: FBT001, FBT002 - include_ignore_attribute: bool = False, # noqa: FBT001, FBT002 + include_row_id: bool = False, # noqa: FBT002 + include_ignore_attribute: bool = False, # noqa: FBT002 ) -> tuple[pd.DataFrame, pd.Series | None, list[bool], list[str]]: """Returns dataset content as dataframes. @@ -766,8 +766,8 @@ def get_data( # noqa: C901 logger.info(f"Going to remove the following attributes: {to_exclude}") keep = np.array([column not in to_exclude for column in attribute_names]) data = data.drop(columns=to_exclude) - categorical_mask = [cat for cat, k in zip(categorical_mask, keep) if k] - attribute_names = [att for att, k in zip(attribute_names, keep) if k] + categorical_mask = [cat for cat, k in zip(categorical_mask, keep, strict=False) if k] + attribute_names = [att for att, k in zip(attribute_names, keep, strict=False) if k] if target is None: return data, None, categorical_mask, attribute_names @@ -863,8 +863,8 @@ def get_features_by_type( # noqa: C901 self, data_type: str, exclude: list[str] | None = None, - exclude_ignore_attribute: bool = True, # noqa: FBT002, FBT001 - exclude_row_id_attribute: bool = True, # noqa: FBT002, FBT001 + exclude_ignore_attribute: bool = True, # noqa: FBT002 + exclude_row_id_attribute: bool = True, # noqa: FBT002 ) -> list[int]: """ Return indices of features of a given type, e.g. all nominal features. diff --git a/openml/datasets/functions.py b/openml/datasets/functions.py index ac5466a44..3ac657ea0 100644 --- a/openml/datasets/functions.py +++ b/openml/datasets/functions.py @@ -9,8 +9,7 @@ from functools import partial from pathlib import Path from pyexpat import ExpatError -from typing import TYPE_CHECKING, Any -from typing_extensions import Literal +from typing import TYPE_CHECKING, Any, Literal import arff import minio.error @@ -259,7 +258,7 @@ def _validated_data_attributes( def check_datasets_active( dataset_ids: list[int], - raise_error_if_not_exist: bool = True, # noqa: FBT001, FBT002 + raise_error_if_not_exist: bool = True, # noqa: FBT002 ) -> dict[int, bool]: """ Check if the dataset ids provided are active. @@ -293,7 +292,7 @@ def check_datasets_active( def _name_to_id( dataset_name: str, version: int | None = None, - error_if_multiple: bool = False, # noqa: FBT001, FBT002 + error_if_multiple: bool = False, # noqa: FBT002 ) -> int: """Attempt to find the dataset id of the dataset with the given name. @@ -341,8 +340,8 @@ def _name_to_id( def get_datasets( dataset_ids: list[str | int], - download_data: bool = False, # noqa: FBT001, FBT002 - download_qualities: bool = False, # noqa: FBT001, FBT002 + download_data: bool = False, # noqa: FBT002 + download_qualities: bool = False, # noqa: FBT002 ) -> list[OpenMLDataset]: """Download datasets. @@ -377,14 +376,14 @@ def get_datasets( @openml.utils.thread_safe_if_oslo_installed def get_dataset( # noqa: C901, PLR0912 dataset_id: int | str, - download_data: bool = False, # noqa: FBT002, FBT001 + download_data: bool = False, # noqa: FBT002 version: int | None = None, - error_if_multiple: bool = False, # noqa: FBT002, FBT001 + error_if_multiple: bool = False, # noqa: FBT002 cache_format: Literal["pickle", "feather"] = "pickle", - download_qualities: bool = False, # noqa: FBT002, FBT001 - download_features_meta_data: bool = False, # noqa: FBT002, FBT001 - download_all_files: bool = False, # noqa: FBT002, FBT001 - force_refresh_cache: bool = False, # noqa: FBT001, FBT002 + download_qualities: bool = False, # noqa: FBT002 + download_features_meta_data: bool = False, # noqa: FBT002 + download_all_files: bool = False, # noqa: FBT002 + force_refresh_cache: bool = False, # noqa: FBT002 ) -> OpenMLDataset: """Download the OpenML dataset representation, optionally also download actual data file. @@ -1116,7 +1115,7 @@ def _get_dataset_description(did_cache_dir: Path, dataset_id: int) -> dict[str, def _get_dataset_parquet( description: dict | OpenMLDataset, cache_directory: Path | None = None, - download_all_files: bool = False, # noqa: FBT001, FBT002 + download_all_files: bool = False, # noqa: FBT002 ) -> Path | None: """Return the path to the local parquet file of the dataset. If is not cached, it is downloaded. @@ -1418,7 +1417,7 @@ def _get_online_dataset_arff(dataset_id: int) -> str | None: str or None A string representation of an ARFF file. Or None if file already exists. """ - dataset_xml = openml._api_calls._perform_api_call("data/%d" % dataset_id, "get") + dataset_xml = openml._api_calls._perform_api_call(f"data/{dataset_id}", "get") # build a dict from the xml. # use the url from the dataset description and return the ARFF string return openml._api_calls._download_text_file( @@ -1439,7 +1438,7 @@ def _get_online_dataset_format(dataset_id: int) -> str: str Dataset format. """ - dataset_xml = openml._api_calls._perform_api_call("data/%d" % dataset_id, "get") + dataset_xml = openml._api_calls._perform_api_call(f"data/{dataset_id}", "get") # build a dict from the xml and get the format from the dataset description return xmltodict.parse(dataset_xml)["oml:data_set_description"]["oml:format"].lower() # type: ignore diff --git a/openml/evaluations/__init__.py b/openml/evaluations/__init__.py index dbff47037..b56d0c2d5 100644 --- a/openml/evaluations/__init__.py +++ b/openml/evaluations/__init__.py @@ -5,7 +5,7 @@ __all__ = [ "OpenMLEvaluation", - "list_evaluations", "list_evaluation_measures", + "list_evaluations", "list_evaluations_setups", ] diff --git a/openml/evaluations/functions.py b/openml/evaluations/functions.py index 7747294d7..0b9f190b4 100644 --- a/openml/evaluations/functions.py +++ b/openml/evaluations/functions.py @@ -5,8 +5,8 @@ import json from functools import partial from itertools import chain -from typing import Any -from typing_extensions import Literal, overload +from typing import Any, Literal +from typing_extensions import overload import numpy as np import pandas as pd @@ -228,7 +228,7 @@ def __list_evaluations(api_call: str) -> list[OpenMLEvaluation]: # Minimalistic check if the XML is useful if "oml:evaluations" not in evals_dict: raise ValueError( - "Error in return XML, does not contain " f'"oml:evaluations": {evals_dict!s}', + f'Error in return XML, does not contain "oml:evaluations": {evals_dict!s}', ) assert isinstance(evals_dict["oml:evaluations"]["oml:evaluation"], list), type( @@ -339,7 +339,7 @@ def list_evaluations_setups( tag: str | None = None, per_fold: bool | None = None, sort_order: str | None = None, - parameters_in_separate_columns: bool = False, # noqa: FBT001, FBT002 + parameters_in_separate_columns: bool = False, # noqa: FBT002 ) -> pd.DataFrame: """List all run-evaluation pairs matching all of the given filters and their hyperparameter settings. diff --git a/openml/extensions/__init__.py b/openml/extensions/__init__.py index b49865e0e..979986182 100644 --- a/openml/extensions/__init__.py +++ b/openml/extensions/__init__.py @@ -1,16 +1,15 @@ # License: BSD 3-Clause -from typing import List, Type # noqa: F401 from .extension_interface import Extension from .functions import get_extension_by_flow, get_extension_by_model, register_extension -extensions = [] # type: List[Type[Extension]] +extensions: list[type[Extension]] = [] __all__ = [ "Extension", - "register_extension", - "get_extension_by_model", "get_extension_by_flow", + "get_extension_by_model", + "register_extension", ] diff --git a/openml/extensions/extension_interface.py b/openml/extensions/extension_interface.py index 2a336eb52..e391d109a 100644 --- a/openml/extensions/extension_interface.py +++ b/openml/extensions/extension_interface.py @@ -63,8 +63,8 @@ def can_handle_model(cls, model: Any) -> bool: def flow_to_model( self, flow: OpenMLFlow, - initialize_with_defaults: bool = False, # noqa: FBT001, FBT002 - strict_version: bool = True, # noqa: FBT002, FBT001 + initialize_with_defaults: bool = False, # noqa: FBT002 + strict_version: bool = True, # noqa: FBT002 ) -> Any: """Instantiate a model from the flow representation. diff --git a/openml/extensions/functions.py b/openml/extensions/functions.py index 06902325e..44df5ec69 100644 --- a/openml/extensions/functions.py +++ b/openml/extensions/functions.py @@ -42,7 +42,7 @@ def register_extension(extension: type[Extension]) -> None: def get_extension_by_flow( flow: OpenMLFlow, - raise_if_no_extension: bool = False, # noqa: FBT001, FBT002 + raise_if_no_extension: bool = False, # noqa: FBT002 ) -> Extension | None: """Get an extension which can handle the given flow. @@ -91,7 +91,7 @@ def get_extension_by_flow( def get_extension_by_model( model: Any, - raise_if_no_extension: bool = False, # noqa: FBT001, FBT002 + raise_if_no_extension: bool = False, # noqa: FBT002 ) -> Extension | None: """Get an extension which can handle the given flow. diff --git a/openml/flows/__init__.py b/openml/flows/__init__.py index ce32fec7d..d455249de 100644 --- a/openml/flows/__init__.py +++ b/openml/flows/__init__.py @@ -12,10 +12,10 @@ __all__ = [ "OpenMLFlow", - "get_flow", - "list_flows", - "get_flow_id", - "flow_exists", "assert_flows_equal", "delete_flow", + "flow_exists", + "get_flow", + "get_flow_id", + "list_flows", ] diff --git a/openml/flows/flow.py b/openml/flows/flow.py index 02d24e78b..7dd84fdee 100644 --- a/openml/flows/flow.py +++ b/openml/flows/flow.py @@ -3,8 +3,9 @@ import logging from collections import OrderedDict +from collections.abc import Hashable, Sequence from pathlib import Path -from typing import Any, Hashable, Sequence, cast +from typing import Any, cast import xmltodict @@ -169,7 +170,7 @@ def extension(self) -> Extension: """The extension of the flow (e.g., sklearn).""" if self._extension is None: self._extension = cast( - Extension, get_extension_by_flow(self, raise_if_no_extension=True) + "Extension", get_extension_by_flow(self, raise_if_no_extension=True) ) return self._extension @@ -408,7 +409,7 @@ def _parse_publish_response(self, xml_response: dict) -> None: """Parse the id from the xml_response and assign it to self.""" self.flow_id = int(xml_response["oml:upload_flow"]["oml:id"]) - def publish(self, raise_error_if_exists: bool = False) -> OpenMLFlow: # noqa: FBT001, FBT002 + def publish(self, raise_error_if_exists: bool = False) -> OpenMLFlow: # noqa: FBT002 """Publish this flow to OpenML server. Raises a PyOpenMLError if the flow exists on the server, but @@ -435,7 +436,7 @@ def publish(self, raise_error_if_exists: bool = False) -> OpenMLFlow: # noqa: F if not flow_id: if self.flow_id: raise openml.exceptions.PyOpenMLError( - "Flow does not exist on the server, " "but 'flow.flow_id' is not None.", + "Flow does not exist on the server, but 'flow.flow_id' is not None.", ) super().publish() assert self.flow_id is not None # for mypy @@ -445,7 +446,7 @@ def publish(self, raise_error_if_exists: bool = False) -> OpenMLFlow: # noqa: F raise openml.exceptions.PyOpenMLError(error_message) elif self.flow_id is not None and self.flow_id != flow_id: raise openml.exceptions.PyOpenMLError( - "Local flow_id does not match server flow_id: " f"'{self.flow_id}' vs '{flow_id}'", + f"Local flow_id does not match server flow_id: '{self.flow_id}' vs '{flow_id}'", ) flow = openml.flows.functions.get_flow(flow_id) @@ -517,7 +518,7 @@ def get_subflow(self, structure: list[str]) -> OpenMLFlow: sub_identifier = structure[0] if sub_identifier not in self.components: raise ValueError( - f"Flow {self.name} does not contain component with " f"identifier {sub_identifier}", + f"Flow {self.name} does not contain component with identifier {sub_identifier}", ) if len(structure) == 1: return self.components[sub_identifier] # type: ignore diff --git a/openml/flows/functions.py b/openml/flows/functions.py index 9906958e5..6c2393f10 100644 --- a/openml/flows/functions.py +++ b/openml/flows/functions.py @@ -5,7 +5,7 @@ import re from collections import OrderedDict from functools import partial -from typing import Any, Dict +from typing import Any import dateutil.parser import pandas as pd @@ -31,7 +31,7 @@ def _get_cached_flows() -> OrderedDict: flows = OrderedDict() # type: 'OrderedDict[int, OpenMLFlow]' flow_cache_dir = openml.utils._create_cache_directory(FLOWS_CACHE_DIR_NAME) - directory_content = os.listdir(flow_cache_dir) + directory_content = os.listdir(flow_cache_dir) # noqa: PTH208 directory_content.sort() # Find all flow ids for which we have downloaded # the flow description @@ -66,11 +66,11 @@ def _get_cached_flow(fid: int) -> OpenMLFlow: return _create_flow_from_xml(fh.read()) except OSError as e: openml.utils._remove_cache_dir_for_id(FLOWS_CACHE_DIR_NAME, fid_cache_dir) - raise OpenMLCacheException("Flow file for fid %d not cached" % fid) from e + raise OpenMLCacheException(f"Flow file for fid {fid} not cached") from e @openml.utils.thread_safe_if_oslo_installed -def get_flow(flow_id: int, reinstantiate: bool = False, strict_version: bool = True) -> OpenMLFlow: # noqa: FBT001, FBT002 +def get_flow(flow_id: int, reinstantiate: bool = False, strict_version: bool = True) -> OpenMLFlow: # noqa: FBT002 """Download the OpenML flow for a given flow ID. Parameters @@ -124,7 +124,7 @@ def _get_flow_description(flow_id: int) -> OpenMLFlow: xml_file = ( openml.utils._create_cache_directory_for_id(FLOWS_CACHE_DIR_NAME, flow_id) / "flow.xml" ) - flow_xml = openml._api_calls._perform_api_call("flow/%d" % flow_id, request_method="get") + flow_xml = openml._api_calls._perform_api_call(f"flow/{flow_id}", request_method="get") with xml_file.open("w", encoding="utf8") as fh: fh.write(flow_xml) @@ -245,7 +245,7 @@ def flow_exists(name: str, external_version: str) -> int | bool: def get_flow_id( model: Any | None = None, name: str | None = None, - exact_version: bool = True, # noqa: FBT001, FBT002 + exact_version: bool = True, # noqa: FBT002 ) -> int | bool | list[int]: """Retrieves the flow id for a model or a flow name. @@ -364,9 +364,9 @@ def assert_flows_equal( # noqa: C901, PLR0912, PLR0913, PLR0915 flow1: OpenMLFlow, flow2: OpenMLFlow, ignore_parameter_values_on_older_children: str | None = None, - ignore_parameter_values: bool = False, # noqa: FBT001, FBT002 - ignore_custom_name_if_none: bool = False, # noqa: FBT001, FBT002 - check_description: bool = True, # noqa: FBT001, FBT002 + ignore_parameter_values: bool = False, # noqa: FBT002 + ignore_custom_name_if_none: bool = False, # noqa: FBT002 + check_description: bool = True, # noqa: FBT002 ) -> None: """Check equality of two flows. @@ -417,7 +417,7 @@ def assert_flows_equal( # noqa: C901, PLR0912, PLR0913, PLR0915 attr1 = getattr(flow1, key, None) attr2 = getattr(flow2, key, None) if key == "components": - if not (isinstance(attr1, Dict) and isinstance(attr2, Dict)): + if not (isinstance(attr1, dict) and isinstance(attr2, dict)): raise TypeError("Cannot compare components because they are not dictionary.") for name in set(attr1.keys()).union(attr2.keys()): @@ -456,9 +456,9 @@ def assert_flows_equal( # noqa: C901, PLR0912, PLR0913, PLR0915 ) if ignore_parameter_values_on_older_children: - assert ( - flow1.upload_date is not None - ), "Flow1 has no upload date that allows us to compare age of children." + assert flow1.upload_date is not None, ( + "Flow1 has no upload date that allows us to compare age of children." + ) upload_date_current_flow = dateutil.parser.parse(flow1.upload_date) upload_date_parent_flow = dateutil.parser.parse( ignore_parameter_values_on_older_children, @@ -493,8 +493,8 @@ def assert_flows_equal( # noqa: C901, PLR0912, PLR0913, PLR0915 # iterating over the parameter's meta info list for param in params1: if ( - isinstance(flow1.parameters_meta_info[param], Dict) - and isinstance(flow2.parameters_meta_info[param], Dict) + isinstance(flow1.parameters_meta_info[param], dict) + and isinstance(flow2.parameters_meta_info[param], dict) and "data_type" in flow1.parameters_meta_info[param] and "data_type" in flow2.parameters_meta_info[param] ): diff --git a/openml/runs/__init__.py b/openml/runs/__init__.py index 6d3dca504..2f068a2e6 100644 --- a/openml/runs/__init__.py +++ b/openml/runs/__init__.py @@ -19,14 +19,14 @@ "OpenMLRun", "OpenMLRunTrace", "OpenMLTraceIteration", - "run_model_on_task", - "run_flow_on_task", + "delete_run", "get_run", - "list_runs", - "get_runs", "get_run_trace", - "run_exists", + "get_runs", "initialize_model_from_run", "initialize_model_from_trace", - "delete_run", + "list_runs", + "run_exists", + "run_flow_on_task", + "run_model_on_task", ] diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 666b75c37..5a21b8bc1 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -62,9 +62,9 @@ def run_model_on_task( # noqa: PLR0913 avoid_duplicate_runs: bool | None = None, flow_tags: list[str] | None = None, seed: int | None = None, - add_local_measures: bool = True, # noqa: FBT001, FBT002 - upload_flow: bool = False, # noqa: FBT001, FBT002 - return_flow: bool = False, # noqa: FBT001, FBT002 + add_local_measures: bool = True, # noqa: FBT002 + upload_flow: bool = False, # noqa: FBT002 + return_flow: bool = False, # noqa: FBT002 n_jobs: int | None = None, ) -> OpenMLRun | tuple[OpenMLRun, OpenMLFlow]: """Run the model on the dataset defined by the task. @@ -181,8 +181,8 @@ def run_flow_on_task( # noqa: C901, PLR0912, PLR0915, PLR0913 avoid_duplicate_runs: bool | None = None, flow_tags: list[str] | None = None, seed: int | None = None, - add_local_measures: bool = True, # noqa: FBT001, FBT002 - upload_flow: bool = False, # noqa: FBT001, FBT002 + add_local_measures: bool = True, # noqa: FBT002 + upload_flow: bool = False, # noqa: FBT002 n_jobs: int | None = None, ) -> OpenMLRun: """Run the model provided by the flow on the dataset defined by task. @@ -353,7 +353,7 @@ def get_run_trace(run_id: int) -> OpenMLRunTrace: ------- openml.runs.OpenMLTrace """ - trace_xml = openml._api_calls._perform_api_call("run/trace/%d" % run_id, "get") + trace_xml = openml._api_calls._perform_api_call(f"run/trace/{run_id}", "get") return OpenMLRunTrace.trace_from_xml(trace_xml) @@ -608,7 +608,7 @@ def _calculate_local_measure( # type: ignore index=tst_idx, prediction=prediction, truth=truth, - proba=dict(zip(task.class_labels, pred_prob)), + proba=dict(zip(task.class_labels, pred_prob, strict=False)), ) else: raise ValueError("The task has no class labels") @@ -798,7 +798,7 @@ def get_runs(run_ids: list[int]) -> list[OpenMLRun]: @openml.utils.thread_safe_if_oslo_installed -def get_run(run_id: int, ignore_cache: bool = False) -> OpenMLRun: # noqa: FBT002, FBT001 +def get_run(run_id: int, ignore_cache: bool = False) -> OpenMLRun: # noqa: FBT002 """Gets run corresponding to run_id. Parameters @@ -828,14 +828,14 @@ def get_run(run_id: int, ignore_cache: bool = False) -> OpenMLRun: # noqa: FBT0 raise OpenMLCacheException(message="dummy") except OpenMLCacheException: - run_xml = openml._api_calls._perform_api_call("run/%d" % run_id, "get") + run_xml = openml._api_calls._perform_api_call(f"run/{run_id}", "get") with run_file.open("w", encoding="utf8") as fh: fh.write(run_xml) return _create_run_from_xml(run_xml) -def _create_run_from_xml(xml: str, from_server: bool = True) -> OpenMLRun: # noqa: PLR0915, PLR0912, C901, FBT001, FBT002 +def _create_run_from_xml(xml: str, from_server: bool = True) -> OpenMLRun: # noqa: PLR0915, PLR0912, C901, FBT002 """Create a run object from xml returned from server. Parameters @@ -977,7 +977,7 @@ def obtain_field(xml_obj, fieldname, from_server, cast=None): # type: ignore evaluations[key] = value if "description" not in files and from_server is True: - raise ValueError("No description file for run %d in run description XML" % run_id) + raise ValueError(f"No description file for run {run_id} in run description XML") if "predictions" not in files and from_server is True: task = openml.tasks.get_task(task_id) @@ -988,7 +988,7 @@ def obtain_field(xml_obj, fieldname, from_server, cast=None): # type: ignore # a run can consist without predictions. But for now let's keep it # Matthias: yes, it should stay as long as we do not really handle # this stuff - raise ValueError("No prediction files for run %d in run description XML" % run_id) + raise ValueError(f"No prediction files for run {run_id} in run description XML") tags = openml.utils.extract_xml_tags("oml:tag", run) @@ -1037,7 +1037,7 @@ def list_runs( # noqa: PLR0913 uploader: list | None = None, tag: str | None = None, study: int | None = None, - display_errors: bool = False, # noqa: FBT001, FBT002 + display_errors: bool = False, # noqa: FBT002 task_type: TaskType | int | None = None, ) -> pd.DataFrame: """ @@ -1171,7 +1171,7 @@ def _list_runs( # noqa: PLR0913, C901 if uploader is not None: api_call += f"/uploader/{','.join([str(int(i)) for i in uploader])}" if study is not None: - api_call += "/study/%d" % study + api_call += f"/study/{study}" if display_errors: api_call += "/show_errors/true" if tag is not None: diff --git a/openml/runs/run.py b/openml/runs/run.py index 945264131..b6997fb53 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -4,12 +4,11 @@ import pickle import time from collections import OrderedDict +from collections.abc import Callable, Sequence from pathlib import Path from typing import ( TYPE_CHECKING, Any, - Callable, - Sequence, ) import arff @@ -280,7 +279,7 @@ def _get_repr_body_fields(self) -> Sequence[tuple[str, str | int | list[str]]]: ] @classmethod - def from_filesystem(cls, directory: str | Path, expect_model: bool = True) -> OpenMLRun: # noqa: FBT001, FBT002 + def from_filesystem(cls, directory: str | Path, expect_model: bool = True) -> OpenMLRun: # noqa: FBT002 """ The inverse of the to_filesystem method. Instantiates an OpenMLRun object based on files stored on the file system. @@ -347,7 +346,7 @@ def from_filesystem(cls, directory: str | Path, expect_model: bool = True) -> Op def to_filesystem( self, directory: str | Path, - store_model: bool = True, # noqa: FBT001, FBT002 + store_model: bool = True, # noqa: FBT002 ) -> None: """ The inverse of the from_filesystem method. Serializes a run @@ -365,7 +364,7 @@ def to_filesystem( model. """ if self.data_content is None or self.model is None: - raise ValueError("Run should have been executed (and contain " "model / predictions)") + raise ValueError("Run should have been executed (and contain model / predictions)") directory = Path(directory) directory.mkdir(exist_ok=True, parents=True) @@ -517,7 +516,7 @@ def get_metric_fn(self, sklearn_fn: Callable, kwargs: dict | None = None) -> np. # TODO: make this a stream reader else: raise ValueError( - "Run should have been locally executed or " "contain outputfile reference.", + "Run should have been locally executed or contain outputfile reference.", ) # Need to know more about the task to compute scores correctly @@ -528,11 +527,11 @@ def get_metric_fn(self, sklearn_fn: Callable, kwargs: dict | None = None) -> np. task.task_type_id in [TaskType.SUPERVISED_CLASSIFICATION, TaskType.LEARNING_CURVE] and "correct" not in attribute_names ): - raise ValueError('Attribute "correct" should be set for ' "classification task runs") + raise ValueError('Attribute "correct" should be set for classification task runs') if task.task_type_id == TaskType.SUPERVISED_REGRESSION and "truth" not in attribute_names: - raise ValueError('Attribute "truth" should be set for ' "regression task runs") + raise ValueError('Attribute "truth" should be set for regression task runs') if task.task_type_id != TaskType.CLUSTERING and "prediction" not in attribute_names: - raise ValueError('Attribute "predict" should be set for ' "supervised task runs") + raise ValueError('Attribute "prediction" should be set for supervised task runs') def _attribute_list_to_dict(attribute_list): # type: ignore # convenience function: Creates a mapping to map from the name of @@ -566,7 +565,7 @@ def _attribute_list_to_dict(attribute_list): # type: ignore pred = predictions_arff["attributes"][predicted_idx][1] corr = predictions_arff["attributes"][correct_idx][1] raise ValueError( - "Predicted and Correct do not have equal values:" f" {pred!s} Vs. {corr!s}", + f"Predicted and Correct do not have equal values: {pred!s} Vs. {corr!s}", ) # TODO: these could be cached @@ -602,7 +601,7 @@ def _attribute_list_to_dict(attribute_list): # type: ignore values_correct[rep][fold][samp].append(correct) scores = [] - for rep in values_predict: + for rep in values_predict: # noqa: PLC0206 for fold in values_predict[rep]: last_sample = len(values_predict[rep][fold]) - 1 y_pred = values_predict[rep][fold][last_sample] diff --git a/openml/runs/trace.py b/openml/runs/trace.py index bc9e1b5d6..708cdd8f1 100644 --- a/openml/runs/trace.py +++ b/openml/runs/trace.py @@ -3,9 +3,10 @@ import json from collections import OrderedDict +from collections.abc import Iterator from dataclasses import dataclass from pathlib import Path -from typing import IO, Any, Iterator +from typing import IO, Any from typing_extensions import Self import arff @@ -149,9 +150,7 @@ def get_selected_iteration(self, fold: int, repeat: int) -> int: for r, f, i in self.trace_iterations: if r == repeat and f == fold and self.trace_iterations[(r, f, i)].selected is True: return i - raise ValueError( - "Could not find the selected iteration for rep/fold %d/%d" % (repeat, fold), - ) + raise ValueError(f"Could not find the selected iteration for rep/fold {repeat}/{fold}") @classmethod def generate( @@ -185,8 +184,7 @@ def generate( raise ValueError("Trace content is empty.") if len(attributes) != len(content[0]): raise ValueError( - "Trace_attributes and trace_content not compatible:" - f" {attributes} vs {content[0]}", + f"Trace_attributes and trace_content not compatible: {attributes} vs {content[0]}", ) return cls._trace_from_arff_struct( diff --git a/openml/setups/__init__.py b/openml/setups/__init__.py index dd38cb9b7..fa4072059 100644 --- a/openml/setups/__init__.py +++ b/openml/setups/__init__.py @@ -4,10 +4,10 @@ from .setup import OpenMLParameter, OpenMLSetup __all__ = [ - "OpenMLSetup", "OpenMLParameter", + "OpenMLSetup", "get_setup", + "initialize_model", "list_setups", "setup_exists", - "initialize_model", ] diff --git a/openml/setups/functions.py b/openml/setups/functions.py index 374911901..4bf279ed1 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -2,11 +2,11 @@ from __future__ import annotations from collections import OrderedDict +from collections.abc import Iterable from functools import partial from itertools import chain from pathlib import Path -from typing import Any, Iterable -from typing_extensions import Literal +from typing import Any, Literal import pandas as pd import xmltodict @@ -94,7 +94,7 @@ def _get_cached_setup(setup_id: int) -> OpenMLSetup: except OSError as e: raise openml.exceptions.OpenMLCacheException( - "Setup file for setup id %d not cached" % setup_id, + f"Setup file for setup id {setup_id} not cached", ) from e diff --git a/openml/study/__init__.py b/openml/study/__init__.py index b7d77fec4..37a6d376a 100644 --- a/openml/study/__init__.py +++ b/openml/study/__init__.py @@ -19,8 +19,8 @@ from .study import OpenMLBenchmarkSuite, OpenMLStudy __all__ = [ - "OpenMLStudy", "OpenMLBenchmarkSuite", + "OpenMLStudy", "attach_to_study", "attach_to_suite", "create_benchmark_suite", @@ -33,6 +33,6 @@ "get_suite", "list_studies", "list_suites", - "update_suite_status", "update_study_status", + "update_suite_status", ] diff --git a/openml/study/functions.py b/openml/study/functions.py index 4e16879d7..bb24ddcff 100644 --- a/openml/study/functions.py +++ b/openml/study/functions.py @@ -1,5 +1,4 @@ # License: BSD 3-Clause -# ruff: noqa: PLR0913 from __future__ import annotations import warnings @@ -422,7 +421,7 @@ def detach_from_study(study_id: int, run_ids: list[int]) -> int: new size of the study (in terms of explicitly linked entities) """ # Interestingly, there's no need to tell the server about the entity type, it knows by itself - uri = "study/%d/detach" % study_id + uri = f"study/{study_id}/detach" post_variables = {"ids": ",".join(str(x) for x in run_ids)} # type: openml._api_calls.DATA_TYPE result_xml = openml._api_calls._perform_api_call( call=uri, diff --git a/openml/study/study.py b/openml/study/study.py index 83bbf0497..de4aac0f4 100644 --- a/openml/study/study.py +++ b/openml/study/study.py @@ -2,7 +2,8 @@ # TODO(eddiebergman): Begging for dataclassses to shorten this all from __future__ import annotations -from typing import Any, Sequence +from collections.abc import Sequence +from typing import Any from openml.base import OpenMLBase from openml.config import get_server_base_url diff --git a/openml/tasks/__init__.py b/openml/tasks/__init__.py index f6df3a8d4..34c994e3a 100644 --- a/openml/tasks/__init__.py +++ b/openml/tasks/__init__.py @@ -19,17 +19,17 @@ ) __all__ = [ - "OpenMLTask", - "OpenMLSupervisedTask", - "OpenMLClusteringTask", - "OpenMLRegressionTask", "OpenMLClassificationTask", + "OpenMLClusteringTask", "OpenMLLearningCurveTask", + "OpenMLRegressionTask", + "OpenMLSplit", + "OpenMLSupervisedTask", + "OpenMLTask", + "TaskType", "create_task", + "delete_task", "get_task", "get_tasks", "list_tasks", - "OpenMLSplit", - "TaskType", - "delete_task", ] diff --git a/openml/tasks/functions.py b/openml/tasks/functions.py index e9b879ae4..c60e0c483 100644 --- a/openml/tasks/functions.py +++ b/openml/tasks/functions.py @@ -38,7 +38,7 @@ def _get_cached_tasks() -> dict[int, OpenMLTask]: OpenMLTask. """ task_cache_dir = openml.utils._create_cache_directory(TASKS_CACHE_DIR_NAME) - directory_content = os.listdir(task_cache_dir) + directory_content = os.listdir(task_cache_dir) # noqa: PTH208 directory_content.sort() # Find all dataset ids for which we have downloaded the dataset @@ -329,7 +329,7 @@ def __list_tasks(api_call: str) -> pd.DataFrame: # noqa: C901, PLR0912 except KeyError as e: if tid is not None: warnings.warn( - "Invalid xml for task %d: %s\nFrom %s" % (tid, e, task_), + f"Invalid xml for task {tid}: {e}\nFrom {task_}", RuntimeWarning, stacklevel=2, ) @@ -388,7 +388,7 @@ def get_tasks( @openml.utils.thread_safe_if_oslo_installed def get_task( task_id: int, - download_splits: bool = False, # noqa: FBT001, FBT002 + download_splits: bool = False, # noqa: FBT002 **get_dataset_kwargs: Any, ) -> OpenMLTask: """Download OpenML task for a given task ID. @@ -444,7 +444,7 @@ def _get_task_description(task_id: int) -> OpenMLTask: except OpenMLCacheException: _cache_dir = openml.utils._create_cache_directory_for_id(TASKS_CACHE_DIR_NAME, task_id) xml_file = _cache_dir / "task.xml" - task_xml = openml._api_calls._perform_api_call("task/%d" % task_id, "get") + task_xml = openml._api_calls._perform_api_call(f"task/{task_id}", "get") with xml_file.open("w", encoding="utf8") as fh: fh.write(task_xml) diff --git a/openml/tasks/split.py b/openml/tasks/split.py index 4e781df35..464e41b2a 100644 --- a/openml/tasks/split.py +++ b/openml/tasks/split.py @@ -18,7 +18,7 @@ class Split(NamedTuple): test: np.ndarray -class OpenMLSplit: +class OpenMLSplit: # noqa: PLW1641 """OpenML Split object. This class manages train-test splits for a dataset across multiple diff --git a/openml/tasks/task.py b/openml/tasks/task.py index 395b52482..d4998970c 100644 --- a/openml/tasks/task.py +++ b/openml/tasks/task.py @@ -5,9 +5,10 @@ import warnings from abc import ABC +from collections.abc import Sequence from enum import Enum from pathlib import Path -from typing import TYPE_CHECKING, Any, Sequence +from typing import TYPE_CHECKING, Any from typing_extensions import TypedDict import openml._api_calls diff --git a/openml/testing.py b/openml/testing.py index d1da16876..8d3bbbd5b 100644 --- a/openml/testing.py +++ b/openml/testing.py @@ -80,7 +80,7 @@ def setUp(self, n_levels: int = 1, tmpdir_suffix: str = "") -> None: for _ in range(n_levels): static_cache_dir = static_cache_dir.parent.absolute() - content = os.listdir(static_cache_dir) + content = os.listdir(static_cache_dir) # noqa: PTH208 if "files" in content: static_cache_dir = static_cache_dir / "files" else: @@ -166,7 +166,11 @@ def _delete_entity_from_tracker(cls, entity_type: str, entity: int) -> None: delete_index = next( i for i, (id_, _) in enumerate( - zip(TestBase.publish_tracker[entity_type], TestBase.flow_name_tracker), + zip( + TestBase.publish_tracker[entity_type], + TestBase.flow_name_tracker, + strict=False, + ), ) if id_ == entity ) @@ -352,9 +356,9 @@ def create_request_response( __all__ = [ - "TestBase", - "SimpleImputer", "CustomImputer", + "SimpleImputer", + "TestBase", "check_task_existence", "create_request_response", ] diff --git a/openml/utils.py b/openml/utils.py index 7e72e7aee..3680bc0ff 100644 --- a/openml/utils.py +++ b/openml/utils.py @@ -4,10 +4,11 @@ import contextlib import shutil import warnings +from collections.abc import Callable, Mapping, Sized from functools import wraps from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Mapping, Sized, TypeVar, overload -from typing_extensions import Literal, ParamSpec +from typing import TYPE_CHECKING, Any, Literal, TypeVar, overload +from typing_extensions import ParamSpec import numpy as np import xmltodict @@ -103,7 +104,7 @@ def _get_rest_api_type_alias(oml_object: OpenMLBase) -> str: return api_type_alias -def _tag_openml_base(oml_object: OpenMLBase, tag: str, untag: bool = False) -> None: # noqa: FBT001, FBT002 +def _tag_openml_base(oml_object: OpenMLBase, tag: str, untag: bool = False) -> None: # noqa: FBT002 api_type_alias = _get_rest_api_type_alias(oml_object) if oml_object.id is None: raise openml.exceptions.ObjectNotPublishedError( @@ -198,7 +199,7 @@ def _delete_entity(entity_type: str, entity_id: int) -> bool: if entity_type not in legal_entities: raise ValueError(f"Can't delete a {entity_type}") - url_suffix = "%s/%d" % (entity_type, entity_id) + url_suffix = f"{entity_type}/{entity_id}" try: result_xml = openml._api_calls._perform_api_call(url_suffix, "delete") result = xmltodict.parse(result_xml) @@ -344,7 +345,7 @@ def _create_cache_directory(key: str) -> Path: return cache_dir -def _get_cache_dir_for_id(key: str, id_: int, create: bool = False) -> Path: # noqa: FBT001, FBT002 +def _get_cache_dir_for_id(key: str, id_: int, create: bool = False) -> Path: # noqa: FBT002 cache_dir = _create_cache_directory(key) if create else _get_cache_dir_for_key(key) return Path(cache_dir) / str(id_) diff --git a/pyproject.toml b/pyproject.toml index 14309c2d5..93a6ffbfa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -141,7 +141,7 @@ markers = [ # https://github.com/charliermarsh/ruff [tool.ruff] -target-version = "py38" +target-version = "py310" line-length = 100 output-format = "grouped" src = ["openml", "tests", "examples"] @@ -274,9 +274,11 @@ ignore = [ "S101", # Use of assert detected. "W292", # No newline at end of file "PLC1901", # "" can be simplified to be falsey - "TCH003", # Move stdlib import into TYPE_CHECKING + "TC003", # Move stdlib import into TYPE_CHECKING "COM812", # Trailing comma missing (handled by linter, ruff recommend disabling if using formatter) "N803", # Argument should be lowercase (but we accept things like `X`) + "PLC0415", # Allow imports inside functions / non-top-level scope + "FBT001", # Allow Boolean-typed positional argument in function definition # TODO(@eddibergman): These should be enabled "D100", # Missing docstring in public module @@ -307,7 +309,7 @@ force-wrap-aliases = true convention = "numpy" [tool.mypy] -python_version = "3.8" +python_version = "3.10" packages = ["openml", "tests"] show_error_codes = true diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 000000000..000969b80 --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"""Package for scripts and utilities.""" From 5d3cf0c499b7aa9819137a868bb917bd709a4953 Mon Sep 17 00:00:00 2001 From: Eman Abdelhaleem <101830347+EmanAbdelhaleem@users.noreply.github.com> Date: Wed, 14 Jan 2026 13:38:43 +0200 Subject: [PATCH 20/46] [BUG] skip failed tests (#1613) #### Metadata Reference Issue: Temporarily fix issue #1586 #### Details `mark.xfail` with ` reason="failures_issue_1544"` and `"strict = False"` for all failed tests as a temporary fix. --- tests/test_evaluations/test_evaluation_functions.py | 1 + tests/test_flows/test_flow.py | 1 + tests/test_flows/test_flow_functions.py | 3 +++ tests/test_runs/test_run_functions.py | 1 + tests/test_study/test_study_functions.py | 1 + tests/test_tasks/test_task_functions.py | 1 + 6 files changed, 8 insertions(+) diff --git a/tests/test_evaluations/test_evaluation_functions.py b/tests/test_evaluations/test_evaluation_functions.py index 7009217d6..ee7c306a1 100644 --- a/tests/test_evaluations/test_evaluation_functions.py +++ b/tests/test_evaluations/test_evaluation_functions.py @@ -258,6 +258,7 @@ def test_list_evaluations_setups_filter_flow(self): assert all(elem in columns for elem in keys) @pytest.mark.production() + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_list_evaluations_setups_filter_task(self): self.use_production_server() task_id = [6] diff --git a/tests/test_flows/test_flow.py b/tests/test_flows/test_flow.py index 99cee6f87..527ad1f8c 100644 --- a/tests/test_flows/test_flow.py +++ b/tests/test_flows/test_flow.py @@ -78,6 +78,7 @@ def test_get_flow(self): assert len(subflow_3.components) == 0 @pytest.mark.production() + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_get_structure(self): # also responsible for testing: flow.get_subflow # We need to use the production server here because 4024 is not the diff --git a/tests/test_flows/test_flow_functions.py b/tests/test_flows/test_flow_functions.py index 46bc36a94..2339b27c8 100644 --- a/tests/test_flows/test_flow_functions.py +++ b/tests/test_flows/test_flow_functions.py @@ -280,6 +280,7 @@ def test_are_flows_equal_ignore_if_older(self): "No known models with list of lists parameters in older versions.", ) @pytest.mark.uses_test_server() + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_sklearn_to_flow_list_of_lists(self): from sklearn.preprocessing import OrdinalEncoder @@ -337,6 +338,7 @@ def test_get_flow_reinstantiate_model_no_extension(self): reason="Requires scikit-learn!=0.19.1, because target flow is from that version.", ) @pytest.mark.production() + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_get_flow_with_reinstantiate_strict_with_wrong_version_raises_exception(self): self.use_production_server() flow = 8175 @@ -527,6 +529,7 @@ def test_delete_flow_success(mock_delete, test_files_directory, test_api_key): @mock.patch.object(requests.Session, "delete") +@pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_delete_unknown_flow(mock_delete, test_files_directory, test_api_key): openml.config.start_using_configuration_for_example() content_file = test_files_directory / "mock_responses" / "flows" / "flow_delete_not_exist.xml" diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index db54151d1..8f2c505b7 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -1567,6 +1567,7 @@ def test_get_runs_list_by_filters(self): assert len(runs) == 2 @pytest.mark.production() + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_get_runs_list_by_tag(self): # We don't have tagged runs on the test server self.use_production_server() diff --git a/tests/test_study/test_study_functions.py b/tests/test_study/test_study_functions.py index 839e74cf3..4b662524b 100644 --- a/tests/test_study/test_study_functions.py +++ b/tests/test_study/test_study_functions.py @@ -13,6 +13,7 @@ class TestStudyFunctions(TestBase): _multiprocess_can_split_ = True @pytest.mark.production() + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_get_study_old(self): self.use_production_server() diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py index 3a2b9ea0a..d44717177 100644 --- a/tests/test_tasks/test_task_functions.py +++ b/tests/test_tasks/test_task_functions.py @@ -57,6 +57,7 @@ def test__get_estimation_procedure_list(self): assert estimation_procedures[0]["task_type_id"] == TaskType.SUPERVISED_CLASSIFICATION @pytest.mark.production() + @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_list_clustering_task(self): self.use_production_server() # as shown by #383, clustering tasks can give list/dict casting problems From 645ef01d8d2627c0900be6c87175ad68c55bc446 Mon Sep 17 00:00:00 2001 From: Eman Abdelhaleem <101830347+EmanAbdelhaleem@users.noreply.github.com> Date: Thu, 15 Jan 2026 00:05:52 +0200 Subject: [PATCH 21/46] [ENH] Improve `NotImplementedError` Messages (#1574) #### Metadata * Reference Issue: fixes #1537 --- openml/datasets/dataset.py | 13 +++++++++++-- openml/runs/functions.py | 22 +++++++++++++++++++--- openml/runs/run.py | 7 ++++++- openml/study/study.py | 16 ++++++++++++++-- openml/tasks/functions.py | 15 +++++++++++++-- openml/tasks/task.py | 9 +++++++-- 6 files changed, 70 insertions(+), 12 deletions(-) diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index 9f6a79aaa..a77fd1040 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -420,7 +420,11 @@ def _get_arff(self, format: str) -> dict: # noqa: A002 file_size = filepath.stat().st_size if file_size > MB_120: raise NotImplementedError( - f"File {filename} too big for {file_size}-bit system ({bits} bytes).", + f"File '{filename}' ({file_size / 1e6:.1f} MB)" + f"exceeds the maximum supported size of 120 MB. " + f"This limitation applies to {bits}-bit systems. " + f"Large dataset handling is currently not fully supported. " + f"Please consider using a smaller dataset" ) if format.lower() == "arff": @@ -780,7 +784,12 @@ def get_data( # noqa: C901 # All the assumptions below for the target are dependant on the number of targets being 1 n_targets = len(target_names) if n_targets > 1: - raise NotImplementedError(f"Number of targets {n_targets} not implemented.") + raise NotImplementedError( + f"Multi-target prediction is not yet supported." + f"Found {n_targets} target columns: {target_names}. " + f"Currently, only single-target datasets are supported. " + f"Please select a single target column." + ) target_name = target_names[0] x = data.drop(columns=[target_name]) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 5a21b8bc1..503788dbd 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -755,7 +755,12 @@ def _run_task_get_arffcontent_parallel_helper( # noqa: PLR0913 test_x = None test_y = None else: - raise NotImplementedError(task.task_type) + raise NotImplementedError( + f"Task type '{task.task_type}' is not supported. " + f"Only OpenMLSupervisedTask and OpenMLClusteringTask are currently implemented. " + f"Task details: task_id={getattr(task, 'task_id', 'unknown')}, " + f"task_class={task.__class__.__name__}" + ) config.logger.info( f"Going to run model {model!s} on " @@ -982,7 +987,13 @@ def obtain_field(xml_obj, fieldname, from_server, cast=None): # type: ignore if "predictions" not in files and from_server is True: task = openml.tasks.get_task(task_id) if task.task_type_id == TaskType.SUBGROUP_DISCOVERY: - raise NotImplementedError("Subgroup discovery tasks are not yet supported.") + raise NotImplementedError( + f"Subgroup discovery tasks are not yet supported. " + f"Task ID: {task_id}. Please check the OpenML documentation" + f"for supported task types. " + f"Currently supported task types: Classification, Regression," + f"Clustering, and Learning Curve." + ) # JvR: actually, I am not sure whether this error should be raised. # a run can consist without predictions. But for now let's keep it @@ -1282,7 +1293,12 @@ def format_prediction( # noqa: PLR0913 if isinstance(task, OpenMLRegressionTask): return [repeat, fold, index, prediction, truth] - raise NotImplementedError(f"Formatting for {type(task)} is not supported.") + raise NotImplementedError( + f"Formatting for {type(task)} is not supported." + f"Supported task types: OpenMLClassificationTask, OpenMLRegressionTask," + f"and OpenMLLearningCurveTask. " + f"Please ensure your task is one of these types." + ) def delete_run(run_id: int) -> bool: diff --git a/openml/runs/run.py b/openml/runs/run.py index b6997fb53..eff011408 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -479,7 +479,12 @@ def _generate_arff_dict(self) -> OrderedDict[str, Any]: ] else: - raise NotImplementedError(f"Task type {task.task_type!s} is not yet supported.") + raise NotImplementedError( + f"Task type '{task.task_type}' is not yet supported. " + f"Supported task types: Classification, Regression, Clustering, Learning Curve. " + f"Task ID: {task.task_id}. " + f"Please check the OpenML documentation for supported task types." + ) return arff_dict diff --git a/openml/study/study.py b/openml/study/study.py index de4aac0f4..7a9c80bbe 100644 --- a/openml/study/study.py +++ b/openml/study/study.py @@ -176,11 +176,23 @@ def _to_dict(self) -> dict[str, dict]: def push_tag(self, tag: str) -> None: """Add a tag to the study.""" - raise NotImplementedError("Tags for studies is not (yet) supported.") + raise NotImplementedError( + "Tag management for studies is not yet supported. " + "The OpenML Python SDK does not currently provide functionality" + "for adding tags to studies." + "For updates on this feature, please refer to the GitHub issues at: " + "https://github.com/openml/openml-python/issues" + ) def remove_tag(self, tag: str) -> None: """Remove a tag from the study.""" - raise NotImplementedError("Tags for studies is not (yet) supported.") + raise NotImplementedError( + "Tag management for studies is not yet supported. " + "The OpenML Python SDK does not currently provide functionality" + "for removing tags from studies. " + "For updates on this feature, please refer to the GitHub issues at: " + "https://github.com/openml/openml-python/issues" + ) class OpenMLStudy(BaseStudy): diff --git a/openml/tasks/functions.py b/openml/tasks/functions.py index c60e0c483..3df2861c0 100644 --- a/openml/tasks/functions.py +++ b/openml/tasks/functions.py @@ -528,7 +528,12 @@ def _create_task_from_xml(xml: str) -> OpenMLTask: TaskType.LEARNING_CURVE: OpenMLLearningCurveTask, }.get(task_type) if cls is None: - raise NotImplementedError(f"Task type {common_kwargs['task_type']} not supported.") + raise NotImplementedError( + f"Task type '{common_kwargs['task_type']}' is not supported. " + f"Supported task types: SUPERVISED_CLASSIFICATION," + f"SUPERVISED_REGRESSION, CLUSTERING, LEARNING_CURVE." + f"Please check the OpenML documentation for available task types." + ) return cls(**common_kwargs) # type: ignore @@ -584,7 +589,13 @@ def create_task( elif task_type == TaskType.SUPERVISED_REGRESSION: task_cls = OpenMLRegressionTask # type: ignore else: - raise NotImplementedError(f"Task type {task_type:d} not supported.") + raise NotImplementedError( + f"Task type ID {task_type:d} is not supported. " + f"Supported task type IDs: {TaskType.SUPERVISED_CLASSIFICATION.value}," + f"{TaskType.SUPERVISED_REGRESSION.value}, " + f"{TaskType.CLUSTERING.value}, {TaskType.LEARNING_CURVE.value}. " + f"Please refer to the TaskType enum for valid task type identifiers." + ) return task_cls( task_type_id=task_type, diff --git a/openml/tasks/task.py b/openml/tasks/task.py index d4998970c..b297a105c 100644 --- a/openml/tasks/task.py +++ b/openml/tasks/task.py @@ -291,7 +291,12 @@ def get_X_and_y(self) -> tuple[pd.DataFrame, pd.Series | pd.DataFrame | None]: TaskType.SUPERVISED_REGRESSION, TaskType.LEARNING_CURVE, ): - raise NotImplementedError(self.task_type) + raise NotImplementedError( + f"Task type '{self.task_type}' is not implemented for get_X_and_y(). " + f"Supported types: SUPERVISED_CLASSIFICATION, SUPERVISED_REGRESSION," + f"LEARNING_CURVE." + f"Task ID: {getattr(self, 'task_id', 'unknown')}. " + ) X, y, _, _ = dataset.get_data(target=self.target_name) return X, y @@ -383,7 +388,7 @@ def __init__( # noqa: PLR0913 self.cost_matrix = cost_matrix if cost_matrix is not None: - raise NotImplementedError("Costmatrix") + raise NotImplementedError("Costmatrix functionality is not yet implemented.") class OpenMLRegressionTask(OpenMLSupervisedTask): From 99928f8b945b107fb3f576c122e697d2ae6610be Mon Sep 17 00:00:00 2001 From: Om Swastik Panda Date: Fri, 16 Jan 2026 00:36:20 +0530 Subject: [PATCH 22/46] [ENH] added version flag to openml cli (#1555) Fixes #1539 --- openml/cli.py | 8 +++++++ tests/test_openml/test_cli.py | 44 +++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 tests/test_openml/test_cli.py diff --git a/openml/cli.py b/openml/cli.py index 4949cc89a..0afb089c2 100644 --- a/openml/cli.py +++ b/openml/cli.py @@ -10,6 +10,7 @@ from urllib.parse import urlparse from openml import config +from openml.__version__ import __version__ def is_hex(string_: str) -> bool: @@ -331,6 +332,13 @@ def main() -> None: subroutines = {"configure": configure} parser = argparse.ArgumentParser() + # Add a global --version flag to display installed version and exit + parser.add_argument( + "--version", + action="version", + version=f"%(prog)s {__version__}", + help="Show the OpenML version and exit", + ) subparsers = parser.add_subparsers(dest="subroutine") parser_configure = subparsers.add_parser( diff --git a/tests/test_openml/test_cli.py b/tests/test_openml/test_cli.py new file mode 100644 index 000000000..eb213b561 --- /dev/null +++ b/tests/test_openml/test_cli.py @@ -0,0 +1,44 @@ +# License: BSD 3-Clause +from __future__ import annotations + +import shutil +import subprocess +import sys + +import openml +import pytest + + +def test_cli_version_prints_package_version(): + # Invoke the CLI via module to avoid relying on console script installation + result = subprocess.run( + [sys.executable, "-m", "openml.cli", "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + + # Ensure successful exit and version present in stdout only + assert result.returncode == 0 + assert result.stderr == "" + assert openml.__version__ in result.stdout + + +def test_console_script_version_prints_package_version(): + # Try to locate the console script; skip if not installed in PATH + console = shutil.which("openml") + if console is None: + pytest.skip("'openml' console script not found in PATH") + + result = subprocess.run( + [console, "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + + assert result.returncode == 0 + assert result.stderr == "" + assert openml.__version__ in result.stdout From cf8e9dbd89284842f33dd31bf0cc3ed03ba0d7a1 Mon Sep 17 00:00:00 2001 From: Satvik Mishra <112589278+satvshr@users.noreply.github.com> Date: Thu, 22 Jan 2026 04:33:29 +0530 Subject: [PATCH 23/46] [BUG] remove accidental skip of `test_get_flow_with_reinstantiate_strict_with_wrong_version_raises_exception` (#1618) #### Metadata * Reference Issue: fixes #1617 * New Tests Added: No * Documentation Updated: No --- tests/test_flows/test_flow_functions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_flows/test_flow_functions.py b/tests/test_flows/test_flow_functions.py index 2339b27c8..875ba8517 100644 --- a/tests/test_flows/test_flow_functions.py +++ b/tests/test_flows/test_flow_functions.py @@ -338,7 +338,6 @@ def test_get_flow_reinstantiate_model_no_extension(self): reason="Requires scikit-learn!=0.19.1, because target flow is from that version.", ) @pytest.mark.production() - @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_get_flow_with_reinstantiate_strict_with_wrong_version_raises_exception(self): self.use_production_server() flow = 8175 From d421b9ec58bc49e4114dba77768edd4bf641391c Mon Sep 17 00:00:00 2001 From: Satvik Mishra <112589278+satvshr@users.noreply.github.com> Date: Mon, 26 Jan 2026 17:00:51 +0530 Subject: [PATCH 24/46] [BUG] Test Failures caused because of pandas 3 (#1628) #### Metadata * Reference Issue: fixes #1627 * What does this PR implement/fix? Explain your changes. This PR fixes the 7 recurring bugs across all current PRs because of pandas 3: 1. `test_get_data_pandas` bug: Solved type error for dataframe columns having `str` datatype for `pandas==3` and `object` for older versions. 2. `test_get_sparse_dataset_dataframe`, `test_get_sparse_dataset_rowid_and_ignore_and_target`, and `test_get_sparse_dataset_dataframe_with_target` bug: typecasting `type_` to a np array. 3. bugs in `test_flow_functions.py`: `ext_version` can now be `nan` because of `pandas 3`. --- .github/workflows/test.yml | 18 ++++++++++++++++-- openml/datasets/dataset.py | 2 +- tests/test_datasets/test_dataset.py | 15 +++++++++------ tests/test_flows/test_flow_functions.py | 3 ++- 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d65cc3796..b10721f55 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,7 +22,7 @@ concurrency: jobs: test: - name: (${{ matrix.os }},Py${{ matrix.python-version }},sk${{ matrix.scikit-learn }},sk-only:${{ matrix.sklearn-only }}) + name: (${{ matrix.os }},Py${{ matrix.python-version }},sk${{ matrix.scikit-learn }}${{ matrix.pandas-version != '' && format(',pd:{0}', matrix.pandas-version) || '' }},sk-only:${{ matrix.sklearn-only }}) runs-on: ${{ matrix.os }} strategy: @@ -64,6 +64,14 @@ jobs: sklearn-only: "false" code-cov: true + # Pandas 2 run + - os: ubuntu-latest + python-version: "3.12" + scikit-learn: "1.5.*" + sklearn-only: "false" + pandas-version: "2.*" + code-cov: false + steps: - uses: actions/checkout@v6 with: @@ -74,10 +82,16 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install test dependencies and scikit-learn + - name: Install test dependencies, scikit-learn, and optional pandas + shell: bash run: | python -m pip install --upgrade pip pip install -e .[test] scikit-learn==${{ matrix.scikit-learn }} + + if [ "${{ matrix.pandas-version }}" != "" ]; then + echo "Installing specific pandas version: ${{ matrix.pandas-version }}" + pip install "pandas==${{ matrix.pandas-version }}" + fi - name: Store repository status id: status-before diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index a77fd1040..d9eee278d 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -488,7 +488,7 @@ def _parse_data_from_arff( # noqa: C901, PLR0912, PLR0915 try: # checks if the strings which should be the class labels # can be encoded into integers - pd.factorize(type_)[0] + pd.factorize(np.array(type_))[0] except ValueError as e: raise ValueError( "Categorical data needs to be numeric when using sparse ARFF." diff --git a/tests/test_datasets/test_dataset.py b/tests/test_datasets/test_dataset.py index 6dc4c7d5d..b13bac30b 100644 --- a/tests/test_datasets/test_dataset.py +++ b/tests/test_datasets/test_dataset.py @@ -102,21 +102,24 @@ def test_get_data_pandas(self): assert isinstance(data, pd.DataFrame) assert data.shape[1] == len(self.titanic.features) assert data.shape[0] == 1309 + # Dynamically detect what this version of Pandas calls string columns. + str_dtype = data["name"].dtype.name + col_dtype = { "pclass": "uint8", "survived": "category", - "name": "object", + "name": str_dtype, "sex": "category", "age": "float64", "sibsp": "uint8", "parch": "uint8", - "ticket": "object", + "ticket": str_dtype, "fare": "float64", - "cabin": "object", + "cabin": str_dtype, "embarked": "category", - "boat": "object", + "boat": str_dtype, "body": "float64", - "home.dest": "object", + "home.dest": str_dtype, } for col_name in data.columns: assert data[col_name].dtype.name == col_dtype[col_name] @@ -357,7 +360,7 @@ def setUp(self): def test_get_sparse_dataset_dataframe_with_target(self): X, y, _, attribute_names = self.sparse_dataset.get_data(target="class") assert isinstance(X, pd.DataFrame) - assert isinstance(X.dtypes[0], pd.SparseDtype) + assert isinstance(X.dtypes.iloc[0], pd.SparseDtype) assert X.shape == (600, 20000) assert isinstance(y, pd.Series) diff --git a/tests/test_flows/test_flow_functions.py b/tests/test_flows/test_flow_functions.py index 875ba8517..5aa99cd62 100644 --- a/tests/test_flows/test_flow_functions.py +++ b/tests/test_flows/test_flow_functions.py @@ -41,8 +41,9 @@ def _check_flow(self, flow): assert isinstance(flow["full_name"], str) assert isinstance(flow["version"], str) # There are some runs on openml.org that can have an empty external version + ext_version = flow["external_version"] ext_version_str_or_none = ( - isinstance(flow["external_version"], str) or flow["external_version"] is None + isinstance(ext_version, str) or ext_version is None or pd.isna(ext_version) ) assert ext_version_str_or_none From 0769ff590835671467587e98aa7917b81f4f2e35 Mon Sep 17 00:00:00 2001 From: Shrivaths S Nair <142079253+JATAYU000@users.noreply.github.com> Date: Sun, 15 Feb 2026 17:58:28 +0530 Subject: [PATCH 25/46] [ENH] Added `ReprMixin` to share `__repr__` formatting (#1595) * Reference Issue: Fixes #1591 * New Tests Added: No * Documentation Updated: Yes (docstring) * Change Log Entry: Adds `ReprMixin` in `utils` --- openml/utils.py | 66 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/openml/utils.py b/openml/utils.py index 3680bc0ff..bbc71d753 100644 --- a/openml/utils.py +++ b/openml/utils.py @@ -2,12 +2,20 @@ from __future__ import annotations import contextlib +import re import shutil import warnings -from collections.abc import Callable, Mapping, Sized +from abc import ABC, abstractmethod +from collections.abc import Callable, Iterable, Mapping, Sequence, Sized from functools import wraps from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal, TypeVar, overload +from typing import ( + TYPE_CHECKING, + Any, + Literal, + TypeVar, + overload, +) from typing_extensions import ParamSpec import numpy as np @@ -470,3 +478,57 @@ def update(self, length: int) -> None: self._progress_bar.update(length) if self._progress_bar.total <= self._progress_bar.n: self._progress_bar.close() + + +class ReprMixin(ABC): + """A mixin class that provides a customizable string representation for OpenML objects. + + This mixin standardizes the __repr__ output format across OpenML classes. + Classes inheriting from this mixin should implement the + _get_repr_body_fields method to specify which fields to display. + """ + + def __repr__(self) -> str: + body_fields = self._get_repr_body_fields() + return self._apply_repr_template(body_fields) + + @abstractmethod + def _get_repr_body_fields(self) -> Sequence[tuple[str, str | int | list[str] | None]]: + """Collect all information to display in the __repr__ body. + + Returns + ------- + body_fields : List[Tuple[str, Union[str, int, List[str]]]] + A list of (name, value) pairs to display in the body of the __repr__. + E.g.: [('metric', 'accuracy'), ('dataset', 'iris')] + If value is a List of str, then each item of the list will appear in a separate row. + """ + # Should be implemented in the base class. + + def _apply_repr_template( + self, + body_fields: Iterable[tuple[str, str | int | list[str] | None]], + ) -> str: + """Generates the header and formats the body for string representation of the object. + + Parameters + ---------- + body_fields: List[Tuple[str, str]] + A list of (name, value) pairs to display in the body of the __repr__. + """ + # We add spaces between capitals, e.g. ClassificationTask -> Classification Task + name_with_spaces = re.sub( + r"(\w)([A-Z])", + r"\1 \2", + self.__class__.__name__[len("OpenML") :], + ) + header_text = f"OpenML {name_with_spaces}" + header = f"{header_text}\n{'=' * len(header_text)}\n" + + _body_fields: list[tuple[str, str | int | list[str]]] = [ + (k, "None" if v is None else v) for k, v in body_fields + ] + longest_field_name_length = max(len(name) for name, _ in _body_fields) + field_line_format = f"{{:.<{longest_field_name_length}}}: {{}}" + body = "\n".join(field_line_format.format(name, value) for name, value in _body_fields) + return header + body From 06ac6d00cd7ef839d9afcc375560b935fbdb0336 Mon Sep 17 00:00:00 2001 From: Satvik Mishra <112589278+satvshr@users.noreply.github.com> Date: Sun, 15 Feb 2026 21:04:55 +0530 Subject: [PATCH 26/46] [ENH] Extend `Extension` class test suite (#1560) #### Metadata * Reference Issue: fixes #1545 * New Tests Added: Yes * Documentation Updated: No * Change Log Entry: Add tests for extension interface contract and extension registry edge cases #### Details * What does this PR implement/fix? Explain your changes. This PR adds unit tests for the OpenML Extension interface and for extension registry behavior. The tests added are the 7 tests mentioned in #1545 * Why is this change necessary? What is the problem it solves? Previously, only the non-abstract registry helpers (`get_extension_by_model`, `get_extension_by_flow`) were covered. The abstract `Extension` interface itself was not tested. --- tests/test_extensions/test_functions.py | 239 +++++++++++++++++++----- 1 file changed, 192 insertions(+), 47 deletions(-) diff --git a/tests/test_extensions/test_functions.py b/tests/test_extensions/test_functions.py index ac4610a15..90fbaa9f1 100644 --- a/tests/test_extensions/test_functions.py +++ b/tests/test_extensions/test_functions.py @@ -1,12 +1,14 @@ # License: BSD 3-Clause from __future__ import annotations -import inspect +from collections import OrderedDict +import inspect +import numpy as np import pytest - +from unittest.mock import patch import openml.testing -from openml.extensions import get_extension_by_flow, get_extension_by_model, register_extension +from openml.extensions import Extension, get_extension_by_flow, get_extension_by_model, register_extension class DummyFlow: @@ -40,54 +42,197 @@ def can_handle_model(model): return False -def _unregister(): - # "Un-register" the test extensions - while True: - rem_dum_ext1 = False - rem_dum_ext2 = False - try: - openml.extensions.extensions.remove(DummyExtension1) - rem_dum_ext1 = True - except ValueError: - pass - try: - openml.extensions.extensions.remove(DummyExtension2) - rem_dum_ext2 = True - except ValueError: - pass - if not rem_dum_ext1 and not rem_dum_ext2: - break +class DummyExtension(Extension): + @classmethod + def can_handle_flow(cls, flow): + return isinstance(flow, DummyFlow) + + @classmethod + def can_handle_model(cls, model): + return isinstance(model, DummyModel) + + def flow_to_model( + self, + flow, + initialize_with_defaults=False, + strict_version=True, + ): + if not isinstance(flow, DummyFlow): + raise ValueError("Invalid flow") + + model = DummyModel() + model.defaults = initialize_with_defaults + model.strict_version = strict_version + return model + + def model_to_flow(self, model): + if not isinstance(model, DummyModel): + raise ValueError("Invalid model") + return DummyFlow() + + def get_version_information(self): + return ["dummy==1.0"] + + def create_setup_string(self, model): + return "DummyModel()" + + def is_estimator(self, model): + return isinstance(model, DummyModel) + + def seed_model(self, model, seed): + model.seed = seed + return model + + def _run_model_on_fold( + self, + model, + task, + X_train, + rep_no, + fold_no, + y_train=None, + X_test=None, + ): + preds = np.zeros(len(X_train)) + probs = None + measures = OrderedDict() + trace = None + return preds, probs, measures, trace + + def obtain_parameter_values(self, flow, model=None): + return [] + + def check_if_model_fitted(self, model): + return False + + def instantiate_model_from_hpo_class(self, model, trace_iteration): + return DummyModel() + class TestInit(openml.testing.TestBase): - def setUp(self): - super().setUp() - _unregister() def test_get_extension_by_flow(self): - assert get_extension_by_flow(DummyFlow()) is None - with pytest.raises(ValueError, match="No extension registered which can handle flow:"): - get_extension_by_flow(DummyFlow(), raise_if_no_extension=True) - register_extension(DummyExtension1) - assert isinstance(get_extension_by_flow(DummyFlow()), DummyExtension1) - register_extension(DummyExtension2) - assert isinstance(get_extension_by_flow(DummyFlow()), DummyExtension1) - register_extension(DummyExtension1) - with pytest.raises( - ValueError, match="Multiple extensions registered which can handle flow:" - ): - get_extension_by_flow(DummyFlow()) + # We replace the global list with a new empty list [] ONLY for this block + with patch("openml.extensions.extensions", []): + assert get_extension_by_flow(DummyFlow()) is None + + with pytest.raises(ValueError, match="No extension registered which can handle flow:"): + get_extension_by_flow(DummyFlow(), raise_if_no_extension=True) + + register_extension(DummyExtension1) + assert isinstance(get_extension_by_flow(DummyFlow()), DummyExtension1) + + register_extension(DummyExtension2) + assert isinstance(get_extension_by_flow(DummyFlow()), DummyExtension1) + + register_extension(DummyExtension1) + with pytest.raises( + ValueError, match="Multiple extensions registered which can handle flow:" + ): + get_extension_by_flow(DummyFlow()) def test_get_extension_by_model(self): - assert get_extension_by_model(DummyModel()) is None - with pytest.raises(ValueError, match="No extension registered which can handle model:"): - get_extension_by_model(DummyModel(), raise_if_no_extension=True) - register_extension(DummyExtension1) - assert isinstance(get_extension_by_model(DummyModel()), DummyExtension1) - register_extension(DummyExtension2) - assert isinstance(get_extension_by_model(DummyModel()), DummyExtension1) - register_extension(DummyExtension1) - with pytest.raises( - ValueError, match="Multiple extensions registered which can handle model:" - ): - get_extension_by_model(DummyModel()) + # Again, we start with a fresh empty list automatically + with patch("openml.extensions.extensions", []): + assert get_extension_by_model(DummyModel()) is None + + with pytest.raises(ValueError, match="No extension registered which can handle model:"): + get_extension_by_model(DummyModel(), raise_if_no_extension=True) + + register_extension(DummyExtension1) + assert isinstance(get_extension_by_model(DummyModel()), DummyExtension1) + + register_extension(DummyExtension2) + assert isinstance(get_extension_by_model(DummyModel()), DummyExtension1) + + register_extension(DummyExtension1) + with pytest.raises( + ValueError, match="Multiple extensions registered which can handle model:" + ): + get_extension_by_model(DummyModel()) + + +def test_flow_to_model_with_defaults(): + """Test flow_to_model with initialize_with_defaults=True.""" + ext = DummyExtension() + flow = DummyFlow() + + model = ext.flow_to_model(flow, initialize_with_defaults=True) + + assert isinstance(model, DummyModel) + assert model.defaults is True + +def test_flow_to_model_strict_version(): + """Test flow_to_model with strict_version parameter.""" + ext = DummyExtension() + flow = DummyFlow() + + model_strict = ext.flow_to_model(flow, strict_version=True) + model_non_strict = ext.flow_to_model(flow, strict_version=False) + + assert isinstance(model_strict, DummyModel) + assert model_strict.strict_version is True + + assert isinstance(model_non_strict, DummyModel) + assert model_non_strict.strict_version is False + +def test_model_to_flow_conversion(): + """Test converting a model back to flow representation.""" + ext = DummyExtension() + model = DummyModel() + + flow = ext.model_to_flow(model) + + assert isinstance(flow, DummyFlow) + + +def test_invalid_flow_raises_error(): + """Test that invalid flow raises appropriate error.""" + class InvalidFlow: + pass + + ext = DummyExtension() + flow = InvalidFlow() + + with pytest.raises(ValueError, match="Invalid flow"): + ext.flow_to_model(flow) + + +@patch("openml.extensions.extensions", []) +def test_extension_not_found_error_message(): + """Test error message contains helpful information.""" + class UnknownModel: + pass + + with pytest.raises(ValueError, match="No extension registered"): + get_extension_by_model(UnknownModel(), raise_if_no_extension=True) + + +def test_register_same_extension_twice(): + """Test behavior when registering same extension twice.""" + # Using a context manager here to isolate the list + with patch("openml.extensions.extensions", []): + register_extension(DummyExtension) + register_extension(DummyExtension) + + matches = [ + ext for ext in openml.extensions.extensions + if ext is DummyExtension + ] + assert len(matches) == 2 + + +@patch("openml.extensions.extensions", []) +def test_extension_priority_order(): + """Test that extensions are checked in registration order.""" + class DummyExtensionA(DummyExtension): + pass + class DummyExtensionB(DummyExtension): + pass + + register_extension(DummyExtensionA) + register_extension(DummyExtensionB) + + assert openml.extensions.extensions[0] is DummyExtensionA + assert openml.extensions.extensions[1] is DummyExtensionB \ No newline at end of file From aa04b30cc4732199f9242269dce75cbb93e2062a Mon Sep 17 00:00:00 2001 From: Rohan Sen Date: Sun, 15 Feb 2026 21:14:59 +0530 Subject: [PATCH 27/46] [ENH] replaced hardcoded test server admin key with env variable and secrets (#1568) #### Metadata * Reference Issue: #1529 * New Tests Added: Yes * Documentation Updated: No * Change Log Entry: #### Details this PR is made to remove the hardcoded test server admin key from the codebase and replace it with environment variable-based authentication. ## Summary - in `openml/config.py` Added a new environment variable constant for the test server admin key: ```python OPENML_TEST_SERVER_ADMIN_KEY_ENV_VAR = "OPENML_TEST_SERVER_ADMIN_KEY" ``` - Testing Base Class Updated in `openml/testing.py`. Modified the `TestBase` class to read the admin key from an environment variable instead of using a hardcoded value: **Before**: ```python admin_key = "abc" ``` **After**: ```python admin_key = os.environ.get(openml.config.OPENML_TEST_SERVER_ADMIN_KEY_ENV_VAR) ``` **Note**: - The admin key is now `None` by default when the environment variable is not set - Tests requiring the admin key will fail gracefully if the key is not available Also in the tests, Added `pytest.skipif` decorators to tests that require admin privileges in the following test files: #### `tests/test_openml/test_config.py` **Test**: `test_switch_to_example_configuration` **Added decorator**: ```python @pytest.mark.skipif( not os.environ.get(openml.config.OPENML_TEST_SERVER_ADMIN_KEY_ENV_VAR), reason="Test requires admin key. Set OPENML_TEST_SERVER_ADMIN_KEY environment variable.", ) ``` #### `tests/test_datasets/test_dataset_functions.py` **Test**: `test_data_status` **Added decorator**: ```python @pytest.mark.skipif( not os.environ.get(openml.config.OPENML_TEST_SERVER_ADMIN_KEY_ENV_VAR), reason="Test requires admin key. Set OPENML_TEST_SERVER_ADMIN_KEY environment variable.", ) ``` **Note**: - These tests will be automatically skipped if the admin key is not provided - Clear skip reason is displayed when tests are skipped - No failures or errors when running tests locally without the admin key ### 4. CI Configuration Update (`.github/workflows/test.yml`) Added the environment variable to all test execution steps in the GitHub Actions workflow: **Steps updated**: - Run tests on Ubuntu Test - Run tests on Ubuntu Production - Run tests on Windows **Added to each step**: ```yaml env: OPENML_TEST_SERVER_ADMIN_KEY: ${{ secrets.OPENML_TEST_SERVER_ADMIN_KEY }} ``` **Impact**: - CI will use the secret stored in GitHub repository settings - Tests requiring admin privileges will run in CI - The actual key value is never exposed in logs or code @PGijsbers this requires someone to put the admin key in the github secrets which would be a critical step. # Update on reviews the configurations should be done from openml config files in `./.openml/config` for directory level configurations, instead of the added responsibility of a new `.env` file and dependencies. in case of local testing the concerned tests would be skipped in case no key is provided. --- .github/workflows/test.yml | 6 ++++++ CONTRIBUTING.md | 11 +++++++++++ openml/config.py | 1 + openml/testing.py | 2 +- tests/test_datasets/test_dataset_functions.py | 4 ++++ tests/test_openml/test_config.py | 2 +- 6 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b10721f55..29ada2298 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -106,6 +106,8 @@ jobs: - name: Run tests on Ubuntu Test if: matrix.os == 'ubuntu-latest' + env: + OPENML_TEST_SERVER_ADMIN_KEY: ${{ secrets.OPENML_TEST_SERVER_ADMIN_KEY }} run: | if [ "${{ matrix.code-cov }}" = "true" ]; then codecov="--cov=openml --long --cov-report=xml" @@ -121,6 +123,8 @@ jobs: - name: Run tests on Ubuntu Production if: matrix.os == 'ubuntu-latest' + env: + OPENML_TEST_SERVER_ADMIN_KEY: ${{ secrets.OPENML_TEST_SERVER_ADMIN_KEY }} run: | if [ "${{ matrix.code-cov }}" = "true" ]; then codecov="--cov=openml --long --cov-report=xml" @@ -136,6 +140,8 @@ jobs: - name: Run tests on Windows if: matrix.os == 'windows-latest' + env: + OPENML_TEST_SERVER_ADMIN_KEY: ${{ secrets.OPENML_TEST_SERVER_ADMIN_KEY }} run: | # we need a separate step because of the bash-specific if-statement in the previous one. pytest -n 4 --durations=20 --dist load -sv --reruns 5 --reruns-delay 1 -m "not uses_test_server" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 35ab30b4a..3a18b63f2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,6 +96,17 @@ To test your new contribution, add [unit tests](https://github.com/openml/openml * Please ensure that the example is run on the test server by beginning with the call to `openml.config.start_using_configuration_for_example()`, which is done by default for tests derived from `TestBase`. * Add the `@pytest.mark.sklearn` marker to your unit tests if they have a dependency on scikit-learn. +#### Running Tests That Require Admin Privileges + +Some tests require admin privileges on the test server and will be automatically skipped unless you provide an admin API key. For regular contributors, the tests will skip gracefully. For core contributors who need to run these tests locally, you can set up the key by exporting the variable as below before running the tests: + +```bash +# For windows +$env:OPENML_TEST_SERVER_ADMIN_KEY = "admin-key" +# For linux/mac +export OPENML_TEST_SERVER_ADMIN_KEY="admin-key" +``` + ### Pull Request Checklist You can go to the `openml-python` GitHub repository to create the pull request by [comparing the branch](https://github.com/openml/openml-python/compare) from your fork with the `develop` branch of the `openml-python` repository. When creating a pull request, make sure to follow the comments and structured provided by the template on GitHub. diff --git a/openml/config.py b/openml/config.py index e6104fd7f..9758b6fff 100644 --- a/openml/config.py +++ b/openml/config.py @@ -25,6 +25,7 @@ OPENML_CACHE_DIR_ENV_VAR = "OPENML_CACHE_DIR" OPENML_SKIP_PARQUET_ENV_VAR = "OPENML_SKIP_PARQUET" +OPENML_TEST_SERVER_ADMIN_KEY_ENV_VAR = "OPENML_TEST_SERVER_ADMIN_KEY" _TEST_SERVER_NORMAL_USER_KEY = "normaluser" diff --git a/openml/testing.py b/openml/testing.py index 8d3bbbd5b..304a4e0be 100644 --- a/openml/testing.py +++ b/openml/testing.py @@ -48,7 +48,7 @@ class TestBase(unittest.TestCase): } flow_name_tracker: ClassVar[list[str]] = [] test_server = "https://test.openml.org/api/v1/xml" - admin_key = "abc" + admin_key = os.environ.get(openml.config.OPENML_TEST_SERVER_ADMIN_KEY_ENV_VAR) user_key = openml.config._TEST_SERVER_NORMAL_USER_KEY # creating logger for tracking files uploaded to test server diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index c41664ba7..d80743a8c 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -599,6 +599,10 @@ def _assert_status_of_dataset(self, *, did: int, status: str): assert len(result) == 1 assert result[did]["status"] == status + @pytest.mark.skipif( + not os.environ.get(openml.config.OPENML_TEST_SERVER_ADMIN_KEY_ENV_VAR), + reason="Test requires admin key. Set OPENML_TEST_SERVER_ADMIN_KEY environment variable.", + ) @pytest.mark.flaky() @pytest.mark.uses_test_server() def test_data_status(self): diff --git a/tests/test_openml/test_config.py b/tests/test_openml/test_config.py index 7ef223504..c5ddc4ecc 100644 --- a/tests/test_openml/test_config.py +++ b/tests/test_openml/test_config.py @@ -110,7 +110,7 @@ class TestConfigurationForExamples(openml.testing.TestBase): def test_switch_to_example_configuration(self): """Verifies the test configuration is loaded properly.""" # Below is the default test key which would be used anyway, but just for clarity: - openml.config.apikey = TestBase.admin_key + openml.config.apikey = "any-api-key" openml.config.server = self.production_server openml.config.start_using_configuration_for_example() From 5b85b778af0b6ab3a15b3f2326e8b5726c2ce8c8 Mon Sep 17 00:00:00 2001 From: Eman Abdelhaleem <101830347+EmanAbdelhaleem@users.noreply.github.com> Date: Sun, 15 Feb 2026 18:26:51 +0200 Subject: [PATCH 28/46] [DOC] Enhance Docstrings of Flows Core Public Functions (#1569) #### Metadata * Reference Issue: #1538 #### Details enhance the docstrings of flows core public functions, add examples, parameter default, parameter type..etc --- openml/flows/functions.py | 230 ++++++++++++++++++++++++++++---------- 1 file changed, 172 insertions(+), 58 deletions(-) diff --git a/openml/flows/functions.py b/openml/flows/functions.py index 6c2393f10..0a2058890 100644 --- a/openml/flows/functions.py +++ b/openml/flows/functions.py @@ -71,23 +71,59 @@ def _get_cached_flow(fid: int) -> OpenMLFlow: @openml.utils.thread_safe_if_oslo_installed def get_flow(flow_id: int, reinstantiate: bool = False, strict_version: bool = True) -> OpenMLFlow: # noqa: FBT002 - """Download the OpenML flow for a given flow ID. + """Fetch an OpenMLFlow by its server-assigned ID. + + Queries the OpenML REST API for the flow metadata and returns an + :class:`OpenMLFlow` instance. If the flow is already cached locally, + the cached copy is returned. Optionally the flow can be re-instantiated + into a concrete model instance using the registered extension. Parameters ---------- flow_id : int The OpenML flow id. - - reinstantiate: bool - Whether to reinstantiate the flow to a model instance. - - strict_version : bool, default=True - Whether to fail if version requirements are not fulfilled. + reinstantiate : bool, optional (default=False) + If True, convert the flow description into a concrete model instance + using the flow's extension (e.g., sklearn). If conversion fails and + ``strict_version`` is True, an exception will be raised. + strict_version : bool, optional (default=True) + When ``reinstantiate`` is True, whether to enforce exact version + requirements for the extension/model. If False, a new flow may + be returned when versions differ. Returns ------- - flow : OpenMLFlow - the flow + OpenMLFlow + The flow object with metadata; ``model`` may be populated when + ``reinstantiate=True``. + + Raises + ------ + OpenMLCacheException + When cached flow files are corrupted or cannot be read. + OpenMLServerException + When the REST API call fails. + + Side Effects + ------------ + - Writes to ``openml.config.cache_directory/flows/{flow_id}/flow.xml`` + when the flow is downloaded from the server. + + Preconditions + ------------- + - Network access to the OpenML server is required unless the flow is cached. + - For private flows, ``openml.config.apikey`` must be set. + + Notes + ----- + Results are cached to speed up subsequent calls. When ``reinstantiate`` is + True and version mismatches occur, a new flow may be returned to reflect + the converted model (only when ``strict_version`` is False). + + Examples + -------- + >>> import openml + >>> flow = openml.flows.get_flow(5) # doctest: +SKIP """ flow_id = int(flow_id) flow = _get_flow_description(flow_id) @@ -138,32 +174,47 @@ def list_flows( tag: str | None = None, uploader: str | None = None, ) -> pd.DataFrame: - """ - Return a list of all flows which are on OpenML. - (Supports large amount of results) + """List flows available on the OpenML server. + + This function supports paging and filtering and returns a pandas + DataFrame with one row per flow and columns for id, name, version, + external_version, full_name and uploader. Parameters ---------- offset : int, optional - the number of flows to skip, starting from the first + Number of flows to skip, starting from the first (for paging). size : int, optional - the maximum number of flows to return + Maximum number of flows to return. tag : str, optional - the tag to include - kwargs: dict, optional - Legal filter operators: uploader. + Only return flows having this tag. + uploader : str, optional + Only return flows uploaded by this user. Returns ------- - flows : dataframe - Each row maps to a dataset - Each column contains the following information: - - flow id - - full name - - name - - version - - external version - - uploader + pandas.DataFrame + Rows correspond to flows. Columns include ``id``, ``full_name``, + ``name``, ``version``, ``external_version``, and ``uploader``. + + Raises + ------ + OpenMLServerException + When the API call fails. + + Side Effects + ------------ + - None: results are fetched and returned; Read-only operation. + + Preconditions + ------------- + - Network access is required to list flows unless cached mechanisms are + used by the underlying API helper. + + Examples + -------- + >>> import openml + >>> flows = openml.flows.list_flows(size=100) # doctest: +SKIP """ listing_call = partial(_list_flows, tag=tag, uploader=uploader) batches = openml.utils._list_all(listing_call, offset=offset, limit=size) @@ -206,25 +257,35 @@ def _list_flows(limit: int, offset: int, **kwargs: Any) -> pd.DataFrame: def flow_exists(name: str, external_version: str) -> int | bool: - """Retrieves the flow id. + """Check whether a flow (name + external_version) exists on the server. - A flow is uniquely identified by name + external_version. + The OpenML server defines uniqueness of flows by the pair + ``(name, external_version)``. This helper queries the server and + returns the corresponding flow id when present. Parameters ---------- - name : string - Name of the flow - external_version : string + name : str + Flow name (e.g., ``sklearn.tree._classes.DecisionTreeClassifier(1)``). + external_version : str Version information associated with flow. Returns ------- - flow_exist : int or bool - flow id iff exists, False otherwise - - Notes - ----- - see https://www.openml.org/api_docs/#!/flow/get_flow_exists_name_version + int or bool + The flow id if the flow exists on the server, otherwise ``False``. + + Raises + ------ + ValueError + If ``name`` or ``external_version`` are empty or not strings. + OpenMLServerException + When the API request fails. + + Examples + -------- + >>> import openml + >>> openml.flows.flow_exists("weka.JRip", "Weka_3.9.0_10153") # doctest: +SKIP """ if not (isinstance(name, str) and len(name) > 0): raise ValueError("Argument 'name' should be a non-empty string") @@ -247,35 +308,58 @@ def get_flow_id( name: str | None = None, exact_version: bool = True, # noqa: FBT002 ) -> int | bool | list[int]: - """Retrieves the flow id for a model or a flow name. + """Retrieve flow id(s) for a model instance or a flow name. - Provide either a model or a name to this function. Depending on the input, it does + Provide either a concrete ``model`` (which will be converted to a flow by + the appropriate extension) or a flow ``name``. Behavior depends on + ``exact_version``: - * ``model`` and ``exact_version == True``: This helper function first queries for the necessary - extension. Second, it uses that extension to convert the model into a flow. Third, it - executes ``flow_exists`` to potentially obtain the flow id the flow is published to the - server. - * ``model`` and ``exact_version == False``: This helper function first queries for the - necessary extension. Second, it uses that extension to convert the model into a flow. Third - it calls ``list_flows`` and filters the returned values based on the flow name. - * ``name``: Ignores ``exact_version`` and calls ``list_flows``, then filters the returned - values based on the flow name. + - ``model`` + ``exact_version=True``: convert ``model`` to a flow and call + :func:`flow_exists` to get a single flow id (or False). + - ``model`` + ``exact_version=False``: convert ``model`` to a flow and + return all server flow ids with the same flow name. + - ``name``: ignore ``exact_version`` and return all server flow ids that + match ``name``. Parameters ---------- - model : object - Any model. Must provide either ``model`` or ``name``. - name : str - Name of the flow. Must provide either ``model`` or ``name``. - exact_version : bool - Whether to return the flow id of the exact version or all flow ids where the name - of the flow matches. This is only taken into account for a model where a version number - is available (requires ``model`` to be set). + model : object, optional + A model instance that can be handled by a registered extension. Either + ``model`` or ``name`` must be provided. + name : str, optional + Flow name to query for. Either ``model`` or ``name`` must be provided. + exact_version : bool, optional (default=True) + When True and ``model`` is provided, only return the id for the exact + external version. When False, return a list of matching ids. Returns ------- - int or bool, List - flow id iff exists, ``False`` otherwise, List if ``exact_version is False`` + int or bool or list[int] + If ``exact_version`` is True: the flow id if found, otherwise ``False``. + If ``exact_version`` is False: a list of matching flow ids (may be empty). + + Raises + ------ + ValueError + If neither ``model`` nor ``name`` is provided, or if both are provided. + OpenMLServerException + If underlying API calls fail. + + Side Effects + ------------ + - May call server APIs (``flow/exists``, ``flow/list``) and therefore + depends on network access and API keys for private flows. + + Examples + -------- + >>> import openml + >>> # Lookup by flow name + >>> openml.flows.get_flow_id(name="weka.JRip") # doctest: +SKIP + >>> # Lookup by model instance (requires a registered extension) + >>> import sklearn + >>> import openml_sklearn + >>> clf = sklearn.tree.DecisionTreeClassifier() + >>> openml.flows.get_flow_id(model=clf) # doctest: +SKIP """ if model is not None and name is not None: raise ValueError("Must provide either argument `model` or argument `name`, but not both.") @@ -391,6 +475,21 @@ def assert_flows_equal( # noqa: C901, PLR0912, PLR0913, PLR0915 check_description : bool Whether to ignore matching of flow descriptions. + + Raises + ------ + TypeError + When either argument is not an :class:`OpenMLFlow`. + ValueError + When a relevant mismatch is found between the two flows. + + Examples + -------- + >>> import openml + >>> f1 = openml.flows.get_flow(5) # doctest: +SKIP + >>> f2 = openml.flows.get_flow(5) # doctest: +SKIP + >>> openml.flows.assert_flows_equal(f1, f2) # doctest: +SKIP + >>> # If flows differ, a ValueError is raised """ if not isinstance(flow1, OpenMLFlow): raise TypeError(f"Argument 1 must be of type OpenMLFlow, but is {type(flow1)}") @@ -550,5 +649,20 @@ def delete_flow(flow_id: int) -> bool: ------- bool True if the deletion was successful. False otherwise. + + Raises + ------ + OpenMLServerException + If the server-side deletion fails due to permissions or other errors. + + Side Effects + ------------ + - Removes the flow from the OpenML server (if permitted). + + Examples + -------- + >>> import openml + >>> # Deletes flow 23 if you are the uploader and it's not linked to runs + >>> openml.flows.delete_flow(23) # doctest: +SKIP """ return openml.utils._delete_entity("flow", flow_id) From aba25866becc7cd231ba2dffd0535d8566c49178 Mon Sep 17 00:00:00 2001 From: Eman Abdelhaleem <101830347+EmanAbdelhaleem@users.noreply.github.com> Date: Sun, 15 Feb 2026 18:30:24 +0200 Subject: [PATCH 29/46] [ENH] improved simple assertion error message in `evalutation/functions.py` (#1600) #### Details fixed a simple assertion error message in `evalutation/functions.py` --- openml/evaluations/functions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openml/evaluations/functions.py b/openml/evaluations/functions.py index 0b9f190b4..61c95a480 100644 --- a/openml/evaluations/functions.py +++ b/openml/evaluations/functions.py @@ -231,8 +231,9 @@ def __list_evaluations(api_call: str) -> list[OpenMLEvaluation]: f'Error in return XML, does not contain "oml:evaluations": {evals_dict!s}', ) - assert isinstance(evals_dict["oml:evaluations"]["oml:evaluation"], list), type( - evals_dict["oml:evaluations"], + assert isinstance(evals_dict["oml:evaluations"]["oml:evaluation"], list), ( + "Expected 'oml:evaluation' to be a list, but got" + f"{type(evals_dict['oml:evaluations']['oml:evaluation']).__name__}. " ) uploader_ids = list( From f7014e74fb4e6f3c418172fede0a870af2919eba Mon Sep 17 00:00:00 2001 From: Rohan Sen Date: Mon, 16 Feb 2026 01:46:20 +0530 Subject: [PATCH 30/46] [ENH] dataclass refactor of openmlparameter and openmlsetup classes (#1582) #### Metadata * Reference Issue: fixes #1541 * New Tests Added: No * Documentation Updated: No #### Details Edited the OpenMLParameter in `openml/setups/setup.py` to use `@dataclass` decorator. This significantly reduces the boilerplate code in the following places: - OpenMLSetup **Before:** ```python class OpenMLSetup: """Setup object (a.k.a. Configuration)....""" def __init__(self, setup_id: int, flow_id: int, parameters: dict[int, Any] | None): if not isinstance(setup_id, int): raise ValueError("setup id should be int") if not isinstance(flow_id, int): raise ValueError("flow id should be int") if parameters is not None and not isinstance(parameters, dict): raise ValueError("parameters should be dict") self.setup_id = setup_id self.flow_id = flow_id self.parameters = parameters ``` **After:** ```python @dataclass class OpenMLSetup: """Setup object (a.k.a. Configuration)....""" setup_id: int flow_id: int parameters: dict[int, Any] | None def __post_init__(self) -> None: if not isinstance(self.setup_id, int): raise ValueError("setup id should be int") if not isinstance(self.flow_id, int): raise ValueError("flow id should be int") if self.parameters is not None and not isinstance(self.parameters, dict): raise ValueError("parameters should be dict") ``` - OpenMLParameter **Before:** ```python class OpenMLParameter: """Parameter object (used in setup)....""" def __init__( # noqa: PLR0913 self, input_id: int, flow_id: int, flow_name: str, full_name: str, parameter_name: str, data_type: str, default_value: str, value: str, ): self.id = input_id self.flow_id = flow_id self.flow_name = flow_name self.full_name = full_name self.parameter_name = parameter_name self.data_type = data_type self.default_value = default_value self.value = value ``` **After:** ```python @dataclass class OpenMLParameter: """Parameter object (used in setup)....""" input_id: int flow_id: int flow_name: str full_name: str parameter_name: str data_type: str default_value: str value: str def __post_init__(self) -> None: # Map input_id to id for backward compatibility self.id = self.input_id ``` ## Tests For tests, I have used `xfail` temporarily to bypass the preexisting test failures in `tests\test_setups\test_setup_functions.py`. --- openml/setups/setup.py | 64 ++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 37 deletions(-) diff --git a/openml/setups/setup.py b/openml/setups/setup.py index 0960ad4c1..170838138 100644 --- a/openml/setups/setup.py +++ b/openml/setups/setup.py @@ -1,12 +1,14 @@ # License: BSD 3-Clause from __future__ import annotations +from dataclasses import asdict, dataclass from typing import Any import openml.config import openml.flows +@dataclass class OpenMLSetup: """Setup object (a.k.a. Configuration). @@ -20,20 +22,20 @@ class OpenMLSetup: The setting of the parameters """ - def __init__(self, setup_id: int, flow_id: int, parameters: dict[int, Any] | None): - if not isinstance(setup_id, int): + setup_id: int + flow_id: int + parameters: dict[int, Any] | None + + def __post_init__(self) -> None: + if not isinstance(self.setup_id, int): raise ValueError("setup id should be int") - if not isinstance(flow_id, int): + if not isinstance(self.flow_id, int): raise ValueError("flow id should be int") - if parameters is not None and not isinstance(parameters, dict): + if self.parameters is not None and not isinstance(self.parameters, dict): raise ValueError("parameters should be dict") - self.setup_id = setup_id - self.flow_id = flow_id - self.parameters = parameters - def _to_dict(self) -> dict[str, Any]: return { "setup_id": self.setup_id, @@ -66,6 +68,7 @@ def __repr__(self) -> str: return header + body +@dataclass class OpenMLParameter: """Parameter object (used in setup). @@ -91,37 +94,24 @@ class OpenMLParameter: If the parameter was set, the value that it was set to. """ - def __init__( # noqa: PLR0913 - self, - input_id: int, - flow_id: int, - flow_name: str, - full_name: str, - parameter_name: str, - data_type: str, - default_value: str, - value: str, - ): - self.id = input_id - self.flow_id = flow_id - self.flow_name = flow_name - self.full_name = full_name - self.parameter_name = parameter_name - self.data_type = data_type - self.default_value = default_value - self.value = value + input_id: int + flow_id: int + flow_name: str + full_name: str + parameter_name: str + data_type: str + default_value: str + value: str + + def __post_init__(self) -> None: + # Map input_id to id for backward compatibility + self.id = self.input_id def _to_dict(self) -> dict[str, Any]: - return { - "id": self.id, - "flow_id": self.flow_id, - "flow_name": self.flow_name, - "full_name": self.full_name, - "parameter_name": self.parameter_name, - "data_type": self.data_type, - "default_value": self.default_value, - "value": self.value, - } + result = asdict(self) + # Replaces input_id with id for backward compatibility + result["id"] = result.pop("input_id") + return result def __repr__(self) -> str: header = "OpenML Parameter" From ef242afab029be718caa9ba58045d119e9a8e458 Mon Sep 17 00:00:00 2001 From: Akarsh Kushwaha <136301822+Akarshkushwaha@users.noreply.github.com> Date: Mon, 16 Feb 2026 18:14:45 +0530 Subject: [PATCH 31/46] Update docs to reference main instead of develop (#1634) #### Metadata * Reference Issue: Fixes #1549 * New Tests Added: NA * Documentation Updated: Yes * Change Log Entry: Updated `PULL_REQUEST_TEMPLATE.md`, `CONTRIBUTING.md`, and `README.md` to reference the `main` branch instead of `develop`. #### Details * **What does this PR implement/fix? Explain your changes.** This PR updates the contribution documentation and the PR template to correctly reference the `main` branch as the default/target branch. The previous documentation incorrectly instructed contributors to use the `develop` branch, which does not exist in this repository. * **Why is this change necessary? What is the problem it solves?** The instructions were outdated for new contributors, as they referred to a non-existent `develop` branch. This corrects the workflow to align with the repository's actual structure (using `main`). * **How can I reproduce the issue this PR is solving and its solution?** Navigate to the previous version of `CONTRIBUTING.md` or `PULL_REQUEST_TEMPLATE.md` and observe the references to `develop`. Check the repository branches to confirm `develop` does not exist. * **Any other comments?** I have also verified the changes by running `pre-commit` locally to ensure no formatting issues were introduced. --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- CONTRIBUTING.md | 12 ++++++------ README.md | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 5584e6438..89ad09697 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -5,7 +5,7 @@ the contribution guidelines: https://github.com/openml/openml-python/blob/main/C Please make sure that: * the title of the pull request is descriptive -* this pull requests is against the `develop` branch +* this pull requests is against the `main` branch * for any new functionality, consider adding a relevant example * add unit tests for new functionalities * collect files uploaded to test server using _mark_entity_for_removal() diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3a18b63f2..d194525ef 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,7 +44,7 @@ To contribute to the openml-python package, follow these steps: 0. Determine how you want to contribute (see above). 1. Set up your local development environment. - 1. Fork and clone the `openml-python` repository. Then, create a new branch from the ``develop`` branch. If you are new to `git`, see our [detailed documentation](#basic-git-workflow), or rely on your favorite IDE. + 1. Fork and clone the `openml-python` repository. Then, create a new branch from the ``main`` branch. If you are new to `git`, see our [detailed documentation](#basic-git-workflow), or rely on your favorite IDE. 2. [Install the local dependencies](#install-local-dependencies) to run the tests for your contribution. 3. [Test your installation](#testing-your-installation) to ensure everything is set up correctly. 4. Implement your contribution. If contributing to the documentation, see [here](#contributing-to-the-documentation). @@ -91,7 +91,7 @@ pytest tests/test_datasets/test_dataset.py::OpenMLDatasetTest pytest tests/test_datasets/test_dataset.py::OpenMLDatasetTest::test_get_data ``` -To test your new contribution, add [unit tests](https://github.com/openml/openml-python/tree/develop/tests), and, if needed, [examples](https://github.com/openml/openml-python/tree/develop/examples) for any new functionality being introduced. Some notes on unit tests and examples: +To test your new contribution, add [unit tests](https://github.com/openml/openml-python/tree/main/tests), and, if needed, [examples](https://github.com/openml/openml-python/tree/main/examples) for any new functionality being introduced. Some notes on unit tests and examples: * If a unit test contains an upload to the test server, please ensure that it is followed by a file collection for deletion, to prevent the test server from bulking up. For example, `TestBase._mark_entity_for_removal('data', dataset.dataset_id)`, `TestBase._mark_entity_for_removal('flow', (flow.flow_id, flow.name))`. * Please ensure that the example is run on the test server by beginning with the call to `openml.config.start_using_configuration_for_example()`, which is done by default for tests derived from `TestBase`. * Add the `@pytest.mark.sklearn` marker to your unit tests if they have a dependency on scikit-learn. @@ -109,7 +109,7 @@ export OPENML_TEST_SERVER_ADMIN_KEY="admin-key" ### Pull Request Checklist -You can go to the `openml-python` GitHub repository to create the pull request by [comparing the branch](https://github.com/openml/openml-python/compare) from your fork with the `develop` branch of the `openml-python` repository. When creating a pull request, make sure to follow the comments and structured provided by the template on GitHub. +You can go to the `openml-python` GitHub repository to create the pull request by [comparing the branch](https://github.com/openml/openml-python/compare) from your fork with the `main` branch of the `openml-python` repository. When creating a pull request, make sure to follow the comments and structured provided by the template on GitHub. **An incomplete contribution** -- where you expect to do more work before receiving a full review -- should be submitted as a `draft`. These may be useful @@ -127,7 +127,7 @@ in the PR description. The preferred workflow for contributing to openml-python is to fork the [main repository](https://github.com/openml/openml-python) on -GitHub, clone, check out the branch `develop`, and develop on a new branch +GitHub, clone, check out the branch `main`, and develop on a new branch branch. Steps: 0. Make sure you have git installed, and a GitHub account. @@ -148,7 +148,7 @@ local disk: 3. Switch to the ``develop`` branch: ```bash - git checkout develop + git checkout main ``` 3. Create a ``feature`` branch to hold your development changes: @@ -157,7 +157,7 @@ local disk: git checkout -b feature/my-feature ``` - Always use a ``feature`` branch. It's good practice to never work on the ``main`` or ``develop`` branch! + Always use a ``feature`` branch. It's good practice to never work on the ``main`` branch! To make the nature of your pull request easily visible, please prepend the name of the branch with the type of changes you want to merge, such as ``feature`` if it contains a new feature, ``fix`` for a bugfix, ``doc`` for documentation and ``maint`` for other maintenance on the package. 4. Develop the feature on your feature branch. Add changed files using ``git add`` and then ``git commit`` files: diff --git a/README.md b/README.md index c44e42981..974c9fa53 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) -[Installation](https://openml.github.io/openml-python/main/#how-to-get-openml-for-python) | [Documentation](https://openml.github.io/openml-python) | [Contribution guidelines](https://github.com/openml/openml-python/blob/develop/CONTRIBUTING.md) +[Installation](https://openml.github.io/openml-python/main/#how-to-get-openml-for-python) | [Documentation](https://openml.github.io/openml-python) | [Contribution guidelines](https://github.com/openml/openml-python/blob/main/CONTRIBUTING.md) OpenML-Python provides an easy-to-use and straightforward Python interface for [OpenML](http://openml.org), an online platform for open science collaboration in machine learning. @@ -94,7 +94,7 @@ Bibtex entry: We welcome contributions from both new and experienced developers! If you would like to contribute to OpenML-Python, please read our -[Contribution Guidelines](https://github.com/openml/openml-python/blob/develop/CONTRIBUTING.md). +[Contribution Guidelines](https://github.com/openml/openml-python/blob/main/CONTRIBUTING.md). If you are new to open-source development, a great way to get started is by looking at issues labeled **"good first issue"** in our GitHub issue tracker. From d18ca42a73e4361baba3f32f25a98ecba7837e85 Mon Sep 17 00:00:00 2001 From: Shrivaths S Nair <142079253+JATAYU000@users.noreply.github.com> Date: Mon, 16 Feb 2026 18:38:58 +0530 Subject: [PATCH 32/46] [ENH] Add `get_cache_size` Utility Function (#1565) #### Metadata * Reference Issue: Fixes #1561 * New Tests Added: Yes * Documentation Updated: Yes (Doc string) * Change Log Entry: Add new function `get_cache_size()` in `utils` #### Details * What does this PR implement/fix? Implements a `get_cache_size()` function which returns the total size of the `openml` cache directory in bytes. --- openml/utils.py | 12 ++++++++++++ tests/test_utils/test_utils.py | 27 +++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/openml/utils.py b/openml/utils.py index bbc71d753..30dc4e53c 100644 --- a/openml/utils.py +++ b/openml/utils.py @@ -436,6 +436,18 @@ def safe_func(*args: P.args, **kwargs: P.kwargs) -> R: return func +def get_cache_size() -> int: + """Calculate the size of OpenML cache directory + + Returns + ------- + cache_size: int + Total size of cache in bytes + """ + path = Path(config.get_cache_directory()) + return sum(f.stat().st_size for f in path.rglob("*") if f.is_file()) + + def _create_lockfiles_dir() -> Path: path = Path(config.get_cache_directory()) / "locks" # TODO(eddiebergman): Not sure why this is allowed to error and ignore??? diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index a1cdb55ea..8dbdd30b5 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -152,3 +152,30 @@ def test_correct_test_server_download_state(): task = openml.tasks.get_task(119) dataset = task.get_dataset() assert len(dataset.features) == dataset.get_data()[0].shape[1] + +@unittest.mock.patch("openml.config.get_cache_directory") +def test_get_cache_size(config_mock,tmp_path): + """ + Test that the OpenML cache size utility correctly reports the cache directory + size before and after fetching a dataset. + + This test uses a temporary directory (tmp_path) as the cache location by + patching the configuration via config_mock. It verifies two conditions: + empty cache and after dataset fetch. + + Parameters + ---------- + config_mock : unittest.mock.Mock + A mock that overrides the configured cache directory to point to tmp_path. + tmp_path : pathlib.Path + A pytest-provided temporary directory used as an isolated cache location. + """ + + config_mock.return_value = tmp_path + cache_size = openml.utils.get_cache_size() + assert cache_size == 0 + sub_dir = tmp_path / "subdir" + sub_dir.mkdir() + (sub_dir / "nested_file.txt").write_bytes(b"b" * 100) + + assert openml.utils.get_cache_size() == 100 \ No newline at end of file From fefea5949833a4a42fb8cdfec98f26b1bb8b03b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20Kir=C3=A1ly?= Date: Mon, 16 Feb 2026 23:05:50 +0100 Subject: [PATCH 33/46] [ENH] move `utils` module to folder (#1612) This is a minimal refactor preparatory PR. It changes the `utils` module from a file to a folder, in anticipation of other PR that may add further utils - to avoid that everyone works on the same file. --- openml/utils/__init__.py | 39 +++++++++++++++++++++++++++ openml/{utils.py => utils/_openml.py} | 3 +-- 2 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 openml/utils/__init__.py rename openml/{utils.py => utils/_openml.py} (99%) diff --git a/openml/utils/__init__.py b/openml/utils/__init__.py new file mode 100644 index 000000000..1e74a3684 --- /dev/null +++ b/openml/utils/__init__.py @@ -0,0 +1,39 @@ +"""Utilities module.""" + +from openml.utils._openml import ( + ProgressBar, + ReprMixin, + _create_cache_directory, + _create_cache_directory_for_id, + _create_lockfiles_dir, + _delete_entity, + _get_cache_dir_for_id, + _get_cache_dir_for_key, + _get_rest_api_type_alias, + _list_all, + _remove_cache_dir_for_id, + _tag_entity, + _tag_openml_base, + extract_xml_tags, + get_cache_size, + thread_safe_if_oslo_installed, +) + +__all__ = [ + "ProgressBar", + "ReprMixin", + "_create_cache_directory", + "_create_cache_directory_for_id", + "_create_lockfiles_dir", + "_delete_entity", + "_get_cache_dir_for_id", + "_get_cache_dir_for_key", + "_get_rest_api_type_alias", + "_list_all", + "_remove_cache_dir_for_id", + "_tag_entity", + "_tag_openml_base", + "extract_xml_tags", + "get_cache_size", + "thread_safe_if_oslo_installed", +] diff --git a/openml/utils.py b/openml/utils/_openml.py similarity index 99% rename from openml/utils.py rename to openml/utils/_openml.py index 30dc4e53c..f18dbe3e0 100644 --- a/openml/utils.py +++ b/openml/utils/_openml.py @@ -26,8 +26,7 @@ import openml import openml._api_calls import openml.exceptions - -from . import config +from openml import config # Avoid import cycles: https://mypy.readthedocs.io/en/latest/common_issues.html#import-cycles if TYPE_CHECKING: From f585699c3c476f818c1373fdc352d03da3b390f8 Mon Sep 17 00:00:00 2001 From: Jigyasu Date: Tue, 17 Feb 2026 14:19:31 +0530 Subject: [PATCH 34/46] [DOC] Developer Environment Setup Docs (#1638) Adds documentation for setting up a developer environment, covering API v1, API v2, and python SDK. --- docs/developer_setup.md | 210 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 docs/developer_setup.md diff --git a/docs/developer_setup.md b/docs/developer_setup.md new file mode 100644 index 000000000..0886492ea --- /dev/null +++ b/docs/developer_setup.md @@ -0,0 +1,210 @@ +# OpenML Local Development Environment Setup + +This guide outlines the standard procedures for setting up a local development environment for the OpenML ecosystem. It covers the configuration of the backend servers (API v1 and API v2) and the Python Client SDK. + +OpenML currently has two backend architecture: + +* **API v1**: The PHP-based server currently serving production traffic. +* **API v2**: The Python-based server (FastAPI) currently under active development. + +> Note on Migration: API v1 is projected to remain operational through at least 2026. API v2 is the target architecture for future development. + +## 1. API v1 Setup (PHP Backend) + +This section details the deployment of the legacy PHP backend. + +### Prerequisites + +* **Docker**: Docker Desktop (Ensure the daemon is running). +* **Version Control**: Git. + +### Installation Steps + +#### 1. Clone the Repository + +Retrieve the OpenML services source code: + +```bash +git clone https://github.com/openml/services +cd services +``` + +#### 2. Configure File Permissions + +To ensure the containerized PHP service can write to the local filesystem, initialize the data directory permissions. + +From the repository root: + +```bash +chown -R www-data:www-data data/php +``` + +If the `www-data` user does not exist on the host system, grant full permissions as a fallback: + +```bash +chmod -R 777 data/php +``` + +#### 3. Launch Services + +Initialize the container stack: + +```bash +docker compose --profile all up -d +``` + +#### Warning: Container Conflicts + +If API v2 (Python backend) containers are present on the system, name conflicts may occur. To resolve this, stop and remove existing containers before launching API v1: + +```bash +docker compose --profile all down +docker compose --profile all up -d +``` + +#### 4. Verification + +Validate the deployment by accessing the flow endpoint. A successful response will return structured JSON data. + +* **Endpoint**: http://localhost:8080/api/v1/json/flow/181 + +### Client Configuration + +To direct the `openml-python` client to the local API v1 instance, modify the configuration as shown below. The API key corresponds to the default key located in `services/config/php/.env`. + +```python +import openml +from openml_sklearn.extension import SklearnExtension +from sklearn.neighbors import KNeighborsClassifier + +# Configure client to use local Docker instance +openml.config.server = "http://localhost:8080/api/v1/xml" +openml.config.apikey = "AD000000000000000000000000000000" + +# Test flow publication +clf = KNeighborsClassifier(n_neighbors=3) +extension = SklearnExtension() +knn_flow = extension.model_to_flow(clf) + +knn_flow.publish() +``` + +## 2. API v2 Setup (Python Backend) + +This section details the deployment of the FastAPI backend. + +### Prerequisites + +* **Docker**: Docker Desktop (Ensure the daemon is running). +* **Version Control**: Git. + +### Installation Steps + +#### 1. Clone the Repository + +Retrieve the API v2 source code: + +```bash +git clone https://github.com/openml/server-api +cd server-api +``` + +#### 2. Launch Services + +Build and start the container stack: + +```bash +docker compose --profile all up +``` + +#### 3. Verification + +Validate the deployment using the following endpoints: + +* **Task Endpoint**: http://localhost:8001/tasks/31 +* **Swagger UI (Documentation)**: http://localhost:8001/docs + +## 3. Python SDK (`openml-python`) Setup + +This section outlines the environment setup for contributing to the OpenML Python client. + +### Installation Steps + +#### 1. Clone the Repository + +```bash +git clone https://github.com/openml/openml-python +cd openml-python +``` + +#### 2. Environment Initialization + +Create an isolated virtual environment (example using Conda): + +```bash +conda create -n openml-python-dev python=3.12 +conda activate openml-python-dev +``` + +#### 3. Install Dependencies + +Install the package in editable mode, including development and documentation dependencies: + +```bash +python -m pip install -e ".[dev,docs]" +``` + +#### 4. Configure Quality Gates + +Install pre-commit hooks to enforce coding standards: + +```bash +pre-commit install +pre-commit run --all-files +``` + +## 4. Testing Guidelines + +The OpenML Python SDK utilizes `pytest` markers to categorize tests based on dependencies and execution context. + +| Marker | Description | +|-------------------|-----------------------------------------------------------------------------| +| `sklearn` | Tests requiring `scikit-learn`. Skipped if the library is missing. | +| `production` | Tests that interact with the live OpenML server (real API calls). | +| `uses_test_server` | Tests requiring the OpenML test server environment. | + +### Execution Examples + +Run the full test suite: + +```bash +pytest +``` + +Run a specific subset (e.g., `scikit-learn` tests): + +```bash +pytest -m sklearn +``` + +Exclude production tests (local only): + +```bash +pytest -m "not production" +``` + +### Admin Privilege Tests + +Certain tests require administrative privileges on the test server. These are skipped automatically unless an admin API key is provided via environment variables. + +#### Windows (PowerShell): + +```shell +$env:OPENML_TEST_SERVER_ADMIN_KEY = "admin-key" +``` + +#### Linux/macOS: + +```bash +export OPENML_TEST_SERVER_ADMIN_KEY="admin-key" +``` From da993f74df36eae7c6f0c08ee0597515df4c7a0a Mon Sep 17 00:00:00 2001 From: Armaghan Shakir Date: Tue, 17 Feb 2026 13:52:39 +0500 Subject: [PATCH 35/46] [DOC] Link to developer setup from documentation page (#1635) Adds link to developer setup from documentation page. --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index 0dba42557..419cc249e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -65,6 +65,7 @@ nav: - Advanced User Guide: details.md - API: reference/ - Contributing: contributing.md + - Developer Setup: developer_setup.md markdown_extensions: - pymdownx.highlight: From 099a1dc664734aeb268a0ee8113d4c61667292d6 Mon Sep 17 00:00:00 2001 From: Aniruth Karthik Date: Wed, 18 Feb 2026 16:42:45 +0530 Subject: [PATCH 36/46] [MNT] register pytest marker `test_server` and change `production` to `production_server` (#1632) * registers `test_server` marker, fixes #1631. * renames `production` marker to `production_server` --- .github/workflows/test.yml | 10 +- docs/developer_setup.md | 6 +- openml/cli.py | 8 +- pyproject.toml | 4 +- tests/conftest.py | 2 +- tests/test_datasets/test_dataset.py | 20 +-- tests/test_datasets/test_dataset_functions.py | 130 +++++++++--------- .../test_evaluation_functions.py | 24 ++-- tests/test_flows/test_flow.py | 22 +-- tests/test_flows/test_flow_functions.py | 30 ++-- tests/test_openml/test_api_calls.py | 6 +- tests/test_openml/test_config.py | 6 +- tests/test_runs/test_run.py | 12 +- tests/test_runs/test_run_functions.py | 90 ++++++------ tests/test_setups/test_setup_functions.py | 20 +-- tests/test_study/test_study_functions.py | 22 +-- tests/test_tasks/test_classification_task.py | 6 +- tests/test_tasks/test_clustering_task.py | 8 +- tests/test_tasks/test_learning_curve_task.py | 6 +- tests/test_tasks/test_regression_task.py | 4 +- tests/test_tasks/test_supervised_task.py | 2 +- tests/test_tasks/test_task.py | 4 +- tests/test_tasks/test_task_functions.py | 38 ++--- tests/test_tasks/test_task_methods.py | 4 +- tests/test_utils/test_utils.py | 20 +-- 25 files changed, 252 insertions(+), 252 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 29ada2298..7fa3450ca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -114,9 +114,9 @@ jobs: fi if [ "${{ matrix.sklearn-only }}" = "true" ]; then - marks="sklearn and not production and not uses_test_server" + marks="sklearn and not production_server and not test_server" else - marks="not production and not uses_test_server" + marks="not production_server and not test_server" fi pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks" @@ -131,9 +131,9 @@ jobs: fi if [ "${{ matrix.sklearn-only }}" = "true" ]; then - marks="sklearn and production and not uses_test_server" + marks="sklearn and production_server and not test_server" else - marks="production and not uses_test_server" + marks="production_server and not test_server" fi pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks" @@ -143,7 +143,7 @@ jobs: env: OPENML_TEST_SERVER_ADMIN_KEY: ${{ secrets.OPENML_TEST_SERVER_ADMIN_KEY }} run: | # we need a separate step because of the bash-specific if-statement in the previous one. - pytest -n 4 --durations=20 --dist load -sv --reruns 5 --reruns-delay 1 -m "not uses_test_server" + pytest -n 4 --durations=20 --dist load -sv --reruns 5 --reruns-delay 1 -m "not test_server" - name: Check for files left behind by test if: matrix.os != 'windows-latest' && always() diff --git a/docs/developer_setup.md b/docs/developer_setup.md index 0886492ea..55a73fef9 100644 --- a/docs/developer_setup.md +++ b/docs/developer_setup.md @@ -170,8 +170,8 @@ The OpenML Python SDK utilizes `pytest` markers to categorize tests based on dep | Marker | Description | |-------------------|-----------------------------------------------------------------------------| | `sklearn` | Tests requiring `scikit-learn`. Skipped if the library is missing. | -| `production` | Tests that interact with the live OpenML server (real API calls). | -| `uses_test_server` | Tests requiring the OpenML test server environment. | +| `production_server`| Tests that interact with the live OpenML server (real API calls). | +| `test_server` | Tests requiring the OpenML test server environment. | ### Execution Examples @@ -190,7 +190,7 @@ pytest -m sklearn Exclude production tests (local only): ```bash -pytest -m "not production" +pytest -m "not production_server" ``` ### Admin Privilege Tests diff --git a/openml/cli.py b/openml/cli.py index 0afb089c2..cbcc38f4a 100644 --- a/openml/cli.py +++ b/openml/cli.py @@ -102,15 +102,15 @@ def check_apikey(apikey: str) -> str: def configure_server(value: str) -> None: def check_server(server: str) -> str: - is_shorthand = server in ["test", "production"] + is_shorthand = server in ["test", "production_server"] if is_shorthand or looks_like_url(server): return "" - return "Must be 'test', 'production' or a url." + return "Must be 'test', 'production_server' or a url." def replace_shorthand(server: str) -> str: if server == "test": return "https://test.openml.org/api/v1/xml" - if server == "production": + if server == "production_server": return "https://www.openml.org/api/v1/xml" return server @@ -119,7 +119,7 @@ def replace_shorthand(server: str) -> str: value=value, check_with_message=check_server, intro_message="Specify which server you wish to connect to.", - input_message="Specify a url or use 'test' or 'production' as a shorthand: ", + input_message="Specify a url or use 'test' or 'production_server' as a shorthand: ", sanitize=replace_shorthand, ) diff --git a/pyproject.toml b/pyproject.toml index 93a6ffbfa..47013271d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -133,10 +133,10 @@ filterwarnings=[ "ignore:the matrix subclass:PendingDeprecationWarning" ] markers = [ - "server: anything that connects to a server", "upload: anything that uploads to a server", - "production: any interaction with the production server", + "production_server: any interaction with the production server", "cache: anything that interacts with the (test) cache", + "test_server: tests that require the OpenML test server", ] # https://github.com/charliermarsh/ruff diff --git a/tests/conftest.py b/tests/conftest.py index bd974f3f3..4fffa9f38 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -272,7 +272,7 @@ def as_robot() -> Iterator[None]: @pytest.fixture(autouse=True) def with_server(request): - if "production" in request.keywords: + if "production_server" in request.keywords: openml.config.server = "https://www.openml.org/api/v1/xml" openml.config.apikey = None yield diff --git a/tests/test_datasets/test_dataset.py b/tests/test_datasets/test_dataset.py index b13bac30b..c651845fb 100644 --- a/tests/test_datasets/test_dataset.py +++ b/tests/test_datasets/test_dataset.py @@ -18,7 +18,7 @@ import pytest -@pytest.mark.production() +@pytest.mark.production_server() class OpenMLDatasetTest(TestBase): _multiprocess_can_split_ = True @@ -281,7 +281,7 @@ def test_equality_comparison(self): self.assertNotEqual(self.titanic, "Wrong_object") -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_tagging(): dataset = openml.datasets.get_dataset(125, download_data=False) @@ -298,7 +298,7 @@ def test_tagging(): datasets = openml.datasets.list_datasets(tag=tag) assert datasets.empty -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_get_feature_with_ontology_data_id_11(): # test on car dataset, which has built-in ontology references dataset = openml.datasets.get_dataset(11) @@ -307,7 +307,7 @@ def test_get_feature_with_ontology_data_id_11(): assert len(dataset.features[2].ontologies) >= 1 assert len(dataset.features[3].ontologies) >= 1 -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_add_remove_ontology_to_dataset(): did = 1 feature_index = 1 @@ -315,7 +315,7 @@ def test_add_remove_ontology_to_dataset(): openml.datasets.functions.data_feature_add_ontology(did, feature_index, ontology) openml.datasets.functions.data_feature_remove_ontology(did, feature_index, ontology) -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_add_same_ontology_multiple_features(): did = 1 ontology = "https://www.openml.org/unittest/" + str(time()) @@ -324,7 +324,7 @@ def test_add_same_ontology_multiple_features(): openml.datasets.functions.data_feature_add_ontology(did, i, ontology) -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_add_illegal_long_ontology(): did = 1 ontology = "http://www.google.com/" + ("a" * 257) @@ -336,7 +336,7 @@ def test_add_illegal_long_ontology(): -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_add_illegal_url_ontology(): did = 1 ontology = "not_a_url" + str(time()) @@ -347,7 +347,7 @@ def test_add_illegal_url_ontology(): assert e.code == 1106 -@pytest.mark.production() +@pytest.mark.production_server() class OpenMLDatasetTestSparse(TestBase): _multiprocess_can_split_ = True @@ -408,7 +408,7 @@ def test_get_sparse_categorical_data_id_395(self): assert len(feature.nominal_values) == 25 -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test__read_features(mocker, workdir, static_cache_dir): """Test we read the features from the xml if no cache pickle is available. This test also does some simple checks to verify that the features are read correctly @@ -440,7 +440,7 @@ def test__read_features(mocker, workdir, static_cache_dir): assert pickle_mock.dump.call_count == 1 -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test__read_qualities(static_cache_dir, workdir, mocker): """Test we read the qualities from the xml if no cache pickle is available. This test also does some minor checks to ensure that the qualities are read correctly. diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index d80743a8c..41e89d950 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -107,7 +107,7 @@ def _check_datasets(self, datasets): for did in datasets: self._check_dataset(datasets[did]) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_tag_untag_dataset(self): tag = "test_tag_%d" % random.randint(1, 1000000) all_tags = _tag_entity("data", 1, tag) @@ -115,12 +115,12 @@ def test_tag_untag_dataset(self): all_tags = _tag_entity("data", 1, tag, untag=True) assert tag not in all_tags - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_list_datasets_length(self): datasets = openml.datasets.list_datasets() assert len(datasets) >= 100 - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_list_datasets_paginate(self): size = 10 max = 100 @@ -135,12 +135,12 @@ def test_list_datasets_paginate(self): categories=["in_preparation", "active", "deactivated"], ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_list_datasets_empty(self): datasets = openml.datasets.list_datasets(tag="NoOneWouldUseThisTagAnyway") assert datasets.empty - @pytest.mark.production() + @pytest.mark.production_server() def test_check_datasets_active(self): # Have to test on live because there is no deactivated dataset on the test server. self.use_production_server() @@ -159,7 +159,7 @@ def test_check_datasets_active(self): ) openml.config.server = self.test_server - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_illegal_character_tag(self): dataset = openml.datasets.get_dataset(1) tag = "illegal_tag&" @@ -169,7 +169,7 @@ def test_illegal_character_tag(self): except openml.exceptions.OpenMLServerException as e: assert e.code == 477 - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_illegal_length_tag(self): dataset = openml.datasets.get_dataset(1) tag = "a" * 65 @@ -179,7 +179,7 @@ def test_illegal_length_tag(self): except openml.exceptions.OpenMLServerException as e: assert e.code == 477 - @pytest.mark.production() + @pytest.mark.production_server() def test__name_to_id_with_deactivated(self): """Check that an activated dataset is returned if an earlier deactivated one exists.""" self.use_production_server() @@ -187,19 +187,19 @@ def test__name_to_id_with_deactivated(self): assert openml.datasets.functions._name_to_id("anneal") == 2 openml.config.server = self.test_server - @pytest.mark.production() + @pytest.mark.production_server() def test__name_to_id_with_multiple_active(self): """With multiple active datasets, retrieve the least recent active.""" self.use_production_server() assert openml.datasets.functions._name_to_id("iris") == 61 - @pytest.mark.production() + @pytest.mark.production_server() def test__name_to_id_with_version(self): """With multiple active datasets, retrieve the least recent active.""" self.use_production_server() assert openml.datasets.functions._name_to_id("iris", version=3) == 969 - @pytest.mark.production() + @pytest.mark.production_server() def test__name_to_id_with_multiple_active_error(self): """With multiple active datasets, retrieve the least recent active.""" self.use_production_server() @@ -211,7 +211,7 @@ def test__name_to_id_with_multiple_active_error(self): error_if_multiple=True, ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test__name_to_id_name_does_not_exist(self): """With multiple active datasets, retrieve the least recent active.""" self.assertRaisesRegex( @@ -221,7 +221,7 @@ def test__name_to_id_name_does_not_exist(self): dataset_name="does_not_exist", ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test__name_to_id_version_does_not_exist(self): """With multiple active datasets, retrieve the least recent active.""" self.assertRaisesRegex( @@ -232,7 +232,7 @@ def test__name_to_id_version_does_not_exist(self): version=100000, ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_datasets_by_name(self): # did 1 and 2 on the test server: dids = ["anneal", "kr-vs-kp"] @@ -240,7 +240,7 @@ def test_get_datasets_by_name(self): assert len(datasets) == 2 _assert_datasets_retrieved_successfully([1, 2]) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_datasets_by_mixed(self): # did 1 and 2 on the test server: dids = ["anneal", 2] @@ -248,14 +248,14 @@ def test_get_datasets_by_mixed(self): assert len(datasets) == 2 _assert_datasets_retrieved_successfully([1, 2]) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_datasets(self): dids = [1, 2] datasets = openml.datasets.get_datasets(dids) assert len(datasets) == 2 _assert_datasets_retrieved_successfully([1, 2]) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_dataset_by_name(self): dataset = openml.datasets.get_dataset("anneal") assert type(dataset) == OpenMLDataset @@ -274,7 +274,7 @@ def test_get_dataset_download_all_files(self): # test_get_dataset_lazy raise NotImplementedError - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_dataset_uint8_dtype(self): dataset = openml.datasets.get_dataset(1) assert type(dataset) == OpenMLDataset @@ -282,7 +282,7 @@ def test_get_dataset_uint8_dtype(self): df, _, _, _ = dataset.get_data() assert df["carbon"].dtype == "uint8" - @pytest.mark.production() + @pytest.mark.production_server() def test_get_dataset_cannot_access_private_data(self): # Issue324 Properly handle private datasets when trying to access them self.use_production_server() @@ -293,7 +293,7 @@ def test_dataset_by_name_cannot_access_private_data(self): self.use_production_server() self.assertRaises(OpenMLPrivateDatasetError, openml.datasets.get_dataset, "NAME_GOES_HERE") - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_dataset_lazy_all_functions(self): """Test that all expected functionality is available without downloading the dataset.""" dataset = openml.datasets.get_dataset(1) @@ -323,28 +323,28 @@ def ensure_absence_of_real_data(): assert classes == ["1", "2", "3", "4", "5", "U"] ensure_absence_of_real_data() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_dataset_sparse(self): dataset = openml.datasets.get_dataset(102) X, *_ = dataset.get_data() assert isinstance(X, pd.DataFrame) assert all(isinstance(col, pd.SparseDtype) for col in X.dtypes) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_download_rowid(self): # Smoke test which checks that the dataset has the row-id set correctly did = 44 dataset = openml.datasets.get_dataset(did) assert dataset.row_id_attribute == "Counter" - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test__get_dataset_description(self): description = _get_dataset_description(self.workdir, 2) assert isinstance(description, dict) description_xml_path = os.path.join(self.workdir, "description.xml") assert os.path.exists(description_xml_path) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test__getarff_path_dataset_arff(self): openml.config.set_root_cache_directory(self.static_cache_dir) description = _get_dataset_description(self.workdir, 2) @@ -408,7 +408,7 @@ def test__download_minio_file_works_with_bucket_subdirectory(self): @mock.patch("openml._api_calls._download_minio_file") - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test__get_dataset_parquet_is_cached(self, patch): openml.config.set_root_cache_directory(self.static_cache_dir) patch.side_effect = RuntimeError( @@ -449,21 +449,21 @@ def test__getarff_md5_issue(self): openml.config.connection_n_retries = n - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test__get_dataset_features(self): features_file = _get_dataset_features_file(self.workdir, 2) assert isinstance(features_file, Path) features_xml_path = self.workdir / "features.xml" assert features_xml_path.exists() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test__get_dataset_qualities(self): qualities = _get_dataset_qualities_file(self.workdir, 2) assert isinstance(qualities, Path) qualities_xml_path = self.workdir / "qualities.xml" assert qualities_xml_path.exists() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_dataset_force_refresh_cache(self): did_cache_dir = _create_cache_directory_for_id( DATASETS_CACHE_DIR_NAME, @@ -486,7 +486,7 @@ def test_get_dataset_force_refresh_cache(self): did_cache_dir, ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_dataset_force_refresh_cache_clean_start(self): did_cache_dir = _create_cache_directory_for_id( DATASETS_CACHE_DIR_NAME, @@ -523,14 +523,14 @@ def test_deletion_of_cache_dir(self): # get_dataset_description is the only data guaranteed to be downloaded @mock.patch("openml.datasets.functions._get_dataset_description") - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_deletion_of_cache_dir_faulty_download(self, patch): patch.side_effect = Exception("Boom!") self.assertRaisesRegex(Exception, "Boom!", openml.datasets.get_dataset, dataset_id=1) datasets_cache_dir = os.path.join(self.workdir, "org", "openml", "test", "datasets") assert len(os.listdir(datasets_cache_dir)) == 0 - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_publish_dataset(self): # lazy loading not possible as we need the arff-file. openml.datasets.get_dataset(3, download_data=True) @@ -556,7 +556,7 @@ def test_publish_dataset(self): ) assert isinstance(dataset.dataset_id, int) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test__retrieve_class_labels(self): openml.config.set_root_cache_directory(self.static_cache_dir) labels = openml.datasets.get_dataset(2).retrieve_class_labels() @@ -573,7 +573,7 @@ def test__retrieve_class_labels(self): labels = custom_ds.retrieve_class_labels(target_name=custom_ds.features[31].name) assert labels == ["COIL", "SHEET"] - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_upload_dataset_with_url(self): dataset = OpenMLDataset( f"{self._get_sentinel()}-UploadTestWithURL", @@ -604,7 +604,7 @@ def _assert_status_of_dataset(self, *, did: int, status: str): reason="Test requires admin key. Set OPENML_TEST_SERVER_ADMIN_KEY environment variable.", ) @pytest.mark.flaky() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_data_status(self): dataset = OpenMLDataset( f"{self._get_sentinel()}-UploadTestWithURL", @@ -696,7 +696,7 @@ def test_attributes_arff_from_df_unknown_dtype(self): with pytest.raises(ValueError, match=err_msg): attributes_arff_from_df(df) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_create_dataset_numpy(self): data = np.array([[1, 2, 3], [1.2, 2.5, 3.8], [2, 5, 8], [0, 1, 0]]).T @@ -730,7 +730,7 @@ def test_create_dataset_numpy(self): ), "Uploaded arff does not match original one" assert _get_online_dataset_format(dataset.id) == "arff", "Wrong format for dataset" - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_create_dataset_list(self): data = [ ["a", "sunny", 85.0, 85.0, "FALSE", "no"], @@ -785,7 +785,7 @@ def test_create_dataset_list(self): ), "Uploaded ARFF does not match original one" assert _get_online_dataset_format(dataset.id) == "arff", "Wrong format for dataset" - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_create_dataset_sparse(self): # test the scipy.sparse.coo_matrix sparse_data = scipy.sparse.coo_matrix( @@ -888,7 +888,7 @@ def test_create_invalid_dataset(self): param["data"] = data[0] self.assertRaises(ValueError, create_dataset, **param) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_online_dataset_arff(self): dataset_id = 100 # Australian # lazy loading not used as arff file is checked. @@ -904,7 +904,7 @@ def test_get_online_dataset_arff(self): return_type=arff.DENSE if d_format == "arff" else arff.COO, ), "ARFF files are not equal" - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_topic_api_error(self): # Check server exception when non-admin accessses apis self.assertRaisesRegex( @@ -923,7 +923,7 @@ def test_topic_api_error(self): topic="business", ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_online_dataset_format(self): # Phoneme dataset dataset_id = 77 @@ -933,7 +933,7 @@ def test_get_online_dataset_format(self): dataset_id ), "The format of the ARFF files is different" - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_create_dataset_pandas(self): data = [ ["a", "sunny", 85.0, 85.0, "FALSE", "no"], @@ -1158,7 +1158,7 @@ def test_ignore_attributes_dataset(self): paper_url=paper_url, ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_publish_fetch_ignore_attribute(self): """Test to upload and retrieve dataset and check ignore_attributes""" data = [ @@ -1277,7 +1277,7 @@ def test_create_dataset_row_id_attribute_error(self): paper_url=paper_url, ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_create_dataset_row_id_attribute_inference(self): # meta-information name = f"{self._get_sentinel()}-pandas_testing_dataset" @@ -1368,13 +1368,13 @@ def test_create_dataset_attributes_auto_without_df(self): paper_url=paper_url, ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_list_qualities(self): qualities = openml.datasets.list_qualities() assert isinstance(qualities, list) is True assert all(isinstance(q, str) for q in qualities) is True - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_dataset_cache_format_pickle(self): dataset = openml.datasets.get_dataset(1) dataset.get_data() @@ -1390,7 +1390,7 @@ def test_get_dataset_cache_format_pickle(self): assert len(categorical) == X.shape[1] assert len(attribute_names) == X.shape[1] - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_dataset_cache_format_feather(self): # This test crashed due to using the parquet file by default, which is downloaded # from minio. However, there is a mismatch between OpenML test server and minio IDs. @@ -1423,7 +1423,7 @@ def test_get_dataset_cache_format_feather(self): assert len(categorical) == X.shape[1] assert len(attribute_names) == X.shape[1] - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_data_edit_non_critical_field(self): # Case 1 # All users can edit non-critical fields of datasets @@ -1445,7 +1445,7 @@ def test_data_edit_non_critical_field(self): edited_dataset = openml.datasets.get_dataset(did) assert edited_dataset.description == desc - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_data_edit_critical_field(self): # Case 2 # only owners (or admin) can edit all critical fields of datasets @@ -1472,7 +1472,7 @@ def test_data_edit_critical_field(self): os.path.join(self.workdir, "org", "openml", "test", "datasets", str(did)), ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_data_edit_requires_field(self): # Check server exception when no field to edit is provided self.assertRaisesRegex( @@ -1485,7 +1485,7 @@ def test_data_edit_requires_field(self): data_id=64, # blood-transfusion-service-center ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_data_edit_requires_valid_dataset(self): # Check server exception when unknown dataset is provided self.assertRaisesRegex( @@ -1496,7 +1496,7 @@ def test_data_edit_requires_valid_dataset(self): description="xor operation dataset", ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_data_edit_cannot_edit_critical_field_if_dataset_has_task(self): # Need to own a dataset to be able to edit meta-data # Will be creating a forked version of an existing dataset to allow the unit test user @@ -1523,7 +1523,7 @@ def test_data_edit_cannot_edit_critical_field_if_dataset_has_task(self): default_target_attribute="y", ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_edit_data_user_cannot_edit_critical_field_of_other_users_dataset(self): # Check server exception when a non-owner or non-admin tries to edit critical fields self.assertRaisesRegex( @@ -1535,7 +1535,7 @@ def test_edit_data_user_cannot_edit_critical_field_of_other_users_dataset(self): default_target_attribute="y", ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_data_fork(self): did = 1 result = fork_dataset(did) @@ -1549,7 +1549,7 @@ def test_data_fork(self): ) - @pytest.mark.production() + @pytest.mark.production_server() def test_list_datasets_with_high_size_parameter(self): # Testing on prod since concurrent deletion of uploded datasets make the test fail self.use_production_server() @@ -1827,7 +1827,7 @@ def all_datasets(): return openml.datasets.list_datasets() -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_list_datasets(all_datasets: pd.DataFrame): # We can only perform a smoke test here because we test on dynamic # data from the internet... @@ -1836,49 +1836,49 @@ def test_list_datasets(all_datasets: pd.DataFrame): _assert_datasets_have_id_and_valid_status(all_datasets) -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_list_datasets_by_tag(all_datasets: pd.DataFrame): tag_datasets = openml.datasets.list_datasets(tag="study_14") assert 0 < len(tag_datasets) < len(all_datasets) _assert_datasets_have_id_and_valid_status(tag_datasets) -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_list_datasets_by_size(): datasets = openml.datasets.list_datasets(size=5) assert len(datasets) == 5 _assert_datasets_have_id_and_valid_status(datasets) -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_list_datasets_by_number_instances(all_datasets: pd.DataFrame): small_datasets = openml.datasets.list_datasets(number_instances="5..100") assert 0 < len(small_datasets) <= len(all_datasets) _assert_datasets_have_id_and_valid_status(small_datasets) -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_list_datasets_by_number_features(all_datasets: pd.DataFrame): wide_datasets = openml.datasets.list_datasets(number_features="50..100") assert 8 <= len(wide_datasets) < len(all_datasets) _assert_datasets_have_id_and_valid_status(wide_datasets) -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_list_datasets_by_number_classes(all_datasets: pd.DataFrame): five_class_datasets = openml.datasets.list_datasets(number_classes="5") assert 3 <= len(five_class_datasets) < len(all_datasets) _assert_datasets_have_id_and_valid_status(five_class_datasets) -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_list_datasets_by_number_missing_values(all_datasets: pd.DataFrame): na_datasets = openml.datasets.list_datasets(number_missing_values="5..100") assert 5 <= len(na_datasets) < len(all_datasets) _assert_datasets_have_id_and_valid_status(na_datasets) -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_list_datasets_combined_filters(all_datasets: pd.DataFrame): combined_filter_datasets = openml.datasets.list_datasets( tag="study_14", @@ -1951,7 +1951,7 @@ def isolate_for_test(): ("with_data", "with_qualities", "with_features"), itertools.product([True, False], repeat=3), ) -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_get_dataset_lazy_behavior( isolate_for_test, with_data: bool, with_qualities: bool, with_features: bool ): @@ -1978,7 +1978,7 @@ def test_get_dataset_lazy_behavior( ) -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_get_dataset_with_invalid_id() -> None: INVALID_ID = 123819023109238 # Well, at some point this will probably be valid... with pytest.raises(OpenMLServerNoResult, match="Unknown dataset") as e: @@ -2006,7 +2006,7 @@ def test_read_features_from_xml_with_whitespace() -> None: assert dict[1].nominal_values == [" - 50000.", " 50000+."] -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_get_dataset_parquet(requests_mock, test_files_directory): # Parquet functionality is disabled on the test server # There is no parquet-copy of the test server yet. diff --git a/tests/test_evaluations/test_evaluation_functions.py b/tests/test_evaluations/test_evaluation_functions.py index ee7c306a1..e15556d7b 100644 --- a/tests/test_evaluations/test_evaluation_functions.py +++ b/tests/test_evaluations/test_evaluation_functions.py @@ -50,7 +50,7 @@ def _check_list_evaluation_setups(self, **kwargs): self.assertSequenceEqual(sorted(list1), sorted(list2)) return evals_setups - @pytest.mark.production() + @pytest.mark.production_server() def test_evaluation_list_filter_task(self): self.use_production_server() @@ -70,7 +70,7 @@ def test_evaluation_list_filter_task(self): assert evaluations[run_id].value is not None assert evaluations[run_id].values is None - @pytest.mark.production() + @pytest.mark.production_server() def test_evaluation_list_filter_uploader_ID_16(self): self.use_production_server() @@ -85,7 +85,7 @@ def test_evaluation_list_filter_uploader_ID_16(self): assert len(evaluations) > 50 - @pytest.mark.production() + @pytest.mark.production_server() def test_evaluation_list_filter_uploader_ID_10(self): self.use_production_server() @@ -104,7 +104,7 @@ def test_evaluation_list_filter_uploader_ID_10(self): assert evaluations[run_id].value is not None assert evaluations[run_id].values is None - @pytest.mark.production() + @pytest.mark.production_server() def test_evaluation_list_filter_flow(self): self.use_production_server() @@ -124,7 +124,7 @@ def test_evaluation_list_filter_flow(self): assert evaluations[run_id].value is not None assert evaluations[run_id].values is None - @pytest.mark.production() + @pytest.mark.production_server() def test_evaluation_list_filter_run(self): self.use_production_server() @@ -144,7 +144,7 @@ def test_evaluation_list_filter_run(self): assert evaluations[run_id].value is not None assert evaluations[run_id].values is None - @pytest.mark.production() + @pytest.mark.production_server() def test_evaluation_list_limit(self): self.use_production_server() @@ -155,7 +155,7 @@ def test_evaluation_list_limit(self): ) assert len(evaluations) == 100 - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_list_evaluations_empty(self): evaluations = openml.evaluations.list_evaluations("unexisting_measure") if len(evaluations) > 0: @@ -163,7 +163,7 @@ def test_list_evaluations_empty(self): assert isinstance(evaluations, dict) - @pytest.mark.production() + @pytest.mark.production_server() def test_evaluation_list_per_fold(self): self.use_production_server() size = 1000 @@ -201,7 +201,7 @@ def test_evaluation_list_per_fold(self): assert evaluations[run_id].value is not None assert evaluations[run_id].values is None - @pytest.mark.production() + @pytest.mark.production_server() def test_evaluation_list_sort(self): self.use_production_server() size = 10 @@ -233,13 +233,13 @@ def test_evaluation_list_sort(self): test_output = sorted(unsorted_output, reverse=True) assert test_output[:size] == sorted_output - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_list_evaluation_measures(self): measures = openml.evaluations.list_evaluation_measures() assert isinstance(measures, list) is True assert all(isinstance(s, str) for s in measures) is True - @pytest.mark.production() + @pytest.mark.production_server() def test_list_evaluations_setups_filter_flow(self): self.use_production_server() flow_id = [405] @@ -257,7 +257,7 @@ def test_list_evaluations_setups_filter_flow(self): keys = list(evals["parameters"].values[0].keys()) assert all(elem in columns for elem in keys) - @pytest.mark.production() + @pytest.mark.production_server() @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_list_evaluations_setups_filter_task(self): self.use_production_server() diff --git a/tests/test_flows/test_flow.py b/tests/test_flows/test_flow.py index 527ad1f8c..b942c0ab9 100644 --- a/tests/test_flows/test_flow.py +++ b/tests/test_flows/test_flow.py @@ -44,7 +44,7 @@ def setUp(self): def tearDown(self): super().tearDown() - @pytest.mark.production() + @pytest.mark.production_server() def test_get_flow(self): # We need to use the production server here because 4024 is not the # test server @@ -77,7 +77,7 @@ def test_get_flow(self): assert subflow_3.parameters["L"] == "-1" assert len(subflow_3.components) == 0 - @pytest.mark.production() + @pytest.mark.production_server() @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_get_structure(self): # also responsible for testing: flow.get_subflow @@ -103,7 +103,7 @@ def test_get_structure(self): subflow = flow.get_subflow(structure) assert subflow.flow_id == sub_flow_id - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_tagging(self): flows = openml.flows.list_flows(size=1) flow_id = flows["id"].iloc[0] @@ -121,7 +121,7 @@ def test_tagging(self): flows = openml.flows.list_flows(tag=tag) assert len(flows) == 0 - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_from_xml_to_xml(self): # Get the raw xml thing # TODO maybe get this via get_flow(), which would have to be refactored @@ -181,7 +181,7 @@ def test_to_xml_from_xml(self): assert new_flow is not flow @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_publish_flow(self): flow = openml.OpenMLFlow( name="sklearn.dummy.DummyClassifier", @@ -223,7 +223,7 @@ def test_publish_existing_flow(self, flow_exists_mock): ) @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_publish_flow_with_similar_components(self): clf = sklearn.ensemble.VotingClassifier( [("lr", sklearn.linear_model.LogisticRegression(solver="lbfgs"))], @@ -274,7 +274,7 @@ def test_publish_flow_with_similar_components(self): TestBase.logger.info(f"collected from {__file__.split('/')[-1]}: {flow3.flow_id}") @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_semi_legal_flow(self): # TODO: Test if parameters are set correctly! # should not throw error as it contains two differentiable forms of @@ -366,7 +366,7 @@ def test_illegal_flow(self): ) self.assertRaises(ValueError, self.extension.model_to_flow, illegal) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_nonexisting_flow_exists(self): def get_sentinel(): # Create a unique prefix for the flow. Necessary because the flow @@ -384,7 +384,7 @@ def get_sentinel(): assert not flow_id @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_existing_flow_exists(self): # create a flow nb = sklearn.naive_bayes.GaussianNB() @@ -425,7 +425,7 @@ def test_existing_flow_exists(self): assert downloaded_flow_id == flow.flow_id @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_sklearn_to_upload_to_flow(self): iris = sklearn.datasets.load_iris() X = iris.data @@ -565,7 +565,7 @@ def test_extract_tags(self): tags = openml.utils.extract_xml_tags("oml:tag", flow_dict["oml:flow"]) assert tags == ["OpenmlWeka", "weka"] - @pytest.mark.production() + @pytest.mark.production_server() def test_download_non_scikit_learn_flows(self): self.use_production_server() diff --git a/tests/test_flows/test_flow_functions.py b/tests/test_flows/test_flow_functions.py index 5aa99cd62..c9af3bf8f 100644 --- a/tests/test_flows/test_flow_functions.py +++ b/tests/test_flows/test_flow_functions.py @@ -47,7 +47,7 @@ def _check_flow(self, flow): ) assert ext_version_str_or_none - @pytest.mark.production() + @pytest.mark.production_server() def test_list_flows(self): self.use_production_server() # We can only perform a smoke test here because we test on dynamic @@ -58,7 +58,7 @@ def test_list_flows(self): for flow in flows.to_dict(orient="index").values(): self._check_flow(flow) - @pytest.mark.production() + @pytest.mark.production_server() def test_list_flows_output_format(self): self.use_production_server() # We can only perform a smoke test here because we test on dynamic @@ -67,13 +67,13 @@ def test_list_flows_output_format(self): assert isinstance(flows, pd.DataFrame) assert len(flows) >= 1500 - @pytest.mark.production() + @pytest.mark.production_server() def test_list_flows_empty(self): self.use_production_server() flows = openml.flows.list_flows(tag="NoOneEverUsesThisTag123") assert flows.empty - @pytest.mark.production() + @pytest.mark.production_server() def test_list_flows_by_tag(self): self.use_production_server() flows = openml.flows.list_flows(tag="weka") @@ -81,7 +81,7 @@ def test_list_flows_by_tag(self): for flow in flows.to_dict(orient="index").values(): self._check_flow(flow) - @pytest.mark.production() + @pytest.mark.production_server() def test_list_flows_paginate(self): self.use_production_server() size = 10 @@ -280,7 +280,7 @@ def test_are_flows_equal_ignore_if_older(self): reason="OrdinalEncoder introduced in 0.20. " "No known models with list of lists parameters in older versions.", ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_sklearn_to_flow_list_of_lists(self): from sklearn.preprocessing import OrdinalEncoder @@ -301,7 +301,7 @@ def test_sklearn_to_flow_list_of_lists(self): assert server_flow.parameters["categories"] == "[[0, 1], [0, 1]]" assert server_flow.model.categories == flow.model.categories - @pytest.mark.production() + @pytest.mark.production_server() def test_get_flow1(self): # Regression test for issue #305 # Basically, this checks that a flow without an external version can be loaded @@ -310,7 +310,7 @@ def test_get_flow1(self): assert flow.external_version is None @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_flow_reinstantiate_model(self): model = ensemble.RandomForestClassifier(n_estimators=33) extension = openml.extensions.get_extension_by_model(model) @@ -322,7 +322,7 @@ def test_get_flow_reinstantiate_model(self): downloaded_flow = openml.flows.get_flow(flow.flow_id, reinstantiate=True) assert isinstance(downloaded_flow.model, sklearn.ensemble.RandomForestClassifier) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_flow_reinstantiate_model_no_extension(self): # Flow 10 is a WEKA flow self.assertRaisesRegex( @@ -338,7 +338,7 @@ def test_get_flow_reinstantiate_model_no_extension(self): Version(sklearn.__version__) == Version("0.19.1"), reason="Requires scikit-learn!=0.19.1, because target flow is from that version.", ) - @pytest.mark.production() + @pytest.mark.production_server() def test_get_flow_with_reinstantiate_strict_with_wrong_version_raises_exception(self): self.use_production_server() flow = 8175 @@ -359,7 +359,7 @@ def test_get_flow_with_reinstantiate_strict_with_wrong_version_raises_exception( # Because scikit-learn dropped min_impurity_split hyperparameter in 1.0, # and the requested flow is from 1.0.0 exactly. ) - @pytest.mark.production() + @pytest.mark.production_server() def test_get_flow_reinstantiate_flow_not_strict_post_1(self): self.use_production_server() flow = openml.flows.get_flow(flow_id=19190, reinstantiate=True, strict_version=False) @@ -373,7 +373,7 @@ def test_get_flow_reinstantiate_flow_not_strict_post_1(self): reason="Requires scikit-learn 0.23.2 or ~0.24.", # Because these still have min_impurity_split, but with new scikit-learn module structure." ) - @pytest.mark.production() + @pytest.mark.production_server() def test_get_flow_reinstantiate_flow_not_strict_023_and_024(self): self.use_production_server() flow = openml.flows.get_flow(flow_id=18587, reinstantiate=True, strict_version=False) @@ -385,7 +385,7 @@ def test_get_flow_reinstantiate_flow_not_strict_023_and_024(self): Version(sklearn.__version__) > Version("0.23"), reason="Requires scikit-learn<=0.23, because the scikit-learn module structure changed.", ) - @pytest.mark.production() + @pytest.mark.production_server() def test_get_flow_reinstantiate_flow_not_strict_pre_023(self): self.use_production_server() flow = openml.flows.get_flow(flow_id=8175, reinstantiate=True, strict_version=False) @@ -393,7 +393,7 @@ def test_get_flow_reinstantiate_flow_not_strict_pre_023(self): assert "sklearn==0.19.1" not in flow.dependencies @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_flow_id(self): if self.long_version: list_all = openml.utils._list_all @@ -428,7 +428,7 @@ def test_get_flow_id(self): pytest.skip(reason="Not sure why there should only be one version of this flow.") assert flow_ids_exact_version_True == flow_ids_exact_version_False - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_delete_flow(self): flow = openml.OpenMLFlow( name="sklearn.dummy.DummyClassifier", diff --git a/tests/test_openml/test_api_calls.py b/tests/test_openml/test_api_calls.py index a295259ef..c8d5be25b 100644 --- a/tests/test_openml/test_api_calls.py +++ b/tests/test_openml/test_api_calls.py @@ -15,14 +15,14 @@ class TestConfig(openml.testing.TestBase): - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_too_long_uri(self): with pytest.raises(openml.exceptions.OpenMLServerError, match="URI too long!"): openml.datasets.list_datasets(data_id=list(range(10000))) @unittest.mock.patch("time.sleep") @unittest.mock.patch("requests.Session") - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_retry_on_database_error(self, Session_class_mock, _): response_mock = unittest.mock.Mock() response_mock.text = ( @@ -117,7 +117,7 @@ def test_download_minio_failure(mock_minio, tmp_path: Path) -> None: ("task/42", "delete"), # 460 ], ) -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_authentication_endpoints_requiring_api_key_show_relevant_help_link( endpoint: str, method: str, diff --git a/tests/test_openml/test_config.py b/tests/test_openml/test_config.py index c5ddc4ecc..fc7221716 100644 --- a/tests/test_openml/test_config.py +++ b/tests/test_openml/test_config.py @@ -106,7 +106,7 @@ def test_setup_with_config(self): class TestConfigurationForExamples(openml.testing.TestBase): - @pytest.mark.production() + @pytest.mark.production_server() def test_switch_to_example_configuration(self): """Verifies the test configuration is loaded properly.""" # Below is the default test key which would be used anyway, but just for clarity: @@ -118,7 +118,7 @@ def test_switch_to_example_configuration(self): assert openml.config.apikey == TestBase.user_key assert openml.config.server == self.test_server - @pytest.mark.production() + @pytest.mark.production_server() def test_switch_from_example_configuration(self): """Verifies the previous configuration is loaded after stopping.""" # Below is the default test key which would be used anyway, but just for clarity: @@ -143,7 +143,7 @@ def test_example_configuration_stop_before_start(self): openml.config.stop_using_configuration_for_example, ) - @pytest.mark.production() + @pytest.mark.production_server() def test_example_configuration_start_twice(self): """Checks that the original config can be returned to if `start..` is called twice.""" openml.config.apikey = TestBase.user_key diff --git a/tests/test_runs/test_run.py b/tests/test_runs/test_run.py index 1a66b76c0..17349fca8 100644 --- a/tests/test_runs/test_run.py +++ b/tests/test_runs/test_run.py @@ -25,7 +25,7 @@ class TestRun(TestBase): # Splitting not helpful, these test's don't rely on the server and take # less than 1 seconds - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_tagging(self): runs = openml.runs.list_runs(size=1) assert not runs.empty, "Test server state is incorrect" @@ -119,7 +119,7 @@ def _check_array(array, type_): assert run_prime_trace_content is None @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_to_from_filesystem_vanilla(self): model = Pipeline( [ @@ -155,7 +155,7 @@ def test_to_from_filesystem_vanilla(self): @pytest.mark.sklearn() @pytest.mark.flaky() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_to_from_filesystem_search(self): model = Pipeline( [ @@ -190,7 +190,7 @@ def test_to_from_filesystem_search(self): ) @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_to_from_filesystem_no_model(self): model = Pipeline( [("imputer", SimpleImputer(strategy="mean")), ("classifier", DummyClassifier())], @@ -296,7 +296,7 @@ def assert_run_prediction_data(task, run, model): assert_method(y_test, saved_y_test) @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_publish_with_local_loaded_flow(self): """ Publish a run tied to a local flow after it has first been saved to @@ -340,7 +340,7 @@ def test_publish_with_local_loaded_flow(self): openml.runs.get_run(loaded_run.run_id) @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_offline_and_online_run_identical(self): extension = SklearnExtension() diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 8f2c505b7..e29558314 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -398,7 +398,7 @@ def _check_sample_evaluations( assert evaluation < max_time_allowed @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_run_regression_on_classif_task(self): task_id = 259 # collins; crossvalidation; has numeric targets @@ -415,7 +415,7 @@ def test_run_regression_on_classif_task(self): ) @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_check_erronous_sklearn_flow_fails(self): task_id = 115 # diabetes; crossvalidation task = openml.tasks.get_task(task_id) @@ -628,7 +628,7 @@ def _run_and_upload_regression( ) @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_run_and_upload_logistic_regression(self): lr = LogisticRegression(solver="lbfgs", max_iter=1000) task_id = self.TEST_SERVER_TASK_SIMPLE["task_id"] @@ -637,7 +637,7 @@ def test_run_and_upload_logistic_regression(self): self._run_and_upload_classification(lr, task_id, n_missing_vals, n_test_obs, "62501") @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_run_and_upload_linear_regression(self): lr = LinearRegression() task_id = self.TEST_SERVER_TASK_REGRESSION["task_id"] @@ -668,7 +668,7 @@ def test_run_and_upload_linear_regression(self): self._run_and_upload_regression(lr, task_id, n_missing_vals, n_test_obs, "62501") @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_run_and_upload_pipeline_dummy_pipeline(self): pipeline1 = Pipeline( steps=[ @@ -686,7 +686,7 @@ def test_run_and_upload_pipeline_dummy_pipeline(self): Version(sklearn.__version__) < Version("0.20"), reason="columntransformer introduction in 0.20.0", ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_run_and_upload_column_transformer_pipeline(self): import sklearn.compose import sklearn.impute @@ -799,7 +799,7 @@ def test_run_and_upload_knn_pipeline(self, warnings_mock): assert call_count == 3 @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_run_and_upload_gridsearch(self): estimator_name = ( "base_estimator" if Version(sklearn.__version__) < Version("1.4") else "estimator" @@ -822,7 +822,7 @@ def test_run_and_upload_gridsearch(self): assert len(run.trace.trace_iterations) == 9 @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_run_and_upload_randomsearch(self): randomsearch = RandomizedSearchCV( RandomForestClassifier(n_estimators=5), @@ -855,7 +855,7 @@ def test_run_and_upload_randomsearch(self): assert len(trace.trace_iterations) == 5 @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_run_and_upload_maskedarrays(self): # This testcase is important for 2 reasons: # 1) it verifies the correct handling of masked arrays (not all @@ -883,7 +883,7 @@ def test_run_and_upload_maskedarrays(self): ########################################################################## @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_learning_curve_task_1(self): task_id = 801 # diabates dataset num_test_instances = 6144 # for learning curve @@ -908,7 +908,7 @@ def test_learning_curve_task_1(self): self._check_sample_evaluations(run.sample_evaluations, num_repeats, num_folds, num_samples) @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_learning_curve_task_2(self): task_id = 801 # diabates dataset num_test_instances = 6144 # for learning curve @@ -949,7 +949,7 @@ def test_learning_curve_task_2(self): Version(sklearn.__version__) < Version("0.21"), reason="Pipelines don't support indexing (used for the assert check)", ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_initialize_cv_from_run(self): randomsearch = Pipeline( [ @@ -1024,7 +1024,7 @@ def _test_local_evaluations(self, run): assert alt_scores[idx] <= 1 @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_local_run_swapped_parameter_order_model(self): clf = DecisionTreeClassifier() australian_task = 595 # Australian; crossvalidation @@ -1044,7 +1044,7 @@ def test_local_run_swapped_parameter_order_model(self): Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_local_run_swapped_parameter_order_flow(self): # construct sci-kit learn classifier clf = Pipeline( @@ -1073,7 +1073,7 @@ def test_local_run_swapped_parameter_order_flow(self): Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_local_run_metric_score(self): # construct sci-kit learn classifier clf = Pipeline( @@ -1096,7 +1096,7 @@ def test_local_run_metric_score(self): self._test_local_evaluations(run) - @pytest.mark.production() + @pytest.mark.production_server() def test_online_run_metric_score(self): self.use_production_server() @@ -1111,7 +1111,7 @@ def test_online_run_metric_score(self): Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_initialize_model_from_run(self): clf = sklearn.pipeline.Pipeline( steps=[ @@ -1173,7 +1173,7 @@ def test_initialize_model_from_run(self): Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test__run_exists(self): # would be better to not sentinel these clfs, # so we do not have to perform the actual runs @@ -1229,7 +1229,7 @@ def test__run_exists(self): assert run_ids, (run_ids, clf) @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_run_with_illegal_flow_id(self): # check the case where the user adds an illegal flow id to a # non-existing flo @@ -1249,7 +1249,7 @@ def test_run_with_illegal_flow_id(self): ) @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_run_with_illegal_flow_id_after_load(self): # Same as `test_run_with_illegal_flow_id`, but test this error is also # caught if the run is stored to and loaded from disk first. @@ -1281,7 +1281,7 @@ def test_run_with_illegal_flow_id_after_load(self): TestBase.logger.info(f"collected from test_run_functions: {loaded_run.run_id}") @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_run_with_illegal_flow_id_1(self): # Check the case where the user adds an illegal flow id to an existing # flow. Comes to a different value error than the previous test @@ -1307,7 +1307,7 @@ def test_run_with_illegal_flow_id_1(self): ) @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_run_with_illegal_flow_id_1_after_load(self): # Same as `test_run_with_illegal_flow_id_1`, but test this error is # also caught if the run is stored to and loaded from disk first. @@ -1350,7 +1350,7 @@ def test_run_with_illegal_flow_id_1_after_load(self): Version(sklearn.__version__) < Version("0.20"), reason="OneHotEncoder cannot handle mixed type DataFrame as input", ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test__run_task_get_arffcontent(self): task = openml.tasks.get_task(7) # kr-vs-kp; crossvalidation num_instances = 3196 @@ -1407,7 +1407,7 @@ def test__create_trace_from_arff(self): trace_arff = arff.load(arff_file) OpenMLRunTrace.trace_from_arff(trace_arff) - @pytest.mark.production() + @pytest.mark.production_server() def test_get_run(self): # this run is not available on test self.use_production_server() @@ -1442,7 +1442,7 @@ def _check_run(self, run): assert isinstance(run, dict) assert len(run) == 8, str(run) - @pytest.mark.production() + @pytest.mark.production_server() def test_get_runs_list(self): # TODO: comes from live, no such lists on test self.use_production_server() @@ -1451,12 +1451,12 @@ def test_get_runs_list(self): for run in runs.to_dict(orient="index").values(): self._check_run(run) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_list_runs_empty(self): runs = openml.runs.list_runs(task=[0]) assert runs.empty - @pytest.mark.production() + @pytest.mark.production_server() def test_get_runs_list_by_task(self): # TODO: comes from live, no such lists on test self.use_production_server() @@ -1475,7 +1475,7 @@ def test_get_runs_list_by_task(self): assert run["task_id"] in task_ids self._check_run(run) - @pytest.mark.production() + @pytest.mark.production_server() def test_get_runs_list_by_uploader(self): # TODO: comes from live, no such lists on test self.use_production_server() @@ -1497,7 +1497,7 @@ def test_get_runs_list_by_uploader(self): assert run["uploader"] in uploader_ids self._check_run(run) - @pytest.mark.production() + @pytest.mark.production_server() def test_get_runs_list_by_flow(self): # TODO: comes from live, no such lists on test self.use_production_server() @@ -1516,7 +1516,7 @@ def test_get_runs_list_by_flow(self): assert run["flow_id"] in flow_ids self._check_run(run) - @pytest.mark.production() + @pytest.mark.production_server() def test_get_runs_pagination(self): # TODO: comes from live, no such lists on test self.use_production_server() @@ -1529,7 +1529,7 @@ def test_get_runs_pagination(self): for run in runs.to_dict(orient="index").values(): assert run["uploader"] in uploader_ids - @pytest.mark.production() + @pytest.mark.production_server() def test_get_runs_list_by_filters(self): # TODO: comes from live, no such lists on test self.use_production_server() @@ -1566,7 +1566,7 @@ def test_get_runs_list_by_filters(self): ) assert len(runs) == 2 - @pytest.mark.production() + @pytest.mark.production_server() @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_get_runs_list_by_tag(self): # We don't have tagged runs on the test server @@ -1580,7 +1580,7 @@ def test_get_runs_list_by_tag(self): Version(sklearn.__version__) < Version("0.20"), reason="columntransformer introduction in 0.20.0", ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_run_on_dataset_with_missing_labels_dataframe(self): # Check that _run_task_get_arffcontent works when one of the class # labels only declared in the arff file, but is not present in the @@ -1617,7 +1617,7 @@ def test_run_on_dataset_with_missing_labels_dataframe(self): Version(sklearn.__version__) < Version("0.20"), reason="columntransformer introduction in 0.20.0", ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_run_on_dataset_with_missing_labels_array(self): # Check that _run_task_get_arffcontent works when one of the class # labels only declared in the arff file, but is not present in the @@ -1656,7 +1656,7 @@ def test_run_on_dataset_with_missing_labels_array(self): # repeat, fold, row_id, 6 confidences, prediction and correct label assert len(row) == 12 - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_cached_run(self): openml.config.set_root_cache_directory(self.static_cache_dir) openml.runs.functions._get_cached_run(1) @@ -1667,7 +1667,7 @@ def test_get_uncached_run(self): openml.runs.functions._get_cached_run(10) @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_run_flow_on_task_downloaded_flow(self): model = sklearn.ensemble.RandomForestClassifier(n_estimators=33) flow = self.extension.model_to_flow(model) @@ -1687,7 +1687,7 @@ def test_run_flow_on_task_downloaded_flow(self): TestBase._mark_entity_for_removal("run", run.run_id) TestBase.logger.info(f"collected from {__file__.split('/')[-1]}: {run.run_id}") - @pytest.mark.production() + @pytest.mark.production_server() def test_format_prediction_non_supervised(self): # non-supervised tasks don't exist on the test server self.use_production_server() @@ -1698,7 +1698,7 @@ def test_format_prediction_non_supervised(self): ): format_prediction(clustering, *ignored_input) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_format_prediction_classification_no_probabilities(self): classification = openml.tasks.get_task( self.TEST_SERVER_TASK_SIMPLE["task_id"], @@ -1708,7 +1708,7 @@ def test_format_prediction_classification_no_probabilities(self): with pytest.raises(ValueError, match="`proba` is required for classification task"): format_prediction(classification, *ignored_input, proba=None) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_format_prediction_classification_incomplete_probabilities(self): classification = openml.tasks.get_task( self.TEST_SERVER_TASK_SIMPLE["task_id"], @@ -1719,7 +1719,7 @@ def test_format_prediction_classification_incomplete_probabilities(self): with pytest.raises(ValueError, match="Each class should have a predicted probability"): format_prediction(classification, *ignored_input, proba=incomplete_probabilities) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_format_prediction_task_without_classlabels_set(self): classification = openml.tasks.get_task( self.TEST_SERVER_TASK_SIMPLE["task_id"], @@ -1730,7 +1730,7 @@ def test_format_prediction_task_without_classlabels_set(self): with pytest.raises(ValueError, match="The classification task must have class labels set"): format_prediction(classification, *ignored_input, proba={}) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_format_prediction_task_learning_curve_sample_not_set(self): learning_curve = openml.tasks.get_task(801, download_data=False) # diabetes;crossvalidation probabilities = {c: 0.2 for c in learning_curve.class_labels} @@ -1738,7 +1738,7 @@ def test_format_prediction_task_learning_curve_sample_not_set(self): with pytest.raises(ValueError, match="`sample` can not be none for LearningCurveTask"): format_prediction(learning_curve, *ignored_input, sample=None, proba=probabilities) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_format_prediction_task_regression(self): task_meta_data = self.TEST_SERVER_TASK_REGRESSION["task_meta_data"] _task_id = check_task_existence(**task_meta_data) @@ -1773,7 +1773,7 @@ def test_format_prediction_task_regression(self): reason="SimpleImputer doesn't handle mixed type DataFrame as input", ) @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_delete_run(self): rs = np.random.randint(1, 2**31 - 1) clf = sklearn.pipeline.Pipeline( @@ -1874,7 +1874,7 @@ def test_delete_unknown_run(mock_delete, test_files_directory, test_api_key): reason="couldn't perform local tests successfully w/o bloating RAM", ) @mock.patch("openml_sklearn.SklearnExtension._prevent_optimize_n_jobs") -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test__run_task_get_arffcontent_2(parallel_mock): """Tests if a run executed in parallel is collated correctly.""" task = openml.tasks.get_task(7) # Supervised Classification on kr-vs-kp @@ -1965,7 +1965,7 @@ def test__run_task_get_arffcontent_2(parallel_mock): (-1, "threading", 10), # the threading backend does preserve mocks even with parallelizing ] ) -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_joblib_backends(parallel_mock, n_jobs, backend, call_count): """Tests evaluation of a run using various joblib backends and n_jobs.""" if backend is None: diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index a0469f9a5..0df3a0b3b 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -35,7 +35,7 @@ def setUp(self): super().setUp() @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_nonexisting_setup_exists(self): # first publish a non-existing flow sentinel = get_sentinel() @@ -83,7 +83,7 @@ def _existing_setup_exists(self, classif): assert setup_id == run.setup_id @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_existing_setup_exists_1(self): def side_effect(self): self.var_smoothing = 1e-9 @@ -99,13 +99,13 @@ def side_effect(self): self._existing_setup_exists(nb) @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_exisiting_setup_exists_2(self): # Check a flow with one hyperparameter self._existing_setup_exists(sklearn.naive_bayes.GaussianNB()) @pytest.mark.sklearn() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_existing_setup_exists_3(self): # Check a flow with many hyperparameters self._existing_setup_exists( @@ -118,7 +118,7 @@ def test_existing_setup_exists_3(self): ), ) - @pytest.mark.production() + @pytest.mark.production_server() def test_get_setup(self): self.use_production_server() # no setups in default test server @@ -135,7 +135,7 @@ def test_get_setup(self): else: assert len(current.parameters) == num_params[idx] - @pytest.mark.production() + @pytest.mark.production_server() def test_setup_list_filter_flow(self): self.use_production_server() @@ -147,7 +147,7 @@ def test_setup_list_filter_flow(self): for setup_id in setups: assert setups[setup_id].flow_id == flow_id - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_list_setups_empty(self): setups = openml.setups.list_setups(setup=[0]) if len(setups) > 0: @@ -155,7 +155,7 @@ def test_list_setups_empty(self): assert isinstance(setups, dict) - @pytest.mark.production() + @pytest.mark.production_server() def test_list_setups_output_format(self): self.use_production_server() flow_id = 6794 @@ -168,7 +168,7 @@ def test_list_setups_output_format(self): assert isinstance(setups, pd.DataFrame) assert len(setups) == 10 - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_setuplist_offset(self): size = 10 setups = openml.setups.list_setups(offset=0, size=size) @@ -180,7 +180,7 @@ def test_setuplist_offset(self): assert len(all) == size * 2 - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_cached_setup(self): openml.config.set_root_cache_directory(self.static_cache_dir) openml.setups.functions._get_cached_setup(1) diff --git a/tests/test_study/test_study_functions.py b/tests/test_study/test_study_functions.py index 4b662524b..2a2d276ec 100644 --- a/tests/test_study/test_study_functions.py +++ b/tests/test_study/test_study_functions.py @@ -12,7 +12,7 @@ class TestStudyFunctions(TestBase): _multiprocess_can_split_ = True - @pytest.mark.production() + @pytest.mark.production_server() @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_get_study_old(self): self.use_production_server() @@ -24,7 +24,7 @@ def test_get_study_old(self): assert len(study.setups) == 30 assert study.runs is None - @pytest.mark.production() + @pytest.mark.production_server() def test_get_study_new(self): self.use_production_server() @@ -35,7 +35,7 @@ def test_get_study_new(self): assert len(study.setups) == 1253 assert len(study.runs) == 1693 - @pytest.mark.production() + @pytest.mark.production_server() def test_get_openml100(self): self.use_production_server() @@ -45,7 +45,7 @@ def test_get_openml100(self): assert isinstance(study_2, openml.study.OpenMLBenchmarkSuite) assert study.study_id == study_2.study_id - @pytest.mark.production() + @pytest.mark.production_server() def test_get_study_error(self): self.use_production_server() @@ -54,7 +54,7 @@ def test_get_study_error(self): ): openml.study.get_study(99) - @pytest.mark.production() + @pytest.mark.production_server() def test_get_suite(self): self.use_production_server() @@ -65,7 +65,7 @@ def test_get_suite(self): assert study.runs is None assert study.setups is None - @pytest.mark.production() + @pytest.mark.production_server() def test_get_suite_error(self): self.use_production_server() @@ -74,7 +74,7 @@ def test_get_suite_error(self): ): openml.study.get_suite(123) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_publish_benchmark_suite(self): fixture_alias = None fixture_name = "unit tested benchmark suite" @@ -143,16 +143,16 @@ def _test_publish_empty_study_is_allowed(self, explicit: bool): assert study_downloaded.main_entity_type == "run" assert study_downloaded.runs is None - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_publish_empty_study_explicit(self): self._test_publish_empty_study_is_allowed(explicit=True) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_publish_empty_study_implicit(self): self._test_publish_empty_study_is_allowed(explicit=False) @pytest.mark.flaky() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_publish_study(self): # get some random runs to attach run_list = openml.evaluations.list_evaluations("predictive_accuracy", size=10) @@ -222,7 +222,7 @@ def test_publish_study(self): res = openml.study.delete_study(study.id) assert res - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_study_attach_illegal(self): run_list = openml.runs.list_runs(size=10) assert len(run_list) == 10 diff --git a/tests/test_tasks/test_classification_task.py b/tests/test_tasks/test_classification_task.py index fed0c0a00..65dcebc1d 100644 --- a/tests/test_tasks/test_classification_task.py +++ b/tests/test_tasks/test_classification_task.py @@ -18,7 +18,7 @@ def setUp(self, n_levels: int = 1): self.task_type = TaskType.SUPERVISED_CLASSIFICATION self.estimation_procedure = 5 - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_download_task(self): task = super().test_download_task() assert task.task_id == self.task_id @@ -26,13 +26,13 @@ def test_download_task(self): assert task.dataset_id == 20 assert task.estimation_procedure_id == self.estimation_procedure - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_class_labels(self): task = get_task(self.task_id) assert task.class_labels == ["tested_negative", "tested_positive"] -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_get_X_and_Y(): task = get_task(119) X, Y = task.get_X_and_y() diff --git a/tests/test_tasks/test_clustering_task.py b/tests/test_tasks/test_clustering_task.py index 2bbb015c6..29f5663c4 100644 --- a/tests/test_tasks/test_clustering_task.py +++ b/tests/test_tasks/test_clustering_task.py @@ -20,15 +20,15 @@ def setUp(self, n_levels: int = 1): self.task_type = TaskType.CLUSTERING self.estimation_procedure = 17 - @pytest.mark.production() + @pytest.mark.production_server() def test_get_dataset(self): # no clustering tasks on test server self.use_production_server() task = openml.tasks.get_task(self.task_id) task.get_dataset() - @pytest.mark.production() - @pytest.mark.uses_test_server() + @pytest.mark.production_server() + @pytest.mark.test_server() def test_download_task(self): # no clustering tasks on test server self.use_production_server() @@ -37,7 +37,7 @@ def test_download_task(self): assert task.task_type_id == TaskType.CLUSTERING assert task.dataset_id == 36 - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_upload_task(self): compatible_datasets = self._get_compatible_rand_dataset() for i in range(100): diff --git a/tests/test_tasks/test_learning_curve_task.py b/tests/test_tasks/test_learning_curve_task.py index fbcbfe9bf..465d9c0be 100644 --- a/tests/test_tasks/test_learning_curve_task.py +++ b/tests/test_tasks/test_learning_curve_task.py @@ -18,7 +18,7 @@ def setUp(self, n_levels: int = 1): self.task_type = TaskType.LEARNING_CURVE self.estimation_procedure = 13 - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_X_and_Y(self): X, Y = super().test_get_X_and_Y() assert X.shape == (768, 8) @@ -27,14 +27,14 @@ def test_get_X_and_Y(self): assert isinstance(Y, pd.Series) assert pd.api.types.is_categorical_dtype(Y) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_download_task(self): task = super().test_download_task() assert task.task_id == self.task_id assert task.task_type_id == TaskType.LEARNING_CURVE assert task.dataset_id == 20 - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_class_labels(self): task = get_task(self.task_id) assert task.class_labels == ["tested_negative", "tested_positive"] diff --git a/tests/test_tasks/test_regression_task.py b/tests/test_tasks/test_regression_task.py index a834cdf0f..26d7dc94b 100644 --- a/tests/test_tasks/test_regression_task.py +++ b/tests/test_tasks/test_regression_task.py @@ -49,7 +49,7 @@ def setUp(self, n_levels: int = 1): self.task_type = TaskType.SUPERVISED_REGRESSION - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_X_and_Y(self): X, Y = super().test_get_X_and_Y() assert X.shape == (194, 32) @@ -58,7 +58,7 @@ def test_get_X_and_Y(self): assert isinstance(Y, pd.Series) assert pd.api.types.is_numeric_dtype(Y) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_download_task(self): task = super().test_download_task() assert task.task_id == self.task_id diff --git a/tests/test_tasks/test_supervised_task.py b/tests/test_tasks/test_supervised_task.py index 3f7b06ee4..99df3cace 100644 --- a/tests/test_tasks/test_supervised_task.py +++ b/tests/test_tasks/test_supervised_task.py @@ -28,7 +28,7 @@ def setUpClass(cls): def setUp(self, n_levels: int = 1): super().setUp() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_X_and_Y(self) -> tuple[pd.DataFrame, pd.Series]: task = get_task(self.task_id) X, Y = task.get_X_and_y() diff --git a/tests/test_tasks/test_task.py b/tests/test_tasks/test_task.py index b77782847..1d0df1210 100644 --- a/tests/test_tasks/test_task.py +++ b/tests/test_tasks/test_task.py @@ -32,11 +32,11 @@ def setUpClass(cls): def setUp(self, n_levels: int = 1): super().setUp() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_download_task(self): return get_task(self.task_id) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_upload_task(self): # We don't know if the task in question already exists, so we try a few times. Checking # beforehand would not be an option because a concurrent unit test could potentially diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py index d44717177..da1f24cdc 100644 --- a/tests/test_tasks/test_task_functions.py +++ b/tests/test_tasks/test_task_functions.py @@ -26,7 +26,7 @@ def setUp(self): def tearDown(self): super().tearDown() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test__get_cached_tasks(self): openml.config.set_root_cache_directory(self.static_cache_dir) tasks = openml.tasks.functions._get_cached_tasks() @@ -34,7 +34,7 @@ def test__get_cached_tasks(self): assert len(tasks) == 3 assert isinstance(next(iter(tasks.values())), OpenMLTask) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test__get_cached_task(self): openml.config.set_root_cache_directory(self.static_cache_dir) task = openml.tasks.functions._get_cached_task(1) @@ -49,14 +49,14 @@ def test__get_cached_task_not_cached(self): 2, ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test__get_estimation_procedure_list(self): estimation_procedures = openml.tasks.functions._get_estimation_procedure_list() assert isinstance(estimation_procedures, list) assert isinstance(estimation_procedures[0], dict) assert estimation_procedures[0]["task_type_id"] == TaskType.SUPERVISED_CLASSIFICATION - @pytest.mark.production() + @pytest.mark.production_server() @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_list_clustering_task(self): self.use_production_server() @@ -73,7 +73,7 @@ def _check_task(self, task): assert isinstance(task["status"], str) assert task["status"] in ["in_preparation", "active", "deactivated"] - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_list_tasks_by_type(self): num_curves_tasks = 198 # number is flexible, check server if fails ttid = TaskType.LEARNING_CURVE @@ -83,18 +83,18 @@ def test_list_tasks_by_type(self): assert ttid == task["ttid"] self._check_task(task) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_list_tasks_length(self): ttid = TaskType.LEARNING_CURVE tasks = openml.tasks.list_tasks(task_type=ttid) assert len(tasks) > 100 - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_list_tasks_empty(self): tasks = openml.tasks.list_tasks(tag="NoOneWillEverUseThisTag") assert tasks.empty - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_list_tasks_by_tag(self): num_basic_tasks = 100 # number is flexible, check server if fails tasks = openml.tasks.list_tasks(tag="OpenML100") @@ -102,14 +102,14 @@ def test_list_tasks_by_tag(self): for task in tasks.to_dict(orient="index").values(): self._check_task(task) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_list_tasks(self): tasks = openml.tasks.list_tasks() assert len(tasks) >= 900 for task in tasks.to_dict(orient="index").values(): self._check_task(task) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_list_tasks_paginate(self): size = 10 max = 100 @@ -119,7 +119,7 @@ def test_list_tasks_paginate(self): for task in tasks.to_dict(orient="index").values(): self._check_task(task) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_list_tasks_per_type_paginate(self): size = 40 max = 100 @@ -136,7 +136,7 @@ def test_list_tasks_per_type_paginate(self): assert j == task["ttid"] self._check_task(task) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test__get_task(self): openml.config.set_root_cache_directory(self.static_cache_dir) openml.tasks.get_task(1882) @@ -144,14 +144,14 @@ def test__get_task(self): @unittest.skip( "Please await outcome of discussion: https://github.com/openml/OpenML/issues/776", ) - @pytest.mark.production() + @pytest.mark.production_server() def test__get_task_live(self): self.use_production_server() # Test the following task as it used to throw an Unicode Error. # https://github.com/openml/openml-python/issues/378 openml.tasks.get_task(34536) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_task(self): task = openml.tasks.get_task(1, download_data=True) # anneal; crossvalidation assert isinstance(task, OpenMLTask) @@ -165,7 +165,7 @@ def test_get_task(self): os.path.join(self.workdir, "org", "openml", "test", "datasets", "1", "dataset.arff") ) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_task_lazy(self): task = openml.tasks.get_task(2, download_data=False) # anneal; crossvalidation assert isinstance(task, OpenMLTask) @@ -188,7 +188,7 @@ def test_get_task_lazy(self): ) @mock.patch("openml.tasks.functions.get_dataset") - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_removal_upon_download_failure(self, get_dataset): class WeirdException(Exception): pass @@ -206,13 +206,13 @@ def assert_and_raise(*args, **kwargs): # Now the file should no longer exist assert not os.path.exists(os.path.join(os.getcwd(), "tasks", "1", "tasks.xml")) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_task_with_cache(self): openml.config.set_root_cache_directory(self.static_cache_dir) task = openml.tasks.get_task(1) assert isinstance(task, OpenMLTask) - @pytest.mark.production() + @pytest.mark.production_server() def test_get_task_different_types(self): self.use_production_server() # Regression task @@ -222,7 +222,7 @@ def test_get_task_different_types(self): # Issue 538, get_task failing with clustering task. openml.tasks.functions.get_task(126033) - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_download_split(self): task = openml.tasks.get_task(1) # anneal; crossvalidation split = task.download_split() diff --git a/tests/test_tasks/test_task_methods.py b/tests/test_tasks/test_task_methods.py index 6b8804b9f..9316d0876 100644 --- a/tests/test_tasks/test_task_methods.py +++ b/tests/test_tasks/test_task_methods.py @@ -16,7 +16,7 @@ def setUp(self): def tearDown(self): super().tearDown() - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_tagging(self): task = openml.tasks.get_task(1) # anneal; crossvalidation # tags can be at most 64 alphanumeric (+ underscore) chars @@ -32,7 +32,7 @@ def test_tagging(self): tasks = openml.tasks.list_tasks(tag=tag) assert len(tasks) == 0 - @pytest.mark.uses_test_server() + @pytest.mark.test_server() def test_get_train_and_test_split_indices(self): openml.config.set_root_cache_directory(self.static_cache_dir) task = openml.tasks.get_task(1882) diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index 8dbdd30b5..38e004bfb 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -48,18 +48,18 @@ def _mocked_perform_api_call(call, request_method): return openml._api_calls._download_text_file(url) -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_list_all(): openml.utils._list_all(listing_call=openml.tasks.functions._list_tasks) -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_list_all_for_tasks(min_number_tasks_on_test_server): tasks = openml.tasks.list_tasks(size=min_number_tasks_on_test_server) assert min_number_tasks_on_test_server == len(tasks) -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_list_all_with_multiple_batches(min_number_tasks_on_test_server): # By setting the batch size one lower than the minimum we guarantee at least two # batches and at the same time do as few batches (roundtrips) as possible. @@ -72,7 +72,7 @@ def test_list_all_with_multiple_batches(min_number_tasks_on_test_server): assert min_number_tasks_on_test_server <= sum(len(batch) for batch in batches) -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_list_all_for_datasets(min_number_datasets_on_test_server): datasets = openml.datasets.list_datasets( size=min_number_datasets_on_test_server, @@ -83,14 +83,14 @@ def test_list_all_for_datasets(min_number_datasets_on_test_server): _check_dataset(dataset) -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_list_all_for_flows(min_number_flows_on_test_server): flows = openml.flows.list_flows(size=min_number_flows_on_test_server) assert min_number_flows_on_test_server == len(flows) @pytest.mark.flaky() # Other tests might need to upload runs first -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_list_all_for_setups(min_number_setups_on_test_server): # TODO apparently list_setups function does not support kwargs setups = openml.setups.list_setups(size=min_number_setups_on_test_server) @@ -98,14 +98,14 @@ def test_list_all_for_setups(min_number_setups_on_test_server): @pytest.mark.flaky() # Other tests might need to upload runs first -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_list_all_for_runs(min_number_runs_on_test_server): runs = openml.runs.list_runs(size=min_number_runs_on_test_server) assert min_number_runs_on_test_server == len(runs) @pytest.mark.flaky() # Other tests might need to upload runs first -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_list_all_for_evaluations(min_number_evaluations_on_test_server): # TODO apparently list_evaluations function does not support kwargs evaluations = openml.evaluations.list_evaluations( @@ -116,7 +116,7 @@ def test_list_all_for_evaluations(min_number_evaluations_on_test_server): @unittest.mock.patch("openml._api_calls._perform_api_call", side_effect=_mocked_perform_api_call) -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_list_all_few_results_available(_perform_api_call): datasets = openml.datasets.list_datasets(size=1000, data_name="iris", data_version=1) assert len(datasets) == 1, "only one iris dataset version 1 should be present" @@ -141,7 +141,7 @@ def test__create_cache_directory(config_mock, tmp_path): openml.utils._create_cache_directory("ghi") -@pytest.mark.uses_test_server() +@pytest.mark.test_server() def test_correct_test_server_download_state(): """This test verifies that the test server downloads the data from the correct source. From ede1497dfedd7fd419b347a35fe90e9b9fb52ffd Mon Sep 17 00:00:00 2001 From: Pieter Gijsbers Date: Thu, 19 Feb 2026 10:16:57 +0100 Subject: [PATCH 37/46] [ENH] Allow using a local test server (#1630) Update the tests to allow connecting to a local test server instead of a remote one (requires https://github.com/openml/services/pull/13). Running the tests locally: - Locally start the services (as defined in https://github.com/openml/services/pull/13) using `docker compose --profile "rest-api" --profile "evaluation-engine" up -d`. Startup can take a few minutes, as currently the PHP container still builds the ES indices from scratch. I noticed that the `start_period` for some services isn't sufficient on my M1 Mac, possibly due to some containers requiring Rosetta to run, slowing things down. You can recognize this by the services reporting "Error" while the container remains running. To avoid this, you can either increase the `start_period` of the services (mostly elastic search and php api), or you can simply run the command again (the services are then already in healthy state and the services that depended on it can start successfully). The following containers should run: openml-test-database, openml-php-rest-api, openml-nginx, openml-evaluation-engine, openml-elasticsearch, openml-minio - Update the `openml/config.py`'s `TEST_SERVER_URL` variable to `"http://localhost:8000"`. - Run the tests (`python -m pytest -m "not production" tests`). This PR builds off unmerged PR https://github.com/openml/openml-python/pull/1620. --------- Co-authored-by: Armaghan Shakir --- openml/cli.py | 2 +- openml/config.py | 7 +++- openml/tasks/functions.py | 11 +++--- openml/testing.py | 2 +- tests/conftest.py | 2 +- tests/files/localhost_8000 | 1 + tests/test_datasets/test_dataset_functions.py | 37 +++++++------------ tests/test_flows/test_flow_functions.py | 15 +++----- tests/test_openml/test_config.py | 2 +- tests/test_runs/test_run_functions.py | 13 ++++--- tests/test_tasks/test_task_functions.py | 32 ++++++++-------- 11 files changed, 56 insertions(+), 68 deletions(-) create mode 120000 tests/files/localhost_8000 diff --git a/openml/cli.py b/openml/cli.py index cbcc38f4a..c33578f6e 100644 --- a/openml/cli.py +++ b/openml/cli.py @@ -109,7 +109,7 @@ def check_server(server: str) -> str: def replace_shorthand(server: str) -> str: if server == "test": - return "https://test.openml.org/api/v1/xml" + return f"{config.TEST_SERVER_URL}/api/v1/xml" if server == "production_server": return "https://www.openml.org/api/v1/xml" return server diff --git a/openml/config.py b/openml/config.py index 9758b6fff..638b45650 100644 --- a/openml/config.py +++ b/openml/config.py @@ -28,6 +28,8 @@ OPENML_TEST_SERVER_ADMIN_KEY_ENV_VAR = "OPENML_TEST_SERVER_ADMIN_KEY" _TEST_SERVER_NORMAL_USER_KEY = "normaluser" +TEST_SERVER_URL = "https://test.openml.org" + class _Config(TypedDict): apikey: str @@ -214,7 +216,7 @@ class ConfigurationForExamples: _last_used_server = None _last_used_key = None _start_last_called = False - _test_server = "https://test.openml.org/api/v1/xml" + _test_server = f"{TEST_SERVER_URL}/api/v1/xml" _test_apikey = _TEST_SERVER_NORMAL_USER_KEY @classmethod @@ -470,7 +472,8 @@ def get_cache_directory() -> str: """ url_suffix = urlparse(server).netloc - reversed_url_suffix = os.sep.join(url_suffix.split(".")[::-1]) # noqa: PTH118 + url_parts = url_suffix.replace(":", "_").split(".")[::-1] + reversed_url_suffix = os.sep.join(url_parts) # noqa: PTH118 return os.path.join(_root_cache_directory, reversed_url_suffix) # noqa: PTH118 diff --git a/openml/tasks/functions.py b/openml/tasks/functions.py index 3df2861c0..2bf1a40f4 100644 --- a/openml/tasks/functions.py +++ b/openml/tasks/functions.py @@ -415,9 +415,10 @@ def get_task( if not isinstance(task_id, int): raise TypeError(f"Task id should be integer, is {type(task_id)}") - cache_key_dir = openml.utils._create_cache_directory_for_id(TASKS_CACHE_DIR_NAME, task_id) - tid_cache_dir = cache_key_dir / str(task_id) - tid_cache_dir_existed = tid_cache_dir.exists() + task_cache_directory = openml.utils._create_cache_directory_for_id( + TASKS_CACHE_DIR_NAME, task_id + ) + task_cache_directory_existed = task_cache_directory.exists() try: task = _get_task_description(task_id) dataset = get_dataset(task.dataset_id, **get_dataset_kwargs) @@ -431,8 +432,8 @@ def get_task( if download_splits and isinstance(task, OpenMLSupervisedTask): task.download_split() except Exception as e: - if not tid_cache_dir_existed: - openml.utils._remove_cache_dir_for_id(TASKS_CACHE_DIR_NAME, tid_cache_dir) + if not task_cache_directory_existed: + openml.utils._remove_cache_dir_for_id(TASKS_CACHE_DIR_NAME, task_cache_directory) raise e return task diff --git a/openml/testing.py b/openml/testing.py index 304a4e0be..9f694f9bf 100644 --- a/openml/testing.py +++ b/openml/testing.py @@ -47,7 +47,7 @@ class TestBase(unittest.TestCase): "user": [], } flow_name_tracker: ClassVar[list[str]] = [] - test_server = "https://test.openml.org/api/v1/xml" + test_server = f"{openml.config.TEST_SERVER_URL}/api/v1/xml" admin_key = os.environ.get(openml.config.OPENML_TEST_SERVER_ADMIN_KEY_ENV_VAR) user_key = openml.config._TEST_SERVER_NORMAL_USER_KEY diff --git a/tests/conftest.py b/tests/conftest.py index 4fffa9f38..2a7a6dcc7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -277,7 +277,7 @@ def with_server(request): openml.config.apikey = None yield return - openml.config.server = "https://test.openml.org/api/v1/xml" + openml.config.server = f"{openml.config.TEST_SERVER_URL}/api/v1/xml" openml.config.apikey = TestBase.user_key yield diff --git a/tests/files/localhost_8000 b/tests/files/localhost_8000 new file mode 120000 index 000000000..334c709ef --- /dev/null +++ b/tests/files/localhost_8000 @@ -0,0 +1 @@ +org/openml/test \ No newline at end of file diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index 41e89d950..151a9ac23 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -527,19 +527,12 @@ def test_deletion_of_cache_dir(self): def test_deletion_of_cache_dir_faulty_download(self, patch): patch.side_effect = Exception("Boom!") self.assertRaisesRegex(Exception, "Boom!", openml.datasets.get_dataset, dataset_id=1) - datasets_cache_dir = os.path.join(self.workdir, "org", "openml", "test", "datasets") + datasets_cache_dir = os.path.join(openml.config.get_cache_directory(), "datasets") assert len(os.listdir(datasets_cache_dir)) == 0 @pytest.mark.test_server() def test_publish_dataset(self): - # lazy loading not possible as we need the arff-file. - openml.datasets.get_dataset(3, download_data=True) - file_path = os.path.join( - openml.config.get_cache_directory(), - "datasets", - "3", - "dataset.arff", - ) + arff_file_path = self.static_cache_dir / "org" / "openml" / "test" / "datasets" / "2" / "dataset.arff" dataset = OpenMLDataset( "anneal", "test", @@ -547,7 +540,7 @@ def test_publish_dataset(self): version=1, licence="public", default_target_attribute="class", - data_file=file_path, + data_file=arff_file_path, ) dataset.publish() TestBase._mark_entity_for_removal("data", dataset.dataset_id) @@ -890,7 +883,7 @@ def test_create_invalid_dataset(self): @pytest.mark.test_server() def test_get_online_dataset_arff(self): - dataset_id = 100 # Australian + dataset_id = 128 # iris -- one of the few datasets without parquet file # lazy loading not used as arff file is checked. dataset = openml.datasets.get_dataset(dataset_id, download_data=True) decoder = arff.ArffDecoder() @@ -1468,8 +1461,9 @@ def test_data_edit_critical_field(self): raise e time.sleep(10) # Delete the cache dir to get the newer version of the dataset + shutil.rmtree( - os.path.join(self.workdir, "org", "openml", "test", "datasets", str(did)), + os.path.join(openml.config.get_cache_directory(), "datasets", str(did)), ) @pytest.mark.test_server() @@ -1734,7 +1728,6 @@ def test_delete_dataset(self): @mock.patch.object(requests.Session, "delete") def test_delete_dataset_not_owned(mock_delete, test_files_directory, test_api_key): - openml.config.start_using_configuration_for_example() content_file = ( test_files_directory / "mock_responses" / "datasets" / "data_delete_not_owned.xml" ) @@ -1749,14 +1742,13 @@ def test_delete_dataset_not_owned(mock_delete, test_files_directory, test_api_ke ): openml.datasets.delete_dataset(40_000) - dataset_url = "https://test.openml.org/api/v1/xml/data/40000" + dataset_url = f"{openml.config.TEST_SERVER_URL}/api/v1/xml/data/40000" assert dataset_url == mock_delete.call_args.args[0] assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key") @mock.patch.object(requests.Session, "delete") def test_delete_dataset_with_run(mock_delete, test_files_directory, test_api_key): - openml.config.start_using_configuration_for_example() content_file = ( test_files_directory / "mock_responses" / "datasets" / "data_delete_has_tasks.xml" ) @@ -1771,14 +1763,13 @@ def test_delete_dataset_with_run(mock_delete, test_files_directory, test_api_key ): openml.datasets.delete_dataset(40_000) - dataset_url = "https://test.openml.org/api/v1/xml/data/40000" + dataset_url = f"{openml.config.TEST_SERVER_URL}/api/v1/xml/data/40000" assert dataset_url == mock_delete.call_args.args[0] assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key") @mock.patch.object(requests.Session, "delete") def test_delete_dataset_success(mock_delete, test_files_directory, test_api_key): - openml.config.start_using_configuration_for_example() content_file = ( test_files_directory / "mock_responses" / "datasets" / "data_delete_successful.xml" ) @@ -1790,14 +1781,13 @@ def test_delete_dataset_success(mock_delete, test_files_directory, test_api_key) success = openml.datasets.delete_dataset(40000) assert success - dataset_url = "https://test.openml.org/api/v1/xml/data/40000" + dataset_url = f"{openml.config.TEST_SERVER_URL}/api/v1/xml/data/40000" assert dataset_url == mock_delete.call_args.args[0] assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key") @mock.patch.object(requests.Session, "delete") def test_delete_unknown_dataset(mock_delete, test_files_directory, test_api_key): - openml.config.start_using_configuration_for_example() content_file = ( test_files_directory / "mock_responses" / "datasets" / "data_delete_not_exist.xml" ) @@ -1812,7 +1802,7 @@ def test_delete_unknown_dataset(mock_delete, test_files_directory, test_api_key) ): openml.datasets.delete_dataset(9_999_999) - dataset_url = "https://test.openml.org/api/v1/xml/data/9999999" + dataset_url = f"{openml.config.TEST_SERVER_URL}/api/v1/xml/data/9999999" assert dataset_url == mock_delete.call_args.args[0] assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key") @@ -1907,9 +1897,8 @@ def _dataset_features_is_downloaded(did: int): def _dataset_data_file_is_downloaded(did: int): - parquet_present = _dataset_file_is_downloaded(did, "dataset.pq") - arff_present = _dataset_file_is_downloaded(did, "dataset.arff") - return parquet_present or arff_present + cache_directory = Path(openml.config.get_cache_directory()) / "datasets" / str(did) + return any(f.suffix in (".pq", ".arff") for f in cache_directory.iterdir()) def _assert_datasets_retrieved_successfully( @@ -2014,7 +2003,7 @@ def test_get_dataset_parquet(requests_mock, test_files_directory): test_files_directory / "mock_responses" / "datasets" / "data_description_61.xml" ) # While the mocked example is from production, unit tests by default connect to the test server. - requests_mock.get("https://test.openml.org/api/v1/xml/data/61", text=content_file.read_text()) + requests_mock.get(f"{openml.config.TEST_SERVER_URL}/api/v1/xml/data/61", text=content_file.read_text()) dataset = openml.datasets.get_dataset(61, download_data=True) assert dataset._parquet_url is not None assert dataset.parquet_file is not None diff --git a/tests/test_flows/test_flow_functions.py b/tests/test_flows/test_flow_functions.py index c9af3bf8f..ce0d5e782 100644 --- a/tests/test_flows/test_flow_functions.py +++ b/tests/test_flows/test_flow_functions.py @@ -453,7 +453,6 @@ def test_delete_flow(self): @mock.patch.object(requests.Session, "delete") def test_delete_flow_not_owned(mock_delete, test_files_directory, test_api_key): - openml.config.start_using_configuration_for_example() content_file = test_files_directory / "mock_responses" / "flows" / "flow_delete_not_owned.xml" mock_delete.return_value = create_request_response( status_code=412, @@ -466,14 +465,13 @@ def test_delete_flow_not_owned(mock_delete, test_files_directory, test_api_key): ): openml.flows.delete_flow(40_000) - flow_url = "https://test.openml.org/api/v1/xml/flow/40000" + flow_url = f"{openml.config.TEST_SERVER_URL}/api/v1/xml/flow/40000" assert flow_url == mock_delete.call_args.args[0] assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key") @mock.patch.object(requests.Session, "delete") def test_delete_flow_with_run(mock_delete, test_files_directory, test_api_key): - openml.config.start_using_configuration_for_example() content_file = test_files_directory / "mock_responses" / "flows" / "flow_delete_has_runs.xml" mock_delete.return_value = create_request_response( status_code=412, @@ -486,14 +484,13 @@ def test_delete_flow_with_run(mock_delete, test_files_directory, test_api_key): ): openml.flows.delete_flow(40_000) - flow_url = "https://test.openml.org/api/v1/xml/flow/40000" + flow_url = f"{openml.config.TEST_SERVER_URL}/api/v1/xml/flow/40000" assert flow_url == mock_delete.call_args.args[0] assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key") @mock.patch.object(requests.Session, "delete") def test_delete_subflow(mock_delete, test_files_directory, test_api_key): - openml.config.start_using_configuration_for_example() content_file = test_files_directory / "mock_responses" / "flows" / "flow_delete_is_subflow.xml" mock_delete.return_value = create_request_response( status_code=412, @@ -506,14 +503,13 @@ def test_delete_subflow(mock_delete, test_files_directory, test_api_key): ): openml.flows.delete_flow(40_000) - flow_url = "https://test.openml.org/api/v1/xml/flow/40000" + flow_url = f"{openml.config.TEST_SERVER_URL}/api/v1/xml/flow/40000" assert flow_url == mock_delete.call_args.args[0] assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key") @mock.patch.object(requests.Session, "delete") def test_delete_flow_success(mock_delete, test_files_directory, test_api_key): - openml.config.start_using_configuration_for_example() content_file = test_files_directory / "mock_responses" / "flows" / "flow_delete_successful.xml" mock_delete.return_value = create_request_response( status_code=200, @@ -523,7 +519,7 @@ def test_delete_flow_success(mock_delete, test_files_directory, test_api_key): success = openml.flows.delete_flow(33364) assert success - flow_url = "https://test.openml.org/api/v1/xml/flow/33364" + flow_url = f"{openml.config.TEST_SERVER_URL}/api/v1/xml/flow/33364" assert flow_url == mock_delete.call_args.args[0] assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key") @@ -531,7 +527,6 @@ def test_delete_flow_success(mock_delete, test_files_directory, test_api_key): @mock.patch.object(requests.Session, "delete") @pytest.mark.xfail(reason="failures_issue_1544", strict=False) def test_delete_unknown_flow(mock_delete, test_files_directory, test_api_key): - openml.config.start_using_configuration_for_example() content_file = test_files_directory / "mock_responses" / "flows" / "flow_delete_not_exist.xml" mock_delete.return_value = create_request_response( status_code=412, @@ -544,6 +539,6 @@ def test_delete_unknown_flow(mock_delete, test_files_directory, test_api_key): ): openml.flows.delete_flow(9_999_999) - flow_url = "https://test.openml.org/api/v1/xml/flow/9999999" + flow_url = f"{openml.config.TEST_SERVER_URL}/api/v1/xml/flow/9999999" assert flow_url == mock_delete.call_args.args[0] assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key") diff --git a/tests/test_openml/test_config.py b/tests/test_openml/test_config.py index fc7221716..13b06223a 100644 --- a/tests/test_openml/test_config.py +++ b/tests/test_openml/test_config.py @@ -78,7 +78,7 @@ def test_get_config_as_dict(self): config = openml.config.get_config_as_dict() _config = {} _config["apikey"] = TestBase.user_key - _config["server"] = "https://test.openml.org/api/v1/xml" + _config["server"] = f"{openml.config.TEST_SERVER_URL}/api/v1/xml" _config["cachedir"] = self.workdir _config["avoid_duplicate_runs"] = False _config["connection_n_retries"] = 20 diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index e29558314..9bc8d74fa 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -1813,7 +1813,6 @@ def test_initialize_model_from_run_nonstrict(self): @mock.patch.object(requests.Session, "delete") def test_delete_run_not_owned(mock_delete, test_files_directory, test_api_key): - openml.config.start_using_configuration_for_example() content_file = test_files_directory / "mock_responses" / "runs" / "run_delete_not_owned.xml" mock_delete.return_value = create_request_response( status_code=412, @@ -1826,14 +1825,13 @@ def test_delete_run_not_owned(mock_delete, test_files_directory, test_api_key): ): openml.runs.delete_run(40_000) - run_url = "https://test.openml.org/api/v1/xml/run/40000" + run_url = f"{openml.config.TEST_SERVER_URL}/api/v1/xml/run/40000" assert run_url == mock_delete.call_args.args[0] assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key") @mock.patch.object(requests.Session, "delete") def test_delete_run_success(mock_delete, test_files_directory, test_api_key): - openml.config.start_using_configuration_for_example() content_file = test_files_directory / "mock_responses" / "runs" / "run_delete_successful.xml" mock_delete.return_value = create_request_response( status_code=200, @@ -1843,14 +1841,13 @@ def test_delete_run_success(mock_delete, test_files_directory, test_api_key): success = openml.runs.delete_run(10591880) assert success - run_url = "https://test.openml.org/api/v1/xml/run/10591880" + run_url = f"{openml.config.TEST_SERVER_URL}/api/v1/xml/run/10591880" assert run_url == mock_delete.call_args.args[0] assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key") @mock.patch.object(requests.Session, "delete") def test_delete_unknown_run(mock_delete, test_files_directory, test_api_key): - openml.config.start_using_configuration_for_example() content_file = test_files_directory / "mock_responses" / "runs" / "run_delete_not_exist.xml" mock_delete.return_value = create_request_response( status_code=412, @@ -1863,7 +1860,7 @@ def test_delete_unknown_run(mock_delete, test_files_directory, test_api_key): ): openml.runs.delete_run(9_999_999) - run_url = "https://test.openml.org/api/v1/xml/run/9999999" + run_url = f"{openml.config.TEST_SERVER_URL}/api/v1/xml/run/9999999" assert run_url == mock_delete.call_args.args[0] assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key") @@ -1873,6 +1870,10 @@ def test_delete_unknown_run(mock_delete, test_files_directory, test_api_key): Version(sklearn.__version__) < Version("0.21"), reason="couldn't perform local tests successfully w/o bloating RAM", ) +@unittest.skipIf( + Version(sklearn.__version__) >= Version("1.8"), + reason="predictions differ significantly", + ) @mock.patch("openml_sklearn.SklearnExtension._prevent_optimize_n_jobs") @pytest.mark.test_server() def test__run_task_get_arffcontent_2(parallel_mock): diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py index da1f24cdc..df3c0a3b6 100644 --- a/tests/test_tasks/test_task_functions.py +++ b/tests/test_tasks/test_task_functions.py @@ -96,7 +96,9 @@ def test_list_tasks_empty(self): @pytest.mark.test_server() def test_list_tasks_by_tag(self): - num_basic_tasks = 100 # number is flexible, check server if fails + # Server starts with 99 active tasks with the tag, and one 'in_preparation', + # so depending on the processing of the last dataset, there may be 99 or 100 matches. + num_basic_tasks = 99 tasks = openml.tasks.list_tasks(tag="OpenML100") assert len(tasks) >= num_basic_tasks for task in tasks.to_dict(orient="index").values(): @@ -156,13 +158,13 @@ def test_get_task(self): task = openml.tasks.get_task(1, download_data=True) # anneal; crossvalidation assert isinstance(task, OpenMLTask) assert os.path.exists( - os.path.join(self.workdir, "org", "openml", "test", "tasks", "1", "task.xml") + os.path.join(openml.config.get_cache_directory(), "tasks", "1", "task.xml") ) assert not os.path.exists( - os.path.join(self.workdir, "org", "openml", "test", "tasks", "1", "datasplits.arff") + os.path.join(openml.config.get_cache_directory(), "tasks", "1", "datasplits.arff") ) assert os.path.exists( - os.path.join(self.workdir, "org", "openml", "test", "datasets", "1", "dataset.arff") + os.path.join(openml.config.get_cache_directory(), "datasets", "1", "dataset_1.pq") ) @pytest.mark.test_server() @@ -170,21 +172,21 @@ def test_get_task_lazy(self): task = openml.tasks.get_task(2, download_data=False) # anneal; crossvalidation assert isinstance(task, OpenMLTask) assert os.path.exists( - os.path.join(self.workdir, "org", "openml", "test", "tasks", "2", "task.xml") + os.path.join(openml.config.get_cache_directory(), "tasks", "2", "task.xml") ) assert task.class_labels == ["1", "2", "3", "4", "5", "U"] assert not os.path.exists( - os.path.join(self.workdir, "org", "openml", "test", "tasks", "2", "datasplits.arff") + os.path.join(openml.config.get_cache_directory(), "tasks", "2", "datasplits.arff") ) # Since the download_data=False is propagated to get_dataset assert not os.path.exists( - os.path.join(self.workdir, "org", "openml", "test", "datasets", "2", "dataset.arff") + os.path.join(openml.config.get_cache_directory(), "datasets", "2", "dataset.arff") ) task.download_split() assert os.path.exists( - os.path.join(self.workdir, "org", "openml", "test", "tasks", "2", "datasplits.arff") + os.path.join(openml.config.get_cache_directory(), "tasks", "2", "datasplits.arff") ) @mock.patch("openml.tasks.functions.get_dataset") @@ -228,7 +230,7 @@ def test_download_split(self): split = task.download_split() assert type(split) == OpenMLSplit assert os.path.exists( - os.path.join(self.workdir, "org", "openml", "test", "tasks", "1", "datasplits.arff") + os.path.join(openml.config.get_cache_directory(), "tasks", "1", "datasplits.arff") ) def test_deletion_of_cache_dir(self): @@ -244,7 +246,6 @@ def test_deletion_of_cache_dir(self): @mock.patch.object(requests.Session, "delete") def test_delete_task_not_owned(mock_delete, test_files_directory, test_api_key): - openml.config.start_using_configuration_for_example() content_file = test_files_directory / "mock_responses" / "tasks" / "task_delete_not_owned.xml" mock_delete.return_value = create_request_response( status_code=412, @@ -257,14 +258,13 @@ def test_delete_task_not_owned(mock_delete, test_files_directory, test_api_key): ): openml.tasks.delete_task(1) - task_url = "https://test.openml.org/api/v1/xml/task/1" + task_url = f"{openml.config.TEST_SERVER_URL}/api/v1/xml/task/1" assert task_url == mock_delete.call_args.args[0] assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key") @mock.patch.object(requests.Session, "delete") def test_delete_task_with_run(mock_delete, test_files_directory, test_api_key): - openml.config.start_using_configuration_for_example() content_file = test_files_directory / "mock_responses" / "tasks" / "task_delete_has_runs.xml" mock_delete.return_value = create_request_response( status_code=412, @@ -277,14 +277,13 @@ def test_delete_task_with_run(mock_delete, test_files_directory, test_api_key): ): openml.tasks.delete_task(3496) - task_url = "https://test.openml.org/api/v1/xml/task/3496" + task_url = f"{openml.config.TEST_SERVER_URL}/api/v1/xml/task/3496" assert task_url == mock_delete.call_args.args[0] assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key") @mock.patch.object(requests.Session, "delete") def test_delete_success(mock_delete, test_files_directory, test_api_key): - openml.config.start_using_configuration_for_example() content_file = test_files_directory / "mock_responses" / "tasks" / "task_delete_successful.xml" mock_delete.return_value = create_request_response( status_code=200, @@ -294,14 +293,13 @@ def test_delete_success(mock_delete, test_files_directory, test_api_key): success = openml.tasks.delete_task(361323) assert success - task_url = "https://test.openml.org/api/v1/xml/task/361323" + task_url = f"{openml.config.TEST_SERVER_URL}/api/v1/xml/task/361323" assert task_url == mock_delete.call_args.args[0] assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key") @mock.patch.object(requests.Session, "delete") def test_delete_unknown_task(mock_delete, test_files_directory, test_api_key): - openml.config.start_using_configuration_for_example() content_file = test_files_directory / "mock_responses" / "tasks" / "task_delete_not_exist.xml" mock_delete.return_value = create_request_response( status_code=412, @@ -314,6 +312,6 @@ def test_delete_unknown_task(mock_delete, test_files_directory, test_api_key): ): openml.tasks.delete_task(9_999_999) - task_url = "https://test.openml.org/api/v1/xml/task/9999999" + task_url = f"{openml.config.TEST_SERVER_URL}/api/v1/xml/task/9999999" assert task_url == mock_delete.call_args.args[0] assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key") From 1bc9f15bc2d4e659d70415df1828d41a2ae0494c Mon Sep 17 00:00:00 2001 From: Om Swastik Panda Date: Fri, 20 Feb 2026 16:22:33 +0530 Subject: [PATCH 38/46] [ENH] Add `OpenMLAuthenticationError` for clearer API key error handling (#1570) ## Overview This PR introduces a new **`OpenMLAuthenticationError`** exception to clearly distinguish **authentication errors** (invalid or missing API key) from **authorization errors** (valid API key without sufficient permissions). --- ## Changes ### **New Exception** * Added **`OpenMLAuthenticationError`** in `exceptions.py` * Inherits from `OpenMLServerError` for consistency * Automatically appends helpful guidance with links to: * Getting an API key: [https://www.openml.org/](https://www.openml.org/) * OpenML authentication documentation * Includes a clear docstring explaining the difference between authentication and authorization errors --- ### **Updated Error Handling** * Updated `_api_calls.py` to: * Import and raise `OpenMLAuthenticationError` for authentication failures --- ### **Tests Updated** * Updated `test_authentication_endpoints_requiring_api_key_show_relevant_help_link` * Now expects `OpenMLAuthenticationError` instead of `OpenMLNotAuthorizedError` * Continues to assert that helpful guidance is included in the error message --- Fixes #1562 --- openml/_api_calls.py | 10 +++------- openml/exceptions.py | 23 +++++++++++++++++++++++ tests/test_openml/test_api_calls.py | 2 +- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/openml/_api_calls.py b/openml/_api_calls.py index 9e53bd9fa..5da635c70 100644 --- a/openml/_api_calls.py +++ b/openml/_api_calls.py @@ -22,8 +22,8 @@ from . import config from .__version__ import __version__ from .exceptions import ( + OpenMLAuthenticationError, OpenMLHashException, - OpenMLNotAuthorizedError, OpenMLServerError, OpenMLServerException, OpenMLServerNoResult, @@ -515,11 +515,7 @@ def __parse_server_exception( 400, # run/42 delete 460, # task/42 delete ]: - msg = ( - f"The API call {url} requires authentication via an API key.\nPlease configure " - "OpenML-Python to use your API as described in this example:" - "\nhttps://openml.github.io/openml-python/latest/examples/Basics/introduction_tutorial/#authentication" - ) - return OpenMLNotAuthorizedError(message=msg) + msg = f"The API call {url} requires authentication via an API key." + return OpenMLAuthenticationError(message=msg) return OpenMLServerException(code=code, message=full_message, url=url) diff --git a/openml/exceptions.py b/openml/exceptions.py index fe63b8a58..1c1343ff3 100644 --- a/openml/exceptions.py +++ b/openml/exceptions.py @@ -63,5 +63,28 @@ class OpenMLNotAuthorizedError(OpenMLServerError): """Indicates an authenticated user is not authorized to execute the requested action.""" +class OpenMLAuthenticationError(OpenMLServerError): + """Exception raised when API authentication fails. + + This typically occurs when: + - No API key is configured + - The API key is invalid or expired + - The API key format is incorrect + + This is different from authorization (OpenMLNotAuthorizedError), which occurs + when a valid API key lacks permissions for the requested operation. + """ + + def __init__(self, message: str): + help_text = ( + "\n\nTo fix this:\n" + "1. Get your API key from https://www.openml.org/\n" + " (you'll need to register for a free account if you don't have one)\n" + "2. Configure your API key by following the authentication guide:\n" + " https://openml.github.io/openml-python/latest/examples/Basics/introduction_tutorial/#authentication" + ) + super().__init__(message + help_text) + + class ObjectNotPublishedError(PyOpenMLError): """Indicates an object has not been published yet.""" diff --git a/tests/test_openml/test_api_calls.py b/tests/test_openml/test_api_calls.py index c8d5be25b..3f30f38ba 100644 --- a/tests/test_openml/test_api_calls.py +++ b/tests/test_openml/test_api_calls.py @@ -124,5 +124,5 @@ def test_authentication_endpoints_requiring_api_key_show_relevant_help_link( ) -> None: # We need to temporarily disable the API key to test the error message with openml.config.overwrite_config_context({"apikey": None}): - with pytest.raises(openml.exceptions.OpenMLNotAuthorizedError, match=API_TOKEN_HELP_LINK): + with pytest.raises(openml.exceptions.OpenMLAuthenticationError, match=API_TOKEN_HELP_LINK): openml._api_calls._perform_api_call(call=endpoint, request_method=method, data=None) From 7feb2a328b68e416cb554cbcf24e091c1c9453e2 Mon Sep 17 00:00:00 2001 From: Om Swastik Panda Date: Sat, 21 Feb 2026 00:01:37 +0530 Subject: [PATCH 39/46] [MNT] Remove redundant `__init__`s in `OpenMLTask` descendants by adding ClassVar (#1588) Fixes #1578 --- openml/tasks/functions.py | 4 ++ openml/tasks/task.py | 148 +++++++++++--------------------------- 2 files changed, 47 insertions(+), 105 deletions(-) diff --git a/openml/tasks/functions.py b/openml/tasks/functions.py index 2bf1a40f4..3fbc7adee 100644 --- a/openml/tasks/functions.py +++ b/openml/tasks/functions.py @@ -426,6 +426,9 @@ def get_task( # Including class labels as part of task meta data handles # the case where data download was initially disabled if isinstance(task, (OpenMLClassificationTask, OpenMLLearningCurveTask)): + assert task.target_name is not None, ( + "Supervised tasks must define a target feature before retrieving class labels." + ) task.class_labels = dataset.retrieve_class_labels(task.target_name) # Clustering tasks do not have class labels # and do not offer download_split @@ -599,6 +602,7 @@ def create_task( ) return task_cls( + task_id=None, task_type_id=task_type, task_type="None", # TODO: refactor to get task type string from ID. data_set_id=dataset_id, diff --git a/openml/tasks/task.py b/openml/tasks/task.py index b297a105c..385b1f949 100644 --- a/openml/tasks/task.py +++ b/openml/tasks/task.py @@ -1,6 +1,4 @@ # License: BSD 3-Clause -# TODO(eddbergman): Seems like a lot of the subclasses could just get away with setting -# a `ClassVar` for whatever changes as their `__init__` defaults, less duplicated code. from __future__ import annotations import warnings @@ -8,7 +6,7 @@ from collections.abc import Sequence from enum import Enum from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from typing_extensions import TypedDict import openml._api_calls @@ -71,31 +69,45 @@ class OpenMLTask(OpenMLBase): Refers to the URL of the data splits used for the OpenML task. """ + DEFAULT_ESTIMATION_PROCEDURE_ID: ClassVar[int] = 1 + def __init__( # noqa: PLR0913 self, task_id: int | None, task_type_id: TaskType, task_type: str, data_set_id: int, - estimation_procedure_id: int = 1, + estimation_procedure_id: int | None = None, estimation_procedure_type: str | None = None, estimation_parameters: dict[str, str] | None = None, evaluation_measure: str | None = None, data_splits_url: str | None = None, + target_name: str | None = None, ): self.task_id = int(task_id) if task_id is not None else None self.task_type_id = task_type_id self.task_type = task_type self.dataset_id = int(data_set_id) + self.target_name = target_name + resolved_estimation_procedure_id = self._resolve_estimation_procedure_id( + estimation_procedure_id, + ) self.evaluation_measure = evaluation_measure self.estimation_procedure: _EstimationProcedure = { "type": estimation_procedure_type, "parameters": estimation_parameters, "data_splits_url": data_splits_url, } - self.estimation_procedure_id = estimation_procedure_id + self.estimation_procedure_id = resolved_estimation_procedure_id self.split: OpenMLSplit | None = None + def _resolve_estimation_procedure_id(self, estimation_procedure_id: int | None) -> int: + return ( + estimation_procedure_id + if estimation_procedure_id is not None + else self.DEFAULT_ESTIMATION_PROCEDURE_ID + ) + @classmethod def _entity_letter(cls) -> str: return "t" @@ -129,7 +141,8 @@ def _get_repr_body_fields(self) -> Sequence[tuple[str, str | int | list[str]]]: if class_labels is not None: fields["# of Classes"] = len(class_labels) - if hasattr(self, "cost_matrix"): + cost_matrix = getattr(self, "cost_matrix", None) + if cost_matrix is not None: fields["Cost Matrix"] = "Available" # determines the order in which the information will be printed @@ -250,13 +263,15 @@ class OpenMLSupervisedTask(OpenMLTask, ABC): Refers to the unique identifier of task. """ + DEFAULT_ESTIMATION_PROCEDURE_ID: ClassVar[int] = 1 + def __init__( # noqa: PLR0913 self, task_type_id: TaskType, task_type: str, data_set_id: int, target_name: str, - estimation_procedure_id: int = 1, + estimation_procedure_id: int | None = None, estimation_procedure_type: str | None = None, estimation_parameters: dict[str, str] | None = None, evaluation_measure: str | None = None, @@ -273,10 +288,9 @@ def __init__( # noqa: PLR0913 estimation_parameters=estimation_parameters, evaluation_measure=evaluation_measure, data_splits_url=data_splits_url, + target_name=target_name, ) - self.target_name = target_name - def get_X_and_y(self) -> tuple[pd.DataFrame, pd.Series | pd.DataFrame | None]: """Get data associated with the current task. @@ -331,6 +345,8 @@ class OpenMLClassificationTask(OpenMLSupervisedTask): Parameters ---------- + task_id : Union[int, None] + ID of the Classification task (if it already exists on OpenML). task_type_id : TaskType ID of the Classification task type. task_type : str @@ -339,7 +355,7 @@ class OpenMLClassificationTask(OpenMLSupervisedTask): ID of the OpenML dataset associated with the Classification task. target_name : str Name of the target variable. - estimation_procedure_id : int, default=None + estimation_procedure_id : int, default=1 ID of the estimation procedure for the Classification task. estimation_procedure_type : str, default=None Type of the estimation procedure. @@ -349,21 +365,21 @@ class OpenMLClassificationTask(OpenMLSupervisedTask): Name of the evaluation measure. data_splits_url : str, default=None URL of the data splits for the Classification task. - task_id : Union[int, None] - ID of the Classification task (if it already exists on OpenML). class_labels : List of str, default=None A list of class labels (for classification tasks). cost_matrix : array, default=None A cost matrix (for classification tasks). """ + DEFAULT_ESTIMATION_PROCEDURE_ID: ClassVar[int] = 1 + def __init__( # noqa: PLR0913 self, task_type_id: TaskType, task_type: str, data_set_id: int, target_name: str, - estimation_procedure_id: int = 1, + estimation_procedure_id: int | None = None, estimation_procedure_type: str | None = None, estimation_parameters: dict[str, str] | None = None, evaluation_measure: str | None = None, @@ -373,20 +389,19 @@ def __init__( # noqa: PLR0913 cost_matrix: np.ndarray | None = None, ): super().__init__( - task_id=task_id, task_type_id=task_type_id, task_type=task_type, data_set_id=data_set_id, + target_name=target_name, estimation_procedure_id=estimation_procedure_id, estimation_procedure_type=estimation_procedure_type, estimation_parameters=estimation_parameters, evaluation_measure=evaluation_measure, - target_name=target_name, data_splits_url=data_splits_url, + task_id=task_id, ) self.class_labels = class_labels self.cost_matrix = cost_matrix - if cost_matrix is not None: raise NotImplementedError("Costmatrix functionality is not yet implemented.") @@ -396,6 +411,8 @@ class OpenMLRegressionTask(OpenMLSupervisedTask): Parameters ---------- + task_id : Union[int, None] + ID of the OpenML Regression task. task_type_id : TaskType Task type ID of the OpenML Regression task. task_type : str @@ -404,7 +421,7 @@ class OpenMLRegressionTask(OpenMLSupervisedTask): ID of the OpenML dataset. target_name : str Name of the target feature used in the Regression task. - estimation_procedure_id : int, default=None + estimation_procedure_id : int, default=7 ID of the OpenML estimation procedure. estimation_procedure_type : str, default=None Type of the OpenML estimation procedure. @@ -412,37 +429,11 @@ class OpenMLRegressionTask(OpenMLSupervisedTask): Parameters used by the OpenML estimation procedure. data_splits_url : str, default=None URL of the OpenML data splits for the Regression task. - task_id : Union[int, None] - ID of the OpenML Regression task. evaluation_measure : str, default=None Evaluation measure used in the Regression task. """ - def __init__( # noqa: PLR0913 - self, - task_type_id: TaskType, - task_type: str, - data_set_id: int, - target_name: str, - estimation_procedure_id: int = 7, - estimation_procedure_type: str | None = None, - estimation_parameters: dict[str, str] | None = None, - data_splits_url: str | None = None, - task_id: int | None = None, - evaluation_measure: str | None = None, - ): - super().__init__( - task_id=task_id, - task_type_id=task_type_id, - task_type=task_type, - data_set_id=data_set_id, - estimation_procedure_id=estimation_procedure_id, - estimation_procedure_type=estimation_procedure_type, - estimation_parameters=estimation_parameters, - evaluation_measure=evaluation_measure, - target_name=target_name, - data_splits_url=data_splits_url, - ) + DEFAULT_ESTIMATION_PROCEDURE_ID: ClassVar[int] = 7 class OpenMLClusteringTask(OpenMLTask): @@ -450,16 +441,16 @@ class OpenMLClusteringTask(OpenMLTask): Parameters ---------- + task_id : Union[int, None] + ID of the OpenML clustering task. task_type_id : TaskType Task type ID of the OpenML clustering task. task_type : str Task type of the OpenML clustering task. data_set_id : int ID of the OpenML dataset used in clustering the task. - estimation_procedure_id : int, default=None + estimation_procedure_id : int, default=17 ID of the OpenML estimation procedure. - task_id : Union[int, None] - ID of the OpenML clustering task. estimation_procedure_type : str, default=None Type of the OpenML estimation procedure used in the clustering task. estimation_parameters : dict, default=None @@ -473,32 +464,7 @@ class OpenMLClusteringTask(OpenMLTask): feature set for the clustering task. """ - def __init__( # noqa: PLR0913 - self, - task_type_id: TaskType, - task_type: str, - data_set_id: int, - estimation_procedure_id: int = 17, - task_id: int | None = None, - estimation_procedure_type: str | None = None, - estimation_parameters: dict[str, str] | None = None, - data_splits_url: str | None = None, - evaluation_measure: str | None = None, - target_name: str | None = None, - ): - super().__init__( - task_id=task_id, - task_type_id=task_type_id, - task_type=task_type, - data_set_id=data_set_id, - evaluation_measure=evaluation_measure, - estimation_procedure_id=estimation_procedure_id, - estimation_procedure_type=estimation_procedure_type, - estimation_parameters=estimation_parameters, - data_splits_url=data_splits_url, - ) - - self.target_name = target_name + DEFAULT_ESTIMATION_PROCEDURE_ID: ClassVar[int] = 17 def get_X(self) -> pd.DataFrame: """Get data associated with the current task. @@ -534,6 +500,8 @@ class OpenMLLearningCurveTask(OpenMLClassificationTask): Parameters ---------- + task_id : Union[int, None] + ID of the Learning Curve task. task_type_id : TaskType ID of the Learning Curve task. task_type : str @@ -542,7 +510,7 @@ class OpenMLLearningCurveTask(OpenMLClassificationTask): ID of the dataset that this task is associated with. target_name : str Name of the target feature in the dataset. - estimation_procedure_id : int, default=None + estimation_procedure_id : int, default=13 ID of the estimation procedure to use for evaluating models. estimation_procedure_type : str, default=None Type of the estimation procedure. @@ -550,8 +518,6 @@ class OpenMLLearningCurveTask(OpenMLClassificationTask): Additional parameters for the estimation procedure. data_splits_url : str, default=None URL of the file containing the data splits for Learning Curve task. - task_id : Union[int, None] - ID of the Learning Curve task. evaluation_measure : str, default=None Name of the evaluation measure to use for evaluating models. class_labels : list of str, default=None @@ -560,32 +526,4 @@ class OpenMLLearningCurveTask(OpenMLClassificationTask): Cost matrix for Learning Curve tasks. """ - def __init__( # noqa: PLR0913 - self, - task_type_id: TaskType, - task_type: str, - data_set_id: int, - target_name: str, - estimation_procedure_id: int = 13, - estimation_procedure_type: str | None = None, - estimation_parameters: dict[str, str] | None = None, - data_splits_url: str | None = None, - task_id: int | None = None, - evaluation_measure: str | None = None, - class_labels: list[str] | None = None, - cost_matrix: np.ndarray | None = None, - ): - super().__init__( - task_id=task_id, - task_type_id=task_type_id, - task_type=task_type, - data_set_id=data_set_id, - estimation_procedure_id=estimation_procedure_id, - estimation_procedure_type=estimation_procedure_type, - estimation_parameters=estimation_parameters, - evaluation_measure=evaluation_measure, - target_name=target_name, - data_splits_url=data_splits_url, - class_labels=class_labels, - cost_matrix=cost_matrix, - ) + DEFAULT_ESTIMATION_PROCEDURE_ID: ClassVar[int] = 13 From dbd432c218cbc3182807684be73951a3749bffde Mon Sep 17 00:00:00 2001 From: "P. Clawmogorov" Date: Tue, 3 Mar 2026 15:50:55 +0100 Subject: [PATCH 40/46] [BUG] race condition in OpenMLSplitTest when running tests in parallel (#1643) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem When running tests in parallel with pytest-xdist (e.g., "pytest -n 3 tests/test_tasks/test_split.py"), one test under OpenMLSplitTest fails intermittently with an EOFError during pickle.load(). This was identified in CI job/63346513831 and reproduces roughly 1 out of 10 runs locally. ## Analysis The root cause is that all test instances share the same pickle cache file path (`self.pd_filename`). When multiple workers run concurrently: 1. Worker A creates the pickle cache file during test execution 2. Worker B reads the pickle cache file 3. Worker A's tearDown() deletes the file 4. Worker B's pickle.load() encounters a partially deleted file → EOFError This is a classic race condition on shared filesystem state. ## Solution Use `tempfile.mkdtemp()` to create a unique temporary directory for each test instance, then copy the ARFF source file there. This ensures: - Each test worker has its own isolated pickle cache file - No shared state between parallel workers - Automatic cleanup via shutil.rmtree() in tearDown() The fix is minimal (10 insertions, 3 deletions) and doesn't change the test logic - only the test isolation mechanism. ## Benchmarks / Testing Ran 5 consecutive parallel test executions: ``` pytest -n 4 tests/test_tasks/test_split.py # 5 times ``` All 15 test runs (3 tests × 5 runs) passed successfully. Before the fix, failures occurred ~10% of the time with parallel execution. Fixes #1641 --- tests/test_tasks/test_split.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test_tasks/test_split.py b/tests/test_tasks/test_split.py index 12cb632d9..7023c7d05 100644 --- a/tests/test_tasks/test_split.py +++ b/tests/test_tasks/test_split.py @@ -3,6 +3,8 @@ import inspect import os +import shutil +import tempfile from pathlib import Path import numpy as np @@ -19,7 +21,7 @@ def setUp(self): __file__ = inspect.getfile(OpenMLSplitTest) self.directory = os.path.dirname(__file__) # This is for dataset - self.arff_filepath = ( + source_arff = ( Path(self.directory).parent / "files" / "org" @@ -29,13 +31,18 @@ def setUp(self): / "1882" / "datasplits.arff" ) + # Use a unique temp directory for each test to avoid race conditions + # when running tests in parallel (see issue #1641) + self._temp_dir = tempfile.TemporaryDirectory() + self.arff_filepath = Path(self._temp_dir.name) / "datasplits.arff" + shutil.copy(source_arff, self.arff_filepath) self.pd_filename = self.arff_filepath.with_suffix(".pkl.py3") def tearDown(self): + # Clean up the entire temp directory try: - os.remove(self.pd_filename) + self._temp_dir.cleanup() except (OSError, FileNotFoundError): - # Replaced bare except. Not sure why these exceptions are acceptable. pass def test_eq(self): From db26db9209f3021c22126f6244f1276bee98e0d2 Mon Sep 17 00:00:00 2001 From: Om Swastik Panda Date: Tue, 3 Mar 2026 23:26:11 +0530 Subject: [PATCH 41/46] [ENH] Replace `asserts` with proper `if else` Exception handling (#1589) Fixes #1581 --- openml/runs/functions.py | 24 +++++--- openml/runs/run.py | 128 ++++++++++++++++++--------------------- openml/runs/trace.py | 15 ++++- 3 files changed, 87 insertions(+), 80 deletions(-) diff --git a/openml/runs/functions.py b/openml/runs/functions.py index 503788dbd..b991fb5ec 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -376,7 +376,8 @@ def initialize_model_from_run(run_id: int, *, strict_version: bool = True) -> An run = get_run(run_id) # TODO(eddiebergman): I imagine this is None if it's not published, # might need to raise an explicit error for that - assert run.setup_id is not None + if run.setup_id is None: + raise ValueError(f"Run {run_id} has no associated setup_id. Cannot initialize model.") return initialize_model(setup_id=run.setup_id, strict_version=strict_version) @@ -416,7 +417,8 @@ def initialize_model_from_trace( run = get_run(run_id) # TODO(eddiebergman): I imagine this is None if it's not published, # might need to raise an explicit error for that - assert run.flow_id is not None + if run.flow_id is None: + raise ValueError(f"Run {run_id} has no associated flow_id. Cannot initialize model.") flow = get_flow(run.flow_id) run_trace = get_run_trace(run_id) @@ -576,8 +578,10 @@ def _calculate_local_measure( # type: ignore _user_defined_measures_fold[openml_name] = sklearn_fn(_test_y, _pred_y) if isinstance(task, (OpenMLClassificationTask, OpenMLLearningCurveTask)): - assert test_y is not None - assert proba_y is not None + if test_y is None: + raise ValueError("test_y cannot be None for classification tasks.") + if proba_y is None: + raise ValueError("proba_y cannot be None for classification tasks.") for i, tst_idx in enumerate(test_indices): if task.class_labels is not None: @@ -622,7 +626,8 @@ def _calculate_local_measure( # type: ignore ) elif isinstance(task, OpenMLRegressionTask): - assert test_y is not None + if test_y is None: + raise ValueError("test_y cannot be None for regression tasks.") for i, _ in enumerate(test_indices): truth = test_y.iloc[i] if isinstance(test_y, pd.Series) else test_y[i] arff_line = format_prediction( @@ -743,7 +748,8 @@ def _run_task_get_arffcontent_parallel_helper( # noqa: PLR0913 if isinstance(task, OpenMLSupervisedTask): x, y = task.get_X_and_y() - assert isinstance(y, (pd.Series, pd.DataFrame)) + if not isinstance(y, (pd.Series, pd.DataFrame)): + raise TypeError(f"y must be a pandas Series or DataFrame, got {type(y).__name__}") train_x = x.iloc[train_indices] train_y = y.iloc[train_indices] test_x = x.iloc[test_indices] @@ -1213,7 +1219,11 @@ def __list_runs(api_call: str) -> pd.DataFrame: f'"http://openml.org/openml": {runs_dict}', ) - assert isinstance(runs_dict["oml:runs"]["oml:run"], list), type(runs_dict["oml:runs"]) + if not isinstance(runs_dict["oml:runs"]["oml:run"], list): + raise TypeError( + f"Expected runs_dict['oml:runs']['oml:run'] to be a list, " + f"got {type(runs_dict['oml:runs']['oml:run']).__name__}" + ) runs = { int(r["oml:run_id"]): { diff --git a/openml/runs/run.py b/openml/runs/run.py index eff011408..086e9c046 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -389,6 +389,57 @@ def to_filesystem( if self.trace is not None: self.trace._to_filesystem(directory) + def _get_arff_attributes_for_task(self, task: OpenMLTask) -> list[tuple[str, Any]]: + """Get ARFF attributes based on task type. + + Parameters + ---------- + task : OpenMLTask + The task for which to generate attributes. + + Returns + ------- + list[tuple[str, Any]] + List of attribute tuples (name, type). + """ + instance_specifications = [ + ("repeat", "NUMERIC"), + ("fold", "NUMERIC"), + ] + + if isinstance(task, (OpenMLLearningCurveTask, OpenMLClassificationTask)): + instance_specifications.append(("sample", "NUMERIC")) + + instance_specifications.append(("row_id", "NUMERIC")) + + if isinstance(task, (OpenMLLearningCurveTask, OpenMLClassificationTask)): + class_labels = task.class_labels + if class_labels is None: + raise ValueError("The task has no class labels") + + prediction_confidences = [ + ("confidence." + class_labels[i], "NUMERIC") for i in range(len(class_labels)) + ] + prediction_and_true = [("prediction", class_labels), ("correct", class_labels)] + return instance_specifications + prediction_and_true + prediction_confidences + + if isinstance(task, OpenMLRegressionTask): + return [*instance_specifications, ("prediction", "NUMERIC"), ("truth", "NUMERIC")] + + if isinstance(task, OpenMLClusteringTask): + return [*instance_specifications, ("cluster", "NUMERIC")] + + supported_task_types = [ + TaskType.SUPERVISED_CLASSIFICATION, + TaskType.SUPERVISED_REGRESSION, + TaskType.CLUSTERING, + TaskType.LEARNING_CURVE, + ] + raise NotImplementedError( + f"Task type {task.task_type!s} for task_id {getattr(task, 'task_id', None)!s} " + f"is not yet supported. Supported task types are: {supported_task_types!r}" + ) + def _generate_arff_dict(self) -> OrderedDict[str, Any]: """Generates the arff dictionary for uploading predictions to the server. @@ -406,7 +457,8 @@ def _generate_arff_dict(self) -> OrderedDict[str, Any]: if self.data_content is None: raise ValueError("Run has not been executed.") if self.flow is None: - assert self.flow_id is not None, "Run has no associated flow id!" + if self.flow_id is None: + raise ValueError("Run has no associated flow id!") self.flow = get_flow(self.flow_id) if self.description_text is None: @@ -417,74 +469,7 @@ def _generate_arff_dict(self) -> OrderedDict[str, Any]: arff_dict["data"] = self.data_content arff_dict["description"] = self.description_text arff_dict["relation"] = f"openml_task_{task.task_id}_predictions" - - if isinstance(task, OpenMLLearningCurveTask): - class_labels = task.class_labels - instance_specifications = [ - ("repeat", "NUMERIC"), - ("fold", "NUMERIC"), - ("sample", "NUMERIC"), - ("row_id", "NUMERIC"), - ] - - arff_dict["attributes"] = instance_specifications - if class_labels is not None: - arff_dict["attributes"] = ( - arff_dict["attributes"] - + [("prediction", class_labels), ("correct", class_labels)] - + [ - ("confidence." + class_labels[i], "NUMERIC") - for i in range(len(class_labels)) - ] - ) - else: - raise ValueError("The task has no class labels") - - elif isinstance(task, OpenMLClassificationTask): - class_labels = task.class_labels - instance_specifications = [ - ("repeat", "NUMERIC"), - ("fold", "NUMERIC"), - ("sample", "NUMERIC"), # Legacy - ("row_id", "NUMERIC"), - ] - - arff_dict["attributes"] = instance_specifications - if class_labels is not None: - prediction_confidences = [ - ("confidence." + class_labels[i], "NUMERIC") for i in range(len(class_labels)) - ] - prediction_and_true = [("prediction", class_labels), ("correct", class_labels)] - arff_dict["attributes"] = ( - arff_dict["attributes"] + prediction_and_true + prediction_confidences - ) - else: - raise ValueError("The task has no class labels") - - elif isinstance(task, OpenMLRegressionTask): - arff_dict["attributes"] = [ - ("repeat", "NUMERIC"), - ("fold", "NUMERIC"), - ("row_id", "NUMERIC"), - ("prediction", "NUMERIC"), - ("truth", "NUMERIC"), - ] - - elif isinstance(task, OpenMLClusteringTask): - arff_dict["attributes"] = [ - ("repeat", "NUMERIC"), - ("fold", "NUMERIC"), - ("row_id", "NUMERIC"), - ("cluster", "NUMERIC"), - ] - - else: - raise NotImplementedError( - f"Task type '{task.task_type}' is not yet supported. " - f"Supported task types: Classification, Regression, Clustering, Learning Curve. " - f"Task ID: {task.task_id}. " - f"Please check the OpenML documentation for supported task types." - ) + arff_dict["attributes"] = self._get_arff_attributes_for_task(task) return arff_dict @@ -641,7 +626,10 @@ def _get_file_elements(self) -> dict: if self.parameter_settings is None: if self.flow is None: - assert self.flow_id is not None # for mypy + if self.flow_id is None: + raise ValueError( + "Run has no associated flow_id and cannot obtain parameter values." + ) self.flow = openml.flows.get_flow(self.flow_id) self.parameter_settings = self.flow.extension.obtain_parameter_values( self.flow, diff --git a/openml/runs/trace.py b/openml/runs/trace.py index 708cdd8f1..f76bd04e8 100644 --- a/openml/runs/trace.py +++ b/openml/runs/trace.py @@ -94,7 +94,8 @@ def get_parameters(self) -> dict[str, Any]: for param, value in self.setup_string.items() } - assert self.parameters is not None + if self.parameters is None: + raise ValueError("Parameters must be set before calling get_parameters().") return {param[len(PREFIX) :]: value for param, value in self.parameters.items()} @@ -490,13 +491,21 @@ def merge_traces(cls, traces: list[OpenMLRunTrace]) -> OpenMLRunTrace: for iteration in trace: key = (iteration.repeat, iteration.fold, iteration.iteration) - assert iteration.parameters is not None + if iteration.parameters is None: + raise ValueError( + f"Iteration parameters cannot be None for repeat {iteration.repeat}, " + f"fold {iteration.fold}, iteration {iteration.iteration}" + ) param_keys = iteration.parameters.keys() if previous_iteration is not None: trace_itr = merged_trace[previous_iteration] - assert trace_itr.parameters is not None + if trace_itr.parameters is None: + raise ValueError( + f"Trace iteration parameters cannot be None " + f"for iteration {previous_iteration}" + ) trace_itr_keys = trace_itr.parameters.keys() if list(param_keys) != list(trace_itr_keys): From 39daaef65306100e1b05f5863cad7b4b5d1e0c89 Mon Sep 17 00:00:00 2001 From: Omkar Kabde Date: Tue, 3 Mar 2026 23:32:02 +0530 Subject: [PATCH 42/46] [MNT] Fix race condition in `OpenMLSplit._from_arff_file` (#1656) Fixes #1641 This PR adds separate temp directories per test in test_split.py to avoid race conditions when running with multiple workers. cc @geetu040 I made this PR because I suspect #1643 is made by an OpenClaw bot. --- tests/test_tasks/test_split.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_tasks/test_split.py b/tests/test_tasks/test_split.py index 7023c7d05..e3320ae80 100644 --- a/tests/test_tasks/test_split.py +++ b/tests/test_tasks/test_split.py @@ -20,7 +20,6 @@ class OpenMLSplitTest(TestBase): def setUp(self): __file__ = inspect.getfile(OpenMLSplitTest) self.directory = os.path.dirname(__file__) - # This is for dataset source_arff = ( Path(self.directory).parent / "files" From e95675a96ca24285e6ac413eb029d04251386374 Mon Sep 17 00:00:00 2001 From: Omkar Kabde Date: Fri, 6 Mar 2026 18:25:45 +0530 Subject: [PATCH 43/46] [MNT] Update setup list test assertion (#1652) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #1645 ``` import openml openml.config.server = "https://www.openml.org/api/v1/xml" setups = openml.setups.list_setups(flow=5873) ``` Screenshot 2026-02-18 at 7 12 32 PM Current number of setups is 2. > 1. For now, please keep using the production server for this test as the test server is not guaranteed to have setups (yet) > 2. Make the assertion based on a minimum. It is possible that people upload more runs for the flow (and consequently, more setups). It is also possible they delete them, but not so likely. So I would check a lower bound that is lower than what is currently on the production server. cc @geetu040 @PGijsbers so should we use the lower bound as 1? ``` assert len(setups) >= 1 ``` --- tests/test_setups/test_setup_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index 0df3a0b3b..5480ce6f8 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -143,7 +143,7 @@ def test_setup_list_filter_flow(self): setups = openml.setups.list_setups(flow=flow_id) - assert len(setups) > 0 # TODO: please adjust 0 + assert len(setups) >= 2 for setup_id in setups: assert setups[setup_id].flow_id == flow_id From 8cc642991de4e0028ec8dfb2b768b6ecc407a245 Mon Sep 17 00:00:00 2001 From: Satvik Mishra <112589278+satvshr@users.noreply.github.com> Date: Mon, 9 Mar 2026 22:52:54 +0530 Subject: [PATCH 44/46] [MNT] Dockerized tests for CI runs using localhost (#1629) #### Metadata * Reference Issue: fixes #1614, stacks on #1630 * New Tests Added: No * Documentation Updated: No #### Details * What does this PR implement/fix? Explain your changes. This PR implements the setting up of the v1 and v2 test servers in CI using docker via `localhost`. --------- Co-authored-by: PGijsbers Co-authored-by: Armaghan Shakir --- .github/workflows/test.yml | 50 ++++++-- tests/conftest.py | 4 +- tests/test_datasets/test_dataset_functions.py | 42 ++++++- tests/test_flows/test_flow.py | 22 +++- tests/test_flows/test_flow_functions.py | 9 ++ tests/test_openml/test_api_calls.py | 5 + tests/test_runs/test_run.py | 20 ++++ tests/test_runs/test_run_functions.py | 112 +++++++++++++++++- tests/test_setups/test_setup_functions.py | 18 ++- tests/test_tasks/test_task_functions.py | 8 ++ 10 files changed, 271 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7fa3450ca..7d5d48ac0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -101,6 +101,23 @@ jobs: echo "BEFORE=$git_status" >> $GITHUB_ENV echo "Repository status before tests: $git_status" + - name: Clone Services + if: matrix.os == 'ubuntu-latest' + run: | + git clone --depth 1 https://github.com/openml/services.git + + - name: Start Docker Services + if: matrix.os == 'ubuntu-latest' + working-directory: ./services + run: | + docker compose --profile rest-api --profile minio up -d + + echo "Waiting for PHP API to boot..." + timeout 60s bash -c 'until [ "$(docker inspect -f {{.State.Health.Status}} openml-php-rest-api)" == "healthy" ]; do sleep 5; done' + + echo "Final Verification: Gateway Connectivity..." + curl -sSfL http://localhost:8000/api/v1/xml/data/1 | head -n 15 + - name: Show installed dependencies run: python -m pip list @@ -108,15 +125,16 @@ jobs: if: matrix.os == 'ubuntu-latest' env: OPENML_TEST_SERVER_ADMIN_KEY: ${{ secrets.OPENML_TEST_SERVER_ADMIN_KEY }} + OPENML_USE_LOCAL_SERVICES: "true" run: | if [ "${{ matrix.code-cov }}" = "true" ]; then codecov="--cov=openml --long --cov-report=xml" fi if [ "${{ matrix.sklearn-only }}" = "true" ]; then - marks="sklearn and not production_server and not test_server" + marks="sklearn and not production_server" else - marks="not production_server and not test_server" + marks="not production_server" fi pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks" @@ -125,15 +143,16 @@ jobs: if: matrix.os == 'ubuntu-latest' env: OPENML_TEST_SERVER_ADMIN_KEY: ${{ secrets.OPENML_TEST_SERVER_ADMIN_KEY }} + OPENML_USE_LOCAL_SERVICES: "true" run: | if [ "${{ matrix.code-cov }}" = "true" ]; then codecov="--cov=openml --long --cov-report=xml" fi if [ "${{ matrix.sklearn-only }}" = "true" ]; then - marks="sklearn and production_server and not test_server" + marks="sklearn and production_server" else - marks="production_server and not test_server" + marks="production_server" fi pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks" @@ -145,6 +164,20 @@ jobs: run: | # we need a separate step because of the bash-specific if-statement in the previous one. pytest -n 4 --durations=20 --dist load -sv --reruns 5 --reruns-delay 1 -m "not test_server" + - name: Upload coverage + if: matrix.code-cov && always() + uses: codecov/codecov-action@v4 + with: + files: coverage.xml + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true + verbose: true + + - name: Cleanup Docker setup + if: matrix.os == 'ubuntu-latest' && always() + run: | + sudo rm -rf services + - name: Check for files left behind by test if: matrix.os != 'windows-latest' && always() run: | @@ -157,15 +190,6 @@ jobs: exit 1 fi - - name: Upload coverage - if: matrix.code-cov && always() - uses: codecov/codecov-action@v4 - with: - files: coverage.xml - token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: true - verbose: true - dummy_windows_py_sk024: name: (windows-latest, Py, sk0.24.*, sk-only:false) runs-on: ubuntu-latest diff --git a/tests/conftest.py b/tests/conftest.py index 2a7a6dcc7..d1bc23d4f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -272,6 +272,8 @@ def as_robot() -> Iterator[None]: @pytest.fixture(autouse=True) def with_server(request): + if os.getenv("OPENML_USE_LOCAL_SERVICES") == "true": + openml.config.TEST_SERVER_URL = "http://localhost:8000" if "production_server" in request.keywords: openml.config.server = "https://www.openml.org/api/v1/xml" openml.config.apikey = None @@ -306,4 +308,4 @@ def workdir(tmp_path): original_cwd = Path.cwd() os.chdir(tmp_path) yield tmp_path - os.chdir(original_cwd) + os.chdir(original_cwd) \ No newline at end of file diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index 151a9ac23..10517a3e1 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -530,6 +530,10 @@ def test_deletion_of_cache_dir_faulty_download(self, patch): datasets_cache_dir = os.path.join(openml.config.get_cache_directory(), "datasets") assert len(os.listdir(datasets_cache_dir)) == 0 + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.test_server() def test_publish_dataset(self): arff_file_path = self.static_cache_dir / "org" / "openml" / "test" / "datasets" / "2" / "dataset.arff" @@ -566,6 +570,10 @@ def test__retrieve_class_labels(self): labels = custom_ds.retrieve_class_labels(target_name=custom_ds.features[31].name) assert labels == ["COIL", "SHEET"] + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.test_server() def test_upload_dataset_with_url(self): dataset = OpenMLDataset( @@ -689,6 +697,10 @@ def test_attributes_arff_from_df_unknown_dtype(self): with pytest.raises(ValueError, match=err_msg): attributes_arff_from_df(df) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.test_server() def test_create_dataset_numpy(self): data = np.array([[1, 2, 3], [1.2, 2.5, 3.8], [2, 5, 8], [0, 1, 0]]).T @@ -723,6 +735,10 @@ def test_create_dataset_numpy(self): ), "Uploaded arff does not match original one" assert _get_online_dataset_format(dataset.id) == "arff", "Wrong format for dataset" + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.test_server() def test_create_dataset_list(self): data = [ @@ -778,6 +794,10 @@ def test_create_dataset_list(self): ), "Uploaded ARFF does not match original one" assert _get_online_dataset_format(dataset.id) == "arff", "Wrong format for dataset" + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.test_server() def test_create_dataset_sparse(self): # test the scipy.sparse.coo_matrix @@ -926,6 +946,10 @@ def test_get_online_dataset_format(self): dataset_id ), "The format of the ARFF files is different" + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.test_server() def test_create_dataset_pandas(self): data = [ @@ -1151,6 +1175,10 @@ def test_ignore_attributes_dataset(self): paper_url=paper_url, ) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.test_server() def test_publish_fetch_ignore_attribute(self): """Test to upload and retrieve dataset and check ignore_attributes""" @@ -1270,6 +1298,10 @@ def test_create_dataset_row_id_attribute_error(self): paper_url=paper_url, ) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.test_server() def test_create_dataset_row_id_attribute_inference(self): # meta-information @@ -1438,6 +1470,10 @@ def test_data_edit_non_critical_field(self): edited_dataset = openml.datasets.get_dataset(did) assert edited_dataset.description == desc + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.test_server() def test_data_edit_critical_field(self): # Case 2 @@ -1490,6 +1526,10 @@ def test_data_edit_requires_valid_dataset(self): description="xor operation dataset", ) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.test_server() def test_data_edit_cannot_edit_critical_field_if_dataset_has_task(self): # Need to own a dataset to be able to edit meta-data @@ -2008,4 +2048,4 @@ def test_get_dataset_parquet(requests_mock, test_files_directory): assert dataset._parquet_url is not None assert dataset.parquet_file is not None assert os.path.isfile(dataset.parquet_file) - assert dataset.data_file is None # is alias for arff path + assert dataset.data_file is None # is alias for arff path \ No newline at end of file diff --git a/tests/test_flows/test_flow.py b/tests/test_flows/test_flow.py index b942c0ab9..6f0de0a43 100644 --- a/tests/test_flows/test_flow.py +++ b/tests/test_flows/test_flow.py @@ -5,6 +5,7 @@ import copy import hashlib import re +import os import time from packaging.version import Version from unittest import mock @@ -33,7 +34,6 @@ from openml.testing import SimpleImputer, TestBase - class TestFlow(TestBase): _multiprocess_can_split_ = True @@ -180,6 +180,10 @@ def test_to_xml_from_xml(self): openml.flows.functions.assert_flows_equal(new_flow, flow) assert new_flow is not flow + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_publish_flow(self): @@ -222,6 +226,10 @@ def test_publish_existing_flow(self, flow_exists_mock): f"collected from {__file__.split('/')[-1]}: {flow.flow_id}", ) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_publish_flow_with_similar_components(self): @@ -273,6 +281,10 @@ def test_publish_flow_with_similar_components(self): TestBase._mark_entity_for_removal("flow", flow3.flow_id, flow3.name) TestBase.logger.info(f"collected from {__file__.split('/')[-1]}: {flow3.flow_id}") + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_semi_legal_flow(self): @@ -383,6 +395,10 @@ def get_sentinel(): flow_id = openml.flows.flow_exists(name, version) assert not flow_id + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_existing_flow_exists(self): @@ -424,6 +440,10 @@ def test_existing_flow_exists(self): ) assert downloaded_flow_id == flow.flow_id + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_sklearn_to_upload_to_flow(self): diff --git a/tests/test_flows/test_flow_functions.py b/tests/test_flows/test_flow_functions.py index ce0d5e782..035fabe4a 100644 --- a/tests/test_flows/test_flow_functions.py +++ b/tests/test_flows/test_flow_functions.py @@ -12,6 +12,7 @@ from unittest import mock from unittest.mock import patch +import os import pandas as pd import pytest import requests @@ -309,6 +310,10 @@ def test_get_flow1(self): flow = openml.flows.get_flow(1) assert flow.external_version is None + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_get_flow_reinstantiate_model(self): @@ -392,6 +397,10 @@ def test_get_flow_reinstantiate_flow_not_strict_pre_023(self): assert flow.flow_id is None assert "sklearn==0.19.1" not in flow.dependencies + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_get_flow_id(self): diff --git a/tests/test_openml/test_api_calls.py b/tests/test_openml/test_api_calls.py index 3f30f38ba..28d94d43a 100644 --- a/tests/test_openml/test_api_calls.py +++ b/tests/test_openml/test_api_calls.py @@ -7,6 +7,7 @@ import minio import pytest +import os import openml from openml.config import ConfigurationForExamples @@ -20,6 +21,10 @@ def test_too_long_uri(self): with pytest.raises(openml.exceptions.OpenMLServerError, match="URI too long!"): openml.datasets.list_datasets(data_id=list(range(10000))) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @unittest.mock.patch("time.sleep") @unittest.mock.patch("requests.Session") @pytest.mark.test_server() diff --git a/tests/test_runs/test_run.py b/tests/test_runs/test_run.py index 17349fca8..92db1817e 100644 --- a/tests/test_runs/test_run.py +++ b/tests/test_runs/test_run.py @@ -118,6 +118,10 @@ def _check_array(array, type_): else: assert run_prime_trace_content is None + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_to_from_filesystem_vanilla(self): @@ -153,6 +157,10 @@ def test_to_from_filesystem_vanilla(self): f"collected from {__file__.split('/')[-1]}: {run_prime.run_id}", ) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.flaky() @pytest.mark.test_server() @@ -189,6 +197,10 @@ def test_to_from_filesystem_search(self): f"collected from {__file__.split('/')[-1]}: {run_prime.run_id}", ) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_to_from_filesystem_no_model(self): @@ -295,6 +307,10 @@ def assert_run_prediction_data(task, run, model): assert_method(y_pred, saved_y_pred) assert_method(y_test, saved_y_test) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_publish_with_local_loaded_flow(self): @@ -339,6 +355,10 @@ def test_publish_with_local_loaded_flow(self): assert openml.flows.flow_exists(flow.name, flow.external_version) openml.runs.get_run(loaded_run.run_id) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_offline_and_online_run_identical(self): diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 9bc8d74fa..19cc1badf 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -397,6 +397,10 @@ def _check_sample_evaluations( assert evaluation > 0 assert evaluation < max_time_allowed + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_regression_on_classif_task(self): @@ -414,6 +418,10 @@ def test_run_regression_on_classif_task(self): task=task, ) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_check_erronous_sklearn_flow_fails(self): @@ -627,6 +635,10 @@ def _run_and_upload_regression( sentinel=sentinel, ) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_and_upload_logistic_regression(self): @@ -636,6 +648,10 @@ def test_run_and_upload_logistic_regression(self): n_test_obs = self.TEST_SERVER_TASK_SIMPLE["n_test_obs"] self._run_and_upload_classification(lr, task_id, n_missing_vals, n_test_obs, "62501") + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_and_upload_linear_regression(self): @@ -667,6 +683,10 @@ def test_run_and_upload_linear_regression(self): n_test_obs = self.TEST_SERVER_TASK_REGRESSION["n_test_obs"] self._run_and_upload_regression(lr, task_id, n_missing_vals, n_test_obs, "62501") + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_and_upload_pipeline_dummy_pipeline(self): @@ -681,6 +701,10 @@ def test_run_and_upload_pipeline_dummy_pipeline(self): n_test_obs = self.TEST_SERVER_TASK_SIMPLE["n_test_obs"] self._run_and_upload_classification(pipeline1, task_id, n_missing_vals, n_test_obs, "62501") + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -798,6 +822,10 @@ def test_run_and_upload_knn_pipeline(self, warnings_mock): call_count += 1 assert call_count == 3 + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_and_upload_gridsearch(self): @@ -821,6 +849,10 @@ def test_run_and_upload_gridsearch(self): ) assert len(run.trace.trace_iterations) == 9 + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_and_upload_randomsearch(self): @@ -854,6 +886,10 @@ def test_run_and_upload_randomsearch(self): trace = openml.runs.get_run_trace(run.run_id) assert len(trace.trace_iterations) == 5 + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_and_upload_maskedarrays(self): @@ -882,6 +918,10 @@ def test_run_and_upload_maskedarrays(self): ########################################################################## + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_learning_curve_task_1(self): @@ -907,6 +947,10 @@ def test_learning_curve_task_1(self): ) self._check_sample_evaluations(run.sample_evaluations, num_repeats, num_folds, num_samples) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_learning_curve_task_2(self): @@ -944,6 +988,10 @@ def test_learning_curve_task_2(self): ) self._check_sample_evaluations(run.sample_evaluations, num_repeats, num_folds, num_samples) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.21"), @@ -1023,6 +1071,10 @@ def _test_local_evaluations(self, run): assert alt_scores[idx] >= 0 assert alt_scores[idx] <= 1 + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_local_run_swapped_parameter_order_model(self): @@ -1039,6 +1091,10 @@ def test_local_run_swapped_parameter_order_model(self): self._test_local_evaluations(run) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1068,6 +1124,10 @@ def test_local_run_swapped_parameter_order_flow(self): self._test_local_evaluations(run) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1106,6 +1166,10 @@ def test_online_run_metric_score(self): self._test_local_evaluations(run) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1168,6 +1232,10 @@ def test_initialize_model_from_run(self): assert flowS.components["Imputer"].parameters["strategy"] == '"most_frequent"' assert flowS.components["VarianceThreshold"].parameters["threshold"] == "0.05" + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1228,6 +1296,10 @@ def test__run_exists(self): run_ids = run_exists(task.task_id, setup_exists) assert run_ids, (run_ids, clf) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_with_illegal_flow_id(self): @@ -1248,6 +1320,10 @@ def test_run_with_illegal_flow_id(self): avoid_duplicate_runs=True, ) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_with_illegal_flow_id_after_load(self): @@ -1306,6 +1382,10 @@ def test_run_with_illegal_flow_id_1(self): avoid_duplicate_runs=True, ) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_with_illegal_flow_id_1_after_load(self): @@ -1345,6 +1425,10 @@ def test_run_with_illegal_flow_id_1_after_load(self): loaded_run.publish, ) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1575,6 +1659,10 @@ def test_get_runs_list_by_tag(self): runs = openml.runs.list_runs(tag="curves", size=2) assert len(runs) >= 1 + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1612,6 +1700,10 @@ def test_run_on_dataset_with_missing_labels_dataframe(self): # repeat, fold, row_id, 6 confidences, prediction and correct label assert len(row) == 12 + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1666,6 +1758,10 @@ def test_get_uncached_run(self): with pytest.raises(openml.exceptions.OpenMLCacheException): openml.runs.functions._get_cached_run(10) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_flow_on_task_downloaded_flow(self): @@ -1767,7 +1863,10 @@ def test_format_prediction_task_regression(self): self.assertListEqual(res, [0] * 5) - + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", @@ -1865,6 +1964,10 @@ def test_delete_unknown_run(mock_delete, test_files_directory, test_api_key): assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key") +@pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", +) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.21"), @@ -1947,6 +2050,10 @@ def test__run_task_get_arffcontent_2(parallel_mock): ) +@pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", +) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.21"), @@ -2016,6 +2123,7 @@ def test_joblib_backends(parallel_mock, n_jobs, backend, call_count): n_jobs=n_jobs, ) from openml_sklearn import SklearnExtension + extension = SklearnExtension() with parallel_backend(backend, n_jobs=n_jobs): res = openml.runs.functions._run_task_get_arffcontent( @@ -2032,4 +2140,4 @@ def test_joblib_backends(parallel_mock, n_jobs, backend, call_count): # *_time_millis_* not recorded when n_jobs = -1 assert len(res[2]["predictive_accuracy"][0]) == 10 assert len(res[3]["predictive_accuracy"][0]) == 10 - assert parallel_mock.call_count == call_count + assert parallel_mock.call_count == call_count \ No newline at end of file diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index 5480ce6f8..600d3edbc 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -4,7 +4,7 @@ import hashlib import time import unittest.mock - +import os import pandas as pd import pytest import sklearn.base @@ -34,6 +34,10 @@ def setUp(self): self.extension = SklearnExtension() super().setUp() + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_nonexisting_setup_exists(self): @@ -82,6 +86,10 @@ def _existing_setup_exists(self, classif): setup_id = openml.setups.setup_exists(flow) assert setup_id == run.setup_id + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_existing_setup_exists_1(self): @@ -98,12 +106,20 @@ def side_effect(self): nb = sklearn.naive_bayes.GaussianNB() self._existing_setup_exists(nb) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_exisiting_setup_exists_2(self): # Check a flow with one hyperparameter self._existing_setup_exists(sklearn.naive_bayes.GaussianNB()) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_existing_setup_exists_3(self): diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py index df3c0a3b6..2ed61ec0f 100644 --- a/tests/test_tasks/test_task_functions.py +++ b/tests/test_tasks/test_task_functions.py @@ -167,6 +167,10 @@ def test_get_task(self): os.path.join(openml.config.get_cache_directory(), "datasets", "1", "dataset_1.pq") ) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.test_server() def test_get_task_lazy(self): task = openml.tasks.get_task(2, download_data=False) # anneal; crossvalidation @@ -224,6 +228,10 @@ def test_get_task_different_types(self): # Issue 538, get_task failing with clustering task. openml.tasks.functions.get_task(126033) + @pytest.mark.skipif( + os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", + reason="Pending resolution of #1657", + ) @pytest.mark.test_server() def test_download_split(self): task = openml.tasks.get_task(1) # anneal; crossvalidation From 25ba6f857e9f4493440a4c1897bdc6eaa1ab5169 Mon Sep 17 00:00:00 2001 From: Pieter Gijsbers Date: Tue, 10 Mar 2026 12:11:39 +0100 Subject: [PATCH 45/46] [MNT] Update CI/CD local server deployment and dependency matrix (#1697) Originally started in #1629, this PR spins up the services within CI uses that as a local test server for the ubuntu-based tests, closing #1614. It also updates the test matrix to make sure included dependencies are only those that have a release on PyPI, and further restriction pandas 3.x installs to only scikit-learn 1.7 and up. Older scikit-learn versions do not play well with scikit-learn 1.6 or below. Finally, it updates some tests to reflect the new test database image state. --------- Co-authored-by: Satvik Mishra <112589278+satvshr@users.noreply.github.com> Co-authored-by: Armaghan Shakir --- .github/workflows/test.yml | 57 ++++++--- openml/tasks/task.py | 37 +++--- pyproject.toml | 1 + tests/conftest.py | 9 +- tests/test_datasets/test_dataset_functions.py | 40 ------- tests/test_flows/test_flow.py | 20 ---- tests/test_flows/test_flow_functions.py | 8 -- tests/test_openml/test_api_calls.py | 4 - tests/test_runs/test_run.py | 21 +--- tests/test_runs/test_run_functions.py | 110 +----------------- tests/test_setups/test_setup_functions.py | 16 --- tests/test_study/test_study_functions.py | 2 +- tests/test_tasks/test_task_functions.py | 8 -- tests/test_utils/test_utils.py | 4 +- 14 files changed, 76 insertions(+), 261 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7d5d48ac0..dc0995fc6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,15 +34,27 @@ jobs: sklearn-only: ["true"] exclude: - # incompatible version combinations + # (python, sklearn) combinations for which there is no PyPI release + # scikit-learn 1.3 - python-version: "3.13" scikit-learn: "1.3.*" - - python-version: "3.13" - scikit-learn: "1.4.*" - python-version: "3.14" scikit-learn: "1.3.*" + # scikit-learn 1.4 + - python-version: "3.13" + scikit-learn: "1.4.*" - python-version: "3.14" scikit-learn: "1.4.*" + # scikit-learn 1.5 + - python-version: "3.14" + scikit-learn: "1.5.*" + # scikit-learn 1.6 + - python-version: "3.14" + scikit-learn: "1.6.*" + # scikit-learn 1.7 is installed with pandas 3 + - python-version: "3.10" + scikit-learn: "1.7.*" + include: # Full test run on ubuntu, 3.14 @@ -64,14 +76,6 @@ jobs: sklearn-only: "false" code-cov: true - # Pandas 2 run - - os: ubuntu-latest - python-version: "3.12" - scikit-learn: "1.5.*" - sklearn-only: "false" - pandas-version: "2.*" - code-cov: false - steps: - uses: actions/checkout@v6 with: @@ -82,15 +86,21 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install test dependencies, scikit-learn, and optional pandas + - name: Install test dependencies, scikit-learn, and pandas shell: bash run: | python -m pip install --upgrade pip pip install -e .[test] scikit-learn==${{ matrix.scikit-learn }} - - if [ "${{ matrix.pandas-version }}" != "" ]; then - echo "Installing specific pandas version: ${{ matrix.pandas-version }}" - pip install "pandas==${{ matrix.pandas-version }}" + + # scikit-learn 1.7+ requires pandas 3.x, earlier versions use pandas 2.x + version="${{ matrix.scikit-learn }}" + major=$(echo "$version" | cut -d. -f1) + minor=$(echo "$version" | cut -d. -f2) + + if [[ "$major" -gt 1 ]] || { [[ "$major" -eq 1 ]] && [[ "$minor" -ge 7 ]]; }; then + pip install "pandas==3.*" + else + pip install "pandas==2.*" fi - name: Store repository status @@ -103,14 +113,18 @@ jobs: - name: Clone Services if: matrix.os == 'ubuntu-latest' + id: clone-services run: | git clone --depth 1 https://github.com/openml/services.git - name: Start Docker Services + id: start-services if: matrix.os == 'ubuntu-latest' working-directory: ./services run: | - docker compose --profile rest-api --profile minio up -d + chmod -R a+rw ./data + chmod -R a+rw ./logs + docker compose --profile rest-api --profile minio --profile evaluation-engine up -d echo "Waiting for PHP API to boot..." timeout 60s bash -c 'until [ "$(docker inspect -f {{.State.Health.Status}} openml-php-rest-api)" == "healthy" ]; do sleep 5; done' @@ -118,6 +132,8 @@ jobs: echo "Final Verification: Gateway Connectivity..." curl -sSfL http://localhost:8000/api/v1/xml/data/1 | head -n 15 + docker container ls + - name: Show installed dependencies run: python -m pip list @@ -173,8 +189,13 @@ jobs: fail_ci_if_error: true verbose: true + - name: Dump server logs + if: always() && steps.start-services.outcome == 'success' + run: | + docker logs openml-php-rest-api -t + - name: Cleanup Docker setup - if: matrix.os == 'ubuntu-latest' && always() + if: always() && steps.clone-services.outcome == 'success' run: | sudo rm -rf services diff --git a/openml/tasks/task.py b/openml/tasks/task.py index 385b1f949..cb1c80e8f 100644 --- a/openml/tasks/task.py +++ b/openml/tasks/task.py @@ -1,14 +1,16 @@ # License: BSD 3-Clause from __future__ import annotations +import logging import warnings from abc import ABC from collections.abc import Sequence from enum import Enum -from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar from typing_extensions import TypedDict +import arff + import openml._api_calls import openml.config from openml import datasets @@ -22,6 +24,9 @@ import pandas as pd +logger = logging.getLogger(__name__) + + # TODO(eddiebergman): Should use `auto()` but might be too late if these numbers are used # and stored on server. class TaskType(Enum): @@ -178,18 +183,6 @@ def get_train_test_split_indices( return self.split.get(repeat=repeat, fold=fold, sample=sample) - def _download_split(self, cache_file: Path) -> None: - # TODO(eddiebergman): Not sure about this try to read and error approach - try: - with cache_file.open(encoding="utf8"): - pass - except OSError: - split_url = self.estimation_procedure["data_splits_url"] - openml._api_calls._download_text_file( - source=str(split_url), - output_path=str(cache_file), - ) - def download_split(self) -> OpenMLSplit: """Download the OpenML split for a given task.""" # TODO(eddiebergman): Can this every be `None`? @@ -199,9 +192,23 @@ def download_split(self) -> OpenMLSplit: try: split = OpenMLSplit._from_arff_file(cached_split_file) - except OSError: + logger.debug("Loaded file from cache: %s", str(cached_split_file)) + except (OSError, arff.BadDataFormat): + logger.info("Failed to load file from cache: %s", str(cached_split_file)) + if cached_split_file.exists(): + logger.debug("Cleaning up old file") + cached_split_file.unlink() # Next, download and cache the associated split file - self._download_split(cached_split_file) + split_url = self.estimation_procedure["data_splits_url"] + openml._api_calls._download_text_file( + source=str(split_url), + output_path=str(cached_split_file), + ) + if cached_split_file.exists(): + logger.info("New file created of size %d", cached_split_file.stat().st_size) + else: + logger.info("Failed to create new file") + split = OpenMLSplit._from_arff_file(cached_split_file) return split diff --git a/pyproject.toml b/pyproject.toml index 47013271d..8c463968b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,6 +126,7 @@ version = {attr = "openml.__version__.__version__"} # https://docs.pytest.org/en/7.2.x/reference/reference.html#ini-options-ref [tool.pytest.ini_options] +log_level="DEBUG" testpaths = ["tests"] minversion = "7.0" xfail_strict = true diff --git a/tests/conftest.py b/tests/conftest.py index d1bc23d4f..1967f1fad 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -286,12 +286,19 @@ def with_server(request): @pytest.fixture(autouse=True) def with_test_cache(test_files_directory, request): + # Skip this fixture for TestBase subclasses - they manage their own cache directory + # in setUp()/tearDown(). Having both mechanisms fight over the global config + # causes race conditions. + if request.instance is not None and isinstance(request.instance, TestBase): + yield + return + if not test_files_directory.exists(): raise ValueError( f"Cannot find test cache dir, expected it to be {test_files_directory!s}!", ) _root_cache_directory = openml.config._root_cache_directory - tmp_cache = test_files_directory / request.node.name + tmp_cache = test_files_directory / request.node.nodeid.replace("/", ".").replace("::", ".") openml.config.set_root_cache_directory(tmp_cache) yield openml.config.set_root_cache_directory(_root_cache_directory) diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index 10517a3e1..974fb36ef 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -530,10 +530,6 @@ def test_deletion_of_cache_dir_faulty_download(self, patch): datasets_cache_dir = os.path.join(openml.config.get_cache_directory(), "datasets") assert len(os.listdir(datasets_cache_dir)) == 0 - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.test_server() def test_publish_dataset(self): arff_file_path = self.static_cache_dir / "org" / "openml" / "test" / "datasets" / "2" / "dataset.arff" @@ -570,10 +566,6 @@ def test__retrieve_class_labels(self): labels = custom_ds.retrieve_class_labels(target_name=custom_ds.features[31].name) assert labels == ["COIL", "SHEET"] - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.test_server() def test_upload_dataset_with_url(self): dataset = OpenMLDataset( @@ -697,10 +689,6 @@ def test_attributes_arff_from_df_unknown_dtype(self): with pytest.raises(ValueError, match=err_msg): attributes_arff_from_df(df) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.test_server() def test_create_dataset_numpy(self): data = np.array([[1, 2, 3], [1.2, 2.5, 3.8], [2, 5, 8], [0, 1, 0]]).T @@ -735,10 +723,6 @@ def test_create_dataset_numpy(self): ), "Uploaded arff does not match original one" assert _get_online_dataset_format(dataset.id) == "arff", "Wrong format for dataset" - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.test_server() def test_create_dataset_list(self): data = [ @@ -794,10 +778,6 @@ def test_create_dataset_list(self): ), "Uploaded ARFF does not match original one" assert _get_online_dataset_format(dataset.id) == "arff", "Wrong format for dataset" - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.test_server() def test_create_dataset_sparse(self): # test the scipy.sparse.coo_matrix @@ -946,10 +926,6 @@ def test_get_online_dataset_format(self): dataset_id ), "The format of the ARFF files is different" - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.test_server() def test_create_dataset_pandas(self): data = [ @@ -1175,10 +1151,6 @@ def test_ignore_attributes_dataset(self): paper_url=paper_url, ) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.test_server() def test_publish_fetch_ignore_attribute(self): """Test to upload and retrieve dataset and check ignore_attributes""" @@ -1298,10 +1270,6 @@ def test_create_dataset_row_id_attribute_error(self): paper_url=paper_url, ) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.test_server() def test_create_dataset_row_id_attribute_inference(self): # meta-information @@ -1470,10 +1438,6 @@ def test_data_edit_non_critical_field(self): edited_dataset = openml.datasets.get_dataset(did) assert edited_dataset.description == desc - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.test_server() def test_data_edit_critical_field(self): # Case 2 @@ -1526,10 +1490,6 @@ def test_data_edit_requires_valid_dataset(self): description="xor operation dataset", ) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.test_server() def test_data_edit_cannot_edit_critical_field_if_dataset_has_task(self): # Need to own a dataset to be able to edit meta-data diff --git a/tests/test_flows/test_flow.py b/tests/test_flows/test_flow.py index 6f0de0a43..4e391fd3b 100644 --- a/tests/test_flows/test_flow.py +++ b/tests/test_flows/test_flow.py @@ -180,10 +180,6 @@ def test_to_xml_from_xml(self): openml.flows.functions.assert_flows_equal(new_flow, flow) assert new_flow is not flow - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_publish_flow(self): @@ -226,10 +222,6 @@ def test_publish_existing_flow(self, flow_exists_mock): f"collected from {__file__.split('/')[-1]}: {flow.flow_id}", ) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_publish_flow_with_similar_components(self): @@ -281,10 +273,6 @@ def test_publish_flow_with_similar_components(self): TestBase._mark_entity_for_removal("flow", flow3.flow_id, flow3.name) TestBase.logger.info(f"collected from {__file__.split('/')[-1]}: {flow3.flow_id}") - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_semi_legal_flow(self): @@ -395,10 +383,6 @@ def get_sentinel(): flow_id = openml.flows.flow_exists(name, version) assert not flow_id - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_existing_flow_exists(self): @@ -440,10 +424,6 @@ def test_existing_flow_exists(self): ) assert downloaded_flow_id == flow.flow_id - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_sklearn_to_upload_to_flow(self): diff --git a/tests/test_flows/test_flow_functions.py b/tests/test_flows/test_flow_functions.py index 035fabe4a..14bb78060 100644 --- a/tests/test_flows/test_flow_functions.py +++ b/tests/test_flows/test_flow_functions.py @@ -310,10 +310,6 @@ def test_get_flow1(self): flow = openml.flows.get_flow(1) assert flow.external_version is None - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_get_flow_reinstantiate_model(self): @@ -397,10 +393,6 @@ def test_get_flow_reinstantiate_flow_not_strict_pre_023(self): assert flow.flow_id is None assert "sklearn==0.19.1" not in flow.dependencies - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_get_flow_id(self): diff --git a/tests/test_openml/test_api_calls.py b/tests/test_openml/test_api_calls.py index 28d94d43a..7ece4309a 100644 --- a/tests/test_openml/test_api_calls.py +++ b/tests/test_openml/test_api_calls.py @@ -21,10 +21,6 @@ def test_too_long_uri(self): with pytest.raises(openml.exceptions.OpenMLServerError, match="URI too long!"): openml.datasets.list_datasets(data_id=list(range(10000))) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @unittest.mock.patch("time.sleep") @unittest.mock.patch("requests.Session") @pytest.mark.test_server() diff --git a/tests/test_runs/test_run.py b/tests/test_runs/test_run.py index 92db1817e..22a8bc936 100644 --- a/tests/test_runs/test_run.py +++ b/tests/test_runs/test_run.py @@ -118,10 +118,6 @@ def _check_array(array, type_): else: assert run_prime_trace_content is None - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_to_from_filesystem_vanilla(self): @@ -157,10 +153,6 @@ def test_to_from_filesystem_vanilla(self): f"collected from {__file__.split('/')[-1]}: {run_prime.run_id}", ) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.flaky() @pytest.mark.test_server() @@ -197,10 +189,6 @@ def test_to_from_filesystem_search(self): f"collected from {__file__.split('/')[-1]}: {run_prime.run_id}", ) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_to_from_filesystem_no_model(self): @@ -307,10 +295,6 @@ def assert_run_prediction_data(task, run, model): assert_method(y_pred, saved_y_pred) assert_method(y_test, saved_y_test) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_publish_with_local_loaded_flow(self): @@ -355,12 +339,9 @@ def test_publish_with_local_loaded_flow(self): assert openml.flows.flow_exists(flow.name, flow.external_version) openml.runs.get_run(loaded_run.run_id) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() + @pytest.mark.skip(reason="https://github.com/openml/openml-python/issues/1586") def test_offline_and_online_run_identical(self): extension = SklearnExtension() diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 19cc1badf..8d5a00f9b 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -397,10 +397,6 @@ def _check_sample_evaluations( assert evaluation > 0 assert evaluation < max_time_allowed - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_regression_on_classif_task(self): @@ -418,10 +414,6 @@ def test_run_regression_on_classif_task(self): task=task, ) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_check_erronous_sklearn_flow_fails(self): @@ -635,10 +627,6 @@ def _run_and_upload_regression( sentinel=sentinel, ) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_and_upload_logistic_regression(self): @@ -648,10 +636,6 @@ def test_run_and_upload_logistic_regression(self): n_test_obs = self.TEST_SERVER_TASK_SIMPLE["n_test_obs"] self._run_and_upload_classification(lr, task_id, n_missing_vals, n_test_obs, "62501") - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_and_upload_linear_regression(self): @@ -683,10 +667,6 @@ def test_run_and_upload_linear_regression(self): n_test_obs = self.TEST_SERVER_TASK_REGRESSION["n_test_obs"] self._run_and_upload_regression(lr, task_id, n_missing_vals, n_test_obs, "62501") - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_and_upload_pipeline_dummy_pipeline(self): @@ -701,10 +681,6 @@ def test_run_and_upload_pipeline_dummy_pipeline(self): n_test_obs = self.TEST_SERVER_TASK_SIMPLE["n_test_obs"] self._run_and_upload_classification(pipeline1, task_id, n_missing_vals, n_test_obs, "62501") - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -822,10 +798,6 @@ def test_run_and_upload_knn_pipeline(self, warnings_mock): call_count += 1 assert call_count == 3 - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_and_upload_gridsearch(self): @@ -849,10 +821,6 @@ def test_run_and_upload_gridsearch(self): ) assert len(run.trace.trace_iterations) == 9 - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_and_upload_randomsearch(self): @@ -886,10 +854,6 @@ def test_run_and_upload_randomsearch(self): trace = openml.runs.get_run_trace(run.run_id) assert len(trace.trace_iterations) == 5 - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_and_upload_maskedarrays(self): @@ -918,10 +882,6 @@ def test_run_and_upload_maskedarrays(self): ########################################################################## - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_learning_curve_task_1(self): @@ -947,10 +907,6 @@ def test_learning_curve_task_1(self): ) self._check_sample_evaluations(run.sample_evaluations, num_repeats, num_folds, num_samples) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_learning_curve_task_2(self): @@ -988,10 +944,6 @@ def test_learning_curve_task_2(self): ) self._check_sample_evaluations(run.sample_evaluations, num_repeats, num_folds, num_samples) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.21"), @@ -1071,10 +1023,6 @@ def _test_local_evaluations(self, run): assert alt_scores[idx] >= 0 assert alt_scores[idx] <= 1 - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_local_run_swapped_parameter_order_model(self): @@ -1091,11 +1039,8 @@ def test_local_run_swapped_parameter_order_model(self): self._test_local_evaluations(run) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() + @pytest.mark.skip("https://github.com/openml/openml-python/issues/1586") @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", @@ -1124,10 +1069,7 @@ def test_local_run_swapped_parameter_order_flow(self): self._test_local_evaluations(run) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) + @pytest.mark.skip(reason="https://github.com/openml/openml-python/issues/1586") @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1166,10 +1108,6 @@ def test_online_run_metric_score(self): self._test_local_evaluations(run) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1232,10 +1170,6 @@ def test_initialize_model_from_run(self): assert flowS.components["Imputer"].parameters["strategy"] == '"most_frequent"' assert flowS.components["VarianceThreshold"].parameters["threshold"] == "0.05" - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1296,10 +1230,6 @@ def test__run_exists(self): run_ids = run_exists(task.task_id, setup_exists) assert run_ids, (run_ids, clf) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_with_illegal_flow_id(self): @@ -1320,10 +1250,6 @@ def test_run_with_illegal_flow_id(self): avoid_duplicate_runs=True, ) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_with_illegal_flow_id_after_load(self): @@ -1382,10 +1308,6 @@ def test_run_with_illegal_flow_id_1(self): avoid_duplicate_runs=True, ) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_with_illegal_flow_id_1_after_load(self): @@ -1425,10 +1347,6 @@ def test_run_with_illegal_flow_id_1_after_load(self): loaded_run.publish, ) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1659,10 +1577,6 @@ def test_get_runs_list_by_tag(self): runs = openml.runs.list_runs(tag="curves", size=2) assert len(runs) >= 1 - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1700,10 +1614,6 @@ def test_run_on_dataset_with_missing_labels_dataframe(self): # repeat, fold, row_id, 6 confidences, prediction and correct label assert len(row) == 12 - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), @@ -1758,10 +1668,6 @@ def test_get_uncached_run(self): with pytest.raises(openml.exceptions.OpenMLCacheException): openml.runs.functions._get_cached_run(10) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_run_flow_on_task_downloaded_flow(self): @@ -1863,10 +1769,6 @@ def test_format_prediction_task_regression(self): self.assertListEqual(res, [0] * 5) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @unittest.skipIf( Version(sklearn.__version__) < Version("0.20"), reason="SimpleImputer doesn't handle mixed type DataFrame as input", @@ -1964,10 +1866,6 @@ def test_delete_unknown_run(mock_delete, test_files_directory, test_api_key): assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key") -@pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", -) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.21"), @@ -2050,10 +1948,6 @@ def test__run_task_get_arffcontent_2(parallel_mock): ) -@pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", -) @pytest.mark.sklearn() @unittest.skipIf( Version(sklearn.__version__) < Version("0.21"), diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index 600d3edbc..30943ea70 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -34,10 +34,6 @@ def setUp(self): self.extension = SklearnExtension() super().setUp() - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_nonexisting_setup_exists(self): @@ -86,10 +82,6 @@ def _existing_setup_exists(self, classif): setup_id = openml.setups.setup_exists(flow) assert setup_id == run.setup_id - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_existing_setup_exists_1(self): @@ -106,20 +98,12 @@ def side_effect(self): nb = sklearn.naive_bayes.GaussianNB() self._existing_setup_exists(nb) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_exisiting_setup_exists_2(self): # Check a flow with one hyperparameter self._existing_setup_exists(sklearn.naive_bayes.GaussianNB()) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.sklearn() @pytest.mark.test_server() def test_existing_setup_exists_3(self): diff --git a/tests/test_study/test_study_functions.py b/tests/test_study/test_study_functions.py index 2a2d276ec..7dc6b6d2a 100644 --- a/tests/test_study/test_study_functions.py +++ b/tests/test_study/test_study_functions.py @@ -227,7 +227,7 @@ def test_study_attach_illegal(self): run_list = openml.runs.list_runs(size=10) assert len(run_list) == 10 run_list_more = openml.runs.list_runs(size=20) - assert len(run_list_more) == 20 + assert len(run_list_more) > 10 # a fresh db should have 15 evaluated runs study = openml.study.create_study( alias=None, diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py index 2ed61ec0f..df3c0a3b6 100644 --- a/tests/test_tasks/test_task_functions.py +++ b/tests/test_tasks/test_task_functions.py @@ -167,10 +167,6 @@ def test_get_task(self): os.path.join(openml.config.get_cache_directory(), "datasets", "1", "dataset_1.pq") ) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.test_server() def test_get_task_lazy(self): task = openml.tasks.get_task(2, download_data=False) # anneal; crossvalidation @@ -228,10 +224,6 @@ def test_get_task_different_types(self): # Issue 538, get_task failing with clustering task. openml.tasks.functions.get_task(126033) - @pytest.mark.skipif( - os.getenv("OPENML_USE_LOCAL_SERVICES") == "true", - reason="Pending resolution of #1657", - ) @pytest.mark.test_server() def test_download_split(self): task = openml.tasks.get_task(1) # anneal; crossvalidation diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index 38e004bfb..75f24ebf0 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -34,7 +34,7 @@ def min_number_setups_on_test_server() -> int: @pytest.fixture() def min_number_runs_on_test_server() -> int: """After a reset at least 21 runs are on the test server""" - return 21 + return 15 @pytest.fixture() @@ -178,4 +178,4 @@ def test_get_cache_size(config_mock,tmp_path): sub_dir.mkdir() (sub_dir / "nested_file.txt").write_bytes(b"b" * 100) - assert openml.utils.get_cache_size() == 100 \ No newline at end of file + assert openml.utils.get_cache_size() == 100 From 8a5532fe4444da1947143cae7379b6c55554782b Mon Sep 17 00:00:00 2001 From: Satvik Mishra <112589278+satvshr@users.noreply.github.com> Date: Thu, 12 Mar 2026 14:26:12 +0530 Subject: [PATCH 46/46] [ENH] Improve Global Config Architecture (#1577) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### Metadata * Reference Issue: fixes #1564 * New Tests Added: No * Documentation Updated: No * Change Log Entry: Replaces global variables with an instance of a dataclass `OpenMLConfig`. #### Details * What does this PR implement/fix? Explain your changes. * Mainly Introduces an `OpenMLConfig` dataclass and an `OpenMLConfigManager` class to accumulate all config functions and varialbles into one class, and a single module-level instance `_config` in `openml.config`. * Reads/parses configuration defaults from `OpenMLConfig`. * Adds `__getattr__` so attribute reads on openml.config forward to the dataclass (preserving read compatibility). * Changes `openml/__init__.py` to route to the instance of `OpenMLConfigManager` when `openml.config` is called. * Moved `ConfigurationForExamples` inside `OpenMLConfigManager` to preserve API. * Why is this change necessary? What is the problem it solves? * consolidates defaults and runtime state into one place, * simplifies consistent parsing and writing of the config file, * removes the use of global variables. --------- Co-authored-by: Franz Király Co-authored-by: Armaghan Shakir --- openml/__init__.py | 9 +- openml/_api_calls.py | 21 +- openml/_config.py | 459 +++++++++++++++ openml/base.py | 1 - openml/cli.py | 19 +- openml/config.py | 529 ------------------ openml/datasets/dataset.py | 6 +- openml/datasets/functions.py | 6 +- openml/evaluations/evaluation.py | 1 - openml/runs/functions.py | 13 +- openml/setups/functions.py | 5 +- openml/setups/setup.py | 1 - openml/study/functions.py | 1 - openml/study/study.py | 4 +- openml/tasks/task.py | 1 - openml/utils/_openml.py | 7 +- .../test_evaluations_example.py | 6 +- tests/test_openml/test_api_calls.py | 1 - tests/test_openml/test_config.py | 7 +- 19 files changed, 516 insertions(+), 581 deletions(-) create mode 100644 openml/_config.py delete mode 100644 openml/config.py diff --git a/openml/__init__.py b/openml/__init__.py index ae5db261f..9a457c146 100644 --- a/openml/__init__.py +++ b/openml/__init__.py @@ -18,9 +18,11 @@ # License: BSD 3-Clause from __future__ import annotations +from typing import TYPE_CHECKING + from . import ( _api_calls, - config, + _config as _config_module, datasets, evaluations, exceptions, @@ -49,6 +51,11 @@ OpenMLTask, ) +if TYPE_CHECKING: + from ._config import OpenMLConfigManager + +config: OpenMLConfigManager = _config_module.__config + def populate_cache( task_ids: list[int] | None = None, diff --git a/openml/_api_calls.py b/openml/_api_calls.py index 5da635c70..179c814e7 100644 --- a/openml/_api_calls.py +++ b/openml/_api_calls.py @@ -12,6 +12,7 @@ import xml import zipfile from pathlib import Path +from typing import cast import minio import requests @@ -19,7 +20,8 @@ import xmltodict from urllib3 import ProxyManager -from . import config +import openml + from .__version__ import __version__ from .exceptions import ( OpenMLAuthenticationError, @@ -70,7 +72,7 @@ def resolve_env_proxies(url: str) -> str | None: def _create_url_from_endpoint(endpoint: str) -> str: - url = config.server + url = cast("str", openml.config.server) if not url.endswith("/"): url += "/" url += endpoint @@ -171,7 +173,7 @@ def _download_minio_file( bucket_name=bucket, object_name=object_name, file_path=str(destination), - progress=ProgressBar() if config.show_progress else None, + progress=ProgressBar() if openml.config.show_progress else None, request_headers=_HEADERS, ) if destination.is_file() and destination.suffix == ".zip": @@ -300,7 +302,8 @@ def _file_id_to_url(file_id: int, filename: str | None = None) -> str: Presents the URL how to download a given file id filename is optional """ - openml_url = config.server.split("/api/") + openml_server = cast("str", openml.config.server) + openml_url = openml_server.split("/api/") url = openml_url[0] + f"/data/download/{file_id!s}" if filename is not None: url += "/" + filename @@ -316,7 +319,7 @@ def _read_url_files( and sending file_elements as files """ data = {} if data is None else data - data["api_key"] = config.apikey + data["api_key"] = openml.config.apikey if file_elements is None: file_elements = {} # Using requests.post sets header 'Accept-encoding' automatically to @@ -336,8 +339,8 @@ def __read_url( md5_checksum: str | None = None, ) -> requests.Response: data = {} if data is None else data - if config.apikey: - data["api_key"] = config.apikey + if openml.config.apikey: + data["api_key"] = openml.config.apikey return _send_request( request_method=request_method, url=url, @@ -362,10 +365,10 @@ def _send_request( # noqa: C901, PLR0912 files: FILE_ELEMENTS_TYPE | None = None, md5_checksum: str | None = None, ) -> requests.Response: - n_retries = max(1, config.connection_n_retries) + n_retries = max(1, openml.config.connection_n_retries) response: requests.Response | None = None - delay_method = _human_delay if config.retry_policy == "human" else _robot_delay + delay_method = _human_delay if openml.config.retry_policy == "human" else _robot_delay # Error to raise in case of retrying too often. Will be set to the last observed exception. retry_raise_e: Exception | None = None diff --git a/openml/_config.py b/openml/_config.py new file mode 100644 index 000000000..a7034b9b4 --- /dev/null +++ b/openml/_config.py @@ -0,0 +1,459 @@ +"""Store module level information like the API key, cache directory and the server""" + +# License: BSD 3-Clause +from __future__ import annotations + +import configparser +import logging +import logging.handlers +import os +import platform +import shutil +import warnings +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass, field, fields, replace +from io import StringIO +from pathlib import Path +from typing import Any, ClassVar, Literal, cast +from urllib.parse import urlparse + +logger = logging.getLogger(__name__) +openml_logger = logging.getLogger("openml") + + +def _resolve_default_cache_dir() -> Path: + user_defined_cache_dir = os.environ.get("OPENML_CACHE_DIR") + if user_defined_cache_dir is not None: + return Path(user_defined_cache_dir) + + if platform.system().lower() != "linux": + return Path("~", ".openml").expanduser() + + xdg_cache_home = os.environ.get("XDG_CACHE_HOME") + if xdg_cache_home is None: + return Path("~", ".cache", "openml").expanduser() + + cache_dir = Path(xdg_cache_home) / "openml" + if cache_dir.exists(): + return cache_dir + + heuristic_dir_for_backwards_compat = Path(xdg_cache_home) / "org" / "openml" + if not heuristic_dir_for_backwards_compat.exists(): + return cache_dir + + root_dir_to_delete = Path(xdg_cache_home) / "org" + openml_logger.warning( + "An old cache directory was found at '%s'. This directory is no longer used by " + "OpenML-Python. To silence this warning you would need to delete the old cache " + "directory. The cached files will then be located in '%s'.", + root_dir_to_delete, + cache_dir, + ) + return Path(xdg_cache_home) + + +@dataclass +class OpenMLConfig: + """Dataclass storing the OpenML configuration.""" + + apikey: str | None = "" + server: str = "https://www.openml.org/api/v1/xml" + cachedir: Path = field(default_factory=_resolve_default_cache_dir) + avoid_duplicate_runs: bool = False + retry_policy: Literal["human", "robot"] = "human" + connection_n_retries: int = 5 + show_progress: bool = False + + def __setattr__(self, name: str, value: Any) -> None: + if name == "apikey" and not isinstance(value, (type(None), str)): + raise TypeError("apikey must be a string or None") + + super().__setattr__(name, value) + + +class OpenMLConfigManager: + """The OpenMLConfigManager manages the configuration of the openml-python package.""" + + def __init__(self) -> None: + self.console_handler: logging.StreamHandler | None = None + self.file_handler: logging.handlers.RotatingFileHandler | None = None + + self.OPENML_CACHE_DIR_ENV_VAR = "OPENML_CACHE_DIR" + self.OPENML_SKIP_PARQUET_ENV_VAR = "OPENML_SKIP_PARQUET" + self._TEST_SERVER_NORMAL_USER_KEY = "normaluser" + self.OPENML_TEST_SERVER_ADMIN_KEY_ENV_VAR = "OPENML_TEST_SERVER_ADMIN_KEY" + self.TEST_SERVER_URL = "https://test.openml.org" + + self._config: OpenMLConfig = OpenMLConfig() + # for legacy test `test_non_writable_home` + self._defaults: dict[str, Any] = OpenMLConfig().__dict__.copy() + self._root_cache_directory: Path = self._config.cachedir + + self.logger = logger + self.openml_logger = openml_logger + + self._examples = ConfigurationForExamples(self) + + self._setup() + + def __getattr__(self, name: str) -> Any: + if hasattr(self._config, name): + return getattr(self._config, name) + raise AttributeError(f"{type(self).__name__!r} object has no attribute {name!r}") + + _FIELDS: ClassVar[set[str]] = {f.name for f in fields(OpenMLConfig)} + + def __setattr__(self, name: str, value: Any) -> None: + # during __init__ before _config exists + if name in { + "_config", + "_root_cache_directory", + "console_handler", + "file_handler", + "logger", + "openml_logger", + "_examples", + "OPENML_CACHE_DIR_ENV_VAR", + "OPENML_SKIP_PARQUET_ENV_VAR", + "_TEST_SERVER_NORMAL_USER_KEY", + }: + return object.__setattr__(self, name, value) + + if name in self._FIELDS: + # write into dataclass, not manager (prevents shadowing) + if name == "cachedir": + object.__setattr__(self, "_root_cache_directory", Path(value)) + object.__setattr__(self, "_config", replace(self._config, **{name: value})) + return None + + object.__setattr__(self, name, value) + return None + + def _create_log_handlers(self, create_file_handler: bool = True) -> None: # noqa: FBT002 + if self.console_handler is not None or self.file_handler is not None: + self.logger.debug("Requested to create log handlers, but they are already created.") + return + + message_format = "[%(levelname)s] [%(asctime)s:%(name)s] %(message)s" + output_formatter = logging.Formatter(message_format, datefmt="%H:%M:%S") + + self.console_handler = logging.StreamHandler() + self.console_handler.setFormatter(output_formatter) + + if create_file_handler: + one_mb = 2**20 + log_path = self._root_cache_directory / "openml_python.log" + self.file_handler = logging.handlers.RotatingFileHandler( + log_path, + maxBytes=one_mb, + backupCount=1, + delay=True, + ) + self.file_handler.setFormatter(output_formatter) + + def _convert_log_levels(self, log_level: int) -> tuple[int, int]: + openml_to_python = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} + python_to_openml = { + logging.DEBUG: 2, + logging.INFO: 1, + logging.WARNING: 0, + logging.CRITICAL: 0, + logging.ERROR: 0, + } + openml_level = python_to_openml.get(log_level, log_level) + python_level = openml_to_python.get(log_level, log_level) + return openml_level, python_level + + def _set_level_register_and_store(self, handler: logging.Handler, log_level: int) -> None: + _oml_level, py_level = self._convert_log_levels(log_level) + handler.setLevel(py_level) + + if self.openml_logger.level > py_level or self.openml_logger.level == logging.NOTSET: + self.openml_logger.setLevel(py_level) + + if handler not in self.openml_logger.handlers: + self.openml_logger.addHandler(handler) + + def set_console_log_level(self, console_output_level: int) -> None: + """Set the log level for console output.""" + assert self.console_handler is not None + self._set_level_register_and_store(self.console_handler, console_output_level) + + def set_file_log_level(self, file_output_level: int) -> None: + """Set the log level for file output.""" + assert self.file_handler is not None + self._set_level_register_and_store(self.file_handler, file_output_level) + + def get_server_base_url(self) -> str: + """Get the base URL of the OpenML server (i.e., without /api).""" + domain, _ = self._config.server.split("/api", maxsplit=1) + return domain.replace("api", "www") + + def set_retry_policy( + self, value: Literal["human", "robot"], n_retries: int | None = None + ) -> None: + """Set the retry policy for server connections.""" + default_retries_by_policy = {"human": 5, "robot": 50} + + if value not in default_retries_by_policy: + raise ValueError( + f"Detected retry_policy '{value}' but must be one of " + f"{list(default_retries_by_policy.keys())}", + ) + if n_retries is not None and not isinstance(n_retries, int): + raise TypeError( + f"`n_retries` must be of type `int` or `None` but is `{type(n_retries)}`." + ) + + if isinstance(n_retries, int) and n_retries < 1: + raise ValueError(f"`n_retries` is '{n_retries}' but must be positive.") + + self._config = replace( + self._config, + retry_policy=value, + connection_n_retries=( + default_retries_by_policy[value] if n_retries is None else n_retries + ), + ) + + def _handle_xdg_config_home_backwards_compatibility(self, xdg_home: str) -> Path: + config_dir = Path(xdg_home) / "openml" + + backwards_compat_config_file = Path(xdg_home) / "config" + if not backwards_compat_config_file.exists(): + return config_dir + + try: + self._parse_config(backwards_compat_config_file) + except Exception: # noqa: BLE001 + return config_dir + + correct_config_location = config_dir / "config" + try: + shutil.copy(backwards_compat_config_file, correct_config_location) + self.openml_logger.warning( + "An openml configuration file was found at the old location " + f"at {backwards_compat_config_file}. We have copied it to the new " + f"location at {correct_config_location}. " + "\nTo silence this warning please verify that the configuration file " + f"at {correct_config_location} is correct and delete the file at " + f"{backwards_compat_config_file}." + ) + return config_dir + except Exception as e: # noqa: BLE001 + self.openml_logger.warning( + "While attempting to perform a backwards compatible fix, we " + f"failed to copy the openml config file at " + f"{backwards_compat_config_file}' to {correct_config_location}" + f"\n{type(e)}: {e}", + "\n\nTo silence this warning, please copy the file " + "to the new location and delete the old file at " + f"{backwards_compat_config_file}.", + ) + return backwards_compat_config_file + + def determine_config_file_path(self) -> Path: + """Determine the path to the openml configuration file.""" + if platform.system().lower() == "linux": + xdg_home = os.environ.get("XDG_CONFIG_HOME") + if xdg_home is not None: + config_dir = self._handle_xdg_config_home_backwards_compatibility(xdg_home) + else: + config_dir = Path("~", ".config", "openml") + else: + config_dir = Path("~") / ".openml" + + config_dir = Path(config_dir).expanduser().resolve() + return config_dir / "config" + + def _parse_config(self, config_file: str | Path) -> dict[str, Any]: + config_file = Path(config_file) + config = configparser.RawConfigParser(defaults=OpenMLConfig().__dict__) # type: ignore + + config_file_ = StringIO() + config_file_.write("[FAKE_SECTION]\n") + try: + with config_file.open("r") as fh: + for line in fh: + config_file_.write(line) + except FileNotFoundError: + self.logger.info( + "No config file found at %s, using default configuration.", config_file + ) + except OSError as e: + self.logger.info("Error opening file %s: %s", config_file, e.args[0]) + config_file_.seek(0) + config.read_file(config_file_) + configuration = dict(config.items("FAKE_SECTION")) + for boolean_field in ["avoid_duplicate_runs", "show_progress"]: + if isinstance(config["FAKE_SECTION"][boolean_field], str): + configuration[boolean_field] = config["FAKE_SECTION"].getboolean(boolean_field) # type: ignore + return configuration # type: ignore + + def start_using_configuration_for_example(self) -> None: + """Sets the configuration to connect to the test server with valid apikey.""" + return self._examples.start_using_configuration_for_example() + + def stop_using_configuration_for_example(self) -> None: + """Store the configuration as it was before `start_use_example_configuration`.""" + return self._examples.stop_using_configuration_for_example() + + def _setup(self, config: dict[str, Any] | None = None) -> None: + config_file = self.determine_config_file_path() + config_dir = config_file.parent + + try: + if not config_dir.exists(): + config_dir.mkdir(exist_ok=True, parents=True) + except PermissionError: + self.openml_logger.warning( + f"No permission to create OpenML directory at {config_dir}!" + " This can result in OpenML-Python not working properly." + ) + + if config is None: + config = self._parse_config(config_file) + + self._config = replace( + self._config, + apikey=config["apikey"], + server=config["server"], + show_progress=config["show_progress"], + avoid_duplicate_runs=config["avoid_duplicate_runs"], + retry_policy=config["retry_policy"], + connection_n_retries=int(config["connection_n_retries"]), + ) + + user_defined_cache_dir = os.environ.get(self.OPENML_CACHE_DIR_ENV_VAR) + if user_defined_cache_dir is not None: + short_cache_dir = Path(user_defined_cache_dir) + else: + short_cache_dir = Path(config["cachedir"]) + + self._root_cache_directory = short_cache_dir.expanduser().resolve() + self._config = replace(self._config, cachedir=self._root_cache_directory) + + try: + cache_exists = self._root_cache_directory.exists() + if not cache_exists: + self._root_cache_directory.mkdir(exist_ok=True, parents=True) + self._create_log_handlers() + except PermissionError: + self.openml_logger.warning( + f"No permission to create OpenML directory at {self._root_cache_directory}!" + " This can result in OpenML-Python not working properly." + ) + self._create_log_handlers(create_file_handler=False) + + def set_field_in_config_file(self, field: str, value: Any) -> None: + """Set a field in the configuration file.""" + if not hasattr(OpenMLConfig(), field): + raise ValueError( + f"Field '{field}' is not valid and must be one of " + f"'{OpenMLConfig().__dict__.keys()}'." + ) + + self._config = replace(self._config, **{field: value}) + config_file = self.determine_config_file_path() + existing = self._parse_config(config_file) + with config_file.open("w") as fh: + for f in OpenMLConfig().__dict__: + v = value if f == field else existing.get(f) + if v is not None: + fh.write(f"{f} = {v}\n") + + def get_config_as_dict(self) -> dict[str, Any]: + """Get the current configuration as a dictionary.""" + return self._config.__dict__.copy() + + def get_cache_directory(self) -> str: + """Get the cache directory for the current server.""" + url_suffix = urlparse(self._config.server).netloc + url_parts = url_suffix.replace(":", "_").split(".")[::-1] + reversed_url_suffix = os.sep.join(url_parts) # noqa: PTH118 + return os.path.join(self._root_cache_directory, reversed_url_suffix) # noqa: PTH118 + + def set_root_cache_directory(self, root_cache_directory: str | Path) -> None: + """Set the root cache directory.""" + self._root_cache_directory = Path(root_cache_directory) + self._config = replace(self._config, cachedir=self._root_cache_directory) + + @contextmanager + def overwrite_config_context(self, config: dict[str, Any]) -> Iterator[dict[str, Any]]: + """Overwrite the current configuration within a context manager.""" + existing_config = self.get_config_as_dict() + merged_config = {**existing_config, **config} + + self._setup(merged_config) + yield merged_config + self._setup(existing_config) + + +class ConfigurationForExamples: + """Allows easy switching to and from a test configuration, used for examples.""" + + _last_used_server = None + _last_used_key = None + _start_last_called = False + + def __init__(self, manager: OpenMLConfigManager): + self._manager = manager + self._test_apikey = manager._TEST_SERVER_NORMAL_USER_KEY + self._test_server = f"{manager.TEST_SERVER_URL}/api/v1/xml" + + def start_using_configuration_for_example(self) -> None: + """Sets the configuration to connect to the test server with valid apikey. + + To configuration as was before this call is stored, and can be recovered + by using the `stop_use_example_configuration` method. + """ + if ( + self._start_last_called + and self._manager._config.server == self._test_server + and self._manager._config.apikey == self._test_apikey + ): + # Method is called more than once in a row without modifying the server or apikey. + # We don't want to save the current test configuration as a last used configuration. + return + + self._last_used_server = self._manager._config.server + self._last_used_key = self._manager._config.apikey + type(self)._start_last_called = True + + # Test server key for examples + self._manager._config = replace( + self._manager._config, + server=self._test_server, + apikey=self._test_apikey, + ) + warnings.warn( + f"Switching to the test server {self._test_server} to not upload results to " + "the live server. Using the test server may result in reduced performance of the " + "API!", + stacklevel=2, + ) + + def stop_using_configuration_for_example(self) -> None: + """Return to configuration as it was before `start_use_example_configuration`.""" + if not type(self)._start_last_called: + # We don't want to allow this because it will (likely) result in the `server` and + # `apikey` variables being set to None. + raise RuntimeError( + "`stop_use_example_configuration` called without a saved config." + "`start_use_example_configuration` must be called first.", + ) + + self._manager._config = replace( + self._manager._config, + server=cast("str", self._last_used_server), + apikey=cast("str", self._last_used_key), + ) + type(self)._start_last_called = False + + +__config = OpenMLConfigManager() + + +def __getattr__(name: str) -> Any: + return getattr(__config, name) diff --git a/openml/base.py b/openml/base.py index a282be8eb..ddee71196 100644 --- a/openml/base.py +++ b/openml/base.py @@ -9,7 +9,6 @@ import xmltodict import openml._api_calls -import openml.config from .utils import _get_rest_api_type_alias, _tag_openml_base diff --git a/openml/cli.py b/openml/cli.py index c33578f6e..838f774d1 100644 --- a/openml/cli.py +++ b/openml/cli.py @@ -6,10 +6,11 @@ import string import sys from collections.abc import Callable +from dataclasses import fields from pathlib import Path from urllib.parse import urlparse -from openml import config +import openml from openml.__version__ import __version__ @@ -59,17 +60,17 @@ def wait_until_valid_input( def print_configuration() -> None: - file = config.determine_config_file_path() + file = openml.config.determine_config_file_path() header = f"File '{file}' contains (or defaults to):" print(header) - max_key_length = max(map(len, config.get_config_as_dict())) - for field, value in config.get_config_as_dict().items(): + max_key_length = max(map(len, openml.config.get_config_as_dict())) + for field, value in openml.config.get_config_as_dict().items(): print(f"{field.ljust(max_key_length)}: {value}") def verbose_set(field: str, value: str) -> None: - config.set_field_in_config_file(field, value) + openml.config.set_field_in_config_file(field, value) print(f"{field} set to '{value}'.") @@ -82,7 +83,7 @@ def check_apikey(apikey: str) -> str: return "" instructions = ( - f"Your current API key is set to: '{config.apikey}'. " + f"Your current API key is set to: '{openml.config.apikey}'. " "You can get an API key at https://new.openml.org. " "You must create an account if you don't have one yet:\n" " 1. Log in with the account.\n" @@ -109,7 +110,7 @@ def check_server(server: str) -> str: def replace_shorthand(server: str) -> str: if server == "test": - return f"{config.TEST_SERVER_URL}/api/v1/xml" + return f"{openml.config.TEST_SERVER_URL}/api/v1/xml" if server == "production_server": return "https://www.openml.org/api/v1/xml" return server @@ -347,7 +348,9 @@ def main() -> None: "'https://openml.github.io/openml-python/main/usage.html#configuration'.", ) - configurable_fields = [f for f in config._defaults if f not in ["max_retries"]] + configurable_fields = [ + f.name for f in fields(openml._config.OpenMLConfig) if f.name not in ["max_retries"] + ] parser_configure.add_argument( "field", diff --git a/openml/config.py b/openml/config.py deleted file mode 100644 index 638b45650..000000000 --- a/openml/config.py +++ /dev/null @@ -1,529 +0,0 @@ -"""Store module level information like the API key, cache directory and the server""" - -# License: BSD 3-Clause -from __future__ import annotations - -import configparser -import logging -import logging.handlers -import os -import platform -import shutil -import warnings -from collections.abc import Iterator -from contextlib import contextmanager -from io import StringIO -from pathlib import Path -from typing import Any, Literal, cast -from typing_extensions import TypedDict -from urllib.parse import urlparse - -logger = logging.getLogger(__name__) -openml_logger = logging.getLogger("openml") -console_handler: logging.StreamHandler | None = None -file_handler: logging.handlers.RotatingFileHandler | None = None - -OPENML_CACHE_DIR_ENV_VAR = "OPENML_CACHE_DIR" -OPENML_SKIP_PARQUET_ENV_VAR = "OPENML_SKIP_PARQUET" -OPENML_TEST_SERVER_ADMIN_KEY_ENV_VAR = "OPENML_TEST_SERVER_ADMIN_KEY" -_TEST_SERVER_NORMAL_USER_KEY = "normaluser" - -TEST_SERVER_URL = "https://test.openml.org" - - -class _Config(TypedDict): - apikey: str - server: str - cachedir: Path - avoid_duplicate_runs: bool - retry_policy: Literal["human", "robot"] - connection_n_retries: int - show_progress: bool - - -def _create_log_handlers(create_file_handler: bool = True) -> None: # noqa: FBT002 - """Creates but does not attach the log handlers.""" - global console_handler, file_handler # noqa: PLW0603 - if console_handler is not None or file_handler is not None: - logger.debug("Requested to create log handlers, but they are already created.") - return - - message_format = "[%(levelname)s] [%(asctime)s:%(name)s] %(message)s" - output_formatter = logging.Formatter(message_format, datefmt="%H:%M:%S") - - console_handler = logging.StreamHandler() - console_handler.setFormatter(output_formatter) - - if create_file_handler: - one_mb = 2**20 - log_path = _root_cache_directory / "openml_python.log" - file_handler = logging.handlers.RotatingFileHandler( - log_path, - maxBytes=one_mb, - backupCount=1, - delay=True, - ) - file_handler.setFormatter(output_formatter) - - -def _convert_log_levels(log_level: int) -> tuple[int, int]: - """Converts a log level that's either defined by OpenML/Python to both specifications.""" - # OpenML verbosity level don't match Python values directly: - openml_to_python = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} - python_to_openml = { - logging.DEBUG: 2, - logging.INFO: 1, - logging.WARNING: 0, - logging.CRITICAL: 0, - logging.ERROR: 0, - } - # Because the dictionaries share no keys, we use `get` to convert as necessary: - openml_level = python_to_openml.get(log_level, log_level) - python_level = openml_to_python.get(log_level, log_level) - return openml_level, python_level - - -def _set_level_register_and_store(handler: logging.Handler, log_level: int) -> None: - """Set handler log level, register it if needed, save setting to config file if specified.""" - _oml_level, py_level = _convert_log_levels(log_level) - handler.setLevel(py_level) - - if openml_logger.level > py_level or openml_logger.level == logging.NOTSET: - openml_logger.setLevel(py_level) - - if handler not in openml_logger.handlers: - openml_logger.addHandler(handler) - - -def set_console_log_level(console_output_level: int) -> None: - """Set console output to the desired level and register it with openml logger if needed.""" - global console_handler # noqa: PLW0602 - assert console_handler is not None - _set_level_register_and_store(console_handler, console_output_level) - - -def set_file_log_level(file_output_level: int) -> None: - """Set file output to the desired level and register it with openml logger if needed.""" - global file_handler # noqa: PLW0602 - assert file_handler is not None - _set_level_register_and_store(file_handler, file_output_level) - - -# Default values (see also https://github.com/openml/OpenML/wiki/Client-API-Standards) -_user_path = Path("~").expanduser().absolute() - - -def _resolve_default_cache_dir() -> Path: - user_defined_cache_dir = os.environ.get(OPENML_CACHE_DIR_ENV_VAR) - if user_defined_cache_dir is not None: - return Path(user_defined_cache_dir) - - if platform.system().lower() != "linux": - return _user_path / ".openml" - - xdg_cache_home = os.environ.get("XDG_CACHE_HOME") - if xdg_cache_home is None: - return Path("~", ".cache", "openml") - - # This is the proper XDG_CACHE_HOME directory, but - # we unfortunately had a problem where we used XDG_CACHE_HOME/org, - # we check heuristically if this old directory still exists and issue - # a warning if it does. There's too much data to move to do this for the user. - - # The new cache directory exists - cache_dir = Path(xdg_cache_home) / "openml" - if cache_dir.exists(): - return cache_dir - - # The old cache directory *does not* exist - heuristic_dir_for_backwards_compat = Path(xdg_cache_home) / "org" / "openml" - if not heuristic_dir_for_backwards_compat.exists(): - return cache_dir - - root_dir_to_delete = Path(xdg_cache_home) / "org" - openml_logger.warning( - "An old cache directory was found at '%s'. This directory is no longer used by " - "OpenML-Python. To silence this warning you would need to delete the old cache " - "directory. The cached files will then be located in '%s'.", - root_dir_to_delete, - cache_dir, - ) - return Path(xdg_cache_home) - - -_defaults: _Config = { - "apikey": "", - "server": "https://www.openml.org/api/v1/xml", - "cachedir": _resolve_default_cache_dir(), - "avoid_duplicate_runs": False, - "retry_policy": "human", - "connection_n_retries": 5, - "show_progress": False, -} - -# Default values are actually added here in the _setup() function which is -# called at the end of this module -server = _defaults["server"] - - -def get_server_base_url() -> str: - """Return the base URL of the currently configured server. - - Turns ``"https://api.openml.org/api/v1/xml"`` in ``"https://www.openml.org/"`` - and ``"https://test.openml.org/api/v1/xml"`` in ``"https://test.openml.org/"`` - - Returns - ------- - str - """ - domain, _path = server.split("/api", maxsplit=1) - return domain.replace("api", "www") - - -apikey: str = _defaults["apikey"] -show_progress: bool = _defaults["show_progress"] -# The current cache directory (without the server name) -_root_cache_directory: Path = Path(_defaults["cachedir"]) -avoid_duplicate_runs = _defaults["avoid_duplicate_runs"] - -retry_policy: Literal["human", "robot"] = _defaults["retry_policy"] -connection_n_retries: int = _defaults["connection_n_retries"] - - -def set_retry_policy(value: Literal["human", "robot"], n_retries: int | None = None) -> None: - global retry_policy # noqa: PLW0603 - global connection_n_retries # noqa: PLW0603 - default_retries_by_policy = {"human": 5, "robot": 50} - - if value not in default_retries_by_policy: - raise ValueError( - f"Detected retry_policy '{value}' but must be one of " - f"{list(default_retries_by_policy.keys())}", - ) - if n_retries is not None and not isinstance(n_retries, int): - raise TypeError(f"`n_retries` must be of type `int` or `None` but is `{type(n_retries)}`.") - - if isinstance(n_retries, int) and n_retries < 1: - raise ValueError(f"`n_retries` is '{n_retries}' but must be positive.") - - retry_policy = value - connection_n_retries = default_retries_by_policy[value] if n_retries is None else n_retries - - -class ConfigurationForExamples: - """Allows easy switching to and from a test configuration, used for examples.""" - - _last_used_server = None - _last_used_key = None - _start_last_called = False - _test_server = f"{TEST_SERVER_URL}/api/v1/xml" - _test_apikey = _TEST_SERVER_NORMAL_USER_KEY - - @classmethod - def start_using_configuration_for_example(cls) -> None: - """Sets the configuration to connect to the test server with valid apikey. - - To configuration as was before this call is stored, and can be recovered - by using the `stop_use_example_configuration` method. - """ - global server # noqa: PLW0603 - global apikey # noqa: PLW0603 - - if cls._start_last_called and server == cls._test_server and apikey == cls._test_apikey: - # Method is called more than once in a row without modifying the server or apikey. - # We don't want to save the current test configuration as a last used configuration. - return - - cls._last_used_server = server - cls._last_used_key = apikey - cls._start_last_called = True - - # Test server key for examples - server = cls._test_server - apikey = cls._test_apikey - warnings.warn( - f"Switching to the test server {server} to not upload results to the live server. " - "Using the test server may result in reduced performance of the API!", - stacklevel=2, - ) - - @classmethod - def stop_using_configuration_for_example(cls) -> None: - """Return to configuration as it was before `start_use_example_configuration`.""" - if not cls._start_last_called: - # We don't want to allow this because it will (likely) result in the `server` and - # `apikey` variables being set to None. - raise RuntimeError( - "`stop_use_example_configuration` called without a saved config." - "`start_use_example_configuration` must be called first.", - ) - - global server # noqa: PLW0603 - global apikey # noqa: PLW0603 - - server = cast("str", cls._last_used_server) - apikey = cast("str", cls._last_used_key) - cls._start_last_called = False - - -def _handle_xdg_config_home_backwards_compatibility( - xdg_home: str, -) -> Path: - # NOTE(eddiebergman): A previous bug results in the config - # file being located at `${XDG_CONFIG_HOME}/config` instead - # of `${XDG_CONFIG_HOME}/openml/config`. As to maintain backwards - # compatibility, where users may already may have had a configuration, - # we copy it over an issue a warning until it's deleted. - # As a heurisitic to ensure that it's "our" config file, we try parse it first. - config_dir = Path(xdg_home) / "openml" - - backwards_compat_config_file = Path(xdg_home) / "config" - if not backwards_compat_config_file.exists(): - return config_dir - - # If it errors, that's a good sign it's not ours and we can - # safely ignore it, jumping out of this block. This is a heurisitc - try: - _parse_config(backwards_compat_config_file) - except Exception: # noqa: BLE001 - return config_dir - - # Looks like it's ours, lets try copy it to the correct place - correct_config_location = config_dir / "config" - try: - # We copy and return the new copied location - shutil.copy(backwards_compat_config_file, correct_config_location) - openml_logger.warning( - "An openml configuration file was found at the old location " - f"at {backwards_compat_config_file}. We have copied it to the new " - f"location at {correct_config_location}. " - "\nTo silence this warning please verify that the configuration file " - f"at {correct_config_location} is correct and delete the file at " - f"{backwards_compat_config_file}." - ) - return config_dir - except Exception as e: # noqa: BLE001 - # We failed to copy and its ours, return the old one. - openml_logger.warning( - "While attempting to perform a backwards compatible fix, we " - f"failed to copy the openml config file at " - f"{backwards_compat_config_file}' to {correct_config_location}" - f"\n{type(e)}: {e}", - "\n\nTo silence this warning, please copy the file " - "to the new location and delete the old file at " - f"{backwards_compat_config_file}.", - ) - return backwards_compat_config_file - - -def determine_config_file_path() -> Path: - if platform.system().lower() == "linux": - xdg_home = os.environ.get("XDG_CONFIG_HOME") - if xdg_home is not None: - config_dir = _handle_xdg_config_home_backwards_compatibility(xdg_home) - else: - config_dir = Path("~", ".config", "openml") - else: - config_dir = Path("~") / ".openml" - - # Still use os.path.expanduser to trigger the mock in the unit test - config_dir = Path(config_dir).expanduser().resolve() - return config_dir / "config" - - -def _setup(config: _Config | None = None) -> None: - """Setup openml package. Called on first import. - - Reads the config file and sets up apikey, server, cache appropriately. - key and server can be set by the user simply using - openml.config.apikey = THEIRKEY - openml.config.server = SOMESERVER - We could also make it a property but that's less clear. - """ - global apikey # noqa: PLW0603 - global server # noqa: PLW0603 - global _root_cache_directory # noqa: PLW0603 - global avoid_duplicate_runs # noqa: PLW0603 - global show_progress # noqa: PLW0603 - - config_file = determine_config_file_path() - config_dir = config_file.parent - - # read config file, create directory for config file - try: - if not config_dir.exists(): - config_dir.mkdir(exist_ok=True, parents=True) - except PermissionError: - openml_logger.warning( - f"No permission to create OpenML directory at {config_dir}!" - " This can result in OpenML-Python not working properly." - ) - - if config is None: - config = _parse_config(config_file) - - avoid_duplicate_runs = config["avoid_duplicate_runs"] - apikey = config["apikey"] - server = config["server"] - show_progress = config["show_progress"] - n_retries = int(config["connection_n_retries"]) - - set_retry_policy(config["retry_policy"], n_retries) - - user_defined_cache_dir = os.environ.get(OPENML_CACHE_DIR_ENV_VAR) - if user_defined_cache_dir is not None: - short_cache_dir = Path(user_defined_cache_dir) - else: - short_cache_dir = Path(config["cachedir"]) - _root_cache_directory = short_cache_dir.expanduser().resolve() - - try: - cache_exists = _root_cache_directory.exists() - # create the cache subdirectory - if not cache_exists: - _root_cache_directory.mkdir(exist_ok=True, parents=True) - _create_log_handlers() - except PermissionError: - openml_logger.warning( - f"No permission to create OpenML directory at {_root_cache_directory}!" - " This can result in OpenML-Python not working properly." - ) - _create_log_handlers(create_file_handler=False) - - -def set_field_in_config_file(field: str, value: Any) -> None: - """Overwrites the `field` in the configuration file with the new `value`.""" - if field not in _defaults: - raise ValueError(f"Field '{field}' is not valid and must be one of '{_defaults.keys()}'.") - - # TODO(eddiebergman): This use of globals has gone too far - globals()[field] = value - config_file = determine_config_file_path() - config = _parse_config(config_file) - with config_file.open("w") as fh: - for f in _defaults: - # We can't blindly set all values based on globals() because when the user - # sets it through config.FIELD it should not be stored to file. - # There doesn't seem to be a way to avoid writing defaults to file with configparser, - # because it is impossible to distinguish from an explicitly set value that matches - # the default value, to one that was set to its default because it was omitted. - value = globals()[f] if f == field else config.get(f) # type: ignore - if value is not None: - fh.write(f"{f} = {value}\n") - - -def _parse_config(config_file: str | Path) -> _Config: - """Parse the config file, set up defaults.""" - config_file = Path(config_file) - config = configparser.RawConfigParser(defaults=_defaults) # type: ignore - - # The ConfigParser requires a [SECTION_HEADER], which we do not expect in our config file. - # Cheat the ConfigParser module by adding a fake section header - config_file_ = StringIO() - config_file_.write("[FAKE_SECTION]\n") - try: - with config_file.open("r") as fh: - for line in fh: - config_file_.write(line) - except FileNotFoundError: - logger.info("No config file found at %s, using default configuration.", config_file) - except OSError as e: - logger.info("Error opening file %s: %s", config_file, e.args[0]) - config_file_.seek(0) - config.read_file(config_file_) - configuration = dict(config.items("FAKE_SECTION")) - for boolean_field in ["avoid_duplicate_runs", "show_progress"]: - if isinstance(config["FAKE_SECTION"][boolean_field], str): - configuration[boolean_field] = config["FAKE_SECTION"].getboolean(boolean_field) # type: ignore - return configuration # type: ignore - - -def get_config_as_dict() -> _Config: - return { - "apikey": apikey, - "server": server, - "cachedir": _root_cache_directory, - "avoid_duplicate_runs": avoid_duplicate_runs, - "connection_n_retries": connection_n_retries, - "retry_policy": retry_policy, - "show_progress": show_progress, - } - - -# NOTE: For backwards compatibility, we keep the `str` -def get_cache_directory() -> str: - """Get the current cache directory. - - This gets the cache directory for the current server relative - to the root cache directory that can be set via - ``set_root_cache_directory()``. The cache directory is the - ``root_cache_directory`` with additional information on which - subdirectory to use based on the server name. By default it is - ``root_cache_directory / org / openml / www`` for the standard - OpenML.org server and is defined as - ``root_cache_directory / top-level domain / second-level domain / - hostname`` - ``` - - Returns - ------- - cachedir : string - The current cache directory. - - """ - url_suffix = urlparse(server).netloc - url_parts = url_suffix.replace(":", "_").split(".")[::-1] - reversed_url_suffix = os.sep.join(url_parts) # noqa: PTH118 - return os.path.join(_root_cache_directory, reversed_url_suffix) # noqa: PTH118 - - -def set_root_cache_directory(root_cache_directory: str | Path) -> None: - """Set module-wide base cache directory. - - Sets the root cache directory, wherin the cache directories are - created to store content from different OpenML servers. For example, - by default, cached data for the standard OpenML.org server is stored - at ``root_cache_directory / org / openml / www``, and the general - pattern is ``root_cache_directory / top-level domain / second-level - domain / hostname``. - - Parameters - ---------- - root_cache_directory : string - Path to use as cache directory. - - See Also - -------- - get_cache_directory - """ - global _root_cache_directory # noqa: PLW0603 - _root_cache_directory = Path(root_cache_directory) - - -start_using_configuration_for_example = ( - ConfigurationForExamples.start_using_configuration_for_example -) -stop_using_configuration_for_example = ConfigurationForExamples.stop_using_configuration_for_example - - -@contextmanager -def overwrite_config_context(config: dict[str, Any]) -> Iterator[_Config]: - """A context manager to temporarily override variables in the configuration.""" - existing_config = get_config_as_dict() - merged_config = {**existing_config, **config} - - _setup(merged_config) # type: ignore - yield merged_config # type: ignore - - _setup(existing_config) - - -__all__ = [ - "get_cache_directory", - "get_config_as_dict", - "set_root_cache_directory", - "start_using_configuration_for_example", - "stop_using_configuration_for_example", -] - -_setup() diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index d9eee278d..59d6205ba 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -17,8 +17,8 @@ import scipy.sparse import xmltodict +import openml from openml.base import OpenMLBase -from openml.config import OPENML_SKIP_PARQUET_ENV_VAR from .data_feature import OpenMLDataFeature @@ -375,7 +375,9 @@ def _download_data(self) -> None: # import required here to avoid circular import. from .functions import _get_dataset_arff, _get_dataset_parquet - skip_parquet = os.environ.get(OPENML_SKIP_PARQUET_ENV_VAR, "false").casefold() == "true" + skip_parquet = ( + os.environ.get(openml.config.OPENML_SKIP_PARQUET_ENV_VAR, "false").casefold() == "true" + ) if self._parquet_url is not None and not skip_parquet: parquet_file = _get_dataset_parquet(self) self.parquet_file = None if parquet_file is None else str(parquet_file) diff --git a/openml/datasets/functions.py b/openml/datasets/functions.py index 3ac657ea0..432938520 100644 --- a/openml/datasets/functions.py +++ b/openml/datasets/functions.py @@ -19,9 +19,9 @@ import xmltodict from scipy.sparse import coo_matrix +import openml import openml._api_calls import openml.utils -from openml.config import OPENML_SKIP_PARQUET_ENV_VAR from openml.exceptions import ( OpenMLHashException, OpenMLPrivateDatasetError, @@ -492,7 +492,9 @@ def get_dataset( # noqa: C901, PLR0912 qualities_file = _get_dataset_qualities_file(did_cache_dir, dataset_id) parquet_file = None - skip_parquet = os.environ.get(OPENML_SKIP_PARQUET_ENV_VAR, "false").casefold() == "true" + skip_parquet = ( + os.environ.get(openml.config.OPENML_SKIP_PARQUET_ENV_VAR, "false").casefold() == "true" + ) download_parquet = "oml:parquet_url" in description and not skip_parquet if download_parquet and (download_data or download_all_files): try: diff --git a/openml/evaluations/evaluation.py b/openml/evaluations/evaluation.py index 5db087024..87df8454a 100644 --- a/openml/evaluations/evaluation.py +++ b/openml/evaluations/evaluation.py @@ -3,7 +3,6 @@ from dataclasses import asdict, dataclass -import openml.config import openml.datasets import openml.flows import openml.runs diff --git a/openml/runs/functions.py b/openml/runs/functions.py index b991fb5ec..d87bd3e18 100644 --- a/openml/runs/functions.py +++ b/openml/runs/functions.py @@ -18,7 +18,6 @@ import openml import openml._api_calls import openml.utils -from openml import config from openml.exceptions import ( OpenMLCacheException, OpenMLRunsExistError, @@ -45,7 +44,7 @@ # Avoid import cycles: https://mypy.readthedocs.io/en/latest/common_issues.html#import-cycles if TYPE_CHECKING: - from openml.config import _Config + from openml._config import _Config from openml.extensions.extension_interface import Extension # get_dict is in run.py to avoid circular imports @@ -107,7 +106,7 @@ def run_model_on_task( # noqa: PLR0913 """ if avoid_duplicate_runs is None: avoid_duplicate_runs = openml.config.avoid_duplicate_runs - if avoid_duplicate_runs and not config.apikey: + if avoid_duplicate_runs and not openml.config.apikey: warnings.warn( "avoid_duplicate_runs is set to True, but no API key is set. " "Please set your API key in the OpenML configuration file, see" @@ -336,7 +335,7 @@ def run_flow_on_task( # noqa: C901, PLR0912, PLR0915, PLR0913 message = f"Executed Task {task.task_id} with Flow id:{run.flow_id}" else: message = f"Executed Task {task.task_id} on local Flow with name {flow.name}." - config.logger.info(message) + openml.config.logger.info(message) return run @@ -530,7 +529,7 @@ def _run_task_get_arffcontent( # noqa: PLR0915, PLR0912, C901 # The forked child process may not copy the configuration state of OpenML from the parent. # Current configuration setup needs to be copied and passed to the child processes. - _config = config.get_config_as_dict() + _config = openml.config.get_config_as_dict() # Execute runs in parallel # assuming the same number of tasks as workers (n_jobs), the total compute time for this # statement will be similar to the slowest run @@ -738,7 +737,7 @@ def _run_task_get_arffcontent_parallel_helper( # noqa: PLR0913 """ # Sets up the OpenML instantiated in the child process to match that of the parent's # if configuration=None, loads the default - config._setup(configuration) + openml.config._setup(configuration) train_indices, test_indices = task.get_train_test_split_indices( repeat=rep_no, @@ -768,7 +767,7 @@ def _run_task_get_arffcontent_parallel_helper( # noqa: PLR0913 f"task_class={task.__class__.__name__}" ) - config.logger.info( + openml.config.logger.info( f"Going to run model {model!s} on " f"dataset {openml.datasets.get_dataset(task.dataset_id).name} " f"for repeat {rep_no} fold {fold_no} sample {sample_no}" diff --git a/openml/setups/functions.py b/openml/setups/functions.py index 4bf279ed1..a24d3a456 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -14,7 +14,6 @@ import openml import openml.exceptions import openml.utils -from openml import config from openml.flows import OpenMLFlow, flow_exists from .setup import OpenMLParameter, OpenMLSetup @@ -84,7 +83,7 @@ def _get_cached_setup(setup_id: int) -> OpenMLSetup: OpenMLCacheException If the setup file for the given setup ID is not cached. """ - cache_dir = Path(config.get_cache_directory()) + cache_dir = Path(openml.config.get_cache_directory()) setup_cache_dir = cache_dir / "setups" / str(setup_id) try: setup_file = setup_cache_dir / "description.xml" @@ -112,7 +111,7 @@ def get_setup(setup_id: int) -> OpenMLSetup: ------- OpenMLSetup (an initialized openml setup object) """ - setup_dir = Path(config.get_cache_directory()) / "setups" / str(setup_id) + setup_dir = Path(openml.config.get_cache_directory()) / "setups" / str(setup_id) setup_dir.mkdir(exist_ok=True, parents=True) setup_file = setup_dir / "description.xml" diff --git a/openml/setups/setup.py b/openml/setups/setup.py index 170838138..0c3a3cb6b 100644 --- a/openml/setups/setup.py +++ b/openml/setups/setup.py @@ -4,7 +4,6 @@ from dataclasses import asdict, dataclass from typing import Any -import openml.config import openml.flows diff --git a/openml/study/functions.py b/openml/study/functions.py index bb24ddcff..7268ea97c 100644 --- a/openml/study/functions.py +++ b/openml/study/functions.py @@ -9,7 +9,6 @@ import xmltodict import openml._api_calls -import openml.config import openml.utils from openml.study.study import OpenMLBenchmarkSuite, OpenMLStudy diff --git a/openml/study/study.py b/openml/study/study.py index 7a9c80bbe..803c6455b 100644 --- a/openml/study/study.py +++ b/openml/study/study.py @@ -5,8 +5,8 @@ from collections.abc import Sequence from typing import Any +import openml from openml.base import OpenMLBase -from openml.config import get_server_base_url class BaseStudy(OpenMLBase): @@ -111,7 +111,7 @@ def _get_repr_body_fields(self) -> Sequence[tuple[str, str | int | list[str]]]: fields["ID"] = self.study_id fields["Study URL"] = self.openml_url if self.creator is not None: - fields["Creator"] = f"{get_server_base_url()}/u/{self.creator}" + fields["Creator"] = f"{openml.config.get_server_base_url()}/u/{self.creator}" if self.creation_date is not None: fields["Upload Time"] = self.creation_date.replace("T", " ") if self.data is not None: diff --git a/openml/tasks/task.py b/openml/tasks/task.py index cb1c80e8f..ab3cb3da4 100644 --- a/openml/tasks/task.py +++ b/openml/tasks/task.py @@ -12,7 +12,6 @@ import arff import openml._api_calls -import openml.config from openml import datasets from openml.base import OpenMLBase from openml.utils import _create_cache_directory_for_id diff --git a/openml/utils/_openml.py b/openml/utils/_openml.py index f18dbe3e0..2bf54690e 100644 --- a/openml/utils/_openml.py +++ b/openml/utils/_openml.py @@ -26,7 +26,6 @@ import openml import openml._api_calls import openml.exceptions -from openml import config # Avoid import cycles: https://mypy.readthedocs.io/en/latest/common_issues.html#import-cycles if TYPE_CHECKING: @@ -336,7 +335,7 @@ def _list_all( # noqa: C901 def _get_cache_dir_for_key(key: str) -> Path: - return Path(config.get_cache_directory()) / key + return Path(openml.config.get_cache_directory()) / key def _create_cache_directory(key: str) -> Path: @@ -443,12 +442,12 @@ def get_cache_size() -> int: cache_size: int Total size of cache in bytes """ - path = Path(config.get_cache_directory()) + path = Path(openml.config.get_cache_directory()) return sum(f.stat().st_size for f in path.rglob("*") if f.is_file()) def _create_lockfiles_dir() -> Path: - path = Path(config.get_cache_directory()) / "locks" + path = Path(openml.config.get_cache_directory()) / "locks" # TODO(eddiebergman): Not sure why this is allowed to error and ignore??? with contextlib.suppress(OSError): path.mkdir(exist_ok=True, parents=True) diff --git a/tests/test_evaluations/test_evaluations_example.py b/tests/test_evaluations/test_evaluations_example.py index a9ad7e8c1..b321f475d 100644 --- a/tests/test_evaluations/test_evaluations_example.py +++ b/tests/test_evaluations/test_evaluations_example.py @@ -3,14 +3,13 @@ import unittest -from openml.config import overwrite_config_context - +import openml class TestEvaluationsExample(unittest.TestCase): def test_example_python_paper(self): # Example script which will appear in the upcoming OpenML-Python paper # This test ensures that the example will keep running! - with overwrite_config_context( + with openml.config.overwrite_config_context( # noqa: F823 { "server": "https://www.openml.org/api/v1/xml", "apikey": None, @@ -18,7 +17,6 @@ def test_example_python_paper(self): ): import matplotlib.pyplot as plt import numpy as np - import openml df = openml.evaluations.list_evaluations_setups( "predictive_accuracy", diff --git a/tests/test_openml/test_api_calls.py b/tests/test_openml/test_api_calls.py index 7ece4309a..f2a81be9f 100644 --- a/tests/test_openml/test_api_calls.py +++ b/tests/test_openml/test_api_calls.py @@ -10,7 +10,6 @@ import os import openml -from openml.config import ConfigurationForExamples import openml.testing from openml._api_calls import _download_minio_bucket, API_TOKEN_HELP_LINK diff --git a/tests/test_openml/test_config.py b/tests/test_openml/test_config.py index 13b06223a..f3feca784 100644 --- a/tests/test_openml/test_config.py +++ b/tests/test_openml/test_config.py @@ -12,7 +12,7 @@ import pytest -import openml.config +import openml import openml.testing from openml.testing import TestBase @@ -37,7 +37,7 @@ def safe_environ_patcher(key: str, value: Any) -> Iterator[None]: class TestConfig(openml.testing.TestBase): @unittest.mock.patch("openml.config.openml_logger.warning") - @unittest.mock.patch("openml.config._create_log_handlers") + @unittest.mock.patch("openml._config.OpenMLConfigManager._create_log_handlers") @unittest.skipIf(os.name == "nt", "https://github.com/openml/openml-python/issues/1033") @unittest.skipIf( platform.uname().release.endswith(("-Microsoft", "microsoft-standard-WSL2")), @@ -127,7 +127,6 @@ def test_switch_from_example_configuration(self): openml.config.start_using_configuration_for_example() openml.config.stop_using_configuration_for_example() - assert openml.config.apikey == TestBase.user_key assert openml.config.server == self.production_server @@ -136,7 +135,7 @@ def test_example_configuration_stop_before_start(self): error_regex = ".*stop_use_example_configuration.*start_use_example_configuration.*first" # Tests do not reset the state of this class. Thus, we ensure it is in # the original state before the test. - openml.config.ConfigurationForExamples._start_last_called = False + openml.config._examples._start_last_called = False self.assertRaisesRegex( RuntimeError, error_regex,