Release v0.2.0 (#155)

* test files

* move results

* versions

* fixes

Author: MatthewMiddlehurst

MANIFEST.in

@@ -3,9 +3,10 @@ recursive-include examples *
 recursive-include results *
 recursive-include tsml_eval *.py
 recursive-include tsml_eval/datasets *.ts *.csv
-recursive-exclude tsml_eval/publications *.csv *.txt *.ipynb
-recursive-exclude tsml_eval/utils *.csv
+recursive-include tsml_eval/publications *.csv *.txt *.ipynb
+recursive-include tsml_eval/testing/_test_result_files *.csv
 include .coveragerc
+include conftest.py
 include LICENSE
 include MANIFEST.in
 include pyproject.toml
@@ -20,6 +21,3 @@ exclude .gitignore
 exclude .pre-commit-config.yaml
 exclude .readthedocs.yml
 exclude sweep.yaml
-
-# these are in _wip currently, may want to remove in the future
-recursive-include tsml_eval/_wip *.csv *.ipynb *.sh *.txt

README.md

@@ -14,7 +14,7 @@
 `tsml-eval` contains benchmarking and evaluation tools for time series machine learning
 algorithms.
 
-The current release of `tsml-eval` is v0.1.1.
+The current release of `tsml-eval` is v0.2.0.
 
 ## Installation
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tsml-eval"
-version = "0.1.1"
+version = "0.2.0"
 description = "A package for benchmarking time series machine learning tools."
 authors = [
     {name = "Matthew Middlehurst", email = "[email protected]"},

tsml_eval/__init__.py

@@ -1,3 +1,3 @@
 """tsml-eval."""
 
-__version__ = "0.1.1"
+__version__ = "0.2.0"

tsml_eval/estimators/classification/hybrid/hivecote_from_file.py

@@ -126,7 +126,7 @@ def _fit(self, X, y):
                 if not self.skip_y_check and len(np.unique(y)) != int(line2[5]):
                     raise ValueError(
                         f"n_classes of {path + file_name} does not match X, "
-                        f"expected {len(np.unique(y))}, got {line2[6]}"
+                        f"expected {len(np.unique(y))}, got {line2[5]}"
                     )
 
                 for j in range(n_instances):
@@ -154,7 +154,7 @@ def _fit(self, X, y):
 
         # add a weight to the weight list based on the files accuracy
         for acc in acc_list:
-            self.weights_.append(acc ** self._alpha)
+            self.weights_.append(acc**self._alpha)
 
         self._use_classifier = [True for _ in range(len(self.classifiers))]
         if self.acc_filter is not None:
@@ -166,11 +166,13 @@ def _fit(self, X, y):
 
             if self._acc_filter[0] != "train" and self._acc_filter[0] != "test":
                 raise ValueError(
-                    f"acc_filter[0] must be 'train' or 'test', got {self._acc_filter[0]}"
+                    f"acc_filter[0] must be 'train' or 'test', got "
+                    f"{self._acc_filter[0]}"
                 )
             elif self._acc_filter[1] <= 0 or self._acc_filter[1] >= 1:
                 raise ValueError(
-                    f"acc_filter[1] must be in between 0 and 1, got {self._acc_filter[1]}"
+                    "acc_filter[1] must be in between 0 and 1, got "
+                    f"{self._acc_filter[1]}"
                 )
 
             if self._acc_filter[0] == "test":
@@ -179,10 +181,10 @@ def _fit(self, X, y):
                 if self.random_state is not None:
                     file_name = f"testResample{self.random_state}.csv"
                 else:
-                    file_name = f"testResample.csv"
+                    file_name = "testResample.csv"
 
                 acc_list = []
-                for i, path in enumerate(self.classifiers):
+                for path in self.classifiers:
                     f = open(path + file_name, "r")
                     lines = f.readlines()
                     line2 = lines[2].split(",")
@@ -213,7 +215,7 @@ def _predict_proba(self, X):
         if self.random_state is not None:
             file_name = f"testResample{self.random_state}.csv"
         else:
-            file_name = f"testResample.csv"
+            file_name = "testResample.csv"
 
         dists = np.zeros((n_instances, self.n_classes_))
 
@@ -226,19 +228,15 @@ def _predict_proba(self, X):
             line2 = lines[2].split(",")
 
             # verify file matches data
-            if len(lines) - 3 != n_instances:  # verify n_instances
-                print(
-                    "ERROR n_instances does not match in: ",
-                    path + file_name,
-                    len(lines) - 3,
-                    n_instances,
+            if len(lines) - 3 != n_instances:
+                raise ValueError(
+                    f"n_instances of {path + file_name} does not match X, "
+                    f"expected {X.shape[0]}, got {len(lines) - 3}"
                 )
-            if self.n_classes_ != int(line2[5]):  # verify n_classes
-                print(
-                    "ERROR n_classes does not match in: ",
-                    path + file_name,
-                    self.n_classes_,
-                    line2[5],
+            if self.n_classes_ != int(line2[5]):
+                raise ValueError(
+                    f"n_classes of {path + file_name} does not match X, "
+                    f"expected {self.n_classes_}, got {line2[5]}"
                 )
 
             #   apply this files weights to the probabilities in the test file
@@ -335,10 +333,10 @@ def get_test_params(cls, parameter_set="default"):
             `MyClass(**params)` or `MyClass(**params[i])` creates a valid test instance.
             `create_test_instance` uses the first (or only) dictionary in `params`.
         """
-        from tsml_eval.estimators.classification.hybrid.tests.test_hivecote import _TEST_RESULTS_PATH
+        from tsml_eval.testing.test_utils import _TEST_RESULTS_PATH
 
         file_paths = [
-            _TEST_RESULTS_PATH + "TestResults/Test1/",
-            _TEST_RESULTS_PATH + "TestResults/Test2/",
+            _TEST_RESULTS_PATH + "/classification/TestResults/Test1/",
+            _TEST_RESULTS_PATH + "/classification/TestResults/Test2/",
         ]
         return {"classifiers": file_paths, "skip_y_check": True, "random_state": 0}

tsml_eval/estimators/classification/hybrid/tests/test_hivecote.py

@@ -2,29 +2,28 @@
 
 __author__ = ["MatthewMiddlehurst"]
 
-import os
-from pathlib import Path
 
 import numpy as np
 from aeon.datasets import load_arrow_head, load_italy_power_demand
 from aeon.utils._testing.estimator_checks import _assert_array_almost_equal
 from aeon.utils.estimator_checks import check_estimator
 
-from tsml_eval.estimators.classification.hybrid.hivecote_from_file import \
-    FromFileHIVECOTE
+from tsml_eval.estimators.classification.hybrid.hivecote_from_file import (
+    FromFileHIVECOTE,
+)
+from tsml_eval.testing.test_utils import _TEST_RESULTS_PATH
 
-_TEST_RESULTS_PATH = os.path.dirname(Path(__file__).parent.parent.parent.parent.parent) + "/tsml_eval/estimators/classification/hybrid/tests/test_files/"
 
 def test_hivecote_from_file():
     """Test HIVE-COTE from file with ItalyPowerDemand results."""
     X_train, y_train = load_italy_power_demand(split="train")
     X_test, _ = load_italy_power_demand(split="test")
 
     file_paths = [
-        _TEST_RESULTS_PATH + "/ItalyPowerDemand/Arsenal/",
-        _TEST_RESULTS_PATH + "/ItalyPowerDemand/DrCIF/",
-        _TEST_RESULTS_PATH + "/ItalyPowerDemand/STC/",
-        _TEST_RESULTS_PATH + "/ItalyPowerDemand/TDE/",
+        _TEST_RESULTS_PATH + "/classification/Arsenal/Predictions/ItalyPowerDemand/",
+        _TEST_RESULTS_PATH + "/classification/DrCIF/Predictions/ItalyPowerDemand/",
+        _TEST_RESULTS_PATH + "/classification/STC/Predictions/ItalyPowerDemand/",
+        _TEST_RESULTS_PATH + "/classification/TDE/Predictions/ItalyPowerDemand/",
     ]
 
     hc2 = FromFileHIVECOTE(classifiers=file_paths, random_state=0)
@@ -65,10 +64,10 @@ def test_tuned_hivecote_from_file():
     X_test, _ = load_arrow_head(split="test")
 
     file_paths = [
-        _TEST_RESULTS_PATH + "/ArrowHead/Arsenal/",
-        _TEST_RESULTS_PATH + "/ArrowHead/DrCIF/",
-        _TEST_RESULTS_PATH + "/ArrowHead/STC/",
-        _TEST_RESULTS_PATH + "/ArrowHead/TDE/",
+        _TEST_RESULTS_PATH + "/classification/Arsenal/Predictions/ArrowHead/",
+        _TEST_RESULTS_PATH + "/classification/DrCIF/Predictions/ArrowHead/",
+        _TEST_RESULTS_PATH + "/classification/STC/Predictions/ArrowHead/",
+        _TEST_RESULTS_PATH + "/classification/TDE/Predictions/ArrowHead/",
     ]
 
     hc2 = FromFileHIVECOTE(classifiers=file_paths, tune_alpha=True, random_state=0)

tsml_eval/estimators/clustering/consensus/tests/test_from_file_clustering.py

@@ -1,17 +1,14 @@
-import os
-from pathlib import Path
-
 import numpy as np
 from aeon.datasets import load_arrow_head
 from sklearn.metrics import rand_score
 
-from tsml_eval.estimators.clustering.consensus.ivc_from_file import \
-    FromFileIterativeVotingClustering
-from tsml_eval.estimators.clustering.consensus.simple_vote_from_file import \
-    FromFileSimpleVote
-
-
-_TEST_RESULTS_PATH = os.path.dirname(Path(__file__).parent.parent.parent.parent.parent) + "/tsml_eval/estimators/clustering/consensus/tests/test_files/"
+from tsml_eval.estimators.clustering.consensus.ivc_from_file import (
+    FromFileIterativeVotingClustering,
+)
+from tsml_eval.estimators.clustering.consensus.simple_vote_from_file import (
+    FromFileSimpleVote,
+)
+from tsml_eval.testing.test_utils import _TEST_RESULTS_PATH
 
 
 def test_from_file_simple_vote():
@@ -20,9 +17,9 @@ def test_from_file_simple_vote():
     X_test, y_test = load_arrow_head(split="test")
 
     file_paths = [
-        _TEST_RESULTS_PATH + "/ArrowHead/PAM-DTW/",
-        _TEST_RESULTS_PATH + "/ArrowHead/PAM-ERP/",
-        _TEST_RESULTS_PATH + "/ArrowHead/PAM-MSM/",
+        _TEST_RESULTS_PATH + "/clustering/PAM-DTW/Predictions/ArrowHead/",
+        _TEST_RESULTS_PATH + "/clustering/PAM-ERP/Predictions/ArrowHead/",
+        _TEST_RESULTS_PATH + "/clustering/PAM-MSM/Predictions/ArrowHead/",
     ]
 
     sv = FromFileSimpleVote(clusterers=file_paths, n_clusters=3, random_state=0)
@@ -43,12 +40,14 @@ def test_from_file_iterative_voting_clustering():
     X_test, y_test = load_arrow_head(split="test")
 
     file_paths = [
-        _TEST_RESULTS_PATH + "/ArrowHead/PAM-DTW/",
-        _TEST_RESULTS_PATH + "/ArrowHead/PAM-ERP/",
-        _TEST_RESULTS_PATH + "/ArrowHead/PAM-MSM/",
+        _TEST_RESULTS_PATH + "/clustering/PAM-DTW/Predictions/ArrowHead/",
+        _TEST_RESULTS_PATH + "/clustering/PAM-ERP/Predictions/ArrowHead/",
+        _TEST_RESULTS_PATH + "/clustering/PAM-MSM/Predictions/ArrowHead/",
     ]
 
-    ivc = FromFileIterativeVotingClustering(clusterers=file_paths, n_clusters=3, max_iterations=100, random_state=0)
+    ivc = FromFileIterativeVotingClustering(
+        clusterers=file_paths, n_clusters=3, max_iterations=100, random_state=0
+    )
     ivc.fit(X_train, y_train)
     preds = ivc.predict(X_test)