Reduce qr test-cases (#395)

cTatu · web-flow · commit 7a1101427361 · 2022-08-05T14:24:08.000+02:00
diff --git a/Dockerfile b/Dockerfile
@@ -5,7 +5,9 @@ COPY . dislib/
 
 ENV PYTHONPATH=$PYTHONPATH:/dislib
 ENV LC_ALL=C.UTF-8
-RUN pip3 install -r /dislib/requirements.txt
+RUN python3 -m pip install --upgrade -r /dislib/requirements.txt
+
+ENV COMPSS_LOAD_SOURCE false
 
 # Expose SSH port and run SSHD
 EXPOSE 22
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -22,7 +22,7 @@ pipeline {
         stage('build') {
             steps {
                 setGithubCommitStatus('pending', 'The Jenkins build is in progress')
-                sh 'git lfs pull origin'
+                sh 'git pull origin'
                 sh 'docker rm -f dislib &> /dev/null || true'
                 sh 'docker rmi -f bscwdc/dislib &> /dev/null || true'
                 sh 'docker build --pull --no-cache --tag bscwdc/dislib .'
diff --git a/dislib/data/io.py b/dislib/data/io.py
@@ -481,7 +481,7 @@ def _read_svmlight(lines, out_blocks, col_size, n_features, store_sparse):
     tmp_file.writelines(lines)
     tmp_file.seek(0)
 
-    x, y = load_svmlight_file(tmp_file, n_features)
+    x, y = load_svmlight_file(tmp_file, n_features=n_features)
     if not store_sparse:
         x = x.toarray()
 
diff --git a/requirements.txt b/requirements.txt
@@ -1,6 +1,8 @@
-scikit-learn>=0.22.1, <=0.24.1
+scikit-learn>=1.1.1
 scipy>=1.3.0
-numpy>=1.18.1, <=1.19.5
+numpy>=1.18.1
 numpydoc>=0.8.0
 cvxpy>=1.1.5
-cbor2>=5.4.0
+cbor2>=5.4.0
+pandas>=0.24.2
+matplotlib>=2.2.3
diff --git a/tests/__init__.py b/tests/__init__.py
@@ -0,0 +1,12 @@
+from time import time
+import unittest
+
+
+class BaseTimedTestCase(unittest.TestCase):
+    def setUp(self):
+        self.start_time = time()
+
+    def tearDown(self):
+        self.end_time = time()
+        print("Test %s took: %.3f seconds" %
+              (self.id(), self.end_time - self.start_time))
diff --git a/tests/test_als.py b/tests/test_als.py
@@ -6,6 +6,7 @@
 import dislib as ds
 from dislib.recommendation import ALS
 import dislib.data.util.model as utilmodel
+from tests import BaseTimedTestCase
 
 
 def load_movielens(train_ratio=0.9):
@@ -42,7 +43,7 @@ def load_movielens(train_ratio=0.9):
     return train_arr, test_arr
 
 
-class ALSTest(unittest.TestCase):
+class ALSTest(BaseTimedTestCase):
     def test_init_params(self):
         # Test all parameters
         seed = 666
diff --git a/tests/test_array.py b/tests/test_array.py
@@ -9,6 +9,7 @@
 import pandas as pd
 import dislib as ds
 from math import ceil
+from tests import BaseTimedTestCase
 
 
 def _sum_and_mult(arr, a=0, axis=0, b=1):
@@ -91,7 +92,7 @@ def _gen_irregular_arrays(fmt, shape=None, block_size=None):
         return x[1:, 1:], x_sp[1:, 1:]
 
 
-class DataLoadingTest(unittest.TestCase):
+class DataLoadingTest(BaseTimedTestCase):
 
     @parameterized.expand([(_gen_random_arrays("dense", (6, 10), (4, 3))
                             + ((6, 10), (4, 3))),
@@ -295,7 +296,7 @@ def test_load_mdcrd_file(self):
         self.assertTrue(_validate_array(x))
 
 
-class LoadBlocksRechunkTest(unittest.TestCase):
+class LoadBlocksRechunkTest(BaseTimedTestCase):
     def test_rechunk_new_block_size_exception(self):
         """ Tests that load_blocks_rechunk function throws an exception
         when the block_size returned of the rechunk is greater than the shape
@@ -347,7 +348,7 @@ def test_rechunk_block_size_exception(self):
             ds.data.load_blocks_rechunk(blocks, (20, 20), (25, 25), (5, 5))
 
 
-class LoadHStackNpyFilesTest(unittest.TestCase):
+class LoadHStackNpyFilesTest(BaseTimedTestCase):
     folder = 'load_hstack_npy_files_test_folder'
     arrays = [np.random.rand(3, 4) for _ in range(5)]
 
@@ -372,7 +373,7 @@ def test_load_hstack_npy_files_2(self):
         self.assertTrue(np.allclose(x.collect(), np.hstack(self.arrays)))
 
 
-class SaveTxtTest(unittest.TestCase):
+class SaveTxtTest(BaseTimedTestCase):
     folder = 'save_txt_test_folder'
 
     def tearDown(self):
@@ -408,7 +409,7 @@ def test_save_txt_merge_rows(self, x, x_np):
         self.assertTrue(_equal_arrays(np.vstack(h_blocks), x_np))
 
 
-class SaveNpyTest(unittest.TestCase):
+class SaveNpyTest(BaseTimedTestCase):
     folder = 'save_npy_test_folder'
 
     def tearDown(self):
@@ -454,7 +455,7 @@ def test_load_npy_files(self):
             ds.data.load_npy_files('save_npy_test_folder')
 
 
-class ArrayTest(unittest.TestCase):
+class ArrayTest(BaseTimedTestCase):
 
     @parameterized.expand([_gen_random_arrays("dense"),
                            _gen_random_arrays("sparse")])
@@ -1086,7 +1087,7 @@ def test_median(self):
             x_csr.median()
 
 
-class MathTest(unittest.TestCase):
+class MathTest(BaseTimedTestCase):
 
     @parameterized.expand([((21, 33), (10, 15), False),
                            ((5, 10), (8, 1), False),
diff --git a/tests/test_csvm.py b/tests/test_csvm.py
@@ -8,9 +8,10 @@
 import dislib as ds
 from dislib.classification import CascadeSVM
 import dislib.data.util.model as utilmodel
+from tests import BaseTimedTestCase
 
 
-class CSVMTest(unittest.TestCase):
+class CSVMTest(BaseTimedTestCase):
     def test_init_params(self):
         """ Test constructor parameters"""
         cascade_arity = 3
diff --git a/tests/test_data_utils.py b/tests/test_data_utils.py
@@ -6,8 +6,10 @@
 from dislib.data.util import pad, pad_last_blocks_with_zeros, \
     compute_bottom_right_shape, remove_last_columns, sync_obj
 
+from tests import BaseTimedTestCase
 
-class DataUtilsTest(unittest.TestCase):
+
+class DataUtilsTest(BaseTimedTestCase):
 
     @parameterized.expand([
         ((3, 4), (3, 3), (3, 1)),
diff --git a/tests/test_daura.py b/tests/test_daura.py
@@ -1,12 +1,11 @@
-import unittest
-
 import numpy as np
 
 import dislib as ds
 from dislib.cluster import Daura
+from tests import BaseTimedTestCase
 
 
-class GaussianMixtureTest(unittest.TestCase):
+class GaussianMixtureTest(BaseTimedTestCase):
 
     def test_init_params(self):
         """Tests that Daura fit_predict"""
diff --git a/tests/test_dbscan.py b/tests/test_dbscan.py
@@ -11,9 +11,10 @@
 import dislib as ds
 from dislib.cluster import DBSCAN
 from dislib.cluster.dbscan.base import _arrange_samples, _rearrange_labels
+from tests import BaseTimedTestCase
 
 
-class ArrangeTest(unittest.TestCase):
+class ArrangeTest(BaseTimedTestCase):
 
     def test_arrange(self):
         """ Tests the arrange method with toy data."""
@@ -160,7 +161,7 @@ def test_arrange_sparse(self):
             self.assertTrue(np.array_equal(samples_sp, samples_d))
 
 
-class DBSCANTest(unittest.TestCase):
+class DBSCANTest(BaseTimedTestCase):
     def test_n_clusters_blobs(self):
         """ Tests that DBSCAN finds the correct number of clusters with blob
         data.
diff --git a/tests/test_decision_tree.py b/tests/test_decision_tree.py
@@ -7,9 +7,10 @@
 import dislib.trees.decision_tree as dt
 from dislib.trees import RfClassifierDataset, transform_to_rf_dataset
 from dislib.trees.decision_tree import _InnerNodeInfo, _LeafInfo
+from tests import BaseTimedTestCase
 
 
-class DecisionTreeTest(unittest.TestCase):
+class DecisionTreeTest(BaseTimedTestCase):
     def test_decision_tree(self):
         x1 = np.array(
             [
diff --git a/tests/test_gm.py b/tests/test_gm.py
@@ -12,9 +12,10 @@
 import dislib as ds
 from dislib.cluster import GaussianMixture
 import dislib.data.util.model as utilmodel
+from tests import BaseTimedTestCase
 
 
-class GaussianMixtureTest(unittest.TestCase):
+class GaussianMixtureTest(BaseTimedTestCase):
 
     def test_init_params(self):
         """Tests that GaussianMixture params are set"""
diff --git a/tests/test_gridsearch.py b/tests/test_gridsearch.py
@@ -1,4 +1,3 @@
-import unittest
 import numpy as np
 
 from sklearn import clone, datasets
@@ -13,9 +12,10 @@
 from dislib.regression import LinearRegression
 from dislib.model_selection import GridSearchCV, KFold
 from dislib.utils import shuffle
+from tests import BaseTimedTestCase
 
 
-class GridSearchCVTest(unittest.TestCase):
+class GridSearchCVTest(BaseTimedTestCase):
 
     def test_estimators_compatibility(self):
         """Tests that dislib estimators are compatible with GridSearchCV.
diff --git a/tests/test_kfold.py b/tests/test_kfold.py
@@ -4,9 +4,10 @@
 
 import dislib as ds
 from dislib.model_selection import KFold
+from tests import BaseTimedTestCase
 
 
-class KFoldTest(unittest.TestCase):
+class KFoldTest(BaseTimedTestCase):
 
     def test_split(self):
         """Tests KFold.split() method"""
diff --git a/tests/test_kmeans.py b/tests/test_kmeans.py
@@ -8,9 +8,10 @@
 import dislib as ds
 from dislib.cluster import KMeans
 import dislib.data.util.model as utilmodel
+from tests import BaseTimedTestCase
 
 
-class KMeansTest(unittest.TestCase):
+class KMeansTest(BaseTimedTestCase):
     def test_init_params(self):
         """ Tests that KMeans object correctly sets the initialization
         parameters """
diff --git a/tests/test_knn.py b/tests/test_knn.py
@@ -6,9 +6,10 @@
 
 import dislib as ds
 from dislib.neighbors import NearestNeighbors
+from tests import BaseTimedTestCase
 
 
-class NearestNeighborsTest(unittest.TestCase):
+class NearestNeighborsTest(BaseTimedTestCase):
     def test_kneighbors(self):
         """ Tests kneighbors against scikit-learn """
         x = np.random.random((1500, 5))
diff --git a/tests/test_lasso.py b/tests/test_lasso.py
@@ -1,14 +1,13 @@
-import unittest
-
 import numpy as np
 from sklearn.metrics import r2_score
 
 import dislib as ds
 from dislib.regression import Lasso
 import dislib.data.util.model as utilmodel
+from tests import BaseTimedTestCase
 
 
-class LassoTest(unittest.TestCase):
+class LassoTest(BaseTimedTestCase):
 
     def test_fit_predict(self):
         """ Tests fit and predicts methods """
diff --git a/tests/test_linear_regression.py b/tests/test_linear_regression.py
@@ -7,9 +7,10 @@
 from dislib.regression import LinearRegression
 from dislib.data import random_array
 import dislib.data.util.model as utilmodel
+from tests import BaseTimedTestCase
 
 
-class LinearRegressionTest(unittest.TestCase):
+class LinearRegressionTest(BaseTimedTestCase):
 
     def test_univariate(self):
         """Tests fit() and predict(), univariate."""
diff --git a/tests/test_matmul.py b/tests/test_matmul.py
@@ -5,6 +5,7 @@
 from scipy import sparse as sp
 
 import dislib as ds
+from tests import BaseTimedTestCase
 
 
 def _validate_array(x):
@@ -43,7 +44,7 @@ def _equal_arrays(x1, x2):
     return np.allclose(x1, x2)
 
 
-class MatmulTest(unittest.TestCase):
+class MatmulTest(BaseTimedTestCase):
 
     @parameterized.expand([((20, 30), (30, 10), False),
                            ((1, 10), (10, 7), False),
diff --git a/tests/test_pca.py b/tests/test_pca.py
@@ -6,9 +6,10 @@
 
 import dislib as ds
 from dislib.decomposition import PCA
+from tests import BaseTimedTestCase
 
 
-class PCATest(unittest.TestCase):
+class PCATest(BaseTimedTestCase):
 
     @parameterized.expand([("eig"),
                            ("svd")])
diff --git a/tests/test_preproc.py b/tests/test_preproc.py
@@ -11,8 +11,10 @@
 import dislib as ds
 from dislib.preprocessing import StandardScaler, MinMaxScaler
 
+from tests import BaseTimedTestCase
 
-class ScalerTest(unittest.TestCase):
+
+class ScalerTest(BaseTimedTestCase):
     def setUp(self) -> None:
         self.x, _ = make_blobs(n_samples=1500, n_features=8,
                                random_state=170, cluster_std=33)
diff --git a/tests/test_qr.py b/tests/test_qr.py
diff --git a/tests/test_randomizedsearch.py b/tests/test_randomizedsearch.py
diff --git a/tests/test_rf_classifier.py b/tests/test_rf_classifier.py
diff --git a/tests/test_rf_dataset.py b/tests/test_rf_dataset.py
diff --git a/tests/test_rf_regressor.py b/tests/test_rf_regressor.py
diff --git a/tests/test_tsqr.py b/tests/test_tsqr.py
diff --git a/tests/test_utils.py b/tests/test_utils.py
