Merge branch 'main' into feature/aqua_ms_changes_2

Author: mrDzurb

.github/workflows/run-unittests-py39-py310.yml renamed to .github/workflows/run-unittests-py310-py311.yml

@@ -1,4 +1,4 @@
-name: "[Py3.9-3.11] - All Unit Tests"
+name: "[Py3.10-3.11] - All Unit Tests"
 
 on:
   workflow_dispatch:
@@ -33,15 +33,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11"]
         name: ["unitary", "slow_tests"]
         include:
           - name: "unitary"
             test-path: "tests/unitary"
             # `model` tests running in "slow_tests",
             # `feature_store` tests has its own test suite
-            # `forecast` tests not supported in python 3.9,3.10 (automlx dependency). Tests are running in python3.8 test env, see run-unittests-py38-cov-report.yml
-            # 'pii' tests run only with py3.8, 'datapane' library conflicts with pandas>2.2.0, which used in py3.9/3.10 setup
+            # `forecast` tests not run in this suite
             # 'hpo' tests hangs if run together with all unitary tests. Tests running in separate command before running all unitary
             ignore-path: |
               --ignore tests/unitary/with_extras/model \

.github/workflows/run-unittests-py38-cov-report.yml renamed to .github/workflows/run-unittests-py39-cov-report.yml

@@ -1,4 +1,4 @@
-name: "[Py3.8][COV REPORT] - All Unit Tests"
+name: "[Py3.9][COV REPORT] - All Unit Tests"
 
 on:
   workflow_dispatch:
@@ -26,7 +26,7 @@ env:
 
 jobs:
   test:
-    name: python 3.8, ${{ matrix.name }}
+    name: python 3.9, ${{ matrix.name }}
     runs-on: ubuntu-latest
     timeout-minutes: 90
 
@@ -58,7 +58,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.9"
           cache: "pip"
           cache-dependency-path: |
             pyproject.toml
@@ -71,7 +71,7 @@ jobs:
         name: "Test env setup"
         timeout-minutes: 30
 
-      # Installing pii deps for python3.8 test setup only, it will not work with python3.9/3.10, because
+      # Installing pii deps for python3.9 test setup only, it will not work with python3.9/3.10, because
       # 'datapane' library conflicts with pandas>2.2.0, which used in py3.9/3.10 setup
       - name: "Install PII dependencies"
         run: |

ads/opctl/operator/lowcode/forecast/model/automlx.py

@@ -249,17 +249,18 @@ def _generate_report(self):
                 self.explain_model()
 
                 global_explanation_section = None
-                if self.spec.explanations_accuracy_mode != SpeedAccuracyMode.AUTOMLX:
-                    # Convert the global explanation data to a DataFrame
-                    global_explanation_df = pd.DataFrame(self.global_explanation)
 
-                    self.formatted_global_explanation = (
-                        global_explanation_df / global_explanation_df.sum(axis=0) * 100
-                    )
-                    self.formatted_global_explanation = self.formatted_global_explanation.rename(
-                        {self.spec.datetime_column.name: ForecastOutputColumns.DATE},
-                        axis=1,
-                    )
+                # Convert the global explanation data to a DataFrame
+                global_explanation_df = pd.DataFrame(self.global_explanation)
+
+                self.formatted_global_explanation = (
+                    global_explanation_df / global_explanation_df.sum(axis=0) * 100
+                )
+
+                self.formatted_global_explanation.rename(
+                    columns={self.spec.datetime_column.name: ForecastOutputColumns.DATE},
+                    inplace=True,
+                )
 
                 aggregate_local_explanations = pd.DataFrame()
                 for s_id, local_ex_df in self.local_explanation.items():
@@ -428,7 +429,9 @@ def explain_model(self):
                     # Use the MLExplainer class from AutoMLx to generate explanations
                     explainer = automlx.MLExplainer(
                         self.models[s_id]["model"],
-                        self.datasets.additional_data.get_data_for_series(series_id=s_id)
+                        self.datasets.additional_data.get_data_for_series(
+                            series_id=s_id
+                        )
                         .drop(self.spec.datetime_column.name, axis=1)
                         .head(-self.spec.horizon)
                         if self.spec.additional_data
@@ -463,6 +466,13 @@ def explain_model(self):
 
                     # Store the explanations in the local_explanation dictionary
                     self.local_explanation[s_id] = explanations_df
+
+                    self.global_explanation[s_id] = dict(
+                        zip(
+                            self.local_explanation[s_id].columns,
+                            np.nanmean((self.local_explanation[s_id]), axis=0),
+                        )
+                    )
                 else:
                     # Fall back to the default explanation generation method
                     super().explain_model()

ads/opctl/operator/lowcode/forecast/model/base_model.py

@@ -503,6 +503,7 @@ def _save_report(
                         f2.write(f1.read())
 
         # forecast csv report
+        # todo: add test data into forecast.csv
         # if self.spec.test_data is not None:
         #     test_data_dict = test_data.get_dict_by_series()
         #     for series_id, test_data_values in test_data_dict.items():
@@ -772,14 +773,6 @@ def explain_model(self):
                     logger.warn(
                         "No explanations generated. Ensure that additional data has been provided."
                     )
-                elif (
-                    self.spec.model == SupportedModels.AutoMLX
-                    and self.spec.explanations_accuracy_mode
-                    == SpeedAccuracyMode.AUTOMLX
-                ):
-                    logger.warning(
-                        "Global explanations not available for AutoMLX models with inherent explainability"
-                    )
                 else:
                     self.global_explanation[s_id] = dict(
                         zip(

pyproject.toml

@@ -45,7 +45,6 @@ classifiers = [
   "Intended Audience :: Developers",
   "License :: OSI Approved :: Universal Permissive License (UPL)",
   "Operating System :: OS Independent",
-  "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
@@ -66,8 +65,7 @@ dependencies = [
   "numpy>=1.19.2,<2.0.0",
   "oci>=2.144.1",
   "ocifs>=1.1.3",
-  "pandas>1.2.1; python_version<'3.9'", # starting pandas v2.1.0 requires-python = '>=3.9'
-  "pandas>=2.2.0; python_version>='3.9'",
+  "pandas>=2.2.0",
   "psutil>=5.7.2",
   "python_jsonschema_objects>=0.3.13",
   "requests",
@@ -145,7 +143,7 @@ torch = [
   "torchvision"
 ]
 viz = [
-  "bokeh>=3.0.0,<3.2.0",  # starting 3.2.0 bokeh not supporting python3.8; relax after ADS will drop py3.8 support
+  "bokeh",
   "folium>=0.12.1",
   "graphviz<0.17",
   "scipy>=1.5.4",
@@ -203,7 +201,7 @@ pii = [
   "scrubadub_spacy",
   "spacy-transformers==1.2.5",
   "spacy==3.6.1",
-  "report-creator>=1.0.32",
+  "report-creator>=1.0.37",
 ]
 llm = ["langchain>=0.2", "langchain-community", "langchain_openai", "pydantic>=2,<3", "evaluate>=0.4.0"]
 aqua = ["jupyter_server"]
@@ -227,10 +225,8 @@ testsuite = [
   "pdfplumber",
   "py4j",
   "pyarrow>=15.0.0",
-  "statsmodels; python_version=='3.8'",
-  "statsmodels>=0.14.1; python_version>='3.9'",  # cython3.0 compatibility added in v0.14.1
-  "tables",
-  "tables>3.9.0; python_version>='3.9'",
+  "statsmodels>=0.14.1",
+  "tables>3.9.0",
   "xlrd>=1.2.0",
 ]