[RAPTOR-3968] Allow DRUM multiclass with only 2 labels (#237), release 1.4.5

* [RAPTOR-3968] Allow DRUM multiclass with only 2 labels

* fix transform test

* remove un-needed print

* change to full release instead of RC

Author: Gregory Meltzer

custom_model_runner/CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+#### [1.4.5] - 2020-12-02
+##### Changes
+- Allow multiclass to function with only 2 labels
+
 #### [1.4.4] - 2020-11-24
 ##### Added
 - New `transform` target type for performing pre-/post- processing on features/targets

custom_model_runner/datarobot_drum/drum/args_parser.py

@@ -211,8 +211,8 @@ def are_both_labels_present(arg):
     @staticmethod
     def _reg_arg_multiclass_labels(*parsers):
         class RequiredLength(argparse.Action):
-            ERROR_MESSAGE = "Multiclass classification requires at least 3 labels."
-            MIN_LABELS = 3
+            ERROR_MESSAGE = "Multiclass classification requires at least 2 labels."
+            MIN_LABELS = 2
 
             def __call__(self, parser, namespace, values, option_string=None):
                 if len(values) < self.MIN_LABELS:

custom_model_runner/datarobot_drum/drum/artifact_predictors/keras_predictor.py

@@ -73,12 +73,18 @@ def predict(self, data, model, **kwargs):
         if self.target_type.value in TargetType.CLASSIFICATION.value:
             if predictions.shape[1] == 1:
                 if self.target_type == TargetType.MULTICLASS:
-                    raise DrumCommonException(
-                        "Target type '{}' predictions must return the "
-                        "probability distribution for all class labels".format(self.target_type)
-                    )
-                predictions = pd.DataFrame(predictions, columns=[self.positive_class_label])
-                predictions[self.negative_class_label] = 1 - predictions[self.positive_class_label]
+                    if len(self.class_labels) > 2:
+                        raise DrumCommonException(
+                            "Target type '{}' predictions must return the "
+                            "probability distribution for all class labels".format(self.target_type)
+                        )
+                    pos_label = self.class_labels[1]
+                    neg_label = self.class_labels[0]
+                else:
+                    pos_label = self.positive_class_label
+                    neg_label = self.negative_class_label
+                predictions = pd.DataFrame(predictions, columns=[pos_label])
+                predictions[neg_label] = 1 - predictions[pos_label]
             else:
                 predictions = pd.DataFrame(predictions, columns=self.class_labels)
         elif self.target_type in [TargetType.REGRESSION, TargetType.ANOMALY]:

custom_model_runner/datarobot_drum/drum/artifact_predictors/torch_predictor.py

@@ -77,12 +77,18 @@ def predict(self, data, model, **kwargs):
         if self.target_type.value in TargetType.CLASSIFICATION.value:
             if predictions.shape[1] == 1:
                 if self.target_type == TargetType.MULTICLASS:
-                    raise DrumCommonException(
-                        "Target type '{}' predictions must return the "
-                        "probability distribution for all class labels".format(self.target_type)
-                    )
-                predictions = pd.DataFrame(predictions, columns=[self.positive_class_label])
-                predictions[self.negative_class_label] = 1 - predictions[self.positive_class_label]
+                    if len(self.class_labels) > 2:
+                        raise DrumCommonException(
+                            "Target type '{}' predictions must return the "
+                            "probability distribution for all class labels".format(self.target_type)
+                        )
+                    pos_label = self.class_labels[1]
+                    neg_label = self.class_labels[0]
+                else:
+                    pos_label = self.positive_class_label
+                    neg_label = self.negative_class_label
+                predictions = pd.DataFrame(predictions, columns=[pos_label])
+                predictions[neg_label] = 1 - predictions[pos_label]
             else:
                 predictions = pd.DataFrame(predictions, columns=self.class_labels)
         elif self.target_type in [TargetType.REGRESSION, TargetType.ANOMALY]:

custom_model_runner/datarobot_drum/drum/artifact_predictors/xgboost_predictor.py

@@ -83,7 +83,7 @@ def predict(self, data, model, **kwargs):
         if self.target_type.value in TargetType.CLASSIFICATION.value:
             if xgboost_native:
                 predictions = model.predict(data)
-                if self.target_type == TargetType.BINARY:
+                if self.target_type == TargetType.BINARY or len(self.class_labels) == 2:
                     negative_preds = 1 - predictions
                     predictions = np.concatenate(
                         (negative_preds.reshape(-1, 1), predictions.reshape(-1, 1)), axis=1

custom_model_runner/datarobot_drum/drum/description.py

@@ -1,3 +1,3 @@
-version = "1.4.4"
+version = "1.4.5"
 __version__ = version
 project_name = "datarobot-drum"

custom_model_runner/datarobot_drum/drum/drum.py

@@ -455,7 +455,7 @@ def _prepare_fit_pipeline(self, run_language):
                         self.options.negative_class_label,
                     ) = possible_class_labels
                 elif self.target_type == TargetType.MULTICLASS:
-                    if len(possible_class_labels) <= 2:
+                    if len(possible_class_labels) < 2:
                         raise DrumCommonException(
                             "Target type {} requires more than 2 class labels. Detected {}: {}".format(
                                 TargetType.MULTICLASS,

public_dropin_environments/java_codegen/dr_requirements.txt

@@ -1,2 +1,2 @@
 pyarrow==0.14.1
-datarobot-drum==1.4.4
+datarobot-drum==1.4.5

public_dropin_environments/java_codegen/env_info.json

@@ -3,6 +3,6 @@
   "name": "[DataRobot] Java Drop-In (DR Codegen, H2O)",
   "description": "This template can be used as an environment for DataRobot generated scoring code or models that implement the either the IClassificationPredictor or IRegressionPredictor interface from the datarobot-prediction package and for H2O models exported as POJO or MOJO.",
   "programmingLanguage": "java",
-  "environmentVersionId": "5fc33e141d41c812de88c04d",
+  "environmentVersionId": "5fc7f2e5e9790c6fd6032869",
   "isPublic": true
 }

public_dropin_environments/python3_keras/dr_requirements.txt

@@ -1,2 +1,2 @@
 pyarrow==0.14.1
-datarobot-drum==1.4.4
+datarobot-drum==1.4.5

public_dropin_environments/python3_keras/env_info.json

@@ -3,6 +3,6 @@
   "name": "[DataRobot] Python 3 Keras Drop-In",
   "description": "This template environment can be used to create artifact-only keras custom models. This environment contains keras backed by tensorflow and only requires your model artifact as a .h5 file and optionally a custom.py file.",
   "programmingLanguage": "python",
-  "environmentVersionId": "5fc33e141d41c812de88c04e",
+  "environmentVersionId": "5fc7f2e5e9790c6fd6032868",
   "isPublic": true
 }

public_dropin_environments/python3_pmml/dr_requirements.txt

@@ -1,2 +1,2 @@
 pyarrow==0.14.1
-datarobot-drum==1.4.4
+datarobot-drum==1.4.5

public_dropin_environments/python3_pmml/env_info.json

@@ -3,6 +3,6 @@
   "name": "[DataRobot] Python 3 PMML Drop-In",
   "description": "This template environment can be used to create artifact-only PMML custom models. This environment contains PyPMML and only requires your model artifact as a .pmml file and optionally a custom.py file.",
   "programmingLanguage": "python",
-  "environmentVersionId": "5fc33e141d41c812de88c04f",
+  "environmentVersionId": "5fc7f2e5e9790c6fd603286a",
   "isPublic": true
 }

public_dropin_environments/python3_pytorch/dr_requirements.txt

@@ -1,2 +1,2 @@
 pyarrow==0.14.1
-datarobot-drum==1.4.4
+datarobot-drum==1.4.5

public_dropin_environments/python3_pytorch/env_info.json

@@ -3,6 +3,6 @@
   "name": "[DataRobot] Python 3 PyTorch Drop-In",
   "description": "This template environment can be used to create artifact-only PyTorch custom models. This environment contains PyTorch and requires only your model artifact as a .pth file, any other code needed to deserialize your model, and optionally a custom.py file.",
   "programmingLanguage": "python",
-  "environmentVersionId": "5fc33e141d41c812de88c04a",
+  "environmentVersionId": "5fc7f2e5e9790c6fd6032864",
   "isPublic": true
 }

public_dropin_environments/python3_sklearn/dr_requirements.txt

@@ -1,2 +1,2 @@
 pyarrow==0.14.1
-datarobot-drum==1.4.4
+datarobot-drum==1.4.5

public_dropin_environments/python3_sklearn/env_info.json

@@ -3,6 +3,6 @@
   "name": "[DataRobot] Python 3 Scikit-Learn Drop-In",
   "description": "This template environment can be used to create artifact-only scikit-learn custom models. This environment contains scikit-learn and only requires your model artifact as a .pkl file and optionally a custom.py file.",
   "programmingLanguage": "python",
-  "environmentVersionId": "5fc33e141d41c812de88c04b",
+  "environmentVersionId": "5fc7f2e5e9790c6fd6032867",
   "isPublic": true
 }

public_dropin_environments/python3_xgboost/dr_requirements.txt

@@ -1,2 +1,2 @@
 pyarrow==0.14.1
-datarobot-drum==1.4.4
+datarobot-drum==1.4.5

public_dropin_environments/python3_xgboost/env_info.json

@@ -3,6 +3,6 @@
   "name": "[DataRobot] Python 3 XGBoost Drop-In",
   "description": "This template environment can be used to create artifact-only xgboost custom models. This environment contains xgboost and only requires your model artifact as a .pkl file and optionally a custom.py file.",
   "programmingLanguage": "python",
-  "environmentVersionId": "5fc33e141d41c812de88c04c",
+  "environmentVersionId": "5fc7f2e5e9790c6fd6032865",
   "isPublic": true
 }

public_dropin_environments/r_lang/dr_requirements.txt

@@ -2,4 +2,4 @@ numpy>=1.16.0,<1.19.0
 pandas==1.1.0
 rpy2<=3.3.6
 pyarrow==0.14.1
-datarobot-drum[R]==1.4.4
+datarobot-drum[R]==1.4.5

public_dropin_environments/r_lang/env_info.json

@@ -3,6 +3,6 @@
   "name": "[DataRobot] R Drop-In",
   "description": "This template environment can be used to create artifact-only R custom models that use the caret library. Your custom model archive need only contain your model artifacts if you use the environment correctly.",
   "programmingLanguage": "r",
-  "environmentVersionId": "5fc33e141d41c812de88c049",
+  "environmentVersionId": "5fc7f2e5e9790c6fd6032866",
   "isPublic": true
 }

tests/conftest.py

@@ -15,6 +15,7 @@
     MULTI_ARTIFACT,
     MULTICLASS,
     MULTICLASS_NUM_LABELS,
+    MULTICLASS_BINARY,
     NO_CUSTOM,
     POJO,
     PYPMML,
@@ -72,6 +73,7 @@
     (None, MULTICLASS_NUM_LABELS): os.path.join(
         TESTS_DATA_PATH, "skyserver_sql2_27_2018_6_51_39_pm_num_class.csv"
     ),
+    (None, MULTICLASS_BINARY): os.path.join(TESTS_DATA_PATH, "iris_binary_training.csv"),
     (None, SPARSE): os.path.join(TESTS_DATA_PATH, "sparse.mtx"),
     (None, SPARSE_TARGET): os.path.join(TESTS_DATA_PATH, "sparse_target.csv"),
     (None, BINARY_BOOL): os.path.join(TESTS_DATA_PATH, "10k_diabetes_sample.csv"),
@@ -106,6 +108,7 @@
     REGRESSION: "MEDV",
     BINARY_TEXT: "Churn",
     MULTICLASS: "class",
+    MULTICLASS_BINARY: "Species",
     MULTICLASS_NUM_LABELS: "class",
     SPARSE: "my_target",
     BINARY_BOOL: "readmitted",
@@ -120,6 +123,8 @@
     ANOMALY: "anomaly",
     UNSTRUCTURED: "unstructured",
     MULTICLASS: "multiclass",
+    MULTICLASS_BINARY: "multiclass",
+    MULTICLASS_NUM_LABELS: "multiclass",
     BINARY_BOOL: "binary",
     TRANSFORM: "transform",
 }
@@ -150,6 +155,16 @@
     (KERAS, BINARY_TEXT): ["False", "True"],
     (POJO, MULTICLASS): ["GALAXY", "QSO", "STAR"],
     (MOJO, MULTICLASS): ["GALAXY", "QSO", "STAR"],
+    (SKLEARN_BINARY, MULTICLASS_BINARY): ["Iris-setosa", "Iris-versicolor"],
+    (SKLEARN, MULTICLASS_BINARY): ["Iris-setosa", "Iris-versicolor"],
+    (XGB, MULTICLASS_BINARY): ["Iris-setosa", "Iris-versicolor"],
+    (KERAS, MULTICLASS_BINARY): ["Iris-setosa", "Iris-versicolor"],
+    (RDS, MULTICLASS_BINARY): ["Iris-setosa", "Iris-versicolor"],
+    (PYPMML, MULTICLASS_BINARY): ["Iris-setosa", "Iris-versicolor"],
+    (PYTORCH, MULTICLASS_BINARY): ["Iris-setosa", "Iris-versicolor"],
+    (CODEGEN, MULTICLASS_BINARY): ["yes", "no"],
+    (MOJO, MULTICLASS_BINARY): ["yes", "no"],
+    (POJO, MULTICLASS_BINARY): ["yes", "no"],
 }
 
 _artifacts = {
@@ -215,6 +230,21 @@
     (SKLEARN_TRANSFORM_DENSE, TRANSFORM): os.path.join(
         TESTS_ARTIFACTS_PATH, "sklearn_transform_dense.pkl"
     ),
+    (SKLEARN, MULTICLASS_BINARY): os.path.join(TESTS_ARTIFACTS_PATH, "sklearn_bin.pkl"),
+    (KERAS, MULTICLASS_BINARY): os.path.join(TESTS_ARTIFACTS_PATH, "keras_bin.h5"),
+    (XGB, MULTICLASS_BINARY): os.path.join(TESTS_ARTIFACTS_PATH, "xgb_bin.pkl"),
+    (PYTORCH, MULTICLASS_BINARY): [
+        os.path.join(TESTS_ARTIFACTS_PATH, "torch_bin.pth"),
+        os.path.join(TESTS_ARTIFACTS_PATH, "PyTorch.py"),
+    ],
+    (RDS, MULTICLASS_BINARY): os.path.join(TESTS_ARTIFACTS_PATH, "r_bin.rds"),
+    (CODEGEN, MULTICLASS_BINARY): os.path.join(TESTS_ARTIFACTS_PATH, "java_bin.jar"),
+    (POJO, MULTICLASS_BINARY): os.path.join(
+        TESTS_ARTIFACTS_PATH,
+        "XGBoost_grid__1_AutoML_20200717_163214_model_159.java",
+    ),
+    (MOJO, MULTICLASS_BINARY): os.path.join(TESTS_ARTIFACTS_PATH, "mojo_bin.zip"),
+    (PYPMML, MULTICLASS_BINARY): os.path.join(TESTS_ARTIFACTS_PATH, "iris_bin.pmml"),
 }
 
 _custom_filepaths = {
@@ -306,7 +336,7 @@ def _foo(problem):
 @pytest.fixture(scope="session")
 def get_target_type():
     def _foo(problem):
-        return _target_types[problem]
+        return _target_types.get(problem, problem)
 
     return _foo
 

tests/drum/constants.py

@@ -46,6 +46,7 @@
 MULTICLASS = "multiclass"
 TRANSFORM = "transform"
 MULTICLASS_NUM_LABELS = "multiclass_num_labels"
+MULTICLASS_BINARY = "multiclass_binary"  # special case for testing multiclass with only 2 classes
 SPARSE = "sparse"
 SPARSE_TARGET = "sparse_target"
 

tests/drum/drum_server_utils.py

@@ -78,7 +78,7 @@ def __init__(
             ArgumentsOptions.MAIN_COMMAND, custom_model_dir, target_type, self.server_address
         )
         if labels:
-            cmd = _cmd_add_class_labels(cmd, labels)
+            cmd = _cmd_add_class_labels(cmd, labels, target_type=target_type)
         if docker:
             cmd += " --docker {}".format(docker)
             if memory:

tests/drum/test_args_parser.py

@@ -45,7 +45,7 @@ def test_valid_class_labels(self, request, valid_labels, as_file, parser):
 
     @pytest.mark.parametrize("as_file", [True, False])
     def test_too_few_labels(self, as_file, parser):
-        labels = list("AB")
+        labels = list("A")
         with NamedTemporaryFile() as f:
             if as_file:
                 f.write("\n".join(labels).encode("utf-8"))
@@ -54,5 +54,5 @@ def test_too_few_labels(self, as_file, parser):
             else:
                 args = ["dummy", "--class-labels", *labels]
 
-            with pytest.raises(argparse.ArgumentTypeError, match="at least 3"):
+            with pytest.raises(argparse.ArgumentTypeError, match="at least 2"):
                 parser.parse_args(args)

tests/drum/test_fit.py

@@ -17,14 +17,14 @@
     KERAS,
     MULTICLASS,
     MULTICLASS_NUM_LABELS,
+    MULTICLASS_BINARY,
     PYTHON,
     PYTORCH,
     PYTORCH_MULTICLASS,
     R_FIT,
     RDS,
     REGRESSION,
     BINARY_TEXT,
-    TRANSFORM,
     SIMPLE,
     SKLEARN,
     SKLEARN_BINARY,
@@ -37,6 +37,7 @@
     WEIGHTS_CSV,
     XGB,
     BINARY_BOOL,
+    TRANSFORM,
 )
 from .utils import _cmd_add_class_labels, _create_custom_model_dir, _exec_shell_cmd
 
@@ -106,7 +107,9 @@ def test_fit_for_use_output_and_nested(
         if use_output:
             cmd += " --output {}".format(output)
         if problem == BINARY:
-            cmd = _cmd_add_class_labels(cmd, resources.class_labels(framework, problem))
+            cmd = _cmd_add_class_labels(
+                cmd, resources.class_labels(framework, problem), target_type=problem
+            )
         if docker:
             cmd += " --docker {} ".format(docker)
 
@@ -123,6 +126,7 @@ def test_fit_for_use_output_and_nested(
             (RDS, BINARY_TEXT, None),
             (RDS, REGRESSION, None),
             (RDS, MULTICLASS, None),
+            (RDS, MULTICLASS_BINARY, None),
             (SKLEARN_BINARY, BINARY_TEXT, DOCKER_PYTHON_SKLEARN),
             (SKLEARN_REGRESSION, REGRESSION, DOCKER_PYTHON_SKLEARN),
             (SKLEARN_ANOMALY, ANOMALY, DOCKER_PYTHON_SKLEARN),
@@ -131,18 +135,22 @@ def test_fit_for_use_output_and_nested(
             (SKLEARN_REGRESSION, REGRESSION, None),
             (SKLEARN_ANOMALY, ANOMALY, None),
             (SKLEARN_MULTICLASS, MULTICLASS, None),
+            (SKLEARN_MULTICLASS, MULTICLASS_BINARY, None),
             (SKLEARN_MULTICLASS, MULTICLASS_NUM_LABELS, None),
             (SKLEARN_TRANSFORM, REGRESSION, None),
             (SKLEARN_TRANSFORM, BINARY, None),
             (XGB, BINARY_TEXT, None),
             (XGB, REGRESSION, None),
             (XGB, MULTICLASS, None),
+            (XGB, MULTICLASS_BINARY, None),
             (KERAS, BINARY_TEXT, None),
             (KERAS, REGRESSION, None),
             (KERAS, MULTICLASS, None),
+            (KERAS, MULTICLASS_BINARY, None),
             (PYTORCH, BINARY_TEXT, None),
             (PYTORCH, REGRESSION, None),
             (PYTORCH_MULTICLASS, MULTICLASS, None),
+            (PYTORCH_MULTICLASS, MULTICLASS_BINARY, None),
         ],
     )
     @pytest.mark.parametrize("weights", [WEIGHTS_CSV, WEIGHTS_ARGS, None])
@@ -175,14 +183,9 @@ def test_fit(
             weights, input_dataset, r_fit=language == R_FIT
         )
 
-        if problem in [BINARY_TEXT, BINARY_BOOL]:
-            target_type = BINARY
-        elif problem == MULTICLASS_NUM_LABELS:
-            target_type = MULTICLASS
-        elif framework == SKLEARN_TRANSFORM:
-            target_type = TRANSFORM
-        else:
-            target_type = problem
+        target_type = (
+            resources.target_types(problem) if framework != SKLEARN_TRANSFORM else TRANSFORM
+        )
 
         cmd = "{} fit --target-type {} --code-dir {} --input {} --verbose ".format(
             ArgumentsOptions.MAIN_COMMAND, target_type, custom_model_dir, input_dataset
@@ -193,7 +196,9 @@ def test_fit(
             cmd += " --unsupervised"
 
         if problem in [BINARY, MULTICLASS]:
-            cmd = _cmd_add_class_labels(cmd, resources.class_labels(framework, problem))
+            cmd = _cmd_add_class_labels(
+                cmd, resources.class_labels(framework, problem), target_type=target_type
+            )
         if docker:
             cmd += " --docker {} ".format(docker)
 

tests/drum/test_fit_variety.py

@@ -47,7 +47,7 @@ def test_fit_variety(framework, variety_resources, resources, variety_data_names
         cmd += " --target {}".format(target)
 
     if problem == BINARY:
-        cmd = _cmd_add_class_labels(cmd, class_labels)
+        cmd = _cmd_add_class_labels(cmd, class_labels, target_type=problem)
 
     p, _, err = _exec_shell_cmd(
         cmd,