From fd292abbfe9c18d0c1e4c9ebfc29dac51d999805 Mon Sep 17 00:00:00 2001 From: Liz Johnson Date: Tue, 25 Feb 2025 15:36:19 -0800 Subject: [PATCH 1/4] added unit tests and finished validation method in evaluation.py --- ads/aqua/evaluation/evaluation.py | 60 ++++++++++++++++- .../with_extras/aqua/test_evaluation.py | 65 ++++++++++++++++++- 2 files changed, 123 insertions(+), 2 deletions(-) diff --git a/ads/aqua/evaluation/evaluation.py b/ads/aqua/evaluation/evaluation.py index 13adf0bdb..5d3c2aa30 100644 --- a/ads/aqua/evaluation/evaluation.py +++ b/ads/aqua/evaluation/evaluation.py @@ -75,6 +75,7 @@ CreateAquaEvaluationDetails, ) from ads.aqua.evaluation.errors import EVALUATION_JOB_EXIT_CODE_MESSAGE +from ads.aqua.model.constants import ModelCustomMetadataFields from ads.aqua.ui import AquaContainerConfig from ads.common.auth import default_signer from ads.common.object_storage_details import ObjectStorageDetails @@ -183,6 +184,26 @@ def create( evaluation_source = ModelDeployment.from_id( create_aqua_evaluation_details.evaluation_source_id ) + try: + if Tags.MULTIMODEL_TYPE_TAG in evaluation_source.freeform_tags: + multi_model_id = evaluation_source.freeform_tags.get( + Tags.AQUA_MODEL_ID_TAG, UNKNOWN + ) + + if not multi_model_id: + raise AquaRuntimeError( + f"Invalid multi model deployment {multi_model_id}." + f"Make sure the {Tags.AQUA_MODEL_ID_TAG} tag is added to the deployment." + ) + + aqua_model = DataScienceModel.from_id(multi_model_id) + AquaEvaluationApp.validate_name_multi_model( + aqua_model, create_aqua_evaluation_details + ) + + except (AquaRuntimeError, AquaValueError) as err: + raise AquaValueError(f"{err}") from err + try: if ( evaluation_source.runtime.type @@ -550,6 +571,43 @@ def create( parameters=AquaEvalParams(), ) + @staticmethod + def validate_name_multi_model( + evaluation_source: DataScienceModel, + create_aqua_evaluation_details: CreateAquaEvaluationDetails, + ): + user_model_parameters = create_aqua_evaluation_details.model_parameters + if "name" not in user_model_parameters: + logger.debug( + f"User did not input model name for multi model deployment evaluation with evaluation source ID: {create_aqua_evaluation_details.evaluation_source_id}" + ) + raise AquaValueError( + "Provide the model name. For evaluation, a single model needs to be targeted using the name in the multi model deployment." + ) + + custom_metadata_list = evaluation_source.custom_metadata_list + user_model_name = user_model_parameters.get("name") + + model_group_count = int( + custom_metadata_list.get( + ModelCustomMetadataFields.MULTIMODEL_GROUP_COUNT + ).value + ) + + model_names = [ + custom_metadata_list.get(f"model-name-{idx}").value + for idx in range(model_group_count) + ] + + if user_model_name not in model_names: + valid_model_names = ", ".join(map(str, model_names)) + logger.debug( + f"User input for model name was {user_model_name}, expected {valid_model_names} evaluation source ID: {create_aqua_evaluation_details.evaluation_source_id}" + ) + raise AquaValueError( + f"Provide the correct model name. The valid model names for this Model Deployment are {valid_model_names}." + ) + def _build_evaluation_runtime( self, evaluation_id: str, @@ -1392,7 +1450,7 @@ def _fetch_jobrun( ) except Exception as e: logger.debug( - f"Failed to retreive job run: {jobrun_id}. " f"DEBUG INFO: {str(e)}" + f"Failed to retreive job run: {jobrun_id}. DEBUG INFO: {str(e)}" ) jobrun = None diff --git a/tests/unitary/with_extras/aqua/test_evaluation.py b/tests/unitary/with_extras/aqua/test_evaluation.py index ef3475184..45ab2dc84 100644 --- a/tests/unitary/with_extras/aqua/test_evaluation.py +++ b/tests/unitary/with_extras/aqua/test_evaluation.py @@ -34,6 +34,7 @@ AquaEvalMetrics, AquaEvalReport, AquaEvaluationSummary, + CreateAquaEvaluationDetails, ) from ads.aqua.extension.base_handler import AquaAPIhandler from ads.jobs.ads_job import DataScienceJob, DataScienceJobRun, Job @@ -353,6 +354,7 @@ class TestDataset: COMPARTMENT_ID = "ocid1.compartment.oc1.." EVAL_ID = "ocid1.datasciencemodel.oc1.iad." INVALID_EVAL_ID = "ocid1.datasciencemodel.oc1.phx." + MODEL_DEPLOYMENT_ID = "ocid1.datasciencemodeldeployment.oc1.." class TestAquaEvaluation(unittest.TestCase): @@ -449,7 +451,7 @@ def test_create_evaluation( mock_from_id.return_value = foundation_model experiment = MagicMock() - experiment.id = "test_experiment_id" + experiment.id = "ocid1.datasciencemodelversionset.oc1.iad.amaaaaaav66vvniakngdzelb5hcgjd6yvfejksu2excidvvi3s5s5whtmdea" mock_mvs_create.return_value = experiment evaluation_model = MagicMock() @@ -533,6 +535,67 @@ def test_create_evaluation( "time_created": f"{oci_dsc_model.time_created}", } + @parameterized.expand( + [ + ( + {"name": "model_one"}, + None + ), + ( + {}, + "Provide the model name. For evaluation, a single model needs to be targeted using the name in the multi model deployment." + ), + ( + {"name": "wrong_model_name"}, + "Provide the correct model name. The valid model names for this Model Deployment are model_one, model_two, model_three." + ) + ]) + @patch("ads.aqua.evaluation.evaluation.AquaEvaluationApp.create") + def test_validate_multi_model_evaluation( + self, + mock_model_parameters, + expected_message, + mock_model + ): + curr_dir = os.path.dirname(__file__) + + eval_model_freeform_tags = {"ftag1": "fvalue1", "ftag2": "fvalue2"} + eval_model_defined_tags = {"dtag1": "dvalue1", "dtag2": "dvalue2"} + + eval_model_freeform_tags[Tags.MULTIMODEL_TYPE_TAG] = "true" + eval_model_freeform_tags[Tags.AQUA_TAG] = "active" + + create_aqua_evaluation_details = dict( # noqa: C408 + evaluation_source_id= TestDataset.MODEL_DEPLOYMENT_ID, + evaluation_name="test_evaluation_name", + dataset_path="oci://dataset_bucket@namespace/prefix/dataset.jsonl", + report_path="oci://report_bucket@namespace/prefix/", + model_parameters=mock_model_parameters, + shape_name="VM.Standard.E3.Flex", + block_storage_size=1, + experiment_name="test_experiment_name", + memory_in_gbs=1, + ocpus=1, + freeform_tags=eval_model_freeform_tags, + defined_tags=eval_model_defined_tags, + ) + + + aqua_multi_model = os.path.join( + curr_dir, "test_data/deployment/aqua_multi_model.yaml" + ) + + mock_model = DataScienceModel.from_yaml( + uri=aqua_multi_model + ) + + mock_create_aqua_evaluation_details = MagicMock(**create_aqua_evaluation_details, spec=CreateAquaEvaluationDetails) + + try: + AquaEvaluationApp.validate_name_multi_model(mock_model, mock_create_aqua_evaluation_details) + except Exception as e: + self.assertEqual(str(e), expected_message) + def test_get_service_model_name(self): # get service model name from fine tuned model deployment source = ModelDeployment().with_freeform_tags( From a4e2da02fbc48893e37d5df4805f96015d9c0eb1 Mon Sep 17 00:00:00 2001 From: Liz Johnson Date: Tue, 25 Feb 2025 16:02:41 -0800 Subject: [PATCH 2/4] fixed model parameter name --- ads/aqua/evaluation/evaluation.py | 4 ++-- tests/unitary/with_extras/aqua/test_evaluation.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ads/aqua/evaluation/evaluation.py b/ads/aqua/evaluation/evaluation.py index 5d3c2aa30..29bc90ee2 100644 --- a/ads/aqua/evaluation/evaluation.py +++ b/ads/aqua/evaluation/evaluation.py @@ -577,7 +577,7 @@ def validate_name_multi_model( create_aqua_evaluation_details: CreateAquaEvaluationDetails, ): user_model_parameters = create_aqua_evaluation_details.model_parameters - if "name" not in user_model_parameters: + if "model" not in user_model_parameters: logger.debug( f"User did not input model name for multi model deployment evaluation with evaluation source ID: {create_aqua_evaluation_details.evaluation_source_id}" ) @@ -586,7 +586,7 @@ def validate_name_multi_model( ) custom_metadata_list = evaluation_source.custom_metadata_list - user_model_name = user_model_parameters.get("name") + user_model_name = user_model_parameters.get("model") model_group_count = int( custom_metadata_list.get( diff --git a/tests/unitary/with_extras/aqua/test_evaluation.py b/tests/unitary/with_extras/aqua/test_evaluation.py index 45ab2dc84..9e419285c 100644 --- a/tests/unitary/with_extras/aqua/test_evaluation.py +++ b/tests/unitary/with_extras/aqua/test_evaluation.py @@ -538,7 +538,7 @@ def test_create_evaluation( @parameterized.expand( [ ( - {"name": "model_one"}, + {"model": "model_one"}, None ), ( @@ -546,7 +546,7 @@ def test_create_evaluation( "Provide the model name. For evaluation, a single model needs to be targeted using the name in the multi model deployment." ), ( - {"name": "wrong_model_name"}, + {"model": "wrong_model_name"}, "Provide the correct model name. The valid model names for this Model Deployment are model_one, model_two, model_three." ) ]) From b25fa2e3d419135c090e3dbeefe0d80853d3fcb1 Mon Sep 17 00:00:00 2001 From: Liz Johnson Date: Tue, 25 Feb 2025 17:55:52 -0800 Subject: [PATCH 3/4] added docstring, fixed PR comments --- ads/aqua/evaluation/evaluation.py | 45 ++++++++++++++----- .../with_extras/aqua/test_evaluation.py | 11 ++--- 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/ads/aqua/evaluation/evaluation.py b/ads/aqua/evaluation/evaluation.py index 29bc90ee2..33bd2bf24 100644 --- a/ads/aqua/evaluation/evaluation.py +++ b/ads/aqua/evaluation/evaluation.py @@ -31,6 +31,7 @@ Tags, ) from ads.aqua.common.errors import ( + AquaError, AquaFileExistsError, AquaFileNotFoundError, AquaMissingKeyError, @@ -197,11 +198,11 @@ def create( ) aqua_model = DataScienceModel.from_id(multi_model_id) - AquaEvaluationApp.validate_name_multi_model( + AquaEvaluationApp.validate_model_name( aqua_model, create_aqua_evaluation_details ) - except (AquaRuntimeError, AquaValueError) as err: + except AquaError as err: raise AquaValueError(f"{err}") from err try: @@ -572,18 +573,28 @@ def create( ) @staticmethod - def validate_name_multi_model( + def validate_model_name( evaluation_source: DataScienceModel, create_aqua_evaluation_details: CreateAquaEvaluationDetails, - ): + ) -> None: + """ + Validates the user input of the model name when creating an Aqua evaluation. + + Parameters + ---------- + evaluation_source: DataScienceModel + The DataScienceModel Object which contains all metadata + about each model in a single and multi model deployment. + create_aqua_evaluation_details: CreateAquaEvaluationDetails + The CreateAquaEvaluationDetails data class which contains all + required and optional fields to create the aqua evaluation. + + Raises + ------- + AquaValueError: + - When the user fails to specify any input for the model name. + - When the user supplies a model name that does not match the model name set in the DataScienceModel metadata.""" user_model_parameters = create_aqua_evaluation_details.model_parameters - if "model" not in user_model_parameters: - logger.debug( - f"User did not input model name for multi model deployment evaluation with evaluation source ID: {create_aqua_evaluation_details.evaluation_source_id}" - ) - raise AquaValueError( - "Provide the model name. For evaluation, a single model needs to be targeted using the name in the multi model deployment." - ) custom_metadata_list = evaluation_source.custom_metadata_list user_model_name = user_model_parameters.get("model") @@ -599,8 +610,18 @@ def validate_name_multi_model( for idx in range(model_group_count) ] + valid_model_names = ", ".join(map(str, model_names)) + + if "model" not in user_model_parameters: + logger.debug( + f"User did not input model name for multi model deployment evaluation with evaluation source ID: {create_aqua_evaluation_details.evaluation_source_id}" + ) + raise AquaValueError( + f"Provide the model name. For evaluation, a single model needs to be targeted using the name in the multi model deployment. The valid model names for this Model Deployment are {valid_model_names}." + ) + if user_model_name not in model_names: - valid_model_names = ", ".join(map(str, model_names)) + logger.debug( f"User input for model name was {user_model_name}, expected {valid_model_names} evaluation source ID: {create_aqua_evaluation_details.evaluation_source_id}" ) diff --git a/tests/unitary/with_extras/aqua/test_evaluation.py b/tests/unitary/with_extras/aqua/test_evaluation.py index 9e419285c..29c1cc58e 100644 --- a/tests/unitary/with_extras/aqua/test_evaluation.py +++ b/tests/unitary/with_extras/aqua/test_evaluation.py @@ -17,6 +17,7 @@ from ads.aqua.common import utils from ads.aqua.common.enums import Tags from ads.aqua.common.errors import ( + AquaError, AquaFileNotFoundError, AquaMissingKeyError, AquaRuntimeError, @@ -451,7 +452,7 @@ def test_create_evaluation( mock_from_id.return_value = foundation_model experiment = MagicMock() - experiment.id = "ocid1.datasciencemodelversionset.oc1.iad.amaaaaaav66vvniakngdzelb5hcgjd6yvfejksu2excidvvi3s5s5whtmdea" + experiment.id = "test_experiment_id" mock_mvs_create.return_value = experiment evaluation_model = MagicMock() @@ -543,7 +544,7 @@ def test_create_evaluation( ), ( {}, - "Provide the model name. For evaluation, a single model needs to be targeted using the name in the multi model deployment." + "Provide the model name. For evaluation, a single model needs to be targeted using the name in the multi model deployment. The valid model names for this Model Deployment are model_one, model_two, model_three." ), ( {"model": "wrong_model_name"}, @@ -551,7 +552,7 @@ def test_create_evaluation( ) ]) @patch("ads.aqua.evaluation.evaluation.AquaEvaluationApp.create") - def test_validate_multi_model_evaluation( + def test_validate_model_name( self, mock_model_parameters, expected_message, @@ -592,8 +593,8 @@ def test_validate_multi_model_evaluation( mock_create_aqua_evaluation_details = MagicMock(**create_aqua_evaluation_details, spec=CreateAquaEvaluationDetails) try: - AquaEvaluationApp.validate_name_multi_model(mock_model, mock_create_aqua_evaluation_details) - except Exception as e: + AquaEvaluationApp.validate_model_name(mock_model, mock_create_aqua_evaluation_details) + except AquaError as e: self.assertEqual(str(e), expected_message) def test_get_service_model_name(self): From 7e9d46e54c1caa29e37a7e0bea91a19f58f058c3 Mon Sep 17 00:00:00 2001 From: Liz Johnson Date: Wed, 26 Feb 2025 13:37:31 -0800 Subject: [PATCH 4/4] fixed PR comments --- ads/aqua/evaluation/evaluation.py | 50 ++++++++++--------- .../with_extras/aqua/test_evaluation.py | 6 +-- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/ads/aqua/evaluation/evaluation.py b/ads/aqua/evaluation/evaluation.py index 33bd2bf24..47497aa26 100644 --- a/ads/aqua/evaluation/evaluation.py +++ b/ads/aqua/evaluation/evaluation.py @@ -185,25 +185,22 @@ def create( evaluation_source = ModelDeployment.from_id( create_aqua_evaluation_details.evaluation_source_id ) - try: - if Tags.MULTIMODEL_TYPE_TAG in evaluation_source.freeform_tags: - multi_model_id = evaluation_source.freeform_tags.get( - Tags.AQUA_MODEL_ID_TAG, UNKNOWN - ) - if not multi_model_id: - raise AquaRuntimeError( - f"Invalid multi model deployment {multi_model_id}." - f"Make sure the {Tags.AQUA_MODEL_ID_TAG} tag is added to the deployment." - ) + if Tags.MULTIMODEL_TYPE_TAG in evaluation_source.freeform_tags: + multi_model_id = evaluation_source.freeform_tags.get( + Tags.AQUA_MODEL_ID_TAG, UNKNOWN + ) - aqua_model = DataScienceModel.from_id(multi_model_id) - AquaEvaluationApp.validate_model_name( - aqua_model, create_aqua_evaluation_details + if not multi_model_id: + raise AquaRuntimeError( + f"Invalid multi model deployment {multi_model_id}." + f"Make sure the {Tags.AQUA_MODEL_ID_TAG} tag is added to the deployment." ) - except AquaError as err: - raise AquaValueError(f"{err}") from err + aqua_model = DataScienceModel.from_id(multi_model_id) + AquaEvaluationApp.validate_model_name( + aqua_model, create_aqua_evaluation_details + ) try: if ( @@ -593,24 +590,31 @@ def validate_model_name( ------- AquaValueError: - When the user fails to specify any input for the model name. - - When the user supplies a model name that does not match the model name set in the DataScienceModel metadata.""" + - When the user supplies a model name that does not match the model name set in the DataScienceModel metadata. + - When the DataScienceModel metadata lacks core attributes for validating the name""" user_model_parameters = create_aqua_evaluation_details.model_parameters custom_metadata_list = evaluation_source.custom_metadata_list user_model_name = user_model_parameters.get("model") - model_group_count = int( - custom_metadata_list.get( - ModelCustomMetadataFields.MULTIMODEL_GROUP_COUNT - ).value - ) + model_count = custom_metadata_list.get(ModelCustomMetadataFields.MULTIMODEL_GROUP_COUNT) + + if model_count and custom_metadata_list: + model_group_count = int(model_count.value) + else: + logger.debug( + f"The ModelCustomMetadataFields.MULTIMODEL_GROUP_COUNT or custom_metadata_list (ModelCustomMetadata) is missing from the metadata in evaluation source ID: {create_aqua_evaluation_details.evaluation_source_id}" + ) + raise AquaRuntimeError( + "Recreate the model deployment and retry the evaluation. An issue occured when initalizing the model group during deployment." + ) model_names = [ - custom_metadata_list.get(f"model-name-{idx}").value + custom_metadata_list.get(f"model-name-{idx}") for idx in range(model_group_count) ] - valid_model_names = ", ".join(map(str, model_names)) + valid_model_names = ", ".join(name.value for name in model_names if name is not None) if "model" not in user_model_parameters: logger.debug( diff --git a/tests/unitary/with_extras/aqua/test_evaluation.py b/tests/unitary/with_extras/aqua/test_evaluation.py index 29c1cc58e..1a88edc50 100644 --- a/tests/unitary/with_extras/aqua/test_evaluation.py +++ b/tests/unitary/with_extras/aqua/test_evaluation.py @@ -38,6 +38,7 @@ CreateAquaEvaluationDetails, ) from ads.aqua.extension.base_handler import AquaAPIhandler +from ads.aqua.model.constants import ModelCustomMetadataFields from ads.jobs.ads_job import DataScienceJob, DataScienceJobRun, Job from ads.model import DataScienceModel from ads.model.deployment.model_deployment import ModelDeployment @@ -538,10 +539,6 @@ def test_create_evaluation( @parameterized.expand( [ - ( - {"model": "model_one"}, - None - ), ( {}, "Provide the model name. For evaluation, a single model needs to be targeted using the name in the multi model deployment. The valid model names for this Model Deployment are model_one, model_two, model_three." @@ -595,6 +592,7 @@ def test_validate_model_name( try: AquaEvaluationApp.validate_model_name(mock_model, mock_create_aqua_evaluation_details) except AquaError as e: + print(str(e)) self.assertEqual(str(e), expected_message) def test_get_service_model_name(self):