diff --git a/ads/aqua/evaluation/evaluation.py b/ads/aqua/evaluation/evaluation.py index 13adf0bdb..47497aa26 100644 --- a/ads/aqua/evaluation/evaluation.py +++ b/ads/aqua/evaluation/evaluation.py @@ -31,6 +31,7 @@ Tags, ) from ads.aqua.common.errors import ( + AquaError, AquaFileExistsError, AquaFileNotFoundError, AquaMissingKeyError, @@ -75,6 +76,7 @@ CreateAquaEvaluationDetails, ) from ads.aqua.evaluation.errors import EVALUATION_JOB_EXIT_CODE_MESSAGE +from ads.aqua.model.constants import ModelCustomMetadataFields from ads.aqua.ui import AquaContainerConfig from ads.common.auth import default_signer from ads.common.object_storage_details import ObjectStorageDetails @@ -183,6 +185,23 @@ def create( evaluation_source = ModelDeployment.from_id( create_aqua_evaluation_details.evaluation_source_id ) + + if Tags.MULTIMODEL_TYPE_TAG in evaluation_source.freeform_tags: + multi_model_id = evaluation_source.freeform_tags.get( + Tags.AQUA_MODEL_ID_TAG, UNKNOWN + ) + + if not multi_model_id: + raise AquaRuntimeError( + f"Invalid multi model deployment {multi_model_id}." + f"Make sure the {Tags.AQUA_MODEL_ID_TAG} tag is added to the deployment." + ) + + aqua_model = DataScienceModel.from_id(multi_model_id) + AquaEvaluationApp.validate_model_name( + aqua_model, create_aqua_evaluation_details + ) + try: if ( evaluation_source.runtime.type @@ -550,6 +569,70 @@ def create( parameters=AquaEvalParams(), ) + @staticmethod + def validate_model_name( + evaluation_source: DataScienceModel, + create_aqua_evaluation_details: CreateAquaEvaluationDetails, + ) -> None: + """ + Validates the user input of the model name when creating an Aqua evaluation. + + Parameters + ---------- + evaluation_source: DataScienceModel + The DataScienceModel Object which contains all metadata + about each model in a single and multi model deployment. + create_aqua_evaluation_details: CreateAquaEvaluationDetails + The CreateAquaEvaluationDetails data class which contains all + required and optional fields to create the aqua evaluation. + + Raises + ------- + AquaValueError: + - When the user fails to specify any input for the model name. + - When the user supplies a model name that does not match the model name set in the DataScienceModel metadata. + - When the DataScienceModel metadata lacks core attributes for validating the name""" + user_model_parameters = create_aqua_evaluation_details.model_parameters + + custom_metadata_list = evaluation_source.custom_metadata_list + user_model_name = user_model_parameters.get("model") + + model_count = custom_metadata_list.get(ModelCustomMetadataFields.MULTIMODEL_GROUP_COUNT) + + if model_count and custom_metadata_list: + model_group_count = int(model_count.value) + else: + logger.debug( + f"The ModelCustomMetadataFields.MULTIMODEL_GROUP_COUNT or custom_metadata_list (ModelCustomMetadata) is missing from the metadata in evaluation source ID: {create_aqua_evaluation_details.evaluation_source_id}" + ) + raise AquaRuntimeError( + "Recreate the model deployment and retry the evaluation. An issue occured when initalizing the model group during deployment." + ) + + model_names = [ + custom_metadata_list.get(f"model-name-{idx}") + for idx in range(model_group_count) + ] + + valid_model_names = ", ".join(name.value for name in model_names if name is not None) + + if "model" not in user_model_parameters: + logger.debug( + f"User did not input model name for multi model deployment evaluation with evaluation source ID: {create_aqua_evaluation_details.evaluation_source_id}" + ) + raise AquaValueError( + f"Provide the model name. For evaluation, a single model needs to be targeted using the name in the multi model deployment. The valid model names for this Model Deployment are {valid_model_names}." + ) + + if user_model_name not in model_names: + + logger.debug( + f"User input for model name was {user_model_name}, expected {valid_model_names} evaluation source ID: {create_aqua_evaluation_details.evaluation_source_id}" + ) + raise AquaValueError( + f"Provide the correct model name. The valid model names for this Model Deployment are {valid_model_names}." + ) + def _build_evaluation_runtime( self, evaluation_id: str, @@ -1392,7 +1475,7 @@ def _fetch_jobrun( ) except Exception as e: logger.debug( - f"Failed to retreive job run: {jobrun_id}. " f"DEBUG INFO: {str(e)}" + f"Failed to retreive job run: {jobrun_id}. DEBUG INFO: {str(e)}" ) jobrun = None diff --git a/tests/unitary/with_extras/aqua/test_evaluation.py b/tests/unitary/with_extras/aqua/test_evaluation.py index ef3475184..1a88edc50 100644 --- a/tests/unitary/with_extras/aqua/test_evaluation.py +++ b/tests/unitary/with_extras/aqua/test_evaluation.py @@ -17,6 +17,7 @@ from ads.aqua.common import utils from ads.aqua.common.enums import Tags from ads.aqua.common.errors import ( + AquaError, AquaFileNotFoundError, AquaMissingKeyError, AquaRuntimeError, @@ -34,8 +35,10 @@ AquaEvalMetrics, AquaEvalReport, AquaEvaluationSummary, + CreateAquaEvaluationDetails, ) from ads.aqua.extension.base_handler import AquaAPIhandler +from ads.aqua.model.constants import ModelCustomMetadataFields from ads.jobs.ads_job import DataScienceJob, DataScienceJobRun, Job from ads.model import DataScienceModel from ads.model.deployment.model_deployment import ModelDeployment @@ -353,6 +356,7 @@ class TestDataset: COMPARTMENT_ID = "ocid1.compartment.oc1.." EVAL_ID = "ocid1.datasciencemodel.oc1.iad." INVALID_EVAL_ID = "ocid1.datasciencemodel.oc1.phx." + MODEL_DEPLOYMENT_ID = "ocid1.datasciencemodeldeployment.oc1.." class TestAquaEvaluation(unittest.TestCase): @@ -533,6 +537,64 @@ def test_create_evaluation( "time_created": f"{oci_dsc_model.time_created}", } + @parameterized.expand( + [ + ( + {}, + "Provide the model name. For evaluation, a single model needs to be targeted using the name in the multi model deployment. The valid model names for this Model Deployment are model_one, model_two, model_three." + ), + ( + {"model": "wrong_model_name"}, + "Provide the correct model name. The valid model names for this Model Deployment are model_one, model_two, model_three." + ) + ]) + @patch("ads.aqua.evaluation.evaluation.AquaEvaluationApp.create") + def test_validate_model_name( + self, + mock_model_parameters, + expected_message, + mock_model + ): + curr_dir = os.path.dirname(__file__) + + eval_model_freeform_tags = {"ftag1": "fvalue1", "ftag2": "fvalue2"} + eval_model_defined_tags = {"dtag1": "dvalue1", "dtag2": "dvalue2"} + + eval_model_freeform_tags[Tags.MULTIMODEL_TYPE_TAG] = "true" + eval_model_freeform_tags[Tags.AQUA_TAG] = "active" + + create_aqua_evaluation_details = dict( # noqa: C408 + evaluation_source_id= TestDataset.MODEL_DEPLOYMENT_ID, + evaluation_name="test_evaluation_name", + dataset_path="oci://dataset_bucket@namespace/prefix/dataset.jsonl", + report_path="oci://report_bucket@namespace/prefix/", + model_parameters=mock_model_parameters, + shape_name="VM.Standard.E3.Flex", + block_storage_size=1, + experiment_name="test_experiment_name", + memory_in_gbs=1, + ocpus=1, + freeform_tags=eval_model_freeform_tags, + defined_tags=eval_model_defined_tags, + ) + + + aqua_multi_model = os.path.join( + curr_dir, "test_data/deployment/aqua_multi_model.yaml" + ) + + mock_model = DataScienceModel.from_yaml( + uri=aqua_multi_model + ) + + mock_create_aqua_evaluation_details = MagicMock(**create_aqua_evaluation_details, spec=CreateAquaEvaluationDetails) + + try: + AquaEvaluationApp.validate_model_name(mock_model, mock_create_aqua_evaluation_details) + except AquaError as e: + print(str(e)) + self.assertEqual(str(e), expected_message) + def test_get_service_model_name(self): # get service model name from fine tuned model deployment source = ModelDeployment().with_freeform_tags(