Deploy the model just if the previous model AucRoc is lower

[fabiopuddu77]

Hello,

Thank you for your support, I am talking about this script
vertex-ai-samples/notebooks/official/pipelines/automl_tabular_classification_beans.ipynb

Is there a way to list just the deployed models and match the previous AucRoc, because I can list just the models in the registry.

`@component(
base_image="gcr.io/deeplearning-platform-release/tf2-cpu.2-6:latest",
packages_to_install=["google-cloud-aiplatform"],
)

def classification_model_eval_metrics(
project: str,
location: str,
staging_bucket: str,
thresholds_dict_str: str,
model: Input[Artifact],
metrics: Output[Metrics],
metricsc: Output[ClassificationMetrics],
) -> NamedTuple("Outputs", [("dep_decision", str)]): # Return parameter.

import json
import logging

from google.cloud import aiplatform as aip

aip.init(project=project, staging_bucket=staging_bucket, location=location)


def retrieve_existing_model_metrics() -> int:
    """
    Retrieve the auRoc_existing metric from the evaluation of the existing model.
    """
    # Retrieve the existing model and its evaluation
    models = aip.Model.list(filter=f'labels.notebook={NOTEBOOK}')
    model = models[1]
    evaluation = model.get_model_evaluation().to_dict()

    # Extract the log loss metric
    auRoc_existing = evaluation['metrics']['auRoc']

    return auRoc_existing


# Fetch model eval info
def get_eval_info(model):
    response = model.list_model_evaluations()
    metrics_list = []
    metrics_string_list = []
    for evaluation in response:
        evaluation = evaluation.to_dict()
        print("model_evaluation")
        print(" name:", evaluation["name"])
        print(" metrics_schema_uri:", evaluation["metricsSchemaUri"])
        metrics = evaluation["metrics"]
        for metric in metrics.keys():
            logging.info("metric: %s, value: %s", metric, metrics[metric])
        metrics_str = json.dumps(metrics)
        metrics_list.append(metrics)
        metrics_string_list.append(metrics_str)

    return (
        evaluation["name"],
        metrics_list,
        metrics_string_list,
    )

# Use the given metrics threshold(s) to determine whether the model is
# accurate enough to deploy.
def classification_thresholds_check(metrics_dict, thresholds_dict):
    for k, v in thresholds_dict.items():
        logging.info("k {}, v {}".format(k, v))
        if k in ["auRoc", "auPrc"]:  # higher is better
            if metrics_dict[k] > v and metrics_dict[k] > retrieve_existing_model_metrics():   # if under threshold, and under previous model aurRoc don't deploy
                logging.info("AucRoc model trained: {} ; threshold: {}; AucRoc existing model: {}; returning True, Deploy the model".format(metrics_dict[k], v, retrieve_existing_model_metrics()))
                return True
logging.info("threshold checks not passed, don't deploy.")
return False

def log_metrics(metrics_list, metricsc):
    test_confusion_matrix = metrics_list[0]["confusionMatrix"]
    logging.info("rows: %s", test_confusion_matrix["rows"])

    # log the ROC curve
    fpr = []
    tpr = []
    thresholds = []
    for item in metrics_list[0]["confidenceMetrics"]:
        fpr.append(item.get("falsePositiveRate", 0.0))
        tpr.append(item.get("recall", 0.0))
        thresholds.append(item.get("confidenceThreshold", 0.0))
    print(f"fpr: {fpr}")
    print(f"tpr: {tpr}")
    print(f"thresholds: {thresholds}")
    metricsc.log_roc_curve(fpr, tpr, thresholds)

    # log the confusion matrix
    annotations = []
    for item in test_confusion_matrix["annotationSpecs"]:
        annotations.append(item["displayName"])
    logging.info("confusion matrix annotations: %s", annotations)
    metricsc.log_confusion_matrix(
        annotations,
        test_confusion_matrix["rows"],
    )

    # log textual metrics info as well
    for metric in metrics_list[0].keys():
        if metric != "confidenceMetrics":
            val_string = json.dumps(metrics_list[0][metric])
            metrics.log_metric(metric, val_string)

logging.getLogger().setLevel(logging.INFO)

# extract the model resource name from the input Model Artifact
model_resource_path = model.metadata["resourceName"]
logging.info("model path: %s", model_resource_path)

# Get the trained model resource
model = aip.Model(model_resource_path)

# Get model evaluation metrics from the the trained model
eval_name, metrics_list, metrics_str_list = get_eval_info(model)
logging.info("got evaluation name: %s", eval_name)
logging.info("got metrics list: %s", metrics_list)
log_metrics(metrics_list, metricsc)

thresholds_dict = json.loads(thresholds_dict_str)
deploy = classification_thresholds_check(metrics_list[0], thresholds_dict)
if deploy:
    dep_decision = "true"
else:
    dep_decision = "false"
logging.info("deployment decision is %s", dep_decision)

return (dep_decision,)

compiler.Compiler().compile(
classification_model_eval_metrics, "tabular_eval_component.yaml"
)`

Then I checked that if deploy is false I get an error:

ERROR 2024-04-10T11:30:01.606136975Z [resource.labels.taskName: workerpool0-0] /opt/conda/lib/python3.7/runpy.py:109: FutureWarning: Python 3.7 has reached end-of-life. KFP will drop support for Python 3.7 on April 23, 2024. To use new versions of the KFP SDK after that date, you will need to upgrade to Python >= 3.8. See https://devguide.python.org/versions/ for more details. ERROR 2024-04-10T11:30:01.606150371Z [resource.labels.taskName: workerpool0-0] __import__(pkg_name) ERROR 2024-04-10T11:30:01.606162923Z [resource.labels.taskName: workerpool0-0] Traceback (most recent call last): ERROR 2024-04-10T11:30:01.606177750Z [resource.labels.taskName: workerpool0-0] File "/opt/conda/lib/python3.7/runpy.py", line 193, in _run_module_as_main ERROR 2024-04-10T11:30:01.606191014Z [resource.labels.taskName: workerpool0-0] "__main__", mod_spec) ERROR 2024-04-10T11:30:01.606203029Z [resource.labels.taskName: workerpool0-0] File "/opt/conda/lib/python3.7/runpy.py", line 85, in _run_code ERROR 2024-04-10T11:30:01.606214414Z [resource.labels.taskName: workerpool0-0] exec(code, run_globals) ERROR 2024-04-10T11:30:01.606226670Z [resource.labels.taskName: workerpool0-0] File "/opt/conda/lib/python3.7/site-packages/kfp/dsl/executor_main.py", line 109, in <module> ERROR 2024-04-10T11:30:01.606238530Z [resource.labels.taskName: workerpool0-0] executor_main() ERROR 2024-04-10T11:30:01.606249872Z [resource.labels.taskName: workerpool0-0] File "/opt/conda/lib/python3.7/site-packages/kfp/dsl/executor_main.py", line 101, in executor_main ERROR 2024-04-10T11:30:01.606266251Z [resource.labels.taskName: workerpool0-0] output_file = executor.execute() ERROR 2024-04-10T11:30:01.606277959Z [resource.labels.taskName: workerpool0-0] File "/opt/conda/lib/python3.7/site-packages/kfp/dsl/executor.py", line 362, in execute ERROR 2024-04-10T11:30:01.606289417Z [resource.labels.taskName: workerpool0-0] return self.write_executor_output(result) ERROR 2024-04-10T11:30:01.606300357Z [resource.labels.taskName: workerpool0-0] File "/opt/conda/lib/python3.7/site-packages/kfp/dsl/executor.py", line 254, in write_executor_output ERROR 2024-04-10T11:30:01.606311640Z [resource.labels.taskName: workerpool0-0] if len(self.return_annotation._fields) != len(func_output): ERROR 2024-04-10T11:30:01.606323337Z [resource.labels.taskName: workerpool0-0] TypeError: object of type 'bool' has no len()