diff --git a/.pipelines/azdo-ci-build-train.yml b/.pipelines/azdo-ci-build-train.yml index c2453d4d..6ed9e1a9 100644 --- a/.pipelines/azdo-ci-build-train.yml +++ b/.pipelines/azdo-ci-build-train.yml @@ -11,7 +11,7 @@ trigger: - ml_service/util/create_scoring_image.py variables: -- group: devopsforai-aml-vg +- group: devopsforai-aml # Choose from default, build_train_pipeline_with_r.py, or build_train_pipeline_with_r_on_dbricks.py - name: build-train-script value: 'build_train_pipeline.py' @@ -34,38 +34,29 @@ stages: - template: azdo-base-pipeline.yml - script: | # Invoke the Python building and publishing a training pipeline - python3 $(Build.SourcesDirectory)/ml_service/pipelines/$(build-train-script) + python3 $(Build.SourcesDirectory)/ml_service/pipelines/build_train_pipeline.py + # Invoke the Python building and publishing an ACI deployment pipeline + python3 $(Build.SourcesDirectory)/ml_service/pipelines/build_deploy_pipeline.py + # Invoke the Python building and publishing an AKS deployment pipeline + python3 $(Build.SourcesDirectory)/ml_service/pipelines/build_deploy_prod_pipeline.py failOnStderr: 'false' env: SP_APP_SECRET: '$(SP_APP_SECRET)' displayName: 'Publish Azure Machine Learning Pipeline' -- stage: 'Trigger_AML_Pipeline' - displayName: 'Train, evaluate, register model via previously published AML pipeline' - jobs: - - job: "Invoke_Model_Pipeline" - condition: and(succeeded(), eq(coalesce(variables['auto-trigger-training'], 'true'), 'true')) - displayName: "Invoke Model Pipeline and evaluate results to register" - pool: - vmImage: 'ubuntu-latest' - container: mcr.microsoft.com/mlops/python:latest - timeoutInMinutes: 0 - steps: - - script: | - python $(Build.SourcesDirectory)/ml_service/pipelines/run_train_pipeline.py - displayName: 'Trigger Training Pipeline' - env: - SP_APP_SECRET: '$(SP_APP_SECRET)' + - task: CopyFiles@2 displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)' inputs: SourceFolder: '$(Build.SourcesDirectory)' TargetFolder: '$(Build.ArtifactStagingDirectory)' Contents: | - code/scoring/** + code/** + ml_service/pipelines/** - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact' inputs: ArtifactName: 'mlops-pipelines' publishLocation: 'container' pathtoPublish: '$(Build.ArtifactStagingDirectory)' - TargetPath: '$(Build.ArtifactStagingDirectory)' \ No newline at end of file + TargetPath: '$(Build.ArtifactStagingDirectory)' + diff --git a/ci-build.yml b/ci-build.yml new file mode 100644 index 00000000..b3c6f901 --- /dev/null +++ b/ci-build.yml @@ -0,0 +1,71 @@ +pr: none +trigger: + branches: + include: + - master + paths: + exclude: + - docs/ + - environment_setup/ + - charts/ + - ml_service/util/create_scoring_image.py + +variables: +- group: devopsforai-aml +# Choose from default, build_train_pipeline_with_r.py, or build_train_pipeline_with_r_on_dbricks.py +- name: build-train-script + value: 'build_train_pipeline.py' +# Automatically triggers the train, evaluate, register pipeline after the CI steps. +# Uncomment to set to false or add same variable name at queue time with value of false to disable. +# - name: auto-trigger-training +# value: false + +stages: +- stage: 'Model_CI' + displayName: 'Model CI' + jobs: + - job: "Model_CI_Pipeline" + displayName: "Model CI Pipeline" + pool: + vmImage: 'ubuntu-latest' + container: mcr.microsoft.com/mlops/python:latest + timeoutInMinutes: 0 + steps: + - template: azdo-base-pipeline.yml + - script: | + # Invoke the Python building and publishing a training pipeline + python3 $(Build.SourcesDirectory)/ml_service/pipelines/$(build-train-script) + failOnStderr: 'false' + env: + SP_APP_SECRET: '$(SP_APP_SECRET)' + displayName: 'Publish Azure Machine Learning Pipeline' +- stage: 'Trigger_AML_Pipeline' + displayName: 'Train, evaluate, register model via previously published AML pipeline' + jobs: + - job: "Invoke_Model_Pipeline" + condition: and(succeeded(), eq(coalesce(variables['auto-trigger-training'], 'true'), 'true')) + displayName: "Invoke Model Pipeline and evaluate results to register" + pool: + vmImage: 'ubuntu-latest' + container: mcr.microsoft.com/mlops/python:latest + timeoutInMinutes: 0 + steps: + - script: | + python $(Build.SourcesDirectory)/ml_service/pipelines/run_train_pipeline.py + displayName: 'Trigger Training Pipeline' + env: + SP_APP_SECRET: '$(SP_APP_SECRET)' + - task: CopyFiles@2 + displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + Contents: | + code/scoring/** + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact' + inputs: + ArtifactName: 'dspe-pipelines' + publishLocation: 'container' + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + TargetPath: '$(Build.ArtifactStagingDirectory)' \ No newline at end of file diff --git a/code/deploy/deploy_model.py b/code/deploy/deploy_model.py new file mode 100644 index 00000000..9fc03b4d --- /dev/null +++ b/code/deploy/deploy_model.py @@ -0,0 +1,75 @@ +from azureml.core.model import InferenceConfig +from azureml.core.webservice.aci import AciServiceDeploymentConfiguration +from azureml.core import Run, Model +import argparse + +parser = argparse.ArgumentParser("deploy") +parser.add_argument( + "--release_id", + type=str, + help="The ID of the release triggering this pipeline run", +) +parser.add_argument( + "--model_name", + type=str, + help="Name of the Model", + default="sklearn_abn_ca_model.pkl", +) +parser.add_argument( + "--service_name", + type=str, + help="Name of the datastore", + default="abn-ca-service" +) + +args = parser.parse_args() + +model_name = args.model_name +release_id = args.release_id +service_name = args.service_name + +print(model_name) +print(release_id) +print(service_name) + +# Get workspace +run = Run.get_context() +exp = run.experiment +ws = run.experiment.workspace + +# Get workspace +inference_config = InferenceConfig(runtime="python", + entry_script="score.py", + conda_file="conda_dependencies.yml", + source_directory="./deploy/scoring/") +print(inference_config) + +# Do something with imagecnfig image_config = ContainerImage + +# Get latest model +model_list = Model.list(ws, name=model_name) +model = next( + filter( + lambda x: x.created_time == max( + model.created_time for model in model_list), + model_list, + ) +) + +# to do: get config from file +deployment_config = AciServiceDeploymentConfiguration(cpu_cores=1, + memory_gb=4, + location='westeurope', + collect_model_data=True) + +service = Model.deploy(workspace=ws, + name=service_name, + models=[model], + inference_config=inference_config, + deployment_config=deployment_config, + deployment_target=None, + overwrite=True) + +print(service.state) + +service.wait_for_deployment(show_output=True) diff --git a/code/deploy/deploy_prod_model.py b/code/deploy/deploy_prod_model.py new file mode 100644 index 00000000..727aa743 --- /dev/null +++ b/code/deploy/deploy_prod_model.py @@ -0,0 +1,70 @@ +from azureml.core.model import InferenceConfig +from azureml.core.webservice import AksWebservice +from azureml.core.compute import ComputeTarget +from azureml.core import Run, Model +import argparse + +parser = argparse.ArgumentParser("deploy") +parser.add_argument( + "--release_id", + type=str, + help="The ID of the release triggering this pipeline run", +) +parser.add_argument( + "--model_name", + type=str, + help="Name of the Model", + default="sklearn_abn_ca_model.pkl", +) +parser.add_argument( + "--service_name", + type=str, + help="Name of the datastore", + default="prod-abn-ca-service" +) + +args = parser.parse_args() + +model_name = args.model_name +release_id = args.release_id +service_name = args.service_name + +print(model_name) +print(release_id) +print(service_name) + +# Get workspace +run = Run.get_context() +exp = run.experiment +ws = run.experiment.workspace + +# Get workspace +inference_config = InferenceConfig(runtime="python", + entry_script="score.py", + conda_file="conda_dependencies.yml", + source_directory="./deploy/scoring/") +print(inference_config) + +# Do something with imagecnfig image_config = ContainerImage + +# Get latest model +model_list = Model.list(ws, name=model_name) +model = next( + filter( + lambda x: x.created_time == max( + model.created_time for model in model_list), + model_list, + ) +) + +aks_target = ComputeTarget(ws, "one-dspe-aks") +deployment_config = AksWebservice.deploy_configuration(cpu_cores=1, + memory_gb=4) +service = Model.deploy(workspace=ws, + name=service_name, + models=[model], + inference_config=inference_config, + deployment_config=deployment_config, + deployment_target=aks_target, + overwrite=True) +service.wait_for_deployment(show_output=True) diff --git a/code/scoring/conda_dependencies.yml b/code/deploy/scoring/conda_dependencies.yml similarity index 96% rename from code/scoring/conda_dependencies.yml rename to code/deploy/scoring/conda_dependencies.yml index 60c8dd92..4a515257 100644 --- a/code/scoring/conda_dependencies.yml +++ b/code/deploy/scoring/conda_dependencies.yml @@ -25,7 +25,7 @@ dependencies: - pip: # Required packages for AzureML execution, history, and data preparation. - - azureml-sdk==1.0.72 + - azureml-sdk[notebooks] - scipy==1.3.1 - scikit-learn==0.21.3 - pandas==0.25.3 @@ -33,4 +33,5 @@ dependencies: - joblib==0.14.0 - gunicorn==19.9.0 - flask==1.1.1 - \ No newline at end of file + - azure-ml-api-sdk + diff --git a/code/scoring/deployment_config_aci.yml b/code/deploy/scoring/deployment_config_aci.yml similarity index 100% rename from code/scoring/deployment_config_aci.yml rename to code/deploy/scoring/deployment_config_aci.yml diff --git a/code/scoring/deployment_config_aks.yml b/code/deploy/scoring/deployment_config_aks.yml similarity index 100% rename from code/scoring/deployment_config_aks.yml rename to code/deploy/scoring/deployment_config_aks.yml diff --git a/code/scoring/inference_config.yml b/code/deploy/scoring/inference_config.yml similarity index 100% rename from code/scoring/inference_config.yml rename to code/deploy/scoring/inference_config.yml diff --git a/code/scoring/score.py b/code/deploy/scoring/score.py similarity index 97% rename from code/scoring/score.py rename to code/deploy/scoring/score.py index dafe6bee..396e638e 100644 --- a/code/scoring/score.py +++ b/code/deploy/scoring/score.py @@ -34,7 +34,7 @@ def init(): # load the model from file into a global object model_path = Model.get_model_path( - model_name="sklearn_regression_model.pkl") + model_name="sklearn_abn_ca_model.pkl") model = joblib.load(model_path) diff --git a/code/training/train.py b/code/training/train.py index d703964f..bb40e661 100644 --- a/code/training/train.py +++ b/code/training/train.py @@ -35,6 +35,7 @@ parser = argparse.ArgumentParser("train") +print(parser) parser.add_argument( "--release_id", type=str, diff --git a/environment_setup/iac-create-environment.yml b/environment_setup/iac-create-environment.yml index 2dd00694..96a01d9e 100644 --- a/environment_setup/iac-create-environment.yml +++ b/environment_setup/iac-create-environment.yml @@ -17,20 +17,20 @@ pool: vmImage: 'ubuntu-latest' variables: -- group: devopsforai-aml-vg +- group: devopsforai-aml steps: - task: AzureResourceGroupDeployment@2 inputs: - azureSubscription: 'AzureResourceConnection' + azureSubscription: 'DspeResourceConnection' action: 'Create Or Update Resource Group' - resourceGroupName: '$(BASE_NAME)-AML-RG' + resourceGroupName: 'sandbox-nl02328-024-rg' location: $(LOCATION) templateLocation: 'Linked artifact' csmFile: '$(Build.SourcesDirectory)/environment_setup/arm-templates/cloud-environment.json' overrideParameters: '-baseName $(BASE_NAME) -location $(LOCATION)' deploymentMode: 'Incremental' - displayName: 'Deploy MLOps resources to Azure' + displayName: 'Deploy ABN-ca resources to Azure' \ No newline at end of file diff --git a/ml_service/pipelines/build_deploy_pipeline.py b/ml_service/pipelines/build_deploy_pipeline.py new file mode 100644 index 00000000..9f1d5a7d --- /dev/null +++ b/ml_service/pipelines/build_deploy_pipeline.py @@ -0,0 +1,116 @@ +from azureml.pipeline.core.graph import PipelineParameter +from azureml.pipeline.steps import PythonScriptStep +from azureml.pipeline.core import Pipeline # , PipelineData +from azureml.core.runconfig import RunConfiguration, CondaDependencies +# from azureml.core import Datastore +import os +import sys +from dotenv import load_dotenv +sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 +from workspace import get_workspace +from attach_compute import get_compute + + +def main(): + load_dotenv() + workspace_name = os.environ.get("BASE_NAME")+"-AML-WS" + resource_group = os.environ.get("RESOURCE_GROUP") + subscription_id = os.environ.get("SUBSCRIPTION_ID") + tenant_id = os.environ.get("TENANT_ID") + app_id = os.environ.get("SP_APP_ID") + app_secret = os.environ.get("SP_APP_SECRET") + deploy_script_path = os.environ.get("DEPLOY_SCRIPT_PATH") + vm_size = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU") + compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME") + model_name = os.environ.get("MODEL_NAME") + build_id = os.environ.get("BUILD_BUILDID") + pipeline_name = os.environ.get("DEPLOY_PIPELINE_NAME") + service_name = os.environ.get("DEPLOY_SERVICE_NAME") + sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN") + + # Get Azure machine learning workspace + aml_workspace = get_workspace( + workspace_name, + resource_group, + subscription_id, + tenant_id, + app_id, + app_secret) + print(aml_workspace) + + # Get Azure machine learning cluster + aml_compute = get_compute( + aml_workspace, + compute_name, + vm_size) + if aml_compute is not None: + print(aml_compute) + + conda_dependencies = CondaDependencies.create( + conda_packages=[ + 'numpy', + 'pandas', + 'scikit-learn' + ], + pip_packages=[ + 'azureml-core==1.0.72.*', + 'azureml-sdk==1.0.72.*', + 'azure-storage', + 'azure-storage-blob', + 'azureml-dataprep', + 'azureml-datadrift==1.0.72.*' + ], + pin_sdk_version=False + ) + + print(conda_dependencies.serialize_to_string()) + + run_config = RunConfiguration( + framework='Python', + conda_dependencies=conda_dependencies + ) + run_config.environment.docker.enabled = True + + model_name = PipelineParameter( + name="model_name", default_value=model_name + ) + print(model_name) + release_id = PipelineParameter( + name="release_id", default_value="0" + ) + print(release_id) + service_name = PipelineParameter( + name="service_name", default_value=service_name + ) + print(service_name) + + deploy_step = PythonScriptStep( + name="Deploy Model", + script_name=deploy_script_path, + compute_target=aml_compute, + source_directory=sources_directory_train, + arguments=[ + "--release_id", release_id, + "--model_name", model_name, + "--service_name", service_name + ], + runconfig=run_config, + allow_reuse=False, + ) + print("Step Deploy created") + + steps = [deploy_step] + + deploy_pipeline = Pipeline(workspace=aml_workspace, steps=steps) + deploy_pipeline.validate() + published_pipeline = deploy_pipeline.publish( + name=pipeline_name, + description="Model deploy pipeline", + version=build_id + ) + print(f'Published pipeline: {published_pipeline.name}') + print(f'for build {published_pipeline.version}') + + +if __name__ == '__main__': + main() diff --git a/ml_service/pipelines/build_deploy_prod_pipeline.py b/ml_service/pipelines/build_deploy_prod_pipeline.py new file mode 100644 index 00000000..966e66d9 --- /dev/null +++ b/ml_service/pipelines/build_deploy_prod_pipeline.py @@ -0,0 +1,116 @@ +from azureml.pipeline.core.graph import PipelineParameter +from azureml.pipeline.steps import PythonScriptStep +from azureml.pipeline.core import Pipeline # , PipelineData +from azureml.core.runconfig import RunConfiguration, CondaDependencies +# from azureml.core import Datastore +import os +import sys +from dotenv import load_dotenv +sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 +from workspace import get_workspace +from attach_compute import get_compute + + +def main(): + load_dotenv() + workspace_name = os.environ.get("BASE_NAME")+"-AML-WS" + resource_group = os.environ.get("RESOURCE_GROUP") + subscription_id = os.environ.get("SUBSCRIPTION_ID") + tenant_id = os.environ.get("TENANT_ID") + app_id = os.environ.get("SP_APP_ID") + app_secret = os.environ.get("SP_APP_SECRET") + deploy_script_path = os.environ.get("DEPLOY_PROD_SCRIPT_PATH") + vm_size = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU") + compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME") + model_name = os.environ.get("MODEL_NAME") + build_id = os.environ.get("BUILD_BUILDID") + pipeline_name = os.environ.get("DEPLOY_PROD_PIPELINE_NAME") + service_name = os.environ.get("DEPLOY_PROD_SERVICE_NAME") + sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN") + + # Get Azure machine learning workspace + aml_workspace = get_workspace( + workspace_name, + resource_group, + subscription_id, + tenant_id, + app_id, + app_secret) + print(aml_workspace) + + # Get Azure machine learning cluster + aml_compute = get_compute( + aml_workspace, + compute_name, + vm_size) + if aml_compute is not None: + print(aml_compute) + + conda_dependencies = CondaDependencies.create( + conda_packages=[ + 'numpy', + 'pandas', + 'scikit-learn' + ], + pip_packages=[ + 'azureml-core==1.0.72.*', + 'azureml-sdk==1.0.72.*', + 'azure-storage', + 'azure-storage-blob', + 'azureml-dataprep', + 'azureml-datadrift==1.0.72.*' + ], + pin_sdk_version=False + ) + + print(conda_dependencies.serialize_to_string()) + + run_config = RunConfiguration( + framework='Python', + conda_dependencies=conda_dependencies + ) + run_config.environment.docker.enabled = True + + model_name = PipelineParameter( + name="model_name", default_value=model_name + ) + print(model_name) + release_id = PipelineParameter( + name="release_id", default_value="0" + ) + print(release_id) + service_name = PipelineParameter( + name="service_name", default_value=service_name + ) + print(service_name) + + deploy_step = PythonScriptStep( + name="Deploy Prod Model", + script_name=deploy_script_path, + compute_target=aml_compute, + source_directory=sources_directory_train, + arguments=[ + "--release_id", release_id, + "--model_name", model_name, + "--service_name", service_name + ], + runconfig=run_config, + allow_reuse=False, + ) + print("Step Deploy Prod created") + + steps = [deploy_step] + + deploy_pipeline = Pipeline(workspace=aml_workspace, steps=steps) + deploy_pipeline.validate() + published_pipeline = deploy_pipeline.publish( + name=pipeline_name, + description="Model deploy Prod pipeline", + version=build_id + ) + print(f'Published pipeline: {published_pipeline.name}') + print(f'for build {published_pipeline.version}') + + +if __name__ == '__main__': + main() diff --git a/ml_service/pipelines/build_train_pipeline.py b/ml_service/pipelines/build_train_pipeline.py index 481c68e5..802c138e 100644 --- a/ml_service/pipelines/build_train_pipeline.py +++ b/ml_service/pipelines/build_train_pipeline.py @@ -14,7 +14,7 @@ def main(): load_dotenv() workspace_name = os.environ.get("BASE_NAME")+"-AML-WS" - resource_group = os.environ.get("BASE_NAME")+"-AML-RG" + resource_group = os.environ.get("RESOURCE_GROUP") subscription_id = os.environ.get("SUBSCRIPTION_ID") tenant_id = os.environ.get("TENANT_ID") app_id = os.environ.get("SP_APP_ID") @@ -46,12 +46,28 @@ def main(): if aml_compute is not None: print(aml_compute) - run_config = RunConfiguration(conda_dependencies=CondaDependencies.create( - conda_packages=['numpy', 'pandas', - 'scikit-learn', 'tensorflow', 'keras'], - pip_packages=['azure', 'azureml-core', - 'azure-storage', - 'azure-storage-blob']) + conda_dependencies = CondaDependencies.create( + conda_packages=[ + 'numpy', + 'pandas', + 'scikit-learn' + ], + pip_packages=[ + 'azureml-core==1.0.72.*', + 'azureml-sdk==1.0.72.*', + 'azure-storage', + 'azure-storage-blob', + 'azureml-dataprep', + 'azureml-datadrift==1.0.72.*' + ], + pin_sdk_version=False + ) + + print(conda_dependencies.serialize_to_string()) + + run_config = RunConfiguration( + framework='Python', + conda_dependencies=conda_dependencies ) run_config.environment.docker.enabled = True diff --git a/ml_service/pipelines/run_deploy_pipeline.py b/ml_service/pipelines/run_deploy_pipeline.py new file mode 100644 index 00000000..b8954ff6 --- /dev/null +++ b/ml_service/pipelines/run_deploy_pipeline.py @@ -0,0 +1,66 @@ +import os +from azureml.pipeline.core import PublishedPipeline +from azureml.core import Workspace +from azureml.core.authentication import ServicePrincipalAuthentication +from dotenv import load_dotenv + + +def main(): + load_dotenv() + workspace_name = os.environ.get("BASE_NAME")+"-AML-WS" + resource_group = os.environ.get("RESOURCE_GROUP") + subscription_id = os.environ.get("SUBSCRIPTION_ID") + tenant_id = os.environ.get("TENANT_ID") + experiment_name = os.environ.get("EXPERIMENT_NAME") + model_name = os.environ.get("MODEL_NAME") + app_id = os.environ.get('SP_APP_ID') + app_secret = os.environ.get('SP_APP_SECRET') + release_id = os.environ.get('RELEASE_RELEASEID') + build_id = os.environ.get('BUILD_BUILDID') + pipeline_name = os.environ.get('DEPLOY_PIPELINE_NAME') + service_name = os.environ.get('DEPLOY_SERVICE_NAME') + + service_principal = ServicePrincipalAuthentication( + tenant_id=tenant_id, + service_principal_id=app_id, + service_principal_password=app_secret) + + aml_workspace = Workspace.get( + name=workspace_name, + subscription_id=subscription_id, + resource_group=resource_group, + auth=service_principal + ) + + # Find the pipeline that was published by the specified build ID + pipelines = PublishedPipeline.list(aml_workspace) + matched_pipes = [] + + for p in pipelines: + if p.version == build_id and p.name == pipeline_name: + matched_pipes.append(p) + + if(len(matched_pipes) > 1): + published_pipeline = None + raise Exception(f"Multiple active pipelines are published for build {build_id}.") # NOQA: E501 + elif(len(matched_pipes) == 0): + published_pipeline = None + raise KeyError(f"Unable to find a published pipeline for this build {build_id}") # NOQA: E501 + else: + published_pipeline = matched_pipes[0] + + pipeline_parameters = {"model_name": model_name, + "release_id": release_id, + "service_name": service_name} + + response = published_pipeline.submit( + aml_workspace, + experiment_name, + pipeline_parameters) + + run_id = response.id + print("Pipeline run initiated ", run_id) + + +if __name__ == "__main__": + main() diff --git a/ml_service/pipelines/run_deploy_prod_pipeline.py b/ml_service/pipelines/run_deploy_prod_pipeline.py new file mode 100644 index 00000000..b88a2349 --- /dev/null +++ b/ml_service/pipelines/run_deploy_prod_pipeline.py @@ -0,0 +1,66 @@ +import os +from azureml.pipeline.core import PublishedPipeline +from azureml.core import Workspace +from azureml.core.authentication import ServicePrincipalAuthentication +from dotenv import load_dotenv + + +def main(): + load_dotenv() + workspace_name = os.environ.get("BASE_NAME")+"-AML-WS" + resource_group = os.environ.get("RESOURCE_GROUP") + subscription_id = os.environ.get("SUBSCRIPTION_ID") + tenant_id = os.environ.get("TENANT_ID") + experiment_name = os.environ.get("EXPERIMENT_NAME") + model_name = os.environ.get("MODEL_NAME") + app_id = os.environ.get('SP_APP_ID') + app_secret = os.environ.get('SP_APP_SECRET') + release_id = os.environ.get('RELEASE_RELEASEID') + build_id = os.environ.get('BUILD_BUILDID') + pipeline_name = os.environ.get('DEPLOY_PROD_PIPELINE_NAME') + service_name = os.environ.get('DEPLOY_PROD_SERVICE_NAME') + + service_principal = ServicePrincipalAuthentication( + tenant_id=tenant_id, + service_principal_id=app_id, + service_principal_password=app_secret) + + aml_workspace = Workspace.get( + name=workspace_name, + subscription_id=subscription_id, + resource_group=resource_group, + auth=service_principal + ) + + # Find the pipeline that was published by the specified build ID + pipelines = PublishedPipeline.list(aml_workspace) + matched_pipes = [] + + for p in pipelines: + if p.version == build_id and p.name == pipeline_name: + matched_pipes.append(p) + + if(len(matched_pipes) > 1): + published_pipeline = None + raise Exception(f"Multiple active pipelines are published for build {build_id}.") # NOQA: E501 + elif(len(matched_pipes) == 0): + published_pipeline = None + raise KeyError(f"Unable to find a published pipeline for this build {build_id}") # NOQA: E501 + else: + published_pipeline = matched_pipes[0] + + pipeline_parameters = {"model_name": model_name, + "release_id": release_id, + "service_name": service_name} + + response = published_pipeline.submit( + aml_workspace, + experiment_name, + pipeline_parameters) + + run_id = response.id + print("Pipeline run initiated ", run_id) + + +if __name__ == "__main__": + main() diff --git a/ml_service/pipelines/run_train_pipeline.py b/ml_service/pipelines/run_train_pipeline.py index 1d942a8c..96a2d2ad 100644 --- a/ml_service/pipelines/run_train_pipeline.py +++ b/ml_service/pipelines/run_train_pipeline.py @@ -8,14 +8,16 @@ def main(): load_dotenv() workspace_name = os.environ.get("BASE_NAME")+"-AML-WS" - resource_group = os.environ.get("BASE_NAME")+"-AML-RG" + resource_group = os.environ.get("RESOURCE_GROUP") subscription_id = os.environ.get("SUBSCRIPTION_ID") tenant_id = os.environ.get("TENANT_ID") experiment_name = os.environ.get("EXPERIMENT_NAME") model_name = os.environ.get("MODEL_NAME") app_id = os.environ.get('SP_APP_ID') app_secret = os.environ.get('SP_APP_SECRET') + release_id = os.environ.get('RELEASE_RELEASEID') build_id = os.environ.get('BUILD_BUILDID') + pipeline_name = os.environ.get('TRAINING_PIPELINE_NAME') service_principal = ServicePrincipalAuthentication( tenant_id=tenant_id, @@ -34,7 +36,7 @@ def main(): matched_pipes = [] for p in pipelines: - if p.version == build_id: + if p.version == build_id and p.name == pipeline_name: matched_pipes.append(p) if(len(matched_pipes) > 1): @@ -46,7 +48,8 @@ def main(): else: published_pipeline = matched_pipes[0] - pipeline_parameters = {"model_name": model_name} + pipeline_parameters = {"model_name": model_name, + "release_id": release_id} response = published_pipeline.submit( aml_workspace, diff --git a/tests/unit/code_test.py b/tests/unit/code_test.py index b22b186c..0d0ccc31 100644 --- a/tests/unit/code_test.py +++ b/tests/unit/code_test.py @@ -8,7 +8,7 @@ # a utility function common_scoring.next_saturday def test_get_workspace(): workspace_name = os.environ.get("BASE_NAME")+"-AML-WS" - resource_group = os.environ.get("BASE_NAME")+"-AML-RG" + resource_group = os.environ.get("RESOURCE_GROUP") subscription_id = os.environ.get("SUBSCRIPTION_ID") tenant_id = os.environ.get("TENANT_ID") app_id = os.environ.get("SP_APP_ID")