From 6f5a002ded07f06a44c11988778d159f16c55f45 Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Thu, 14 Nov 2019 11:38:26 +0100 Subject: [PATCH 01/60] Change variables and inputs names [skip ci] --- environment_setup/iac-create-environment.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/environment_setup/iac-create-environment.yml b/environment_setup/iac-create-environment.yml index 2dd00694..04abce83 100644 --- a/environment_setup/iac-create-environment.yml +++ b/environment_setup/iac-create-environment.yml @@ -17,13 +17,13 @@ pool: vmImage: 'ubuntu-latest' variables: -- group: devopsforai-aml-vg +- group: devopsforai-aml steps: - task: AzureResourceGroupDeployment@2 inputs: - azureSubscription: 'AzureResourceConnection' + azureSubscription: 'DspeResourceConnection' action: 'Create Or Update Resource Group' resourceGroupName: '$(BASE_NAME)-AML-RG' location: $(LOCATION) @@ -31,6 +31,6 @@ steps: csmFile: '$(Build.SourcesDirectory)/environment_setup/arm-templates/cloud-environment.json' overrideParameters: '-baseName $(BASE_NAME) -location $(LOCATION)' deploymentMode: 'Incremental' - displayName: 'Deploy MLOps resources to Azure' + displayName: 'Deploy ABN-ca resources to Azure' \ No newline at end of file From 114e419da6b41ee7475a7c625dacf9daa18b705d Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Thu, 14 Nov 2019 11:49:45 +0100 Subject: [PATCH 02/60] Change resource-group name --- environment_setup/iac-create-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment_setup/iac-create-environment.yml b/environment_setup/iac-create-environment.yml index 04abce83..96a01d9e 100644 --- a/environment_setup/iac-create-environment.yml +++ b/environment_setup/iac-create-environment.yml @@ -25,7 +25,7 @@ steps: inputs: azureSubscription: 'DspeResourceConnection' action: 'Create Or Update Resource Group' - resourceGroupName: '$(BASE_NAME)-AML-RG' + resourceGroupName: 'sandbox-nl02328-024-rg' location: $(LOCATION) templateLocation: 'Linked artifact' csmFile: '$(Build.SourcesDirectory)/environment_setup/arm-templates/cloud-environment.json' From 2937b699c7d7f89e9664a18836c8854dd2456f5d Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Thu, 14 Nov 2019 12:03:15 +0100 Subject: [PATCH 03/60] Change resource group to 22 --- environment_setup/iac-create-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment_setup/iac-create-environment.yml b/environment_setup/iac-create-environment.yml index 96a01d9e..0843a9b8 100644 --- a/environment_setup/iac-create-environment.yml +++ b/environment_setup/iac-create-environment.yml @@ -25,7 +25,7 @@ steps: inputs: azureSubscription: 'DspeResourceConnection' action: 'Create Or Update Resource Group' - resourceGroupName: 'sandbox-nl02328-024-rg' + resourceGroupName: 'sandbox-nl02328-022-rg' location: $(LOCATION) templateLocation: 'Linked artifact' csmFile: '$(Build.SourcesDirectory)/environment_setup/arm-templates/cloud-environment.json' From af220aded979d4e9710b8ca21e16437b3d4b9ed2 Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Thu, 14 Nov 2019 13:13:38 +0100 Subject: [PATCH 04/60] Change resource-group to 23 --- environment_setup/iac-create-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment_setup/iac-create-environment.yml b/environment_setup/iac-create-environment.yml index 0843a9b8..9c90c621 100644 --- a/environment_setup/iac-create-environment.yml +++ b/environment_setup/iac-create-environment.yml @@ -25,7 +25,7 @@ steps: inputs: azureSubscription: 'DspeResourceConnection' action: 'Create Or Update Resource Group' - resourceGroupName: 'sandbox-nl02328-022-rg' + resourceGroupName: 'sandbox-nl02328-023-rg' location: $(LOCATION) templateLocation: 'Linked artifact' csmFile: '$(Build.SourcesDirectory)/environment_setup/arm-templates/cloud-environment.json' From 5f43ada971c85de6bee64f98a9f2dca3cfc0b58d Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Thu, 14 Nov 2019 13:28:26 +0100 Subject: [PATCH 05/60] Change service connection to Azure --- environment_setup/iac-create-environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environment_setup/iac-create-environment.yml b/environment_setup/iac-create-environment.yml index 9c90c621..123510a6 100644 --- a/environment_setup/iac-create-environment.yml +++ b/environment_setup/iac-create-environment.yml @@ -17,13 +17,13 @@ pool: vmImage: 'ubuntu-latest' variables: -- group: devopsforai-aml +- group: devopsforai-aml-vg steps: - task: AzureResourceGroupDeployment@2 inputs: - azureSubscription: 'DspeResourceConnection' + azureSubscription: 'AzureResourceConnection' action: 'Create Or Update Resource Group' resourceGroupName: 'sandbox-nl02328-023-rg' location: $(LOCATION) From 84c3cd4dc908ac2bc2debd8f9293f0ae1dfbe49f Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Thu, 14 Nov 2019 13:31:07 +0100 Subject: [PATCH 06/60] Change variables to vg --- environment_setup/iac-create-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment_setup/iac-create-environment.yml b/environment_setup/iac-create-environment.yml index 123510a6..d694a34c 100644 --- a/environment_setup/iac-create-environment.yml +++ b/environment_setup/iac-create-environment.yml @@ -23,7 +23,7 @@ variables: steps: - task: AzureResourceGroupDeployment@2 inputs: - azureSubscription: 'AzureResourceConnection' + azureSubscription: 'DspeResourceConnection' action: 'Create Or Update Resource Group' resourceGroupName: 'sandbox-nl02328-023-rg' location: $(LOCATION) From a10910c868c243f8e523911e9922a3be67c050b9 Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Thu, 14 Nov 2019 13:34:15 +0100 Subject: [PATCH 07/60] deploy resources to 24 sandbox group --- environment_setup/iac-create-environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environment_setup/iac-create-environment.yml b/environment_setup/iac-create-environment.yml index d694a34c..96a01d9e 100644 --- a/environment_setup/iac-create-environment.yml +++ b/environment_setup/iac-create-environment.yml @@ -17,7 +17,7 @@ pool: vmImage: 'ubuntu-latest' variables: -- group: devopsforai-aml-vg +- group: devopsforai-aml steps: @@ -25,7 +25,7 @@ steps: inputs: azureSubscription: 'DspeResourceConnection' action: 'Create Or Update Resource Group' - resourceGroupName: 'sandbox-nl02328-023-rg' + resourceGroupName: 'sandbox-nl02328-024-rg' location: $(LOCATION) templateLocation: 'Linked artifact' csmFile: '$(Build.SourcesDirectory)/environment_setup/arm-templates/cloud-environment.json' From 3cdb50581f7f85743f4a077e4731e88586b42c57 Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Thu, 14 Nov 2019 14:44:10 +0100 Subject: [PATCH 08/60] Set up CI with Azure Pipelines [skip ci] --- ci-build.yml | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 ci-build.yml diff --git a/ci-build.yml b/ci-build.yml new file mode 100644 index 00000000..b3c6f901 --- /dev/null +++ b/ci-build.yml @@ -0,0 +1,71 @@ +pr: none +trigger: + branches: + include: + - master + paths: + exclude: + - docs/ + - environment_setup/ + - charts/ + - ml_service/util/create_scoring_image.py + +variables: +- group: devopsforai-aml +# Choose from default, build_train_pipeline_with_r.py, or build_train_pipeline_with_r_on_dbricks.py +- name: build-train-script + value: 'build_train_pipeline.py' +# Automatically triggers the train, evaluate, register pipeline after the CI steps. +# Uncomment to set to false or add same variable name at queue time with value of false to disable. +# - name: auto-trigger-training +# value: false + +stages: +- stage: 'Model_CI' + displayName: 'Model CI' + jobs: + - job: "Model_CI_Pipeline" + displayName: "Model CI Pipeline" + pool: + vmImage: 'ubuntu-latest' + container: mcr.microsoft.com/mlops/python:latest + timeoutInMinutes: 0 + steps: + - template: azdo-base-pipeline.yml + - script: | + # Invoke the Python building and publishing a training pipeline + python3 $(Build.SourcesDirectory)/ml_service/pipelines/$(build-train-script) + failOnStderr: 'false' + env: + SP_APP_SECRET: '$(SP_APP_SECRET)' + displayName: 'Publish Azure Machine Learning Pipeline' +- stage: 'Trigger_AML_Pipeline' + displayName: 'Train, evaluate, register model via previously published AML pipeline' + jobs: + - job: "Invoke_Model_Pipeline" + condition: and(succeeded(), eq(coalesce(variables['auto-trigger-training'], 'true'), 'true')) + displayName: "Invoke Model Pipeline and evaluate results to register" + pool: + vmImage: 'ubuntu-latest' + container: mcr.microsoft.com/mlops/python:latest + timeoutInMinutes: 0 + steps: + - script: | + python $(Build.SourcesDirectory)/ml_service/pipelines/run_train_pipeline.py + displayName: 'Trigger Training Pipeline' + env: + SP_APP_SECRET: '$(SP_APP_SECRET)' + - task: CopyFiles@2 + displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + Contents: | + code/scoring/** + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact' + inputs: + ArtifactName: 'dspe-pipelines' + publishLocation: 'container' + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + TargetPath: '$(Build.ArtifactStagingDirectory)' \ No newline at end of file From 112263c4ed3f11b897e456e144e8f6054e0bb335 Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Thu, 14 Nov 2019 14:53:45 +0100 Subject: [PATCH 09/60] Set up CI with Azure Pipelines [skip ci] --- .pipelines/azdo-ci-build-train.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/azdo-ci-build-train.yml b/.pipelines/azdo-ci-build-train.yml index c2453d4d..086a1d06 100644 --- a/.pipelines/azdo-ci-build-train.yml +++ b/.pipelines/azdo-ci-build-train.yml @@ -11,7 +11,7 @@ trigger: - ml_service/util/create_scoring_image.py variables: -- group: devopsforai-aml-vg +- group: devopsforai-aml # Choose from default, build_train_pipeline_with_r.py, or build_train_pipeline_with_r_on_dbricks.py - name: build-train-script value: 'build_train_pipeline.py' From 7fef4ebb0034080db6f996c446d4d65cab3202d8 Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Thu, 14 Nov 2019 15:08:39 +0100 Subject: [PATCH 10/60] change resoure_group name --- tests/unit/code_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/code_test.py b/tests/unit/code_test.py index b22b186c..b9efed32 100644 --- a/tests/unit/code_test.py +++ b/tests/unit/code_test.py @@ -8,7 +8,7 @@ # a utility function common_scoring.next_saturday def test_get_workspace(): workspace_name = os.environ.get("BASE_NAME")+"-AML-WS" - resource_group = os.environ.get("BASE_NAME")+"-AML-RG" + resource_group = "RESOURCE_GROUP" subscription_id = os.environ.get("SUBSCRIPTION_ID") tenant_id = os.environ.get("TENANT_ID") app_id = os.environ.get("SP_APP_ID") From 6d2945b431e577947c1d5f3b69198241e6a3ee99 Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Thu, 14 Nov 2019 15:15:01 +0100 Subject: [PATCH 11/60] get resource-group name from variables --- tests/unit/code_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/code_test.py b/tests/unit/code_test.py index b9efed32..0d0ccc31 100644 --- a/tests/unit/code_test.py +++ b/tests/unit/code_test.py @@ -8,7 +8,7 @@ # a utility function common_scoring.next_saturday def test_get_workspace(): workspace_name = os.environ.get("BASE_NAME")+"-AML-WS" - resource_group = "RESOURCE_GROUP" + resource_group = os.environ.get("RESOURCE_GROUP") subscription_id = os.environ.get("SUBSCRIPTION_ID") tenant_id = os.environ.get("TENANT_ID") app_id = os.environ.get("SP_APP_ID") From 54fc36d7660333565e27c28a4417a5344fee3265 Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Thu, 14 Nov 2019 16:06:14 +0100 Subject: [PATCH 12/60] change resource group name --- ml_service/pipelines/build_train_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ml_service/pipelines/build_train_pipeline.py b/ml_service/pipelines/build_train_pipeline.py index 481c68e5..1c7c6a82 100644 --- a/ml_service/pipelines/build_train_pipeline.py +++ b/ml_service/pipelines/build_train_pipeline.py @@ -14,7 +14,7 @@ def main(): load_dotenv() workspace_name = os.environ.get("BASE_NAME")+"-AML-WS" - resource_group = os.environ.get("BASE_NAME")+"-AML-RG" + resource_group = os.environ.get("RESOURCE_GROUP") subscription_id = os.environ.get("SUBSCRIPTION_ID") tenant_id = os.environ.get("TENANT_ID") app_id = os.environ.get("SP_APP_ID") From f98ecc27c72013b4a3a5a39b8a245e9f8fa5e9bc Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Thu, 14 Nov 2019 17:08:03 +0100 Subject: [PATCH 13/60] change resource group name --- ml_service/pipelines/run_train_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ml_service/pipelines/run_train_pipeline.py b/ml_service/pipelines/run_train_pipeline.py index 1d942a8c..2492672d 100644 --- a/ml_service/pipelines/run_train_pipeline.py +++ b/ml_service/pipelines/run_train_pipeline.py @@ -8,7 +8,7 @@ def main(): load_dotenv() workspace_name = os.environ.get("BASE_NAME")+"-AML-WS" - resource_group = os.environ.get("BASE_NAME")+"-AML-RG" + resource_group = os.environ.get("RESOURCE_GROUP") subscription_id = os.environ.get("SUBSCRIPTION_ID") tenant_id = os.environ.get("TENANT_ID") experiment_name = os.environ.get("EXPERIMENT_NAME") From eaa8109cc63e7258e9b304d2a461428ab044a59c Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Fri, 15 Nov 2019 10:18:07 +0100 Subject: [PATCH 14/60] Copy pipelines folder --- .pipelines/azdo-ci-build-train.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/azdo-ci-build-train.yml b/.pipelines/azdo-ci-build-train.yml index 086a1d06..77dbc7a4 100644 --- a/.pipelines/azdo-ci-build-train.yml +++ b/.pipelines/azdo-ci-build-train.yml @@ -62,6 +62,7 @@ stages: TargetFolder: '$(Build.ArtifactStagingDirectory)' Contents: | code/scoring/** + ml_service/pipelines/** - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact' inputs: From ea2c3029b125b67a9b86d2240cdb93ca65ac54d2 Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Fri, 15 Nov 2019 11:10:32 +0100 Subject: [PATCH 15/60] Update run_train_pipeline.py --- ml_service/pipelines/run_train_pipeline.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ml_service/pipelines/run_train_pipeline.py b/ml_service/pipelines/run_train_pipeline.py index 2492672d..adc59d37 100644 --- a/ml_service/pipelines/run_train_pipeline.py +++ b/ml_service/pipelines/run_train_pipeline.py @@ -4,7 +4,6 @@ from azureml.core.authentication import ServicePrincipalAuthentication from dotenv import load_dotenv - def main(): load_dotenv() workspace_name = os.environ.get("BASE_NAME")+"-AML-WS" From 952b06078376220e878a7428d67b70a5c0c821ab Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Fri, 15 Nov 2019 11:16:11 +0100 Subject: [PATCH 16/60] Update run_train_pipeline.py --- ml_service/pipelines/run_train_pipeline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ml_service/pipelines/run_train_pipeline.py b/ml_service/pipelines/run_train_pipeline.py index adc59d37..2492672d 100644 --- a/ml_service/pipelines/run_train_pipeline.py +++ b/ml_service/pipelines/run_train_pipeline.py @@ -4,6 +4,7 @@ from azureml.core.authentication import ServicePrincipalAuthentication from dotenv import load_dotenv + def main(): load_dotenv() workspace_name = os.environ.get("BASE_NAME")+"-AML-WS" From 963b786aaf90f0fc2feadd9139fa717c911bdb01 Mon Sep 17 00:00:00 2001 From: evilkova Date: Fri, 15 Nov 2019 14:08:37 +0100 Subject: [PATCH 17/60] add deploment script for deployment pipeline --- ml_service/pipelines/run_deploy_pipeline.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 ml_service/pipelines/run_deploy_pipeline.py diff --git a/ml_service/pipelines/run_deploy_pipeline.py b/ml_service/pipelines/run_deploy_pipeline.py new file mode 100644 index 00000000..89e7d26e --- /dev/null +++ b/ml_service/pipelines/run_deploy_pipeline.py @@ -0,0 +1 @@ +kjbk,jb \ No newline at end of file From 9c205db0965b1b487b8321f4d32ea80963939c77 Mon Sep 17 00:00:00 2001 From: evilkova Date: Fri, 15 Nov 2019 14:09:51 +0100 Subject: [PATCH 18/60] add deployment script for the pipeline --- ml_service/pipelines/run_deploy_pipeline.py | 67 ++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/ml_service/pipelines/run_deploy_pipeline.py b/ml_service/pipelines/run_deploy_pipeline.py index 89e7d26e..b8954ff6 100644 --- a/ml_service/pipelines/run_deploy_pipeline.py +++ b/ml_service/pipelines/run_deploy_pipeline.py @@ -1 +1,66 @@ -kjbk,jb \ No newline at end of file +import os +from azureml.pipeline.core import PublishedPipeline +from azureml.core import Workspace +from azureml.core.authentication import ServicePrincipalAuthentication +from dotenv import load_dotenv + + +def main(): + load_dotenv() + workspace_name = os.environ.get("BASE_NAME")+"-AML-WS" + resource_group = os.environ.get("RESOURCE_GROUP") + subscription_id = os.environ.get("SUBSCRIPTION_ID") + tenant_id = os.environ.get("TENANT_ID") + experiment_name = os.environ.get("EXPERIMENT_NAME") + model_name = os.environ.get("MODEL_NAME") + app_id = os.environ.get('SP_APP_ID') + app_secret = os.environ.get('SP_APP_SECRET') + release_id = os.environ.get('RELEASE_RELEASEID') + build_id = os.environ.get('BUILD_BUILDID') + pipeline_name = os.environ.get('DEPLOY_PIPELINE_NAME') + service_name = os.environ.get('DEPLOY_SERVICE_NAME') + + service_principal = ServicePrincipalAuthentication( + tenant_id=tenant_id, + service_principal_id=app_id, + service_principal_password=app_secret) + + aml_workspace = Workspace.get( + name=workspace_name, + subscription_id=subscription_id, + resource_group=resource_group, + auth=service_principal + ) + + # Find the pipeline that was published by the specified build ID + pipelines = PublishedPipeline.list(aml_workspace) + matched_pipes = [] + + for p in pipelines: + if p.version == build_id and p.name == pipeline_name: + matched_pipes.append(p) + + if(len(matched_pipes) > 1): + published_pipeline = None + raise Exception(f"Multiple active pipelines are published for build {build_id}.") # NOQA: E501 + elif(len(matched_pipes) == 0): + published_pipeline = None + raise KeyError(f"Unable to find a published pipeline for this build {build_id}") # NOQA: E501 + else: + published_pipeline = matched_pipes[0] + + pipeline_parameters = {"model_name": model_name, + "release_id": release_id, + "service_name": service_name} + + response = published_pipeline.submit( + aml_workspace, + experiment_name, + pipeline_parameters) + + run_id = response.id + print("Pipeline run initiated ", run_id) + + +if __name__ == "__main__": + main() From 543270b08d76d18cf466ffc061d16e74d36ac452 Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Fri, 15 Nov 2019 14:40:31 +0100 Subject: [PATCH 19/60] Add deployment pipeline --- .pipelines/azdo-ci-build-train.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.pipelines/azdo-ci-build-train.yml b/.pipelines/azdo-ci-build-train.yml index 77dbc7a4..ea4a836e 100644 --- a/.pipelines/azdo-ci-build-train.yml +++ b/.pipelines/azdo-ci-build-train.yml @@ -55,6 +55,19 @@ stages: displayName: 'Trigger Training Pipeline' env: SP_APP_SECRET: '$(SP_APP_SECRET)' + - job: "Invoke_Deploy_Pipeline" + condition: and(succeeded(), eq(coalesce(variables['auto-trigger-training'], 'true'), 'true')) + displayName: "Invoke Deploy Pipeline" + pool: + vmImage: 'ubuntu-latest' + container: mcr.microsoft.com/mlops/python:latest + timeoutInMinutes: 0 + steps: + - script: | + python $(Build.SourcesDirectory)/ml_service/pipelines/run_deploy_pipeline.py + displayName: 'Trigger Deployment Pipeline' + env: + SP_APP_SECRET: '$(SP_APP_SECRET)' - task: CopyFiles@2 displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)' inputs: From d6f3678d107a18fe1ccb442339c45ae2fefecaf5 Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Fri, 15 Nov 2019 14:59:22 +0100 Subject: [PATCH 20/60] join train and deploy pipelines in 1 job --- .pipelines/azdo-ci-build-train.yml | 33 ++++++------------------------ 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/.pipelines/azdo-ci-build-train.yml b/.pipelines/azdo-ci-build-train.yml index ea4a836e..01bf2868 100644 --- a/.pipelines/azdo-ci-build-train.yml +++ b/.pipelines/azdo-ci-build-train.yml @@ -34,40 +34,19 @@ stages: - template: azdo-base-pipeline.yml - script: | # Invoke the Python building and publishing a training pipeline - python3 $(Build.SourcesDirectory)/ml_service/pipelines/$(build-train-script) + python3 $(Build.SourcesDirectory)/ml_service/pipelines/build_train_pipeline.py failOnStderr: 'false' env: SP_APP_SECRET: '$(SP_APP_SECRET)' displayName: 'Publish Azure Machine Learning Pipeline' -- stage: 'Trigger_AML_Pipeline' - displayName: 'Train, evaluate, register model via previously published AML pipeline' - jobs: - - job: "Invoke_Model_Pipeline" - condition: and(succeeded(), eq(coalesce(variables['auto-trigger-training'], 'true'), 'true')) - displayName: "Invoke Model Pipeline and evaluate results to register" - pool: - vmImage: 'ubuntu-latest' - container: mcr.microsoft.com/mlops/python:latest - timeoutInMinutes: 0 - steps: - - script: | - python $(Build.SourcesDirectory)/ml_service/pipelines/run_train_pipeline.py - displayName: 'Trigger Training Pipeline' - env: - SP_APP_SECRET: '$(SP_APP_SECRET)' - - job: "Invoke_Deploy_Pipeline" - condition: and(succeeded(), eq(coalesce(variables['auto-trigger-training'], 'true'), 'true')) - displayName: "Invoke Deploy Pipeline" - pool: - vmImage: 'ubuntu-latest' - container: mcr.microsoft.com/mlops/python:latest - timeoutInMinutes: 0 - steps: - script: | - python $(Build.SourcesDirectory)/ml_service/pipelines/run_deploy_pipeline.py - displayName: 'Trigger Deployment Pipeline' + # Invoke the Python building and publishing a deployment pipeline + python3 $(Build.SourcesDirectory)/ml_service/pipelines/build_deploy_pipeline.py + failOnStderr: 'false' env: SP_APP_SECRET: '$(SP_APP_SECRET)' + displayName: 'Publish Azure Machine Learning Deployment Pipeline' + - task: CopyFiles@2 displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)' inputs: From 7605a595ca247d78cde198581fdefc7a3873e450 Mon Sep 17 00:00:00 2001 From: evilkova Date: Fri, 15 Nov 2019 15:06:50 +0100 Subject: [PATCH 21/60] Add build deploy script --- ml_service/pipelines/build_deploy_pipeline.py | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 ml_service/pipelines/build_deploy_pipeline.py diff --git a/ml_service/pipelines/build_deploy_pipeline.py b/ml_service/pipelines/build_deploy_pipeline.py new file mode 100644 index 00000000..ca4592ff --- /dev/null +++ b/ml_service/pipelines/build_deploy_pipeline.py @@ -0,0 +1,90 @@ +from azureml.pipeline.core.graph import PipelineParameter +from azureml.pipeline.steps import PythonScriptStep +from azureml.pipeline.core import Pipeline # , PipelineData +from azureml.core.runconfig import RunConfiguration, CondaDependencies +# from azureml.core import Datastore +import os +import sys +from dotenv import load_dotenv +sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 +from workspace import get_workspace +from attach_compute import get_compute + + +def main(): + load_dotenv() + workspace_name = os.environ.get("BASE_NAME")+"-AML-WS" + resource_group = os.environ.get("RESOURCE_GROUP") + subscription_id = os.environ.get("SUBSCRIPTION_ID") + tenant_id = os.environ.get("TENANT_ID") + app_id = os.environ.get("SP_APP_ID") + app_secret = os.environ.get("SP_APP_SECRET") + deploy_script_path = os.environ.get("DEPLOY_SCRIPT_PATH") + evaluate_script_path = os.environ.get("EVALUATE_SCRIPT_PATH") + vm_size = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU") + compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME") + model_name = os.environ.get("MODEL_NAME") + build_id = os.environ.get("BUILD_BUILDID") + pipeline_name = os.environ.get("DEPLOY_PIPELINE_NAME") + service_name = os.environ.get('DEPLOY_SERVICE_NAME') + + # Get Azure machine learning workspace + aml_workspace = get_workspace( + workspace_name, + resource_group, + subscription_id, + tenant_id, + app_id, + app_secret) + print(aml_workspace) + + # Get Azure machine learning cluster + aml_compute = get_compute( + aml_workspace, + compute_name, + vm_size) + if aml_compute is not None: + print(aml_compute) + + run_config = RunConfiguration(conda_dependencies=CondaDependencies.create( + conda_packages=['numpy', 'pandas', + 'scikit-learn', 'tensorflow', 'keras'], + pip_packages=['azure', 'azureml-core', + 'azure-storage', + 'azure-storage-blob']) + ) + run_config.environment.docker.enabled = True + + model_name = PipelineParameter( + name="model_name", default_value=model_name) + release_id = PipelineParameter( + name="release_id", default_value="0" + ) + + deploy_step = PythonScriptStep( + name="Deploy Model", + script_name=deploy_script_path, + compute_target=aml_compute, + arguments=[ + "--release_id", release_id, + "--model_name", model_name, + ], + runconfig=run_config, + allow_reuse=False, + ) + print("Step Deploy created") + + + deploy_pipeline = Pipeline(workspace=aml_workspace, steps=steps) + deploy_pipeline.validate() + published_pipeline = deploy_pipeline.publish( + name=pipeline_name, + description="Model deploy pipeline", + version=build_id + ) + print(f'Published pipeline: {published_pipeline.name}') + print(f'for build {published_pipeline.version}') + + +if __name__ == '__main__': + main() From c48666bd0a9c029708cb73f3626847677033fc15 Mon Sep 17 00:00:00 2001 From: evilkova Date: Fri, 15 Nov 2019 16:09:08 +0100 Subject: [PATCH 22/60] add sevice name to deployment build --- ml_service/pipelines/build_deploy_pipeline.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/ml_service/pipelines/build_deploy_pipeline.py b/ml_service/pipelines/build_deploy_pipeline.py index ca4592ff..cf200c9c 100644 --- a/ml_service/pipelines/build_deploy_pipeline.py +++ b/ml_service/pipelines/build_deploy_pipeline.py @@ -20,13 +20,13 @@ def main(): app_id = os.environ.get("SP_APP_ID") app_secret = os.environ.get("SP_APP_SECRET") deploy_script_path = os.environ.get("DEPLOY_SCRIPT_PATH") - evaluate_script_path = os.environ.get("EVALUATE_SCRIPT_PATH") vm_size = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU") compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME") model_name = os.environ.get("MODEL_NAME") build_id = os.environ.get("BUILD_BUILDID") pipeline_name = os.environ.get("DEPLOY_PIPELINE_NAME") - service_name = os.environ.get('DEPLOY_SERVICE_NAME') + service_name = os.environ.get("DEPLOY_SERVICE_NAME") + sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN") # Get Azure machine learning workspace aml_workspace = get_workspace( @@ -48,7 +48,7 @@ def main(): run_config = RunConfiguration(conda_dependencies=CondaDependencies.create( conda_packages=['numpy', 'pandas', - 'scikit-learn', 'tensorflow', 'keras'], + 'scikit-learn'], pip_packages=['azure', 'azureml-core', 'azure-storage', 'azure-storage-blob']) @@ -56,24 +56,31 @@ def main(): run_config.environment.docker.enabled = True model_name = PipelineParameter( - name="model_name", default_value=model_name) + name="model_name", default_value=model_name + ) release_id = PipelineParameter( name="release_id", default_value="0" ) + service_name = PipelineParameter( + name="service_name", default_value=service_name + ) deploy_step = PythonScriptStep( name="Deploy Model", script_name=deploy_script_path, compute_target=aml_compute, + source_directory=sources_directory_train, arguments=[ "--release_id", release_id, "--model_name", model_name, + "--service_name", service_name ], runconfig=run_config, allow_reuse=False, ) print("Step Deploy created") + steps = [deploy_step] deploy_pipeline = Pipeline(workspace=aml_workspace, steps=steps) deploy_pipeline.validate() From 86d30c4ed857e324d16c58cc035ac10efeea7f80 Mon Sep 17 00:00:00 2001 From: evilkova Date: Fri, 15 Nov 2019 16:15:06 +0100 Subject: [PATCH 23/60] delete white space --- ml_service/pipelines/build_deploy_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ml_service/pipelines/build_deploy_pipeline.py b/ml_service/pipelines/build_deploy_pipeline.py index cf200c9c..4b0325fb 100644 --- a/ml_service/pipelines/build_deploy_pipeline.py +++ b/ml_service/pipelines/build_deploy_pipeline.py @@ -81,7 +81,7 @@ def main(): print("Step Deploy created") steps = [deploy_step] - + deploy_pipeline = Pipeline(workspace=aml_workspace, steps=steps) deploy_pipeline.validate() published_pipeline = deploy_pipeline.publish( From 3494ebf611bb5f3f76bed27fddc42788053bf096 Mon Sep 17 00:00:00 2001 From: evilkova Date: Fri, 15 Nov 2019 16:32:38 +0100 Subject: [PATCH 24/60] print for testing --- ml_service/pipelines/build_deploy_pipeline.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ml_service/pipelines/build_deploy_pipeline.py b/ml_service/pipelines/build_deploy_pipeline.py index 4b0325fb..888c1df1 100644 --- a/ml_service/pipelines/build_deploy_pipeline.py +++ b/ml_service/pipelines/build_deploy_pipeline.py @@ -58,12 +58,15 @@ def main(): model_name = PipelineParameter( name="model_name", default_value=model_name ) + print(model_name) release_id = PipelineParameter( name="release_id", default_value="0" ) + print(release_id) service_name = PipelineParameter( name="service_name", default_value=service_name ) + print(service_name) deploy_step = PythonScriptStep( name="Deploy Model", From fc99db625d50c36b45c8d53a7432db30ea88cab7 Mon Sep 17 00:00:00 2001 From: evilkova Date: Fri, 15 Nov 2019 16:43:35 +0100 Subject: [PATCH 25/60] change configuration for deployment --- ml_service/pipelines/build_train_pipeline.py | 28 +++++++++++++++----- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/ml_service/pipelines/build_train_pipeline.py b/ml_service/pipelines/build_train_pipeline.py index 1c7c6a82..802c138e 100644 --- a/ml_service/pipelines/build_train_pipeline.py +++ b/ml_service/pipelines/build_train_pipeline.py @@ -46,12 +46,28 @@ def main(): if aml_compute is not None: print(aml_compute) - run_config = RunConfiguration(conda_dependencies=CondaDependencies.create( - conda_packages=['numpy', 'pandas', - 'scikit-learn', 'tensorflow', 'keras'], - pip_packages=['azure', 'azureml-core', - 'azure-storage', - 'azure-storage-blob']) + conda_dependencies = CondaDependencies.create( + conda_packages=[ + 'numpy', + 'pandas', + 'scikit-learn' + ], + pip_packages=[ + 'azureml-core==1.0.72.*', + 'azureml-sdk==1.0.72.*', + 'azure-storage', + 'azure-storage-blob', + 'azureml-dataprep', + 'azureml-datadrift==1.0.72.*' + ], + pin_sdk_version=False + ) + + print(conda_dependencies.serialize_to_string()) + + run_config = RunConfiguration( + framework='Python', + conda_dependencies=conda_dependencies ) run_config.environment.docker.enabled = True From 7074137a2b9b3f5f4096ecb9d24f7b00eb0f3005 Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Tue, 19 Nov 2019 10:49:53 +0100 Subject: [PATCH 26/60] run training and deployment in one script in yaml --- .pipelines/azdo-ci-build-train.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.pipelines/azdo-ci-build-train.yml b/.pipelines/azdo-ci-build-train.yml index 01bf2868..cf91ffa6 100644 --- a/.pipelines/azdo-ci-build-train.yml +++ b/.pipelines/azdo-ci-build-train.yml @@ -35,17 +35,12 @@ stages: - script: | # Invoke the Python building and publishing a training pipeline python3 $(Build.SourcesDirectory)/ml_service/pipelines/build_train_pipeline.py - failOnStderr: 'false' - env: - SP_APP_SECRET: '$(SP_APP_SECRET)' - displayName: 'Publish Azure Machine Learning Pipeline' - - script: | # Invoke the Python building and publishing a deployment pipeline python3 $(Build.SourcesDirectory)/ml_service/pipelines/build_deploy_pipeline.py failOnStderr: 'false' env: SP_APP_SECRET: '$(SP_APP_SECRET)' - displayName: 'Publish Azure Machine Learning Deployment Pipeline' + displayName: 'Publish Azure Machine Learning Pipeline' - task: CopyFiles@2 displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)' From b656ae4cafc53d9f409851c69b64d69c21cf9603 Mon Sep 17 00:00:00 2001 From: evilkova Date: Tue, 19 Nov 2019 11:20:24 +0100 Subject: [PATCH 27/60] add deploy_model script --- code/deploy/deploy_model.py | 68 +++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 code/deploy/deploy_model.py diff --git a/code/deploy/deploy_model.py b/code/deploy/deploy_model.py new file mode 100644 index 00000000..18866bb4 --- /dev/null +++ b/code/deploy/deploy_model.py @@ -0,0 +1,68 @@ +from azureml.core.model import InferenceConfig +from azureml.core.webservice.aci import AciServiceDeploymentConfiguration +from azureml.core import Run, Model +import argparse + +parser = argparse.ArgumentParser("deploy") +parser.add_argument( + "--release_id", + type=str, + help="The ID of the release triggering this pipeline run", +) +parser.add_argument( + "--model_name", + type=str, + help="Name of the Model", + default="sklearn_abn_ca_model.pkl", +) +parser.add_argument( + "--service_name", + type=str, + help="Name of the datastore", + default="abn-ca-service" +) + +args = parser.parse_args() + +model_name = args.model_name +release_id = args.release_id +service_name = args.service_name + +# Get workspace +run = Run.get_context() +exp = run.experiment +ws = run.experiment.workspace + +# Get workspace +inference_config = InferenceConfig(runtime="python", + entry_script="../scoring/score.py", + conda_file="../scoring/conda_dependencies.yml", + source_directory="../scoring/") + +# Do something with imagecnfig image_config = ContainerImage + +# Get latest model +model_list = Model.list(ws, name=model_name) +model = next( + filter( + lambda x: x.created_time == max( + model.created_time for model in model_list), + model_list, + ) +) + +# to do: get config from file +deployment_config = AciServiceDeploymentConfiguration(cpu_cores=1, + memory_gb=4, + location='westeurope', + collect_model_data=True) + +service = Model.deploy(workspace=ws, + name=service_name, + models=[model], + inference_config=inference_config, + deployment_config=deployment_config, + deployment_target=None, + overwrite=True) + +service.wait_for_deployment(show_output=True) From 7a4bd094bb13b697d266a12e288b09e745b555e3 Mon Sep 17 00:00:00 2001 From: evilkova Date: Tue, 19 Nov 2019 11:40:03 +0100 Subject: [PATCH 28/60] fix long path in deploy model script --- code/deploy/deploy_model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/code/deploy/deploy_model.py b/code/deploy/deploy_model.py index 18866bb4..a8b3a0fe 100644 --- a/code/deploy/deploy_model.py +++ b/code/deploy/deploy_model.py @@ -36,7 +36,8 @@ # Get workspace inference_config = InferenceConfig(runtime="python", entry_script="../scoring/score.py", - conda_file="../scoring/conda_dependencies.yml", + conda_file="../scoring/" + "conda_dependencies.yml", source_directory="../scoring/") # Do something with imagecnfig image_config = ContainerImage From c69589b7753846bab0f865f9dfa012dc7c0313c5 Mon Sep 17 00:00:00 2001 From: evilkova Date: Tue, 19 Nov 2019 13:24:40 +0100 Subject: [PATCH 29/60] add pipeline name to run train pipeline --- ml_service/pipelines/run_train_pipeline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ml_service/pipelines/run_train_pipeline.py b/ml_service/pipelines/run_train_pipeline.py index 2492672d..6d49ed72 100644 --- a/ml_service/pipelines/run_train_pipeline.py +++ b/ml_service/pipelines/run_train_pipeline.py @@ -16,6 +16,7 @@ def main(): app_id = os.environ.get('SP_APP_ID') app_secret = os.environ.get('SP_APP_SECRET') build_id = os.environ.get('BUILD_BUILDID') + pipeline_name = os.environ.get('TRAINING_PIPELINE_NAME') service_principal = ServicePrincipalAuthentication( tenant_id=tenant_id, @@ -34,7 +35,7 @@ def main(): matched_pipes = [] for p in pipelines: - if p.version == build_id: + if p.version == build_id and p.name == pipeline_name: matched_pipes.append(p) if(len(matched_pipes) > 1): From ad10a84fce48bf467ff8e34afbbcf9070b3ea22e Mon Sep 17 00:00:00 2001 From: evilkova Date: Tue, 19 Nov 2019 14:04:53 +0100 Subject: [PATCH 30/60] move scoring folder to deploy folder --- code/deploy/deploy_model.py | 6 +++--- code/{ => deploy}/scoring/conda_dependencies.yml | 0 code/{ => deploy}/scoring/deployment_config_aci.yml | 0 code/{ => deploy}/scoring/deployment_config_aks.yml | 0 code/{ => deploy}/scoring/inference_config.yml | 0 code/{ => deploy}/scoring/score.py | 0 6 files changed, 3 insertions(+), 3 deletions(-) rename code/{ => deploy}/scoring/conda_dependencies.yml (100%) rename code/{ => deploy}/scoring/deployment_config_aci.yml (100%) rename code/{ => deploy}/scoring/deployment_config_aks.yml (100%) rename code/{ => deploy}/scoring/inference_config.yml (100%) rename code/{ => deploy}/scoring/score.py (100%) diff --git a/code/deploy/deploy_model.py b/code/deploy/deploy_model.py index a8b3a0fe..5344c34a 100644 --- a/code/deploy/deploy_model.py +++ b/code/deploy/deploy_model.py @@ -35,10 +35,10 @@ # Get workspace inference_config = InferenceConfig(runtime="python", - entry_script="../scoring/score.py", - conda_file="../scoring/" + entry_script="./scoring/score.py", + conda_file="./scoring/" "conda_dependencies.yml", - source_directory="../scoring/") + source_directory="./scoring/") # Do something with imagecnfig image_config = ContainerImage diff --git a/code/scoring/conda_dependencies.yml b/code/deploy/scoring/conda_dependencies.yml similarity index 100% rename from code/scoring/conda_dependencies.yml rename to code/deploy/scoring/conda_dependencies.yml diff --git a/code/scoring/deployment_config_aci.yml b/code/deploy/scoring/deployment_config_aci.yml similarity index 100% rename from code/scoring/deployment_config_aci.yml rename to code/deploy/scoring/deployment_config_aci.yml diff --git a/code/scoring/deployment_config_aks.yml b/code/deploy/scoring/deployment_config_aks.yml similarity index 100% rename from code/scoring/deployment_config_aks.yml rename to code/deploy/scoring/deployment_config_aks.yml diff --git a/code/scoring/inference_config.yml b/code/deploy/scoring/inference_config.yml similarity index 100% rename from code/scoring/inference_config.yml rename to code/deploy/scoring/inference_config.yml diff --git a/code/scoring/score.py b/code/deploy/scoring/score.py similarity index 100% rename from code/scoring/score.py rename to code/deploy/scoring/score.py From d42d0f7b3b568568c613e4eb39805ac6cfea43d8 Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Tue, 19 Nov 2019 14:07:51 +0100 Subject: [PATCH 31/60] Copy deploy and training folders --- .pipelines/azdo-ci-build-train.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/azdo-ci-build-train.yml b/.pipelines/azdo-ci-build-train.yml index cf91ffa6..28d25b82 100644 --- a/.pipelines/azdo-ci-build-train.yml +++ b/.pipelines/azdo-ci-build-train.yml @@ -48,7 +48,8 @@ stages: SourceFolder: '$(Build.SourcesDirectory)' TargetFolder: '$(Build.ArtifactStagingDirectory)' Contents: | - code/scoring/** + code/deploy/** + code/trainig/** ml_service/pipelines/** - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact' From 7ed6b44a5b7b0701971000a5d4ae501a3d68e712 Mon Sep 17 00:00:00 2001 From: evilkova Date: Tue, 19 Nov 2019 14:33:37 +0100 Subject: [PATCH 32/60] change path to scoring directory --- code/deploy/deploy_model.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/code/deploy/deploy_model.py b/code/deploy/deploy_model.py index 5344c34a..d25ce47a 100644 --- a/code/deploy/deploy_model.py +++ b/code/deploy/deploy_model.py @@ -35,10 +35,9 @@ # Get workspace inference_config = InferenceConfig(runtime="python", - entry_script="./scoring/score.py", - conda_file="./scoring/" - "conda_dependencies.yml", - source_directory="./scoring/") + entry_script="score.py", + conda_file="conda_dependencies.yml", + source_directory="./deploy/scoring/") # Do something with imagecnfig image_config = ContainerImage From af89e7e4823ea77049a723dcfe714ba1ac2df962 Mon Sep 17 00:00:00 2001 From: evilkova Date: Tue, 19 Nov 2019 16:05:36 +0100 Subject: [PATCH 33/60] add print to deploy model script --- code/deploy/deploy_model.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/code/deploy/deploy_model.py b/code/deploy/deploy_model.py index d25ce47a..1ddaffa2 100644 --- a/code/deploy/deploy_model.py +++ b/code/deploy/deploy_model.py @@ -28,6 +28,10 @@ release_id = args.release_id service_name = args.service_name +print(model_name) +print(release_id) +print(service_name) + # Get workspace run = Run.get_context() exp = run.experiment @@ -38,6 +42,7 @@ entry_script="score.py", conda_file="conda_dependencies.yml", source_directory="./deploy/scoring/") +print(inference_config) # Do something with imagecnfig image_config = ContainerImage From c1861f3c09f80ff1e097d6df2b4a9b3d1de410a4 Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Tue, 19 Nov 2019 16:34:03 +0100 Subject: [PATCH 34/60] add azureml dependencies --- code/deploy/scoring/conda_dependencies.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/code/deploy/scoring/conda_dependencies.yml b/code/deploy/scoring/conda_dependencies.yml index 60c8dd92..4a515257 100644 --- a/code/deploy/scoring/conda_dependencies.yml +++ b/code/deploy/scoring/conda_dependencies.yml @@ -25,7 +25,7 @@ dependencies: - pip: # Required packages for AzureML execution, history, and data preparation. - - azureml-sdk==1.0.72 + - azureml-sdk[notebooks] - scipy==1.3.1 - scikit-learn==0.21.3 - pandas==0.25.3 @@ -33,4 +33,5 @@ dependencies: - joblib==0.14.0 - gunicorn==19.9.0 - flask==1.1.1 - \ No newline at end of file + - azure-ml-api-sdk + From 7a4b6d5721a52e85611095d9f657db0258b7357d Mon Sep 17 00:00:00 2001 From: evilkova Date: Tue, 19 Nov 2019 16:56:54 +0100 Subject: [PATCH 35/60] delete overwrite parameter from deploy function --- code/deploy/deploy_model.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/code/deploy/deploy_model.py b/code/deploy/deploy_model.py index 1ddaffa2..c13ad148 100644 --- a/code/deploy/deploy_model.py +++ b/code/deploy/deploy_model.py @@ -67,7 +67,6 @@ models=[model], inference_config=inference_config, deployment_config=deployment_config, - deployment_target=None, - overwrite=True) + deployment_target=None) service.wait_for_deployment(show_output=True) From e05b02a92c1055a40db83261d9eda68870e9e73f Mon Sep 17 00:00:00 2001 From: evilkova Date: Wed, 20 Nov 2019 09:29:45 +0100 Subject: [PATCH 36/60] change conda dependences --- code/deploy/deploy_model.py | 3 +- code/deploy/scoring/conda_dependencies.yml | 36 ++++++++++++++++------ 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/code/deploy/deploy_model.py b/code/deploy/deploy_model.py index c13ad148..1ddaffa2 100644 --- a/code/deploy/deploy_model.py +++ b/code/deploy/deploy_model.py @@ -67,6 +67,7 @@ models=[model], inference_config=inference_config, deployment_config=deployment_config, - deployment_target=None) + deployment_target=None, + overwrite=True) service.wait_for_deployment(show_output=True) diff --git a/code/deploy/scoring/conda_dependencies.yml b/code/deploy/scoring/conda_dependencies.yml index 4a515257..325ff70f 100644 --- a/code/deploy/scoring/conda_dependencies.yml +++ b/code/deploy/scoring/conda_dependencies.yml @@ -1,37 +1,53 @@ # Conda environment specification. The dependencies defined in this file will + # be automatically provisioned for managed runs. These include runs against + # the localdocker, remotedocker, and cluster compute targets. + # Note that this file is NOT used to automatically manage dependencies for the + # local compute target. To provision these dependencies locally, run: + # conda env update --file conda_dependencies.yml + # Details about the Conda environment file format: + # https://conda.io/docs/using/envs.html#create-environment-file-by-hand + # For managing Spark packages and configuration, see spark_dependencies.yml. + + # Version of this configuration file's structure and semantics in AzureML. + # This directive is stored in a comment to preserve the Conda file structure. + # [AzureMlVersion] = 2 + name: project_environment dependencies: # The python interpreter version. + # Currently Azure ML Workbench only supports 3.5.2 and later. -- python=3.7.5 + +- python=3.6.2 # Required by azureml-defaults, installed separately through Conda to + # get a prebuilt version and not require build tools for the install. -- psutil=5.6 #latest + +- psutil=5.3 - pip: # Required packages for AzureML execution, history, and data preparation. - - azureml-sdk[notebooks] - - scipy==1.3.1 + - azureml-sdk[notebooks] # add the version to lock it ==0.1.74 + - scipy==1.0.0 - scikit-learn==0.21.3 - - pandas==0.25.3 - - numpy==1.17.3 - - joblib==0.14.0 - - gunicorn==19.9.0 + - pandas==0.23.1 + - numpy==1.14.5 + - joblib==0.13.2 + - gunicorn==19.9.0 - flask==1.1.1 - - azure-ml-api-sdk - + - azure-ml-api-sdk \ No newline at end of file From ae853cc7160bb5a96a21d13dd1345d0040127cca Mon Sep 17 00:00:00 2001 From: evilkova Date: Wed, 20 Nov 2019 11:51:11 +0100 Subject: [PATCH 37/60] change conda dependencies in deploy pipeline --- code/deploy/scoring/conda_dependencies.yml | 36 ++++++------------- ml_service/pipelines/build_deploy_pipeline.py | 28 +++++++++++---- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/code/deploy/scoring/conda_dependencies.yml b/code/deploy/scoring/conda_dependencies.yml index 325ff70f..4a515257 100644 --- a/code/deploy/scoring/conda_dependencies.yml +++ b/code/deploy/scoring/conda_dependencies.yml @@ -1,53 +1,37 @@ # Conda environment specification. The dependencies defined in this file will - # be automatically provisioned for managed runs. These include runs against - # the localdocker, remotedocker, and cluster compute targets. - # Note that this file is NOT used to automatically manage dependencies for the - # local compute target. To provision these dependencies locally, run: - # conda env update --file conda_dependencies.yml - # Details about the Conda environment file format: - # https://conda.io/docs/using/envs.html#create-environment-file-by-hand - # For managing Spark packages and configuration, see spark_dependencies.yml. - - # Version of this configuration file's structure and semantics in AzureML. - # This directive is stored in a comment to preserve the Conda file structure. - # [AzureMlVersion] = 2 - name: project_environment dependencies: # The python interpreter version. - # Currently Azure ML Workbench only supports 3.5.2 and later. - -- python=3.6.2 +- python=3.7.5 # Required by azureml-defaults, installed separately through Conda to - # get a prebuilt version and not require build tools for the install. - -- psutil=5.3 +- psutil=5.6 #latest - pip: # Required packages for AzureML execution, history, and data preparation. - - azureml-sdk[notebooks] # add the version to lock it ==0.1.74 - - scipy==1.0.0 + - azureml-sdk[notebooks] + - scipy==1.3.1 - scikit-learn==0.21.3 - - pandas==0.23.1 - - numpy==1.14.5 - - joblib==0.13.2 - - gunicorn==19.9.0 + - pandas==0.25.3 + - numpy==1.17.3 + - joblib==0.14.0 + - gunicorn==19.9.0 - flask==1.1.1 - - azure-ml-api-sdk \ No newline at end of file + - azure-ml-api-sdk + diff --git a/ml_service/pipelines/build_deploy_pipeline.py b/ml_service/pipelines/build_deploy_pipeline.py index 888c1df1..9f1d5a7d 100644 --- a/ml_service/pipelines/build_deploy_pipeline.py +++ b/ml_service/pipelines/build_deploy_pipeline.py @@ -46,12 +46,28 @@ def main(): if aml_compute is not None: print(aml_compute) - run_config = RunConfiguration(conda_dependencies=CondaDependencies.create( - conda_packages=['numpy', 'pandas', - 'scikit-learn'], - pip_packages=['azure', 'azureml-core', - 'azure-storage', - 'azure-storage-blob']) + conda_dependencies = CondaDependencies.create( + conda_packages=[ + 'numpy', + 'pandas', + 'scikit-learn' + ], + pip_packages=[ + 'azureml-core==1.0.72.*', + 'azureml-sdk==1.0.72.*', + 'azure-storage', + 'azure-storage-blob', + 'azureml-dataprep', + 'azureml-datadrift==1.0.72.*' + ], + pin_sdk_version=False + ) + + print(conda_dependencies.serialize_to_string()) + + run_config = RunConfiguration( + framework='Python', + conda_dependencies=conda_dependencies ) run_config.environment.docker.enabled = True From 2f0674046981a5247a23e5d0724f32827941e1b4 Mon Sep 17 00:00:00 2001 From: evilkova Date: Wed, 20 Nov 2019 13:16:10 +0100 Subject: [PATCH 38/60] print parser to find release id --- code/training/train.py | 1 + ml_service/pipelines/build_train_pipeline.py | 1 + 2 files changed, 2 insertions(+) diff --git a/code/training/train.py b/code/training/train.py index d703964f..bb40e661 100644 --- a/code/training/train.py +++ b/code/training/train.py @@ -35,6 +35,7 @@ parser = argparse.ArgumentParser("train") +print(parser) parser.add_argument( "--release_id", type=str, diff --git a/ml_service/pipelines/build_train_pipeline.py b/ml_service/pipelines/build_train_pipeline.py index 802c138e..d1e928e4 100644 --- a/ml_service/pipelines/build_train_pipeline.py +++ b/ml_service/pipelines/build_train_pipeline.py @@ -117,6 +117,7 @@ def main(): ) print(f'Published pipeline: {published_pipeline.name}') print(f'for build {published_pipeline.version}') + print(f'with release id {release_id}') if __name__ == '__main__': From 6d40199fbb8d680387ffe087f4d2bd8eaf5fbe2a Mon Sep 17 00:00:00 2001 From: evilkova Date: Wed, 20 Nov 2019 13:56:32 +0100 Subject: [PATCH 39/60] add release id to run pipeline --- ml_service/pipelines/build_train_pipeline.py | 1 - ml_service/pipelines/run_train_pipeline.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/ml_service/pipelines/build_train_pipeline.py b/ml_service/pipelines/build_train_pipeline.py index d1e928e4..802c138e 100644 --- a/ml_service/pipelines/build_train_pipeline.py +++ b/ml_service/pipelines/build_train_pipeline.py @@ -117,7 +117,6 @@ def main(): ) print(f'Published pipeline: {published_pipeline.name}') print(f'for build {published_pipeline.version}') - print(f'with release id {release_id}') if __name__ == '__main__': diff --git a/ml_service/pipelines/run_train_pipeline.py b/ml_service/pipelines/run_train_pipeline.py index 6d49ed72..ef52bbf3 100644 --- a/ml_service/pipelines/run_train_pipeline.py +++ b/ml_service/pipelines/run_train_pipeline.py @@ -15,6 +15,7 @@ def main(): model_name = os.environ.get("MODEL_NAME") app_id = os.environ.get('SP_APP_ID') app_secret = os.environ.get('SP_APP_SECRET') + release_id = os.environ.get('RELEASE_RELEASEID') build_id = os.environ.get('BUILD_BUILDID') pipeline_name = os.environ.get('TRAINING_PIPELINE_NAME') From d5a6cb5aaf337bdf9651fcf23e6cc895e5cc927c Mon Sep 17 00:00:00 2001 From: evilkova Date: Wed, 20 Nov 2019 14:05:49 +0100 Subject: [PATCH 40/60] add release id to pipeline --- ml_service/pipelines/run_train_pipeline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ml_service/pipelines/run_train_pipeline.py b/ml_service/pipelines/run_train_pipeline.py index ef52bbf3..96a2d2ad 100644 --- a/ml_service/pipelines/run_train_pipeline.py +++ b/ml_service/pipelines/run_train_pipeline.py @@ -48,7 +48,8 @@ def main(): else: published_pipeline = matched_pipes[0] - pipeline_parameters = {"model_name": model_name} + pipeline_parameters = {"model_name": model_name, + "release_id": release_id} response = published_pipeline.submit( aml_workspace, From 80c7c583227b1cbe21ef8345af1d0a90dbaafa1f Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Wed, 20 Nov 2019 15:51:21 +0100 Subject: [PATCH 41/60] Update azdo-ci-build-train.yml for Azure Pipelines --- .pipelines/azdo-ci-build-train.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/azdo-ci-build-train.yml b/.pipelines/azdo-ci-build-train.yml index 28d25b82..78c89702 100644 --- a/.pipelines/azdo-ci-build-train.yml +++ b/.pipelines/azdo-ci-build-train.yml @@ -57,4 +57,5 @@ stages: ArtifactName: 'mlops-pipelines' publishLocation: 'container' pathtoPublish: '$(Build.ArtifactStagingDirectory)' - TargetPath: '$(Build.ArtifactStagingDirectory)' \ No newline at end of file + TargetPath: '$(Build.ArtifactStagingDirectory)' + From f129f8490187319ec3aa005340bdcdd5551ba074 Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Wed, 20 Nov 2019 15:55:55 +0100 Subject: [PATCH 42/60] Update azdo-ci-build-train.yml for Azure Pipelines --- .pipelines/azdo-ci-build-train.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/azdo-ci-build-train.yml b/.pipelines/azdo-ci-build-train.yml index 78c89702..9dccf3a7 100644 --- a/.pipelines/azdo-ci-build-train.yml +++ b/.pipelines/azdo-ci-build-train.yml @@ -49,7 +49,7 @@ stages: TargetFolder: '$(Build.ArtifactStagingDirectory)' Contents: | code/deploy/** - code/trainig/** + code/training/** ml_service/pipelines/** - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact' From 5d0819230271f344a9d4bb43668a12d326bc1063 Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Wed, 20 Nov 2019 15:57:42 +0100 Subject: [PATCH 43/60] Update azdo-ci-build-train.yml for Azure Pipelines --- .pipelines/azdo-ci-build-train.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.pipelines/azdo-ci-build-train.yml b/.pipelines/azdo-ci-build-train.yml index 9dccf3a7..eb1311f3 100644 --- a/.pipelines/azdo-ci-build-train.yml +++ b/.pipelines/azdo-ci-build-train.yml @@ -48,8 +48,7 @@ stages: SourceFolder: '$(Build.SourcesDirectory)' TargetFolder: '$(Build.ArtifactStagingDirectory)' Contents: | - code/deploy/** - code/training/** + code/** ml_service/pipelines/** - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact' From 46af56e2d947758eb900234ac9df16d88bc5507b Mon Sep 17 00:00:00 2001 From: evilkova Date: Wed, 20 Nov 2019 16:59:15 +0100 Subject: [PATCH 44/60] change model name in score script --- code/deploy/deploy_model.py | 2 ++ code/deploy/scoring/score.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/code/deploy/deploy_model.py b/code/deploy/deploy_model.py index 1ddaffa2..9fc03b4d 100644 --- a/code/deploy/deploy_model.py +++ b/code/deploy/deploy_model.py @@ -70,4 +70,6 @@ deployment_target=None, overwrite=True) +print(service.state) + service.wait_for_deployment(show_output=True) diff --git a/code/deploy/scoring/score.py b/code/deploy/scoring/score.py index dafe6bee..396e638e 100644 --- a/code/deploy/scoring/score.py +++ b/code/deploy/scoring/score.py @@ -34,7 +34,7 @@ def init(): # load the model from file into a global object model_path = Model.get_model_path( - model_name="sklearn_regression_model.pkl") + model_name="sklearn_abn_ca_model.pkl") model = joblib.load(model_path) From 69aa057bb36aaa7f17959aaef912b8fd452f0b30 Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Fri, 22 Nov 2019 11:16:22 +0100 Subject: [PATCH 45/60] add aks deployment pipeline --- .pipelines/azdo-ci-build-train.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.pipelines/azdo-ci-build-train.yml b/.pipelines/azdo-ci-build-train.yml index eb1311f3..6ed9e1a9 100644 --- a/.pipelines/azdo-ci-build-train.yml +++ b/.pipelines/azdo-ci-build-train.yml @@ -35,8 +35,10 @@ stages: - script: | # Invoke the Python building and publishing a training pipeline python3 $(Build.SourcesDirectory)/ml_service/pipelines/build_train_pipeline.py - # Invoke the Python building and publishing a deployment pipeline + # Invoke the Python building and publishing an ACI deployment pipeline python3 $(Build.SourcesDirectory)/ml_service/pipelines/build_deploy_pipeline.py + # Invoke the Python building and publishing an AKS deployment pipeline + python3 $(Build.SourcesDirectory)/ml_service/pipelines/build_deploy_prod_pipeline.py failOnStderr: 'false' env: SP_APP_SECRET: '$(SP_APP_SECRET)' From 95ca659d283908f602db9b34d1e0dc13150e14d1 Mon Sep 17 00:00:00 2001 From: evilkova Date: Fri, 22 Nov 2019 11:22:23 +0100 Subject: [PATCH 46/60] add prod deployment --- code/deploy/deploy_prod_model.py | 74 +++++++++++ .../pipelines/build_deploy_prod_pipeline.py | 116 ++++++++++++++++++ .../pipelines/run_deploy_prod_pipeline.py | 66 ++++++++++ 3 files changed, 256 insertions(+) create mode 100644 code/deploy/deploy_prod_model.py create mode 100644 ml_service/pipelines/build_deploy_prod_pipeline.py create mode 100644 ml_service/pipelines/run_deploy_prod_pipeline.py diff --git a/code/deploy/deploy_prod_model.py b/code/deploy/deploy_prod_model.py new file mode 100644 index 00000000..60a9038c --- /dev/null +++ b/code/deploy/deploy_prod_model.py @@ -0,0 +1,74 @@ +from azureml.core.model import InferenceConfig +from azureml.core.compute import AksCompute +from azureml.core.webservice import AksWebservice, Webservice +from azureml.core import Run, Model +import argparse + +parser = argparse.ArgumentParser("deploy") +parser.add_argument( + "--release_id", + type=str, + help="The ID of the release triggering this pipeline run", +) +parser.add_argument( + "--model_name", + type=str, + help="Name of the Model", + default="sklearn_abn_ca_model.pkl", +) +parser.add_argument( + "--service_name", + type=str, + help="Name of the datastore", + default="prod-abn-ca-service" +) + +args = parser.parse_args() + +model_name = args.model_name +release_id = args.release_id +service_name = args.service_name + +print(model_name) +print(release_id) +print(service_name) + +# Get workspace +run = Run.get_context() +exp = run.experiment +ws = run.experiment.workspace + +# Get workspace +inference_config = InferenceConfig(runtime="python", + entry_script="score.py", + conda_file="conda_dependencies.yml", + source_directory="./deploy/scoring/") +print(inference_config) + +# Do something with imagecnfig image_config = ContainerImage + +# Get latest model +model_list = Model.list(ws, name=model_name) +model = next( + filter( + lambda x: x.created_time == max( + model.created_time for model in model_list), + model_list, + ) +) + +# to do: get config from file +aks_target = AksCompute(ws,"dspe-aks") +# If deploying to a cluster configured for dev/test, ensure that it was created with enough +# cores and memory to handle this deployment configuration. Note that memory is also used by +# things such as dependencies and AML components. +deployment_config = AksWebservice.deploy_configuration(cpu_cores = 1, + memory_gb = 4) +service = Model.deploy(workspace=ws, + name=service_name, + models=[model], + inference_config=inference_config, + deployment_config=deployment_config, + deployment_target=aks_target, + overwrite=True) +service.wait_for_deployment(show_output = True) \ No newline at end of file diff --git a/ml_service/pipelines/build_deploy_prod_pipeline.py b/ml_service/pipelines/build_deploy_prod_pipeline.py new file mode 100644 index 00000000..966e66d9 --- /dev/null +++ b/ml_service/pipelines/build_deploy_prod_pipeline.py @@ -0,0 +1,116 @@ +from azureml.pipeline.core.graph import PipelineParameter +from azureml.pipeline.steps import PythonScriptStep +from azureml.pipeline.core import Pipeline # , PipelineData +from azureml.core.runconfig import RunConfiguration, CondaDependencies +# from azureml.core import Datastore +import os +import sys +from dotenv import load_dotenv +sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 +from workspace import get_workspace +from attach_compute import get_compute + + +def main(): + load_dotenv() + workspace_name = os.environ.get("BASE_NAME")+"-AML-WS" + resource_group = os.environ.get("RESOURCE_GROUP") + subscription_id = os.environ.get("SUBSCRIPTION_ID") + tenant_id = os.environ.get("TENANT_ID") + app_id = os.environ.get("SP_APP_ID") + app_secret = os.environ.get("SP_APP_SECRET") + deploy_script_path = os.environ.get("DEPLOY_PROD_SCRIPT_PATH") + vm_size = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU") + compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME") + model_name = os.environ.get("MODEL_NAME") + build_id = os.environ.get("BUILD_BUILDID") + pipeline_name = os.environ.get("DEPLOY_PROD_PIPELINE_NAME") + service_name = os.environ.get("DEPLOY_PROD_SERVICE_NAME") + sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN") + + # Get Azure machine learning workspace + aml_workspace = get_workspace( + workspace_name, + resource_group, + subscription_id, + tenant_id, + app_id, + app_secret) + print(aml_workspace) + + # Get Azure machine learning cluster + aml_compute = get_compute( + aml_workspace, + compute_name, + vm_size) + if aml_compute is not None: + print(aml_compute) + + conda_dependencies = CondaDependencies.create( + conda_packages=[ + 'numpy', + 'pandas', + 'scikit-learn' + ], + pip_packages=[ + 'azureml-core==1.0.72.*', + 'azureml-sdk==1.0.72.*', + 'azure-storage', + 'azure-storage-blob', + 'azureml-dataprep', + 'azureml-datadrift==1.0.72.*' + ], + pin_sdk_version=False + ) + + print(conda_dependencies.serialize_to_string()) + + run_config = RunConfiguration( + framework='Python', + conda_dependencies=conda_dependencies + ) + run_config.environment.docker.enabled = True + + model_name = PipelineParameter( + name="model_name", default_value=model_name + ) + print(model_name) + release_id = PipelineParameter( + name="release_id", default_value="0" + ) + print(release_id) + service_name = PipelineParameter( + name="service_name", default_value=service_name + ) + print(service_name) + + deploy_step = PythonScriptStep( + name="Deploy Prod Model", + script_name=deploy_script_path, + compute_target=aml_compute, + source_directory=sources_directory_train, + arguments=[ + "--release_id", release_id, + "--model_name", model_name, + "--service_name", service_name + ], + runconfig=run_config, + allow_reuse=False, + ) + print("Step Deploy Prod created") + + steps = [deploy_step] + + deploy_pipeline = Pipeline(workspace=aml_workspace, steps=steps) + deploy_pipeline.validate() + published_pipeline = deploy_pipeline.publish( + name=pipeline_name, + description="Model deploy Prod pipeline", + version=build_id + ) + print(f'Published pipeline: {published_pipeline.name}') + print(f'for build {published_pipeline.version}') + + +if __name__ == '__main__': + main() diff --git a/ml_service/pipelines/run_deploy_prod_pipeline.py b/ml_service/pipelines/run_deploy_prod_pipeline.py new file mode 100644 index 00000000..b88a2349 --- /dev/null +++ b/ml_service/pipelines/run_deploy_prod_pipeline.py @@ -0,0 +1,66 @@ +import os +from azureml.pipeline.core import PublishedPipeline +from azureml.core import Workspace +from azureml.core.authentication import ServicePrincipalAuthentication +from dotenv import load_dotenv + + +def main(): + load_dotenv() + workspace_name = os.environ.get("BASE_NAME")+"-AML-WS" + resource_group = os.environ.get("RESOURCE_GROUP") + subscription_id = os.environ.get("SUBSCRIPTION_ID") + tenant_id = os.environ.get("TENANT_ID") + experiment_name = os.environ.get("EXPERIMENT_NAME") + model_name = os.environ.get("MODEL_NAME") + app_id = os.environ.get('SP_APP_ID') + app_secret = os.environ.get('SP_APP_SECRET') + release_id = os.environ.get('RELEASE_RELEASEID') + build_id = os.environ.get('BUILD_BUILDID') + pipeline_name = os.environ.get('DEPLOY_PROD_PIPELINE_NAME') + service_name = os.environ.get('DEPLOY_PROD_SERVICE_NAME') + + service_principal = ServicePrincipalAuthentication( + tenant_id=tenant_id, + service_principal_id=app_id, + service_principal_password=app_secret) + + aml_workspace = Workspace.get( + name=workspace_name, + subscription_id=subscription_id, + resource_group=resource_group, + auth=service_principal + ) + + # Find the pipeline that was published by the specified build ID + pipelines = PublishedPipeline.list(aml_workspace) + matched_pipes = [] + + for p in pipelines: + if p.version == build_id and p.name == pipeline_name: + matched_pipes.append(p) + + if(len(matched_pipes) > 1): + published_pipeline = None + raise Exception(f"Multiple active pipelines are published for build {build_id}.") # NOQA: E501 + elif(len(matched_pipes) == 0): + published_pipeline = None + raise KeyError(f"Unable to find a published pipeline for this build {build_id}") # NOQA: E501 + else: + published_pipeline = matched_pipes[0] + + pipeline_parameters = {"model_name": model_name, + "release_id": release_id, + "service_name": service_name} + + response = published_pipeline.submit( + aml_workspace, + experiment_name, + pipeline_parameters) + + run_id = response.id + print("Pipeline run initiated ", run_id) + + +if __name__ == "__main__": + main() From 9dba3c81891b9259d91bb1a3d71e489530d378cc Mon Sep 17 00:00:00 2001 From: evilkova Date: Fri, 22 Nov 2019 11:30:17 +0100 Subject: [PATCH 47/60] fix linting flake8 in deploy prod script --- code/deploy/deploy_prod_model.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/code/deploy/deploy_prod_model.py b/code/deploy/deploy_prod_model.py index 60a9038c..e3615921 100644 --- a/code/deploy/deploy_prod_model.py +++ b/code/deploy/deploy_prod_model.py @@ -1,6 +1,6 @@ from azureml.core.model import InferenceConfig from azureml.core.compute import AksCompute -from azureml.core.webservice import AksWebservice, Webservice +from azureml.core.webservice import AksWebservice from azureml.core import Run, Model import argparse @@ -58,12 +58,14 @@ ) # to do: get config from file -aks_target = AksCompute(ws,"dspe-aks") -# If deploying to a cluster configured for dev/test, ensure that it was created with enough -# cores and memory to handle this deployment configuration. Note that memory is also used by +aks_target = AksCompute(ws, "dspe-aks") +# If deploying to a cluster configured for dev/test, +# ensure that it was created with enough +# cores and memory to handle this deployment configuration. +# Note that memory is also used by # things such as dependencies and AML components. -deployment_config = AksWebservice.deploy_configuration(cpu_cores = 1, - memory_gb = 4) +deployment_config = AksWebservice.deploy_configuration(cpu_cores=1, + memory_gb=4) service = Model.deploy(workspace=ws, name=service_name, models=[model], @@ -71,4 +73,4 @@ deployment_config=deployment_config, deployment_target=aks_target, overwrite=True) -service.wait_for_deployment(show_output = True) \ No newline at end of file +service.wait_for_deployment(show_output=True) From 89686298619e19abc13ac3e12e530f6ef50e1f3c Mon Sep 17 00:00:00 2001 From: evilkova Date: Fri, 22 Nov 2019 12:06:07 +0100 Subject: [PATCH 48/60] attach dspe-aks cluster --- code/deploy/deploy_prod_model.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/code/deploy/deploy_prod_model.py b/code/deploy/deploy_prod_model.py index e3615921..4ae1c0c2 100644 --- a/code/deploy/deploy_prod_model.py +++ b/code/deploy/deploy_prod_model.py @@ -1,5 +1,5 @@ from azureml.core.model import InferenceConfig -from azureml.core.compute import AksCompute +from azureml.core.compute import AksCompute, ComputeTarget from azureml.core.webservice import AksWebservice from azureml.core import Run, Model import argparse @@ -58,7 +58,13 @@ ) # to do: get config from file -aks_target = AksCompute(ws, "dspe-aks") +purpose = AksCompute.ClusterPurpose.DEV_TEST +attach_config = AksCompute.attach_configuration(resource_group="sandbox-" + "nl02328-024-rg", + cluster_name="dspe-aks", + cluster_purpose=purpose) +aks_target = ComputeTarget.attach(ws, "dspe-aks", attach_config) + # If deploying to a cluster configured for dev/test, # ensure that it was created with enough # cores and memory to handle this deployment configuration. From b382c77116730240503f3dc1e6277d2b9680a004 Mon Sep 17 00:00:00 2001 From: evilkova Date: Fri, 22 Nov 2019 13:43:04 +0100 Subject: [PATCH 49/60] reattachement aks --- code/deploy/deploy_prod_model.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/code/deploy/deploy_prod_model.py b/code/deploy/deploy_prod_model.py index 4ae1c0c2..5af82a81 100644 --- a/code/deploy/deploy_prod_model.py +++ b/code/deploy/deploy_prod_model.py @@ -57,12 +57,20 @@ ) ) +# reattaachemet AKS cluster +compute_target = ComputeTarget(workspace=ws, name="dspe-aks") +compute_target.detach() +compute_target.wait_for_completion(show_output=True) +print("AKS is detached") + # to do: get config from file purpose = AksCompute.ClusterPurpose.DEV_TEST attach_config = AksCompute.attach_configuration(resource_group="sandbox-" "nl02328-024-rg", cluster_name="dspe-aks", cluster_purpose=purpose) +print(attach_config) +print("dspe-aks is attached") aks_target = ComputeTarget.attach(ws, "dspe-aks", attach_config) # If deploying to a cluster configured for dev/test, From 64e82795fb31639890516df10e0f5331a870dd4c Mon Sep 17 00:00:00 2001 From: evilkova Date: Fri, 22 Nov 2019 14:13:50 +0100 Subject: [PATCH 50/60] delete compute taret attachement --- code/deploy/deploy_prod_model.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/code/deploy/deploy_prod_model.py b/code/deploy/deploy_prod_model.py index 5af82a81..57171872 100644 --- a/code/deploy/deploy_prod_model.py +++ b/code/deploy/deploy_prod_model.py @@ -1,5 +1,5 @@ from azureml.core.model import InferenceConfig -from azureml.core.compute import AksCompute, ComputeTarget +from azureml.core.compute import AksCompute from azureml.core.webservice import AksWebservice from azureml.core import Run, Model import argparse @@ -58,28 +58,29 @@ ) # reattaachemet AKS cluster -compute_target = ComputeTarget(workspace=ws, name="dspe-aks") -compute_target.detach() -compute_target.wait_for_completion(show_output=True) -print("AKS is detached") +# compute_target = ComputeTarget(workspace=ws, name="dspe-aks") +# compute_target.detach() +# compute_target.wait_for_completion(show_output=True) +# print("AKS is detached") # to do: get config from file -purpose = AksCompute.ClusterPurpose.DEV_TEST -attach_config = AksCompute.attach_configuration(resource_group="sandbox-" - "nl02328-024-rg", - cluster_name="dspe-aks", - cluster_purpose=purpose) -print(attach_config) -print("dspe-aks is attached") -aks_target = ComputeTarget.attach(ws, "dspe-aks", attach_config) +# purpose = AksCompute.ClusterPurpose.DEV_TEST +# attach_config = AksCompute.attach_configuration(resource_group="sandbox-" +# "nl02328-024-rg", +# cluster_name="dspe-aks", +# cluster_purpose=purpose) +# print(attach_config) +# print("dspe-aks is attached") +# aks_target = ComputeTarget.attach(ws, "dspe-aks", attach_config) # If deploying to a cluster configured for dev/test, # ensure that it was created with enough # cores and memory to handle this deployment configuration. # Note that memory is also used by # things such as dependencies and AML components. +aks_target = AksCompute(ws, "dspe-aks") deployment_config = AksWebservice.deploy_configuration(cpu_cores=1, - memory_gb=4) + memory_gb=1) service = Model.deploy(workspace=ws, name=service_name, models=[model], From bf186e6a5b9edc428a6e04ba747dcd91af5b37d7 Mon Sep 17 00:00:00 2001 From: evilkova Date: Fri, 22 Nov 2019 16:24:51 +0100 Subject: [PATCH 51/60] remove compute configuration --- code/deploy/deploy_prod_model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/deploy/deploy_prod_model.py b/code/deploy/deploy_prod_model.py index 57171872..0f187d64 100644 --- a/code/deploy/deploy_prod_model.py +++ b/code/deploy/deploy_prod_model.py @@ -78,14 +78,14 @@ # cores and memory to handle this deployment configuration. # Note that memory is also used by # things such as dependencies and AML components. -aks_target = AksCompute(ws, "dspe-aks") +#aks_target = AksCompute(ws, "dspe-aks") deployment_config = AksWebservice.deploy_configuration(cpu_cores=1, - memory_gb=1) + memory_gb=4) service = Model.deploy(workspace=ws, name=service_name, models=[model], inference_config=inference_config, deployment_config=deployment_config, - deployment_target=aks_target, + deployment_target="one-dspe-aks", overwrite=True) service.wait_for_deployment(show_output=True) From 3ed98b3d0298b1b8745f25aab8924dba89143fc3 Mon Sep 17 00:00:00 2001 From: evilkova Date: Fri, 22 Nov 2019 16:30:21 +0100 Subject: [PATCH 52/60] fix flake8 --- code/deploy/deploy_prod_model.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/code/deploy/deploy_prod_model.py b/code/deploy/deploy_prod_model.py index 0f187d64..560c15b2 100644 --- a/code/deploy/deploy_prod_model.py +++ b/code/deploy/deploy_prod_model.py @@ -1,5 +1,4 @@ from azureml.core.model import InferenceConfig -from azureml.core.compute import AksCompute from azureml.core.webservice import AksWebservice from azureml.core import Run, Model import argparse @@ -78,7 +77,7 @@ # cores and memory to handle this deployment configuration. # Note that memory is also used by # things such as dependencies and AML components. -#aks_target = AksCompute(ws, "dspe-aks") +# aks_target = AksCompute(ws, "dspe-aks") deployment_config = AksWebservice.deploy_configuration(cpu_cores=1, memory_gb=4) service = Model.deploy(workspace=ws, From ee15e7923d9a15cd918ca757b5b3b65af493a4df Mon Sep 17 00:00:00 2001 From: evilkova Date: Fri, 22 Nov 2019 17:48:41 +0100 Subject: [PATCH 53/60] get compute target for deployment --- code/deploy/deploy_prod_model.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/code/deploy/deploy_prod_model.py b/code/deploy/deploy_prod_model.py index 560c15b2..dc24e99e 100644 --- a/code/deploy/deploy_prod_model.py +++ b/code/deploy/deploy_prod_model.py @@ -1,5 +1,6 @@ from azureml.core.model import InferenceConfig from azureml.core.webservice import AksWebservice +from azureml.core.compute import ComputeTarget from azureml.core import Run, Model import argparse @@ -77,7 +78,8 @@ # cores and memory to handle this deployment configuration. # Note that memory is also used by # things such as dependencies and AML components. -# aks_target = AksCompute(ws, "dspe-aks") +aks_target = ComputeTarget._get(ws, "one-dspe-aks") +print(aks_target) deployment_config = AksWebservice.deploy_configuration(cpu_cores=1, memory_gb=4) service = Model.deploy(workspace=ws, @@ -85,6 +87,6 @@ models=[model], inference_config=inference_config, deployment_config=deployment_config, - deployment_target="one-dspe-aks", + deployment_target=aks_target, overwrite=True) service.wait_for_deployment(show_output=True) From 3081d1f467fd2a5426023038d3629ed990876aac Mon Sep 17 00:00:00 2001 From: evilkova Date: Fri, 22 Nov 2019 18:12:24 +0100 Subject: [PATCH 54/60] delete deployment_target --- code/deploy/deploy_prod_model.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/code/deploy/deploy_prod_model.py b/code/deploy/deploy_prod_model.py index dc24e99e..b4b19cf2 100644 --- a/code/deploy/deploy_prod_model.py +++ b/code/deploy/deploy_prod_model.py @@ -1,6 +1,5 @@ from azureml.core.model import InferenceConfig from azureml.core.webservice import AksWebservice -from azureml.core.compute import ComputeTarget from azureml.core import Run, Model import argparse @@ -78,8 +77,7 @@ # cores and memory to handle this deployment configuration. # Note that memory is also used by # things such as dependencies and AML components. -aks_target = ComputeTarget._get(ws, "one-dspe-aks") -print(aks_target) +# aks_target = ComputeTarget._get(ws, "one-dspe-aks") deployment_config = AksWebservice.deploy_configuration(cpu_cores=1, memory_gb=4) service = Model.deploy(workspace=ws, @@ -87,6 +85,6 @@ models=[model], inference_config=inference_config, deployment_config=deployment_config, - deployment_target=aks_target, + deployment_target=None, overwrite=True) service.wait_for_deployment(show_output=True) From a86d78ec7f773fe1b1f8b6f682996e643f26555b Mon Sep 17 00:00:00 2001 From: evilkova Date: Fri, 22 Nov 2019 18:35:43 +0100 Subject: [PATCH 55/60] connect to aks compute target for deployment --- code/deploy/deploy_prod_model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/code/deploy/deploy_prod_model.py b/code/deploy/deploy_prod_model.py index b4b19cf2..350badc5 100644 --- a/code/deploy/deploy_prod_model.py +++ b/code/deploy/deploy_prod_model.py @@ -1,5 +1,6 @@ from azureml.core.model import InferenceConfig from azureml.core.webservice import AksWebservice +from azureml.core.compute import AksCompute from azureml.core import Run, Model import argparse @@ -77,7 +78,7 @@ # cores and memory to handle this deployment configuration. # Note that memory is also used by # things such as dependencies and AML components. -# aks_target = ComputeTarget._get(ws, "one-dspe-aks") +aks_target = AksCompute(ws, "one-dspe-aks") deployment_config = AksWebservice.deploy_configuration(cpu_cores=1, memory_gb=4) service = Model.deploy(workspace=ws, From ad400408ac3aedd8410997f03f0005fee7fa688e Mon Sep 17 00:00:00 2001 From: evilkova Date: Mon, 25 Nov 2019 14:19:44 +0100 Subject: [PATCH 56/60] get aks cluster from ws --- code/deploy/deploy_prod_model.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/code/deploy/deploy_prod_model.py b/code/deploy/deploy_prod_model.py index 350badc5..b64fad8a 100644 --- a/code/deploy/deploy_prod_model.py +++ b/code/deploy/deploy_prod_model.py @@ -1,6 +1,5 @@ from azureml.core.model import InferenceConfig from azureml.core.webservice import AksWebservice -from azureml.core.compute import AksCompute from azureml.core import Run, Model import argparse @@ -78,7 +77,12 @@ # cores and memory to handle this deployment configuration. # Note that memory is also used by # things such as dependencies and AML components. -aks_target = AksCompute(ws, "one-dspe-aks") +# aks_target = AksCompute(ws, "one-dspe-aks") + +cts = ws.compute_targets +print(cts) +aks_name = 'one-dspe-aks' +aks_target = cts[aks_name] deployment_config = AksWebservice.deploy_configuration(cpu_cores=1, memory_gb=4) service = Model.deploy(workspace=ws, @@ -86,6 +90,6 @@ models=[model], inference_config=inference_config, deployment_config=deployment_config, - deployment_target=None, + deployment_target=aks_target, overwrite=True) service.wait_for_deployment(show_output=True) From ada8fd8727a59e9a2e48cefa2c2748af9f7e4475 Mon Sep 17 00:00:00 2001 From: evilkova Date: Mon, 25 Nov 2019 15:01:55 +0100 Subject: [PATCH 57/60] set authentication credentials --- code/deploy/deploy_prod_model.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/code/deploy/deploy_prod_model.py b/code/deploy/deploy_prod_model.py index b64fad8a..b500088f 100644 --- a/code/deploy/deploy_prod_model.py +++ b/code/deploy/deploy_prod_model.py @@ -1,6 +1,8 @@ from azureml.core.model import InferenceConfig from azureml.core.webservice import AksWebservice +from azureml.core.compute import ComputeTarget from azureml.core import Run, Model +from azure.common.credentials import ServicePrincipalCredentials import argparse parser = argparse.ArgumentParser("deploy") @@ -79,10 +81,28 @@ # things such as dependencies and AML components. # aks_target = AksCompute(ws, "one-dspe-aks") -cts = ws.compute_targets -print(cts) -aks_name = 'one-dspe-aks' -aks_target = cts[aks_name] +# Tenant ID for your Azure subscription +TENANT_ID = '3a15904d-3fd9-4256-a753-beb05cdf0c6d' + +# Your service principal App ID +CLIENT = '21c8db4c-cbae-40a4-98ba-83d5414fb6bd' + +# Your service principal password +KEY = '4X0--sUXdzvFcnfG5aAZIk8kZiwGT-y]' + +credentials = ServicePrincipalCredentials( + client_id=CLIENT, + secret=KEY, + tenant=TENANT_ID +) + +# cts = ws.compute_targets +# print(cts) +# aks_name = 'one-dspe-aks' +# aks_target = cts[aks_name] + +compute_target = ComputeTarget(workspace=ws, name="one-dspe-aks") +print(compute_target) deployment_config = AksWebservice.deploy_configuration(cpu_cores=1, memory_gb=4) service = Model.deploy(workspace=ws, @@ -90,6 +110,6 @@ models=[model], inference_config=inference_config, deployment_config=deployment_config, - deployment_target=aks_target, + deployment_target=compute_target, overwrite=True) service.wait_for_deployment(show_output=True) From e96d9285730e474ed06b636ce022c003047539c2 Mon Sep 17 00:00:00 2001 From: evilkova <57631140+evilkova@users.noreply.github.com> Date: Mon, 25 Nov 2019 15:53:14 +0100 Subject: [PATCH 58/60] delete comments of getting target --- code/deploy/deploy_prod_model.py | 43 -------------------------------- 1 file changed, 43 deletions(-) diff --git a/code/deploy/deploy_prod_model.py b/code/deploy/deploy_prod_model.py index b500088f..72cbc0e6 100644 --- a/code/deploy/deploy_prod_model.py +++ b/code/deploy/deploy_prod_model.py @@ -58,49 +58,6 @@ ) ) -# reattaachemet AKS cluster -# compute_target = ComputeTarget(workspace=ws, name="dspe-aks") -# compute_target.detach() -# compute_target.wait_for_completion(show_output=True) -# print("AKS is detached") - -# to do: get config from file -# purpose = AksCompute.ClusterPurpose.DEV_TEST -# attach_config = AksCompute.attach_configuration(resource_group="sandbox-" -# "nl02328-024-rg", -# cluster_name="dspe-aks", -# cluster_purpose=purpose) -# print(attach_config) -# print("dspe-aks is attached") -# aks_target = ComputeTarget.attach(ws, "dspe-aks", attach_config) - -# If deploying to a cluster configured for dev/test, -# ensure that it was created with enough -# cores and memory to handle this deployment configuration. -# Note that memory is also used by -# things such as dependencies and AML components. -# aks_target = AksCompute(ws, "one-dspe-aks") - -# Tenant ID for your Azure subscription -TENANT_ID = '3a15904d-3fd9-4256-a753-beb05cdf0c6d' - -# Your service principal App ID -CLIENT = '21c8db4c-cbae-40a4-98ba-83d5414fb6bd' - -# Your service principal password -KEY = '4X0--sUXdzvFcnfG5aAZIk8kZiwGT-y]' - -credentials = ServicePrincipalCredentials( - client_id=CLIENT, - secret=KEY, - tenant=TENANT_ID -) - -# cts = ws.compute_targets -# print(cts) -# aks_name = 'one-dspe-aks' -# aks_target = cts[aks_name] - compute_target = ComputeTarget(workspace=ws, name="one-dspe-aks") print(compute_target) deployment_config = AksWebservice.deploy_configuration(cpu_cores=1, From 7835735117762772f37c6e55914ba9e2032c56a4 Mon Sep 17 00:00:00 2001 From: evilkova Date: Mon, 25 Nov 2019 17:05:01 +0100 Subject: [PATCH 59/60] get compute target from dotenv --- code/deploy/deploy_prod_model.py | 46 ++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/code/deploy/deploy_prod_model.py b/code/deploy/deploy_prod_model.py index 72cbc0e6..a2efaa48 100644 --- a/code/deploy/deploy_prod_model.py +++ b/code/deploy/deploy_prod_model.py @@ -1,9 +1,47 @@ from azureml.core.model import InferenceConfig from azureml.core.webservice import AksWebservice -from azureml.core.compute import ComputeTarget from azureml.core import Run, Model -from azure.common.credentials import ServicePrincipalCredentials import argparse +import os +import sys +from dotenv import load_dotenv +sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 +from workspace import get_workspace +from attach_compute import get_compute + +load_dotenv() +workspace_name = os.environ.get("BASE_NAME")+"-AML-WS" +resource_group = os.environ.get("RESOURCE_GROUP") +subscription_id = os.environ.get("SUBSCRIPTION_ID") +tenant_id = os.environ.get("TENANT_ID") +app_id = os.environ.get("SP_APP_ID") +app_secret = os.environ.get("SP_APP_SECRET") +deploy_script_path = os.environ.get("DEPLOY_PROD_SCRIPT_PATH") +vm_size = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU") +compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME") +model_name = os.environ.get("MODEL_NAME") +build_id = os.environ.get("BUILD_BUILDID") +pipeline_name = os.environ.get("DEPLOY_PROD_PIPELINE_NAME") +service_name = os.environ.get("DEPLOY_PROD_SERVICE_NAME") +sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN") + +# Get Azure machine learning workspace +aml_workspace = get_workspace( + workspace_name, + resource_group, + subscription_id, + tenant_id, + app_id, + app_secret) +print(aml_workspace) + +# Get Azure machine learning cluster +aml_compute = get_compute( + aml_workspace, + compute_name, + vm_size) +if aml_compute is not None: + print(aml_compute) parser = argparse.ArgumentParser("deploy") parser.add_argument( @@ -58,8 +96,6 @@ ) ) -compute_target = ComputeTarget(workspace=ws, name="one-dspe-aks") -print(compute_target) deployment_config = AksWebservice.deploy_configuration(cpu_cores=1, memory_gb=4) service = Model.deploy(workspace=ws, @@ -67,6 +103,6 @@ models=[model], inference_config=inference_config, deployment_config=deployment_config, - deployment_target=compute_target, + deployment_target=aml_compute, overwrite=True) service.wait_for_deployment(show_output=True) From 2daa95338246800fd9c34bdad14f9b3bdc0c212d Mon Sep 17 00:00:00 2001 From: evilkova Date: Mon, 25 Nov 2019 17:24:12 +0100 Subject: [PATCH 60/60] get compute target aks cluster --- code/deploy/deploy_prod_model.py | 44 +++----------------------------- 1 file changed, 3 insertions(+), 41 deletions(-) diff --git a/code/deploy/deploy_prod_model.py b/code/deploy/deploy_prod_model.py index a2efaa48..727aa743 100644 --- a/code/deploy/deploy_prod_model.py +++ b/code/deploy/deploy_prod_model.py @@ -1,47 +1,8 @@ from azureml.core.model import InferenceConfig from azureml.core.webservice import AksWebservice +from azureml.core.compute import ComputeTarget from azureml.core import Run, Model import argparse -import os -import sys -from dotenv import load_dotenv -sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 -from workspace import get_workspace -from attach_compute import get_compute - -load_dotenv() -workspace_name = os.environ.get("BASE_NAME")+"-AML-WS" -resource_group = os.environ.get("RESOURCE_GROUP") -subscription_id = os.environ.get("SUBSCRIPTION_ID") -tenant_id = os.environ.get("TENANT_ID") -app_id = os.environ.get("SP_APP_ID") -app_secret = os.environ.get("SP_APP_SECRET") -deploy_script_path = os.environ.get("DEPLOY_PROD_SCRIPT_PATH") -vm_size = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU") -compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME") -model_name = os.environ.get("MODEL_NAME") -build_id = os.environ.get("BUILD_BUILDID") -pipeline_name = os.environ.get("DEPLOY_PROD_PIPELINE_NAME") -service_name = os.environ.get("DEPLOY_PROD_SERVICE_NAME") -sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN") - -# Get Azure machine learning workspace -aml_workspace = get_workspace( - workspace_name, - resource_group, - subscription_id, - tenant_id, - app_id, - app_secret) -print(aml_workspace) - -# Get Azure machine learning cluster -aml_compute = get_compute( - aml_workspace, - compute_name, - vm_size) -if aml_compute is not None: - print(aml_compute) parser = argparse.ArgumentParser("deploy") parser.add_argument( @@ -96,6 +57,7 @@ ) ) +aks_target = ComputeTarget(ws, "one-dspe-aks") deployment_config = AksWebservice.deploy_configuration(cpu_cores=1, memory_gb=4) service = Model.deploy(workspace=ws, @@ -103,6 +65,6 @@ models=[model], inference_config=inference_config, deployment_config=deployment_config, - deployment_target=aml_compute, + deployment_target=aks_target, overwrite=True) service.wait_for_deployment(show_output=True)