diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..90e25cec --- /dev/null +++ b/.env.example @@ -0,0 +1,36 @@ +# Azure Subscription Variables +WORKSPACE_NAME = '' +RESOURCE_GROUP = '' +SUBSCRIPTION_ID = '' +LOCATION = '' +TENANT_ID = '' + +# Azure ML Workspace Variables +EXPERIMENT_NAME = '' +SCRIPT_FOLDER = './' +BLOB_STORE_NAME = '' +# Remote VM Config +REMOTE_VM_NAME = '' +REMOTE_VM_USERNAME = '' +REMOTE_VM_PASSWORD = '' +REMOTE_VM_IP = '' +# AML Compute Cluster Config +AML_CLUSTER_NAME = '' +AML_CLUSTER_VM_SIZE = '' +AML_CLUSTER_MAX_NODES = '' +AML_CLUSTER_MIN_NODES = '' +AML_CLUSTER_PRIORITY = 'lowpriority' +# Training Config +MODEL_NAME = '' +# AML Pipeline Config +TRAINING_PIPELINE_NAME = '' +PIPELINE_CONDA_PATH = 'aml_config/conda_dependencies.yml' +MODEL_PATH = '' +# Image config +IMAGE_NAME = '' +IMAGE_DESCRIPTION = '' +IMAGE_VERSION = '' +# ACI Config +ACI_CPU_CORES = '' +ACI_MEM_GB = '' +ACI_DESCRIPTION = '' \ No newline at end of file diff --git a/.gitignore b/.gitignore index bc6e89d1..3a5a8879 100644 --- a/.gitignore +++ b/.gitignore @@ -103,3 +103,5 @@ venv.bak/ # mypy .mypy_cache/ + +.DS_Store diff --git a/.pipelines/azdo-base-pipeline.yml b/.pipelines/azdo-base-pipeline.yml new file mode 100644 index 00000000..926b404f --- /dev/null +++ b/.pipelines/azdo-base-pipeline.yml @@ -0,0 +1,26 @@ +# this pipeline should be ignored for now +parameters: + pipelineType: 'training' + +steps: +- script: | + flake8 --output-file=$(Build.BinariesDirectory)/lint-testresults.xml --format junit-xml + workingDirectory: '$(Build.SourcesDirectory)' + displayName: 'Run code quality tests' + enabled: 'true' + +- script: | + pytest --junitxml=$(Build.BinariesDirectory)/unit-testresults.xml $(Build.SourcesDirectory)/tests/unit + displayName: 'Run unit tests' + enabled: 'true' + env: + SP_APP_SECRET: '$(SP_APP_SECRET)' + +- task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testResultsFiles: '$(Build.BinariesDirectory)/*-testresults.xml' + testRunTitle: 'Linting & Unit tests' + failTaskOnFailedTests: true + displayName: 'Publish linting and unit test results' + enabled: 'true' diff --git a/.pipelines/azdo-ci-build-train.yml b/.pipelines/azdo-ci-build-train.yml new file mode 100644 index 00000000..09f90909 --- /dev/null +++ b/.pipelines/azdo-ci-build-train.yml @@ -0,0 +1,45 @@ +pr: none +trigger: + branches: + include: + - master + +pool: + vmImage: 'ubuntu-latest' + +container: mcr.microsoft.com/mlops/python:latest + + +variables: +- group: devopsforai-aml-vg + + +steps: +- template: azdo-base-pipeline.yml + +- bash: | + # Invoke the Python building and publishing a training pipeline + python3 $(Build.SourcesDirectory)/ml_service/pipelines/build_train_pipeline.py + failOnStderr: 'false' + env: + SP_APP_SECRET: '$(SP_APP_SECRET)' + displayName: 'Train model using AML with Remote Compute' + enabled: 'true' + +- task: CopyFiles@2 + displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + Contents: | + ml_service/pipelines/?(run_train_pipeline.py|*.json) + code/scoring/** + + +- task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact' + inputs: + ArtifactName: 'mlops-pipelines' + publishLocation: 'container' + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + TargetPath: '$(Build.ArtifactStagingDirectory)' \ No newline at end of file diff --git a/.pipelines/azdo-pr-build-train.yml b/.pipelines/azdo-pr-build-train.yml new file mode 100644 index 00000000..8bf6ca56 --- /dev/null +++ b/.pipelines/azdo-pr-build-train.yml @@ -0,0 +1,18 @@ +trigger: none +pr: + branches: + include: + - master + +pool: + vmImage: 'ubuntu-latest' + +container: mcr.microsoft.com/mlops/python:latest + + +variables: +- group: devopsforai-aml-vg + + +steps: +- template: azdo-base-pipeline.yml \ No newline at end of file diff --git a/README.md b/README.md index 29ebd646..28d2a078 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,18 @@ +--- +page_type: sample +languages: +- python +products: +- azure +- azure-machine-learning-service +- azure-devops +--- + # MLOps with Azure ML [![Build Status](https://dev.azure.com/customai/DevopsForAI-AML/_apis/build/status/Microsoft.MLOpsPython?branchName=master)](https://dev.azure.com/customai/DevopsForAI-AML/_build/latest?definitionId=25&branchName=master) -### Author: Praneet Solanki | Richin Jain MLOps will help you to understand how to build the Continuous Integration and Continuous Delivery pipeline for a ML/AI project. We will be using the Azure DevOps Project for build and release/deployment pipelines along with Azure ML services for model retraining pipeline, model management and operationalization. @@ -25,20 +34,15 @@ To deploy this solution in your subscription, follow the manual instructions in This reference architecture shows how to implement continuous integration (CI), continuous delivery (CD), and retraining pipeline for an AI application using Azure DevOps and Azure Machine Learning. The solution is built on the scikit-learn diabetes dataset but can be easily adapted for any AI scenario and other popular build systems such as Jenkins and Travis. -![Architecture](/docs/images/Architecture_DevOps_AI.png) +![Architecture](/docs/images/main-flow.png) ## Architecture Flow ### Train Model 1. Data Scientist writes/updates the code and push it to git repo. This triggers the Azure DevOps build pipeline (continuous integration). -2. Once the Azure DevOps build pipeline is triggered, it runs following types of tasks: - - Run for new code: Every time new code is committed to the repo, the build pipeline performs data sanity tests and unit tests on the new code. - - One-time run: These tasks runs only for the first time the build pipeline runs. It will programatically create an [Azure ML Service Workspace](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-azure-machine-learning-architecture#workspace), provision [Azure ML Compute](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-set-up-training-targets#amlcompute) (used for model training compute), and publish an [Azure ML Pipeline](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-ml-pipelines). This published Azure ML pipeline is the model training/retraining pipeline. - - > Note: The Publish Azure ML pipeline task currently runs for every code change - -3. The Azure ML Retraining pipeline is triggered once the Azure DevOps build pipeline completes. All the tasks in this pipeline runs on Azure ML Compute created earlier. Following are the tasks in this pipeline: +2. Once the Azure DevOps build pipeline is triggered, it performs code quality checks, data sanity tests, unit tests, builds an [Azure ML Pipeline](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-ml-pipelines) and publishes it in an [Azure ML Service Workspace](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-azure-machine-learning-architecture#workspace). +3. The [Azure ML Pipeline](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-ml-pipelines) is triggered once the Azure DevOps build pipeline completes. All the tasks in this pipeline runs on Azure ML Compute. Following are the tasks in this pipeline: - **Train Model** task executes model training script on Azure ML Compute. It outputs a [model](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-azure-machine-learning-architecture#model) file which is stored in the [run history](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-azure-machine-learning-architecture#run). @@ -50,16 +54,8 @@ This reference architecture shows how to implement continuous integration (CI), Once you have registered your ML model, you can use Azure ML + Azure DevOps to deploy it. -The **Package Model** task packages the new model along with the scoring file and its python dependencies into a [docker image](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-azure-machine-learning-architecture#image) and pushes it to [Azure Container Registry](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-intro). This image is used to deploy the model as [web service](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-azure-machine-learning-architecture#web-service). - -The **Deploy Model** task handles deploying your Azure ML model to the cloud (ACI or AKS). -This pipeline deploys the model scoring image into Staging/QA and PROD environments. - - In the Staging/QA environment, one task creates an [Azure Container Instance](https://docs.microsoft.com/en-us/azure/container-instances/container-instances-overview) and deploys the scoring image as a [web service](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-azure-machine-learning-architecture#web-service) on it. - -The second task invokes the web service by calling its REST endpoint with dummy data. +[Azure DevOps release pipeline](https://docs.microsoft.com/en-us/azure/devops/pipelines/release/?view=azure-devops) packages the new model along with the scoring file and its python dependencies into a [docker image](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-azure-machine-learning-architecture#image) and pushes it to [Azure Container Registry](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-intro). This image is used to deploy the model as [web service](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-azure-machine-learning-architecture#web-service) across QA and Prod environments. The QA environment is running on top of [Azure Container Instances (ACI)](https://azure.microsoft.com/en-us/services/container-instances/) and the Prod environemt is built with [Azure Kubernetes Service (AKS)](https://docs.microsoft.com/en-us/azure/aks/intro-kubernetes). -5. The deployment in production is a [gated release](https://docs.microsoft.com/en-us/azure/devops/pipelines/release/approvals/gates?view=azure-devops). This means that once the model web service deployment in the Staging/QA environment is successful, a notification is sent to approvers to manually review and approve the release. Once the release is approved, the model scoring web service is deployed to [Azure Kubernetes Service(AKS)](https://docs.microsoft.com/en-us/azure/aks/intro-kubernetes) and the deployment is tested. ### Repo Details diff --git a/aml_config/conda_dependencies.yml b/aml_config/conda_dependencies.yml deleted file mode 100644 index 48505e28..00000000 --- a/aml_config/conda_dependencies.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Conda environment specification. The dependencies defined in this file will - -# be automatically provisioned for managed runs. These include runs against - -# the localdocker, remotedocker, and cluster compute targets. - - -# Note that this file is NOT used to automatically manage dependencies for the - -# local compute target. To provision these dependencies locally, run: - -# conda env update --file conda_dependencies.yml - - -# Details about the Conda environment file format: - -# https://conda.io/docs/using/envs.html#create-environment-file-by-hand - - -# For managing Spark packages and configuration, see spark_dependencies.yml. - - -# Version of this configuration file's structure and semantics in AzureML. - -# This directive is stored in a comment to preserve the Conda file structure. - -# [AzureMlVersion] = 2 - - -name: project_environment -dependencies: - # The python interpreter version. - - # Currently Azure ML Workbench only supports 3.5.2 and later. - -- python=3.6.2 - # Required by azureml-defaults, installed separately through Conda to - - # get a prebuilt version and not require build tools for the install. - -- psutil=5.3 - -- pip: - # Required packages for AzureML execution, history, and data preparation. - - azureml-sdk[notebooks] - - pynacl==1.2.1 - - scipy==1.0.0 - - scikit-learn==0.19.1 - - pandas==0.23.1 - - numpy==1.14.5 \ No newline at end of file diff --git a/aml_config/config.json b/aml_config/config.json deleted file mode 100644 index 7105ecf7..00000000 --- a/aml_config/config.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "subscription_id": "<>", - "resource_group": "DevOps_AzureML_Demo", - "workspace_name": "AzureML_Demo_ws", - "location": "southcentralus" -} diff --git a/aml_config/security_config.json b/aml_config/security_config.json deleted file mode 100644 index 777d0f1b..00000000 --- a/aml_config/security_config.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "sp_user" : "<>", - "sp_password" : "<>", - "sp_tenant_id" : "<>", - "remote_vm_name" : "<>", - "remote_vm_username" : "<>", - "remote_vm_password" : "<>", - "remote_vm_ip" : "<>", - "experiment_name" : "devops-ai-demo", - "aml_cluster_name" : "aml-compute", - "model_name" : "sklearn_regression_model.pkl", - "vnet_resourcegroup_name" : "<>", - "vnet_name" : "<>", - "subnet_name" : "<>" -} \ No newline at end of file diff --git a/aml_service/00-WorkSpace.py b/aml_service/00-WorkSpace.py deleted file mode 100644 index f234ed4c..00000000 --- a/aml_service/00-WorkSpace.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -Copyright (C) Microsoft Corporation. All rights reserved.​ - ​ -Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, -royalty-free right to use, copy, and modify the software code provided by us -("Software Code"). You may not sublicense the Software Code or any use of it -(except to your affiliates and to vendors to perform work on your behalf) -through distribution, network access, service agreement, lease, rental, or -otherwise. This license does not purport to express any claim of ownership over -data you may have shared with Microsoft in the creation of the Software Code. -Unless applicable law gives you more rights, Microsoft reserves all other -rights not expressly granted herein, whether by implication, estoppel or -otherwise. ​ - ​ -THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -""" -from azureml.core import Workspace -import os, json, sys -import azureml.core -from azureml.core.authentication import AzureCliAuthentication - -print("SDK Version:", azureml.core.VERSION) -# print('current dir is ' +os.curdir) -with open("aml_config/config.json") as f: - config = json.load(f) - -workspace_name = config["workspace_name"] -resource_group = config["resource_group"] -subscription_id = config["subscription_id"] -location = config["location"] - -cli_auth = AzureCliAuthentication() - -try: - ws = Workspace.get( - name=workspace_name, - subscription_id=subscription_id, - resource_group=resource_group, - auth=cli_auth, - ) - -except: - # this call might take a minute or two. - print("Creating new workspace") - ws = Workspace.create( - name=workspace_name, - subscription_id=subscription_id, - resource_group=resource_group, - # create_resource_group=True, - location=location, - auth=cli_auth, - ) - -# print Workspace details -print(ws.name, ws.resource_group, ws.location, ws.subscription_id, sep="\n") diff --git a/aml_service/01-Experiment.py b/aml_service/01-Experiment.py deleted file mode 100644 index b3543e1c..00000000 --- a/aml_service/01-Experiment.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -Copyright (C) Microsoft Corporation. All rights reserved.​ - ​ -Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, -royalty-free right to use, copy, and modify the software code provided by us -("Software Code"). You may not sublicense the Software Code or any use of it -(except to your affiliates and to vendors to perform work on your behalf) -through distribution, network access, service agreement, lease, rental, or -otherwise. This license does not purport to express any claim of ownership over -data you may have shared with Microsoft in the creation of the Software Code. -Unless applicable law gives you more rights, Microsoft reserves all other -rights not expressly granted herein, whether by implication, estoppel or -otherwise. ​ - ​ -THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -""" -import os -from azureml.core import Experiment -from azureml.core import Workspace -from azureml.core.authentication import AzureCliAuthentication - -cli_auth = AzureCliAuthentication() - - -def getExperiment(): - ws = Workspace.from_config(auth=cli_auth) - script_folder = "." - experiment_name = "devops-ai-demo" - exp = Experiment(workspace=ws, name=experiment_name) - print(exp.name, exp.workspace.name, sep="\n") - return exp - - -if __name__ == "__main__": - exp = getExperiment() diff --git a/aml_service/02-AttachTrainingVM.py b/aml_service/02-AttachTrainingVM.py deleted file mode 100644 index 3fc11c25..00000000 --- a/aml_service/02-AttachTrainingVM.py +++ /dev/null @@ -1,78 +0,0 @@ -""" -Copyright (C) Microsoft Corporation. All rights reserved.​ - ​ -Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, -royalty-free right to use, copy, and modify the software code provided by us -("Software Code"). You may not sublicense the Software Code or any use of it -(except to your affiliates and to vendors to perform work on your behalf) -through distribution, network access, service agreement, lease, rental, or -otherwise. This license does not purport to express any claim of ownership over -data you may have shared with Microsoft in the creation of the Software Code. -Unless applicable law gives you more rights, Microsoft reserves all other -rights not expressly granted herein, whether by implication, estoppel or -otherwise. ​ - ​ -THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -""" - -from azureml.core import Workspace -from azureml.core import Run -from azureml.core import Experiment -from azureml.core.conda_dependencies import CondaDependencies -from azureml.core.runconfig import RunConfiguration -import os, json -from azureml.core.compute import RemoteCompute -from azureml.core.compute import DsvmCompute -from azureml.core.compute_target import ComputeTargetException -from azureml.core.authentication import AzureCliAuthentication - -cli_auth = AzureCliAuthentication() - -# Get workspace -ws = Workspace.from_config(auth=cli_auth) - -# Read the New VM Config -with open("aml_config/security_config.json") as f: - config = json.load(f) - -remote_vm_name = config["remote_vm_name"] -remote_vm_username = config["remote_vm_username"] -remote_vm_password = config["remote_vm_password"] -remote_vm_ip = config["remote_vm_ip"] - -try: - dsvm_compute = RemoteCompute.attach( - ws, - name=remote_vm_name, - username=remote_vm_username, - address=remote_vm_ip, - ssh_port=22, - password=remote_vm_password, - ) - dsvm_compute.wait_for_completion(show_output=True) - -except Exception as e: - print("Caught = {}".format(e.message)) - print("Compute config already attached.") - - -## Create VM if not available -# compute_target_name = remote_vm_name - -# try: -# dsvm_compute = DsvmCompute(workspace=ws, name=compute_target_name) -# print('found existing:', dsvm_compute.name) -# except ComputeTargetException: -# print('creating new.') -# dsvm_config = DsvmCompute.provisioning_configuration(vm_size="Standard_D2_v2") -# dsvm_compute = DsvmCompute.create(ws, name=compute_target_name, provisioning_configuration=dsvm_config) -# dsvm_compute.wait_for_completion(show_output=True) diff --git a/aml_service/03-AttachAmlCluster.py b/aml_service/03-AttachAmlCluster.py deleted file mode 100644 index 1ba3f127..00000000 --- a/aml_service/03-AttachAmlCluster.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -Copyright (C) Microsoft Corporation. All rights reserved.​ - -Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, -royalty-free right to use, copy, and modify the software code provided by us -("Software Code"). You may not sublicense the Software Code or any use of it -(except to your affiliates and to vendors to perform work on your behalf) -through distribution, network access, service agreement, lease, rental, or -otherwise. This license does not purport to express any claim of ownership over -data you may have shared with Microsoft in the creation of the Software Code. -Unless applicable law gives you more rights, Microsoft reserves all other -rights not expressly granted herein, whether by implication, estoppel or -otherwise. ​ - -THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -""" - -from azureml.core import Workspace -from azureml.core.compute import ComputeTarget, AmlCompute -from azureml.core.compute_target import ComputeTargetException -from azureml.core.authentication import AzureCliAuthentication -import os, json - -cli_auth = AzureCliAuthentication() -# Get workspace -ws = Workspace.from_config(auth=cli_auth) - -# Read the New VM Config -with open("aml_config/security_config.json") as f: - config = json.load(f) - -aml_cluster_name = config["aml_cluster_name"] - -# un-comment the below lines if you want to put AML Compute under Vnet. Also update /aml_config/security_config.json -# vnet_resourcegroup_name = config['vnet_resourcegroup_name'] -# vnet_name = config['vnet_name'] -# subnet_name = config['subnet_name'] - -# Verify that cluster does not exist already -try: - cpu_cluster = ComputeTarget(workspace=ws, name=aml_cluster_name) - print("Found existing cluster, use it.") -except ComputeTargetException: - compute_config = AmlCompute.provisioning_configuration( - vm_size="STANDARD_D2_V2", - vm_priority="dedicated", - min_nodes=1, - max_nodes=3, - idle_seconds_before_scaledown="300", - # #Uncomment the below lines for VNet support - # vnet_resourcegroup_name=vnet_resourcegroup_name, - # vnet_name=vnet_name, - # subnet_name=subnet_name - ) - cpu_cluster = ComputeTarget.create(ws, aml_cluster_name, compute_config) - -cpu_cluster.wait_for_completion(show_output=True) diff --git a/aml_service/04-AmlPipelines.py b/aml_service/04-AmlPipelines.py deleted file mode 100644 index 44d389e5..00000000 --- a/aml_service/04-AmlPipelines.py +++ /dev/null @@ -1,211 +0,0 @@ -""" -Copyright (C) Microsoft Corporation. All rights reserved.​ - ​ -Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, -royalty-free right to use, copy, and modify the software code provided by us -("Software Code"). You may not sublicense the Software Code or any use of it -(except to your affiliates and to vendors to perform work on your behalf) -through distribution, network access, service agreement, lease, rental, or -otherwise. This license does not purport to express any claim of ownership over -data you may have shared with Microsoft in the creation of the Software Code. -Unless applicable law gives you more rights, Microsoft reserves all other -rights not expressly granted herein, whether by implication, estoppel or -otherwise. ​ - ​ -THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -""" - -import os, json, requests, datetime -import argparse -from azureml.core import Workspace, Experiment, Datastore -from azureml.core.runconfig import RunConfiguration, CondaDependencies -from azureml.data.data_reference import DataReference -from azureml.pipeline.core import Pipeline, PipelineData, StepSequence -from azureml.pipeline.steps import PythonScriptStep -from azureml.pipeline.core import PublishedPipeline -from azureml.pipeline.core.graph import PipelineParameter -from azureml.core.compute import ComputeTarget - -# from azureml.widgets import RunDetails -from azureml.core.authentication import AzureCliAuthentication - -print("Pipeline SDK-specific imports completed") - -cli_auth = AzureCliAuthentication() - - -parser = argparse.ArgumentParser("Pipeline") -parser.add_argument( - "--pipeline_action", - type=str, - choices=["pipeline-test", "publish"], - help="Determines if pipeline needs to run on small data set \ - or pipeline needs to be republished", - #default="pipeline-test", -) - -args = parser.parse_args() - - -# Get workspace -ws = Workspace.from_config(path="aml_config/config.json", auth=cli_auth) -def_blob_store = Datastore(ws, "workspaceblobstore") - -# Get AML Compute name and Experiment Name -with open("aml_config/security_config.json") as f: - config = json.load(f) - -experiment_name = config["experiment_name"] -aml_cluster_name = config["aml_cluster_name"] -aml_pipeline_name = "training-pipeline" -#model_name = config["model_name"] -model_name = PipelineParameter(name="model_name", default_value="sklearn_regression_model.pkl") - -source_directory = "code" - -# Run Config -# Declare packages dependencies required in the pipeline (these can also be expressed as a YML file) -# cd = CondaDependencies.create(pip_packages=["azureml-defaults", 'tensorflow==1.8.0']) -cd = CondaDependencies("aml_config/conda_dependencies.yml") - -run_config = RunConfiguration(conda_dependencies=cd) -aml_compute = ws.compute_targets[aml_cluster_name] -run_config.environment.docker.enabled = True -run_config.environment.spark.precache_packages = False - -jsonconfigs = PipelineData("jsonconfigs", datastore=def_blob_store) - -# Suffix for all the config files -config_suffix = datetime.datetime.now().strftime("%Y%m%d%H") -print("PipelineData object created") - -# Create python script step to run the training/scoring main script -train = PythonScriptStep( - name="Train New Model", - script_name="training/train.py", - compute_target=aml_compute, - source_directory=source_directory, - arguments=[ - "--config_suffix", config_suffix, - "--json_config", jsonconfigs, - "--model_name", model_name, - ], - runconfig=run_config, - # inputs=[jsonconfigs], - outputs=[jsonconfigs], - allow_reuse=False, -) -print("Step Train created") - -evaluate = PythonScriptStep( - name="Evaluate New Model with Prod Model", - script_name="evaluate/evaluate_model.py", - compute_target=aml_compute, - source_directory=source_directory, - arguments=[ - "--config_suffix", config_suffix, - "--json_config", jsonconfigs, - ], - runconfig=run_config, - inputs=[jsonconfigs], - # outputs=[jsonconfigs], - allow_reuse=False, -) -print("Step Evaluate created") - -register_model = PythonScriptStep( - name="Register New Trained Model", - script_name="register/register_model.py", - compute_target=aml_compute, - source_directory=source_directory, - arguments=[ - "--config_suffix", config_suffix, - "--json_config", jsonconfigs, - "--model_name", model_name, - ], - runconfig=run_config, - inputs=[jsonconfigs], - # outputs=[jsonconfigs], - allow_reuse=False, -) -print("Step register model created") - -# Package model step is moved to Azure DevOps Release Pipeline -# package_model = PythonScriptStep( -# name="Package Model as Scoring Image", -# script_name="scoring/create_scoring_image.py", -# compute_target=aml_compute, -# source_directory=source_directory, -# arguments=["--config_suffix", config_suffix, "--json_config", jsonconfigs], -# runconfig=run_config, -# inputs=[jsonconfigs], -# # outputs=[jsonconfigs], -# allow_reuse=False, -# ) -# print("Packed the model into a Scoring Image") - -# Create Steps dependency such that they run in sequence -evaluate.run_after(train) -register_model.run_after(evaluate) -#package_model.run_after(register_model) - -steps = [register_model] - - -# Build Pipeline -pipeline1 = Pipeline(workspace=ws, steps=steps) -print("Pipeline is built") - -# Validate Pipeline -pipeline1.validate() -print("Pipeline validation complete") - - -# Submit unpublished pipeline with small data set for test -if args.pipeline_action == "pipeline-test": - pipeline_run1 = Experiment(ws, experiment_name).submit( - pipeline1, regenerate_outputs=True - ) - print("Pipeline is submitted for execution") - pipeline_run1.wait_for_completion(show_output=True) - - -# RunDetails(pipeline_run1).show() - - -# Define pipeline parameters -# run_env = PipelineParameter( -# name="dev_flag", -# default_value=True) - -# dbname = PipelineParameter( -# name="dbname", -# default_value='opex') - - -# Publish Pipeline -if args.pipeline_action == "publish": - published_pipeline1 = pipeline1.publish( - name=aml_pipeline_name, description="Model training/retraining pipeline" - ) - print( - "Pipeline is published as rest_endpoint {} ".format( - published_pipeline1.endpoint - ) - ) - # write published pipeline details as build artifact - pipeline_config = {} - pipeline_config["pipeline_name"] = published_pipeline1.name - pipeline_config["rest_endpoint"] = published_pipeline1.endpoint - pipeline_config["experiment_name"] = "published-pipeline-exp" # experiment_name - with open("aml_config/pipeline_config.json", "w") as outfile: - json.dump(pipeline_config, outfile) diff --git a/aml_service/05-TriggerAmlPipeline.py b/aml_service/05-TriggerAmlPipeline.py deleted file mode 100644 index 66838a62..00000000 --- a/aml_service/05-TriggerAmlPipeline.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -Copyright (C) Microsoft Corporation. All rights reserved.​ - ​ -Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, -royalty-free right to use, copy, and modify the software code provided by us -("Software Code"). You may not sublicense the Software Code or any use of it -(except to your affiliates and to vendors to perform work on your behalf) -through distribution, network access, service agreement, lease, rental, or -otherwise. This license does not purport to express any claim of ownership over -data you may have shared with Microsoft in the creation of the Software Code. -Unless applicable law gives you more rights, Microsoft reserves all other -rights not expressly granted herein, whether by implication, estoppel or -otherwise. ​ - ​ -THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -""" - -import os, json, requests, datetime, sys -import argparse -from azureml.core.authentication import AzureCliAuthentication - -try: - with open("aml_config/pipeline_config.json") as f: - config = json.load(f) - with open("aml_config/security_config.json") as f: - security_config = json.load(f) -except: - print("No pipeline config found") - sys.exit(0) - -# Run a published pipeline -cli_auth = AzureCliAuthentication() -aad_token = cli_auth.get_authentication_header() -rest_endpoint1 = config["rest_endpoint"] -experiment_name = config["experiment_name"] -model_name = security_config["model_name"] - -print(rest_endpoint1) - -response = requests.post( - rest_endpoint1, headers=aad_token, - json={"ExperimentName": experiment_name, - "ParameterAssignments": {"model_name":model_name}} -) - -run_id = response.json()["Id"] -print(run_id) -print("Pipeline run initiated") diff --git a/aml_service/10-TrainOnLocal.py b/aml_service/10-TrainOnLocal.py deleted file mode 100644 index d7c71b3b..00000000 --- a/aml_service/10-TrainOnLocal.py +++ /dev/null @@ -1,73 +0,0 @@ -""" -Copyright (C) Microsoft Corporation. All rights reserved.​ - ​ -Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, -royalty-free right to use, copy, and modify the software code provided by us -("Software Code"). You may not sublicense the Software Code or any use of it -(except to your affiliates and to vendors to perform work on your behalf) -through distribution, network access, service agreement, lease, rental, or -otherwise. This license does not purport to express any claim of ownership over -data you may have shared with Microsoft in the creation of the Software Code. -Unless applicable law gives you more rights, Microsoft reserves all other -rights not expressly granted herein, whether by implication, estoppel or -otherwise. ​ - ​ -THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -""" - -from azureml.core.runconfig import RunConfiguration -from azureml.core import Workspace -from azureml.core import Experiment -from azureml.core import ScriptRunConfig -import json -from azureml.core.authentication import AzureCliAuthentication - -cli_auth = AzureCliAuthentication() - -# Get workspace -ws = Workspace.from_config(auth=cli_auth) - -# Attach Experiment -experiment_name = "devops-ai-demo" -exp = Experiment(workspace=ws, name=experiment_name) -print(exp.name, exp.workspace.name, sep="\n") - -# Editing a run configuration property on-fly. -run_config_user_managed = RunConfiguration() -run_config_user_managed.environment.python.user_managed_dependencies = True - -print("Submitting an experiment.") -src = ScriptRunConfig( - source_directory="./code", - script="training/train.py", - run_config=run_config_user_managed, -) -run = exp.submit(src) - -# Shows output of the run on stdout. -run.wait_for_completion(show_output=True, wait_post_processing=True) - -# Raise exception if run fails -if run.get_status() == "Failed": - raise Exception( - "Training on local failed with following run status: {} and logs: \n {}".format( - run.get_status(), run.get_details_with_logs() - ) - ) - -# Writing the run id to /aml_config/run_id.json - -run_id = {} -run_id["run_id"] = run.id -run_id["experiment_name"] = run.experiment.name -with open("aml_config/run_id.json", "w") as outfile: - json.dump(run_id, outfile) diff --git a/aml_service/11-TrainOnLocalEnv.py b/aml_service/11-TrainOnLocalEnv.py deleted file mode 100644 index 544a9d93..00000000 --- a/aml_service/11-TrainOnLocalEnv.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -Copyright (C) Microsoft Corporation. All rights reserved.​ - ​ -Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, -royalty-free right to use, copy, and modify the software code provided by us -("Software Code"). You may not sublicense the Software Code or any use of it -(except to your affiliates and to vendors to perform work on your behalf) -through distribution, network access, service agreement, lease, rental, or -otherwise. This license does not purport to express any claim of ownership over -data you may have shared with Microsoft in the creation of the Software Code. -Unless applicable law gives you more rights, Microsoft reserves all other -rights not expressly granted herein, whether by implication, estoppel or -otherwise. ​ - ​ -THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -""" -## Create a new Conda environment on local and train the model -## System-managed environment - -from azureml.core.conda_dependencies import CondaDependencies -from azureml.core.runconfig import RunConfiguration -from azureml.core import Workspace -from azureml.core import Experiment -from azureml.core import ScriptRunConfig - -from azureml.core.authentication import AzureCliAuthentication - -cli_auth = AzureCliAuthentication() - -# Get workspace -ws = Workspace.from_config(auth=cli_auth) - -# Attach Experiment -experiment_name = "devops-ai-demo" -exp = Experiment(workspace=ws, name=experiment_name) -print(exp.name, exp.workspace.name, sep="\n") - -# Editing a run configuration property on-fly. -run_config_system_managed = RunConfiguration() -# Use a new conda environment that is to be created from the conda_dependencies.yml file -run_config_system_managed.environment.python.user_managed_dependencies = False -# Automatically create the conda environment before the run -run_config_system_managed.prepare_environment = True - -# # add scikit-learn to the conda_dependencies.yml file -# Specify conda dependencies with scikit-learn -# run_config_system_managed.environment.python.conda_dependencies = CondaDependencies.create(conda_packages=['scikit-learn']) - -print("Submitting an experiment to new conda virtual env") -src = ScriptRunConfig( - source_directory="./code", - script="training/train.py", - run_config=run_config_user_managed, -) -run = exp.submit(src) - -# Shows output of the run on stdout. -run.wait_for_completion(show_output=True, wait_post_processing=True) - -# Raise exception if run fails -if run.get_status() == "Failed": - raise Exception( - "Training on local env failed with following run status: {} and logs: \n {}".format( - run.get_status(), run.get_details_with_logs() - ) - ) - -# Writing the run id to /aml_config/run_id.json -run_id = {} -run_id["run_id"] = run.id -run_id["experiment_name"] = run.experiment.name -with open("aml_config/run_id.json", "w") as outfile: - json.dump(run_id, outfile) diff --git a/aml_service/12-TrainOnVM.py b/aml_service/12-TrainOnVM.py deleted file mode 100644 index 788ffd15..00000000 --- a/aml_service/12-TrainOnVM.py +++ /dev/null @@ -1,80 +0,0 @@ -""" -Copyright (C) Microsoft Corporation. All rights reserved.​ - ​ -Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, -royalty-free right to use, copy, and modify the software code provided by us -("Software Code"). You may not sublicense the Software Code or any use of it -(except to your affiliates and to vendors to perform work on your behalf) -through distribution, network access, service agreement, lease, rental, or -otherwise. This license does not purport to express any claim of ownership over -data you may have shared with Microsoft in the creation of the Software Code. -Unless applicable law gives you more rights, Microsoft reserves all other -rights not expressly granted herein, whether by implication, estoppel or -otherwise. ​ - ​ -THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -""" -import os, json -from azureml.core import Workspace -from azureml.core import Experiment -from azureml.core.compute import RemoteCompute -from azureml.core.runconfig import RunConfiguration -from azureml.core import ScriptRunConfig -import azureml.core -from azureml.core.authentication import AzureCliAuthentication - -cli_auth = AzureCliAuthentication() -# Get workspace -ws = Workspace.from_config(auth=cli_auth) - - -# Read the New VM Config -with open("aml_config/security_config.json") as f: - config = json.load(f) -remote_vm_name = config["remote_vm_name"] - - -# Attach Experiment -experiment_name = "devops-ai-demo" -exp = Experiment(workspace=ws, name=experiment_name) -print(exp.name, exp.workspace.name, sep="\n") - -run_config = RunConfiguration() -run_config.target = remote_vm_name - -# replace with your path to the python interpreter in the remote VM found earlier -run_config.environment.python.interpreter_path = "/anaconda/envs/myenv/bin/python" -run_config.environment.python.user_managed_dependencies = True - - -src = ScriptRunConfig( - source_directory="./code", script="training/train.py", run_config=run_config -) -run = exp.submit(src) - -# Shows output of the run on stdout. -run.wait_for_completion(show_output=True, wait_post_processing=True) - -# Raise exception if run fails -if run.get_status() == "Failed": - raise Exception( - "Training on local env failed with following run status: {} and logs: \n {}".format( - run.get_status(), run.get_details_with_logs() - ) - ) - -# Writing the run id to /aml_config/run_id.json -run_id = {} -run_id["run_id"] = run.id -run_id["experiment_name"] = run.experiment.name -with open("aml_config/run_id.json", "w") as outfile: - json.dump(run_id, outfile) diff --git a/aml_service/15-EvaluateModel.py b/aml_service/15-EvaluateModel.py deleted file mode 100644 index 4d266a98..00000000 --- a/aml_service/15-EvaluateModel.py +++ /dev/null @@ -1,93 +0,0 @@ -""" -Copyright (C) Microsoft Corporation. All rights reserved.​ - ​ -Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, -royalty-free right to use, copy, and modify the software code provided by us -("Software Code"). You may not sublicense the Software Code or any use of it -(except to your affiliates and to vendors to perform work on your behalf) -through distribution, network access, service agreement, lease, rental, or -otherwise. This license does not purport to express any claim of ownership over -data you may have shared with Microsoft in the creation of the Software Code. -Unless applicable law gives you more rights, Microsoft reserves all other -rights not expressly granted herein, whether by implication, estoppel or -otherwise. ​ - ​ -THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -""" -import os, json -from azureml.core import Workspace -from azureml.core import Experiment -from azureml.core.model import Model -import azureml.core -from azureml.core import Run -from azureml.core.authentication import AzureCliAuthentication - -cli_auth = AzureCliAuthentication() - -# Get workspace -ws = Workspace.from_config(auth=cli_auth) - -# Paramaterize the matrics on which the models should be compared - -# Add golden data set on which all the model performance can be evaluated - -# Get the latest run_id -with open("aml_config/run_id.json") as f: - config = json.load(f) - -new_model_run_id = config["run_id"] -experiment_name = config["experiment_name"] -exp = Experiment(workspace=ws, name=experiment_name) - - -try: - # Get most recently registered model, we assume that is the model in production. Download this model and compare it with the recently trained model by running test with same data set. - model_list = Model.list(ws) - production_model = next( - filter( - lambda x: x.created_time == max(model.created_time for model in model_list), - model_list, - ) - ) - production_model_run_id = production_model.tags.get("run_id") - run_list = exp.get_runs() - # production_model_run = next(filter(lambda x: x.id == production_model_run_id, run_list)) - - # Get the run history for both production model and newly trained model and compare mse - production_model_run = Run(exp, run_id=production_model_run_id) - new_model_run = Run(exp, run_id=new_model_run_id) - - production_model_mse = production_model_run.get_metrics().get("mse") - new_model_mse = new_model_run.get_metrics().get("mse") - print( - "Current Production model mse: {}, New trained model mse: {}".format( - production_model_mse, new_model_mse - ) - ) - - promote_new_model = False - if new_model_mse < production_model_mse: - promote_new_model = True - print("New trained model performs better, thus it will be registered") -except: - promote_new_model = True - print("This is the first model to be trained, thus nothing to evaluate for now") - -run_id = {} -run_id["run_id"] = "" -# Writing the run id to /aml_config/run_id.json -if promote_new_model: - run_id["run_id"] = new_model_run_id - -run_id["experiment_name"] = experiment_name -with open("aml_config/run_id.json", "w") as outfile: - json.dump(run_id, outfile) diff --git a/aml_service/20-RegisterModel.py b/aml_service/20-RegisterModel.py deleted file mode 100644 index bd9a7bbc..00000000 --- a/aml_service/20-RegisterModel.py +++ /dev/null @@ -1,92 +0,0 @@ -""" -Copyright (C) Microsoft Corporation. All rights reserved.​ - ​ -Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, -royalty-free right to use, copy, and modify the software code provided by us -("Software Code"). You may not sublicense the Software Code or any use of it -(except to your affiliates and to vendors to perform work on your behalf) -through distribution, network access, service agreement, lease, rental, or -otherwise. This license does not purport to express any claim of ownership over -data you may have shared with Microsoft in the creation of the Software Code. -Unless applicable law gives you more rights, Microsoft reserves all other -rights not expressly granted herein, whether by implication, estoppel or -otherwise. ​ - ​ -THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -""" -import os, json, sys -from azureml.core import Workspace -from azureml.core import Run -from azureml.core import Experiment -from azureml.core.model import Model - -from azureml.core.runconfig import RunConfiguration -from azureml.core.authentication import AzureCliAuthentication - -cli_auth = AzureCliAuthentication() - -# Get workspace -ws = Workspace.from_config(auth=cli_auth) - -# Get the latest evaluation result -try: - with open("aml_config/run_id.json") as f: - config = json.load(f) - if not config["run_id"]: - raise Exception("No new model to register as production model perform better") -except: - print("No new model to register as production model perform better") - # raise Exception('No new model to register as production model perform better') - sys.exit(0) - -run_id = config["run_id"] -experiment_name = config["experiment_name"] -exp = Experiment(workspace=ws, name=experiment_name) - -run = Run(experiment=exp, run_id=run_id) -names = run.get_file_names -names() -print("Run ID for last run: {}".format(run_id)) -model_local_dir = "model" -os.makedirs(model_local_dir, exist_ok=True) - -# Download Model to Project root directory -model_name = "sklearn_regression_model.pkl" -run.download_file( - name="./outputs/" + model_name, output_file_path="./model/" + model_name -) -print("Downloaded model {} to Project root directory".format(model_name)) -os.chdir("./model") -model = Model.register( - model_path=model_name, # this points to a local file - model_name=model_name, # this is the name the model is registered as - tags={"area": "diabetes", "type": "regression", "run_id": run_id}, - description="Regression model for diabetes dataset", - workspace=ws, -) -os.chdir("..") -print( - "Model registered: {} \nModel Description: {} \nModel Version: {}".format( - model.name, model.description, model.version - ) -) - -# Remove the evaluate.json as we no longer need it -# os.remove("aml_config/evaluate.json") - -# Writing the registered model details to /aml_config/model.json -model_json = {} -model_json["model_name"] = model.name -model_json["model_version"] = model.version -model_json["run_id"] = run_id -with open("aml_config/model.json", "w") as outfile: - json.dump(model_json, outfile) diff --git a/aml_service/30-CreateScoringImage.py b/aml_service/30-CreateScoringImage.py deleted file mode 100644 index 4c7597e1..00000000 --- a/aml_service/30-CreateScoringImage.py +++ /dev/null @@ -1,111 +0,0 @@ -""" -Copyright (C) Microsoft Corporation. All rights reserved.​ - ​ -Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, -royalty-free right to use, copy, and modify the software code provided by us -("Software Code"). You may not sublicense the Software Code or any use of it -(except to your affiliates and to vendors to perform work on your behalf) -through distribution, network access, service agreement, lease, rental, or -otherwise. This license does not purport to express any claim of ownership over -data you may have shared with Microsoft in the creation of the Software Code. -Unless applicable law gives you more rights, Microsoft reserves all other -rights not expressly granted herein, whether by implication, estoppel or -otherwise. ​ - ​ -THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -""" -import os, json, sys -from azureml.core import Workspace -from azureml.core.image import ContainerImage, Image -from azureml.core.model import Model -from azureml.core.authentication import AzureCliAuthentication - -cli_auth = AzureCliAuthentication() - -# Get workspace -ws = Workspace.from_config(auth=cli_auth) - -# Get the latest model details - -# try: -# with open("aml_config/model.json") as f: -# config = json.load(f) -# except: -# print("No new model to register thus no need to create new scoring image") -# # raise Exception('No new model to register as production model perform better') -# sys.exit(0) - -# model_name = config["model_name"] -# model_version = config["model_version"] - - -# model_list = Model.list(workspace=ws) -# model, = (m for m in model_list if m.version == model_version and m.name == model_name) -# print( -# "Model picked: {} \nModel Description: {} \nModel Version: {}".format( -# model.name, model.description, model.version -# ) -# ) - -try: - with open("aml_config/security_config.json") as f: - security_config = json.load(f) -except: - print("No Security Config found") - sys.exit(0) - -# Run a published pipeline -#model_name = "sklearn_regression_model.pkl" -model_name = security_config["model_name"] -model = Model(ws, name=model_name) - -os.chdir("./code/scoring") -image_name = "diabetes-model-score" - -image_config = ContainerImage.image_configuration( - execution_script="score.py", - runtime="python", - conda_file="conda_dependencies.yml", - description="Image with ridge regression model", - tags={"area": "diabetes", "type": "regression"}, -) - -image = Image.create( - name=image_name, models=[model], image_config=image_config, workspace=ws -) - -image.wait_for_creation(show_output=True) -os.chdir("../..") - -if image.creation_state != "Succeeded": - raise Exception("Image creation status: {image.creation_state}") - -print( - "{}(v.{} [{}]) stored at {} with build log {}".format( - image.name, - image.version, - image.creation_state, - image.image_location, - image.image_build_log_uri, - ) -) - -# Writing the image details to /aml_config/image.json -image_json = {} -image_json["image_name"] = image.name -image_json["image_version"] = image.version -image_json["image_location"] = image.image_location -with open("aml_config/image.json", "w") as outfile: - json.dump(image_json, outfile) - - -# How to fix the schema for a model, like if we have multiple models expecting different schema, diff --git a/aml_service/34-GetScoringImageName.py b/aml_service/34-GetScoringImageName.py deleted file mode 100644 index b5f3a764..00000000 --- a/aml_service/34-GetScoringImageName.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -Copyright (C) Microsoft Corporation. All rights reserved.​ - ​ -Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, -royalty-free right to use, copy, and modify the software code provided by us -("Software Code"). You may not sublicense the Software Code or any use of it -(except to your affiliates and to vendors to perform work on your behalf) -through distribution, network access, service agreement, lease, rental, or -otherwise. This license does not purport to express any claim of ownership over -data you may have shared with Microsoft in the creation of the Software Code. -Unless applicable law gives you more rights, Microsoft reserves all other -rights not expressly granted herein, whether by implication, estoppel or -otherwise. ​ - ​ -THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -""" -import os, json, sys -from azureml.core import Workspace -from azureml.core.authentication import AzureCliAuthentication - -cli_auth = AzureCliAuthentication() - -# Get workspace -ws = Workspace.from_config(auth=cli_auth) - -# Get the latest image details -latest_image = ws.images -name, version = latest_image.get(list(latest_image)[0]).id.split(':') - -# Writing the image details to /aml_config/image.json -image_json = {} -image_json["image_name"] = name -image_json["image_version"] = int(version) -with open("aml_config/image.json", "w") as outfile: - json.dump(image_json, outfile) diff --git a/aml_service/50-deployOnAci.py b/aml_service/50-deployOnAci.py deleted file mode 100644 index 00313380..00000000 --- a/aml_service/50-deployOnAci.py +++ /dev/null @@ -1,88 +0,0 @@ -""" -Copyright (C) Microsoft Corporation. All rights reserved.​ - ​ -Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, -royalty-free right to use, copy, and modify the software code provided by us -("Software Code"). You may not sublicense the Software Code or any use of it -(except to your affiliates and to vendors to perform work on your behalf) -through distribution, network access, service agreement, lease, rental, or -otherwise. This license does not purport to express any claim of ownership over -data you may have shared with Microsoft in the creation of the Software Code. -Unless applicable law gives you more rights, Microsoft reserves all other -rights not expressly granted herein, whether by implication, estoppel or -otherwise. ​ - ​ -THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -""" -import os, json, datetime, sys -from operator import attrgetter -from azureml.core import Workspace -from azureml.core.model import Model -from azureml.core.image import Image -from azureml.core.webservice import Webservice -from azureml.core.webservice import AciWebservice -from azureml.core.authentication import AzureCliAuthentication - -cli_auth = AzureCliAuthentication() -# Get workspace -ws = Workspace.from_config(auth=cli_auth) # Get the Image to deploy details -try: - with open("aml_config/image.json") as f: - config = json.load(f) -except: - print("No new model, thus no deployment on ACI") - # raise Exception('No new model to register as production model perform better') - sys.exit(0) - - -image_name = config["image_name"] -image_version = config["image_version"] - -images = Image.list(workspace=ws) -image, = (m for m in images if m.version == image_version and m.name == image_name) -print( - "From image.json, Image used to deploy webservice on ACI: {}\nImage Version: {}\nImage Location = {}".format( - image.name, image.version, image.image_location - ) -) - -# image = max(images, key=attrgetter('version')) -# print('From Max Version, Image used to deploy webservice on ACI: {}\nImage Version: {}\nImage Location = {}'.format(image.name, image.version, image.image_location)) - - -aciconfig = AciWebservice.deploy_configuration( - cpu_cores=1, - memory_gb=1, - tags={"area": "diabetes", "type": "regression"}, - description="A sample description", -) - -aci_service_name = "aciwebservice" + datetime.datetime.now().strftime("%m%d%H") - -service = Webservice.deploy_from_image( - deployment_config=aciconfig, image=image, name=aci_service_name, workspace=ws -) - -service.wait_for_deployment() -print( - "Deployed ACI Webservice: {} \nWebservice Uri: {}".format( - service.name, service.scoring_uri - ) -) - -# service=Webservice(name ='aciws0622', workspace =ws) -# Writing the ACI details to /aml_config/aci_webservice.json -aci_webservice = {} -aci_webservice["aci_name"] = service.name -aci_webservice["aci_url"] = service.scoring_uri -with open("aml_config/aci_webservice.json", "w") as outfile: - json.dump(aci_webservice, outfile) diff --git a/aml_service/51-deployOnAks.py b/aml_service/51-deployOnAks.py deleted file mode 100644 index 379ea90c..00000000 --- a/aml_service/51-deployOnAks.py +++ /dev/null @@ -1,124 +0,0 @@ -""" -Copyright (C) Microsoft Corporation. All rights reserved.​ - ​ -Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, -royalty-free right to use, copy, and modify the software code provided by us -("Software Code"). You may not sublicense the Software Code or any use of it -(except to your affiliates and to vendors to perform work on your behalf) -through distribution, network access, service agreement, lease, rental, or -otherwise. This license does not purport to express any claim of ownership over -data you may have shared with Microsoft in the creation of the Software Code. -Unless applicable law gives you more rights, Microsoft reserves all other -rights not expressly granted herein, whether by implication, estoppel or -otherwise. ​ - ​ -THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -""" -import os, json, datetime, sys -from operator import attrgetter -from azureml.core import Workspace -from azureml.core.model import Model -from azureml.core.image import Image -from azureml.core.compute import AksCompute, ComputeTarget -from azureml.core.webservice import Webservice, AksWebservice -from azureml.core.authentication import AzureCliAuthentication - -cli_auth = AzureCliAuthentication() -# Get workspace -ws = Workspace.from_config(auth=cli_auth) - -# Get the Image to deploy details -try: - with open("aml_config/image.json") as f: - config = json.load(f) -except: - print("No new model, thus no deployment on ACI") - # raise Exception('No new model to register as production model perform better') - sys.exit(0) - -image_name = config["image_name"] -image_version = config["image_version"] - -images = Image.list(workspace=ws) -image, = (m for m in images if m.version == image_version and m.name == image_name) -print( - "From image.json, Image used to deploy webservice on ACI: {}\nImage Version: {}\nImage Location = {}".format( - image.name, image.version, image.image_location - ) -) - -# image = max(images, key=attrgetter('version')) -# print('From Max Version, Image used to deploy webservice on ACI: {}\nImage Version: {}\nImage Location = {}'.format(image.name, image.version, image.image_location)) - -# Check if AKS already Available -try: - with open("aml_config/aks_webservice.json") as f: - config = json.load(f) - aks_name = config["aks_name"] - aks_service_name = config["aks_service_name"] - compute_list = ws.compute_targets() - aks_target, = (c for c in compute_list if c.name == aks_name) - service = Webservice(name=aks_service_name, workspace=ws) - print( - "Updating AKS service {} with image: {}".format( - aks_service_name, image.image_location - ) - ) - service.update(image=image) -except: - aks_name = "aks" + datetime.datetime.now().strftime("%m%d%H") - aks_service_name = "akswebservice" + datetime.datetime.now().strftime("%m%d%H") - prov_config = AksCompute.provisioning_configuration( - agent_count=6, vm_size="Standard_F4s", location="eastus" - ) - print( - "No AKS found in aks_webservice.json. Creating new Aks: {} and AKS Webservice: {}".format( - aks_name, aks_service_name - ) - ) - # Create the cluster - aks_target = ComputeTarget.create( - workspace=ws, name=aks_name, provisioning_configuration=prov_config - ) - - aks_target.wait_for_completion(show_output=True) - print(aks_target.provisioning_state) - print(aks_target.provisioning_errors) - - # Use the default configuration (can also provide parameters to customize) - aks_config = AksWebservice.deploy_configuration(enable_app_insights=True) - - service = Webservice.deploy_from_image( - workspace=ws, - name=aks_service_name, - image=image, - deployment_config=aks_config, - deployment_target=aks_target, - ) - - service.wait_for_deployment(show_output=True) - print(service.state) - print( - "Deployed AKS Webservice: {} \nWebservice Uri: {}".format( - service.name, service.scoring_uri - ) - ) - - -# Writing the AKS details to /aml_config/aks_webservice.json -aks_webservice = {} -aks_webservice["aks_name"] = aks_name -aks_webservice["aks_service_name"] = service.name -aks_webservice["aks_url"] = service.scoring_uri -aks_webservice["aks_keys"] = service.get_keys() -with open("aml_config/aks_webservice.json", "w") as outfile: - json.dump(aks_webservice, outfile) diff --git a/aml_service/60-AciWebserviceTest.py b/aml_service/60-AciWebserviceTest.py deleted file mode 100644 index a8c40f69..00000000 --- a/aml_service/60-AciWebserviceTest.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -Copyright (C) Microsoft Corporation. All rights reserved.​ - ​ -Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, -royalty-free right to use, copy, and modify the software code provided by us -("Software Code"). You may not sublicense the Software Code or any use of it -(except to your affiliates and to vendors to perform work on your behalf) -through distribution, network access, service agreement, lease, rental, or -otherwise. This license does not purport to express any claim of ownership over -data you may have shared with Microsoft in the creation of the Software Code. -Unless applicable law gives you more rights, Microsoft reserves all other -rights not expressly granted herein, whether by implication, estoppel or -otherwise. ​ - ​ -THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -""" -import numpy -import os, json, datetime, sys -from operator import attrgetter -from azureml.core import Workspace -from azureml.core.model import Model -from azureml.core.image import Image -from azureml.core.webservice import Webservice -from azureml.core.webservice import AciWebservice -from azureml.core.authentication import AzureCliAuthentication - -cli_auth = AzureCliAuthentication() -# Get workspace -ws = Workspace.from_config(auth=cli_auth) -# Get the ACI Details -try: - with open("aml_config/aci_webservice.json") as f: - config = json.load(f) -except: - print("No new model, thus no deployment on ACI") - # raise Exception('No new model to register as production model perform better') - sys.exit(0) - -service_name = config["aci_name"] -# Get the hosted web service -service = Webservice(name=service_name, workspace=ws) - -# Input for Model with all features -input_j = [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [10, 9, 8, 7, 6, 5, 4, 3, 2, 1]] -print(input_j) -test_sample = json.dumps({"data": input_j}) -test_sample = bytes(test_sample, encoding="utf8") -try: - prediction = service.run(input_data=test_sample) - print(prediction) -except Exception as e: - result = str(e) - print(result) - raise Exception("ACI service is not working as expected") diff --git a/aml_service/61-AksWebserviceTest.py b/aml_service/61-AksWebserviceTest.py deleted file mode 100644 index f22982e0..00000000 --- a/aml_service/61-AksWebserviceTest.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -Copyright (C) Microsoft Corporation. All rights reserved.​ - ​ -Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, -royalty-free right to use, copy, and modify the software code provided by us -("Software Code"). You may not sublicense the Software Code or any use of it -(except to your affiliates and to vendors to perform work on your behalf) -through distribution, network access, service agreement, lease, rental, or -otherwise. This license does not purport to express any claim of ownership over -data you may have shared with Microsoft in the creation of the Software Code. -Unless applicable law gives you more rights, Microsoft reserves all other -rights not expressly granted herein, whether by implication, estoppel or -otherwise. ​ - ​ -THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -""" -import numpy -import os, json, datetime, sys -from operator import attrgetter -from azureml.core import Workspace -from azureml.core.model import Model -from azureml.core.image import Image -from azureml.core.webservice import Webservice -from azureml.core.authentication import AzureCliAuthentication - -cli_auth = AzureCliAuthentication() -# Get workspace -ws = Workspace.from_config(auth=cli_auth) - -# Get the AKS Details -try: - with open("aml_config/aks_webservice.json") as f: - config = json.load(f) -except: - print("No new model, thus no deployment on ACI") - # raise Exception('No new model to register as production model perform better') - sys.exit(0) - -service_name = config["aks_service_name"] -# Get the hosted web service -service = Webservice(workspace=ws, name=service_name) - -# Input for Model with all features -input_j = [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [10, 9, 8, 7, 6, 5, 4, 3, 2, 1]] -print(input_j) -test_sample = json.dumps({"data": input_j}) -test_sample = bytes(test_sample, encoding="utf8") -try: - prediction = service.run(input_data=test_sample) - print(prediction) -except Exception as e: - result = str(e) - print(result) - raise Exception("AKS service is not working as expected") - -# Delete aci after test -service.delete() diff --git a/aml_service/helper/azcli.py b/aml_service/helper/azcli.py deleted file mode 100644 index 4affc1b3..00000000 --- a/aml_service/helper/azcli.py +++ /dev/null @@ -1,73 +0,0 @@ -""" -Copyright (C) Microsoft Corporation. All rights reserved.​ - ​ -Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, -royalty-free right to use, copy, and modify the software code provided by us -("Software Code"). You may not sublicense the Software Code or any use of it -(except to your affiliates and to vendors to perform work on your behalf) -through distribution, network access, service agreement, lease, rental, or -otherwise. This license does not purport to express any claim of ownership over -data you may have shared with Microsoft in the creation of the Software Code. -Unless applicable law gives you more rights, Microsoft reserves all other -rights not expressly granted herein, whether by implication, estoppel or -otherwise. ​ - ​ -THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -""" -import subprocess - - -def az_login(sp_user: str, sp_password: str, sp_tenant_id: str): - """ - Uses the provided service principal credentials to log into the azure cli. - This should always be the first step in executing az cli commands. - """ - cmd = "az login --service-principal --username {} --password {} --tenant {}" - out, err = run_cmd(cmd.format(sp_user, sp_password, sp_tenant_id)) - return out, err - - -def run_cmd(cmd: str): - """ - Runs an arbitrary command line command. Works for Linux or Windows. - Returns a tuple of output and error. - """ - proc = subprocess.Popen( - cmd, shell=True, stdout=subprocess.PIPE, universal_newlines=True - ) - output, error = proc.communicate() - if proc.returncode != 0: - print("Following command execution failed: {}".format(cmd)) - raise Exception("Operation Failed. Look at console logs for error info") - return output, error - - -def az_account_set(subscription_id: str): - """ - Sets the correct azure subscription. - This should always be run after the az_login. - """ - cmd = "az account set -s {}" - out, err = run_cmd(cmd.format(subscription_id)) - return out, err - - -def az_acr_create(resource_group: str, acr_name: str): - cmd = "az acr create --resource-group {} --name {} --sku Basic" - out, err = run_cmd(cmd.format(resource_group, acr_name)) - return out, err - - -def az_acr_login(acr_name: str): - cmd = "az acr login --name {}" - out, err = run_cmd(cmd.format(acr_name)) - return out, err diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index c3815408..00000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,68 +0,0 @@ -pool: - vmImage: 'Ubuntu 16.04' -#Your build pipeline references a secret variable named ‘sp_username’. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it secret. See https://go.microsoft.com/fwlink/?linkid=865972 -#Your build pipeline references a secret variable named ‘sp_password’. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it secret. See https://go.microsoft.com/fwlink/?linkid=865972 -#Your build pipeline references a secret variable named ‘sp_tenantid’. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it secret. See https://go.microsoft.com/fwlink/?linkid=865972 -#Your build pipeline references a secret variable named ‘subscription_id’. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it secret. See https://go.microsoft.com/fwlink/?linkid=865972 - -variables: -- group: AzureKeyVaultSecrets - -trigger: -- master -- releases/* -- develop - -steps: -- task: UsePythonVersion@0 - inputs: - versionSpec: '3.6' - architecture: 'x64' - -- task: Bash@3 - displayName: 'Install Requirements' - inputs: - targetType: filePath - filePath: 'environment_setup/install_requirements.sh' - workingDirectory: 'environment_setup' - -- script: | - az login --service-principal -u $(spidentity) -p $(spsecret) --tenant $(sptenant) - - displayName: 'Login to Azure' - -- script: | - sed -i 's#"subscription_id": "<>"#"subscription_id": "$(subscriptionid)"#g' aml_config/config.json - - displayName: 'replace subscription value' - -- script: 'pytest tests/unit/data_test.py' - displayName: 'Data Quality Check' - -- script: 'python aml_service/00-WorkSpace.py' - displayName: 'Get or Create Workspace' - -- script: 'python aml_service/03-AttachAmlCluster.py' - displayName: 'Create AML Compute Cluster' - -- script: 'python aml_service/04-AmlPipelines.py' - displayName: 'Create and Test AML Pipeline' - -- script: 'python aml_service/04-AmlPipelines.py --pipeline_action publish' - displayName: 'Publish AML Pipeline as Endpoint' - -- task: CopyFiles@2 - displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)' - inputs: - SourceFolder: '$(Build.SourcesDirectory)' - TargetFolder: '$(Build.ArtifactStagingDirectory)' - Contents: '**' - -- task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: devops-for-ai' - inputs: - ArtifactName: 'devops-for-ai' - publishLocation: 'container' - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - TargetPath: '$(Build.ArtifactStagingDirectory)' - diff --git a/code/evaluate/evaluate_model.py b/code/evaluate/evaluate_model.py index d3ba5af0..02e048b6 100644 --- a/code/evaluate/evaluate_model.py +++ b/code/evaluate/evaluate_model.py @@ -23,11 +23,9 @@ ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ -import os, json -from azureml.core import Workspace -from azureml.core import Experiment +import os +import json from azureml.core.model import Model -import azureml.core from azureml.core import Run import argparse @@ -68,10 +66,6 @@ with open(train_output_path) as f: config = json.load(f) -# parser = argparse.ArgumentParser() -# parser.add_argument('--train_run_id',type=str,default='',help='Run id of the newly trained model') -# #parser.add_argument('--model_assets_path',type=str,default='outputs',help='Location of trained model.') - new_model_run_id = config["run_id"] # args.train_run_id experiment_name = config["experiment_name"] @@ -79,19 +73,23 @@ try: - # Get most recently registered model, we assume that is the model in production. Download this model and compare it with the recently trained model by running test with same data set. + # Get most recently registered model, we assume that + # is the model in production. + # Download this model and compare it with the recently + # trained model by running test with same data set. model_list = Model.list(ws) production_model = next( filter( - lambda x: x.created_time == max(model.created_time for model in model_list), + lambda x: x.created_time == max( + model.created_time for model in model_list), model_list, ) ) production_model_run_id = production_model.tags.get("run_id") run_list = exp.get_runs() - # production_model_run = next(filter(lambda x: x.id == production_model_run_id, run_list)) - # Get the run history for both production model and newly trained model and compare mse + # Get the run history for both production model and + # newly trained model and compare mse production_model_run = Run(exp, run_id=production_model_run_id) new_model_run = Run(exp, run_id=new_model_run_id) @@ -107,9 +105,10 @@ if new_model_mse < production_model_mse: promote_new_model = True print("New trained model performs better, thus it will be registered") -except: +except Exception: promote_new_model = True - print("This is the first model to be trained, thus nothing to evaluate for now") + print("This is the first model to be trained, \ + thus nothing to evaluate for now") run_id = {} run_id["run_id"] = "" diff --git a/code/register/register_model.py b/code/register/register_model.py index c7d38b82..05f469b7 100644 --- a/code/register/register_model.py +++ b/code/register/register_model.py @@ -23,20 +23,21 @@ ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ -import os, json, sys -from azureml.core import Workspace +import os +import json +import sys from azureml.core import Run -from azureml.core import Experiment from azureml.core.model import Model import argparse -from azureml.core.runconfig import RunConfiguration from azureml.core.authentication import AzureCliAuthentication cli_auth = AzureCliAuthentication() # Get workspace -# ws = Workspace.from_config(auth=cli_auth) +# ws = Workspace.from_config(auth=cli_auth, path='./') + + run = Run.get_context() exp = run.experiment ws = run.experiment.workspace @@ -75,10 +76,10 @@ with open(evaluate_output_path) as f: config = json.load(f) if not config["run_id"]: - raise Exception("No new model to register as production model perform better") -except: + raise Exception( + "No new model to register as production model perform better") +except Exception: print("No new model to register as production model perform better") - # raise Exception('No new model to register as production model perform better') sys.exit(0) run_id = config["run_id"] diff --git a/code/scoring/conda_dependencies.yml b/code/scoring/conda_dependencies.yml index 9bca0710..f13c3c3d 100644 --- a/code/scoring/conda_dependencies.yml +++ b/code/scoring/conda_dependencies.yml @@ -44,6 +44,11 @@ dependencies: # Required packages for AzureML execution, history, and data preparation. - azureml-sdk[notebooks] # add the version to lock it ==0.1.74 - scipy==1.0.0 - - scikit-learn==0.19.1 + - scikit-learn==0.21.3 - pandas==0.23.1 - - numpy==1.14.5 \ No newline at end of file + - numpy==1.14.5 + - joblib==0.13.2 + - gunicorn==19.9.0 + - flask==1.1.1 + - azure-ml-api-sdk + diff --git a/code/scoring/create_scoring_image.py b/code/scoring/create_scoring_image.py deleted file mode 100644 index 1aafade1..00000000 --- a/code/scoring/create_scoring_image.py +++ /dev/null @@ -1,124 +0,0 @@ -""" -Copyright (C) Microsoft Corporation. All rights reserved.​ - ​ -Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, -royalty-free right to use, copy, and modify the software code provided by us -("Software Code"). You may not sublicense the Software Code or any use of it -(except to your affiliates and to vendors to perform work on your behalf) -through distribution, network access, service agreement, lease, rental, or -otherwise. This license does not purport to express any claim of ownership over -data you may have shared with Microsoft in the creation of the Software Code. -Unless applicable law gives you more rights, Microsoft reserves all other -rights not expressly granted herein, whether by implication, estoppel or -otherwise. ​ - ​ -THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -""" -import os, json, sys -import argparse -from azureml.core import Workspace -from azureml.core.image import ContainerImage, Image -from azureml.core import Run -from azureml.core.model import Model -from azureml.core.authentication import AzureCliAuthentication - -cli_auth = AzureCliAuthentication() - -run = Run.get_context() -if "OfflineRun" in run.id: - print("offline run") - # Get workspace - ws = Workspace.from_config(auth=cli_auth) -else: - exp = run.experiment - ws = run.experiment.workspace - -# Get the latest model details - -parser = argparse.ArgumentParser("scoring_image") -parser.add_argument( - "--config_suffix", type=str, help="Datetime suffix for json config files" -) -parser.add_argument( - "--json_config", - type=str, - help="Directory to write all the intermediate json configs", -) -args = parser.parse_args() - -register_model_json = "model_{}.json".format(args.config_suffix) -register_output_path = os.path.join(args.json_config, register_model_json) - - -try: - with open(register_output_path) as f: - config = json.load(f) -except: - print("No new model to register thus no need to create new scoring image") - # raise Exception('No new model to register as production model perform better') - sys.exit(0) - -model_name = config["model_name"] -model_version = config["model_version"] - -model_list = Model.list(workspace=ws) -model, = (m for m in model_list if m.version == model_version and m.name == model_name) -print( - "Model picked: {} \nModel Description: {} \nModel Version: {}".format( - model.name, model.description, model.version - ) -) - -os.chdir("scoring") -image_name = "diabetes-model-score" - -image_config = ContainerImage.image_configuration( - execution_script="score.py", - runtime="python-slim", - conda_file="conda_dependencies.yml", - description="Image with ridge regression model", - tags={"area": "diabetes", "type": "regression"}, -) - -image = Image.create( - name=image_name, models=[model], image_config=image_config, workspace=ws -) - -image.wait_for_creation(show_output=True) -os.chdir("..") - -if image.creation_state != "Succeeded": - raise Exception("Image creation status: {image.creation_state}") - -print( - "{}(v.{} [{}]) stored at {} with build log {}".format( - image.name, - image.version, - image.creation_state, - image.image_location, - image.image_build_log_uri, - ) -) - -# Writing the image details to /aml_config/image.json -image_json = {} -image_json["image_name"] = image.name -image_json["image_version"] = image.version -image_json["image_location"] = image.image_location -# with open("aml_config/image.json", "w") as outfile: -# json.dump(image_json, outfile) -filename = "image_{}.json".format(args.config_suffix) -output_path = os.path.join(args.json_config, filename) -with open(output_path, "w") as outfile: - json.dump(image_json, outfile) - -# How to fix the schema for a model, like if we have multiple models expecting different schema, diff --git a/code/scoring/deployment_config_aci.yml b/code/scoring/deployment_config_aci.yml new file mode 100644 index 00000000..939483b5 --- /dev/null +++ b/code/scoring/deployment_config_aci.yml @@ -0,0 +1,5 @@ +--- +containerResourceRequirements: + cpu: 1 + memoryInGB: 4 +computeType: ACI \ No newline at end of file diff --git a/code/scoring/deployment_config_aks.yml b/code/scoring/deployment_config_aks.yml new file mode 100644 index 00000000..5cc78847 --- /dev/null +++ b/code/scoring/deployment_config_aks.yml @@ -0,0 +1,16 @@ +computeType: AKS +autoScaler: + autoscaleEnabled: True + minReplicas: 1 + maxReplicas: 3 + refreshPeriodInSeconds: 10 + targetUtilization: 70 +authEnabled: True +containerResourceRequirements: + cpu: 1 + memoryInGB: 4 +appInsightsEnabled: False +scoringTimeoutMs: 5000 +maxConcurrentRequestsPerContainer: 2 +maxQueueWaitMs: 5000 +sslEnabled: True \ No newline at end of file diff --git a/code/scoring/inference_config.yml b/code/scoring/inference_config.yml new file mode 100644 index 00000000..3f65cf33 --- /dev/null +++ b/code/scoring/inference_config.yml @@ -0,0 +1,9 @@ +entryScript: score.py +runtime: python +condaFile: conda_dependencies.yml +extraDockerfileSteps: +schemaFile: +sourceDirectory: +enableGpu: False +baseImage: +baseImageRegistry: \ No newline at end of file diff --git a/code/scoring/score.py b/code/scoring/score.py index 994ca24a..dafe6bee 100644 --- a/code/scoring/score.py +++ b/code/scoring/score.py @@ -23,19 +23,18 @@ ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ -import pickle import json import numpy -from sklearn.ensemble import RandomForestClassifier from azureml.core.model import Model +import joblib def init(): global model - from sklearn.externals import joblib # load the model from file into a global object - model_path = Model.get_model_path(model_name="sklearn_regression_model.pkl") + model_path = Model.get_model_path( + model_name="sklearn_regression_model.pkl") model = joblib.load(model_path) diff --git a/code/training/train.py b/code/training/train.py index d94e0855..2b541615 100644 --- a/code/training/train.py +++ b/code/training/train.py @@ -23,8 +23,6 @@ ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ -import pickle -from azureml.core import Workspace from azureml.core.run import Run import os import argparse @@ -35,8 +33,6 @@ from sklearn.externals import joblib import numpy as np import json -import subprocess -from typing import Tuple, List parser = argparse.ArgumentParser("train") @@ -72,8 +68,10 @@ X, y = load_diabetes(return_X_y=True) columns = ["age", "gender", "bmi", "bp", "s1", "s2", "s3", "s4", "s5", "s6"] -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0) -data = {"train": {"X": X_train, "y": y_train}, "test": {"X": X_test, "y": y_test}} +X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.2, random_state=0) +data = {"train": {"X": X_train, "y": y_train}, + "test": {"X": X_test, "y": y_test}} print("Running train.py") @@ -97,16 +95,13 @@ # upload the model file explicitly into artifacts run.upload_file(name="./outputs/" + model_name, path_or_stream=model_name) -print("Uploaded the model {} to experiment {}".format(model_name, run.experiment.name)) +print("Uploaded the model {} to experiment {}".format( + model_name, run.experiment.name)) dirpath = os.getcwd() print(dirpath) print("Following files are uploaded ") print(run.get_file_names()) -# register the model -# run.log_model(file_name = model_name) -# print('Registered the model {} to run history {}'.format(model_name, run.history.name)) - run_id = {} run_id["run_id"] = run.id run_id["experiment_name"] = run.experiment.name @@ -115,4 +110,4 @@ with open(output_path, "w") as outfile: json.dump(run_id, outfile) -run.complete() \ No newline at end of file +run.complete() diff --git a/docs/code_description.md b/docs/code_description.md index 45fb7bb7..ef131408 100644 --- a/docs/code_description.md +++ b/docs/code_description.md @@ -2,59 +2,37 @@ ### Environment Setup -- requirements.txt : It consist of list of python packages which are needed by the train.py to run successfully on host agent (locally). +- `environment_setup/requirements.txt` : It consist of list of python packages which are needed by the train.py to run successfully on host agent (locally). -- install_requirements.sh : This script prepare the python environment i.e. install the Azure ML SDK and the packages specified in requirements.txt +- `environment_setup/install_requirements.sh` : This script prepare the python environment i.e. install the Azure ML SDK and the packages specified in requirements.txt -### Config Files -All the scripts inside the ./aml_config are config files. These are the files where you need to provide details about the subscription, resource group, workspace, conda dependencies, remote vm, AKS etc. +- `environment_setup/iac-*.yml, arm-templates` : Infrastructure as Code piplines to create and delete required resources along with corresponding arm-templates. -- config.json : This is a mandatory config file. Provide the subscription id, resource group name, workspace name and location where you want to create Azure ML services workspace. If you have already created the workspace, provide the existing workspace details in here. +- `environment_setup/Dockerfile` : Dockerfile of a building agent containing Python 3.6 and all required packages. -- conda_dependencies.yml : This is a mandatory file. This files contains the list of dependencies which are needed by the training/scoring script to run. This file is used to prepare environment for the local run(user managed/system managed) and docker run(local/remote). +- `environment_setup/docker-image-pipeline.yml` : An AzDo pipeline building and pushing [microsoft/mlopspython](https://hub.docker.com/_/microsoft-mlops-python) image. -- security_config.json : This file contains the credentials to the remove vm where we want to train the model. This config is used by the script 02-AttachTrainingVM.py to attach remote vm as a compute to the workspace. Attaching remote vm to workspace is one time operation. It is recommended not to publish this file with credentials populated in it. You can put the credentials, run the 02-AttachTrainingVM.py manually and clear the credentials before pushing it to git. +### Pipelines -- aks_webservice.json : This is an optional config. If you already have an AKS attached to your workspace, then provide the details in this file. If not, you do not have to check in this file to git. +- `.pipelines/azdo-base-pipeline.yml` : a pipeline template used by ci-build-train pipeline and pr-build-train pipelines. It contains steps performig linting, data and unit testing. +- `.pipelines/azdo-ci-build-train.yml` : a pipeline triggered when the code is merged into **master**. It profrorms linting, data integrity testing, unit testing, building and publishing an ML pipeline. +- `.pipelines/azdo-pr-build-train.yml` : a pipeline triggered when a **pull request** to the **master** branch is created. It profrorms linting, data integrity testing and unit testing only. -### Build Pipeline Scripts +### ML Services -The script under ./aml_service are used in build pipeline. All the scripts starting with 0 are the one time run scripts. These are the scripts which need to be run only once. There is no harm of running these scripts every time in build pipeline. +- `ml_service/pipelines/build_train_pipeline.py` : builds and publishes an ML training pipeline. +- `ml_service/pipelines/run_train_pipeline.py` : invokes a published ML training pipeline via REST API. +- `ml_service/util` : contains common utility functions used to build and publish an ML training pipeline. -- 00-WorkSpace.py : This is a onetime run script. It reads the workspace details from ./aml_config/config.json file and create (if workspace not available) or get (existing workspace). +### Code -- 01-Experiment.py : This is a onetime run script. It registers the root directory as project. It is not included as a step in build pipeline. +- `code/training/train.py` : a training step of an ML training pipeline. +- `code/evaluate/evaluate_model.py` : an evaluating step of an ML training pipeline. +- `code/evaluate/register_model.py` : registers a new trained model if evaluation shows the new model is more performent than the previous one. -- 02-AttachTrainingVM.py : This is a onetime run script. It attaches a remote VM to the workspace. It reads the config from ./aml_config/security_config.json. It is not included as a step in build pipeline. +### Scoring +- code/scoring/score.py : a scoring script which is about to be packed into a Docker Image along with a model while being deployed to QA/Prod environment. +- code/scoring/conda_dependencies.yml : contains a list of dependencies required by sore.py to be installed in a deployable Docker Image +- code/scoring/inference_config.yml, deployment_config_aci.yml, deployment_config_aks.yml : configuration files for the [AML Model Deploy](https://marketplace.visualstudio.com/items?itemName=ms-air-aiagility.private-vss-services-azureml&ssr=false#overview) pipeline task for ACI and AKS deployment targets. -- 10-TrainOnLocal.py : This scripts triggers the run of ./training/train.py script on the local compute(Host agent in case of build pipeline). If you are training on remote vm, you do not need this script in build pipeline. All the training scripts (1x) generates an output file aml_config/run_id.json which records the run_id and run history name of the training run. run_id.json is used by 20-RegisterModel.py to get the trained model. - -- 11-TrainOnLocalEnv.py : Its functionality is same as 10-TrainOnLocal.py, the only difference is that it creates a virtual environment on local compute and run training script on virtual env. - -- 12-TrainOnVM.py : As we want to train the model on remote VM, this script is included as a task in build pipeline. It submits the training job on remote vm. - -- 15.EvaluateModel.py : It gets the metrics of latest model trained and compares it with the model in production. If the production model still performs better, all below scripts are skipped. - -- 20-RegisterModel.py : It gets the run id from training steps output json and registers the model associated with that run along with tags. This scripts outputs a model.json file which contains model name and version. This script included as build task. - -- 30-CreateScoringImage.py : This takes the model details from last step, creates a scoring webservice docker image and publish the image to ACR. This script included as build task. It writes the image name and version to image.json file. - -### Deployment/Release Scripts -File under the directory ./aml_service starting with 5x and 6x are used in release pipeline. They are basically to deploy the docker image on AKS and ACI and publish webservice on them. - -- 50-deployOnAci.py : This script reads the image.json which is published as an artifact from build pipeline, create aci cluster and deploy the scoring web service on it. It writes the scoring service details to aci_webservice.json - -- 51-deployOnAks.py : This script reads the image.json which is published as an artifact from build pipeline, create aks cluster and deploy the scoring web service on it. If the aks_webservice.json file was checked in with existing aks details, it will update the existing webservice with new Image. It writes the scoring service details to aks_webservice.json - -- 60-AciWebServiceTest.py : Reads the ACI info from aci_webservice.json and test it with sample data. - -- 61-AksWebServiceTest.py : Reads the AKS info from aks_webservice.json and test it with sample data. - -### Training/Scoring Scripts - -- /code/training/train.py : This is the model training code. It uploads the model file to AML Service run id once the training is successful. This script is submitted as run job by all the 1x scripts. - -- /code/scoring/score.py : This is the score file used to create the webservice docker image. There is a conda_dependencies.yml in this directory which is exactly same as the one in aml_config. These two files are needed by the 30-CreateScoringImage.py scripts to be in same root directory while creating the image. - -**Note: In CICD Pipeline, please make sure that the working directory is the root directory of the repo.** diff --git a/docs/getting_started.md b/docs/getting_started.md index bd1c2839..994e220d 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -10,206 +10,205 @@ We use Azure DevOps for running our build(CI), retraining trigger and release (C If you already have Azure DevOps account, create a [new project](https://docs.microsoft.com/en-us/azure/devops/organizations/projects/create-project?view=azure-devops). -#### Enable Azure DevOps Preview -The steps below uses the latest DevOps features. Thus, please enable the feature **New YAML pipeline creation experience** by following the instructions [here](https://docs.microsoft.com/en-us/azure/devops/project/navigation/preview-features?view=azure-devops). -**Note:** Make sure you have the right permissions in Azure DevOps to do so. - -### 3. Create Service Principal to Login to Azure and create resources +### 3. Create Service Principal to Login to Azure To create service principal, register an application entity in Azure Active Directory (Azure AD) and grant it the Contributor or Owner role of the subscription or the resource group where the web service belongs to. See [how to create service principal](https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal) and assign permissions to manage Azure resource. -Please make note the following values after creating a service principal, we will need them in subsequent steps -- Azure subscription id (subscriptionid) -- Service principal username (spidentity)([application id](https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal#get-application-id-and-authentication-key)) -- Service principal password (spsecret) ([auth_key](https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal#get-application-id-and-authentication-key)) -- Service principal [tenant id](https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal#get-tenant-id) (sptenant) - -**Note:** You must have sufficient permissions to register an application with your Azure AD tenant, and assign the application to a role in your Azure subscription. Contact your subscription adminstator if you don't have the permissions. Normally a subscription admin can create a Service principal and can provide you the details. +Please make note of the following values after creating a service principal, we will need them in subsequent steps +- Application (client) ID +- Directory (tenant) ID +- Application Secret -### 4. Store secret in Key Vault and link it as variable group in Azure DevOps to be used by piplines. -Our pipeline require the following variables to autheticate with Azure. -- spidentity -- spsecret -- sptenant -- subscriptionid +**Note:** You must have sufficient permissions to register an application with your Azure AD tenant, and assign the application to a role in your Azure subscription. Contact your subscription adminstator if you don't have the permissions. Normally a subscription admin can create a Service principal and can provide you the details. -We noted the value of these variables in previous steps. -**NOTE:** These values should be treated as secret as they allow access to your subscription. +### 4. Create a Variable Group We make use of variable group inside Azure DevOps to store variables and their values that we want to make available across multiple pipelines. You can either store the values directly in [Azure DevOps](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/variable-groups?view=azure-devops&tabs=designer#create-a-variable-group) or connect to an Azure Key Vault in your subscription. Please refer to the documentation [here](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/variable-groups?view=azure-devops&tabs=designer#create-a-variable-group) to learn more about how to create a variable group and [link](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/variable-groups?view=azure-devops&tabs=designer#use-a-variable-group) it to your pipeline. -Please name your variable group **``AzureKeyVaultSecrets``**, we are using this name within our build yaml file. - -Up until now you shouls have +Please name your variable group **``devopsforai-aml-vg``** as we are using this name within our build yaml file. + +The varibale group should contain the following variables: + +| Variable Name | Suggested Value | +| --- | --- | +| AML_COMPUTE_CLUSTER_CPU_SKU | STANDARD_DS2_V2 | +| AML_COMPUTE_CLUSTER_NAME | train-cluster | +| AML_WORKSPACE_NAME | mlops-AML-WS | +| BASE_NAME | mlops | +| EVALUATE_SCRIPT_PATH | evaluate/evaluate_model.py | +| EXPERIMENT_NAME | mlopspython | +| LOCATION | centralus | +| MODEL_NAME | sklearn_regression_model.pkl | +| REGISTER_SCRIPT_PATH | register/register_model.py | +| RESOURCE_GROUP | mlops-AML-RG | +| SOURCES_DIR_TRAIN | code | +| SP_APP_ID | | +| SP_APP_SECRET | | +| SUBSCRIPTION_ID | | +| TENANT_ID | | +| TRAIN_SCRIPT_PATH | training/train.py | + +Mark **SP_APP_SECRET** variable as a secret one. + +Make sure to select the **Allow access to all pipelines** checkbox in the variable group configuration. + +Up until now you should have: - Forked (or cloned) the repo - Created a devops account or use an existing one - Got service principal details and subscription id -- Set them as variable group within devops +- A variable group with all configuration values -We now have 3 pipelines that we would set up -- **Build Pipeline (azure-pipelines.yml)**: Runs tests and sets up infrastructure -- **Retraining trigger pipeline(/template/retraining-template.json)**: This pipeline triggers Azure ML Pipeline (training/retraining) which trains a new model and publishes model image, if new model performs better -- **Release pipeline(/template/release-template.json)**: This pipeline deploys and tests model image as web service in QA and Prod environment +### 5. Create resources +The easiest way to create all required resources (Resource Group, ML Workspace, Container Registry, Storage Account, etc.) is to leverage an "Infrastructure as Code" [pipeline coming in this repository](../environment_setup/iac-create-environment.yml). This **IaC** pipeline takes care of all required resources basing on these [ARM templates](../environment_setup/arm-templates/cloud-environment.json). The pipeline requires an **Azure Resource Manager** service connection: +![create service connection](./images/create-rm-service-connection.png) -### 5. Set up Build Pipeline -1. Select your devops organization and project by clicking dev.azure.com -2. Once you are in the right devops project, click Pipelines on the left hand menu and select Builds -3. Click **New pipeline** to create new pipeline - ![new build pipeline](./images/new-build-pipeline1.png) -4. On the Connect option page, select **GitHub** - ![build connnect step](./images/build-connect.png) - -5. On the Select option page, select the GitHub repository where you forked the code. -![select repo](./images/build-selectrepo.png) +Give the connection name **``AzureResourceConnection``** as it is referred by the pipeline definition. -6. Authorize Azure Pipelines to access your git account -![select repo](./images/Install_Azure_pipeline.png) +In your DevOps project create a build pipeline from your forked **GitHub** repository: -7. Since the repository contains azure-pipelines.yml at the root level, Azure DevOps recognizes it and auto imports it. Click **Run** and this will start the build pipeline. -![select repo](./images/build-createpipeline1.png) +![build connnect step](./images/build-connect.png) -8. Your build run would look similar to the following image -![select repo](./images/build-run.png) +Refer to an **Existing Azure Pipelines YAML file**: -Great, you now have the build pipeline setup, you can either manually trigger it or it gets automatically triggered everytime there is a change in the master branch. +![configure step](./images/select-iac-pipeline.png) +Having done that, run the pipeline: -**Note:** The build pipeline will perform basic test on the code and provision infrastructure on azure. This can take around 10 mins to complete. +![iac run](./images/run-iac-pipeline.png) -### 6. Set up Retraining trigger release pipeline +Check out created resources in the [Azure Portal](portal.azure.com): -**Note:** For setting up release pipelines, first download the [release-pipelines](../release-pipelines) to your local filesystem so you can import it. +![created resources](./images/created-resources.png) -**Also Note:** If this is the first time you are creating a release pipeline, you would see the following option, click on **New Pipeline** -![import release pipeline](./images/release-new-pipeline.png) +Alternatively, you can also use a [cleaning pipeline](../environment_setup/iac-remove-environment.yml) that removes resources created for this project or you can just delete a resource group in the [Azure Portal](portal.azure.com). -To enable the option to **Import release pipeline**, we must have atleast one release pipeline so let's create one with an empty job. -![import release pipeline](./images/release-empty-job.png) -On the next screen, click on **Save** and then click **Ok** to save the empty release pipeline. -![import release pipeline](./images/release-save-empty.png) +### 6. Set up Build Pipeline -**Steps** +In your [Azure DevOps](https://dev.azure.com) project create and run a new build pipeline refereing to [azdo-ci-build-train.yml](../.pipelines/azdo-ci-build-train.yml) pipeline in your forked **GitHub** repository: -1. Select the Release tab from the menu on the left, then click the New dropdown on top and click on **Import Release pipeline** -![import release pipeline](./images/release-import.png) +![configure ci build pipeline](./images/ci-build-pipeline-configure.png) -1. On the next screen, navigate to **release-pipelines** folder and select **retrainingtrigger.json** pipeline file, click import. You should now see the following screen. Under Stages click on the Retrain stage, where it shows the red error sign. -![release retraining triggger](./images/release-retrainingtrigger.png) +Name the pipeline **ci-build**. Once the pipline is finished, explore the execution logs: - Click on agent job and then from the drop down for Agent Pool on the right side select **Hosted Ubuntu 1604** agent to execute your run and click **Save** button on top right. -![release retraining agent](./images/release-retrainingagent.png) +![ci build logs](./images/ci-build-logs.png) -1. We would now link the variable group we created earlier to this release pipeline. To do so click on the **Variables** tab, then click on **Variable** groups and then select **Link variable group** and select the variable group that we created in previous step and click **Link** followed by **Save** button. -![release retraining artifact](./images/release-link-vg.png) -1. We want the retraining pipeline to be triggered every time build pipeline is complete. To create this dependency, we will link the artifact from build pipeline as a trigger for retraining trigger release pipeline. To do so, click on the **pipeline** tab and then select **Add an artifact** option under Artifacts. -![release pipeline view](./images/release-retrainingpipeline.png) +and checkout a published training pipeline in the **mlops-AML-WS** workspace in [Azure Portal](https://ms.portal.azure.com/): -1. This will open up a pop up window, on this screen: - - for source type, select **Build** - - for project, select your project in Azure DevOps that you created in previous steps. - - For Source select the source build pipeline. If you have forked the git repo, the build pipeline may named ``yourgitusername.MLOpsPython`` - - In the Source alias, replace the auto-populated value with - **``DevOpsForAI``** - - Field **Default version** will get auto populated **Latest**, you can leave them as it is. - - Click on **Add**, and then **Save** the pipeline - ![release retraining artifact](./images/release-retrainingartifact.png) +![training pipeline](./images/training-pipeline.png) -1. Artifact is now added for retraining trigger pipeline, hit the **save** button on top right and then click **ok**. -1. To trigger this pipeline every time build pipeline executes, click on the lighting sign to enable the **Continous Deployment Trigger**, click **Save**. - ![release retraining artifact](./images/release-retrainingtrigger1.png) - -2. If you want to run this pipeline on a schedule, you can set one by clicking on **Schedule set** in Artifacts section. -![release retraining artifact](./images/release-retrainingartifactsuccess.png) +Great, you now have the build pipeline setup, you can either manually trigger it or it gets automatically triggered everytime there is a change in the master branch. The pipeline performs linting, unit testing, builds and publishes an **ML Training Pipeline** in an **ML Workspace** -1. For the first time, we will manually trigger this pipeline. - - Click Releases option on the left hand side and navigate to the release pipeline you just created. - ![release retraining artifact](./images/release-createarelease.png) - - Click **Create Release** - ![release create ](./images/release-create.png) - - On the next screen click on **Create** button, this creates a manual release for you. +### 7. Train the Model - **Note**: This release pipeline will call the published AML pipeline. The AML pipeline will train the model and package it into image. It will take around 10 mins to complete. The next steps need this pipeline to complete successfully. At this point, you can go to the Azure Portal AML WOrkspace resource created inside resource group "DevOps_AzureML_Demo" and click on the **Pipeline** tab to see the running pipeline. +The next step is to invoke the training pipeline created in the previous step. It can be done with a **Release Pipeline**: -### 7. Set up release (Deployment) pipeline +![invoke training pipeline](./images/invoke-training-pipeline.png) -**Note:** For setting up release pipelines, first download the [release-pipelines](../release-pipelines) to your local filesystem so you can import it. +An artifact of this pipeline will be the result of the build pipeline **ci-buid**: -**Also Note:** Before creating this pipeline, make sure that the build pipeline, retraining trigger release pipeline and AML retraining pipeline have been executed, as they will be creating resources during their run like docker images that we will deploy as part of this pipeline. So it is important for them to have successful runs before the setup here. +![artifact invoke pipeline](./images/artifact-invoke-pipeline.png) -Let's set up the release deployment pipeline now. -1. As done in previous step, Select the Release tab from the menu on the left, then click the New dropdown on top and click on **Import Release pipeline** -![import release pipeline](./images/release-import.png) +Configure a pipeline to see values from the previously defined variable group **devopsforai-aml-vg**: -1. On the next screen, navigate to **release-pipelines** folder and select **releasedeployment.json** pipeline file, click import. You should now see the following screen. Under Stages click on the QA environment's **view stage task", where it shows the red error sign. -![release retraining triggger](./images/release-deployment.png) +![retrain pipeline vg](./images/retrain-pipeline-vg.png) - Click on agent job and then from the drop down for Agent Pool on the right side select **Hosted Ubuntu 1604** agent to execute your run and click **Save** button on top right. -![release retraining agent](./images/release-deploymentqaagent.png) +Add an empty stage with name **Invoke Training Pipeline** and make sure that the **Agent Specification** is **ubuntu-16.04**: - Follow the same steps for **Prod Environment** and select **Hosted Ubuntu 1604** for agent pool and save the pipeline. - ![release retraining agent](./images/release-deploymentprodagent.png) +![agent specification](./images/agent-specification.png) -1. We would now link the variable group we created earlier to this release pipeline. To do so click on the **Variables** tab, then click on **Variable** groups and then select **Link variable group** and select the variable group that we created in previous step and click **Link** followed by **Save** button. -![release retraining artifact](./images/release-link-vg.png) +Add a command line step **Run Training Pipeline** with the following script: -1. We now need to add artifact that will trigger this pipeline. We will add two artifacts: - - Build pipeline output as artifact since that contains our configuration and code files that we require in this pipeline. - - ACR artifact to trigger this pipeline everytime there is a new image that gets published to Azure container registry (ACR) as part of retraining pipeline. +```bash +docker run -v $(System.DefaultWorkingDirectory)/_ci-build/mlops-pipelines/ml_service/pipelines:/pipelines \ +-w=/pipelines -e MODEL_NAME=$MODEL_NAME -e EXPERIMENT_NAME=$EXPERIMENT_NAME \ +-e TENANT_ID=$TENANT_ID -e SP_APP_ID=$SP_APP_ID -e SP_APP_SECRET=$SP_APP_SECRET \ +mcr.microsoft.com/mlops/python:latest python run_train_pipeline.py +``` - Here are the steps to add build output as artifact +This release pipeline should be automatically triggered (continuous deployment) whenever a new **ML training pipeline** is published by the **AzDo builder pipeline**. It can also be triggered manually or configured to run on a scheduled basis. Create a new release to trigger the pipeline manually: - - Click on pipeline tab to go back to pipeline view and click **Add an artifact**. This will open a pop up window - - for source type, select **Build** - - for project, select your project in Azure DevOps that you created in previous steps. - - For Source select the source build pipeline. If you have forked the git repo, the build pipeline may named ``yourgitusername.DevOpsForAI`` - - In the Source alias, replace the auto-populated value with - **``DevOpsForAI``** - - Field **Devault version** will get auto populated **Latest**, you can leave them as it is. - - Click on **Add**, and then **Save** the pipeline - ![release retraining artifact](./images/release-retrainingartifact.png) +![create release](./images/create-release.png) - **Here are the steps to add [Azure ML Model as an artifact](https://marketplace.visualstudio.com/items?itemName=ms-air-aiagility.vss-services-azureml)** +Once the release pipeline is completed, check out in the **ML Workspace** that the training pipeline is running: +![running training pipeline](./images/running-training-pipeline.png) - - Install the Azure Machine Learning extension for your DevOps organization from [here](https://marketplace.visualstudio.com/items?itemName=ms-air-aiagility.vss-services-azureml). You need to have admin rights to install it. +The training pipeline will train, evaluate and register a new model. Wait until it is fininshed and make sure there is a new model in the **ML Workspace**: - - Create Service Connection - 1. Go to your DevOps project and click on Project settings on bottom left corner - 2. Under Project Settings -> Pipelines, click on Service connections, click on "New service connection" and select Azure Resource Manager - ![release retraining agent](./images/service-connection.png) - - 3. Provide following info and click Ok once done: - ![release retraining agent](./images/service-connection-add.png) - - - - Click on pipeline tab to go back to pipeline view and click **Add an artifact**. This will open a pop up window - - For Source type, click on **more artifact types** dropdown and select **AzureML Model Artifact** - - For **Service Endpoint**, select an existing endpoint **MLOpsPython**, if you don't see anything in the dropdown, click on **Manage** and [create new **Azure Resource Manager**](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops#create-a-service-connection) service connection for your subscription. - ![release retraining agent](./images/model-artifact.png) - **Note:** You must have sufficient privileges to create a service connection, if not contact your subscription adminstrator. - - For Model Names, select **sklearn_regression_model.pkl**, this is the name of the newly trained model and if the previous pipelines executed properly you will see this model name in the drop down. - - For Default version, keep it to **Latest version** - - For Source alias, keep the default generated name. - - Click Add - - Click on lighting sign to enable the **Continous Deployment Trigger**, click **Save**. - ![release retraining artifact](./images/model-artifact-cd-trigger.png) +![trained model](./images/trained-model.png) + +Good! Now we have a trained model. + +### 8. Deploy the Model + +The final step is to deploy the model across environments with a release pipeline. There will be a **``QA``** environment running on [Azure Container Instances](https://azure.microsoft.com/en-us/services/container-instances/) and a **``Prod``** environment running on [Azure Kubernetes Service](https://azure.microsoft.com/en-us/services/kubernetes-service). + +![deploy model](./images/deploy-model.png) + + +This pipeline leverages the **Azure Machine Learning** extension that should be installed in your organization from the [marketplace](https://marketplace.visualstudio.com/items?itemName=ms-air-aiagility.vss-services-azureml). + +The pipeline consumes two artifacts: the result of the **Build Pipeline** as it contains configuration files and the **model** trained and registered by the ML training pipeline. +Configuration of a code **_ci-build** artifact is similar to what we did in the previous chapter. -1. We now have QA environment continously deployed each time there is a new ml model registered in AML Model Management. You can select pre-deployment conditions for prod environment, normally you don't want it to be auto deployed, so select manual only trigger here. +In order to configure a model artifact there should be a service connection to **mlops-AML-WS** workspace: - ![release retraining artifact](./images/release-deploymentprodtrigger.png) +![workspace connection](./images/workspace-connection.png) - To deploy a release manually, follow the document [here](https://docs.microsoft.com/en-us/azure/devops/pipelines/get-started-designer?view=azure-devops&tabs=new-nav#deploy-a-release) +Add an artifact to the pipeline and select **AzureML Model Artifact** source type. Select the **Service Endpoint** and **Model Names** from the drop down lists: +![model artifact](./images/model-artifact.png) -Congratulations, you now have three pipelines set up end to end. - - Build pipeline: triggered on code change to master branch on GitHub. - - Release Trigger pipeline: triggered on build pipeline execution and registers a new ML model to AML Model Management if better than previous one. - - Release Deployment pipeline: QA environment is auto triggered when there is a new model. - Prod is manual only and user decides when to release to this environment. +Create a stage **QA (ACI)** and add a single task to the job **Azure ML Model Deploy**: + +![deploy aci](./images/deploy-aci.png) + +Specify task parameters as it is shown in the table below: + + +| Parameter | Value | +| --- | --- | +| Display Name | Azure ML Model Deploy | +| Azure ML Workspace | mlops-AML-WS | +| Inference config Path | `$(System.DefaultWorkingDirectory)/_ci-build/mlops-pipelines/code/scoring/inference_config.yml` | +| Model Deployment Target | Azure Container Instance | +| Deployment Name | mlopspython-aci | +| Deployment Configuration file | `$(System.DefaultWorkingDirectory)/_ci-build/mlops-pipelines/code/scoring/deployment_config_aci.yml` | +| Overwrite existing deployment | X | + + +In a similar way create a stage **Prod (AKS** and add a single task to the job **Azure ML Model Deploy**: + +![deploy aks](./images/deploy-aks.png) + +Specify task parameters as it is shown in the table below: + +| Parameter | Value | +| --- | --- | +| Display Name | Azure ML Model Deploy | +| Azure ML Workspace | mlops-AML-WS | +| Inference config Path | `$(System.DefaultWorkingDirectory)/_ci-build/mlops-pipelines/code/scoring/inference_config.yml` | +| Model Deployment Target | Azure Kubernetes Service | +| Select AKS Cluster for Deployment | YOUR_DEPLOYMENT_K8S_CLUSTER | +| Deployment Name | mlopspython-aks | +| Deployment Configuration file | `$(System.DefaultWorkingDirectory)/_ci-build/mlops-pipelines/code/scoring/deployment_config_aks.yml` | +| Overwrite existing deployment | X | + +**Note:** Creating of a Kubernetes cluster on AKS is out of scope of this tutorial, so you should take care of it on your own. + +Save the pipeline and craete a release to trigger it manually. Once the pipeline exection is finished, check out deployments in the **mlops-AML-WS** workspace. + + + +Congratulations! You have three pipelines set up end to end: + - Build pipeline: triggered on code change to master branch on GitHub, performs linting, unit testing and publishing a trainig pipeline + - Release Trigger pipeline: runs a published training pipeline to trian, evaluate and register a model + - Release Deployment pipeline: deploys a model to QA (ACI) and Prod (AKS) environemts + diff --git a/docs/images/agent-specification.png b/docs/images/agent-specification.png new file mode 100644 index 00000000..c71c3b68 Binary files /dev/null and b/docs/images/agent-specification.png differ diff --git a/docs/images/artifact-invoke-pipeline.png b/docs/images/artifact-invoke-pipeline.png new file mode 100644 index 00000000..2a6dcebf Binary files /dev/null and b/docs/images/artifact-invoke-pipeline.png differ diff --git a/docs/images/build-connect.png b/docs/images/build-connect.png index f5d9d61a..79553d80 100644 Binary files a/docs/images/build-connect.png and b/docs/images/build-connect.png differ diff --git a/docs/images/ci-build-logs.png b/docs/images/ci-build-logs.png new file mode 100644 index 00000000..726f70ac Binary files /dev/null and b/docs/images/ci-build-logs.png differ diff --git a/docs/images/ci-build-pipeline-configure.png b/docs/images/ci-build-pipeline-configure.png new file mode 100644 index 00000000..d593d1dc Binary files /dev/null and b/docs/images/ci-build-pipeline-configure.png differ diff --git a/docs/images/create-release.png b/docs/images/create-release.png new file mode 100644 index 00000000..15069b5d Binary files /dev/null and b/docs/images/create-release.png differ diff --git a/docs/images/create-rm-service-connection.png b/docs/images/create-rm-service-connection.png new file mode 100644 index 00000000..629d3c2a Binary files /dev/null and b/docs/images/create-rm-service-connection.png differ diff --git a/docs/images/created-resources.png b/docs/images/created-resources.png new file mode 100644 index 00000000..d5136ee8 Binary files /dev/null and b/docs/images/created-resources.png differ diff --git a/docs/images/deploy-aci.png b/docs/images/deploy-aci.png new file mode 100644 index 00000000..0270143b Binary files /dev/null and b/docs/images/deploy-aci.png differ diff --git a/docs/images/deploy-aks.png b/docs/images/deploy-aks.png new file mode 100644 index 00000000..96d83b8b Binary files /dev/null and b/docs/images/deploy-aks.png differ diff --git a/docs/images/deploy-model.png b/docs/images/deploy-model.png new file mode 100644 index 00000000..8a4cbd06 Binary files /dev/null and b/docs/images/deploy-model.png differ diff --git a/docs/images/invoke-training-pipeline.png b/docs/images/invoke-training-pipeline.png new file mode 100644 index 00000000..21619ae3 Binary files /dev/null and b/docs/images/invoke-training-pipeline.png differ diff --git a/docs/images/main-flow.png b/docs/images/main-flow.png new file mode 100644 index 00000000..a49f7440 Binary files /dev/null and b/docs/images/main-flow.png differ diff --git a/docs/images/model-artifact.png b/docs/images/model-artifact.png index 0681a556..b89390b4 100644 Binary files a/docs/images/model-artifact.png and b/docs/images/model-artifact.png differ diff --git a/docs/images/retrain-pipeline-vg.png b/docs/images/retrain-pipeline-vg.png new file mode 100644 index 00000000..4aa30e9f Binary files /dev/null and b/docs/images/retrain-pipeline-vg.png differ diff --git a/docs/images/run-iac-pipeline.png b/docs/images/run-iac-pipeline.png new file mode 100644 index 00000000..15771246 Binary files /dev/null and b/docs/images/run-iac-pipeline.png differ diff --git a/docs/images/running-training-pipeline.png b/docs/images/running-training-pipeline.png new file mode 100644 index 00000000..0d3af93e Binary files /dev/null and b/docs/images/running-training-pipeline.png differ diff --git a/docs/images/select-iac-pipeline.png b/docs/images/select-iac-pipeline.png new file mode 100644 index 00000000..e165ccc8 Binary files /dev/null and b/docs/images/select-iac-pipeline.png differ diff --git a/docs/images/trained-model.png b/docs/images/trained-model.png new file mode 100644 index 00000000..3753fd7d Binary files /dev/null and b/docs/images/trained-model.png differ diff --git a/docs/images/training-pipeline.png b/docs/images/training-pipeline.png new file mode 100644 index 00000000..cbdaf048 Binary files /dev/null and b/docs/images/training-pipeline.png differ diff --git a/docs/images/workspace-connection.png b/docs/images/workspace-connection.png new file mode 100644 index 00000000..570a724e Binary files /dev/null and b/docs/images/workspace-connection.png differ diff --git a/environment_setup/Dockerfile b/environment_setup/Dockerfile new file mode 100644 index 00000000..b6b3be6a --- /dev/null +++ b/environment_setup/Dockerfile @@ -0,0 +1,13 @@ +FROM conda/miniconda3 + +LABEL org.label-schema.vendor = "Microsoft" \ + org.label-schema.url = "https://hub.docker.com/r/microsoft/mlopspython" \ + org.label-schema.vcs-url = "https://github.com/microsoft/MLOpsPython" + + + +COPY environment_setup/requirements.txt /setup/ + +RUN apt-get update && apt-get install gcc -y && pip install --upgrade -r /setup/requirements.txt + +CMD ["python"] \ No newline at end of file diff --git a/environment_setup/arm-templates/cloud-environment.json b/environment_setup/arm-templates/cloud-environment.json new file mode 100644 index 00000000..e01471d7 --- /dev/null +++ b/environment_setup/arm-templates/cloud-environment.json @@ -0,0 +1,123 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "baseName": { + "type": "string", + "maxLength": 10, + "minLength": 3, + "metadata": { + "description": "The base name to use as prefix to create all the resources." + } + }, + "location": { + "type": "string", + "defaultValue": "eastus", + "allowedValues": [ + "eastus", + "eastus2", + "southcentralus", + "southeastasia", + "westcentralus", + "westeurope", + "westus2" + ], + "metadata": { + "description": "Specifies the location for all resources." + } + } + }, + "variables": { + "amlWorkspaceName": "[concat(parameters('baseName'),'-AML-WS')]", + "storageAccountName": "[concat(toLower(parameters('baseName')), 'amlsa')]", + "storageAccountType": "Standard_LRS", + "keyVaultName": "[concat(parameters('baseName'),'-AML-KV')]", + "tenantId": "[subscription().tenantId]", + "applicationInsightsName": "[concat(parameters('baseName'),'-AML-AI')]", + "containerRegistryName": "[concat(toLower(parameters('baseName')),'amlcr')]" + }, + "resources": [ + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2018-07-01", + "name": "[variables('storageAccountName')]", + "location": "[parameters('location')]", + "sku": { + "name": "[variables('storageAccountType')]" + }, + "kind": "StorageV2", + "properties": { + "encryption": { + "services": { + "blob": { + "enabled": true + }, + "file": { + "enabled": true + } + }, + "keySource": "Microsoft.Storage" + }, + "supportsHttpsTrafficOnly": true + } + }, + { + "type": "Microsoft.KeyVault/vaults", + "apiVersion": "2018-02-14", + "name": "[variables('keyVaultName')]", + "location": "[parameters('location')]", + "properties": { + "tenantId": "[variables('tenantId')]", + "sku": { + "name": "standard", + "family": "A" + }, + "accessPolicies": [] + } + }, + { + "type": "Microsoft.Insights/components", + "apiVersion": "2015-05-01", + "name": "[variables('applicationInsightsName')]", + "location": "[if(or(equals(parameters('location'),'eastus2'),equals(parameters('location'),'westcentralus')),'southcentralus',parameters('location'))]", + "kind": "web", + "properties": { + "Application_Type": "web" + } + }, + { + "type": "Microsoft.ContainerRegistry/registries", + "apiVersion": "2017-10-01", + "name": "[variables('containerRegistryName')]", + "location": "[parameters('location')]", + "sku": { + "name": "Standard" + }, + "properties": { + "adminUserEnabled": true + } + }, + { + "type": "Microsoft.MachineLearningServices/workspaces", + "apiVersion": "2018-11-19", + "name": "[variables('amlWorkspaceName')]", + "location": "[parameters('location')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]", + "[resourceId('Microsoft.KeyVault/vaults', variables('keyVaultName'))]", + "[resourceId('Microsoft.Insights/components', variables('applicationInsightsName'))]", + "[resourceId('Microsoft.ContainerRegistry/registries', variables('containerRegistryName'))]" + ], + "identity": { + "type": "systemAssigned" + }, + "properties": { + "friendlyName": "[variables('amlWorkspaceName')]", + "keyVault": "[resourceId('Microsoft.KeyVault/vaults',variables('keyVaultName'))]", + "applicationInsights": "[resourceId('Microsoft.Insights/components',variables('applicationInsightsName'))]", + "containerRegistry": "[resourceId('Microsoft.ContainerRegistry/registries',variables('containerRegistryName'))]", + "storageAccount": "[resourceId('Microsoft.Storage/storageAccounts/',variables('storageAccountName'))]" + } + } + ] +} \ No newline at end of file diff --git a/environment_setup/docker-image-pipeline.yml b/environment_setup/docker-image-pipeline.yml new file mode 100644 index 00000000..c88884d8 --- /dev/null +++ b/environment_setup/docker-image-pipeline.yml @@ -0,0 +1,29 @@ +resources: +- repo: self + +queue: + name: Hosted Ubuntu 1604 + +trigger: + branches: + include: + - master + + paths: + include: + - environment_setup/* + +variables: + containerRegistry: $[coalesce(variables['acrServiceConnection'], 'acrconnection')] + imageName: $[coalesce(variables['agentImageName'], 'public/mlops/mlopspython')] + +steps: + - task: Docker@2 + displayName: Build and Push + inputs: + command: buildAndPush + containerRegistry: '$(containerRegistry)' + repository: '$(imageName)' + tags: 'latest' + buildContext: '$(Build.SourcesDirectory)' + dockerFile: '$(Build.SourcesDirectory)/environment_setup/Dockerfile' diff --git a/environment_setup/iac-create-environment.yml b/environment_setup/iac-create-environment.yml new file mode 100644 index 00000000..57e5a106 --- /dev/null +++ b/environment_setup/iac-create-environment.yml @@ -0,0 +1,29 @@ +trigger: + branches: + include: + - master + paths: + include: + - environment_setup/arm-templates/* + +pool: + vmImage: 'ubuntu-latest' + +variables: +- group: devopsforai-aml-vg + + +steps: +- task: AzureResourceGroupDeployment@2 + inputs: + azureSubscription: 'AzureResourceConnection' + action: 'Create Or Update Resource Group' + resourceGroupName: '$(RESOURCE_GROUP)' + location: $(LOCATION) + templateLocation: 'Linked artifact' + csmFile: '$(Build.SourcesDirectory)/environment_setup/arm-templates/cloud-environment.json' + overrideParameters: '-baseName $(BASE_NAME)' + deploymentMode: 'Incremental' + displayName: 'Deploy MLOps resources to Azure' + + \ No newline at end of file diff --git a/environment_setup/iac-remove-environment.yml b/environment_setup/iac-remove-environment.yml new file mode 100644 index 00000000..67626223 --- /dev/null +++ b/environment_setup/iac-remove-environment.yml @@ -0,0 +1,25 @@ +trigger: + branches: + include: + - master + paths: + include: + - environment_setup/arm-templates/* + +pool: + vmImage: 'ubuntu-latest' + +variables: +- group: devopsforai-aml-vg + + +steps: +- task: AzureResourceGroupDeployment@2 + inputs: + azureSubscription: 'AzureResourceConnection' + action: 'DeleteRG' + resourceGroupName: '$(RESOURCE_GROUP)' + location: $(LOCATION) + displayName: 'Delete resources in Azure' + + \ No newline at end of file diff --git a/environment_setup/requirements.txt b/environment_setup/requirements.txt index b3c2a14c..8a086c4d 100644 --- a/environment_setup/requirements.txt +++ b/environment_setup/requirements.txt @@ -1,5 +1,8 @@ -scipy==1.0.0 -scikit-learn==0.19.1 -numpy==1.14.5 -pandas==0.23.1 -pytest==4.3.0 \ No newline at end of file +pytest==4.3.0 +requests>=2.22 +azureml>=0.2 +azureml-sdk>=1.0 +python-dotenv>=0.10.3 +flake8 +flake8_formatter_junit_xml +azure-cli==2.0.71 \ No newline at end of file diff --git a/ml_service/pipelines/build_train_pipeline.py b/ml_service/pipelines/build_train_pipeline.py new file mode 100644 index 00000000..a294a1c0 --- /dev/null +++ b/ml_service/pipelines/build_train_pipeline.py @@ -0,0 +1,134 @@ +from azureml.pipeline.core.graph import PipelineParameter +from azureml.pipeline.steps import PythonScriptStep +from azureml.pipeline.core import Pipeline, PipelineData +from azureml.core.runconfig import RunConfiguration, CondaDependencies +from azureml.core import Datastore +import datetime +import os +import sys +from dotenv import load_dotenv +sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 +from workspace import get_workspace +from attach_compute import get_compute +import json + + +def main(): + load_dotenv() + workspace_name = os.environ.get("AML_WORKSPACE_NAME") + resource_group = os.environ.get("RESOURCE_GROUP") + subscription_id = os.environ.get("SUBSCRIPTION_ID") + tenant_id = os.environ.get("TENANT_ID") + app_id = os.environ.get("SP_APP_ID") + app_secret = os.environ.get("SP_APP_SECRET") + sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN") + train_script_path = os.environ.get("TRAIN_SCRIPT_PATH") + evaluate_script_path = os.environ.get("EVALUATE_SCRIPT_PATH") + register_script_path = os.environ.get("REGISTER_SCRIPT_PATH") + vm_size_cpu = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU") + compute_name_cpu = os.environ.get("AML_COMPUTE_CLUSTER_NAME") + model_name = os.environ.get("MODEL_NAME") + + # Get Azure machine learning workspace + aml_workspace = get_workspace( + workspace_name, + resource_group, + subscription_id, + tenant_id, + app_id, + app_secret) + print(aml_workspace) + + # Get Azure machine learning cluster + aml_compute_cpu = get_compute( + aml_workspace, + compute_name_cpu, + vm_size_cpu) + if aml_compute_cpu is not None: + print(aml_compute_cpu) + + run_config = RunConfiguration(conda_dependencies=CondaDependencies.create( + conda_packages=['numpy', 'pandas', + 'scikit-learn', 'tensorflow', 'keras'], + pip_packages=['azure', 'azureml-core', + 'azure-storage', + 'azure-storage-blob']) + ) + run_config.environment.docker.enabled = True + + model_name = PipelineParameter( + name="model_name", default_value=model_name) + def_blob_store = Datastore(aml_workspace, "workspaceblobstore") + jsonconfigs = PipelineData("jsonconfigs", datastore=def_blob_store) + config_suffix = datetime.datetime.now().strftime("%Y%m%d%H") + + train_step = PythonScriptStep( + name="Train Model", + script_name=train_script_path, + compute_target=aml_compute_cpu, + source_directory=sources_directory_train, + arguments=[ + "--config_suffix", config_suffix, + "--json_config", jsonconfigs, + "--model_name", model_name, + ], + runconfig=run_config, + # inputs=[jsonconfigs], + outputs=[jsonconfigs], + allow_reuse=False, + ) + print("Step Train created") + + evaluate_step = PythonScriptStep( + name="Evaluate Model ", + script_name=evaluate_script_path, + compute_target=aml_compute_cpu, + source_directory=sources_directory_train, + arguments=[ + "--config_suffix", config_suffix, + "--json_config", jsonconfigs, + ], + runconfig=run_config, + inputs=[jsonconfigs], + # outputs=[jsonconfigs], + allow_reuse=False, + ) + print("Step Evaluate created") + + register_model_step = PythonScriptStep( + name="Register New Trained Model", + script_name=register_script_path, + compute_target=aml_compute_cpu, + source_directory=sources_directory_train, + arguments=[ + "--config_suffix", config_suffix, + "--json_config", jsonconfigs, + "--model_name", model_name, + ], + runconfig=run_config, + inputs=[jsonconfigs], + # outputs=[jsonconfigs], + allow_reuse=False, + ) + print("Step register model created") + + evaluate_step.run_after(train_step) + register_model_step.run_after(evaluate_step) + steps = [register_model_step] + + train_pipeline = Pipeline(workspace=aml_workspace, steps=steps) + train_pipeline.validate() + published_pipeline = train_pipeline.publish( + name="training-pipeline", + description="Model training/retraining pipeline" + ) + + train_pipeline_json = {} + train_pipeline_json["rest_endpoint"] = published_pipeline.endpoint + json_file_path = "ml_service/pipelines/train_pipeline.json" + with open(json_file_path, "w") as outfile: + json.dump(train_pipeline_json, outfile) + + +if __name__ == '__main__': + main() diff --git a/ml_service/pipelines/run_train_pipeline.py b/ml_service/pipelines/run_train_pipeline.py new file mode 100644 index 00000000..c036aefd --- /dev/null +++ b/ml_service/pipelines/run_train_pipeline.py @@ -0,0 +1,30 @@ +import sys +import os +import json +import requests +from azureml.core.authentication import AzureCliAuthentication + + +try: + with open("train_pipeline.json") as f: + train_pipeline_json = json.load(f) +except Exception: + print("No pipeline json found") + sys.exit(0) + +experiment_name = os.environ.get("EXPERIMENT_NAME") +model_name = os.environ.get("MODEL_NAME") + +cli_auth = AzureCliAuthentication() +token = cli_auth.get_authentication_header() + +rest_endpoint = train_pipeline_json["rest_endpoint"] + +response = requests.post( + rest_endpoint, headers=token, + json={"ExperimentName": experiment_name, + "ParameterAssignments": {"model_name": model_name}} +) + +run_id = response.json()["Id"] +print("Pipeline run initiated ", run_id) diff --git a/ml_service/util/attach_compute.py b/ml_service/util/attach_compute.py new file mode 100644 index 00000000..ff9d0ebd --- /dev/null +++ b/ml_service/util/attach_compute.py @@ -0,0 +1,46 @@ +import os +from dotenv import load_dotenv +from azureml.core import Workspace +from azureml.core.compute import AmlCompute +from azureml.core.compute import ComputeTarget +from azureml.exceptions import ComputeTargetException + + +def get_compute( + workspace: Workspace, + compute_name: str, + vm_size: str +): + # Load the environment variables from .env in case this script + # is called outside an existing process + load_dotenv() + # Verify that cluster does not exist already + try: + if compute_name in workspace.compute_targets: + compute_target = workspace.compute_targets[compute_name] + if compute_target and type(compute_target) is AmlCompute: + print('Found existing compute target ' + compute_name + + ' so using it.') + else: + compute_config = AmlCompute.provisioning_configuration( + vm_size=vm_size, + vm_priority=os.environ.get("AML_CLUSTER_PRIORITY", + 'lowpriority'), + min_nodes=int(os.environ.get("AML_CLUSTER_MIN_NODES", 0)), + max_nodes=int(os.environ.get("AML_CLUSTER_MAX_NODES", 4)), + idle_seconds_before_scaledown="300" + # #Uncomment the below lines for VNet support + # vnet_resourcegroup_name=vnet_resourcegroup_name, + # vnet_name=vnet_name, + # subnet_name=subnet_name + ) + compute_target = ComputeTarget.create(workspace, compute_name, + compute_config) + compute_target.wait_for_completion( + show_output=True, + min_node_count=None, + timeout_in_minutes=10) + return compute_target + except ComputeTargetException: + print('An error occurred trying to provision compute.') + exit() diff --git a/ml_service/util/register_model.py b/ml_service/util/register_model.py new file mode 100644 index 00000000..6f9634ab --- /dev/null +++ b/ml_service/util/register_model.py @@ -0,0 +1,49 @@ +import sys +import os +import os.path +from dotenv import load_dotenv +from azureml.core import Workspace +from azureml.core.model import Model +from azureml.core.authentication import ServicePrincipalAuthentication + +# Load the environment variables from .env in case this script +# is called outside an existing process +load_dotenv() + +TENANT_ID = os.environ.get('TENANT_ID') +APP_ID = os.environ.get('APP_ID') +APP_SECRET = os.environ.get('APP_SECRET') +MODEL_PATH = os.environ.get('MODEL_PATH') +MODEL_NAME = os.environ.get('MODEL_NAME') +WORKSPACE_NAME = os.environ.get('WORKSPACE_NAME') +SUBSCRIPTION_ID = os.environ.get('SUBSCRIPTION_ID') +RESOURCE_GROUP = os.environ.get('RESOURCE_GROUP') + + +if os.path.isfile(MODEL_PATH) is False: + print("The given model path %s is invalid" % (MODEL_PATH)) + sys.exit(1) + +SP_AUTH = ServicePrincipalAuthentication( + tenant_id=TENANT_ID, + service_principal_id=APP_ID, + service_principal_password=APP_SECRET) + +WORKSPACE = Workspace.get( + WORKSPACE_NAME, + SP_AUTH, + SUBSCRIPTION_ID, + RESOURCE_GROUP +) + +try: + MODEL = Model.register( + model_path=MODEL_PATH, + model_name=MODEL_NAME, + description="Forecasting Model", + workspace=WORKSPACE) + + print("Model registered successfully. ID: " + MODEL.id) +except Exception as caught_error: + print("Error while registering the model: " + str(caught_error)) + sys.exit(1) diff --git a/ml_service/util/workspace.py b/ml_service/util/workspace.py new file mode 100644 index 00000000..08d1f67d --- /dev/null +++ b/ml_service/util/workspace.py @@ -0,0 +1,29 @@ +import sys +from azureml.core import Workspace +from azureml.core.authentication import ServicePrincipalAuthentication + + +def get_workspace( + name: str, + resource_group: str, + subscription_id: str, + tenant_id: str, + app_id: str, + app_secret: str): + service_principal = ServicePrincipalAuthentication( + tenant_id=tenant_id, + service_principal_id=app_id, + service_principal_password=app_secret) + + try: + aml_workspace = Workspace.get( + name=name, + subscription_id=subscription_id, + resource_group=resource_group, + auth=service_principal) + + return aml_workspace + except Exception as caught_exception: + print("Error while retrieving Workspace...") + print(str(caught_exception)) + sys.exit(1) diff --git a/model/placeholder b/model/placeholder deleted file mode 100644 index e69de29b..00000000 diff --git a/release-pipelines/releasedeployment.json b/release-pipelines/releasedeployment.json deleted file mode 100644 index 8df01f33..00000000 --- a/release-pipelines/releasedeployment.json +++ /dev/null @@ -1,581 +0,0 @@ -{ - "source": 2, - "revision": 1, - "description": null, - "createdBy": { - "displayName": "Username", - "url": "https://app.vssps.visualstudio.com/Ababa295f-6e98-40b6-9dc1-aa6118e169e2/_apis/Identities/af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "_links": { - "avatar": { - "href": "https://youaccount.visualstudio.com/_apis/GraphProfile/MemberAvatars/aad.ZmZhYjg5YzEtYmIxNC03NGRiLTk3NTAtZDBlMzQ2NGQwNjU0" - } - }, - "id": "af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "uniqueName": "user@email.com", - "imageUrl": "https://youaccount.visualstudio.com/_api/_common/identityImage?id=af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "descriptor": "aad.ZmZhYjg5YzEtYmIxNC03NGRiLTk3NTAtZDBlMzQ2NGQwNjU0" - }, - "createdOn": "2019-03-28T18:56:41.680Z", - "modifiedBy": { - "displayName": "User Name", - "url": "https://app.vssps.visualstudio.com/Ababa295f-6e98-40b6-9dc1-aa6118e169e2/_apis/Identities/af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "_links": { - "avatar": { - "href": "https://youaccount.visualstudio.com/_apis/GraphProfile/MemberAvatars/aad.ZmZhYjg5YzEtYmIxNC03NGRiLTk3NTAtZDBlMzQ2NGQwNjU0" - } - }, - "id": "af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "uniqueName": "user@email.com", - "imageUrl": "https://youaccount.visualstudio.com/_api/_common/identityImage?id=af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "descriptor": "aad.ZmZhYjg5YzEtYmIxNC03NGRiLTk3NTAtZDBlMzQ2NGQwNjU0" - }, - "modifiedOn": "2019-03-28T18:56:41.680Z", - "isDeleted": false, - "variables": {}, - "variableGroups": [], - "environments": [ - { - "id": 8, - "name": "QA Environment", - "rank": 1, - "owner": { - "displayName": "User Name", - "url": "https://app.vssps.visualstudio.com/Ababa295f-6e98-40b6-9dc1-aa6118e169e2/_apis/Identities/af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "_links": { - "avatar": { - "href": "https://youaccount.visualstudio.com/_apis/GraphProfile/MemberAvatars/aad.ZmZhYjg5YzEtYmIxNC03NGRiLTk3NTAtZDBlMzQ2NGQwNjU0" - } - }, - "id": "af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "uniqueName": "user@email.com", - "imageUrl": "https://youaccount.visualstudio.com/_api/_common/identityImage?id=af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "descriptor": "aad.ZmZhYjg5YzEtYmIxNC03NGRiLTk3NTAtZDBlMzQ2NGQwNjU0" - }, - "variables": {}, - "variableGroups": [], - "preDeployApprovals": { - "approvals": [ - { - "rank": 1, - "isAutomated": true, - "isNotificationOn": false, - "id": 22 - } - ], - "approvalOptions": { - "requiredApproverCount": null, - "releaseCreatorCanBeApprover": false, - "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, - "enforceIdentityRevalidation": false, - "timeoutInMinutes": 0, - "executionOrder": 1 - } - }, - "deployStep": { - "id": 25 - }, - "postDeployApprovals": { - "approvals": [ - { - "rank": 1, - "isAutomated": true, - "isNotificationOn": false, - "id": 26 - } - ], - "approvalOptions": { - "requiredApproverCount": null, - "releaseCreatorCanBeApprover": false, - "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, - "enforceIdentityRevalidation": false, - "timeoutInMinutes": 0, - "executionOrder": 2 - } - }, - "deployPhases": [ - { - "deploymentInput": { - "parallelExecution": { - "parallelExecutionType": 0 - }, - "skipArtifactsDownload": false, - "artifactsDownloadInput": { - "downloadInputs": [] - }, - "queueId": 18, - "demands": [], - "enableAccessToken": false, - "timeoutInMinutes": 0, - "jobCancelTimeoutInMinutes": 1, - "condition": "succeeded()", - "overrideInputs": {} - }, - "rank": 1, - "phaseType": 1, - "name": "Agent job", - "refName": null, - "workflowTasks": [ - { - "environment": {}, - "taskId": "33c63b11-352b-45a2-ba1b-54cb568a29ca", - "version": "0.*", - "name": "Use Python 3.6", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "versionSpec": "3.6", - "addToPath": "true", - "architecture": "x64" - } - }, - { - "environment": {}, - "taskId": "6c731c3c-3c68-459a-a5c9-bde6e6595b5b", - "version": "3.*", - "name": "Bash Script", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "targetType": "filePath", - "filePath": "$(System.DefaultWorkingDirectory)/DevOpsForAI/devops-for-ai/environment_setup/install_requirements.sh", - "arguments": "", - "script": "# Write your commands here\n\n# Use the environment variables input below to pass secret variables to this script", - "workingDirectory": "$(System.DefaultWorkingDirectory)/DevOpsForAI/devops-for-ai/environment_setup", - "failOnStderr": "false", - "noProfile": "true", - "noRc": "true" - } - }, - { - "environment": {}, - "taskId": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", - "version": "2.*", - "name": "Login to Azure Subscription", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "script": "az login --service-principal -u $(spidentity) -p $(spsecret) --tenant $(sptenant)\n", - "workingDirectory": "", - "failOnStderr": "false" - } - }, - { - "environment": {}, - "taskId": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", - "version": "2.*", - "name": "New model available, Create Scoring Image", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "script": "python aml_service/30-CreateScoringImage.py", - "workingDirectory": "$(System.DefaultWorkingDirectory)/DevOpsForAI/devops-for-ai", - "failOnStderr": "false" - } - }, - { - "environment": {}, - "taskId": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", - "version": "2.*", - "name": "Get Latest Scoring Image Name & Version", - "refName": "", - "enabled": false, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "script": "python aml_service/34-GetScoringImageName.py", - "workingDirectory": "$(System.DefaultWorkingDirectory)/DevOpsForAI/devops-for-ai", - "failOnStderr": "false" - } - }, - { - "environment": {}, - "taskId": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", - "version": "2.*", - "name": "Deploy new image to ACI", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "script": "python aml_service/50-deployOnAci.py", - "workingDirectory": "$(System.DefaultWorkingDirectory)/DevOpsForAI/devops-for-ai", - "failOnStderr": "false" - } - }, - { - "environment": {}, - "taskId": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", - "version": "2.*", - "name": "Test the image on ACI", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "script": "python aml_service/60-AciWebserviceTest.py", - "workingDirectory": "$(System.DefaultWorkingDirectory)/DevOpsForAI/devops-for-ai", - "failOnStderr": "false" - } - } - ] - } - ], - "environmentOptions": { - "emailNotificationType": "OnlyOnFailure", - "emailRecipients": "release.environment.owner;release.creator", - "skipArtifactsDownload": false, - "timeoutInMinutes": 0, - "enableAccessToken": false, - "publishDeploymentStatus": true, - "badgeEnabled": false, - "autoLinkWorkItems": false, - "pullRequestDeploymentEnabled": false - }, - "demands": [], - "conditions": [ - { - "name": "ReleaseStarted", - "conditionType": 1, - "value": "" - } - ], - "executionPolicy": { - "concurrencyCount": 1, - "queueDepthCount": 0 - }, - "schedules": [], - "currentRelease": { - "id": 0, - "url": "https://youaccount.vsrm.visualstudio.com/c9414c5b-b8f8-4d50-a8bf-eae8dbbb6a2a/_apis/Release/releases/0", - "_links": {} - }, - "retentionPolicy": { - "daysToKeep": 30, - "releasesToKeep": 3, - "retainBuild": true - }, - "processParameters": {}, - "properties": {}, - "preDeploymentGates": { - "id": 0, - "gatesOptions": null, - "gates": [] - }, - "postDeploymentGates": { - "id": 0, - "gatesOptions": null, - "gates": [] - }, - "environmentTriggers": [], - "badgeUrl": "https://youaccount.vsrm.visualstudio.com/_apis/public/Release/badge/c9414c5b-b8f8-4d50-a8bf-eae8dbbb6a2a/5/8" - }, - { - "id": 9, - "name": "Prod Environment", - "rank": 2, - "owner": { - "displayName": "User Name", - "url": "https://app.vssps.visualstudio.com/Ababa295f-6e98-40b6-9dc1-aa6118e169e2/_apis/Identities/af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "_links": { - "avatar": { - "href": "https://youaccount.visualstudio.com/_apis/GraphProfile/MemberAvatars/aad.ZmZhYjg5YzEtYmIxNC03NGRiLTk3NTAtZDBlMzQ2NGQwNjU0" - } - }, - "id": "af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "uniqueName": "user@email.com", - "imageUrl": "https://youaccount.visualstudio.com/_api/_common/identityImage?id=af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "descriptor": "aad.ZmZhYjg5YzEtYmIxNC03NGRiLTk3NTAtZDBlMzQ2NGQwNjU0" - }, - "variables": {}, - "variableGroups": [], - "preDeployApprovals": { - "approvals": [ - { - "rank": 1, - "isAutomated": true, - "isNotificationOn": false, - "id": 23 - } - ], - "approvalOptions": { - "requiredApproverCount": null, - "releaseCreatorCanBeApprover": false, - "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, - "enforceIdentityRevalidation": false, - "timeoutInMinutes": 0, - "executionOrder": 1 - } - }, - "deployStep": { - "id": 24 - }, - "postDeployApprovals": { - "approvals": [ - { - "rank": 1, - "isAutomated": true, - "isNotificationOn": false, - "id": 27 - } - ], - "approvalOptions": { - "requiredApproverCount": null, - "releaseCreatorCanBeApprover": false, - "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, - "enforceIdentityRevalidation": false, - "timeoutInMinutes": 0, - "executionOrder": 2 - } - }, - "deployPhases": [ - { - "deploymentInput": { - "parallelExecution": { - "parallelExecutionType": 0 - }, - "skipArtifactsDownload": false, - "artifactsDownloadInput": { - "downloadInputs": [] - }, - "queueId": 18, - "demands": [], - "enableAccessToken": false, - "timeoutInMinutes": 0, - "jobCancelTimeoutInMinutes": 1, - "condition": "succeeded()", - "overrideInputs": {} - }, - "rank": 1, - "phaseType": 1, - "name": "Agent job", - "refName": null, - "workflowTasks": [ - { - "environment": {}, - "taskId": "33c63b11-352b-45a2-ba1b-54cb568a29ca", - "version": "0.*", - "name": "Use Python 3.6", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "versionSpec": "3.6", - "addToPath": "true", - "architecture": "x64" - } - }, - { - "environment": {}, - "taskId": "6c731c3c-3c68-459a-a5c9-bde6e6595b5b", - "version": "3.*", - "name": "Bash Script", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "targetType": "filePath", - "filePath": "$(System.DefaultWorkingDirectory)/DevOpsForAI/devops-for-ai/environment_setup/install_requirements.sh", - "arguments": "", - "script": "# Write your commands here\n\n# Use the environment variables input below to pass secret variables to this script", - "workingDirectory": "$(System.DefaultWorkingDirectory)/DevOpsForAI/devops-for-ai/environment_setup", - "failOnStderr": "false", - "noProfile": "true", - "noRc": "true" - } - }, - { - "environment": {}, - "taskId": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", - "version": "2.*", - "name": "Login to Azure Subscription", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "script": "az login --service-principal -u $(spidentity) -p $(spsecret) --tenant $(sptenant)", - "workingDirectory": "", - "failOnStderr": "false" - } - }, - { - "environment": {}, - "taskId": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", - "version": "2.*", - "name": "Get Latest Scoring Image Name & Version", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "script": "python aml_service/34-GetScoringImageName.py", - "workingDirectory": "$(System.DefaultWorkingDirectory)/DevOpsForAI/devops-for-ai", - "failOnStderr": "false" - } - }, - { - "environment": {}, - "taskId": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", - "version": "2.*", - "name": "Deploy to AKS", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "script": "python aml_service/51-deployOnAks.py", - "workingDirectory": "$(System.DefaultWorkingDirectory)/DevOpsForAI/devops-for-ai", - "failOnStderr": "false" - } - }, - { - "environment": {}, - "taskId": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", - "version": "2.*", - "name": "Test AKS endpoint", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "script": "python aml_service/61-AksWebserviceTest.py", - "workingDirectory": "$(System.DefaultWorkingDirectory)/DevOpsForAI/devops-for-ai", - "failOnStderr": "false" - } - } - ] - } - ], - "environmentOptions": { - "emailNotificationType": "OnlyOnFailure", - "emailRecipients": "release.environment.owner;release.creator", - "skipArtifactsDownload": false, - "timeoutInMinutes": 0, - "enableAccessToken": false, - "publishDeploymentStatus": true, - "badgeEnabled": false, - "autoLinkWorkItems": false, - "pullRequestDeploymentEnabled": false - }, - "demands": [], - "conditions": [], - "executionPolicy": { - "concurrencyCount": 1, - "queueDepthCount": 0 - }, - "schedules": [], - "currentRelease": { - "id": 0, - "url": "https://youaccount.vsrm.visualstudio.com/c9414c5b-b8f8-4d50-a8bf-eae8dbbb6a2a/_apis/Release/releases/0", - "_links": {} - }, - "retentionPolicy": { - "daysToKeep": 30, - "releasesToKeep": 3, - "retainBuild": true - }, - "processParameters": {}, - "properties": {}, - "preDeploymentGates": { - "id": 0, - "gatesOptions": null, - "gates": [] - }, - "postDeploymentGates": { - "id": 0, - "gatesOptions": null, - "gates": [] - }, - "environmentTriggers": [], - "badgeUrl": "https://youaccount.vsrm.visualstudio.com/_apis/public/Release/badge/c9414c5b-b8f8-4d50-a8bf-eae8dbbb6a2a/5/9" - } - ], - "artifacts": [], - "triggers": [], - "releaseNameFormat": "Release-$(rev:r)", - "tags": [], - "pipelineProcess": { - "type": 1 - }, - "properties": { - "DefinitionCreationSource": { - "$type": "System.String", - "$value": "ReleaseImport" - } - }, - "id": 5, - "name": "releasedeploymentpipeline", - "path": "\\", - "projectReference": null, - "url": "https://youaccount.vsrm.visualstudio.com/c9414c5b-b8f8-4d50-a8bf-eae8dbbb6a2a/_apis/Release/definitions/5", - "_links": { - "self": { - "href": "https://youaccount.vsrm.visualstudio.com/c9414c5b-b8f8-4d50-a8bf-eae8dbbb6a2a/_apis/Release/definitions/5" - }, - "web": { - "href": "https://youaccount.visualstudio.com/c9414c5b-b8f8-4d50-a8bf-eae8dbbb6a2a/_release?definitionId=5" - } - } -} \ No newline at end of file diff --git a/release-pipelines/retrainingtrigger.json b/release-pipelines/retrainingtrigger.json deleted file mode 100644 index 1bcba3ce..00000000 --- a/release-pipelines/retrainingtrigger.json +++ /dev/null @@ -1,291 +0,0 @@ -{ - "source": 2, - "revision": 1, - "description": null, - "createdBy": { - "displayName": "User Name", - "url": "https://spsprodcus3.vssps.visualstudio.com/A127dc0c3-e10b-4004-a104-fa5be489bed1/_apis/Identities/af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "_links": { - "avatar": { - "href": "https://dev.azure.com/userorg/_apis/GraphProfile/MemberAvatars/aad.ZmZhYjg5YzEtYmIxNC03NGRiLTk3NTAtZDBlMzQ2NGQwNjU0" - } - }, - "id": "af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "uniqueName": "user@email.com", - "imageUrl": "https://dev.azure.com/userorg/_api/_common/identityImage?id=af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "descriptor": "aad.ZmZhYjg5YzEtYmIxNC03NGRiLTk3NTAtZDBlMzQ2NGQwNjU0" - }, - "createdOn": "2019-03-29T01:48:19.893Z", - "modifiedBy": { - "displayName": "User Name", - "url": "https://spsprodcus3.vssps.visualstudio.com/A127dc0c3-e10b-4004-a104-fa5be489bed1/_apis/Identities/af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "_links": { - "avatar": { - "href": "https://dev.azure.com/userorg/_apis/GraphProfile/MemberAvatars/aad.ZmZhYjg5YzEtYmIxNC03NGRiLTk3NTAtZDBlMzQ2NGQwNjU0" - } - }, - "id": "af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "uniqueName": "user@email.com", - "imageUrl": "https://dev.azure.com/userorg/_api/_common/identityImage?id=af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "descriptor": "aad.ZmZhYjg5YzEtYmIxNC03NGRiLTk3NTAtZDBlMzQ2NGQwNjU0" - }, - "modifiedOn": "2019-03-29T01:48:19.893Z", - "isDeleted": false, - "variables": {}, - "variableGroups": [ - 7 - ], - "environments": [ - { - "id": 9, - "name": "Retrain", - "rank": 1, - "owner": { - "displayName": "User Name", - "url": "https://spsprodcus3.vssps.visualstudio.com/A127dc0c3-e10b-4004-a104-fa5be489bed1/_apis/Identities/af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "_links": { - "avatar": { - "href": "https://dev.azure.com/userorg/_apis/GraphProfile/MemberAvatars/aad.ZmZhYjg5YzEtYmIxNC03NGRiLTk3NTAtZDBlMzQ2NGQwNjU0" - } - }, - "id": "af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "uniqueName": "user@email.com", - "imageUrl": "https://dev.azure.com/userorg/_api/_common/identityImage?id=af1dae6a-5d55-49bb-a1a1-8e5db902dc1c", - "descriptor": "aad.ZmZhYjg5YzEtYmIxNC03NGRiLTk3NTAtZDBlMzQ2NGQwNjU0" - }, - "variables": {}, - "variableGroups": [], - "preDeployApprovals": { - "approvals": [ - { - "rank": 1, - "isAutomated": true, - "isNotificationOn": false, - "id": 29 - } - ], - "approvalOptions": { - "requiredApproverCount": null, - "releaseCreatorCanBeApprover": false, - "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, - "enforceIdentityRevalidation": false, - "timeoutInMinutes": 0, - "executionOrder": 1 - } - }, - "deployStep": { - "id": 30 - }, - "postDeployApprovals": { - "approvals": [ - { - "rank": 1, - "isAutomated": true, - "isNotificationOn": false, - "id": 31 - } - ], - "approvalOptions": { - "requiredApproverCount": null, - "releaseCreatorCanBeApprover": false, - "autoTriggeredAndPreviousEnvironmentApprovedCanBeSkipped": false, - "enforceIdentityRevalidation": false, - "timeoutInMinutes": 0, - "executionOrder": 2 - } - }, - "deployPhases": [ - { - "deploymentInput": { - "parallelExecution": { - "parallelExecutionType": 0 - }, - "skipArtifactsDownload": false, - "artifactsDownloadInput": { - "downloadInputs": [] - }, - "queueId": 6, - "demands": [], - "enableAccessToken": false, - "timeoutInMinutes": 0, - "jobCancelTimeoutInMinutes": 1, - "condition": "succeeded()", - "overrideInputs": {} - }, - "rank": 1, - "phaseType": 1, - "name": "Agent job", - "refName": null, - "workflowTasks": [ - { - "environment": {}, - "taskId": "33c63b11-352b-45a2-ba1b-54cb568a29ca", - "version": "0.*", - "name": "Use Python 3.6", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "versionSpec": "3.6", - "addToPath": "true", - "architecture": "x64" - } - }, - { - "environment": {}, - "taskId": "6c731c3c-3c68-459a-a5c9-bde6e6595b5b", - "version": "3.*", - "name": "Install Requirements", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "targetType": "filePath", - "filePath": "$(System.DefaultWorkingDirectory)/DevOpsForAI/devops-for-ai/environment_setup/install_requirements.sh", - "arguments": "", - "script": "# Write your commands here\n\n# Use the environment variables input below to pass secret variables to this script", - "workingDirectory": "$(System.DefaultWorkingDirectory)/DevOpsForAI/devops-for-ai/environment_setup", - "failOnStderr": "false", - "noProfile": "true", - "noRc": "true" - } - }, - { - "environment": {}, - "taskId": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", - "version": "2.*", - "name": "Login to Azure Subscription", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "script": "az login --service-principal -u $(spidentity) -p $(spsecret) --tenant $(sptenant)", - "workingDirectory": "", - "failOnStderr": "false" - } - }, - { - "environment": {}, - "taskId": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", - "version": "2.*", - "name": "Run AML Pipeline", - "refName": "", - "enabled": true, - "alwaysRun": false, - "continueOnError": false, - "timeoutInMinutes": 0, - "definitionType": "task", - "overrideInputs": {}, - "condition": "succeeded()", - "inputs": { - "script": "python aml_service/05-TriggerAmlPipeline.py", - "workingDirectory": "$(System.DefaultWorkingDirectory)/DevOpsForAI/devops-for-ai", - "failOnStderr": "false" - } - } - ] - } - ], - "environmentOptions": { - "emailNotificationType": "OnlyOnFailure", - "emailRecipients": "release.environment.owner;release.creator", - "skipArtifactsDownload": false, - "timeoutInMinutes": 0, - "enableAccessToken": false, - "publishDeploymentStatus": true, - "badgeEnabled": false, - "autoLinkWorkItems": false, - "pullRequestDeploymentEnabled": false - }, - "demands": [], - "conditions": [ - { - "name": "ReleaseStarted", - "conditionType": 1, - "value": "" - } - ], - "executionPolicy": { - "concurrencyCount": 1, - "queueDepthCount": 0 - }, - "schedules": [], - "currentRelease": { - "id": 0, - "url": "https://vsrm.dev.azure.com/userorg/420d3eaf-7dbb-46cb-a7c9-93662c745570/_apis/Release/releases/0", - "_links": {} - }, - "retentionPolicy": { - "daysToKeep": 30, - "releasesToKeep": 3, - "retainBuild": true - }, - "processParameters": {}, - "properties": {}, - "preDeploymentGates": { - "id": 0, - "gatesOptions": null, - "gates": [] - }, - "postDeploymentGates": { - "id": 0, - "gatesOptions": null, - "gates": [] - }, - "environmentTriggers": [], - "badgeUrl": "https://vsrm.dev.azure.com/userorg/_apis/public/Release/badge/420d3eaf-7dbb-46cb-a7c9-93662c745570/6/9" - } - ], - "artifacts": [], - "triggers": [ - { - "schedule": { - "jobId": "5efd6865-0305-493a-9ff9-08995bbb72e5", - "timeZoneId": "UTC", - "startHours": 3, - "startMinutes": 0, - "daysToRelease": 21 - }, - "triggerType": 2 - } - ], - "releaseNameFormat": "Release-$(rev:r)", - "tags": [], - "pipelineProcess": { - "type": 1 - }, - "properties": { - "DefinitionCreationSource": { - "$type": "System.String", - "$value": "ReleaseClone" - } - }, - "id": 6, - "name": "retrainingtriggerpipeline", - "path": "\\", - "projectReference": null, - "url": "https://vsrm.dev.azure.com/userorg/420d3eaf-7dbb-46cb-a7c9-93662c745570/_apis/Release/definitions/6", - "_links": { - "self": { - "href": "https://vsrm.dev.azure.com/userorg/420d3eaf-7dbb-46cb-a7c9-93662c745570/_apis/Release/definitions/6" - }, - "web": { - "href": "https://dev.azure.com/userorg/420d3eaf-7dbb-46cb-a7c9-93662c745570/_release?definitionId=6" - } - } -} \ No newline at end of file diff --git a/tests/unit/code_test.py b/tests/unit/code_test.py new file mode 100644 index 00000000..bcdf5e3e --- /dev/null +++ b/tests/unit/code_test.py @@ -0,0 +1,25 @@ +import sys +import os +sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 +from workspace import get_workspace + + +# Just an example of a unit test against +# a utility function common_scoring.next_saturday +def test_get_workspace(): + workspace_name = os.environ.get("AML_WORKSPACE_NAME") + resource_group = os.environ.get("RESOURCE_GROUP") + subscription_id = os.environ.get("SUBSCRIPTION_ID") + tenant_id = os.environ.get("TENANT_ID") + app_id = os.environ.get("SP_APP_ID") + app_secret = os.environ.get("SP_APP_SECRET") + + aml_workspace = get_workspace( + workspace_name, + resource_group, + subscription_id, + tenant_id, + app_id, + app_secret) + + assert aml_workspace.name == workspace_name diff --git a/tests/unit/data_test.py b/tests/unit/data_test.py index ad5c28ba..8b40b8bc 100644 --- a/tests/unit/data_test.py +++ b/tests/unit/data_test.py @@ -34,7 +34,8 @@ def get_absPath(filename): """Returns the path of the notebooks folder""" path = os.path.abspath( os.path.join( - os.path.dirname(__file__), os.path.pardir, os.path.pardir, "data", filename + os.path.dirname( + __file__), os.path.pardir, os.path.pardir, "data", filename ) ) return path @@ -119,6 +120,8 @@ def test_check_distribution(): mean = np.mean(dataset.values, axis=0) std = np.mean(dataset.values, axis=0) assert ( - np.sum(abs(mean - historical_mean) > shift_tolerance * abs(historical_mean)) - or np.sum(abs(std - historical_std) > shift_tolerance * abs(historical_std)) > 0 + np.sum(abs(mean - historical_mean) > + shift_tolerance * abs(historical_mean)) + or np.sum(abs(std - historical_std) > + shift_tolerance * abs(historical_std)) > 0 )