From 5011b69f25c8ccf4afd6c58fbafb6958282d6ce5 Mon Sep 17 00:00:00 2001 From: DaDa0413 Date: Sat, 22 Jan 2022 15:59:31 +0800 Subject: [PATCH 1/7] Add CI yaml file --- .pipelines/azdo-base-pipeline.yml | 26 ++++++++++++++++++ .pipelines/azdo-ci-build-train.yml | 43 ++++++++++++++++++++++++++++++ .pipelines/azdo-pr-build-train.yml | 18 +++++++++++++ 3 files changed, 87 insertions(+) create mode 100644 .pipelines/azdo-base-pipeline.yml create mode 100644 .pipelines/azdo-ci-build-train.yml create mode 100644 .pipelines/azdo-pr-build-train.yml diff --git a/.pipelines/azdo-base-pipeline.yml b/.pipelines/azdo-base-pipeline.yml new file mode 100644 index 00000000..5d60696c --- /dev/null +++ b/.pipelines/azdo-base-pipeline.yml @@ -0,0 +1,26 @@ +# this pipeline should be ignored for now +parameters: + pipelineType: 'training' + +steps: +- script: | + flake8 --output-file=$(Build.BinariesDirectory)/lint-testresults.xml --format junit-xml + workingDirectory: '$(Build.SourcesDirectory)' + displayName: 'Run code quality tests' + enabled: 'true' + +- script: | + pytest --junitxml=$(Build.BinariesDirectory)/unit-testresults.xml $(Build.SourcesDirectory)/tests/unit + displayName: 'Run unit tests' + enabled: 'true' + env: + SP_APP_SECRET: '$(SP_APP_SECRET)' + +- task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testResultsFiles: '$(Build.BinariesDirectory)/*-testresults.xml' + testRunTitle: 'Linting & Unit tests' + failTaskOnFailedTests: true + displayName: 'Publish linting and unit test results' + enabled: 'true' \ No newline at end of file diff --git a/.pipelines/azdo-ci-build-train.yml b/.pipelines/azdo-ci-build-train.yml new file mode 100644 index 00000000..59b2c86b --- /dev/null +++ b/.pipelines/azdo-ci-build-train.yml @@ -0,0 +1,43 @@ +pr: none +trigger: + branches: + include: + - master + +pool: + vmImage: 'ubuntu-latest' + +container: mcr.microsoft.com/mlops/python:latest + + +variables: +- group: devopsforai-aml-vg + + +steps: +- template: azdo-base-pipeline.yml + +- bash: | + # Invoke the Python building and publishing a training pipeline + python3 $(Build.SourcesDirectory)/ml_service/pipelines/build_train_pipeline.py + failOnStderr: 'false' + env: + SP_APP_SECRET: '$(SP_APP_SECRET)' + displayName: 'Train model using AML with Remote Compute' + enabled: 'true' + +- task: CopyFiles@2 + displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + Contents: | + ml_service/pipelines/?(run_train_pipeline.py|*.json) + code/scoring/** +- task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact' + inputs: + ArtifactName: 'mlops-pipelines' + publishLocation: 'container' + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + TargetPath: '$(Build.ArtifactStagingDirectory)' \ No newline at end of file diff --git a/.pipelines/azdo-pr-build-train.yml b/.pipelines/azdo-pr-build-train.yml new file mode 100644 index 00000000..159d85c2 --- /dev/null +++ b/.pipelines/azdo-pr-build-train.yml @@ -0,0 +1,18 @@ +trigger: none +pr: + branches: + include: + - master + +pool: + vmImage: 'ubuntu-latest' + +container: mcr.microsoft.com/mlops/python:latest + + +variables: +- group: devopsforai-aml-vg + + +steps: +- template: azdo-base-pipeline.yml \ No newline at end of file From 146ddfd6c006b964550d7fddf791b78f3fc3ccbd Mon Sep 17 00:00:00 2001 From: DaDa0413 Date: Sat, 22 Jan 2022 16:33:13 +0800 Subject: [PATCH 2/7] Use mypool to avoid request parallel --- .pipelines/azdo-ci-build-train.yml | 5 +++-- .pipelines/azdo-pr-build-train.yml | 5 +++-- environment_setup/iac-remove-environment-pipeline.yml | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.pipelines/azdo-ci-build-train.yml b/.pipelines/azdo-ci-build-train.yml index 59b2c86b..804444ee 100644 --- a/.pipelines/azdo-ci-build-train.yml +++ b/.pipelines/azdo-ci-build-train.yml @@ -4,8 +4,9 @@ trigger: include: - master -pool: - vmImage: 'ubuntu-latest' +# pool: +# vmImage: 'ubuntu-latest' +pool: MyPool container: mcr.microsoft.com/mlops/python:latest diff --git a/.pipelines/azdo-pr-build-train.yml b/.pipelines/azdo-pr-build-train.yml index 159d85c2..29dfb123 100644 --- a/.pipelines/azdo-pr-build-train.yml +++ b/.pipelines/azdo-pr-build-train.yml @@ -4,8 +4,9 @@ pr: include: - master -pool: - vmImage: 'ubuntu-latest' +# pool: +# vmImage: 'ubuntu-latest' +pool: MyPool container: mcr.microsoft.com/mlops/python:latest diff --git a/environment_setup/iac-remove-environment-pipeline.yml b/environment_setup/iac-remove-environment-pipeline.yml index 39ff9e7a..6e02dc5a 100644 --- a/environment_setup/iac-remove-environment-pipeline.yml +++ b/environment_setup/iac-remove-environment-pipeline.yml @@ -2,8 +2,9 @@ pr: none trigger: none -pool: - vmImage: 'ubuntu-latest' +# pool: +# vmImage: 'ubuntu-latest' +pool: MyPool variables: - group: devopsforai-aml-vg From 663bd92c15e1c5e9de4c18e8811324770b3d4046 Mon Sep 17 00:00:00 2001 From: DaDa0413 Date: Sat, 22 Jan 2022 16:37:05 +0800 Subject: [PATCH 3/7] Add iac-create-environment.yml --- environment_setup/iac-create-environment.yml | 28 ++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 environment_setup/iac-create-environment.yml diff --git a/environment_setup/iac-create-environment.yml b/environment_setup/iac-create-environment.yml new file mode 100644 index 00000000..502c9721 --- /dev/null +++ b/environment_setup/iac-create-environment.yml @@ -0,0 +1,28 @@ +trigger: + branches: + include: + - master + paths: + include: + - environment_setup/arm-templates/* + +# pool: +# vmImage: 'ubuntu-latest' +pool: MyPool + +variables: +- group: devopsforai-aml-vg + + +steps: +- task: AzureResourceGroupDeployment@2 + inputs: + azureSubscription: 'AzureResourceConnection' + action: 'Create Or Update Resource Group' + resourceGroupName: '$(BASE_NAME)-AML-RG' + location: $(LOCATION) + templateLocation: 'Linked artifact' + csmFile: '$(Build.SourcesDirectory)/environment_setup/arm-templates/cloud-environment.json' + overrideParameters: '-baseName $(BASE_NAME)' + deploymentMode: 'Incremental' + displayName: 'Deploy MLOps resources to Azure' \ No newline at end of file From c7844feb505f32dc342a16d8de0edcd844637cfd Mon Sep 17 00:00:00 2001 From: DaDa0413 Date: Sat, 22 Jan 2022 16:44:30 +0800 Subject: [PATCH 4/7] Add iac-remove-environment.yml --- environment_setup/iac-remove-environment.yml | 24 ++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 environment_setup/iac-remove-environment.yml diff --git a/environment_setup/iac-remove-environment.yml b/environment_setup/iac-remove-environment.yml new file mode 100644 index 00000000..52c199ce --- /dev/null +++ b/environment_setup/iac-remove-environment.yml @@ -0,0 +1,24 @@ +trigger: + branches: + include: + - master + paths: + include: + - environment_setup/arm-templates/* + +# pool: +# vmImage: 'ubuntu-latest' +pool: MyPool + +variables: +- group: devopsforai-aml-vg + + +steps: +- task: AzureResourceGroupDeployment@2 + inputs: + azureSubscription: 'AzureResourceConnection' + action: 'DeleteRG' + resourceGroupName: '$(BASE_NAME)-AML-RG' + location: $(LOCATION) + displayName: 'Delete resources in Azure' \ No newline at end of file From 603b2ce94c9f365ec918cf86dd15cbe674c464d2 Mon Sep 17 00:00:00 2001 From: DaDa0413 Date: Sat, 22 Jan 2022 16:48:32 +0800 Subject: [PATCH 5/7] Modify azure-resource-connection name --- environment_setup/iac-create-environment.yml | 2 +- environment_setup/iac-remove-environment.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment_setup/iac-create-environment.yml b/environment_setup/iac-create-environment.yml index 502c9721..d44b02b4 100644 --- a/environment_setup/iac-create-environment.yml +++ b/environment_setup/iac-create-environment.yml @@ -17,7 +17,7 @@ variables: steps: - task: AzureResourceGroupDeployment@2 inputs: - azureSubscription: 'AzureResourceConnection' + azureSubscription: 'azure-resource-connection' action: 'Create Or Update Resource Group' resourceGroupName: '$(BASE_NAME)-AML-RG' location: $(LOCATION) diff --git a/environment_setup/iac-remove-environment.yml b/environment_setup/iac-remove-environment.yml index 52c199ce..b7fd3b1b 100644 --- a/environment_setup/iac-remove-environment.yml +++ b/environment_setup/iac-remove-environment.yml @@ -17,7 +17,7 @@ variables: steps: - task: AzureResourceGroupDeployment@2 inputs: - azureSubscription: 'AzureResourceConnection' + azureSubscription: 'azure-resource-connection' action: 'DeleteRG' resourceGroupName: '$(BASE_NAME)-AML-RG' location: $(LOCATION) From 59b287b250eebf1df6a300c3ab0d866cad409e47 Mon Sep 17 00:00:00 2001 From: DaDa0413 Date: Sat, 22 Jan 2022 17:22:57 +0800 Subject: [PATCH 6/7] Use default pool --- .pipelines/azdo-ci-build-train.yml | 2 +- .pipelines/azdo-pr-build-train.yml | 2 +- environment_setup/iac-create-environment.yml | 2 +- environment_setup/iac-remove-environment-pipeline.yml | 2 +- environment_setup/iac-remove-environment.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.pipelines/azdo-ci-build-train.yml b/.pipelines/azdo-ci-build-train.yml index 804444ee..bcebb8ff 100644 --- a/.pipelines/azdo-ci-build-train.yml +++ b/.pipelines/azdo-ci-build-train.yml @@ -6,7 +6,7 @@ trigger: # pool: # vmImage: 'ubuntu-latest' -pool: MyPool +pool: Default container: mcr.microsoft.com/mlops/python:latest diff --git a/.pipelines/azdo-pr-build-train.yml b/.pipelines/azdo-pr-build-train.yml index 29dfb123..dc99bf0a 100644 --- a/.pipelines/azdo-pr-build-train.yml +++ b/.pipelines/azdo-pr-build-train.yml @@ -6,7 +6,7 @@ pr: # pool: # vmImage: 'ubuntu-latest' -pool: MyPool +pool: Default container: mcr.microsoft.com/mlops/python:latest diff --git a/environment_setup/iac-create-environment.yml b/environment_setup/iac-create-environment.yml index d44b02b4..1c9bd429 100644 --- a/environment_setup/iac-create-environment.yml +++ b/environment_setup/iac-create-environment.yml @@ -8,7 +8,7 @@ trigger: # pool: # vmImage: 'ubuntu-latest' -pool: MyPool +pool: Default variables: - group: devopsforai-aml-vg diff --git a/environment_setup/iac-remove-environment-pipeline.yml b/environment_setup/iac-remove-environment-pipeline.yml index 6e02dc5a..90f70039 100644 --- a/environment_setup/iac-remove-environment-pipeline.yml +++ b/environment_setup/iac-remove-environment-pipeline.yml @@ -4,7 +4,7 @@ trigger: none # pool: # vmImage: 'ubuntu-latest' -pool: MyPool +pool: Default variables: - group: devopsforai-aml-vg diff --git a/environment_setup/iac-remove-environment.yml b/environment_setup/iac-remove-environment.yml index b7fd3b1b..f79c1a18 100644 --- a/environment_setup/iac-remove-environment.yml +++ b/environment_setup/iac-remove-environment.yml @@ -8,7 +8,7 @@ trigger: # pool: # vmImage: 'ubuntu-latest' -pool: MyPool +pool: Default variables: - group: devopsforai-aml-vg From 106321f81253092fa5abe02d6e5acc449897d148 Mon Sep 17 00:00:00 2001 From: DaDa0413 Date: Sat, 22 Jan 2022 23:14:41 +0800 Subject: [PATCH 7/7] Use default agent in docker pipeline --- environment_setup/docker-image-pipeline.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/environment_setup/docker-image-pipeline.yml b/environment_setup/docker-image-pipeline.yml index 9f7361ac..a8ba95f2 100644 --- a/environment_setup/docker-image-pipeline.yml +++ b/environment_setup/docker-image-pipeline.yml @@ -2,8 +2,9 @@ resources: - repo: self -pool: - vmImage: 'ubuntu-latest' +# pool: +# vmImage: 'ubuntu-latest' +pool: Default trigger: branches: