diff --git a/.pipelines/azdo-base-pipeline.yml b/.pipelines/azdo-base-pipeline.yml new file mode 100644 index 00000000..5d60696c --- /dev/null +++ b/.pipelines/azdo-base-pipeline.yml @@ -0,0 +1,26 @@ +# this pipeline should be ignored for now +parameters: + pipelineType: 'training' + +steps: +- script: | + flake8 --output-file=$(Build.BinariesDirectory)/lint-testresults.xml --format junit-xml + workingDirectory: '$(Build.SourcesDirectory)' + displayName: 'Run code quality tests' + enabled: 'true' + +- script: | + pytest --junitxml=$(Build.BinariesDirectory)/unit-testresults.xml $(Build.SourcesDirectory)/tests/unit + displayName: 'Run unit tests' + enabled: 'true' + env: + SP_APP_SECRET: '$(SP_APP_SECRET)' + +- task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testResultsFiles: '$(Build.BinariesDirectory)/*-testresults.xml' + testRunTitle: 'Linting & Unit tests' + failTaskOnFailedTests: true + displayName: 'Publish linting and unit test results' + enabled: 'true' \ No newline at end of file diff --git a/.pipelines/azdo-ci-build-train.yml b/.pipelines/azdo-ci-build-train.yml new file mode 100644 index 00000000..bcebb8ff --- /dev/null +++ b/.pipelines/azdo-ci-build-train.yml @@ -0,0 +1,44 @@ +pr: none +trigger: + branches: + include: + - master + +# pool: +# vmImage: 'ubuntu-latest' +pool: Default + +container: mcr.microsoft.com/mlops/python:latest + + +variables: +- group: devopsforai-aml-vg + + +steps: +- template: azdo-base-pipeline.yml + +- bash: | + # Invoke the Python building and publishing a training pipeline + python3 $(Build.SourcesDirectory)/ml_service/pipelines/build_train_pipeline.py + failOnStderr: 'false' + env: + SP_APP_SECRET: '$(SP_APP_SECRET)' + displayName: 'Train model using AML with Remote Compute' + enabled: 'true' + +- task: CopyFiles@2 + displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + Contents: | + ml_service/pipelines/?(run_train_pipeline.py|*.json) + code/scoring/** +- task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact' + inputs: + ArtifactName: 'mlops-pipelines' + publishLocation: 'container' + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + TargetPath: '$(Build.ArtifactStagingDirectory)' \ No newline at end of file diff --git a/.pipelines/azdo-pr-build-train.yml b/.pipelines/azdo-pr-build-train.yml new file mode 100644 index 00000000..dc99bf0a --- /dev/null +++ b/.pipelines/azdo-pr-build-train.yml @@ -0,0 +1,19 @@ +trigger: none +pr: + branches: + include: + - master + +# pool: +# vmImage: 'ubuntu-latest' +pool: Default + +container: mcr.microsoft.com/mlops/python:latest + + +variables: +- group: devopsforai-aml-vg + + +steps: +- template: azdo-base-pipeline.yml \ No newline at end of file diff --git a/environment_setup/docker-image-pipeline.yml b/environment_setup/docker-image-pipeline.yml index 9f7361ac..a8ba95f2 100644 --- a/environment_setup/docker-image-pipeline.yml +++ b/environment_setup/docker-image-pipeline.yml @@ -2,8 +2,9 @@ resources: - repo: self -pool: - vmImage: 'ubuntu-latest' +# pool: +# vmImage: 'ubuntu-latest' +pool: Default trigger: branches: diff --git a/environment_setup/iac-create-environment.yml b/environment_setup/iac-create-environment.yml new file mode 100644 index 00000000..1c9bd429 --- /dev/null +++ b/environment_setup/iac-create-environment.yml @@ -0,0 +1,28 @@ +trigger: + branches: + include: + - master + paths: + include: + - environment_setup/arm-templates/* + +# pool: +# vmImage: 'ubuntu-latest' +pool: Default + +variables: +- group: devopsforai-aml-vg + + +steps: +- task: AzureResourceGroupDeployment@2 + inputs: + azureSubscription: 'azure-resource-connection' + action: 'Create Or Update Resource Group' + resourceGroupName: '$(BASE_NAME)-AML-RG' + location: $(LOCATION) + templateLocation: 'Linked artifact' + csmFile: '$(Build.SourcesDirectory)/environment_setup/arm-templates/cloud-environment.json' + overrideParameters: '-baseName $(BASE_NAME)' + deploymentMode: 'Incremental' + displayName: 'Deploy MLOps resources to Azure' \ No newline at end of file diff --git a/environment_setup/iac-remove-environment-pipeline.yml b/environment_setup/iac-remove-environment-pipeline.yml index 39ff9e7a..90f70039 100644 --- a/environment_setup/iac-remove-environment-pipeline.yml +++ b/environment_setup/iac-remove-environment-pipeline.yml @@ -2,8 +2,9 @@ pr: none trigger: none -pool: - vmImage: 'ubuntu-latest' +# pool: +# vmImage: 'ubuntu-latest' +pool: Default variables: - group: devopsforai-aml-vg diff --git a/environment_setup/iac-remove-environment.yml b/environment_setup/iac-remove-environment.yml new file mode 100644 index 00000000..f79c1a18 --- /dev/null +++ b/environment_setup/iac-remove-environment.yml @@ -0,0 +1,24 @@ +trigger: + branches: + include: + - master + paths: + include: + - environment_setup/arm-templates/* + +# pool: +# vmImage: 'ubuntu-latest' +pool: Default + +variables: +- group: devopsforai-aml-vg + + +steps: +- task: AzureResourceGroupDeployment@2 + inputs: + azureSubscription: 'azure-resource-connection' + action: 'DeleteRG' + resourceGroupName: '$(BASE_NAME)-AML-RG' + location: $(LOCATION) + displayName: 'Delete resources in Azure' \ No newline at end of file