diff --git a/.env.example b/.env.example index 47311d1e..149621af 100644 --- a/.env.example +++ b/.env.example @@ -15,7 +15,7 @@ WORKSPACE_NAME = 'mlops-aml-ws' EXPERIMENT_NAME = 'mlopspython' # AML Compute Cluster Config -AML_ENV_NAME='diabetes_regression_training_env' +AML_ENV_NAME='automobile_training_env' AML_ENV_TRAIN_CONDA_DEP_FILE="conda_dependencies.yml" AML_COMPUTE_CLUSTER_NAME = 'train-cluster' AML_COMPUTE_CLUSTER_CPU_SKU = 'STANDARD_DS2_V2' @@ -23,7 +23,7 @@ AML_CLUSTER_MAX_NODES = '4' AML_CLUSTER_MIN_NODES = '0' AML_CLUSTER_PRIORITY = 'lowpriority' # Training Config -MODEL_NAME = 'diabetes_regression_model.pkl' +MODEL_NAME = 'automobile_model.pkl' MODEL_VERSION = '1' TRAIN_SCRIPT_PATH = 'training/train_aml.py' @@ -33,8 +33,8 @@ TRAINING_PIPELINE_NAME = 'Training Pipeline' MODEL_PATH = '' EVALUATE_SCRIPT_PATH = 'evaluate/evaluate_model.py' REGISTER_SCRIPT_PATH = 'register/register_model.py' -SOURCES_DIR_TRAIN = 'diabetes_regression' -DATASET_NAME = 'diabetes_ds' +SOURCES_DIR_TRAIN = 'automobile' +DATASET_NAME = 'automobile_ds' DATASET_VERSION = 'latest' # Optional. Set it if you have configured non default datastore to point to your data DATASTORE_NAME = '' @@ -61,8 +61,8 @@ USE_GPU_FOR_SCORING = "false" AML_ENV_SCORE_CONDA_DEP_FILE="conda_dependencies_scoring.yml" AML_ENV_SCORECOPY_CONDA_DEP_FILE="conda_dependencies_scorecopy.yml" # AML Compute Cluster Config for parallel batch scoring -AML_ENV_NAME_SCORING='diabetes_regression_scoring_env' -AML_ENV_NAME_SCORE_COPY='diabetes_regression_score_copy_env' +AML_ENV_NAME_SCORING='automobile_scoring_env' +AML_ENV_NAME_SCORE_COPY='automobile_score_copy_env' AML_COMPUTE_CLUSTER_NAME_SCORING = 'score-cluster' AML_COMPUTE_CLUSTER_CPU_SKU_SCORING = 'STANDARD_DS2_V2' AML_CLUSTER_MAX_NODES_SCORING = '4' @@ -74,8 +74,8 @@ BATCHSCORE_COPY_SCRIPT_PATH = 'scoring/parallel_batchscore_copyoutput.py' SCORING_DATASTORE_INPUT_CONTAINER = 'input' -SCORING_DATASTORE_INPUT_FILENAME = 'diabetes_scoring_input.csv' +SCORING_DATASTORE_INPUT_FILENAME = 'automobile_scoring_input.csv' SCORING_DATASTORE_OUTPUT_CONTAINER = 'output' -SCORING_DATASTORE_OUTPUT_FILENAME = 'diabetes_scoring_output.csv' -SCORING_DATASET_NAME = 'diabetes_scoring_ds' -SCORING_PIPELINE_NAME = 'diabetes-scoring-pipeline' +SCORING_DATASTORE_OUTPUT_FILENAME = 'automobile_scoring_output.csv' +SCORING_DATASET_NAME = 'automobile_scoring_ds' +SCORING_PIPELINE_NAME = 'automobile-scoring-pipeline' diff --git a/.pipelines/abtest.yml b/.pipelines/abtest.yml index cf876181..17b5ceb3 100644 --- a/.pipelines/abtest.yml +++ b/.pipelines/abtest.yml @@ -18,7 +18,7 @@ trigger: - ml_service/util/smoke_test_scoring_service.py variables: -- template: diabetes_regression-variables-template.yml +- template: automobile-variables-template.yml - group: 'devopsforai-aml-vg' - name: 'helmVersion' value: 'v3.1.1' diff --git a/.pipelines/diabetes_regression-batchscoring-ci.yml b/.pipelines/automobile-batchscoring-ci.yml similarity index 87% rename from .pipelines/diabetes_regression-batchscoring-ci.yml rename to .pipelines/automobile-batchscoring-ci.yml index 1392fddb..a6f3733b 100644 --- a/.pipelines/diabetes_regression-batchscoring-ci.yml +++ b/.pipelines/automobile-batchscoring-ci.yml @@ -1,4 +1,4 @@ -# Continuous Integration (CI) pipeline that orchestrates the batch scoring of the diabetes_regression model. +# Continuous Integration (CI) pipeline that orchestrates the batch scoring of the automobile model. # Runtime parameters to select artifacts parameters: @@ -28,12 +28,12 @@ trigger: - master paths: include: - - diabetes_regression/scoring/parallel_batchscore.py - - ml_service/pipelines/diabetes_regression_build_parallel_batchscore_pipeline.py + - automobile/scoring/parallel_batchscore.py + - ml_service/pipelines/automobile_build_parallel_batchscore_pipeline.py - ml_service/pipelines/run_parallel_batchscore_pipeline.py variables: -- template: diabetes_regression-variables-template.yml +- template: automobile-variables-template.yml - group: devopsforai-aml-vg pool: @@ -49,7 +49,7 @@ stages: timeoutInMinutes: 0 steps: - template: code-quality-template.yml - - template: diabetes_regression-get-model-id-artifact-template.yml + - template: automobile-get-model-id-artifact-template.yml parameters: projectId: '$(resources.pipeline.model-train-ci.projectID)' pipelineId: '$(resources.pipeline.model-train-ci.pipelineID)' @@ -65,7 +65,7 @@ stages: set -e # fail on error export SUBSCRIPTION_ID=$(az account show --query id -o tsv) # Invoke the Python building and publishing a training pipeline - python -m ml_service.pipelines.diabetes_regression_build_parallel_batchscore_pipeline + python -m ml_service.pipelines.automobile_build_parallel_batchscore_pipeline env: SCORING_DATASTORE_ACCESS_KEY: $(SCORING_DATASTORE_ACCESS_KEY) diff --git a/.pipelines/diabetes_regression-cd.yml b/.pipelines/automobile-cd.yml similarity index 94% rename from .pipelines/diabetes_regression-cd.yml rename to .pipelines/automobile-cd.yml index a691cc47..885896cd 100644 --- a/.pipelines/diabetes_regression-cd.yml +++ b/.pipelines/automobile-cd.yml @@ -1,4 +1,4 @@ -# Continuous Integration (CI) pipeline that orchestrates the deployment of the diabetes_regression model. +# Continuous Integration (CI) pipeline that orchestrates the deployment of the automobile model. # Runtime parameters to select artifacts parameters: @@ -24,7 +24,7 @@ resources: - master variables: -- template: diabetes_regression-variables-template.yml +- template: automobile-variables-template.yml - group: devopsforai-aml-vg stages: @@ -38,7 +38,7 @@ stages: timeoutInMinutes: 0 steps: - download: none - - template: diabetes_regression-get-model-id-artifact-template.yml + - template: automobile-get-model-id-artifact-template.yml parameters: projectId: '$(resources.pipeline.model-train-ci.projectID)' pipelineId: '$(resources.pipeline.model-train-ci.pipelineID)' @@ -84,7 +84,7 @@ stages: container: mlops timeoutInMinutes: 0 steps: - - template: diabetes_regression-get-model-id-artifact-template.yml + - template: automobile-get-model-id-artifact-template.yml parameters: projectId: '$(resources.pipeline.model-train-ci.projectID)' pipelineId: '$(resources.pipeline.model-train-ci.pipelineID)' @@ -130,12 +130,12 @@ stages: container: mlops timeoutInMinutes: 0 steps: - - template: diabetes_regression-get-model-id-artifact-template.yml + - template: automobile-get-model-id-artifact-template.yml parameters: projectId: '$(resources.pipeline.model-train-ci.projectID)' pipelineId: '$(resources.pipeline.model-train-ci.pipelineID)' artifactBuildId: ${{ parameters.artifactBuildId }} - - template: diabetes_regression-package-model-template.yml + - template: automobile-package-model-template.yml parameters: modelId: $(MODEL_NAME):$(get_model.MODEL_VERSION) scoringScriptPath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/score.py' diff --git a/.pipelines/diabetes_regression-ci-image.yml b/.pipelines/automobile-ci-image.yml similarity index 77% rename from .pipelines/diabetes_regression-ci-image.yml rename to .pipelines/automobile-ci-image.yml index d7c925bf..2c1302f5 100644 --- a/.pipelines/diabetes_regression-ci-image.yml +++ b/.pipelines/automobile-ci-image.yml @@ -14,10 +14,10 @@ trigger: include: - ml_service/util/create_scoring_image.py - ml_service/util/Dockerfile - - diabetes_regression/scoring/ + - automobile/scoring/ exclude: - - diabetes_regression/scoring/deployment_config_aci.yml - - diabetes_regression/scoring/deployment_config_aks.yml + - automobile/scoring/deployment_config_aci.yml + - automobile/scoring/deployment_config_aks.yml pool: vmImage: 'ubuntu-latest' @@ -30,7 +30,7 @@ variables: value: 'scoring/scoreB.py' steps: -- template: diabetes_regression-package-model-template.yml +- template: automobile-package-model-template.yml parameters: modelId: $(MODEL_NAME):$(MODEL_VERSION) scoringScriptPath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/$(SCORE_SCRIPT)' diff --git a/.pipelines/diabetes_regression-ci.yml b/.pipelines/automobile-ci.yml similarity index 85% rename from .pipelines/diabetes_regression-ci.yml rename to .pipelines/automobile-ci.yml index 5a539af0..aadacc9e 100644 --- a/.pipelines/diabetes_regression-ci.yml +++ b/.pipelines/automobile-ci.yml @@ -1,4 +1,4 @@ -# Continuous Integration (CI) pipeline that orchestrates the training, evaluation, and registration of the diabetes_regression model. +# Continuous Integration (CI) pipeline that orchestrates the training, evaluation, and registration of the automobile model. resources: containers: @@ -12,13 +12,13 @@ trigger: - master paths: include: - - diabetes_regression/ - - ml_service/pipelines/diabetes_regression_build_train_pipeline.py - - ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py - - ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py + - automobile/ + - ml_service/pipelines/automobile_build_train_pipeline.py + - ml_service/pipelines/automobile_build_train_pipeline_with_r.py + - ml_service/pipelines/automobile_build_train_pipeline_with_r_on_dbricks.py variables: -- template: diabetes_regression-variables-template.yml +- template: automobile-variables-template.yml - group: devopsforai-aml-vg pool: @@ -43,7 +43,7 @@ stages: set -e # fail on error export SUBSCRIPTION_ID=$(az account show --query id -o tsv) # Invoke the Python building and publishing a training pipeline - python -m ml_service.pipelines.diabetes_regression_build_train_pipeline + python -m ml_service.pipelines.automobile_build_train_pipeline displayName: 'Publish Azure Machine Learning Pipeline' - stage: 'Trigger_AML_Pipeline' @@ -94,4 +94,4 @@ stages: container: mlops timeoutInMinutes: 0 steps: - - template: diabetes_regression-publish-model-artifact-template.yml + - template: automobile-publish-model-artifact-template.yml diff --git a/.pipelines/diabetes_regression-get-model-id-artifact-template.yml b/.pipelines/automobile-get-model-id-artifact-template.yml similarity index 100% rename from .pipelines/diabetes_regression-get-model-id-artifact-template.yml rename to .pipelines/automobile-get-model-id-artifact-template.yml diff --git a/.pipelines/diabetes_regression-package-model-template.yml b/.pipelines/automobile-package-model-template.yml similarity index 100% rename from .pipelines/diabetes_regression-package-model-template.yml rename to .pipelines/automobile-package-model-template.yml diff --git a/.pipelines/diabetes_regression-publish-model-artifact-template.yml b/.pipelines/automobile-publish-model-artifact-template.yml similarity index 100% rename from .pipelines/diabetes_regression-publish-model-artifact-template.yml rename to .pipelines/automobile-publish-model-artifact-template.yml diff --git a/.pipelines/diabetes_regression-variables-template.yml b/.pipelines/automobile-variables-template.yml similarity index 90% rename from .pipelines/diabetes_regression-variables-template.yml rename to .pipelines/automobile-variables-template.yml index 502753fb..e5f0f92f 100644 --- a/.pipelines/diabetes_regression-variables-template.yml +++ b/.pipelines/automobile-variables-template.yml @@ -3,7 +3,7 @@ variables: # Source Config # The directory containing the scripts for training, evaluating, and registering the model - name: SOURCES_DIR_TRAIN - value: diabetes_regression + value: automobile # The path to the model training script under SOURCES_DIR_TRAIN - name: TRAIN_SCRIPT_PATH value: training/train_aml.py @@ -22,20 +22,20 @@ variables: - name: EXPERIMENT_NAME value: mlopspython - name: DATASET_NAME - value: diabetes_ds + value: automobile_ds # Uncomment DATASTORE_NAME if you have configured non default datastore to point to your data # - name: DATASTORE_NAME # value: datablobstore - name: DATASET_VERSION value: latest - name: TRAINING_PIPELINE_NAME - value: "diabetes-Training-Pipeline" + value: "automobile-Training-Pipeline" - name: MODEL_NAME - value: diabetes_regression_model.pkl + value: automobile_model.pkl # AML Compute Cluster Config - name: AML_ENV_NAME - value: diabetes_regression_training_env + value: automobile_training_env - name: AML_ENV_TRAIN_CONDA_DEP_FILE value: "conda_dependencies.yml" - name: AML_COMPUTE_CLUSTER_CPU_SKU @@ -51,7 +51,7 @@ variables: # The name for the (docker/webapp) scoring image - name: IMAGE_NAME - value: "diabetestrained" + value: "automobiletrained" # Optional. Used by a training pipeline with R on Databricks - name: DB_CLUSTER_ID @@ -80,9 +80,9 @@ variables: value: "conda_dependencies_scorecopy.yml" # AML Compute Cluster Config for parallel batch scoring - name: AML_ENV_NAME_SCORING - value: diabetes_regression_scoring_env + value: automobile_scoring_env - name: AML_ENV_NAME_SCORE_COPY - value: diabetes_regression_score_copy_env + value: automobile_score_copy_env - name: AML_COMPUTE_CLUSTER_CPU_SKU_SCORING value: STANDARD_DS2_V2 - name: AML_COMPUTE_CLUSTER_NAME_SCORING @@ -113,17 +113,17 @@ variables: value: "input" # Blobname for the input data - include any applicable path in the string - name: SCORING_DATASTORE_INPUT_FILENAME - value: "diabetes_scoring_input.csv" + value: "automobile_scoring_input.csv" # Blob container where the output data for scoring can be found - name: SCORING_DATASTORE_OUTPUT_CONTAINER value: "output" # Blobname for the output data - include any applicable path in the string - name: SCORING_DATASTORE_OUTPUT_FILENAME - value: "diabetes_scoring_output.csv" + value: "automobile_scoring_output.csv" # Dataset name for input data for scoring - name: SCORING_DATASET_NAME - value: "diabetes_scoring_ds" + value: "automobile_scoring_ds" # Scoring pipeline name - name: SCORING_PIPELINE_NAME - value: "diabetes-scoring-pipeline" + value: "automobile-scoring-pipeline" \ No newline at end of file diff --git a/.pipelines/code-quality-template.yml b/.pipelines/code-quality-template.yml index afaf7a9a..1a030b49 100644 --- a/.pipelines/code-quality-template.yml +++ b/.pipelines/code-quality-template.yml @@ -5,7 +5,7 @@ steps: displayName: 'Run lint tests' - script: | - python -m pytest . --cov=diabetes_regression --cov-report=html --cov-report=xml --junitxml=unit-testresults.xml + python -m pytest . --cov=automobile --cov-report=html --cov-report=xml --junitxml=unit-testresults.xml condition: succeededOrFailed() displayName: 'Run unit tests' diff --git a/.pipelines/pr.yml b/.pipelines/pr.yml index 765a5fef..97589932 100644 --- a/.pipelines/pr.yml +++ b/.pipelines/pr.yml @@ -17,7 +17,7 @@ pool: container: mlops variables: -- template: diabetes_regression-variables-template.yml +- template: automobile-variables-template.yml - group: devopsforai-aml-vg steps: diff --git a/diabetes_regression/.amlignore b/automobile/.amlignore similarity index 100% rename from diabetes_regression/.amlignore rename to automobile/.amlignore diff --git a/diabetes_regression/ci_dependencies.yml b/automobile/ci_dependencies.yml similarity index 100% rename from diabetes_regression/ci_dependencies.yml rename to automobile/ci_dependencies.yml diff --git a/diabetes_regression/conda_dependencies.yml b/automobile/conda_dependencies.yml similarity index 97% rename from diabetes_regression/conda_dependencies.yml rename to automobile/conda_dependencies.yml index e214c7b2..3124c554 100644 --- a/diabetes_regression/conda_dependencies.yml +++ b/automobile/conda_dependencies.yml @@ -14,7 +14,7 @@ # This directive is stored in a comment to preserve the Conda file structure. # [AzureMlVersion] = 2 -name: diabetes_regression_training_env +name: automobile_training_env dependencies: # The python interpreter version. # Currently Azure ML Workbench only supports 3.5.2 and later. diff --git a/diabetes_regression/conda_dependencies_scorecopy.yml b/automobile/conda_dependencies_scorecopy.yml similarity index 100% rename from diabetes_regression/conda_dependencies_scorecopy.yml rename to automobile/conda_dependencies_scorecopy.yml diff --git a/diabetes_regression/conda_dependencies_scoring.yml b/automobile/conda_dependencies_scoring.yml similarity index 100% rename from diabetes_regression/conda_dependencies_scoring.yml rename to automobile/conda_dependencies_scoring.yml diff --git a/diabetes_regression/evaluate/evaluate_model.py b/automobile/evaluate/evaluate_model.py similarity index 97% rename from diabetes_regression/evaluate/evaluate_model.py rename to automobile/evaluate/evaluate_model.py index d1ff3c6a..e705c373 100644 --- a/diabetes_regression/evaluate/evaluate_model.py +++ b/automobile/evaluate/evaluate_model.py @@ -34,7 +34,7 @@ # the following code is a good starting point for you # use # python -m evaluate.evaluate_model -# in diabetes_regression folder context +# in automobile folder context # if (run.id.startswith('OfflineRun')): # from dotenv import load_dotenv @@ -42,7 +42,7 @@ # load_dotenv() # sources_dir = os.environ.get("SOURCES_DIR_TRAIN") # if (sources_dir is None): -# sources_dir = 'diabetes_regression' +# sources_dir = 'automobile' # path_to_util = os.path.join(".", sources_dir, "util") # sys.path.append(os.path.abspath(path_to_util)) # NOQA: E402 # from model_helper import get_model @@ -83,7 +83,7 @@ "--model_name", type=str, help="Name of the Model", - default="diabetes_model.pkl", + default="automobile_model.pkl", ) parser.add_argument( diff --git a/diabetes_regression/parameters.json b/automobile/parameters.json similarity index 100% rename from diabetes_regression/parameters.json rename to automobile/parameters.json diff --git a/diabetes_regression/register/register_model.py b/automobile/register/register_model.py similarity index 98% rename from diabetes_regression/register/register_model.py rename to automobile/register/register_model.py index bca55a83..f729d170 100644 --- a/diabetes_regression/register/register_model.py +++ b/automobile/register/register_model.py @@ -70,7 +70,7 @@ def main(): "--model_name", type=str, help="Name of the Model", - default="diabetes_model.pkl", + default="automobile_model.pkl", ) parser.add_argument( @@ -180,7 +180,7 @@ def register_aml_model( build_uri=None ): try: - tagsValue = {"area": "diabetes_regression", + tagsValue = {"area": "automobile", "run_id": run_id, "experiment_name": exp.name} tagsValue.update(model_tags) diff --git a/diabetes_regression/scoring/deployment_config_aci.yml b/automobile/scoring/deployment_config_aci.yml similarity index 100% rename from diabetes_regression/scoring/deployment_config_aci.yml rename to automobile/scoring/deployment_config_aci.yml diff --git a/diabetes_regression/scoring/deployment_config_aks.yml b/automobile/scoring/deployment_config_aks.yml similarity index 100% rename from diabetes_regression/scoring/deployment_config_aks.yml rename to automobile/scoring/deployment_config_aks.yml diff --git a/diabetes_regression/scoring/inference_config.yml b/automobile/scoring/inference_config.yml similarity index 100% rename from diabetes_regression/scoring/inference_config.yml rename to automobile/scoring/inference_config.yml diff --git a/diabetes_regression/scoring/parallel_batchscore.py b/automobile/scoring/parallel_batchscore.py similarity index 100% rename from diabetes_regression/scoring/parallel_batchscore.py rename to automobile/scoring/parallel_batchscore.py diff --git a/diabetes_regression/scoring/parallel_batchscore_copyoutput.py b/automobile/scoring/parallel_batchscore_copyoutput.py similarity index 100% rename from diabetes_regression/scoring/parallel_batchscore_copyoutput.py rename to automobile/scoring/parallel_batchscore_copyoutput.py diff --git a/diabetes_regression/scoring/score.py b/automobile/scoring/score.py similarity index 100% rename from diabetes_regression/scoring/score.py rename to automobile/scoring/score.py diff --git a/diabetes_regression/scoring/scoreA.py b/automobile/scoring/scoreA.py similarity index 100% rename from diabetes_regression/scoring/scoreA.py rename to automobile/scoring/scoreA.py diff --git a/diabetes_regression/scoring/scoreB.py b/automobile/scoring/scoreB.py similarity index 100% rename from diabetes_regression/scoring/scoreB.py rename to automobile/scoring/scoreB.py diff --git a/diabetes_regression/training/R/r_train.r b/automobile/training/R/r_train.r similarity index 100% rename from diabetes_regression/training/R/r_train.r rename to automobile/training/R/r_train.r diff --git a/diabetes_regression/training/R/train_with_r.py b/automobile/training/R/train_with_r.py similarity index 100% rename from diabetes_regression/training/R/train_with_r.py rename to automobile/training/R/train_with_r.py diff --git a/diabetes_regression/training/R/train_with_r_on_databricks.py b/automobile/training/R/train_with_r_on_databricks.py similarity index 100% rename from diabetes_regression/training/R/train_with_r_on_databricks.py rename to automobile/training/R/train_with_r_on_databricks.py diff --git a/diabetes_regression/training/R/weight_data.csv b/automobile/training/R/weight_data.csv similarity index 100% rename from diabetes_regression/training/R/weight_data.csv rename to automobile/training/R/weight_data.csv diff --git a/diabetes_regression/training/test_train.py b/automobile/training/test_train.py similarity index 91% rename from diabetes_regression/training/test_train.py rename to automobile/training/test_train.py index e1a79781..cc7fb270 100644 --- a/diabetes_regression/training/test_train.py +++ b/automobile/training/test_train.py @@ -1,5 +1,5 @@ import numpy as np -from diabetes_regression.training.train import train_model, get_model_metrics +from automobile.training.train import train_model, get_model_metrics def test_train_model(): diff --git a/diabetes_regression/training/train.py b/automobile/training/train.py similarity index 100% rename from diabetes_regression/training/train.py rename to automobile/training/train.py diff --git a/diabetes_regression/training/train_aml.py b/automobile/training/train_aml.py similarity index 100% rename from diabetes_regression/training/train_aml.py rename to automobile/training/train_aml.py diff --git a/diabetes_regression/util/__init__.py b/automobile/util/__init__.py similarity index 100% rename from diabetes_regression/util/__init__.py rename to automobile/util/__init__.py diff --git a/diabetes_regression/util/model_helper.py b/automobile/util/model_helper.py similarity index 100% rename from diabetes_regression/util/model_helper.py rename to automobile/util/model_helper.py diff --git a/data/README.md b/data/README.md deleted file mode 100644 index d43d139c..00000000 --- a/data/README.md +++ /dev/null @@ -1,3 +0,0 @@ -This folder is used for example data, and it is not meant to be used for storing training data. - -Follow steps to [Configure Training Data](../docs/custom_model.md#Configure-Custom-Training) to use your own data for training. \ No newline at end of file diff --git a/data/data_test.py b/data/data_test.py deleted file mode 100644 index 6d7d2ddf..00000000 --- a/data/data_test.py +++ /dev/null @@ -1,127 +0,0 @@ -# test integrity of the input data -""" -Copyright (C) Microsoft Corporation. All rights reserved.​ - ​ -Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, -royalty-free right to use, copy, and modify the software code provided by us -("Software Code"). You may not sublicense the Software Code or any use of it -(except to your affiliates and to vendors to perform work on your behalf) -through distribution, network access, service agreement, lease, rental, or -otherwise. This license does not purport to express any claim of ownership over -data you may have shared with Microsoft in the creation of the Software Code. -Unless applicable law gives you more rights, Microsoft reserves all other -rights not expressly granted herein, whether by implication, estoppel or -otherwise. ​ - ​ -THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -""" -import os -import numpy as np -import pandas as pd - - -# get absolute path of csv files from data folder -def get_absPath(filename): - """Returns the path of the notebooks folder""" - path = os.path.abspath( - os.path.join( - os.path.dirname( - __file__), os.path.pardir, "data", filename - ) - ) - return path - - -# number of features -expected_columns = 10 - -# distribution of features in the training set -historical_mean = np.array( - [ - -3.63962254e-16, - 1.26972339e-16, - -8.01646331e-16, - 1.28856202e-16, - -8.99230414e-17, - 1.29609747e-16, - -4.56397112e-16, - 3.87573332e-16, - -3.84559152e-16, - -3.39848813e-16, - 1.52133484e02, - ] -) -historical_std = np.array( - [ - 4.75651494e-02, - 4.75651494e-02, - 4.75651494e-02, - 4.75651494e-02, - 4.75651494e-02, - 4.75651494e-02, - 4.75651494e-02, - 4.75651494e-02, - 4.75651494e-02, - 4.75651494e-02, - 7.70057459e01, - ] -) - -# maximal relative change in feature mean or standrd deviation -# that we can tolerate -shift_tolerance = 3 - - -def test_check_schema(): - datafile = get_absPath("diabetes.csv") - # check that file exists - assert os.path.exists(datafile) - dataset = pd.read_csv(datafile) - header = dataset[dataset.columns[:-1]] - actual_columns = header.shape[1] - # check header has expected number of columns - assert actual_columns == expected_columns - - -def test_check_bad_schema(): - datafile = get_absPath("diabetes_bad_schema.csv") - # check that file exists - assert os.path.exists(datafile) - dataset = pd.read_csv(datafile) - header = dataset[dataset.columns[:-1]] - actual_columns = header.shape[1] - # check header has expected number of columns - assert actual_columns != expected_columns - - -def test_check_missing_values(): - datafile = get_absPath("diabetes_missing_values.csv") - # check that file exists - assert os.path.exists(datafile) - dataset = pd.read_csv(datafile) - n_nan = np.sum(np.isnan(dataset.values)) - assert n_nan > 0 - - -def test_check_distribution(): - datafile = get_absPath("diabetes_bad_dist.csv") - # check that file exists - assert os.path.exists(datafile) - dataset = pd.read_csv(datafile) - mean = np.mean(dataset.values, axis=0) - std = np.mean(dataset.values, axis=0) - assert ( - np.sum(abs(mean - historical_mean) - > shift_tolerance * abs(historical_mean)) - or np.sum(abs(std - historical_std) - > shift_tolerance * abs(historical_std)) > 0 - ) diff --git a/data/diabetes.csv b/data/diabetes.csv deleted file mode 100644 index 162002a7..00000000 --- a/data/diabetes.csv +++ /dev/null @@ -1,443 +0,0 @@ -AGE,SEX,BMI,BP,S1,S2,S3,S4,S5,S6,Y -0.0380759064334241,0.0506801187398187,0.0616962065186885,0.0218723549949558,-0.0442234984244464,-0.0348207628376986,-0.0434008456520269,-0.00259226199818282,0.0199084208763183,-0.0176461251598052,151.0 --0.001882016527791,-0.044641636506989,-0.0514740612388061,-0.0263278347173518,-0.00844872411121698,-0.019163339748222,0.0744115640787594,-0.0394933828740919,-0.0683297436244215,-0.09220404962683,75.0 -0.0852989062966783,0.0506801187398187,0.0444512133365941,-0.00567061055493425,-0.0455994512826475,-0.0341944659141195,-0.0323559322397657,-0.00259226199818282,0.00286377051894013,-0.0259303389894746,141.0 --0.0890629393522603,-0.044641636506989,-0.0115950145052127,-0.0366564467985606,0.0121905687618,0.0249905933641021,-0.0360375700438527,0.0343088588777263,0.0226920225667445,-0.0093619113301358,206.0 -0.00538306037424807,-0.044641636506989,-0.0363846922044735,0.0218723549949558,0.00393485161259318,0.0155961395104161,0.0081420836051921,-0.00259226199818282,-0.0319914449413559,-0.0466408735636482,135.0 --0.0926954778032799,-0.044641636506989,-0.0406959404999971,-0.0194420933298793,-0.0689906498720667,-0.0792878444118122,0.0412768238419757,-0.076394503750001,-0.0411803851880079,-0.0963461565416647,97.0 --0.0454724779400257,0.0506801187398187,-0.0471628129432825,-0.015999222636143,-0.040095639849843,-0.0248000120604336,0.000778807997017968,-0.0394933828740919,-0.0629129499162512,-0.0383566597339788,138.0 -0.063503675590561,0.0506801187398187,-0.00189470584028465,0.0666296740135272,0.0906198816792644,0.108914381123697,0.0228686348215404,0.0177033544835672,-0.0358167281015492,0.00306440941436832,63.0 -0.0417084448844436,0.0506801187398187,0.0616962065186885,-0.0400993174922969,-0.0139525355440215,0.00620168565673016,-0.0286742944356786,-0.00259226199818282,-0.0149564750249113,0.0113486232440377,110.0 --0.0709002470971626,-0.044641636506989,0.0390621529671896,-0.0332135761048244,-0.0125765826858204,-0.034507614375909,-0.0249926566315915,-0.00259226199818282,0.0677363261102861,-0.0135040182449705,310.0 --0.0963280162542995,-0.044641636506989,-0.0838084234552331,0.0081008722200108,-0.103389471327095,-0.0905611890362353,-0.0139477432193303,-0.076394503750001,-0.0629129499162512,-0.0342145528191441,101.0 -0.0271782910803654,0.0506801187398187,0.0175059114895716,-0.0332135761048244,-0.00707277125301585,0.0459715403040008,-0.0654906724765493,0.0712099797536354,-0.096433222891784,-0.0590671943081523,69.0 -0.0162806757273067,-0.044641636506989,-0.0288400076873072,-0.00911348124867051,-0.00432086553661359,-0.00976888589453599,0.0449584616460628,-0.0394933828740919,-0.0307512098645563,-0.0424987666488135,179.0 -0.00538306037424807,0.0506801187398187,-0.00189470584028465,0.0081008722200108,-0.00432086553661359,-0.0157187066685371,-0.0029028298070691,-0.00259226199818282,0.0383932482116977,-0.0135040182449705,185.0 -0.0453409833354632,-0.044641636506989,-0.0256065714656645,-0.0125563519424068,0.0176943801946045,-6.12835790604833e-05,0.0817748396869335,-0.0394933828740919,-0.0319914449413559,-0.0756356219674911,118.0 --0.0527375548420648,0.0506801187398187,-0.0180618869484982,0.0804011567884723,0.0892439288210632,0.107661787276539,-0.0397192078479398,0.108111100629544,0.0360557900898319,-0.0424987666488135,171.0 --0.00551455497881059,-0.044641636506989,0.0422955891888323,0.0494153205448459,0.0245741444856101,-0.0238605666750649,0.0744115640787594,-0.0394933828740919,0.0522799997967812,0.0279170509033766,166.0 -0.0707687524926,0.0506801187398187,0.0121168511201671,0.0563010619323185,0.034205814493018,0.0494161733836856,-0.0397192078479398,0.0343088588777263,0.027367707542609,-0.00107769750046639,144.0 --0.0382074010379866,-0.044641636506989,-0.0105172024313319,-0.0366564467985606,-0.0373437341334407,-0.0194764882100115,-0.0286742944356786,-0.00259226199818282,-0.0181182673078967,-0.0176461251598052,97.0 --0.0273097856849279,-0.044641636506989,-0.0180618869484982,-0.0400993174922969,-0.00294491267841247,-0.0113346282034837,0.0375951860378887,-0.0394933828740919,-0.0089440189577978,-0.0549250873933176,168.0 --0.0491050163910452,-0.044641636506989,-0.0568631216082106,-0.0435421881860331,-0.0455994512826475,-0.043275771306016,0.000778807997017968,-0.0394933828740919,-0.0119006848015081,0.0154907301588724,68.0 --0.0854304009012408,0.0506801187398187,-0.0223731352440218,0.00121513083253827,-0.0373437341334407,-0.0263657543693812,0.0155053592133662,-0.0394933828740919,-0.072128454601956,-0.0176461251598052,49.0 --0.0854304009012408,-0.044641636506989,-0.00405032998804645,-0.00911348124867051,-0.00294491267841247,0.00776742796567782,0.0228686348215404,-0.0394933828740919,-0.0611765950943345,-0.0135040182449705,68.0 -0.0453409833354632,0.0506801187398187,0.0606183944448076,0.0310533436263482,0.0287020030602135,-0.0473467013092799,-0.0544457590642881,0.0712099797536354,0.133598980013008,0.135611830689079,245.0 --0.0636351701951234,-0.044641636506989,0.0358287167455469,-0.0228849640236156,-0.0304639698424351,-0.0188501912864324,-0.00658446761115617,-0.00259226199818282,-0.0259524244351894,-0.0549250873933176,184.0 --0.067267708646143,0.0506801187398187,-0.0126728265790937,-0.0400993174922969,-0.0153284884022226,0.0046359433477825,-0.0581273968683752,0.0343088588777263,0.0191990330785671,-0.0342145528191441,202.0 --0.107225631607358,-0.044641636506989,-0.0773415510119477,-0.0263278347173518,-0.0896299427450836,-0.0961978613484469,0.0265502726256275,-0.076394503750001,-0.0425721049227942,-0.0052198044153011,137.0 --0.0236772472339084,-0.044641636506989,0.0595405823709267,-0.0400993174922969,-0.0428475455662452,-0.0435889197678055,0.0118237214092792,-0.0394933828740919,-0.0159982677581387,0.0403433716478807,85.0 -0.0526060602375023,-0.044641636506989,-0.0212953231701409,-0.0745280244296595,-0.040095639849843,-0.0376390989938044,-0.00658446761115617,-0.0394933828740919,-0.000609254186102297,-0.0549250873933176,131.0 -0.0671362140415805,0.0506801187398187,-0.00620595413580824,0.063186803319791,-0.0428475455662452,-0.0958847128866574,0.052321737254237,-0.076394503750001,0.0594238004447941,0.0527696923923848,283.0 --0.0600026317441039,-0.044641636506989,0.0444512133365941,-0.0194420933298793,-0.00982467696941811,-0.00757684666200928,0.0228686348215404,-0.0394933828740919,-0.0271286455543265,-0.0093619113301358,129.0 --0.0236772472339084,-0.044641636506989,-0.0654856181992578,-0.081413765817132,-0.0387196869916418,-0.0536096705450705,0.0596850128624111,-0.076394503750001,-0.0371283460104736,-0.0424987666488135,59.0 -0.0344433679824045,0.0506801187398187,0.125287118877662,0.0287580963824284,-0.0538551684318543,-0.0129003705124313,-0.10230705051742,0.108111100629544,0.000271485727907132,0.0279170509033766,341.0 -0.030810829531385,-0.044641636506989,-0.0503962491649252,-0.00222773986119799,-0.0442234984244464,-0.0899348921126563,0.118591217727804,-0.076394503750001,-0.0181182673078967,0.00306440941436832,87.0 -0.0162806757273067,-0.044641636506989,-0.063329994051496,-0.0573136709609782,-0.0579830270064577,-0.0489124436182275,0.0081420836051921,-0.0394933828740919,-0.0594726974107223,-0.0673514081378217,65.0 -0.0489735217864827,0.0506801187398187,-0.030995631835069,-0.0492803060204031,0.0493412959332305,-0.00413221358232442,0.133317768944152,-0.0535158088069373,0.0213108465682448,0.0196328370737072,102.0 -0.0126481372762872,-0.044641636506989,0.0228949718589761,0.0528581912385822,0.00806271018719657,-0.0285577936019079,0.0375951860378887,-0.0394933828740919,0.0547240033481791,-0.0259303389894746,265.0 --0.00914709342983014,-0.044641636506989,0.0110390390462862,-0.0573136709609782,-0.0249601584096305,-0.0429626228442264,0.0302319104297145,-0.0394933828740919,0.01703713241478,-0.0052198044153011,276.0 --0.00188201652779104,0.0506801187398187,0.0713965151836166,0.0976155102571536,0.0878679759628621,0.0754074957122168,-0.0213110188275045,0.0712099797536354,0.0714240327805764,0.0237749439885419,252.0 --0.00188201652779104,0.0506801187398187,0.0142724752679289,-0.0745280244296595,0.00255889875439205,0.00620168565673016,-0.0139477432193303,-0.00259226199818282,0.0191990330785671,0.00306440941436832,90.0 -0.00538306037424807,0.0506801187398187,-0.00836157828357004,0.0218723549949558,0.054845107366035,0.07321545647969,-0.0249926566315915,0.0343088588777263,0.0125531528133893,0.094190761540732,100.0 --0.099960554705319,-0.044641636506989,-0.0676412423470196,-0.108956731367022,-0.0744944613048712,-0.072711726714232,0.0155053592133662,-0.0394933828740919,-0.0498684677352306,-0.0093619113301358,55.0 --0.0600026317441039,0.0506801187398187,-0.0105172024313319,-0.0148515990830405,-0.0497273098572509,-0.0235474182132754,-0.0581273968683752,0.0158582984397717,-0.00991895736315477,-0.0342145528191441,61.0 -0.0199132141783263,-0.044641636506989,-0.0234509473179027,-0.0710851537359232,0.0204462859110067,-0.0100820343563255,0.118591217727804,-0.076394503750001,-0.0425721049227942,0.0734802269665584,92.0 -0.0453409833354632,0.0506801187398187,0.068163078961974,0.0081008722200108,-0.0167044412604238,0.0046359433477825,-0.0765355858888105,0.0712099797536354,0.0324332257796019,-0.0176461251598052,259.0 -0.0271782910803654,0.0506801187398187,-0.0353068801305926,0.0322009670761646,-0.0112006298276192,0.00150445872988718,-0.0102661054152432,-0.00259226199818282,-0.0149564750249113,-0.0507829804784829,53.0 --0.0563700932930843,-0.044641636506989,-0.0115950145052127,-0.0332135761048244,-0.0469754041408486,-0.0476598497710694,0.00446044580110504,-0.0394933828740919,-0.00797939755454164,-0.0880619427119953,190.0 --0.0781653239992017,-0.044641636506989,-0.0730303027164241,-0.0573136709609782,-0.0841261313122791,-0.0742774690231797,-0.0249926566315915,-0.0394933828740919,-0.0181182673078967,-0.0839198357971606,142.0 -0.0671362140415805,0.0506801187398187,-0.041773752573878,0.0115437429137471,0.00255889875439205,0.00588853719494063,0.0412768238419757,-0.0394933828740919,-0.0594726974107223,-0.0217882320746399,75.0 --0.0418399394890061,0.0506801187398187,0.0142724752679289,-0.00567061055493425,-0.0125765826858204,0.00620168565673016,-0.0728539480847234,0.0712099797536354,0.0354619386607697,-0.0135040182449705,142.0 -0.0344433679824045,-0.044641636506989,-0.00728376620968916,0.0149866136074833,-0.0442234984244464,-0.0373259505320149,-0.0029028298070691,-0.0394933828740919,-0.02139368094036,0.00720651632920303,155.0 -0.0598711371395414,0.0506801187398187,0.0164280994156907,0.0287580963824284,-0.0414715927080441,-0.029184090525487,-0.0286742944356786,-0.00259226199818282,-0.00239668149341427,-0.0217882320746399,225.0 --0.0527375548420648,-0.044641636506989,-0.00943939035745095,-0.00567061055493425,0.0397096259258226,0.0447189464568426,0.0265502726256275,-0.00259226199818282,-0.0181182673078967,-0.0135040182449705,59.0 --0.00914709342983014,-0.044641636506989,-0.0159062628007364,0.0700725447072635,0.0121905687618,0.0221722572079963,0.0155053592133662,-0.00259226199818282,-0.0332487872476258,0.0486275854775501,104.0 --0.0491050163910452,-0.044641636506989,0.0250505960067379,0.0081008722200108,0.0204462859110067,0.0177881787429428,0.052321737254237,-0.0394933828740919,-0.0411803851880079,0.00720651632920303,182.0 --0.0418399394890061,-0.044641636506989,-0.0493184370910443,-0.0366564467985606,-0.00707277125301585,-0.0226079728279068,0.0854564774910206,-0.0394933828740919,-0.0664881482228354,0.00720651632920303,128.0 --0.0418399394890061,-0.044641636506989,0.0412177771149514,-0.0263278347173518,-0.0318399227006362,-0.0304366843726451,-0.0360375700438527,0.00294290613320356,0.0336568129023847,-0.0176461251598052,52.0 --0.0273097856849279,-0.044641636506989,-0.063329994051496,-0.0504279295735057,-0.0896299427450836,-0.104339721354975,0.052321737254237,-0.076394503750001,-0.0561575730950062,-0.0673514081378217,37.0 -0.0417084448844436,-0.044641636506989,-0.064407806125377,0.0356438377699009,0.0121905687618,-0.057993749010124,0.181179060397284,-0.076394503750001,-0.000609254186102297,-0.0507829804784829,170.0 -0.063503675590561,0.0506801187398187,-0.0256065714656645,0.0115437429137471,0.0644767773734429,0.048476727998317,0.0302319104297145,-0.00259226199818282,0.0383932482116977,0.0196328370737072,170.0 --0.0709002470971626,-0.044641636506989,-0.00405032998804645,-0.0400993174922969,-0.0662387441556644,-0.0786615474882331,0.052321737254237,-0.076394503750001,-0.0514005352605825,-0.0342145528191441,61.0 --0.0418399394890061,0.0506801187398187,0.00457216660300077,-0.0538708002672419,-0.0442234984244464,-0.0273051997547498,-0.0802172236928976,0.0712099797536354,0.0366457977933988,0.0196328370737072,144.0 --0.0273097856849279,0.0506801187398187,-0.00728376620968916,-0.0400993174922969,-0.0112006298276192,-0.0138398158977999,0.0596850128624111,-0.0394933828740919,-0.0823814832581028,-0.0259303389894746,52.0 --0.034574862586967,-0.044641636506989,-0.0374625042783544,-0.0607565416547144,0.0204462859110067,0.0434663526096845,-0.0139477432193303,-0.00259226199818282,-0.0307512098645563,-0.0714935150526564,128.0 -0.0671362140415805,0.0506801187398187,-0.0256065714656645,-0.0400993174922969,-0.0634868384392622,-0.0598726397808612,-0.0029028298070691,-0.0394933828740919,-0.0191970476139445,0.0113486232440377,71.0 --0.0454724779400257,0.0506801187398187,-0.0245287593917836,0.0597439326260547,0.00531080447079431,0.0149698425868371,-0.0544457590642881,0.0712099797536354,0.0423448954496075,0.0154907301588724,163.0 --0.00914709342983014,0.0506801187398187,-0.0180618869484982,-0.0332135761048244,-0.0208322998350272,0.0121515064307313,-0.0728539480847234,0.0712099797536354,0.000271485727907132,0.0196328370737072,150.0 -0.0417084448844436,0.0506801187398187,-0.0148284507268555,-0.0171468461892456,-0.00569681839481472,0.00839372488925688,-0.0139477432193303,-0.00185423958066465,-0.0119006848015081,0.00306440941436832,97.0 -0.0380759064334241,0.0506801187398187,-0.0299178197611881,-0.0400993174922969,-0.0332158755588373,-0.0241737151368545,-0.0102661054152432,-0.00259226199818282,-0.0129079422541688,0.00306440941436832,160.0 -0.0162806757273067,-0.044641636506989,-0.0460850008694016,-0.00567061055493425,-0.0758704141630723,-0.0614383820898088,-0.0139477432193303,-0.0394933828740919,-0.0514005352605825,0.0196328370737072,178.0 --0.00188201652779104,-0.044641636506989,-0.0697968664947814,-0.0125563519424068,-0.000193006962010205,-0.00914258897095694,0.0707299262746723,-0.0394933828740919,-0.0629129499162512,0.0403433716478807,48.0 --0.00188201652779104,-0.044641636506989,0.0336730925977851,0.125158475807044,0.0245741444856101,0.0262431872112602,-0.0102661054152432,-0.00259226199818282,0.0267142576335128,0.0610539062220542,270.0 -0.063503675590561,0.0506801187398187,-0.00405032998804645,-0.0125563519424068,0.103003457403075,0.0487898764601065,0.056003375058324,-0.00259226199818282,0.0844952822124031,-0.0176461251598052,202.0 -0.0126481372762872,0.0506801187398187,-0.02021751109626,-0.00222773986119799,0.0383336730676214,0.05317395492516,-0.00658446761115617,0.0343088588777263,-0.00514530798026311,-0.0093619113301358,111.0 -0.0126481372762872,0.0506801187398187,0.00241654245523897,0.0563010619323185,0.0273260502020124,0.0171618818193638,0.0412768238419757,-0.0394933828740919,0.00371173823343597,0.0734802269665584,85.0 --0.00914709342983014,0.0506801187398187,-0.030995631835069,-0.0263278347173518,-0.0112006298276192,-0.00100072896442909,-0.0213110188275045,-0.00259226199818282,0.0062093156165054,0.0279170509033766,42.0 --0.0309423241359475,0.0506801187398187,0.0282840322283806,0.0700725447072635,-0.126780669916514,-0.106844909049291,-0.0544457590642881,-0.047980640675551,-0.0307512098645563,0.0154907301588724,170.0 --0.0963280162542995,-0.044641636506989,-0.0363846922044735,-0.0745280244296595,-0.0387196869916418,-0.0276183482165393,0.0155053592133662,-0.0394933828740919,-0.0740888714915354,-0.00107769750046639,200.0 -0.00538306037424807,-0.044641636506989,-0.0579409336820915,-0.0228849640236156,-0.0676146970138656,-0.0683276482491785,-0.0544457590642881,-0.00259226199818282,0.0428956878925287,-0.0839198357971606,252.0 --0.103593093156339,-0.044641636506989,-0.0374625042783544,-0.0263278347173518,0.00255889875439205,0.0199802179754696,0.0118237214092792,-0.00259226199818282,-0.0683297436244215,-0.0259303389894746,113.0 -0.0707687524926,-0.044641636506989,0.0121168511201671,0.0425295791573734,0.0713565416644485,0.0534871033869495,0.052321737254237,-0.00259226199818282,0.0253931349154494,-0.0052198044153011,143.0 -0.0126481372762872,0.0506801187398187,-0.0223731352440218,-0.0297707054110881,0.0108146159035988,0.0284352264437869,-0.0213110188275045,0.0343088588777263,-0.00608024819631442,-0.00107769750046639,51.0 --0.0164121703318693,-0.044641636506989,-0.0353068801305926,-0.0263278347173518,0.0328298616348169,0.0171618818193638,0.100183028707369,-0.0394933828740919,-0.0702093127286876,-0.0797777288823259,52.0 --0.0382074010379866,-0.044641636506989,0.00996122697240527,-0.0469850588797694,-0.0593589798646588,-0.0529833736214915,-0.0102661054152432,-0.0394933828740919,-0.0159982677581387,-0.0424987666488135,210.0 -0.00175052192322852,-0.044641636506989,-0.0396181284261162,-0.100923366426447,-0.0290880169842339,-0.0301235359108556,0.0449584616460628,-0.0501947079281055,-0.0683297436244215,-0.129483011860342,65.0 -0.0453409833354632,-0.044641636506989,0.0713965151836166,0.00121513083253827,-0.00982467696941811,-0.00100072896442909,0.0155053592133662,-0.0394933828740919,-0.0411803851880079,-0.0714935150526564,141.0 --0.0709002470971626,0.0506801187398187,-0.0751859268641859,-0.0400993174922969,-0.051103262715452,-0.015092409744958,-0.0397192078479398,-0.00259226199818282,-0.096433222891784,-0.0342145528191441,55.0 -0.0453409833354632,-0.044641636506989,-0.00620595413580824,0.0115437429137471,0.0631008245152418,0.0162224364339952,0.0965013909032818,-0.0394933828740919,0.0428956878925287,-0.0383566597339788,134.0 --0.0527375548420648,0.0506801187398187,-0.0406959404999971,-0.067642283042187,-0.0318399227006362,-0.0370128020702253,0.0375951860378887,-0.0394933828740919,-0.0345237153303495,0.0693381200517237,42.0 --0.0454724779400257,-0.044641636506989,-0.0482406250171634,-0.0194420933298793,-0.000193006962010205,-0.0160318551303266,0.0670482884705852,-0.0394933828740919,-0.0247911874324607,0.0196328370737072,111.0 -0.0126481372762872,-0.044641636506989,-0.0256065714656645,-0.0400993174922969,-0.0304639698424351,-0.0451546620767532,0.0780932018828464,-0.076394503750001,-0.072128454601956,0.0113486232440377,98.0 -0.0453409833354632,-0.044641636506989,0.0519958978537604,-0.0538708002672419,0.0631008245152418,0.0647604480113727,-0.0102661054152432,0.0343088588777263,0.0372320112089689,0.0196328370737072,164.0 --0.0200447087828888,-0.044641636506989,0.00457216660300077,0.0976155102571536,0.00531080447079431,-0.0207290820571696,0.0633666506664982,-0.0394933828740919,0.0125531528133893,0.0113486232440377,48.0 --0.0491050163910452,-0.044641636506989,-0.064407806125377,-0.10207098997955,-0.00294491267841247,-0.0154055582067476,0.0633666506664982,-0.0472426182580328,-0.0332487872476258,-0.0549250873933176,96.0 --0.0781653239992017,-0.044641636506989,-0.0169840748746173,-0.0125563519424068,-0.000193006962010205,-0.0135266674360104,0.0707299262746723,-0.0394933828740919,-0.0411803851880079,-0.09220404962683,90.0 --0.0709002470971626,-0.044641636506989,-0.0579409336820915,-0.081413765817132,-0.0455994512826475,-0.0288709420636975,-0.0434008456520269,-0.00259226199818282,0.00114379737951254,-0.0052198044153011,162.0 -0.0562385986885218,0.0506801187398187,0.00996122697240527,0.0494153205448459,-0.00432086553661359,-0.0122740735888523,-0.0434008456520269,0.0343088588777263,0.060787754150744,0.0320591578182113,150.0 --0.0273097856849279,-0.044641636506989,0.088641508365711,-0.0251802111642493,0.0218222387692079,0.0425269072243159,-0.0323559322397657,0.0343088588777263,0.00286377051894013,0.0776223338813931,279.0 -0.00175052192322852,0.0506801187398187,-0.00512814206192736,-0.0125563519424068,-0.0153284884022226,-0.0138398158977999,0.0081420836051921,-0.0394933828740919,-0.00608024819631442,-0.0673514081378217,92.0 --0.00188201652779104,-0.044641636506989,-0.064407806125377,0.0115437429137471,0.0273260502020124,0.0375165318356834,-0.0139477432193303,0.0343088588777263,0.0117839003835759,-0.0549250873933176,83.0 -0.0162806757273067,-0.044641636506989,0.0175059114895716,-0.0228849640236156,0.0603489187988395,0.0444057979950531,0.0302319104297145,-0.00259226199818282,0.0372320112089689,-0.00107769750046639,128.0 -0.0162806757273067,0.0506801187398187,-0.0450071887955207,0.063186803319791,0.0108146159035988,-0.00037443204085002,0.0633666506664982,-0.0394933828740919,-0.0307512098645563,0.036201264733046,102.0 --0.0926954778032799,-0.044641636506989,0.0282840322283806,-0.015999222636143,0.0369577202094203,0.0249905933641021,0.056003375058324,-0.0394933828740919,-0.00514530798026311,-0.00107769750046639,302.0 -0.0598711371395414,0.0506801187398187,0.0412177771149514,0.0115437429137471,0.0410855787840237,0.0707102687853738,-0.0360375700438527,0.0343088588777263,-0.0109044358473771,-0.0300724459043093,198.0 --0.0273097856849279,-0.044641636506989,0.0649296427403312,-0.00222773986119799,-0.0249601584096305,-0.0172844489774848,0.0228686348215404,-0.0394933828740919,-0.0611765950943345,-0.063209301222987,95.0 -0.0235457526293458,0.0506801187398187,-0.0320734439089499,-0.0400993174922969,-0.0318399227006362,-0.0216685274425382,-0.0139477432193303,-0.00259226199818282,-0.0109044358473771,0.0196328370737072,53.0 --0.0963280162542995,-0.044641636506989,-0.0762637389380668,-0.0435421881860331,-0.0455994512826475,-0.0348207628376986,0.0081420836051921,-0.0394933828740919,-0.0594726974107223,-0.0839198357971606,134.0 -0.0271782910803654,-0.044641636506989,0.0498402737059986,-0.0550184238203444,-0.00294491267841247,0.0406480164535787,-0.0581273968683752,0.0527594193156808,-0.0529587932392004,-0.0052198044153011,144.0 -0.0199132141783263,0.0506801187398187,0.045529025410475,0.0299057198322448,-0.062110885581061,-0.0558017097775973,-0.0728539480847234,0.0269286347025444,0.0456008084141249,0.0403433716478807,232.0 -0.0380759064334241,0.0506801187398187,-0.00943939035745095,0.0023627543856408,0.00118294589619092,0.0375165318356834,-0.0544457590642881,0.0501763408543672,-0.0259524244351894,0.106617082285236,81.0 -0.0417084448844436,0.0506801187398187,-0.0320734439089499,-0.0228849640236156,-0.0497273098572509,-0.0401442866881206,0.0302319104297145,-0.0394933828740919,-0.126097385560409,0.0154907301588724,104.0 -0.0199132141783263,-0.044641636506989,0.00457216660300077,-0.0263278347173518,0.023198191627409,0.0102726156599941,0.0670482884705852,-0.0394933828740919,-0.0236445575721341,-0.0466408735636482,59.0 --0.0854304009012408,-0.044641636506989,0.0207393477112143,-0.0263278347173518,0.00531080447079431,0.01966706951368,-0.0029028298070691,-0.00259226199818282,-0.0236445575721341,0.00306440941436832,246.0 -0.0199132141783263,0.0506801187398187,0.0142724752679289,0.063186803319791,0.0149424744782022,0.0202933664372591,-0.0470824834561139,0.0343088588777263,0.0466607723568145,0.0900486546258972,297.0 -0.0235457526293458,-0.044641636506989,0.110197749843329,0.063186803319791,0.0135665216200011,-0.0329418720669614,-0.0249926566315915,0.0206554441536399,0.09924022573399,0.0237749439885419,258.0 --0.0309423241359475,0.0506801187398187,0.00133873038135806,-0.00567061055493425,0.0644767773734429,0.0494161733836856,-0.0470824834561139,0.108111100629544,0.0837967663655224,0.00306440941436832,229.0 -0.0489735217864827,0.0506801187398187,0.0584627702970458,0.0700725447072635,0.0135665216200011,0.0206065148990486,-0.0213110188275045,0.0343088588777263,0.0220040504561505,0.0279170509033766,275.0 -0.0598711371395414,-0.044641636506989,-0.0212953231701409,0.0872868981759448,0.0452134373586271,0.0315667110616823,-0.0470824834561139,0.0712099797536354,0.0791210813896579,0.135611830689079,281.0 --0.0563700932930843,0.0506801187398187,-0.0105172024313319,0.0253152256886921,0.023198191627409,0.0400217195299996,-0.0397192078479398,0.0343088588777263,0.0206123307213641,0.0569117993072195,179.0 -0.0162806757273067,-0.044641636506989,-0.0471628129432825,-0.00222773986119799,-0.019456346976826,-0.0429626228442264,0.0339135482338016,-0.0394933828740919,0.027367707542609,0.0279170509033766,200.0 --0.0491050163910452,-0.044641636506989,0.00457216660300077,0.0115437429137471,-0.0373437341334407,-0.0185370428246429,-0.0176293810234174,-0.00259226199818282,-0.0398095943643375,-0.0217882320746399,200.0 -0.063503675590561,-0.044641636506989,0.0175059114895716,0.0218723549949558,0.00806271018719657,0.0215459602844172,-0.0360375700438527,0.0343088588777263,0.0199084208763183,0.0113486232440377,173.0 -0.0489735217864827,0.0506801187398187,0.0810968238485447,0.0218723549949558,0.0438374845004259,0.0641341510877936,-0.0544457590642881,0.0712099797536354,0.0324332257796019,0.0486275854775501,180.0 -0.00538306037424807,0.0506801187398187,0.034750904671666,-0.00108011630809546,0.152537760298315,0.198787989657293,-0.0618090346724622,0.185234443260194,0.0155668445407018,0.0734802269665584,84.0 --0.00551455497881059,-0.044641636506989,0.023972783932857,0.0081008722200108,-0.0345918284170385,-0.0388916928409625,0.0228686348215404,-0.0394933828740919,-0.0159982677581387,-0.0135040182449705,121.0 --0.00551455497881059,0.0506801187398187,-0.00836157828357004,-0.00222773986119799,-0.0332158755588373,-0.0636304213223356,-0.0360375700438527,-0.00259226199818282,0.0805854642386665,0.00720651632920303,161.0 --0.0890629393522603,-0.044641636506989,-0.0611743699037342,-0.0263278347173518,-0.0552311212900554,-0.0545491159304391,0.0412768238419757,-0.076394503750001,-0.0939356455087147,-0.0549250873933176,99.0 -0.0344433679824045,0.0506801187398187,-0.00189470584028465,-0.0125563519424068,0.0383336730676214,0.0137172487396789,0.0780932018828464,-0.0394933828740919,0.00455189046612778,-0.0963461565416647,109.0 --0.0527375548420648,-0.044641636506989,-0.0622521819776151,-0.0263278347173518,-0.00569681839481472,-0.005071658967693,0.0302319104297145,-0.0394933828740919,-0.0307512098645563,-0.0714935150526564,115.0 -0.00901559882526763,-0.044641636506989,0.0164280994156907,0.00465800152627453,0.0094386630453977,0.0105857641217836,-0.0286742944356786,0.0343088588777263,0.0389683660308856,0.11904340302974,268.0 --0.0636351701951234,0.0506801187398187,0.0961861928828773,0.104501251644626,-0.00294491267841247,-0.00475851050590347,-0.00658446761115617,-0.00259226199818282,0.0226920225667445,0.0734802269665584,274.0 --0.0963280162542995,-0.044641636506989,-0.0697968664947814,-0.067642283042187,-0.019456346976826,-0.0107083312799046,0.0155053592133662,-0.0394933828740919,-0.0468794828442166,-0.0797777288823259,158.0 -0.0162806757273067,0.0506801187398187,-0.0212953231701409,-0.00911348124867051,0.034205814493018,0.047850431074738,0.000778807997017968,-0.00259226199818282,-0.0129079422541688,0.0237749439885419,107.0 --0.0418399394890061,0.0506801187398187,-0.0536296853865679,-0.0400993174922969,-0.0841261313122791,-0.0717722813288634,-0.0029028298070691,-0.0394933828740919,-0.072128454601956,-0.0300724459043093,83.0 --0.0745327855481821,-0.044641636506989,0.0433734012627132,-0.0332135761048244,0.0121905687618,0.000251864882729031,0.0633666506664982,-0.0394933828740919,-0.0271286455543265,-0.0466408735636482,103.0 --0.00551455497881059,-0.044641636506989,0.056307146149284,-0.0366564467985606,-0.0483513569990498,-0.0429626228442264,-0.0728539480847234,0.0379989709653172,0.0507815133629732,0.0569117993072195,272.0 --0.0926954778032799,-0.044641636506989,-0.0816527993074713,-0.0573136709609782,-0.0607349327228599,-0.068014499787389,0.0486400994501499,-0.076394503750001,-0.0664881482228354,-0.0217882320746399,85.0 -0.00538306037424807,-0.044641636506989,0.0498402737059986,0.0976155102571536,-0.0153284884022226,-0.0163450035921162,-0.00658446761115617,-0.00259226199818282,0.01703713241478,-0.0135040182449705,280.0 -0.0344433679824045,0.0506801187398187,0.11127556191721,0.076958286094736,-0.0318399227006362,-0.03388131745233,-0.0213110188275045,-0.00259226199818282,0.028016506523264,0.0734802269665584,336.0 -0.0235457526293458,-0.044641636506989,0.0616962065186885,0.0528581912385822,-0.0345918284170385,-0.0489124436182275,-0.0286742944356786,-0.00259226199818282,0.0547240033481791,-0.0052198044153011,281.0 -0.0417084448844436,0.0506801187398187,0.0142724752679289,0.0425295791573734,-0.0304639698424351,-0.00131387742621863,-0.0434008456520269,-0.00259226199818282,-0.0332487872476258,0.0154907301588724,118.0 --0.0273097856849279,-0.044641636506989,0.0476846495582368,-0.0469850588797694,0.034205814493018,0.0572448849284239,-0.0802172236928976,0.13025177315509,0.0450661683362615,0.131469723774244,317.0 -0.0417084448844436,0.0506801187398187,0.0121168511201671,0.0390867084636372,0.054845107366035,0.0444057979950531,0.00446044580110504,-0.00259226199818282,0.0456008084141249,-0.00107769750046639,235.0 --0.0309423241359475,-0.044641636506989,0.00564997867688165,-0.00911348124867051,0.0190703330528056,0.00682798258030921,0.0744115640787594,-0.0394933828740919,-0.0411803851880079,-0.0424987666488135,60.0 -0.030810829531385,0.0506801187398187,0.0466068374843559,-0.015999222636143,0.0204462859110067,0.0506687672308438,-0.0581273968683752,0.0712099797536354,0.0062093156165054,0.00720651632920303,174.0 --0.0418399394890061,-0.044641636506989,0.128520555099304,0.063186803319791,-0.0332158755588373,-0.0326287236051719,0.0118237214092792,-0.0394933828740919,-0.0159982677581387,-0.0507829804784829,259.0 --0.0309423241359475,0.0506801187398187,0.0595405823709267,0.00121513083253827,0.0121905687618,0.0315667110616823,-0.0434008456520269,0.0343088588777263,0.0148227108412663,0.00720651632920303,178.0 --0.0563700932930843,-0.044641636506989,0.0929527566612346,-0.0194420933298793,0.0149424744782022,0.0234248510551544,-0.0286742944356786,0.0254525898675081,0.0260560896336847,0.0403433716478807,128.0 --0.0600026317441039,0.0506801187398187,0.0153502873418098,-0.0194420933298793,0.0369577202094203,0.0481635795365275,0.0191869970174533,-0.00259226199818282,-0.0307512098645563,-0.00107769750046639,96.0 --0.0491050163910452,0.0506801187398187,-0.00512814206192736,-0.0469850588797694,-0.0208322998350272,-0.0204159335953801,-0.0691723102806364,0.0712099797536354,0.061237907519701,-0.0383566597339788,126.0 -0.0235457526293458,-0.044641636506989,0.0703187031097357,0.0253152256886921,-0.0345918284170385,-0.014466112821379,-0.0323559322397657,-0.00259226199818282,-0.0191970476139445,-0.0093619113301358,288.0 -0.00175052192322852,-0.044641636506989,-0.00405032998804645,-0.00567061055493425,-0.00844872411121698,-0.0238605666750649,0.052321737254237,-0.0394933828740919,-0.0089440189577978,-0.0135040182449705,88.0 --0.034574862586967,0.0506801187398187,-0.000816893766403737,0.0700725447072635,0.0397096259258226,0.0669524872438994,-0.0654906724765493,0.108111100629544,0.0267142576335128,0.0734802269665584,292.0 -0.0417084448844436,0.0506801187398187,-0.0439293767216398,0.063186803319791,-0.00432086553661359,0.0162224364339952,-0.0139477432193303,-0.00259226199818282,-0.0345237153303495,0.0113486232440377,71.0 -0.0671362140415805,0.0506801187398187,0.0207393477112143,-0.00567061055493425,0.0204462859110067,0.0262431872112602,-0.0029028298070691,-0.00259226199818282,0.00864028293306308,0.00306440941436832,197.0 --0.0273097856849279,0.0506801187398187,0.0606183944448076,0.0494153205448459,0.0851160702464598,0.0863676918748504,-0.0029028298070691,0.0343088588777263,0.0378144788263439,0.0486275854775501,186.0 --0.0164121703318693,-0.044641636506989,-0.0105172024313319,0.00121513083253827,-0.0373437341334407,-0.0357602082230672,0.0118237214092792,-0.0394933828740919,-0.02139368094036,-0.0342145528191441,25.0 --0.00188201652779104,0.0506801187398187,-0.0331512559828308,-0.0182944697767768,0.0314539087766158,0.0428400556861055,-0.0139477432193303,0.0199174217361217,0.0102256424049578,0.0279170509033766,84.0 --0.0127796318808497,-0.044641636506989,-0.0654856181992578,-0.0699375301828207,0.00118294589619092,0.0168487333575743,-0.0029028298070691,-0.00702039650329191,-0.0307512098645563,-0.0507829804784829,96.0 --0.00551455497881059,-0.044641636506989,0.0433734012627132,0.0872868981759448,0.0135665216200011,0.00714113104209875,-0.0139477432193303,-0.00259226199818282,0.0423448954496075,-0.0176461251598052,195.0 --0.00914709342983014,-0.044641636506989,-0.0622521819776151,-0.0745280244296595,-0.0235842055514294,-0.0132135189742209,0.00446044580110504,-0.0394933828740919,-0.0358167281015492,-0.0466408735636482,53.0 --0.0454724779400257,0.0506801187398187,0.0638518306664503,0.0700725447072635,0.133274420283499,0.131461070372543,-0.0397192078479398,0.108111100629544,0.0757375884575476,0.0859065477110625,217.0 --0.0527375548420648,-0.044641636506989,0.0304396563761424,-0.0745280244296595,-0.0235842055514294,-0.0113346282034837,-0.0029028298070691,-0.00259226199818282,-0.0307512098645563,-0.00107769750046639,172.0 -0.0162806757273067,0.0506801187398187,0.0724743272574975,0.076958286094736,-0.00844872411121698,0.00557538873315109,-0.00658446761115617,-0.00259226199818282,-0.0236445575721341,0.0610539062220542,131.0 -0.0453409833354632,-0.044641636506989,-0.019139699022379,0.0218723549949558,0.0273260502020124,-0.0135266674360104,0.100183028707369,-0.0394933828740919,0.0177634778671173,-0.0135040182449705,214.0 --0.0418399394890061,-0.044641636506989,-0.0665634302731387,-0.0469850588797694,-0.0373437341334407,-0.043275771306016,0.0486400994501499,-0.0394933828740919,-0.0561575730950062,-0.0135040182449705,59.0 --0.0563700932930843,0.0506801187398187,-0.0600965578298533,-0.0366564467985606,-0.0882539898868825,-0.0708328359434948,-0.0139477432193303,-0.0394933828740919,-0.0781409106690696,-0.104630370371334,70.0 -0.0707687524926,-0.044641636506989,0.0692408910358548,0.0379390850138207,0.0218222387692079,0.00150445872988718,-0.0360375700438527,0.0391060045915944,0.0776327891955595,0.106617082285236,220.0 -0.00175052192322852,0.0506801187398187,0.0595405823709267,-0.00222773986119799,0.0617248716570406,0.063194705702425,-0.0581273968683752,0.108111100629544,0.0689822116363026,0.12732761685941,268.0 --0.00188201652779104,-0.044641636506989,-0.0266843835395454,0.0494153205448459,0.0589729659406384,-0.0160318551303266,-0.0470824834561139,0.0712099797536354,0.133598980013008,0.0196328370737072,152.0 -0.0235457526293458,0.0506801187398187,-0.02021751109626,-0.0366564467985606,-0.0139525355440215,-0.015092409744958,0.0596850128624111,-0.0394933828740919,-0.096433222891784,-0.0176461251598052,47.0 --0.0200447087828888,-0.044641636506989,-0.0460850008694016,-0.0986281192858133,-0.0758704141630723,-0.0598726397808612,-0.0176293810234174,-0.0394933828740919,-0.0514005352605825,-0.0466408735636482,74.0 -0.0417084448844436,0.0506801187398187,0.0713965151836166,0.0081008722200108,0.0383336730676214,0.0159092879722056,-0.0176293810234174,0.0343088588777263,0.0734100780491161,0.0859065477110625,295.0 --0.0636351701951234,0.0506801187398187,-0.0794971751597095,-0.00567061055493425,-0.071742555588469,-0.0664487574784414,-0.0102661054152432,-0.0394933828740919,-0.0181182673078967,-0.0549250873933176,101.0 -0.0162806757273067,0.0506801187398187,0.00996122697240527,-0.0435421881860331,-0.0965097070360893,-0.0946321190394993,-0.0397192078479398,-0.0394933828740919,0.01703713241478,0.00720651632920303,151.0 -0.0671362140415805,-0.044641636506989,-0.0385403163522353,-0.0263278347173518,-0.0318399227006362,-0.0263657543693812,0.0081420836051921,-0.0394933828740919,-0.0271286455543265,0.00306440941436832,127.0 -0.0453409833354632,0.0506801187398187,0.0196615356373334,0.0390867084636372,0.0204462859110067,0.0259300387494707,0.0081420836051921,-0.00259226199818282,-0.003303712578677,0.0196328370737072,237.0 -0.0489735217864827,-0.044641636506989,0.0272062201544997,-0.0251802111642493,0.023198191627409,0.0184144756665219,-0.0618090346724622,0.0800662487638535,0.0722236508199124,0.0320591578182113,225.0 -0.0417084448844436,-0.044641636506989,-0.00836157828357004,-0.0263278347173518,0.0245741444856101,0.0162224364339952,0.0707299262746723,-0.0394933828740919,-0.0483617248028919,-0.0300724459043093,81.0 --0.0236772472339084,-0.044641636506989,-0.0159062628007364,-0.0125563519424068,0.0204462859110067,0.0412743133771578,-0.0434008456520269,0.0343088588777263,0.0140724525157685,-0.0093619113301358,151.0 --0.0382074010379866,0.0506801187398187,0.00457216660300077,0.0356438377699009,-0.0112006298276192,0.00588853719494063,-0.0470824834561139,0.0343088588777263,0.0163049527999418,-0.00107769750046639,107.0 -0.0489735217864827,-0.044641636506989,-0.0428515646477589,-0.0538708002672419,0.0452134373586271,0.0500424703072647,0.0339135482338016,-0.00259226199818282,-0.0259524244351894,-0.063209301222987,64.0 -0.0453409833354632,0.0506801187398187,0.00564997867688165,0.0563010619323185,0.0644767773734429,0.0891860280309562,-0.0397192078479398,0.0712099797536354,0.0155668445407018,-0.0093619113301358,138.0 -0.0453409833354632,0.0506801187398187,-0.0353068801305926,0.063186803319791,-0.00432086553661359,-0.00162702588800815,-0.0102661054152432,-0.00259226199818282,0.0155668445407018,0.0569117993072195,185.0 -0.0162806757273067,-0.044641636506989,0.023972783932857,-0.0228849640236156,-0.0249601584096305,-0.0260526059075917,-0.0323559322397657,-0.00259226199818282,0.0372320112089689,0.0320591578182113,265.0 --0.0745327855481821,0.0506801187398187,-0.0180618869484982,0.0081008722200108,-0.019456346976826,-0.0248000120604336,-0.0654906724765493,0.0343088588777263,0.0673172179146849,-0.0176461251598052,101.0 --0.0817978624502212,0.0506801187398187,0.0422955891888323,-0.0194420933298793,0.0397096259258226,0.0575580333902134,-0.0691723102806364,0.108111100629544,0.0471861678860197,-0.0383566597339788,137.0 --0.067267708646143,-0.044641636506989,-0.0547074974604488,-0.0263278347173518,-0.0758704141630723,-0.082106180567918,0.0486400994501499,-0.076394503750001,-0.0868289932162924,-0.104630370371334,143.0 -0.00538306037424807,-0.044641636506989,-0.00297251791416553,0.0494153205448459,0.0741084473808508,0.0707102687853738,0.0449584616460628,-0.00259226199818282,-0.00149858682029207,-0.0093619113301358,141.0 --0.00188201652779104,-0.044641636506989,-0.0665634302731387,0.00121513083253827,-0.00294491267841247,0.00307020103883484,0.0118237214092792,-0.00259226199818282,-0.0202887477516296,-0.0259303389894746,79.0 -0.00901559882526763,-0.044641636506989,-0.0126728265790937,0.0287580963824284,-0.0180803941186249,-0.005071658967693,-0.0470824834561139,0.0343088588777263,0.0233748412798208,-0.0052198044153011,292.0 --0.00551455497881059,0.0506801187398187,-0.041773752573878,-0.0435421881860331,-0.0799982727376757,-0.0761563597939169,-0.0323559322397657,-0.0394933828740919,0.0102256424049578,-0.0093619113301358,178.0 -0.0562385986885218,0.0506801187398187,-0.030995631835069,0.0081008722200108,0.0190703330528056,0.0212328118226277,0.0339135482338016,-0.0394933828740919,-0.0295276227417736,-0.0590671943081523,91.0 -0.00901559882526763,0.0506801187398187,-0.00512814206192736,-0.0641994123484507,0.0699805888062474,0.0838625041805342,-0.0397192078479398,0.0712099797536354,0.0395398780720242,0.0196328370737072,116.0 --0.067267708646143,-0.044641636506989,-0.0590187457559724,0.0322009670761646,-0.051103262715452,-0.0495387405418066,-0.0102661054152432,-0.0394933828740919,0.00200784054982379,0.0237749439885419,86.0 -0.0271782910803654,0.0506801187398187,0.0250505960067379,0.0149866136074833,0.0259500973438113,0.048476727998317,-0.0397192078479398,0.0343088588777263,0.00783714230182385,0.0237749439885419,122.0 --0.0236772472339084,-0.044641636506989,-0.0460850008694016,-0.0332135761048244,0.0328298616348169,0.0362639379885253,0.0375951860378887,-0.00259226199818282,-0.0332487872476258,0.0113486232440377,72.0 -0.0489735217864827,0.0506801187398187,0.00349435452911985,0.0700725447072635,-0.00844872411121698,0.0134041002778894,-0.0544457590642881,0.0343088588777263,0.0133159679089277,0.036201264733046,129.0 --0.0527375548420648,-0.044641636506989,0.0541515220015222,-0.0263278347173518,-0.0552311212900554,-0.03388131745233,-0.0139477432193303,-0.0394933828740919,-0.0740888714915354,-0.0590671943081523,142.0 -0.0417084448844436,-0.044641636506989,-0.0450071887955207,0.0344962143200845,0.0438374845004259,-0.0157187066685371,0.0375951860378887,-0.0144006206784737,0.089898693277671,0.00720651632920303,90.0 -0.0562385986885218,-0.044641636506989,-0.0579409336820915,-0.00796585769556799,0.0520932016496327,0.0491030249218961,0.056003375058324,-0.0214118336448964,-0.0283202425479987,0.0444854785627154,158.0 --0.034574862586967,0.0506801187398187,-0.0557853095343297,-0.015999222636143,-0.00982467696941811,-0.00788999512379879,0.0375951860378887,-0.0394933828740919,-0.0529587932392004,0.0279170509033766,39.0 -0.0816663678456587,0.0506801187398187,0.00133873038135806,0.0356438377699009,0.126394655992494,0.0910649188016934,0.0191869970174533,0.0343088588777263,0.0844952822124031,-0.0300724459043093,196.0 --0.00188201652779104,0.0506801187398187,0.0304396563761424,0.0528581912385822,0.0397096259258226,0.0566185880048449,-0.0397192078479398,0.0712099797536354,0.0253931349154494,0.0279170509033766,222.0 -0.110726675453815,0.0506801187398187,0.00672779075076256,0.0287580963824284,-0.0277120641260328,-0.00726369820021974,-0.0470824834561139,0.0343088588777263,0.00200784054982379,0.0776223338813931,277.0 --0.0309423241359475,-0.044641636506989,0.0466068374843559,0.0149866136074833,-0.0167044412604238,-0.0470335528474903,0.000778807997017968,-0.00259226199818282,0.0634559213720654,-0.0259303389894746,99.0 -0.00175052192322852,0.0506801187398187,0.0261284080806188,-0.00911348124867051,0.0245741444856101,0.038455977221052,-0.0213110188275045,0.0343088588777263,0.00943640914607987,0.00306440941436832,196.0 -0.00901559882526763,-0.044641636506989,0.045529025410475,0.0287580963824284,0.0121905687618,-0.0138398158977999,0.0265502726256275,-0.0394933828740919,0.0461323310394148,0.036201264733046,202.0 -0.030810829531385,-0.044641636506989,0.0401399650410705,0.076958286094736,0.0176943801946045,0.0378296802974729,-0.0286742944356786,0.0343088588777263,-0.00149858682029207,0.11904340302974,155.0 -0.0380759064334241,0.0506801187398187,-0.0180618869484982,0.0666296740135272,-0.051103262715452,-0.0166581520539057,-0.0765355858888105,0.0343088588777263,-0.0119006848015081,-0.0135040182449705,77.0 -0.00901559882526763,-0.044641636506989,0.0142724752679289,0.0149866136074833,0.054845107366035,0.0472241341511589,0.0707299262746723,-0.0394933828740919,-0.0332487872476258,-0.0590671943081523,191.0 -0.0925639831987174,-0.044641636506989,0.0369065288194278,0.0218723549949558,-0.0249601584096305,-0.0166581520539057,0.000778807997017968,-0.0394933828740919,-0.0225121719296605,-0.0217882320746399,70.0 -0.0671362140415805,-0.044641636506989,0.00349435452911985,0.0356438377699009,0.0493412959332305,0.0312535625998928,0.0707299262746723,-0.0394933828740919,-0.000609254186102297,0.0196328370737072,73.0 -0.00175052192322852,-0.044641636506989,-0.0708746785686623,-0.0228849640236156,-0.00156895982021134,-0.00100072896442909,0.0265502726256275,-0.0394933828740919,-0.0225121719296605,0.00720651632920303,49.0 -0.030810829531385,-0.044641636506989,-0.0331512559828308,-0.0228849640236156,-0.0469754041408486,-0.0811667351825494,0.103864666511456,-0.076394503750001,-0.0398095943643375,-0.0549250873933176,65.0 -0.0271782910803654,0.0506801187398187,0.0940305687351156,0.0976155102571536,-0.0345918284170385,-0.0320024266815928,-0.0434008456520269,-0.00259226199818282,0.0366457977933988,0.106617082285236,263.0 -0.0126481372762872,0.0506801187398187,0.0358287167455469,0.0494153205448459,0.0534691545078339,0.0741549018650587,-0.0691723102806364,0.145012221505454,0.0456008084141249,0.0486275854775501,248.0 -0.0744012909436196,-0.044641636506989,0.0315174684500233,0.10105838095089,0.0465893902168282,0.0368902349121043,0.0155053592133662,-0.00259226199818282,0.0336568129023847,0.0444854785627154,296.0 --0.0418399394890061,-0.044641636506989,-0.0654856181992578,-0.0400993174922969,-0.00569681839481472,0.014343545663258,-0.0434008456520269,0.0343088588777263,0.00702686254915195,-0.0135040182449705,214.0 --0.0890629393522603,-0.044641636506989,-0.041773752573878,-0.0194420933298793,-0.0662387441556644,-0.0742774690231797,0.0081420836051921,-0.0394933828740919,0.00114379737951254,-0.0300724459043093,185.0 -0.0235457526293458,0.0506801187398187,-0.0396181284261162,-0.00567061055493425,-0.0483513569990498,-0.0332550205287509,0.0118237214092792,-0.0394933828740919,-0.101643547945512,-0.0673514081378217,78.0 --0.0454724779400257,-0.044641636506989,-0.0385403163522353,-0.0263278347173518,-0.0153284884022226,0.000878161806308105,-0.0323559322397657,-0.00259226199818282,0.00114379737951254,-0.0383566597339788,93.0 --0.0236772472339084,0.0506801187398187,-0.0256065714656645,0.0425295791573734,-0.0538551684318543,-0.0476598497710694,-0.0213110188275045,-0.0394933828740919,0.00114379737951254,0.0196328370737072,252.0 --0.099960554705319,-0.044641636506989,-0.0234509473179027,-0.0641994123484507,-0.0579830270064577,-0.0601857882426507,0.0118237214092792,-0.0394933828740919,-0.0181182673078967,-0.0507829804784829,150.0 --0.0273097856849279,-0.044641636506989,-0.0665634302731387,-0.112399602060758,-0.0497273098572509,-0.0413968805352788,0.000778807997017968,-0.0394933828740919,-0.0358167281015492,-0.0093619113301358,77.0 -0.030810829531385,0.0506801187398187,0.0325952805239042,0.0494153205448459,-0.040095639849843,-0.0435889197678055,-0.0691723102806364,0.0343088588777263,0.0630166151147464,0.00306440941436832,208.0 --0.103593093156339,0.0506801187398187,-0.0460850008694016,-0.0263278347173518,-0.0249601584096305,-0.0248000120604336,0.0302319104297145,-0.0394933828740919,-0.0398095943643375,-0.0549250873933176,77.0 -0.0671362140415805,0.0506801187398187,-0.0299178197611881,0.0574486853821349,-0.000193006962010205,-0.0157187066685371,0.0744115640787594,-0.0505637191368646,-0.0384591123013538,0.00720651632920303,108.0 --0.0527375548420648,-0.044641636506989,-0.0126728265790937,-0.0607565416547144,-0.000193006962010205,0.00808057642746734,0.0118237214092792,-0.00259226199818282,-0.0271286455543265,-0.0507829804784829,160.0 --0.0273097856849279,0.0506801187398187,-0.0159062628007364,-0.0297707054110881,0.00393485161259318,-0.000687580502639557,0.0412768238419757,-0.0394933828740919,-0.0236445575721341,0.0113486232440377,53.0 --0.0382074010379866,0.0506801187398187,0.0713965151836166,-0.0573136709609782,0.153913713156516,0.155886650392127,0.000778807997017968,0.0719480021711535,0.0502764933899896,0.0693381200517237,220.0 -0.00901559882526763,-0.044641636506989,-0.030995631835069,0.0218723549949558,0.00806271018719657,0.00870687335104641,0.00446044580110504,-0.00259226199818282,0.00943640914607987,0.0113486232440377,154.0 -0.0126481372762872,0.0506801187398187,0.000260918307477141,-0.0114087283893043,0.0397096259258226,0.0572448849284239,-0.0397192078479398,0.0560805201945126,0.024052583226893,0.0320591578182113,259.0 -0.0671362140415805,-0.044641636506989,0.0369065288194278,-0.0504279295735057,-0.0235842055514294,-0.034507614375909,0.0486400994501499,-0.0394933828740919,-0.0259524244351894,-0.0383566597339788,90.0 -0.0453409833354632,-0.044641636506989,0.0390621529671896,0.0459724498511097,0.00668675732899544,-0.0241737151368545,0.0081420836051921,-0.0125555646346783,0.0643282330236709,0.0569117993072195,246.0 -0.0671362140415805,0.0506801187398187,-0.0148284507268555,0.0585963091762383,-0.0593589798646588,-0.034507614375909,-0.0618090346724622,0.012906208769699,-0.00514530798026311,0.0486275854775501,124.0 -0.0271782910803654,-0.044641636506989,0.00672779075076256,0.0356438377699009,0.0796122588136553,0.0707102687853738,0.0155053592133662,0.0343088588777263,0.0406722637144977,0.0113486232440377,67.0 -0.0562385986885218,-0.044641636506989,-0.0687190544209005,-0.0687899065952895,-0.000193006962010205,-0.00100072896442909,0.0449584616460628,-0.0376483268302965,-0.0483617248028919,-0.00107769750046639,72.0 -0.0344433679824045,0.0506801187398187,-0.00943939035745095,0.0597439326260547,-0.0359677812752396,-0.00757684666200928,-0.0765355858888105,0.0712099797536354,0.0110081010458725,-0.0217882320746399,257.0 -0.0235457526293458,-0.044641636506989,0.0196615356373334,-0.0125563519424068,0.0837401173882587,0.0387691256828415,0.0633666506664982,-0.00259226199818282,0.0660482061630984,0.0486275854775501,262.0 -0.0489735217864827,0.0506801187398187,0.0746299514052593,0.0666296740135272,-0.00982467696941811,-0.00225332281158722,-0.0434008456520269,0.0343088588777263,0.0336568129023847,0.0196328370737072,275.0 -0.030810829531385,0.0506801187398187,-0.00836157828357004,0.00465800152627453,0.0149424744782022,0.0274957810584184,0.0081420836051921,-0.00812743012956918,-0.0295276227417736,0.0569117993072195,177.0 --0.103593093156339,0.0506801187398187,-0.0234509473179027,-0.0228849640236156,-0.0868780370286814,-0.0677013513255995,-0.0176293810234174,-0.0394933828740919,-0.0781409106690696,-0.0714935150526564,71.0 -0.0162806757273067,0.0506801187398187,-0.0460850008694016,0.0115437429137471,-0.0332158755588373,-0.0160318551303266,-0.0102661054152432,-0.00259226199818282,-0.0439854025655911,-0.0424987666488135,47.0 --0.0600026317441039,0.0506801187398187,0.0541515220015222,-0.0194420933298793,-0.0497273098572509,-0.0489124436182275,0.0228686348215404,-0.0394933828740919,-0.0439854025655911,-0.0052198044153011,187.0 --0.0273097856849279,-0.044641636506989,-0.0353068801305926,-0.0297707054110881,-0.0566070741482565,-0.058620045933703,0.0302319104297145,-0.0394933828740919,-0.0498684677352306,-0.129483011860342,125.0 -0.0417084448844436,-0.044641636506989,-0.0320734439089499,-0.061904165207817,0.0796122588136553,0.0509819156926333,0.056003375058324,-0.00997248617336464,0.0450661683362615,-0.0590671943081523,78.0 --0.0817978624502212,-0.044641636506989,-0.0816527993074713,-0.0400993174922969,0.00255889875439205,-0.0185370428246429,0.0707299262746723,-0.0394933828740919,-0.0109044358473771,-0.09220404962683,51.0 --0.0418399394890061,-0.044641636506989,0.0476846495582368,0.0597439326260547,0.127770608850695,0.128016437292858,-0.0249926566315915,0.108111100629544,0.0638931206368394,0.0403433716478807,258.0 --0.0127796318808497,-0.044641636506989,0.0606183944448076,0.0528581912385822,0.0479653430750293,0.0293746718291555,-0.0176293810234174,0.0343088588777263,0.0702112981933102,0.00720651632920303,215.0 -0.0671362140415805,-0.044641636506989,0.056307146149284,0.0735154154009998,-0.0139525355440215,-0.039204841302752,-0.0323559322397657,-0.00259226199818282,0.0757375884575476,0.036201264733046,303.0 --0.0527375548420648,0.0506801187398187,0.098341817030639,0.0872868981759448,0.0603489187988395,0.0487898764601065,-0.0581273968683752,0.108111100629544,0.0844952822124031,0.0403433716478807,243.0 -0.00538306037424807,-0.044641636506989,0.0595405823709267,-0.0561660474078757,0.0245741444856101,0.0528608064633705,-0.0434008456520269,0.0509143632718854,-0.00421985970694603,-0.0300724459043093,91.0 -0.0816663678456587,-0.044641636506989,0.0336730925977851,0.0081008722200108,0.0520932016496327,0.0566185880048449,-0.0176293810234174,0.0343088588777263,0.0348641930961596,0.0693381200517237,150.0 -0.030810829531385,0.0506801187398187,0.056307146149284,0.076958286094736,0.0493412959332305,-0.0122740735888523,-0.0360375700438527,0.0712099797536354,0.120053382001538,0.0900486546258972,310.0 -0.00175052192322852,-0.044641636506989,-0.0654856181992578,-0.00567061055493425,-0.00707277125301585,-0.0194764882100115,0.0412768238419757,-0.0394933828740919,-0.003303712578677,0.00720651632920303,153.0 --0.0491050163910452,-0.044641636506989,0.160854917315731,-0.0469850588797694,-0.0290880169842339,-0.019789636671801,-0.0470824834561139,0.0343088588777263,0.028016506523264,0.0113486232440377,346.0 --0.0273097856849279,0.0506801187398187,-0.0557853095343297,0.0253152256886921,-0.00707277125301585,-0.0235474182132754,0.052321737254237,-0.0394933828740919,-0.00514530798026311,-0.0507829804784829,63.0 -0.0780338293946392,0.0506801187398187,-0.0245287593917836,-0.0423945646329306,0.00668675732899544,0.0528608064633705,-0.0691723102806364,0.0808042711813717,-0.0371283460104736,0.0569117993072195,89.0 -0.0126481372762872,-0.044641636506989,-0.0363846922044735,0.0425295791573734,-0.0139525355440215,0.0129343775852051,-0.0268334755336351,0.00515697338575809,-0.0439854025655911,0.00720651632920303,50.0 -0.0417084448844436,-0.044641636506989,-0.00836157828357004,-0.0573136709609782,0.00806271018719657,-0.0313761297580137,0.151725957964588,-0.076394503750001,-0.0802365402489018,-0.0176461251598052,39.0 -0.0489735217864827,-0.044641636506989,-0.041773752573878,0.104501251644626,0.0355817673512192,-0.0257394574458021,0.177497422593197,-0.076394503750001,-0.0129079422541688,0.0154907301588724,103.0 --0.0164121703318693,0.0506801187398187,0.127442743025423,0.0976155102571536,0.0163184273364034,0.0174750302811533,-0.0213110188275045,0.0343088588777263,0.0348641930961596,0.00306440941436832,308.0 --0.0745327855481821,0.0506801187398187,-0.0773415510119477,-0.0469850588797694,-0.0469754041408486,-0.0326287236051719,0.00446044580110504,-0.0394933828740919,-0.072128454601956,-0.0176461251598052,116.0 -0.0344433679824045,0.0506801187398187,0.0282840322283806,-0.0332135761048244,-0.0455994512826475,-0.00976888589453599,-0.050764121260201,-0.00259226199818282,-0.0594726974107223,-0.0217882320746399,145.0 --0.034574862586967,0.0506801187398187,-0.0256065714656645,-0.0171468461892456,0.00118294589619092,-0.00287961973516629,0.0081420836051921,-0.015507654304751,0.0148227108412663,0.0403433716478807,74.0 --0.0527375548420648,0.0506801187398187,-0.0622521819776151,0.0115437429137471,-0.00844872411121698,-0.0366996536084358,0.122272855531891,-0.076394503750001,-0.0868289932162924,0.00306440941436832,45.0 -0.0598711371395414,-0.044641636506989,-0.000816893766403737,-0.0848566365108683,0.075484400239052,0.0794784257154807,0.00446044580110504,0.0343088588777263,0.0233748412798208,0.0279170509033766,115.0 -0.063503675590561,0.0506801187398187,0.088641508365711,0.0700725447072635,0.0204462859110067,0.0375165318356834,-0.050764121260201,0.0712099797536354,0.0293004132685869,0.0734802269665584,264.0 -0.00901559882526763,-0.044641636506989,-0.0320734439089499,-0.0263278347173518,0.0424615316422248,-0.0103951828181151,0.159089233572762,-0.076394503750001,-0.0119006848015081,-0.0383566597339788,87.0 -0.00538306037424807,0.0506801187398187,0.0304396563761424,0.0838440274822086,-0.0373437341334407,-0.0473467013092799,0.0155053592133662,-0.0394933828740919,0.00864028293306308,0.0154907301588724,202.0 -0.0380759064334241,0.0506801187398187,0.00888341489852436,0.0425295791573734,-0.0428475455662452,-0.0210422305189592,-0.0397192078479398,-0.00259226199818282,-0.0181182673078967,0.00720651632920303,127.0 -0.0126481372762872,-0.044641636506989,0.00672779075076256,-0.0561660474078757,-0.0758704141630723,-0.0664487574784414,-0.0213110188275045,-0.0376483268302965,-0.0181182673078967,-0.09220404962683,182.0 -0.0744012909436196,0.0506801187398187,-0.02021751109626,0.0459724498511097,0.0741084473808508,0.0328193049088404,-0.0360375700438527,0.0712099797536354,0.106354276741726,0.036201264733046,241.0 -0.0162806757273067,-0.044641636506989,-0.0245287593917836,0.0356438377699009,-0.00707277125301585,-0.00319276819695581,-0.0139477432193303,-0.00259226199818282,0.0155668445407018,0.0154907301588724,66.0 --0.00551455497881059,0.0506801187398187,-0.0115950145052127,0.0115437429137471,-0.0222082526932283,-0.0154055582067476,-0.0213110188275045,-0.00259226199818282,0.0110081010458725,0.0693381200517237,94.0 -0.0126481372762872,-0.044641636506989,0.0261284080806188,0.063186803319791,0.125018703134293,0.0916912157252725,0.0633666506664982,-0.00259226199818282,0.057572856202426,-0.0217882320746399,283.0 --0.034574862586967,-0.044641636506989,-0.0590187457559724,0.00121513083253827,-0.0538551684318543,-0.078035250564654,0.0670482884705852,-0.076394503750001,-0.02139368094036,0.0154907301588724,64.0 -0.0671362140415805,0.0506801187398187,-0.0363846922044735,-0.0848566365108683,-0.00707277125301585,0.01966706951368,-0.0544457590642881,0.0343088588777263,0.00114379737951254,0.0320591578182113,102.0 -0.0380759064334241,0.0506801187398187,-0.0245287593917836,0.00465800152627453,-0.0263361112678317,-0.0263657543693812,0.0155053592133662,-0.0394933828740919,-0.0159982677581387,-0.0259303389894746,200.0 -0.00901559882526763,0.0506801187398187,0.0185837235634525,0.0390867084636372,0.0176943801946045,0.0105857641217836,0.0191869970174533,-0.00259226199818282,0.0163049527999418,-0.0176461251598052,265.0 --0.0926954778032799,0.0506801187398187,-0.0902752958985185,-0.0573136709609782,-0.0249601584096305,-0.0304366843726451,-0.00658446761115617,-0.00259226199818282,0.024052583226893,0.00306440941436832,94.0 -0.0707687524926,-0.044641636506989,-0.00512814206192736,-0.00567061055493425,0.0878679759628621,0.102964560349696,0.0118237214092792,0.0343088588777263,-0.0089440189577978,0.0279170509033766,230.0 --0.0164121703318693,-0.044641636506989,-0.052551873312687,-0.0332135761048244,-0.0442234984244464,-0.0363865051466462,0.0191869970174533,-0.0394933828740919,-0.0683297436244215,-0.0300724459043093,181.0 -0.0417084448844436,0.0506801187398187,-0.0223731352440218,0.0287580963824284,-0.0662387441556644,-0.0451546620767532,-0.0618090346724622,-0.00259226199818282,0.00286377051894013,-0.0549250873933176,156.0 -0.0126481372762872,-0.044641636506989,-0.02021751109626,-0.015999222636143,0.0121905687618,0.0212328118226277,-0.0765355858888105,0.108111100629544,0.0598807230654812,-0.0217882320746399,233.0 --0.0382074010379866,-0.044641636506989,-0.0547074974604488,-0.0779708951233958,-0.0332158755588373,-0.0864902590329714,0.140681044552327,-0.076394503750001,-0.0191970476139445,-0.0052198044153011,60.0 -0.0453409833354632,-0.044641636506989,-0.00620595413580824,-0.015999222636143,0.125018703134293,0.125198101136752,0.0191869970174533,0.0343088588777263,0.0324332257796019,-0.0052198044153011,219.0 -0.0707687524926,0.0506801187398187,-0.0169840748746173,0.0218723549949558,0.0438374845004259,0.0563054395430553,0.0375951860378887,-0.00259226199818282,-0.0702093127286876,-0.0176461251598052,80.0 --0.0745327855481821,0.0506801187398187,0.0552293340754031,-0.0400993174922969,0.0534691545078339,0.05317395492516,-0.0434008456520269,0.0712099797536354,0.061237907519701,-0.0342145528191441,68.0 -0.0598711371395414,0.0506801187398187,0.0767855755530211,0.0253152256886921,0.00118294589619092,0.0168487333575743,-0.0544457590642881,0.0343088588777263,0.0299356483965325,0.0444854785627154,332.0 -0.0744012909436196,-0.044641636506989,0.0185837235634525,0.063186803319791,0.0617248716570406,0.0428400556861055,0.0081420836051921,-0.00259226199818282,0.0580391276638951,-0.0590671943081523,248.0 -0.00901559882526763,-0.044641636506989,-0.0223731352440218,-0.0320659525517218,-0.0497273098572509,-0.0686407967109681,0.0780932018828464,-0.0708593356186146,-0.0629129499162512,-0.0383566597339788,84.0 --0.0709002470971626,-0.044641636506989,0.0929527566612346,0.0126913664668496,0.0204462859110067,0.0425269072243159,0.000778807997017968,0.000359827671889909,-0.0545441527110952,-0.00107769750046639,200.0 -0.0235457526293458,0.0506801187398187,-0.030995631835069,-0.00567061055493425,-0.0167044412604238,0.0177881787429428,-0.0323559322397657,-0.00259226199818282,-0.0740888714915354,-0.0342145528191441,55.0 --0.0527375548420648,0.0506801187398187,0.0390621529671896,-0.0400993174922969,-0.00569681839481472,-0.0129003705124313,0.0118237214092792,-0.0394933828740919,0.0163049527999418,0.00306440941436832,85.0 -0.0671362140415805,-0.044641636506989,-0.0611743699037342,-0.0400993174922969,-0.0263361112678317,-0.024486863598644,0.0339135482338016,-0.0394933828740919,-0.0561575730950062,-0.0590671943081523,89.0 -0.00175052192322852,-0.044641636506989,-0.00836157828357004,-0.0641994123484507,-0.0387196869916418,-0.024486863598644,0.00446044580110504,-0.0394933828740919,-0.0646830224644503,-0.0549250873933176,31.0 -0.0235457526293458,0.0506801187398187,-0.0374625042783544,-0.0469850588797694,-0.0910058956032848,-0.0755300628703378,-0.0323559322397657,-0.0394933828740919,-0.0307512098645563,-0.0135040182449705,129.0 -0.0380759064334241,0.0506801187398187,-0.0137506386529745,-0.015999222636143,-0.0359677812752396,-0.0219816759043277,-0.0139477432193303,-0.00259226199818282,-0.0259524244351894,-0.00107769750046639,83.0 -0.0162806757273067,-0.044641636506989,0.0735521393313785,-0.0412469410453994,-0.00432086553661359,-0.0135266674360104,-0.0139477432193303,-0.00111621716314646,0.0428956878925287,0.0444854785627154,275.0 --0.00188201652779104,0.0506801187398187,-0.0245287593917836,0.0528581912385822,0.0273260502020124,0.0300009687527346,0.0302319104297145,-0.00259226199818282,-0.02139368094036,0.036201264733046,65.0 -0.0126481372762872,-0.044641636506989,0.0336730925977851,0.0333485905259811,0.0300779559184146,0.0271826325966288,-0.0029028298070691,0.00884708547334898,0.0311929907028023,0.0279170509033766,198.0 -0.0744012909436196,-0.044641636506989,0.034750904671666,0.0941726395634173,0.0575970130824372,0.0202933664372591,0.0228686348215404,-0.00259226199818282,0.0738021469200488,-0.0217882320746399,236.0 -0.0417084448844436,0.0506801187398187,-0.0385403163522353,0.0528581912385822,0.0768603530972531,0.116429944206646,-0.0397192078479398,0.0712099797536354,-0.0225121719296605,-0.0135040182449705,253.0 --0.00914709342983014,0.0506801187398187,-0.0396181284261162,-0.0400993174922969,-0.00844872411121698,0.0162224364339952,-0.0654906724765493,0.0712099797536354,0.0177634778671173,-0.0673514081378217,124.0 -0.00901559882526763,0.0506801187398187,-0.00189470584028465,0.0218723549949558,-0.0387196869916418,-0.0248000120604336,-0.00658446761115617,-0.0394933828740919,-0.0398095943643375,-0.0135040182449705,44.0 -0.0671362140415805,0.0506801187398187,-0.030995631835069,0.00465800152627453,0.0245741444856101,0.0356376410649462,-0.0286742944356786,0.0343088588777263,0.0233748412798208,0.0817644407962278,172.0 -0.00175052192322852,-0.044641636506989,-0.0460850008694016,-0.0332135761048244,-0.07311850844667,-0.0814798836443389,0.0449584616460628,-0.0693832907835783,-0.0611765950943345,-0.0797777288823259,114.0 --0.00914709342983014,0.0506801187398187,0.00133873038135806,-0.00222773986119799,0.0796122588136553,0.0700839718617947,0.0339135482338016,-0.00259226199818282,0.0267142576335128,0.0817644407962278,142.0 --0.00551455497881059,-0.044641636506989,0.0649296427403312,0.0356438377699009,-0.00156895982021134,0.0149698425868371,-0.0139477432193303,0.000728838880648992,-0.0181182673078967,0.0320591578182113,109.0 -0.096196521649737,-0.044641636506989,0.0401399650410705,-0.0573136709609782,0.0452134373586271,0.0606895180081088,-0.0213110188275045,0.0361539149215217,0.0125531528133893,0.0237749439885419,180.0 --0.0745327855481821,-0.044641636506989,-0.0234509473179027,-0.00567061055493425,-0.0208322998350272,-0.0141529643595894,0.0155053592133662,-0.0394933828740919,-0.0384591123013538,-0.0300724459043093,144.0 -0.0598711371395414,0.0506801187398187,0.0530737099276413,0.0528581912385822,0.0328298616348169,0.01966706951368,-0.0102661054152432,0.0343088588777263,0.0552050380896167,-0.00107769750046639,163.0 --0.0236772472339084,-0.044641636506989,0.0401399650410705,-0.0125563519424068,-0.00982467696941811,-0.00100072896442909,-0.0029028298070691,-0.00259226199818282,-0.0119006848015081,-0.0383566597339788,147.0 -0.00901559882526763,-0.044641636506989,-0.02021751109626,-0.0538708002672419,0.0314539087766158,0.0206065148990486,0.056003375058324,-0.0394933828740919,-0.0109044358473771,-0.00107769750046639,97.0 -0.0162806757273067,0.0506801187398187,0.0142724752679289,0.00121513083253827,0.00118294589619092,-0.0213553789807487,-0.0323559322397657,0.0343088588777263,0.0749683360277342,0.0403433716478807,220.0 -0.0199132141783263,-0.044641636506989,-0.0342290680567117,0.055153438482502,0.0672286830898452,0.0741549018650587,-0.00658446761115617,0.0328328140426899,0.0247253233428045,0.0693381200517237,190.0 -0.0889314447476978,-0.044641636506989,0.00672779075076256,0.0253152256886921,0.0300779559184146,0.00870687335104641,0.0633666506664982,-0.0394933828740919,0.00943640914607987,0.0320591578182113,109.0 -0.0199132141783263,-0.044641636506989,0.00457216660300077,0.0459724498511097,-0.0180803941186249,-0.0545491159304391,0.0633666506664982,-0.0394933828740919,0.0286607203138089,0.0610539062220542,191.0 --0.0236772472339084,-0.044641636506989,0.0304396563761424,-0.00567061055493425,0.0823641645300576,0.092004364187062,-0.0176293810234174,0.0712099797536354,0.0330470723549341,0.00306440941436832,122.0 -0.096196521649737,-0.044641636506989,0.0519958978537604,0.0792535333386559,0.054845107366035,0.0365770864503148,-0.0765355858888105,0.141322109417863,0.098646374304928,0.0610539062220542,230.0 -0.0235457526293458,0.0506801187398187,0.0616962065186885,0.0620391798699746,0.0245741444856101,-0.0360733566848567,-0.0912621371051588,0.155344535350708,0.133395733837469,0.0817644407962278,242.0 -0.0707687524926,0.0506801187398187,-0.00728376620968916,0.0494153205448459,0.0603489187988395,-0.00444536204411395,-0.0544457590642881,0.108111100629544,0.129019411600168,0.0569117993072195,248.0 -0.030810829531385,-0.044641636506989,0.00564997867688165,0.0115437429137471,0.0782363059554542,0.077912683406533,-0.0434008456520269,0.108111100629544,0.0660482061630984,0.0196328370737072,249.0 --0.00188201652779104,-0.044641636506989,0.0541515220015222,-0.0664946594890845,0.0727324945226497,0.0566185880048449,-0.0434008456520269,0.0848633944777217,0.0844952822124031,0.0486275854775501,192.0 -0.0453409833354632,0.0506801187398187,-0.00836157828357004,-0.0332135761048244,-0.00707277125301585,0.00119131026809764,-0.0397192078479398,0.0343088588777263,0.0299356483965325,0.0279170509033766,131.0 -0.0744012909436196,-0.044641636506989,0.114508998138853,0.0287580963824284,0.0245741444856101,0.0249905933641021,0.0191869970174533,-0.00259226199818282,-0.000609254186102297,-0.0052198044153011,237.0 --0.0382074010379866,-0.044641636506989,0.067085266888093,-0.0607565416547144,-0.0290880169842339,-0.0232342697514859,-0.0102661054152432,-0.00259226199818282,-0.00149858682029207,0.0196328370737072,78.0 --0.0127796318808497,0.0506801187398187,-0.0557853095343297,-0.00222773986119799,-0.0277120641260328,-0.029184090525487,0.0191869970174533,-0.0394933828740919,-0.0170521046047435,0.0444854785627154,135.0 -0.00901559882526763,0.0506801187398187,0.0304396563761424,0.0425295791573734,-0.00294491267841247,0.0368902349121043,-0.0654906724765493,0.0712099797536354,-0.0236445575721341,0.0154907301588724,244.0 -0.0816663678456587,0.0506801187398187,-0.0256065714656645,-0.0366564467985606,-0.0703666027302678,-0.0464072559239113,-0.0397192078479398,-0.00259226199818282,-0.0411803851880079,-0.0052198044153011,199.0 -0.030810829531385,-0.044641636506989,0.104808689473925,0.076958286094736,-0.0112006298276192,-0.0113346282034837,-0.0581273968683752,0.0343088588777263,0.0571041874478439,0.036201264733046,270.0 -0.0271782910803654,0.0506801187398187,-0.00620595413580824,0.0287580963824284,-0.0167044412604238,-0.00162702588800815,-0.0581273968683752,0.0343088588777263,0.0293004132685869,0.0320591578182113,164.0 --0.0600026317441039,0.0506801187398187,-0.0471628129432825,-0.0228849640236156,-0.071742555588469,-0.0576806005483345,-0.00658446761115617,-0.0394933828740919,-0.0629129499162512,-0.0549250873933176,72.0 -0.00538306037424807,-0.044641636506989,-0.0482406250171634,-0.0125563519424068,0.00118294589619092,-0.00663740127664067,0.0633666506664982,-0.0394933828740919,-0.0514005352605825,-0.0590671943081523,96.0 --0.0200447087828888,-0.044641636506989,0.0854080721440683,-0.0366564467985606,0.0919958345374655,0.0894991764927457,-0.0618090346724622,0.145012221505454,0.0809479135112756,0.0527696923923848,306.0 -0.0199132141783263,0.0506801187398187,-0.0126728265790937,0.0700725447072635,-0.0112006298276192,0.00714113104209875,-0.0397192078479398,0.0343088588777263,0.00538436996854573,0.00306440941436832,91.0 --0.0636351701951234,-0.044641636506989,-0.0331512559828308,-0.0332135761048244,0.00118294589619092,0.0240511479787335,-0.0249926566315915,-0.00259226199818282,-0.0225121719296605,-0.0590671943081523,214.0 -0.0271782910803654,-0.044641636506989,-0.00728376620968916,-0.0504279295735057,0.075484400239052,0.0566185880048449,0.0339135482338016,-0.00259226199818282,0.0434431722527813,0.0154907301588724,95.0 --0.0164121703318693,-0.044641636506989,-0.0137506386529745,0.132044217194516,-0.00982467696941811,-0.00381906512053488,0.0191869970174533,-0.0394933828740919,-0.0358167281015492,-0.0300724459043093,216.0 -0.030810829531385,0.0506801187398187,0.0595405823709267,0.0563010619323185,-0.0222082526932283,0.00119131026809764,-0.0323559322397657,-0.00259226199818282,-0.0247911874324607,-0.0176461251598052,263.0 -0.0562385986885218,0.0506801187398187,0.0218171597850952,0.0563010619323185,-0.00707277125301585,0.0181013272047324,-0.0323559322397657,-0.00259226199818282,-0.0236445575721341,0.0237749439885419,178.0 --0.0200447087828888,-0.044641636506989,0.0185837235634525,0.090729768869681,0.00393485161259318,0.00870687335104641,0.0375951860378887,-0.0394933828740919,-0.0578000656756125,0.00720651632920303,113.0 --0.107225631607358,-0.044641636506989,-0.0115950145052127,-0.0400993174922969,0.0493412959332305,0.0644472995495832,-0.0139477432193303,0.0343088588777263,0.00702686254915195,-0.0300724459043093,200.0 -0.0816663678456587,0.0506801187398187,-0.00297251791416553,-0.0332135761048244,0.0424615316422248,0.057871181852003,-0.0102661054152432,0.0343088588777263,-0.000609254186102297,-0.00107769750046639,139.0 -0.00538306037424807,0.0506801187398187,0.0175059114895716,0.0322009670761646,0.127770608850695,0.127390140369279,-0.0213110188275045,0.0712099797536354,0.062575181458056,0.0154907301588724,139.0 -0.0380759064334241,0.0506801187398187,-0.0299178197611881,-0.0745280244296595,-0.0125765826858204,-0.0125872220506418,0.00446044580110504,-0.00259226199818282,0.00371173823343597,-0.0300724459043093,88.0 -0.030810829531385,-0.044641636506989,-0.02021751109626,-0.00567061055493425,-0.00432086553661359,-0.0294972389872765,0.0780932018828464,-0.0394933828740919,-0.0109044358473771,-0.00107769750046639,148.0 -0.00175052192322852,0.0506801187398187,-0.0579409336820915,-0.0435421881860331,-0.0965097070360893,-0.0470335528474903,-0.098625412713333,0.0343088588777263,-0.0611765950943345,-0.0714935150526564,88.0 --0.0273097856849279,0.0506801187398187,0.0606183944448076,0.107944122338362,0.0121905687618,-0.0175975974392743,-0.0029028298070691,-0.00259226199818282,0.0702112981933102,0.135611830689079,243.0 --0.0854304009012408,0.0506801187398187,-0.0406959404999971,-0.0332135761048244,-0.0813742255958769,-0.0695802420963367,-0.00658446761115617,-0.0394933828740919,-0.0578000656756125,-0.0424987666488135,71.0 -0.0126481372762872,0.0506801187398187,-0.0719524906425432,-0.0469850588797694,-0.051103262715452,-0.0971373067338155,0.118591217727804,-0.076394503750001,-0.0202887477516296,-0.0383566597339788,77.0 --0.0527375548420648,-0.044641636506989,-0.0557853095343297,-0.0366564467985606,0.0892439288210632,-0.00319276819695581,0.0081420836051921,0.0343088588777263,0.132372649338676,0.00306440941436832,109.0 --0.0236772472339084,0.0506801187398187,0.045529025410475,0.0218723549949558,0.10988322169408,0.0888728795691667,0.000778807997017968,0.0343088588777263,0.0741925366900307,0.0610539062220542,272.0 --0.0745327855481821,0.0506801187398187,-0.00943939035745095,0.0149866136074833,-0.0373437341334407,-0.0216685274425382,-0.0139477432193303,-0.00259226199818282,-0.0332487872476258,0.0113486232440377,60.0 --0.00551455497881059,0.0506801187398187,-0.0331512559828308,-0.015999222636143,0.00806271018719657,0.0162224364339952,0.0155053592133662,-0.00259226199818282,-0.0283202425479987,-0.0756356219674911,54.0 --0.0600026317441039,0.0506801187398187,0.0498402737059986,0.0184294843012196,-0.0167044412604238,-0.0301235359108556,-0.0176293810234174,-0.00259226199818282,0.049768659920749,-0.0590671943081523,221.0 --0.0200447087828888,-0.044641636506989,-0.084886235529114,-0.0263278347173518,-0.0359677812752396,-0.0341944659141195,0.0412768238419757,-0.0516707527631419,-0.0823814832581028,-0.0466408735636482,90.0 -0.0380759064334241,0.0506801187398187,0.00564997867688165,0.0322009670761646,0.00668675732899544,0.0174750302811533,-0.0249926566315915,0.0343088588777263,0.0148227108412663,0.0610539062220542,311.0 -0.0162806757273067,-0.044641636506989,0.0207393477112143,0.0218723549949558,-0.0139525355440215,-0.0132135189742209,-0.00658446761115617,-0.00259226199818282,0.0133159679089277,0.0403433716478807,281.0 -0.0417084448844436,-0.044641636506989,-0.00728376620968916,0.0287580963824284,-0.0428475455662452,-0.0482861466946485,0.052321737254237,-0.076394503750001,-0.072128454601956,0.0237749439885419,182.0 -0.0199132141783263,0.0506801187398187,0.104808689473925,0.0700725447072635,-0.0359677812752396,-0.0266789028311707,-0.0249926566315915,-0.00259226199818282,0.00371173823343597,0.0403433716478807,321.0 --0.0491050163910452,0.0506801187398187,-0.0245287593917836,6.75072794357462e-05,-0.0469754041408486,-0.0282446451401184,-0.0654906724765493,0.0284046795375808,0.0191990330785671,0.0113486232440377,58.0 -0.00175052192322852,0.0506801187398187,-0.00620595413580824,-0.0194420933298793,-0.00982467696941811,0.00494909180957202,-0.0397192078479398,0.0343088588777263,0.0148227108412663,0.0983328684555666,262.0 -0.0344433679824045,-0.044641636506989,-0.0385403163522353,-0.0125563519424068,0.0094386630453977,0.00526224027136155,-0.00658446761115617,-0.00259226199818282,0.0311929907028023,0.0983328684555666,206.0 --0.0454724779400257,0.0506801187398187,0.137143051690352,-0.015999222636143,0.0410855787840237,0.0318798595234718,-0.0434008456520269,0.0712099797536354,0.0710215779459822,0.0486275854775501,233.0 --0.00914709342983014,0.0506801187398187,0.17055522598066,0.0149866136074833,0.0300779559184146,0.033758750294209,-0.0213110188275045,0.0343088588777263,0.0336568129023847,0.0320591578182113,242.0 --0.0164121703318693,0.0506801187398187,0.00241654245523897,0.0149866136074833,0.0218222387692079,-0.0100820343563255,-0.0249926566315915,0.0343088588777263,0.085533121187439,0.0817644407962278,123.0 --0.00914709342983014,-0.044641636506989,0.0379843408933087,-0.0400993174922969,-0.0249601584096305,-0.00381906512053488,-0.0434008456520269,0.0158582984397717,-0.00514530798026311,0.0279170509033766,167.0 -0.0199132141783263,-0.044641636506989,-0.0579409336820915,-0.0573136709609782,-0.00156895982021134,-0.0125872220506418,0.0744115640787594,-0.0394933828740919,-0.0611765950943345,-0.0756356219674911,63.0 -0.0526060602375023,0.0506801187398187,-0.00943939035745095,0.0494153205448459,0.0507172487914316,-0.019163339748222,-0.0139477432193303,0.0343088588777263,0.119343994203787,-0.0176461251598052,197.0 --0.0273097856849279,0.0506801187398187,-0.0234509473179027,-0.015999222636143,0.0135665216200011,0.0127778033543103,0.0265502726256275,-0.00259226199818282,-0.0109044358473771,-0.0217882320746399,71.0 --0.0745327855481821,-0.044641636506989,-0.0105172024313319,-0.00567061055493425,-0.0662387441556644,-0.0570543036247554,-0.0029028298070691,-0.0394933828740919,-0.0425721049227942,-0.00107769750046639,168.0 --0.107225631607358,-0.044641636506989,-0.0342290680567117,-0.067642283042187,-0.0634868384392622,-0.0705196874817053,0.0081420836051921,-0.0394933828740919,-0.000609254186102297,-0.0797777288823259,140.0 -0.0453409833354632,0.0506801187398187,-0.00297251791416553,0.107944122338362,0.0355817673512192,0.0224854056697859,0.0265502726256275,-0.00259226199818282,0.028016506523264,0.0196328370737072,217.0 --0.00188201652779104,-0.044641636506989,0.068163078961974,-0.00567061055493425,0.119514891701488,0.130208476525385,-0.0249926566315915,0.0867084505215172,0.0461323310394148,-0.00107769750046639,121.0 -0.0199132141783263,0.0506801187398187,0.00996122697240527,0.0184294843012196,0.0149424744782022,0.0447189464568426,-0.0618090346724622,0.0712099797536354,0.00943640914607987,-0.063209301222987,235.0 -0.0162806757273067,0.0506801187398187,0.00241654245523897,-0.00567061055493425,-0.00569681839481472,0.0108989125835731,-0.050764121260201,0.0343088588777263,0.0226920225667445,-0.0383566597339788,245.0 --0.00188201652779104,-0.044641636506989,-0.0385403163522353,0.0218723549949558,-0.108893282759899,-0.115613065979398,0.0228686348215404,-0.076394503750001,-0.0468794828442166,0.0237749439885419,40.0 -0.0162806757273067,-0.044641636506989,0.0261284080806188,0.0585963091762383,-0.0607349327228599,-0.0442152166913845,-0.0139477432193303,-0.0339582147427055,-0.0514005352605825,-0.0259303389894746,52.0 --0.0709002470971626,0.0506801187398187,-0.0891974838246376,-0.0745280244296595,-0.0428475455662452,-0.0257394574458021,-0.0323559322397657,-0.00259226199818282,-0.0129079422541688,-0.0549250873933176,104.0 -0.0489735217864827,-0.044641636506989,0.0606183944448076,-0.0228849640236156,-0.0235842055514294,-0.072711726714232,-0.0434008456520269,-0.00259226199818282,0.104137611358979,0.036201264733046,132.0 -0.00538306037424807,0.0506801187398187,-0.0288400076873072,-0.00911348124867051,-0.0318399227006362,-0.0288709420636975,0.0081420836051921,-0.0394933828740919,-0.0181182673078967,0.00720651632920303,88.0 -0.0344433679824045,0.0506801187398187,-0.0299178197611881,0.00465800152627453,0.0933717873956666,0.0869939887984295,0.0339135482338016,-0.00259226199818282,0.024052583226893,-0.0383566597339788,69.0 -0.0235457526293458,0.0506801187398187,-0.019139699022379,0.0494153205448459,-0.0634868384392622,-0.0611252336280193,0.00446044580110504,-0.0394933828740919,-0.0259524244351894,-0.0135040182449705,219.0 -0.0199132141783263,-0.044641636506989,-0.0406959404999971,-0.015999222636143,-0.00844872411121698,-0.0175975974392743,0.052321737254237,-0.0394933828740919,-0.0307512098645563,0.00306440941436832,72.0 --0.0454724779400257,-0.044641636506989,0.0153502873418098,-0.0745280244296595,-0.0497273098572509,-0.0172844489774848,-0.0286742944356786,-0.00259226199818282,-0.104364820832166,-0.0756356219674911,201.0 -0.0526060602375023,0.0506801187398187,-0.0245287593917836,0.0563010619323185,-0.00707277125301585,-0.005071658967693,-0.0213110188275045,-0.00259226199818282,0.0267142576335128,-0.0383566597339788,110.0 --0.00551455497881059,0.0506801187398187,0.00133873038135806,-0.0848566365108683,-0.0112006298276192,-0.0166581520539057,0.0486400994501499,-0.0394933828740919,-0.0411803851880079,-0.0880619427119953,51.0 -0.00901559882526763,0.0506801187398187,0.0692408910358548,0.0597439326260547,0.0176943801946045,-0.0232342697514859,-0.0470824834561139,0.0343088588777263,0.103292264911524,0.0734802269665584,277.0 --0.0236772472339084,-0.044641636506989,-0.0697968664947814,-0.0641994123484507,-0.0593589798646588,-0.0504781859271752,0.0191869970174533,-0.0394933828740919,-0.0891368600793477,-0.0507829804784829,63.0 --0.0418399394890061,0.0506801187398187,-0.0299178197611881,-0.00222773986119799,0.0218222387692079,0.0365770864503148,0.0118237214092792,-0.00259226199818282,-0.0411803851880079,0.065196013136889,118.0 --0.0745327855481821,-0.044641636506989,-0.0460850008694016,-0.0435421881860331,-0.0290880169842339,-0.0232342697514859,0.0155053592133662,-0.0394933828740919,-0.0398095943643375,-0.0217882320746399,69.0 -0.0344433679824045,-0.044641636506989,0.0185837235634525,0.0563010619323185,0.0121905687618,-0.0545491159304391,-0.0691723102806364,0.0712099797536354,0.130080609521753,0.00720651632920303,273.0 --0.0600026317441039,-0.044641636506989,0.00133873038135806,-0.0297707054110881,-0.00707277125301585,-0.0216685274425382,0.0118237214092792,-0.00259226199818282,0.0318152175007986,-0.0549250873933176,258.0 --0.0854304009012408,0.0506801187398187,-0.030995631835069,-0.0228849640236156,-0.0634868384392622,-0.0542359674686496,0.0191869970174533,-0.0394933828740919,-0.096433222891784,-0.0342145528191441,43.0 -0.0526060602375023,-0.044641636506989,-0.00405032998804645,-0.0309183289641906,-0.0469754041408486,-0.0583068974719135,-0.0139477432193303,-0.0258399681500055,0.0360557900898319,0.0237749439885419,198.0 -0.0126481372762872,-0.044641636506989,0.0153502873418098,-0.0332135761048244,0.0410855787840237,0.0321930079852613,-0.0029028298070691,-0.00259226199818282,0.0450661683362615,-0.0673514081378217,242.0 -0.0598711371395414,0.0506801187398187,0.0228949718589761,0.0494153205448459,0.0163184273364034,0.0118383579689417,-0.0139477432193303,-0.00259226199818282,0.0395398780720242,0.0196328370737072,232.0 --0.0236772472339084,-0.044641636506989,0.045529025410475,0.090729768869681,-0.0180803941186249,-0.0354470597612776,0.0707299262746723,-0.0394933828740919,-0.0345237153303495,-0.0093619113301358,175.0 -0.0162806757273067,-0.044641636506989,-0.0450071887955207,-0.0573136709609782,-0.0345918284170385,-0.05392281900686,0.0744115640787594,-0.076394503750001,-0.0425721049227942,0.0403433716478807,93.0 -0.110726675453815,0.0506801187398187,-0.0331512559828308,-0.0228849640236156,-0.00432086553661359,0.0202933664372591,-0.0618090346724622,0.0712099797536354,0.0155668445407018,0.0444854785627154,168.0 --0.0200447087828888,-0.044641636506989,0.0972640049567582,-0.00567061055493425,-0.00569681839481472,-0.0238605666750649,-0.0213110188275045,-0.00259226199818282,0.0616858488238662,0.0403433716478807,275.0 --0.0164121703318693,-0.044641636506989,0.0541515220015222,0.0700725447072635,-0.0332158755588373,-0.0279314966783289,0.0081420836051921,-0.0394933828740919,-0.0271286455543265,-0.0093619113301358,293.0 -0.0489735217864827,0.0506801187398187,0.1231314947299,0.0838440274822086,-0.104765424185296,-0.10089508827529,-0.0691723102806364,-0.00259226199818282,0.0366457977933988,-0.0300724459043093,281.0 --0.0563700932930843,-0.044641636506989,-0.0805749872335904,-0.0848566365108683,-0.0373437341334407,-0.0370128020702253,0.0339135482338016,-0.0394933828740919,-0.0561575730950062,-0.137767225690012,72.0 -0.0271782910803654,-0.044641636506989,0.0929527566612346,-0.0527231767141394,0.00806271018719657,0.0397085710682101,-0.0286742944356786,0.021024455362399,-0.0483617248028919,0.0196328370737072,140.0 -0.063503675590561,-0.044641636506989,-0.0503962491649252,0.107944122338362,0.0314539087766158,0.0193539210518905,-0.0176293810234174,0.0236075338237126,0.0580391276638951,0.0403433716478807,189.0 --0.0527375548420648,0.0506801187398187,-0.0115950145052127,0.0563010619323185,0.0562210602242361,0.0729023080179005,-0.0397192078479398,0.0712099797536354,0.0305664873984148,-0.0052198044153011,181.0 --0.00914709342983014,0.0506801187398187,-0.0277621956134263,0.0081008722200108,0.0479653430750293,0.0372033833738938,-0.0286742944356786,0.0343088588777263,0.0660482061630984,-0.0424987666488135,209.0 -0.00538306037424807,-0.044641636506989,0.0584627702970458,-0.0435421881860331,-0.07311850844667,-0.0723985782524425,0.0191869970174533,-0.076394503750001,-0.0514005352605825,-0.0259303389894746,136.0 -0.0744012909436196,-0.044641636506989,0.0854080721440683,0.063186803319791,0.0149424744782022,0.0130909518160999,0.0155053592133662,-0.00259226199818282,0.0062093156165054,0.0859065477110625,261.0 --0.0527375548420648,-0.044641636506989,-0.000816893766403737,-0.0263278347173518,0.0108146159035988,0.00714113104209875,0.0486400994501499,-0.0394933828740919,-0.0358167281015492,0.0196328370737072,113.0 -0.0816663678456587,0.0506801187398187,0.00672779075076256,-0.00452298700183173,0.10988322169408,0.117056241130225,-0.0323559322397657,0.0918746074441444,0.0547240033481791,0.00720651632920303,131.0 --0.00551455497881059,-0.044641636506989,0.00888341489852436,-0.0504279295735057,0.0259500973438113,0.0472241341511589,-0.0434008456520269,0.0712099797536354,0.0148227108412663,0.00306440941436832,174.0 --0.0273097856849279,-0.044641636506989,0.0800190117746638,0.09876313370697,-0.00294491267841247,0.0181013272047324,-0.0176293810234174,0.00331191734196264,-0.0295276227417736,0.036201264733046,257.0 --0.0527375548420648,-0.044641636506989,0.0713965151836166,-0.0745280244296595,-0.0153284884022226,-0.00131387742621863,0.00446044580110504,-0.0214118336448964,-0.0468794828442166,0.00306440941436832,55.0 -0.00901559882526763,-0.044641636506989,-0.0245287593917836,-0.0263278347173518,0.0988755988284711,0.0941964034195887,0.0707299262746723,-0.00259226199818282,-0.02139368094036,0.00720651632920303,84.0 --0.0200447087828888,-0.044641636506989,-0.0547074974604488,-0.0538708002672419,-0.0662387441556644,-0.0573674520865449,0.0118237214092792,-0.0394933828740919,-0.0740888714915354,-0.0052198044153011,42.0 -0.0235457526293458,-0.044641636506989,-0.0363846922044735,6.75072794357462e-05,0.00118294589619092,0.0346981956795776,-0.0434008456520269,0.0343088588777263,-0.0332487872476258,0.0610539062220542,146.0 -0.0380759064334241,0.0506801187398187,0.0164280994156907,0.0218723549949558,0.0397096259258226,0.0450320949186321,-0.0434008456520269,0.0712099797536354,0.049768659920749,0.0154907301588724,212.0 --0.0781653239992017,0.0506801187398187,0.077863387626902,0.0528581912385822,0.0782363059554542,0.0644472995495832,0.0265502726256275,-0.00259226199818282,0.0406722637144977,-0.0093619113301358,233.0 -0.00901559882526763,0.0506801187398187,-0.0396181284261162,0.0287580963824284,0.0383336730676214,0.0735286049414796,-0.0728539480847234,0.108111100629544,0.0155668445407018,-0.0466408735636482,91.0 -0.00175052192322852,0.0506801187398187,0.0110390390462862,-0.0194420933298793,-0.0167044412604238,-0.00381906512053488,-0.0470824834561139,0.0343088588777263,0.024052583226893,0.0237749439885419,111.0 --0.0781653239992017,-0.044641636506989,-0.0406959404999971,-0.081413765817132,-0.100637565610693,-0.112794729823292,0.0228686348215404,-0.076394503750001,-0.0202887477516296,-0.0507829804784829,152.0 -0.030810829531385,0.0506801187398187,-0.0342290680567117,0.0436772026071898,0.0575970130824372,0.0688313780146366,-0.0323559322397657,0.057556565029549,0.0354619386607697,0.0859065477110625,120.0 --0.034574862586967,0.0506801187398187,0.00564997867688165,-0.00567061055493425,-0.07311850844667,-0.062690975936967,-0.00658446761115617,-0.0394933828740919,-0.045420957777041,0.0320591578182113,67.0 -0.0489735217864827,0.0506801187398187,0.088641508365711,0.0872868981759448,0.0355817673512192,0.0215459602844172,-0.0249926566315915,0.0343088588777263,0.0660482061630984,0.131469723774244,310.0 --0.0418399394890061,-0.044641636506989,-0.0331512559828308,-0.0228849640236156,0.0465893902168282,0.0415874618389473,0.056003375058324,-0.0247329345237283,-0.0259524244351894,-0.0383566597339788,94.0 --0.00914709342983014,-0.044641636506989,-0.0568631216082106,-0.0504279295735057,0.0218222387692079,0.0453452433804217,-0.0286742944356786,0.0343088588777263,-0.00991895736315477,-0.0176461251598052,183.0 -0.0707687524926,0.0506801187398187,-0.030995631835069,0.0218723549949558,-0.0373437341334407,-0.0470335528474903,0.0339135482338016,-0.0394933828740919,-0.0149564750249113,-0.00107769750046639,66.0 -0.00901559882526763,-0.044641636506989,0.0552293340754031,-0.00567061055493425,0.0575970130824372,0.0447189464568426,-0.0029028298070691,0.0232385226149535,0.0556835477026737,0.106617082285236,173.0 --0.0273097856849279,-0.044641636506989,-0.0600965578298533,-0.0297707054110881,0.0465893902168282,0.0199802179754696,0.122272855531891,-0.0394933828740919,-0.0514005352605825,-0.0093619113301358,72.0 -0.0162806757273067,-0.044641636506989,0.00133873038135806,0.0081008722200108,0.00531080447079431,0.0108989125835731,0.0302319104297145,-0.0394933828740919,-0.045420957777041,0.0320591578182113,49.0 --0.0127796318808497,-0.044641636506989,-0.0234509473179027,-0.0400993174922969,-0.0167044412604238,0.0046359433477825,-0.0176293810234174,-0.00259226199818282,-0.0384591123013538,-0.0383566597339788,64.0 --0.0563700932930843,-0.044641636506989,-0.074108114790305,-0.0504279295735057,-0.0249601584096305,-0.0470335528474903,0.0928197530991947,-0.076394503750001,-0.0611765950943345,-0.0466408735636482,48.0 -0.0417084448844436,0.0506801187398187,0.0196615356373334,0.0597439326260547,-0.00569681839481472,-0.00256647127337676,-0.0286742944356786,-0.00259226199818282,0.0311929907028023,0.00720651632920303,178.0 --0.00551455497881059,0.0506801187398187,-0.0159062628007364,-0.067642283042187,0.0493412959332305,0.0791652772536912,-0.0286742944356786,0.0343088588777263,-0.0181182673078967,0.0444854785627154,104.0 -0.0417084448844436,0.0506801187398187,-0.0159062628007364,0.0172818607481171,-0.0373437341334407,-0.0138398158977999,-0.0249926566315915,-0.0110795197996419,-0.0468794828442166,0.0154907301588724,132.0 --0.0454724779400257,-0.044641636506989,0.0390621529671896,0.00121513083253827,0.0163184273364034,0.0152829910486266,-0.0286742944356786,0.0265596234937854,0.0445283740214053,-0.0259303389894746,220.0 --0.0454724779400257,-0.044641636506989,-0.0730303027164241,-0.081413765817132,0.0837401173882587,0.0278089295202079,0.17381578478911,-0.0394933828740919,-0.00421985970694603,0.00306440941436832,57.0 diff --git a/data/diabetes_bad_dist.csv b/data/diabetes_bad_dist.csv deleted file mode 100644 index 2d7cf434..00000000 --- a/data/diabetes_bad_dist.csv +++ /dev/null @@ -1,3 +0,0 @@ -AGE,SEX,BMI,BP,S1,S2,S3,S4,S5,S6,Y -100,0.0506801187398187,0.0616962065186885,0.0218723549949558,-0.0442234984244464,-0.0348207628376986,-0.0434008456520269,-0.00259226199818282,0.0199084208763183,-0.0176461251598052,151.0 --0.0018820165277910,-0.044641636506989,-0.0514740612388061,-0.0263278347173518,-0.00844872411121698,-0.019163339748222,0.0744115640787594,-0.0394933828740919,-0.0683297436244215,-0.09220404962683,75.0 diff --git a/data/diabetes_bad_schema.csv b/data/diabetes_bad_schema.csv deleted file mode 100644 index b21fca1d..00000000 --- a/data/diabetes_bad_schema.csv +++ /dev/null @@ -1,3 +0,0 @@ -SEX,BMI,BP,S1,S2,S3,S4,S5,S6,Y -0.0506801187398187,0.0616962065186885,0.0218723549949558,-0.0442234984244464,-0.0348207628376986,-0.0434008456520269,-0.00259226199818282,0.0199084208763183,-0.0176461251598052,151.0 --0.04464163650698,-0.0514740612388061,-0.0263278347173518,-0.00844872411121698,-0.019163339748222,0.0744115640787594,-0.0394933828740919,-0.0683297436244215,-0.09220404962683,75.0 diff --git a/data/diabetes_missing_values.csv b/data/diabetes_missing_values.csv deleted file mode 100644 index 44d8a056..00000000 --- a/data/diabetes_missing_values.csv +++ /dev/null @@ -1,3 +0,0 @@ -AGE,SEX,BMI,BP,S1,S2,S3,S4,S5,S6,Y -,,,0.0218723549949558,-0.0442234984244464,-0.0348207628376986,-0.0434008456520269,-0.00259226199818282,0.0199084208763183,-0.0176461251598052,151.0 --0.001882016527791,-0.044641636506989,-0.0514740612388061,-0.0263278347173518,-0.00844872411121698,-0.019163339748222,0.0744115640787594,-0.0394933828740919,-0.0683297436244215,-0.09220404962683,75.0 diff --git a/docs/canary_ab_deployment.md b/docs/canary_ab_deployment.md deleted file mode 100644 index 49edb503..00000000 --- a/docs/canary_ab_deployment.md +++ /dev/null @@ -1,124 +0,0 @@ -# Model deployment to AKS cluster with Canary deployment - -[![Build Status](https://aidemos.visualstudio.com/MLOps/_apis/build/status/microsoft.MLOpsPython-Canary?branchName=master)](https://aidemos.visualstudio.com/MLOps/_build/latest?definitionId=133&branchName=master) - -If your target deployment environment is a Kubernetes cluster and you want to implement [Canary and/or A/B testing deployment strategies](http://adfpractice-fedor.blogspot.com/2019/04/deployment-strategies-with-kubernetes.html) you can follow this sample guide. - -- [Prerequisites](#prerequisites) -- [Install Istio on a K8s cluster](#install-istio-on-a-k8s-cluster) -- [Set up variables](#set-up-variables) -- [Configure a pipeline to build and deploy a scoring Image](#configure-a-pipeline-to-build-and-deploy-a-scoring-image) -- [Build a new Scoring Image](#build-a-new-scoring-image) - -## Prerequisites - -Before continuing with this guide, you will need: - -* An [Azure Kubernetes Service (AKS)](https://azure.microsoft.com/en-us/services/kubernetes-service) cluster - * This does **not** have to be the same cluster as the example in [Getting Started: Deploy the model to Azure Kubernetes Service](/docs/getting_started.md#deploy-the-model-to-azure-kubernetes-service) - * The cluster does not have to be connected to Azure Machine Learning. - * If you want to deploy a new cluster, see [Quickstart: Deploy an Azure Kubernetes Service cluster using the Azure CLI](https://docs.microsoft.com/en-us/azure/aks/kubernetes-walkthrough) -* An Azure Container Registry instance that is authenticated with your Azure Kubernetes Service cluster. - * The chart you will deploy is assuming you are authenticated using a service principal. - * See [Authenticate with Azure Container Registry from Azure Kubernetes Service](https://docs.microsoft.com/en-us/azure/aks/cluster-container-registry-integration#configure-acr-integration-for-existing-aks-clusters) for an authentication guide. -* In Azure DevOps, a service connection to your Kubernetes cluster. - * If you do not currently have a namespace, create one named 'abtesting'. - -## Install Istio on a K8s cluster - -You'll be using the [Istio](https://istio.io) service mesh implementation to control traffic routing between model versions. Follow the instructions at [Install and use Istio in Azure Kubernetes Service (AKS)](https://docs.microsoft.com/azure/aks/servicemesh-istio-install?pivots=client-operating-system-linux). - -After Istio is installed, figure out the Istio gateway endpoint on your K8s cluster: - -```bash -GATEWAY_IP=$(kubectl get svc istio-ingressgateway -n istio-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}') -``` - -You don't need to create any Istio resources (e.g. Gateway or VirtualService) at this point. It will be handled by the AzDo pipeline that builds and deploys a scoring image. - -## Set up variables - -There are some extra variables that you need to setup in ***devopsforai-aml-vg*** variable group (see [getting started](./getting_started.md)): - -| Variable Name | Suggested Value | Short Description | -|---------------------------|-----------------------|-----------------------------------------------------------| -| K8S_AB_SERVICE_CONNECTION | mlops-aks | Name of the service connection to your Kubernetes cluster | -| K8S_AB_NAMESPACE | abtesting | Kubernetes namespace for model deployment | -| IMAGE_REPO_NAME | [Your ACR's DNS name] | Image reposiory name (e.g. mlopspyciamlcr.azurecr.io) | - -## Configure a pipeline to build and deploy a scoring Image - -Import and run the [abtest.yml](./.pipelines/abtest.yml) multistage deployment pipeline. - -After the pipeline completes successfully, you will see a registered Docker image in the ACR repository attached to the Azure ML Service: - -![scoring image](./images/scoring_image.png) - -The pipeline creates Istio Gateway and VirtualService and deploys the scoring image to the Kubernetes cluster. - -```bash -kubectl get deployments --namespace abtesting -NAME READY UP-TO-DATE AVAILABLE AGE -model-green 1/1 1 1 19h -``` - -## Build a new Scoring Image - -Change value of the ***SCORE_SCRIPT*** variable in the [abtest.yml](./.pipelines/abtest.yml) to point to ***scoring/scoreA.py*** and merge it to the master branch. - -**Note:** ***scoreA.py*** and ***scoreB.py*** files used in this tutorial are just mockups returning either "New Model A" or "New Model B" respectively. They are used to demonstrate the concept of testing two scoring images with different models or scoring code. In real life you would implement a scoring file similar to [score.py](./../code/scoring/score.py) (see the [Getting Started](./getting_started.md) guide). - -It will automatically trigger the pipeline and deploy a new scoring image with the following stages implementing ***Canary*** deployment strategy: - -| Stage | Green Weight | Blue Weight | Description | -|------------|--------------|-------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| -| Blue_0 | 100 | 0 | New image (blue) is deployed.
But all traffic (100%) is still routed to the old (green) image. | -| Blue_50 | 50 | 50 | Traffic is split between old (green) and new (blue) images 50/50. | -| Blue_100 | 0 | 100 | All traffic (100%) is routed to the blue image. | -| Blue_Green | 0 | 100 | Old green image is removed. The new blue image is copied as green.
Blue and Green images are equal.
All traffic (100%) is routed to the blue image. | -| Green_100 | 100 | 0 | All traffic (100%) is routed to the green image.
The blue image is removed. | - -**Note:** The pipeline performs the rollout without any pausing. You may want to configure [Approvals and Checks](https://docs.microsoft.com/en-us/azure/devops/pipelines/process/approvals?view=azure-devops&tabs=check-pass) for the stages on your environment for better experience of the model testing. The environment ***abtestenv*** will be added automatically to your AzDo project after the first pipeline run. - -At each stage you can verify how the traffic is routed sending requests to $GATEWAY_IP/score with ***Postman*** or with ***curl***: - -```bash -curl $GATEWAY_IP/score -``` - -You can also emulate a simple load test on the gateway with the ***load_test.sh***: - -```bash -./charts/load_test.sh 10 $GATEWAY_IP/score -``` - -The command above sends 10 requests to the gateway. So if the pipeline has completed stage Blue_50, the result will look like this: - -```bash -"New Model A" -"New Model A" -"New Model A" -"New Model B" -"New Model A" -"New Model B" -"New Model B" -"New Model A" -"New Model A" -"New Model A" -``` - -Regardless of the blue/green weight values set on the cluster, you can perform ***A/B testing*** and send requests directly to either blue or green images: - -```bash -curl --header "x-api-version: blue" $GATEWAY_IP/score -curl --header "x-api-version: green" $GATEWAY_IP/score -``` - -or with a load_test.sh script: - -```bash -./charts/load_test.sh 10 $GATEWAY_IP/score blue -./charts/load_test.sh 10 $GATEWAY_IP/score green -``` - -In this case the Istio Virtual Service analyzes the request header and routes the traffic directly to the specified model version. diff --git a/docs/code_description.md b/docs/code_description.md deleted file mode 100644 index 81abc78f..00000000 --- a/docs/code_description.md +++ /dev/null @@ -1,97 +0,0 @@ -## Repo Details - -### Directory Structure - -High level directory structure for this repository: - -```bash -├── .pipelines <- Azure DevOps YAML pipelines for CI, PR and model training and deployment. -├── bootstrap <- Python script to initialize this repository with a custom project name. -├── charts <- Helm charts to deploy resources on Azure Kubernetes Service(AKS). -├── data <- Initial set of data to train and evaluate model. Not for use to store data. -├── diabetes_regression <- The top-level folder for the ML project. -│ ├── evaluate <- Python script to evaluate trained ML model. -│ ├── register <- Python script to register trained ML model with Azure Machine Learning Service. -│ ├── scoring <- Python score.py to deploy trained ML model. -│ ├── training <- Python script to train ML model. -│ ├── R <- R script to train R based ML model. -│ ├── util <- Python script for various utility operations specific to this ML project. -├── docs <- Extensive markdown documentation for entire project. -├── environment_setup <- The top-level folder for everything related to infrastructure. -│ ├── arm-templates <- Azure Resource Manager(ARM) templates to build infrastructure needed for this project. -│ ├── tf-templates <- Terraform templates to build infrastructure needed for this project. -├── experimentation <- Jupyter notebooks with ML experimentation code. -├── ml_service <- The top-level folder for all Azure Machine Learning resources. -│ ├── pipelines <- Python script that builds Azure Machine Learning pipelines. -│ ├── util <- Python script for various utility operations specific to Azure Machine Learning. -├── .env.example <- Example .env file with environment for local development experience. -├── .gitignore <- A gitignore file specifies intentionally un-tracked files that Git should ignore. -├── LICENSE <- License document for this project. -├── README.md <- The top-level README for developers using this project. -``` - -The repository provides a template with folders structure suitable for maintaining multiple ML projects. There are common folders such as ***.pipelines***, ***environment_setup***, ***ml_service*** and folders containing the code base for each ML project. This repository contains a single sample ML project in the ***diabetes_regression*** folder. This folder is going to be automatically renamed to your project name if you follow the [bootstrap procedure](../bootstrap/README.md). - -### Environment Setup - -- `environment_setup/install_requirements.sh` : This script prepares a local conda environment i.e. install the Azure ML SDK and the packages specified in environment definitions. - -- `environment_setup/iac-*-arm.yml, arm-templates` : Infrastructure as Code piplines to create required resources using ARM, along with corresponding arm-templates. Infrastructure as Code can be deployed with this template or with the Terraform template. - -- `environment_setup/iac-*-tf.yml, tf-templates` : Infrastructure as Code piplines to create required resources using Terraform, along with corresponding tf-templates. Infrastructure as Code can be deployed with this template or with the ARM template. - -- `environment_setup/iac-remove-environment.yml` : Infrastructure as Code piplines to delete the created required resources. - -- `environment_setup/Dockerfile` : Dockerfile of a build agent containing Python 3.6 and all required packages. - -- `environment_setup/docker-image-pipeline.yml` : An AzDo pipeline for building and pushing [microsoft/mlopspython](https://hub.docker.com/_/microsoft-mlops-python) image. - -### Pipelines - -- `.pipelines/abtest.yml` : a pipeline demonstrating [Canary deployment strategy](./docs/canary_ab_deployment.md). -- `.pipelines/code-quality-template.yml` : a pipeline template used by the CI and PR pipelines. It contains steps performing linting, data and unit testing. -- `.pipelines/diabetes_regression-ci-image.yml` : a pipeline building a scoring image for the diabetes regression model. -- `.pipelines/diabetes_regression-ci.yml` : a pipeline triggered when the code is merged into **master**. It performs linting, data integrity testing, unit testing, building and publishing an ML pipeline. -- `.pipelines/diabetes_regression-cd.yml` : a pipeline triggered when the code is merged into **master** and the `.pipelines/diabetes_regression-ci.yml` completes. Deploys the model to ACI, AKS or Webapp. -- `.pipelines/diabetes_regression-package-model-template.yml` : Pipeline template that creates a model package and adds the package location to the environment for subsequent tasks to use. -- `.pipelines/diabetes_regression-get-model-id-artifact-template.yml` : a pipeline template used by the `.pipelines/diabetes_regression-cd.yml` pipeline. It takes the model metadata artifact published by the previous pipeline and gets the model ID. -- `.pipelines/diabetes_regression-publish-model-artifact-template.yml` : a pipeline template used by the `.pipelines/diabetes_regression-ci.yml` pipeline. It finds out if a new model was registered and publishes a pipeline artifact containing the model metadata. -- `.pipelines/helm-*.yml` : pipeline templates used by the `.pipelines/abtest.yml` pipeline. -- `.pipelines/pr.yml` : a pipeline triggered when a **pull request** to the **master** branch is created. It performs linting, data integrity testing and unit testing only. - -### ML Services - -- `ml_service/pipelines/diabetes_regression_build_train_pipeline.py` : builds and publishes an ML training pipeline. It uses Python on ML Compute. -- `ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py` : builds and publishes an ML training pipeline. It uses R on ML Compute. -- `ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py` : builds and publishes an ML training pipeline. It uses R on Databricks Compute. -- `ml_service/pipelines/run_train_pipeline.py` : invokes a published ML training pipeline (Python on ML Compute) via REST API. -- `ml_service/util` : contains common utility functions used to build and publish an ML training pipeline. - -### Environment Definitions - -- `diabetes_regression/conda_dependencies.yml` : Conda environment definition for the environment used for both training and scoring (Docker image in which train.py and score.py are run). -- `diabetes_regression/ci_dependencies.yml` : Conda environment definition for the CI environment. - -### Training Step - -- `diabetes_regression/training/train_aml.py`: a training step of an ML training pipeline. -- `diabetes_regression/training/train.py` : ML functionality called by train_aml.py -- `diabetes_regression/training/R/r_train.r` : training a model with R basing on a sample dataset (weight_data.csv). -- `diabetes_regression/training/R/train_with_r.py` : a python wrapper (ML Pipeline Step) invoking R training script on ML Compute -- `diabetes_regression/training/R/train_with_r_on_databricks.py` : a python wrapper (ML Pipeline Step) invoking R training script on Databricks Compute -- `diabetes_regression/training/R/weight_data.csv` : a sample dataset used by R script (r_train.r) to train a model -- `diabetes_regression/training/R/test_train.py` : a unit test for the training script(s) - -### Evaluation Step - -- `diabetes_regression/evaluate/evaluate_model.py` : an evaluating step which cancels the pipeline in case of non-improvement. - -### Registering Step - -- `diabetes_regression/register/register_model.py` : registers a new trained model if evaluation shows the new model is more performant than the previous one. - -### Scoring - -- `diabetes_regression/scoring/score.py` : a scoring script which is about to be packed into a Docker Image along with a model while being deployed to QA/Prod environment. -- `diabetes_regression/scoring/inference_config.yml`, `deployment_config_aci.yml`, `deployment_config_aks.yml` : configuration files for the [AML Model Deploy](https://marketplace.visualstudio.com/items?itemName=ms-air-aiagility.private-vss-services-azureml&ssr=false#overview) pipeline task for ACI and AKS deployment targets. -- `diabetes_regression/scoring/scoreA.py`, `diabetes_regression/scoring/scoreB.py` : simplified scoring files for the [Canary deployment sample](./docs/canary_ab_deployment.md). diff --git a/docs/custom_container.md b/docs/custom_container.md deleted file mode 100644 index 46e692f9..00000000 --- a/docs/custom_container.md +++ /dev/null @@ -1,113 +0,0 @@ -# Customizing the Azure DevOps job container - -The Model training and deployment pipeline uses a Docker container -on the Azure Pipelines agents to provide a reproducible environment -to run test and deployment code. - The image of the container -`mcr.microsoft.com/mlops/python:latest` is built with this -[Dockerfile](../environment_setup/Dockerfile). - -Additionally mcr.microsoft.com/mlops/python image is also tagged with below tags. - -| Image Tags | Description | -| ----------------------------------------------- | :---------------------------------------------------------------------------------------- | -| mcr.microsoft.com/mlops/python:latest | latest image | -| mcr.microsoft.com/mlops/python:build-[id] | where [id] is Azure Devops build id e.g. mcr.microsoft.com/mlops/python:build-20200325.1 | -| mcr.microsoft.com/mlops/python:amlsdk-[version] | where [version] is aml sdk version e.g. mcr.microsoft.com/mlops/python:amlsdk-1.1.5.1 | -| mcr.microsoft.com/mlops/python:release-[id] | where [id] is github release id e.g. mcr.microsoft.com/mlops/python:release-3.0.0 | | - -In your project you will want to build your own -Docker image that only contains the dependencies and tools required for your -use case. This image will be more likely smaller and therefore faster, and it -will be totally maintained by your team. - -## Provision an Azure Container Registry - -An Azure Container Registry is deployed along your Azure ML Workspace to manage models. -You can use that registry instance to store your MLOps container image as well, or -provision a separate instance. - -## Create a Registry Service Connection - -[Create a service connection](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#sep-docreg) to your Azure Container Registry: - -- As *Connection type*, select *Docker Registry* -- As *Registry type*, select *Azure Container Registry* -- As *Azure container registry*, select your Container registry instance -- As *Service connection name*, enter `acrconnection` - -## Update the environment definition - -Modify the [Dockerfile](../environment_setup/Dockerfile) and/or the -[ci_dependencies.yml](../diabetes_regression/ci_dependencies.yml) CI Conda -environment definition to tailor your environment. -Conda provides a [reusable environment for training and deployment with Azure Machine Learning](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-use-environments). -The Conda environment used for CI should use the same package versions as the Conda environment -used for the Azure ML training and scoring environments (defined in [conda_dependencies.yml](../diabetes_regression/conda_dependencies.yml)). -This enables you to run unit and integration tests using the exact same dependencies as used in the ML pipeline. - -If a package is available in a Conda package repository, then we recommend that -you use the Conda installation rather than the pip installation. Conda packages -typically come with prebuilt binaries that make installation more reliable. - -## Create a container build pipeline - -In your [Azure DevOps](https://dev.azure.com) project create a new build -pipeline referring to the -[environment_setup/docker-image-pipeline.yml](../environment_setup/docker-image-pipeline.yml) -pipeline definition in your forked repository. - -Edit the [environment_setup/docker-image-pipeline.yml](../environment_setup/docker-image-pipeline.yml) file -and modify the string `'public/mlops/python'` with an name suitable to describe your environment, -e.g. `'mlops/diabetes_regression'`. - -Save and run the pipeline, making sure to set the these runtime variables: `amlsdkversion` and `githubrelease`. The values are up to you to set depending on your environment. These will show as tags on your image. - -![Custom Container Vars](./images/custom-container-variables.png) - -This will build and push a container image to your Azure Container Registry with -the name you have just edited. The next step is to modify the build pipeline to run the CI job on a container -run from that image. - -## Modify the model pipeline - -Modify the model pipeline file [diabetes_regression-ci.yml](../.pipelines/diabetes_regression-ci.yml) by replacing this section: - -``` -resources: - containers: - - container: mlops - image: mcr.microsoft.com/mlops/python:latest -``` - -with (using the image name previously defined): - -``` -resources: - containers: - - container: mlops - image: mlops/diabetes_regression - endpoint: acrconnection -``` - -Run the pipeline and ensure your container has been used. - -## Addressing conflicting dependencies - -Especially when working in a team, it's possible for environment changes across branches to interfere with one another. - -For example, if the master branch is using scikit-learn and you create a branch to use Tensorflow instead, and you -decide to remove scikit-learn from the -[ci_dependencies.yml](../diabetes_regression/ci_dependencies.yml) Conda environment definition -and run the [docker-image-pipeline.yml](../environment_setup/docker-image-pipeline.yml) Docker image, -then the master branch will stop building. - -You could leave scikit-learn in addition to Tensorflow in the environment, but that is not ideal, as you would have to take an extra step to remove scikit-learn after merging your branch to master. - -A better approach would be to use a distinct name for your modified environment, such as `mlops/diabetes_regression/tensorflow`. -By changing the name of the image in your branch in both the container build pipeline -[environment_setup/docker-image-pipeline.yml](../environment_setup/docker-image-pipeline.yml) -and the model pipeline file -[diabetes_regression-ci.yml](../.pipelines/diabetes_regression-ci.yml), -and running both pipelines in sequence on your branch, -you avoid any branch conflicts, and the name does not have to be changed after merging to master. diff --git a/docs/custom_model.md b/docs/custom_model.md deleted file mode 100644 index 28a15d78..00000000 --- a/docs/custom_model.md +++ /dev/null @@ -1,124 +0,0 @@ -# Bring your own code with the MLOpsPython repository template - -This document provides steps to follow when using this repository as a template to train models and deploy the models with real-time inference in Azure ML with your own scripts and data. - -1. Follow the MLOpsPython [Getting Started](getting_started.md) guide -1. Bootstrap the project -1. Configure training data -1. [If necessary] Convert your ML experimental code into production ready code -1. Replace the training code -1. [Optional] Update the evaluation code -1. Customize the build agent environment -1. [If appropriate] Replace the score code -1. [If appropriate] Configure batch scoring data - -## Follow the Getting Started guide - -Follow the [Getting Started](getting_started.md) guide to set up the infrastructure and pipelines to execute MLOpsPython. - -Take a look at the [Repo Details](code_description.md) document for a description of the structure of this repository. - -## Bootstrap the project - -Bootstrapping will prepare the directory structure to be used for your project name which includes: - -* renaming files and folders from the base project name `diabetes_regression` to your project name -* fixing imports and absolute path based on your project name -* deleting and cleaning up some directories - -**Note:** Since the bootstrap script will rename the `diabetes_regression` folder to the project name of your choice, we'll refer to your project as `[project name]` when paths are involved. - -To bootstrap from the existing MLOpsPython repository: - -1. Ensure Python 3 is installed locally -1. From a local copy of the code, run the `bootstrap.py` script in the `bootstrap` folder -`python bootstrap.py -d [dirpath] -n [projectname]` - * `[dirpath]` is the absolute path to the root of the directory where MLOpsPython is cloned - * `[projectname]` is the name of your ML project - -# Configure Custom Training - -## Configure training data - -The training ML pipeline uses a [sample diabetes dataset](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_diabetes.html) as training data. - -**Important** Convert the template to use your own Azure ML Dataset for model training via these steps: - -1. [Create a Dataset](https://docs.microsoft.com/azure/machine-learning/how-to-create-register-datasets) in your Azure ML workspace -1. Update the `DATASET_NAME` and `DATASTORE_NAME` variables in `.pipelines/[project name]-variables-template.yml` - -## Convert your ML experimental code into production ready code - -The MLOpsPython template creates an Azure Machine Learning (ML) pipeline that invokes a set of [Azure ML pipeline steps](https://docs.microsoft.com/python/api/azureml-pipeline-steps/azureml.pipeline.steps) (see `ml_service/pipelines/[project name]_build_train_pipeline.py`). If your experiment is currently in a Jupyter notebook, it will need to be refactored into scripts that can be run independently and dropped into the template which the existing Azure ML pipeline steps utilize. - -1. Refactor your experiment code into scripts -1. [Recommended] Prepare unit tests - -Examples of all these scripts are provided in this repository. -See the [Convert ML experimental code to production code tutorial](https://docs.microsoft.com/azure/machine-learning/tutorial-convert-ml-experiment-to-production) for a step by step guide and additional details. - -## Replace training code - -The template contains three scripts in the `[project name]/training` folder. Update these scripts for your experiment code. - -* `train.py` contains the platform-agnostic logic required to do basic data preparation and train the model. This script can be invoked against a static data file for local development. -* `train_aml.py` is the entry script for the ML pipeline step. It invokes the functions in `train.py` in an Azure ML context and adds logging. `train_aml.py` loads parameters for training from `[project name]/parameters.json` and passes them to the training function in `train.py`. If your experiment code can be refactored to match the function signatures in `train.py`, this file shouldn't need many changes. -* `test_train.py` contains tests that guard against functional regressions in `train.py`. Remove this file if you have no tests for your own code. - -Add any dependencies required by training to `[project name]/conda_dependencies.yml]`. This file will be used to generate the environment that the pipeline steps will run in. - -## Update evaluation code - -The MLOpsPython template uses the evaluate_model script to compare the performance of the newly trained model and the current production model based on Mean Squared Error. If the performance of the newly trained model is better than the current production model, then the pipelines continue. Otherwise, the pipelines are canceled. - -To keep the evaluation step, replace all instances of `mse` in `[project name]/evaluate/evaluate_model.py` with the metric that you want. - -To disable the evaluation step, either: - -* set the DevOps pipeline variable `RUN_EVALUATION` to `false` -* uncomment `RUN_EVALUATION` in `.pipelines/[project name]-variables-template.yml` and set the value to `false` - -## Customize the build agent environment - -The DevOps pipeline definitions in the MLOpsPython template run several steps in a Docker container that contains the dependencies required to work through the Getting Started guide. These dependencies may change over time and may not suit your project's needs. To manage your own dependencies, there are a few options: - -* Add a pipeline step to install dependencies required by unit tests to `.pipelines/code-quality-template.yml`. Recommended if you only have a small number of test dependencies. -* Create a new Docker image containing your dependencies. See [docs/custom_container.md](custom_container.md). Recommended if you have a larger number of dependencies, or if the overhead of installing additional dependencies on each run is too high. -* Remove the container references from the pipeline definition files and run the pipelines on self hosted agents with dependencies pre-installed. - -# Configure Custom Scoring - -## Replace score code - -For the model to provide real-time inference capabilities, the score code needs to be replaced. The MLOpsPython template uses the score code to deploy the model to do real-time scoring on ACI, AKS, or Web apps. - -If you want to keep scoring: - -1. Update or replace `[project name]/scoring/score.py` -1. Add any dependencies required by scoring to `[project name]/conda_dependencies.yml` -1. Modify the test cases in the `ml_service/util/smoke_test_scoring_service.py` script to match the schema of the training features in your data -1. Check and modify [project name]/scoring/deployment_config_aks.yml if AKS deployment is planned. The deployment configuration shall suit custom model as well as AKS cluster size. - -# Configure Custom Batch Scoring - -## Configure input and output data - -The batch scoring pipeline is configured to use the default datastore for input and output. It will use sample data for scoring. - -In order to configure your own input datastore and output datastores, you will need to specify an Azure Blob Storage Account and set up input and output containers. - -Configure the variables below in your variable group. - -**Note: The datastore storage resource, input/output containers, and scoring data is not created automatically. Make sure that you have manually provisioned these resources and placed your scoring data in your input container with the proper name.** - - -| Variable Name | Suggested Value | Short description | -| ------------------------ | ------------------------- | --------------------------------------------------------------------------------------------------------------------------- | -| SCORING_DATASTORE_STORAGE_NAME | | [Azure Blob Storage Account](https://docs.microsoft.com/en-us/azure/storage/blobs/) name. | -| SCORING_DATASTORE_ACCESS_KEY | | [Azure Storage Account Key](https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-requests-to-azure-storage). You may want to consider linking this variable to Azure KeyVault to avoid storing the access key in plain text. | -| SCORING_DATASTORE_INPUT_CONTAINER | | The name of the container for input data. Defaults to `input` if not set. | -| SCORING_DATASTORE_OUTPUT_CONTAINER| | The name of the container for output data. Defaults to `output` if not set. | -| SCORING_DATASTORE_INPUT_FILENAME | | The filename of the input data in your container Defaults to `diabetes_scoring_input.csv` if not set. | -| SCORING_DATASET_NAME | | The AzureML Dataset name to use. Defaults to `diabetes_scoring_ds` if not set (optional). | -| SCORING_DATASTORE_OUTPUT_FILENAME | | The filename to use for the output data. The pipeline will create this file. Defaults to `diabetes_scoring_output.csv` if not set (optional). | - diff --git a/docs/development_setup.md b/docs/development_setup.md deleted file mode 100644 index 1c8c2479..00000000 --- a/docs/development_setup.md +++ /dev/null @@ -1,33 +0,0 @@ -## Development environment setup - -### Setup - -Please be aware that the local environment also needs access to the Azure subscription so you have to have Contributor access on the Azure ML Workspace. - -In order to configure the project locally, create a copy of `.env.example` in the root directory and name it `.env`. Fill out all missing values and adjust the existing ones to suit your requirements. - -### Installation - -[Install the Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli). The Azure CLI will be used to log you in interactively. - -Install [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html). - -Install the required Python modules. [`install_requirements.sh`](https://github.com/microsoft/MLOpsPython/blob/master/environment_setup/install_requirements.sh) creates and activates a new conda environment with required Python modules. - -``` -. environment_setup/install_requirements.sh -``` - -### Running local code - -To run your local ML pipeline code on Azure ML, run a command such as the following (in bash, all on one line): - -``` -export BUILD_BUILDID=$(uuidgen); python ml_service/pipelines/diabetes_regression_build_train_pipeline.py && python ml_service/pipelines/run_train_pipeline.py -``` - -BUILD_BUILDID is a variable used to uniquely identify the ML pipeline between the -`diabetes_regression_build_train_pipeline.py` and `run_train_pipeline.py` scripts. In Azure DevOps it is -set to the current build number. In a local environment, we can use a command such as -`uuidgen` so set a different random identifier on each run, ensuring there are -no collisions. diff --git a/docs/getting_started.md b/docs/getting_started.md deleted file mode 100644 index 4ba694d7..00000000 --- a/docs/getting_started.md +++ /dev/null @@ -1,464 +0,0 @@ -# Getting Started with MLOpsPython - -This guide shows how to get MLOpsPython working with a sample ML project **_diabetes_regression_**. The project creates a linear regression model to predict diabetes and has CI/CD DevOps practices enabled for model training and serving when these steps are completed in this getting started guide. - -If you would like to bring your own model code to use this template structure, follow the [custom model](custom_model.md) guide. We recommend completing this getting started guide with the diabetes model through ACI deployment first to ensure everything is working in your environment before converting the template to use your own model code. - -- [Setting up Azure DevOps](#setting-up-azure-devops) - - [Install the Azure Machine Learning extension](#install-the-azure-machine-learning-extension) -- [Get the code](#get-the-code) -- [Create a Variable Group for your Pipeline](#create-a-variable-group-for-your-pipeline) - - [Variable Descriptions](#variable-descriptions) -- [Provisioning resources using Azure Pipelines](#provisioning-resources-using-azure-pipelines) - - [Create an Azure DevOps Service Connection for the Azure Resource Manager](#create-an-azure-devops-service-connection-for-the-azure-resource-manager) - - [Create the IaC Pipeline](#create-the-iac-pipeline) -- [Create an Azure DevOps Service Connection for the Azure ML Workspace](#create-an-azure-devops-service-connection-for-the-azure-ml-workspace) -- [Set up Build, Release Trigger, and Release Multi-Stage Pipeline](#set-up-build-release-trigger-and-release-multi-stage-pipelines) - - [Set up the Model CI Training, Evaluation, and Registration Pipeline](#set-up-the-model-ci-training-evaluation-and-registration-pipeline) - - [Set up the Release Deployment and/or Batch Scoring Pipelines](#set-up-the-release-deployment-andor-batch-scoring-pipelines) -- [Further Exploration](#further-exploration) - - [Deploy the model to Azure Kubernetes Service](#deploy-the-model-to-azure-kubernetes-service) - - [Web Service Authentication on Azure Kubernetes Service](#web-service-authentication-on-azure-kubernetes-service) - - [Deploy the model to Azure App Service (Azure Web App for containers)](#deploy-the-model-to-azure-app-service-azure-web-app-for-containers) - - [Example pipelines using R](#example-pipelines-using-r) - - [Observability and Monitoring](#observability-and-monitoring) - - [Clean up the example resources](#clean-up-the-example-resources) -- [Next Steps: Integrating your project](#next-steps-integrating-your-project) - - [Additional Variables and Configuration](#additional-variables-and-configuration) - - [More variable options](#more-variable-options) - - [Local configuration](#local-configuration) - -## Setting up Azure DevOps - -You'll use Azure DevOps for running the multi-stage pipeline with build, model training, and scoring service release stages. If you don't already have an Azure DevOps organization, create one by following the instructions at [Quickstart: Create an organization or project collection](https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/create-organization?view=azure-devops). - -If you already have an Azure DevOps organization, create a new project using the guide at [Create a project in Azure DevOps and TFS](https://docs.microsoft.com/en-us/azure/devops/organizations/projects/create-project?view=azure-devops). - -### Install the Azure Machine Learning extension - -Install the **Azure Machine Learning** extension to your Azure DevOps organization from the [Visual Studio Marketplace](https://marketplace.visualstudio.com/items?itemName=ms-air-aiagility.vss-services-azureml) by clicking "Get it free" and following the steps. The UI will tell you if try to add it and it's already installed. - -This extension contains the Azure ML pipeline tasks and adds the ability to create Azure ML Workspace service connections. The documentation page on the marketplace includes detailed instructions with screenshots on what capabilities it includes. - -## Get the code - -We recommend using the [repository template](https://github.com/microsoft/MLOpsPython/generate), which effectively forks this repository to your own GitHub location and squashes the history. You can use the resulting repository for this guide and for your own experimentation. - -## Create a Variable Group for your Pipeline - -MLOpsPython requires some variables to be set before you can run any pipelines. You'll need to create a _variable group_ in Azure DevOps to store values that are reused across multiple pipelines or pipeline stages. Either store the values directly in [Azure DevOps](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/variable-groups?view=azure-devops&tabs=designer#create-a-variable-group) or connect to an Azure Key Vault in your subscription. Check out the [Add & use variable groups](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/variable-groups?view=azure-devops&tabs=yaml#use-a-variable-group) documentation to learn more about how to create a variable group and link it to your pipeline. - -Navigate to **Library** in the **Pipelines** section as indicated below: - -![Library Variable Groups](./images/library_variable_groups.png) - -Create a variable group named **`devopsforai-aml-vg`**. The YAML pipeline definitions in this repository refer to this variable group by name. - -The variable group should contain the following required variables. **Azure resources that don't exist yet will be created in the [Provisioning resources using Azure Pipelines](#provisioning-resources-using-azure-pipelines) step below.** - -| Variable Name | Suggested Value | Short description | -| ------------------------ | ------------------------- | --------------------------------------------------------------------------------------------------------------------------- | -| BASE_NAME | [your project name] | Unique naming prefix for created resources - max 10 chars, letters and numbers only | -| LOCATION | centralus | [Azure location](https://azure.microsoft.com/en-us/global-infrastructure/locations/), no spaces. You can list all the region codes by running `az account list-locations -o table` in the Azure CLI | -| RESOURCE_GROUP | mlops-RG | Azure Resource Group name | -| WORKSPACE_NAME | mlops-AML-WS | Azure ML Workspace name | -| AZURE_RM_SVC_CONNECTION | azure-resource-connection | [Azure Resource Manager Service Connection](#create-an-azure-devops-service-connection-for-the-azure-resource-manager) name | -| WORKSPACE_SVC_CONNECTION | aml-workspace-connection | [Azure ML Workspace Service Connection](#create-an-azure-devops-azure-ml-workspace-service-connection) name | -| ACI_DEPLOYMENT_NAME | mlops-aci | [Azure Container Instances](https://azure.microsoft.com/en-us/services/container-instances/) name | | - -Make sure you select the **Allow access to all pipelines** checkbox in the variable group configuration. To do this, first **Save** the variable group, then click **Pipeline Permissions**, then the button with 3 vertical dots, and then **Open access** button. - -More variables are available for further tweaking, but the above variables are all you need to get started with this example. For more information, see the [Additional Variables and Configuration](#additional-variables-and-configuration) section. - -### Variable Descriptions - -**BASE_NAME** is used as a prefix for naming Azure resources and should be unique. When sharing an Azure subscription, the prefix allows you to avoid naming collisions for resources that require unique names, for example, Azure Blob Storage and Registry DNS. Make sure to set BASE_NAME to a unique name so that created resources will have unique names, for example, MyUniqueMLamlcr, MyUniqueML-AML-KV, and so on. The length of the BASE_NAME value shouldn't exceed 10 characters and must contain letters and numbers only. - -**LOCATION** is the name of the [Azure location](https://azure.microsoft.com/en-us/global-infrastructure/locations/) for your resources. There should be no spaces in the name. For example, central, westus, northeurope. You can list all the region codes by running `az account list-locations -o table` in the Azure CLI. - -**RESOURCE_GROUP** is used as the name for the resource group that will hold the Azure resources for the solution. If providing an existing Azure ML Workspace, set this value to the corresponding resource group name. - -**WORKSPACE_NAME** is used for creating the Azure Machine Learning Workspace. *While you should be able to provide an existing Azure ML Workspace if you have one, you will run into problems if this has been provisioned manually and the naming of the associated storage account doesn't follow the convention followed in this repo -- as the environment provisioning will try to associate it with a new Storage Account and this is not supported. To avoid these problems, specify a new workspace/unique name.* - -**AZURE_RM_SVC_CONNECTION** is used by the [Azure Pipeline](../environment_setup/iac-create-environment-pipeline.yml) in Azure DevOps that creates the Azure ML workspace and associated resources through Azure Resource Manager. You'll create the connection in a [step below](#create-an-azure-devops-service-connection-for-the-azure-resource-manager). - -**WORKSPACE_SVC_CONNECTION** is used to reference a [service connection for the Azure ML workspace](#create-an-azure-devops-azure-ml-workspace-service-connection). You'll create the connection after [provisioning the workspace](#provisioning-resources-using-azure-pipelines) in the [Create an Azure DevOps Service Connection for the Azure ML Workspace](#create-an-azure-devops-service-connection-for-the-azure-ml-workspace) section below. - -**ACI_DEPLOYMENT_NAME** is used for naming the scoring service during deployment to [Azure Container Instances](https://azure.microsoft.com/en-us/services/container-instances/). - - -## Provisioning resources using Azure Pipelines - -The easiest way to create all required Azure resources (Resource Group, Azure ML Workspace, Container Registry, and others) is to use the **Infrastructure as Code (IaC)** [pipeline with ARM templates](../environment_setup/iac-create-environment-pipeline-arm.yml) or the [pipeline with Terraform templates](../environment_setup/iac-create-environment-pipeline-tf.yml). The pipeline takes care of setting up all required resources based on these [Azure Resource Manager templates](../environment_setup/arm-templates/cloud-environment.json), or based on these [Terraform templates](../environment_setup/tf-templates). - -**Note:** Since Azure Blob storage account required for batch scoring is optional, the resource provisioning pipelines mentioned above do not create this resource automatically, and manual creation is required before use. - -### Create an Azure DevOps Service Connection for the Azure Resource Manager - -The [IaC provisioning pipeline](../environment_setup/iac-create-environment-pipeline.yml) requires an **Azure Resource Manager** [service connection](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#create-a-service-connection). To create one, in Azure DevOps select **Project Settings**, then **Service Connections**, and create a new one, where: - -- Type is **Azure Resource Manager** -- Authentication method is **Service principal (automatic)** -- Scope level is **Subscription** -- Leave **`Resource Group`** empty after selecting your subscription in the dropdown -- Use the same **`Service Connection Name`** that you used in the variable group you created -- Select **Grant access permission to all pipelines** - -![Create service connection](./images/create-rm-service-connection.png) - -**Note:** Creating the Azure Resource Manager service connection scope requires 'Owner' or 'User Access Administrator' permissions on the subscription. -You'll also need sufficient permissions to register an application with your Azure AD tenant, or you can get the ID and secret of a service principal from your Azure AD Administrator. That principal must have 'Contributor' permissions on the subscription. - -### Create the IaC Pipeline - -In your Azure DevOps project, create a build pipeline from your forked repository: - -![Build connect step](./images/build-connect.png) - -If you are using GitHub, after picking the option above, you'll be asked to authorize to GitHub and select the repo you forked. Then you'll have to select your forked repository on GitHub under the **Repository Access** section, and click **Approve and Install**. - -After the above, and when you're redirected back to Azure DevOps, select the **Existing Azure Pipelines YAML file** option and set the path to [/environment_setup/iac-create-environment-pipeline-arm.yml](../environment_setup/iac-create-environment-pipeline-arm.yml) or to [/environment_setup/iac-create-environment-pipeline-tf.yml](../environment_setup/iac-create-environment-pipeline-tf.yml), depending on if you want to deploy your infrastructure using ARM templates or Terraform: - -![Configure step](./images/select-iac-pipeline.png) - -If you decide to use Terraform, make sure the ['Terraform Build & Release Tasks' from Charles Zipp](https://marketplace.visualstudio.com/items?itemName=charleszipp.azure-pipelines-tasks-terraform) is installed. - -Having done that, run the pipeline: - -![IaC run](./images/run-iac-pipeline.png) - -Check that the newly created resources appear in the [Azure Portal](https://portal.azure.com): - -![Created resources](./images/created-resources.png) - -**Note**: If you have other errors, one good thing to check is what you used in the variable names. If you end up running the pipeline multiple times, you may also run into errors and need to delete the Azure services and re-run the pipeline -- this should include a resource group, a KeyVault, a Storage Account, a Container Registry, an Application Insights and a Machine Learning workspace. - -## Create an Azure DevOps Service Connection for the Azure ML Workspace - -At this point, you should have an Azure ML Workspace created. Similar to the Azure Resource Manager service connection, you need to create an additional one for the Azure ML Workspace. - -Create a new service connection to your Azure ML Workspace using the [Machine Learning Extension](https://marketplace.visualstudio.com/items?itemName=ms-air-aiagility.vss-services-azureml) instructions to enable executing the Azure ML training pipeline. The connection name needs to match `WORKSPACE_SVC_CONNECTION` that you set in the variable group above (e.g., 'aml-workspace-connection'). - -![Created resources](./images/ml-ws-svc-connection.png) - -**Note:** Similar to the Azure Resource Manager service connection you created earlier, creating a service connection with Azure Machine Learning workspace scope requires 'Owner' or 'User Access Administrator' permissions on the Workspace. -You'll need sufficient permissions to register an application with your Azure AD tenant, or you can get the ID and secret of a service principal from your Azure AD Administrator. That principal must have Contributor permissions on the Azure ML Workspace. - -## Set up Build, Release Trigger, and Release Multi-Stage Pipelines - -Now that you've provisioned all the required Azure resources and service connections, you can set up the pipelines for training (Continuous Integration - **CI**) and deploying (Continuous Deployment - **CD**) your machine learning model to production. Additionally, you can set up a pipeline for batch scoring. - -1. **Model CI, training, evaluation, and registration** - triggered on code changes to master branch on GitHub. Runs linting, unit tests, code coverage, and publishes and runs the training pipeline. If a new model is registered after evaluation, it creates a build artifact containing the JSON metadata of the model. Definition: [diabetes_regression-ci.yml](../.pipelines/diabetes_regression-ci.yml). -1. **Release deployment** - consumes the artifact of the previous pipeline and deploys a model to either [Azure Container Instances (ACI)](https://azure.microsoft.com/en-us/services/container-instances/), [Azure Kubernetes Service (AKS)](https://azure.microsoft.com/en-us/services/kubernetes-service), or [Azure App Service](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-deploy-app-service) environments. See [Further Exploration](#further-exploration) for other deployment types. Definition: [diabetes_regression-cd.yml](../.pipelines/diabetes_regression-cd.yml). - 1. **Note:** Edit the pipeline definition to remove unused stages. For example, if you're deploying to Azure Container Instances and Azure Kubernetes Service only, you'll need to delete the unused `Deploy_Webapp` stage. -1. **Batch Scoring Code Continuous Integration** - consumes the artifact of the model training pipeline. Runs linting, unit tests, code coverage, publishes a batch scoring pipeline, and invokes the published batch scoring pipeline to score a model. - -These pipelines use a Docker container on the Azure Pipelines agents to accomplish the pipeline steps. The container image ***mcr.microsoft.com/mlops/python:latest*** is built with [this Dockerfile](../environment_setup/Dockerfile) and has all the necessary dependencies installed for MLOpsPython and ***diabetes_regression***. This image is an example of a custom Docker image with a pre-baked environment. The environment is guaranteed to be the same on any building agent, VM, or local machine. **In your project, you'll want to build your own Docker image that only contains the dependencies and tools required for your use case. Your image will probably be smaller and faster, and it will be maintained by your team.** - -### Set up the Model CI, training, evaluation, and registration pipeline - -In your Azure DevOps project, create and run a new build pipeline based on the [./pipelines/diabetes_regression-ci.yml](../.pipelines/diabetes_regression-ci.yml) -pipeline definition in your forked repository. - -If you plan to use the release deployment pipeline (in the next section), you will need to rename this pipeline to `Model-Train-Register-CI`. - -**Note**: *To rename your pipeline, after you saved it, click **Pipelines** on the left menu on Azure DevOps, then **All** to see all the pipelines, then click the menu with the 3 vertical dots that appears when you hover the name of the new pipeline, and click it to pick **"Rename/move pipeline"**.* - -Start a run of the pipeline if you haven't already, and once the pipeline is finished, check the execution result. Note that the run can take 20 minutes, with time mostly spent in **Trigger ML Training Pipeline > Invoke ML Pipeline** step. You can track the execution of the AML pipeline by opening the AML Workspace user interface. Screenshots are below: - -![Build](./images/model-train-register.png) - -And the pipeline artifacts: - -![Build](./images/model-train-register-artifacts.png) - -Also check the published training pipeline in your newly created AML workspace in [Azure Machine Learning Studio](https://ml.azure.com/): - -![Training pipeline](./images/training-pipeline.png) - -Great, you now have the build pipeline for training set up which automatically triggers every time there's a change in the master branch! - -After the pipeline is finished, you'll also see a new model in the **AML Workspace** model registry section: - -![Trained model](./images/trained-model.png) - -To disable the automatic trigger of the training pipeline, change the `auto-trigger-training` variable as listed in the `.pipelines\diabetes_regression-ci.yml` pipeline to `false`. You can also override the variable at runtime execution of the pipeline. - -The pipeline stages are summarized below: - -#### Model CI - -- Linting (code quality analysis) -- Unit tests and code coverage analysis -- Build and publish _ML Training Pipeline_ in an _ML Workspace_ - -#### Train model - -- Determine the ID of the _ML Training Pipeline_ published in the previous stage. -- Trigger the _ML Training Pipeline_ and waits for it to complete. - - This is an **agentless** job. The CI pipeline can wait for ML pipeline completion for hours or even days without using agent resources. -- Determine if a new model was registered by the _ML Training Pipeline_. - - If the model evaluation step of the AML Pipeline determines that the new model doesn't perform any better than the previous one, the new model won't register and the _ML Training Pipeline_ will be **canceled**. In this case, you'll see a message in the 'Train Model' job under the 'Determine if evaluation succeeded and new model is registered' step saying '**Model was not registered for this run.**' - - See [evaluate_model.py](../diabetes_regression/evaluate/evaluate_model.py#L118) for the evaluation logic. This is a simplified test that just looks at MSE to decide whether or not to register a new model. A more realistic verification would also do some error analysis and verify the inferences/error distribution against a test dataset, for example. - - **Note**: *while it's possible to do an Evaluation Step as part of the ADO pipeline, this evaluation is logically part of the work done by Data Scientists, and as such the recommendation is that this step is done as part of the AML Pipeline and not ADO pipelines.* - - [Additional Variables and Configuration](#additional-variables-and-configuration) for configuring this and other behavior. - -#### Create pipeline artifact - -- Get the info about the registered model -- Create an Azure DevOps pipeline artifact called `model` that contains a `model.json` file containing the model information, for example: - -```json -{ "createdTime": "2021-12-14T13:03:24.494748+00:00", "framework": "Custom", "frameworkVersion": null, "id": "diabetes_regression_model.pkl:1", "name": "diabetes_regression_model.pkl", "version": 1 } -``` - -- Here's [more information on Azure DevOps Artifacts](https://docs.microsoft.com/en-us/azure/devops/pipelines/artifacts/build-artifacts?view=azure-devops&tabs=yaml#explore-download-and-deploy-your-artifacts) and where to find them on the ADO user interface. - -### Set up the Release Deployment and/or Batch Scoring pipelines - ---- -**PRE-REQUISITES** - -In order to use these pipelines: - -1. Follow the steps to set up the Model CI, training, evaluation, and registration pipeline. -1. You **must** rename your model CI/train/eval/register pipeline to `Model-Train-Register-CI`. - -These pipelines rely on the model CI pipeline and reference it by name. - -If you would like to change the name of your model CI pipeline, you must edit this section of yml for the CD and batch scoring pipeline, where it says `source: Model-Train-Register-CI` to use your own name. -``` -trigger: none -resources: - containers: - - container: mlops - image: mcr.microsoft.com/mlops/python:latest - pipelines: - - pipeline: model-train-ci - source: Model-Train-Register-CI # Name of the triggering pipeline - trigger: - branches: - include: - - master -``` - ---- - -The release deployment and batch scoring pipelines have the following behaviors: - -- The pipeline will **automatically trigger** on completion of the `Model-Train-Register-CI` pipeline for the master branch. -- The pipeline will default to using the latest successful build of the `Model-Train-Register-CI` pipeline. It will deploy the model produced by that build. -- You can specify a `Model-Train-Register-CI` build ID when running the pipeline manually. You can find this in the url of the build, and the model registered from that build will also be tagged with the build ID. This is useful to skip model training and registration, and deploy/score a model successfully registered by a `Model-Train-Register-CI` build. - - For example, if you navigate to a specific run of your CI pipeline, the URL should be something like `https://dev.azure.com/yourOrgName/yourProjectName/_build/results?buildId=653&view=results`. **653** is the build ID in this case. See the second screenshot below to verify where this number would be used. - -### Set up the Release Deployment pipeline - -In your Azure DevOps project, create and run a new **build** pipeline based on the [./pipelines/diabetes_regression-cd.yml](../.pipelines/diabetes_regression-cd.yml) -pipeline definition in your forked repository. It is recommended you rename this pipeline to something like `Model-Deploy-CD` for clarity. - -**Note**: *While Azure DevOps supports both Build and Release pipelines, when using YAML you don't usually need to use Release pipelines. This repository assumes the usage only of Build pipelines.* - -Your first run will use the latest model created by the `Model-Train-Register-CI` pipeline. - -Once the pipeline is finished, check the execution result: - -![Build](./images/model-deploy-result.png) - -To specify a particular build's model, set the `Model Train CI Build Id` parameter to the build ID you would like to use: - -![Build](./images/model-deploy-configure.png) - -Once your pipeline run begins, you can see the model name and version downloaded from the `Model-Train-Register-CI` pipeline. The run time will typically be 5-10 minutes. - -![Build](./images/model-deploy-get-artifact-logs.png) - -The pipeline has the following stage: - -#### Deploy to ACI - -- Deploy the model to the QA environment in [Azure Container Instances](https://azure.microsoft.com/en-us/services/container-instances/). -- Smoke test - - The test sends a sample query to the scoring web service and verifies that it returns the expected response. Have a look at the [smoke test code](../ml_service/util/smoke_test_scoring_service.py) for an example. - -- You can verify that an ACI instance was created in the same resource group you specified: - -![Created Resouces ](./images/aci-in-azure-portal.png) - -### Set up the Batch Scoring pipeline - -In your Azure DevOps project, create and run a new build pipeline based on the [.pipelines/diabetes_regression-batchscoring-ci.yml](../.pipelines/diabetes_regression-batchscoring-ci.yml) -pipeline definition in your forked repository. Rename this pipeline to `Batch-Scoring`. - -Once the pipeline is finished, check the execution result: - -![Build](./images/batchscoring-ci-result.png) - -Also check the published batch scoring pipeline in your AML workspace in the [Azure Portal](https://portal.azure.com/): - -![Batch scoring pipeline](./images/batchscoring-pipeline.png) - -Great, you now have the build pipeline set up for batch scoring which automatically triggers every time there's a change in the master branch! - -The pipeline stages are described below in detail -- and you must do further configurations to actually see the batch inferences: - -#### Batch Scoring CI - -- Linting (code quality analysis) -- Unit tests and code coverage analysis -- Build and publish *ML Batch Scoring Pipeline* in an *AML Workspace* - -#### Batch Score model - -- Determine the model to be used based on the model name (required), model version, model tag name and model tag value bound pipeline parameters. - - If run via Azure DevOps pipeline, the batch scoring pipeline will take the model name and version from the `Model-Train-Register-CI` build used as input. - - If run locally without the model version, the batch scoring pipeline will use the model's latest version. -- Trigger the *ML Batch Scoring Pipeline* and wait for it to complete. - - This is an **agentless** job. The CI pipeline can wait for ML pipeline completion for hours or even days without using agent resources. -- Create an Azure ML pipeline with two steps. The pipeline is created by the code in `ml_service\pipelines\diabetes_regression_build_parallel_batchscore_pipeline.py` and has two steps: - - `scoringstep` - this step is a **`ParallelRunStep`** that executes the code in `diabetes_regression\scoring\parallel_batchscore.py` with several different batches of the data to be scored. - - `scorecopystep` - this is a **`PythonScriptStep`** step that copies the output inferences from Azure ML's internal storage into a target location in a another storage account. - - If you run the instructions as defined above with no changes to variables, this step will be **not** executed. You'll see a message in the logs for the corresponding step saying `Missing Parameters`. In this case, you'll be able to find the file with the inferences in the same Storage Account associated with Azure ML, in a location similar to `azureml-blobstore-SomeGuid\azureml\SomeOtherGuid\defaultoutput\parallel_run_step.txt`. One way to find the right path is this: - - Open your experiment in Azure ML (by default called `mlopspython`). - - Open the run that you want to look at (named something like `neat_morning_qc10dzjy` or similar). - - In the graphical pipeline view with 2 steps, click the button to open the details tab: `Show run overview`. - - You'll see two steps (corresponding to `scoringstep`and `scorecopystep` as described above). - - Click the step with the with older "Submitted time". - - Click "Output + logs" at the top, and you'll see something like the following: - ![Outputs of `scoringstep`](./images/batch-child-run-scoringstep.png) - - The `defaultoutput` file will have JSON content with the path to a file called `parallel_run_step.txt` containing the scoring. - -To properly configure this step for your own custom scoring data, you must follow the instructions in [Configure Custom Batch Scoring](custom_model.md#Configure-Custom-Batch-Scoring), which let you specify both the location of the files to score (via the `SCORING_DATASTORE_INPUT_*` configuration variables) and where to store the inferences (via the `SCORING_DATASTORE_OUTPUT_*` configuration variables). - -## Further Exploration - -You should now have a working set of pipelines that can get you started with MLOpsPython. Below are some additional features offered that might suit your scenario. - -### Deploy the model to Azure Kubernetes Service - -MLOpsPython also can deploy to [Azure Kubernetes Service](https://azure.microsoft.com/en-us/services/kubernetes-service). - -Creating a cluster on Azure Kubernetes Service is out of scope of this tutorial, but you can find set up information on the [Quickstart: Deploy an Azure Kubernetes Service (AKS) cluster using the Azure portal](https://docs.microsoft.com/en-us/azure/aks/kubernetes-walkthrough-portal#create-an-aks-cluster) page. - -> **_Note_** -> -> If your target deployment environment is a Kubernetes cluster and you want to implement Canary and/or A/B testing deployment strategies, check out this [tutorial](./canary_ab_deployment.md). - -Keep the Azure Container Instances deployment active because it's a lightweight way to validate changes before deploying to Azure Kubernetes Service. - -In the Variables tab, edit your variable group (`devopsforai-aml-vg`). In the variable group definition, add these variables: - -| Variable Name | Suggested Value | Description | -| ------------------- | --------------- | ----------- | -| AKS_COMPUTE_NAME | aks | The Compute name of the inference cluster, created in the Azure ML Workspace (ml.azure.com). This connection has to be created manually before setting the value! | -| AKS_DEPLOYMENT_NAME | mlops-aks | The name of the deployed aks cluster in your subscripttion. | - -After successfully deploying to Azure Container Instances, the next stage will deploy the model to Kubernetes and run a smoke test. - -Set **AKS_COMPUTE_NAME** to the _Compute name_ of the Inference Cluster that references the Azure Kubernetes Service cluster in your Azure ML Workspace. - -![build](./images/multi-stage-aci-aks.png) - -Consider enabling [manual approvals](https://docs.microsoft.com/en-us/azure/devops/pipelines/process/approvals) before the deployment stages. - -#### Web Service Authentication on Azure Kubernetes Service - -When deploying to Azure Kubernetes Service, key-based authentication is enabled by default. You can also enable token-based authentication. Token-based authentication requires clients to use an Azure Active Directory account to request an authentication token, which is used to make requests to the deployed service. For more details on how to authenticate with ML web service deployed on the AKS service please follow [Smoke Test](../ml_service/util/smoke_test_scoring_service.py) or the Azure documentation on [web service authentication](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-deploy-azure-kubernetes-service#web-service-authentication). - -### Deploy the model to Azure App Service (Azure Web App for containers) - -If you want to deploy your scoring service as an [Azure App Service](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-deploy-app-service) instead of Azure Container Instances or Azure Kubernetes Service, follow these additional steps. - -- First, you'll need to create an App Service Plan using Linux. The simplest way is to run this from your Azure CLI: `az appservice plan create --name nameOfAppServicePlan --resource-group nameOfYourResourceGroup --sku B1 --is-linux`. - -- Second, you'll need to create a webapp in this App Service Plan, and configure it to run a certain container. As currently there is no UI in the Azure Portal to do this, this has to be done from the command line. We'll come back to this. - -- In the Variables tab, edit your variable group (`devopsforai-aml-vg`) and add a variable: - - | Variable Name | Suggested Value | - | ---------------------- | ---------------------- | - | WEBAPP_DEPLOYMENT_NAME | _name of your web app_ | - - Set **WEBAPP_DEPLOYMENT_NAME** to the name of your Azure Web App. You have not yet created this webapp, so just use the name you're planning on giving it. - -- Delete the **ACI_DEPLOYMENT_NAME** or any AKS-related variable. - -- Next, you'll need to run your `Model-Deploy-CD` pipeline - - - The pipeline uses the [Azure ML CLI](../.pipelines/diabetes_regression-package-model-template.yml) to create a scoring image. The image will be registered under an Azure Container Registry instance that belongs to the Azure Machine Learning Service. Any dependencies that the scoring file depends on can also be packaged with the container with an image config. Learn more about how to create a container using the Azure ML SDK with the [Image class](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.core.image.image.image?view=azure-ml-py#create-workspace--name--models--image-config-) API documentation. - - - This pipeline will **fail** on the `Azure Web App on Container Deploy` step, with an error saying the webapp doesn't exist yet. This is expected. Go to the next step. - -- If you want to confirm that the scoring image has been created, open the Azure Container Registry mentioned above, which will be in the Resource Group of the Azure ML workspace, and look for the repositories. You'll have one that was created by the pipeline, called `package`, which was created by the CD pipeline: - - ![Azure Container Registry repository list](./images/container-registry-webapp-image.png) - -- Notedown the name of the Login Server of your Azure Container Registry. It'll be something like `YourAcrName.azurecr.io`. - -- Going back to the Step Two, now you can create a Web App in you App Service Plan using this scoring image but with the `latest` tag. The easiest way to do this is to run this in the Azure CLI: `az webapp create --resource-group yourResourceGroup --plan nameOfAppServicePlan --name nameOfWebApp --deployment-container-image-name YourAcrName.azurecr.io/package:latest` - - Here, `nameOfWebApp` is the same you put in your Azure DevOps `WEBAPP_DEPLOYMENT_NAME` variable. - -From now on, whenever you run the CD pipeline, it will update the image in the container registry and it'll automatically update the one used in the WebApp. CD pipeline runs will now succeed. - -![build](./images/ADO-CD-pipeline-to-webapp.png) - -To confirm, you can open the App Service Plan, open your new WebApp, and open the **Deployment Center**, where you'll see something like: - -![WebApp Deployment Center page](./images/appservice-webapp-deploymentcenter.png) - -If you run into problems, you may have to make sure your webapp has the credentials to pull the image from the Azure Container Registry created by the Infrastructure as Code pipeline. Instructions can be found on the [Configure registry credentials in web app](https://docs.microsoft.com/en-us/azure/devops/pipelines/targets/webapp-on-container-linux?view=azure-devops&tabs=dotnet-core%2Cyaml#configure-registry-credentials-in-web-app) page. - -### Example pipelines using R - -The build pipeline also supports building and publishing Azure ML pipelines using R to train a model. You can enable it by changing the `build-train-script` pipeline variable to either of the following values: - -- `diabetes_regression_build_train_pipeline_with_r.py` to train a model with R on Azure ML Compute. You'll also need to uncomment (include) the `r-essentials` Conda packages in the environment definition YAML `diabetes_regression/conda_dependencies.yml`. -- `diabetes_regression_build_train_pipeline_with_r_on_dbricks.py` to train a model with R on Databricks. You'll need to manually create a Databricks cluster and attach it to the Azure ML Workspace as a compute resource. Set the DB_CLUSTER_ID and DATABRICKS_COMPUTE_NAME variables in your variable group. - -Example ML pipelines using R have a single step to train a model. They don't demonstrate how to evaluate and register a model. The evaluation and registering techniques are shown only in the Python implementation. - -### Observability and Monitoring - -You can explore aspects of model observability in the solution, such as: - -- **Logging**: Navigate to the Application Insights instance linked to the Azure ML Portal, then go to the Logs (Analytics) pane. The following sample query correlates HTTP requests with custom logs generated in `score.py`. This can be used, for example, to analyze query duration vs. scoring batch size: - - ```sql - let Traceinfo=traces - | extend d=parse_json(tostring(customDimensions.Content)) - | project workspace=customDimensions.["Workspace Name"], - service=customDimensions.["Service Name"], - NumberOfPredictions=tostring(d.NumberOfPredictions), - id=tostring(d.RequestId), - TraceParent=tostring(d.TraceParent); - requests - | project timestamp, id, success, resultCode, duration - | join kind=fullouter Traceinfo on id - | project-away id1 - ``` - -- **Distributed tracing**: The smoke test client code sets an HTTP `traceparent` header (per the [W3C Trace Context proposed specification](https://www.w3.org/TR/trace-context-1)), and the `score.py` code logs the header. The query above shows how to surface this value. You can adapt it to your tracing framework. -- **Monitoring**: You can use [Azure Monitor for containers](https://docs.microsoft.com/en-us/azure/azure-monitor/insights/container-insights-overview) to monitor the Azure ML scoring containers' performance. - -### Clean up the example resources - -To remove the resources created for this project, use the [/environment_setup/iac-remove-environment-pipeline.yml](../environment_setup/iac-remove-environment-pipeline.yml) definition or you can just delete the resource group in the [Azure Portal](https://portal.azure.com). - -## Next Steps: Integrating your project - -- The [custom model](custom_model.md) guide includes information on bringing your own code to this repository template. -- We recommend using a [custom container](custom_model.md#customize-the-build-agent-environment) to manage your pipeline environment and dependencies. The container provided with the getting started guide may not be suitable or up to date with your project needs. -- Consider using [Azure Pipelines self-hosted agents](https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/agents?view=azure-devops&tabs=browser#install) to speed up your Azure ML pipeline execution. The Docker container image for the Azure ML pipeline is sizable, and having it cached on the agent between runs can trim several minutes from your runs. Additionally, for secure deployments of Azure Machine Learning, you'll probably need to have a self-hosted agent in a Virtual Network. - -### Additional Variables and Configuration - -#### More variable options - -There are more variables used in the project. They're defined in two places: one for local execution and one for using Azure DevOps Pipelines. - -For using Azure Pipelines, all other variables are stored in the file `.pipelines/diabetes_regression-variables-template.yml`. Using the default values as a starting point, adjust the variables to suit your requirements. - -In the `diabetes_regression` folder, you'll also find the `parameters.json` file that we recommend using to provide parameters for training, evaluation, and scoring scripts. The sample parameter that `diabetes_regression` uses is the ridge regression [_alpha_ hyperparameter](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Ridge.html). We don't provide any serializers for this config file. - -#### Local configuration - -For instructions on how to set up a local development environment, refer to the [Development environment setup instructions](development_setup.md). diff --git a/docs/images/ADO-CD-pipeline-to-webapp.png b/docs/images/ADO-CD-pipeline-to-webapp.png deleted file mode 100644 index aac8c9ee..00000000 Binary files a/docs/images/ADO-CD-pipeline-to-webapp.png and /dev/null differ diff --git a/docs/images/aci-in-azure-portal.png b/docs/images/aci-in-azure-portal.png deleted file mode 100644 index e7bfa8cd..00000000 Binary files a/docs/images/aci-in-azure-portal.png and /dev/null differ diff --git a/docs/images/appservice-webapp-deploymentcenter.png b/docs/images/appservice-webapp-deploymentcenter.png deleted file mode 100644 index b79ff615..00000000 Binary files a/docs/images/appservice-webapp-deploymentcenter.png and /dev/null differ diff --git a/docs/images/batch-child-run-scoringstep.png b/docs/images/batch-child-run-scoringstep.png deleted file mode 100644 index 6b87f52d..00000000 Binary files a/docs/images/batch-child-run-scoringstep.png and /dev/null differ diff --git a/docs/images/batchscoring-ci-result.png b/docs/images/batchscoring-ci-result.png deleted file mode 100644 index d07d41a8..00000000 Binary files a/docs/images/batchscoring-ci-result.png and /dev/null differ diff --git a/docs/images/batchscoring-pipeline.png b/docs/images/batchscoring-pipeline.png deleted file mode 100644 index 2b79fe03..00000000 Binary files a/docs/images/batchscoring-pipeline.png and /dev/null differ diff --git a/docs/images/build-connect.png b/docs/images/build-connect.png deleted file mode 100644 index 79553d80..00000000 Binary files a/docs/images/build-connect.png and /dev/null differ diff --git a/docs/images/ci-build-pipeline-configure.png b/docs/images/ci-build-pipeline-configure.png deleted file mode 100644 index 62953b53..00000000 Binary files a/docs/images/ci-build-pipeline-configure.png and /dev/null differ diff --git a/docs/images/container-registry-webapp-image.png b/docs/images/container-registry-webapp-image.png deleted file mode 100644 index 4ec09f8f..00000000 Binary files a/docs/images/container-registry-webapp-image.png and /dev/null differ diff --git a/docs/images/create-rm-service-connection.png b/docs/images/create-rm-service-connection.png deleted file mode 100644 index e677636a..00000000 Binary files a/docs/images/create-rm-service-connection.png and /dev/null differ diff --git a/docs/images/created-resources.png b/docs/images/created-resources.png deleted file mode 100644 index d5136ee8..00000000 Binary files a/docs/images/created-resources.png and /dev/null differ diff --git a/docs/images/custom-container-variables.png b/docs/images/custom-container-variables.png deleted file mode 100644 index 24a6a92a..00000000 Binary files a/docs/images/custom-container-variables.png and /dev/null differ diff --git a/docs/images/deploy-aci.png b/docs/images/deploy-aci.png deleted file mode 100644 index 0270143b..00000000 Binary files a/docs/images/deploy-aci.png and /dev/null differ diff --git a/docs/images/deploy-aks.png b/docs/images/deploy-aks.png deleted file mode 100644 index 96d83b8b..00000000 Binary files a/docs/images/deploy-aks.png and /dev/null differ diff --git a/docs/images/library_variable_groups.png b/docs/images/library_variable_groups.png deleted file mode 100644 index 1029769c..00000000 Binary files a/docs/images/library_variable_groups.png and /dev/null differ diff --git a/docs/images/ml-lifecycle.png b/docs/images/ml-lifecycle.png deleted file mode 100644 index c06e61d3..00000000 Binary files a/docs/images/ml-lifecycle.png and /dev/null differ diff --git a/docs/images/ml-ws-svc-connection.png b/docs/images/ml-ws-svc-connection.png deleted file mode 100644 index baf52e1f..00000000 Binary files a/docs/images/ml-ws-svc-connection.png and /dev/null differ diff --git a/docs/images/model-artifact-cd-trigger.png b/docs/images/model-artifact-cd-trigger.png deleted file mode 100644 index aca2cfb5..00000000 Binary files a/docs/images/model-artifact-cd-trigger.png and /dev/null differ diff --git a/docs/images/model-artifact.png b/docs/images/model-artifact.png deleted file mode 100644 index b89390b4..00000000 Binary files a/docs/images/model-artifact.png and /dev/null differ diff --git a/docs/images/model-deploy-configure.png b/docs/images/model-deploy-configure.png deleted file mode 100644 index fcd87750..00000000 Binary files a/docs/images/model-deploy-configure.png and /dev/null differ diff --git a/docs/images/model-deploy-get-artifact-logs.png b/docs/images/model-deploy-get-artifact-logs.png deleted file mode 100644 index 2249a8d3..00000000 Binary files a/docs/images/model-deploy-get-artifact-logs.png and /dev/null differ diff --git a/docs/images/model-deploy-result.png b/docs/images/model-deploy-result.png deleted file mode 100644 index cd3d166e..00000000 Binary files a/docs/images/model-deploy-result.png and /dev/null differ diff --git a/docs/images/model-train-register-artifacts.png b/docs/images/model-train-register-artifacts.png deleted file mode 100644 index 0d3eed26..00000000 Binary files a/docs/images/model-train-register-artifacts.png and /dev/null differ diff --git a/docs/images/model-train-register.png b/docs/images/model-train-register.png deleted file mode 100644 index 5ce4ef41..00000000 Binary files a/docs/images/model-train-register.png and /dev/null differ diff --git a/docs/images/multi-stage-aci-aks.png b/docs/images/multi-stage-aci-aks.png deleted file mode 100644 index 0307fbf6..00000000 Binary files a/docs/images/multi-stage-aci-aks.png and /dev/null differ diff --git a/docs/images/multi-stage-aci.png b/docs/images/multi-stage-aci.png deleted file mode 100644 index a96f3195..00000000 Binary files a/docs/images/multi-stage-aci.png and /dev/null differ diff --git a/docs/images/multi-stage-webapp.png b/docs/images/multi-stage-webapp.png deleted file mode 100644 index e6d60ce1..00000000 Binary files a/docs/images/multi-stage-webapp.png and /dev/null differ diff --git a/docs/images/release-task-createimage.PNG b/docs/images/release-task-createimage.PNG deleted file mode 100644 index 8224db18..00000000 Binary files a/docs/images/release-task-createimage.PNG and /dev/null differ diff --git a/docs/images/release-task-webappdeploy.PNG b/docs/images/release-task-webappdeploy.PNG deleted file mode 100644 index 0f23c08d..00000000 Binary files a/docs/images/release-task-webappdeploy.PNG and /dev/null differ diff --git a/docs/images/release-webapp-pipeline.PNG b/docs/images/release-webapp-pipeline.PNG deleted file mode 100644 index 10ffddff..00000000 Binary files a/docs/images/release-webapp-pipeline.PNG and /dev/null differ diff --git a/docs/images/run-iac-pipeline.png b/docs/images/run-iac-pipeline.png deleted file mode 100644 index f2549da8..00000000 Binary files a/docs/images/run-iac-pipeline.png and /dev/null differ diff --git a/docs/images/scoring_image.png b/docs/images/scoring_image.png deleted file mode 100644 index ecb1c245..00000000 Binary files a/docs/images/scoring_image.png and /dev/null differ diff --git a/docs/images/select-iac-pipeline.png b/docs/images/select-iac-pipeline.png deleted file mode 100644 index 695b041f..00000000 Binary files a/docs/images/select-iac-pipeline.png and /dev/null differ diff --git a/docs/images/trained-model.png b/docs/images/trained-model.png deleted file mode 100644 index 5bea4fe2..00000000 Binary files a/docs/images/trained-model.png and /dev/null differ diff --git a/docs/images/training-pipeline.png b/docs/images/training-pipeline.png deleted file mode 100644 index 48854513..00000000 Binary files a/docs/images/training-pipeline.png and /dev/null differ diff --git a/environment_setup/Dockerfile b/environment_setup/Dockerfile index 0dfa36b6..5159ade2 100644 --- a/environment_setup/Dockerfile +++ b/environment_setup/Dockerfile @@ -4,7 +4,7 @@ LABEL org.label-schema.vendor = "Microsoft" \ org.label-schema.url = "https://hub.docker.com/r/microsoft/mlopspython" \ org.label-schema.vcs-url = "https://github.com/microsoft/MLOpsPython" -COPY diabetes_regression/ci_dependencies.yml /setup/ +COPY automobile/ci_dependencies.yml /setup/ # activate environment ENV PATH /usr/local/envs/mlopspython_ci/bin:$PATH diff --git a/environment_setup/install_requirements.sh b/environment_setup/install_requirements.sh index 989e8b1e..9ae27f5d 100755 --- a/environment_setup/install_requirements.sh +++ b/environment_setup/install_requirements.sh @@ -26,6 +26,6 @@ set -eux -conda env create -f diabetes_regression/ci_dependencies.yml +conda env create -f automobile/ci_dependencies.yml conda activate mlopspython_ci diff --git a/experimentation/Diabetes Ridge Regression Experimentation Pipeline.ipynb b/experimentation/Diabetes Ridge Regression Experimentation Pipeline.ipynb deleted file mode 100644 index 8b04a5c5..00000000 --- a/experimentation/Diabetes Ridge Regression Experimentation Pipeline.ipynb +++ /dev/null @@ -1,353 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Experiment with parameters for a Ridge Regression Model on the Diabetes Dataset in an Azure ML Pipeline" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This notebook is for experimenting with different parameters to train a ridge regression model on the Diabetes dataset." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Change out of the experimentation directory\n", - "%cd .." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import azureml.core\n", - "from azureml.core import Workspace" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Load the workspace from the saved config file\n", - "ws = Workspace.from_config()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os, shutil\n", - "\n", - "# Create a folder for the experiment files\n", - "training_folder = 'diabetes-training'\n", - "os.makedirs(training_folder, exist_ok=True)\n", - "\n", - "# Copy the data file into the experiment folder\n", - "shutil.copy('data/diabetes.csv', os.path.join(training_folder, \"diabetes.csv\"))\n", - "\n", - "# Copy the train functions into the experiment folder\n", - "shutil.copy('diabetes_regression/training/train.py', os.path.join(training_folder, \"train.py\"))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%writefile $training_folder/parameters.json\n", - "{\n", - " \"training\":\n", - " {\n", - " \"alpha\": 0.3\n", - " },\n", - " \"evaluation\":\n", - " {\n", - "\n", - " },\n", - " \"scoring\":\n", - " {\n", - " \n", - " }\n", - "}\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%writefile $training_folder/diabetes_training.py\n", - "# Import libraries\n", - "from azureml.core import Run\n", - "import pandas as pd\n", - "import shutil\n", - "import joblib\n", - "\n", - "from train import split_data, train_model\n", - "\n", - "# Get parameters\n", - "parser = argparse.ArgumentParser()\n", - "parser.add_argument('--output_folder', type=str, dest='output_folder', default=\"diabetes_model\", help='output folder')\n", - "args = parser.parse_args()\n", - "output_folder = args.output_folder\n", - "\n", - "# Get the experiment run context\n", - "run = Run.get_context()\n", - "\n", - "# load the diabetes dataset\n", - "print(\"Loading Data...\")\n", - "train_df = pd.read_csv('diabetes.csv')\n", - "\n", - "data = split_data(train_df)\n", - "\n", - "# Specify the parameters to test\n", - "with open(\"parameters.json\") as f:\n", - " pars = json.load(f)\n", - " train_args = pars[\"training\"]\n", - "\n", - "# Log parameters\n", - "for k, v in train_args.items():\n", - " run.log(k, v)\n", - "\n", - "model, metrics = train_model(data, train_args)\n", - "\n", - "# Log metrics\n", - "for k, v in metrics.items():\n", - " run.log(k, v)\n", - "\n", - "# Save the parameters file to the outputs folder\n", - "os.makedirs(output_folder, exist_ok=True)\n", - "shutil.copy('parameters.json', os.path.join(output_folder, 'parameters.json'))\n", - "joblib.dump(value=model, filename= output_folder + \"/model.pkl\")\n", - " \n", - "run.complete()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%writefile $training_folder/register_diabetes.py\n", - "# Import libraries\n", - "import argparse\n", - "import joblib\n", - "from azureml.core import Workspace, Model, Run\n", - "\n", - "# Get parameters\n", - "parser = argparse.ArgumentParser()\n", - "parser.add_argument('--model_folder', type=str, dest='model_folder', default=\"diabetes_model\", help='model location')\n", - "args = parser.parse_args()\n", - "model_folder = args.model_folder\n", - "\n", - "# Get the experiment run context\n", - "run = Run.get_context()\n", - "\n", - "# load the model\n", - "print(\"Loading model from \" + model_folder)\n", - "model_file = model_folder + \"/model.pkl\"\n", - "model = joblib.load(model_file)\n", - "\n", - "Model.register(workspace=run.experiment.workspace,\n", - " model_path = model_file,\n", - " model_name = 'diabetes_model',\n", - " tags={'Training context':'Pipeline'})\n", - "\n", - "run.complete()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azureml.core.compute import ComputeTarget, AmlCompute\n", - "from azureml.core.compute_target import ComputeTargetException\n", - "\n", - "cluster_name = \"aml-cluster\"\n", - "\n", - "# Verify that cluster exists\n", - "try:\n", - " pipeline_cluster = ComputeTarget(workspace=ws, name=cluster_name)\n", - " print('Found existing cluster, use it.')\n", - "except ComputeTargetException:\n", - " # If not, create it\n", - " compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_D2_V2',\n", - " max_nodes=4,\n", - " idle_seconds_before_scaledown=1800)\n", - " pipeline_cluster = ComputeTarget.create(ws, cluster_name, compute_config)\n", - "\n", - "pipeline_cluster.wait_for_completion(show_output=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azureml.core import Environment\n", - "from azureml.core.conda_dependencies import CondaDependencies\n", - "from azureml.core.runconfig import RunConfiguration\n", - "\n", - "# Create a Python environment for the experiment\n", - "diabetes_env = Environment(\"diabetes-pipeline-env\")\n", - "diabetes_env.python.user_managed_dependencies = False # Let Azure ML manage dependencies\n", - "diabetes_env.docker.enabled = True # Use a docker container\n", - "\n", - "# Create a set of package dependencies\n", - "diabetes_packages = CondaDependencies.create(conda_packages=['scikit-learn','pandas'],\n", - " pip_packages=['azureml-sdk'])\n", - "\n", - "# Add the dependencies to the environment\n", - "diabetes_env.python.conda_dependencies = diabetes_packages\n", - "\n", - "# Register the environment (just in case you want to use it again)\n", - "diabetes_env.register(workspace=ws)\n", - "registered_env = Environment.get(ws, 'diabetes-pipeline-env')\n", - "\n", - "# Create a new runconfig object for the pipeline\n", - "pipeline_run_config = RunConfiguration()\n", - "\n", - "# Use the compute you created above. \n", - "pipeline_run_config.target = pipeline_cluster\n", - "\n", - "# Assign the environment to the run configuration\n", - "pipeline_run_config.environment = registered_env\n", - "\n", - "print (\"Run configuration created.\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azureml.pipeline.core import PipelineData\n", - "from azureml.pipeline.steps import PythonScriptStep, EstimatorStep\n", - "from azureml.train.estimator import Estimator\n", - "\n", - "# Get the training dataset\n", - "#diabetes_ds = ws.datasets.get(\"diabetes dataset\")\n", - "\n", - "# Create a PipelineData (temporary Data Reference) for the model folder\n", - "model_folder = PipelineData(\"model_folder\", datastore=ws.get_default_datastore())\n", - "\n", - "estimator = Estimator(source_directory=training_folder,\n", - " compute_target = pipeline_cluster,\n", - " environment_definition=pipeline_run_config.environment,\n", - " entry_script='diabetes_training.py')\n", - "\n", - "# Step 1, run the estimator to train the model\n", - "train_step = EstimatorStep(name = \"Train Model\",\n", - " estimator=estimator, \n", - " estimator_entry_script_arguments=['--output_folder', model_folder],\n", - " outputs=[model_folder],\n", - " compute_target = pipeline_cluster,\n", - " allow_reuse = True)\n", - "\n", - "# Step 2, run the model registration script\n", - "register_step = PythonScriptStep(name = \"Register Model\",\n", - " source_directory = training_folder,\n", - " script_name = \"register_diabetes.py\",\n", - " arguments = ['--model_folder', model_folder],\n", - " inputs=[model_folder],\n", - " compute_target = pipeline_cluster,\n", - " runconfig = pipeline_run_config,\n", - " allow_reuse = True)\n", - "\n", - "print(\"Pipeline steps defined\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azureml.core import Experiment\n", - "from azureml.pipeline.core import Pipeline\n", - "from azureml.widgets import RunDetails\n", - "\n", - "# Construct the pipeline\n", - "pipeline_steps = [train_step, register_step]\n", - "pipeline = Pipeline(workspace = ws, steps=pipeline_steps)\n", - "print(\"Pipeline is built.\")\n", - "\n", - "# Create an experiment and run the pipeline\n", - "experiment = Experiment(workspace = ws, name = 'diabetes-training-pipeline')\n", - "pipeline_run = experiment.submit(pipeline, regenerate_outputs=True)\n", - "print(\"Pipeline submitted for execution.\")\n", - "\n", - "RunDetails(pipeline_run).show()\n", - "pipeline_run.wait_for_completion()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azureml.core import Model\n", - "\n", - "for model in Model.list(ws):\n", - " print(model.name, 'version:', model.version)\n", - " for tag_name in model.tags:\n", - " tag = model.tags[tag_name]\n", - " print ('\\t',tag_name, ':', tag)\n", - " for prop_name in model.properties:\n", - " prop = model.properties[prop_name]\n", - " print ('\\t',prop_name, ':', prop)\n", - " print('\\n')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/experimentation/Diabetes Ridge Regression Parameter Experimentation.ipynb b/experimentation/Diabetes Ridge Regression Parameter Experimentation.ipynb deleted file mode 100644 index aab5e052..00000000 --- a/experimentation/Diabetes Ridge Regression Parameter Experimentation.ipynb +++ /dev/null @@ -1,211 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Experiment with parameters for a Ridge Regression Model on the Diabetes Dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This notebook is for experimenting with different parameters to train a ridge regression model on the Diabetes dataset." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Change out of the experimentation directory\n", - "%cd .." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import azureml.core\n", - "from azureml.core import Workspace" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Load the workspace from the saved config file\n", - "ws = Workspace.from_config()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os, shutil\n", - "\n", - "# Create a folder for the experiment files\n", - "training_folder = 'diabetes-training'\n", - "os.makedirs(training_folder, exist_ok=True)\n", - "\n", - "# Copy the data file into the experiment folder\n", - "shutil.copy('data/diabetes.csv', os.path.join(training_folder, \"diabetes.csv\"))\n", - "\n", - "# Copy the train functions into the experiment folder\n", - "shutil.copy('diabetes_regression/training/train.py', os.path.join(training_folder, \"train.py\"))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%writefile $training_folder/parameters.json\n", - "{\n", - " \"training\":\n", - " {\n", - " \"alpha\": 0.3\n", - " },\n", - " \"evaluation\":\n", - " {\n", - "\n", - " },\n", - " \"scoring\":\n", - " {\n", - " \n", - " }\n", - "}\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%writefile $training_folder/diabetes_training.py\n", - "# Import libraries\n", - "from azureml.core import Run\n", - "import json\n", - "import os\n", - "import pandas as pd\n", - "import shutil\n", - "\n", - "from train import split_data, train_model\n", - "\n", - "# Get the experiment run context\n", - "run = Run.get_context()\n", - "\n", - "# load the diabetes dataset\n", - "print(\"Loading Data...\")\n", - "train_df = pd.read_csv('diabetes.csv')\n", - "\n", - "data = split_data(train_df)\n", - "\n", - "# Specify the parameters to test\n", - "with open(\"parameters.json\") as f:\n", - " pars = json.load(f)\n", - " train_args = pars[\"training\"]\n", - "\n", - "# Log parameters\n", - "for k, v in train_args.items():\n", - " run.log(k, v)\n", - "\n", - "model, metrics = train_model(data, train_args)\n", - "\n", - "# Log metrics\n", - "for k, v in metrics.items():\n", - " run.log(k, v)\n", - "\n", - "# Save the parameters file to the outputs folder\n", - "os.makedirs('outputs', exist_ok=True)\n", - "shutil.copy('parameters.json', os.path.join('outputs', 'parameters.json'))\n", - " \n", - "run.complete()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azureml.train.estimator import Estimator\n", - "from azureml.core import Experiment\n", - "\n", - "# Create an estimator\n", - "estimator = Estimator(source_directory=training_folder,\n", - " entry_script='diabetes_training.py',\n", - " compute_target='local',\n", - " conda_packages=['scikit-learn']\n", - " )\n", - "\n", - "# Create an experiment\n", - "experiment_name = 'diabetes-training'\n", - "experiment = Experiment(workspace = ws, name = experiment_name)\n", - "\n", - "# Run the experiment based on the estimator\n", - "run = experiment.submit(config=estimator)\n", - "run.wait_for_completion(show_output=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "metrics = run.get_metrics()\n", - "for k, v in metrics.items():\n", - " print(k, v)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for file in run.get_file_names():\n", - " print(file)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.6.10 64-bit ('OH3': conda)", - "language": "python", - "name": "python361064bitoh3conda5f7beeba8c1d407187c86667ecfb684f" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.10" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/experimentation/Diabetes Ridge Regression Scoring.ipynb b/experimentation/Diabetes Ridge Regression Scoring.ipynb deleted file mode 100644 index 9ac340ed..00000000 --- a/experimentation/Diabetes Ridge Regression Scoring.ipynb +++ /dev/null @@ -1,114 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Score Data with a Ridge Regression Model Trained on the Diabetes Dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This notebook loads the model trained in the Diabetes Ridge Regression Training notebook, prepares the data, and scores the data." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "import numpy\n", - "from azureml.core.model import Model\n", - "import joblib" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Load Model" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "model_path = Model.get_model_path(model_name=\"sklearn_regression_model.pkl\")\n", - "model = joblib.load(model_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Prepare Data" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "raw_data = '{\"data\":[[1,2,3,4,5,6,7,8,9,10],[10,9,8,7,6,5,4,3,2,1]]}'\n", - "\n", - "data = json.loads(raw_data)[\"data\"]\n", - "data = numpy.array(data)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Score Data" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Test result: {'result': [5113.099642122813, 3713.6329271385353]}\n" - ] - } - ], - "source": [ - "request_headers = {}\n", - "\n", - "result = model.predict(data)\n", - "print(\"Test result: \", {\"result\": result.tolist()})" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python (storedna)", - "language": "python", - "name": "storedna" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.9" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/experimentation/Diabetes Ridge Regression Training.ipynb b/experimentation/Diabetes Ridge Regression Training.ipynb deleted file mode 100644 index fa192115..00000000 --- a/experimentation/Diabetes Ridge Regression Training.ipynb +++ /dev/null @@ -1,401 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Train a Ridge Regression Model on the Diabetes Dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This notebook loads the Diabetes dataset from sklearn, splits the data into training and validation sets, trains a Ridge regression model, validates the model on the validation set, and saves the model." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from sklearn.datasets import load_diabetes\n", - "from sklearn.linear_model import Ridge\n", - "from sklearn.metrics import mean_squared_error\n", - "from sklearn.model_selection import train_test_split\n", - "import joblib\n", - "import pandas as pd" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Load Data" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "sample_data = load_diabetes()\n", - "\n", - "df = pd.DataFrame(\n", - " data=sample_data.data,\n", - " columns=sample_data.feature_names)\n", - "df['Y'] = sample_data.target" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(442, 10)\n" - ] - } - ], - "source": [ - "print(df.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
agesexbmibps1s2s3s4s5s6Y
count4.420000e+024.420000e+024.420000e+024.420000e+024.420000e+024.420000e+024.420000e+024.420000e+024.420000e+024.420000e+02442.000000
mean-3.634285e-161.308343e-16-8.045349e-161.281655e-16-8.835316e-171.327024e-16-4.574646e-163.777301e-16-3.830854e-16-3.412882e-16152.133484
std4.761905e-024.761905e-024.761905e-024.761905e-024.761905e-024.761905e-024.761905e-024.761905e-024.761905e-024.761905e-0277.093005
min-1.072256e-01-4.464164e-02-9.027530e-02-1.123996e-01-1.267807e-01-1.156131e-01-1.023071e-01-7.639450e-02-1.260974e-01-1.377672e-0125.000000
25%-3.729927e-02-4.464164e-02-3.422907e-02-3.665645e-02-3.424784e-02-3.035840e-02-3.511716e-02-3.949338e-02-3.324879e-02-3.317903e-0287.000000
50%5.383060e-03-4.464164e-02-7.283766e-03-5.670611e-03-4.320866e-03-3.819065e-03-6.584468e-03-2.592262e-03-1.947634e-03-1.077698e-03140.500000
75%3.807591e-025.068012e-023.124802e-023.564384e-022.835801e-022.984439e-022.931150e-023.430886e-023.243323e-022.791705e-02211.500000
max1.107267e-015.068012e-021.705552e-011.320442e-011.539137e-011.987880e-011.811791e-011.852344e-011.335990e-011.356118e-01346.000000
\n", - "
" - ], - "text/plain": [ - " age sex bmi bp s1 \\\n", - "count 4.420000e+02 4.420000e+02 4.420000e+02 4.420000e+02 4.420000e+02 \n", - "mean -3.634285e-16 1.308343e-16 -8.045349e-16 1.281655e-16 -8.835316e-17 \n", - "std 4.761905e-02 4.761905e-02 4.761905e-02 4.761905e-02 4.761905e-02 \n", - "min -1.072256e-01 -4.464164e-02 -9.027530e-02 -1.123996e-01 -1.267807e-01 \n", - "25% -3.729927e-02 -4.464164e-02 -3.422907e-02 -3.665645e-02 -3.424784e-02 \n", - "50% 5.383060e-03 -4.464164e-02 -7.283766e-03 -5.670611e-03 -4.320866e-03 \n", - "75% 3.807591e-02 5.068012e-02 3.124802e-02 3.564384e-02 2.835801e-02 \n", - "max 1.107267e-01 5.068012e-02 1.705552e-01 1.320442e-01 1.539137e-01 \n", - "\n", - " s2 s3 s4 s5 s6 \\\n", - "count 4.420000e+02 4.420000e+02 4.420000e+02 4.420000e+02 4.420000e+02 \n", - "mean 1.327024e-16 -4.574646e-16 3.777301e-16 -3.830854e-16 -3.412882e-16 \n", - "std 4.761905e-02 4.761905e-02 4.761905e-02 4.761905e-02 4.761905e-02 \n", - "min -1.156131e-01 -1.023071e-01 -7.639450e-02 -1.260974e-01 -1.377672e-01 \n", - "25% -3.035840e-02 -3.511716e-02 -3.949338e-02 -3.324879e-02 -3.317903e-02 \n", - "50% -3.819065e-03 -6.584468e-03 -2.592262e-03 -1.947634e-03 -1.077698e-03 \n", - "75% 2.984439e-02 2.931150e-02 3.430886e-02 3.243323e-02 2.791705e-02 \n", - "max 1.987880e-01 1.811791e-01 1.852344e-01 1.335990e-01 1.356118e-01 \n", - "\n", - " Y \n", - "count 442.000000 \n", - "mean 152.133484 \n", - "std 77.093005 \n", - "min 25.000000 \n", - "25% 87.000000 \n", - "50% 140.500000 \n", - "75% 211.500000 \n", - "max 346.000000 " - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# All data in a single dataframe\n", - "df.describe()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Split Data into Training and Validation Sets" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "X = df.drop('Y', axis=1).values\n", - "y = df['Y'].values\n", - "\n", - "X_train, X_test, y_train, y_test = train_test_split(\n", - " X, y, test_size=0.2, random_state=0)\n", - "data = {\"train\": {\"X\": X_train, \"y\": y_train},\n", - " \"test\": {\"X\": X_test, \"y\": y_test}}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Train Model on Training Set" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Ridge(alpha=0.5, copy_X=True, fit_intercept=True, max_iter=None,\n", - " normalize=False, random_state=None, solver='auto', tol=0.001)" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# experiment parameters\n", - "args = {\n", - " \"alpha\": 0.5\n", - "}\n", - "\n", - "reg_model = Ridge(**args)\n", - "reg_model.fit(data[\"train\"][\"X\"], data[\"train\"][\"y\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Validate Model on Validation Set" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'mse': 3298.9096058070622}\n" - ] - } - ], - "source": [ - "preds = reg_model.predict(data[\"test\"][\"X\"])\n", - "mse = mean_squared_error(preds, y_test)\n", - "metrics = {\"mse\": mse}\n", - "print(metrics)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Save Model" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['sklearn_regression_model.pkl']" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model_name = \"sklearn_regression_model.pkl\"\n", - "\n", - "joblib.dump(value=reg, filename=model_name)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/ml_service/pipelines/diabetes_regression_build_parallel_batchscore_pipeline.py b/ml_service/pipelines/automobile_build_parallel_batchscore_pipeline.py similarity index 99% rename from ml_service/pipelines/diabetes_regression_build_parallel_batchscore_pipeline.py rename to ml_service/pipelines/automobile_build_parallel_batchscore_pipeline.py index 5a0f0125..5a602d77 100644 --- a/ml_service/pipelines/diabetes_regression_build_parallel_batchscore_pipeline.py +++ b/ml_service/pipelines/automobile_build_parallel_batchscore_pipeline.py @@ -124,7 +124,7 @@ def get_input_dataset(ws: Workspace, ds: Datastore, env: Env) -> Dataset: def get_fallback_input_dataset(ws: Workspace, env: Env) -> Dataset: """ Called when an input datastore does not exist or no input data file exists - at that location. Create a sample dataset using the diabetes dataset from + at that location. Create a sample dataset using the automobile dataset from scikit-learn. Useful when debugging this code in the absence of the input data location Azure blob. diff --git a/ml_service/pipelines/diabetes_regression_build_train_pipeline.py b/ml_service/pipelines/automobile_build_train_pipeline.py similarity index 98% rename from ml_service/pipelines/diabetes_regression_build_train_pipeline.py rename to ml_service/pipelines/automobile_build_train_pipeline.py index 03937186..3d5c3c0f 100644 --- a/ml_service/pipelines/diabetes_regression_build_train_pipeline.py +++ b/ml_service/pipelines/automobile_build_train_pipeline.py @@ -65,7 +65,7 @@ def main(): create_sample_data_csv() # Use a CSV to read in the data set. - file_name = "diabetes.csv" + file_name = "automobile.csv" if not os.path.exists(file_name): raise Exception( @@ -91,7 +91,7 @@ def main(): dataset = dataset.register( workspace=aml_workspace, name=dataset_name, - description="diabetes training data", + description="automobile training data", tags={"format": "CSV"}, create_new_version=True, ) diff --git a/ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py b/ml_service/pipelines/automobile_build_train_pipeline_with_r.py similarity index 94% rename from ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py rename to ml_service/pipelines/automobile_build_train_pipeline_with_r.py index 254f22eb..4a5ad86a 100644 --- a/ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py +++ b/ml_service/pipelines/automobile_build_train_pipeline_with_r.py @@ -26,7 +26,7 @@ def main(): # Create a reusable Azure ML environment # Make sure to include `r-essentials' - # in diabetes_regression/conda_dependencies.yml + # in automobile/conda_dependencies.yml environment = get_environment( aml_workspace, e.aml_env_name, @@ -40,7 +40,7 @@ def main(): name="Train Model", script_name="train_with_r.py", compute_target=aml_compute, - source_directory="diabetes_regression/training/R", + source_directory="automobile/training/R", runconfig=run_config, allow_reuse=False, ) diff --git a/ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py b/ml_service/pipelines/automobile_build_train_pipeline_with_r_on_dbricks.py similarity index 96% rename from ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py rename to ml_service/pipelines/automobile_build_train_pipeline_with_r_on_dbricks.py index ae607b3b..d1c0920e 100644 --- a/ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py +++ b/ml_service/pipelines/automobile_build_train_pipeline_with_r_on_dbricks.py @@ -29,7 +29,7 @@ def main(): name="DBPythonInLocalMachine", num_workers=1, python_script_name="train_with_r_on_databricks.py", - source_directory="diabetes_regression/training/R", + source_directory="automobile/training/R", run_name='DB_Python_R_demo', existing_cluster_id=e.db_cluster_id, compute_target=aml_compute, diff --git a/ml_service/util/create_scoring_image.py b/ml_service/util/create_scoring_image.py index 378cb3b4..c7a51664 100644 --- a/ml_service/util/create_scoring_image.py +++ b/ml_service/util/create_scoring_image.py @@ -27,7 +27,7 @@ model = Model(ws, name=e.model_name, version=e.model_version) sources_dir = e.sources_directory_train if (sources_dir is None): - sources_dir = 'diabetes_regression' + sources_dir = 'automobile' score_script = os.path.join(".", sources_dir, e.score_script) score_file = os.path.basename(score_script) path_to_scoring = os.path.dirname(score_script)