From 5294b827959b41a863a42f26ac9e224d2456e0c7 Mon Sep 17 00:00:00 2001 From: hly001 <55067941+hly001@users.noreply.github.com> Date: Mon, 9 Sep 2019 02:45:42 +0000 Subject: [PATCH 1/2] Set up CI with Azure Pipelines [skip ci] --- environment_setup/iac-create-environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environment_setup/iac-create-environment.yml b/environment_setup/iac-create-environment.yml index 6c2495cb..4ca73d1e 100644 --- a/environment_setup/iac-create-environment.yml +++ b/environment_setup/iac-create-environment.yml @@ -18,8 +18,8 @@ steps: inputs: azureSubscription: 'AzureResourceConnection' action: 'Create Or Update Resource Group' - resourceGroupName: '$(BASE_NAME)-AML-RG' - location: $(LOCATION) + resourceGroupName: 'HLMLOps-AML-RG' + location: 'centralus' templateLocation: 'Linked artifact' csmFile: '$(Build.SourcesDirectory)/environment_setup/arm-templates/cloud-environment.json' overrideParameters: '-baseName $(BASE_NAME)' From 969a9bb539730b9c8aef80eaa9bfc7b1b54171a3 Mon Sep 17 00:00:00 2001 From: hly001 <55067941+hly001@users.noreply.github.com> Date: Mon, 9 Sep 2019 03:58:49 +0000 Subject: [PATCH 2/2] create run_train_pipeline.py Copy from train.py to make run_train_pipeline.py --- code/training/run_train_pipeline.py | 113 ++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 code/training/run_train_pipeline.py diff --git a/code/training/run_train_pipeline.py b/code/training/run_train_pipeline.py new file mode 100644 index 00000000..2b541615 --- /dev/null +++ b/code/training/run_train_pipeline.py @@ -0,0 +1,113 @@ +""" +Copyright (C) Microsoft Corporation. All rights reserved.​ + ​ +Microsoft Corporation (“Microsoft”) grants you a nonexclusive, perpetual, +royalty-free right to use, copy, and modify the software code provided by us +("Software Code"). You may not sublicense the Software Code or any use of it +(except to your affiliates and to vendors to perform work on your behalf) +through distribution, network access, service agreement, lease, rental, or +otherwise. This license does not purport to express any claim of ownership over +data you may have shared with Microsoft in the creation of the Software Code. +Unless applicable law gives you more rights, Microsoft reserves all other +rights not expressly granted herein, whether by implication, estoppel or +otherwise. ​ + ​ +THE SOFTWARE CODE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +MICROSOFT OR ITS LICENSORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +""" +from azureml.core.run import Run +import os +import argparse +from sklearn.datasets import load_diabetes +from sklearn.linear_model import Ridge +from sklearn.metrics import mean_squared_error +from sklearn.model_selection import train_test_split +from sklearn.externals import joblib +import numpy as np +import json + + +parser = argparse.ArgumentParser("train") +parser.add_argument( + "--config_suffix", type=str, help="Datetime suffix for json config files" +) +parser.add_argument( + "--json_config", + type=str, + help="Directory to write all the intermediate json configs", +) +parser.add_argument( + "--model_name", + type=str, + help="Name of the Model", + default="sklearn_regression_model.pkl", +) + +args = parser.parse_args() + +print("Argument 1: %s" % args.config_suffix) +print("Argument 2: %s" % args.json_config) + +model_name = args.model_name + +if not (args.json_config is None): + os.makedirs(args.json_config, exist_ok=True) + print("%s created" % args.json_config) + +run = Run.get_context() +exp = run.experiment +ws = run.experiment.workspace + +X, y = load_diabetes(return_X_y=True) +columns = ["age", "gender", "bmi", "bp", "s1", "s2", "s3", "s4", "s5", "s6"] +X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.2, random_state=0) +data = {"train": {"X": X_train, "y": y_train}, + "test": {"X": X_test, "y": y_test}} + +print("Running train.py") + +# Randomly pic alpha +alphas = np.arange(0.0, 1.0, 0.05) +alpha = alphas[np.random.choice(alphas.shape[0], 1, replace=False)][0] +print(alpha) +run.log("alpha", alpha) +reg = Ridge(alpha=alpha) +reg.fit(data["train"]["X"], data["train"]["y"]) +preds = reg.predict(data["test"]["X"]) +run.log("mse", mean_squared_error(preds, data["test"]["y"])) + + +# Save model as part of the run history + +# model_name = "." + +with open(model_name, "wb") as file: + joblib.dump(value=reg, filename=model_name) + +# upload the model file explicitly into artifacts +run.upload_file(name="./outputs/" + model_name, path_or_stream=model_name) +print("Uploaded the model {} to experiment {}".format( + model_name, run.experiment.name)) +dirpath = os.getcwd() +print(dirpath) +print("Following files are uploaded ") +print(run.get_file_names()) + +run_id = {} +run_id["run_id"] = run.id +run_id["experiment_name"] = run.experiment.name +filename = "run_id_{}.json".format(args.config_suffix) +output_path = os.path.join(args.json_config, filename) +with open(output_path, "w") as outfile: + json.dump(run_id, outfile) + +run.complete()