diff --git a/README.md b/README.md index 28653eb..6b4cc2a 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,14 @@ -# mode_python_sql -This script allows users to download a CSV of raw SQL from queries in a report. +# Mode Python SQL +Dumps the queries from one or more reports to the local file system as SQL files. +Forked from https://github.com/mode/mode_python_sql # Steps to use this script: 1. In Mode, generate API token (under Settings -> Your Name -> API Tokens). 2. Add the token and password values to the mode.yml file. +3. Add the name of your organisation to the mode.yml file. +4. Add each report you want to retrieve queries from to the mode.yml file. 3. Run this script using: - `python demo.py -org={{organization_username}} -reporttoken={{report_token}}` - -For example, for this report https://modeanalytics.com/modeanalytics/reports/eb7e7c23e72f I would run: - -`python demo.py -org=modeanalytics -reporttoken=eb7e7c23e72f` - + `python demo.py [-o output_directory]` diff --git a/demo.py b/demo.py index 012adba..08d47d4 100644 --- a/demo.py +++ b/demo.py @@ -1,66 +1,97 @@ +import json, requests, datetime, ConfigParser, argparse, sys, csv, yaml, os -import json, requests, datetime, ConfigParser, argparse, sys, csv, yaml from requests.auth import HTTPBasicAuth -def get_auth(whichAuth): - with open ('mode.yml', 'r') as f: - mode = yaml.load(f) +############# +# Constants # +############# - token = mode["mode"]["token"] - password = mode["mode"]["password"] - auth = (token, password) - return auth +MODE_URL = 'https://modeanalytics.com' +CONFIG_FILE = 'mode.yml' -def get_response_json(url, auth): - response = requests.get(url, auth=auth) - return response.json() - -def get_mode_results(): - - if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('-org', '--org') - parser.add_argument('-reporttoken', '--reporttoken') - args = parser.parse_args() - - mode_url = 'https://modeanalytics.com' - if(args.org is not None and args.reporttoken is not None): - api_url = '/api/' + args.org + '/reports/' + args.reporttoken - else: - sys.exit('We did not get your -org or -reporttoken parameters.') - - auth = get_auth('mode') - - url = mode_url + api_url - print "API URL: " + url +#################### +# Helper Functions # +#################### - data = get_response_json(url, auth) +def get_api_url(organisation, report_token): + return '/api/' + organisation + '/reports/' + report_token - links = data['_links'] - last_run = links['last_successful_run'] - run_url = last_run['href'] - url = mode_url + run_url + '/query_runs/' - - data = get_response_json(url, auth) - - embedded = data['_embedded'] - query_runs = embedded['query_runs'] - - file = open('sql.csv', 'w') - writer = csv.writer(file, lineterminator='\n') - writer.writerow(["Query Token", "SQL Query"]) - - for x in query_runs: - token = x['query_token'] - raw = x['raw_source'] - print "Query Token: " + token - print raw - print "*************************************" - writer.writerow([token, raw]) - - file.close() - -get_mode_results() +def get_auth(whichAuth, token, password): + auth = (token, password) + return auth +def get_response_json(url, auth): + response = requests.get(url, auth=auth) + return response.json() + + +################# +# Main Function # +################# + +def main(args): + # Read command line arguments + output_directory = args.output_dir + + # Read config file + with open(CONFIG_FILE, 'r') as f: + config = yaml.load(f) + org_name = config['organisation']['name'] + reports = config['organisation']['reports'] + token = config['mode']['token'] + password = config['mode']['password'] + + report_directory = os.path.join(output_directory, 'mode_reports') + if not os.path.isdir(report_directory): + os.makedirs(report_directory) + + if not reports: + print("No reports listed in {}, exiting".format( + CONFIG_FILE + )) + for reporttoken in reports: + # Assemble request information + auth = get_auth('mode', token, password) + api_url = get_api_url(org_name, reporttoken) + url = MODE_URL + api_url + print("Dumping queries from report @ {}".format(url)) + + # Retrieve and parse data + data = get_response_json(url, auth) + links = data['_links'] + last_run = links['last_successful_run'] + run_url = last_run['href'] + url = MODE_URL + run_url + '/query_runs/' + + data = get_response_json(url, auth) + embedded = data['_embedded'] + query_runs = embedded['query_runs'] + + # Write data to output directory + query_directory = os.path.join(report_directory, reporttoken, 'queries') + if not os.path.isdir(query_directory): + os.makedirs(query_directory) + + files_written = 0 + for query_run in query_runs: + output_file = os.path.join( + query_directory, + 'mode_query_{}.sql'.format(query_run['query_token']) + ) + with open(output_file, 'w') as output: + output.write(query_run['raw_source']) + files_written += 1 + + print("Wrote {} files to {}".format( + files_written, query_directory + )) + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-o', '--output-dir', default='./') + + args = parser.parse_args() + + main(args) diff --git a/mode.yml b/mode.yml index 588e8fc..dcb544c 100644 --- a/mode.yml +++ b/mode.yml @@ -1,3 +1,7 @@ mode: token: password: + +organisation: + name: + reports: []