diff --git a/utils/alexa-ruleset-checker.py b/utils/alexa-ruleset-checker.py new file mode 100755 index 000000000000..de5374b1aa0b --- /dev/null +++ b/utils/alexa-ruleset-checker.py @@ -0,0 +1,178 @@ +#! /usr/bin/env python3.3 + +# Copyright 2014 Claudio Moretti +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# +# This little piece of software works by downloading the Alexa Top 1M website list, which freely available, +# then it uses `git diff` to generate a list of XML ruleset files that are in the master branch but not in stable. +# Finally, it compares the two and prints the file name and path of every ruleset file that +# a) is in master but not in stable and +# b) has a target in the Alexa Top1M list +# + +import sys +import csv +import xml.etree.ElementTree as etree +import subprocess +import random +import urllib.request +import urllib.error +import zipfile +import os +import time + +# Variables and constants +sitesList = [] + +# Temporary file containing the `git diff` between master and stable +tmpRulesFileName = "/tmp/rulesDiff-" + format(random.randrange(1,65535)) # Feel free to enlarge if needed + +# URL of the Alexa Top1M +# alexaTop1MURL = "http://s3.amazonaws.com/alexa-static/top-1m.csv.zip" +alexaTop1MURL = "http://127.0.0.1/top-1m.csv.zip" + +# Temporary file name, to aboid conflicts +tmpAlexaFileName = "/tmp/alexa-top1M-" + format(random.randrange(1,65535)) + ".csv" + +# Logfile. Records the same output as the script +logFileName = "/tmp/alexa-ruleset-log-" + format(random.randrange(1,65535)) + ".log" + +# Filename of the CSV file contained in the Alexa zipfile +tmpAlexaZipFileContents = 'top-1m.csv' + +# Absolute path of the git repo (the folder containing src/) +# Remember to change this accordingly to your system, if you ever move the script +# +# By default, it refers to the parent directory of the one containing the script +# because the script was put in utils/ +# +# __NEEDS A TRAILING SLASH__ +# +# gitRepositoryPath = os.path.abspath(os.path.join(os.curdir, os.pardir)) +gitRepositoryPath = os.path.abspath(os.path.join(os.curdir, os.pardir)) + "/" + +# Maximum number of websites to use in the Alexa Top 1M (i.e. it's no longer 1M but maxSitesNumber) +# Set to -1 for 'unlimited' +maxSitesNumber = 1000 + +# Functions +def ruleLookup(target): + try: # list.index(value) throus an exception for a "not found", so if it throws it, it's not found + sitesList.index(target) + return 1 + except: + return 0 + +# Fetch the Alexa Top 1M - http://stackoverflow.com/questions/1517616/stream-large-binary-files-with-urllib2-to-file +try: + print("Retrieving Alexa Top1M from", alexaTop1MURL) + tmpAlexaZipFileName, headers = urllib.request.urlretrieve(alexaTop1MURL) + print("File downloaded and stored in %s" % tmpAlexaZipFileName) +except urllib.error.URLError as e: + print("Failed to download Alexa Top 1M") + sys.exit('Error message: %s' % e) + +# Now unzip it +try: + # Extract in /tmp/ + print("Start extracting %s" % tmpAlexaZipFileName) + tmpAlexaZipFile = zipfile.ZipFile(tmpAlexaZipFileName,'r') + tmpAlexaZipFile.extractall('/tmp/') +except zipfile.BadZipfile: + sys.exit("The zip file %s is corrupted.",tmpAlexaZipFileName) + +try: + # Rename the file to match the file with the random in it + os.rename('/tmp/' + tmpAlexaZipFileContents,tmpAlexaFileName) + print("Alexa Top1M retrieved and stored in %s" % tmpAlexaFileName) +except OSError as e: + print("Failed to rename /tmp/top-1M.csv to %s." % (tmpAlexaFileName)) + sys.exit('Error message: %s' % (e)) + +# Handles reading the Alexa Top 1M and pushing all sites in a list +sitesReader = csv.reader(open(tmpAlexaFileName), delimiter=',', quotechar='"') +for row in sitesReader: + try: + # Since some Alexa sites are not FQDNs, split where there's a "/" and keep ony the first part + siteFQDN = sitesList.append(row[1].split("/",1)[0]) + # print("Line %s: %s" % (sitesReader.line_num, sitesList[len(sitesList) - 1])) # Outputs the current line + if sitesReader.line_num == maxSitesNumber: + break + except csv.Error as e: + sys.exit('file %s, line %d: %s' % (tmpAlexaFileName, sitesReader.line_num, e)) + +# `git diff` the master revision against stable, rules folder only +try: + print("Create git diff between master and stable in %s" % tmpRulesFileName) + tmpRulesFile = open(tmpRulesFileName,"w") + #subprocess.call(['git', 'diff', '--name-status', 'master..remotes/origin/stable', '../src/chrome/content/rules'], stdout=tmpRulesFile) + subprocess.call(['git', 'diff', '--name-status', 'remotes/origin/stable..master', '../src/chrome/content/rules'], stdout=tmpRulesFile) + tmpRulesFile.close() +except OSError as e: + sys.exit('An OSError exception was raised: %s' % (e)) + +rulesList = open(tmpRulesFileName, 'r') +logFile = open(logFileName,'w') +logFile.write("Log file generated on %s.\nPaths are relative to the root directory of the git repo.\n\n" % time.strftime("%Y-%m-%d %H:%M:%S")) + +# Let's keep track of how many rules were added and how many were modified +# Must be declared here or won't be available at the end of the loop +countAddedRules = 0 +countEditedRules = 0 + +# Start parsing the list +for line in rulesList: + try: + # Split into "file mode in commit + file path" + ruleFile = line.split() + found = 0 + # If file mode is "A" (add) or "M" (edited) + if ruleFile[0] == "A" or ruleFile[0] == "M": # If file was added or edited between stable and master, parse + ruleFileObject= open(gitRepositoryPath + ruleFile[1]) + ruleText = etree.parse(ruleFileObject) # ADJUST FILE PATH (here is '../') IF YOU MOVE THE SCRIPT - XXX: Obsolete warning? + for target in ruleText.findall('target'): + FQDN = target.get('host') # URL of the website + if ruleLookup(FQDN) == 1: # Look it up in the sitesList + # Message different according to file mode + if ruleFile[0] == "A": # New + found = "NEW" + countAddedRules = countAddedRules + 1 + break + elif ruleFile[0] == "M": # Edited + found = "EDITED" + countEditedRules = countEditedRules + 1 + break + + # If found, print it TABULATED + if found != 0: + print("%s:\t%s" % (found, ruleFile[1])) + logFile.write("%s:\t%s" % (found, ruleFile[1])) + + # else ignore + # There are some problems with file name encoding. So, for now, just print an error and pass + except FileNotFoundError as e: # Won't happen before line.split() is invoked + print("File not found:", ruleFile[1]) +# logFile.write ("File not found: %s\n" % ruleFile[1]) + logFile.write("%s\n" % e) + pass + except IOError as ioe: #Treated same as FileNotFoundError + print("File not found:", ruleFile[1]) +# logFile.write ("File not found: %s\n" % ruleFile[1]) + logFile.write("%s\n" % e) + pass + + + +# Print our simple statistics +print("\n\nStatistics:\nParsed rules: %s\nNewly added rules: %s\nEdited rules: %d" % (maxSitesNumber, countAddedRules, countEditedRules)) +logFile.write("\n\nStatistics:\nParsed rules: %s\nNewly added rules: %s\nEdited rules: %d" % (maxSitesNumber, countAddedRules, countEditedRules)) +print("\n\nLog file can be found at %s" % logFileName) + +# Close the rules file +rulesList.close() +# And the log file +logFile.close()