This repository was archived by the owner on Apr 19, 2021. It is now read-only.
forked from EFForg/https-everywhere
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmake-sqlite.py
More file actions
executable file
·72 lines (59 loc) · 2.33 KB
/
Copy pathmake-sqlite.py
File metadata and controls
executable file
·72 lines (59 loc) · 2.33 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
#!/usr/bin/python2.7
#
# Builds an sqlite DB containing all the rulesets, indexed by target.
import subprocess
import sqlite3
import sys, re, os
from lxml import etree
def nomes_all(where=sys.argv[1:]):
"""Returns generator to extract all files from a list of files/dirs"""
if not where: where=['.']
for i in where:
if os.path.isfile(i):
yield i
elif os.path.isdir(i):
for r, d, f in os.walk(i):
for fi in f:
yield os.path.join(r, fi)
conn = sqlite3.connect(os.path.join(os.path.dirname(__file__), '../src/defaults/rulesets.sqlite'))
c = conn.cursor()
c.execute('''DROP TABLE IF EXISTS rulesets''')
c.execute('''CREATE TABLE rulesets
(id INTEGER PRIMARY KEY,
contents TEXT)''')
c.execute('''DROP TABLE IF EXISTS targets''')
c.execute('''CREATE TABLE targets
(host TEXT,
ruleset_id INTEGER)''')
c.execute('''DROP TABLE IF EXISTS git_commit''')
c.execute('''CREATE TABLE git_commit
(git_commit TEXT)''')
git_commit = subprocess.check_output("git rev-parse HEAD", shell=True).rstrip("\n")
c.execute('''INSERT INTO git_commit (git_commit) VALUES(?)''', (git_commit,))
parser = etree.XMLParser(remove_blank_text=True)
for fi in nomes_all():
try:
tree = etree.parse(fi, parser)
except Exception as oops:
if fi[-4:] != ".xml":
continue
print("%s failed XML validity: %s\n" % (fi, oops))
if not tree.xpath("/ruleset"):
continue
# Remove comments to save space.
etree.strip_tags(tree,etree.Comment)
targets = tree.xpath("/ruleset/target/@host")
# Strip out the target tags. These aren't necessary in the DB because
# targets are looked up in the target table, which has a foreign key
# pointing into the ruleset table.
etree.strip_tags(tree,'target')
# Store the filename in the `f' attribute so "view source XML" for rules in
# FF version can find it.
tree.xpath("/ruleset")[0].attrib["f"] = os.path.basename(fi).decode(encoding="UTF-8")
c.execute('''INSERT INTO rulesets (contents) VALUES(?)''', (etree.tostring(tree),));
ruleset_id = c.lastrowid
for target in targets:
c.execute('''INSERT INTO targets (host, ruleset_id) VALUES(?, ?)''', (target, ruleset_id));
conn.commit()
conn.execute("VACUUM")
conn.close()