jcs.org
iOS web browser with a focus on security and privacy
1#!/usr/bin/ruby
2#
3# Endless
4# Copyright (c) 2014-2015 joshua stein <jcs@jcs.org>
5#
6# See LICENSE file for redistribution terms.
7#
8
9require "active_support/core_ext/hash/conversions"
10require "plist"
11require "json"
12require "net/https"
13require "uri"
14
15HTTPS_E_TARGETS_PLIST = "Endless/Resources/https-everywhere_targets.plist"
16HTTPS_E_RULES_PLIST = "Endless/Resources/https-everywhere_rules.plist"
17
18URLBLOCKER_JSON = "urlblocker.json"
19URLBLOCKER_TARGETS_PLIST = "Endless/Resources/urlblocker_targets.plist"
20
21# in b64 for some reason
22HSTS_PRELOAD_LIST = "https://chromium.googlesource.com/chromium/src/net/+/master/http/transport_security_state_static.json?format=TEXT"
23HSTS_PRELOAD_HOSTS_PLIST = "Endless/Resources/hsts_preload.plist"
24
25FORCE = (ARGV[0].to_s == "-f")
26
27# convert all HTTPS Everywhere XML rule files into one big rules hash and write
28# it out as a plist, as well as a standalone hash of target URLs -> rule names
29# to another plist
30def convert_https_e
31 https_e_git_commit = `cd https-everywhere && git show -s`.split("\n")[0].
32 gsub(/^commit /, "")[0, 12]
33
34 if File.exists?(HTTPS_E_TARGETS_PLIST)
35 if m = File.open(HTTPS_E_TARGETS_PLIST).gets.to_s.match(/Everywhere (.+) - /)
36 if (m[1] == https_e_git_commit) && !FORCE
37 return
38 end
39 end
40 end
41
42 rules = {}
43 targets = {}
44
45 Dir.glob(File.dirname(__FILE__) +
46 "/https-everywhere/src/chrome/content/rules/*.xml").each do |f|
47 hash = Hash.from_xml(File.read(f))
48
49 raise "no ruleset" if !hash["ruleset"]
50
51 if hash["ruleset"]["default_off"] ||
52 hash["ruleset"]["platform"] == "mixedcontent"
53 next # XXX: should we store these?
54 end
55
56 raise "conflict on #{f}" if rules[hash["ruleset"]["name"]]
57
58 # validate regexps
59 begin
60 r = hash["ruleset"]["rule"]
61 r = [ r ] if !r.is_a?(Array)
62 r.each do |h|
63 Regexp.compile(h["from"])
64 end
65
66 if r = hash["ruleset"]["securecookie"]
67 r = [ r ] if !r.is_a?(Array)
68 r.each do |h|
69 Regexp.compile(h["host"])
70 Regexp.compile(h["name"])
71 end
72 end
73 rescue => e
74 STDERR.puts "error in #{f}: #{e} (#{hash.inspect})"
75 exit 1
76 end
77
78 rules[hash["ruleset"]["name"]] = hash
79
80 hash["ruleset"]["target"].each do |target|
81 if !target.is_a?(Hash)
82 # why do some of these get converted into an array?
83 if target.length != 2 || target[0] != "host"
84 puts f
85 raise target.inspect
86 end
87
88 target = { target[0] => target[1] }
89 end
90
91 if targets[target["host"][1]]
92 raise "rules already exist for #{target["host"]}"
93 end
94
95 targets[target["host"]] = hash["ruleset"]["name"]
96 end
97 end
98
99 File.write(HTTPS_E_TARGETS_PLIST,
100 "<!-- generated from HTTPS Everywhere #{https_e_git_commit} - do not " +
101 "directly edit this file -->\n" +
102 targets.to_plist)
103
104 File.write(HTTPS_E_RULES_PLIST,
105 "<!-- generated from HTTPS Everywhere #{https_e_git_commit} - do not " +
106 "directly edit this file -->\n" +
107 rules.to_plist)
108end
109
110# convert JSON ruleset into a list of target domains and a list of rulesets
111# with information URLs
112def convert_urlblocker
113 targets = {}
114
115 JSON.parse(File.read(URLBLOCKER_JSON)).each do |company,domains|
116 domains.each do |dom|
117 targets[dom] = company
118 end
119 end
120
121 File.write(URLBLOCKER_TARGETS_PLIST,
122 "<!-- generated from #{URLBLOCKER_JSON} - do not directly edit this " +
123 "file -->\n" +
124 targets.to_plist)
125end
126
127def convert_hsts_preload
128 domains = {}
129
130 json = JSON.parse(Net::HTTP.get(URI(HSTS_PRELOAD_LIST)).unpack("m0").first)
131 json["entries"].each do |entry|
132 domains[entry["name"]] = {
133 "include_subdomains" => !!entry["include_subdomains"]
134 }
135 end
136
137 File.write(HSTS_PRELOAD_HOSTS_PLIST,
138 "<!-- generated from #{HSTS_PRELOAD_LIST} - do not directly edit this " +
139 "file -->\n" +
140 domains.to_plist)
141end
142
143convert_https_e
144convert_urlblocker
145convert_hsts_preload