Add mach command to update public domain list and use a HashSet instead of a Vec to lookup public domains

1 files changed, 23 insertions, 0 deletions

diff --git a/python/servo/bootstrap_commands.py b/python/servo/bootstrap_commands.py
index 659826fea66..373dc4ebbf0 100644
--- a/python/servo/bootstrap_commands.py
+++ b/python/servo/bootstrap_commands.py
@@ -299,6 +299,29 @@ class MachCommands(CommandBase):
             print("Unable to parse chromium HSTS preload list, has the format changed?")
             sys.exit(1)
 
+    @Command('update-pub-domains',
+             description='Download the public domains list and update resources/public_domains.txt',
+             category='bootstrap')
+    def bootstrap_pub_suffix(self, force=False):
+        list_url = "https://publicsuffix.org/list/public_suffix_list.dat"
+        dst_filename = path.join(self.context.topdir, "resources", "public_domains.txt")
+        not_implemented_case = re.compile(r'^[^*]+\*')
+
+        try:
+            content = download_bytes("Public suffix list", list_url)
+        except urllib2.URLError:
+            print("Unable to download the public suffix list; are you connected to the internet?")
+            sys.exit(1)
+
+        lines = [l.strip() for l in content.decode("utf8").split("\n")]
+        suffixes = [l for l in lines if not l.startswith("//") and not l == ""]
+
+        with open(dst_filename, "wb") as fo:
+            for suffix in suffixes:
+                if not_implemented_case.match(suffix):
+                    print("Warning: the new list contains a case that servo can't handle: %s" % suffix)
+                fo.write(suffix.encode("idna") + "\n")
+
     @Command('clean-nightlies',
              description='Clean unused nightly builds of Rust and Cargo',
              category='bootstrap')