Auto merge of #24303 - servo:script-codegen, r=nox

WebIDL codegen: Replace cmake with a single Python script

When [playing around with Cargo’s new timing visualization](https://internals.rust-lang.org/t/exploring-crate-graph-build-times-with-cargo-build-ztimings/10975/21), I was surprised to see the `script` crate’s build script take 76 seconds. I did not expect WebIDL bindings generation to be *that* computationally intensive.

It turns out almost all of this time is overhead. The build script uses CMake to generate bindings for each WebIDL file in parallel, but that causes a lot of work to be repeated 366 times:

* Starting up a Python VM
* Importing (parts of) the Python standard library
* Importing ~16k lines of our Python code
* Recompiling the latter to bytecode, since we used `python -B` to disable writing `.pyc` files
* Deserializing with `cPickle` and recreating in memory the results   of parsing all WebIDL files

----

This commit remove the use of CMake and cPickle for the `script` crate. Instead, all WebIDL bindings generation is done sequentially in a single Python process. This takes 2 to 3 seconds.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/24303)
<!-- Reviewable:end -->

5 files changed, 119 insertions, 284 deletions

diff --git a/components/script/dom/bindings/codegen/BindingGen.py b/components/script/dom/bindings/codegen/BindingGen.py
deleted file mode 100644
index 63cc68e46e5..00000000000
--- a/components/script/dom/bindings/codegen/BindingGen.py
+++ /dev/null
@@ -1,54 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-import sys
-import os
-sys.path.append(os.path.join(".", "parser"))
-sys.path.append(os.path.join(".", "ply"))
-import cPickle
-from Configuration import Configuration
-from CodegenRust import CGBindingRoot, replaceFileIfChanged
-
-
-def generate_binding_rs(config, outputprefix, webidlfile):
-    """
-    |config| Is the configuration object.
-    |outputprefix| is a prefix to use for the header guards and filename.
-    """
-
-    filename = outputprefix + ".rs"
-    module = CGBindingRoot(config, outputprefix, webidlfile).define()
-    if not module:
-        print "Skipping empty module: %s" % (filename)
-    elif replaceFileIfChanged(filename, module):
-        print "Generating binding implementation: %s" % (filename)
-
-
-def main():
-    # Parse arguments.
-    from optparse import OptionParser
-    usagestring = "usage: %prog configFile outputdir outputPrefix webIDLFile"
-    o = OptionParser(usage=usagestring)
-    (options, args) = o.parse_args()
-
-    if len(args) != 4:
-        o.error(usagestring)
-    configFile = os.path.normpath(args[0])
-    outputdir = args[1]
-    outputPrefix = args[2]
-    webIDLFile = os.path.normpath(args[3])
-
-    # Load the parsing results
-    resultsPath = os.path.join(outputdir, 'ParserResults.pkl')
-    with open(resultsPath, 'rb') as f:
-        parserData = cPickle.load(f)
-
-    # Create the configuration data.
-    config = Configuration(configFile, parserData)
-
-    # Generate the prototype classes.
-    generate_binding_rs(config, outputPrefix, webIDLFile)
-
-if __name__ == '__main__':
-    main()
diff --git a/components/script/dom/bindings/codegen/CodegenRust.py b/components/script/dom/bindings/codegen/CodegenRust.py
index 3cd4078814b..c02499f222f 100644
--- a/components/script/dom/bindings/codegen/CodegenRust.py
+++ b/components/script/dom/bindings/codegen/CodegenRust.py
@@ -54,32 +54,6 @@ RUST_KEYWORDS = {"abstract", "alignof", "as", "become", "box", "break", "const",
                  "use", "virtual", "where", "while", "yield"}
 
 
-def replaceFileIfChanged(filename, newContents):
-    """
-    Read a copy of the old file, so that we don't touch it if it hasn't changed.
-    Returns True if the file was updated, false otherwise.
-    """
-    # XXXjdm This doesn't play well with make right now.
-    #       Force the file to always be updated, or else changing CodegenRust.py
-    #       will cause many autogenerated bindings to be regenerated perpetually
-    #       until the result is actually different.
-
-    # oldFileContents = ""
-    # try:
-    #     with open(filename, 'rb') as oldFile:
-    #         oldFileContents = ''.join(oldFile.readlines())
-    # except:
-    #     pass
-
-    # if newContents == oldFileContents:
-    #     return False
-
-    with open(filename, 'wb') as f:
-        f.write(newContents)
-
-    return True
-
-
 def toStringBool(arg):
     return str(not not arg).lower()
 
diff --git a/components/script/dom/bindings/codegen/GlobalGen.py b/components/script/dom/bindings/codegen/GlobalGen.py
deleted file mode 100644
index 1850a41d5f3..00000000000
--- a/components/script/dom/bindings/codegen/GlobalGen.py
+++ /dev/null
@@ -1,143 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-# We do one global pass over all the WebIDL to generate our prototype enum
-# and generate information for subsequent phases.
-
-import sys
-import os
-import json
-sys.path.append(os.path.join(".", "parser"))
-sys.path.append(os.path.join(".", "ply"))
-import WebIDL
-import cPickle
-from Configuration import Configuration
-from CodegenRust import GlobalGenRoots, replaceFileIfChanged
-
-
-def generate_file(config, name, filename):
-    root = getattr(GlobalGenRoots, name)(config)
-    code = root.define()
-
-    if replaceFileIfChanged(filename, code):
-        print "Generating %s" % (filename)
-    else:
-        print "%s hasn't changed - not touching it" % (filename)
-
-
-def main():
-    # Parse arguments.
-    from optparse import OptionParser
-    usageString = "usage: %prog [options] configFile outputdir webidldir cssProperties.json docServoDir [files]"
-    o = OptionParser(usage=usageString)
-    o.add_option("--cachedir", dest='cachedir', default=None,
-                 help="Directory in which to cache lex/parse tables.")
-    o.add_option("--filelist", dest='filelist', default=None,
-                 help="A file containing the list (one per line) of webidl files to process.")
-    (options, args) = o.parse_args()
-
-    if len(args) < 2:
-        o.error(usageString)
-
-    configFile = args[0]
-    outputdir = args[1]
-    baseDir = args[2]
-    css_properties_json = args[3]
-    doc_servo = args[4]
-    if options.filelist is not None:
-        fileList = [l.strip() for l in open(options.filelist).xreadlines()]
-    else:
-        fileList = args[3:]
-
-    # Parse the WebIDL.
-    parser = WebIDL.Parser(options.cachedir)
-    for filename in fileList:
-        fullPath = os.path.normpath(os.path.join(baseDir, filename))
-        with open(fullPath, 'rb') as f:
-            lines = f.readlines()
-        parser.parse(''.join(lines), fullPath)
-
-    add_css_properties_attributes(fileList, css_properties_json, parser)
-
-    parserResults = parser.finish()
-
-    # Write the parser results out to a pickle.
-    resultsPath = os.path.join(outputdir, 'ParserResults.pkl')
-    with open(resultsPath, 'wb') as resultsFile:
-        cPickle.dump(parserResults, resultsFile, -1)
-
-    # Load the configuration.
-    config = Configuration(configFile, parserResults)
-
-    to_generate = [
-        ('PrototypeList', 'PrototypeList.rs'),
-        ('RegisterBindings', 'RegisterBindings.rs'),
-        ('InterfaceObjectMap', 'InterfaceObjectMap.rs'),
-        ('InterfaceObjectMapData', 'InterfaceObjectMapData.json'),
-        ('InterfaceTypes', 'InterfaceTypes.rs'),
-        ('InheritTypes', 'InheritTypes.rs'),
-        ('Bindings', os.path.join('Bindings', 'mod.rs')),
-        ('UnionTypes', 'UnionTypes.rs'),
-    ]
-
-    for name, filename in to_generate:
-        generate_file(config, name, os.path.join(outputdir, filename))
-
-    generate_file(config, 'SupportedDomApis', os.path.join(doc_servo, 'apis.html'))
-
-
-def add_css_properties_attributes(webidl_files, css_properties_json, parser):
-    for filename in webidl_files:
-        if os.path.basename(filename) == "CSSStyleDeclaration.webidl":
-            break
-    else:
-        return
-
-    css_properties = json.load(open(css_properties_json, "rb"))
-    idl = "partial interface CSSStyleDeclaration {\n%s\n};\n" % "\n".join(
-        "  [%sCEReactions, SetterThrows] attribute [TreatNullAs=EmptyString] DOMString %s;" % (
-            ('Pref="%s", ' % data["pref"] if data["pref"] else ""),
-            attribute_name
-        )
-        for (kind, properties_list) in sorted(css_properties.items())
-        for (property_name, data) in sorted(properties_list.items())
-        for attribute_name in attribute_names(property_name)
-    )
-    parser.parse(idl.encode("utf-8"), "CSSStyleDeclaration_generated.webidl")
-
-
-def attribute_names(property_name):
-    # https://drafts.csswg.org/cssom/#dom-cssstyledeclaration-dashed-attribute
-    if property_name != "float":
-        yield property_name
-    else:
-        yield "_float"
-
-    # https://drafts.csswg.org/cssom/#dom-cssstyledeclaration-camel-cased-attribute
-    if "-" in property_name:
-        yield "".join(camel_case(property_name))
-
-    # https://drafts.csswg.org/cssom/#dom-cssstyledeclaration-webkit-cased-attribute
-    if property_name.startswith("-webkit-"):
-        yield "".join(camel_case(property_name), True)
-
-
-# https://drafts.csswg.org/cssom/#css-property-to-idl-attribute
-def camel_case(chars, webkit_prefixed=False):
-    if webkit_prefixed:
-        chars = chars[1:]
-    next_is_uppercase = False
-    for c in chars:
-        if c == '-':
-            next_is_uppercase = True
-        elif next_is_uppercase:
-            next_is_uppercase = False
-            # Should be ASCII-uppercase, but all non-custom CSS property names are within ASCII
-            yield c.upper()
-        else:
-            yield c
-
-
-if __name__ == '__main__':
-    main()
diff --git a/components/script/dom/bindings/codegen/pythonpath.py b/components/script/dom/bindings/codegen/pythonpath.py
deleted file mode 100644
index 67739c3625f..00000000000
--- a/components/script/dom/bindings/codegen/pythonpath.py
+++ /dev/null
@@ -1,61 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-"""
-Run a python script, adding extra directories to the python path.
-"""
-
-
-def main(args):
-    def usage():
-        print >>sys.stderr, "pythonpath.py -I directory script.py [args...]"
-        sys.exit(150)
-
-    paths = []
-
-    while True:
-        try:
-            arg = args[0]
-        except IndexError:
-            usage()
-
-        if arg == '-I':
-            args.pop(0)
-            try:
-                path = args.pop(0)
-            except IndexError:
-                usage()
-
-            paths.append(os.path.abspath(path))
-            continue
-
-        if arg.startswith('-I'):
-            paths.append(os.path.abspath(args.pop(0)[2:]))
-            continue
-
-        if arg.startswith('-D'):
-            os.chdir(args.pop(0)[2:])
-            continue
-
-        break
-
-    script = args[0]
-
-    sys.path[0:0] = [os.path.abspath(os.path.dirname(script))] + paths
-    sys.argv = args
-    sys.argc = len(args)
-
-    frozenglobals['__name__'] = '__main__'
-    frozenglobals['__file__'] = script
-
-    execfile(script, frozenglobals)
-
-# Freeze scope here ... why this makes things work I have no idea ...
-frozenglobals = globals()
-
-import sys
-import os
-
-if __name__ == '__main__':
-    main(sys.argv[1:])
diff --git a/components/script/dom/bindings/codegen/run.py b/components/script/dom/bindings/codegen/run.py
new file mode 100644
index 00000000000..130d35e5268
--- /dev/null
+++ b/components/script/dom/bindings/codegen/run.py
@@ -0,0 +1,119 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+import json
+
+
+def main():
+    os.chdir(os.path.join(os.path.dirname(__file__)))
+    sys.path[0:0] = ["./parser", "./ply"]
+
+    css_properties_json, out_dir = sys.argv[1:]
+    doc_servo = "../../../../../target/doc/servo"
+    webidls_dir = "../../webidls"
+    config_file = "Bindings.conf"
+
+    import WebIDL
+    from Configuration import Configuration
+    from CodegenRust import CGBindingRoot
+
+    parser = WebIDL.Parser(make_dir(os.path.join(out_dir, "cache")))
+    webidls = [name for name in os.listdir(webidls_dir) if name.endswith(".webidl")]
+    for webidl in webidls:
+        filename = os.path.join(webidls_dir, webidl)
+        with open(filename, "rb") as f:
+            parser.parse(f.read(), filename)
+
+    add_css_properties_attributes(css_properties_json, parser)
+    parser_results = parser.finish()
+    config = Configuration(config_file, parser_results)
+    make_dir(os.path.join(out_dir, "Bindings"))
+
+    for name, filename in [
+        ("PrototypeList", "PrototypeList.rs"),
+        ("RegisterBindings", "RegisterBindings.rs"),
+        ("InterfaceObjectMap", "InterfaceObjectMap.rs"),
+        ("InterfaceObjectMapData", "InterfaceObjectMapData.json"),
+        ("InterfaceTypes", "InterfaceTypes.rs"),
+        ("InheritTypes", "InheritTypes.rs"),
+        ("Bindings", "Bindings/mod.rs"),
+        ("UnionTypes", "UnionTypes.rs"),
+    ]:
+        generate(config, name, os.path.join(out_dir, filename))
+    make_dir(doc_servo)
+    generate(config, "SupportedDomApis", os.path.join(doc_servo, "apis.html"))
+
+    for webidl in webidls:
+        filename = os.path.join(webidls_dir, webidl)
+        prefix = "Bindings/%sBinding" % webidl[:-len(".webidl")]
+        module = CGBindingRoot(config, prefix, filename).define()
+        if module:
+            with open(os.path.join(out_dir, prefix + ".rs"), "wb") as f:
+                f.write(module)
+
+
+def make_dir(path):
+    if not os.path.exists(path):
+        os.makedirs(path)
+    return path
+
+
+def generate(config, name, filename):
+    from CodegenRust import GlobalGenRoots
+    root = getattr(GlobalGenRoots, name)(config)
+    code = root.define()
+    with open(filename, "wb") as f:
+        f.write(code)
+
+
+def add_css_properties_attributes(css_properties_json, parser):
+    css_properties = json.load(open(css_properties_json, "rb"))
+    idl = "partial interface CSSStyleDeclaration {\n%s\n};\n" % "\n".join(
+        "  [%sCEReactions, SetterThrows] attribute [TreatNullAs=EmptyString] DOMString %s;" % (
+            ('Pref="%s", ' % data["pref"] if data["pref"] else ""),
+            attribute_name
+        )
+        for (kind, properties_list) in sorted(css_properties.items())
+        for (property_name, data) in sorted(properties_list.items())
+        for attribute_name in attribute_names(property_name)
+    )
+    parser.parse(idl.encode("utf-8"), "CSSStyleDeclaration_generated.webidl")
+
+
+def attribute_names(property_name):
+    # https://drafts.csswg.org/cssom/#dom-cssstyledeclaration-dashed-attribute
+    if property_name != "float":
+        yield property_name
+    else:
+        yield "_float"
+
+    # https://drafts.csswg.org/cssom/#dom-cssstyledeclaration-camel-cased-attribute
+    if "-" in property_name:
+        yield "".join(camel_case(property_name))
+
+    # https://drafts.csswg.org/cssom/#dom-cssstyledeclaration-webkit-cased-attribute
+    if property_name.startswith("-webkit-"):
+        yield "".join(camel_case(property_name), True)
+
+
+# https://drafts.csswg.org/cssom/#css-property-to-idl-attribute
+def camel_case(chars, webkit_prefixed=False):
+    if webkit_prefixed:
+        chars = chars[1:]
+    next_is_uppercase = False
+    for c in chars:
+        if c == '-':
+            next_is_uppercase = True
+        elif next_is_uppercase:
+            next_is_uppercase = False
+            # Should be ASCII-uppercase, but all non-custom CSS property names are within ASCII
+            yield c.upper()
+        else:
+            yield c
+
+
+if __name__ == "__main__":
+    main()