Auto merge of #6201 - glennw:jquery-runner, r=metajack

<!-- Reviewable:start -->
[<img src="https://reviewable.io/review_button.png" height=40 alt="Review on Reviewable"/>](https://reviewable.io/reviews/servo/servo/6201)
<!-- Reviewable:end -->

6 files changed, 356 insertions, 38 deletions

diff --git a/python/servo/command_base.py b/python/servo/command_base.py
index 185b716165d..80811e30a04 100644
--- a/python/servo/command_base.py
+++ b/python/servo/command_base.py
@@ -121,6 +121,44 @@ class CommandBase(object):
             self._cargo_build_id = open(filename).read().strip()
         return self._cargo_build_id
 
+    def get_binary_path(self, release, dev):
+        base_path = path.join("components", "servo", "target")
+        release_path = path.join(base_path, "release", "servo")
+        dev_path = path.join(base_path, "debug", "servo")
+
+        # Prefer release if both given
+        if release and dev:
+            dev = False
+
+        release_exists = path.exists(release_path)
+        dev_exists = path.exists(dev_path)
+
+        if not release_exists and not dev_exists:
+            print("No Servo binary found. Please run './mach build' and try again.")
+            sys.exit()
+
+        if release and release_exists:
+            return release_path
+
+        if dev and dev_exists:
+            return dev_path
+
+        if not dev and not release and release_exists and dev_exists:
+            print("You have multiple profiles built. Please specify which "
+                  "one to run with '--release' or '--dev'.")
+            sys.exit()
+
+        if not dev and not release:
+            if release_exists:
+                return release_path
+            else:
+                return dev_path
+
+        print("The %s profile is not built. Please run './mach build%s' "
+              "and try again." % ("release" if release else "dev",
+                                   " --release" if release else ""))
+        sys.exit()
+
     def build_env(self, gonk=False, hosts_file_path=None):
         """Return an extended environment dictionary."""
         env = os.environ.copy()
diff --git a/python/servo/post_build_commands.py b/python/servo/post_build_commands.py
index 0dae99e0a87..f6c4c015ba1 100644
--- a/python/servo/post_build_commands.py
+++ b/python/servo/post_build_commands.py
@@ -27,44 +27,6 @@ def read_file(filename, if_exists=False):
 @CommandProvider
 class MachCommands(CommandBase):
 
-    def get_binary_path(self, release, dev):
-        base_path = path.join("components", "servo", "target")
-        release_path = path.join(base_path, "release", "servo")
-        dev_path = path.join(base_path, "debug", "servo")
-
-        # Prefer release if both given
-        if release and dev:
-            dev = False
-
-        release_exists = path.exists(release_path)
-        dev_exists = path.exists(dev_path)
-
-        if not release_exists and not dev_exists:
-            print("No Servo binary found. Please run './mach build' and try again.")
-            sys.exit()
-
-        if release and release_exists:
-            return release_path
-
-        if dev and dev_exists:
-            return dev_path
-
-        if not dev and not release and release_exists and dev_exists:
-            print("You have multiple profiles built. Please specify which "
-                  "one to run with '--release' or '--dev'.")
-            sys.exit()
-
-        if not dev and not release:
-            if release_exists:
-                return release_path
-            else:
-                return dev_path
-
-        print("The %s profile is not built. Please run './mach build%s' "
-              "and try again." % ("release" if release else "dev",
-                                   " --release" if release else ""))
-        sys.exit()
-
     @Command('run',
              description='Run Servo',
              category='post-build')
diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py
index 81361b0081f..44220022db4 100644
--- a/python/servo/testing_commands.py
+++ b/python/servo/testing_commands.py
@@ -225,6 +225,26 @@ class MachCommands(CommandBase):
         execfile(run_file, run_globals)
         return run_globals["update_tests"](**kwargs)
 
+    @Command('test-jquery',
+             description='Run the jQuery test suite',
+             category='testing')
+    @CommandArgument('--release', '-r', action='store_true',
+                     help='Run the release build')
+    @CommandArgument('--dev', '-d', action='store_true',
+                     help='Run the dev build')
+    def test_jquery(self, release, dev):
+        return self.jquery_test_runner("test", release, dev)
+
+    @Command('update-jquery',
+             description='Update the jQuery test suite expected results',
+             category='testing')
+    @CommandArgument('--release', '-r', action='store_true',
+                     help='Run the release build')
+    @CommandArgument('--dev', '-d', action='store_true',
+                     help='Run the dev build')
+    def update_jquery(self):
+        return self.jquery_test_runner("update", release, dev)
+
     @Command('test-css',
              description='Run the web platform tests',
              category='testing',
@@ -291,3 +311,24 @@ class MachCommands(CommandBase):
             return default
 
         return path
+
+    def jquery_test_runner(self, cmd, release, dev):
+        self.ensure_bootstrapped()
+        base_dir = path.abspath(path.join("tests", "jquery"))
+        jquery_dir = path.join(base_dir, "jquery")
+        run_file = path.join(base_dir, "run_jquery.py")
+
+        # Clone the jQuery repository if it doesn't exist
+        if not os.path.isdir(jquery_dir):
+            subprocess.check_call(
+                ["git", "clone", "-b", "servo", "--depth", "1", "https://github.com/servo/jquery", jquery_dir])
+
+        # Run pull in case the jQuery repo was updated since last test run
+        subprocess.check_call(
+            ["git", "-C", jquery_dir, "pull"])
+
+        # Check that a release servo build exists
+        bin_path = path.abspath(self.get_binary_path(release, dev))
+
+        return subprocess.check_call(
+            [run_file, cmd, bin_path, base_dir])
diff --git a/tests/jquery/.gitignore b/tests/jquery/.gitignore
new file mode 100644
index 00000000000..fc1787458a2
--- /dev/null
+++ b/tests/jquery/.gitignore
@@ -0,0 +1 @@
+jquery/
diff --git a/tests/jquery/expected_selector.txt b/tests/jquery/expected_selector.txt
new file mode 100644
index 00000000000..eddb132445f
--- /dev/null
+++ b/tests/jquery/expected_selector.txt
@@ -0,0 +1,14 @@
+[jQuery test] [15/0/15] jQuery.uniqueSort
+[jQuery test] [0/1/1] Iframe dispatch should not affect jQuery (#13936)
+[jQuery test] [4/0/4] class - jQuery only
+[jQuery test] [0/2/2] attributes - jQuery.attr
+[jQuery test] [5/0/5] name
+[jQuery test] [4/0/4] selectors with comma
+[jQuery test] [16/0/16] jQuery.contains
+[jQuery test] [27/0/27] child and adjacent
+[jQuery test] [7/0/7] element - jQuery only
+[jQuery test] [0/2/2] Sizzle cache collides with multiple Sizzles on a page
+[jQuery test] [1/0/1] disconnected nodes
+[jQuery test] [50/4/54] attributes
+[jQuery test] [3/0/3] disconnected nodes - jQuery only
+[jQuery test] [26/0/26] id
diff --git a/tests/jquery/run_jquery.py b/tests/jquery/run_jquery.py
new file mode 100755
index 00000000000..ac5398a12d1
--- /dev/null
+++ b/tests/jquery/run_jquery.py
@@ -0,0 +1,262 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import re
+import subprocess
+import sys
+import BaseHTTPServer
+import SimpleHTTPServer
+import SocketServer
+import threading
+import urlparse
+
+# List of jQuery modules that will be tested.
+# TODO(gw): Disabled most of them as something has been
+# introduced very recently that causes the resource task
+# to panic - and hard fail doesn't exit the servo
+# process when this happens.
+# See https://github.com/servo/servo/issues/6210 and
+#     https://github.com/servo/servo/issues/6211
+JQUERY_MODULES = [
+    #"ajax",            # panics
+    #"attributes",
+    #"callbacks",
+    #"core",            # mozjs crash
+    #"css",
+    #"data",
+    #"deferred",
+    #"dimensions",
+    #"effects",
+    #"event",           # panics
+    #"manipulation",    # mozjs crash
+    #"offset",
+    #"queue",
+    "selector",
+    #"serialize",
+    #"support",
+    #"traversing",
+    #"wrap"
+]
+
+# Port to run the HTTP server on for jQuery.
+TEST_SERVER_PORT = 8192
+
+# A regex for matching console.log output lines from the test runner.
+REGEX_PATTERN = "^\[jQuery test\] \[([0-9]+)/([0-9]+)/([0-9]+)] (.*)"
+
+# The result of a single test group.
+class TestResult:
+    def __init__(self, success, fail, total, text):
+        self.success = int(success)
+        self.fail = int(fail)
+        self.total = int(total)
+        self.text = text
+
+    def __key(self):
+        return (self.success, self.fail, self.total, self.text)
+
+    def __eq__(self, other):
+        return self.__key() == other.__key()
+
+    def __ne__(self, other):
+        return self.__key() != other.__key()
+
+    def __hash__(self):
+        return hash(self.__key())
+
+    def __repr__(self):
+        return "ok={0} fail={1} total={2}".format(self.success, self.fail, self.total)
+
+
+# Parse a line, producing a TestResult.
+# Throws if unable to parse.
+def parse_line_to_result(line):
+    match = re.match(REGEX_PATTERN, line)
+    success, fail, total, name = match.groups()
+    return name, TestResult(success, fail, total, line)
+
+
+# Parse an entire buffer of lines to a dictionary
+# of test results, keyed by the test name.
+def parse_string_to_results(buffer):
+    test_results = {}
+    lines = buffer.splitlines()
+    for line in lines:
+        name, test_result = parse_line_to_result(line)
+        test_results[name] = test_result
+    return test_results
+
+
+# Run servo and print / parse the results for a specific jQuery test module.
+def run_servo(servo_exe, module):
+    url = "http://localhost:{0}/jquery/test/?module={1}".format(TEST_SERVER_PORT, module)
+    args = [ servo_exe, url, "-z", "-f" ]
+    proc = subprocess.Popen(args, stdout=subprocess.PIPE)
+    while True:
+        line = proc.stdout.readline()
+        if len(line) == 0:
+            break
+        line = line.rstrip()
+        try:
+            name, test_result = parse_line_to_result(line)
+            yield name, test_result
+        except AttributeError:
+            pass
+
+# Build the filename for an expected results file.
+def module_filename(module):
+    return 'expected_{0}.txt'.format(module)
+
+# Read an existing set of expected results to compare against.
+def read_existing_results(module):
+    with open(module_filename(module), 'r') as file:
+        buffer = file.read()
+        return parse_string_to_results(buffer)
+
+# Write a set of results to file
+def write_results(module, results):
+    with open(module_filename(module), 'w') as file:
+        for result in test_results.itervalues():
+            file.write(result.text + '\n')
+
+# Print usage if command line args are incorrect
+def print_usage():
+    print("USAGE: {0} servo_binary jquery_base_dir test|update".format(sys.argv[0]))
+
+# Run a simple HTTP server to serve up the jQuery test suite
+def run_http_server():
+    class ThreadingSimpleServer(SocketServer.ThreadingMixIn,
+                                BaseHTTPServer.HTTPServer):
+        allow_reuse_address = True
+
+    class RequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
+        # TODO(gw): HACK copy the fixed version from python
+        # main repo - due to https://bugs.python.org/issue23112
+        def send_head(self):
+            path = self.translate_path(self.path)
+            f = None
+            if os.path.isdir(path):
+                parts = urlparse.urlsplit(self.path)
+                if not parts.path.endswith('/'):
+                    # redirect browser - doing basically what apache does
+                    self.send_response(301)
+                    new_parts = (parts[0], parts[1], parts[2] + '/',
+                                 parts[3], parts[4])
+                    new_url = urlparse.urlunsplit(new_parts)
+                    self.send_header("Location", new_url)
+                    self.end_headers()
+                    return None
+                for index in "index.html", "index.htm":
+                    index = os.path.join(path, index)
+                    if os.path.exists(index):
+                        path = index
+                        break
+                else:
+                    return self.list_directory(path)
+            ctype = self.guess_type(path)
+            try:
+                # Always read in binary mode. Opening files in text mode may cause
+                # newline translations, making the actual size of the content
+                # transmitted *less* than the content-length!
+                f = open(path, 'rb')
+            except IOError:
+                self.send_error(404, "File not found")
+                return None
+            try:
+                self.send_response(200)
+                self.send_header("Content-type", ctype)
+                fs = os.fstat(f.fileno())
+                self.send_header("Content-Length", str(fs[6]))
+                self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
+                self.end_headers()
+                return f
+            except:
+                f.close()
+                raise
+
+        def log_message(self, format, *args):
+            return
+
+    server = ThreadingSimpleServer(('', TEST_SERVER_PORT), RequestHandler)
+    while True:
+        sys.stdout.flush()
+        server.handle_request()
+
+if __name__ == '__main__':
+    if len(sys.argv) == 4:
+        cmd = sys.argv[1]
+        servo_exe = sys.argv[2]
+        base_dir = sys.argv[3]
+        os.chdir(base_dir)
+
+        # Ensure servo binary can be found
+        if not os.path.isfile(servo_exe):
+            print("Unable to find {0}. This script expects an existing build of Servo.".format(servo_exe))
+            sys.exit(1)
+
+        # Start the test server
+        httpd_thread = threading.Thread(target=run_http_server)
+        httpd_thread.setDaemon(True)
+        httpd_thread.start()
+
+        if cmd == "test":
+            print("Testing jQuery on Servo!")
+            test_count = 0
+            unexpected_count = 0
+
+            individual_success = 0
+            individual_total = 0
+
+            # Test each module separately
+            for module in JQUERY_MODULES:
+                print("\t{0}".format(module))
+
+                prev_test_results = read_existing_results(module)
+                for name, current_result in run_servo(servo_exe, module):
+                    test_count += 1
+                    individual_success += current_result.success
+                    individual_total += current_result.total
+
+                    # If this test was in the previous results, compare them.
+                    if name in prev_test_results:
+                        prev_result = prev_test_results[name]
+                        if prev_result == current_result:
+                            print("\t\tOK: {0}".format(name))
+                        else:
+                            unexpected_count += 1
+                            print("\t\tFAIL: {0}: WAS {1} NOW {2}".format(name, prev_result, current_result))
+                        del prev_test_results[name]
+                    else:
+                        # There was a new test that wasn't expected
+                        unexpected_count += 1
+                        print("\t\tNEW: {0}".format(current_result.text))
+
+                # Check what's left over, these are tests that were expected but didn't run this time.
+                for name in prev_test_results:
+                    test_count += 1
+                    unexpected_count += 1
+                    print("\t\tMISSING: {0}".format(prev_test_results[name].text))
+
+            print("\tRan {0} test groups. {1} unexpected results.".format(test_count, unexpected_count))
+            print("\t{0} tests succeeded of {1} ({2:.2f}%)".format(individual_success,
+                                                                   individual_total,
+                                                                   100.0 * individual_success / individual_total))
+            if unexpected_count > 0:
+                sys.exit(1)
+        elif cmd == "update":
+            print("Updating jQuery expected results")
+            for module in JQUERY_MODULES:
+                print("\t{0}".format(module))
+                test_results = {}
+                for name, test_result in run_servo(servo_exe, module):
+                    print("\t\t{0} {1}".format(name, test_result))
+                    test_results[name] = test_result
+                write_results(module, test_results)
+        else:
+            print_usage()
+    else:
+        print_usage()