diff options
Diffstat (limited to 'tests/jquery/run_jquery.py')
| -rwxr-xr-x | tests/jquery/run_jquery.py | 262 |
1 files changed, 262 insertions, 0 deletions
diff --git a/tests/jquery/run_jquery.py b/tests/jquery/run_jquery.py new file mode 100755 index 00000000000..ac5398a12d1 --- /dev/null +++ b/tests/jquery/run_jquery.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import re +import subprocess +import sys +import BaseHTTPServer +import SimpleHTTPServer +import SocketServer +import threading +import urlparse + +# List of jQuery modules that will be tested. +# TODO(gw): Disabled most of them as something has been +# introduced very recently that causes the resource task +# to panic - and hard fail doesn't exit the servo +# process when this happens. +# See https://github.com/servo/servo/issues/6210 and +# https://github.com/servo/servo/issues/6211 +JQUERY_MODULES = [ + #"ajax", # panics + #"attributes", + #"callbacks", + #"core", # mozjs crash + #"css", + #"data", + #"deferred", + #"dimensions", + #"effects", + #"event", # panics + #"manipulation", # mozjs crash + #"offset", + #"queue", + "selector", + #"serialize", + #"support", + #"traversing", + #"wrap" +] + +# Port to run the HTTP server on for jQuery. +TEST_SERVER_PORT = 8192 + +# A regex for matching console.log output lines from the test runner. +REGEX_PATTERN = "^\[jQuery test\] \[([0-9]+)/([0-9]+)/([0-9]+)] (.*)" + +# The result of a single test group. +class TestResult: + def __init__(self, success, fail, total, text): + self.success = int(success) + self.fail = int(fail) + self.total = int(total) + self.text = text + + def __key(self): + return (self.success, self.fail, self.total, self.text) + + def __eq__(self, other): + return self.__key() == other.__key() + + def __ne__(self, other): + return self.__key() != other.__key() + + def __hash__(self): + return hash(self.__key()) + + def __repr__(self): + return "ok={0} fail={1} total={2}".format(self.success, self.fail, self.total) + + +# Parse a line, producing a TestResult. +# Throws if unable to parse. +def parse_line_to_result(line): + match = re.match(REGEX_PATTERN, line) + success, fail, total, name = match.groups() + return name, TestResult(success, fail, total, line) + + +# Parse an entire buffer of lines to a dictionary +# of test results, keyed by the test name. +def parse_string_to_results(buffer): + test_results = {} + lines = buffer.splitlines() + for line in lines: + name, test_result = parse_line_to_result(line) + test_results[name] = test_result + return test_results + + +# Run servo and print / parse the results for a specific jQuery test module. +def run_servo(servo_exe, module): + url = "http://localhost:{0}/jquery/test/?module={1}".format(TEST_SERVER_PORT, module) + args = [ servo_exe, url, "-z", "-f" ] + proc = subprocess.Popen(args, stdout=subprocess.PIPE) + while True: + line = proc.stdout.readline() + if len(line) == 0: + break + line = line.rstrip() + try: + name, test_result = parse_line_to_result(line) + yield name, test_result + except AttributeError: + pass + +# Build the filename for an expected results file. +def module_filename(module): + return 'expected_{0}.txt'.format(module) + +# Read an existing set of expected results to compare against. +def read_existing_results(module): + with open(module_filename(module), 'r') as file: + buffer = file.read() + return parse_string_to_results(buffer) + +# Write a set of results to file +def write_results(module, results): + with open(module_filename(module), 'w') as file: + for result in test_results.itervalues(): + file.write(result.text + '\n') + +# Print usage if command line args are incorrect +def print_usage(): + print("USAGE: {0} servo_binary jquery_base_dir test|update".format(sys.argv[0])) + +# Run a simple HTTP server to serve up the jQuery test suite +def run_http_server(): + class ThreadingSimpleServer(SocketServer.ThreadingMixIn, + BaseHTTPServer.HTTPServer): + allow_reuse_address = True + + class RequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): + # TODO(gw): HACK copy the fixed version from python + # main repo - due to https://bugs.python.org/issue23112 + def send_head(self): + path = self.translate_path(self.path) + f = None + if os.path.isdir(path): + parts = urlparse.urlsplit(self.path) + if not parts.path.endswith('/'): + # redirect browser - doing basically what apache does + self.send_response(301) + new_parts = (parts[0], parts[1], parts[2] + '/', + parts[3], parts[4]) + new_url = urlparse.urlunsplit(new_parts) + self.send_header("Location", new_url) + self.end_headers() + return None + for index in "index.html", "index.htm": + index = os.path.join(path, index) + if os.path.exists(index): + path = index + break + else: + return self.list_directory(path) + ctype = self.guess_type(path) + try: + # Always read in binary mode. Opening files in text mode may cause + # newline translations, making the actual size of the content + # transmitted *less* than the content-length! + f = open(path, 'rb') + except IOError: + self.send_error(404, "File not found") + return None + try: + self.send_response(200) + self.send_header("Content-type", ctype) + fs = os.fstat(f.fileno()) + self.send_header("Content-Length", str(fs[6])) + self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) + self.end_headers() + return f + except: + f.close() + raise + + def log_message(self, format, *args): + return + + server = ThreadingSimpleServer(('', TEST_SERVER_PORT), RequestHandler) + while True: + sys.stdout.flush() + server.handle_request() + +if __name__ == '__main__': + if len(sys.argv) == 4: + cmd = sys.argv[1] + servo_exe = sys.argv[2] + base_dir = sys.argv[3] + os.chdir(base_dir) + + # Ensure servo binary can be found + if not os.path.isfile(servo_exe): + print("Unable to find {0}. This script expects an existing build of Servo.".format(servo_exe)) + sys.exit(1) + + # Start the test server + httpd_thread = threading.Thread(target=run_http_server) + httpd_thread.setDaemon(True) + httpd_thread.start() + + if cmd == "test": + print("Testing jQuery on Servo!") + test_count = 0 + unexpected_count = 0 + + individual_success = 0 + individual_total = 0 + + # Test each module separately + for module in JQUERY_MODULES: + print("\t{0}".format(module)) + + prev_test_results = read_existing_results(module) + for name, current_result in run_servo(servo_exe, module): + test_count += 1 + individual_success += current_result.success + individual_total += current_result.total + + # If this test was in the previous results, compare them. + if name in prev_test_results: + prev_result = prev_test_results[name] + if prev_result == current_result: + print("\t\tOK: {0}".format(name)) + else: + unexpected_count += 1 + print("\t\tFAIL: {0}: WAS {1} NOW {2}".format(name, prev_result, current_result)) + del prev_test_results[name] + else: + # There was a new test that wasn't expected + unexpected_count += 1 + print("\t\tNEW: {0}".format(current_result.text)) + + # Check what's left over, these are tests that were expected but didn't run this time. + for name in prev_test_results: + test_count += 1 + unexpected_count += 1 + print("\t\tMISSING: {0}".format(prev_test_results[name].text)) + + print("\tRan {0} test groups. {1} unexpected results.".format(test_count, unexpected_count)) + print("\t{0} tests succeeded of {1} ({2:.2f}%)".format(individual_success, + individual_total, + 100.0 * individual_success / individual_total)) + if unexpected_count > 0: + sys.exit(1) + elif cmd == "update": + print("Updating jQuery expected results") + for module in JQUERY_MODULES: + print("\t{0}".format(module)) + test_results = {} + for name, test_result in run_servo(servo_exe, module): + print("\t\t{0} {1}".format(name, test_result)) + test_results[name] = test_result + write_results(module, test_results) + else: + print_usage() + else: + print_usage() |