diff options
Diffstat (limited to 'tests')
21 files changed, 724 insertions, 66 deletions
diff --git a/tests/wpt/harness/README.rst b/tests/wpt/harness/README.rst index 63f18aca613..fc650eec45a 100644 --- a/tests/wpt/harness/README.rst +++ b/tests/wpt/harness/README.rst @@ -231,7 +231,7 @@ The web-platform-test harness knows about several keys: Any value indicates that the test is disabled. `type` - The test type e.g. `testharness` or `reftest`. + The test type e.g. `testharness`, `reftest`, or `wdspec`. `reftype` The type of comparison for reftests; either `==` or `!=`. diff --git a/tests/wpt/harness/docs/expectation.rst b/tests/wpt/harness/docs/expectation.rst index 76aeb49c0dc..6a0c77684a3 100644 --- a/tests/wpt/harness/docs/expectation.rst +++ b/tests/wpt/harness/docs/expectation.rst @@ -203,6 +203,10 @@ When used for expectation data, manifests have the following format: the (sub)test is disabled and should either not be run (for tests) or that its results should be ignored (subtests). + * A key ``restart-after`` which can be set to any value to indicate that + the runner should restart the browser after running this test (e.g. to + clear out unwanted state). + * Variables ``debug``, ``os``, ``version``, ``processor`` and ``bits`` that describe the configuration of the browser under test. ``debug`` is a boolean indicating whether a build is a debug diff --git a/tests/wpt/harness/wptrunner/browsers/chrome.py b/tests/wpt/harness/wptrunner/browsers/chrome.py index a711fddd0b4..18491359456 100644 --- a/tests/wpt/harness/wptrunner/browsers/chrome.py +++ b/tests/wpt/harness/wptrunner/browsers/chrome.py @@ -3,7 +3,7 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. from .base import Browser, ExecutorBrowser, require_arg -from .webdriver import ChromedriverLocalServer +from ..webdriver_server import ChromeDriverServer from ..executors import executor_kwargs as base_executor_kwargs from ..executors.executorselenium import (SeleniumTestharnessExecutor, SeleniumRefTestExecutor) @@ -49,32 +49,33 @@ def env_options(): class ChromeBrowser(Browser): """Chrome is backed by chromedriver, which is supplied through - ``browsers.webdriver.ChromedriverLocalServer``.""" + ``wptrunner.webdriver.ChromeDriverServer``. + """ def __init__(self, logger, binary, webdriver_binary="chromedriver"): """Creates a new representation of Chrome. The `binary` argument gives the browser binary to use for testing.""" Browser.__init__(self, logger) self.binary = binary - self.driver = ChromedriverLocalServer(self.logger, binary=webdriver_binary) + self.server = ChromeDriverServer(self.logger, binary=webdriver_binary) def start(self): - self.driver.start() + self.server.start(block=False) def stop(self): - self.driver.stop() + self.server.stop() def pid(self): - return self.driver.pid + return self.server.pid def is_alive(self): # TODO(ato): This only indicates the driver is alive, # and doesn't say anything about whether a browser session # is active. - return self.driver.is_alive() + return self.server.is_alive() def cleanup(self): self.stop() def executor_browser(self): - return ExecutorBrowser, {"webdriver_url": self.driver.url} + return ExecutorBrowser, {"webdriver_url": self.server.url} diff --git a/tests/wpt/harness/wptrunner/browsers/firefox.py b/tests/wpt/harness/wptrunner/browsers/firefox.py index 92d70764903..dcd7a4f1d17 100644 --- a/tests/wpt/harness/wptrunner/browsers/firefox.py +++ b/tests/wpt/harness/wptrunner/browsers/firefox.py @@ -13,18 +13,27 @@ from mozprofile.permissions import ServerLocations from mozrunner import FirefoxRunner from mozcrash import mozcrash -from .base import get_free_port, Browser, ExecutorBrowser, require_arg, cmd_arg, browser_command +from .base import (get_free_port, + Browser, + ExecutorBrowser, + require_arg, + cmd_arg, + browser_command) from ..executors import executor_kwargs as base_executor_kwargs -from ..executors.executormarionette import MarionetteTestharnessExecutor, MarionetteRefTestExecutor +from ..executors.executormarionette import (MarionetteTestharnessExecutor, + MarionetteRefTestExecutor, + MarionetteWdspecExecutor) from ..environment import hostnames + here = os.path.join(os.path.split(__file__)[0]) __wptrunner__ = {"product": "firefox", "check_args": "check_args", "browser": "FirefoxBrowser", "executor": {"testharness": "MarionetteTestharnessExecutor", - "reftest": "MarionetteRefTestExecutor"}, + "reftest": "MarionetteRefTestExecutor", + "wdspec": "MarionetteWdspecExecutor"}, "browser_kwargs": "browser_kwargs", "executor_kwargs": "executor_kwargs", "env_options": "env_options", @@ -62,6 +71,8 @@ def executor_kwargs(test_type, server_config, cache_manager, run_info_data, executor_kwargs["timeout_multiplier"] = 2 elif run_info_data["debug"] or run_info_data.get("asan"): executor_kwargs["timeout_multiplier"] = 3 + if test_type == "wdspec": + executor_kwargs["webdriver_binary"] = kwargs.get("webdriver_binary") return executor_kwargs @@ -80,6 +91,7 @@ def run_info_extras(**kwargs): def update_properties(): return ["debug", "e10s", "os", "version", "processor", "bits"], {"debug", "e10s"} + class FirefoxBrowser(Browser): used_ports = set() init_timeout = 60 diff --git a/tests/wpt/harness/wptrunner/environment.py b/tests/wpt/harness/wptrunner/environment.py index b3d0e301fd3..732b785589b 100644 --- a/tests/wpt/harness/wptrunner/environment.py +++ b/tests/wpt/harness/wptrunner/environment.py @@ -32,7 +32,6 @@ def do_delayed_imports(logger, test_paths): global serve, sslutils serve_root = serve_path(test_paths) - sys.path.insert(0, serve_root) failed = [] @@ -45,7 +44,6 @@ def do_delayed_imports(logger, test_paths): try: import sslutils except ImportError: - raise failed.append("sslutils") if failed: diff --git a/tests/wpt/harness/wptrunner/executors/base.py b/tests/wpt/harness/wptrunner/executors/base.py index 0c8c2a1bc50..f0ce1665888 100644 --- a/tests/wpt/harness/wptrunner/executors/base.py +++ b/tests/wpt/harness/wptrunner/executors/base.py @@ -63,6 +63,7 @@ class TestharnessResultConverter(object): [test.subtest_result_cls(name, self.test_codes[status], message, stack) for name, status, message, stack in subtest_results]) + testharness_result_converter = TestharnessResultConverter() @@ -71,11 +72,24 @@ def reftest_result_converter(self, test, result): extra=result.get("extra")), []) +def pytest_result_converter(self, test, data): + harness_data, subtest_data = data + + if subtest_data is None: + subtest_data = [] + + harness_result = test.result_cls(*harness_data) + subtest_results = [test.subtest_result_cls(*item) for item in subtest_data] + + return (harness_result, subtest_results) + + class ExecutorException(Exception): def __init__(self, status, message): self.status = status self.message = message + class TestExecutor(object): __metaclass__ = ABCMeta @@ -116,11 +130,13 @@ class TestExecutor(object): :param runner: TestRunner instance that is going to run the tests""" self.runner = runner - self.protocol.setup(runner) + if self.protocol is not None: + self.protocol.setup(runner) def teardown(self): """Run cleanup steps after tests have finished""" - self.protocol.teardown() + if self.protocol is not None: + self.protocol.teardown() def run_test(self, test): """Run a particular test. @@ -137,6 +153,7 @@ class TestExecutor(object): if result is Stop: return result + # log result of parent test if result[0].status == "ERROR": self.logger.debug(result[0].message) @@ -144,7 +161,6 @@ class TestExecutor(object): self.runner.send_message("test_ended", test, result) - def server_url(self, protocol): return "%s://%s:%s" % (protocol, self.server_config["host"], @@ -191,6 +207,7 @@ class RefTestExecutor(TestExecutor): self.screenshot_cache = screenshot_cache + class RefTestImplementation(object): def __init__(self, executor): self.timeout_multiplier = executor.timeout_multiplier @@ -288,6 +305,11 @@ class RefTestImplementation(object): self.screenshot_cache[key] = hash_val, data return True, data + +class WdspecExecutor(TestExecutor): + convert_result = pytest_result_converter + + class Protocol(object): def __init__(self, executor, browser): self.executor = executor diff --git a/tests/wpt/harness/wptrunner/executors/executormarionette.py b/tests/wpt/harness/wptrunner/executors/executormarionette.py index c50bcf87f87..c4b1cba689a 100644 --- a/tests/wpt/harness/wptrunner/executors/executormarionette.py +++ b/tests/wpt/harness/wptrunner/executors/executormarionette.py @@ -3,9 +3,9 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. import hashlib +import httplib import os import socket -import sys import threading import time import traceback @@ -13,10 +13,15 @@ import urlparse import uuid from collections import defaultdict +from ..wpttest import WdspecResult, WdspecSubtestResult + +errors = None marionette = None +webdriver = None here = os.path.join(os.path.split(__file__)[0]) +from . import pytestrunner from .base import (ExecutorException, Protocol, RefTestExecutor, @@ -25,22 +30,29 @@ from .base import (ExecutorException, TestharnessExecutor, testharness_result_converter, reftest_result_converter, - strip_server) + strip_server, + WdspecExecutor) from ..testrunner import Stop +from ..webdriver_server import GeckoDriverServer # Extra timeout to use after internal test timeout at which the harness # should force a timeout extra_timeout = 5 # seconds + def do_delayed_imports(): - global marionette - global errors + global errors, marionette, webdriver + + # Marionette client used to be called marionette, recently it changed + # to marionette_driver for unfathomable reasons try: import marionette from marionette import errors except ImportError: from marionette_driver import marionette, errors + import webdriver + class MarionetteProtocol(Protocol): def __init__(self, executor, browser): @@ -54,8 +66,10 @@ class MarionetteProtocol(Protocol): """Connect to browser via Marionette.""" Protocol.setup(self, runner) - self.logger.debug("Connecting to marionette on port %i" % self.marionette_port) - self.marionette = marionette.Marionette(host='localhost', port=self.marionette_port) + self.logger.debug("Connecting to Marionette on port %i" % self.marionette_port) + self.marionette = marionette.Marionette(host='localhost', + port=self.marionette_port, + socket_timeout=None) # XXX Move this timeout somewhere self.logger.debug("Waiting for Marionette connection") @@ -97,10 +111,10 @@ class MarionetteProtocol(Protocol): pass del self.marionette + @property def is_alive(self): - """Check if the marionette connection is still active""" + """Check if the Marionette connection is still active.""" try: - # Get a simple property over the connection self.marionette.current_window_handle except Exception: return False @@ -126,12 +140,18 @@ class MarionetteProtocol(Protocol): "document.title = '%s'" % threading.current_thread().name.replace("'", '"')) def wait(self): + socket_timeout = self.marionette.client.sock.gettimeout() + if socket_timeout: + self.marionette.set_script_timeout((socket_timeout / 2) * 1000) + while True: try: - self.marionette.execute_async_script(""); + self.marionette.execute_async_script("") except errors.ScriptTimeoutException: + self.logger.debug("Script timed out") pass except (socket.timeout, IOError): + self.logger.debug("Socket closed") break except Exception as e: self.logger.error(traceback.format_exc(e)) @@ -213,7 +233,63 @@ class MarionetteProtocol(Protocol): with self.marionette.using_context(self.marionette.CONTEXT_CHROME): self.marionette.execute_script(script) -class MarionetteRun(object): + +class RemoteMarionetteProtocol(Protocol): + def __init__(self, executor, browser): + do_delayed_imports() + Protocol.__init__(self, executor, browser) + self.session = None + self.webdriver_binary = executor.webdriver_binary + self.marionette_port = browser.marionette_port + self.server = None + + def setup(self, runner): + """Connect to browser via the Marionette HTTP server.""" + try: + self.server = GeckoDriverServer( + self.logger, self.marionette_port, binary=self.webdriver_binary) + self.server.start(block=False) + self.logger.info( + "WebDriver HTTP server listening at %s" % self.server.url) + + self.logger.info( + "Establishing new WebDriver session with %s" % self.server.url) + self.session = webdriver.Session( + self.server.host, self.server.port, self.server.base_path) + except Exception: + self.logger.error(traceback.format_exc()) + self.executor.runner.send_message("init_failed") + else: + self.executor.runner.send_message("init_succeeded") + + def teardown(self): + try: + if self.session.session_id is not None: + self.session.end() + except Exception: + pass + if self.server is not None and self.server.is_alive: + self.server.stop() + + @property + def is_alive(self): + """Test that the Marionette connection is still alive. + + Because the remote communication happens over HTTP we need to + make an explicit request to the remote. It is allowed for + WebDriver spec tests to not have a WebDriver session, since this + may be what is tested. + + An HTTP request to an invalid path that results in a 404 is + proof enough to us that the server is alive and kicking. + """ + conn = httplib.HTTPConnection(self.server.host, self.server.port) + conn.request("HEAD", self.server.base_path + "invalid") + res = conn.getresponse() + return res.status == 404 + + +class ExecuteAsyncScriptRun(object): def __init__(self, logger, func, marionette, url, timeout): self.logger = logger self.result = None @@ -277,8 +353,8 @@ class MarionetteRun(object): class MarionetteTestharnessExecutor(TestharnessExecutor): - def __init__(self, browser, server_config, timeout_multiplier=1, close_after_done=True, - debug_info=None): + def __init__(self, browser, server_config, timeout_multiplier=1, + close_after_done=True, debug_info=None, **kwargs): """Marionette-based executor for testharness.js tests""" TestharnessExecutor.__init__(self, browser, server_config, timeout_multiplier=timeout_multiplier, @@ -295,7 +371,7 @@ class MarionetteTestharnessExecutor(TestharnessExecutor): do_delayed_imports() def is_alive(self): - return self.protocol.is_alive() + return self.protocol.is_alive def on_environment_change(self, new_environment): self.protocol.on_environment_change(self.last_environment, new_environment) @@ -307,11 +383,11 @@ class MarionetteTestharnessExecutor(TestharnessExecutor): timeout = (test.timeout * self.timeout_multiplier if self.debug_info is None else None) - success, data = MarionetteRun(self.logger, - self.do_testharness, - self.protocol.marionette, - self.test_url(test), - timeout).run() + success, data = ExecuteAsyncScriptRun(self.logger, + self.do_testharness, + self.protocol.marionette, + self.test_url(test), + timeout).run() if success: return self.convert_result(test, data) @@ -338,7 +414,9 @@ class MarionetteTestharnessExecutor(TestharnessExecutor): class MarionetteRefTestExecutor(RefTestExecutor): def __init__(self, browser, server_config, timeout_multiplier=1, - screenshot_cache=None, close_after_done=True, debug_info=None): + screenshot_cache=None, close_after_done=True, + debug_info=None, **kwargs): + """Marionette-based executor for reftests""" RefTestExecutor.__init__(self, browser, @@ -358,7 +436,7 @@ class MarionetteRefTestExecutor(RefTestExecutor): self.wait_script = f.read() def is_alive(self): - return self.protocol.is_alive() + return self.protocol.is_alive def on_environment_change(self, new_environment): self.protocol.on_environment_change(self.last_environment, new_environment) @@ -376,7 +454,6 @@ class MarionetteRefTestExecutor(RefTestExecutor): self.has_window = True result = self.implementation.run_test(test) - return self.convert_result(test, result) def screenshot(self, test, viewport_size, dpi): @@ -388,7 +465,7 @@ class MarionetteRefTestExecutor(RefTestExecutor): test_url = self.test_url(test) - return MarionetteRun(self.logger, + return ExecuteAsyncScriptRun(self.logger, self._screenshot, self.protocol.marionette, test_url, @@ -405,3 +482,78 @@ class MarionetteRefTestExecutor(RefTestExecutor): screenshot = screenshot.split(",", 1)[1] return screenshot + + +class WdspecRun(object): + def __init__(self, func, session, path, timeout): + self.func = func + self.result = None + self.session = session + self.path = path + self.timeout = timeout + self.result_flag = threading.Event() + + def run(self): + """Runs function in a thread and interrupts it if it exceeds the + given timeout. Returns (True, (Result, [SubtestResult ...])) in + case of success, or (False, (status, extra information)) in the + event of failure. + """ + + executor = threading.Thread(target=self._run) + executor.start() + + flag = self.result_flag.wait(self.timeout) + if self.result is None: + assert not flag + self.result = False, ("EXTERNAL-TIMEOUT", None) + + return self.result + + def _run(self): + try: + self.result = True, self.func(self.session, self.path, self.timeout) + except (socket.timeout, IOError): + self.result = False, ("CRASH", None) + except Exception as e: + message = getattr(e, "message") + if message: + message += "\n" + message += traceback.format_exc(e) + self.result = False, ("ERROR", message) + finally: + self.result_flag.set() + + +class MarionetteWdspecExecutor(WdspecExecutor): + def __init__(self, browser, server_config, webdriver_binary, + timeout_multiplier=1, close_after_done=True, debug_info=None): + WdspecExecutor.__init__(self, browser, server_config, + timeout_multiplier=timeout_multiplier, + debug_info=debug_info) + self.webdriver_binary = webdriver_binary + self.protocol = RemoteMarionetteProtocol(self, browser) + + def is_alive(self): + return self.protocol.is_alive + + def on_environment_change(self, new_environment): + pass + + def do_test(self, test): + timeout = test.timeout * self.timeout_multiplier + extra_timeout + + success, data = WdspecRun(self.do_wdspec, + self.protocol.session, + test.path, + timeout).run() + + if success: + return self.convert_result(test, data) + + return (test.result_cls(*data), []) + + def do_wdspec(self, session, path, timeout): + harness_result = ("OK", None) + subtest_results = pytestrunner.run(path, session, timeout=timeout) + return (harness_result, subtest_results) diff --git a/tests/wpt/harness/wptrunner/executors/executorservo.py b/tests/wpt/harness/wptrunner/executors/executorservo.py index b3421ffe625..068061b958d 100644 --- a/tests/wpt/harness/wptrunner/executors/executorservo.py +++ b/tests/wpt/harness/wptrunner/executors/executorservo.py @@ -71,7 +71,7 @@ class ServoTestharnessExecutor(ProcessTestExecutor): self.result_flag = threading.Event() args = [render_arg(self.browser.render_backend), "--hard-fail", "-u", "Servo/wptrunner", - "-z", self.test_url(test)] + "-Z", "replace-surrogates", "-z", self.test_url(test)] for stylesheet in self.browser.user_stylesheets: args += ["--user-stylesheet", stylesheet] for pref, value in test.environment.get('prefs', {}).iteritems(): @@ -204,7 +204,7 @@ class ServoRefTestExecutor(ProcessTestExecutor): debug_args, command = browser_command( self.binary, [render_arg(self.browser.render_backend), "--hard-fail", "--exit", - "-u", "Servo/wptrunner", "-Z", "disable-text-aa,load-webfonts-synchronously", + "-u", "Servo/wptrunner", "-Z", "disable-text-aa,load-webfonts-synchronously,replace-surrogates", "--output=%s" % output_path, full_url], self.debug_info) diff --git a/tests/wpt/harness/wptrunner/executors/executorservodriver.py b/tests/wpt/harness/wptrunner/executors/executorservodriver.py index 52154d09ba0..fceeb58fad2 100644 --- a/tests/wpt/harness/wptrunner/executors/executorservodriver.py +++ b/tests/wpt/harness/wptrunner/executors/executorservodriver.py @@ -14,16 +14,24 @@ from .base import (Protocol, RefTestImplementation, TestharnessExecutor, strip_server) -import webdriver +from .. import webdriver from ..testrunner import Stop +webdriver = None + here = os.path.join(os.path.split(__file__)[0]) extra_timeout = 5 +def do_delayed_imports(): + global webdriver + import webdriver + + class ServoWebDriverProtocol(Protocol): def __init__(self, executor, browser, capabilities, **kwargs): + do_delayed_imports() Protocol.__init__(self, executor, browser) self.capabilities = capabilities self.host = browser.webdriver_host @@ -34,10 +42,11 @@ class ServoWebDriverProtocol(Protocol): """Connect to browser via WebDriver.""" self.runner = runner + url = "http://%s:%d" % (self.host, self.port) session_started = False try: self.session = webdriver.Session(self.host, self.port, - extension=webdriver.ServoExtensions) + extension=webdriver.servo.ServoCommandExtensions) self.session.start() except: self.logger.warning( @@ -62,7 +71,7 @@ class ServoWebDriverProtocol(Protocol): def is_alive(self): try: # Get a simple property over the connection - self.session.handle + self.session.window_handle # TODO what exception? except Exception: return False diff --git a/tests/wpt/harness/wptrunner/executors/pytestrunner/__init__.py b/tests/wpt/harness/wptrunner/executors/pytestrunner/__init__.py new file mode 100644 index 00000000000..de3a34a794b --- /dev/null +++ b/tests/wpt/harness/wptrunner/executors/pytestrunner/__init__.py @@ -0,0 +1,6 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from . import fixtures +from .runner import run diff --git a/tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py b/tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py new file mode 100644 index 00000000000..77afb4a3684 --- /dev/null +++ b/tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py @@ -0,0 +1,58 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import pytest + + +"""pytest fixtures for use in Python-based WPT tests. + +The purpose of test fixtures is to provide a fixed baseline upon which +tests can reliably and repeatedly execute. +""" + + +class Session(object): + """Fixture to allow access to wptrunner's existing WebDriver session + in tests. + + The session is not created by default to enable testing of session + creation. However, a module-scoped session will be implicitly created + at the first call to a WebDriver command. This means methods such as + `session.send_command` and `session.session_id` are possible to use + without having a session. + + To illustrate implicit session creation:: + + def test_session_scope(session): + # at this point there is no session + assert session.session_id is None + + # window_id is a WebDriver command, + # and implicitly creates the session for us + assert session.window_id is not None + + # we now have a session + assert session.session_id is not None + + You can also access the session in custom fixtures defined in the + tests, such as a setup function:: + + @pytest.fixture(scope="function") + def setup(request, session): + session.url = "https://example.org" + + def test_something(setup, session): + assert session.url == "https://example.org" + + The session is closed when the test module goes out of scope by an + implicit call to `session.end`. + """ + + def __init__(self, client): + self.client = client + + @pytest.fixture(scope="module") + def session(self, request): + request.addfinalizer(self.client.end) + return self.client diff --git a/tests/wpt/harness/wptrunner/executors/pytestrunner/runner.py b/tests/wpt/harness/wptrunner/executors/pytestrunner/runner.py new file mode 100644 index 00000000000..8aa575ff8b7 --- /dev/null +++ b/tests/wpt/harness/wptrunner/executors/pytestrunner/runner.py @@ -0,0 +1,113 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Provides interface to deal with pytest. + +Usage:: + + session = webdriver.client.Session("127.0.0.1", "4444", "/") + harness_result = ("OK", None) + subtest_results = pytestrunner.run("/path/to/test", session.url) + return (harness_result, subtest_results) +""" + +import errno +import shutil +import tempfile + +from . import fixtures + + +pytest = None + + +def do_delayed_imports(): + global pytest + import pytest + + +def run(path, session, timeout=0): + """Run Python test at ``path`` in pytest. The provided ``session`` + is exposed as a fixture available in the scope of the test functions. + + :param path: Path to the test file. + :param session: WebDriver session to expose. + :param timeout: Duration before interrupting potentially hanging + tests. If 0, there is no timeout. + + :returns: List of subtest results, which are tuples of (test id, + status, message, stacktrace). + """ + + if pytest is None: + do_delayed_imports() + + recorder = SubtestResultRecorder() + plugins = [recorder, + fixtures.Session(session)] + + # TODO(ato): Deal with timeouts + + with TemporaryDirectory() as cache: + pytest.main(["--strict", # turn warnings into errors + "--verbose", # show each individual subtest + "--capture", "no", # enable stdout/stderr from tests + "--basetemp", cache, # temporary directory + path], + plugins=plugins) + + return recorder.results + + +class SubtestResultRecorder(object): + def __init__(self): + self.results = [] + + def pytest_runtest_logreport(self, report): + if report.passed and report.when == "call": + self.record_pass(report) + elif report.failed: + if report.when != "call": + self.record_error(report) + else: + self.record_fail(report) + elif report.skipped: + self.record_skip(report) + + def record_pass(self, report): + self.record(report.nodeid, "PASS") + + def record_fail(self, report): + self.record(report.nodeid, "FAIL", stack=report.longrepr) + + def record_error(self, report): + # error in setup/teardown + if report.when != "call": + message = "%s error" % report.when + self.record(report.nodeid, "ERROR", message, report.longrepr) + + def record_skip(self, report): + self.record(report.nodeid, "ERROR", + "In-test skip decorators are disallowed, " + "please use WPT metadata to ignore tests.") + + def record(self, test, status, message=None, stack=None): + if stack is not None: + stack = str(stack) + new_result = (test, status, message, stack) + self.results.append(new_result) + + +class TemporaryDirectory(object): + def __enter__(self): + self.path = tempfile.mkdtemp(prefix="pytest-") + return self.path + + def __exit__(self, *args): + try: + shutil.rmtree(self.path) + except OSError as e: + # no such file or directory + if e.errno != errno.ENOENT: + raise diff --git a/tests/wpt/harness/wptrunner/manifestexpected.py b/tests/wpt/harness/wptrunner/manifestexpected.py index b46a1ef9597..c0e22a843fd 100644 --- a/tests/wpt/harness/wptrunner/manifestexpected.py +++ b/tests/wpt/harness/wptrunner/manifestexpected.py @@ -29,10 +29,10 @@ def data_cls_getter(output_node, visited_node): raise ValueError -def disabled(node): - """Boolean indicating whether the test is disabled""" +def bool_prop(name, node): + """Boolean property""" try: - return node.get("disabled") + return node.get(name) except KeyError: return None @@ -109,7 +109,11 @@ class ExpectedManifest(ManifestItem): @property def disabled(self): - return disabled(self) + return bool_prop("disabled", self) + + @property + def restart_after(self): + return bool_prop("restart-after", self) @property def tags(self): @@ -123,7 +127,11 @@ class ExpectedManifest(ManifestItem): class DirectoryManifest(ManifestItem): @property def disabled(self): - return disabled(self) + return bool_prop("disabled", self) + + @property + def restart_after(self): + return bool_prop("restart-after", self) @property def tags(self): @@ -164,7 +172,11 @@ class TestNode(ManifestItem): @property def disabled(self): - return disabled(self) + return bool_prop("disabled", self) + + @property + def restart_after(self): + return bool_prop("restart-after", self) @property def tags(self): diff --git a/tests/wpt/harness/wptrunner/testrunner.py b/tests/wpt/harness/wptrunner/testrunner.py index b5617827a40..77d2a885083 100644 --- a/tests/wpt/harness/wptrunner/testrunner.py +++ b/tests/wpt/harness/wptrunner/testrunner.py @@ -524,7 +524,8 @@ class TestRunnerManager(threading.Thread): self.test = None - restart_before_next = (file_result.status in ("CRASH", "EXTERNAL-TIMEOUT") or + restart_before_next = (test.restart_after or + file_result.status in ("CRASH", "EXTERNAL-TIMEOUT") or subtest_unexpected or is_unexpected) if (self.pause_after_test or diff --git a/tests/wpt/harness/wptrunner/webdriver_server.py b/tests/wpt/harness/wptrunner/webdriver_server.py new file mode 100644 index 00000000000..68d3fb7a3bf --- /dev/null +++ b/tests/wpt/harness/wptrunner/webdriver_server.py @@ -0,0 +1,206 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import abc +import errno +import os +import platform +import socket +import threading +import time +import traceback +import urlparse + +import mozprocess + + +__all__ = ["SeleniumServer", "ChromeDriverServer", + "GeckoDriverServer", "WebDriverServer"] + + +class WebDriverServer(object): + __metaclass__ = abc.ABCMeta + + default_base_path = "/" + _used_ports = set() + + def __init__(self, logger, binary, host="127.0.0.1", port=None, + base_path="", env=None): + self.logger = logger + self.binary = binary + self.host = host + if base_path == "": + self.base_path = self.default_base_path + else: + self.base_path = base_path + self.env = os.environ.copy() if env is None else env + + self._port = port + self._cmd = None + self._proc = None + + @abc.abstractmethod + def make_command(self): + """Returns the full command for starting the server process as a list.""" + + def start(self, block=True): + try: + self._run(block) + except KeyboardInterrupt: + self.stop() + + def _run(self, block): + self._cmd = self.make_command() + self._proc = mozprocess.ProcessHandler( + self._cmd, + processOutputLine=self.on_output, + env=self.env, + storeOutput=False) + + try: + self._proc.run() + except OSError as e: + if e.errno == errno.ENOENT: + raise IOError( + "WebDriver HTTP server executable not found: %s" % self.binary) + raise + + self.logger.debug( + "Waiting for server to become accessible: %s" % self.url) + try: + wait_for_service((self.host, self.port)) + except: + self.logger.error( + "WebDriver HTTP server was not accessible " + "within the timeout:\n%s" % traceback.format_exc()) + raise + + if block: + self._proc.wait() + + def stop(self): + if self.is_alive: + return self._proc.kill() + return not self.is_alive + + @property + def is_alive(self): + return (self._proc is not None and + self._proc.proc is not None and + self._proc.poll() is None) + + def on_output(self, line): + self.logger.process_output(self.pid, + line.decode("utf8", "replace"), + command=" ".join(self._cmd)) + + @property + def pid(self): + if self._proc is not None: + return self._proc.pid + + @property + def url(self): + return "http://%s:%i%s" % (self.host, self.port, self.base_path) + + @property + def port(self): + if self._port is None: + self._port = self._find_next_free_port() + return self._port + + @staticmethod + def _find_next_free_port(): + port = get_free_port(4444, exclude=WebDriverServer._used_ports) + WebDriverServer._used_ports.add(port) + return port + + +class SeleniumServer(WebDriverServer): + default_base_path = "/wd/hub" + + def make_command(self): + return ["java", "-jar", self.binary, "-port", str(self.port)] + + +class ChromeDriverServer(WebDriverServer): + default_base_path = "/wd/hub" + + def __init__(self, logger, binary="chromedriver", port=None, + base_path=""): + WebDriverServer.__init__( + self, logger, binary, port=port, base_path=base_path) + + def make_command(self): + return [self.binary, + cmd_arg("port", str(self.port)), + cmd_arg("url-base", self.base_path) if self.base_path else ""] + + +class GeckoDriverServer(WebDriverServer): + def __init__(self, logger, marionette_port=2828, binary="wires", + host="127.0.0.1", port=None): + env = os.environ.copy() + env["RUST_BACKTRACE"] = "1" + WebDriverServer.__init__(self, logger, binary, host=host, port=port, env=env) + self.marionette_port = marionette_port + + def make_command(self): + return [self.binary, + "--connect-existing", + "--marionette-port", str(self.marionette_port), + "--webdriver-host", self.host, + "--webdriver-port", str(self.port)] + + +def cmd_arg(name, value=None): + prefix = "-" if platform.system() == "Windows" else "--" + rv = prefix + name + if value is not None: + rv += "=" + value + return rv + + +def get_free_port(start_port, exclude=None): + """Get the first port number after start_port (inclusive) that is + not currently bound. + + :param start_port: Integer port number at which to start testing. + :param exclude: Set of port numbers to skip""" + port = start_port + while True: + if exclude and port in exclude: + port += 1 + continue + s = socket.socket() + try: + s.bind(("127.0.0.1", port)) + except socket.error: + port += 1 + else: + return port + finally: + s.close() + + +def wait_for_service(addr, timeout=15): + """Waits until network service given as a tuple of (host, port) becomes + available or the `timeout` duration is reached, at which point + ``socket.error`` is raised.""" + end = time.time() + timeout + while end > time.time(): + so = socket.socket() + try: + so.connect(addr) + except socket.timeout: + pass + except socket.error as e: + if e[0] != errno.ECONNREFUSED: + raise + else: + return True + finally: + so.close() + time.sleep(0.5) + raise socket.error("Service is unavailable: %s:%i" % addr) diff --git a/tests/wpt/harness/wptrunner/wptcommandline.py b/tests/wpt/harness/wptrunner/wptcommandline.py index 9979d69fd38..3b059378e09 100644 --- a/tests/wpt/harness/wptrunner/wptcommandline.py +++ b/tests/wpt/harness/wptrunner/wptcommandline.py @@ -10,6 +10,7 @@ from collections import OrderedDict from distutils.spawn import find_executable import config +import wpttest def abs_path(path): @@ -25,6 +26,7 @@ def url_or_path(path): else: return abs_path(path) + def require_arg(kwargs, name, value_func=None): if value_func is None: value_func = lambda x: x is not None @@ -101,8 +103,8 @@ def create_parser(product_choices=None): test_selection_group = parser.add_argument_group("Test Selection") test_selection_group.add_argument("--test-types", action="store", - nargs="*", default=["testharness", "reftest"], - choices=["testharness", "reftest"], + nargs="*", default=wpttest.enabled_tests, + choices=wpttest.enabled_tests, help="Test types to run") test_selection_group.add_argument("--include", action="append", help="URL prefix to include") @@ -159,8 +161,8 @@ def create_parser(product_choices=None): gecko_group = parser.add_argument_group("Gecko-specific") gecko_group.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path, help="Path to the folder containing browser prefs") - gecko_group.add_argument("--e10s", dest="gecko_e10s", action="store_true", - help="Run tests with electrolysis preferences") + gecko_group.add_argument("--disable-e10s", dest="gecko_e10s", action="store_false", default=True, + help="Run tests without electrolysis preferences") b2g_group = parser.add_argument_group("B2G-specific") b2g_group.add_argument("--b2g-no-backup", action="store_true", default=False, @@ -343,12 +345,14 @@ def check_args(kwargs): return kwargs + def check_args_update(kwargs): set_from_config(kwargs) if kwargs["product"] is None: kwargs["product"] = "firefox" + def create_parser_update(product_choices=None): from mozlog.structured import commandline diff --git a/tests/wpt/harness/wptrunner/wptlogging.py b/tests/wpt/harness/wptrunner/wptlogging.py index 9e6e737d3e9..047e025fad1 100644 --- a/tests/wpt/harness/wptrunner/wptlogging.py +++ b/tests/wpt/harness/wptrunner/wptlogging.py @@ -74,6 +74,7 @@ class LoggingWrapper(StringIO): instead""" def __init__(self, queue, prefix=None): + StringIO.__init__(self) self.queue = queue self.prefix = prefix @@ -94,6 +95,7 @@ class LoggingWrapper(StringIO): def flush(self): pass + class CaptureIO(object): def __init__(self, logger, do_capture): self.logger = logger diff --git a/tests/wpt/harness/wptrunner/wpttest.py b/tests/wpt/harness/wptrunner/wpttest.py index 73a50bb129e..9832f72654e 100644 --- a/tests/wpt/harness/wptrunner/wpttest.py +++ b/tests/wpt/harness/wptrunner/wpttest.py @@ -12,6 +12,8 @@ import mozinfo from wptmanifest.parser import atoms atom_reset = atoms["Reset"] +enabled_tests = set(["testharness", "reftest", "wdspec"]) + class Result(object): def __init__(self, status, message, expected=None, extra=None): @@ -22,6 +24,9 @@ class Result(object): self.expected = expected self.extra = extra + def __repr__(self): + return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.status) + class SubtestResult(object): def __init__(self, name, status, message, stack=None, expected=None): @@ -33,20 +38,33 @@ class SubtestResult(object): self.stack = stack self.expected = expected + def __repr__(self): + return "<%s.%s %s %s>" % (self.__module__, self.__class__.__name__, self.name, self.status) + class TestharnessResult(Result): default_expected = "OK" statuses = set(["OK", "ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"]) +class TestharnessSubtestResult(SubtestResult): + default_expected = "PASS" + statuses = set(["PASS", "FAIL", "TIMEOUT", "NOTRUN"]) + + class ReftestResult(Result): default_expected = "PASS" statuses = set(["PASS", "FAIL", "ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"]) -class TestharnessSubtestResult(SubtestResult): +class WdspecResult(Result): + default_expected = "OK" + statuses = set(["OK", "ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"]) + + +class WdspecSubtestResult(SubtestResult): default_expected = "PASS" - statuses = set(["PASS", "FAIL", "TIMEOUT", "NOTRUN"]) + statuses = set(["PASS", "FAIL", "ERROR"]) def get_run_info(metadata_root, product, **kwargs): @@ -82,6 +100,7 @@ class RunInfo(dict): mozinfo.find_and_update_from_json(*dirs) + class B2GRunInfo(RunInfo): def __init__(self, *args, **kwargs): RunInfo.__init__(self, *args, **kwargs) @@ -115,7 +134,6 @@ class Test(object): path=manifest_item.path, protocol="https" if hasattr(manifest_item, "https") and manifest_item.https else "http") - @property def id(self): return self.url @@ -141,7 +159,6 @@ class Test(object): if subtest_meta is not None: yield subtest_meta - def disabled(self, subtest=None): for meta in self.itermeta(subtest): disabled = meta.disabled @@ -150,6 +167,14 @@ class Test(object): return None @property + def restart_after(self): + for meta in self.itermeta(None): + restart_after = meta.restart_after + if restart_after is not None: + return True + return False + + @property def tags(self): tags = set() for meta in self.itermeta(): @@ -191,6 +216,9 @@ class Test(object): except KeyError: return default + def __repr__(self): + return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.id) + class TestharnessTest(Test): result_cls = TestharnessResult @@ -293,12 +321,18 @@ class ReftestTest(Test): return ("reftype", "refurl") +class WdspecTest(Test): + result_cls = WdspecResult + subtest_result_cls = WdspecSubtestResult + test_type = "wdspec" + + manifest_test_cls = {"reftest": ReftestTest, "testharness": TestharnessTest, - "manual": ManualTest} + "manual": ManualTest, + "wdspec": WdspecTest} def from_manifest(manifest_test, inherit_metadata, test_metadata): test_cls = manifest_test_cls[manifest_test.item_type] - return test_cls.from_manifest(manifest_test, inherit_metadata, test_metadata) diff --git a/tests/wpt/metadata-css/cssom-1_dev/html/escape.htm.ini b/tests/wpt/metadata-css/cssom-1_dev/html/escape.htm.ini index a46acb9468d..ca3b93d8303 100644 --- a/tests/wpt/metadata-css/cssom-1_dev/html/escape.htm.ini +++ b/tests/wpt/metadata-css/cssom-1_dev/html/escape.htm.ini @@ -1,3 +1,14 @@ [escape.htm] type: testharness - expected: CRASH + [Null bytes] + expected: FAIL + bug: https://github.com/servo/servo/issues/10685 + + [Various tests] + expected: FAIL + bug: https://github.com/servo/servo/issues/10685 + + [Surrogates] + expected: FAIL + bug: https://github.com/servo/servo/issues/6564 + diff --git a/tests/wpt/metadata/webgl/conformance-1.0.3/conformance/programs/invalid-UTF-16.html.ini b/tests/wpt/metadata/webgl/conformance-1.0.3/conformance/programs/invalid-UTF-16.html.ini deleted file mode 100644 index 889bd615088..00000000000 --- a/tests/wpt/metadata/webgl/conformance-1.0.3/conformance/programs/invalid-UTF-16.html.ini +++ /dev/null @@ -1,3 +0,0 @@ -[invalid-UTF-16.html] - type: testharness - expected: CRASH diff --git a/tests/wpt/metadata/webstorage/storage_setitem.html.ini b/tests/wpt/metadata/webstorage/storage_setitem.html.ini index 93dfb482b17..0a0f43019b2 100644 --- a/tests/wpt/metadata/webstorage/storage_setitem.html.ini +++ b/tests/wpt/metadata/webstorage/storage_setitem.html.ini @@ -1,3 +1,19 @@ [storage_setitem.html] type: testharness - expected: CRASH + expected: TIMEOUT + [localStorage[\] = "�"] + expected: FAIL + bug: https://github.com/servo/servo/issues/6564 + + [localStorage[\] = "�a"] + expected: FAIL + bug: https://github.com/servo/servo/issues/6564 + + [localStorage[\] = "a�"] + expected: FAIL + bug: https://github.com/servo/servo/issues/6564 + + [localStorage["0"\]] + expected: TIMEOUT + bug: https://github.com/servo/servo/issues/10686 + |