Auto merge of #29639 - mrobinson:reorganize-wpt-python, r=mukilan

Reorganize Servo's WPT Python scripts

This change moves all of Servo's WPT Python support scripts into one directory as they were previously scattered throughout the directory structure. This should allow more code reuse and make it easier to understand how everything fits together.

The changes:

- `tests/wpt/update` → `python/wpt/importer`
- `etc/ci/upstream-wpt-changes/wptupstreamer` → `python/wpt/exporter`
- `etc/ci/upstream-wpt-changes/test.py` → `python/wpt/test.py`
- `etc/ci/upstream-wpt-changes/tests` → `python/wpt/tests`
- `tests/wpt/servowpt.py` → - `python/wpt/update.py` - `python/wpt/run.py`
- `tests/wpt/manifestupdate.py` → `python/wpt/manifestupdate.py`

This change also removes
 - The ability to run the `update-wpt` and `test-wpt` commands without using `mach`. These didn't work very well, because it was difficult to get all of the wptrunner and mach dependencies installed outside of the Python virtualenv. It's simpler if they are always run through `mach`.
- The old WPT change sync / upstreaming script that was no longer used. This allows this code to pass `test-tidy` now.

<!-- Please describe your changes on the following line: -->

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `___` with appropriate data: -->
- [x] `./mach build -d` does not report any errors
- [x] `./mach test-tidy` does not report any errors
- [x] There are tests for these changes

<!-- Also, please make sure that "Allow edits from maintainers" checkbox is checked, so that we can help you if you get stuck somewhere along the way.-->

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

1 files changed, 326 insertions, 0 deletions

diff --git a/python/wpt/run.py b/python/wpt/run.py
new file mode 100644
index 00000000000..5a75eee2c7a
--- /dev/null
+++ b/python/wpt/run.py
@@ -0,0 +1,326 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+# pylint: disable=missing-docstring
+
+import dataclasses
+import json
+import multiprocessing
+import os
+import re
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+
+from typing import List, NamedTuple, Optional, Union
+
+import mozlog
+import mozlog.formatters
+
+from . import SERVO_ROOT, WPT_PATH, WPT_TOOLS_PATH, update_args_for_layout_2020
+from .grouping_formatter import (
+    ServoFormatter, ServoHandler,
+    UnexpectedResult, UnexpectedSubtestResult
+)
+from wptrunner import wptcommandline
+from wptrunner import wptrunner
+
+
+CERTS_PATH = os.path.join(WPT_TOOLS_PATH, "certs")
+TRACKER_API = "https://build.servo.org/intermittent-tracker"
+TRACKER_API_ENV_VAR = "INTERMITTENT_TRACKER_API"
+TRACKER_DASHBOARD_SECRET_ENV_VAR = "INTERMITTENT_TRACKER_DASHBOARD_SECRET"
+
+
+def determine_build_type(kwargs: dict, target_dir: str):
+    if kwargs["release"]:
+        return "release"
+    elif kwargs["debug"]:
+        return "debug"
+    elif os.path.exists(os.path.join(target_dir, "debug")):
+        return "debug"
+    elif os.path.exists(os.path.join(target_dir, "release")):
+        return "release"
+    return "debug"
+
+
+def set_if_none(args: dict, key: str, value):
+    if key not in args or args[key] is None:
+        args[key] = value
+
+
+def run_tests(**kwargs):
+    # By default, Rayon selects the number of worker threads based on the
+    # available CPU count. This doesn't work very well when running tests on CI,
+    # since we run so many Servo processes in parallel. The result is a lot of
+    # extra timeouts. Instead, force Rayon to assume we are running on a 2 CPU
+    # environment.
+    os.environ["RAYON_RS_NUM_CPUS"] = "2"
+    os.environ["RUST_BACKTRACE"] = "1"
+    os.environ["HOST_FILE"] = os.path.join(SERVO_ROOT, "tests", "wpt", "hosts")
+
+    set_if_none(kwargs, "product", "servo")
+    set_if_none(kwargs, "config", os.path.join(WPT_PATH, "config.ini"))
+    set_if_none(kwargs, "include_manifest", os.path.join(WPT_PATH, "include.ini"))
+    set_if_none(kwargs, "manifest_update", False)
+    set_if_none(kwargs, "processes", multiprocessing.cpu_count())
+
+    set_if_none(kwargs, "ca_cert_path", os.path.join(CERTS_PATH, "cacert.pem"))
+    set_if_none(
+        kwargs, "host_key_path", os.path.join(CERTS_PATH, "web-platform.test.key")
+    )
+    set_if_none(
+        kwargs, "host_cert_path", os.path.join(CERTS_PATH, "web-platform.test.pem")
+    )
+
+    kwargs["user_stylesheets"].append(os.path.join(SERVO_ROOT, "resources", "ahem.css"))
+
+    if "CARGO_TARGET_DIR" in os.environ:
+        target_dir = os.path.join(os.environ["CARGO_TARGET_DIR"])
+    else:
+        target_dir = os.path.join(SERVO_ROOT, "target")
+    default_binary_path = os.path.join(
+        target_dir, determine_build_type(kwargs, target_dir), "servo"
+    )
+    if sys.platform == "win32":
+        target_dir += ".exe"
+
+    set_if_none(kwargs, "binary", default_binary_path)
+    set_if_none(kwargs, "webdriver_binary", default_binary_path)
+
+    if kwargs.pop("rr_chaos"):
+        kwargs["debugger"] = "rr"
+        kwargs["debugger_args"] = "record --chaos"
+        kwargs["repeat_until_unexpected"] = True
+        # TODO: Delete rr traces from green test runs?
+
+    prefs = kwargs.pop("prefs")
+    if prefs:
+        kwargs["binary_args"] = ["--pref=" + pref for pref in prefs]
+
+    if not kwargs.get("no_default_test_types"):
+        test_types = {
+            "servo": ["testharness", "reftest", "wdspec"],
+            "servodriver": ["testharness", "reftest"],
+        }
+        product = kwargs.get("product") or "servo"
+        kwargs["test_types"] = test_types[product]
+
+    filter_intermittents_output = kwargs.pop("filter_intermittents", None)
+    unexpected_raw_log_output_file = kwargs.pop("log_raw_unexpected", None)
+    raw_log_outputs = kwargs.get("log_raw", [])
+
+    wptcommandline.check_args(kwargs)
+    update_args_for_layout_2020(kwargs)
+
+    mozlog.commandline.log_formatters["servo"] = (
+        ServoFormatter,
+        "Servo's grouping output formatter",
+    )
+
+    use_mach_logging = False
+    if len(kwargs["test_list"]) == 1:
+        file_ext = os.path.splitext(kwargs["test_list"][0])[1].lower()
+        if file_ext in [".htm", ".html", ".js", ".xhtml", ".xht", ".py"]:
+            use_mach_logging = True
+
+    if use_mach_logging:
+        logger = wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
+    else:
+        logger = wptrunner.setup_logging(kwargs, {"servo": sys.stdout})
+
+    handler = ServoHandler()
+    logger.add_handler(handler)
+
+    wptrunner.run_tests(**kwargs)
+    return_value = 0 if not handler.unexpected_results else 1
+
+    # Filter intermittents if that was specified on the command-line.
+    if handler.unexpected_results and filter_intermittents_output:
+        # Copy the list of unexpected results from the first run, so that we
+        # can access them after the tests are rerun (which changes
+        # `handler.unexpected_results`). After rerunning some tests will be
+        # marked as flaky but otherwise the contents of this original list
+        # won't change.
+        unexpected_results = list(handler.unexpected_results)
+
+        # This isn't strictly necessary since `handler.suite_start()` clears
+        # the state, but make sure that we are starting with a fresh handler.
+        handler.reset_state()
+
+        print(80 * "=")
+        print(f"Rerunning {len(unexpected_results)} tests "
+              "with unexpected results to detect flaky tests.")
+        unexpected_results_tests = [result.path for result in unexpected_results]
+        kwargs["test_list"] = unexpected_results_tests
+        kwargs["include"] = unexpected_results_tests
+        kwargs["pause_after_test"] = False
+        wptrunner.run_tests(**kwargs)
+
+        # Use the second run to mark tests from the first run as flaky, but
+        # discard the results otherwise.
+        # TODO: It might be a good idea to send the new results to the
+        # dashboard if they were also unexpected.
+        stable_tests = [result.path for result in handler.unexpected_results]
+        for result in unexpected_results:
+            result.flaky = result.path not in stable_tests
+
+        all_filtered = filter_intermittents(unexpected_results,
+                                            filter_intermittents_output)
+        return_value = 0 if all_filtered else 1
+
+    # Write the unexpected-only raw log if that was specified on the command-line.
+    if unexpected_raw_log_output_file:
+        if not raw_log_outputs:
+            print("'--log-raw-unexpected' not written without '--log-raw'.")
+        else:
+            write_unexpected_only_raw_log(
+                handler.unexpected_results,
+                raw_log_outputs[0].name,
+                unexpected_raw_log_output_file
+            )
+
+    return return_value
+
+
+class GithubContextInformation(NamedTuple):
+    build_url: Optional[str]
+    pull_url: Optional[str]
+    branch_name: Optional[str]
+
+
+class TrackerDashboardFilter():
+    def __init__(self):
+        base_url = os.environ.get(TRACKER_API_ENV_VAR, TRACKER_API)
+        self.headers = {
+            "Content-Type": "application/json"
+        }
+        if TRACKER_DASHBOARD_SECRET_ENV_VAR in os.environ and os.environ[TRACKER_DASHBOARD_SECRET_ENV_VAR]:
+            self.url = f"{base_url}/dashboard/attempts"
+            secret = os.environ[TRACKER_DASHBOARD_SECRET_ENV_VAR]
+            self.headers["Authorization"] = f"Bearer {secret}"
+        else:
+            self.url = f"{base_url}/dashboard/query"
+
+    @staticmethod
+    def get_github_context_information() -> GithubContextInformation:
+        github_context = json.loads(os.environ.get("GITHUB_CONTEXT", "{}"))
+        if not github_context:
+            return GithubContextInformation(None, None, None)
+
+        repository = github_context['repository']
+        repo_url = f"https://github.com/{repository}"
+
+        run_id = github_context['run_id']
+        build_url = f"{repo_url}/actions/runs/{run_id}"
+
+        commit_title = github_context["event"]["head_commit"]["message"]
+        match = re.match(r"^Auto merge of #(\d+)", commit_title)
+        pr_url = f"{repo_url}/pull/{match.group(1)}" if match else None
+
+        return GithubContextInformation(
+            build_url,
+            pr_url,
+            github_context["ref_name"]
+        )
+
+    def make_data_from_result(
+        self,
+        result: Union[UnexpectedResult, UnexpectedSubtestResult],
+    ) -> dict:
+        data = {
+            'path': result.path,
+            'subtest': None,
+            'expected': result.expected,
+            'actual': result.actual,
+            'time': result.time // 1000,
+            'message': result.message,
+            'stack': result.stack,
+        }
+        if isinstance(result, UnexpectedSubtestResult):
+            data["subtest"] = result.subtest
+        return data
+
+    def report_failures(self, unexpected_results: List[UnexpectedResult]):
+        attempts = []
+        for result in unexpected_results:
+            attempts.append(self.make_data_from_result(result))
+            for subtest_result in result.unexpected_subtest_results:
+                attempts.append(self.make_data_from_result(subtest_result))
+
+        context = self.get_github_context_information()
+        try:
+            request = urllib.request.Request(
+                url=self.url,
+                method='POST',
+                data=json.dumps({
+                    'branch': context.branch_name,
+                    'build_url': context.build_url,
+                    'pull_url': context.pull_url,
+                    'attempts': attempts
+                }).encode('utf-8'),
+                headers=self.headers)
+
+            known_intermittents = dict()
+            with urllib.request.urlopen(request) as response:
+                for test in json.load(response)["known"]:
+                    known_intermittents[test["path"]] = \
+                        [issue["number"] for issue in test["issues"]]
+
+        except urllib.error.HTTPError as e:
+            print(e)
+            print(e.readlines())
+            raise(e)
+
+        for result in unexpected_results:
+            result.issues = known_intermittents.get(result.path, [])
+
+
+def filter_intermittents(
+    unexpected_results: List[UnexpectedResult],
+    output_path: str
+) -> bool:
+    print(f"Filtering {len(unexpected_results)} "
+          "unexpected results for known intermittents")
+    dashboard = TrackerDashboardFilter()
+    dashboard.report_failures(unexpected_results)
+
+    def add_result(output, text, results: List[UnexpectedResult], filter_func) -> None:
+        filtered = [str(result) for result in filter(filter_func, results)]
+        if filtered:
+            output += [f"{text} ({len(results)}): ", *filtered]
+
+    def is_stable_and_unexpected(result):
+        return not result.flaky and not result.issues
+
+    output: List[str] = []
+    add_result(output, "Flaky unexpected results", unexpected_results,
+               lambda result: result.flaky)
+    add_result(output, "Stable unexpected results that are known-intermittent",
+               unexpected_results, lambda result: not result.flaky and result.issues)
+    add_result(output, "Stable unexpected results",
+               unexpected_results, is_stable_and_unexpected)
+    print("\n".join(output))
+
+    with open(output_path, "w", encoding="utf-8") as file:
+        json.dump([dataclasses.asdict(result) for result in unexpected_results], file)
+
+    return not any([is_stable_and_unexpected(result) for result in unexpected_results])
+
+
+def write_unexpected_only_raw_log(
+    unexpected_results: List[UnexpectedResult],
+    raw_log_file: str,
+    filtered_raw_log_file: str
+):
+    tests = [result.path for result in unexpected_results]
+    print(f"Writing unexpected-only raw log to {filtered_raw_log_file}")
+
+    with open(filtered_raw_log_file, "w", encoding="utf-8") as output:
+        with open(raw_log_file) as input:
+            for line in input.readlines():
+                data = json.loads(line)
+                if data["action"] in ["suite_start", "suite_end"] or \
+                        ("test" in data and data["test"] in tests):
+                    output.write(line)