diff options
| author | Martin Robinson <mrobinson@igalia.com> | 2023-04-10 13:56:47 +0200 |
|---|---|---|
| committer | Martin Robinson <mrobinson@igalia.com> | 2023-04-10 23:04:17 +0200 |
| commit | bc3abf995313ece462233c914f4aa61b76566e66 (patch) | |
| tree | 48d83efc5f52ab9118dc09dd229ffee4ade9ceb3 | |
| parent | d579bd91b8e82606907dd789f13f6ecaee6a9b18 (diff) | |
| download | servo-bc3abf995313ece462233c914f4aa61b76566e66.tar.gz servo-bc3abf995313ece462233c914f4aa61b76566e66.zip | |
Remove more Taskcluster and Treeherder integration
Servo no longer uses Taskcluster and Treeherder, so this change removes
script references to those services and support files.
25 files changed, 11 insertions, 2174 deletions
diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 57c4a7700d3..933683c713c 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -63,8 +63,8 @@ jobs: - name: Bootstrap run: | python3 -m pip install --upgrade pip virtualenv - brew bundle install --verbose --no-upgrade --file=etc/taskcluster/macos/Brewfile - brew bundle install --verbose --no-upgrade --file=etc/taskcluster/macos/Brewfile-build + brew bundle install --verbose --no-upgrade --file=etc/homebrew/Brewfile + brew bundle install --verbose --no-upgrade --file=etc/homebrew/Brewfile-build rm -rf /usr/local/etc/openssl rm -rf /usr/local/etc/openssl@1.1 brew install openssl@1.1 gnu-tar @@ -133,7 +133,7 @@ jobs: # run: | # gtar -xzf target.tar.gz # python3 -m pip install --upgrade pip virtualenv - # brew bundle install --verbose --no-upgrade --file=etc/taskcluster/macos/Brewfile + # brew bundle install --verbose --no-upgrade --file=etc/homebrew/Brewfile # - name: Smoketest # run: python3 ./mach smoketest # - name: Run tests diff --git a/.gitignore b/.gitignore index 8b09f9790d0..221b769edf2 100644 --- a/.gitignore +++ b/.gitignore @@ -75,4 +75,4 @@ support/hololens/.vs/ layout_trace* # Package managers -etc/taskcluster/macos/Brewfile.lock.json +etc/homebrew/Brewfile.lock.json diff --git a/.taskcluster.yml b/.taskcluster.yml deleted file mode 100644 index b2c277f6f09..00000000000 --- a/.taskcluster.yml +++ /dev/null @@ -1,120 +0,0 @@ -version: 1 - -# If and when switching to `reporting: checks-v1` here, also change the `statuses` route to `checks` -# in `CONFIG.routes_for_all_subtasks` in `etc/taskcluster/decision_task.py` - -policy: - # https://docs.taskcluster.net/docs/reference/integrations/taskcluster-github/docs/taskcluster-yml-v1#pull-requests - pullRequests: public - - # NOTE: when updating this consider whether the daily hook needs similar changes: -# https://tools.taskcluster.net/hooks/project-servo/daily -tasks: - $let: - task_common: - provisionerId: - $if: "taskcluster_root_url == 'https://taskcluster.net'" - then: aws-provisioner-v1 - else: proj-servo - created: {$fromNow: ''} - deadline: {$fromNow: '1 day'} - priority: high - extra: - treeherder: - machine: {platform: Linux} - labels: [x64] - symbol: Decision - payload: - maxRunTime: {$eval: '20 * 60'} - # https://github.com/servo/taskcluster-bootstrap-docker-images#decision-task - image: "servobrowser/taskcluster-bootstrap:decision-task@\ - sha256:7471a998e4462638c8d3e2cf0b4a99c9a5c8ca9f2ec0ae01cc069473b35cde10" - features: - taskclusterProxy: true - artifacts: - public/repo.bundle: - type: file - path: /repo.bundle - expires: {$fromNow: '1 day'} - env: - GIT_URL: ${event.repository.clone_url} - TASK_FOR: ${tasks_for} - command: - - /bin/bash - - '--login' - - '-e' - - '-c' - - >- - git init repo && - cd repo && - git fetch --depth 1 "$GIT_URL" "$GIT_REF" && - git reset --hard "$GIT_SHA" && - python3 etc/taskcluster/decision_task.py - in: - - $if: "tasks_for == 'github-push'" - then: - $let: - branch: - $if: "event.ref[:11] == 'refs/heads/'" - then: "${event.ref[11:]}" - in: - $if: "branch in ['auto', 'try', 'master'] || branch[:4] == 'try-'" - then: - $mergeDeep: - - {$eval: "task_common"} - - metadata: - name: "Servo: GitHub push decision task" - description: "" - owner: ${event.pusher.name}@users.noreply.github.com - source: ${event.compare} - workerType: - $if: "taskcluster_root_url == 'https://taskcluster.net'" - then: servo-docker-worker - else: docker - scopes: - - "assume:repo:github.com/servo/servo:branch:${branch}" - routes: - $let: - treeherder_repo: - $if: "branch[:4] == 'try-'" - then: "servo-try" - else: "servo-${branch}" - in: - - "tc-treeherder.v2._/${treeherder_repo}.${event.after}" - - "tc-treeherder-staging.v2._/${treeherder_repo}.${event.after}" - payload: - env: - GIT_REF: ${event.ref} - GIT_SHA: ${event.after} - TASK_OWNER: ${event.pusher.name}@users.noreply.github.com - TASK_SOURCE: ${event.compare} - - $if: >- - tasks_for == 'github-pull-request' && - event['action'] in ['opened', 'reopened', 'synchronize'] - then: - $mergeDeep: - - {$eval: "task_common"} - - metadata: - name: "Servo: GitHub PR decision task" - description: "" - owner: ${event.sender.login}@users.noreply.github.com - source: ${event.pull_request.url} - workerType: - $if: "taskcluster_root_url == 'https://taskcluster.net'" - then: servo-docker-untrusted - else: docker-untrusted - scopes: - - "assume:repo:github.com/servo/servo:pull-request" - routes: - - "tc-treeherder.v2._/servo-prs.${event.pull_request.head.sha}" - - "tc-treeherder-staging.v2._/servo-prs.${event.pull_request.head.sha}" - payload: - env: - # We use the merge commit made by GitHub, not the PR’s branch - GIT_REF: refs/pull/${event.pull_request.number}/merge - # `event.pull_request.merge_commit_sha` is tempting but not what we want: - # https://github.com/servo/servo/pull/22597#issuecomment-451518810 - GIT_SHA: FETCH_HEAD - TASK_OWNER: ${event.sender.login}@users.noreply.github.com - TASK_SOURCE: ${event.pull_request.url} - diff --git a/README.md b/README.md index 819fdf13263..771ae02f3fd 100644 --- a/README.md +++ b/README.md @@ -58,8 +58,8 @@ NOTE: run these steps after you've cloned the project locally. ``` sh cd servo -brew bundle install --file=etc/taskcluster/macos/Brewfile -brew bundle install --file=etc/taskcluster/macos/Brewfile-build +brew bundle install --file=etc/homebrew/Brewfile +brew bundle install --file=etc/homebrew/Brewfile-build pip install virtualenv ``` diff --git a/etc/ci/update-wpt-checkout b/etc/ci/update-wpt-checkout index 58f5cc386a9..4ff7f7c125e 100755 --- a/etc/ci/update-wpt-checkout +++ b/etc/ci/update-wpt-checkout @@ -101,11 +101,6 @@ function unsafe_open_pull_request() { # is unnecessary. git checkout "${BRANCH_NAME}" || return 0 - if [[ -z "${WPT_SYNC_TOKEN+set}" && "${TASKCLUSTER_PROXY_URL+set}" == "set" ]]; then - SECRET_RESPONSE=$(curl ${TASKCLUSTER_PROXY_URL}/api/secrets/v1/secret/project/servo/wpt-sync) - WPT_SYNC_TOKEN=`echo "${SECRET_RESPONSE}" | jq --raw-output '.secret.token'` - fi - if [[ -z "${WPT_SYNC_TOKEN+set}" ]]; then echo "Github auth token missing from WPT_SYNC_TOKEN." return 1 diff --git a/etc/taskcluster/macos/Brewfile b/etc/homebrew/Brewfile index c11e49b6c30..c11e49b6c30 100644 --- a/etc/taskcluster/macos/Brewfile +++ b/etc/homebrew/Brewfile diff --git a/etc/taskcluster/macos/Brewfile-build b/etc/homebrew/Brewfile-build index c78fe2cea3c..c78fe2cea3c 100644 --- a/etc/taskcluster/macos/Brewfile-build +++ b/etc/homebrew/Brewfile-build diff --git a/etc/taskcluster/macos/Brewfile-wpt-update b/etc/homebrew/Brewfile-wpt-update index f563d5e7bf9..f563d5e7bf9 100644 --- a/etc/taskcluster/macos/Brewfile-wpt-update +++ b/etc/homebrew/Brewfile-wpt-update diff --git a/etc/taskcluster/README.md b/etc/taskcluster/README.md deleted file mode 100644 index e91b526dfcc..00000000000 --- a/etc/taskcluster/README.md +++ /dev/null @@ -1,277 +0,0 @@ -# Testing Servo on Taskcluster - -## In-tree and out-of-tree configuration - -Where possible, we prefer keeping Taskcluster-related configuration and code in this directory, -set up CI so that testing of a given git branch uses the version in that branch. -That way, anyone can make changes (such installing a new system dependency -[in a `Dockerfile`](#docker-images)) in the same PR that relies on those changes. - -For some things however that is not practical, -or some deployment step that mutates global state is required. -That configuration is split between the [mozilla/community-tc-config] and -[servo/taskcluster-config] repositories, -managed by the Taskcluster team and the Servo team respectively. - -[mozilla/community-tc-config]: https://github.com/mozilla/community-tc-config/blob/master/config/projects.yml -[servo/taskcluster-config]: https://github.com/servo/taskcluster-config/tree/master/config - - -## Homu - -When a pull request is reviewed and the appropriate command is given, -[Homu] creates a merge commit of `master` and the PR’s branch, and pushes it to the `auto` branch. -One or more CI system (through their own means) get notified of this push by GitHub, -start testing the merge commit, and use the [GitHub Status API] to report results. - -Through a [Webhook], Homu gets notified of changes to these statues. -If all of the required statuses are reported successful, -Homu pushes its merge commit to the `master` branch -and goes on to testing the next pull request in its queue. - -[Homu]: https://github.com/servo/servo/wiki/Homu -[GitHub Status API]: https://developer.github.com/v3/repos/statuses/ -[Webhook]: https://developer.github.com/webhooks/ - - -## Taskcluster − GitHub integration - -Taskcluster is very flexible and not necessarily tied to GitHub, -but it does have an optional [GitHub integration service] that you can enable -on a repository [as a GitHub App]. -When enabled, this service gets notified for every push, pull request, or GitHub release. -It then schedules some tasks based on reading [`.taskcluster.yml`] in the corresponding commit. - -This file contains templates for creating one or more tasks, -but the logic it can support is fairly limited. -So a common pattern is to have it only run a single initial task called a *decision task* -that can have complex logic based on code and data in the repository -to build an arbitrary [task graph]. - -[GitHub integration service]: https://community-tc.services.mozilla.com/docs/manual/using/github -[as a GitHub App]: https://github.com/apps/community-tc-integration/ -[`.taskcluster.yml`]: https://community-tc.services.mozilla.com/docs/reference/integrations/taskcluster-github/docs/taskcluster-yml-v1 -[task graph]: https://community-tc.services.mozilla.com/docs/manual/using/task-graph - - -## Servo’s decision task - -This repository’s [`.taskcluster.yml`][tc.yml] schedules a single task -that runs the Python 3 script [`etc/taskcluster/decision_task.py`](decision_task.py). -It is called a *decision task* as it is responsible for deciding what other tasks to schedule. - -The Docker image that runs the decision task -is hosted on Docker Hub at [`servobrowser/taskcluster-bootstrap`][hub]. -It is built by [Docker Hub automated builds] based on a `Dockerfile` -in the [`taskcluster-bootstrap-docker-images`] GitHub repository. -Hopefully, this image does not need to be modified often -as it only needs to clone the repository and run Python. - -[tc.yml]: ../../../.taskcluster.yml -[hub]: https://hub.docker.com/r/servobrowser/taskcluster-bootstrap/ -[Docker Hub automated builds]: https://docs.docker.com/docker-hub/builds/ -[`taskcluster-bootstrap-docker-images`]: https://github.com/servo/taskcluster-bootstrap-docker-images/ - - -## Docker images - -[Similar to Firefox][firefox], Servo’s decision task supports running other tasks -in Docker images built on-demand, based on `Dockerfile`s in the main repository. -Modifying a `Dockerfile` and relying on those new changes -can be done in the same pull request or commit. - -To avoid rebuilding images on every pull request, -they are cached based on a hash of the source `Dockerfile`. -For now, to support this hashing, we make `Dockerfile`s be self-contained (with one exception). -Images are built without a [context], -so instructions like [`COPY`] cannot be used because there is nothing to copy from. -The exception is that the decision task adds support for a non-standard include directive: -when a `Dockerfile` first line is `% include` followed by a filename, -that line is replaced with the content of that file. - -For example, -[`etc/taskcluster/docker/build.dockerfile`](docker/build.dockerfile) starts like so: - -```Dockerfile -% include base.dockerfile - -RUN \ - apt-get install -qy --no-install-recommends \ -# […] -``` - -[firefox]: https://firefox-source-docs.mozilla.org/taskcluster/taskcluster/docker-images.html -[context]: https://docs.docker.com/engine/reference/commandline/build/#extended-description -[`COPY`]: https://docs.docker.com/engine/reference/builder/#copy - - -## Build artifacts - -[web-platform-tests] (WPT) is large enough that running all of a it takes a long time. -So it supports *chunking*, -such as multiple chunks of the test suite can be run in parallel on different machines. -As of this writing, -Servo’s current Buildbot setup for this has each machine start by compiling its own copy of Servo. -On Taskcluster with a decision task, -we can have a single build task save its resulting binary executable as an [artifact], -together with multiple testing tasks that each depend on the build task -(wait until it successfully finishes before they can start) -and start by downloading the artifact that was saved earlier. - -The logic for all this is in [`decision_task.py`](decision_task.py) -and can be modified in any pull request. - -[web-platform-tests]: https://github.com/web-platform-tests/wpt -[artifact]: https://community-tc.services.mozilla.com/docs/manual/using/artifacts - - -## Log artifacts - -Taskcluster automatically save the `stdio` output of a task as an artifact, -and has special support for showing and streaming that output while the task is still running. - -Servo’s decision task additionally looks for `*.log` arguments to its tasks’s commands, -assumes they instruct a program to create a log file with that name, -and saves those log files as individual artifacts. - -For example, WPT tasks have a `filtered-wpt-errorsummary.log` artifact -that is typically the most relevant output when such a task fails. - - -## Scopes and roles - -[Scopes] are what Taskcluster calls permissions. -They control access to everything. - -Anyone logged in in the [web UI] has (access to) a set of scopes, -which is visible on the [credentials] page -(reachable from clicking on one’s own name on the top-right of any page). - -A running task has a set of scopes allowing it access to various functionality and APIs. -It can grant those scopes (and at most only those) to sub-tasks that it schedules -(if it has the scope allowing it to schedule new tasks in the first place). - -[Roles] represent each a set of scopes. -They can be granted to… things, -and then configured separately to modify what scopes they [expand] to. - -For example, when Taskcluster-GitHub schedules tasks based on the `.taskcluster.yml` file -in a push to the `auto` branch of this repository, -those tasks are granted the scope `assume:repo:github.com/servo/servo:branch:auto`. -Scopes that start with `assume:` are special, -they expand to the scopes defined in the matching roles. -In this case, the [`repo:github.com/servo/servo:branch:*`][branches] role matches. - -The [`project:servo:decision-task/base`][base] -and [`project:servo:decision-task/trusted`][trusted] roles -centralize the set of scopes granted to the decision task. -This avoids maintaining them seprately in the `repo:…` roles, -in the `hook-id:…` role, -and in the `taskcluster.yml` file. -Only the `base` role is granted to tasks executed when a pull request is opened. -These tasks are less trusted because they run before the code has been reviewed, -and anyone can open a PR. - -Members of the [@servo/taskcluster-admins] GitHub team are granted -the scope `assume:project-admin:servo`, which is necessary to deploy changes -to those roles from the [servo/taskcluster-config] repository. - -[Scopes]: https://community-tc.services.mozilla.com/docs/manual/design/apis/hawk/scopes -[web UI]: https://community-tc.services.mozilla.com/ -[credentials]: https://community-tc.services.mozilla.com/profile -[Roles]: https://community-tc.services.mozilla.com/docs/manual/design/apis/hawk/roles -[expand]: https://community-tc.services.mozilla.com/docs/reference/platform/taskcluster-auth/docs/roles -[branches]: https://community-tc.services.mozilla.com/auth/roles/repo%3Agithub.com%2Fservo%2Fservo%3Abranch%3A* -[base]: https://community-tc.services.mozilla.com/auth/roles/project%3Aservo%3Adecision-task%2Fbase -[trusted]: https://community-tc.services.mozilla.com/auth/roles/project%3Aservo%3Adecision-task%2Ftrusted -[@servo/taskcluster-admins]: https://github.com/orgs/servo/teams/taskcluster-admins - - -## Daily tasks - -The [`project-servo/daily`] hook in Taskcluster’s [Hooks service] -is used to run some tasks automatically ever 24 hours. -In this case as well we use a decision task. -The `decision_task.py` script can differentiate this from a GitHub push -based on the `$TASK_FOR` environment variable. -Daily tasks can also be triggered manually. - -Scopes available to the daily decision task need to be both requested in the hook definition -and granted through the [`hook-id:project-servo/daily`] role. - -Because they do not have something similar to GitHub statuses that link to them, -daily tasks are indexed under the [`project.servo.daily`] namespace. - -[`project.servo.daily`]: https://tools.taskcluster.net/index/project.servo.daily -[`project-servo/daily`]: https://github.com/servo/taskcluster-config/blob/master/config/hooks.yml -[Hooks service]: https://community-tc.services.mozilla.com/docs/manual/using/scheduled-tasks -[`hook-id:project-servo/daily`]: https://github.com/servo/taskcluster-config/blob/master/config/roles.yml - - -## Servo’s worker pools - -Each task is assigned to a “worker pool”. -Servo has several, for the different environments a task can run in: - -* `docker` and `docker-untrusted` provide a Linux environment with full `root` privileges, - in a Docker container running a [Docker image](#docker-images) of the task’s choice, - in a short-lived virtual machine, - on Google Cloud Platform. - - Instances are started automatically as needed - when the existing capacity is insufficient to execute queued tasks. - They terminate themselves after being idle without work for a while, - or unconditionally after a few days. - Because these workers are short-lived, - we don’t need to worry about evicting old entries from Cargo’s or rustup’s download cache, - for example. - - [The Taskcluster team manages][mozilla/community-tc-config] - the configuration and VM image for these two pools. - The latter has fewer scopes. It runs automated testing of pull requests - as soon as they’re opened or updated, before any review. - -* `win2016` runs Windows Server 2016 on AWS EC2. - Like with Docker tasks, workers are short-lived and started automatically. - The configuration and VM image for this pool - is [managed by the Servo team][servo/taskcluster-config]. - - Tasks run as an unprivileged user. - Because creating an new the VM image is slow and deploying it mutates global state, - when a tool does not require system-wide installation - we prefer having each task obtain it as needed by extracting an archive in a directory. - See calls of `with_directory_mount` and `with_repacked_msi` in - [`decision_task.py`](decision_task.py) and [`decisionlib.py`](decisionlib.py). - -* `macos` runs, you guessed it, macOS. - Tasks run on dedicated hardware provided long-term by Macstadium. - The system-wide configuration of those machines - is [managed by the Servo team][servo/taskcluster-config] through SaltStack. - There is a task-owned (but preserved across tasks) install of Homebrew, - with `Brewfile`s [in this repository](macos/). - - This [Workers] page lists the current state of each macOS worker. - (A similar page exists for other each worker pools, but as of this writing it has - [usability issues](https://github.com/taskcluster/taskcluster/issues/1972) - with short-lived workers.) - -[Workers]: https://community-tc.services.mozilla.com/provisioners/proj-servo/worker-types/macos - - -## Taskcluster − Treeherder integration - -See [`treeherder.md`](treeherder.md). - - -## Self-service, IRC, and Bugzilla - -Taskcluster is designed to be “self-service” as much as possible. -Between this repository [servo/taskcluster-config] and [mozilla/community-tc-config], -anyone should be able to submit PRs for any part of the configuration. - -Feel free to ask for help on the `#servo` or `#taskcluster` channels on Mozilla IRC. - -For issue reports or feature requests on various bits of Taskcluster *software*, -file bugs [in Mozilla’s Bugzilla, under `Taskcluster`][bug]. - -[bug]: https://bugzilla.mozilla.org/enter_bug.cgi?product=Taskcluster diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py deleted file mode 100644 index 89039137cb7..00000000000 --- a/etc/taskcluster/decision_task.py +++ /dev/null @@ -1,103 +0,0 @@ -# coding: utf8 - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -import os.path -import decisionlib -import functools -from decisionlib import CONFIG, SHARED -from urllib.request import urlopen - - -def main(task_for): - with decisionlib.make_repo_bundle(): - tasks(task_for) - - -def tasks(task_for): - if CONFIG.git_ref.startswith("refs/heads/"): - branch = CONFIG.git_ref[len("refs/heads/"):] - CONFIG.treeherder_repository_name = "servo-" + ( - branch if not branch.startswith("try-") else "try" - ) - - # Work around a tc-github bug/limitation: - # https://bugzilla.mozilla.org/show_bug.cgi?id=1548781#c4 - if task_for.startswith("github"): - # https://github.com/taskcluster/taskcluster/blob/21f257dc8/services/github/config.yml#L14 - CONFIG.routes_for_all_subtasks.append("statuses") - - if task_for == "github-push": - all_tests = [] - by_branch_name = { - "auto": all_tests, - "try": all_tests, - "try-taskcluster": [ - # Add functions here as needed, in your push to that branch - ], - "master": [], - - # The "try-*" keys match those in `servo_try_choosers` in Homu’s config: - # https://github.com/servo/saltfs/blob/master/homu/map.jinja - - "try-mac": [], - "try-linux": [], - "try-windows": [], - "try-arm": [], - "try-wpt": [], - "try-wpt-2020": [], - "try-wpt-mac": [], - "test-wpt": [], - } - - elif task_for == "github-pull-request": - CONFIG.treeherder_repository_name = "servo-prs" - CONFIG.index_read_only = True - CONFIG.docker_image_build_worker_type = None - - # We want the merge commit that GitHub creates for the PR. - # The event does contain a `pull_request.merge_commit_sha` key, but it is wrong: - # https://github.com/servo/servo/pull/22597#issuecomment-451518810 - CONFIG.git_sha_is_current_head() - - elif task_for == "try-windows-ami": - CONFIG.git_sha_is_current_head() - CONFIG.windows_worker_type = os.environ["NEW_AMI_WORKER_TYPE"] - - # https://tools.taskcluster.net/hooks/project-servo/daily - elif task_for == "daily": - daily_tasks_setup() - - -ping_on_daily_task_failure = "SimonSapin, nox, emilio" -build_artifacts_expire_in = "1 week" -build_dependencies_artifacts_expire_in = "1 month" -log_artifacts_expire_in = "1 year" - - -def daily_tasks_setup(): - # Unlike when reacting to a GitHub push event, - # the commit hash is not known until we clone the repository. - CONFIG.git_sha_is_current_head() - - # On failure, notify a few people on IRC - # https://docs.taskcluster.net/docs/reference/core/taskcluster-notify/docs/usage - notify_route = "notify.irc-channel.#servo.on-failed" - CONFIG.routes_for_all_subtasks.append(notify_route) - CONFIG.scopes_for_all_subtasks.append("queue:route:" + notify_route) - CONFIG.task_name_template = "Servo daily: %s. On failure, ping: " + ping_on_daily_task_failure - - -CONFIG.task_name_template = "Servo: %s" -CONFIG.docker_images_expire_in = build_dependencies_artifacts_expire_in -CONFIG.repacked_msi_files_expire_in = build_dependencies_artifacts_expire_in -CONFIG.index_prefix = "project.servo" -CONFIG.default_provisioner_id = "proj-servo" -CONFIG.docker_image_build_worker_type = "docker" - -CONFIG.windows_worker_type = "win2016" - -if __name__ == "__main__": # pragma: no cover - main(task_for=os.environ["TASK_FOR"]) diff --git a/etc/taskcluster/decisionlib.py b/etc/taskcluster/decisionlib.py deleted file mode 100644 index 22d9a2f22a0..00000000000 --- a/etc/taskcluster/decisionlib.py +++ /dev/null @@ -1,851 +0,0 @@ -# coding: utf8 - -# Copyright 2018 The Servo Project Developers. See the COPYRIGHT -# file at the top-level directory of this distribution. -# -# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or -# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license -# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your -# option. This file may not be copied, modified, or distributed -# except according to those terms. - -""" -Project-independent library for Taskcluster decision tasks -""" - -import base64 -import contextlib -import datetime -import hashlib -import json -import os -import re -import subprocess -import sys -import taskcluster - - -# Public API -__all__ = [ - "CONFIG", "SHARED", "Task", "DockerWorkerTask", - "GenericWorkerTask", "WindowsGenericWorkerTask", - "make_repo_bundle", -] - - -class Config: - """ - Global configuration, for users of the library to modify. - """ - def __init__(self): - self.task_name_template = "%s" - self.index_prefix = "garbage.servo-decisionlib" - self.index_read_only = False - self.scopes_for_all_subtasks = [] - self.routes_for_all_subtasks = [] - self.docker_image_build_worker_type = None - self.docker_images_expire_in = "1 month" - self.repacked_msi_files_expire_in = "1 month" - self.treeherder_repository_name = None - - # Set by docker-worker: - # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/environment - self.decision_task_id = os.environ.get("TASK_ID") - - # Set in the decision task’s payload, such as defined in .taskcluster.yml - self.task_owner = os.environ.get("TASK_OWNER") - self.task_source = os.environ.get("TASK_SOURCE") - self.git_url = os.environ.get("GIT_URL") - self.git_ref = os.environ.get("GIT_REF") - self.git_sha = os.environ.get("GIT_SHA") - self.git_bundle_shallow_ref = "refs/heads/shallow" - - self.tc_root_url = os.environ.get("TASKCLUSTER_ROOT_URL") - self.default_provisioner_id = "proj-example" - - - def tree_hash(self): - if not hasattr(self, "_tree_hash"): - # Use the SHA-1 hash of the git "tree" object rather than the commit. - # A `@bors-servo retry` command creates a new merge commit with a different commit hash - # but with the same tree hash. - output = subprocess.check_output(["git", "show", "-s", "--format=%T", "HEAD"]) - self._tree_hash = output.decode("utf-8").strip() - return self._tree_hash - - def git_sha_is_current_head(self): - output = subprocess.check_output(["git", "rev-parse", "HEAD"]) - self.git_sha = output.decode("utf8").strip() - - -class Shared: - """ - Global shared state. - """ - def __init__(self): - self.now = datetime.datetime.utcnow() - self.found_or_created_indexed_tasks = {} - - options = {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]} - self.queue_service = taskcluster.Queue(options) - self.index_service = taskcluster.Index(options) - - def from_now_json(self, offset): - """ - Same as `taskcluster.fromNowJSON`, but uses the creation time of `self` for “now”. - """ - return taskcluster.stringDate(taskcluster.fromNow(offset, dateObj=self.now)) - - -CONFIG = Config() -SHARED = Shared() - - -def chaining(op, attr): - def method(self, *args, **kwargs): - op(self, attr, *args, **kwargs) - return self - return method - - -def append_to_attr(self, attr, *args): getattr(self, attr).extend(args) -def prepend_to_attr(self, attr, *args): getattr(self, attr)[0:0] = list(args) -def update_attr(self, attr, **kwargs): getattr(self, attr).update(kwargs) - - -class Task: - """ - A task definition, waiting to be created. - - Typical is to use chain the `with_*` methods to set or extend this object’s attributes, - then call the `crate` or `find_or_create` method to schedule a task. - - This is an abstract class that needs to be specialized for different worker implementations. - """ - def __init__(self, name): - self.name = name - self.description = "" - self.scheduler_id = "taskcluster-github" - self.provisioner_id = CONFIG.default_provisioner_id - self.worker_type = "github-worker" - self.deadline_in = "1 day" - self.expires_in = "1 year" - self.index_and_artifacts_expire_in = self.expires_in - self.dependencies = [] - self.scopes = [] - self.routes = [] - self.extra = {} - self.treeherder_required = False - self.priority = None # Defaults to 'lowest' - self.git_fetch_url = CONFIG.git_url - self.git_fetch_ref = CONFIG.git_ref - self.git_checkout_sha = CONFIG.git_sha - - # All `with_*` methods return `self`, so multiple method calls can be chained. - with_description = chaining(setattr, "description") - with_scheduler_id = chaining(setattr, "scheduler_id") - with_provisioner_id = chaining(setattr, "provisioner_id") - with_worker_type = chaining(setattr, "worker_type") - with_deadline_in = chaining(setattr, "deadline_in") - with_expires_in = chaining(setattr, "expires_in") - with_index_and_artifacts_expire_in = chaining(setattr, "index_and_artifacts_expire_in") - with_priority = chaining(setattr, "priority") - - with_dependencies = chaining(append_to_attr, "dependencies") - with_scopes = chaining(append_to_attr, "scopes") - with_routes = chaining(append_to_attr, "routes") - - with_extra = chaining(update_attr, "extra") - - def with_index_at(self, index_path): - self.routes.append("index.%s.%s" % (CONFIG.index_prefix, index_path)) - return self - - def with_treeherder_required(self): - self.treeherder_required = True - return self - - def with_treeherder(self, category, symbol, group_name=None, group_symbol=None): - assert len(symbol) <= 25, symbol - self.name = "%s: %s" % (category, self.name) - - # The message schema does not allow spaces in the platfrom or in labels, - # but the UI shows them in that order separated by spaces. - # So massage the metadata to get the UI to show the string we want. - # `labels` defaults to ["opt"] if not provided or empty, - # so use a more neutral underscore instead. - parts = category.split(" ") - platform = parts[0] - labels = parts[1:] or ["_"] - - # https://github.com/mozilla/treeherder/blob/master/schemas/task-treeherder-config.yml - self.with_extra(treeherder=dict_update_if_truthy( - { - "machine": {"platform": platform}, - "labels": labels, - "symbol": symbol, - }, - groupName=group_name, - groupSymbol=group_symbol, - )) - - if CONFIG.treeherder_repository_name: - assert CONFIG.git_sha - suffix = ".v2._/%s.%s" % (CONFIG.treeherder_repository_name, CONFIG.git_sha) - self.with_routes( - "tc-treeherder" + suffix, - "tc-treeherder-staging" + suffix, - ) - - self.treeherder_required = False # Taken care of - return self - - def build_worker_payload(self): # pragma: no cover - """ - Overridden by sub-classes to return a dictionary in a worker-specific format, - which is used as the `payload` property in a task definition request - passed to the Queue’s `createTask` API. - - <https://docs.taskcluster.net/docs/reference/platform/taskcluster-queue/references/api#createTask> - """ - raise NotImplementedError - - def create(self): - """ - Call the Queue’s `createTask` API to schedule a new task, and return its ID. - - <https://docs.taskcluster.net/docs/reference/platform/taskcluster-queue/references/api#createTask> - """ - worker_payload = self.build_worker_payload() - assert not self.treeherder_required, \ - "make sure to call with_treeherder() for this task: %s" % self.name - - assert CONFIG.decision_task_id - assert CONFIG.task_owner - assert CONFIG.task_source - - def dedup(xs): - seen = set() - return [x for x in xs if not (x in seen or seen.add(x))] - - queue_payload = { - "taskGroupId": CONFIG.decision_task_id, - "dependencies": dedup([CONFIG.decision_task_id] + self.dependencies), - "schedulerId": self.scheduler_id, - "provisionerId": self.provisioner_id, - "workerType": self.worker_type, - - "created": SHARED.from_now_json(""), - "deadline": SHARED.from_now_json(self.deadline_in), - "expires": SHARED.from_now_json(self.expires_in), - "metadata": { - "name": CONFIG.task_name_template % self.name, - "description": self.description, - "owner": CONFIG.task_owner, - "source": CONFIG.task_source, - }, - - "payload": worker_payload, - } - scopes = self.scopes + CONFIG.scopes_for_all_subtasks - routes = self.routes + CONFIG.routes_for_all_subtasks - if any(r.startswith("index.") for r in routes): - self.extra.setdefault("index", {})["expires"] = \ - SHARED.from_now_json(self.index_and_artifacts_expire_in) - dict_update_if_truthy( - queue_payload, - scopes=scopes, - routes=routes, - extra=self.extra, - priority=self.priority, - ) - - task_id = taskcluster.slugId() - SHARED.queue_service.createTask(task_id, queue_payload) - print("Scheduled %s: %s" % (task_id, self.name)) - return task_id - - @staticmethod - def find(index_path): - full_index_path = "%s.%s" % (CONFIG.index_prefix, index_path) - task_id = SHARED.index_service.findTask(full_index_path)["taskId"] - print("Found task %s indexed at %s" % (task_id, full_index_path)) - return task_id - - def find_or_create(self, index_path): - """ - Try to find a task in the Index and return its ID. - - The index path used is `{CONFIG.index_prefix}.{index_path}`. - `index_path` defaults to `by-task-definition.{sha256}` - with a hash of the worker payload and worker type. - - If no task is found in the index, - it is created with a route to add it to the index at that same path if it succeeds. - - <https://docs.taskcluster.net/docs/reference/core/taskcluster-index/references/api#findTask> - """ - task_id = SHARED.found_or_created_indexed_tasks.get(index_path) - if task_id is not None: - return task_id - - try: - task_id = Task.find(index_path) - except taskcluster.TaskclusterRestFailure as e: - if e.status_code != 404: # pragma: no cover - raise - if not CONFIG.index_read_only: - self.with_index_at(index_path) - task_id = self.create() - - SHARED.found_or_created_indexed_tasks[index_path] = task_id - return task_id - - def with_curl_script(self, url, file_path): - return self \ - .with_script(""" - curl --compressed --retry 5 --connect-timeout 10 -Lf "%s" -o "%s" - """ % (url, file_path)) - - def with_curl_artifact_script(self, task_id, artifact_name, out_directory=""): - queue_service = CONFIG.tc_root_url + "/api/queue" - return self \ - .with_dependencies(task_id) \ - .with_curl_script( - queue_service + "/v1/task/%s/artifacts/public/%s" % (task_id, artifact_name), - os.path.join(out_directory, url_basename(artifact_name)), - ) - - def with_repo_bundle(self, **kwargs): - self.git_fetch_url = "../repo.bundle" - self.git_fetch_ref = CONFIG.git_bundle_shallow_ref - self.git_checkout_sha = "FETCH_HEAD" - return self \ - .with_curl_artifact_script(CONFIG.decision_task_id, "repo.bundle") \ - .with_repo(**kwargs) - - -class GenericWorkerTask(Task): - """ - Task definition for a worker type that runs the `generic-worker` implementation. - - This is an abstract class that needs to be specialized for different operating systems. - - <https://github.com/taskcluster/generic-worker> - """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.max_run_time_minutes = 30 - self.env = {} - self.features = {} - self.mounts = [] - self.artifacts = [] - - with_max_run_time_minutes = chaining(setattr, "max_run_time_minutes") - with_mounts = chaining(append_to_attr, "mounts") - with_env = chaining(update_attr, "env") - - def build_command(self): # pragma: no cover - """ - Overridden by sub-classes to return the `command` property of the worker payload, - in the format appropriate for the operating system. - """ - raise NotImplementedError - - def build_worker_payload(self): - """ - Return a `generic-worker` worker payload. - - <https://docs.taskcluster.net/docs/reference/workers/generic-worker/docs/payload> - """ - worker_payload = { - "command": self.build_command(), - "maxRunTime": self.max_run_time_minutes * 60 - } - return dict_update_if_truthy( - worker_payload, - env=self.env, - mounts=self.mounts, - features=self.features, - artifacts=[ - { - "type": type_, - "path": path, - "name": "public/" + url_basename(path), - "expires": SHARED.from_now_json(self.index_and_artifacts_expire_in), - } - for type_, path in self.artifacts - ], - ) - - def with_artifacts(self, *paths, type="file"): - """ - Add each path in `paths` as a task artifact - that expires in `self.index_and_artifacts_expire_in`. - - `type` can be `"file"` or `"directory"`. - - Paths are relative to the task’s home directory. - """ - for path in paths: - if (type, path) in self.artifacts: - raise ValueError("Duplicate artifact: " + path) # pragma: no cover - self.artifacts.append(tuple((type, path))) - return self - - def with_features(self, *names): - """ - Enable the given `generic-worker` features. - - <https://github.com/taskcluster/generic-worker/blob/master/native_windows.yml> - """ - self.features.update({name: True for name in names}) - return self - - def _mount_content(self, url_or_artifact_name, task_id, sha256): - if task_id: - content = {"taskId": task_id, "artifact": url_or_artifact_name} - else: - content = {"url": url_or_artifact_name} - if sha256: - content["sha256"] = sha256 - return content - - def with_file_mount(self, url_or_artifact_name, task_id=None, sha256=None, path=None): - """ - Make `generic-worker` download a file before the task starts - and make it available at `path` (which is relative to the task’s home directory). - - If `sha256` is provided, `generic-worker` will hash the downloaded file - and check it against the provided signature. - - If `task_id` is provided, this task will depend on that task - and `url_or_artifact_name` is the name of an artifact of that task. - """ - return self.with_mounts({ - "file": path or url_basename(url_or_artifact_name), - "content": self._mount_content(url_or_artifact_name, task_id, sha256), - }) - - def with_directory_mount(self, url_or_artifact_name, task_id=None, sha256=None, path=None): - """ - Make `generic-worker` download an archive before the task starts, - and uncompress it at `path` (which is relative to the task’s home directory). - - `url_or_artifact_name` must end in one of `.rar`, `.tar.bz2`, `.tar.gz`, or `.zip`. - The archive must be in the corresponding format. - - If `sha256` is provided, `generic-worker` will hash the downloaded archive - and check it against the provided signature. - - If `task_id` is provided, this task will depend on that task - and `url_or_artifact_name` is the name of an artifact of that task. - """ - supported_formats = ["rar", "tar.bz2", "tar.gz", "zip"] - for fmt in supported_formats: - suffix = "." + fmt - if url_or_artifact_name.endswith(suffix): - return self.with_mounts({ - "directory": path or url_basename(url_or_artifact_name[:-len(suffix)]), - "content": self._mount_content(url_or_artifact_name, task_id, sha256), - "format": fmt, - }) - raise ValueError( - "%r does not appear to be in one of the supported formats: %r" - % (url_or_artifact_name, ", ".join(supported_formats)) - ) # pragma: no cover - - -class WindowsGenericWorkerTask(GenericWorkerTask): - """ - Task definition for a `generic-worker` task running on Windows. - - Scripts are written as `.bat` files executed with `cmd.exe`. - """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.scripts = [] - self.rdp_info_artifact_name = None - - with_script = chaining(append_to_attr, "scripts") - with_early_script = chaining(prepend_to_attr, "scripts") - - def build_worker_payload(self): - if self.rdp_info_artifact_name: - rdp_scope = "generic-worker:allow-rdp:%s/%s" % (self.provisioner_id, self.worker_type) - self.scopes.append(rdp_scope) - self.scopes.append("generic-worker:os-group:proj-servo/win2016/Administrators") - self.scopes.append("generic-worker:run-as-administrator:proj-servo/win2016") - self.with_features("runAsAdministrator") - return dict_update_if_truthy( - super().build_worker_payload(), - rdpInfo=self.rdp_info_artifact_name, - osGroups=["Administrators"] - ) - - def with_rdp_info(self, *, artifact_name): - """ - Enable RDP access to this task’s environment. - - See `rdpInfo` in - <https://community-tc.services.mozilla.com/docs/reference/workers/generic-worker/multiuser-windows-payload> - """ - assert not artifact_name.startswith("public/") - self.rdp_info_artifact_name = artifact_name - - def build_command(self): - return [deindent(s) for s in self.scripts] - - def with_path_from_homedir(self, *paths): - """ - Interpret each path in `paths` as relative to the task’s home directory, - and add it to the `PATH` environment variable. - """ - for p in paths: - self.with_early_script("set PATH=%HOMEDRIVE%%HOMEPATH%\\{};%PATH%".format(p)) - return self - - def with_repo(self, sparse_checkout=None): - """ - Make a clone the git repository at the start of the task. - This uses `CONFIG.git_url`, `CONFIG.git_ref`, and `CONFIG.git_sha`, - and creates the clone in a `repo` directory in the task’s home directory. - - If `sparse_checkout` is given, it must be a list of path patterns - to be used in `.git/info/sparse-checkout`. - See <https://git-scm.com/docs/git-read-tree#_sparse_checkout>. - """ - git = """ - git init repo - cd repo - """ - if sparse_checkout: - self.with_mounts({ - "file": "sparse-checkout", - "content": {"raw": "\n".join(sparse_checkout)}, - }) - git += """ - git config core.sparsecheckout true - copy ..\\sparse-checkout .git\\info\\sparse-checkout - type .git\\info\\sparse-checkout - """ - git += """ - git fetch --no-tags {} {} - git reset --hard {} - """.format( - assert_truthy(self.git_fetch_url), - assert_truthy(self.git_fetch_ref), - assert_truthy(self.git_checkout_sha), - ) - return self \ - .with_git() \ - .with_script(git) - - def with_git(self): - """ - Make the task download `git-for-windows` and make it available for `git` commands. - - This is implied by `with_repo`. - """ - return self \ - .with_path_from_homedir("git\\cmd") \ - .with_directory_mount( - "https://github.com/git-for-windows/git/releases/download/" + - "v2.24.0.windows.2/MinGit-2.24.0.2-64-bit.zip", - sha256="c33aec6ae68989103653ca9fb64f12cabccf6c61d0dde30c50da47fc15cf66e2", - path="git", - ) - - def with_curl_script(self, url, file_path): - self.with_curl() - return super().with_curl_script(url, file_path) - - def with_curl(self): - return self \ - .with_path_from_homedir("curl\\curl-7.73.0-win64-mingw\\bin") \ - .with_directory_mount( - "https://curl.haxx.se/windows/dl-7.73.0/curl-7.73.0-win64-mingw.zip", - sha256="2e1ffdb6c25c8648a1243bb3a268120be442399b1c93d7da309bba235ecdab9a", - path="curl", - ) - - def with_rustup(self): - """ - Download rustup.rs and make it available to task commands, - but does not download any default toolchain. - """ - return self \ - .with_path_from_homedir(".cargo\\bin") \ - .with_early_script( - "%HOMEDRIVE%%HOMEPATH%\\rustup-init.exe --default-toolchain none --profile=minimal -y" - ) \ - .with_file_mount("https://win.rustup.rs/x86_64", path="rustup-init.exe") - - def with_repacked_msi(self, url, sha256, path): - """ - Download an MSI file from `url`, extract the files in it with `lessmsi`, - and make them available in the directory at `path` (relative to the task’s home directory). - - `sha256` is required and the MSI file must have that hash. - - The file extraction (and recompression in a ZIP file) is done in a separate task, - wich is indexed based on `sha256` and cached for `CONFIG.repacked_msi_files_expire_in`. - - <https://github.com/activescott/lessmsi> - """ - repack_task = ( - WindowsGenericWorkerTask("MSI repack: " + url) - .with_worker_type(self.worker_type) - .with_max_run_time_minutes(20) - .with_file_mount(url, sha256=sha256, path="input.msi") - .with_directory_mount( - "https://github.com/activescott/lessmsi/releases/download/" + - "v1.6.1/lessmsi-v1.6.1.zip", - sha256="540b8801e08ec39ba26a100c855898f455410cecbae4991afae7bb2b4df026c7", - path="lessmsi" - ) - .with_directory_mount( - "https://www.7-zip.org/a/7za920.zip", - sha256="2a3afe19c180f8373fa02ff00254d5394fec0349f5804e0ad2f6067854ff28ac", - path="7zip", - ) - .with_path_from_homedir("lessmsi", "7zip") - .with_script(""" - lessmsi x input.msi extracted\\ - cd extracted\\SourceDir - 7za a repacked.zip * - """) - .with_artifacts("extracted/SourceDir/repacked.zip") - .with_index_and_artifacts_expire_in(CONFIG.repacked_msi_files_expire_in) - .find_or_create("repacked-msi." + sha256) - ) - return self \ - .with_dependencies(repack_task) \ - .with_directory_mount("public/repacked.zip", task_id=repack_task, path=path) - - def with_python3(self): - """ - For Python 3, use `with_directory_mount` and the "embeddable zip file" distribution - from python.org. - You may need to remove `python37._pth` from the ZIP in order to work around - <https://bugs.python.org/issue34841>. - """ - return ( - self - .with_curl_script( - "https://www.python.org/ftp/python/3.7.3/python-3.7.3-amd64.exe", - "do-the-python.exe" - ) - .with_script("do-the-python.exe /quiet TargetDir=%HOMEDRIVE%%HOMEPATH%\\python3") - .with_path_from_homedir("python3", "python3\\Scripts") - .with_script("pip install virtualenv==20.2.1") - ) - - -class UnixTaskMixin(Task): - def with_repo(self, alternate_object_dir=""): - """ - Make a clone the git repository at the start of the task. - This uses `CONFIG.git_url`, `CONFIG.git_ref`, and `CONFIG.git_sha` - - * generic-worker: creates the clone in a `repo` directory - in the task’s directory. - - * docker-worker: creates the clone in a `/repo` directory - at the root of the Docker container’s filesystem. - `git` and `ca-certificate` need to be installed in the Docker image. - - """ - # Not using $GIT_ALTERNATE_OBJECT_DIRECTORIES since it causes - # "object not found - no match for id" errors when Cargo fetches git dependencies - return self \ - .with_script(""" - git init repo - cd repo - echo "{alternate}" > .git/objects/info/alternates - time git fetch --no-tags {} {} - time git reset --hard {} - """.format( - assert_truthy(self.git_fetch_url), - assert_truthy(self.git_fetch_ref), - assert_truthy(self.git_checkout_sha), - alternate=alternate_object_dir, - )) - - -class DockerWorkerTask(UnixTaskMixin, Task): - """ - Task definition for a worker type that runs the `generic-worker` implementation. - - Scripts are interpreted with `bash`. - - <https://github.com/taskcluster/docker-worker> - """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.docker_image = "ubuntu:bionic-20180821" - self.max_run_time_minutes = 30 - self.scripts = [] - self.env = {} - self.caches = {} - self.features = {} - self.capabilities = {} - self.artifacts = [] - - with_docker_image = chaining(setattr, "docker_image") - with_max_run_time_minutes = chaining(setattr, "max_run_time_minutes") - with_script = chaining(append_to_attr, "scripts") - with_early_script = chaining(prepend_to_attr, "scripts") - with_caches = chaining(update_attr, "caches") - with_env = chaining(update_attr, "env") - with_capabilities = chaining(update_attr, "capabilities") - - def with_artifacts(self, *paths): - for path in paths: - if path in self.artifacts: - raise ValueError("Duplicate artifact: " + path) # pragma: no cover - self.artifacts.append(path) - return self - - def build_worker_payload(self): - """ - Return a `docker-worker` worker payload. - - <https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/payload> - """ - worker_payload = { - "image": self.docker_image, - "maxRunTime": self.max_run_time_minutes * 60, - "command": [ - "/bin/bash", "--login", "-x", "-e", "-o", "pipefail", "-c", - deindent("\n".join(self.scripts)) - ], - } - return dict_update_if_truthy( - worker_payload, - env=self.env, - cache=self.caches, - features=self.features, - capabilities=self.capabilities, - artifacts={ - "public/" + url_basename(path): { - "type": "file", - "path": path, - "expires": SHARED.from_now_json(self.index_and_artifacts_expire_in), - } - for path in self.artifacts - }, - ) - - def with_features(self, *names): - """ - Enable the given `docker-worker` features. - - <https://github.com/taskcluster/docker-worker/blob/master/docs/features.md> - """ - self.features.update({name: True for name in names}) - return self - - def with_dockerfile(self, dockerfile): - """ - Build a Docker image based on the given `Dockerfile`, and use it for this task. - - `dockerfile` is a path in the filesystem where this code is running. - Some non-standard syntax is supported, see `expand_dockerfile`. - - The image is indexed based on a hash of the expanded `Dockerfile`, - and cached for `CONFIG.docker_images_expire_in`. - - Images are built without any *context*. - <https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#understand-build-context> - """ - basename = os.path.basename(dockerfile) - suffix = ".dockerfile" - assert basename.endswith(suffix) - image_name = basename[:-len(suffix)] - - dockerfile_contents = expand_dockerfile(dockerfile) - digest = hashlib.sha256(dockerfile_contents).hexdigest() - - image_build_task = ( - DockerWorkerTask("Docker image: " + image_name) - .with_worker_type(CONFIG.docker_image_build_worker_type or self.worker_type) - .with_max_run_time_minutes(30) - .with_index_and_artifacts_expire_in(CONFIG.docker_images_expire_in) - .with_features("dind") - .with_env(DOCKERFILE=dockerfile_contents) - .with_artifacts("/image.tar.lz4") - .with_script(""" - echo "$DOCKERFILE" | docker build -t taskcluster-built - - docker save taskcluster-built | lz4 > /image.tar.lz4 - """) - .with_docker_image( - # https://github.com/servo/taskcluster-bootstrap-docker-images#image-builder - "servobrowser/taskcluster-bootstrap:image-builder@sha256:" \ - "0a7d012ce444d62ffb9e7f06f0c52fedc24b68c2060711b313263367f7272d9d" - ) - .find_or_create("docker-image." + digest) - ) - - return self \ - .with_dependencies(image_build_task) \ - .with_docker_image({ - "type": "task-image", - "path": "public/image.tar.lz4", - "taskId": image_build_task, - }) - - -def expand_dockerfile(dockerfile): - """ - Read the file at path `dockerfile`, - and transitively expand the non-standard `% include` header if it is present. - """ - with open(dockerfile, "rb") as f: - dockerfile_contents = f.read() - - include_marker = b"% include" - if not dockerfile_contents.startswith(include_marker): - return dockerfile_contents - - include_line, _, rest = dockerfile_contents.partition(b"\n") - included = include_line[len(include_marker):].strip().decode("utf8") - path = os.path.join(os.path.dirname(dockerfile), included) - return b"\n".join([expand_dockerfile(path), rest]) - - -def assert_truthy(x): - assert x - return x - - -def dict_update_if_truthy(d, **kwargs): - for key, value in kwargs.items(): - if value: - d[key] = value - return d - - -def deindent(string): - return re.sub("\n +", "\n ", string).strip() - - -def url_basename(url): - return url.rpartition("/")[-1] - - -@contextlib.contextmanager -def make_repo_bundle(): - subprocess.check_call(["git", "config", "user.name", "Decision task"]) - subprocess.check_call(["git", "config", "user.email", "nobody@mozilla.com"]) - tree = subprocess.check_output(["git", "show", CONFIG.git_sha, "--pretty=%T", "--no-patch"]) - message = "Shallow version of commit " + CONFIG.git_sha - commit = subprocess.check_output(["git", "commit-tree", tree.strip(), "-m", message]) - subprocess.check_call(["git", "update-ref", CONFIG.git_bundle_shallow_ref, commit.strip()]) - subprocess.check_call(["git", "show-ref"]) - create = ["git", "bundle", "create", "../repo.bundle", CONFIG.git_bundle_shallow_ref] - with subprocess.Popen(create) as p: - yield - exit_code = p.wait() - if exit_code: - sys.exit(exit_code) diff --git a/etc/taskcluster/docker/base.dockerfile b/etc/taskcluster/docker/base.dockerfile deleted file mode 100644 index 18a2dbac6a0..00000000000 --- a/etc/taskcluster/docker/base.dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM ubuntu:20.04 - -ENV \ - # - # Some APT packages like 'tzdata' wait for user input on install by default. - # https://stackoverflow.com/questions/44331836/apt-get-install-tzdata-noninteractive - DEBIAN_FRONTEND=noninteractive \ - LANG=C.UTF-8 \ - LANGUAGE=C.UTF-8 \ - LC_ALL=C.UTF-8 - -RUN \ - apt-get update -q && \ - apt-get install -qy --no-install-recommends \ - # - # Cloning the repository - git \ - ca-certificates \ - # - # Running mach with Python 3 - python3 \ - python3-pip \ - python3-dev \ - virtualenv \ - # - # Compiling C modules when installing Python packages in a virtualenv - gcc \ - # - # Installing rustup and sccache (build dockerfile) or fetching build artifacts (run tasks) - curl \ - # Setting the default locale - locales \ - locales-all diff --git a/etc/taskcluster/docker/build.dockerfile b/etc/taskcluster/docker/build.dockerfile deleted file mode 100644 index 60ae5606cdb..00000000000 --- a/etc/taskcluster/docker/build.dockerfile +++ /dev/null @@ -1,53 +0,0 @@ -% include base.dockerfile - -RUN \ - apt-get install -qy --no-install-recommends \ - # - # Testing decisionlib (see etc/taskcluster/mock.py) - python3-coverage \ - # - # Multiple C/C++ dependencies built from source - g++ \ - make \ - cmake \ - # - # Fontconfig - gperf \ - # - # ANGLE - xorg-dev \ - # - # mozjs (SpiderMonkey) - autoconf2.13 \ - # - # Bindgen (for SpiderMonkey bindings) - clang \ - llvm \ - llvm-dev \ - # - # GStreamer - libpcre3-dev \ - # - # OpenSSL - libssl-dev \ - # - # blurz - libdbus-1-dev \ - # - # sampling profiler - libunwind-dev \ - # - # x11 integration - libxcb-render-util0-dev \ - libxcb-shape0-dev \ - libxcb-xfixes0-dev \ - # - && \ - # - # Install the version of rustup that is current when this Docker image is being built: - # We want at least 1.21 (increment in this comment to force an image rebuild). - curl https://sh.rustup.rs -sSf | sh -s -- --profile=minimal -y && \ - # - # There are no sccache binary releases that include this commit, so we install a particular - # git commit instead. - ~/.cargo/bin/cargo install sccache --git https://github.com/mozilla/sccache/ --rev e66c9c15142a7e583d6ab80bd614bdffb2ebcc47 diff --git a/etc/taskcluster/docker/run-android-emulator.dockerfile b/etc/taskcluster/docker/run-android-emulator.dockerfile deleted file mode 100644 index 92eb116ef6b..00000000000 --- a/etc/taskcluster/docker/run-android-emulator.dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -% include base.dockerfile - -RUN \ - apt-get install -qy --no-install-recommends \ - # - # Multiple Android-related tools are in Java - openjdk-8-jdk-headless \ - # - # Emulator dependencies - libgl1 \ - libpulse0 diff --git a/etc/taskcluster/docker/run.dockerfile b/etc/taskcluster/docker/run.dockerfile deleted file mode 100644 index f02b4fdff97..00000000000 --- a/etc/taskcluster/docker/run.dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -% include base.dockerfile - -# Servo’s runtime dependencies: -RUN apt-get install -qy --no-install-recommends \ - libgl1 \ - libssl1.1 \ - libdbus-1-3 \ - libxcb-shape0-dev \ - gstreamer1.0-plugins-good \ - gstreamer1.0-plugins-bad \ - gstreamer1.0-libav \ - gstreamer1.0-gl \ - libunwind8 diff --git a/etc/taskcluster/docker/wpt-update.dockerfile b/etc/taskcluster/docker/wpt-update.dockerfile deleted file mode 100644 index 0a8edde7bc9..00000000000 --- a/etc/taskcluster/docker/wpt-update.dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -% include run.dockerfile - -RUN apt-get install -qy --no-install-recommends \ - python3 \ - jq diff --git a/etc/taskcluster/mock.py b/etc/taskcluster/mock.py deleted file mode 100755 index 0bce7891876..00000000000 --- a/etc/taskcluster/mock.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2018 The Servo Project Developers. See the COPYRIGHT -# file at the top-level directory of this distribution. -# -# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or -# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license -# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your -# option. This file may not be copied, modified, or distributed -# except according to those terms. - -''''set -e -python3 -m coverage run $0 -python3 -m coverage report -m --fail-under 100 -exit - -Run the decision task with fake Taskcluster APIs, to catch Python errors before pushing. -''' - -import os -import sys -from unittest.mock import MagicMock - - -class TaskclusterRestFailure(Exception): - status_code = 404 - - -class Index: - __init__ = insertTask = lambda *_, **__: None - - def findTask(self, path): - if decision_task.CONFIG.git_ref == "refs/heads/master": - return {"taskId": "<from index>"} - raise TaskclusterRestFailure - - -stringDate = str -slugId = b"<new id>".lower -sys.exit = Queue = fromNow = MagicMock() -sys.modules["taskcluster"] = sys.modules[__name__] -sys.dont_write_bytecode = True -os.environ.update(**{k: k for k in "TASK_ID TASK_OWNER TASK_SOURCE GIT_URL GIT_SHA".split()}) -os.environ["GIT_REF"] = "refs/heads/auto" -os.environ["TASKCLUSTER_ROOT_URL"] = "https://community-tc.services.mozilla.com" -os.environ["TASKCLUSTER_PROXY_URL"] = "http://taskcluster" -os.environ["NEW_AMI_WORKER_TYPE"] = "-" -import decision_task # noqa: E402 -decision_task.decisionlib.subprocess = MagicMock() - -print("\n# Push:") -decision_task.main("github-push") - -print("\n# Push with hot caches:") -decision_task.main("github-push") - -print("\n# Push to master:") -decision_task.CONFIG.git_ref = "refs/heads/master" -decision_task.main("github-push") - -print("\n# Daily:") -decision_task.main("daily") - -print("\n# Try AMI:") -decision_task.main("try-windows-ami") - -print("\n# PR:") -decision_task.main("github-pull-request") - -print() diff --git a/etc/taskcluster/simulate_github_events.py b/etc/taskcluster/simulate_github_events.py deleted file mode 100755 index c99436b9106..00000000000 --- a/etc/taskcluster/simulate_github_events.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -''''set -e -cd "$(dirname $0)" -exec ../../python/_virtualenv/bin/python "$(basename $0)" -''' - -try: - import jsone -except ImportError: - import sys - sys.exit("pip install git+https://github.com/taskcluster/json-e") - -import yaml -import json - -template = yaml.load(open("../../.taskcluster.yml").read().decode("utf8")) -repo = dict( - repository=dict( - clone_url="https://github.com/servo/servo.git", - ), -) -contexts = [ - dict( - tasks_for="github-release", - event=repo, - ), - dict( - tasks_for="github-pull-request", - event=dict( - action="comment", - **repo - ), - ), - dict( - tasks_for="github-push", - event=dict( - ref="refs/heads/master", - compare="https://github.com/servo/servo/compare/1753cda...de09c8f", - after="de09c8fb6ef87dec5932d5fab4adcb421d291a54", - pusher=dict( - name="bors-servo", - ), - **repo - ), - ), - dict( - tasks_for="github-pull-request", - event=dict( - action="synchronize", - pull_request=dict( - number=22583, - url="https://github.com/servo/servo/pull/22583", - head=dict( - sha="51a422c9ef47420eb69c802643b7686bdb498652", - ), - merge_commit_sha="876fcf7a5fe971a9ac0a4ce117906c552c08c095", - ), - sender=dict( - login="jdm", - ), - **repo - ), - ), -] -for context in contexts: - print(context["tasks_for"]) - print(json.dumps(jsone.render(template, context), indent=2)) diff --git a/etc/taskcluster/treeherder.md b/etc/taskcluster/treeherder.md deleted file mode 100644 index 63223dddfc6..00000000000 --- a/etc/taskcluster/treeherder.md +++ /dev/null @@ -1,103 +0,0 @@ -# Treeherder for Servo - -Treeherder is tool for visualizing the status of “trees”, -meaning branches in various source repositories. -It shows each push to the repository with the corresponding commits -as well as the CI jobs that were started for that push. -While it is possible to write other tools that submit job data, -CI integration is easiest with Taskcluster. - -* [Production instance](https://treeherder.mozilla.org/) -* [Staging instance](https://treeherder.allizom.org/) -* [Source code](https://github.com/mozilla/treeherder/) - - -## Trees / repositories / branches - -Treeherders knows a about a number of *repostories*. -Mercurial on Mozilla’s servers and git on GitHub are supported. -Despite the name, in the GitHub case -each Treeherder repository maps to one branch in a git repository. -They are configured in the [`repository.json`] file. -As of this writing there are four for `github.com/servo/servo`, -named after the corresponding branch: - -[`repository.json`]: https://github.com/mozilla/treeherder/blob/master/treeherder/model/fixtures/repository.json - -* [`servo-master`](https://treeherder.mozilla.org/#/jobs?repo=servo-master) -* [`servo-auto`](https://treeherder.mozilla.org/#/jobs?repo=servo-auto) -* [`servo-try`](https://treeherder.mozilla.org/#/jobs?repo=servo-try) -* [`servo-try-taskcluster`](https://treeherder.mozilla.org/#/jobs?repo=servo-try-taskcluster) - -In the UI, the “Repos” button near the top right corner allows switching. - -`servo-auto` is the relevant one when a pull request is approved with Homu for landing, -since the `auto` branch is where it pushes a merge commit for testing. - - -## Data flow / how it all works - -(This section is mostly useful for future changes or troubleshooting.) - -Changes to the Treeherder repository are deployed to Staging -soon (minutes) after they are merged on GitHub, -and to Production manually at some point later. -See [current deployment status](https://whatsdeployed.io/s-dqv). - -Once a configuration change with a new repository/branch is deployed, -Treeherder will show it in its UI and start recording push and job data in its database. -This data comes from [Pulse], Mozilla’s shared message queue that coordinates separate services. -The [Pulse Inspector] shows messages as they come (though not in the past), -which can be useful for debugging. -Note that you need to add at least one “Binding”, -or the “Start Listening” button won’t do anything. - -[Pulse]: https://wiki.mozilla.org/Auto-tools/Projects/Pulse -[Pulse Inspector]: https://community-tc.services.mozilla.com/pulse-messages - - -### Push data - -When [taskcluster-github] is [enabled] on a repository, -it recieves [webhooks] from GitHub for various events -such as a push to a branch of the repository. - -In addition to starting Taskcluster tasks based on `.taskcluster.yml` in the repository, -in [`api.js`] it creates [Pulse messages] corresponding to those events. -Treeherder consumes messages from the `exchange/taskcluster-github/v1/push` exchange -(among others) in [`push_loader.py`]. -In Pulse Inspector, these messages for the Servo repository can be seen -by specifying the [`primary.servo.servo`] routing key pattern. - -[taskcluster-github]: https://github.com/taskcluster/taskcluster/tree/master/services/github -[enabled]: https://github.com/apps/community-tc-integration/ -[webhooks]: https://developer.github.com/webhooks/ -[Pulse messages]: https://community-tc.services.mozilla.com/docs/reference/integrations/github/exchanges -[`api.js`]: https://github.com/taskcluster/taskcluster/blob/master/services/github/src/api.js -[`push_loader.py`]: https://github.com/mozilla/treeherder/blob/master/treeherder/etl/push_loader.py -[`primary.servo.servo`]: https://community-tc.services.mozilla.com/pulse-messages?bindings%5B0%5D%5Bexchange%5D=exchange%2Ftaskcluster-github%2Fv1%2Fpush&bindings%5B0%5D%5BroutingKeyPattern%5D=primary.servo.servo - - -### (Taskcluster) job data - -The Taskcluster Queue generates a number of [Pulse messages about tasks]. -Each value of the `routes` array in the task definition, with a `route.` prefix prepended, -is additional routing key for those messages. - -Treeherder reads those messages -if they have an appropriate route ([see in Pulse inspector][inspector1]), -However, it will drop an incoming message -if the `extra.treeherder` object in the task definition doesn’t conform to the [schema]. -Such schema validation errors are logged, but those logs are not easy to access. -Ask on IRC on `#taskcluster`. - -Finally, Treeherder reads that latter kind of message in [`job_loader.py`]. - - - -[Pulse messages about tasks]: https://community-tc.services.mozilla.com/docs/reference/platform/taskcluster-queue/references/events -[taskcluster-treeherder]: https://github.com/taskcluster/taskcluster-treeherder/ -[other messages]: https://community-tc.services.mozilla.com/docs/reference/integrations/taskcluster-treeherder#job-pulse-messages -[schema]: https://schemas.taskcluster.net/treeherder/v1/task-treeherder-config.json -[`job_loader.py`]: https://github.com/mozilla/treeherder/blob/master/treeherder/etl/job_loader.py -[inspector1]: https://tools.taskcluster.net/pulse-inspector?bindings%5B0%5D%5Bexchange%5D=exchange%2Ftaskcluster-queue%2Fv1%2Ftask-defined&bindings%5B0%5D%5BroutingKeyPattern%5D=route.tc-treeherder.%23 diff --git a/python/servo/package_commands.py b/python/servo/package_commands.py index f6010c0d3a0..53910dcd644 100644 --- a/python/servo/package_commands.py +++ b/python/servo/package_commands.py @@ -21,7 +21,6 @@ import shutil import subprocess import sys import tempfile -import urllib import xml from mach.decorators import ( @@ -98,15 +97,6 @@ else: raise e -def get_taskcluster_secret(name): - url = ( - os.environ.get("TASKCLUSTER_PROXY_URL", "http://taskcluster") - + "/api/secrets/v1/secret/project/servo/" - + name - ) - return json.load(urllib.request.urlopen(url))["secret"] - - def otool(s): o = subprocess.Popen(['/usr/bin/otool', '-L', s], stdout=subprocess.PIPE) for line in map(lambda s: s.decode('ascii'), o.stdout): @@ -601,23 +591,16 @@ class PackageCommands(CommandBase): @CommandArgument('platform', choices=PACKAGES.keys(), help='Package platform type to upload') - @CommandArgument('--secret-from-taskcluster', - action='store_true', - help='Retrieve the appropriate secrets from taskcluster.') @CommandArgument('--secret-from-environment', action='store_true', help='Retrieve the appropriate secrets from the environment.') - def upload_nightly(self, platform, secret_from_taskcluster, secret_from_environment): + def upload_nightly(self, platform, secret_from_environment): import boto3 def get_s3_secret(): aws_access_key = None aws_secret_access_key = None - if secret_from_taskcluster: - secret = get_taskcluster_secret("s3-upload-credentials") - aws_access_key = secret["aws_access_key_id"] - aws_secret_access_key = secret["aws_secret_access_key"] - elif secret_from_environment: + if secret_from_environment: secret = json.loads(os.environ['S3_UPLOAD_CREDENTIALS']) aws_access_key = secret["aws_access_key_id"] aws_secret_access_key = secret["aws_secret_access_key"] @@ -758,10 +741,7 @@ class PackageCommands(CommandBase): '--message=Version Bump: {}'.format(brew_version), ]) - if secret_from_taskcluster: - token = get_taskcluster_secret('github-homebrew-token')["token"] - else: - token = os.environ['GITHUB_HOMEBREW_TOKEN'] + token = os.environ['GITHUB_HOMEBREW_TOKEN'] push_url = 'https://{}@github.com/servo/homebrew-servo.git' # TODO(aneeshusa): Use subprocess.DEVNULL with Python 3.3+ @@ -804,8 +784,6 @@ def setup_uwp_signing(ms_app_store, publisher): if ms_app_store: return ["/p:AppxPackageSigningEnabled=false"] - is_tc = "TASKCLUSTER_PROXY_URL" in os.environ - def run_powershell_cmd(cmd): try: return ( @@ -818,10 +796,7 @@ def setup_uwp_signing(ms_app_store, publisher): exit(1) pfx = None - if is_tc: - print("Packaging on TC. Using secret certificate") - pfx = get_taskcluster_secret("windows-codesign-cert/latest")["pfx"]["base64"] - elif 'CODESIGN_CERT' in os.environ: + if 'CODESIGN_CERT' in os.environ: pfx = os.environ['CODESIGN_CERT'] if pfx: @@ -832,10 +807,7 @@ def setup_uwp_signing(ms_app_store, publisher): # Powershell command that lists all certificates for publisher cmd = '(dir cert: -Recurse | Where-Object {$_.Issuer -eq "' + publisher + '"}).Thumbprint' certs = list(set(run_powershell_cmd(cmd).splitlines())) - if not certs and is_tc: - print("Error: No certificate installed for publisher " + publisher) - exit(1) - if not certs and not is_tc: + if not certs: print("No certificate installed for publisher " + publisher) print("Creating and installing a temporary certificate") # PowerShell command that creates and install signing certificate for publisher diff --git a/servo-tidy.toml b/servo-tidy.toml index af3eee3aacb..762eb169145 100644 --- a/servo-tidy.toml +++ b/servo-tidy.toml @@ -123,9 +123,6 @@ files = [ "./tests/wpt/mozilla/tests/css/pre_with_tab.html", "./tests/wpt/mozilla/tests/mozilla/textarea_placeholder.html", # Python 3 syntax causes "E901 SyntaxError" when flake8 runs in Python 2 - "./etc/taskcluster/decision_task.py", - "./etc/taskcluster/decisionlib.py", - "./tests/wpt/reftests-report/gen.py", "./components/style/properties/build.py", ] # Directories that are ignored for the non-WPT tidy check. diff --git a/tests/wpt/reftests-report/gen.py b/tests/wpt/reftests-report/gen.py deleted file mode 100755 index ba07ca26469..00000000000 --- a/tests/wpt/reftests-report/gen.py +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env python3 - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -import gzip -import json -import os -import re -import sys -import urllib.request -from html import escape as html_escape - - -TASKCLUSTER_ROOT_URL = "https://community-tc.services.mozilla.com" - - -def fetch(url): - url = TASKCLUSTER_ROOT_URL + "/api/" + url - print("Fetching " + url) - response = urllib.request.urlopen(url) - assert response.getcode() == 200 - encoding = response.info().get("Content-Encoding") - if not encoding: - return response - elif encoding == "gzip": - return gzip.GzipFile(fileobj=response) - else: - raise ValueError("Unsupported Content-Encoding: %s" % encoding) - - -def fetch_json(url): - with fetch(url) as response: - return json.load(response) - - -def task(platform, chunk, key): - return "index/v1/task/project.servo.%s_wpt_%s.%s" % (platform, chunk, key) - - -def failing_reftests(platform, key): - chunk_1_task_id = fetch_json(task(platform, 1, key))["taskId"] - name = fetch_json("queue/v1/task/" + chunk_1_task_id)["metadata"]["name"] - match = re.search("WPT chunk (\d+) / (\d+)", name) - assert match.group(1) == "1" - total_chunks = int(match.group(2)) - - for chunk in range(1, total_chunks + 1): - with fetch(task(platform, chunk, key) + "/artifacts/public/test-wpt.log") as response: - yield from parse(response) - - -def parse(file_like): - seen = set() - for line in file_like: - message = json.loads(line) - status = message.get("status") - if status not in {None, "OK", "PASS"}: - screenshots = message.get("extra", {}).get("reftest_screenshots") - if screenshots: - url = message["test"] - assert url.startswith("/") - yield url[1:], message.get("expected") == "PASS", screenshots - - -def main(source, commit_sha=None): - failures = Directory() - - if commit_sha: - title = "<h1>Layout 2020 regressions in commit <code>%s</code></h1>" % commit_sha - failures_2013 = {url for url, _, _ in failing_reftests("linux_x64", source)} - for url, _expected_pass, screenshots in failing_reftests("linux_x64_2020", source): - if url not in failures_2013: - failures.add(url, screenshots) - else: - title = "Unexpected failures" - with open(source, "rb") as file_obj: - for url, expected_pass, screenshots in parse(file_obj): - if expected_pass: - failures.add(url, screenshots) - - here = os.path.dirname(__file__) - with open(os.path.join(here, "prism.js")) as f: - prism_js = f.read() - with open(os.path.join(here, "prism.css")) as f: - prism_css = f.read() - with open(os.path.join(here, "report.html"), "w", encoding="utf-8") as html: - os.chdir(os.path.join(here, "..")) - html.write(""" - <!doctype html> - <meta charset=utf-8> - <title>WPT reftests failures report</title> - <link rel=stylesheet href=prism.css> - <style> - ul { padding-left: 1em } - li { list-style: "⯈ " } - li.expanded { list-style: "⯆ " } - li:not(.expanded) > ul, li:not(.expanded) > div { display: none } - li > div { display: grid; grid-gap: 1em; grid-template-columns: 1fr 1fr } - li > div > p { grid-column: span 2 } - li > div > img { grid-row: 2; width: 300px; box-shadow: 0 0 10px } - li > div > img:hover { transform: scale(3); transform-origin: 0 0 } - li > div > pre { grid-row: 3; font-size: 12px !important } - pre code { white-space: pre-wrap !important } - <h1>%s</h1> - </style> - %s - """ % (prism_css, title)) - failures.write(html) - html.write(""" - <script> - for (let li of document.getElementsByTagName("li")) { - li.addEventListener('click', event => { - li.classList.toggle("expanded") - event.stopPropagation() - }) - } - %s - </script> - """ % prism_js) - - -class Directory: - def __init__(self): - self.count = 0 - self.contents = {} - - def add(self, path, screenshots): - self.count += 1 - first, _, rest = path.partition("/") - if rest: - self.contents.setdefault(first, Directory()).add(rest, screenshots) - else: - assert path not in self.contents - self.contents[path] = screenshots - - def write(self, html): - html.write("<ul>\n") - for k, v in self.contents.items(): - html.write("<li><code>%s</code>\n" % k) - if isinstance(v, Directory): - html.write("<strong>%s</strong>\n" % v.count) - v.write(html) - else: - a, rel, b = v - html.write("<div>\n<p><code>%s</code> %s <code>%s</code></p>\n" - % (a["url"], rel, b["url"])) - for side in [a, b]: - html.write("<img src='data:image/png;base64,%s'>\n" % side["screenshot"]) - url = side["url"] - prefix = "/_mozilla/" - if url.startswith(prefix): - filename = "mozilla/tests/" + url[len(prefix):] - elif url == "about:blank": - src = "" - filename = None - else: - filename = "web-platform-tests" + url - if filename: - with open(filename, encoding="utf-8") as f: - src = html_escape(f.read()) - html.write("<pre><code class=language-html>%s</code></pre>\n" % src) - html.write("</li>\n") - html.write("</ul>\n") - - -if __name__ == "__main__": - sys.exit(main(*sys.argv[1:])) diff --git a/tests/wpt/reftests-report/prism.css b/tests/wpt/reftests-report/prism.css deleted file mode 100644 index 6fa6fcc5be6..00000000000 --- a/tests/wpt/reftests-report/prism.css +++ /dev/null @@ -1,141 +0,0 @@ -/* PrismJS 1.19.0 -https://prismjs.com/download.html#themes=prism&languages=markup+css+clike+javascript */ -/** - * prism.js default theme for JavaScript, CSS and HTML - * Based on dabblet (http://dabblet.com) - * @author Lea Verou - */ - -code[class*="language-"], -pre[class*="language-"] { - color: black; - background: none; - text-shadow: 0 1px white; - font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace; - font-size: 1em; - text-align: left; - white-space: pre; - word-spacing: normal; - word-break: normal; - word-wrap: normal; - line-height: 1.5; - - -moz-tab-size: 4; - -o-tab-size: 4; - tab-size: 4; - - -webkit-hyphens: none; - -moz-hyphens: none; - -ms-hyphens: none; - hyphens: none; -} - -pre[class*="language-"]::-moz-selection, pre[class*="language-"] ::-moz-selection, -code[class*="language-"]::-moz-selection, code[class*="language-"] ::-moz-selection { - text-shadow: none; - background: #b3d4fc; -} - -pre[class*="language-"]::selection, pre[class*="language-"] ::selection, -code[class*="language-"]::selection, code[class*="language-"] ::selection { - text-shadow: none; - background: #b3d4fc; -} - -@media print { - code[class*="language-"], - pre[class*="language-"] { - text-shadow: none; - } -} - -/* Code blocks */ -pre[class*="language-"] { - padding: 1em; - margin: .5em 0; - overflow: auto; -} - -:not(pre) > code[class*="language-"], -pre[class*="language-"] { - background: #f5f2f0; -} - -/* Inline code */ -:not(pre) > code[class*="language-"] { - padding: .1em; - border-radius: .3em; - white-space: normal; -} - -.token.comment, -.token.prolog, -.token.doctype, -.token.cdata { - color: slategray; -} - -.token.punctuation { - color: #999; -} - -.token.namespace { - opacity: .7; -} - -.token.property, -.token.tag, -.token.boolean, -.token.number, -.token.constant, -.token.symbol, -.token.deleted { - color: #905; -} - -.token.selector, -.token.attr-name, -.token.string, -.token.char, -.token.builtin, -.token.inserted { - color: #690; -} - -.token.operator, -.token.entity, -.token.url, -.language-css .token.string, -.style .token.string { - color: #9a6e3a; - background: hsla(0, 0%, 100%, .5); -} - -.token.atrule, -.token.attr-value, -.token.keyword { - color: #07a; -} - -.token.function, -.token.class-name { - color: #DD4A68; -} - -.token.regex, -.token.important, -.token.variable { - color: #e90; -} - -.token.important, -.token.bold { - font-weight: bold; -} -.token.italic { - font-style: italic; -} - -.token.entity { - cursor: help; -} diff --git a/tests/wpt/reftests-report/prism.js b/tests/wpt/reftests-report/prism.js deleted file mode 100644 index a43970450d6..00000000000 --- a/tests/wpt/reftests-report/prism.js +++ /dev/null @@ -1,7 +0,0 @@ -/* PrismJS 1.19.0 -https://prismjs.com/download.html#themes=prism&languages=markup+css+clike+javascript */ -var _self="undefined"!=typeof window?window:"undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?self:{},Prism=function(u){var c=/\blang(?:uage)?-([\w-]+)\b/i,r=0,C={manual:u.Prism&&u.Prism.manual,disableWorkerMessageHandler:u.Prism&&u.Prism.disableWorkerMessageHandler,util:{encode:function e(r){return r instanceof _?new _(r.type,e(r.content),r.alias):Array.isArray(r)?r.map(e):r.replace(/&/g,"&").replace(/</g,"<").replace(/\u00a0/g," ")},type:function(e){return Object.prototype.toString.call(e).slice(8,-1)},objId:function(e){return e.__id||Object.defineProperty(e,"__id",{value:++r}),e.__id},clone:function n(e,t){var a,r,i=C.util.type(e);switch(t=t||{},i){case"Object":if(r=C.util.objId(e),t[r])return t[r];for(var o in a={},t[r]=a,e)e.hasOwnProperty(o)&&(a[o]=n(e[o],t));return a;case"Array":return r=C.util.objId(e),t[r]?t[r]:(a=[],t[r]=a,e.forEach(function(e,r){a[r]=n(e,t)}),a);default:return e}},getLanguage:function(e){for(;e&&!c.test(e.className);)e=e.parentElement;return e?(e.className.match(c)||[,"none"])[1].toLowerCase():"none"},currentScript:function(){if("undefined"==typeof document)return null;if("currentScript"in document)return document.currentScript;try{throw new Error}catch(e){var r=(/at [^(\r\n]*\((.*):.+:.+\)$/i.exec(e.stack)||[])[1];if(r){var n=document.getElementsByTagName("script");for(var t in n)if(n[t].src==r)return n[t]}return null}}},languages:{extend:function(e,r){var n=C.util.clone(C.languages[e]);for(var t in r)n[t]=r[t];return n},insertBefore:function(n,e,r,t){var a=(t=t||C.languages)[n],i={};for(var o in a)if(a.hasOwnProperty(o)){if(o==e)for(var l in r)r.hasOwnProperty(l)&&(i[l]=r[l]);r.hasOwnProperty(o)||(i[o]=a[o])}var s=t[n];return t[n]=i,C.languages.DFS(C.languages,function(e,r){r===s&&e!=n&&(this[e]=i)}),i},DFS:function e(r,n,t,a){a=a||{};var i=C.util.objId;for(var o in r)if(r.hasOwnProperty(o)){n.call(r,o,r[o],t||o);var l=r[o],s=C.util.type(l);"Object"!==s||a[i(l)]?"Array"!==s||a[i(l)]||(a[i(l)]=!0,e(l,n,o,a)):(a[i(l)]=!0,e(l,n,null,a))}}},plugins:{},highlightAll:function(e,r){C.highlightAllUnder(document,e,r)},highlightAllUnder:function(e,r,n){var t={callback:n,container:e,selector:'code[class*="language-"], [class*="language-"] code, code[class*="lang-"], [class*="lang-"] code'};C.hooks.run("before-highlightall",t),t.elements=Array.prototype.slice.apply(t.container.querySelectorAll(t.selector)),C.hooks.run("before-all-elements-highlight",t);for(var a,i=0;a=t.elements[i++];)C.highlightElement(a,!0===r,t.callback)},highlightElement:function(e,r,n){var t=C.util.getLanguage(e),a=C.languages[t];e.className=e.className.replace(c,"").replace(/\s+/g," ")+" language-"+t;var i=e.parentNode;i&&"pre"===i.nodeName.toLowerCase()&&(i.className=i.className.replace(c,"").replace(/\s+/g," ")+" language-"+t);var o={element:e,language:t,grammar:a,code:e.textContent};function l(e){o.highlightedCode=e,C.hooks.run("before-insert",o),o.element.innerHTML=o.highlightedCode,C.hooks.run("after-highlight",o),C.hooks.run("complete",o),n&&n.call(o.element)}if(C.hooks.run("before-sanity-check",o),!o.code)return C.hooks.run("complete",o),void(n&&n.call(o.element));if(C.hooks.run("before-highlight",o),o.grammar)if(r&&u.Worker){var s=new Worker(C.filename);s.onmessage=function(e){l(e.data)},s.postMessage(JSON.stringify({language:o.language,code:o.code,immediateClose:!0}))}else l(C.highlight(o.code,o.grammar,o.language));else l(C.util.encode(o.code))},highlight:function(e,r,n){var t={code:e,grammar:r,language:n};return C.hooks.run("before-tokenize",t),t.tokens=C.tokenize(t.code,t.grammar),C.hooks.run("after-tokenize",t),_.stringify(C.util.encode(t.tokens),t.language)},matchGrammar:function(e,r,n,t,a,i,o){for(var l in n)if(n.hasOwnProperty(l)&&n[l]){var s=n[l];s=Array.isArray(s)?s:[s];for(var u=0;u<s.length;++u){if(o&&o==l+","+u)return;var c=s[u],g=c.inside,f=!!c.lookbehind,h=!!c.greedy,d=0,m=c.alias;if(h&&!c.pattern.global){var p=c.pattern.toString().match(/[imsuy]*$/)[0];c.pattern=RegExp(c.pattern.source,p+"g")}c=c.pattern||c;for(var y=t,v=a;y<r.length;v+=r[y].length,++y){var k=r[y];if(r.length>e.length)return;if(!(k instanceof _)){if(h&&y!=r.length-1){if(c.lastIndex=v,!(S=c.exec(e)))break;for(var b=S.index+(f&&S[1]?S[1].length:0),w=S.index+S[0].length,A=y,P=v,x=r.length;A<x&&(P<w||!r[A].type&&!r[A-1].greedy);++A)(P+=r[A].length)<=b&&(++y,v=P);if(r[y]instanceof _)continue;O=A-y,k=e.slice(v,P),S.index-=v}else{c.lastIndex=0;var S=c.exec(k),O=1}if(S){f&&(d=S[1]?S[1].length:0);w=(b=S.index+d)+(S=S[0].slice(d)).length;var E=k.slice(0,b),N=k.slice(w),j=[y,O];E&&(++y,v+=E.length,j.push(E));var L=new _(l,g?C.tokenize(S,g):S,m,S,h);if(j.push(L),N&&j.push(N),Array.prototype.splice.apply(r,j),1!=O&&C.matchGrammar(e,r,n,y,v,!0,l+","+u),i)break}else if(i)break}}}}},tokenize:function(e,r){var n=[e],t=r.rest;if(t){for(var a in t)r[a]=t[a];delete r.rest}return C.matchGrammar(e,n,r,0,0,!1),n},hooks:{all:{},add:function(e,r){var n=C.hooks.all;n[e]=n[e]||[],n[e].push(r)},run:function(e,r){var n=C.hooks.all[e];if(n&&n.length)for(var t,a=0;t=n[a++];)t(r)}},Token:_};function _(e,r,n,t,a){this.type=e,this.content=r,this.alias=n,this.length=0|(t||"").length,this.greedy=!!a}if(u.Prism=C,_.stringify=function r(e,n){if("string"==typeof e)return e;if(Array.isArray(e)){var t="";return e.forEach(function(e){t+=r(e,n)}),t}var a={type:e.type,content:r(e.content,n),tag:"span",classes:["token",e.type],attributes:{},language:n},i=e.alias;i&&(Array.isArray(i)?Array.prototype.push.apply(a.classes,i):a.classes.push(i)),C.hooks.run("wrap",a);var o="";for(var l in a.attributes)o+=" "+l+'="'+(a.attributes[l]||"").replace(/"/g,""")+'"';return"<"+a.tag+' class="'+a.classes.join(" ")+'"'+o+">"+a.content+"</"+a.tag+">"},!u.document)return u.addEventListener&&(C.disableWorkerMessageHandler||u.addEventListener("message",function(e){var r=JSON.parse(e.data),n=r.language,t=r.code,a=r.immediateClose;u.postMessage(C.highlight(t,C.languages[n],n)),a&&u.close()},!1)),C;var e=C.util.currentScript();function n(){C.manual||C.highlightAll()}if(e&&(C.filename=e.src,e.hasAttribute("data-manual")&&(C.manual=!0)),!C.manual){var t=document.readyState;"loading"===t||"interactive"===t&&e&&e.defer?document.addEventListener("DOMContentLoaded",n):window.requestAnimationFrame?window.requestAnimationFrame(n):window.setTimeout(n,16)}return C}(_self);"undefined"!=typeof module&&module.exports&&(module.exports=Prism),"undefined"!=typeof global&&(global.Prism=Prism); -Prism.languages.markup={comment:/<!--[\s\S]*?-->/,prolog:/<\?[\s\S]+?\?>/,doctype:{pattern:/<!DOCTYPE(?:[^>"'[\]]|"[^"]*"|'[^']*')+(?:\[(?:(?!<!--)[^"'\]]|"[^"]*"|'[^']*'|<!--[\s\S]*?-->)*\]\s*)?>/i,greedy:!0},cdata:/<!\[CDATA\[[\s\S]*?]]>/i,tag:{pattern:/<\/?(?!\d)[^\s>\/=$<%]+(?:\s(?:\s*[^\s>\/=]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+(?=[\s>]))|(?=[\s/>])))+)?\s*\/?>/i,greedy:!0,inside:{tag:{pattern:/^<\/?[^\s>\/]+/i,inside:{punctuation:/^<\/?/,namespace:/^[^\s>\/:]+:/}},"attr-value":{pattern:/=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+)/i,inside:{punctuation:[/^=/,{pattern:/^(\s*)["']|["']$/,lookbehind:!0}]}},punctuation:/\/?>/,"attr-name":{pattern:/[^\s>\/]+/,inside:{namespace:/^[^\s>\/:]+:/}}}},entity:/&#?[\da-z]{1,8};/i},Prism.languages.markup.tag.inside["attr-value"].inside.entity=Prism.languages.markup.entity,Prism.hooks.add("wrap",function(a){"entity"===a.type&&(a.attributes.title=a.content.replace(/&/,"&"))}),Object.defineProperty(Prism.languages.markup.tag,"addInlined",{value:function(a,e){var s={};s["language-"+e]={pattern:/(^<!\[CDATA\[)[\s\S]+?(?=\]\]>$)/i,lookbehind:!0,inside:Prism.languages[e]},s.cdata=/^<!\[CDATA\[|\]\]>$/i;var n={"included-cdata":{pattern:/<!\[CDATA\[[\s\S]*?\]\]>/i,inside:s}};n["language-"+e]={pattern:/[\s\S]+/,inside:Prism.languages[e]};var t={};t[a]={pattern:RegExp("(<__[\\s\\S]*?>)(?:<!\\[CDATA\\[[\\s\\S]*?\\]\\]>\\s*|[\\s\\S])*?(?=<\\/__>)".replace(/__/g,a),"i"),lookbehind:!0,greedy:!0,inside:n},Prism.languages.insertBefore("markup","cdata",t)}}),Prism.languages.xml=Prism.languages.extend("markup",{}),Prism.languages.html=Prism.languages.markup,Prism.languages.mathml=Prism.languages.markup,Prism.languages.svg=Prism.languages.markup; -!function(s){var e=/("|')(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/;s.languages.css={comment:/\/\*[\s\S]*?\*\//,atrule:{pattern:/@[\w-]+[\s\S]*?(?:;|(?=\s*\{))/,inside:{rule:/^@[\w-]+/,"selector-function-argument":{pattern:/(\bselector\s*\((?!\s*\))\s*)(?:[^()]|\((?:[^()]|\([^()]*\))*\))+?(?=\s*\))/,lookbehind:!0,alias:"selector"}}},url:{pattern:RegExp("url\\((?:"+e.source+"|[^\n\r()]*)\\)","i"),inside:{function:/^url/i,punctuation:/^\(|\)$/}},selector:RegExp("[^{}\\s](?:[^{};\"']|"+e.source+")*?(?=\\s*\\{)"),string:{pattern:e,greedy:!0},property:/[-_a-z\xA0-\uFFFF][-\w\xA0-\uFFFF]*(?=\s*:)/i,important:/!important\b/i,function:/[-a-z0-9]+(?=\()/i,punctuation:/[(){};:,]/},s.languages.css.atrule.inside.rest=s.languages.css;var t=s.languages.markup;t&&(t.tag.addInlined("style","css"),s.languages.insertBefore("inside","attr-value",{"style-attr":{pattern:/\s*style=("|')(?:\\[\s\S]|(?!\1)[^\\])*\1/i,inside:{"attr-name":{pattern:/^\s*style/i,inside:t.tag.inside},punctuation:/^\s*=\s*['"]|['"]\s*$/,"attr-value":{pattern:/.+/i,inside:s.languages.css}},alias:"language-css"}},t.tag))}(Prism); -Prism.languages.clike={comment:[{pattern:/(^|[^\\])\/\*[\s\S]*?(?:\*\/|$)/,lookbehind:!0},{pattern:/(^|[^\\:])\/\/.*/,lookbehind:!0,greedy:!0}],string:{pattern:/(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,greedy:!0},"class-name":{pattern:/(\b(?:class|interface|extends|implements|trait|instanceof|new)\s+|\bcatch\s+\()[\w.\\]+/i,lookbehind:!0,inside:{punctuation:/[.\\]/}},keyword:/\b(?:if|else|while|do|for|return|in|instanceof|function|new|try|throw|catch|finally|null|break|continue)\b/,boolean:/\b(?:true|false)\b/,function:/\w+(?=\()/,number:/\b0x[\da-f]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?/i,operator:/[<>]=?|[!=]=?=?|--?|\+\+?|&&?|\|\|?|[?*/~^%]/,punctuation:/[{}[\];(),.:]/}; -Prism.languages.javascript=Prism.languages.extend("clike",{"class-name":[Prism.languages.clike["class-name"],{pattern:/(^|[^$\w\xA0-\uFFFF])[_$A-Z\xA0-\uFFFF][$\w\xA0-\uFFFF]*(?=\.(?:prototype|constructor))/,lookbehind:!0}],keyword:[{pattern:/((?:^|})\s*)(?:catch|finally)\b/,lookbehind:!0},{pattern:/(^|[^.]|\.\.\.\s*)\b(?:as|async(?=\s*(?:function\b|\(|[$\w\xA0-\uFFFF]|$))|await|break|case|class|const|continue|debugger|default|delete|do|else|enum|export|extends|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|undefined|var|void|while|with|yield)\b/,lookbehind:!0}],number:/\b(?:(?:0[xX](?:[\dA-Fa-f](?:_[\dA-Fa-f])?)+|0[bB](?:[01](?:_[01])?)+|0[oO](?:[0-7](?:_[0-7])?)+)n?|(?:\d(?:_\d)?)+n|NaN|Infinity)\b|(?:\b(?:\d(?:_\d)?)+\.?(?:\d(?:_\d)?)*|\B\.(?:\d(?:_\d)?)+)(?:[Ee][+-]?(?:\d(?:_\d)?)+)?/,function:/#?[_$a-zA-Z\xA0-\uFFFF][$\w\xA0-\uFFFF]*(?=\s*(?:\.\s*(?:apply|bind|call)\s*)?\()/,operator:/--|\+\+|\*\*=?|=>|&&|\|\||[!=]==|<<=?|>>>?=?|[-+*/%&|^!=<>]=?|\.{3}|\?[.?]?|[~:]/}),Prism.languages.javascript["class-name"][0].pattern=/(\b(?:class|interface|extends|implements|instanceof|new)\s+)[\w.\\]+/,Prism.languages.insertBefore("javascript","keyword",{regex:{pattern:/((?:^|[^$\w\xA0-\uFFFF."'\])\s])\s*)\/(?:\[(?:[^\]\\\r\n]|\\.)*]|\\.|[^/\\\[\r\n])+\/[gimyus]{0,6}(?=(?:\s|\/\*[\s\S]*?\*\/)*(?:$|[\r\n,.;:})\]]|\/\/))/,lookbehind:!0,greedy:!0},"function-variable":{pattern:/#?[_$a-zA-Z\xA0-\uFFFF][$\w\xA0-\uFFFF]*(?=\s*[=:]\s*(?:async\s*)?(?:\bfunction\b|(?:\((?:[^()]|\([^()]*\))*\)|[_$a-zA-Z\xA0-\uFFFF][$\w\xA0-\uFFFF]*)\s*=>))/,alias:"function"},parameter:[{pattern:/(function(?:\s+[_$A-Za-z\xA0-\uFFFF][$\w\xA0-\uFFFF]*)?\s*\(\s*)(?!\s)(?:[^()]|\([^()]*\))+?(?=\s*\))/,lookbehind:!0,inside:Prism.languages.javascript},{pattern:/[_$a-z\xA0-\uFFFF][$\w\xA0-\uFFFF]*(?=\s*=>)/i,inside:Prism.languages.javascript},{pattern:/(\(\s*)(?!\s)(?:[^()]|\([^()]*\))+?(?=\s*\)\s*=>)/,lookbehind:!0,inside:Prism.languages.javascript},{pattern:/((?:\b|\s|^)(?!(?:as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|undefined|var|void|while|with|yield)(?![$\w\xA0-\uFFFF]))(?:[_$A-Za-z\xA0-\uFFFF][$\w\xA0-\uFFFF]*\s*)\(\s*)(?!\s)(?:[^()]|\([^()]*\))+?(?=\s*\)\s*\{)/,lookbehind:!0,inside:Prism.languages.javascript}],constant:/\b[A-Z](?:[A-Z_]|\dx?)*\b/}),Prism.languages.insertBefore("javascript","string",{"template-string":{pattern:/`(?:\\[\s\S]|\${(?:[^{}]|{(?:[^{}]|{[^}]*})*})+}|(?!\${)[^\\`])*`/,greedy:!0,inside:{"template-punctuation":{pattern:/^`|`$/,alias:"string"},interpolation:{pattern:/((?:^|[^\\])(?:\\{2})*)\${(?:[^{}]|{(?:[^{}]|{[^}]*})*})+}/,lookbehind:!0,inside:{"interpolation-punctuation":{pattern:/^\${|}$/,alias:"punctuation"},rest:Prism.languages.javascript}},string:/[\s\S]+/}}}),Prism.languages.markup&&Prism.languages.markup.tag.addInlined("script","javascript"),Prism.languages.js=Prism.languages.javascript; diff --git a/tests/wpt/update/fetchlogs.py b/tests/wpt/update/fetchlogs.py deleted file mode 100644 index 385f2c54174..00000000000 --- a/tests/wpt/update/fetchlogs.py +++ /dev/null @@ -1,99 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -import argparse -import cStringIO -import gzip -import json -import os -import requests -import six.moves.urllib as urllib - -treeherder_base = "https://treeherder.mozilla.org/" - -"""Simple script for downloading structured logs from treeherder. - -For the moment this is specialised to work with web-platform-tests -logs; in due course it should move somewhere generic and get hooked -up to mach or similar""" - -# Interpretation of the "job" list from -# https://github.com/mozilla/treeherder-service/blob/master/treeherder/webapp/api/utils.py#L18 - -def create_parser(): - parser = argparse.ArgumentParser() - parser.add_argument("branch", action="store", - help="Branch on which jobs ran") - parser.add_argument("commit", - action="store", - help="Commit hash for push") - - return parser - -def download(url, prefix, dest, force_suffix=True): - if dest is None: - dest = "." - - if prefix and not force_suffix: - name = os.path.join(dest, prefix + ".log") - else: - name = None - counter = 0 - - while not name or os.path.exists(name): - counter += 1 - sep = "" if not prefix else "-" - name = os.path.join(dest, prefix + sep + str(counter) + ".log") - - with open(name, "wb") as f: - resp = requests.get(url, stream=True) - for chunk in resp.iter_content(1024): - f.write(chunk) - -def get_blobber_url(branch, job): - job_id = job["id"] - resp = requests.get(urllib.parse.urljoin(treeherder_base, - "/api/project/%s/artifact/?job_id=%i&name=Job%%20Info" % (branch, - job_id))) - job_data = resp.json() - - if job_data: - assert len(job_data) == 1 - job_data = job_data[0] - try: - details = job_data["blob"]["job_details"] - for item in details: - if item["value"] == "wpt_raw.log": - return item["url"] - except: - return None - - -def get_structured_logs(branch, commit, dest=None): - resp = requests.get(urllib.parse.urljoin(treeherder_base, "/api/project/%s/resultset/?revision=%s" % (branch, commit))) - - revision_data = resp.json() - - result_set = revision_data["results"][0]["id"] - - resp = requests.get(urllib.parse.urljoin(treeherder_base, "/api/project/%s/jobs/?result_set_id=%s&count=2000&exclusion_profile=false" % (branch, result_set))) - - job_data = resp.json() - - for result in job_data["results"]: - job_type_name = result["job_type_name"] - if job_type_name.startswith("W3C Web Platform"): - url = get_blobber_url(branch, result) - if url: - prefix = result["platform"] # platform - download(url, prefix, None) - -def main(): - parser = create_parser() - args = parser.parse_args() - - get_structured_logs(args.branch, args.commit) - -if __name__ == "__main__": - main() |