diff options
25 files changed, 11 insertions, 2174 deletions
diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 57c4a7700d3..933683c713c 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -63,8 +63,8 @@ jobs: - name: Bootstrap run: | python3 -m pip install --upgrade pip virtualenv - brew bundle install --verbose --no-upgrade --file=etc/taskcluster/macos/Brewfile - brew bundle install --verbose --no-upgrade --file=etc/taskcluster/macos/Brewfile-build + brew bundle install --verbose --no-upgrade --file=etc/homebrew/Brewfile + brew bundle install --verbose --no-upgrade --file=etc/homebrew/Brewfile-build rm -rf /usr/local/etc/openssl rm -rf /usr/local/etc/openssl@1.1 brew install openssl@1.1 gnu-tar @@ -133,7 +133,7 @@ jobs: # run: | # gtar -xzf target.tar.gz # python3 -m pip install --upgrade pip virtualenv - # brew bundle install --verbose --no-upgrade --file=etc/taskcluster/macos/Brewfile + # brew bundle install --verbose --no-upgrade --file=etc/homebrew/Brewfile # - name: Smoketest # run: python3 ./mach smoketest # - name: Run tests diff --git a/.gitignore b/.gitignore index 8b09f9790d0..221b769edf2 100644 --- a/.gitignore +++ b/.gitignore @@ -75,4 +75,4 @@ support/hololens/.vs/ layout_trace* # Package managers -etc/taskcluster/macos/Brewfile.lock.json +etc/homebrew/Brewfile.lock.json diff --git a/.taskcluster.yml b/.taskcluster.yml deleted file mode 100644 index b2c277f6f09..00000000000 --- a/.taskcluster.yml +++ /dev/null @@ -1,120 +0,0 @@ -version: 1 - -# If and when switching to `reporting: checks-v1` here, also change the `statuses` route to `checks` -# in `CONFIG.routes_for_all_subtasks` in `etc/taskcluster/decision_task.py` - -policy: - # https://docs.taskcluster.net/docs/reference/integrations/taskcluster-github/docs/taskcluster-yml-v1#pull-requests - pullRequests: public - - # NOTE: when updating this consider whether the daily hook needs similar changes: -# https://tools.taskcluster.net/hooks/project-servo/daily -tasks: - $let: - task_common: - provisionerId: - $if: "taskcluster_root_url == 'https://taskcluster.net'" - then: aws-provisioner-v1 - else: proj-servo - created: {$fromNow: ''} - deadline: {$fromNow: '1 day'} - priority: high - extra: - treeherder: - machine: {platform: Linux} - labels: [x64] - symbol: Decision - payload: - maxRunTime: {$eval: '20 * 60'} - # https://github.com/servo/taskcluster-bootstrap-docker-images#decision-task - image: "servobrowser/taskcluster-bootstrap:decision-task@\ - sha256:7471a998e4462638c8d3e2cf0b4a99c9a5c8ca9f2ec0ae01cc069473b35cde10" - features: - taskclusterProxy: true - artifacts: - public/repo.bundle: - type: file - path: /repo.bundle - expires: {$fromNow: '1 day'} - env: - GIT_URL: ${event.repository.clone_url} - TASK_FOR: ${tasks_for} - command: - - /bin/bash - - '--login' - - '-e' - - '-c' - - >- - git init repo && - cd repo && - git fetch --depth 1 "$GIT_URL" "$GIT_REF" && - git reset --hard "$GIT_SHA" && - python3 etc/taskcluster/decision_task.py - in: - - $if: "tasks_for == 'github-push'" - then: - $let: - branch: - $if: "event.ref[:11] == 'refs/heads/'" - then: "${event.ref[11:]}" - in: - $if: "branch in ['auto', 'try', 'master'] || branch[:4] == 'try-'" - then: - $mergeDeep: - - {$eval: "task_common"} - - metadata: - name: "Servo: GitHub push decision task" - description: "" - owner: ${event.pusher.name}@users.noreply.github.com - source: ${event.compare} - workerType: - $if: "taskcluster_root_url == 'https://taskcluster.net'" - then: servo-docker-worker - else: docker - scopes: - - "assume:repo:github.com/servo/servo:branch:${branch}" - routes: - $let: - treeherder_repo: - $if: "branch[:4] == 'try-'" - then: "servo-try" - else: "servo-${branch}" - in: - - "tc-treeherder.v2._/${treeherder_repo}.${event.after}" - - "tc-treeherder-staging.v2._/${treeherder_repo}.${event.after}" - payload: - env: - GIT_REF: ${event.ref} - GIT_SHA: ${event.after} - TASK_OWNER: ${event.pusher.name}@users.noreply.github.com - TASK_SOURCE: ${event.compare} - - $if: >- - tasks_for == 'github-pull-request' && - event['action'] in ['opened', 'reopened', 'synchronize'] - then: - $mergeDeep: - - {$eval: "task_common"} - - metadata: - name: "Servo: GitHub PR decision task" - description: "" - owner: ${event.sender.login}@users.noreply.github.com - source: ${event.pull_request.url} - workerType: - $if: "taskcluster_root_url == 'https://taskcluster.net'" - then: servo-docker-untrusted - else: docker-untrusted - scopes: - - "assume:repo:github.com/servo/servo:pull-request" - routes: - - "tc-treeherder.v2._/servo-prs.${event.pull_request.head.sha}" - - "tc-treeherder-staging.v2._/servo-prs.${event.pull_request.head.sha}" - payload: - env: - # We use the merge commit made by GitHub, not the PR’s branch - GIT_REF: refs/pull/${event.pull_request.number}/merge - # `event.pull_request.merge_commit_sha` is tempting but not what we want: - # https://github.com/servo/servo/pull/22597#issuecomment-451518810 - GIT_SHA: FETCH_HEAD - TASK_OWNER: ${event.sender.login}@users.noreply.github.com - TASK_SOURCE: ${event.pull_request.url} - diff --git a/README.md b/README.md index 819fdf13263..771ae02f3fd 100644 --- a/README.md +++ b/README.md @@ -58,8 +58,8 @@ NOTE: run these steps after you've cloned the project locally. ``` sh cd servo -brew bundle install --file=etc/taskcluster/macos/Brewfile -brew bundle install --file=etc/taskcluster/macos/Brewfile-build +brew bundle install --file=etc/homebrew/Brewfile +brew bundle install --file=etc/homebrew/Brewfile-build pip install virtualenv ``` diff --git a/etc/ci/update-wpt-checkout b/etc/ci/update-wpt-checkout index 58f5cc386a9..4ff7f7c125e 100755 --- a/etc/ci/update-wpt-checkout +++ b/etc/ci/update-wpt-checkout @@ -101,11 +101,6 @@ function unsafe_open_pull_request() { # is unnecessary. git checkout "${BRANCH_NAME}" || return 0 - if [[ -z "${WPT_SYNC_TOKEN+set}" && "${TASKCLUSTER_PROXY_URL+set}" == "set" ]]; then - SECRET_RESPONSE=$(curl ${TASKCLUSTER_PROXY_URL}/api/secrets/v1/secret/project/servo/wpt-sync) - WPT_SYNC_TOKEN=`echo "${SECRET_RESPONSE}" | jq --raw-output '.secret.token'` - fi - if [[ -z "${WPT_SYNC_TOKEN+set}" ]]; then echo "Github auth token missing from WPT_SYNC_TOKEN." return 1 diff --git a/etc/taskcluster/macos/Brewfile b/etc/homebrew/Brewfile index c11e49b6c30..c11e49b6c30 100644 --- a/etc/taskcluster/macos/Brewfile +++ b/etc/homebrew/Brewfile diff --git a/etc/taskcluster/macos/Brewfile-build b/etc/homebrew/Brewfile-build index c78fe2cea3c..c78fe2cea3c 100644 --- a/etc/taskcluster/macos/Brewfile-build +++ b/etc/homebrew/Brewfile-build diff --git a/etc/taskcluster/macos/Brewfile-wpt-update b/etc/homebrew/Brewfile-wpt-update index f563d5e7bf9..f563d5e7bf9 100644 --- a/etc/taskcluster/macos/Brewfile-wpt-update +++ b/etc/homebrew/Brewfile-wpt-update diff --git a/etc/taskcluster/README.md b/etc/taskcluster/README.md deleted file mode 100644 index e91b526dfcc..00000000000 --- a/etc/taskcluster/README.md +++ /dev/null @@ -1,277 +0,0 @@ -# Testing Servo on Taskcluster - -## In-tree and out-of-tree configuration - -Where possible, we prefer keeping Taskcluster-related configuration and code in this directory, -set up CI so that testing of a given git branch uses the version in that branch. -That way, anyone can make changes (such installing a new system dependency -[in a `Dockerfile`](#docker-images)) in the same PR that relies on those changes. - -For some things however that is not practical, -or some deployment step that mutates global state is required. -That configuration is split between the [mozilla/community-tc-config] and -[servo/taskcluster-config] repositories, -managed by the Taskcluster team and the Servo team respectively. - -[mozilla/community-tc-config]: https://github.com/mozilla/community-tc-config/blob/master/config/projects.yml -[servo/taskcluster-config]: https://github.com/servo/taskcluster-config/tree/master/config - - -## Homu - -When a pull request is reviewed and the appropriate command is given, -[Homu] creates a merge commit of `master` and the PR’s branch, and pushes it to the `auto` branch. -One or more CI system (through their own means) get notified of this push by GitHub, -start testing the merge commit, and use the [GitHub Status API] to report results. - -Through a [Webhook], Homu gets notified of changes to these statues. -If all of the required statuses are reported successful, -Homu pushes its merge commit to the `master` branch -and goes on to testing the next pull request in its queue. - -[Homu]: https://github.com/servo/servo/wiki/Homu -[GitHub Status API]: https://developer.github.com/v3/repos/statuses/ -[Webhook]: https://developer.github.com/webhooks/ - - -## Taskcluster − GitHub integration - -Taskcluster is very flexible and not necessarily tied to GitHub, -but it does have an optional [GitHub integration service] that you can enable -on a repository [as a GitHub App]. -When enabled, this service gets notified for every push, pull request, or GitHub release. -It then schedules some tasks based on reading [`.taskcluster.yml`] in the corresponding commit. - -This file contains templates for creating one or more tasks, -but the logic it can support is fairly limited. -So a common pattern is to have it only run a single initial task called a *decision task* -that can have complex logic based on code and data in the repository -to build an arbitrary [task graph]. - -[GitHub integration service]: https://community-tc.services.mozilla.com/docs/manual/using/github -[as a GitHub App]: https://github.com/apps/community-tc-integration/ -[`.taskcluster.yml`]: https://community-tc.services.mozilla.com/docs/reference/integrations/taskcluster-github/docs/taskcluster-yml-v1 -[task graph]: https://community-tc.services.mozilla.com/docs/manual/using/task-graph - - -## Servo’s decision task - -This repository’s [`.taskcluster.yml`][tc.yml] schedules a single task -that runs the Python 3 script [`etc/taskcluster/decision_task.py`](decision_task.py). -It is called a *decision task* as it is responsible for deciding what other tasks to schedule. - -The Docker image that runs the decision task -is hosted on Docker Hub at [`servobrowser/taskcluster-bootstrap`][hub]. -It is built by [Docker Hub automated builds] based on a `Dockerfile` -in the [`taskcluster-bootstrap-docker-images`] GitHub repository. -Hopefully, this image does not need to be modified often -as it only needs to clone the repository and run Python. - -[tc.yml]: ../../../.taskcluster.yml -[hub]: https://hub.docker.com/r/servobrowser/taskcluster-bootstrap/ -[Docker Hub automated builds]: https://docs.docker.com/docker-hub/builds/ -[`taskcluster-bootstrap-docker-images`]: https://github.com/servo/taskcluster-bootstrap-docker-images/ - - -## Docker images - -[Similar to Firefox][firefox], Servo’s decision task supports running other tasks -in Docker images built on-demand, based on `Dockerfile`s in the main repository. -Modifying a `Dockerfile` and relying on those new changes -can be done in the same pull request or commit. - -To avoid rebuilding images on every pull request, -they are cached based on a hash of the source `Dockerfile`. -For now, to support this hashing, we make `Dockerfile`s be self-contained (with one exception). -Images are built without a [context], -so instructions like [`COPY`] cannot be used because there is nothing to copy from. -The exception is that the decision task adds support for a non-standard include directive: -when a `Dockerfile` first line is `% include` followed by a filename, -that line is replaced with the content of that file. - -For example, -[`etc/taskcluster/docker/build.dockerfile`](docker/build.dockerfile) starts like so: - -```Dockerfile -% include base.dockerfile - -RUN \ - apt-get install -qy --no-install-recommends \ -# […] -``` - -[firefox]: https://firefox-source-docs.mozilla.org/taskcluster/taskcluster/docker-images.html -[context]: https://docs.docker.com/engine/reference/commandline/build/#extended-description -[`COPY`]: https://docs.docker.com/engine/reference/builder/#copy - - -## Build artifacts - -[web-platform-tests] (WPT) is large enough that running all of a it takes a long time. -So it supports *chunking*, -such as multiple chunks of the test suite can be run in parallel on different machines. -As of this writing, -Servo’s current Buildbot setup for this has each machine start by compiling its own copy of Servo. -On Taskcluster with a decision task, -we can have a single build task save its resulting binary executable as an [artifact], -together with multiple testing tasks that each depend on the build task -(wait until it successfully finishes before they can start) -and start by downloading the artifact that was saved earlier. - -The logic for all this is in [`decision_task.py`](decision_task.py) -and can be modified in any pull request. - -[web-platform-tests]: https://github.com/web-platform-tests/wpt -[artifact]: https://community-tc.services.mozilla.com/docs/manual/using/artifacts - - -## Log artifacts - -Taskcluster automatically save the `stdio` output of a task as an artifact, -and has special support for showing and streaming that output while the task is still running. - -Servo’s decision task additionally looks for `*.log` arguments to its tasks’s commands, -assumes they instruct a program to create a log file with that name, -and saves those log files as individual artifacts. - -For example, WPT tasks have a `filtered-wpt-errorsummary.log` artifact -that is typically the most relevant output when such a task fails. - - -## Scopes and roles - -[Scopes] are what Taskcluster calls permissions. -They control access to everything. - -Anyone logged in in the [web UI] has (access to) a set of scopes, -which is visible on the [credentials] page -(reachable from clicking on one’s own name on the top-right of any page). - -A running task has a set of scopes allowing it access to various functionality and APIs. -It can grant those scopes (and at most only those) to sub-tasks that it schedules -(if it has the scope allowing it to schedule new tasks in the first place). - -[Roles] represent each a set of scopes. -They can be granted to… things, -and then configured separately to modify what scopes they [expand] to. - -For example, when Taskcluster-GitHub schedules tasks based on the `.taskcluster.yml` file -in a push to the `auto` branch of this repository, -those tasks are granted the scope `assume:repo:github.com/servo/servo:branch:auto`. -Scopes that start with `assume:` are special, -they expand to the scopes defined in the matching roles. -In this case, the [`repo:github.com/servo/servo:branch:*`][branches] role matches. - -The [`project:servo:decision-task/base`][base] -and [`project:servo:decision-task/trusted`][trusted] roles -centralize the set of scopes granted to the decision task. -This avoids maintaining them seprately in the `repo:…` roles, -in the `hook-id:…` role, -and in the `taskcluster.yml` file. -Only the `base` role is granted to tasks executed when a pull request is opened. -These tasks are less trusted because they run before the code has been reviewed, -and anyone can open a PR. - -Members of the [@servo/taskcluster-admins] GitHub team are granted -the scope `assume:project-admin:servo`, which is necessary to deploy changes -to those roles from the [servo/taskcluster-config] repository. - -[Scopes]: https://community-tc.services.mozilla.com/docs/manual/design/apis/hawk/scopes -[web UI]: https://community-tc.services.mozilla.com/ -[credentials]: https://community-tc.services.mozilla.com/profile -[Roles]: https://community-tc.services.mozilla.com/docs/manual/design/apis/hawk/roles -[expand]: https://community-tc.services.mozilla.com/docs/reference/platform/taskcluster-auth/docs/roles -[branches]: https://community-tc.services.mozilla.com/auth/roles/repo%3Agithub.com%2Fservo%2Fservo%3Abranch%3A* -[base]: https://community-tc.services.mozilla.com/auth/roles/project%3Aservo%3Adecision-task%2Fbase -[trusted]: https://community-tc.services.mozilla.com/auth/roles/project%3Aservo%3Adecision-task%2Ftrusted -[@servo/taskcluster-admins]: https://github.com/orgs/servo/teams/taskcluster-admins - - -## Daily tasks - -The [`project-servo/daily`] hook in Taskcluster’s [Hooks service] -is used to run some tasks automatically ever 24 hours. -In this case as well we use a decision task. -The `decision_task.py` script can differentiate this from a GitHub push -based on the `$TASK_FOR` environment variable. -Daily tasks can also be triggered manually. - -Scopes available to the daily decision task need to be both requested in the hook definition -and granted through the [`hook-id:project-servo/daily`] role. - -Because they do not have something similar to GitHub statuses that link to them, -daily tasks are indexed under the [`project.servo.daily`] namespace. - -[`project.servo.daily`]: https://tools.taskcluster.net/index/project.servo.daily -[`project-servo/daily`]: https://github.com/servo/taskcluster-config/blob/master/config/hooks.yml -[Hooks service]: https://community-tc.services.mozilla.com/docs/manual/using/scheduled-tasks -[`hook-id:project-servo/daily`]: https://github.com/servo/taskcluster-config/blob/master/config/roles.yml - - -## Servo’s worker pools - -Each task is assigned to a “worker pool”. -Servo has several, for the different environments a task can run in: - -* `docker` and `docker-untrusted` provide a Linux environment with full `root` privileges, - in a Docker container running a [Docker image](#docker-images) of the task’s choice, - in a short-lived virtual machine, - on Google Cloud Platform. - - Instances are started automatically as needed - when the existing capacity is insufficient to execute queued tasks. - They terminate themselves after being idle without work for a while, - or unconditionally after a few days. - Because these workers are short-lived, - we don’t need to worry about evicting old entries from Cargo’s or rustup’s download cache, - for example. - - [The Taskcluster team manages][mozilla/community-tc-config] - the configuration and VM image for these two pools. - The latter has fewer scopes. It runs automated testing of pull requests - as soon as they’re opened or updated, before any review. - -* `win2016` runs Windows Server 2016 on AWS EC2. - Like with Docker tasks, workers are short-lived and started automatically. - The configuration and VM image for this pool - is [managed by the Servo team][servo/taskcluster-config]. - - Tasks run as an unprivileged user. - Because creating an new the VM image is slow and deploying it mutates global state, - when a tool does not require system-wide installation - we prefer having each task obtain it as needed by extracting an archive in a directory. - See calls of `with_directory_mount` and `with_repacked_msi` in - [`decision_task.py`](decision_task.py) and [`decisionlib.py`](decisionlib.py). - -* `macos` runs, you guessed it, macOS. - Tasks run on dedicated hardware provided long-term by Macstadium. - The system-wide configuration of those machines - is [managed by the Servo team][servo/taskcluster-config] through SaltStack. - There is a task-owned (but preserved across tasks) install of Homebrew, - with `Brewfile`s [in this repository](macos/). - - This [Workers] page lists the current state of each macOS worker. - (A similar page exists for other each worker pools, but as of this writing it has - [usability issues](https://github.com/taskcluster/taskcluster/issues/1972) - with short-lived workers.) - -[Workers]: https://community-tc.services.mozilla.com/provisioners/proj-servo/worker-types/macos - - -## Taskcluster − Treeherder integration - -See [`treeherder.md`](treeherder.md). - - -## Self-service, IRC, and Bugzilla - -Taskcluster is designed to be “self-service” as much as possible. -Between this repository [servo/taskcluster-config] and [mozilla/community-tc-config], -anyone should be able to submit PRs for any part of the configuration. - -Feel free to ask for help on the `#servo` or `#taskcluster` channels on Mozilla IRC. - -For issue reports or feature requests on various bits of Taskcluster *software*, -file bugs [in Mozilla’s Bugzilla, under `Taskcluster`][bug]. - -[bug]: https://bugzilla.mozilla.org/enter_bug.cgi?product=Taskcluster diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py deleted file mode 100644 index 89039137cb7..00000000000 --- a/etc/taskcluster/decision_task.py +++ /dev/null @@ -1,103 +0,0 @@ -# coding: utf8 - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -import os.path -import decisionlib -import functools -from decisionlib import CONFIG, SHARED -from urllib.request import urlopen - - -def main(task_for): - with decisionlib.make_repo_bundle(): - tasks(task_for) - - -def tasks(task_for): - if CONFIG.git_ref.startswith("refs/heads/"): - branch = CONFIG.git_ref[len("refs/heads/"):] - CONFIG.treeherder_repository_name = "servo-" + ( - branch if not branch.startswith("try-") else "try" - ) - - # Work around a tc-github bug/limitation: - # https://bugzilla.mozilla.org/show_bug.cgi?id=1548781#c4 - if task_for.startswith("github"): - # https://github.com/taskcluster/taskcluster/blob/21f257dc8/services/github/config.yml#L14 - CONFIG.routes_for_all_subtasks.append("statuses") - - if task_for == "github-push": - all_tests = [] - by_branch_name = { - "auto": all_tests, - "try": all_tests, - "try-taskcluster": [ - # Add functions here as needed, in your push to that branch - ], - "master": [], - - # The "try-*" keys match those in `servo_try_choosers` in Homu’s config: - # https://github.com/servo/saltfs/blob/master/homu/map.jinja - - "try-mac": [], - "try-linux": [], - "try-windows": [], - "try-arm": [], - "try-wpt": [], - "try-wpt-2020": [], - "try-wpt-mac": [], - "test-wpt": [], - } - - elif task_for == "github-pull-request": - CONFIG.treeherder_repository_name = "servo-prs" - CONFIG.index_read_only = True - CONFIG.docker_image_build_worker_type = None - - # We want the merge commit that GitHub creates for the PR. - # The event does contain a `pull_request.merge_commit_sha` key, but it is wrong: - # https://github.com/servo/servo/pull/22597#issuecomment-451518810 - CONFIG.git_sha_is_current_head() - - elif task_for == "try-windows-ami": - CONFIG.git_sha_is_current_head() - CONFIG.windows_worker_type = os.environ["NEW_AMI_WORKER_TYPE"] - - # https://tools.taskcluster.net/hooks/project-servo/daily - elif task_for == "daily": - daily_tasks_setup() - - -ping_on_daily_task_failure = "SimonSapin, nox, emilio" -build_artifacts_expire_in = "1 week" -build_dependencies_artifacts_expire_in = "1 month" -log_artifacts_expire_in = "1 year" - - -def daily_tasks_setup(): - # Unlike when reacting to a GitHub push event, - # the commit hash is not known until we clone the repository. - CONFIG.git_sha_is_current_head() - - # On failure, notify a few people on IRC - # https://docs.taskcluster.net/docs/reference/core/taskcluster-notify/docs/usage - notify_route = "notify.irc-channel.#servo.on-failed" - CONFIG.routes_for_all_subtasks.append(notify_route) - CONFIG.scopes_for_all_subtasks.append("queue:route:" + notify_route) - CONFIG.task_name_template = "Servo daily: %s. On failure, ping: " + ping_on_daily_task_failure - - -CONFIG.task_name_template = "Servo: %s" -CONFIG.docker_images_expire_in = build_dependencies_artifacts_expire_in -CONFIG.repacked_msi_files_expire_in = build_dependencies_artifacts_expire_in -CONFIG.index_prefix = "project.servo" -CONFIG.default_provisioner_id = "proj-servo" -CONFIG.docker_image_build_worker_type = "docker" - -CONFIG.windows_worker_type = "win2016" - -if __name__ == "__main__": # pragma: no cover - main(task_for=os.environ["TASK_FOR"]) diff --git a/etc/taskcluster/decisionlib.py b/etc/taskcluster/decisionlib.py deleted file mode 100644 index 22d9a2f22a0..00000000000 --- a/etc/taskcluster/decisionlib.py +++ /dev/null @@ -1,851 +0,0 @@ -# coding: utf8 - -# Copyright 2018 The Servo Project Developers. See the COPYRIGHT -# file at the top-level directory of this distribution. -# -# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or -# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license -# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your -# option. This file may not be copied, modified, or distributed -# except according to those terms. - -""" -Project-independent library for Taskcluster decision tasks -""" - -import base64 -import contextlib -import datetime -import hashlib -import json -import os -import re -import subprocess -import sys -import taskcluster - - -# Public API -__all__ = [ - "CONFIG", "SHARED", "Task", "DockerWorkerTask", - "GenericWorkerTask", "WindowsGenericWorkerTask", - "make_repo_bundle", -] - - -class Config: - """ - Global configuration, for users of the library to modify. - """ - def __init__(self): - self.task_name_template = "%s" - self.index_prefix = "garbage.servo-decisionlib" - self.index_read_only = False - self.scopes_for_all_subtasks = [] - self.routes_for_all_subtasks = [] - self.docker_image_build_worker_type = None - self.docker_images_expire_in = "1 month" - self.repacked_msi_files_expire_in = "1 month" - self.treeherder_repository_name = None - - # Set by docker-worker: - # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/environment - self.decision_task_id = os.environ.get("TASK_ID") - - # Set in the decision task’s payload, such as defined in .taskcluster.yml - self.task_owner = os.environ.get("TASK_OWNER") - self.task_source = os.environ.get("TASK_SOURCE") - self.git_url = os.environ.get("GIT_URL") - self.git_ref = os.environ.get("GIT_REF") - self.git_sha = os.environ.get("GIT_SHA") - self.git_bundle_shallow_ref = "refs/heads/shallow" - - self.tc_root_url = os.environ.get("TASKCLUSTER_ROOT_URL") - self.default_provisioner_id = "proj-example" - - - def tree_hash(self): - if not hasattr(self, "_tree_hash"): - # Use the SHA-1 hash of the git "tree" object rather than the commit. - # A `@bors-servo retry` command creates a new merge commit with a different commit hash - # but with the same tree hash. - output = subprocess.check_output(["git", "show", "-s", "--format=%T", "HEAD"]) - self._tree_hash = output.decode("utf-8").strip() - return self._tree_hash - - def git_sha_is_current_head(self): - output = subprocess.check_output(["git", "rev-parse", "HEAD"]) - self.git_sha = output.decode("utf8").strip() - - -class Shared: - """ - Global shared state. - """ - def __init__(self): - self.now = datetime.datetime.utcnow() - self.found_or_created_indexed_tasks = {} - - options = {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]} - self.queue_service = taskcluster.Queue(options) - self.index_service = taskcluster.Index(options) - - def from_now_json(self, offset): - """ - Same as `taskcluster.fromNowJSON`, but uses the creation time of `self` for “now”. - """ - return taskcluster.stringDate(taskcluster.fromNow(offset, dateObj=self.now)) - - -CONFIG = Config() -SHARED = Shared() - - -def chaining(op, attr): - def method(self, *args, **kwargs): - op(self, attr, *args, **kwargs) - return self - return method - - -def append_to_attr(self, attr, *args): getattr(self, attr).extend(args) -def prepend_to_attr(self, attr, *args): getattr(self, attr)[0:0] = list(args) -def update_attr(self, attr, **kwargs): getattr(self, attr).update(kwargs) - - -class Task: - """ - A task definition, waiting to be created. - - Typical is to use chain the `with_*` methods to set or extend this object’s attributes, - then call the `crate` or `find_or_create` method to schedule a task. - - This is an abstract class that needs to be specialized for different worker implementations. - """ - def __init__(self, name): - self.name = name - self.description = "" - self.scheduler_id = "taskcluster-github" - self.provisioner_id = CONFIG.default_provisioner_id - self.worker_type = "github-worker" - self.deadline_in = "1 day" - self.expires_in = "1 year" - self.index_and_artifacts_expire_in = self.expires_in - self.dependencies = [] - self.scopes = [] - self.routes = [] - self.extra = {} - self.treeherder_required = False - self.priority = None # Defaults to 'lowest' - self.git_fetch_url = CONFIG.git_url - self.git_fetch_ref = CONFIG.git_ref - self.git_checkout_sha = CONFIG.git_sha - - # All `with_*` methods return `self`, so multiple method calls can be chained. - with_description = chaining(setattr, "description") - with_scheduler_id = chaining(setattr, "scheduler_id") - with_provisioner_id = chaining(setattr, "provisioner_id") - with_worker_type = chaining(setattr, "worker_type") - with_deadline_in = chaining(setattr, "deadline_in") - with_expires_in = chaining(setattr, "expires_in") - with_index_and_artifacts_expire_in = chaining(setattr, "index_and_artifacts_expire_in") - with_priority = chaining(setattr, "priority") - - with_dependencies = chaining(append_to_attr, "dependencies") - with_scopes = chaining(append_to_attr, "scopes") - with_routes = chaining(append_to_attr, "routes") - - with_extra = chaining(update_attr, "extra") - - def with_index_at(self, index_path): - self.routes.append("index.%s.%s" % (CONFIG.index_prefix, index_path)) - return self - - def with_treeherder_required(self): - self.treeherder_required = True - return self - - def with_treeherder(self, category, symbol, group_name=None, group_symbol=None): - assert len(symbol) <= 25, symbol - self.name = "%s: %s" % (category, self.name) - - # The message schema does not allow spaces in the platfrom or in labels, - # but the UI shows them in that order separated by spaces. - # So massage the metadata to get the UI to show the string we want. - # `labels` defaults to ["opt"] if not provided or empty, - # so use a more neutral underscore instead. - parts = category.split(" ") - platform = parts[0] - labels = parts[1:] or ["_"] - - # https://github.com/mozilla/treeherder/blob/master/schemas/task-treeherder-config.yml - self.with_extra(treeherder=dict_update_if_truthy( - { - "machine": {"platform": platform}, - "labels": labels, - "symbol": symbol, - }, - groupName=group_name, - groupSymbol=group_symbol, - )) - - if CONFIG.treeherder_repository_name: - assert CONFIG.git_sha - suffix = ".v2._/%s.%s" % (CONFIG.treeherder_repository_name, CONFIG.git_sha) - self.with_routes( - "tc-treeherder" + suffix, - "tc-treeherder-staging" + suffix, - ) - - self.treeherder_required = False # Taken care of - return self - - def build_worker_payload(self): # pragma: no cover - """ - Overridden by sub-classes to return a dictionary in a worker-specific format, - which is used as the `payload` property in a task definition request - passed to the Queue’s `createTask` API. - - <https://docs.taskcluster.net/docs/reference/platform/taskcluster-queue/references/api#createTask> - """ - raise NotImplementedError - - def create(self): - """ - Call the Queue’s `createTask` API to schedule a new task, and return its ID. - - <https://docs.taskcluster.net/docs/reference/platform/taskcluster-queue/references/api#createTask> - """ - worker_payload = self.build_worker_payload() - assert not self.treeherder_required, \ - "make sure to call with_treeherder() for this task: %s" % self.name - - assert CONFIG.decision_task_id - assert CONFIG.task_owner - assert CONFIG.task_source - - def dedup(xs): - seen = set() - return [x for x in xs if not (x in seen or seen.add(x))] - - queue_payload = { - "taskGroupId": CONFIG.decision_task_id, - "dependencies": dedup([CONFIG.decision_task_id] + self.dependencies), - "schedulerId": self.scheduler_id, - "provisionerId": self.provisioner_id, - "workerType": self.worker_type, - - "created": SHARED.from_now_json(""), - "deadline": SHARED.from_now_json(self.deadline_in), - "expires": SHARED.from_now_json(self.expires_in), - "metadata": { - "name": CONFIG.task_name_template % self.name, - "description": self.description, - "owner": CONFIG.task_owner, - "source": CONFIG.task_source, - }, - - "payload": worker_payload, - } - scopes = self.scopes + CONFIG.scopes_for_all_subtasks - routes = self.routes + CONFIG.routes_for_all_subtasks - if any(r.startswith("index.") for r in routes): - self.extra.setdefault("index", {})["expires"] = \ - SHARED.from_now_json(self.index_and_artifacts_expire_in) - dict_update_if_truthy( - queue_payload, - scopes=scopes, - routes=routes, - extra=self.extra, - priority=self.priority, - ) - - task_id = taskcluster.slugId() - SHARED.queue_service.createTask(task_id, queue_payload) - print("Scheduled %s: %s" % (task_id, self.name)) - return task_id - - @staticmethod - def find(index_path): - full_index_path = "%s.%s" % (CONFIG.index_prefix, index_path) - task_id = SHARED.index_service.findTask(full_index_path)["taskId"] - print("Found task %s indexed at %s" % (task_id, full_index_path)) - return task_id - - def find_or_create(self, index_path): - """ - Try to find a task in the Index and return its ID. - - The index path used is `{CONFIG.index_prefix}.{index_path}`. - `index_path` defaults to `by-task-definition.{sha256}` - with a hash of the worker payload and worker type. - - If no task is found in the index, - it is created with a route to add it to the index at that same path if it succeeds. - - <https://docs.taskcluster.net/docs/reference/core/taskcluster-index/references/api#findTask> - """ - task_id = SHARED.found_or_created_indexed_tasks.get(index_path) - if task_id is not None: - return task_id - - try: - task_id = Task.find(index_path) - except taskcluster.TaskclusterRestFailure as e: - if e.status_code != 404: # pragma: no cover - raise - if not CONFIG.index_read_only: - self.with_index_at(index_path) - task_id = self.create() - - SHARED.found_or_created_indexed_tasks[index_path] = task_id - return task_id - - def with_curl_script(self, url, file_path): - return self \ - .with_script(""" - curl --compressed --retry 5 --connect-timeout 10 -Lf "%s" -o "%s" - """ % (url, file_path)) - - def with_curl_artifact_script(self, task_id, artifact_name, out_directory=""): - queue_service = CONFIG.tc_root_url + "/api/queue" - return self \ - .with_dependencies(task_id) \ - .with_curl_script( - queue_service + "/v1/task/%s/artifacts/public/%s" % (task_id, artifact_name), - os.path.join(out_directory, url_basename(artifact_name)), - ) - - def with_repo_bundle(self, **kwargs): - self.git_fetch_url = "../repo.bundle" - self.git_fetch_ref = CONFIG.git_bundle_shallow_ref - self.git_checkout_sha = "FETCH_HEAD" - return self \ - .with_curl_artifact_script(CONFIG.decision_task_id, "repo.bundle") \ - .with_repo(**kwargs) - - -class GenericWorkerTask(Task): - """ - Task definition for a worker type that runs the `generic-worker` implementation. - - This is an abstract class that needs to be specialized for different operating systems. - - <https://github.com/taskcluster/generic-worker> - """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.max_run_time_minutes = 30 - self.env = {} - self.features = {} - self.mounts = [] - self.artifacts = [] - - with_max_run_time_minutes = chaining(setattr, "max_run_time_minutes") - with_mounts = chaining(append_to_attr, "mounts") - with_env = chaining(update_attr, "env") - - def build_command(self): # pragma: no cover - """ - Overridden by sub-classes to return the `command` property of the worker payload, - in the format appropriate for the operating system. - """ - raise NotImplementedError - - def build_worker_payload(self): - """ - Return a `generic-worker` worker payload. - - <https://docs.taskcluster.net/docs/reference/workers/generic-worker/docs/payload> - """ - worker_payload = { - "command": self.build_command(), - "maxRunTime": self.max_run_time_minutes * 60 - } - return dict_update_if_truthy( - worker_payload, - env=self.env, - mounts=self.mounts, - features=self.features, - artifacts=[ - { - "type": type_, - "path": path, - "name": "public/" + url_basename(path), - "expires": SHARED.from_now_json(self.index_and_artifacts_expire_in), - } - for type_, path in self.artifacts - ], - ) - - def with_artifacts(self, *paths, type="file"): - """ - Add each path in `paths` as a task artifact - that expires in `self.index_and_artifacts_expire_in`. - - `type` can be `"file"` or `"directory"`. - - Paths are relative to the task’s home directory. - """ - for path in paths: - if (type, path) in self.artifacts: - raise ValueError("Duplicate artifact: " + path) # pragma: no cover - self.artifacts.append(tuple((type, path))) - return self - - def with_features(self, *names): - """ - Enable the given `generic-worker` features. - - <https://github.com/taskcluster/generic-worker/blob/master/native_windows.yml> - """ - self.features.update({name: True for name in names}) - return self - - def _mount_content(self, url_or_artifact_name, task_id, sha256): - if task_id: - content = {"taskId": task_id, "artifact": url_or_artifact_name} - else: - content = {"url": url_or_artifact_name} - if sha256: - content["sha256"] = sha256 - return content - - def with_file_mount(self, url_or_artifact_name, task_id=None, sha256=None, path=None): - """ - Make `generic-worker` download a file before the task starts - and make it available at `path` (which is relative to the task’s home directory). - - If `sha256` is provided, `generic-worker` will hash the downloaded file - and check it against the provided signature. - - If `task_id` is provided, this task will depend on that task - and `url_or_artifact_name` is the name of an artifact of that task. - """ - return self.with_mounts({ - "file": path or url_basename(url_or_artifact_name), - "content": self._mount_content(url_or_artifact_name, task_id, sha256), - }) - - def with_directory_mount(self, url_or_artifact_name, task_id=None, sha256=None, path=None): - """ - Make `generic-worker` download an archive before the task starts, - and uncompress it at `path` (which is relative to the task’s home directory). - - `url_or_artifact_name` must end in one of `.rar`, `.tar.bz2`, `.tar.gz`, or `.zip`. - The archive must be in the corresponding format. - - If `sha256` is provided, `generic-worker` will hash the downloaded archive - and check it against the provided signature. - - If `task_id` is provided, this task will depend on that task - and `url_or_artifact_name` is the name of an artifact of that task. - """ - supported_formats = ["rar", "tar.bz2", "tar.gz", "zip"] - for fmt in supported_formats: - suffix = "." + fmt - if url_or_artifact_name.endswith(suffix): - return self.with_mounts({ - "directory": path or url_basename(url_or_artifact_name[:-len(suffix)]), - "content": self._mount_content(url_or_artifact_name, task_id, sha256), - "format": fmt, - }) - raise ValueError( - "%r does not appear to be in one of the supported formats: %r" - % (url_or_artifact_name, ", ".join(supported_formats)) - ) # pragma: no cover - - -class WindowsGenericWorkerTask(GenericWorkerTask): - """ - Task definition for a `generic-worker` task running on Windows. - - Scripts are written as `.bat` files executed with `cmd.exe`. - """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.scripts = [] - self.rdp_info_artifact_name = None - - with_script = chaining(append_to_attr, "scripts") - with_early_script = chaining(prepend_to_attr, "scripts") - - def build_worker_payload(self): - if self.rdp_info_artifact_name: - rdp_scope = "generic-worker:allow-rdp:%s/%s" % (self.provisioner_id, self.worker_type) - self.scopes.append(rdp_scope) - self.scopes.append("generic-worker:os-group:proj-servo/win2016/Administrators") - self.scopes.append("generic-worker:run-as-administrator:proj-servo/win2016") - self.with_features("runAsAdministrator") - return dict_update_if_truthy( - super().build_worker_payload(), - rdpInfo=self.rdp_info_artifact_name, - osGroups=["Administrators"] - ) - - def with_rdp_info(self, *, artifact_name): - """ - Enable RDP access to this task’s environment. - - See `rdpInfo` in - <https://community-tc.services.mozilla.com/docs/reference/workers/generic-worker/multiuser-windows-payload> - """ - assert not artifact_name.startswith("public/") - self.rdp_info_artifact_name = artifact_name - - def build_command(self): - return [deindent(s) for s in self.scripts] - - def with_path_from_homedir(self, *paths): - """ - Interpret each path in `paths` as relative to the task’s home directory, - and add it to the `PATH` environment variable. - """ - for p in paths: - self.with_early_script("set PATH=%HOMEDRIVE%%HOMEPATH%\\{};%PATH%".format(p)) - return self - - def with_repo(self, sparse_checkout=None): - """ - Make a clone the git repository at the start of the task. - This uses `CONFIG.git_url`, `CONFIG.git_ref`, and `CONFIG.git_sha`, - and creates the clone in a `repo` directory in the task’s home directory. - - If `sparse_checkout` is given, it must be a list of path patterns - to be used in `.git/info/sparse-checkout`. - See <https://git-scm.com/docs/git-read-tree#_sparse_checkout>. - """ - git = """ - git init repo - cd repo - """ - if sparse_checkout: - self.with_mounts({ - "file": "sparse-checkout", - "content": {"raw": "\n".join(sparse_checkout)}, - }) - git += """ - git config core.sparsecheckout true - copy ..\\sparse-checkout .git\\info\\sparse-checkout - type .git\\info\\sparse-checkout - """ - git += """ - git fetch --no-tags {} {} - git reset --hard {} - """.format( - assert_truthy(self.git_fetch_url), - assert_truthy(self.git_fetch_ref), - assert_truthy(self.git_checkout_sha), - ) - return self \ - .with_git() \ - .with_script(git) - - def with_git(self): - """ - Make the task download `git-for-windows` and make it available for `git` commands. - - This is implied by `with_repo`. - """ - return self \ - .with_path_from_homedir("git\\cmd") \ - .with_directory_mount( - "https://github.com/git-for-windows/git/releases/download/" + - "v2.24.0.windows.2/MinGit-2.24.0.2-64-bit.zip", - sha256="c33aec6ae68989103653ca9fb64f12cabccf6c61d0dde30c50da47fc15cf66e2", - path="git", - ) - - def with_curl_script(self, url, file_path): - self.with_curl() - return super().with_curl_script(url, file_path) - - def with_curl(self): - return self \ - .with_path_from_homedir("curl\\curl-7.73.0-win64-mingw\\bin") \ - .with_directory_mount( - "https://curl.haxx.se/windows/dl-7.73.0/curl-7.73.0-win64-mingw.zip", - sha256="2e1ffdb6c25c8648a1243bb3a268120be442399b1c93d7da309bba235ecdab9a", - path="curl", - ) - - def with_rustup(self): - """ - Download rustup.rs and make it available to task commands, - but does not download any default toolchain. - """ - return self \ - .with_path_from_homedir(".cargo\\bin") \ - .with_early_script( - "%HOMEDRIVE%%HOMEPATH%\\rustup-init.exe --default-toolchain none --profile=minimal -y" - ) \ - .with_file_mount("https://win.rustup.rs/x86_64", path="rustup-init.exe") - - def with_repacked_msi(self, url, sha256, path): - """ - Download an MSI file from `url`, extract the files in it with `lessmsi`, - and make them available in the directory at `path` (relative to the task’s home directory). - - `sha256` is required and the MSI file must have that hash. - - The file extraction (and recompression in a ZIP file) is done in a separate task, - wich is indexed based on `sha256` and cached for `CONFIG.repacked_msi_files_expire_in`. - - <https://github.com/activescott/lessmsi> - """ - repack_task = ( - WindowsGenericWorkerTask("MSI repack: " + url) - .with_worker_type(self.worker_type) - .with_max_run_time_minutes(20) - .with_file_mount(url, sha256=sha256, path="input.msi") - .with_directory_mount( - "https://github.com/activescott/lessmsi/releases/download/" + - "v1.6.1/lessmsi-v1.6.1.zip", - sha256="540b8801e08ec39ba26a100c855898f455410cecbae4991afae7bb2b4df026c7", - path="lessmsi" - ) - .with_directory_mount( - "https://www.7-zip.org/a/7za920.zip", - sha256="2a3afe19c180f8373fa02ff00254d5394fec0349f5804e0ad2f6067854ff28ac", - path="7zip", - ) - .with_path_from_homedir("lessmsi", "7zip") - .with_script(""" - lessmsi x input.msi extracted\\ - cd extracted\\SourceDir - 7za a repacked.zip * - """) - .with_artifacts("extracted/SourceDir/repacked.zip") - .with_index_and_artifacts_expire_in(CONFIG.repacked_msi_files_expire_in) - .find_or_create("repacked-msi." + sha256) - ) - return self \ - .with_dependencies(repack_task) \ - .with_directory_mount("public/repacked.zip", task_id=repack_task, path=path) - - def with_python3(self): - """ - For Python 3, use `with_directory_mount` and the "embeddable zip file" distribution - from python.org. - You may need to remove `python37._pth` from the ZIP in order to work around - <https://bugs.python.org/issue34841>. - """ - return ( - self - .with_curl_script( - "https://www.python.org/ftp/python/3.7.3/python-3.7.3-amd64.exe", - "do-the-python.exe" - ) - .with_script("do-the-python.exe /quiet TargetDir=%HOMEDRIVE%%HOMEPATH%\\python3") - .with_path_from_homedir("python3", "python3\\Scripts") - .with_script("pip install virtualenv==20.2.1") - ) - - -class UnixTaskMixin(Task): - def with_repo(self, alternate_object_dir=""): - """ - Make a clone the git repository at the start of the task. - This uses `CONFIG.git_url`, `CONFIG.git_ref`, and `CONFIG.git_sha` - - * generic-worker: creates the clone in a `repo` directory - in the task’s directory. - - * docker-worker: creates the clone in a `/repo` directory - at the root of the Docker container’s filesystem. - `git` and `ca-certificate` need to be installed in the Docker image. - - """ - # Not using $GIT_ALTERNATE_OBJECT_DIRECTORIES since it causes - # "object not found - no match for id" errors when Cargo fetches git dependencies - return self \ - .with_script(""" - git init repo - cd repo - echo "{alternate}" > .git/objects/info/alternates - time git fetch --no-tags {} {} - time git reset --hard {} - """.format( - assert_truthy(self.git_fetch_url), - assert_truthy(self.git_fetch_ref), - assert_truthy(self.git_checkout_sha), - alternate=alternate_object_dir, - )) - - -class DockerWorkerTask(UnixTaskMixin, Task): - """ - Task definition for a worker type that runs the `generic-worker` implementation. - - Scripts are interpreted with `bash`. - - <https://github.com/taskcluster/docker-worker> - """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.docker_image = "ubuntu:bionic-20180821" - self.max_run_time_minutes = 30 - self.scripts = [] - self.env = {} - self.caches = {} - self.features = {} - self.capabilities = {} - self.artifacts = [] - - with_docker_image = chaining(setattr, "docker_image") - with_max_run_time_minutes = chaining(setattr, "max_run_time_minutes") - with_script = chaining(append_to_attr, "scripts") - with_early_script = chaining(prepend_to_attr, "scripts") - with_caches = chaining(update_attr, "caches") - with_env = chaining(update_attr, "env") - with_capabilities = chaining(update_attr, "capabilities") - - def with_artifacts(self, *paths): - for path in paths: - if path in self.artifacts: - raise ValueError("Duplicate artifact: " + path) # pragma: no cover - self.artifacts.append(path) - return self - - def build_worker_payload(self): - """ - Return a `docker-worker` worker payload. - - <https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/payload> - """ - worker_payload = { - "image": self.docker_image, - "maxRunTime": self.max_run_time_minutes * 60, - "command": [ - "/bin/bash", "--login", "-x", "-e", "-o", "pipefail", "-c", - deindent("\n".join(self.scripts)) - ], - } - return dict_update_if_truthy( - worker_payload, - env=self.env, - cache=self.caches, - features=self.features, - capabilities=self.capabilities, - artifacts={ - "public/" + url_basename(path): { - "type": "file", - "path": path, - "expires": SHARED.from_now_json(self.index_and_artifacts_expire_in), - } - for path in self.artifacts - }, - ) - - def with_features(self, *names): - """ - Enable the given `docker-worker` features. - - <https://github.com/taskcluster/docker-worker/blob/master/docs/features.md> - """ - self.features.update({name: True for name in names}) - return self - - def with_dockerfile(self, dockerfile): - """ - Build a Docker image based on the given `Dockerfile`, and use it for this task. - - `dockerfile` is a path in the filesystem where this code is running. - Some non-standard syntax is supported, see `expand_dockerfile`. - - The image is indexed based on a hash of the expanded `Dockerfile`, - and cached for `CONFIG.docker_images_expire_in`. - - Images are built without any *context*. - <https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#understand-build-context> - """ - basename = os.path.basename(dockerfile) - suffix = ".dockerfile" - assert basename.endswith(suffix) - image_name = basename[:-len(suffix)] - - dockerfile_contents = expand_dockerfile(dockerfile) - digest = hashlib.sha256(dockerfile_contents).hexdigest() - - image_build_task = ( - DockerWorkerTask("Docker image: " + image_name) - .with_worker_type(CONFIG.docker_image_build_worker_type or self.worker_type) - .with_max_run_time_minutes(30) - .with_index_and_artifacts_expire_in(CONFIG.docker_images_expire_in) - .with_features("dind") - .with_env(DOCKERFILE=dockerfile_contents) - .with_artifacts("/image.tar.lz4") - .with_script(""" - echo "$DOCKERFILE" | docker build -t taskcluster-built - - docker save taskcluster-built | lz4 > /image.tar.lz4 - """) - .with_docker_image( - # https://github.com/servo/taskcluster-bootstrap-docker-images#image-builder - "servobrowser/taskcluster-bootstrap:image-builder@sha256:" \ - "0a7d012ce444d62ffb9e7f06f0c52fedc24b68c2060711b313263367f7272d9d" - ) - .find_or_create("docker-image." + digest) - ) - - return self \ - .with_dependencies(image_build_task) \ - .with_docker_image({ - "type": "task-image", - "path": "public/image.tar.lz4", - "taskId": image_build_task, - }) - - -def expand_dockerfile(dockerfile): - """ - Read the file at path `dockerfile`, - and transitively expand the non-standard `% include` header if it is present. - """ - with open(dockerfile, "rb") as f: - dockerfile_contents = f.read() - - include_marker = b"% include" - if not dockerfile_contents.startswith(include_marker): - return dockerfile_contents - - include_line, _, rest = dockerfile_contents.partition(b"\n") - included = include_line[len(include_marker):].strip().decode("utf8") - path = os.path.join(os.path.dirname(dockerfile), included) - return b"\n".join([expand_dockerfile(path), rest]) - - -def assert_truthy(x): - assert x - return x - - -def dict_update_if_truthy(d, **kwargs): - for key, value in kwargs.items(): - if value: - d[key] = value - return d - - -def deindent(string): - return re.sub("\n +", "\n ", string).strip() - - -def url_basename(url): - return url.rpartition("/")[-1] - - -@contextlib.contextmanager -def make_repo_bundle(): - subprocess.check_call(["git", "config", "user.name", "Decision task"]) - subprocess.check_call(["git", "config", "user.email", "nobody@mozilla.com"]) - tree = subprocess.check_output(["git", "show", CONFIG.git_sha, "--pretty=%T", "--no-patch"]) - message = "Shallow version of commit " + CONFIG.git_sha - commit = subprocess.check_output(["git", "commit-tree", tree.strip(), "-m", message]) - subprocess.check_call(["git", "update-ref", CONFIG.git_bundle_shallow_ref, commit.strip()]) - subprocess.check_call(["git", "show-ref"]) - create = ["git", "bundle", "create", "../repo.bundle", CONFIG.git_bundle_shallow_ref] - with subprocess.Popen(create) as p: - yield - exit_code = p.wait() - if exit_code: - sys.exit(exit_code) diff --git a/etc/taskcluster/docker/base.dockerfile b/etc/taskcluster/docker/base.dockerfile deleted file mode 100644 index 18a2dbac6a0..00000000000 --- a/etc/taskcluster/docker/base.dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM ubuntu:20.04 - -ENV \ - # - # Some APT packages like 'tzdata' wait for user input on install by default. - # https://stackoverflow.com/questions/44331836/apt-get-install-tzdata-noninteractive - DEBIAN_FRONTEND=noninteractive \ - LANG=C.UTF-8 \ - LANGUAGE=C.UTF-8 \ - LC_ALL=C.UTF-8 - -RUN \ - apt-get update -q && \ - apt-get install -qy --no-install-recommends \ - # - # Cloning the repository - git \ - ca-certificates \ - # - # Running mach with Python 3 - python3 \ - python3-pip \ - python3-dev \ - virtualenv \ - # - # Compiling C modules when installing Python packages in a virtualenv - gcc \ - # - # Installing rustup and sccache (build dockerfile) or fetching build artifacts (run tasks) - curl \ - # Setting the default locale - locales \ - locales-all diff --git a/etc/taskcluster/docker/build.dockerfile b/etc/taskcluster/docker/build.dockerfile deleted file mode 100644 index 60ae5606cdb..00000000000 --- a/etc/taskcluster/docker/build.dockerfile +++ /dev/null @@ -1,53 +0,0 @@ -% include base.dockerfile - -RUN \ - apt-get install -qy --no-install-recommends \ - # - # Testing decisionlib (see etc/taskcluster/mock.py) - python3-coverage \ - # - # Multiple C/C++ dependencies built from source - g++ \ - make \ - cmake \ - # - # Fontconfig - gperf \ - # - # ANGLE - xorg-dev \ - # - # mozjs (SpiderMonkey) - autoconf2.13 \ - # - # Bindgen (for SpiderMonkey bindings) - clang \ - llvm \ - llvm-dev \ - # - # GStreamer - libpcre3-dev \ - # - # OpenSSL - libssl-dev \ - # - # blurz - libdbus-1-dev \ - # - # sampling profiler - libunwind-dev \ - # - # x11 integration - libxcb-render-util0-dev \ - libxcb-shape0-dev \ - libxcb-xfixes0-dev \ - # - && \ - # - # Install the version of rustup that is current when this Docker image is being built: - # We want at least 1.21 (increment in this comment to force an image rebuild). - curl https://sh.rustup.rs -sSf | sh -s -- --profile=minimal -y && \ - # - # There are no sccache binary releases that include this commit, so we install a particular - # git commit instead. - ~/.cargo/bin/cargo install sccache --git https://github.com/mozilla/sccache/ --rev e66c9c15142a7e583d6ab80bd614bdffb2ebcc47 diff --git a/etc/taskcluster/docker/run-android-emulator.dockerfile b/etc/taskcluster/docker/run-android-emulator.dockerfile deleted file mode 100644 index 92eb116ef6b..00000000000 --- a/etc/taskcluster/docker/run-android-emulator.dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -% include base.dockerfile - -RUN \ - apt-get install -qy --no-install-recommends \ - # - # Multiple Android-related tools are in Java - openjdk-8-jdk-headless \ - # - # Emulator dependencies - libgl1 \ - libpulse0 diff --git a/etc/taskcluster/docker/run.dockerfile b/etc/taskcluster/docker/run.dockerfile deleted file mode 100644 index f02b4fdff97..00000000000 --- a/etc/taskcluster/docker/run.dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -% include base.dockerfile - -# Servo’s runtime dependencies: -RUN apt-get install -qy --no-install-recommends \ - libgl1 \ - libssl1.1 \ - libdbus-1-3 \ - libxcb-shape0-dev \ - gstreamer1.0-plugins-good \ - gstreamer1.0-plugins-bad \ - gstreamer1.0-libav \ - gstreamer1.0-gl \ - libunwind8 diff --git a/etc/taskcluster/docker/wpt-update.dockerfile b/etc/taskcluster/docker/wpt-update.dockerfile deleted file mode 100644 index 0a8edde7bc9..00000000000 --- a/etc/taskcluster/docker/wpt-update.dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -% include run.dockerfile - -RUN apt-get install -qy --no-install-recommends \ - python3 \ - jq diff --git a/etc/taskcluster/mock.py b/etc/taskcluster/mock.py deleted file mode 100755 index 0bce7891876..00000000000 --- a/etc/taskcluster/mock.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2018 The Servo Project Developers. See the COPYRIGHT -# file at the top-level directory of this distribution. -# -# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or -# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license -# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your -# option. This file may not be copied, modified, or distributed -# except according to those terms. - -''''set -e -python3 -m coverage run $0 -python3 -m coverage report -m --fail-under 100 -exit - -Run the decision task with fake Taskcluster APIs, to catch Python errors before pushing. -''' - -import os -import sys -from unittest.mock import MagicMock - - -class TaskclusterRestFailure(Exception): - status_code = 404 - - -class Index: - __init__ = insertTask = lambda *_, **__: None - - def findTask(self, path): - if decision_task.CONFIG.git_ref == "refs/heads/master": - return {"taskId": "<from index>"} - raise TaskclusterRestFailure - - -stringDate = str -slugId = b"<new id>".lower -sys.exit = Queue = fromNow = MagicMock() -sys.modules["taskcluster"] = sys.modules[__name__] -sys.dont_write_bytecode = True -os.environ.update(**{k: k for k in "TASK_ID TASK_OWNER TASK_SOURCE GIT_URL GIT_SHA".split()}) -os.environ["GIT_REF"] = "refs/heads/auto" -os.environ["TASKCLUSTER_ROOT_URL"] = "https://community-tc.services.mozilla.com" -os.environ["TASKCLUSTER_PROXY_URL"] = "http://taskcluster" -os.environ["NEW_AMI_WORKER_TYPE"] = "-" -import decision_task # noqa: E402 -decision_task.decisionlib.subprocess = MagicMock() - -print("\n# Push:") -decision_task.main("github-push") - -print("\n# Push with hot caches:") -decision_task.main("github-push") - -print("\n# Push to master:") -decision_task.CONFIG.git_ref = "refs/heads/master" -decision_task.main("github-push") - -print("\n# Daily:") -decision_task.main("daily") - -print("\n# Try AMI:") -decision_task.main("try-windows-ami") - -print("\n# PR:") -decision_task.main("github-pull-request") - -print() diff --git a/etc/taskcluster/simulate_github_events.py b/etc/taskcluster/simulate_github_events.py deleted file mode 100755 index c99436b9106..00000000000 --- a/etc/taskcluster/simulate_github_events.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -''''set -e -cd "$(dirname $0)" -exec ../../python/_virtualenv/bin/python "$(basename $0)" -''' - -try: - import jsone -except ImportError: - import sys - sys.exit("pip install git+https://github.com/taskcluster/json-e") - -import yaml -import json - -template = yaml.load(open("../../.taskcluster.yml").read().decode("utf8")) -repo = dict( - repository=dict( - clone_url="https://github.com/servo/servo.git", - ), -) -contexts = [ - dict( - tasks_for="github-release", - event=repo, - ), - dict( - tasks_for="github-pull-request", - event=dict( - action="comment", - **repo - ), - ), - dict( - tasks_for="github-push", - event=dict( - ref="refs/heads/master", - compare="https://github.com/servo/servo/compare/1753cda...de09c8f", - after="de09c8fb6ef87dec5932d5fab4adcb421d291a54", - pusher=dict( - name="bors-servo", - ), - **repo - ), - ), - dict( - tasks_for="github-pull-request", - event=dict( - action="synchronize", - pull_request=dict( - number=22583, - url="https://github.com/servo/servo/pull/22583", - head=dict( - sha="51a422c9ef47420eb69c802643b7686bdb498652", - ), - merge_commit_sha="876fcf7a5fe971a9ac0a4ce117906c552c08c095", - ), - sender=dict( - login="jdm", - ), - **repo - ), - ), -] -for context in contexts: - print(context["tasks_for"]) - print(json.dumps(jsone.render(template, context), indent=2)) diff --git a/etc/taskcluster/treeherder.md b/etc/taskcluster/treeherder.md deleted file mode 100644 index 63223dddfc6..00000000000 --- a/etc/taskcluster/treeherder.md +++ /dev/null @@ -1,103 +0,0 @@ -# Treeherder for Servo - -Treeherder is tool for visualizing the status of “trees”, -meaning branches in various source repositories. -It shows each push to the repository with the corresponding commits -as well as the CI jobs that were started for that push. -While it is possible to write other tools that submit job data, -CI integration is easiest with Taskcluster. - -* [Production instance](https://treeherder.mozilla.org/) -* [Staging instance](https://treeherder.allizom.org/) -* [Source code](https://github.com/mozilla/treeherder/) - - -## Trees / repositories / branches - -Treeherders knows a about a number of *repostories*. -Mercurial on Mozilla’s servers and git on GitHub are supported. -Despite the name, in the GitHub case -each Treeherder repository maps to one branch in a git repository. -They are configured in the [`repository.json`] file. -As of this writing there are four for `github.com/servo/servo`, -named after the corresponding branch: - -[`repository.json`]: https://github.com/mozilla/treeherder/blob/master/treeherder/model/fixtures/repository.json - -* [`servo-master`](https://treeherder.mozilla.org/#/jobs?repo=servo-master) -* [`servo-auto`](https://treeherder.mozilla.org/#/jobs?repo=servo-auto) -* [`servo-try`](https://treeherder.mozilla.org/#/jobs?repo=servo-try) -* [`servo-try-taskcluster`](https://treeherder.mozilla.org/#/jobs?repo=servo-try-taskcluster) - -In the UI, the “Repos” button near the top right corner allows switching. - -`servo-auto` is the relevant one when a pull request is approved with Homu for landing, -since the `auto` branch is where it pushes a merge commit for testing. - - -## Data flow / how it all works - -(This section is mostly useful for future changes or troubleshooting.) - -Changes to the Treeherder repository are deployed to Staging -soon (minutes) after they are merged on GitHub, -and to Production manually at some point later. -See [current deployment status](https://whatsdeployed.io/s-dqv). - -Once a configuration change with a new repository/branch is deployed, -Treeherder will show it in its UI and start recording push and job data in its database. -This data comes from [Pulse], Mozilla’s shared message queue that coordinates separate services. -The [Pulse Inspector] shows messages as they come (though not in the past), -which can be useful for debugging. -Note that you need to add at least one “Binding”, -or the “Start Listening” button won’t do anything. - -[Pulse]: https://wiki.mozilla.org/Auto-tools/Projects/Pulse -[Pulse Inspector]: https://community-tc.services.mozilla.com/pulse-messages - - -### Push data - -When [taskcluster-github] is [enabled] on a repository, -it recieves [webhooks] from GitHub for various events -such as a push to a branch of the repository. - -In addition to starting Taskcluster tasks based on `.taskcluster.yml` in the repository, -in [`api.js`] it creates [Pulse messages] corresponding to those events. -Treeherder consumes messages from the `exchange/taskcluster-github/v1/push` exchange -(among others) in [`push_loader.py`]. -In Pulse Inspector, these messages for the Servo repository can be seen -by specifying the [`primary.servo.servo`] routing key pattern. - -[taskcluster-github]: https://github.com/taskcluster/taskcluster/tree/master/services/github -[enabled]: https://github.com/apps/community-tc-integration/ -[webhooks]: https://developer.github.com/webhooks/ -[Pulse messages]: https://community-tc.services.mozilla.com/docs/reference/integrations/github/exchanges -[`api.js`]: https://github.com/taskcluster/taskcluster/blob/master/services/github/src/api.js -[`push_loader.py`]: https://github.com/mozilla/treeherder/blob/master/treeherder/etl/push_loader.py -[`primary.servo.servo`]: https://community-tc.services.mozilla.com/pulse-messages?bindings%5B0%5D%5Bexchange%5D=exchange%2Ftaskcluster-github%2Fv1%2Fpush&bindings%5B0%5D%5BroutingKeyPattern%5D=primary.servo.servo - - -### (Taskcluster) job data - -The Taskcluster Queue generates a number of [Pulse messages about tasks]. -Each value of the `routes` array in the task definition, with a `route.` prefix prepended, -is additional routing key for those messages. - -Treeherder reads those messages -if they have an appropriate route ([see in Pulse inspector][inspector1]), -However, it will drop an incoming message -if the `extra.treeherder` object in the task definition doesn’t conform to the [schema]. -Such schema validation errors are logged, but those logs are not easy to access. -Ask on IRC on `#taskcluster`. - -Finally, Treeherder reads that latter kind of message in [`job_loader.py`]. - - - -[Pulse messages about tasks]: https://community-tc.services.mozilla.com/docs/reference/platform/taskcluster-queue/references/events -[taskcluster-treeherder]: https://github.com/taskcluster/taskcluster-treeherder/ -[other messages]: https://community-tc.services.mozilla.com/docs/reference/integrations/taskcluster-treeherder#job-pulse-messages -[schema]: https://schemas.taskcluster.net/treeherder/v1/task-treeherder-config.json -[`job_loader.py`]: https://github.com/mozilla/treeherder/blob/master/treeherder/etl/job_loader.py -[inspector1]: https://tools.taskcluster.net/pulse-inspector?bindings%5B0%5D%5Bexchange%5D=exchange%2Ftaskcluster-queue%2Fv1%2Ftask-defined&bindings%5B0%5D%5BroutingKeyPattern%5D=route.tc-treeherder.%23 diff --git a/python/servo/package_commands.py b/python/servo/package_commands.py index f6010c0d3a0..53910dcd644 100644 --- a/python/servo/package_commands.py +++ b/python/servo/package_commands.py @@ -21,7 +21,6 @@ import shutil import subprocess import sys import tempfile -import urllib import xml from mach.decorators import ( @@ -98,15 +97,6 @@ else: raise e -def get_taskcluster_secret(name): - url = ( - os.environ.get("TASKCLUSTER_PROXY_URL", "http://taskcluster") - + "/api/secrets/v1/secret/project/servo/" - + name - ) - return json.load(urllib.request.urlopen(url))["secret"] - - def otool(s): o = subprocess.Popen(['/usr/bin/otool', '-L', s], stdout=subprocess.PIPE) for line in map(lambda s: s.decode('ascii'), o.stdout): @@ -601,23 +591,16 @@ class PackageCommands(CommandBase): @CommandArgument('platform', choices=PACKAGES.keys(), help='Package platform type to upload') - @CommandArgument('--secret-from-taskcluster', - action='store_true', - help='Retrieve the appropriate secrets from taskcluster.') @CommandArgument('--secret-from-environment', action='store_true', help='Retrieve the appropriate secrets from the environment.') - def upload_nightly(self, platform, secret_from_taskcluster, secret_from_environment): + def upload_nightly(self, platform, secret_from_environment): import boto3 def get_s3_secret(): aws_access_key = None aws_secret_access_key = None - if secret_from_taskcluster: - secret = get_taskcluster_secret("s3-upload-credentials") - aws_access_key = secret["aws_access_key_id"] - aws_secret_access_key = secret["aws_secret_access_key"] - elif secret_from_environment: + if secret_from_environment: secret = json.loads(os.environ['S3_UPLOAD_CREDENTIALS']) aws_access_key = secret["aws_access_key_id"] aws_secret_access_key = secret["aws_secret_access_key"] @@ -758,10 +741,7 @@ class PackageCommands(CommandBase): '--message=Version Bump: {}'.format(brew_version), ]) - if secret_from_taskcluster: - token = get_taskcluster_secret('github-homebrew-token')["token"] - else: - token = os.environ['GITHUB_HOMEBREW_TOKEN'] + token = os.environ['GITHUB_HOMEBREW_TOKEN'] push_url = 'https://{}@github.com/servo/homebrew-servo.git' # TODO(aneeshusa): Use subprocess.DEVNULL with Python 3.3+ @@ -804,8 +784,6 @@ def setup_uwp_signing(ms_app_store, publisher): if ms_app_store: return ["/p:AppxPackageSigningEnabled=false"] - is_tc = "TASKCLUSTER_PROXY_URL" in os.environ - def run_powershell_cmd(cmd): try: return ( @@ -818,10 +796,7 @@ def setup_uwp_signing(ms_app_store, publisher): exit(1) pfx = None - if is_tc: - print("Packaging on TC. Using secret certificate") - pfx = get_taskcluster_secret("windows-codesign-cert/latest")["pfx"]["base64"] - elif 'CODESIGN_CERT' in os.environ: + if 'CODESIGN_CERT' in os.environ: pfx = os.environ['CODESIGN_CERT'] if pfx: @@ -832,10 +807,7 @@ def setup_uwp_signing(ms_app_store, publisher): # Powershell command that lists all certificates for publisher cmd = '(dir cert: -Recurse | Where-Object {$_.Issuer -eq "' + publisher + '"}).Thumbprint' certs = list(set(run_powershell_cmd(cmd).splitlines())) - if not certs and is_tc: - print("Error: No certificate installed for publisher " + publisher) - exit(1) - if not certs and not is_tc: + if not certs: print("No certificate installed for publisher " + publisher) print("Creating and installing a temporary certificate") # PowerShell command that creates and install signing certificate for publisher diff --git a/servo-tidy.toml b/servo-tidy.toml index af3eee3aacb..762eb169145 100644 --- a/servo-tidy.toml +++ b/servo-tidy.toml @@ -123,9 +123,6 @@ files = [ "./tests/wpt/mozilla/tests/css/pre_with_tab.html", "./tests/wpt/mozilla/tests/mozilla/textarea_placeholder.html", # Python 3 syntax causes "E901 SyntaxError" when flake8 runs in Python 2 - "./etc/taskcluster/decision_task.py", - "./etc/taskcluster/decisionlib.py", - "./tests/wpt/reftests-report/gen.py", "./components/style/properties/build.py", ] # Directories that are ignored for the non-WPT tidy check. diff --git a/tests/wpt/reftests-report/gen.py b/tests/wpt/reftests-report/gen.py deleted file mode 100755 index ba07ca26469..00000000000 --- a/tests/wpt/reftests-report/gen.py +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env python3 - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -import gzip -import json -import os -import re -import sys -import urllib.request -from html import escape as html_escape - - -TASKCLUSTER_ROOT_URL = "https://community-tc.services.mozilla.com" - - -def fetch(url): - url = TASKCLUSTER_ROOT_URL + "/api/" + url - print("Fetching " + url) - response = urllib.request.urlopen(url) - assert response.getcode() == 200 - encoding = response.info().get("Content-Encoding") - if not encoding: - return response - elif encoding == "gzip": - return gzip.GzipFile(fileobj=response) - else: - raise ValueError("Unsupported Content-Encoding: %s" % encoding) - - -def fetch_json(url): - with fetch(url) as response: - return json.load(response) - - -def task(platform, chunk, key): - return "index/v1/task/project.servo.%s_wpt_%s.%s" % (platform, chunk, key) - - -def failing_reftests(platform, key): - chunk_1_task_id = fetch_json(task(platform, 1, key))["taskId"] - name = fetch_json("queue/v1/task/" + chunk_1_task_id)["metadata"]["name"] - match = re.search("WPT chunk (\d+) / (\d+)", name) - assert match.group(1) == "1" - total_chunks = int(match.group(2)) - - for chunk in range(1, total_chunks + 1): - with fetch(task(platform, chunk, key) + "/artifacts/public/test-wpt.log") as response: - yield from parse(response) - - -def parse(file_like): - seen = set() - for line in file_like: - message = json.loads(line) - status = message.get("status") - if status not in {None, "OK", "PASS"}: - screenshots = message.get("extra", {}).get("reftest_screenshots") - if screenshots: - url = message["test"] - assert url.startswith("/") - yield url[1:], message.get("expected") == "PASS", screenshots - - -def main(source, commit_sha=None): - failures = Directory() - - if commit_sha: - title = "<h1>Layout 2020 regressions in commit <code>%s</code></h1>" % commit_sha - failures_2013 = {url for url, _, _ in failing_reftests("linux_x64", source)} - for url, _expected_pass, screenshots in failing_reftests("linux_x64_2020", source): - if url not in failures_2013: - failures.add(url, screenshots) - else: - title = "Unexpected failures" - with open(source, "rb") as file_obj: - for url, expected_pass, screenshots in parse(file_obj): - if expected_pass: - failures.add(url, screenshots) - - here = os.path.dirname(__file__) - with open(os.path.join(here, "prism.js")) as f: - prism_js = f.read() - with open(os.path.join(here, "prism.css")) as f: - prism_css = f.read() - with open(os.path.join(here, "report.html"), "w", encoding="utf-8") as html: - os.chdir(os.path.join(here, "..")) - html.write(""" - <!doctype html> - <meta charset=utf-8> - <title>WPT reftests failures report</title> - <link rel=stylesheet href=prism.css> - <style> - ul { padding-left: 1em } - li { list-style: "⯈ " } - li.expanded { list-style: "⯆ " } - li:not(.expanded) > ul, li:not(.expanded) > div { display: none } - li > div { display: grid; grid-gap: 1em; grid-template-columns: 1fr 1fr } - li > div > p { grid-column: span 2 } - li > div > img { grid-row: 2; width: 300px; box-shadow: 0 0 10px } - li > div > img:hover { transform: scale(3); transform-origin: 0 0 } - li > div > pre { grid-row: 3; font-size: 12px !important } - pre code { white-space: pre-wrap !important } - <h1>%s</h1> - </style> - %s - """ % (prism_css, title)) - failures.write(html) - html.write(""" - <script> - for (let li of document.getElementsByTagName("li")) { - li.addEventListener('click', event => { - li.classList.toggle("expanded") - event.stopPropagation() - }) - } - %s - </script> - """ % prism_js) - - -class Directory: - def __init__(self): - self.count = 0 - self.contents = {} - - def add(self, path, screenshots): - self.count += 1 - first, _, rest = path.partition("/") - if rest: - self.contents.setdefault(first, Directory()).add(rest, screenshots) - else: - assert path not in self.contents - self.contents[path] = screenshots - - def write(self, html): - html.write("<ul>\n") - for k, v in self.contents.items(): - html.write("<li><code>%s</code>\n" % k) - if isinstance(v, Directory): - html.write("<strong>%s</strong>\n" % v.count) - v.write(html) - else: - a, rel, b = v - html.write("<div>\n<p><code>%s</code> %s <code>%s</code></p>\n" - % (a["url"], rel, b["url"])) - for side in [a, b]: - html.write("<img src='data:image/png;base64,%s'>\n" % side["screenshot"]) - url = side["url"] - prefix = "/_mozilla/" - if url.startswith(prefix): - filename = "mozilla/tests/" + url[len(prefix):] - elif url == "about:blank": - src = "" - filename = None - else: - filename = "web-platform-tests" + url - if filename: - with open(filename, encoding="utf-8") as f: - src = html_escape(f.read()) - html.write("<pre><code class=language-html>%s</code></pre>\n" % src) - html.write("</li>\n") - html.write("</ul>\n") - - -if __name__ == "__main__": - sys.exit(main(*sys.argv[1:])) diff --git a/tests/wpt/reftests-report/prism.css b/tests/wpt/reftests-report/prism.css deleted file mode 100644 index 6fa6fcc5be6..00000000000 --- a/tests/wpt/reftests-report/prism.css +++ /dev/null @@ -1,141 +0,0 @@ -/* PrismJS 1.19.0 -https://prismjs.com/download.html#themes=prism&languages=markup+css+clike+javascript */ -/** - * prism.js default theme for JavaScript, CSS and HTML - * Based on dabblet (http://dabblet.com) - * @author Lea Verou - */ - -code[class*="language-"], -pre[class*="language-"] { - color: black; - background: none; - text-shadow: 0 1px white; - font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace; - font-size: 1em; - text-align: left; - white-space: pre; - word-spacing: normal; - word-break: normal; - word-wrap: normal; - line-height: 1.5; - - -moz-tab-size: 4; - -o-tab-size: 4; - tab-size: 4; - - -webkit-hyphens: none; - -moz-hyphens: none; - -ms-hyphens: none; - hyphens: none; -} - -pre[class*="language-"]::-moz-selection, pre[class*="language-"] ::-moz-selection, -code[class*="language-"]::-moz-selection, code[class*="language-"] ::-moz-selection { - text-shadow: none; - background: #b3d4fc; -} - -pre[class*="language-"]::selection, pre[class*="language-"] ::selection, -code[class*="language-"]::selection, code[class*="language-"] ::selection { - text-shadow: none; - background: #b3d4fc; -} - -@media print { - code[class*="language-"], - pre[class*="language-"] { - text-shadow: none; - } -} - -/* Code blocks */ -pre[class*="language-"] { - padding: 1em; - margin: .5em 0; - overflow: auto; -} - -:not(pre) > code[class*="language-"], -pre[class*="language-"] { - background: #f5f2f0; -} - -/* Inline code */ -:not(pre) > code[class*="language-"] { - padding: .1em; - border-radius: .3em; - white-space: normal; -} - -.token.comment, -.token.prolog, -.token.doctype, -.token.cdata { - color: slategray; -} - -.token.punctuation { - color: #999; -} - -.token.namespace { - opacity: .7; -} - -.token.property, -.token.tag, -.token.boolean, -.token.number, -.token.constant, -.token.symbol, -.token.deleted { - color: #905; -} - -.token.selector, -.token.attr-name, -.token.string, -.token.char, -.token.builtin, -.token.inserted { - color: #690; -} - -.token.operator, -.token.entity, -.token.url, -.language-css .token.string, -.style .token.string { - color: #9a6e3a; - background: hsla(0, 0%, 100%, .5); -} - -.token.atrule, -.token.attr-value, -.token.keyword { - color: #07a; -} - -.token.function, -.token.class-name { - color: #DD4A68; -} - -.token.regex, -.token.important, -.token.variable { - color: #e90; -} - -.token.important, -.token.bold { - font-weight: bold; -} -.token.italic { - font-style: italic; -} - -.token.entity { - cursor: help; -} diff --git a/tests/wpt/reftests-report/prism.js b/tests/wpt/reftests-report/prism.js deleted file mode 100644 index a43970450d6..00000000000 --- a/tests/wpt/reftests-report/prism.js +++ /dev/null @@ -1,7 +0,0 @@ -/* PrismJS 1.19.0 -https://prismjs.com/download.html#themes=prism&languages=markup+css+clike+javascript */ -var _self="undefined"!=typeof window?window:"undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?self:{},Prism=function(u){var c=/\blang(?:uage)?-([\w-]+)\b/i,r=0,C={manual:u.Prism&&u.Prism.manual,disableWorkerMessageHandler:u.Prism&&u.Prism.disableWorkerMessageHandler,util:{encode:function e(r){return r instanceof _?new _(r.type,e(r.content),r.alias):Array.isArray(r)?r.map(e):r.replace(/&/g,"&").replace(/</g,"<").replace(/\u00a0/g," ")},type:function(e){return Object.prototype.toString.call(e).slice(8,-1)},objId:function(e){return e.__id||Object.defineProperty(e,"__id",{value:++r}),e.__id},clone:function n(e,t){var a,r,i=C.util.type(e);switch(t=t||{},i){case"Object":if(r=C.util.objId(e),t[r])return t[r];for(var o in a={},t[r]=a,e)e.hasOwnProperty(o)&&(a[o]=n(e[o],t));return a;case"Array":return r=C.util.objId(e),t[r]?t[r]:(a=[],t[r]=a,e.forEach(function(e,r){a[r]=n(e,t)}),a);default:return e}},getLanguage:function(e){for(;e&&!c.test(e.className);)e=e.parentElement;return e?(e.className.match(c)||[,"none"])[1].toLowerCase():"none"},currentScript:function(){if("undefined"==typeof document)return null;if("currentScript"in document)return document.currentScript;try{throw new Error}catch(e){var r=(/at [^(\r\n]*\((.*):.+:.+\)$/i.exec(e.stack)||[])[1];if(r){var n=document.getElementsByTagName("script");for(var t in n)if(n[t].src==r)return n[t]}return null}}},languages:{extend:function(e,r){var n=C.util.clone(C.languages[e]);for(var t in r)n[t]=r[t];return n},insertBefore:function(n,e,r,t){var a=(t=t||C.languages)[n],i={};for(var o in a)if(a.hasOwnProperty(o)){if(o==e)for(var l in r)r.hasOwnProperty(l)&&(i[l]=r[l]);r.hasOwnProperty(o)||(i[o]=a[o])}var s=t[n];return t[n]=i,C.languages.DFS(C.languages,function(e,r){r===s&&e!=n&&(this[e]=i)}),i},DFS:function e(r,n,t,a){a=a||{};var i=C.util.objId;for(var o in r)if(r.hasOwnProperty(o)){n.call(r,o,r[o],t||o);var l=r[o],s=C.util.type(l);"Object"!==s||a[i(l)]?"Array"!==s||a[i(l)]||(a[i(l)]=!0,e(l,n,o,a)):(a[i(l)]=!0,e(l,n,null,a))}}},plugins:{},highlightAll:function(e,r){C.highlightAllUnder(document,e,r)},highlightAllUnder:function(e,r,n){var t={callback:n,container:e,selector:'code[class*="language-"], [class*="language-"] code, code[class*="lang-"], [class*="lang-"] code'};C.hooks.run("before-highlightall",t),t.elements=Array.prototype.slice.apply(t.container.querySelectorAll(t.selector)),C.hooks.run("before-all-elements-highlight",t);for(var a,i=0;a=t.elements[i++];)C.highlightElement(a,!0===r,t.callback)},highlightElement:function(e,r,n){var t=C.util.getLanguage(e),a=C.languages[t];e.className=e.className.replace(c,"").replace(/\s+/g," ")+" language-"+t;var i=e.parentNode;i&&"pre"===i.nodeName.toLowerCase()&&(i.className=i.className.replace(c,"").replace(/\s+/g," ")+" language-"+t);var o={element:e,language:t,grammar:a,code:e.textContent};function l(e){o.highlightedCode=e,C.hooks.run("before-insert",o),o.element.innerHTML=o.highlightedCode,C.hooks.run("after-highlight",o),C.hooks.run("complete",o),n&&n.call(o.element)}if(C.hooks.run("before-sanity-check",o),!o.code)return C.hooks.run("complete",o),void(n&&n.call(o.element));if(C.hooks.run("before-highlight",o),o.grammar)if(r&&u.Worker){var s=new Worker(C.filename);s.onmessage=function(e){l(e.data)},s.postMessage(JSON.stringify({language:o.language,code:o.code,immediateClose:!0}))}else l(C.highlight(o.code,o.grammar,o.language));else l(C.util.encode(o.code))},highlight:function(e,r,n){var t={code:e,grammar:r,language:n};return C.hooks.run("before-tokenize",t),t.tokens=C.tokenize(t.code,t.grammar),C.hooks.run("after-tokenize",t),_.stringify(C.util.encode(t.tokens),t.language)},matchGrammar:function(e,r,n,t,a,i,o){for(var l in n)if(n.hasOwnProperty(l)&&n[l]){var s=n[l];s=Array.isArray(s)?s:[s];for(var u=0;u<s.length;++u){if(o&&o==l+","+u)return;var c=s[u],g=c.inside,f=!!c.lookbehind,h=!!c.greedy,d=0,m=c.alias;if(h&&!c.pattern.global){var p=c.pattern.toString().match(/[imsuy]*$/)[0];c.pattern=RegExp(c.pattern.source,p+"g")}c=c.pattern||c;for(var y=t,v=a;y<r.length;v+=r[y].length,++y){var k=r[y];if(r.length>e.length)return;if(!(k instanceof _)){if(h&&y!=r.length-1){if(c.lastIndex=v,!(S=c.exec(e)))break;for(var b=S.index+(f&&S[1]?S[1].length:0),w=S.index+S[0].length,A=y,P=v,x=r.length;A<x&&(P<w||!r[A].type&&!r[A-1].greedy);++A)(P+=r[A].length)<=b&&(++y,v=P);if(r[y]instanceof _)continue;O=A-y,k=e.slice(v,P),S.index-=v}else{c.lastIndex=0;var S=c.exec(k),O=1}if(S){f&&(d=S[1]?S[1].length:0);w=(b=S.index+d)+(S=S[0].slice(d)).length;var E=k.slice(0,b),N=k.slice(w),j=[y,O];E&&(++y,v+=E.length,j.push(E));var L=new _(l,g?C.tokenize(S,g):S,m,S,h);if(j.push(L),N&&j.push(N),Array.prototype.splice.apply(r,j),1!=O&&C.matchGrammar(e,r,n,y,v,!0,l+","+u),i)break}else if(i)break}}}}},tokenize:function(e,r){var n=[e],t=r.rest;if(t){for(var a in t)r[a]=t[a];delete r.rest}return C.matchGrammar(e,n,r,0,0,!1),n},hooks:{all:{},add:function(e,r){var n=C.hooks.all;n[e]=n[e]||[],n[e].push(r)},run:function(e,r){var n=C.hooks.all[e];if(n&&n.length)for(var t,a=0;t=n[a++];)t(r)}},Token:_};function _(e,r,n,t,a){this.type=e,this.content=r,this.alias=n,this.length=0|(t||"").length,this.greedy=!!a}if(u.Prism=C,_.stringify=function r(e,n){if("string"==typeof e)return e;if(Array.isArray(e)){var t="";return e.forEach(function(e){t+=r(e,n)}),t}var a={type:e.type,content:r(e.content,n),tag:"span",classes:["token",e.type],attributes:{},language:n},i=e.alias;i&&(Array.isArray(i)?Array.prototype.push.apply(a.classes,i):a.classes.push(i)),C.hooks.run("wrap",a);var o="";for(var l in a.attributes)o+=" "+l+'="'+(a.attributes[l]||"").replace(/"/g,""")+'"';return"<"+a.tag+' class="'+a.classes.join(" ")+'"'+o+">"+a.content+"</"+a.tag+">"},!u.document)return u.addEventListener&&(C.disableWorkerMessageHandler||u.addEventListener("message",function(e){var r=JSON.parse(e.data),n=r.language,t=r.code,a=r.immediateClose;u.postMessage(C.highlight(t,C.languages[n],n)),a&&u.close()},!1)),C;var e=C.util.currentScript();function n(){C.manual||C.highlightAll()}if(e&&(C.filename=e.src,e.hasAttribute("data-manual")&&(C.manual=!0)),!C.manual){var t=document.readyState;"loading"===t||"interactive"===t&&e&&e.defer?document.addEventListener("DOMContentLoaded",n):window.requestAnimationFrame?window.requestAnimationFrame(n):window.setTimeout(n,16)}return C}(_self);"undefined"!=typeof module&&module.exports&&(module.exports=Prism),"undefined"!=typeof global&&(global.Prism=Prism); -Prism.languages.markup={comment:/<!--[\s\S]*?-->/,prolog:/<\?[\s\S]+?\?>/,doctype:{pattern:/<!DOCTYPE(?:[^>"'[\]]|"[^"]*"|'[^']*')+(?:\[(?:(?!<!--)[^"'\]]|"[^"]*"|'[^']*'|<!--[\s\S]*?-->)*\]\s*)?>/i,greedy:!0},cdata:/<!\[CDATA\[[\s\S]*?]]>/i,tag:{pattern:/<\/?(?!\d)[^\s>\/=$<%]+(?:\s(?:\s*[^\s>\/=]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+(?=[\s>]))|(?=[\s/>])))+)?\s*\/?>/i,greedy:!0,inside:{tag:{pattern:/^<\/?[^\s>\/]+/i,inside:{punctuation:/^<\/?/,namespace:/^[^\s>\/:]+:/}},"attr-value":{pattern:/=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+)/i,inside:{punctuation:[/^=/,{pattern:/^(\s*)["']|["']$/,lookbehind:!0}]}},punctuation:/\/?>/,"attr-name":{pattern:/[^\s>\/]+/,inside:{namespace:/^[^\s>\/:]+:/}}}},entity:/&#?[\da-z]{1,8};/i},Prism.languages.markup.tag.inside["attr-value"].inside.entity=Prism.languages.markup.entity,Prism.hooks.add("wrap",function(a){"entity"===a.type&&(a.attributes.title=a.content.replace(/&/,"&"))}),Object.defineProperty(Prism.languages.markup.tag,"addInlined",{value:function(a,e){var s={};s["language-"+e]={pattern:/(^<!\[CDATA\[)[\s\S]+?(?=\]\]>$)/i,lookbehind:!0,inside:Prism.languages[e]},s.cdata=/^<!\[CDATA\[|\]\]>$/i;var n={"included-cdata":{pattern:/<!\[CDATA\[[\s\S]*?\]\]>/i,inside:s}};n["language-"+e]={pattern:/[\s\S]+/,inside:Prism.languages[e]};var t={};t[a]={pattern:RegExp("(<__[\\s\\S]*?>)(?:<!\\[CDATA\\[[\\s\\S]*?\\]\\]>\\s*|[\\s\\S])*?(?=<\\/__>)".replace(/__/g,a),"i"),lookbehind:!0,greedy:!0,inside:n},Prism.languages.insertBefore("markup","cdata",t)}}),Prism.languages.xml=Prism.languages.extend("markup",{}),Prism.languages.html=Prism.languages.markup,Prism.languages.mathml=Prism.languages.markup,Prism.languages.svg=Prism.languages.markup; -!function(s){var e=/("|')(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/;s.languages.css={comment:/\/\*[\s\S]*?\*\//,atrule:{pattern:/@[\w-]+[\s\S]*?(?:;|(?=\s*\{))/,inside:{rule:/^@[\w-]+/,"selector-function-argument":{pattern:/(\bselector\s*\((?!\s*\))\s*)(?:[^()]|\((?:[^()]|\([^()]*\))*\))+?(?=\s*\))/,lookbehind:!0,alias:"selector"}}},url:{pattern:RegExp("url\\((?:"+e.source+"|[^\n\r()]*)\\)","i"),inside:{function:/^url/i,punctuation:/^\(|\)$/}},selector:RegExp("[^{}\\s](?:[^{};\"']|"+e.source+")*?(?=\\s*\\{)"),string:{pattern:e,greedy:!0},property:/[-_a-z\xA0-\uFFFF][-\w\xA0-\uFFFF]*(?=\s*:)/i,important:/!important\b/i,function:/[-a-z0-9]+(?=\()/i,punctuation:/[(){};:,]/},s.languages.css.atrule.inside.rest=s.languages.css;var t=s.languages.markup;t&&(t.tag.addInlined("style","css"),s.languages.insertBefore("inside","attr-value",{"style-attr":{pattern:/\s*style=("|')(?:\\[\s\S]|(?!\1)[^\\])*\1/i,inside:{"attr-name":{pattern:/^\s*style/i,inside:t.tag.inside},punctuation:/^\s*=\s*['"]|['"]\s*$/,"attr-value":{pattern:/.+/i,inside:s.languages.css}},alias:"language-css"}},t.tag))}(Prism); -Prism.languages.clike={comment:[{pattern:/(^|[^\\])\/\*[\s\S]*?(?:\*\/|$)/,lookbehind:!0},{pattern:/(^|[^\\:])\/\/.*/,lookbehind:!0,greedy:!0}],string:{pattern:/(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,greedy:!0},"class-name":{pattern:/(\b(?:class|interface|extends|implements|trait|instanceof|new)\s+|\bcatch\s+\()[\w.\\]+/i,lookbehind:!0,inside:{punctuation:/[.\\]/}},keyword:/\b(?:if|else|while|do|for|return|in|instanceof|function|new|try|throw|catch|finally|null|break|continue)\b/,boolean:/\b(?:true|false)\b/,function:/\w+(?=\()/,number:/\b0x[\da-f]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?/i,operator:/[<>]=?|[!=]=?=?|--?|\+\+?|&&?|\|\|?|[?*/~^%]/,punctuation:/[{}[\];(),.:]/}; -Prism.languages.javascript=Prism.languages.extend("clike",{"class-name":[Prism.languages.clike["class-name"],{pattern:/(^|[^$\w\xA0-\uFFFF])[_$A-Z\xA0-\uFFFF][$\w\xA0-\uFFFF]*(?=\.(?:prototype|constructor))/,lookbehind:!0}],keyword:[{pattern:/((?:^|})\s*)(?:catch|finally)\b/,lookbehind:!0},{pattern:/(^|[^.]|\.\.\.\s*)\b(?:as|async(?=\s*(?:function\b|\(|[$\w\xA0-\uFFFF]|$))|await|break|case|class|const|continue|debugger|default|delete|do|else|enum|export|extends|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|undefined|var|void|while|with|yield)\b/,lookbehind:!0}],number:/\b(?:(?:0[xX](?:[\dA-Fa-f](?:_[\dA-Fa-f])?)+|0[bB](?:[01](?:_[01])?)+|0[oO](?:[0-7](?:_[0-7])?)+)n?|(?:\d(?:_\d)?)+n|NaN|Infinity)\b|(?:\b(?:\d(?:_\d)?)+\.?(?:\d(?:_\d)?)*|\B\.(?:\d(?:_\d)?)+)(?:[Ee][+-]?(?:\d(?:_\d)?)+)?/,function:/#?[_$a-zA-Z\xA0-\uFFFF][$\w\xA0-\uFFFF]*(?=\s*(?:\.\s*(?:apply|bind|call)\s*)?\()/,operator:/--|\+\+|\*\*=?|=>|&&|\|\||[!=]==|<<=?|>>>?=?|[-+*/%&|^!=<>]=?|\.{3}|\?[.?]?|[~:]/}),Prism.languages.javascript["class-name"][0].pattern=/(\b(?:class|interface|extends|implements|instanceof|new)\s+)[\w.\\]+/,Prism.languages.insertBefore("javascript","keyword",{regex:{pattern:/((?:^|[^$\w\xA0-\uFFFF."'\])\s])\s*)\/(?:\[(?:[^\]\\\r\n]|\\.)*]|\\.|[^/\\\[\r\n])+\/[gimyus]{0,6}(?=(?:\s|\/\*[\s\S]*?\*\/)*(?:$|[\r\n,.;:})\]]|\/\/))/,lookbehind:!0,greedy:!0},"function-variable":{pattern:/#?[_$a-zA-Z\xA0-\uFFFF][$\w\xA0-\uFFFF]*(?=\s*[=:]\s*(?:async\s*)?(?:\bfunction\b|(?:\((?:[^()]|\([^()]*\))*\)|[_$a-zA-Z\xA0-\uFFFF][$\w\xA0-\uFFFF]*)\s*=>))/,alias:"function"},parameter:[{pattern:/(function(?:\s+[_$A-Za-z\xA0-\uFFFF][$\w\xA0-\uFFFF]*)?\s*\(\s*)(?!\s)(?:[^()]|\([^()]*\))+?(?=\s*\))/,lookbehind:!0,inside:Prism.languages.javascript},{pattern:/[_$a-z\xA0-\uFFFF][$\w\xA0-\uFFFF]*(?=\s*=>)/i,inside:Prism.languages.javascript},{pattern:/(\(\s*)(?!\s)(?:[^()]|\([^()]*\))+?(?=\s*\)\s*=>)/,lookbehind:!0,inside:Prism.languages.javascript},{pattern:/((?:\b|\s|^)(?!(?:as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|undefined|var|void|while|with|yield)(?![$\w\xA0-\uFFFF]))(?:[_$A-Za-z\xA0-\uFFFF][$\w\xA0-\uFFFF]*\s*)\(\s*)(?!\s)(?:[^()]|\([^()]*\))+?(?=\s*\)\s*\{)/,lookbehind:!0,inside:Prism.languages.javascript}],constant:/\b[A-Z](?:[A-Z_]|\dx?)*\b/}),Prism.languages.insertBefore("javascript","string",{"template-string":{pattern:/`(?:\\[\s\S]|\${(?:[^{}]|{(?:[^{}]|{[^}]*})*})+}|(?!\${)[^\\`])*`/,greedy:!0,inside:{"template-punctuation":{pattern:/^`|`$/,alias:"string"},interpolation:{pattern:/((?:^|[^\\])(?:\\{2})*)\${(?:[^{}]|{(?:[^{}]|{[^}]*})*})+}/,lookbehind:!0,inside:{"interpolation-punctuation":{pattern:/^\${|}$/,alias:"punctuation"},rest:Prism.languages.javascript}},string:/[\s\S]+/}}}),Prism.languages.markup&&Prism.languages.markup.tag.addInlined("script","javascript"),Prism.languages.js=Prism.languages.javascript; diff --git a/tests/wpt/update/fetchlogs.py b/tests/wpt/update/fetchlogs.py deleted file mode 100644 index 385f2c54174..00000000000 --- a/tests/wpt/update/fetchlogs.py +++ /dev/null @@ -1,99 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -import argparse -import cStringIO -import gzip -import json -import os -import requests -import six.moves.urllib as urllib - -treeherder_base = "https://treeherder.mozilla.org/" - -"""Simple script for downloading structured logs from treeherder. - -For the moment this is specialised to work with web-platform-tests -logs; in due course it should move somewhere generic and get hooked -up to mach or similar""" - -# Interpretation of the "job" list from -# https://github.com/mozilla/treeherder-service/blob/master/treeherder/webapp/api/utils.py#L18 - -def create_parser(): - parser = argparse.ArgumentParser() - parser.add_argument("branch", action="store", - help="Branch on which jobs ran") - parser.add_argument("commit", - action="store", - help="Commit hash for push") - - return parser - -def download(url, prefix, dest, force_suffix=True): - if dest is None: - dest = "." - - if prefix and not force_suffix: - name = os.path.join(dest, prefix + ".log") - else: - name = None - counter = 0 - - while not name or os.path.exists(name): - counter += 1 - sep = "" if not prefix else "-" - name = os.path.join(dest, prefix + sep + str(counter) + ".log") - - with open(name, "wb") as f: - resp = requests.get(url, stream=True) - for chunk in resp.iter_content(1024): - f.write(chunk) - -def get_blobber_url(branch, job): - job_id = job["id"] - resp = requests.get(urllib.parse.urljoin(treeherder_base, - "/api/project/%s/artifact/?job_id=%i&name=Job%%20Info" % (branch, - job_id))) - job_data = resp.json() - - if job_data: - assert len(job_data) == 1 - job_data = job_data[0] - try: - details = job_data["blob"]["job_details"] - for item in details: - if item["value"] == "wpt_raw.log": - return item["url"] - except: - return None - - -def get_structured_logs(branch, commit, dest=None): - resp = requests.get(urllib.parse.urljoin(treeherder_base, "/api/project/%s/resultset/?revision=%s" % (branch, commit))) - - revision_data = resp.json() - - result_set = revision_data["results"][0]["id"] - - resp = requests.get(urllib.parse.urljoin(treeherder_base, "/api/project/%s/jobs/?result_set_id=%s&count=2000&exclusion_profile=false" % (branch, result_set))) - - job_data = resp.json() - - for result in job_data["results"]: - job_type_name = result["job_type_name"] - if job_type_name.startswith("W3C Web Platform"): - url = get_blobber_url(branch, result) - if url: - prefix = result["platform"] # platform - download(url, prefix, None) - -def main(): - parser = create_parser() - args = parser.parse_args() - - get_structured_logs(args.branch, args.commit) - -if __name__ == "__main__": - main() |