diff options
| -rw-r--r-- | .taskcluster.yml | 3 | ||||
| -rwxr-xr-x | etc/ci/upload_docs.sh | 45 | ||||
| -rw-r--r-- | etc/taskcluster/decision_task.py | 143 | ||||
| -rw-r--r-- | etc/taskcluster/decisionlib.py | 16 | ||||
| -rwxr-xr-x | etc/taskcluster/mock.py | 19 | ||||
| -rw-r--r-- | python/servo/post_build_commands.py | 20 |
6 files changed, 154 insertions, 92 deletions
diff --git a/.taskcluster.yml b/.taskcluster.yml index ea33b12034f..10559de409c 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -25,8 +25,7 @@ tasks: owner: &task_owner ${event.pusher.name}@users.noreply.github.com source: &task_source ${event.compare} scopes: - # Granted to role "repo:github.com/servo/servo:branch:*" - - "assume:project:servo:decision-task/trusted" + - "assume:repo:github.com/servo/servo:branch:${event.ref[11:]}" routes: # len("refs/heads/") == 11, so event.ref[11:] is the branch name - "tc-treeherder.v2.servo/servo-${event.ref[11:]}.${event.after}" diff --git a/etc/ci/upload_docs.sh b/etc/ci/upload_docs.sh deleted file mode 100755 index 4aab401bdee..00000000000 --- a/etc/ci/upload_docs.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -# Helper script to upload docs to doc.servo.org. -# Requires ghp-import (from pip) -# GitHub API token must be passed in environment var TOKEN - -set -o errexit -set -o nounset -set -o pipefail - -cd "$(dirname ${0})/../.." - -# Clean up any traces of previous doc builds. -./etc/ci/clean_build_artifacts.sh - -env CC=gcc-5 CXX=g++-5 ./mach doc -# etc/doc.servo.org/index.html overwrites $(mach rust-root)/doc/index.html -# Use recursive copy here to avoid `cp` returning an error code -# when it encounters directories. -cp -r etc/doc.servo.org/* target/doc/ - -python components/style/properties/build.py servo html regular - -cd components/script -cmake . -cmake --build . --target supported-apis -echo "Copying apis.html." -cp apis.html ../../target/doc/servo/ -echo "Copied apis.html." -cd ../.. - -echo "Starting ghp-import." -ghp-import -n target/doc -echo "Finished ghp-import." -git push -qf \ - "https://${TOKEN}@github.com/servo/doc.servo.org.git" gh-pages \ - &>/dev/null -echo "Finished git push." - -# Clean up the traces of the current doc build. -./etc/ci/clean_build_artifacts.sh diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 60434d4c06b..04d08b197df 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -8,33 +8,75 @@ import os.path from decisionlib import * -def main(task_for, mock=False): +def main(task_for): if task_for == "github-push": - if CONFIG.git_ref in ["refs/heads/auto", "refs/heads/try", "refs/heads/try-taskcluster"]: - CONFIG.treeherder_repo_name = "servo-" + CONFIG.git_ref.split("/")[-1] - - linux_tidy_unit() - android_arm32_dev() - android_arm32_release() - android_x86_release() - windows_unit() - macos_unit() - - # These are disabled in a "real" decision task, - # but should still run when testing this Python code. (See `mock.py`.) - if mock: - windows_release() - linux_wpt() - linux_build_task("Indexed by task definition").find_or_create() - android_x86_wpt() + # FIXME https://github.com/servo/servo/issues/22325 implement these: + macos_wpt = magicleap_dev = linux_arm32_dev = linux_arm64_dev = \ + android_arm32_dev_from_macos = lambda: None + # FIXME still buggy: + linux_wpt = lambda: None # Shadows the existing top-level function + + all_tests = [ + linux_tidy_unit_docs, + windows_unit, + macos_unit, + magicleap_dev, + android_arm32_dev, + android_arm32_release, + android_x86_release, + linux_arm32_dev, + linux_arm64_dev, + linux_wpt, + macos_wpt, + ] + by_branch_name = { + "auto": all_tests, + "try": all_tests, + "try-taskcluster": [ + # Add functions here as needed, in your push to that branch + ], + "master": [ + # Also show these tasks in https://treeherder.mozilla.org/#/jobs?repo=servo-auto + lambda: CONFIG.treeherder_repository_names.append("servo-auto"), + upload_docs, + ], + + # The "try-*" keys match those in `servo_try_choosers` in Homu’s config: + # https://github.com/servo/saltfs/blob/master/homu/map.jinja + + "try-mac": [macos_unit], + "try-linux": [linux_tidy_unit_docs], + "try-windows": [windows_unit], + "try-magicleap": [magicleap_dev], + "try-arm": [linux_arm32_dev, linux_arm64_dev], + "try-wpt": [linux_wpt], + "try-wpt-mac": [macos_wpt], + "try-wpt-android": [android_x86_wpt], + "try-android": [ + android_arm32_dev, + android_arm32_dev_from_macos, + android_x86_wpt + ], + } + assert CONFIG.git_ref.startswith("refs/heads/") + branch = CONFIG.git_ref[len("refs/heads/"):] + CONFIG.treeherder_repository_names.append("servo-" + branch) + for function in by_branch_name.get(branch, []): + function() # https://tools.taskcluster.net/hooks/project-servo/daily elif task_for == "daily": daily_tasks_setup() with_rust_nightly() - else: # pragma: no cover - raise ValueError("Unrecognized $TASK_FOR value: %r", task_for) + +# These are disabled in a "real" decision task, +# but should still run when testing this Python code. (See `mock.py`.) +def mocked_only(): + windows_release() + linux_wpt() + android_x86_wpt() + linux_build_task("Indexed by task definition").find_or_create() ping_on_daily_task_failure = "SimonSapin, nox, emilio" @@ -69,10 +111,10 @@ windows_sparse_checkout = [ ] -def linux_tidy_unit(): +def linux_tidy_unit_docs(): return ( - linux_build_task("Tidy + dev build + unit") - .with_treeherder("Linux x64") + linux_build_task("Tidy + dev build + unit tests + docs") + .with_treeherder("Linux x64", "Tidy+Unit+Doc") .with_script(""" ./mach test-tidy --no-progress --all ./mach build --dev @@ -81,24 +123,61 @@ def linux_tidy_unit(): ./mach build --dev --libsimpleservo ./mach build --dev --no-default-features --features default-except-unstable ./mach test-tidy --no-progress --self-test + ./etc/memory_reports_over_time.py --test ./etc/taskcluster/mock.py ./etc/ci/lockfile_changed.sh ./etc/ci/check_no_panic.sh - """).create() + + ./mach doc + cd target/doc + git init + time git add . + git -c user.name="Taskcluster" -c user.email="" \ + commit -q -m "Rebuild Servo documentation" + git bundle create docs.bundle HEAD + """) + .with_artifacts("/repo/target/doc/docs.bundle") + .find_or_create("docs." + CONFIG.git_sha) ) +def upload_docs(): + docs_build_task_id = Task.find("docs." + CONFIG.git_sha) + return ( + linux_task("Upload docs to GitHub Pages") + .with_treeherder("Linux x64", "DocUpload") + .with_dockerfile(dockerfile_path("base")) + .with_curl_artifact_script(docs_build_task_id, "docs.bundle") + .with_features("taskclusterProxy") + .with_scopes("secrets:get:project/servo/doc.servo.org") + .with_env(PY="""if 1: + import urllib, json + url = "http://taskcluster/secrets/v1/secret/project/servo/doc.servo.org" + token = json.load(urllib.urlopen(url))["secret"]["token"] + open("/root/.git-credentials", "w").write("https://git:%s@github.com/" % token) + """) + .with_script(""" + python -c "$PY" + git init --bare + git config credential.helper store + git fetch --quiet docs.bundle + git push --force https://github.com/servo/doc.servo.org FETCH_HEAD:gh-pages + """) + .create() + ) + def macos_unit(): return ( macos_build_task("Dev build + unit tests") - .with_treeherder("macOS x64") + .with_treeherder("macOS x64", "Unit") .with_script(""" ./mach build --dev ./mach test-unit ./mach package --dev ./etc/ci/lockfile_changed.sh - """).create() + """) + .create() ) @@ -136,7 +215,7 @@ def android_arm32_dev(): def android_arm32_release(): return ( android_build_task("Release build") - .with_treeherder("Android ARMv7") + .with_treeherder("Android ARMv7", "Release") .with_script("./mach build --android --release") .with_artifacts( "/repo/target/android/armv7-linux-androideabi/release/servoapp.apk", @@ -149,7 +228,7 @@ def android_arm32_release(): def android_x86_release(): return ( android_build_task("Release build") - .with_treeherder("Android x86") + .with_treeherder("Android x86", "Release") .with_script("./mach build --target i686-linux-android --release") .with_artifacts( "/repo/target/android/i686-linux-android/release/servoapp.apk", @@ -185,7 +264,7 @@ def android_x86_wpt(): def windows_unit(): return ( windows_build_task("Dev build + unit tests") - .with_treeherder("Windows x64") + .with_treeherder("Windows x64", "Unit") .with_script( # Not necessary as this would be done at the start of `build`, # but this allows timing it separately. @@ -204,7 +283,7 @@ def windows_unit(): def windows_release(): return ( windows_build_task("Release build") - .with_treeherder("Windows x64") + .with_treeherder("Windows x64", "Release") .with_script("mach build --release", "mach package --release") .with_artifacts("repo/target/release/msi/Servo.exe", @@ -224,7 +303,7 @@ def linux_wpt(): def linux_release_build(): return ( linux_build_task("Release build") - .with_treeherder("Linux x64") + .with_treeherder("Linux x64", "Release") .with_script(""" ./mach build --release --with-debug-assertions -p servo ./etc/ci/lockfile_changed.sh @@ -241,7 +320,7 @@ def linux_release_build(): def wpt_chunk(release_build_task, total_chunks, this_chunk): task = ( linux_task("WPT chunk %s / %s" % (this_chunk, total_chunks)) - .with_treeherder("Linux x64", "WPT %s" % this_chunk) + .with_treeherder("Linux x64", "WPT-%s" % this_chunk) .with_dockerfile(dockerfile_path("run")) .with_repo() .with_curl_artifact_script(release_build_task, "target.tar.gz") diff --git a/etc/taskcluster/decisionlib.py b/etc/taskcluster/decisionlib.py index 77197b1bd0b..553113db326 100644 --- a/etc/taskcluster/decisionlib.py +++ b/etc/taskcluster/decisionlib.py @@ -43,7 +43,7 @@ class Config: self.docker_image_buil_worker_type = None self.docker_images_expire_in = "1 month" self.repacked_msi_files_expire_in = "1 month" - self.treeherder_repo_name = None + self.treeherder_repository_names = [] # Set by docker-worker: # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/environment @@ -156,9 +156,9 @@ class Task: "symbol": symbol, }) - if CONFIG.treeherder_repo_name: + for repo in CONFIG.treeherder_repository_names: assert CONFIG.git_sha - suffix = ".v2._/%s.%s" % (CONFIG.treeherder_repo_name, CONFIG.git_sha) + suffix = ".v2._/%s.%s" % (repo, CONFIG.git_sha) self.with_routes( "tc-treeherder" + suffix, "tc-treeherder-staging" + suffix, @@ -223,6 +223,11 @@ class Task: print("Scheduled %s" % self.name) return task_id + @staticmethod + def find(index_path): + full_index_path = "%s.%s" % (CONFIG.index_prefix, index_path) + return SHARED.index_service.findTask(full_index_path)["taskId"] + def find_or_create(self, index_path=None): """ Try to find a task in the Index and return its ID. @@ -240,18 +245,17 @@ class Task: worker_type = self.worker_type index_by = json.dumps([worker_type, self.build_worker_payload()]).encode("utf-8") index_path = "by-task-definition." + hashlib.sha256(index_by).hexdigest() - index_path = "%s.%s" % (CONFIG.index_prefix, index_path) task_id = SHARED.found_or_created_indexed_tasks.get(index_path) if task_id is not None: return task_id try: - task_id = SHARED.index_service.findTask(index_path)["taskId"] + task_id = Task.find(index_path) except taskcluster.TaskclusterRestFailure as e: if e.status_code != 404: # pragma: no cover raise - self.routes.append("index." + index_path) + self.routes.append("index.%s.%s" % (CONFIG.index_prefix, index_path)) task_id = self.create() SHARED.found_or_created_indexed_tasks[index_path] = task_id diff --git a/etc/taskcluster/mock.py b/etc/taskcluster/mock.py index d3e19d88080..60720f73ddd 100755 --- a/etc/taskcluster/mock.py +++ b/etc/taskcluster/mock.py @@ -31,7 +31,9 @@ class TaskclusterRestFailure(Exception): class Index: __init__ = insertTask = lambda *_, **__: None - def findTask(self, _): + def findTask(self, path): + if decision_task.CONFIG.git_ref == "refs/heads/master": + return {"taskId": ""} raise TaskclusterRestFailure @@ -45,10 +47,19 @@ os.environ["GIT_REF"] = "refs/heads/auto" import decision_task print("\n# Push:") -decision_task.main("github-push", mock=True) +decision_task.main("github-push") print("\n# Push with hot caches:") -decision_task.main("github-push", mock=True) +decision_task.main("github-push") + +print("\n# Mocked only:") +decision_task.mocked_only() + +print("\n# Push to master:") +decision_task.CONFIG.git_ref = "refs/heads/master" +decision_task.main("github-push") print("\n# Daily:") -decision_task.main("daily", mock=True) +decision_task.main("daily") + +print() diff --git a/python/servo/post_build_commands.py b/python/servo/post_build_commands.py index dc2731c0318..4050f33f932 100644 --- a/python/servo/post_build_commands.py +++ b/python/servo/post_build_commands.py @@ -13,6 +13,7 @@ import json import os import os.path as path import subprocess +import sys from shutil import copytree, rmtree, copy2 from mach.decorators import ( @@ -263,10 +264,23 @@ class PostBuildCommands(CommandBase): else: copy2(full_name, destination) - return self.call_rustup_run( + returncode = self.call_rustup_run( ["cargo", "doc", "--manifest-path", self.ports_servo_manifest()] + params, - env=self.build_env() - ) + env=self.build_env()) + if returncode: + return returncode + + static = path.join(self.context.topdir, "etc", "doc.servo.org") + for name in os.listdir(static): + copy2(path.join(static, name), path.join(docs, name)) + + build = path.join(self.context.topdir, "components", "style", "properties", "build.py") + subprocess.check_call([sys.executable, build, "servo", "html"]) + + script = path.join(self.context.topdir, "components", "script") + subprocess.check_call(["cmake", "."], cwd=script) + subprocess.check_call(["cmake", "--build", ".", "--target", "supported-apis"], cwd=script) + copy2(path.join(script, "apis.html"), path.join(docs, "servo", "apis.html")) @Command('browse-doc', description='Generate documentation and open it in a web browser', |