From 203a06ff24966d3e5ee20b9c190addb9e05d3809 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Thu, 21 Nov 2019 11:27:33 +0100 Subject: =?UTF-8?q?Fix=20Brewfile=E2=80=99s=20for=20running=20Servo=20with?= =?UTF-8?q?out=20bulding=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- etc/taskcluster/decision_task.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'etc/taskcluster/decision_task.py') diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 8fd12d7086f..aa483ee4dc0 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -587,8 +587,8 @@ def update_wpt(): ) return ( with_homebrew(update_task, [ - "etc/taskcluster/macos/Brewfile-wpt", - "etc/taskcluster/macos/Brewfile-gstreamer", + "etc/taskcluster/macos/Brewfile-wpt-update", + "etc/taskcluster/macos/Brewfile", ]) # Pushing the new changes to the git remote requires a full repo clone. .with_repo(shallow=False, alternate_object_dir="/var/cache/servo.git/objects") @@ -628,7 +628,7 @@ def macos_wpt(): build_task = macos_release_build_with_debug_assertions(priority=priority) def macos_run_task(name): task = macos_task(name).with_python2() - return with_homebrew(task, ["etc/taskcluster/macos/Brewfile-gstreamer"]) + return with_homebrew(task, ["etc/taskcluster/macos/Brewfile"]) wpt_chunks( "macOS x64", macos_run_task, @@ -909,7 +909,7 @@ def macos_build_task(name): return ( with_homebrew(build_task, [ "etc/taskcluster/macos/Brewfile", - "etc/taskcluster/macos/Brewfile-gstreamer", + "etc/taskcluster/macos/Brewfile-build", ]) .with_script(""" export OPENSSL_INCLUDE_DIR="$(brew --prefix openssl)/include" -- cgit v1.2.3 From b96a36c1316111754de699698b9af9a3fac89509 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Thu, 21 Nov 2019 23:20:28 +0100 Subject: Move `mkdir` call out of `with_curl_script` --- etc/taskcluster/decision_task.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'etc/taskcluster/decision_task.py') diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index aa483ee4dc0..841bc54882c 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -378,7 +378,7 @@ def android_x86_release(): def android_x86_wpt(): build_task = android_x86_release() - return ( + task = ( linux_task("WPT") .with_treeherder("Android x86") .with_provisioner_id("proj-servo") @@ -387,7 +387,12 @@ def android_x86_wpt(): .with_scopes("project:servo:docker-worker-kvm:capability:privileged") .with_dockerfile(dockerfile_path("run-android-emulator")) .with_repo() - .with_curl_artifact_script(build_task, "servoapp.apk", "target/android/i686-linux-android/release") + ) + apk_dir = "target/android/i686-linux-android/release" + return ( + task + .with_script("mkdir -p " + apk_dir) + .with_curl_artifact_script(build_task, "servoapp.apk", apk_dir) .with_script(""" ./mach bootstrap-android --accept-all-licences --emulator-x86 ./mach test-android-startup --release -- cgit v1.2.3 From 55a8158d17bb593c74ca771c3ca2d52d6cafafaa Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Thu, 21 Nov 2019 22:08:57 +0100 Subject: =?UTF-8?q?Decision=20taks:=20add=20artifacts=20of=20the=20reposit?= =?UTF-8?q?ory=E2=80=99s=20contents?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- etc/taskcluster/decision_task.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'etc/taskcluster/decision_task.py') diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 841bc54882c..9500656fb2c 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -10,6 +10,11 @@ from decisionlib import CONFIG, SHARED def main(task_for): + with decisionlib.make_repo_bundle(): + tasks(task_for) + + +def tasks(task_for): if CONFIG.git_ref.startswith("refs/heads/"): branch = CONFIG.git_ref[len("refs/heads/"):] CONFIG.treeherder_repository_name = "servo-" + ( -- cgit v1.2.3 From 3a8bb531a07aaee60cec110853d1b54d53d438c8 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Fri, 22 Nov 2019 15:04:30 +0100 Subject: Fetch repository from artifacts rather than GitHub --- etc/taskcluster/decision_task.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'etc/taskcluster/decision_task.py') diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 9500656fb2c..1771474b323 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -171,7 +171,7 @@ def linux_tidy_unit_untrusted(): .with_max_run_time_minutes(60) .with_dockerfile(dockerfile_path("build")) .with_env(**build_env, **unix_build_env, **linux_build_env) - .with_repo() + .with_repo_bundle() .with_script("rustup set profile minimal") # required by components/script_plugins: .with_script("rustup component add rustc-dev") @@ -391,7 +391,7 @@ def android_x86_wpt(): .with_capabilities(privileged=True) .with_scopes("project:servo:docker-worker-kvm:capability:privileged") .with_dockerfile(dockerfile_path("run-android-emulator")) - .with_repo() + .with_repo_bundle() ) apk_dir = "target/android/i686-linux-android/release" return ( @@ -560,7 +560,7 @@ def linux_wpt(): .find_or_create("build.linux_x64_release_w_assertions" + CONFIG.task_id()) ) def linux_run_task(name): - return linux_task(name).with_dockerfile(dockerfile_path("run")) + return linux_task(name).with_dockerfile(dockerfile_path("run")).with_repo_bundle() wpt_chunks("Linux x64", linux_run_task, release_build_task, repo_dir="/repo", total_chunks=4, processes=12) @@ -594,14 +594,14 @@ def update_wpt(): .with_scopes("secrets:get:project/servo/wpt-sync") .with_index_and_artifacts_expire_in(log_artifacts_expire_in) .with_max_run_time_minutes(6 * 60) + # Not using the bundle, pushing the new changes to the git remote requires a full repo. + .with_repo(shallow=False, alternate_object_dir="/var/cache/servo.git/objects") ) return ( with_homebrew(update_task, [ "etc/taskcluster/macos/Brewfile-wpt-update", "etc/taskcluster/macos/Brewfile", ]) - # Pushing the new changes to the git remote requires a full repo clone. - .with_repo(shallow=False, alternate_object_dir="/var/cache/servo.git/objects") .with_curl_artifact_script(build_task, "target.tar.gz") .with_script(""" export PKG_CONFIG_PATH="$(brew --prefix libffi)/lib/pkgconfig/" @@ -637,21 +637,21 @@ def macos_wpt(): priority = "high" if CONFIG.git_ref == "refs/heads/auto" else None build_task = macos_release_build_with_debug_assertions(priority=priority) def macos_run_task(name): - task = macos_task(name).with_python2() + task = macos_task(name).with_python2() \ + .with_repo_bundle(alternate_object_dir="/var/cache/servo.git/objects") return with_homebrew(task, ["etc/taskcluster/macos/Brewfile"]) wpt_chunks( "macOS x64", macos_run_task, build_task, repo_dir="repo", - repo_kwargs=dict(alternate_object_dir="/var/cache/servo.git/objects"), total_chunks=30, processes=4, ) def wpt_chunks(platform, make_chunk_task, build_task, total_chunks, processes, - repo_dir, chunks="all", repo_kwargs={}): + repo_dir, chunks="all"): if chunks == "all": chunks = range(total_chunks + 1) for this_chunk in chunks: @@ -660,7 +660,6 @@ def wpt_chunks(platform, make_chunk_task, build_task, total_chunks, processes, this_chunk, total_chunks, width=len(str(total_chunks)), )) .with_treeherder(platform, "WPT-%s" % this_chunk) - .with_repo(**repo_kwargs) .with_curl_artifact_script(build_task, "target.tar.gz") .with_script("tar -xzf target.tar.gz") .with_index_and_artifacts_expire_in(log_artifacts_expire_in) @@ -807,7 +806,7 @@ def linux_build_task(name, *, build_env=build_env, install_rustc_dev=True): .with_max_run_time_minutes(60) .with_dockerfile(dockerfile_path("build")) .with_env(**build_env, **unix_build_env, **linux_build_env) - .with_repo() + .with_repo_bundle() .with_script("rustup set profile minimal") ) if install_rustc_dev: @@ -850,7 +849,7 @@ def windows_build_task(name, package=True, arch="x86_64"): **windows_build_env[arch], **windows_build_env["all"] ) - .with_repo(sparse_checkout=windows_sparse_checkout) + .with_repo_bundle(sparse_checkout=windows_sparse_checkout) .with_python2() .with_directory_mount( "https://www.python.org/ftp/python/3.7.3/python-3.7.3-embed-amd64.zip", @@ -903,7 +902,7 @@ def macos_build_task(name): # https://github.com/servo/servo/issues/24735 .with_max_run_time_minutes(60 * 2) .with_env(**build_env, **unix_build_env, **macos_build_env) - .with_repo(alternate_object_dir="/var/cache/servo.git/objects") + .with_repo_bundle(alternate_object_dir="/var/cache/servo.git/objects") .with_python2() .with_rustup() # Since macOS workers are long-lived and ~/.rustup kept across tasks: -- cgit v1.2.3 From 4dbdd255a1739707d7a8347c515f563811938557 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Fri, 22 Nov 2019 15:36:54 +0100 Subject: Remove support for shallow clones. The bundle is already shallow. --- etc/taskcluster/decision_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'etc/taskcluster/decision_task.py') diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 1771474b323..0244f955bfe 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -595,7 +595,7 @@ def update_wpt(): .with_index_and_artifacts_expire_in(log_artifacts_expire_in) .with_max_run_time_minutes(6 * 60) # Not using the bundle, pushing the new changes to the git remote requires a full repo. - .with_repo(shallow=False, alternate_object_dir="/var/cache/servo.git/objects") + .with_repo(alternate_object_dir="/var/cache/servo.git/objects") ) return ( with_homebrew(update_task, [ -- cgit v1.2.3 From 77089ef44e8766b3d5366b0db4e8a608ee799657 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Thu, 21 Nov 2019 10:11:17 +0100 Subject: Fix updating the GitHub Status as soon as any TC task fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … rather than only when the entire task group is resolved. This allows Homu to more quickly be notified of a failure, and move on to the next PR in the queue sooner. --- etc/taskcluster/decision_task.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'etc/taskcluster/decision_task.py') diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 0244f955bfe..ecfa82507f9 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -21,6 +21,12 @@ def tasks(task_for): branch if not branch.startswith("try-") else "try" ) + # Work around a tc-github bug/limitation: + # https://bugzilla.mozilla.org/show_bug.cgi?id=1548781#c4 + if task_for.startswith("github"): + # https://github.com/taskcluster/taskcluster/blob/21f257dc8/services/github/config.yml#L14 + CONFIG.routes_for_all_subtasks.append("statuses") + # The magicleap build is broken until there's a surfman back end magicleap_dev = lambda: None magicleap_nightly = lambda: None -- cgit v1.2.3