diff options
| -rw-r--r-- | etc/taskcluster/decision_task.py | 23 | ||||
| -rw-r--r-- | etc/taskcluster/decisionlib.py | 65 |
2 files changed, 48 insertions, 40 deletions
diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 9500656fb2c..1771474b323 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -171,7 +171,7 @@ def linux_tidy_unit_untrusted(): .with_max_run_time_minutes(60) .with_dockerfile(dockerfile_path("build")) .with_env(**build_env, **unix_build_env, **linux_build_env) - .with_repo() + .with_repo_bundle() .with_script("rustup set profile minimal") # required by components/script_plugins: .with_script("rustup component add rustc-dev") @@ -391,7 +391,7 @@ def android_x86_wpt(): .with_capabilities(privileged=True) .with_scopes("project:servo:docker-worker-kvm:capability:privileged") .with_dockerfile(dockerfile_path("run-android-emulator")) - .with_repo() + .with_repo_bundle() ) apk_dir = "target/android/i686-linux-android/release" return ( @@ -560,7 +560,7 @@ def linux_wpt(): .find_or_create("build.linux_x64_release_w_assertions" + CONFIG.task_id()) ) def linux_run_task(name): - return linux_task(name).with_dockerfile(dockerfile_path("run")) + return linux_task(name).with_dockerfile(dockerfile_path("run")).with_repo_bundle() wpt_chunks("Linux x64", linux_run_task, release_build_task, repo_dir="/repo", total_chunks=4, processes=12) @@ -594,14 +594,14 @@ def update_wpt(): .with_scopes("secrets:get:project/servo/wpt-sync") .with_index_and_artifacts_expire_in(log_artifacts_expire_in) .with_max_run_time_minutes(6 * 60) + # Not using the bundle, pushing the new changes to the git remote requires a full repo. + .with_repo(shallow=False, alternate_object_dir="/var/cache/servo.git/objects") ) return ( with_homebrew(update_task, [ "etc/taskcluster/macos/Brewfile-wpt-update", "etc/taskcluster/macos/Brewfile", ]) - # Pushing the new changes to the git remote requires a full repo clone. - .with_repo(shallow=False, alternate_object_dir="/var/cache/servo.git/objects") .with_curl_artifact_script(build_task, "target.tar.gz") .with_script(""" export PKG_CONFIG_PATH="$(brew --prefix libffi)/lib/pkgconfig/" @@ -637,21 +637,21 @@ def macos_wpt(): priority = "high" if CONFIG.git_ref == "refs/heads/auto" else None build_task = macos_release_build_with_debug_assertions(priority=priority) def macos_run_task(name): - task = macos_task(name).with_python2() + task = macos_task(name).with_python2() \ + .with_repo_bundle(alternate_object_dir="/var/cache/servo.git/objects") return with_homebrew(task, ["etc/taskcluster/macos/Brewfile"]) wpt_chunks( "macOS x64", macos_run_task, build_task, repo_dir="repo", - repo_kwargs=dict(alternate_object_dir="/var/cache/servo.git/objects"), total_chunks=30, processes=4, ) def wpt_chunks(platform, make_chunk_task, build_task, total_chunks, processes, - repo_dir, chunks="all", repo_kwargs={}): + repo_dir, chunks="all"): if chunks == "all": chunks = range(total_chunks + 1) for this_chunk in chunks: @@ -660,7 +660,6 @@ def wpt_chunks(platform, make_chunk_task, build_task, total_chunks, processes, this_chunk, total_chunks, width=len(str(total_chunks)), )) .with_treeherder(platform, "WPT-%s" % this_chunk) - .with_repo(**repo_kwargs) .with_curl_artifact_script(build_task, "target.tar.gz") .with_script("tar -xzf target.tar.gz") .with_index_and_artifacts_expire_in(log_artifacts_expire_in) @@ -807,7 +806,7 @@ def linux_build_task(name, *, build_env=build_env, install_rustc_dev=True): .with_max_run_time_minutes(60) .with_dockerfile(dockerfile_path("build")) .with_env(**build_env, **unix_build_env, **linux_build_env) - .with_repo() + .with_repo_bundle() .with_script("rustup set profile minimal") ) if install_rustc_dev: @@ -850,7 +849,7 @@ def windows_build_task(name, package=True, arch="x86_64"): **windows_build_env[arch], **windows_build_env["all"] ) - .with_repo(sparse_checkout=windows_sparse_checkout) + .with_repo_bundle(sparse_checkout=windows_sparse_checkout) .with_python2() .with_directory_mount( "https://www.python.org/ftp/python/3.7.3/python-3.7.3-embed-amd64.zip", @@ -903,7 +902,7 @@ def macos_build_task(name): # https://github.com/servo/servo/issues/24735 .with_max_run_time_minutes(60 * 2) .with_env(**build_env, **unix_build_env, **macos_build_env) - .with_repo(alternate_object_dir="/var/cache/servo.git/objects") + .with_repo_bundle(alternate_object_dir="/var/cache/servo.git/objects") .with_python2() .with_rustup() # Since macOS workers are long-lived and ~/.rustup kept across tasks: diff --git a/etc/taskcluster/decisionlib.py b/etc/taskcluster/decisionlib.py index e206ae8aa97..a28efdd7a8c 100644 --- a/etc/taskcluster/decisionlib.py +++ b/etc/taskcluster/decisionlib.py @@ -58,6 +58,7 @@ class Config: self.git_url = os.environ.get("GIT_URL") self.git_ref = os.environ.get("GIT_REF") self.git_sha = os.environ.get("GIT_SHA") + self.git_bundle_shallow_ref = "refs/heads/shallow" self.tc_root_url = os.environ.get("TASKCLUSTER_ROOT_URL") self.default_provisioner_id = "proj-example" @@ -147,6 +148,9 @@ class Task: self.extra = {} self.treeherder_required = False self.priority = None # Defaults to 'lowest' + self.git_fetch_url = CONFIG.git_url + self.git_fetch_ref = CONFIG.git_ref + self.git_checkout_sha = CONFIG.git_sha # All `with_*` methods return `self`, so multiple method calls can be chained. with_description = chaining(setattr, "description") @@ -321,6 +325,14 @@ class Task: os.path.join(out_directory, url_basename(artifact_name)), ) + def with_repo_bundle(self, **kwargs): + self.git_fetch_url = "../repo.bundle" + self.git_fetch_ref = CONFIG.git_bundle_shallow_ref + self.git_checkout_sha = "FETCH_HEAD" + return self \ + .with_curl_artifact_script(CONFIG.decision_task_id, "repo.bundle") \ + .with_repo(**kwargs) + class GenericWorkerTask(Task): """ @@ -500,13 +512,17 @@ class WindowsGenericWorkerTask(GenericWorkerTask): type .git\\info\\sparse-checkout """ git += """ - git fetch --no-tags {depth} %GIT_URL% %GIT_REF% - git reset --hard %GIT_SHA% - """.format(depth="--depth 30" if shallow else "") + git fetch --no-tags {depth} {} {} + git reset --hard {} + """.format( + assert_truthy(self.git_fetch_url), + assert_truthy(self.git_fetch_ref), + assert_truthy(self.git_checkout_sha), + depth="--depth 30" if shallow else "", + ) return self \ .with_git() \ - .with_script(git) \ - .with_env(**git_env()) + .with_script(git) def with_git(self): """ @@ -613,7 +629,7 @@ class WindowsGenericWorkerTask(GenericWorkerTask): class UnixTaskMixin(Task): - def with_repo(self, shallow=True, alternate_object_dir=None): + def with_repo(self, shallow=True, alternate_object_dir=""): """ Make a shallow clone the git repository at the start of the task. This uses `CONFIG.git_url`, `CONFIG.git_ref`, and `CONFIG.git_sha` @@ -628,22 +644,19 @@ class UnixTaskMixin(Task): """ # Not using $GIT_ALTERNATE_OBJECT_DIRECTORIES since it causes # "object not found - no match for id" errors when Cargo fetches git dependencies - if alternate_object_dir: - self.with_env(ALTERNATE_OBJDIR=alternate_object_dir) return self \ - .with_env(**git_env()) \ - .with_early_script(""" + .with_script(""" git init repo cd repo - {alternate} - time git fetch --no-tags {depth} "$GIT_URL" "$GIT_REF" - time git reset --hard "$GIT_SHA" + echo "{alternate}" > .git/objects/info/alternates + time git fetch --no-tags {depth} {} {} + time git reset --hard {} """.format( + assert_truthy(self.git_fetch_url), + assert_truthy(self.git_fetch_ref), + assert_truthy(self.git_checkout_sha), depth="--depth 30" if shallow else "", - alternate=( - """echo "$ALTERNATE_OBJDIR" > .git/objects/info/alternates""" - if alternate_object_dir else "" - ) + alternate=alternate_object_dir, )) @@ -821,15 +834,10 @@ def expand_dockerfile(dockerfile): return b"\n".join([expand_dockerfile(path), rest]) -def git_env(): - assert CONFIG.git_url - assert CONFIG.git_ref - assert CONFIG.git_sha - return { - "GIT_URL": CONFIG.git_url, - "GIT_REF": CONFIG.git_ref, - "GIT_SHA": CONFIG.git_sha, - } +def assert_truthy(x): + assert x + return x + def dict_update_if_truthy(d, **kwargs): for key, value in kwargs.items(): @@ -853,9 +861,10 @@ def make_repo_bundle(): tree = subprocess.check_output(["git", "show", CONFIG.git_sha, "--pretty=%T", "--no-patch"]) message = "Shallow version of commit " + CONFIG.git_sha commit = subprocess.check_output(["git", "commit-tree", tree.strip(), "-m", message]) - subprocess.check_call(["git", "update-ref", "refs/heads/shallow", commit.strip()]) + subprocess.check_call(["git", "update-ref", CONFIG.git_bundle_shallow_ref, commit.strip()]) subprocess.check_call(["git", "show-ref"]) - with subprocess.Popen(["git", "bundle", "create", "../repo.bundle", "refs/heads/shallow"]) as p: + create = ["git", "bundle", "create", "../repo.bundle", CONFIG.git_bundle_shallow_ref] + with subprocess.Popen(create) as p: yield exit_code = p.wait() if exit_code: |