Auto merge of #24811 - servo:fail-fast, r=jdmhomu-tmp

Fix updating the GitHub Status as soon as any TC task fails

… rather than only when the entire task group is resolved. This allows Homu to more quickly be notified of a failure, and move on to the next PR in the queue sooner.

(Plus drive-by Brewfile fix.)

1 files changed, 89 insertions, 57 deletions

diff --git a/etc/taskcluster/decisionlib.py b/etc/taskcluster/decisionlib.py
index acd8556ad79..ef585657a32 100644
--- a/etc/taskcluster/decisionlib.py
+++ b/etc/taskcluster/decisionlib.py
@@ -14,6 +14,7 @@ Project-independent library for Taskcluster decision tasks
 """
 
 import base64
+import contextlib
 import datetime
 import hashlib
 import json
@@ -28,6 +29,7 @@ import taskcluster
 __all__ = [
     "CONFIG", "SHARED", "Task", "DockerWorkerTask",
     "GenericWorkerTask", "WindowsGenericWorkerTask", "MacOsGenericWorkerTask",
+    "make_repo_bundle",
 ]
 
 
@@ -56,6 +58,7 @@ class Config:
         self.git_url = os.environ.get("GIT_URL")
         self.git_ref = os.environ.get("GIT_REF")
         self.git_sha = os.environ.get("GIT_SHA")
+        self.git_bundle_shallow_ref = "refs/heads/shallow"
 
         self.tc_root_url = os.environ.get("TASKCLUSTER_ROOT_URL")
         self.default_provisioner_id = "proj-example"
@@ -145,6 +148,9 @@ class Task:
         self.extra = {}
         self.treeherder_required = False
         self.priority = None  # Defaults to 'lowest'
+        self.git_fetch_url = CONFIG.git_url
+        self.git_fetch_ref = CONFIG.git_ref
+        self.git_checkout_sha = CONFIG.git_sha
 
     # All `with_*` methods return `self`, so multiple method calls can be chained.
     with_description = chaining(setattr, "description")
@@ -221,9 +227,14 @@ class Task:
         assert CONFIG.decision_task_id
         assert CONFIG.task_owner
         assert CONFIG.task_source
+
+        def dedup(xs):
+            seen = set()
+            return [x for x in xs if not (x in seen or seen.add(x))]
+
         queue_payload = {
             "taskGroupId": CONFIG.decision_task_id,
-            "dependencies": [CONFIG.decision_task_id] + self.dependencies,
+            "dependencies": dedup([CONFIG.decision_task_id] + self.dependencies),
             "schedulerId": self.scheduler_id,
             "provisionerId": self.provisioner_id,
             "workerType": self.worker_type,
@@ -299,6 +310,29 @@ class Task:
         SHARED.found_or_created_indexed_tasks[index_path] = task_id
         return task_id
 
+    def with_curl_script(self, url, file_path):
+        return self \
+        .with_script("""
+            curl --retry 5 --connect-timeout 10 -Lf "%s" -o "%s"
+        """ % (url, file_path))
+
+    def with_curl_artifact_script(self, task_id, artifact_name, out_directory=""):
+        queue_service = CONFIG.tc_root_url + "/api/queue"
+        return self \
+        .with_dependencies(task_id) \
+        .with_curl_script(
+            queue_service + "/v1/task/%s/artifacts/public/%s" % (task_id, artifact_name),
+            os.path.join(out_directory, url_basename(artifact_name)),
+        )
+
+    def with_repo_bundle(self, **kwargs):
+        self.git_fetch_url = "../repo.bundle"
+        self.git_fetch_ref = CONFIG.git_bundle_shallow_ref
+        self.git_checkout_sha = "FETCH_HEAD"
+        return self \
+        .with_curl_artifact_script(CONFIG.decision_task_id, "repo.bundle") \
+        .with_repo(**kwargs)
+
 
 class GenericWorkerTask(Task):
     """
@@ -453,9 +487,9 @@ class WindowsGenericWorkerTask(GenericWorkerTask):
             self.with_early_script("set PATH=%HOMEDRIVE%%HOMEPATH%\\{};%PATH%".format(p))
         return self
 
-    def with_repo(self, sparse_checkout=None, shallow=True):
+    def with_repo(self, sparse_checkout=None):
         """
-        Make a shallow clone the git repository at the start of the task.
+        Make a clone the git repository at the start of the task.
         This uses `CONFIG.git_url`, `CONFIG.git_ref`, and `CONFIG.git_sha`,
         and creates the clone in a `repo` directory in the task’s home directory.
 
@@ -478,13 +512,16 @@ class WindowsGenericWorkerTask(GenericWorkerTask):
                 type .git\\info\\sparse-checkout
             """
         git += """
-            git fetch --no-tags {depth} %GIT_URL% %GIT_REF%
-            git reset --hard %GIT_SHA%
-        """.format(depth="--depth 30" if shallow else "")
+            git fetch --no-tags {} {}
+            git reset --hard {}
+        """.format(
+            assert_truthy(self.git_fetch_url),
+            assert_truthy(self.git_fetch_ref),
+            assert_truthy(self.git_checkout_sha),
+        )
         return self \
         .with_git() \
-        .with_script(git) \
-        .with_env(**git_env())
+        .with_script(git)
 
     def with_git(self):
         """
@@ -501,6 +538,19 @@ class WindowsGenericWorkerTask(GenericWorkerTask):
             path="git",
         )
 
+    def with_curl_script(self, url, file_path):
+        self.with_curl()
+        return super().with_curl_script(url, file_path)
+
+    def with_curl(self):
+        return self \
+        .with_path_from_homedir("curl\\curl-7.67.0-win64-mingw\\bin") \
+        .with_directory_mount(
+            "https://curl.haxx.se/windows/dl-7.67.0_4/curl-7.67.0_4-win64-mingw.zip",
+            sha256="1d50deeac7f945ed75149e6300f6d21f007a6b942ab851a119ed76cdef27d714",
+            path="curl",
+        )
+
     def with_rustup(self):
         """
         Download rustup.rs and make it available to task commands,
@@ -578,13 +628,9 @@ class WindowsGenericWorkerTask(GenericWorkerTask):
 
 
 class UnixTaskMixin(Task):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.curl_scripts_count = 0
-
-    def with_repo(self, shallow=True, alternate_object_dir=None):
+    def with_repo(self, alternate_object_dir=""):
         """
-        Make a shallow clone the git repository at the start of the task.
+        Make a clone the git repository at the start of the task.
         This uses `CONFIG.git_url`, `CONFIG.git_ref`, and `CONFIG.git_sha`
 
         * generic-worker: creates the clone in a `repo` directory
@@ -597,46 +643,20 @@ class UnixTaskMixin(Task):
         """
         # Not using $GIT_ALTERNATE_OBJECT_DIRECTORIES since it causes
         # "object not found - no match for id" errors when Cargo fetches git dependencies
-        if alternate_object_dir:
-            self.with_env(ALTERNATE_OBJDIR=alternate_object_dir)
         return self \
-        .with_env(**git_env()) \
-        .with_early_script("""
+        .with_script("""
             git init repo
             cd repo
-            {alternate}
-            time git fetch --no-tags {depth} "$GIT_URL" "$GIT_REF"
-            time git reset --hard "$GIT_SHA"
+            echo "{alternate}" > .git/objects/info/alternates
+            time git fetch --no-tags {} {}
+            time git reset --hard {}
         """.format(
-            depth="--depth 30" if shallow else "",
-            alternate=(
-                """echo "$ALTERNATE_OBJDIR" > .git/objects/info/alternates"""
-                if alternate_object_dir else ""
-            )
+            assert_truthy(self.git_fetch_url),
+            assert_truthy(self.git_fetch_ref),
+            assert_truthy(self.git_checkout_sha),
+            alternate=alternate_object_dir,
         ))
 
-    def with_curl_script(self, url, file_path):
-        self.curl_scripts_count += 1
-        n = self.curl_scripts_count
-        return self \
-        .with_env(**{
-            "CURL_%s_URL" % n: url,
-            "CURL_%s_PATH" % n: file_path,
-        }) \
-        .with_script("""
-            mkdir -p $(dirname "$CURL_{n}_PATH")
-            curl --retry 5 --connect-timeout 10 -Lf "$CURL_{n}_URL" -o "$CURL_{n}_PATH"
-        """.format(n=n))
-
-    def with_curl_artifact_script(self, task_id, artifact_name, out_directory=""):
-        queue_service = CONFIG.tc_root_url + "/api/queue"
-        return self \
-        .with_dependencies(task_id) \
-        .with_curl_script(
-            queue_service + "/v1/task/%s/artifacts/public/%s" % (task_id, artifact_name),
-            os.path.join(out_directory, url_basename(artifact_name)),
-        )
-
 
 class MacOsGenericWorkerTask(UnixTaskMixin, GenericWorkerTask):
     """
@@ -812,15 +832,10 @@ def expand_dockerfile(dockerfile):
     return b"\n".join([expand_dockerfile(path), rest])
 
 
-def git_env():
-    assert CONFIG.git_url
-    assert CONFIG.git_ref
-    assert CONFIG.git_sha
-    return {
-        "GIT_URL": CONFIG.git_url,
-        "GIT_REF": CONFIG.git_ref,
-        "GIT_SHA": CONFIG.git_sha,
-    }
+def assert_truthy(x):
+    assert x
+    return x
+
 
 def dict_update_if_truthy(d, **kwargs):
     for key, value in kwargs.items():
@@ -835,3 +850,20 @@ def deindent(string):
 
 def url_basename(url):
     return url.rpartition("/")[-1]
+
+
+@contextlib.contextmanager
+def make_repo_bundle():
+    subprocess.check_call(["git", "config", "user.name", "Decision task"])
+    subprocess.check_call(["git", "config", "user.email", "nobody@mozilla.com"])
+    tree = subprocess.check_output(["git", "show", CONFIG.git_sha, "--pretty=%T", "--no-patch"])
+    message = "Shallow version of commit " + CONFIG.git_sha
+    commit = subprocess.check_output(["git", "commit-tree", tree.strip(), "-m", message])
+    subprocess.check_call(["git", "update-ref", CONFIG.git_bundle_shallow_ref, commit.strip()])
+    subprocess.check_call(["git", "show-ref"])
+    create = ["git", "bundle", "create", "../repo.bundle", CONFIG.git_bundle_shallow_ref]
+    with subprocess.Popen(create) as p:
+        yield
+        exit_code = p.wait()
+        if exit_code:
+            sys.exit(exit_code)