Auto merge of #22444 - servo:tc, r=SimonSapin

More Taskcluster/Treeherder tweaks

* Fix/add more tasks showing up on Treeherder
* Repeat fewer tasks on `@bors-servo retry`

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/22444)
<!-- Reviewable:end -->

3 files changed, 45 insertions, 24 deletions

diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py
index d27a7111fc9..dd1fbd06fea 100644
--- a/etc/taskcluster/decision_task.py
+++ b/etc/taskcluster/decision_task.py
@@ -5,10 +5,15 @@
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 import os.path
-from decisionlib import *
+import decisionlib
+from decisionlib import CONFIG, SHARED
 
 
 def main(task_for):
+    assert CONFIG.git_ref.startswith("refs/heads/")
+    branch = CONFIG.git_ref[len("refs/heads/"):]
+    CONFIG.treeherder_repository_name = "servo-" + branch
+
     if task_for == "github-push":
         # FIXME https://github.com/servo/servo/issues/22325 implement these:
         macos_wpt = magicleap_dev = linux_arm32_dev = linux_arm64_dev = \
@@ -34,8 +39,6 @@ def main(task_for):
                 # Add functions here as needed, in your push to that branch
             ],
             "master": [
-                # Also show these tasks in https://treeherder.mozilla.org/#/jobs?repo=servo-auto
-                lambda: CONFIG.treeherder_repository_names.append("servo-auto"),
                 upload_docs,
             ],
 
@@ -56,9 +59,6 @@ def main(task_for):
                 android_x86_wpt
             ],
         }
-        assert CONFIG.git_ref.startswith("refs/heads/")
-        branch = CONFIG.git_ref[len("refs/heads/"):]
-        CONFIG.treeherder_repository_names.append("servo-" + branch)
         for function in by_branch_name.get(branch, []):
             function()
 
@@ -74,7 +74,7 @@ def mocked_only():
     windows_release()
     linux_wpt()
     android_x86_wpt()
-    linux_build_task("Indexed by task definition").find_or_create()
+    decisionlib.DockerWorkerTask("Indexed by task definition").find_or_create()
 
 
 ping_on_daily_task_failure = "SimonSapin, nox, emilio"
@@ -141,7 +141,7 @@ def linux_tidy_unit_docs():
 
 
 def upload_docs():
-    docs_build_task_id = Task.find("docs." + CONFIG.git_sha)
+    docs_build_task_id = decisionlib.Task.find("docs." + CONFIG.git_sha)
     return (
         linux_task("Upload docs to GitHub Pages")
         .with_treeherder("Linux x64", "DocUpload")
@@ -175,7 +175,7 @@ def macos_unit():
             ./mach package --dev
             ./etc/ci/lockfile_changed.sh
         """)
-        .create()
+        .find_or_create("macos_unit." + CONFIG.git_sha)
     )
 
 
@@ -187,7 +187,8 @@ def with_rust_nightly():
         modified_build_env["RUSTFLAGS"] = " ".join(flags)
 
     return (
-        linux_build_task("Linux x64: with Rust Nightly", build_env=modified_build_env)
+        linux_build_task("with Rust Nightly", build_env=modified_build_env)
+        .with_treeherder("Linux x64", "RustNightly")
         .with_script("""
             echo "nightly" > rust-toolchain
             ./mach build --dev
@@ -206,7 +207,7 @@ def android_arm32_dev():
             ./etc/ci/lockfile_changed.sh
             python ./etc/ci/check_dynamic_symbols.py
         """)
-        .create()
+        .find_or_create("android_arm32_dev." + CONFIG.git_sha)
     )
 
 
@@ -239,7 +240,7 @@ def android_x86_release():
 def android_x86_wpt():
     build_task = android_x86_release()
     return (
-        DockerWorkerTask("WPT")
+        linux_task("WPT")
         .with_treeherder("Android x86")
         .with_provisioner_id("proj-servo")
         .with_worker_type("docker-worker-kvm")
@@ -255,7 +256,7 @@ def android_x86_wpt():
                 /_mozilla/mozilla/DOMParser.html \
                 /_mozilla/mozilla/webgl/context_creation_error.html
         """)
-        .create()
+        .find_or_create("android_x86_release." + CONFIG.git_sha)
     )
 
 
@@ -368,7 +369,7 @@ def wpt_chunk(release_build_task, total_chunks, this_chunk):
         for word in script.split()
         if word.endswith(".log")
     ])
-    return task.create()
+    return task.find_or_create("linux_wpt_%s.%s" % (this_chunk, CONFIG.git_sha))
 
 
 def daily_tasks_setup():
@@ -402,18 +403,28 @@ def dockerfile_path(name):
 
 
 def linux_task(name):
-    return DockerWorkerTask(name).with_worker_type("servo-docker-worker")
+    return (
+        decisionlib.DockerWorkerTask(name)
+        .with_worker_type("servo-docker-worker")
+        .with_treeherder_required()
+    )
 
 
 def windows_task(name):
-    return WindowsGenericWorkerTask(name).with_worker_type("servo-win2016")
+    return (
+        decisionlib.WindowsGenericWorkerTask(name)
+        .with_worker_type("servo-win2016")
+        .with_treeherder_required()
+    )
+
 
 
 def macos_task(name):
     return (
-        MacOsGenericWorkerTask(name)
+        decisionlib.MacOsGenericWorkerTask(name)
         .with_provisioner_id("proj-servo")
         .with_worker_type("macos")
+        .with_treeherder_required()
     )
 
 
diff --git a/etc/taskcluster/decisionlib.py b/etc/taskcluster/decisionlib.py
index 553113db326..7a4892cd901 100644
--- a/etc/taskcluster/decisionlib.py
+++ b/etc/taskcluster/decisionlib.py
@@ -43,7 +43,7 @@ class Config:
         self.docker_image_buil_worker_type = None
         self.docker_images_expire_in = "1 month"
         self.repacked_msi_files_expire_in = "1 month"
-        self.treeherder_repository_names = []
+        self.treeherder_repository_name = None
 
         # Set by docker-worker:
         # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/environment
@@ -119,6 +119,7 @@ class Task:
         self.scopes = []
         self.routes = []
         self.extra = {}
+        self.treeherder_required = False
 
     # All `with_*` methods return `self`, so multiple method calls can be chained.
     with_description = chaining(setattr, "description")
@@ -135,6 +136,10 @@ class Task:
 
     with_extra = chaining(update_attr, "extra")
 
+    def with_treeherder_required(self):
+        self.treeherder_required = True
+        return self
+
     def with_treeherder(self, category, symbol=None):
         symbol = symbol or self.name
         assert len(symbol) <= 25, symbol
@@ -156,14 +161,15 @@ class Task:
             "symbol": symbol,
         })
 
-        for repo in CONFIG.treeherder_repository_names:
+        if CONFIG.treeherder_repository_name:
             assert CONFIG.git_sha
-            suffix = ".v2._/%s.%s" % (repo, CONFIG.git_sha)
+            suffix = ".v2._/%s.%s" % (CONFIG.treeherder_repository_name, CONFIG.git_sha)
             self.with_routes(
                 "tc-treeherder" + suffix,
                 "tc-treeherder-staging" + suffix,
             )
 
+        self.treeherder_required = False  # Taken care of
         return self
 
     def build_worker_payload(self):  # pragma: no cover
@@ -183,6 +189,8 @@ class Task:
         <https://docs.taskcluster.net/docs/reference/platform/taskcluster-queue/references/api#createTask>
         """
         worker_payload = self.build_worker_payload()
+        assert not self.treeherder_required, \
+            "make sure to call with_treeherder() for this task: %s" % self.name
 
         assert CONFIG.decision_task_id
         assert CONFIG.task_owner
@@ -220,13 +228,15 @@ class Task:
 
         task_id = taskcluster.slugId().decode("utf8")
         SHARED.queue_service.createTask(task_id, queue_payload)
-        print("Scheduled %s" % self.name)
+        print("Scheduled %s: %s" % (task_id, self.name))
         return task_id
 
     @staticmethod
     def find(index_path):
         full_index_path = "%s.%s" % (CONFIG.index_prefix, index_path)
-        return SHARED.index_service.findTask(full_index_path)["taskId"]
+        task_id = SHARED.index_service.findTask(full_index_path)["taskId"]
+        print("Found task %s indexed at %s" % (task_id, full_index_path))
+        return task_id
 
     def find_or_create(self, index_path=None):
         """
diff --git a/etc/taskcluster/mock.py b/etc/taskcluster/mock.py
index 60720f73ddd..bfc963ebaf8 100755
--- a/etc/taskcluster/mock.py
+++ b/etc/taskcluster/mock.py
@@ -33,12 +33,12 @@ class Index:
 
     def findTask(self, path):
         if decision_task.CONFIG.git_ref == "refs/heads/master":
-            return {"taskId": ""}
+            return {"taskId": "<from index>"}
         raise TaskclusterRestFailure
 
 
 stringDate = str
-slugId = b"id".lower
+slugId = b"<new id>".lower
 Queue = fromNow = MagicMock()
 sys.modules["taskcluster"] = sys.modules[__name__]
 sys.dont_write_bytecode = True