bootstrap: Improve pip dependency resolution (#31026)

We can now use the "new" pip resolver which should prevent the
installation of conflicting packages. Also, take this opportunity to
make bootstrap more resilient. Hash all dependencies to detect
situations where a newer marker file has been installed, but for an
older branch. This should ensure that dependencies are up to date even
when switching back and forth between older and new branches.

This also updates some dependencies to be the same as the ones used for
WPT tests, which is an issue caught be the resolver.

Fixes #10611.

2 files changed, 41 insertions, 30 deletions

diff --git a/python/mach_bootstrap.py b/python/mach_bootstrap.py
index ad7278c4514..053f7245798 100644
--- a/python/mach_bootstrap.py
+++ b/python/mach_bootstrap.py
@@ -2,6 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
+import hashlib
 import os
 import platform
 import site
@@ -108,6 +109,42 @@ def _process_exec(args):
         sys.exit(1)
 
 
+def install_virtual_env_requirements(project_path: str, python: str, virtualenv_path: str):
+    requirements_paths = [
+        os.path.join(project_path, "python", "requirements.txt"),
+        os.path.join(project_path, WPT_TOOLS_PATH, "requirements_tests.txt",),
+        os.path.join(project_path, WPT_RUNNER_PATH, "requirements.txt",),
+    ]
+
+    requirements_hasher = hashlib.sha256()
+    for path in requirements_paths:
+        with open(path, 'rb') as file:
+            requirements_hasher.update(file.read())
+
+    try:
+        marker_path = os.path.join(virtualenv_path, "requirements.sha256")
+        with open(marker_path, 'r') as marker_file:
+            marker_hash = marker_file.read()
+    except FileNotFoundError:
+        marker_hash = None
+
+    requirements_hash = requirements_hasher.hexdigest()
+
+    if marker_hash != requirements_hash:
+        print(" * Upgrading pip...")
+        _process_exec([python, "-m", "pip", "install", "--upgrade", "pip"])
+
+        print(" * Installing Python requirements...")
+        _process_exec([python, "-m", "pip", "install", "-I",
+                       "-r", requirements_paths[0],
+                       "-r", requirements_paths[1],
+                       "-r", requirements_paths[2]])
+        with open(marker_path, "w") as marker_file:
+            marker_file.write(requirements_hash)
+    else:
+        print(" * Python requirements up to date.")
+
+
 def _activate_virtualenv(topdir):
     virtualenv_path = os.path.join(topdir, "python", "_venv%d.%d" % (sys.version_info[0], sys.version_info[1]))
     python = sys.executable
@@ -134,33 +171,7 @@ def _activate_virtualenv(topdir):
         # Otherwise pip may still try to write to the wrong site-packages directory.
         python = os.path.join(venv_script_path, "python")
 
-    # TODO: Right now, we iteratively install all the requirements by invoking
-    # `pip install` each time. If it were the case that there were conflicting
-    # requirements, we wouldn't know about them. Once
-    # https://github.com/pypa/pip/issues/988 is addressed, then we can just
-    # chain each of the requirements files into the same `pip install` call
-    # and it will check for conflicts.
-    requirements_paths = [
-        os.path.join("python", "requirements.txt"),
-        os.path.join(WPT_TOOLS_PATH, "requirements_tests.txt",),
-        os.path.join(WPT_RUNNER_PATH, "requirements.txt",),
-    ]
-
-    for req_rel_path in requirements_paths:
-        req_path = os.path.join(topdir, req_rel_path)
-        marker_file = req_rel_path.replace(os.path.sep, '-')
-        marker_path = os.path.join(virtualenv_path, marker_file)
-
-        try:
-            if os.path.getmtime(req_path) + 10 < os.path.getmtime(marker_path):
-                continue
-        except OSError:
-            pass
-
-        print(f" * Installing Python requirements from {req_path}...")
-        _process_exec([python, "-m", "pip", "install", "-I", "-r", req_path])
-
-        open(marker_path, 'w').close()
+    install_virtual_env_requirements(topdir, python, virtualenv_path)
 
 
 def _ensure_case_insensitive_if_windows():
diff --git a/python/requirements.txt b/python/requirements.txt
index 959ef9966b5..7c56901599f 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -3,8 +3,8 @@
 
 blessings == 1.7
 distro == 1.4
-mozinfo == 1.2.1
-mozlog == 7.1.0
+mozinfo == 1.2.3
+mozlog == 8.0.0
 setuptools == 68.2.2; python_version >= "3.8"
 setuptools == 65.5.1; python_version < "3.8"
 toml == 0.9.2
@@ -27,7 +27,7 @@ pyOpenSSL == 23.0.0
 PyGithub == 1.58.1
 
 # For Python3 compatibility
-six == 1.15
+six == 1.16
 
 # For sending build notifications.
 notify-py == 0.3.42