build: Speed up first run after build on macOS (#32928)

GStreamer has to process plugins each time they are added when
initializing. When those files have changed, this triggers macOS
security protections which can add many seconds to access time. This
change eliminates that problem after the first packaging of libraries by
skipping packaging if everything is up-to-date and not overwriting the
dylibs everytime.

In addition, it moves a lot of the code for packaging GStreamer
libraries on macOS into the `gstreamer` module and adds type-safety and
comments to the Python.

Signed-off-by: Martin Robinson <mrobinson@igalia.com>

1 files changed, 150 insertions, 6 deletions

diff --git a/python/servo/gstreamer.py b/python/servo/gstreamer.py
index 3c63b7dbd5d..b7e68a0772f 100644
--- a/python/servo/gstreamer.py
+++ b/python/servo/gstreamer.py
@@ -7,8 +7,11 @@
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
-import os
+import os.path
+import shutil
+import subprocess
 import sys
+from typing import Set
 
 GSTREAMER_BASE_LIBS = [
     # gstreamer
@@ -157,11 +160,6 @@ def windows_plugins():
     return [f"{lib}.dll" for lib in libs]
 
 
-def macos_gst_root():
-    return os.path.join(
-        "/", "Library", "Frameworks", "GStreamer.framework", "Versions", "1.0")
-
-
 def macos_plugins():
     plugins = [
         *GSTREAMER_PLUGIN_LIBS,
@@ -185,5 +183,151 @@ pub(crate) static GSTREAMER_PLUGINS: &[&'static str] = &[
 ''' % ',\n'.join(map(lambda x: '"' + x + '"', plugins)))
 
 
+def is_macos_system_library(library_path: str) -> bool:
+    """Returns true if if the given dependency line from otool refers to
+       a system library that should not be packaged."""
+    return (library_path.startswith("/System/Library")
+            or library_path.startswith("/usr/lib")
+            or ".asan." in library_path)
+
+
+def rewrite_dependencies_to_be_relative(binary: str, dependency_lines: Set[str], relative_path: str):
+    """Given a path to a binary (either an executable or a dylib), rewrite the
+       the given dependency lines to be found at the given relative path to
+       the executable in which they are used. In our case, this is typically servoshell."""
+    for dependency_line in dependency_lines:
+        if is_macos_system_library(dependency_line) or dependency_line.startswith("@rpath/"):
+            continue
+
+        new_path = os.path.join("@executable_path", relative_path, os.path.basename(dependency_line))
+        arguments = ['install_name_tool', '-change', dependency_line, new_path, binary]
+        try:
+            subprocess.check_call(arguments)
+        except subprocess.CalledProcessError as exception:
+            print(f"{arguments} install_name_tool exited with return value {exception.returncode}")
+
+
+def make_rpath_path_absolute(dylib_path_from_otool: str, rpath: str):
+    """Given a dylib dependency from otool, resolve the path into a full path if it
+       contains `@rpath`."""
+    if not dylib_path_from_otool.startswith("@rpath/"):
+        return dylib_path_from_otool
+
+    # Not every dependency is in the same directory as the binary that is references. For
+    # instance, plugins dylibs can be found in "gstreamer-1.0".
+    path_relative_to_rpath = dylib_path_from_otool.replace('@rpath/', '')
+    for relative_directory in ["", "..", "gstreamer-1.0"]:
+        full_path = os.path.join(rpath, relative_directory, path_relative_to_rpath)
+        if os.path.exists(full_path):
+            return os.path.normpath(full_path)
+
+    raise Exception("Unable to satisfy rpath dependency: " + dylib_path_from_otool)
+
+
+def find_non_system_dependencies_with_otool(binary_path: str) -> Set[str]:
+    """Given a binary path, find all dylib dependency lines that do not refer to
+       system libraries."""
+    process = subprocess.Popen(['/usr/bin/otool', '-L', binary_path], stdout=subprocess.PIPE)
+    output = set()
+
+    for line in map(lambda line: line.decode('utf8'), process.stdout):
+        if not line.startswith("\t"):
+            continue
+        dependency = line.split(' ', 1)[0][1:]
+
+        # No need to do any processing for system libraries. They should be
+        # present on all macOS systems.
+        if not is_macos_system_library(dependency):
+            output.add(dependency)
+    return output
+
+
+def package_gstreamer_dylibs(binary_path: str, library_target_directory: str, cross_compilation_target: str = None):
+    """Copy all GStreamer dependencies to the "lib" subdirectory of a built version of
+       Servo. Also update any transitive shared library paths so that they are relative to
+       this subdirectory."""
+
+    # This import only works when called from `mach`.
+    import servo.platform
+
+    gstreamer_root = servo.platform.get().gstreamer_root(cross_compilation_target)
+    gstreamer_version = servo.platform.macos.GSTREAMER_PLUGIN_VERSION
+    gstreamer_root_libs = os.path.join(gstreamer_root, "lib")
+
+    # This is the relative path from the directory we are packaging the dylibs into and
+    # the binary we are packaging them for.
+    relative_path = os.path.relpath(library_target_directory, os.path.dirname(binary_path)) + "/"
+
+    # This might be None if we are cross-compiling.
+    if not gstreamer_root:
+        return True
+
+    # Detect when the packaged library versions do not reflect our current version of GStreamer,
+    # by writing a marker file with the packaged GStreamer version into the target directory.
+    marker_file = os.path.join(library_target_directory, f".gstreamer-{gstreamer_version}")
+
+    print()
+    if os.path.exists(library_target_directory) and os.path.exists(marker_file):
+        print(" • GStreamer packaging is up-to-date")
+        return True
+
+    if os.path.exists(library_target_directory):
+        print(" • Packaged GStreamer is out of date. Rebuilding into {library_target_directory}")
+        shutil.rmtree(library_target_directory)
+    else:
+        print(f" • Packaging GStreamer into {library_target_directory}")
+
+    os.makedirs(library_target_directory, exist_ok=True)
+    try:
+        # Collect all the initial binary dependencies for Servo and the plugins that it uses,
+        # which are loaded dynmically at runtime and don't appear in `otool` output.
+        binary_dependencies = set(find_non_system_dependencies_with_otool(binary_path))
+        binary_dependencies.update(
+            [os.path.join(gstreamer_root_libs, "gstreamer-1.0", plugin)
+                for plugin in macos_plugins()]
+        )
+
+        rewrite_dependencies_to_be_relative(binary_path, binary_dependencies, relative_path)
+
+        number_copied = 0
+        pending_to_be_copied = binary_dependencies
+        already_copied = set()
+
+        while pending_to_be_copied:
+            checking = set(pending_to_be_copied)
+            pending_to_be_copied.clear()
+
+            for otool_dependency in checking:
+                already_copied.add(otool_dependency)
+
+                original_dylib_path = make_rpath_path_absolute(otool_dependency, gstreamer_root_libs)
+                transitive_dependencies = set(find_non_system_dependencies_with_otool(original_dylib_path))
+
+                # First copy the dylib into the directory where we are collecting them all for
+                # packaging, and rewrite its dependencies to be relative to the executable we
+                # are packaging them for.
+                new_dylib_path = os.path.join(library_target_directory, os.path.basename(original_dylib_path))
+                if not os.path.exists(new_dylib_path):
+                    number_copied += 1
+                    shutil.copyfile(original_dylib_path, new_dylib_path)
+                    rewrite_dependencies_to_be_relative(new_dylib_path, transitive_dependencies, relative_path)
+
+                # Now queue up any transitive dependencies for processing in further iteration loops.
+                transitive_dependencies.difference_update(already_copied)
+                pending_to_be_copied.update(transitive_dependencies)
+
+    except Exception as exception:
+        print(f"ERROR: could not package required dylibs: {exception}")
+        raise exception
+
+    with open(marker_file, "w") as file:
+        file.write(gstreamer_version)
+
+    if number_copied:
+        print(f" • Processed {number_copied} GStreamer dylibs. ")
+        print("   This can cause the startup to be slow due to macOS security protections.")
+    return True
+
+
 if __name__ == "__main__":
     write_plugin_list(sys.argv[1])