diff options
Diffstat (limited to 'python/servo')
| -rw-r--r-- | python/servo/build_commands.py | 107 | ||||
| -rw-r--r-- | python/servo/gstreamer.py | 156 | ||||
| -rw-r--r-- | python/servo/package_commands.py | 8 | ||||
| -rw-r--r-- | python/servo/platform/macos.py | 2 |
4 files changed, 160 insertions, 113 deletions
diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py index 93a432041df..61cede7e7b3 100644 --- a/python/servo/build_commands.py +++ b/python/servo/build_commands.py @@ -31,11 +31,12 @@ from mach.decorators import ( from mach.registrar import Registrar import servo.platform +import servo.platform.macos import servo.util import servo.visual_studio from servo.command_base import BuildType, CommandBase, call, check_call -from servo.gstreamer import windows_dlls, windows_plugins, macos_plugins +from servo.gstreamer import windows_dlls, windows_plugins, package_gstreamer_dylibs SUPPORTED_ASAN_TARGETS = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu"] @@ -154,8 +155,10 @@ class MachCommands(CommandBase): assert os.path.exists(servo_bin_dir) if self.enable_media: - print("Packaging gstreamer dylibs") - if not package_gstreamer_dylibs(self.cross_compile_target, built_binary): + library_target_directory = path.join(path.dirname(built_binary), "lib/") + if not package_gstreamer_dylibs(built_binary, + library_target_directory, + self.cross_compile_target): return 1 # On the Mac, set a lovely icon. This makes it easier to pick out the Servo binary in tools @@ -293,104 +296,6 @@ class MachCommands(CommandBase): print(f"[Warning] Could not generate notification: {e}", file=sys.stderr) -def otool(s): - o = subprocess.Popen(['/usr/bin/otool', '-L', s], stdout=subprocess.PIPE) - for line in map(lambda s: s.decode('ascii'), o.stdout): - if line[0] == '\t': - yield line.split(' ', 1)[0][1:] - - -def install_name_tool(binary, *args): - try: - subprocess.check_call(['install_name_tool', *args, binary]) - except subprocess.CalledProcessError as e: - print("install_name_tool exited with return value %d" % e.returncode) - - -def change_link_name(binary, old, new): - install_name_tool(binary, '-change', old, f"@executable_path/{new}") - - -def is_system_library(lib): - return lib.startswith("/System/Library") or lib.startswith("/usr/lib") or ".asan." in lib - - -def is_relocatable_library(lib): - return lib.startswith("@rpath/") - - -def change_non_system_libraries_path(libraries, relative_path, binary): - for lib in libraries: - if is_system_library(lib) or is_relocatable_library(lib): - continue - new_path = path.join(relative_path, path.basename(lib)) - change_link_name(binary, lib, new_path) - - -def resolve_rpath(lib, rpath_root): - if not is_relocatable_library(lib): - return lib - - rpaths = ['', '../', 'gstreamer-1.0/'] - for rpath in rpaths: - full_path = rpath_root + lib.replace('@rpath/', rpath) - if path.exists(full_path): - return path.normpath(full_path) - - raise Exception("Unable to satisfy rpath dependency: " + lib) - - -def copy_dependencies(binary_path, lib_path, gst_lib_dir): - relative_path = path.relpath(lib_path, path.dirname(binary_path)) + "/" - - # Update binary libraries - binary_dependencies = set(otool(binary_path)) - change_non_system_libraries_path(binary_dependencies, relative_path, binary_path) - - plugins = [os.path.join(gst_lib_dir, "gstreamer-1.0", plugin) for plugin in macos_plugins()] - binary_dependencies = binary_dependencies.union(plugins) - - # Update dependencies libraries - need_checked = binary_dependencies - checked = set() - while need_checked: - checking = set(need_checked) - need_checked = set() - for f in checking: - # No need to check these for their dylibs - if is_system_library(f): - continue - full_path = resolve_rpath(f, gst_lib_dir) - need_relinked = set(otool(full_path)) - new_path = path.join(lib_path, path.basename(full_path)) - if not path.exists(new_path): - shutil.copyfile(full_path, new_path) - change_non_system_libraries_path(need_relinked, relative_path, new_path) - need_checked.update(need_relinked) - checked.update(checking) - need_checked.difference_update(checked) - - -def package_gstreamer_dylibs(cross_compilation_target, servo_bin): - gst_root = servo.platform.get().gstreamer_root(cross_compilation_target) - - # This might be None if we are cross-compiling. - if not gst_root: - return True - - lib_dir = path.join(path.dirname(servo_bin), "lib") - if os.path.exists(lib_dir): - shutil.rmtree(lib_dir) - os.mkdir(lib_dir) - try: - copy_dependencies(servo_bin, lib_dir, path.join(gst_root, 'lib', '')) - except Exception as e: - print("ERROR: could not package required dylibs") - print(e) - return False - return True - - def copy_windows_dlls_to_build_directory(servo_binary: str, target_triple: str) -> bool: servo_exe_dir = os.path.dirname(servo_binary) assert os.path.exists(servo_exe_dir) diff --git a/python/servo/gstreamer.py b/python/servo/gstreamer.py index 3c63b7dbd5d..b7e68a0772f 100644 --- a/python/servo/gstreamer.py +++ b/python/servo/gstreamer.py @@ -7,8 +7,11 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -import os +import os.path +import shutil +import subprocess import sys +from typing import Set GSTREAMER_BASE_LIBS = [ # gstreamer @@ -157,11 +160,6 @@ def windows_plugins(): return [f"{lib}.dll" for lib in libs] -def macos_gst_root(): - return os.path.join( - "/", "Library", "Frameworks", "GStreamer.framework", "Versions", "1.0") - - def macos_plugins(): plugins = [ *GSTREAMER_PLUGIN_LIBS, @@ -185,5 +183,151 @@ pub(crate) static GSTREAMER_PLUGINS: &[&'static str] = &[ ''' % ',\n'.join(map(lambda x: '"' + x + '"', plugins))) +def is_macos_system_library(library_path: str) -> bool: + """Returns true if if the given dependency line from otool refers to + a system library that should not be packaged.""" + return (library_path.startswith("/System/Library") + or library_path.startswith("/usr/lib") + or ".asan." in library_path) + + +def rewrite_dependencies_to_be_relative(binary: str, dependency_lines: Set[str], relative_path: str): + """Given a path to a binary (either an executable or a dylib), rewrite the + the given dependency lines to be found at the given relative path to + the executable in which they are used. In our case, this is typically servoshell.""" + for dependency_line in dependency_lines: + if is_macos_system_library(dependency_line) or dependency_line.startswith("@rpath/"): + continue + + new_path = os.path.join("@executable_path", relative_path, os.path.basename(dependency_line)) + arguments = ['install_name_tool', '-change', dependency_line, new_path, binary] + try: + subprocess.check_call(arguments) + except subprocess.CalledProcessError as exception: + print(f"{arguments} install_name_tool exited with return value {exception.returncode}") + + +def make_rpath_path_absolute(dylib_path_from_otool: str, rpath: str): + """Given a dylib dependency from otool, resolve the path into a full path if it + contains `@rpath`.""" + if not dylib_path_from_otool.startswith("@rpath/"): + return dylib_path_from_otool + + # Not every dependency is in the same directory as the binary that is references. For + # instance, plugins dylibs can be found in "gstreamer-1.0". + path_relative_to_rpath = dylib_path_from_otool.replace('@rpath/', '') + for relative_directory in ["", "..", "gstreamer-1.0"]: + full_path = os.path.join(rpath, relative_directory, path_relative_to_rpath) + if os.path.exists(full_path): + return os.path.normpath(full_path) + + raise Exception("Unable to satisfy rpath dependency: " + dylib_path_from_otool) + + +def find_non_system_dependencies_with_otool(binary_path: str) -> Set[str]: + """Given a binary path, find all dylib dependency lines that do not refer to + system libraries.""" + process = subprocess.Popen(['/usr/bin/otool', '-L', binary_path], stdout=subprocess.PIPE) + output = set() + + for line in map(lambda line: line.decode('utf8'), process.stdout): + if not line.startswith("\t"): + continue + dependency = line.split(' ', 1)[0][1:] + + # No need to do any processing for system libraries. They should be + # present on all macOS systems. + if not is_macos_system_library(dependency): + output.add(dependency) + return output + + +def package_gstreamer_dylibs(binary_path: str, library_target_directory: str, cross_compilation_target: str = None): + """Copy all GStreamer dependencies to the "lib" subdirectory of a built version of + Servo. Also update any transitive shared library paths so that they are relative to + this subdirectory.""" + + # This import only works when called from `mach`. + import servo.platform + + gstreamer_root = servo.platform.get().gstreamer_root(cross_compilation_target) + gstreamer_version = servo.platform.macos.GSTREAMER_PLUGIN_VERSION + gstreamer_root_libs = os.path.join(gstreamer_root, "lib") + + # This is the relative path from the directory we are packaging the dylibs into and + # the binary we are packaging them for. + relative_path = os.path.relpath(library_target_directory, os.path.dirname(binary_path)) + "/" + + # This might be None if we are cross-compiling. + if not gstreamer_root: + return True + + # Detect when the packaged library versions do not reflect our current version of GStreamer, + # by writing a marker file with the packaged GStreamer version into the target directory. + marker_file = os.path.join(library_target_directory, f".gstreamer-{gstreamer_version}") + + print() + if os.path.exists(library_target_directory) and os.path.exists(marker_file): + print(" • GStreamer packaging is up-to-date") + return True + + if os.path.exists(library_target_directory): + print(" • Packaged GStreamer is out of date. Rebuilding into {library_target_directory}") + shutil.rmtree(library_target_directory) + else: + print(f" • Packaging GStreamer into {library_target_directory}") + + os.makedirs(library_target_directory, exist_ok=True) + try: + # Collect all the initial binary dependencies for Servo and the plugins that it uses, + # which are loaded dynmically at runtime and don't appear in `otool` output. + binary_dependencies = set(find_non_system_dependencies_with_otool(binary_path)) + binary_dependencies.update( + [os.path.join(gstreamer_root_libs, "gstreamer-1.0", plugin) + for plugin in macos_plugins()] + ) + + rewrite_dependencies_to_be_relative(binary_path, binary_dependencies, relative_path) + + number_copied = 0 + pending_to_be_copied = binary_dependencies + already_copied = set() + + while pending_to_be_copied: + checking = set(pending_to_be_copied) + pending_to_be_copied.clear() + + for otool_dependency in checking: + already_copied.add(otool_dependency) + + original_dylib_path = make_rpath_path_absolute(otool_dependency, gstreamer_root_libs) + transitive_dependencies = set(find_non_system_dependencies_with_otool(original_dylib_path)) + + # First copy the dylib into the directory where we are collecting them all for + # packaging, and rewrite its dependencies to be relative to the executable we + # are packaging them for. + new_dylib_path = os.path.join(library_target_directory, os.path.basename(original_dylib_path)) + if not os.path.exists(new_dylib_path): + number_copied += 1 + shutil.copyfile(original_dylib_path, new_dylib_path) + rewrite_dependencies_to_be_relative(new_dylib_path, transitive_dependencies, relative_path) + + # Now queue up any transitive dependencies for processing in further iteration loops. + transitive_dependencies.difference_update(already_copied) + pending_to_be_copied.update(transitive_dependencies) + + except Exception as exception: + print(f"ERROR: could not package required dylibs: {exception}") + raise exception + + with open(marker_file, "w") as file: + file.write(gstreamer_version) + + if number_copied: + print(f" • Processed {number_copied} GStreamer dylibs. ") + print(" This can cause the startup to be slow due to macOS security protections.") + return True + + if __name__ == "__main__": write_plugin_list(sys.argv[1]) diff --git a/python/servo/package_commands.py b/python/servo/package_commands.py index a9470b7b372..284f2df86ef 100644 --- a/python/servo/package_commands.py +++ b/python/servo/package_commands.py @@ -22,6 +22,7 @@ import shutil import subprocess import sys +import servo.gstreamer from mach.decorators import ( CommandArgument, CommandProvider, @@ -38,8 +39,6 @@ from servo.command_base import ( is_macosx, is_windows, ) -from servo.build_commands import copy_dependencies -from servo.gstreamer import macos_gst_root from servo.util import delete, get_target_dir PACKAGES = { @@ -218,10 +217,9 @@ class PackageCommands(CommandBase): change_prefs(dir_to_resources, "macosx") - print("Finding dylibs and relinking") + print("Packaging GStreamer...") dmg_binary = path.join(content_dir, "servo") - dir_to_gst_lib = path.join(macos_gst_root(), 'lib', '') - copy_dependencies(dmg_binary, lib_dir, dir_to_gst_lib) + servo.gstreamer.package_gstreamer_dylibs(dmg_binary, lib_dir) print("Adding version to Credits.rtf") version_command = [binary_path, '--version'] diff --git a/python/servo/platform/macos.py b/python/servo/platform/macos.py index eeb2ee26c1c..de20fe35ad5 100644 --- a/python/servo/platform/macos.py +++ b/python/servo/platform/macos.py @@ -16,6 +16,7 @@ from .. import util from .base import Base URL_BASE = "https://github.com/servo/servo-build-deps/releases/download/macOS" +GSTREAMER_PLUGIN_VERSION = "1.22.3" GSTREAMER_URL = f"{URL_BASE}/gstreamer-1.0-1.22.3-universal.pkg" GSTREAMER_DEVEL_URL = f"{URL_BASE}/gstreamer-1.0-devel-1.22.3-universal.pkg" GSTREAMER_ROOT = "/Library/Frameworks/GStreamer.framework/Versions/1.0" @@ -34,7 +35,6 @@ class MacOS(Base): def is_gstreamer_installed(self, cross_compilation_target: Optional[str]) -> bool: # Servo only supports the official GStreamer distribution on MacOS. - # Make sure we use the right `pkg-config`. return not cross_compilation_target and os.path.exists(GSTREAMER_ROOT) def _platform_bootstrap(self, _force: bool) -> bool: |