diff options
| -rw-r--r-- | .github/workflows/mac.yml | 13 | ||||
| -rw-r--r-- | README.md | 3 | ||||
| -rw-r--r-- | components/servo/lib.rs | 5 | ||||
| -rw-r--r-- | etc/homebrew/Brewfile | 10 | ||||
| -rw-r--r-- | etc/homebrew/Brewfile-build | 12 | ||||
| -rwxr-xr-x | etc/install_macos_gstreamer.sh | 19 | ||||
| -rw-r--r-- | python/servo/bootstrap.py | 18 | ||||
| -rw-r--r-- | python/servo/build_commands.py | 131 | ||||
| -rw-r--r-- | python/servo/command_base.py | 30 | ||||
| -rw-r--r-- | python/servo/gstreamer.py | 20 | ||||
| -rw-r--r-- | python/servo/package_commands.py | 81 |
11 files changed, 198 insertions, 144 deletions
diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index bff564716d7..09ac5ab9fbf 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -66,18 +66,11 @@ jobs: - name: Bootstrap run: | python3 -m pip install --upgrade pip virtualenv - brew bundle install --verbose --no-upgrade --file=etc/homebrew/Brewfile - brew bundle install --verbose --no-upgrade --file=etc/homebrew/Brewfile-build - rm -rf /usr/local/etc/openssl - rm -rf /usr/local/etc/openssl@1.1 - brew install openssl@1.1 gnu-tar + bash etc/install_macos_gstreamer.sh + brew install gnu-tar - name: Release build run: | - export OPENSSL_INCLUDE_DIR="$(brew --prefix openssl)/include" - export OPENSSL_LIB_DIR="$(brew --prefix openssl)/lib" - export PKG_CONFIG_PATH="$(brew --prefix libffi)/lib/pkgconfig/" - export PKG_CONFIG_PATH="$(brew --prefix zlib)/lib/pkgconfig/:$PKG_CONFIG_PATH" - python3 ./mach build --release --media-stack=dummy --with-${{ env.LAYOUT }} + python3 ./mach build --release --with-${{ env.LAYOUT }} - name: Smoketest run: python3 ./mach smoketest - name: Unit tests diff --git a/README.md b/README.md index 771ae02f3fd..5e85abaf920 100644 --- a/README.md +++ b/README.md @@ -58,8 +58,7 @@ NOTE: run these steps after you've cloned the project locally. ``` sh cd servo -brew bundle install --file=etc/homebrew/Brewfile -brew bundle install --file=etc/homebrew/Brewfile-build +bash etc/install_macos_gstreamer.sh pip install virtualenv ``` diff --git a/components/servo/lib.rs b/components/servo/lib.rs index 131a53e656c..65b84721006 100644 --- a/components/servo/lib.rs +++ b/components/servo/lib.rs @@ -177,6 +177,11 @@ mod media_platform { } else { let mut plugin_dir = std::env::current_exe().unwrap(); plugin_dir.pop(); + + if cfg!(target_os = "macos") { + plugin_dir.push("lib"); + } + plugin_dir }; diff --git a/etc/homebrew/Brewfile b/etc/homebrew/Brewfile deleted file mode 100644 index c11e49b6c30..00000000000 --- a/etc/homebrew/Brewfile +++ /dev/null @@ -1,10 +0,0 @@ -# Runtime dependencies - -brew "gnutls" -brew "gstreamer" -brew "gst-plugins-base" -brew "gst-libav" -brew "gst-plugins-bad" -brew "gst-plugins-good" -brew "gst-rtsp-server" -brew "openssl" diff --git a/etc/homebrew/Brewfile-build b/etc/homebrew/Brewfile-build deleted file mode 100644 index c78fe2cea3c..00000000000 --- a/etc/homebrew/Brewfile-build +++ /dev/null @@ -1,12 +0,0 @@ -# Build dependencies (that are not also runtime dependencies) - -brew "autoconf@2.13" -brew "automake" -brew "cmake" -brew "pkg-config" -brew "llvm" -brew "yasm" -brew "zlib" - -# For sccache -brew "openssl@1.1" diff --git a/etc/install_macos_gstreamer.sh b/etc/install_macos_gstreamer.sh new file mode 100755 index 00000000000..874b94852c1 --- /dev/null +++ b/etc/install_macos_gstreamer.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +set -o errexit +set -o nounset +set -o pipefail + +VERSION=1.22.2 +URL_BASE=https://github.com/servo/servo-build-deps/releases/download/macOS + +cd /tmp +curl -L "${URL_BASE}/gstreamer-1.0-${VERSION}-universal.pkg" -o gstreamer.pkg +curl -L "${URL_BASE}/gstreamer-1.0-devel-${VERSION}-universal.pkg" \ + -o gstreamer-dev.pkg +sudo installer -pkg 'gstreamer.pkg' -target / +sudo installer -pkg 'gstreamer-dev.pkg' -target / diff --git a/python/servo/bootstrap.py b/python/servo/bootstrap.py index 3e1ec06f53e..4a20564d1fb 100644 --- a/python/servo/bootstrap.py +++ b/python/servo/bootstrap.py @@ -11,11 +11,29 @@ import distro import subprocess import six import urllib + +from os import path from subprocess import PIPE from zipfile import BadZipfile import servo.packages as packages from servo.util import extract, download_file, host_triple +from servo.gstreamer import macos_gst_root + + +def check_macos_gstreamer_lib(): + try: + env = os.environ.copy() + gst_root = macos_gst_root() + env["PATH"] = path.join(gst_root, "bin") + env["PKG_CONFIG_PATH"] = path.join(gst_root, "lib", "pkgconfig") + has_gst = subprocess.call(["pkg-config", "--atleast-version=1.21", "gstreamer-1.0"], + stdout=PIPE, stderr=PIPE, env=env) == 0 + gst_lib_dir = subprocess.check_output(["pkg-config", "--variable=libdir", "gstreamer-1.0"], + env=env) + return has_gst and gst_lib_dir.startswith(bytes(gst_root, 'utf-8')) + except FileNotFoundError: + return False def check_gstreamer_lib(): diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py index 135a48ef7ed..f5c6e2aa749 100644 --- a/python/servo/build_commands.py +++ b/python/servo/build_commands.py @@ -34,7 +34,7 @@ from mach.registrar import Registrar from mach_bootstrap import _get_exec_path from servo.command_base import CommandBase, cd, call, check_call, append_to_path_env, gstreamer_root -from servo.gstreamer import windows_dlls, windows_plugins, macos_dylibs, macos_plugins +from servo.gstreamer import windows_dlls, windows_plugins, macos_plugins from servo.util import host_triple @@ -625,13 +625,20 @@ class MachCommands(CommandBase): status = 1 elif sys.platform == "darwin": - servo_exe_dir = os.path.dirname( - self.get_binary_path(release, dev, target=target, simpleservo=libsimpleservo) - ) - assert os.path.exists(servo_exe_dir) + servo_path = self.get_binary_path(release, dev, target=target, simpleservo=libsimpleservo) + servo_bin_dir = os.path.dirname(servo_path) + assert os.path.exists(servo_bin_dir) - if has_media_stack and not package_gstreamer_dylibs(servo_exe_dir): - return 1 + if has_media_stack: + gst_root = gstreamer_root(target, env) + if not package_gstreamer_dylibs(gst_root, servo_path): + return 1 + + # On Mac we use the relocatable dylibs from offical gstreamer + # .pkg distribution. We need to add an LC_RPATH to the servo binary + # to allow the dynamic linker to be able to locate these dylibs + # See `man dyld` for more info + add_rpath_to_binary(servo_path, "@executable_path/lib/") # On the Mac, set a lovely icon. This makes it easier to pick out the Servo binary in tools # like Instruments.app. @@ -791,22 +798,102 @@ def angle_root(target, nuget_env): return angle_default_path -def package_gstreamer_dylibs(servo_exe_dir): - missing = [] - gst_dylibs = macos_dylibs() + macos_plugins() - for gst_lib in gst_dylibs: - try: - dest_path = os.path.join(servo_exe_dir, os.path.basename(gst_lib)) - if os.path.isfile(dest_path): - os.remove(dest_path) - shutil.copy(gst_lib, servo_exe_dir) - except Exception as e: - print(e) - missing += [str(gst_lib)] +def otool(s): + o = subprocess.Popen(['/usr/bin/otool', '-L', s], stdout=subprocess.PIPE) + for line in map(lambda s: s.decode('ascii'), o.stdout): + if line[0] == '\t': + yield line.split(' ', 1)[0][1:] - for gst_lib in missing: - print("ERROR: could not find required GStreamer DLL: " + gst_lib) - return not missing + +def install_name_tool(binary, *args): + try: + subprocess.check_call(['install_name_tool', *args, binary]) + except subprocess.CalledProcessError as e: + print("install_name_tool exited with return value %d" % e.returncode) + + +def change_link_name(binary, old, new): + install_name_tool(binary, '-change', old, f"@executable_path/{new}") + + +def add_rpath_to_binary(binary, relative_path): + install_name_tool(binary, "-add_rpath", relative_path) + + +def change_rpath_in_binary(binary, old, new): + install_name_tool(binary, "-rpath", old, new) + + +def is_system_library(lib): + return lib.startswith("/System/Library") or lib.startswith("/usr/lib") + + +def is_relocatable_library(lib): + return lib.startswith("@rpath/") + + +def change_non_system_libraries_path(libraries, relative_path, binary): + for lib in libraries: + if is_system_library(lib) or is_relocatable_library(lib): + continue + new_path = path.join(relative_path, path.basename(lib)) + change_link_name(binary, lib, new_path) + + +def resolve_rpath(lib, rpath_root): + if not is_relocatable_library(lib): + return lib + + rpaths = ['', '../', 'gstreamer-1.0/'] + for rpath in rpaths: + full_path = rpath_root + lib.replace('@rpath/', rpath) + if path.exists(full_path): + return path.normpath(full_path) + + raise Exception("Unable to satisfy rpath dependency: " + lib) + + +def copy_dependencies(binary_path, lib_path, gst_root): + relative_path = path.relpath(lib_path, path.dirname(binary_path)) + "/" + + # Update binary libraries + binary_dependencies = set(otool(binary_path)) + binary_dependencies = binary_dependencies.union(macos_plugins()) + change_non_system_libraries_path(binary_dependencies, relative_path, binary_path) + + # Update dependencies libraries + need_checked = binary_dependencies + checked = set() + while need_checked: + checking = set(need_checked) + need_checked = set() + for f in checking: + # No need to check these for their dylibs + if is_system_library(f): + continue + full_path = resolve_rpath(f, gst_root) + need_relinked = set(otool(full_path)) + new_path = path.join(lib_path, path.basename(full_path)) + if not path.exists(new_path): + shutil.copyfile(full_path, new_path) + change_non_system_libraries_path(need_relinked, relative_path, new_path) + need_checked.update(need_relinked) + checked.update(checking) + need_checked.difference_update(checked) + + +def package_gstreamer_dylibs(gst_root, servo_bin): + lib_dir = path.join(path.dirname(servo_bin), "lib") + if os.path.exists(lib_dir): + shutil.rmtree(lib_dir) + os.mkdir(lib_dir) + try: + copy_dependencies(servo_bin, lib_dir, path.join(gst_root, 'lib', '')) + except Exception as e: + print("ERROR: could not package required dylibs") + print(e) + return False + return True def package_gstreamer_dlls(env, servo_exe_dir, target, uwp): diff --git a/python/servo/command_base.py b/python/servo/command_base.py index 76c32bbd85c..db2070f5b68 100644 --- a/python/servo/command_base.py +++ b/python/servo/command_base.py @@ -36,11 +36,12 @@ import toml from xml.etree.ElementTree import XML from servo.util import download_file -from .bootstrap import check_gstreamer_lib +from .bootstrap import check_gstreamer_lib, check_macos_gstreamer_lib from mach.decorators import CommandArgument from mach.registrar import Registrar from servo.packages import WINDOWS_MSVC as msvc_deps from servo.util import host_triple +from servo.gstreamer import macos_gst_root BIN_SUFFIX = ".exe" if sys.platform == "win32" else "" NIGHTLY_REPOSITORY_URL = "https://servo-builds2.s3.amazonaws.com/" @@ -240,6 +241,8 @@ def gstreamer_root(target, env, topdir=None): return gst_default_path elif is_linux(): return path.join(topdir, "support", "linux", "gstreamer", "gst") + elif is_macosx(): + return macos_gst_root() return None @@ -541,6 +544,16 @@ class CommandBase(object): return False if "media-dummy" in features: return False + + if is_macosx(): + if check_macos_gstreamer_lib(): + # We override homebrew gstreamer if installed and + # always use pkgconfig from official gstreamer framework + return True + else: + raise Exception("Official GStreamer framework not found (we need at least 1.21)." + "Please see installation instructions in README.md") + try: if check_gstreamer_lib(): return False @@ -660,14 +673,17 @@ install them, let us know by filing a bug!") env["HARFBUZZ_SYS_NO_PKG_CONFIG"] = "true" if is_build and self.needs_gstreamer_env(target or host_triple(), env, uwp, features): - gstpath = gstreamer_root(target or host_triple(), env, self.get_top_dir()) - extra_path += [path.join(gstpath, "bin")] - libpath = path.join(gstpath, "lib") + gst_root = gstreamer_root(target or host_triple(), env, self.get_top_dir()) + bin_path = path.join(gst_root, "bin") + lib_path = path.join(gst_root, "lib") + pkg_config_path = path.join(lib_path, "pkgconfig") # we append in the reverse order so that system gstreamer libraries # do not get precedence - extra_path = [libpath] + extra_path - extra_lib = [libpath] + extra_lib - append_to_path_env(path.join(libpath, "pkgconfig"), env, "PKG_CONFIG_PATH") + extra_path = [bin_path] + extra_path + extra_lib = [lib_path] + extra_lib + append_to_path_env(pkg_config_path, env, "PKG_CONFIG_PATH") + if is_macosx(): + env["OPENSSL_INCLUDE_DIR"] = path.join(gst_root, "Headers") if is_linux(): distrib, version, _ = distro.linux_distribution() diff --git a/python/servo/gstreamer.py b/python/servo/gstreamer.py index 1b7739d0f62..42da291e9f0 100644 --- a/python/servo/gstreamer.py +++ b/python/servo/gstreamer.py @@ -9,7 +9,6 @@ import os import sys -import platform GSTREAMER_DYLIBS = [ # gstreamer @@ -121,20 +120,9 @@ def windows_plugins(uwp): return [f"{lib}.dll" for lib in libs] -def macos_lib_dir(): - # homebrew use /opt/homebrew on macos ARM, use /usr/local on Intel - if platform.machine() == 'arm64': - return os.path.join('/', 'opt', 'homebrew', 'lib') - return os.path.join('/', 'usr', 'local', 'lib') - - -def macos_dylibs(): - dylibs = [ - *[f"lib{lib}-1.0.0.dylib" for lib in GSTREAMER_DYLIBS], - "libnice.dylib", - "libnice.10.dylib", - ] - return [os.path.join(macos_lib_dir(), lib) for lib in dylibs] +def macos_gst_root(): + return os.path.join( + "/", "Library", "Frameworks", "GStreamer.framework", "Versions", "1.0") def macos_plugins(): @@ -148,7 +136,7 @@ def macos_plugins(): ] def plugin_path(plugin): - return os.path.join(macos_lib_dir(), 'gstreamer-1.0', f"lib{plugin}.dylib") + return os.path.join(macos_gst_root(), 'lib', 'gstreamer-1.0', f"lib{plugin}.dylib") # These plugins depend on the particular version of GStreamer that is installed # on the system that is building servo. diff --git a/python/servo/package_commands.py b/python/servo/package_commands.py index 4f19c8dea19..1f4e87f9830 100644 --- a/python/servo/package_commands.py +++ b/python/servo/package_commands.py @@ -40,7 +40,8 @@ from servo.command_base import ( is_macosx, is_windows, ) -from servo.gstreamer import macos_dylibs, macos_plugins +from servo.build_commands import copy_dependencies, change_rpath_in_binary +from servo.gstreamer import macos_gst_root from servo.util import delete # Note: mako cannot be imported at the top level because it breaks mach bootstrap @@ -99,66 +100,11 @@ else: raise e -def otool(s): - o = subprocess.Popen(['/usr/bin/otool', '-L', s], stdout=subprocess.PIPE) - for line in map(lambda s: s.decode('ascii'), o.stdout): - if line[0] == '\t': - yield line.split(' ', 1)[0][1:] - - def listfiles(directory): return [f for f in os.listdir(directory) if path.isfile(path.join(directory, f))] -def install_name_tool(old, new, binary): - try: - subprocess.check_call(['install_name_tool', '-change', old, '@executable_path/' + new, binary]) - except subprocess.CalledProcessError as e: - print("install_name_tool exited with return value %d" % e.returncode) - - -def is_system_library(lib): - return lib.startswith("/System/Library") or lib.startswith("/usr/lib") - - -def change_non_system_libraries_path(libraries, relative_path, binary): - for lib in libraries: - if is_system_library(lib): - continue - new_path = path.join(relative_path, path.basename(lib)) - install_name_tool(lib, new_path, binary) - - -def copy_dependencies(binary_path, lib_path): - relative_path = path.relpath(lib_path, path.dirname(binary_path)) + "/" - - # Update binary libraries - binary_dependencies = set(otool(binary_path)) - binary_dependencies = binary_dependencies.union(macos_dylibs()) - binary_dependencies = binary_dependencies.union(macos_plugins()) - change_non_system_libraries_path(binary_dependencies, relative_path, binary_path) - - # Update dependencies libraries - need_checked = binary_dependencies - checked = set() - while need_checked: - checking = set(need_checked) - need_checked = set() - for f in checking: - # No need to check these for their dylibs - if is_system_library(f): - continue - need_relinked = set(otool(f)) - new_path = path.join(lib_path, path.basename(f)) - if not path.exists(new_path): - shutil.copyfile(f, new_path) - change_non_system_libraries_path(need_relinked, relative_path, new_path) - need_checked.update(need_relinked) - checked.update(checking) - need_checked.difference_update(checked) - - def copy_windows_dependencies(binary_path, destination): for f in os.listdir(binary_path): if os.path.isfile(path.join(binary_path, f)) and f.endswith(".dll"): @@ -336,15 +282,16 @@ class PackageCommands(CommandBase): shutil.copy2(path.join(dir_to_root, 'Info.plist'), path.join(dir_to_app, 'Contents', 'Info.plist')) content_dir = path.join(dir_to_app, 'Contents', 'MacOS') - os.makedirs(content_dir) + lib_dir = path.join(content_dir, 'lib') + os.makedirs(lib_dir) shutil.copy2(binary_path, content_dir) change_prefs(dir_to_resources, "macosx") print("Finding dylibs and relinking") - # TODO(mrobinson): GStreamer dependencies don't need to be packaged - # with servo until the media backend is re-enabled. - # copy_dependencies(path.join(content_dir, 'servo'), content_dir) + dmg_binary = path.join(content_dir, "servo") + dir_to_gst_lib = path.join(macos_gst_root(), 'lib', '') + copy_dependencies(dmg_binary, lib_dir, dir_to_gst_lib) print("Adding version to Credits.rtf") version_command = [binary_path, '--version'] @@ -401,13 +348,17 @@ class PackageCommands(CommandBase): os.remove(tar_path) shutil.copytree(path.join(dir_to_root, 'resources'), path.join(dir_to_brew, 'resources')) os.makedirs(path.join(dir_to_brew, 'bin')) - shutil.copy2(binary_path, path.join(dir_to_brew, 'bin', 'servo')) # Note that in the context of Homebrew, libexec is reserved for private use by the formula # and therefore is not symlinked into HOMEBREW_PREFIX. - os.makedirs(path.join(dir_to_brew, 'libexec')) - # TODO(mrobinson): GStreamer dependencies don't need to be packaged - # with servo until the media backend is re-enabled. - # copy_dependencies(path.join(dir_to_brew, 'bin', 'servo'), path.join(dir_to_brew, 'libexec')) + # The 'lib' sub-directory within 'libexec' is necessary to satisfy + # rpath relative install_names in the gstreamer packages + brew_servo_bin = path.join(dir_to_brew, 'bin', 'servo') + shutil.copy2(binary_path, brew_servo_bin) + change_rpath_in_binary( + brew_servo_bin, '@executable_path/lib/', '@executable_path/../libexec/lib/') + dir_to_lib = path.join(dir_to_brew, 'libexec', 'lib') + os.makedirs(dir_to_lib) + copy_dependencies(brew_servo_bin, dir_to_lib, dir_to_gst_lib) archive_deterministically(dir_to_brew, tar_path, prepend_path='servo/') delete(dir_to_brew) print("Packaged Servo into " + tar_path) |