diff options
author | Martin Robinson <mrobinson@igalia.com> | 2023-05-18 15:09:30 +0200 |
---|---|---|
committer | Martin Robinson <mrobinson@igalia.com> | 2023-05-19 13:08:43 +0200 |
commit | 5be14ecc3cc92309637604b48bdcf249b110ac16 (patch) | |
tree | ef5f13d8e79866eda1a84c847837938c884ebb2c /python/servo | |
parent | e09f85e17bd504a4c1c218ad14fa1a0ecbcaa839 (diff) | |
download | servo-5be14ecc3cc92309637604b48bdcf249b110ac16.tar.gz servo-5be14ecc3cc92309637604b48bdcf249b110ac16.zip |
Start organizing platform-specific Python code
This starts to split platform-specific Python code into its own module,
which should help to tidy up our mach commands and make things more
reusable.
Diffstat (limited to 'python/servo')
-rw-r--r-- | python/servo/bootstrap.py | 314 | ||||
-rw-r--r-- | python/servo/bootstrap_commands.py | 21 | ||||
-rw-r--r-- | python/servo/build_commands.py | 2 | ||||
-rw-r--r-- | python/servo/command_base.py | 37 | ||||
-rw-r--r-- | python/servo/packages.py | 14 | ||||
-rw-r--r-- | python/servo/platform/__init__.py | 64 | ||||
-rw-r--r-- | python/servo/platform/base.py | 34 | ||||
-rw-r--r-- | python/servo/platform/linux.py | 171 | ||||
-rw-r--r-- | python/servo/platform/macos.py | 36 | ||||
-rw-r--r-- | python/servo/platform/windows.py | 98 | ||||
-rw-r--r-- | python/servo/testing_commands.py | 2 | ||||
-rw-r--r-- | python/servo/util.py | 39 |
12 files changed, 442 insertions, 390 deletions
diff --git a/python/servo/bootstrap.py b/python/servo/bootstrap.py deleted file mode 100644 index 4a20564d1fb..00000000000 --- a/python/servo/bootstrap.py +++ /dev/null @@ -1,314 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -from __future__ import absolute_import, print_function - -from distutils.spawn import find_executable -from distutils.version import LooseVersion -import os -import distro -import subprocess -import six -import urllib - -from os import path -from subprocess import PIPE -from zipfile import BadZipfile - -import servo.packages as packages -from servo.util import extract, download_file, host_triple -from servo.gstreamer import macos_gst_root - - -def check_macos_gstreamer_lib(): - try: - env = os.environ.copy() - gst_root = macos_gst_root() - env["PATH"] = path.join(gst_root, "bin") - env["PKG_CONFIG_PATH"] = path.join(gst_root, "lib", "pkgconfig") - has_gst = subprocess.call(["pkg-config", "--atleast-version=1.21", "gstreamer-1.0"], - stdout=PIPE, stderr=PIPE, env=env) == 0 - gst_lib_dir = subprocess.check_output(["pkg-config", "--variable=libdir", "gstreamer-1.0"], - env=env) - return has_gst and gst_lib_dir.startswith(bytes(gst_root, 'utf-8')) - except FileNotFoundError: - return False - - -def check_gstreamer_lib(): - return subprocess.call(["pkg-config", "--atleast-version=1.16", "gstreamer-1.0"], - stdout=PIPE, stderr=PIPE) == 0 - - -def run_as_root(command, force=False): - if os.geteuid() != 0: - command.insert(0, 'sudo') - if force: - command.append('-y') - return subprocess.call(command) - - -def install_linux_deps(context, pkgs_ubuntu, pkgs_fedora, pkgs_void, force): - install = False - pkgs = [] - if context.distro in ['Ubuntu', 'Debian GNU/Linux']: - command = ['apt-get', 'install'] - pkgs = pkgs_ubuntu - if subprocess.call(['dpkg', '-s'] + pkgs, stdout=PIPE, stderr=PIPE) != 0: - install = True - elif context.distro in ['CentOS', 'CentOS Linux', 'Fedora', 'Fedora Linux']: - installed_pkgs = str(subprocess.check_output(['rpm', '-qa'])).replace('\n', '|') - pkgs = pkgs_fedora - for p in pkgs: - command = ['dnf', 'install'] - if "|{}".format(p) not in installed_pkgs: - install = True - break - elif context.distro == 'void': - installed_pkgs = str(subprocess.check_output(['xbps-query', '-l'])) - pkgs = pkgs_void - for p in pkgs: - command = ['xbps-install', '-A'] - if "ii {}-".format(p) not in installed_pkgs: - install = force = True - break - - if not install: - return False - - print("Installing missing dependencies...") - if run_as_root(command + pkgs, force) != 0: - raise Exception("Installation of dependencies failed.") - return True - - -def gstreamer(context, force=False): - cur = os.curdir - gstdir = os.path.join(cur, "support", "linux", "gstreamer") - if not os.path.isdir(os.path.join(gstdir, "gst", "lib")): - subprocess.check_call(["bash", "gstreamer.sh"], cwd=gstdir) - return True - return False - - -def bootstrap_gstreamer(context, force=False): - if not gstreamer(context, force): - print("gstreamer is already set up") - return 0 - - -def linux(context, force=False): - # Please keep these in sync with the packages in README.md - pkgs_apt = ['git', 'curl', 'autoconf', 'libx11-dev', 'libfreetype6-dev', - 'libgl1-mesa-dri', 'libglib2.0-dev', 'xorg-dev', 'gperf', 'g++', - 'build-essential', 'cmake', 'libssl-dev', - 'liblzma-dev', 'libxmu6', 'libxmu-dev', - "libxcb-render0-dev", "libxcb-shape0-dev", "libxcb-xfixes0-dev", - 'libgles2-mesa-dev', 'libegl1-mesa-dev', 'libdbus-1-dev', - 'libharfbuzz-dev', 'ccache', 'clang', 'libunwind-dev', - 'libgstreamer1.0-dev', 'libgstreamer-plugins-base1.0-dev', - 'libgstreamer-plugins-bad1.0-dev', 'autoconf2.13', - 'libunwind-dev', 'llvm-dev'] - pkgs_dnf = ['libtool', 'gcc-c++', 'libXi-devel', 'freetype-devel', - 'libunwind-devel', 'mesa-libGL-devel', 'mesa-libEGL-devel', - 'glib2-devel', 'libX11-devel', 'libXrandr-devel', 'gperf', - 'fontconfig-devel', 'cabextract', 'ttmkfdir', 'expat-devel', - 'rpm-build', 'openssl-devel', 'cmake', - 'libXcursor-devel', 'libXmu-devel', - 'dbus-devel', 'ncurses-devel', 'harfbuzz-devel', 'ccache', - 'clang', 'clang-libs', 'llvm', 'autoconf213', 'python3-devel', - 'gstreamer1-devel', 'gstreamer1-plugins-base-devel', - 'gstreamer1-plugins-bad-free-devel', 'libjpeg-turbo-devel', - 'zlib', 'libjpeg'] - pkgs_xbps = ['libtool', 'gcc', 'libXi-devel', 'freetype-devel', - 'libunwind-devel', 'MesaLib-devel', 'glib-devel', 'pkg-config', - 'libX11-devel', 'libXrandr-devel', 'gperf', 'bzip2-devel', - 'fontconfig-devel', 'cabextract', 'expat-devel', 'cmake', - 'cmake', 'libXcursor-devel', 'libXmu-devel', 'dbus-devel', - 'ncurses-devel', 'harfbuzz-devel', 'ccache', 'glu-devel', - 'clang', 'gstreamer1-devel', 'autoconf213', - 'gst-plugins-base1-devel', 'gst-plugins-bad1-devel'] - - installed_something = install_linux_deps(context, pkgs_apt, pkgs_dnf, - pkgs_xbps, force) - - if not check_gstreamer_lib(): - installed_something |= gstreamer(context, force) - - if not installed_something: - print("Dependencies were already installed!") - - return 0 - - -def windows_msvc(context, force=False): - '''Bootstrapper for MSVC building on Windows.''' - - deps_dir = os.path.join(context.sharedir, "msvc-dependencies") - deps_url = "https://github.com/servo/servo-build-deps/releases/download/msvc-deps/" - - def version(package): - return packages.WINDOWS_MSVC[package] - - def package_dir(package): - return os.path.join(deps_dir, package, version(package)) - - def check_cmake(version): - cmake_path = find_executable("cmake") - if cmake_path: - cmake = subprocess.Popen([cmake_path, "--version"], stdout=PIPE) - cmake_version_output = six.ensure_str(cmake.stdout.read()).splitlines()[0] - cmake_version = cmake_version_output.replace("cmake version ", "") - if LooseVersion(cmake_version) >= LooseVersion(version): - return True - return False - - def prepare_file(zip_path, full_spec): - if not os.path.isfile(zip_path): - zip_url = "{}{}.zip".format(deps_url, urllib.parse.quote(full_spec)) - download_file(full_spec, zip_url, zip_path) - - print("Extracting {}...".format(full_spec), end='') - try: - extract(zip_path, deps_dir) - except BadZipfile: - print("\nError: %s.zip is not a valid zip file, redownload..." % full_spec) - os.remove(zip_path) - prepare_file(zip_path, full_spec) - else: - print("done") - - to_install = {} - for package in packages.WINDOWS_MSVC: - # Don't install CMake if it already exists in PATH - if package == "cmake" and check_cmake(version("cmake")): - continue - - if not os.path.isdir(package_dir(package)): - to_install[package] = version(package) - - if not to_install: - return 0 - - print("Installing missing MSVC dependencies...") - for package in to_install: - full_spec = '{}-{}'.format(package, version(package)) - - parent_dir = os.path.dirname(package_dir(package)) - if not os.path.isdir(parent_dir): - os.makedirs(parent_dir) - - zip_path = package_dir(package) + ".zip" - prepare_file(zip_path, full_spec) - - extracted_path = os.path.join(deps_dir, full_spec) - os.rename(extracted_path, package_dir(package)) - - return 0 - - -LINUX_SPECIFIC_BOOTSTRAPPERS = { - "gstreamer": bootstrap_gstreamer, -} - - -def get_linux_distribution(): - distrib = six.ensure_str(distro.name()) - version = six.ensure_str(distro.version()) - - if distrib in ['LinuxMint', 'Linux Mint', 'KDE neon']: - if '.' in version: - major, _ = version.split('.', 1) - else: - major = version - - if major == '22': - base_version = '22.04' - elif major == '21': - base_version = '21.04' - elif major == '20': - base_version = '20.04' - elif major == '19': - base_version = '18.04' - elif major == '18': - base_version = '16.04' - else: - raise Exception('unsupported version of %s: %s' % (distrib, version)) - - distrib, version = 'Ubuntu', base_version - elif distrib == 'Pop!_OS': - if '.' in version: - major, _ = version.split('.', 1) - else: - major = version - - if major == '22': - base_version = '22.04' - elif major == '21': - base_version = '21.04' - elif major == '20': - base_version = '20.04' - elif major == '19': - base_version = '18.04' - elif major == '18': - base_version = '16.04' - else: - raise Exception('unsupported version of %s: %s' % (distrib, version)) - - distrib, version = 'Ubuntu', base_version - elif distrib.lower() == 'elementary': - if version == '5.0': - base_version = '18.04' - elif version[0:3] == '0.4': - base_version = '16.04' - else: - raise Exception('unsupported version of %s: %s' % (distrib, version)) - distrib, version = 'Ubuntu', base_version - elif distrib.lower() == 'ubuntu': - if version > '22.04': - print('WARNING: unsupported version of %s: %s' % (distrib, version)) - # Fixme: we should allow checked/supported versions only - elif distrib.lower() not in [ - 'centos', - 'centos linux', - 'debian gnu/linux', - 'fedora', - 'fedora linux', - 'void', - 'nixos', - 'arch', - 'arch linux', - ]: - raise Exception('mach bootstrap does not support %s, please file a bug' % distrib) - - return distrib, version - - -def bootstrap(context, force=False, specific=None): - '''Dispatches to the right bootstrapping function for the OS.''' - - bootstrapper = None - if "windows-msvc" in host_triple(): - bootstrapper = windows_msvc - elif "linux-gnu" in host_triple(): - distrib, version = get_linux_distribution() - - if distrib.lower() == 'nixos': - print('NixOS does not need bootstrap, it will automatically enter a nix-shell') - print('Just run ./mach build') - print('') - print('You will need to run a nix-shell if you are trying to run any of the built binaries') - print('To enter the nix-shell manually use:') - print(' $ nix-shell etc/shell.nix') - return - - context.distro = distrib - context.distro_version = version - bootstrapper = LINUX_SPECIFIC_BOOTSTRAPPERS.get(specific, linux) - - if bootstrapper is None: - print('Bootstrap support is not yet available for your OS.') - return 1 - - return bootstrapper(context, force=force) diff --git a/python/servo/bootstrap_commands.py b/python/servo/bootstrap_commands.py index aac03c8d645..f75be592e74 100644 --- a/python/servo/bootstrap_commands.py +++ b/python/servo/bootstrap_commands.py @@ -27,7 +27,8 @@ from mach.decorators import ( Command, ) -import servo.bootstrap as bootstrap +import servo.platform + from servo.command_base import CommandBase, cd, check_call from servo.util import delete, download_bytes, download_file, extract, check_hash @@ -41,10 +42,15 @@ class MachCommands(CommandBase): action='store_true', help='Boostrap without confirmation') def bootstrap(self, force=False): - # This entry point isn't actually invoked, ./mach bootstrap is directly - # called by mach (see mach_bootstrap.bootstrap_command_only) so that + # Note: This entry point isn't actually invoked by ./mach bootstrap. + # ./mach bootstrap calls mach_bootstrap.bootstrap_command_only so that # it can install dependencies without needing mach's dependencies - return bootstrap.bootstrap(self.context, force=force) + try: + servo.platform.get().bootstrap(self.context.sharedir, force) + except NotImplementedError as exception: + print(exception) + return 1 + return 0 @Command('bootstrap-gstreamer', description='Set up a local copy of the gstreamer libraries (linux only).', @@ -53,7 +59,12 @@ class MachCommands(CommandBase): action='store_true', help='Boostrap without confirmation') def bootstrap_gstreamer(self, force=False): - return bootstrap.bootstrap(self.context, force=force, specific="gstreamer") + try: + servo.platform.get().bootstrap_gstreamer(self.context.sharedir, force) + except NotImplementedError as exception: + print(exception) + return 1 + return 0 @Command('bootstrap-android', description='Install the Android SDK and NDK.', diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py index 6cd4bdc7c45..1f69637ad85 100644 --- a/python/servo/build_commands.py +++ b/python/servo/build_commands.py @@ -35,7 +35,7 @@ from mach.registrar import Registrar from mach_bootstrap import _get_exec_path from servo.command_base import CommandBase, cd, call, check_call, append_to_path_env, gstreamer_root from servo.gstreamer import windows_dlls, windows_plugins, macos_plugins -from servo.util import host_triple +from servo.platform import host_triple @CommandProvider diff --git a/python/servo/command_base.py b/python/servo/command_base.py index db2070f5b68..048286e7957 100644 --- a/python/servo/command_base.py +++ b/python/servo/command_base.py @@ -33,14 +33,12 @@ from os import path from subprocess import PIPE import toml +import servo.platform from xml.etree.ElementTree import XML -from servo.util import download_file -from .bootstrap import check_gstreamer_lib, check_macos_gstreamer_lib +from servo.util import download_file, get_default_cache_dir from mach.decorators import CommandArgument from mach.registrar import Registrar -from servo.packages import WINDOWS_MSVC as msvc_deps -from servo.util import host_triple from servo.gstreamer import macos_gst_root BIN_SUFFIX = ".exe" if sys.platform == "win32" else "" @@ -287,9 +285,7 @@ class CommandBase(object): # Handle missing/default items self.config.setdefault("tools", {}) - default_cache_dir = os.environ.get("SERVO_CACHE_DIR", - path.join(context.topdir, ".servo")) - self.config["tools"].setdefault("cache-dir", default_cache_dir) + self.config["tools"].setdefault("cache-dir", get_default_cache_dir(context.topdir)) resolverelative("tools", "cache-dir") default_cargo_home = os.environ.get("CARGO_HOME", @@ -545,23 +541,23 @@ class CommandBase(object): if "media-dummy" in features: return False + # MacOS always needs the GStreamer environment variable, but should + # also check that the Servo-specific version is downloaded and available. if is_macosx(): - if check_macos_gstreamer_lib(): - # We override homebrew gstreamer if installed and - # always use pkgconfig from official gstreamer framework + if servo.platform.get().is_gstreamer_installed(): return True else: raise Exception("Official GStreamer framework not found (we need at least 1.21)." "Please see installation instructions in README.md") try: - if check_gstreamer_lib(): + if servo.platform.get().is_gstreamer_installed(): return False except Exception: # Some systems don't have pkg-config; we can't probe in this case # and must hope for the best return False - effective_target = target or host_triple() + effective_target = target or servo.platform.host_triple() if "x86_64" not in effective_target or "android" in effective_target: # We don't build gstreamer for non-x86_64 / android yet return False @@ -581,7 +577,7 @@ install them, let us know by filing a bug!") """Some commands, like test-wpt, don't use a full build env, but may still need dynamic search paths. This command sets that up""" if not android and self.needs_gstreamer_env(None, os.environ): - gstpath = gstreamer_root(host_triple(), os.environ, self.get_top_dir()) + gstpath = gstreamer_root(servo.platform.host_triple(), os.environ, self.get_top_dir()) if gstpath is None: return os.environ["LD_LIBRARY_PATH"] = path.join(gstpath, "lib") @@ -590,10 +586,11 @@ install them, let us know by filing a bug!") os.environ["GST_PLUGIN_SCANNER"] = path.join(gstpath, "libexec", "gstreamer-1.0", "gst-plugin-scanner") def msvc_package_dir(self, package): - return path.join(self.context.sharedir, "msvc-dependencies", package, msvc_deps[package]) + return path.join(self.context.sharedir, "msvc-dependencies", package, + servo.platform.windows.DEPENDENCIES[package]) def vs_dirs(self): - assert 'windows' in host_triple() + assert 'windows' in servo.platform.host_triple() vsinstalldir = os.environ.get('VSINSTALLDIR') vs_version = os.environ.get('VisualStudioVersion') if vsinstalldir and vs_version: @@ -622,7 +619,7 @@ install them, let us know by filing a bug!") env['PATH'] = env['PATH'].encode('ascii', 'ignore') extra_path = [] extra_lib = [] - if "msvc" in (target or host_triple()): + if "msvc" in (target or servo.platform.host_triple()): extra_path += [path.join(self.msvc_package_dir("cmake"), "bin")] extra_path += [path.join(self.msvc_package_dir("llvm"), "bin")] extra_path += [path.join(self.msvc_package_dir("ninja"), "bin")] @@ -636,7 +633,7 @@ install them, let us know by filing a bug!") env["TARGET_CFLAGS"] += " -DWINAPI_FAMILY=WINAPI_FAMILY_APP" env["TARGET_CXXFLAGS"] += " -DWINAPI_FAMILY=WINAPI_FAMILY_APP" - arch = (target or host_triple()).split('-')[0] + arch = (target or servo.platform.host_triple()).split('-')[0] vcpkg_arch = { "x86_64": "x64-windows", "i686": "x86-windows", @@ -672,8 +669,8 @@ install them, let us know by filing a bug!") # Always build harfbuzz from source env["HARFBUZZ_SYS_NO_PKG_CONFIG"] = "true" - if is_build and self.needs_gstreamer_env(target or host_triple(), env, uwp, features): - gst_root = gstreamer_root(target or host_triple(), env, self.get_top_dir()) + if is_build and self.needs_gstreamer_env(target or servo.platform.host_triple(), env, uwp, features): + gst_root = gstreamer_root(target or servo.platform.host_triple(), env, self.get_top_dir()) bin_path = path.join(gst_root, "bin") lib_path = path.join(gst_root, "lib") pkg_config_path = path.join(lib_path, "pkgconfig") @@ -1018,7 +1015,7 @@ install them, let us know by filing a bug!") if self.context.bootstrapped: return - target_platform = target or host_triple() + target_platform = target or servo.platform.host_triple() # Always check if all needed MSVC dependencies are installed if "msvc" in target_platform: diff --git a/python/servo/packages.py b/python/servo/packages.py deleted file mode 100644 index 3b375eb2877..00000000000 --- a/python/servo/packages.py +++ /dev/null @@ -1,14 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at https://mozilla.org/MPL/2.0/. - -WINDOWS_MSVC = { - "cmake": "3.14.3", - "llvm": "15.0.5", - "moztools": "3.2", - "ninja": "1.7.1", - "nuget": "08-08-2019", - "openssl": "111.3.0+1.1.1c-vs2017-2019-09-18", - "gstreamer-uwp": "1.16.0.5", - "openxr-loader-uwp": "1.0", -} diff --git a/python/servo/platform/__init__.py b/python/servo/platform/__init__.py new file mode 100644 index 00000000000..84968550d00 --- /dev/null +++ b/python/servo/platform/__init__.py @@ -0,0 +1,64 @@ +# Copyright 2023 The Servo Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution. +# +# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import platform + +from .base import Base +from .windows import Windows + + +def host_platform(): + os_type = platform.system().lower() + if os_type == "linux": + os_type = "unknown-linux-gnu" + elif os_type == "darwin": + os_type = "apple-darwin" + elif os_type == "android": + os_type = "linux-androideabi" + elif os_type == "windows": + os_type = "pc-windows-msvc" + elif os_type == "freebsd": + os_type = "unknown-freebsd" + else: + os_type = "unknown" + return os_type + + +def host_triple(): + os_type = host_platform() + cpu_type = platform.machine().lower() + if cpu_type in ["i386", "i486", "i686", "i768", "x86"]: + cpu_type = "i686" + elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]: + cpu_type = "x86_64" + elif cpu_type == "arm": + cpu_type = "arm" + elif cpu_type == "aarch64": + cpu_type = "aarch64" + else: + cpu_type = "unknown" + return f"{cpu_type}-{os_type}" + + +def get(): + # We import the concrete platforms in if-statements here, because + # each one might have platform-specific imports which might not + # resolve on all platforms. + # TODO(mrobinson): We should do this for Windows too, once we + # stop relying on platform-specific code outside of this module. + # pylint: disable=import-outside-toplevel + if "windows-msvc" in host_triple(): + return Windows() + if "linux-gnu" in host_triple(): + from .linux import Linux + return Linux() + if "apple-darwin" in host_triple(): + from .macos import MacOS + return MacOS() + return Base() diff --git a/python/servo/platform/base.py b/python/servo/platform/base.py new file mode 100644 index 00000000000..316c998b1db --- /dev/null +++ b/python/servo/platform/base.py @@ -0,0 +1,34 @@ +# Copyright 2023 The Servo Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution. +# +# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import subprocess + + +class Base: + def _platform_bootstrap(self, _cache_dir: str, _force: bool) -> bool: + raise NotImplementedError("Bootstrap installation detection not yet available.") + + def _platform_bootstrap_gstreamer(self, _cache_dir: str, _force: bool) -> bool: + raise NotImplementedError("GStreamer bootstrap support is not yet available for your OS.") + + def _platform_is_gstreamer_installed(self) -> bool: + return subprocess.call( + ["pkg-config", "--atleast-version=1.16", "gstreamer-1.0"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0 + + def bootstrap(self, cache_dir: str, force: bool): + if not self._platform_bootstrap(cache_dir, force): + print("Dependencies were already installed!") + + def bootstrap_gstreamer(self, cache_dir: str, force: bool): + if not self._platform_bootstrap_gstreamer(cache_dir, force): + print("Dependencies were already installed!") + + def is_gstreamer_installed(self) -> bool: + return self._platform_is_gstreamer_installed() diff --git a/python/servo/platform/linux.py b/python/servo/platform/linux.py new file mode 100644 index 00000000000..da1a83a3b27 --- /dev/null +++ b/python/servo/platform/linux.py @@ -0,0 +1,171 @@ +# Copyright 2023 The Servo Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution. +# +# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import os +import subprocess + +from typing import Tuple + +import distro +import six + +from .base import Base + +# Please keep these in sync with the packages in README.md +APT_PKGS = ['git', 'curl', 'autoconf', 'libx11-dev', 'libfreetype6-dev', + 'libgl1-mesa-dri', 'libglib2.0-dev', 'xorg-dev', 'gperf', 'g++', + 'build-essential', 'cmake', 'libssl-dev', + 'liblzma-dev', 'libxmu6', 'libxmu-dev', + "libxcb-render0-dev", "libxcb-shape0-dev", "libxcb-xfixes0-dev", + 'libgles2-mesa-dev', 'libegl1-mesa-dev', 'libdbus-1-dev', + 'libharfbuzz-dev', 'ccache', 'clang', 'libunwind-dev', + 'libgstreamer1.0-dev', 'libgstreamer-plugins-base1.0-dev', + 'libgstreamer-plugins-bad1.0-dev', 'autoconf2.13', + 'libunwind-dev', 'llvm-dev'] +DNF_PKGS = ['libtool', 'gcc-c++', 'libXi-devel', 'freetype-devel', + 'libunwind-devel', 'mesa-libGL-devel', 'mesa-libEGL-devel', + 'glib2-devel', 'libX11-devel', 'libXrandr-devel', 'gperf', + 'fontconfig-devel', 'cabextract', 'ttmkfdir', 'expat-devel', + 'rpm-build', 'openssl-devel', 'cmake', + 'libXcursor-devel', 'libXmu-devel', + 'dbus-devel', 'ncurses-devel', 'harfbuzz-devel', 'ccache', + 'clang', 'clang-libs', 'llvm', 'autoconf213', 'python3-devel', + 'gstreamer1-devel', 'gstreamer1-plugins-base-devel', + 'gstreamer1-plugins-bad-free-devel', 'libjpeg-turbo-devel', + 'zlib', 'libjpeg'] +XBPS_PKGS = ['libtool', 'gcc', 'libXi-devel', 'freetype-devel', + 'libunwind-devel', 'MesaLib-devel', 'glib-devel', 'pkg-config', + 'libX11-devel', 'libXrandr-devel', 'gperf', 'bzip2-devel', + 'fontconfig-devel', 'cabextract', 'expat-devel', 'cmake', + 'cmake', 'libXcursor-devel', 'libXmu-devel', 'dbus-devel', + 'ncurses-devel', 'harfbuzz-devel', 'ccache', 'glu-devel', + 'clang', 'gstreamer1-devel', 'autoconf213', + 'gst-plugins-base1-devel', 'gst-plugins-bad1-devel'] + + +class Linux(Base): + def __init__(self): + (self.distro, self.version) = Linux.get_distro_and_version() + + @staticmethod + def get_distro_and_version() -> Tuple[str, str]: + distrib = six.ensure_str(distro.name()) + version = six.ensure_str(distro.version()) + + if distrib in ['LinuxMint', 'Linux Mint', 'KDE neon', 'Pop!_OS']: + if '.' in version: + major, _ = version.split('.', 1) + else: + major = version + + distrib = 'Ubuntu' + if major == '22': + version = '22.04' + elif major == '21': + version = '21.04' + elif major == '20': + version = '20.04' + elif major == '19': + version = '18.04' + elif major == '18': + version = '16.04' + + if distrib.lower() == 'elementary': + distrib = 'Ubuntu' + if version == '5.0': + version = '18.04' + elif version[0:3] == '0.4': + version = '16.04' + + return (distrib, version) + + def _platform_bootstrap(self, _cache_dir: str, force: bool) -> bool: + if self.distro.lower() == 'nixos': + print('NixOS does not need bootstrap, it will automatically enter a nix-shell') + print('Just run ./mach build') + print('') + print('You will need to run a nix-shell if you are trying ' + 'to run any of the built binaries') + print('To enter the nix-shell manually use:') + print(' $ nix-shell etc/shell.nix') + return False + + if self.distro.lower() == 'ubuntu' and self.version > '22.04': + print(f"WARNING: unsupported version of {self.distro}: {self.version}") + + # FIXME: Better version checking for these distributions. + if self.distro.lower() not in [ + 'arch linux', + 'arch', + 'centos linux', + 'centos', + 'debian gnu/linux', + 'fedora linux', + 'fedora', + 'nixos', + 'ubuntu', + 'void', + ]: + raise NotImplementedError("mach bootstrap does not support " + f"{self.distro}, please file a bug") + + installed_something = self.install_non_gstreamer_dependencies(force) + installed_something |= self._platform_bootstrap_gstreamer(_cache_dir, force) + return installed_something + + def install_non_gstreamer_dependencies(self, force: bool) -> bool: + install = False + pkgs = [] + if self.distro in ['Ubuntu', 'Debian GNU/Linux']: + command = ['apt-get', 'install'] + pkgs = APT_PKGS + if subprocess.call(['dpkg', '-s'] + pkgs, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) != 0: + install = True + elif self.distro in ['CentOS', 'CentOS Linux', 'Fedora', 'Fedora Linux']: + installed_pkgs = str(subprocess.check_output(['rpm', '-qa'])).replace('\n', '|') + pkgs = DNF_PKGS + for pkg in pkgs: + command = ['dnf', 'install'] + if "|{}".format(pkg) not in installed_pkgs: + install = True + break + elif self.distro == 'void': + installed_pkgs = str(subprocess.check_output(['xbps-query', '-l'])) + pkgs = XBPS_PKGS + for pkg in pkgs: + command = ['xbps-install', '-A'] + if "ii {}-".format(pkg) not in installed_pkgs: + install = force = True + break + + if not install: + return False + + def run_as_root(command, force=False): + if os.geteuid() != 0: + command.insert(0, 'sudo') + if force: + command.append('-y') + return subprocess.call(command) + + print("Installing missing dependencies...") + if run_as_root(command + pkgs, force) != 0: + raise Exception("Installation of dependencies failed.") + return True + + def _platform_bootstrap_gstreamer(self, _cache_dir: str, _force: bool) -> bool: + if self.is_gstreamer_installed(): + return False + + gstdir = os.path.join(os.curdir, "support", "linux", "gstreamer") + if not os.path.isdir(os.path.join(gstdir, "gst", "lib")): + subprocess.check_call(["bash", "gstreamer.sh"], cwd=gstdir) + return True + return False diff --git a/python/servo/platform/macos.py b/python/servo/platform/macos.py new file mode 100644 index 00000000000..6189140cabf --- /dev/null +++ b/python/servo/platform/macos.py @@ -0,0 +1,36 @@ +# Copyright 2023 The Servo Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution. +# +# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import os +import subprocess + +from .base import Base +from ..gstreamer import macos_gst_root + + +class MacOS(Base): + def __init__(self): + pass + + def _platform_is_gstreamer_installed(self) -> bool: + # We override homebrew gstreamer if installed and always use pkgconfig + # from official gstreamer framework. + try: + gst_root = macos_gst_root() + env = os.environ.copy() + env["PATH"] = os.path.join(gst_root, "bin") + env["PKG_CONFIG_PATH"] = os.path.join(gst_root, "lib", "pkgconfig") + has_gst = subprocess.call( + ["pkg-config", "--atleast-version=1.21", "gstreamer-1.0"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) == 0 + gst_lib_dir = subprocess.check_output( + ["pkg-config", "--variable=libdir", "gstreamer-1.0"], env=env) + return has_gst and gst_lib_dir.startswith(bytes(gst_root, 'utf-8')) + except FileNotFoundError: + return False diff --git a/python/servo/platform/windows.py b/python/servo/platform/windows.py new file mode 100644 index 00000000000..cea8de5a5dd --- /dev/null +++ b/python/servo/platform/windows.py @@ -0,0 +1,98 @@ +# Copyright 2023 The Servo Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution. +# +# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import os +import shutil +import subprocess +import urllib +import zipfile + +from distutils.version import LooseVersion + +import six +from .base import Base +from ..util import extract, download_file + +DEPS_URL = "https://github.com/servo/servo-build-deps/releases/download/msvc-deps/" +DEPENDENCIES = { + "cmake": "3.14.3", + "llvm": "15.0.5", + "moztools": "3.2", + "ninja": "1.7.1", + "nuget": "08-08-2019", + "openssl": "111.3.0+1.1.1c-vs2017-2019-09-18", + "gstreamer-uwp": "1.16.0.5", + "openxr-loader-uwp": "1.0", +} + + +class Windows(Base): + def __init__(self): + pass + + @staticmethod + def cmake_already_installed(required_version: str) -> bool: + cmake_path = shutil.which("cmake") + if not cmake_path: + return False + + output = subprocess.check_output([cmake_path, "--version"]) + cmake_version_output = six.ensure_str(output).splitlines()[0] + installed_version = cmake_version_output.replace("cmake version ", "") + return LooseVersion(installed_version) >= LooseVersion(required_version) + + @classmethod + def prepare_file(cls, deps_dir: str, zip_path: str, full_spec: str): + if not os.path.isfile(zip_path): + zip_url = "{}{}.zip".format(DEPS_URL, urllib.parse.quote(full_spec)) + download_file(full_spec, zip_url, zip_path) + + print("Extracting {}...".format(full_spec), end='') + try: + extract(zip_path, deps_dir) + except zipfile.BadZipfile: + print("\nError: %s.zip is not a valid zip file, redownload..." % full_spec) + os.remove(zip_path) + cls.prepare_file(deps_dir, zip_path, full_spec) + else: + print("done") + + def _platform_bootstrap(self, cache_dir: str, _force: bool = False) -> bool: + deps_dir = os.path.join(cache_dir, "msvc-dependencies") + + def get_package_dir(package, version) -> str: + return os.path.join(deps_dir, package, version) + + to_install = {} + for (package, version) in DEPENDENCIES.items(): + # Don't install CMake if it already exists in PATH + if package == "cmake" and self.cmake_already_installed(version): + continue + + if not os.path.isdir(get_package_dir(package, version)): + to_install[package] = version + + if not to_install: + return False + + print("Installing missing MSVC dependencies...") + for (package, version) in to_install.items(): + full_spec = '{}-{}'.format(package, version) + + package_dir = get_package_dir(package, version) + parent_dir = os.path.dirname(package_dir) + if not os.path.isdir(parent_dir): + os.makedirs(parent_dir) + + self.prepare_file(deps_dir, package_dir + ".zip", full_spec) + + extracted_path = os.path.join(deps_dir, full_spec) + os.rename(extracted_path, package_dir) + + return True diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index 9209e9240d8..41a577cadc2 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -40,7 +40,7 @@ from servo.command_base import ( call, check_call, check_output, ) from servo_tidy_tests import test_tidy -from servo.util import host_triple +from servo.platform import host_triple SCRIPT_PATH = os.path.split(__file__)[0] PROJECT_TOPLEVEL_PATH = os.path.abspath(os.path.join(SCRIPT_PATH, "..", "..")) diff --git a/python/servo/util.py b/python/servo/util.py index c41cb1e1640..4151bbd4929 100644 --- a/python/servo/util.py +++ b/python/servo/util.py @@ -12,7 +12,6 @@ from __future__ import absolute_import, print_function, unicode_literals import hashlib import os import os.path -import platform import shutil import stat import sys @@ -59,40 +58,6 @@ def delete(path): os.remove(path) -def host_platform(): - os_type = platform.system().lower() - if os_type == "linux": - os_type = "unknown-linux-gnu" - elif os_type == "darwin": - os_type = "apple-darwin" - elif os_type == "android": - os_type = "linux-androideabi" - elif os_type == "windows": - os_type = "pc-windows-msvc" - elif os_type == "freebsd": - os_type = "unknown-freebsd" - else: - os_type = "unknown" - return os_type - - -def host_triple(): - os_type = host_platform() - cpu_type = platform.machine().lower() - if cpu_type in ["i386", "i486", "i686", "i768", "x86"]: - cpu_type = "i686" - elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]: - cpu_type = "x86_64" - elif cpu_type == "arm": - cpu_type = "arm" - elif cpu_type == "aarch64": - cpu_type = "aarch64" - else: - cpu_type = "unknown" - - return "{}-{}".format(cpu_type, os_type) - - def download(desc, src, writer, start_byte=0): if start_byte: print("Resuming download of {} ...".format(src)) @@ -229,3 +194,7 @@ def check_hash(filename, expected, algorithm): if hasher.hexdigest() != expected: print("Incorrect {} hash for {}".format(algorithm, filename)) sys.exit(1) + + +def get_default_cache_dir(topdir): + return os.environ.get("SERVO_CACHE_DIR", os.path.join(topdir, ".servo")) |