aboutsummaryrefslogtreecommitdiffstats
path: root/python/servo
diff options
context:
space:
mode:
Diffstat (limited to 'python/servo')
-rw-r--r--python/servo/build_commands.py32
-rw-r--r--python/servo/command_base.py127
-rw-r--r--python/servo/devenv_commands.py9
-rw-r--r--python/servo/platform/__init__.py26
-rw-r--r--python/servo/platform/base.py100
-rw-r--r--python/servo/platform/linux.py53
-rw-r--r--python/servo/platform/macos.py86
-rw-r--r--python/servo/platform/windows.py97
-rw-r--r--python/servo/post_build_commands.py21
-rw-r--r--python/servo/testing_commands.py16
-rw-r--r--python/servo/util.py66
11 files changed, 400 insertions, 233 deletions
diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py
index c2e3459cfbd..5c016b82200 100644
--- a/python/servo/build_commands.py
+++ b/python/servo/build_commands.py
@@ -24,6 +24,8 @@ import zipfile
from time import time
import notifypy
+import servo.platform
+import servo.util
from mach.decorators import (
CommandArgument,
@@ -33,9 +35,8 @@ from mach.decorators import (
from mach.registrar import Registrar
from mach_bootstrap import _get_exec_path
-from servo.command_base import CommandBase, cd, call, check_call, append_to_path_env, gstreamer_root
+from servo.command_base import CommandBase, cd, call, check_call
from servo.gstreamer import windows_dlls, windows_plugins, macos_plugins
-from servo.platform import host_triple
@CommandProvider
@@ -92,7 +93,7 @@ class MachCommands(CommandBase):
features += media_stack
has_media_stack = media_stack[0] == "media-gstreamer"
- target_path = base_path = self.get_target_dir()
+ target_path = base_path = servo.util.get_target_dir()
if android:
target_path = path.join(target_path, "android")
base_path = path.join(target_path, target)
@@ -141,8 +142,8 @@ class MachCommands(CommandBase):
build_start = time()
env["CARGO_TARGET_DIR"] = target_path
- host = host_triple()
- target_triple = target or host_triple()
+ host = servo.platform.host_triple()
+ target_triple = target or servo.platform.host_triple()
if 'apple-darwin' in host and target_triple == host:
if 'CXXFLAGS' not in env:
env['CXXFLAGS'] = ''
@@ -191,7 +192,7 @@ class MachCommands(CommandBase):
# Ensure that the NuGet ANGLE package containing libEGL is accessible
# to the Rust linker.
- append_to_path_env(angle_root(target_triple, env), env, "LIB")
+ servo.util.append_paths_to_env(env, "LIB", angle_root(target_triple, env))
# Don't want to mix non-UWP libraries with vendored UWP libraries.
if "gstreamer" in env['LIB']:
@@ -224,12 +225,6 @@ class MachCommands(CommandBase):
print(stderr.decode(encoding))
exit(1)
- # Ensure that GStreamer libraries are accessible when linking.
- if 'windows' in target_triple:
- gst_root = gstreamer_root(target_triple, env)
- if gst_root:
- append_to_path_env(os.path.join(gst_root, "lib"), env, "LIB")
-
if android:
if "ANDROID_NDK" not in env:
print("Please set the ANDROID_NDK environment variable.")
@@ -511,9 +506,8 @@ class MachCommands(CommandBase):
assert os.path.exists(servo_bin_dir)
if has_media_stack:
- gst_root = gstreamer_root(target, env)
print("Packaging gstreamer dylibs")
- if not package_gstreamer_dylibs(gst_root, servo_path):
+ if not package_gstreamer_dylibs(target, servo_path):
return 1
# On Mac we use the relocatable dylibs from offical gstreamer
@@ -776,7 +770,13 @@ def copy_dependencies(binary_path, lib_path, gst_root):
need_checked.difference_update(checked)
-def package_gstreamer_dylibs(gst_root, servo_bin):
+def package_gstreamer_dylibs(cross_compilation_target, servo_bin):
+ gst_root = servo.platform.get().gstreamer_root(cross_compilation_target)
+
+ # This might be None if we are cross-compiling.
+ if not gst_root:
+ return True
+
lib_dir = path.join(path.dirname(servo_bin), "lib")
if os.path.exists(lib_dir):
shutil.rmtree(lib_dir)
@@ -791,7 +791,7 @@ def package_gstreamer_dylibs(gst_root, servo_bin):
def package_gstreamer_dlls(env, servo_exe_dir, target, uwp):
- gst_root = gstreamer_root(target, env)
+ gst_root = servo.platform.get().gstreamer_root(cross_compilation_target=target)
if not gst_root:
print("Could not find GStreamer installation directory.")
return False
diff --git a/python/servo/command_base.py b/python/servo/command_base.py
index e7536ad4e30..e6209b0cf23 100644
--- a/python/servo/command_base.py
+++ b/python/servo/command_base.py
@@ -34,12 +34,12 @@ from subprocess import PIPE
import toml
import servo.platform
+import servo.util as util
from xml.etree.ElementTree import XML
from servo.util import download_file, get_default_cache_dir
from mach.decorators import CommandArgument
from mach.registrar import Registrar
-from servo.gstreamer import macos_gst_root
BIN_SUFFIX = ".exe" if sys.platform == "win32" else ""
NIGHTLY_REPOSITORY_URL = "https://servo-builds2.s3.amazonaws.com/"
@@ -213,37 +213,6 @@ def is_linux():
return sys.platform.startswith('linux')
-def append_to_path_env(string, env, name):
- variable = ""
- if name in env:
- variable = six.ensure_str(env[name])
- if len(variable) > 0:
- variable += os.pathsep
- variable += string
- env[name] = variable
-
-
-def gstreamer_root(target, env, topdir=None):
- if is_windows():
- arch = {
- "x86_64": "X86_64",
- "x86": "X86",
- "aarch64": "ARM64",
- }
- gst_x64 = arch[target.split('-')[0]]
- gst_default_path = path.join("C:\\gstreamer\\1.0", gst_x64)
- gst_env = "GSTREAMER_1_0_ROOT_" + gst_x64
- if env.get(gst_env) is not None:
- return env.get(gst_env)
- elif os.path.exists(path.join(gst_default_path, "bin", "ffi-7.dll")):
- return gst_default_path
- elif is_linux():
- return path.join(topdir, "support", "linux", "gstreamer", "gst")
- elif is_macosx():
- return macos_gst_root()
- return None
-
-
class BuildNotFound(Exception):
def __init__(self, message):
self.message = message
@@ -343,14 +312,8 @@ class CommandBase(object):
def get_top_dir(self):
return self.context.topdir
- def get_target_dir(self):
- if "CARGO_TARGET_DIR" in os.environ:
- return os.environ["CARGO_TARGET_DIR"]
- else:
- return path.join(self.context.topdir, "target")
-
def get_apk_path(self, release):
- base_path = self.get_target_dir()
+ base_path = util.get_target_dir()
base_path = path.join(base_path, "android", self.config["android"]["target"])
apk_name = "servoapp.apk"
build_type = "release" if release else "debug"
@@ -359,7 +322,7 @@ class CommandBase(object):
def get_binary_path(self, release, dev, target=None, android=False, simpleservo=False):
# TODO(autrilla): this function could still use work - it shouldn't
# handle quitting, or printing. It should return the path, or an error.
- base_path = self.get_target_dir()
+ base_path = util.get_target_dir()
binary_name = "servo" + BIN_SUFFIX
@@ -532,56 +495,6 @@ class CommandBase(object):
return self.get_executable(destination_folder)
- def needs_gstreamer_env(self, target, env, uwp=False, features=[]):
- if uwp:
- return False
- if "media-dummy" in features:
- return False
-
- # MacOS always needs the GStreamer environment variable, but should
- # also check that the Servo-specific version is downloaded and available.
- if is_macosx():
- if servo.platform.get().is_gstreamer_installed():
- return True
- else:
- raise Exception("Official GStreamer framework not found (we need at least 1.21)."
- "Please see installation instructions in README.md")
-
- try:
- if servo.platform.get().is_gstreamer_installed():
- return False
- except Exception:
- # Some systems don't have pkg-config; we can't probe in this case
- # and must hope for the best
- return False
- effective_target = target or servo.platform.host_triple()
- if "x86_64" not in effective_target or "android" in effective_target:
- # We don't build gstreamer for non-x86_64 / android yet
- return False
- if is_linux() or is_windows():
- if path.isdir(gstreamer_root(effective_target, env, self.get_top_dir())):
- return True
- else:
- raise Exception("Your system's gstreamer libraries are out of date \
-(we need at least 1.16). Please run ./mach bootstrap-gstreamer")
- else:
- raise Exception("Your system's gstreamer libraries are out of date \
-(we need at least 1.16). If you're unable to \
-install them, let us know by filing a bug!")
- return False
-
- def set_run_env(self, android=False):
- """Some commands, like test-wpt, don't use a full build env,
- but may still need dynamic search paths. This command sets that up"""
- if not android and self.needs_gstreamer_env(None, os.environ):
- gstpath = gstreamer_root(servo.platform.host_triple(), os.environ, self.get_top_dir())
- if gstpath is None:
- return
- os.environ["LD_LIBRARY_PATH"] = path.join(gstpath, "lib")
- os.environ["GST_PLUGIN_SYSTEM_PATH"] = path.join(gstpath, "lib", "gstreamer-1.0")
- os.environ["PKG_CONFIG_PATH"] = path.join(gstpath, "lib", "pkgconfig")
- os.environ["GST_PLUGIN_SCANNER"] = path.join(gstpath, "libexec", "gstreamer-1.0", "gst-plugin-scanner")
-
def msvc_package_dir(self, package):
return path.join(self.context.sharedir, "msvc-dependencies", package,
servo.platform.windows.DEPENDENCIES[package])
@@ -606,6 +519,10 @@ install them, let us know by filing a bug!")
def build_env(self, hosts_file_path=None, target=None, is_build=False, test_unit=False, uwp=False, features=None):
"""Return an extended environment dictionary."""
env = os.environ.copy()
+
+ if not features or "media-dummy" not in features:
+ servo.platform.get().set_gstreamer_environment_variables_if_necessary(env, cross_compilation_target=target)
+
if sys.platform == "win32" and type(env['PATH']) == six.text_type:
# On win32, the virtualenv's activate_this.py script sometimes ends up
# turning os.environ['PATH'] into a unicode string. This doesn't work
@@ -615,7 +532,6 @@ install them, let us know by filing a bug!")
# it in any case.
env['PATH'] = env['PATH'].encode('ascii', 'ignore')
extra_path = []
- extra_lib = []
if "msvc" in (target or servo.platform.host_triple()):
extra_path += [path.join(self.msvc_package_dir("cmake"), "bin")]
extra_path += [path.join(self.msvc_package_dir("llvm"), "bin")]
@@ -666,19 +582,6 @@ install them, let us know by filing a bug!")
# Always build harfbuzz from source
env["HARFBUZZ_SYS_NO_PKG_CONFIG"] = "true"
- if is_build and self.needs_gstreamer_env(target or servo.platform.host_triple(), env, uwp, features):
- gst_root = gstreamer_root(target or servo.platform.host_triple(), env, self.get_top_dir())
- bin_path = path.join(gst_root, "bin")
- lib_path = path.join(gst_root, "lib")
- pkg_config_path = path.join(lib_path, "pkgconfig")
- # we append in the reverse order so that system gstreamer libraries
- # do not get precedence
- extra_path = [bin_path] + extra_path
- extra_lib = [lib_path] + extra_lib
- append_to_path_env(pkg_config_path, env, "PKG_CONFIG_PATH")
- if is_macosx():
- env["OPENSSL_INCLUDE_DIR"] = path.join(gst_root, "Headers")
-
if is_linux():
distrib, version, _ = distro.linux_distribution()
distrib = six.ensure_str(distrib)
@@ -687,17 +590,13 @@ install them, let us know by filing a bug!")
env["HARFBUZZ_SYS_NO_PKG_CONFIG"] = "true"
if extra_path:
- append_to_path_env(os.pathsep.join(extra_path), env, "PATH")
+ util.append_paths_to_env(env, "PATH", extra_path)
if self.config["build"]["incremental"]:
env["CARGO_INCREMENTAL"] = "1"
elif self.config["build"]["incremental"] is not None:
env["CARGO_INCREMENTAL"] = "0"
- if extra_lib:
- path_var = "DYLD_LIBRARY_PATH" if sys.platform == "darwin" else "LD_LIBRARY_PATH"
- append_to_path_env(os.pathsep.join(extra_lib), env, path_var)
-
# Paths to Android build tools:
if self.config["android"]["sdk"]:
env["ANDROID_SDK"] = self.config["android"]["sdk"]
@@ -737,6 +636,14 @@ install them, let us know by filing a bug!")
# This wrapper script is in bash and doesn't work on Windows
# where we want to run doctests as part of `./mach test-unit`
env['RUSTDOC'] = path.join(self.context.topdir, 'etc', 'rustdoc-with-private')
+ elif "msvc" in servo.platform.host_triple():
+ # on MSVC, we need some DLLs in the path. They were copied
+ # in to the servo.exe build dir, so just point PATH to that.
+ util.prepend_paths_to_env(env, "PATH", path.dirname(self.get_binary_path(False, False)))
+
+ # FIXME: https://github.com/servo/servo/issues/26192
+ if test_unit and "apple-darwin" not in servo.platform.host_triple():
+ env["RUST_BACKTRACE"] = "1"
if self.config["build"]["rustflags"]:
env['RUSTFLAGS'] = env.get('RUSTFLAGS', "") + " " + self.config["build"]["rustflags"]
@@ -1067,7 +974,7 @@ install them, let us know by filing a bug!")
def ensure_clobbered(self, target_dir=None):
if target_dir is None:
- target_dir = self.get_target_dir()
+ target_dir = util.get_target_dir()
auto = True if os.environ.get('AUTOCLOBBER', False) else False
src_clobber = os.path.join(self.context.topdir, 'CLOBBER')
target_clobber = os.path.join(target_dir, 'CLOBBER')
diff --git a/python/servo/devenv_commands.py b/python/servo/devenv_commands.py
index f69a1c8231f..77b95ba743f 100644
--- a/python/servo/devenv_commands.py
+++ b/python/servo/devenv_commands.py
@@ -48,9 +48,8 @@ class MachCommands(CommandBase):
self.ensure_bootstrapped(target=target)
self.ensure_clobbered()
- env = self.build_env()
- status = self.run_cargo_build_like_command("check", params, env=env, features=features, **kwargs)
+ status = self.run_cargo_build_like_command("check", params, features=features, **kwargs)
if status == 0:
print('Finished checking, binary NOT updated. Consider ./mach build before ./mach run')
@@ -142,9 +141,8 @@ class MachCommands(CommandBase):
self.ensure_bootstrapped(target=target)
self.ensure_clobbered()
- env = self.build_env()
- return self.run_cargo_build_like_command("fix", params, env=env, features=features, **kwargs)
+ return self.run_cargo_build_like_command("fix", params, features=features, **kwargs)
@Command('cargo-clippy',
description='Run "cargo clippy"',
@@ -166,9 +164,8 @@ class MachCommands(CommandBase):
self.ensure_bootstrapped(target=target)
self.ensure_clobbered()
- env = self.build_env()
- return self.run_cargo_build_like_command("clippy", params, env=env, features=features, **kwargs)
+ return self.run_cargo_build_like_command("clippy", params, features=features, **kwargs)
@Command('grep',
description='`git grep` for selected directories.',
diff --git a/python/servo/platform/__init__.py b/python/servo/platform/__init__.py
index 84968550d00..e06bebaf863 100644
--- a/python/servo/platform/__init__.py
+++ b/python/servo/platform/__init__.py
@@ -9,9 +9,10 @@
import platform
-from .base import Base
from .windows import Windows
+__platform__ = None
+
def host_platform():
os_type = platform.system().lower()
@@ -47,18 +48,27 @@ def host_triple():
def get():
+ # pylint: disable=global-statement
+ global __platform__
+ if __platform__:
+ return __platform__
+
# We import the concrete platforms in if-statements here, because
# each one might have platform-specific imports which might not
# resolve on all platforms.
# TODO(mrobinson): We should do this for Windows too, once we
# stop relying on platform-specific code outside of this module.
# pylint: disable=import-outside-toplevel
- if "windows-msvc" in host_triple():
- return Windows()
- if "linux-gnu" in host_triple():
+ triple = host_triple()
+ if "windows-msvc" in triple:
+ __platform__ = Windows(triple)
+ elif "linux-gnu" in triple:
from .linux import Linux
- return Linux()
- if "apple-darwin" in host_triple():
+ __platform__ = Linux(triple)
+ elif "apple-darwin" in triple:
from .macos import MacOS
- return MacOS()
- return Base()
+ __platform__ = MacOS(triple)
+ else:
+ from .base import Base
+ __platform__ = Base(triple)
+ return __platform__
diff --git a/python/servo/platform/base.py b/python/servo/platform/base.py
index 316c998b1db..b0ac64386bd 100644
--- a/python/servo/platform/base.py
+++ b/python/servo/platform/base.py
@@ -7,28 +7,104 @@
# option. This file may not be copied, modified, or distributed
# except according to those terms.
+import os
import subprocess
+from typing import Dict, Optional
+
+from .. import util
class Base:
+ def __init__(self, triple: str):
+ self.environ = os.environ.copy()
+ self.triple = triple
+ self.is_windows = False
+ self.is_linux = False
+ self.is_macos = False
+
+ def set_gstreamer_environment_variables_if_necessary(
+ self, env: Dict[str, str], cross_compilation_target: Optional[str], check_installation=True
+ ):
+ # Environment variables are not needed when cross-compiling on any
+ # platform other than Windows. UWP doesn't support GStreamer. GStreamer
+ # for Android is handled elsewhere.
+ if cross_compilation_target and (
+ not self.is_windows
+ or "uwp" in cross_compilation_target
+ or "android" in cross_compilation_target
+ ):
+ return
+
+ # We may not need to update environment variables if GStreamer is installed
+ # for the system on Linux.
+ gstreamer_root = self.gstreamer_root(cross_compilation_target)
+ if gstreamer_root:
+ util.prepend_paths_to_env(env, "PATH", os.path.join(gstreamer_root, "bin"))
+ util.prepend_paths_to_env(
+ env, "PKG_CONFIG_PATH", os.path.join(gstreamer_root, "lib", "pkgconfig")
+ )
+ util.prepend_paths_to_env(
+ env,
+ self.library_path_variable_name(),
+ os.path.join(gstreamer_root, "lib"),
+ )
+ env["GST_PLUGIN_SCANNER"] = os.path.join(
+ gstreamer_root,
+ "libexec",
+ "gstreamer-1.0",
+ f"gst-plugin-scanner{self.executable_suffix()}",
+ )
+ env["GST_PLUGIN_SYSTEM_PATH"] = os.path.join(gstreamer_root, "lib", "gstreamer-1.0")
+ if self.is_macos:
+ env["OPENSSL_INCLUDE_DIR"] = os.path.join(gstreamer_root, "Headers")
+
+ # If we are not cross-compiling GStreamer must be installed for the system. In
+ # the cross-compilation case, we might be picking it up from another directory.
+ if check_installation and not self.is_gstreamer_installed(cross_compilation_target):
+ raise FileNotFoundError(
+ "GStreamer libraries not found (>= version 1.16)."
+ "Please see installation instructions in README.md"
+ )
+
+ def gstreamer_root(self, _cross_compilation_target: Optional[str]) -> Optional[str]:
+ raise NotImplementedError("Do not know how to get GStreamer path for platform.")
+
+ def library_path_variable_name(self):
+ raise NotImplementedError("Do not know how to set library path for platform.")
+
+ def executable_suffix(self):
+ return ""
+
def _platform_bootstrap(self, _cache_dir: str, _force: bool) -> bool:
raise NotImplementedError("Bootstrap installation detection not yet available.")
- def _platform_bootstrap_gstreamer(self, _cache_dir: str, _force: bool) -> bool:
- raise NotImplementedError("GStreamer bootstrap support is not yet available for your OS.")
+ def _platform_bootstrap_gstreamer(self, _force: bool) -> bool:
+ raise NotImplementedError(
+ "GStreamer bootstrap support is not yet available for your OS."
+ )
- def _platform_is_gstreamer_installed(self) -> bool:
- return subprocess.call(
- ["pkg-config", "--atleast-version=1.16", "gstreamer-1.0"],
- stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0
+ def is_gstreamer_installed(self, cross_compilation_target: Optional[str]) -> bool:
+ env = os.environ.copy()
+ self.set_gstreamer_environment_variables_if_necessary(
+ env, cross_compilation_target, check_installation=False)
+ return (
+ subprocess.call(
+ ["pkg-config", "--atleast-version=1.16", "gstreamer-1.0"],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env=env,
+ )
+ == 0
+ )
def bootstrap(self, cache_dir: str, force: bool):
if not self._platform_bootstrap(cache_dir, force):
print("Dependencies were already installed!")
- def bootstrap_gstreamer(self, cache_dir: str, force: bool):
- if not self._platform_bootstrap_gstreamer(cache_dir, force):
- print("Dependencies were already installed!")
-
- def is_gstreamer_installed(self) -> bool:
- return self._platform_is_gstreamer_installed()
+ def bootstrap_gstreamer(self, _cache_dir: str, force: bool):
+ if not self._platform_bootstrap_gstreamer(force):
+ root = self.gstreamer_root(None)
+ if root:
+ print(f"GStreamer found at: {root}")
+ else:
+ print("GStreamer already installed system-wide.")
diff --git a/python/servo/platform/linux.py b/python/servo/platform/linux.py
index da1a83a3b27..b4a8c031f37 100644
--- a/python/servo/platform/linux.py
+++ b/python/servo/platform/linux.py
@@ -9,12 +9,12 @@
import os
import subprocess
-
-from typing import Tuple
+import tempfile
+from typing import Optional, Tuple
import distro
import six
-
+from .. import util
from .base import Base
# Please keep these in sync with the packages in README.md
@@ -48,11 +48,21 @@ XBPS_PKGS = ['libtool', 'gcc', 'libXi-devel', 'freetype-devel',
'clang', 'gstreamer1-devel', 'autoconf213',
'gst-plugins-base1-devel', 'gst-plugins-bad1-devel']
+GSTREAMER_URL = \
+ "https://github.com/servo/servo-build-deps/releases/download/linux/gstreamer-1.16-x86_64-linux-gnu.20190515.tar.gz"
+PREPACKAGED_GSTREAMER_ROOT = \
+ os.path.join(util.get_target_dir(), "dependencies", "gstreamer")
+
class Linux(Base):
- def __init__(self):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.is_linux = True
(self.distro, self.version) = Linux.get_distro_and_version()
+ def library_path_variable_name(self):
+ return "LD_LIBRARY_PATH"
+
@staticmethod
def get_distro_and_version() -> Tuple[str, str]:
distrib = six.ensure_str(distro.name())
@@ -116,7 +126,7 @@ class Linux(Base):
f"{self.distro}, please file a bug")
installed_something = self.install_non_gstreamer_dependencies(force)
- installed_something |= self._platform_bootstrap_gstreamer(_cache_dir, force)
+ installed_something |= self._platform_bootstrap_gstreamer(force)
return installed_something
def install_non_gstreamer_dependencies(self, force: bool) -> bool:
@@ -157,15 +167,34 @@ class Linux(Base):
print("Installing missing dependencies...")
if run_as_root(command + pkgs, force) != 0:
- raise Exception("Installation of dependencies failed.")
+ raise EnvironmentError("Installation of dependencies failed.")
return True
- def _platform_bootstrap_gstreamer(self, _cache_dir: str, _force: bool) -> bool:
- if self.is_gstreamer_installed():
+ def gstreamer_root(self, cross_compilation_target: Optional[str]) -> Optional[str]:
+ if cross_compilation_target:
+ return None
+ if os.path.exists(PREPACKAGED_GSTREAMER_ROOT):
+ return PREPACKAGED_GSTREAMER_ROOT
+ # GStreamer might be installed system-wide, but we do not return a root in this
+ # case because we don't have to update environment variables.
+ return None
+
+ def _platform_bootstrap_gstreamer(self, force: bool) -> bool:
+ if not force and self.is_gstreamer_installed(cross_compilation_target=None):
return False
- gstdir = os.path.join(os.curdir, "support", "linux", "gstreamer")
- if not os.path.isdir(os.path.join(gstdir, "gst", "lib")):
- subprocess.check_call(["bash", "gstreamer.sh"], cwd=gstdir)
+ with tempfile.TemporaryDirectory() as temp_dir:
+ file_name = os.path.join(temp_dir, GSTREAMER_URL.rsplit('/', maxsplit=1)[-1])
+ util.download_file("Pre-packaged GStreamer binaries", GSTREAMER_URL, file_name)
+
+ print(f"Installing GStreamer packages to {PREPACKAGED_GSTREAMER_ROOT}...")
+ os.makedirs(PREPACKAGED_GSTREAMER_ROOT, exist_ok=True)
+
+ # Extract, but strip one component from the output, because the package includes
+ # a toplevel directory called "./gst/" and we'd like to have the same directory
+ # structure on all platforms.
+ subprocess.check_call(["tar", "xf", file_name, "-C", PREPACKAGED_GSTREAMER_ROOT,
+ "--strip-components=2"])
+
+ assert self.is_gstreamer_installed(cross_compilation_target=None)
return True
- return False
diff --git a/python/servo/platform/macos.py b/python/servo/platform/macos.py
index 6189140cabf..0f5ca632b2f 100644
--- a/python/servo/platform/macos.py
+++ b/python/servo/platform/macos.py
@@ -9,28 +9,76 @@
import os
import subprocess
+import tempfile
+from typing import Optional
+from .. import util
from .base import Base
-from ..gstreamer import macos_gst_root
+
+URL_BASE = "https://github.com/servo/servo-build-deps/releases/download/macOS"
+GSTREAMER_URL = f"{URL_BASE}/gstreamer-1.0-1.22.2-universal.pkg"
+GSTREAMER_DEVEL_URL = f"{URL_BASE}/gstreamer-1.0-devel-1.22.2-universal.pkg"
+GSTREAMER_ROOT = "/Library/Frameworks/GStreamer.framework/Versions/1.0"
class MacOS(Base):
- def __init__(self):
- pass
-
- def _platform_is_gstreamer_installed(self) -> bool:
- # We override homebrew gstreamer if installed and always use pkgconfig
- # from official gstreamer framework.
- try:
- gst_root = macos_gst_root()
- env = os.environ.copy()
- env["PATH"] = os.path.join(gst_root, "bin")
- env["PKG_CONFIG_PATH"] = os.path.join(gst_root, "lib", "pkgconfig")
- has_gst = subprocess.call(
- ["pkg-config", "--atleast-version=1.21", "gstreamer-1.0"],
- stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) == 0
- gst_lib_dir = subprocess.check_output(
- ["pkg-config", "--variable=libdir", "gstreamer-1.0"], env=env)
- return has_gst and gst_lib_dir.startswith(bytes(gst_root, 'utf-8'))
- except FileNotFoundError:
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.is_macos = True
+
+ def library_path_variable_name(self):
+ return "DYLD_LIBRARY_PATH"
+
+ def gstreamer_root(self, cross_compilation_target: Optional[str]) -> Optional[str]:
+ # We do not support building with gstreamer while cross-compiling on MacOS.
+ if cross_compilation_target or not os.path.exists(GSTREAMER_ROOT):
+ return None
+ return GSTREAMER_ROOT
+
+ def is_gstreamer_installed(self, cross_compilation_target: Optional[str]) -> bool:
+ if not super().is_gstreamer_installed(cross_compilation_target):
+ return False
+
+ # Servo only supports the official GStreamer distribution on MacOS.
+ env = os.environ.copy()
+ self.set_gstreamer_environment_variables_if_necessary(
+ env, cross_compilation_target, check_installation=False
+ )
+ gst_lib_dir = subprocess.check_output(
+ ["pkg-config", "--variable=libdir", "gstreamer-1.0"], env=env
+ )
+ if not gst_lib_dir.startswith(bytes(GSTREAMER_ROOT, "utf-8")):
+ print("GStreamer is installed, but not the official packages.\n"
+ "Run `./mach bootstrap-gtstreamer` or install packages from "
+ "https://gstreamer.freedesktop.org/")
return False
+ return True
+
+ def _platform_bootstrap_gstreamer(self, force: bool) -> bool:
+ if not force and self.is_gstreamer_installed(cross_compilation_target=None):
+ return False
+
+ with tempfile.TemporaryDirectory() as temp_dir:
+ libs_pkg = os.path.join(temp_dir, GSTREAMER_URL.rsplit("/", maxsplit=1)[-1])
+ devel_pkg = os.path.join(
+ temp_dir, GSTREAMER_DEVEL_URL.rsplit("/", maxsplit=1)[-1]
+ )
+
+ util.download_file("GStreamer libraries", GSTREAMER_URL, libs_pkg)
+ util.download_file(
+ "GStreamer development support", GSTREAMER_DEVEL_URL, devel_pkg
+ )
+
+ print("Installing GStreamer packages...")
+ subprocess.check_call(
+ [
+ "sudo",
+ "sh",
+ "-c",
+ f"installer -pkg '{libs_pkg}' -target / &&"
+ f"installer -pkg '{devel_pkg}' -target /",
+ ]
+ )
+
+ assert self.is_gstreamer_installed(cross_compilation_target=None)
+ return True
diff --git a/python/servo/platform/windows.py b/python/servo/platform/windows.py
index cea8de5a5dd..8ea2a0a5650 100644
--- a/python/servo/platform/windows.py
+++ b/python/servo/platform/windows.py
@@ -10,14 +10,15 @@
import os
import shutil
import subprocess
+import tempfile
+from typing import Optional
import urllib
import zipfile
-
from distutils.version import LooseVersion
import six
+from .. import util
from .base import Base
-from ..util import extract, download_file
DEPS_URL = "https://github.com/servo/servo-build-deps/releases/download/msvc-deps/"
DEPENDENCIES = {
@@ -31,10 +32,22 @@ DEPENDENCIES = {
"openxr-loader-uwp": "1.0",
}
+URL_BASE = "https://gstreamer.freedesktop.org/data/pkg/windows/1.16.0/"
+GSTREAMER_URL = f"{URL_BASE}/gstreamer-1.0-msvc-x86_64-1.16.0.msi"
+GSTREAMER_DEVEL_URL = f"{URL_BASE}/gstreamer-1.0-devel-msvc-x86_64-1.16.0.msi"
+DEPENDENCIES_DIR = os.path.join(util.get_target_dir(), "dependencies")
+
class Windows(Base):
- def __init__(self):
- pass
+ def __init__(self, triple: str):
+ super().__init__(triple)
+ self.is_windows = True
+
+ def executable_suffix(self):
+ return ".exe"
+
+ def library_path_variable_name(self):
+ return "LIB"
@staticmethod
def cmake_already_installed(required_version: str) -> bool:
@@ -51,11 +64,11 @@ class Windows(Base):
def prepare_file(cls, deps_dir: str, zip_path: str, full_spec: str):
if not os.path.isfile(zip_path):
zip_url = "{}{}.zip".format(DEPS_URL, urllib.parse.quote(full_spec))
- download_file(full_spec, zip_url, zip_path)
+ util.download_file(full_spec, zip_url, zip_path)
- print("Extracting {}...".format(full_spec), end='')
+ print("Extracting {}...".format(full_spec), end="")
try:
- extract(zip_path, deps_dir)
+ util.extract(zip_path, deps_dir)
except zipfile.BadZipfile:
print("\nError: %s.zip is not a valid zip file, redownload..." % full_spec)
os.remove(zip_path)
@@ -70,7 +83,7 @@ class Windows(Base):
return os.path.join(deps_dir, package, version)
to_install = {}
- for (package, version) in DEPENDENCIES.items():
+ for package, version in DEPENDENCIES.items():
# Don't install CMake if it already exists in PATH
if package == "cmake" and self.cmake_already_installed(version):
continue
@@ -82,8 +95,8 @@ class Windows(Base):
return False
print("Installing missing MSVC dependencies...")
- for (package, version) in to_install.items():
- full_spec = '{}-{}'.format(package, version)
+ for package, version in to_install.items():
+ full_spec = "{}-{}".format(package, version)
package_dir = get_package_dir(package, version)
parent_dir = os.path.dirname(package_dir)
@@ -96,3 +109,67 @@ class Windows(Base):
os.rename(extracted_path, package_dir)
return True
+
+ def gstreamer_root(self, cross_compilation_target: Optional[str]) -> Optional[str]:
+ build_target_triple = cross_compilation_target or self.triple
+ gst_arch_names = {
+ "x86_64": "X86_64",
+ "x86": "X86",
+ "aarch64": "ARM64",
+ }
+ gst_arch_name = gst_arch_names[build_target_triple.split("-")[0]]
+
+ # The bootstraped version of GStreamer always takes precedance of the installed vesion.
+ prepackaged_root = os.path.join(
+ DEPENDENCIES_DIR, "gstreamer", "1.0", gst_arch_name
+ )
+ if os.path.exists(os.path.join(prepackaged_root, "bin", "ffi-7.dll")):
+ return prepackaged_root
+
+ # The installed version of GStreamer often sets an environment variable pointing to
+ # the install location.
+ root_from_env = os.environ.get(f"GSTREAMER_1_0_ROOT_{gst_arch_name}")
+ if root_from_env:
+ return root_from_env
+
+ # If all else fails, look for an installation in the default install directory.
+ default_root = os.path.join("C:\\gstreamer\\1.0", gst_arch_name)
+ if os.path.exists(os.path.join(default_root, "bin", "ffi-7.dll")):
+ return default_root
+
+ return None
+
+ def is_gstreamer_installed(self, cross_compilation_target: Optional[str]) -> bool:
+ return self.gstreamer_root(cross_compilation_target) is not None
+
+ def _platform_bootstrap_gstreamer(self, force: bool) -> bool:
+ if not force and self.is_gstreamer_installed(cross_compilation_target=None):
+ return False
+
+ if "x86_64" not in self.triple:
+ print("Bootstrapping gstreamer not supported on "
+ "non-x86-64 Windows. Please install manually")
+ return False
+
+ with tempfile.TemporaryDirectory() as temp_dir:
+ libs_msi = os.path.join(temp_dir, GSTREAMER_URL.rsplit("/", maxsplit=1)[-1])
+ devel_msi = os.path.join(
+ temp_dir, GSTREAMER_DEVEL_URL.rsplit("/", maxsplit=1)[-1]
+ )
+
+ util.download_file("GStreamer libraries", GSTREAMER_URL, libs_msi)
+ util.download_file(
+ "GStreamer development support", GSTREAMER_DEVEL_URL, devel_msi
+ )
+
+ print(f"Installing GStreamer packages to {DEPENDENCIES_DIR}...")
+ os.makedirs(DEPENDENCIES_DIR, exist_ok=True)
+ common_args = [
+ f"TARGETDIR={DEPENDENCIES_DIR}", # Install destination
+ "/qn", # Quiet mode
+ ]
+ subprocess.check_call(["msiexec", "/a", libs_msi] + common_args)
+ subprocess.check_call(["msiexec", "/a", devel_msi] + common_args)
+
+ assert self.is_gstreamer_installed(cross_compilation_target=None)
+ return True
diff --git a/python/servo/post_build_commands.py b/python/servo/post_build_commands.py
index 74af93fb0c9..07f104630a1 100644
--- a/python/servo/post_build_commands.py
+++ b/python/servo/post_build_commands.py
@@ -15,6 +15,8 @@ import os.path as path
import subprocess
from shutil import copytree, rmtree, copy2
+import servo.util
+
from mach.decorators import (
CommandArgument,
CommandProvider,
@@ -79,9 +81,15 @@ class PostBuildCommands(CommandBase):
help="Command-line arguments to be passed through to Servo")
def run(self, params, release=False, dev=False, android=None, debug=False, debugger=None,
headless=False, software=False, bin=None, emulator=False, usb=False, nightly=None):
- self.set_run_env(android is not None)
env = self.build_env()
env["RUST_BACKTRACE"] = "1"
+ if software:
+ if not is_linux():
+ print("Software rendering is only supported on Linux at the moment.")
+ return
+
+ env['LIBGL_ALWAYS_SOFTWARE'] = "1"
+ os.environ.update(env)
# Make --debugger imply --debug
if debugger:
@@ -129,13 +137,6 @@ class PostBuildCommands(CommandBase):
if headless:
args.append('-z')
- if software:
- if not is_linux():
- print("Software rendering is only supported on Linux at the moment.")
- return
-
- env['LIBGL_ALWAYS_SOFTWARE'] = "1"
-
# Borrowed and modified from:
# http://hg.mozilla.org/mozilla-central/file/c9cfa9b91dea/python/mozbuild/mozbuild/mach_commands.py#l883
if debug:
@@ -251,7 +252,7 @@ class PostBuildCommands(CommandBase):
toolchain_path = path.dirname(path.dirname(rustc_path))
rust_docs = path.join(toolchain_path, "share", "doc", "rust", "html")
- docs = path.join(self.get_target_dir(), "doc")
+ docs = path.join(servo.util.get_target_dir(), "doc")
if not path.exists(docs):
os.makedirs(docs)
@@ -293,4 +294,4 @@ class PostBuildCommands(CommandBase):
self.doc([])
import webbrowser
webbrowser.open("file://" + path.abspath(path.join(
- self.get_target_dir(), "doc", "servo", "index.html")))
+ servo.util.get_target_dir(), "doc", "servo", "index.html")))
diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py
index 18780b2ac01..f632ce30a1c 100644
--- a/python/servo/testing_commands.py
+++ b/python/servo/testing_commands.py
@@ -40,7 +40,6 @@ from servo.command_base import (
call, check_call, check_output,
)
from servo_tidy_tests import test_tidy
-from servo.platform import host_triple
SCRIPT_PATH = os.path.split(__file__)[0]
PROJECT_TOPLEVEL_PATH = os.path.abspath(os.path.join(SCRIPT_PATH, "..", ".."))
@@ -247,16 +246,6 @@ class MachCommands(CommandBase):
packages.discard('stylo')
- env = self.build_env(test_unit=True)
- # FIXME: https://github.com/servo/servo/issues/26192
- if "apple-darwin" not in host_triple():
- env["RUST_BACKTRACE"] = "1"
-
- if "msvc" in host_triple():
- # on MSVC, we need some DLLs in the path. They were copied
- # in to the servo.exe build dir, so just point PATH to that.
- env["PATH"] = "%s%s%s" % (path.dirname(self.get_binary_path(False, False)), os.pathsep, env["PATH"])
-
if len(packages) > 0 or len(in_crate_packages) > 0:
args = []
for crate in packages:
@@ -270,7 +259,7 @@ class MachCommands(CommandBase):
err = self.run_cargo_build_like_command("bench" if bench else "test",
args,
- env=env,
+ env=self.build_env(test_unit=True),
with_layout_2020=with_layout_2020,
**kwargs)
if err:
@@ -393,7 +382,8 @@ class MachCommands(CommandBase):
return self._test_wpt(android=True, **kwargs)
def _test_wpt(self, android=False, **kwargs):
- self.set_run_env(android)
+ if not android:
+ os.environ.update(self.build_env())
return wpt.run.run_tests(**kwargs)
# Helper to ensure all specified paths are handled, otherwise dispatch to appropriate test suite.
diff --git a/python/servo/util.py b/python/servo/util.py
index 4151bbd4929..43769e181de 100644
--- a/python/servo/util.py
+++ b/python/servo/util.py
@@ -17,9 +17,12 @@ import stat
import sys
import time
import urllib
+import urllib.request
import zipfile
+from typing import Dict, List, Union
-from io import BytesIO
+import six
+from io import BufferedIOBase, BytesIO
from socket import error as socket_error
try:
@@ -27,6 +30,8 @@ try:
except ImportError:
HAS_SNI = False
+SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__))
+SERVO_ROOT = os.path.abspath(os.path.join(SCRIPT_PATH, "..", ".."))
HAS_SNI_AND_RECENT_PYTHON = HAS_SNI and sys.version_info >= (2, 7, 9)
@@ -58,17 +63,17 @@ def delete(path):
os.remove(path)
-def download(desc, src, writer, start_byte=0):
+def download(description: str, url: str, writer: BufferedIOBase, start_byte: int = 0):
if start_byte:
- print("Resuming download of {} ...".format(src))
+ print("Resuming download of {} ...".format(url))
else:
- print("Downloading {} ...".format(src))
+ print("Downloading {} ...".format(url))
dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty())
try:
- req = urllib.request.Request(src)
+ req = urllib.request.Request(url)
if start_byte:
- req = urllib.request.Request(src, headers={'Range': 'bytes={}-'.format(start_byte)})
+ req = urllib.request.Request(url, headers={'Range': 'bytes={}-'.format(start_byte)})
resp = urllib.request.urlopen(req, **get_urlopen_kwargs())
fsize = None
@@ -79,7 +84,7 @@ def download(desc, src, writer, start_byte=0):
chunk_size = 64 * 1024
previous_progress_line = None
- previous_progress_line_time = 0
+ previous_progress_line_time = 0.0
while True:
chunk = resp.read(chunk_size)
if not chunk:
@@ -88,7 +93,7 @@ def download(desc, src, writer, start_byte=0):
if not dumb:
if fsize is not None:
pct = recved * 100.0 / fsize
- progress_line = "\rDownloading %s: %5.1f%%" % (desc, pct)
+ progress_line = "\rDownloading %s: %5.1f%%" % (description, pct)
now = time.time()
duration = now - previous_progress_line_time
if progress_line != previous_progress_line and duration > .1:
@@ -102,13 +107,13 @@ def download(desc, src, writer, start_byte=0):
if not dumb:
print()
except urllib.error.HTTPError as e:
- print("Download failed ({}): {} - {}".format(e.code, e.reason, src))
+ print("Download failed ({}): {} - {}".format(e.code, e.reason, url))
if e.code == 403:
print("No Rust compiler binary available for this platform. "
"Please see https://github.com/servo/servo/#prerequisites")
sys.exit(1)
except urllib.error.URLError as e:
- print("Error downloading {}: {}. The failing URL was: {}".format(desc, e.reason, src))
+ print("Error downloading {}: {}. The failing URL was: {}".format(description, e.reason, url))
sys.exit(1)
except socket_error as e:
print("Looks like there's a connectivity issue, check your Internet connection. {}".format(e))
@@ -118,22 +123,22 @@ def download(desc, src, writer, start_byte=0):
raise
-def download_bytes(desc, src):
+def download_bytes(description: str, url: str):
content_writer = BytesIO()
- download(desc, src, content_writer)
+ download(description, url, content_writer)
return content_writer.getvalue()
-def download_file(desc, src, dst):
- tmp_path = dst + ".part"
+def download_file(description: str, url: str, destination_path: str):
+ tmp_path = destination_path + ".part"
try:
start_byte = os.path.getsize(tmp_path)
with open(tmp_path, 'ab') as fd:
- download(desc, src, fd, start_byte=start_byte)
+ download(description, url, fd, start_byte=start_byte)
except os.error:
with open(tmp_path, 'wb') as fd:
- download(desc, src, fd)
- os.rename(tmp_path, dst)
+ download(description, url, fd)
+ os.rename(tmp_path, destination_path)
# https://stackoverflow.com/questions/39296101/python-zipfile-removes-execute-permissions-from-binaries
@@ -198,3 +203,30 @@ def check_hash(filename, expected, algorithm):
def get_default_cache_dir(topdir):
return os.environ.get("SERVO_CACHE_DIR", os.path.join(topdir, ".servo"))
+
+
+def append_paths_to_env(env: Dict[str, str], key: str, paths: Union[str, List[str]]):
+ if isinstance(paths, list):
+ paths = os.pathsep.join(paths)
+
+ existing_value = env.get(key, None)
+ if existing_value:
+ new_value = six.ensure_str(existing_value) + os.pathsep + paths
+ else:
+ new_value = paths
+ env[key] = new_value
+
+
+def prepend_paths_to_env(env: Dict[str, str], key: str, paths: Union[str, List[str]]):
+ if isinstance(paths, list):
+ paths = os.pathsep.join(paths)
+
+ existing_value = env.get(key, None)
+ new_value = paths
+ if existing_value:
+ new_value += os.pathsep + six.ensure_str(existing_value)
+ env[key] = new_value
+
+
+def get_target_dir():
+ return os.environ.get("CARGO_TARGET_DIR", os.path.join(SERVO_ROOT, "target"))
generated by cgit v1.2.3 (git 2.25.1) at 2025-04-30 00:10:58 +0000