Implement `bootstrap-gstreamer` for all platforms

This change makes it so that the Platform classes can now handle
installing GStreamer dependencies and properly setting up the
environment including when cross-compiling. For Windows and Linux
is now installed into `target/dependencies/gstreamer` when not installed
system-wide. In addition:

1. Creating and moving existing environment path append helpers to
   `util.py`.
2. Combining the `set_run_env` and `build_dev` functions and moving
   some outside code into them so that it can be shared. Now code that
   used to call `set_run_env` calls `build_dev` and then
   `os.environ.update(...)`.
3. Adding Python typing information in many places.

Signed-off-by: Martin Robinson <mrobinson@igalia.com>

5 files changed, 301 insertions, 61 deletions

diff --git a/python/servo/platform/__init__.py b/python/servo/platform/__init__.py
index 84968550d00..e06bebaf863 100644
--- a/python/servo/platform/__init__.py
+++ b/python/servo/platform/__init__.py
@@ -9,9 +9,10 @@
 
 import platform
 
-from .base import Base
 from .windows import Windows
 
+__platform__ = None
+
 
 def host_platform():
     os_type = platform.system().lower()
@@ -47,18 +48,27 @@ def host_triple():
 
 
 def get():
+    # pylint: disable=global-statement
+    global __platform__
+    if __platform__:
+        return __platform__
+
     # We import the concrete platforms in if-statements here, because
     # each one might have platform-specific imports which might not
     # resolve on all platforms.
     # TODO(mrobinson): We should do this for Windows too, once we
     # stop relying on platform-specific code outside of this module.
     # pylint: disable=import-outside-toplevel
-    if "windows-msvc" in host_triple():
-        return Windows()
-    if "linux-gnu" in host_triple():
+    triple = host_triple()
+    if "windows-msvc" in triple:
+        __platform__ = Windows(triple)
+    elif "linux-gnu" in triple:
         from .linux import Linux
-        return Linux()
-    if "apple-darwin" in host_triple():
+        __platform__ = Linux(triple)
+    elif "apple-darwin" in triple:
         from .macos import MacOS
-        return MacOS()
-    return Base()
+        __platform__ = MacOS(triple)
+    else:
+        from .base import Base
+        __platform__ = Base(triple)
+    return __platform__
diff --git a/python/servo/platform/base.py b/python/servo/platform/base.py
index 316c998b1db..b0ac64386bd 100644
--- a/python/servo/platform/base.py
+++ b/python/servo/platform/base.py
@@ -7,28 +7,104 @@
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
+import os
 import subprocess
+from typing import Dict, Optional
+
+from .. import util
 
 
 class Base:
+    def __init__(self, triple: str):
+        self.environ = os.environ.copy()
+        self.triple = triple
+        self.is_windows = False
+        self.is_linux = False
+        self.is_macos = False
+
+    def set_gstreamer_environment_variables_if_necessary(
+        self, env: Dict[str, str], cross_compilation_target: Optional[str], check_installation=True
+    ):
+        # Environment variables are not needed when cross-compiling on any
+        # platform other than Windows. UWP doesn't support GStreamer. GStreamer
+        # for Android is handled elsewhere.
+        if cross_compilation_target and (
+            not self.is_windows
+            or "uwp" in cross_compilation_target
+            or "android" in cross_compilation_target
+        ):
+            return
+
+        # We may not need to update environment variables if GStreamer is installed
+        # for the system on Linux.
+        gstreamer_root = self.gstreamer_root(cross_compilation_target)
+        if gstreamer_root:
+            util.prepend_paths_to_env(env, "PATH", os.path.join(gstreamer_root, "bin"))
+            util.prepend_paths_to_env(
+                env, "PKG_CONFIG_PATH", os.path.join(gstreamer_root, "lib", "pkgconfig")
+            )
+            util.prepend_paths_to_env(
+                env,
+                self.library_path_variable_name(),
+                os.path.join(gstreamer_root, "lib"),
+            )
+            env["GST_PLUGIN_SCANNER"] = os.path.join(
+                gstreamer_root,
+                "libexec",
+                "gstreamer-1.0",
+                f"gst-plugin-scanner{self.executable_suffix()}",
+            )
+            env["GST_PLUGIN_SYSTEM_PATH"] = os.path.join(gstreamer_root, "lib", "gstreamer-1.0")
+            if self.is_macos:
+                env["OPENSSL_INCLUDE_DIR"] = os.path.join(gstreamer_root, "Headers")
+
+        # If we are not cross-compiling GStreamer must be installed for the system. In
+        # the cross-compilation case, we might be picking it up from another directory.
+        if check_installation and not self.is_gstreamer_installed(cross_compilation_target):
+            raise FileNotFoundError(
+                "GStreamer libraries not found (>= version 1.16)."
+                "Please see installation instructions in README.md"
+            )
+
+    def gstreamer_root(self, _cross_compilation_target: Optional[str]) -> Optional[str]:
+        raise NotImplementedError("Do not know how to get GStreamer path for platform.")
+
+    def library_path_variable_name(self):
+        raise NotImplementedError("Do not know how to set library path for platform.")
+
+    def executable_suffix(self):
+        return ""
+
     def _platform_bootstrap(self, _cache_dir: str, _force: bool) -> bool:
         raise NotImplementedError("Bootstrap installation detection not yet available.")
 
-    def _platform_bootstrap_gstreamer(self, _cache_dir: str, _force: bool) -> bool:
-        raise NotImplementedError("GStreamer bootstrap support is not yet available for your OS.")
+    def _platform_bootstrap_gstreamer(self, _force: bool) -> bool:
+        raise NotImplementedError(
+            "GStreamer bootstrap support is not yet available for your OS."
+        )
 
-    def _platform_is_gstreamer_installed(self) -> bool:
-        return subprocess.call(
-            ["pkg-config", "--atleast-version=1.16", "gstreamer-1.0"],
-            stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0
+    def is_gstreamer_installed(self, cross_compilation_target: Optional[str]) -> bool:
+        env = os.environ.copy()
+        self.set_gstreamer_environment_variables_if_necessary(
+            env, cross_compilation_target, check_installation=False)
+        return (
+            subprocess.call(
+                ["pkg-config", "--atleast-version=1.16", "gstreamer-1.0"],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                env=env,
+            )
+            == 0
+        )
 
     def bootstrap(self, cache_dir: str, force: bool):
         if not self._platform_bootstrap(cache_dir, force):
             print("Dependencies were already installed!")
 
-    def bootstrap_gstreamer(self, cache_dir: str, force: bool):
-        if not self._platform_bootstrap_gstreamer(cache_dir, force):
-            print("Dependencies were already installed!")
-
-    def is_gstreamer_installed(self) -> bool:
-        return self._platform_is_gstreamer_installed()
+    def bootstrap_gstreamer(self, _cache_dir: str, force: bool):
+        if not self._platform_bootstrap_gstreamer(force):
+            root = self.gstreamer_root(None)
+            if root:
+                print(f"GStreamer found at: {root}")
+            else:
+                print("GStreamer already installed system-wide.")
diff --git a/python/servo/platform/linux.py b/python/servo/platform/linux.py
index da1a83a3b27..b4a8c031f37 100644
--- a/python/servo/platform/linux.py
+++ b/python/servo/platform/linux.py
@@ -9,12 +9,12 @@
 
 import os
 import subprocess
-
-from typing import Tuple
+import tempfile
+from typing import Optional, Tuple
 
 import distro
 import six
-
+from .. import util
 from .base import Base
 
 # Please keep these in sync with the packages in README.md
@@ -48,11 +48,21 @@ XBPS_PKGS = ['libtool', 'gcc', 'libXi-devel', 'freetype-devel',
              'clang', 'gstreamer1-devel', 'autoconf213',
              'gst-plugins-base1-devel', 'gst-plugins-bad1-devel']
 
+GSTREAMER_URL = \
+    "https://github.com/servo/servo-build-deps/releases/download/linux/gstreamer-1.16-x86_64-linux-gnu.20190515.tar.gz"
+PREPACKAGED_GSTREAMER_ROOT = \
+    os.path.join(util.get_target_dir(), "dependencies", "gstreamer")
+
 
 class Linux(Base):
-    def __init__(self):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.is_linux = True
         (self.distro, self.version) = Linux.get_distro_and_version()
 
+    def library_path_variable_name(self):
+        return "LD_LIBRARY_PATH"
+
     @staticmethod
     def get_distro_and_version() -> Tuple[str, str]:
         distrib = six.ensure_str(distro.name())
@@ -116,7 +126,7 @@ class Linux(Base):
                                       f"{self.distro}, please file a bug")
 
         installed_something = self.install_non_gstreamer_dependencies(force)
-        installed_something |= self._platform_bootstrap_gstreamer(_cache_dir, force)
+        installed_something |= self._platform_bootstrap_gstreamer(force)
         return installed_something
 
     def install_non_gstreamer_dependencies(self, force: bool) -> bool:
@@ -157,15 +167,34 @@ class Linux(Base):
 
         print("Installing missing dependencies...")
         if run_as_root(command + pkgs, force) != 0:
-            raise Exception("Installation of dependencies failed.")
+            raise EnvironmentError("Installation of dependencies failed.")
         return True
 
-    def _platform_bootstrap_gstreamer(self, _cache_dir: str, _force: bool) -> bool:
-        if self.is_gstreamer_installed():
+    def gstreamer_root(self, cross_compilation_target: Optional[str]) -> Optional[str]:
+        if cross_compilation_target:
+            return None
+        if os.path.exists(PREPACKAGED_GSTREAMER_ROOT):
+            return PREPACKAGED_GSTREAMER_ROOT
+        # GStreamer might be installed system-wide, but we do not return a root in this
+        # case because we don't have to update environment variables.
+        return None
+
+    def _platform_bootstrap_gstreamer(self, force: bool) -> bool:
+        if not force and self.is_gstreamer_installed(cross_compilation_target=None):
             return False
 
-        gstdir = os.path.join(os.curdir, "support", "linux", "gstreamer")
-        if not os.path.isdir(os.path.join(gstdir, "gst", "lib")):
-            subprocess.check_call(["bash", "gstreamer.sh"], cwd=gstdir)
+        with tempfile.TemporaryDirectory() as temp_dir:
+            file_name = os.path.join(temp_dir, GSTREAMER_URL.rsplit('/', maxsplit=1)[-1])
+            util.download_file("Pre-packaged GStreamer binaries", GSTREAMER_URL, file_name)
+
+            print(f"Installing GStreamer packages to {PREPACKAGED_GSTREAMER_ROOT}...")
+            os.makedirs(PREPACKAGED_GSTREAMER_ROOT, exist_ok=True)
+
+            # Extract, but strip one component from the output, because the package includes
+            # a toplevel directory called "./gst/" and we'd like to have the same directory
+            # structure on all platforms.
+            subprocess.check_call(["tar", "xf", file_name, "-C", PREPACKAGED_GSTREAMER_ROOT,
+                                   "--strip-components=2"])
+
+            assert self.is_gstreamer_installed(cross_compilation_target=None)
             return True
-        return False
diff --git a/python/servo/platform/macos.py b/python/servo/platform/macos.py
index 6189140cabf..0f5ca632b2f 100644
--- a/python/servo/platform/macos.py
+++ b/python/servo/platform/macos.py
@@ -9,28 +9,76 @@
 
 import os
 import subprocess
+import tempfile
+from typing import Optional
 
+from .. import util
 from .base import Base
-from ..gstreamer import macos_gst_root
+
+URL_BASE = "https://github.com/servo/servo-build-deps/releases/download/macOS"
+GSTREAMER_URL = f"{URL_BASE}/gstreamer-1.0-1.22.2-universal.pkg"
+GSTREAMER_DEVEL_URL = f"{URL_BASE}/gstreamer-1.0-devel-1.22.2-universal.pkg"
+GSTREAMER_ROOT = "/Library/Frameworks/GStreamer.framework/Versions/1.0"
 
 
 class MacOS(Base):
-    def __init__(self):
-        pass
-
-    def _platform_is_gstreamer_installed(self) -> bool:
-        # We override homebrew gstreamer if installed and always use pkgconfig
-        # from official gstreamer framework.
-        try:
-            gst_root = macos_gst_root()
-            env = os.environ.copy()
-            env["PATH"] = os.path.join(gst_root, "bin")
-            env["PKG_CONFIG_PATH"] = os.path.join(gst_root, "lib", "pkgconfig")
-            has_gst = subprocess.call(
-                ["pkg-config", "--atleast-version=1.21", "gstreamer-1.0"],
-                stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) == 0
-            gst_lib_dir = subprocess.check_output(
-                ["pkg-config", "--variable=libdir", "gstreamer-1.0"], env=env)
-            return has_gst and gst_lib_dir.startswith(bytes(gst_root, 'utf-8'))
-        except FileNotFoundError:
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.is_macos = True
+
+    def library_path_variable_name(self):
+        return "DYLD_LIBRARY_PATH"
+
+    def gstreamer_root(self, cross_compilation_target: Optional[str]) -> Optional[str]:
+        # We do not support building with gstreamer while cross-compiling on MacOS.
+        if cross_compilation_target or not os.path.exists(GSTREAMER_ROOT):
+            return None
+        return GSTREAMER_ROOT
+
+    def is_gstreamer_installed(self, cross_compilation_target: Optional[str]) -> bool:
+        if not super().is_gstreamer_installed(cross_compilation_target):
+            return False
+
+        # Servo only supports the official GStreamer distribution on MacOS.
+        env = os.environ.copy()
+        self.set_gstreamer_environment_variables_if_necessary(
+            env, cross_compilation_target, check_installation=False
+        )
+        gst_lib_dir = subprocess.check_output(
+            ["pkg-config", "--variable=libdir", "gstreamer-1.0"], env=env
+        )
+        if not gst_lib_dir.startswith(bytes(GSTREAMER_ROOT, "utf-8")):
+            print("GStreamer is installed, but not the official packages.\n"
+                  "Run `./mach bootstrap-gtstreamer` or install packages from "
+                  "https://gstreamer.freedesktop.org/")
             return False
+        return True
+
+    def _platform_bootstrap_gstreamer(self, force: bool) -> bool:
+        if not force and self.is_gstreamer_installed(cross_compilation_target=None):
+            return False
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            libs_pkg = os.path.join(temp_dir, GSTREAMER_URL.rsplit("/", maxsplit=1)[-1])
+            devel_pkg = os.path.join(
+                temp_dir, GSTREAMER_DEVEL_URL.rsplit("/", maxsplit=1)[-1]
+            )
+
+            util.download_file("GStreamer libraries", GSTREAMER_URL, libs_pkg)
+            util.download_file(
+                "GStreamer development support", GSTREAMER_DEVEL_URL, devel_pkg
+            )
+
+            print("Installing GStreamer packages...")
+            subprocess.check_call(
+                [
+                    "sudo",
+                    "sh",
+                    "-c",
+                    f"installer -pkg '{libs_pkg}' -target / &&"
+                    f"installer -pkg '{devel_pkg}' -target /",
+                ]
+            )
+
+            assert self.is_gstreamer_installed(cross_compilation_target=None)
+            return True
diff --git a/python/servo/platform/windows.py b/python/servo/platform/windows.py
index cea8de5a5dd..8ea2a0a5650 100644
--- a/python/servo/platform/windows.py
+++ b/python/servo/platform/windows.py
@@ -10,14 +10,15 @@
 import os
 import shutil
 import subprocess
+import tempfile
+from typing import Optional
 import urllib
 import zipfile
-
 from distutils.version import LooseVersion
 
 import six
+from .. import util
 from .base import Base
-from ..util import extract, download_file
 
 DEPS_URL = "https://github.com/servo/servo-build-deps/releases/download/msvc-deps/"
 DEPENDENCIES = {
@@ -31,10 +32,22 @@ DEPENDENCIES = {
     "openxr-loader-uwp": "1.0",
 }
 
+URL_BASE = "https://gstreamer.freedesktop.org/data/pkg/windows/1.16.0/"
+GSTREAMER_URL = f"{URL_BASE}/gstreamer-1.0-msvc-x86_64-1.16.0.msi"
+GSTREAMER_DEVEL_URL = f"{URL_BASE}/gstreamer-1.0-devel-msvc-x86_64-1.16.0.msi"
+DEPENDENCIES_DIR = os.path.join(util.get_target_dir(), "dependencies")
+
 
 class Windows(Base):
-    def __init__(self):
-        pass
+    def __init__(self, triple: str):
+        super().__init__(triple)
+        self.is_windows = True
+
+    def executable_suffix(self):
+        return ".exe"
+
+    def library_path_variable_name(self):
+        return "LIB"
 
     @staticmethod
     def cmake_already_installed(required_version: str) -> bool:
@@ -51,11 +64,11 @@ class Windows(Base):
     def prepare_file(cls, deps_dir: str, zip_path: str, full_spec: str):
         if not os.path.isfile(zip_path):
             zip_url = "{}{}.zip".format(DEPS_URL, urllib.parse.quote(full_spec))
-            download_file(full_spec, zip_url, zip_path)
+            util.download_file(full_spec, zip_url, zip_path)
 
-        print("Extracting {}...".format(full_spec), end='')
+        print("Extracting {}...".format(full_spec), end="")
         try:
-            extract(zip_path, deps_dir)
+            util.extract(zip_path, deps_dir)
         except zipfile.BadZipfile:
             print("\nError: %s.zip is not a valid zip file, redownload..." % full_spec)
             os.remove(zip_path)
@@ -70,7 +83,7 @@ class Windows(Base):
             return os.path.join(deps_dir, package, version)
 
         to_install = {}
-        for (package, version) in DEPENDENCIES.items():
+        for package, version in DEPENDENCIES.items():
             # Don't install CMake if it already exists in PATH
             if package == "cmake" and self.cmake_already_installed(version):
                 continue
@@ -82,8 +95,8 @@ class Windows(Base):
             return False
 
         print("Installing missing MSVC dependencies...")
-        for (package, version) in to_install.items():
-            full_spec = '{}-{}'.format(package, version)
+        for package, version in to_install.items():
+            full_spec = "{}-{}".format(package, version)
 
             package_dir = get_package_dir(package, version)
             parent_dir = os.path.dirname(package_dir)
@@ -96,3 +109,67 @@ class Windows(Base):
             os.rename(extracted_path, package_dir)
 
         return True
+
+    def gstreamer_root(self, cross_compilation_target: Optional[str]) -> Optional[str]:
+        build_target_triple = cross_compilation_target or self.triple
+        gst_arch_names = {
+            "x86_64": "X86_64",
+            "x86": "X86",
+            "aarch64": "ARM64",
+        }
+        gst_arch_name = gst_arch_names[build_target_triple.split("-")[0]]
+
+        # The bootstraped version of GStreamer always takes precedance of the installed vesion.
+        prepackaged_root = os.path.join(
+            DEPENDENCIES_DIR, "gstreamer", "1.0", gst_arch_name
+        )
+        if os.path.exists(os.path.join(prepackaged_root, "bin", "ffi-7.dll")):
+            return prepackaged_root
+
+        # The installed version of GStreamer often sets an environment variable pointing to
+        # the install location.
+        root_from_env = os.environ.get(f"GSTREAMER_1_0_ROOT_{gst_arch_name}")
+        if root_from_env:
+            return root_from_env
+
+        # If all else fails, look for an installation in the default install directory.
+        default_root = os.path.join("C:\\gstreamer\\1.0", gst_arch_name)
+        if os.path.exists(os.path.join(default_root, "bin", "ffi-7.dll")):
+            return default_root
+
+        return None
+
+    def is_gstreamer_installed(self, cross_compilation_target: Optional[str]) -> bool:
+        return self.gstreamer_root(cross_compilation_target) is not None
+
+    def _platform_bootstrap_gstreamer(self, force: bool) -> bool:
+        if not force and self.is_gstreamer_installed(cross_compilation_target=None):
+            return False
+
+        if "x86_64" not in self.triple:
+            print("Bootstrapping gstreamer not supported on "
+                  "non-x86-64 Windows. Please install manually")
+            return False
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            libs_msi = os.path.join(temp_dir, GSTREAMER_URL.rsplit("/", maxsplit=1)[-1])
+            devel_msi = os.path.join(
+                temp_dir, GSTREAMER_DEVEL_URL.rsplit("/", maxsplit=1)[-1]
+            )
+
+            util.download_file("GStreamer libraries", GSTREAMER_URL, libs_msi)
+            util.download_file(
+                "GStreamer development support", GSTREAMER_DEVEL_URL, devel_msi
+            )
+
+            print(f"Installing GStreamer packages to {DEPENDENCIES_DIR}...")
+            os.makedirs(DEPENDENCIES_DIR, exist_ok=True)
+            common_args = [
+                f"TARGETDIR={DEPENDENCIES_DIR}",  # Install destination
+                "/qn",  # Quiet mode
+            ]
+            subprocess.check_call(["msiexec", "/a", libs_msi] + common_args)
+            subprocess.check_call(["msiexec", "/a", devel_msi] + common_args)
+
+            assert self.is_gstreamer_installed(cross_compilation_target=None)
+            return True