diff options
author | Mukilan Thiyagarajan <mukilan@igalia.com> | 2024-08-26 18:38:21 +0530 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-08-26 13:08:21 +0000 |
commit | b6d5ac09b0b2acbb0f5b00232e53d0111a159063 (patch) | |
tree | 18842738e78794ba096a0af44b16d37695cff6d7 | |
parent | 4397d8a02156a009d16d8b79796b1e54ca635624 (diff) | |
download | servo-b6d5ac09b0b2acbb0f5b00232e53d0111a159063.tar.gz servo-b6d5ac09b0b2acbb0f5b00232e53d0111a159063.zip |
mach: introduce `BuildTarget` abstraction (#33114)
Introduce a new `BuildTarget` abstraction to centralize the code for
supporting different ways of choosing the build target (e.g --android,
--target x86_64-linux-android , --target aarch64-linux-ohos). This
is currently handled in an adhoc fashion in different commands (
mach package, install, run) leading to a proliferation of keyword
parameters for the commands and duplicated logic.
The patch introduces a new `allow_target_configuration` decorator to
do the validation and parsing of these parameters into the appropriate
`BuildTarget` subclass, which is now stored as an instance attribute
of the CommandBase class. All the code that previously relied on
`self.cross_compile_target` has been switched to use the BuildTarget.
Signed-off-by: Mukilan Thiyagarajan <mukilan@igalia.com>
-rw-r--r-- | .github/workflows/android.yml | 2 | ||||
-rw-r--r-- | python/servo/build_commands.py | 76 | ||||
-rw-r--r-- | python/servo/command_base.py | 438 | ||||
-rw-r--r-- | python/servo/devenv_commands.py | 3 | ||||
-rw-r--r-- | python/servo/gstreamer.py | 9 | ||||
-rw-r--r-- | python/servo/package_commands.py | 51 | ||||
-rw-r--r-- | python/servo/platform/base.py | 15 | ||||
-rw-r--r-- | python/servo/platform/build_target.py | 372 | ||||
-rw-r--r-- | python/servo/platform/linux.py | 7 | ||||
-rw-r--r-- | python/servo/platform/macos.py | 18 | ||||
-rw-r--r-- | python/servo/platform/windows.py | 23 | ||||
-rw-r--r-- | python/servo/post_build_commands.py | 12 | ||||
-rw-r--r-- | python/servo/testing_commands.py | 9 |
13 files changed, 522 insertions, 513 deletions
diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index eb88fd240ab..6f991c10587 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -85,7 +85,7 @@ jobs: APK_SIGNING_KEY_ALIAS: ${{ secrets.APK_SIGNING_KEY_ALIAS }} APK_SIGNING_KEY_PASS: ${{ secrets.APK_SIGNING_KEY_PASS }} run: | - python3 ./mach build --use-crown --locked --android --target ${{ matrix.arch }} --${{ inputs.profile }} + python3 ./mach build --use-crown --locked --target ${{ matrix.arch }} --${{ inputs.profile }} cp -r target/cargo-timings target/cargo-timings-android-${{ matrix.arch }} # TODO: This is disabled since APK crashes during startup. # See https://github.com/servo/servo/issues/31134 diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py index 61cede7e7b3..121aa493459 100644 --- a/python/servo/build_commands.py +++ b/python/servo/build_commands.py @@ -13,13 +13,10 @@ import os.path as path import pathlib import shutil import stat -import subprocess import sys -import urllib from time import time -from typing import Dict, Optional -import zipfile +from typing import Optional import notifypy @@ -37,6 +34,7 @@ import servo.visual_studio from servo.command_base import BuildType, CommandBase, call, check_call from servo.gstreamer import windows_dlls, windows_plugins, package_gstreamer_dylibs +from servo.platform.build_target import BuildTarget SUPPORTED_ASAN_TARGETS = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu"] @@ -83,7 +81,7 @@ class MachCommands(CommandBase): self.ensure_clobbered() host = servo.platform.host_triple() - target_triple = self.cross_compile_target or servo.platform.host_triple() + target_triple = self.target.triple() if with_asan: if target_triple not in SUPPORTED_ASAN_TARGETS: @@ -133,21 +131,15 @@ class MachCommands(CommandBase): "rustc", opts, env=env, verbose=verbose, **kwargs) if status == 0: - built_binary = self.get_binary_path( - build_type, - target=self.cross_compile_target, - android=self.is_android_build, - asan=with_asan - ) - - if self.is_android_build and not no_package: - rv = Registrar.dispatch("package", context=self.context, build_type=build_type, - target=self.cross_compile_target, flavor=None) + built_binary = self.get_binary_path(build_type, asan=with_asan) + + if not no_package and self.target.needs_packaging(): + rv = Registrar.dispatch("package", context=self.context, build_type=build_type, flavor=None) if rv: return rv if sys.platform == "win32": - if not copy_windows_dlls_to_build_directory(built_binary, target_triple): + if not copy_windows_dlls_to_build_directory(built_binary, self.target): status = 1 elif sys.platform == "darwin": @@ -156,9 +148,7 @@ class MachCommands(CommandBase): if self.enable_media: library_target_directory = path.join(path.dirname(built_binary), "lib/") - if not package_gstreamer_dylibs(built_binary, - library_target_directory, - self.cross_compile_target): + if not package_gstreamer_dylibs(built_binary, library_target_directory, self.target): return 1 # On the Mac, set a lovely icon. This makes it easier to pick out the Servo binary in tools @@ -184,40 +174,6 @@ class MachCommands(CommandBase): return status - def download_and_build_android_dependencies_if_needed(self, env: Dict[str, str]): - if not self.is_android_build: - return - - # Build the name of the package containing all GStreamer dependencies - # according to the build target. - android_lib = self.config["android"]["lib"] - gst_lib = f"gst-build-{android_lib}" - gst_lib_zip = f"gstreamer-{android_lib}-1.16.0-20190517-095630.zip" - gst_lib_path = os.path.join(self.target_path, "gstreamer", gst_lib) - pkg_config_path = os.path.join(gst_lib_path, "pkgconfig") - env["PKG_CONFIG_PATH"] = pkg_config_path - if not os.path.exists(gst_lib_path): - # Download GStreamer dependencies if they have not already been downloaded - # This bundle is generated with `libgstreamer_android_gen` - # Follow these instructions to build and deploy new binaries - # https://github.com/servo/libgstreamer_android_gen#build - gst_url = f"https://servo-deps-2.s3.amazonaws.com/gstreamer/{gst_lib_zip}" - print(f"Downloading GStreamer dependencies ({gst_url})") - - urllib.request.urlretrieve(gst_url, gst_lib_zip) - zip_ref = zipfile.ZipFile(gst_lib_zip, "r") - zip_ref.extractall(os.path.join(self.target_path, "gstreamer")) - os.remove(gst_lib_zip) - - # Change pkgconfig info to make all GStreamer dependencies point - # to the libgstreamer_android.so bundle. - for each in os.listdir(pkg_config_path): - if each.endswith('.pc'): - print(f"Setting pkgconfig info for {each}") - target_path = os.path.join(pkg_config_path, each) - expr = f"s#libdir=.*#libdir={gst_lib_path}#g" - subprocess.call(["perl", "-i", "-pe", expr, target_path]) - @Command('clean', description='Clean the target/ and python/_venv[version]/ directories', category='build') @@ -296,7 +252,7 @@ class MachCommands(CommandBase): print(f"[Warning] Could not generate notification: {e}", file=sys.stderr) -def copy_windows_dlls_to_build_directory(servo_binary: str, target_triple: str) -> bool: +def copy_windows_dlls_to_build_directory(servo_binary: str, target: BuildTarget) -> bool: servo_exe_dir = os.path.dirname(servo_binary) assert os.path.exists(servo_exe_dir) @@ -317,18 +273,18 @@ def copy_windows_dlls_to_build_directory(servo_binary: str, target_triple: str) find_and_copy_built_dll("libGLESv2.dll") print(" • Copying GStreamer DLLs to binary directory...") - if not package_gstreamer_dlls(servo_exe_dir, target_triple): + if not package_gstreamer_dlls(servo_exe_dir, target): return False print(" • Copying MSVC DLLs to binary directory...") - if not package_msvc_dlls(servo_exe_dir, target_triple): + if not package_msvc_dlls(servo_exe_dir, target): return False return True -def package_gstreamer_dlls(servo_exe_dir: str, target: str): - gst_root = servo.platform.get().gstreamer_root(cross_compilation_target=target) +def package_gstreamer_dlls(servo_exe_dir: str, target: BuildTarget): + gst_root = servo.platform.get().gstreamer_root(target) if not gst_root: print("Could not find GStreamer installation directory.") return False @@ -366,7 +322,7 @@ def package_gstreamer_dlls(servo_exe_dir: str, target: str): return not missing -def package_msvc_dlls(servo_exe_dir: str, target: str): +def package_msvc_dlls(servo_exe_dir: str, target: BuildTarget): def copy_file(dll_path: Optional[str]) -> bool: if not dll_path or not os.path.exists(dll_path): print(f"WARNING: Could not find DLL at {dll_path}", file=sys.stderr) @@ -383,7 +339,7 @@ def package_msvc_dlls(servo_exe_dir: str, target: str): "x86_64": "x64", "i686": "x86", "aarch64": "arm64", - }[target.split('-')[0]] + }[target.triple().split('-')[0]] for msvc_redist_dir in servo.visual_studio.find_msvc_redist_dirs(vs_platform): if copy_file(os.path.join(msvc_redist_dir, "msvcp140.dll")) and \ diff --git a/python/servo/command_base.py b/python/servo/command_base.py index a9b62f9558e..016022c6c41 100644 --- a/python/servo/command_base.py +++ b/python/servo/command_base.py @@ -10,17 +10,13 @@ from __future__ import annotations import contextlib -import errno -import json -import pathlib from enum import Enum -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional import functools import gzip import itertools import locale import os -import platform import re import shutil import subprocess @@ -35,16 +31,17 @@ from glob import glob from os import path from subprocess import PIPE from xml.etree.ElementTree import XML -from packaging.version import parse as parse_version import toml from mach.decorators import CommandArgument, CommandArgumentGroup from mach.registrar import Registrar +from servo.platform.build_target import BuildTarget, AndroidTarget, OpenHarmonyTarget +from servo.util import download_file, get_default_cache_dir + import servo.platform import servo.util as util -from servo.util import download_file, get_default_cache_dir NIGHTLY_REPOSITORY_URL = "https://servo-builds2.s3.amazonaws.com/" ASAN_LEAK_SUPPRESSION_FILE = "support/suppressed_leaks_for_asan.txt" @@ -256,9 +253,10 @@ class CommandBase(object): self.context = context self.enable_media = False self.features = [] - self.cross_compile_target = None - self.is_android_build = False - self.target_path = util.get_target_dir() + + # Default to native build target. This will later be overriden + # by `configure_build_target` + self.target = BuildTarget.from_triple(None) def get_env_bool(var, default): # Contents of env vars are strings by default. This returns the @@ -317,9 +315,6 @@ class CommandBase(object): self.config.setdefault("ohos", {}) self.config["ohos"].setdefault("ndk", "") - # Set default android target - self.setup_configuration_for_android_target("armv7-linux-androideabi") - _rust_toolchain = None def rust_toolchain(self): @@ -339,28 +334,16 @@ class CommandBase(object): apk_name = "servoapp.apk" return path.join(base_path, build_type.directory_name(), apk_name) - def get_binary_path(self, build_type: BuildType, target=None, android=False, asan=False): - if target is None and asan: - target = servo.platform.host_triple() - + def get_binary_path(self, build_type: BuildType, asan: bool = False): base_path = util.get_target_dir() - - if android: - base_path = path.join(base_path, self.config["android"]["target"]) - elif target: - base_path = path.join(base_path, target) - if android or (target is not None and "-ohos" in target): - return path.join(base_path, build_type.directory_name(), "libservoshell.so") - - binary_name = f"servo{servo.platform.get().executable_suffix()}" + if asan or self.target.is_cross_build(): + base_path = path.join(base_path, self.target.triple()) + binary_name = self.target.binary_name() binary_path = path.join(base_path, build_type.directory_name(), binary_name) if not path.exists(binary_path): - if target is None: - print("WARNING: Fallback to host-triplet prefixed target dirctory for binary path.") - return self.get_binary_path(build_type, target=servo.platform.host_triple(), android=android) - else: - raise BuildNotFound('No Servo binary found. Perhaps you forgot to run `./mach build`?') + raise BuildNotFound('No Servo binary found. Perhaps you forgot to run `./mach build`?') + return binary_path def detach_volume(self, mounted_volume): @@ -491,9 +474,9 @@ class CommandBase(object): # If we are installing on MacOS and Windows, we need to make sure that GStreamer's # `pkg-config` is on the path and takes precedence over other `pkg-config`s. - if self.enable_media and not self.is_android_build: + if self.enable_media: platform = servo.platform.get() - gstreamer_root = platform.gstreamer_root(cross_compilation_target=self.cross_compile_target) + gstreamer_root = platform.gstreamer_root(self.target) if gstreamer_root: util.prepend_paths_to_env(env, "PATH", os.path.join(gstreamer_root, "bin")) @@ -532,272 +515,10 @@ class CommandBase(object): # Suppress known false-positives during memory leak sanitizing. env["LSAN_OPTIONS"] = f"{env.get('LSAN_OPTIONS', '')}:suppressions={ASAN_LEAK_SUPPRESSION_FILE}" - self.build_android_env_if_needed(env) - self.build_ohos_env_if_needed(env) + self.target.configure_build_environment(env, self.config, self.context.topdir) return env - def build_android_env_if_needed(self, env: Dict[str, str]): - if not self.is_android_build: - return - - # Paths to Android build tools: - if self.config["android"]["sdk"]: - env["ANDROID_SDK_ROOT"] = self.config["android"]["sdk"] - if self.config["android"]["ndk"]: - env["ANDROID_NDK_ROOT"] = self.config["android"]["ndk"] - - toolchains = path.join(self.context.topdir, "android-toolchains") - for kind in ["sdk", "ndk"]: - default = os.path.join(toolchains, kind) - if os.path.isdir(default): - env.setdefault(f"ANDROID_{kind.upper()}_ROOT", default) - - if "IN_NIX_SHELL" in env and ("ANDROID_NDK_ROOT" not in env or "ANDROID_SDK_ROOT" not in env): - print("Please set SERVO_ANDROID_BUILD=1 when starting the Nix shell to include the Android SDK/NDK.") - sys.exit(1) - if "ANDROID_NDK_ROOT" not in env: - print("Please set the ANDROID_NDK_ROOT environment variable.") - sys.exit(1) - if "ANDROID_SDK_ROOT" not in env: - print("Please set the ANDROID_SDK_ROOT environment variable.") - sys.exit(1) - - android_platform = self.config["android"]["platform"] - android_toolchain_name = self.config["android"]["toolchain_name"] - android_lib = self.config["android"]["lib"] - - android_api = android_platform.replace('android-', '') - - # Check if the NDK version is 26 - if not os.path.isfile(path.join(env["ANDROID_NDK_ROOT"], 'source.properties')): - print("ANDROID_NDK should have file `source.properties`.") - print("The environment variable ANDROID_NDK_ROOT may be set at a wrong path.") - sys.exit(1) - with open(path.join(env["ANDROID_NDK_ROOT"], 'source.properties'), encoding="utf8") as ndk_properties: - lines = ndk_properties.readlines() - if lines[1].split(' = ')[1].split('.')[0] != '26': - print("Servo currently only supports NDK r26c.") - sys.exit(1) - - # Android builds also require having the gcc bits on the PATH and various INCLUDE - # path munging if you do not want to install a standalone NDK. See: - # https://dxr.mozilla.org/mozilla-central/source/build/autoconf/android.m4#139-161 - os_type = platform.system().lower() - if os_type not in ["linux", "darwin"]: - raise Exception("Android cross builds are only supported on Linux and macOS.") - - cpu_type = platform.machine().lower() - host_suffix = "unknown" - if cpu_type in ["i386", "i486", "i686", "i768", "x86"]: - host_suffix = "x86" - elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]: - host_suffix = "x86_64" - host = os_type + "-" + host_suffix - - host_cc = env.get('HOST_CC') or shutil.which("clang") - host_cxx = env.get('HOST_CXX') or shutil.which("clang++") - - llvm_toolchain = path.join(env['ANDROID_NDK_ROOT'], "toolchains", "llvm", "prebuilt", host) - env['PATH'] = (env['PATH'] + ':' + path.join(llvm_toolchain, "bin")) - - def to_ndk_bin(prog): - return path.join(llvm_toolchain, "bin", prog) - - # This workaround is due to an issue in the x86_64 Android NDK that introduces - # an undefined reference to the symbol '__extendsftf2'. - # See https://github.com/termux/termux-packages/issues/8029#issuecomment-1369150244 - if "x86_64" in self.cross_compile_target: - libclangrt_filename = subprocess.run( - [to_ndk_bin(f"x86_64-linux-android{android_api}-clang"), "--print-libgcc-file-name"], - check=True, - capture_output=True, - encoding="utf8" - ).stdout - env['RUSTFLAGS'] = env.get('RUSTFLAGS', "") - env["RUSTFLAGS"] += f"-C link-arg={libclangrt_filename}" - - env["RUST_TARGET"] = self.cross_compile_target - env['HOST_CC'] = host_cc - env['HOST_CXX'] = host_cxx - env['HOST_CFLAGS'] = '' - env['HOST_CXXFLAGS'] = '' - env['TARGET_CC'] = to_ndk_bin("clang") - env['TARGET_CPP'] = to_ndk_bin("clang") + " -E" - env['TARGET_CXX'] = to_ndk_bin("clang++") - - env['TARGET_AR'] = to_ndk_bin("llvm-ar") - env['TARGET_RANLIB'] = to_ndk_bin("llvm-ranlib") - env['TARGET_OBJCOPY'] = to_ndk_bin("llvm-objcopy") - env['TARGET_YASM'] = to_ndk_bin("yasm") - env['TARGET_STRIP'] = to_ndk_bin("llvm-strip") - env['RUST_FONTCONFIG_DLOPEN'] = "on" - - env["LIBCLANG_PATH"] = path.join(llvm_toolchain, "lib") - env["CLANG_PATH"] = to_ndk_bin("clang") - - # A cheat-sheet for some of the build errors caused by getting the search path wrong... - # - # fatal error: 'limits' file not found - # -- add -I cxx_include - # unknown type name '__locale_t' (when running bindgen in mozjs_sys) - # -- add -isystem sysroot_include - # error: use of undeclared identifier 'UINTMAX_C' - # -- add -D__STDC_CONSTANT_MACROS - # - # Also worth remembering: autoconf uses C for its configuration, - # even for C++ builds, so the C flags need to line up with the C++ flags. - env['TARGET_CFLAGS'] = "--target=" + android_toolchain_name - env['TARGET_CXXFLAGS'] = "--target=" + android_toolchain_name - - # These two variables are needed for the mozjs compilation. - env['ANDROID_API_LEVEL'] = android_api - env["ANDROID_NDK_HOME"] = env["ANDROID_NDK_ROOT"] - - # The two variables set below are passed by our custom - # support/android/toolchain.cmake to the NDK's CMake toolchain file - env["ANDROID_ABI"] = android_lib - env["ANDROID_PLATFORM"] = android_platform - env["NDK_CMAKE_TOOLCHAIN_FILE"] = path.join( - env['ANDROID_NDK_ROOT'], "build", "cmake", "android.toolchain.cmake") - env["CMAKE_TOOLCHAIN_FILE"] = path.join( - self.context.topdir, "support", "android", "toolchain.cmake") - - # Set output dir for gradle aar files - env["AAR_OUT_DIR"] = path.join(self.context.topdir, "target", "android", "aar") - if not os.path.exists(env['AAR_OUT_DIR']): - os.makedirs(env['AAR_OUT_DIR']) - - env['TARGET_PKG_CONFIG_SYSROOT_DIR'] = path.join(llvm_toolchain, 'sysroot') - - def build_ohos_env_if_needed(self, env: Dict[str, str]): - if not (self.cross_compile_target and self.cross_compile_target.endswith('-ohos')): - return - - # Paths to OpenHarmony SDK and build tools: - # Note: `OHOS_SDK_NATIVE` is the CMake variable name the `hvigor` build-system - # uses for the native directory of the SDK, so we use the same name to be consistent. - if "OHOS_SDK_NATIVE" not in env and self.config["ohos"]["ndk"]: - env["OHOS_SDK_NATIVE"] = self.config["ohos"]["ndk"] - - if "OHOS_SDK_NATIVE" not in env: - print("Please set the OHOS_SDK_NATIVE environment variable to the location of the `native` directory " - "in the OpenHarmony SDK.") - sys.exit(1) - - ndk_root = pathlib.Path(env["OHOS_SDK_NATIVE"]) - - if not ndk_root.is_dir(): - print(f"OHOS_SDK_NATIVE is not set to a valid directory: `{ndk_root}`") - sys.exit(1) - - ndk_root = ndk_root.resolve() - package_info = ndk_root.joinpath("oh-uni-package.json") - try: - with open(package_info) as meta_file: - meta = json.load(meta_file) - ohos_api_version = int(meta['apiVersion']) - ohos_sdk_version = parse_version(meta['version']) - if ohos_sdk_version < parse_version('4.0'): - print("Warning: mach build currently assumes at least the OpenHarmony 4.0 SDK is used.") - print(f"Info: The OpenHarmony SDK {ohos_sdk_version} is targeting API-level {ohos_api_version}") - except Exception as e: - print(f"Failed to read metadata information from {package_info}") - print(f"Exception: {e}") - - # The OpenHarmony SDK for Windows hosts currently does not contain a libclang shared library, - # which is required by `bindgen` (see issue - # https://gitee.com/openharmony/third_party_llvm-project/issues/I8H50W). Using upstream `clang` is currently - # also not easily possible, since `libcxx` support still needs to be upstreamed ( - # https://github.com/llvm/llvm-project/pull/73114). - os_type = platform.system().lower() - if os_type not in ["linux", "darwin"]: - raise Exception("OpenHarmony builds are currently only supported on Linux and macOS Hosts.") - - llvm_toolchain = ndk_root.joinpath("llvm") - llvm_bin = llvm_toolchain.joinpath("bin") - ohos_sysroot = ndk_root.joinpath("sysroot") - if not (llvm_toolchain.is_dir() and llvm_bin.is_dir()): - print(f"Expected to find `llvm` and `llvm/bin` folder under $OHOS_SDK_NATIVE at `{llvm_toolchain}`") - sys.exit(1) - if not ohos_sysroot.is_dir(): - print(f"Could not find OpenHarmony sysroot in {ndk_root}") - sys.exit(1) - - # Note: We don't use the `<target_triple>-clang` wrappers on purpose, since - # a) the OH 4.0 SDK does not have them yet AND - # b) the wrappers in the newer SDKs are bash scripts, which can cause problems - # on windows, depending on how the wrapper is called. - # Instead, we ensure that all the necessary flags for the c-compiler are set - # via environment variables such as `TARGET_CFLAGS`. - def to_sdk_llvm_bin(prog: str): - if is_windows(): - prog = prog + '.exe' - llvm_prog = llvm_bin.joinpath(prog) - if not llvm_prog.is_file(): - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), llvm_prog) - return str(llvm_bin.joinpath(prog)) - - # CC and CXX should already be set to appropriate host compilers by `build_env()` - env['HOST_CC'] = env['CC'] - env['HOST_CXX'] = env['CXX'] - env['TARGET_AR'] = to_sdk_llvm_bin("llvm-ar") - env['TARGET_RANLIB'] = to_sdk_llvm_bin("llvm-ranlib") - env['TARGET_READELF'] = to_sdk_llvm_bin("llvm-readelf") - env['TARGET_OBJCOPY'] = to_sdk_llvm_bin("llvm-objcopy") - env['TARGET_STRIP'] = to_sdk_llvm_bin("llvm-strip") - - rust_target_triple = str(self.cross_compile_target).replace('-', '_') - ndk_clang = to_sdk_llvm_bin(f"{self.cross_compile_target}-clang") - ndk_clangxx = to_sdk_llvm_bin(f"{self.cross_compile_target}-clang++") - env[f'CC_{rust_target_triple}'] = ndk_clang - env[f'CXX_{rust_target_triple}'] = ndk_clangxx - # The clang target name is different from the LLVM target name - clang_target_triple = str(self.cross_compile_target).replace('-unknown-', '-') - clang_target_triple_underscore = clang_target_triple.replace('-', '_') - env[f'CC_{clang_target_triple_underscore}'] = ndk_clang - env[f'CXX_{clang_target_triple_underscore}'] = ndk_clangxx - # rustc linker - env[f'CARGO_TARGET_{rust_target_triple.upper()}_LINKER'] = ndk_clang - # We could also use a cross-compile wrapper - env["RUSTFLAGS"] += f' -Clink-arg=--target={clang_target_triple}' - env["RUSTFLAGS"] += f' -Clink-arg=--sysroot={ohos_sysroot}' - - env['HOST_CFLAGS'] = '' - env['HOST_CXXFLAGS'] = '' - ohos_cflags = ['-D__MUSL__', f' --target={clang_target_triple}', f' --sysroot={ohos_sysroot}'] - if clang_target_triple.startswith('armv7-'): - ohos_cflags.extend(['-march=armv7-a', '-mfloat-abi=softfp', '-mtune=generic-armv7-a', '-mthumb']) - ohos_cflags_str = " ".join(ohos_cflags) - env['TARGET_CFLAGS'] = ohos_cflags_str - env['TARGET_CPPFLAGS'] = '-D__MUSL__' - env['TARGET_CXXFLAGS'] = ohos_cflags_str - - # CMake related flags - cmake_toolchain_file = ndk_root.joinpath("build", "cmake", "ohos.toolchain.cmake") - if cmake_toolchain_file.is_file(): - env[f'CMAKE_TOOLCHAIN_FILE_{rust_target_triple}'] = str(cmake_toolchain_file) - else: - print( - f"Warning: Failed to find the OpenHarmony CMake Toolchain file - Expected it at {cmake_toolchain_file}") - env[f'CMAKE_C_COMPILER_{rust_target_triple}'] = ndk_clang - env[f'CMAKE_CXX_COMPILER_{rust_target_triple}'] = ndk_clangxx - - # pkg-config - pkg_config_path = '{}:{}'.format(str(ohos_sysroot.joinpath("usr", "lib", "pkgconfig")), - str(ohos_sysroot.joinpath("usr", "share", "pkgconfig"))) - env[f'PKG_CONFIG_SYSROOT_DIR_{rust_target_triple}'] = str(ohos_sysroot) - env[f'PKG_CONFIG_PATH_{rust_target_triple}'] = pkg_config_path - - # bindgen / libclang-sys - env["LIBCLANG_PATH"] = path.join(llvm_toolchain, "lib") - env["CLANG_PATH"] = ndk_clangxx - env[f'CXXSTDLIB_{clang_target_triple_underscore}'] = "c++" - bindgen_extra_clangs_args_var = f'BINDGEN_EXTRA_CLANG_ARGS_{rust_target_triple}' - bindgen_extra_clangs_args = env.get(bindgen_extra_clangs_args_var, "") - bindgen_extra_clangs_args = bindgen_extra_clangs_args + " " + ohos_cflags_str - env[bindgen_extra_clangs_args_var] = bindgen_extra_clangs_args - @staticmethod def common_command_arguments(build_configuration=False, build_type=False): decorators = [] @@ -884,7 +605,7 @@ class CommandBase(object): kwargs.pop('profile', None) if build_configuration: - self.configure_cross_compilation(kwargs['target'], kwargs['android'], kwargs['win_arm64']) + self.configure_build_target(kwargs) self.features = kwargs.get("features", None) or [] self.enable_media = self.is_media_enabled(kwargs['media_stack']) @@ -898,6 +619,16 @@ class CommandBase(object): return decorator_function + @staticmethod + def allow_target_configuration(original_function): + def target_configuration_decorator(self, *args, **kwargs): + self.configure_build_target(kwargs, suppress_log=True) + kwargs.pop('target', False) + kwargs.pop('android', False) + return original_function(self, *args, **kwargs) + + return target_configuration_decorator + def configure_build_type(self, release: bool, dev: bool, prod: bool, profile: Optional[str]) -> BuildType: option_count = release + dev + prod + (profile is not None) @@ -926,33 +657,32 @@ class CommandBase(object): else: return BuildType.custom(profile) - def configure_cross_compilation( - self, - cross_compile_target: Optional[str], - android: Optional[str], - win_arm64: Optional[str]): - # Force the UWP-enabled target if the convenience UWP flags are passed. - if android is None: - android = self.config["build"]["android"] - if android: - if not cross_compile_target: - cross_compile_target = self.config["android"]["target"] - assert cross_compile_target - assert self.setup_configuration_for_android_target(cross_compile_target) - elif cross_compile_target: - # If a target was specified, it might also be an android target, - # so set up the configuration in that case. - self.setup_configuration_for_android_target(cross_compile_target) - - self.cross_compile_target = cross_compile_target - self.is_android_build = (cross_compile_target and "android" in cross_compile_target) - self.target_path = servo.util.get_target_dir() - if self.is_android_build: - assert self.cross_compile_target - self.target_path = path.join(self.target_path, "android", self.cross_compile_target) - - if self.cross_compile_target: - print(f"Targeting '{self.cross_compile_target}' for cross-compilation") + def configure_build_target(self, kwargs: Dict[str, Any], suppress_log: bool = False): + if hasattr(self.context, 'target'): + # This call is for a dispatched command and we've already configured + # the target, so just use it. + self.target = self.context.target + return + + android = kwargs.get('android') or self.config["build"]["android"] + target_triple = kwargs.get('target') + + if android and target_triple: + print("Please specify either --target or --android.") + sys.exit(1) + + # Set the default Android target + if android and not target_triple: + target_triple = "armv7-linux-androideabi" + + self.target = BuildTarget.from_triple(target_triple) + + self.context.target = self.target + if self.target.is_cross_build() and not suppress_log: + print(f"Targeting '{self.target.triple()}' for cross-compilation") + + def is_android(self): + return isinstance(self.target, AndroidTarget) def is_media_enabled(self, media_stack: Optional[str]): """Determine whether media is enabled based on the value of the build target @@ -962,7 +692,7 @@ class CommandBase(object): if self.config["build"]["media-stack"] != "auto": media_stack = self.config["build"]["media-stack"] assert media_stack - elif not self.cross_compile_target: + elif not self.target.is_cross_build(): media_stack = "gstreamer" else: media_stack = "dummy" @@ -971,8 +701,8 @@ class CommandBase(object): # Once we drop support for this platform (it's currently needed for wpt.fyi runners), # we can remove this workaround and officially only support Ubuntu 22.04 and up. platform = servo.platform.get() - if not self.cross_compile_target and platform.is_linux and \ - not platform.is_gstreamer_installed(self.cross_compile_target): + if not self.target.is_cross_build() and platform.is_linux and \ + not platform.is_gstreamer_installed(self.target): return False return media_stack != "dummy" @@ -988,12 +718,10 @@ class CommandBase(object): ): env = env or self.build_env() - # Android GStreamer integration is handled elsewhere. # NB: On non-Linux platforms we cannot check whether GStreamer is installed until # environment variables are set via `self.build_env()`. platform = servo.platform.get() - if self.enable_media and not self.is_android_build and \ - not platform.is_gstreamer_installed(self.cross_compile_target): + if self.enable_media and not platform.is_gstreamer_installed(self.target): raise FileNotFoundError( "GStreamer libraries not found (>= version 1.18)." "Please see installation instructions in README.md" @@ -1007,9 +735,9 @@ class CommandBase(object): ] if target_override: args += ["--target", target_override] - elif self.cross_compile_target: - args += ["--target", self.cross_compile_target] - if self.is_android_build or '-ohos' in self.cross_compile_target: + elif self.target.is_cross_build(): + args += ["--target", self.target.triple()] + if type(self.target) in [AndroidTarget, OpenHarmonyTarget]: # Note: in practice `cargo rustc` should just be used unconditionally. assert command != 'build', "For Android / OpenHarmony `cargo rustc` must be used instead of cargo build" if command == 'rustc': @@ -1066,57 +794,17 @@ class CommandBase(object): return sdk_adb return "emulator" - def setup_configuration_for_android_target(self, target: str): - """If cross-compilation targets Android, configure the Android - build by writing the appropriate toolchain configuration values - into the stored configuration.""" - if target == "armv7-linux-androideabi": - self.config["android"]["platform"] = "android-30" - self.config["android"]["target"] = target - self.config["android"]["toolchain_prefix"] = "arm-linux-androideabi" - self.config["android"]["arch"] = "arm" - self.config["android"]["lib"] = "armeabi-v7a" - self.config["android"]["toolchain_name"] = "armv7a-linux-androideabi30" - return True - elif target == "aarch64-linux-android": - self.config["android"]["platform"] = "android-30" - self.config["android"]["target"] = target - self.config["android"]["toolchain_prefix"] = target - self.config["android"]["arch"] = "arm64" - self.config["android"]["lib"] = "arm64-v8a" - self.config["android"]["toolchain_name"] = "aarch64-linux-androideabi30" - return True - elif target == "i686-linux-android": - # https://github.com/jemalloc/jemalloc/issues/1279 - self.config["android"]["platform"] = "android-30" - self.config["android"]["target"] = target - self.config["android"]["toolchain_prefix"] = target - self.config["android"]["arch"] = "x86" - self.config["android"]["lib"] = "x86" - self.config["android"]["toolchain_name"] = "i686-linux-android30" - return True - elif target == "x86_64-linux-android": - self.config["android"]["platform"] = "android-30" - self.config["android"]["target"] = target - self.config["android"]["toolchain_prefix"] = target - self.config["android"]["arch"] = "x86_64" - self.config["android"]["lib"] = "x86_64" - self.config["android"]["toolchain_name"] = "x86_64-linux-android30" - return True - return False - def ensure_bootstrapped(self): if self.context.bootstrapped: return servo.platform.get().passive_bootstrap() - needs_toolchain_install = self.cross_compile_target and \ - self.cross_compile_target not in \ + needs_toolchain_install = self.target.triple() not in \ check_output(["rustup", "target", "list", "--installed"], cwd=self.context.topdir).decode() if needs_toolchain_install: - check_call(["rustup", "target", "add", self.cross_compile_target], + check_call(["rustup", "target", "add", self.target.triple()], cwd=self.context.topdir) self.context.bootstrapped = True diff --git a/python/servo/devenv_commands.py b/python/servo/devenv_commands.py index 60df5eb1052..0c30bc1af5b 100644 --- a/python/servo/devenv_commands.py +++ b/python/servo/devenv_commands.py @@ -205,7 +205,6 @@ class MachCommands(CommandBase): print(logfile + " doesn't exist") return -1 - self.cross_compile_target = target env = self.build_env() ndk_stack = path.join(env["ANDROID_NDK"], "ndk-stack") self.setup_configuration_for_android_target(target) @@ -226,8 +225,6 @@ class MachCommands(CommandBase): @CommandArgument('--target', action='store', default="armv7-linux-androideabi", help="Build target") def ndk_gdb(self, release, target): - self.cross_compile_target = target - self.setup_configuration_for_android_target(target) env = self.build_env() ndk_gdb = path.join(env["ANDROID_NDK"], "ndk-gdb") adb_path = path.join(env["ANDROID_SDK"], "platform-tools", "adb") diff --git a/python/servo/gstreamer.py b/python/servo/gstreamer.py index fea93660dbb..72b80edbf4a 100644 --- a/python/servo/gstreamer.py +++ b/python/servo/gstreamer.py @@ -13,6 +13,11 @@ import subprocess import sys from typing import Set +# This file is called as a script from components/servo/build.rs, so +# we need to explicitly modify the search path here. +sys.path[0:0] = [os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))] +from servo.platform.build_target import BuildTarget # noqa: E402 + GSTREAMER_BASE_LIBS = [ # gstreamer "gstbase", @@ -242,7 +247,7 @@ def find_non_system_dependencies_with_otool(binary_path: str) -> Set[str]: return output -def package_gstreamer_dylibs(binary_path: str, library_target_directory: str, cross_compilation_target: str = None): +def package_gstreamer_dylibs(binary_path: str, library_target_directory: str, target: BuildTarget): """Copy all GStreamer dependencies to the "lib" subdirectory of a built version of Servo. Also update any transitive shared library paths so that they are relative to this subdirectory.""" @@ -250,7 +255,7 @@ def package_gstreamer_dylibs(binary_path: str, library_target_directory: str, cr # This import only works when called from `mach`. import servo.platform - gstreamer_root = servo.platform.get().gstreamer_root(cross_compilation_target) + gstreamer_root = servo.platform.get().gstreamer_root(target) gstreamer_version = servo.platform.macos.GSTREAMER_PLUGIN_VERSION gstreamer_root_libs = os.path.join(gstreamer_root, "lib") diff --git a/python/servo/package_commands.py b/python/servo/package_commands.py index 4ffebe92cff..27a69b48cfb 100644 --- a/python/servo/package_commands.py +++ b/python/servo/package_commands.py @@ -132,31 +132,21 @@ class PackageCommands(CommandBase): default=None, help='Package using the given Gradle flavor') @CommandBase.common_command_arguments(build_configuration=False, build_type=True) - def package(self, build_type: BuildType, android=None, target=None, flavor=None, with_asan=False): - if android is None: - android = self.config["build"]["android"] - if target and android: - print("Please specify either --target or --android.") - sys.exit(1) - if not android: - android = self.setup_configuration_for_android_target(target) - else: - target = self.config["android"]["target"] - - self.cross_compile_target = target + @CommandBase.allow_target_configuration + def package(self, build_type: BuildType, flavor=None, with_asan=False): env = self.build_env() - binary_path = self.get_binary_path(build_type, target=target, android=android, asan=with_asan) + binary_path = self.get_binary_path(build_type, asan=with_asan) dir_to_root = self.get_top_dir() target_dir = path.dirname(binary_path) - if android: - android_target = self.config["android"]["target"] - if "aarch64" in android_target: + if self.is_android(): + target_triple = self.target.triple() + if "aarch64" in target_triple: arch_string = "Arm64" - elif "armv7" in android_target: + elif "armv7" in target_triple: arch_string = "Armv7" - elif "i686" in android_target: + elif "i686" in target_triple: arch_string = "x86" - elif "x86_64" in android_target: + elif "x86_64" in target_triple: arch_string = "x64" else: arch_string = "Arm" @@ -211,7 +201,7 @@ class PackageCommands(CommandBase): print("Packaging GStreamer...") dmg_binary = path.join(content_dir, "servo") - servo.gstreamer.package_gstreamer_dylibs(dmg_binary, lib_dir) + servo.gstreamer.package_gstreamer_dylibs(dmg_binary, lib_dir, self.target) print("Adding version to Credits.rtf") version_command = [binary_path, '--version'] @@ -368,30 +358,25 @@ class PackageCommands(CommandBase): default=None, help='Install the given target platform') @CommandBase.common_command_arguments(build_configuration=False, build_type=True) - def install(self, build_type: BuildType, android=False, emulator=False, usb=False, target=None, with_asan=False): - if target and android: - print("Please specify either --target or --android.") - sys.exit(1) - if not android: - android = self.setup_configuration_for_android_target(target) - self.cross_compile_target = target - + @CommandBase.allow_target_configuration + def install(self, build_type: BuildType, emulator=False, usb=False, with_asan=False): env = self.build_env() try: - binary_path = self.get_binary_path(build_type, android=android, asan=with_asan) + binary_path = self.get_binary_path(build_type, asan=with_asan) except BuildNotFound: print("Servo build not found. Building servo...") result = Registrar.dispatch( - "build", context=self.context, build_type=build_type, android=android, + "build", context=self.context, build_type=build_type ) if result: return result try: - binary_path = self.get_binary_path(build_type, android=android, asan=with_asan) + binary_path = self.get_binary_path(build_type, asan=with_asan) except BuildNotFound: print("Rebuilding Servo did not solve the missing build problem.") return 1 - if android: + + if self.is_android(): pkg_path = self.get_apk_path(build_type) exec_command = [self.android_adb_path(env)] if emulator and usb: @@ -409,7 +394,7 @@ class PackageCommands(CommandBase): if not path.exists(pkg_path): print("Servo package not found. Packaging servo...") result = Registrar.dispatch( - "package", context=self.context, build_type=build_type, android=android, + "package", context=self.context, build_type=build_type ) if result != 0: return result diff --git a/python/servo/platform/base.py b/python/servo/platform/base.py index 3cfbd09b618..6fb2f535f4e 100644 --- a/python/servo/platform/base.py +++ b/python/servo/platform/base.py @@ -12,6 +12,8 @@ import shutil import subprocess from typing import Optional +from .build_target import BuildTarget + class Base: def __init__(self, triple: str): @@ -21,7 +23,7 @@ class Base: self.is_linux = False self.is_macos = False - def gstreamer_root(self, _cross_compilation_target: Optional[str]) -> Optional[str]: + def gstreamer_root(self, target: BuildTarget) -> Optional[str]: raise NotImplementedError("Do not know how to get GStreamer path for platform.") def executable_suffix(self) -> str: @@ -30,13 +32,13 @@ class Base: def _platform_bootstrap(self, _force: bool) -> bool: raise NotImplementedError("Bootstrap installation detection not yet available.") - def _platform_bootstrap_gstreamer(self, _force: bool) -> bool: + def _platform_bootstrap_gstreamer(self, _target: BuildTarget, _force: bool) -> bool: raise NotImplementedError( "GStreamer bootstrap support is not yet available for your OS." ) - def is_gstreamer_installed(self, cross_compilation_target: Optional[str]) -> bool: - gstreamer_root = self.gstreamer_root(cross_compilation_target) + def is_gstreamer_installed(self, target: BuildTarget) -> bool: + gstreamer_root = self.gstreamer_root(target) if gstreamer_root: pkg_config = os.path.join(gstreamer_root, "bin", "pkg-config") else: @@ -100,8 +102,9 @@ class Base: return False def bootstrap_gstreamer(self, force: bool): - if not self._platform_bootstrap_gstreamer(force): - root = self.gstreamer_root(None) + target = BuildTarget.from_triple(self.triple) + if not self._platform_bootstrap_gstreamer(target, force): + root = self.gstreamer_root(target) if root: print(f"GStreamer found at: {root}") else: diff --git a/python/servo/platform/build_target.py b/python/servo/platform/build_target.py new file mode 100644 index 00000000000..bf795386b5a --- /dev/null +++ b/python/servo/platform/build_target.py @@ -0,0 +1,372 @@ +# Copyright 2024 The Servo Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution. +# +# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import errno +import json +import os +import pathlib +import platform +import shutil +import subprocess +import sys + +from os import path +from packaging.version import parse as parse_version +from typing import Any, Dict, Optional + +import servo.platform + + +class BuildTarget(object): + def __init__(self, target_triple: str): + self.target_triple = target_triple + + @staticmethod + def from_triple(target_triple: Optional[str]) -> 'BuildTarget': + host_triple = servo.platform.host_triple() + if target_triple: + if 'android' in target_triple: + return AndroidTarget(target_triple) + elif 'ohos' in target_triple: + return OpenHarmonyTarget(target_triple) + elif target_triple != host_triple: + raise Exception(f"Unknown build target {target_triple}") + return BuildTarget(host_triple) + + def triple(self) -> str: + return self.target_triple + + def binary_name(self) -> str: + return f"servo{servo.platform.get().executable_suffix()}" + + def configure_build_environment(self, env: Dict[str, str], config: Dict[str, Any], topdir: pathlib.Path): + pass + + def is_cross_build(self) -> bool: + return False + + def needs_packaging(self) -> bool: + return False + + +class CrossBuildTarget(BuildTarget): + def is_cross_build(self) -> bool: + return True + + +class AndroidTarget(CrossBuildTarget): + def ndk_configuration(self) -> Dict[str, str]: + target = self.triple() + config = {} + if target == "armv7-linux-androideabi": + config["platform"] = "android-30" + config["target"] = target + config["toolchain_prefix"] = "arm-linux-androideabi" + config["arch"] = "arm" + config["lib"] = "armeabi-v7a" + config["toolchain_name"] = "armv7a-linux-androideabi30" + elif target == "aarch64-linux-android": + config["platform"] = "android-30" + config["target"] = target + config["toolchain_prefix"] = target + config["arch"] = "arm64" + config["lib"] = "arm64-v8a" + config["toolchain_name"] = "aarch64-linux-androideabi30" + elif target == "i686-linux-android": + # https://github.com/jemalloc/jemalloc/issues/1279 + config["platform"] = "android-30" + config["target"] = target + config["toolchain_prefix"] = target + config["arch"] = "x86" + config["lib"] = "x86" + config["toolchain_name"] = "i686-linux-android30" + elif target == "x86_64-linux-android": + config["platform"] = "android-30" + config["target"] = target + config["toolchain_prefix"] = target + config["arch"] = "x86_64" + config["lib"] = "x86_64" + config["toolchain_name"] = "x86_64-linux-android30" + else: + raise Exception(f"Unknown android target {target}") + + return config + + def configure_build_environment(self, env: Dict[str, str], config: Dict[str, Any], topdir: pathlib.Path): + # Paths to Android build tools: + if config["android"]["sdk"]: + env["ANDROID_SDK_ROOT"] = config["android"]["sdk"] + if config["android"]["ndk"]: + env["ANDROID_NDK_ROOT"] = config["android"]["ndk"] + + toolchains = path.join(topdir, "android-toolchains") + for kind in ["sdk", "ndk"]: + default = os.path.join(toolchains, kind) + if os.path.isdir(default): + env.setdefault(f"ANDROID_{kind.upper()}_ROOT", default) + + if "IN_NIX_SHELL" in env and ("ANDROID_NDK_ROOT" not in env or "ANDROID_SDK_ROOT" not in env): + print("Please set SERVO_ANDROID_BUILD=1 when starting the Nix shell to include the Android SDK/NDK.") + sys.exit(1) + if "ANDROID_NDK_ROOT" not in env: + print("Please set the ANDROID_NDK_ROOT environment variable.") + sys.exit(1) + if "ANDROID_SDK_ROOT" not in env: + print("Please set the ANDROID_SDK_ROOT environment variable.") + sys.exit(1) + + ndk_configuration = self.ndk_configuration() + android_platform = ndk_configuration["platform"] + android_toolchain_name = ndk_configuration["toolchain_name"] + android_lib = ndk_configuration["lib"] + + android_api = android_platform.replace('android-', '') + + # Check if the NDK version is 26 + if not os.path.isfile(path.join(env["ANDROID_NDK_ROOT"], 'source.properties')): + print("ANDROID_NDK should have file `source.properties`.") + print("The environment variable ANDROID_NDK_ROOT may be set at a wrong path.") + sys.exit(1) + with open(path.join(env["ANDROID_NDK_ROOT"], 'source.properties'), encoding="utf8") as ndk_properties: + lines = ndk_properties.readlines() + if lines[1].split(' = ')[1].split('.')[0] != '26': + print("Servo currently only supports NDK r26c.") + sys.exit(1) + + # Android builds also require having the gcc bits on the PATH and various INCLUDE + # path munging if you do not want to install a standalone NDK. See: + # https://dxr.mozilla.org/mozilla-central/source/build/autoconf/android.m4#139-161 + os_type = platform.system().lower() + if os_type not in ["linux", "darwin"]: + raise Exception("Android cross builds are only supported on Linux and macOS.") + + cpu_type = platform.machine().lower() + host_suffix = "unknown" + if cpu_type in ["i386", "i486", "i686", "i768", "x86"]: + host_suffix = "x86" + elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]: + host_suffix = "x86_64" + host = os_type + "-" + host_suffix + + host_cc = env.get('HOST_CC') or shutil.which("clang") + host_cxx = env.get('HOST_CXX') or shutil.which("clang++") + + llvm_toolchain = path.join(env['ANDROID_NDK_ROOT'], "toolchains", "llvm", "prebuilt", host) + env['PATH'] = (env['PATH'] + ':' + path.join(llvm_toolchain, "bin")) + + def to_ndk_bin(prog): + return path.join(llvm_toolchain, "bin", prog) + + # This workaround is due to an issue in the x86_64 Android NDK that introduces + # an undefined reference to the symbol '__extendsftf2'. + # See https://github.com/termux/termux-packages/issues/8029#issuecomment-1369150244 + if "x86_64" in self.triple(): + libclangrt_filename = subprocess.run( + [to_ndk_bin(f"x86_64-linux-android{android_api}-clang"), "--print-libgcc-file-name"], + check=True, + capture_output=True, + encoding="utf8" + ).stdout + env['RUSTFLAGS'] = env.get('RUSTFLAGS', "") + env["RUSTFLAGS"] += f"-C link-arg={libclangrt_filename}" + + env["RUST_TARGET"] = self.triple() + env['HOST_CC'] = host_cc + env['HOST_CXX'] = host_cxx + env['HOST_CFLAGS'] = '' + env['HOST_CXXFLAGS'] = '' + env['TARGET_CC'] = to_ndk_bin("clang") + env['TARGET_CPP'] = to_ndk_bin("clang") + " -E" + env['TARGET_CXX'] = to_ndk_bin("clang++") + + env['TARGET_AR'] = to_ndk_bin("llvm-ar") + env['TARGET_RANLIB'] = to_ndk_bin("llvm-ranlib") + env['TARGET_OBJCOPY'] = to_ndk_bin("llvm-objcopy") + env['TARGET_YASM'] = to_ndk_bin("yasm") + env['TARGET_STRIP'] = to_ndk_bin("llvm-strip") + env['RUST_FONTCONFIG_DLOPEN'] = "on" + + env["LIBCLANG_PATH"] = path.join(llvm_toolchain, "lib") + env["CLANG_PATH"] = to_ndk_bin("clang") + + # A cheat-sheet for some of the build errors caused by getting the search path wrong... + # + # fatal error: 'limits' file not found + # -- add -I cxx_include + # unknown type name '__locale_t' (when running bindgen in mozjs_sys) + # -- add -isystem sysroot_include + # error: use of undeclared identifier 'UINTMAX_C' + # -- add -D__STDC_CONSTANT_MACROS + # + # Also worth remembering: autoconf uses C for its configuration, + # even for C++ builds, so the C flags need to line up with the C++ flags. + env['TARGET_CFLAGS'] = "--target=" + android_toolchain_name + env['TARGET_CXXFLAGS'] = "--target=" + android_toolchain_name + + # These two variables are needed for the mozjs compilation. + env['ANDROID_API_LEVEL'] = android_api + env["ANDROID_NDK_HOME"] = env["ANDROID_NDK_ROOT"] + + # The two variables set below are passed by our custom + # support/android/toolchain.cmake to the NDK's CMake toolchain file + env["ANDROID_ABI"] = android_lib + env["ANDROID_PLATFORM"] = android_platform + env["NDK_CMAKE_TOOLCHAIN_FILE"] = path.join( + env['ANDROID_NDK_ROOT'], "build", "cmake", "android.toolchain.cmake") + env["CMAKE_TOOLCHAIN_FILE"] = path.join(topdir, "support", "android", "toolchain.cmake") + + # Set output dir for gradle aar files + env["AAR_OUT_DIR"] = path.join(topdir, "target", "android", "aar") + if not os.path.exists(env['AAR_OUT_DIR']): + os.makedirs(env['AAR_OUT_DIR']) + + env['TARGET_PKG_CONFIG_SYSROOT_DIR'] = path.join(llvm_toolchain, 'sysroot') + + def binary_name(self) -> str: + return "libservoshell.so" + + def is_cross_build(self) -> bool: + return True + + def needs_packaging(self) -> bool: + return True + + +class OpenHarmonyTarget(CrossBuildTarget): + def configure_build_environment(self, env: Dict[str, str], config: Dict[str, Any], topdir: pathlib.Path): + # Paths to OpenHarmony SDK and build tools: + # Note: `OHOS_SDK_NATIVE` is the CMake variable name the `hvigor` build-system + # uses for the native directory of the SDK, so we use the same name to be consistent. + if "OHOS_SDK_NATIVE" not in env and config["ohos"]["ndk"]: + env["OHOS_SDK_NATIVE"] = config["ohos"]["ndk"] + + if "OHOS_SDK_NATIVE" not in env: + print("Please set the OHOS_SDK_NATIVE environment variable to the location of the `native` directory " + "in the OpenHarmony SDK.") + sys.exit(1) + + ndk_root = pathlib.Path(env["OHOS_SDK_NATIVE"]) + + if not ndk_root.is_dir(): + print(f"OHOS_SDK_NATIVE is not set to a valid directory: `{ndk_root}`") + sys.exit(1) + + ndk_root = ndk_root.resolve() + package_info = ndk_root.joinpath("oh-uni-package.json") + try: + with open(package_info) as meta_file: + meta = json.load(meta_file) + ohos_api_version = int(meta['apiVersion']) + ohos_sdk_version = parse_version(meta['version']) + if ohos_sdk_version < parse_version('4.0'): + print("Warning: mach build currently assumes at least the OpenHarmony 4.0 SDK is used.") + print(f"Info: The OpenHarmony SDK {ohos_sdk_version} is targeting API-level {ohos_api_version}") + except Exception as e: + print(f"Failed to read metadata information from {package_info}") + print(f"Exception: {e}") + + # The OpenHarmony SDK for Windows hosts currently does not contain a libclang shared library, + # which is required by `bindgen` (see issue + # https://gitee.com/openharmony/third_party_llvm-project/issues/I8H50W). Using upstream `clang` is currently + # also not easily possible, since `libcxx` support still needs to be upstreamed ( + # https://github.com/llvm/llvm-project/pull/73114). + os_type = platform.system().lower() + if os_type not in ["linux", "darwin"]: + raise Exception("OpenHarmony builds are currently only supported on Linux and macOS Hosts.") + + llvm_toolchain = ndk_root.joinpath("llvm") + llvm_bin = llvm_toolchain.joinpath("bin") + ohos_sysroot = ndk_root.joinpath("sysroot") + if not (llvm_toolchain.is_dir() and llvm_bin.is_dir()): + print(f"Expected to find `llvm` and `llvm/bin` folder under $OHOS_SDK_NATIVE at `{llvm_toolchain}`") + sys.exit(1) + if not ohos_sysroot.is_dir(): + print(f"Could not find OpenHarmony sysroot in {ndk_root}") + sys.exit(1) + + # Note: We don't use the `<target_triple>-clang` wrappers on purpose, since + # a) the OH 4.0 SDK does not have them yet AND + # b) the wrappers in the newer SDKs are bash scripts, which can cause problems + # on windows, depending on how the wrapper is called. + # Instead, we ensure that all the necessary flags for the c-compiler are set + # via environment variables such as `TARGET_CFLAGS`. + def to_sdk_llvm_bin(prog: str): + if sys.platform == 'win32': + prog = prog + '.exe' + llvm_prog = llvm_bin.joinpath(prog) + if not llvm_prog.is_file(): + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), llvm_prog) + return str(llvm_bin.joinpath(prog)) + + # CC and CXX should already be set to appropriate host compilers by `build_env()` + env['HOST_CC'] = env['CC'] + env['HOST_CXX'] = env['CXX'] + env['TARGET_AR'] = to_sdk_llvm_bin("llvm-ar") + env['TARGET_RANLIB'] = to_sdk_llvm_bin("llvm-ranlib") + env['TARGET_READELF'] = to_sdk_llvm_bin("llvm-readelf") + env['TARGET_OBJCOPY'] = to_sdk_llvm_bin("llvm-objcopy") + env['TARGET_STRIP'] = to_sdk_llvm_bin("llvm-strip") + + target_triple = self.triple() + rust_target_triple = str(target_triple).replace('-', '_') + ndk_clang = to_sdk_llvm_bin(f"{target_triple}-clang") + ndk_clangxx = to_sdk_llvm_bin(f"{target_triple}-clang++") + env[f'CC_{rust_target_triple}'] = ndk_clang + env[f'CXX_{rust_target_triple}'] = ndk_clangxx + # The clang target name is different from the LLVM target name + clang_target_triple = str(target_triple).replace('-unknown-', '-') + clang_target_triple_underscore = clang_target_triple.replace('-', '_') + env[f'CC_{clang_target_triple_underscore}'] = ndk_clang + env[f'CXX_{clang_target_triple_underscore}'] = ndk_clangxx + # rustc linker + env[f'CARGO_TARGET_{rust_target_triple.upper()}_LINKER'] = ndk_clang + # We could also use a cross-compile wrapper + env["RUSTFLAGS"] += f' -Clink-arg=--target={clang_target_triple}' + env["RUSTFLAGS"] += f' -Clink-arg=--sysroot={ohos_sysroot}' + + env['HOST_CFLAGS'] = '' + env['HOST_CXXFLAGS'] = '' + ohos_cflags = ['-D__MUSL__', f' --target={clang_target_triple}', f' --sysroot={ohos_sysroot}'] + if clang_target_triple.startswith('armv7-'): + ohos_cflags.extend(['-march=armv7-a', '-mfloat-abi=softfp', '-mtune=generic-armv7-a', '-mthumb']) + ohos_cflags_str = " ".join(ohos_cflags) + env['TARGET_CFLAGS'] = ohos_cflags_str + env['TARGET_CPPFLAGS'] = '-D__MUSL__' + env['TARGET_CXXFLAGS'] = ohos_cflags_str + + # CMake related flags + cmake_toolchain_file = ndk_root.joinpath("build", "cmake", "ohos.toolchain.cmake") + if cmake_toolchain_file.is_file(): + env[f'CMAKE_TOOLCHAIN_FILE_{rust_target_triple}'] = str(cmake_toolchain_file) + else: + print( + f"Warning: Failed to find the OpenHarmony CMake Toolchain file - Expected it at {cmake_toolchain_file}") + env[f'CMAKE_C_COMPILER_{rust_target_triple}'] = ndk_clang + env[f'CMAKE_CXX_COMPILER_{rust_target_triple}'] = ndk_clangxx + + # pkg-config + pkg_config_path = '{}:{}'.format(str(ohos_sysroot.joinpath("usr", "lib", "pkgconfig")), + str(ohos_sysroot.joinpath("usr", "share", "pkgconfig"))) + env[f'PKG_CONFIG_SYSROOT_DIR_{rust_target_triple}'] = str(ohos_sysroot) + env[f'PKG_CONFIG_PATH_{rust_target_triple}'] = pkg_config_path + + # bindgen / libclang-sys + env["LIBCLANG_PATH"] = path.join(llvm_toolchain, "lib") + env["CLANG_PATH"] = ndk_clangxx + env[f'CXXSTDLIB_{clang_target_triple_underscore}'] = "c++" + bindgen_extra_clangs_args_var = f'BINDGEN_EXTRA_CLANG_ARGS_{rust_target_triple}' + bindgen_extra_clangs_args = env.get(bindgen_extra_clangs_args_var, "") + bindgen_extra_clangs_args = bindgen_extra_clangs_args + " " + ohos_cflags_str + env[bindgen_extra_clangs_args_var] = bindgen_extra_clangs_args + + def binary_name(self) -> str: + return "libservoshell.so" + + def needs_packaging(self) -> bool: + return True diff --git a/python/servo/platform/linux.py b/python/servo/platform/linux.py index 6ed34ee9892..1397a4f16de 100644 --- a/python/servo/platform/linux.py +++ b/python/servo/platform/linux.py @@ -7,12 +7,13 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. +import distro import os import subprocess from typing import Optional, Tuple -import distro from .base import Base +from .build_target import BuildTarget # Please keep these in sync with the packages on the wiki, using the instructions below # https://github.com/servo/servo/wiki/Building @@ -211,10 +212,10 @@ class Linux(Base): raise EnvironmentError("Installation of dependencies failed.") return True - def gstreamer_root(self, cross_compilation_target: Optional[str]) -> Optional[str]: + def gstreamer_root(self, _target: BuildTarget) -> Optional[str]: return None - def _platform_bootstrap_gstreamer(self, _force: bool) -> bool: + def _platform_bootstrap_gstreamer(self, _target: BuildTarget, _force: bool) -> bool: raise EnvironmentError( "Bootstrapping GStreamer on Linux is not supported. " + "Please install it using your distribution package manager.") diff --git a/python/servo/platform/macos.py b/python/servo/platform/macos.py index de20fe35ad5..a1655dc613e 100644 --- a/python/servo/platform/macos.py +++ b/python/servo/platform/macos.py @@ -14,6 +14,7 @@ from typing import Optional from .. import util from .base import Base +from .build_target import BuildTarget URL_BASE = "https://github.com/servo/servo-build-deps/releases/download/macOS" GSTREAMER_PLUGIN_VERSION = "1.22.3" @@ -27,15 +28,15 @@ class MacOS(Base): super().__init__(*args, **kwargs) self.is_macos = True - def gstreamer_root(self, cross_compilation_target: Optional[str]) -> Optional[str]: + def gstreamer_root(self, target: BuildTarget) -> Optional[str]: # We do not support building with gstreamer while cross-compiling on MacOS. - if cross_compilation_target or not os.path.exists(GSTREAMER_ROOT): + if target.is_cross_build() or not os.path.exists(GSTREAMER_ROOT): return None return GSTREAMER_ROOT - def is_gstreamer_installed(self, cross_compilation_target: Optional[str]) -> bool: + def is_gstreamer_installed(self, target: BuildTarget) -> bool: # Servo only supports the official GStreamer distribution on MacOS. - return not cross_compilation_target and os.path.exists(GSTREAMER_ROOT) + return not target.is_cross_build() and os.path.exists(GSTREAMER_ROOT) def _platform_bootstrap(self, _force: bool) -> bool: installed_something = False @@ -49,11 +50,12 @@ class MacOS(Base): except subprocess.CalledProcessError as e: print("Could not run homebrew. Is it installed?") raise e - installed_something |= self._platform_bootstrap_gstreamer(False) + target = BuildTarget.from_triple(None) + installed_something |= self._platform_bootstrap_gstreamer(target, False) return installed_something - def _platform_bootstrap_gstreamer(self, force: bool) -> bool: - if not force and self.is_gstreamer_installed(cross_compilation_target=None): + def _platform_bootstrap_gstreamer(self, target: BuildTarget, force: bool) -> bool: + if not force and self.is_gstreamer_installed(target): return False with tempfile.TemporaryDirectory() as temp_dir: @@ -78,5 +80,5 @@ class MacOS(Base): ] ) - assert self.is_gstreamer_installed(cross_compilation_target=None) + assert self.is_gstreamer_installed(target) return True diff --git a/python/servo/platform/windows.py b/python/servo/platform/windows.py index dced63a8fb5..52b7d4f0a2c 100644 --- a/python/servo/platform/windows.py +++ b/python/servo/platform/windows.py @@ -14,8 +14,10 @@ from typing import Optional import urllib import zipfile -from .. import util +from servo import util + from .base import Base +from .build_target import BuildTarget DEPS_URL = "https://github.com/servo/servo-build-deps/releases/download/msvc-deps" DEPENDENCIES = { @@ -57,7 +59,7 @@ class Windows(Base): else: print("done") - def _platform_bootstrap(self, force: bool = False) -> bool: + def _platform_bootstrap(self, force: bool) -> bool: installed_something = self.passive_bootstrap() try: @@ -77,7 +79,8 @@ class Windows(Base): print("Could not run chocolatey. Follow manual build setup instructions.") raise e - installed_something |= self._platform_bootstrap_gstreamer(force) + target = BuildTarget.from_triple(None) + installed_something |= self._platform_bootstrap_gstreamer(target, force) return installed_something def passive_bootstrap(self) -> bool: @@ -103,8 +106,8 @@ class Windows(Base): return True - def gstreamer_root(self, cross_compilation_target: Optional[str]) -> Optional[str]: - build_target_triple = cross_compilation_target or self.triple + def gstreamer_root(self, target: BuildTarget) -> Optional[str]: + build_target_triple = target.triple() gst_arch_names = { "x86_64": "X86_64", "x86": "X86", @@ -132,11 +135,11 @@ class Windows(Base): return None - def is_gstreamer_installed(self, cross_compilation_target: Optional[str]) -> bool: - return self.gstreamer_root(cross_compilation_target) is not None + def is_gstreamer_installed(self, target: BuildTarget) -> bool: + return self.gstreamer_root(target) is not None - def _platform_bootstrap_gstreamer(self, force: bool) -> bool: - if not force and self.is_gstreamer_installed(cross_compilation_target=None): + def _platform_bootstrap_gstreamer(self, target: BuildTarget, force: bool) -> bool: + if not force and self.is_gstreamer_installed(target): return False if "x86_64" not in self.triple: @@ -171,5 +174,5 @@ class Windows(Base): "msiexec.exe", "-ArgumentList", f"@({quoted_arguments})", ").ExitCode" ]) - assert self.is_gstreamer_installed(cross_compilation_target=None) + assert self.is_gstreamer_installed(target) return True diff --git a/python/servo/post_build_commands.py b/python/servo/post_build_commands.py index 9a085c93f51..50ce11ce2fb 100644 --- a/python/servo/post_build_commands.py +++ b/python/servo/post_build_commands.py @@ -82,7 +82,8 @@ class PostBuildCommands(CommandBase): 'params', nargs='...', help="Command-line arguments to be passed through to Servo") @CommandBase.common_command_arguments(build_configuration=False, build_type=True) - def run(self, params, build_type: BuildType, android=None, debugger=False, debugger_cmd=None, + @CommandBase.allow_target_configuration + def run(self, params, build_type: BuildType, debugger=False, debugger_cmd=None, headless=False, software=False, bin=None, emulator=False, usb=False, nightly=None, with_asan=False): env = self.build_env() env["RUST_BACKTRACE"] = "1" @@ -98,10 +99,7 @@ class PostBuildCommands(CommandBase): if debugger_cmd: debugger = True - if android is None: - android = self.config["build"]["android"] - - if android: + if self.is_android(): if debugger: print("Android on-device debugging is not supported by mach yet. See") print("https://github.com/servo/servo/wiki/Building-for-Android#debugging-on-device") @@ -136,7 +134,9 @@ class PostBuildCommands(CommandBase): shell.communicate(bytes("\n".join(script) + "\n", "utf8")) return shell.wait() - args = [bin or self.get_nightly_binary_path(nightly) or self.get_binary_path(build_type, asan=with_asan)] + args = [bin + or self.get_nightly_binary_path(nightly) + or self.get_binary_path(build_type, asan=with_asan)] if headless: args.append('-z') diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index b05a89a7a30..d3d09745eee 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -321,12 +321,10 @@ class MachCommands(CommandBase): ) return self._test_wpt(build_type=build_type, android=True, **kwargs) - def _test_wpt(self, build_type: BuildType, with_asan=False, android=False, **kwargs): - if not android: - os.environ.update(self.build_env()) - + @CommandBase.allow_target_configuration + def _test_wpt(self, build_type: BuildType, with_asan=False, **kwargs): # TODO(mrobinson): Why do we pass the wrong binary path in when running WPT on Android? - binary_path = self.get_binary_path(build_type=build_type, asan=with_asan) + binary_path = self.get_binary_path(build_type, asan=with_asan) return_value = wpt.run.run_tests(binary_path, **kwargs) return return_value if not kwargs["always_succeed"] else 0 @@ -388,7 +386,6 @@ class MachCommands(CommandBase): avd = "servo-x86" target = "i686-linux-android" print("Assuming --target " + target) - self.cross_compile_target = target env = self.build_env() os.environ["PATH"] = env["PATH"] |