diff options
| -rw-r--r-- | python/servo/bootstrap.py | 110 | ||||
| -rw-r--r-- | python/servo/bootstrap_commands.py | 15 | ||||
| -rw-r--r-- | python/servo/bootstrapper/__init__.py | 3 | ||||
| -rw-r--r-- | python/servo/bootstrapper/base.py | 54 | ||||
| -rw-r--r-- | python/servo/bootstrapper/bootstrap.py | 40 | ||||
| -rw-r--r-- | python/servo/bootstrapper/windows_gnu.py | 72 | ||||
| -rw-r--r-- | python/servo/bootstrapper/windows_msvc.py | 82 | ||||
| -rw-r--r-- | python/servo/command_base.py | 18 | ||||
| -rw-r--r-- | python/servo/packages.py (renamed from python/servo/bootstrapper/packages.py) | 24 |
9 files changed, 137 insertions, 281 deletions
diff --git a/python/servo/bootstrap.py b/python/servo/bootstrap.py new file mode 100644 index 00000000000..ec8a0a6c449 --- /dev/null +++ b/python/servo/bootstrap.py @@ -0,0 +1,110 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function + +from distutils.spawn import find_executable +import os +import subprocess +import sys + +import servo.packages as packages +from servo.util import extract, download_file, host_triple + + +def windows_gnu(context, force=False): + '''Bootstrapper for msys2 based environments for building in Windows.''' + + if not find_executable('pacman'): + print( + 'The Windows GNU bootstrapper only works with msys2 with pacman. ' + 'Get msys2 at http://msys2.github.io/' + ) + return 1 + + # Ensure repositories are up to date + command = ['pacman', '--sync', '--refresh'] + subprocess.check_call(command) + + # Install packages + command = ['pacman', '--sync', '--needed'] + if force: + command.append('--noconfirm') + subprocess.check_call(command + list(packages.WINDOWS_GNU)) + + # Downgrade GCC to 5.4.0-1 + gcc_pkgs = ["gcc", "gcc-ada", "gcc-fortran", "gcc-libgfortran", "gcc-libs", "gcc-objc"] + gcc_version = "5.4.0-1" + mingw_url = "http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-{}-{}-any.pkg.tar.xz" + gcc_list = [mingw_url.format(gcc, gcc_version) for gcc in gcc_pkgs] + + # Note: `--upgrade` also does downgrades + downgrade_command = ['pacman', '--upgrade'] + if force: + downgrade_command.append('--noconfirm') + subprocess.check_call(downgrade_command + gcc_list) + + +def windows_msvc(context, force=False): + '''Bootstrapper for MSVC building on Windows.''' + + deps_dir = os.path.join(context.sharedir, "msvc-dependencies") + deps_url = "https://servo-rust.s3.amazonaws.com/msvc-deps/" + + def version(package): + return packages.WINDOWS_MSVC[package] + + def package_dir(package): + return os.path.join(deps_dir, package, version(package)) + + to_install = {} + for package in packages.WINDOWS_MSVC: + # Don't install CMake if it already exists in PATH + if package == "cmake" and find_executable(package): + continue + + if not os.path.isdir(package_dir(package)): + to_install[package] = version(package) + + if not to_install: + return 0 + + print("Installing missing MSVC dependencies...") + for package in to_install: + full_spec = '{}-{}'.format(package, version(package)) + + parent_dir = os.path.dirname(package_dir(package)) + if not os.path.isdir(parent_dir): + os.makedirs(parent_dir) + + zip_path = package_dir(package) + ".zip" + if not os.path.isfile(zip_path): + zip_url = "{}{}.zip".format(deps_url, full_spec) + download_file(full_spec, zip_url, zip_path) + + print("Extracting {}...".format(full_spec), end='') + extract(zip_path, deps_dir) + print("done") + + extracted_path = os.path.join(deps_dir, full_spec) + os.rename(extracted_path, package_dir(package)) + + return 0 + + +def bootstrap(context, force=False): + '''Dispatches to the right bootstrapping function for the OS.''' + + bootstrapper = None + + if "windows-gnu" in host_triple(): + bootstrapper = windows_gnu + elif "windows-msvc" in host_triple(): + bootstrapper = windows_msvc + + if bootstrapper is None: + print('Bootstrap support is not yet available for your OS.') + return 1 + + return bootstrapper(context, force=force) diff --git a/python/servo/bootstrap_commands.py b/python/servo/bootstrap_commands.py index 9128c1dccff..8975f174d84 100644 --- a/python/servo/bootstrap_commands.py +++ b/python/servo/bootstrap_commands.py @@ -7,7 +7,7 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -from __future__ import print_function, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import base64 import json @@ -24,6 +24,7 @@ from mach.decorators import ( Command, ) +import servo.bootstrap as bootstrap from servo.command_base import CommandBase, BIN_SUFFIX from servo.util import download_bytes, download_file, extract, host_triple @@ -44,17 +45,11 @@ class MachCommands(CommandBase): @Command('bootstrap', description='Install required packages for building.', category='bootstrap') - @CommandArgument('--interactive', "-i", - action='store_true', - help='Need to answer any (Y/n) interactive prompts.') @CommandArgument('--force', '-f', action='store_true', - help='Force reinstall packages') - def bootstrap(self, interactive=False, force=False): - from servo.bootstrapper.bootstrap import Bootstrapper - - bootstrapper = Bootstrapper(self.context) - bootstrapper.bootstrap(interactive=interactive, force=force) + help='Boostrap without confirmation') + def bootstrap(self, force=False): + return bootstrap.bootstrap(self.context, force=force) @Command('bootstrap-rust', description='Download the Rust compiler', diff --git a/python/servo/bootstrapper/__init__.py b/python/servo/bootstrapper/__init__.py deleted file mode 100644 index 6fbe8159b2d..00000000000 --- a/python/servo/bootstrapper/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/python/servo/bootstrapper/base.py b/python/servo/bootstrapper/base.py deleted file mode 100644 index b988e06c53a..00000000000 --- a/python/servo/bootstrapper/base.py +++ /dev/null @@ -1,54 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import print_function, unicode_literals - -import distutils -import subprocess - - -class BaseBootstrapper(object): - """Base class for system bootstrappers.""" - - def __init__(self, interactive=False): - self.package_manager_updated = False - self.interactive = interactive - - def ensure_system_packages(self): - ''' - Check for missing packages. - ''' - raise NotImplementedError('%s must implement ensure_system_packages()' % - __name__) - - def install_system_packages(self): - ''' - Install packages required to build Servo. - ''' - raise NotImplementedError('%s must implement install_system_packages()' % - __name__) - - def which(self, name): - """Python implementation of which. - - It returns the path of an executable or None if it couldn't be found. - """ - return distutils.spawn.find_executable(name) - - def check_output(self, *args, **kwargs): - """Run subprocess.check_output.""" - return subprocess.check_output(*args, **kwargs) - - def _ensure_package_manager_updated(self): - if self.package_manager_updated: - return - - self._update_package_manager() - self.package_manager_updated = True - - def _update_package_manager(self): - """Updates the package manager's manifests/package list. - - This should be defined in child classes. - """ diff --git a/python/servo/bootstrapper/bootstrap.py b/python/servo/bootstrapper/bootstrap.py deleted file mode 100644 index 4cd1fc7bc5c..00000000000 --- a/python/servo/bootstrapper/bootstrap.py +++ /dev/null @@ -1,40 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import print_function - -import sys - -from windows_gnu import WindowsGnuBootstrapper -from windows_msvc import WindowsMsvcBootstrapper - - -class Bootstrapper(object): - """Main class that performs system bootstrap.""" - - def __init__(self, context): - self.instance = None - cls = None - args = {} - - if sys.platform.startswith('msys'): - cls = WindowsGnuBootstrapper - - elif sys.platform.startswith('win32'): - cls = WindowsMsvcBootstrapper - - if cls is None: - sys.exit('Bootstrap support is not yet available for your OS.') - - self.instance = cls(**args) - self.instance.context = context - - def bootstrap(self, interactive=False, force=False): - self.instance.interactive = interactive - self.instance.force = force - - if force: - self.instance.install_system_packages() - else: - self.instance.ensure_system_packages() diff --git a/python/servo/bootstrapper/windows_gnu.py b/python/servo/bootstrapper/windows_gnu.py deleted file mode 100644 index 4980baa3bb1..00000000000 --- a/python/servo/bootstrapper/windows_gnu.py +++ /dev/null @@ -1,72 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import sys -import subprocess - -from base import BaseBootstrapper -from packages import WINDOWS_GNU as deps - - -class WindowsGnuBootstrapper(BaseBootstrapper): - '''Bootstrapper for msys2 based environments for building in Windows.''' - - def __init__(self, **kwargs): - BaseBootstrapper.__init__(self, **kwargs) - - if not self.which('pacman'): - raise NotImplementedError('The Windows bootstrapper only works with msys2 with pacman. Get msys2 at ' - 'http://msys2.github.io/') - - def ensure_system_packages(self): - install_packages = [] - for p in deps: - command = ['pacman', '-Qs', p] - if self.run_check(command): - install_packages += [p] - if install_packages: - install_packages(install_packages) - - def install_system_packages(self, packages=deps): - self._ensure_package_manager_updated() - self.pacman_install(*packages) - - def _update_package_manager(self): - self.pacman_update() - - def run(self, command): - subprocess.check_call(command, stdin=sys.stdin) - - def run_check(self, command): - return subprocess.call(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - def pacman_update(self): - command = ['pacman', '--sync', '--refresh'] - self.run(command) - - def pacman_upgrade(self): - command = ['pacman', '--sync', '--refresh', '--sysupgrade'] - self.run(command) - - def pacman_install(self, *packages): - command = ['pacman', '--sync'] - if not self.force: - command.append('--needed') - if not self.interactive: - command.append('--noconfirm') - command.extend(packages) - self.run(command) - - # downgrade GCC to 5.4.0-1 - gcc_type = ["gcc", "gcc-ada", "gcc-fortran", "gcc-libgfortran", "gcc-libs", "gcc-objc"] - gcc_version = "5.4.0-1" - mingw_url = "http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-{}-{}-any.pkg.tar.xz" - gcc_list = [] - for gcc in gcc_type: - gcc_list += [mingw_url.format(gcc, gcc_version)] - downgrade_command = ['pacman', '-U'] - if not self.interactive: - downgrade_command.append('--noconfirm') - downgrade_command.extend(gcc_list) - self.run(downgrade_command) diff --git a/python/servo/bootstrapper/windows_msvc.py b/python/servo/bootstrapper/windows_msvc.py deleted file mode 100644 index 0545928f453..00000000000 --- a/python/servo/bootstrapper/windows_msvc.py +++ /dev/null @@ -1,82 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import os -import shutil -from distutils import spawn - -from servo.util import extract, download_file -from base import BaseBootstrapper -from packages import WINDOWS_MSVC as deps - - -class WindowsMsvcBootstrapper(BaseBootstrapper): - '''Bootstrapper for MSVC building on Windows.''' - - def __init__(self, **kwargs): - BaseBootstrapper.__init__(self, **kwargs) - - def ensure_system_packages(self): - self.install_system_packages() - - def install_system_packages(self, packages=deps): - - deps_dir = os.path.join(self.context.sharedir, "msvc-dependencies") - deps_url = "https://servo-rust.s3.amazonaws.com/msvc-deps/" - first_run = True - - if self.force: - if os.path.isdir(deps_dir): - shutil.rmtree(deps_dir) - - if not os.path.isdir(deps_dir): - os.makedirs(deps_dir) - - # Read file with installed dependencies, if exist - installed_deps_file = os.path.join(deps_dir, "installed-dependencies.txt") - if os.path.exists(installed_deps_file): - installed_deps = [l.strip() for l in open(installed_deps_file)] - else: - installed_deps = [] - - # list of dependencies that need to be updated - update_deps = list(set(packages) - set(installed_deps)) - - for dep in packages: - dep_name = dep.split("-")[0] - - # Don't download CMake if already exists in PATH - if dep_name == "cmake": - if spawn.find_executable(dep_name): - continue - - dep_dir = os.path.join(deps_dir, dep_name) - # if not installed or need to be updated - if not os.path.exists(dep_dir) or dep in update_deps: - if first_run: - print "Installing missing MSVC dependencies..." - first_run = False - - dep_version_dir = os.path.join(deps_dir, dep) - - if os.path.exists(dep_version_dir): - shutil.rmtree(dep_version_dir) - - dep_zip = dep_version_dir + ".zip" - if not os.path.isfile(dep_zip): - download_file(dep, "%s%s.zip" % (deps_url, dep), dep_zip) - - print "Extracting %s..." % dep, - extract(dep_zip, deps_dir) - print "done" - - # Delete directory if exist - if os.path.exists(dep_dir): - shutil.rmtree(dep_dir) - os.rename(dep_version_dir, dep_dir) - - # Write in installed-dependencies.txt file - with open(installed_deps_file, 'w') as installed_file: - for line in packages: - installed_file.write(line + "\n") diff --git a/python/servo/command_base.py b/python/servo/command_base.py index 00bc38de070..e80c9cb018b 100644 --- a/python/servo/command_base.py +++ b/python/servo/command_base.py @@ -19,12 +19,12 @@ from subprocess import PIPE import sys import tarfile +from mach.registrar import Registrar import toml -from mach.registrar import Registrar +from servo.packages import WINDOWS_MSVC as msvc_deps from servo.util import host_triple - BIN_SUFFIX = ".exe" if sys.platform == "win32" else "" @@ -384,14 +384,18 @@ class CommandBase(object): if "msvc" in (target or host_triple()): msvc_x64 = "64" if "x86_64" in (target or host_triple()) else "" msvc_deps_dir = path.join(self.context.sharedir, "msvc-dependencies") - extra_path += [path.join(msvc_deps_dir, "cmake", "bin")] - extra_path += [path.join(msvc_deps_dir, "ninja", "bin")] + + def package_dir(package): + return path.join(msvc_deps_dir, package, msvc_deps[package]) + + extra_path += [path.join(package_dir("cmake"), "bin")] + extra_path += [path.join(package_dir("ninja"), "bin")] # Link openssl - env["OPENSSL_INCLUDE_DIR"] = path.join(msvc_deps_dir, "openssl", "include") - env["OPENSSL_LIB_DIR"] = path.join(msvc_deps_dir, "openssl", "lib" + msvc_x64) + env["OPENSSL_INCLUDE_DIR"] = path.join(package_dir("openssl"), "include") + env["OPENSSL_LIB_DIR"] = path.join(package_dir("openssl"), "lib" + msvc_x64) env["OPENSSL_LIBS"] = "ssleay32MD:libeay32MD" # Link moztools - env["MOZTOOLS_PATH"] = path.join(msvc_deps_dir, "moztools", "bin") + env["MOZTOOLS_PATH"] = path.join(package_dir("moztools"), "bin") if is_windows(): if not os.environ.get("NATIVE_WIN32_PYTHON"): diff --git a/python/servo/bootstrapper/packages.py b/python/servo/packages.py index 7e9a7cd9c0e..9e4906dc5cf 100644 --- a/python/servo/bootstrapper/packages.py +++ b/python/servo/packages.py @@ -2,9 +2,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -# Listed all packages for different platforms in one file - -WINDOWS_GNU = [ +WINDOWS_GNU = set([ + "diffutils", + "make", "mingw-w64-x86_64-toolchain", "mingw-w64-x86_64-freetype", "mingw-w64-x86_64-icu", @@ -12,17 +12,15 @@ WINDOWS_GNU = [ "mingw-w64-x86_64-ca-certificates", "mingw-w64-x86_64-expat", "mingw-w64-x86_64-cmake", - "tar", - "diffutils", "patch", "patchutils", - "make", "python2-setuptools", -] + "tar", +]) -WINDOWS_MSVC = [ - "cmake-3.6.1", - "ninja-1.7.1", - "openssl-1.0.1t-vs2015", - "moztools-0.0.1-5", -] +WINDOWS_MSVC = { + "cmake": "3.6.1", + "moztools": "0.0.1-5", + "ninja": "1.7.1", + "openssl": "1.0.1t-vs2015", +} |