diff options
| -rw-r--r-- | python/mach_bootstrap.py | 10 | ||||
| -rw-r--r-- | python/requirements-salt.txt | 6 | ||||
| -rw-r--r-- | python/requirements.txt | 3 | ||||
| -rw-r--r-- | python/servo/bootstrap.py | 229 | ||||
| -rw-r--r-- | python/servo/bootstrap_commands.py | 111 | ||||
| -rw-r--r-- | python/servo/bootstrapper/__init__.py | 3 | ||||
| -rw-r--r-- | python/servo/bootstrapper/base.py | 62 | ||||
| -rw-r--r-- | python/servo/bootstrapper/bootstrap.py | 42 | ||||
| -rw-r--r-- | python/servo/bootstrapper/windows_gnu.py | 75 | ||||
| -rw-r--r-- | python/servo/bootstrapper/windows_msvc.py | 86 | ||||
| -rw-r--r-- | python/servo/build_commands.py | 3 | ||||
| -rw-r--r-- | python/servo/command_base.py | 64 | ||||
| -rw-r--r-- | python/servo/packages.py (renamed from python/servo/bootstrapper/packages.py) | 24 | ||||
| -rw-r--r-- | python/servo/testing_commands.py | 3 | ||||
| -rw-r--r-- | python/servo/util.py | 151 |
15 files changed, 430 insertions, 442 deletions
diff --git a/python/mach_bootstrap.py b/python/mach_bootstrap.py index d41bb1f5099..fa612ddd868 100644 --- a/python/mach_bootstrap.py +++ b/python/mach_bootstrap.py @@ -118,7 +118,11 @@ def _activate_virtualenv(topdir): if not virtualenv: sys.exit("Python virtualenv is not installed. Please install it prior to running mach.") - process = Popen([virtualenv, "-p", python, virtualenv_path], stdout=PIPE, stderr=PIPE) + process = Popen( + [virtualenv, "-p", python, "--system-site-packages", virtualenv_path], + stdout=PIPE, + stderr=PIPE + ) process.wait() if process.returncode: out, err = process.communicate() @@ -153,7 +157,7 @@ def _activate_virtualenv(topdir): if not pip: sys.exit("Python pip is either not installed or not found in virtualenv.") - process = Popen([pip, "install", "-q", "-U", "pip"], stdout=PIPE, stderr=PIPE) + process = Popen([pip, "install", "-q", "-I", "-U", "pip"], stdout=PIPE, stderr=PIPE) process.wait() if process.returncode: out, err = process.communicate() @@ -175,7 +179,7 @@ def _activate_virtualenv(topdir): if not pip: sys.exit("Python pip is either not installed or not found in virtualenv.") - process = Popen([pip, "install", "-q", "-r", req_path], stdout=PIPE, stderr=PIPE) + process = Popen([pip, "install", "-q", "-I", "-r", req_path], stdout=PIPE, stderr=PIPE) process.wait() if process.returncode: out, err = process.communicate() diff --git a/python/requirements-salt.txt b/python/requirements-salt.txt new file mode 100644 index 00000000000..1486b590053 --- /dev/null +++ b/python/requirements-salt.txt @@ -0,0 +1,6 @@ +# Ensure all versions are pinned for repeatability, +# since `--system-site-packages` is enabled + +# For boostrapping, make sure versions match those in saltfs +salt == 2016.3.4 +GitPython == 0.3.2 diff --git a/python/requirements.txt b/python/requirements.txt index 8248a21e930..41b25ccdb5a 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,3 +1,6 @@ +# Ensure all versions are pinned for repeatability, +# since `--system-site-packages` is enabled + blessings == 1.6 mach == 0.6.0 mozdebug == 0.1 diff --git a/python/servo/bootstrap.py b/python/servo/bootstrap.py new file mode 100644 index 00000000000..54862a885bd --- /dev/null +++ b/python/servo/bootstrap.py @@ -0,0 +1,229 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function + +from distutils.spawn import find_executable +import json +import os +import platform +import shutil +import subprocess + +import servo.packages as packages +from servo.util import extract, download_file, host_triple + + +def salt(context, force=False): + # Ensure Salt is installed in the virtualenv + # It's not instaled globally because it's a large, non-required dependency, + # and the installation fails on Windows + print("Checking Salt installation...", end='') + reqs_path = os.path.join(context.topdir, 'python', 'requirements-salt.txt') + process = subprocess.Popen( + ["pip", "install", "-q", "-I", "-r", reqs_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + process.wait() + if process.returncode: + out, err = process.communicate() + print('failed to install Salt via pip:') + print('Output: {}\nError: {}'.format(out, err)) + return 1 + print("done") + + salt_root = os.path.join(context.sharedir, 'salt') + config_dir = os.path.join(salt_root, 'etc', 'salt') + pillar_dir = os.path.join(config_dir, 'pillars') + + # In order to allow `mach bootstrap` to work from any CWD, + # the `root_dir` must be an absolute path. + # We place it under `context.sharedir` because + # Salt caches data (e.g. gitfs files) in its `var` subdirectory. + # Hence, dynamically generate the config with an appropriate `root_dir` + # and serialize it as JSON (which is valid YAML). + config = { + 'fileserver_backend': ['git'], + 'gitfs_env_whitelist': 'base', + 'gitfs_provider': 'gitpython', + 'gitfs_remotes': [ + 'https://github.com/servo/saltfs.git', + ], + 'hash_type': 'sha384', + 'master': 'localhost', + 'root_dir': salt_root, + 'state_output': 'changes', + 'state_tabular': True, + } + + if not os.path.exists(config_dir): + os.makedirs(config_dir, mode=0o700) + with open(os.path.join(config_dir, 'minion'), 'w') as config_file: + config_file.write(json.dumps(config) + '\n') + + # Similarly, the pillar data is created dynamically + # and temporarily serialized to disk. + # This dynamism is not yet used, but will be in the future + # to enable Android bootstrapping by using + # context.sharedir as a location for Android packages. + pillar = { + 'top.sls': { + 'base': { + '*': ['bootstrap'], + }, + }, + 'bootstrap.sls': { + 'fully_managed': False, + }, + } + if os.path.exists(pillar_dir): + shutil.rmtree(pillar_dir) + os.makedirs(pillar_dir, mode=0o700) + for filename in pillar: + with open(os.path.join(pillar_dir, filename), 'w') as pillar_file: + pillar_file.write(json.dumps(pillar[filename]) + '\n') + + cmd = [ + 'sudo', + # sudo escapes from the venv, need to use full path + find_executable('salt-call'), + '--local', + '--config-dir={}'.format(config_dir), + '--pillar-root={}'.format(pillar_dir), + 'state.apply', + 'servo-build-dependencies', + ] + + if not force: + print('Running bootstrap in dry-run mode to show changes') + # Because `test=True` mode runs each state individually without + # considering how required/previous states affect the system, + # it will often report states with requisites as failing due + # to the requisites not actually being run, + # even though these are spurious and will succeed during + # the actual highstate. + # Hence `--retcode-passthrough` is not helpful in dry-run mode, + # so only detect failures of the actual salt-call binary itself. + retcode = subprocess.call(cmd + ['test=True']) + if retcode != 0: + print('Something went wrong while bootstrapping') + return retcode + + proceed = raw_input( + 'Proposed changes are above, proceed with bootstrap? [y/N]: ' + ) + if proceed.lower() not in ['y', 'yes']: + return 0 + + print('') + + print('Running Salt bootstrap') + retcode = subprocess.call(cmd + ['--retcode-passthrough']) + if retcode == 0: + print('Salt bootstrapping complete') + else: + print('Salt bootstrapping encountered errors') + return retcode + + +def windows_gnu(context, force=False): + '''Bootstrapper for msys2 based environments for building in Windows.''' + + if not find_executable('pacman'): + print( + 'The Windows GNU bootstrapper only works with msys2 with pacman. ' + 'Get msys2 at http://msys2.github.io/' + ) + return 1 + + # Ensure repositories are up to date + command = ['pacman', '--sync', '--refresh'] + subprocess.check_call(command) + + # Install packages + command = ['pacman', '--sync', '--needed'] + if force: + command.append('--noconfirm') + subprocess.check_call(command + list(packages.WINDOWS_GNU)) + + # Downgrade GCC to 5.4.0-1 + gcc_pkgs = ["gcc", "gcc-ada", "gcc-fortran", "gcc-libgfortran", "gcc-libs", "gcc-objc"] + gcc_version = "5.4.0-1" + mingw_url = "http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-{}-{}-any.pkg.tar.xz" + gcc_list = [mingw_url.format(gcc, gcc_version) for gcc in gcc_pkgs] + + # Note: `--upgrade` also does downgrades + downgrade_command = ['pacman', '--upgrade'] + if force: + downgrade_command.append('--noconfirm') + subprocess.check_call(downgrade_command + gcc_list) + + +def windows_msvc(context, force=False): + '''Bootstrapper for MSVC building on Windows.''' + + deps_dir = os.path.join(context.sharedir, "msvc-dependencies") + deps_url = "https://servo-rust.s3.amazonaws.com/msvc-deps/" + + def version(package): + return packages.WINDOWS_MSVC[package] + + def package_dir(package): + return os.path.join(deps_dir, package, version(package)) + + to_install = {} + for package in packages.WINDOWS_MSVC: + # Don't install CMake if it already exists in PATH + if package == "cmake" and find_executable(package): + continue + + if not os.path.isdir(package_dir(package)): + to_install[package] = version(package) + + if not to_install: + return 0 + + print("Installing missing MSVC dependencies...") + for package in to_install: + full_spec = '{}-{}'.format(package, version(package)) + + parent_dir = os.path.dirname(package_dir(package)) + if not os.path.isdir(parent_dir): + os.makedirs(parent_dir) + + zip_path = package_dir(package) + ".zip" + if not os.path.isfile(zip_path): + zip_url = "{}{}.zip".format(deps_url, full_spec) + download_file(full_spec, zip_url, zip_path) + + print("Extracting {}...".format(full_spec), end='') + extract(zip_path, deps_dir) + print("done") + + extracted_path = os.path.join(deps_dir, full_spec) + os.rename(extracted_path, package_dir(package)) + + return 0 + + +def bootstrap(context, force=False): + '''Dispatches to the right bootstrapping function for the OS.''' + + bootstrapper = None + + if "windows-gnu" in host_triple(): + bootstrapper = windows_gnu + elif "windows-msvc" in host_triple(): + bootstrapper = windows_msvc + elif "linux-gnu" in host_triple(): + distro, version, _ = platform.linux_distribution() + if distro == 'Ubuntu' and version == '14.04': + bootstrapper = salt + + if bootstrapper is None: + print('Bootstrap support is not yet available for your OS.') + return 1 + + return bootstrapper(context, force=force) diff --git a/python/servo/bootstrap_commands.py b/python/servo/bootstrap_commands.py index 9bc7c60fd4a..8975f174d84 100644 --- a/python/servo/bootstrap_commands.py +++ b/python/servo/bootstrap_commands.py @@ -7,8 +7,7 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -from __future__ import print_function, unicode_literals -from socket import error as socket_error +from __future__ import absolute_import, print_function, unicode_literals import base64 import json @@ -17,9 +16,6 @@ import os.path as path import re import shutil import sys -import StringIO -import tarfile -import zipfile import urllib2 from mach.decorators import ( @@ -28,93 +24,9 @@ from mach.decorators import ( Command, ) -from servo.command_base import CommandBase, host_triple, BIN_SUFFIX - - -def download(desc, src, writer, start_byte=0): - if start_byte: - print("Resuming download of %s..." % desc) - else: - print("Downloading %s..." % desc) - dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty()) - - try: - req = urllib2.Request(src) - if start_byte: - req = urllib2.Request(src, headers={'Range': 'bytes={}-'.format(start_byte)}) - resp = urllib2.urlopen(req) - - fsize = None - if resp.info().getheader('Content-Length'): - fsize = int(resp.info().getheader('Content-Length').strip()) + start_byte - - recved = start_byte - chunk_size = 8192 - - while True: - chunk = resp.read(chunk_size) - if not chunk: - break - recved += len(chunk) - if not dumb: - if fsize is not None: - pct = recved * 100.0 / fsize - print("\rDownloading %s: %5.1f%%" % (desc, pct), end="") - - sys.stdout.flush() - writer.write(chunk) - - if not dumb: - print() - except urllib2.HTTPError, e: - print("Download failed (%d): %s - %s" % (e.code, e.reason, src)) - if e.code == 403: - print("No Rust compiler binary available for this platform. " - "Please see https://github.com/servo/servo/#prerequisites") - sys.exit(1) - except urllib2.URLError, e: - print("Error downloading Rust compiler: %s. The failing URL was: %s" % (e.reason, src)) - sys.exit(1) - except socket_error, e: - print("Looks like there's a connectivity issue, check your Internet connection. %s" % (e)) - sys.exit(1) - except KeyboardInterrupt: - writer.flush() - raise - - -def download_file(desc, src, dst): - tmp_path = dst + ".part" - try: - start_byte = os.path.getsize(tmp_path) - with open(tmp_path, 'ab') as fd: - download(desc, src, fd, start_byte=start_byte) - except os.error: - with open(tmp_path, 'wb') as fd: - download(desc, src, fd) - os.rename(tmp_path, dst) - - -def download_bytes(desc, src): - content_writer = StringIO.StringIO() - download(desc, src, content_writer) - return content_writer.getvalue() - - -def extract(src, dst, movedir=None): - if src.endswith(".zip"): - zipfile.ZipFile(src).extractall(dst) - else: - tarfile.open(src).extractall(dst) - - if movedir: - for f in os.listdir(movedir): - frm = path.join(movedir, f) - to = path.join(dst, f) - os.rename(frm, to) - os.rmdir(movedir) - - os.remove(src) +import servo.bootstrap as bootstrap +from servo.command_base import CommandBase, BIN_SUFFIX +from servo.util import download_bytes, download_file, extract, host_triple @CommandProvider @@ -133,20 +45,11 @@ class MachCommands(CommandBase): @Command('bootstrap', description='Install required packages for building.', category='bootstrap') - @CommandArgument('--interactive', "-i", - action='store_true', - help='Need to answer any (Y/n) interactive prompts.') - @CommandArgument('--android', - action='store_true', - help='Install required packages for Android') @CommandArgument('--force', '-f', action='store_true', - help='Force reinstall packages') - def bootstrap(self, android=False, interactive=False, force=False): - from servo.bootstrapper.bootstrap import Bootstrapper - - bootstrapper = Bootstrapper(self.context) - bootstrapper.bootstrap(android=android, interactive=interactive, force=force) + help='Boostrap without confirmation') + def bootstrap(self, force=False): + return bootstrap.bootstrap(self.context, force=force) @Command('bootstrap-rust', description='Download the Rust compiler', diff --git a/python/servo/bootstrapper/__init__.py b/python/servo/bootstrapper/__init__.py deleted file mode 100644 index 6fbe8159b2d..00000000000 --- a/python/servo/bootstrapper/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/python/servo/bootstrapper/base.py b/python/servo/bootstrapper/base.py deleted file mode 100644 index 40f3d1f5ebc..00000000000 --- a/python/servo/bootstrapper/base.py +++ /dev/null @@ -1,62 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import print_function, unicode_literals - -import distutils -import subprocess - - -class BaseBootstrapper(object): - """Base class for system bootstrappers.""" - - def __init__(self, interactive=False): - self.package_manager_updated = False - self.interactive = interactive - - def ensure_system_packages(self): - ''' - Check for missing packages. - ''' - raise NotImplementedError('%s must implement ensure_system_packages()' % - __name__) - - def install_system_packages(self): - ''' - Install packages required to build Servo. - ''' - raise NotImplementedError('%s must implement install_system_packages()' % - __name__) - - def install_mobile_android_packages(self): - ''' - Install packages required to build Servo for Android. - ''' - raise NotImplementedError('Cannot bootstrap Servo for Android: ' - '%s does not yet implement install_mobile_android_packages()' - % __name__) - - def which(self, name): - """Python implementation of which. - - It returns the path of an executable or None if it couldn't be found. - """ - return distutils.spawn.find_executable(name) - - def check_output(self, *args, **kwargs): - """Run subprocess.check_output.""" - return subprocess.check_output(*args, **kwargs) - - def _ensure_package_manager_updated(self): - if self.package_manager_updated: - return - - self._update_package_manager() - self.package_manager_updated = True - - def _update_package_manager(self): - """Updates the package manager's manifests/package list. - - This should be defined in child classes. - """ diff --git a/python/servo/bootstrapper/bootstrap.py b/python/servo/bootstrapper/bootstrap.py deleted file mode 100644 index d07dc12fc3c..00000000000 --- a/python/servo/bootstrapper/bootstrap.py +++ /dev/null @@ -1,42 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import print_function - -import sys - -from windows_gnu import WindowsGnuBootstrapper -from windows_msvc import WindowsMsvcBootstrapper - - -class Bootstrapper(object): - """Main class that performs system bootstrap.""" - - def __init__(self, context): - self.instance = None - cls = None - args = {} - - if sys.platform.startswith('msys'): - cls = WindowsGnuBootstrapper - - elif sys.platform.startswith('win32'): - cls = WindowsMsvcBootstrapper - - if cls is None: - sys.exit('Bootstrap support is not yet available for your OS.') - - self.instance = cls(**args) - self.instance.context = context - - def bootstrap(self, android=False, interactive=False, force=False): - self.instance.interactive = interactive - self.instance.force = force - - if android: - self.instance.install_mobile_android_packages() - elif force: - self.instance.install_system_packages() - else: - self.instance.ensure_system_packages() diff --git a/python/servo/bootstrapper/windows_gnu.py b/python/servo/bootstrapper/windows_gnu.py deleted file mode 100644 index 794f1790cd1..00000000000 --- a/python/servo/bootstrapper/windows_gnu.py +++ /dev/null @@ -1,75 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import sys -import subprocess - -from base import BaseBootstrapper -from packages import WINDOWS_GNU as deps - - -class WindowsGnuBootstrapper(BaseBootstrapper): - '''Bootstrapper for msys2 based environments for building in Windows.''' - - def __init__(self, **kwargs): - BaseBootstrapper.__init__(self, **kwargs) - - if not self.which('pacman'): - raise NotImplementedError('The Windows bootstrapper only works with msys2 with pacman. Get msys2 at ' - 'http://msys2.github.io/') - - def ensure_system_packages(self): - install_packages = [] - for p in deps: - command = ['pacman', '-Qs', p] - if self.run_check(command): - install_packages += [p] - if install_packages: - install_packages(install_packages) - - def install_system_packages(self, packages=deps): - self._ensure_package_manager_updated() - self.pacman_install(*packages) - - def install_mobile_android_packages(self): - sys.exit('We do not support building Android on Windows. Sorry!') - - def _update_package_manager(self): - self.pacman_update() - - def run(self, command): - subprocess.check_call(command, stdin=sys.stdin) - - def run_check(self, command): - return subprocess.call(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - def pacman_update(self): - command = ['pacman', '--sync', '--refresh'] - self.run(command) - - def pacman_upgrade(self): - command = ['pacman', '--sync', '--refresh', '--sysupgrade'] - self.run(command) - - def pacman_install(self, *packages): - command = ['pacman', '--sync'] - if not self.force: - command.append('--needed') - if not self.interactive: - command.append('--noconfirm') - command.extend(packages) - self.run(command) - - # downgrade GCC to 5.4.0-1 - gcc_type = ["gcc", "gcc-ada", "gcc-fortran", "gcc-libgfortran", "gcc-libs", "gcc-objc"] - gcc_version = "5.4.0-1" - mingw_url = "http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-{}-{}-any.pkg.tar.xz" - gcc_list = [] - for gcc in gcc_type: - gcc_list += [mingw_url.format(gcc, gcc_version)] - downgrade_command = ['pacman', '-U'] - if not self.interactive: - downgrade_command.append('--noconfirm') - downgrade_command.extend(gcc_list) - self.run(downgrade_command) diff --git a/python/servo/bootstrapper/windows_msvc.py b/python/servo/bootstrapper/windows_msvc.py deleted file mode 100644 index 11ff765c99d..00000000000 --- a/python/servo/bootstrapper/windows_msvc.py +++ /dev/null @@ -1,86 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import os -import sys -import shutil -from distutils import spawn - -from base import BaseBootstrapper -from packages import WINDOWS_MSVC as deps - - -class WindowsMsvcBootstrapper(BaseBootstrapper): - '''Bootstrapper for MSVC building on Windows.''' - - def __init__(self, **kwargs): - BaseBootstrapper.__init__(self, **kwargs) - - def ensure_system_packages(self): - self.install_system_packages() - - def install_system_packages(self, packages=deps): - from servo.bootstrap_commands import extract, download_file - - deps_dir = os.path.join(self.context.sharedir, "msvc-dependencies") - deps_url = "https://servo-rust.s3.amazonaws.com/msvc-deps/" - first_run = True - - if self.force: - if os.path.isdir(deps_dir): - shutil.rmtree(deps_dir) - - if not os.path.isdir(deps_dir): - os.makedirs(deps_dir) - - # Read file with installed dependencies, if exist - installed_deps_file = os.path.join(deps_dir, "installed-dependencies.txt") - if os.path.exists(installed_deps_file): - installed_deps = [l.strip() for l in open(installed_deps_file)] - else: - installed_deps = [] - - # list of dependencies that need to be updated - update_deps = list(set(packages) - set(installed_deps)) - - for dep in packages: - dep_name = dep.split("-")[0] - - # Don't download CMake if already exists in PATH - if dep_name == "cmake": - if spawn.find_executable(dep_name): - continue - - dep_dir = os.path.join(deps_dir, dep_name) - # if not installed or need to be updated - if not os.path.exists(dep_dir) or dep in update_deps: - if first_run: - print "Installing missing MSVC dependencies..." - first_run = False - - dep_version_dir = os.path.join(deps_dir, dep) - - if os.path.exists(dep_version_dir): - shutil.rmtree(dep_version_dir) - - dep_zip = dep_version_dir + ".zip" - if not os.path.isfile(dep_zip): - download_file(dep, "%s%s.zip" % (deps_url, dep), dep_zip) - - print "Extracting %s..." % dep, - extract(dep_zip, deps_dir) - print "done" - - # Delete directory if exist - if os.path.exists(dep_dir): - shutil.rmtree(dep_dir) - os.rename(dep_version_dir, dep_dir) - - # Write in installed-dependencies.txt file - with open(installed_deps_file, 'w') as installed_file: - for line in packages: - installed_file.write(line + "\n") - - def install_mobile_android_packages(self): - sys.exit('We do not support building Android on Windows. Sorry!') diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py index 6e7ec8d24c5..5fbb7ba164d 100644 --- a/python/servo/build_commands.py +++ b/python/servo/build_commands.py @@ -24,7 +24,8 @@ from mach.decorators import ( Command, ) -from servo.command_base import CommandBase, cd, call, BIN_SUFFIX, host_triple, find_dep_path_newest +from servo.command_base import CommandBase, cd, call, BIN_SUFFIX, find_dep_path_newest +from servo.util import host_triple def format_duration(seconds): diff --git a/python/servo/command_base.py b/python/servo/command_base.py index 80fe675da1c..e80c9cb018b 100644 --- a/python/servo/command_base.py +++ b/python/servo/command_base.py @@ -18,11 +18,12 @@ import subprocess from subprocess import PIPE import sys import tarfile -import platform +from mach.registrar import Registrar import toml -from mach.registrar import Registrar +from servo.packages import WINDOWS_MSVC as msvc_deps +from servo.util import host_triple BIN_SUFFIX = ".exe" if sys.platform == "win32" else "" @@ -107,51 +108,6 @@ def archive_deterministically(dir_to_archive, dest_archive, prepend_path=None): os.rename(temp_file, dest_archive) -def host_triple(): - os_type = platform.system().lower() - if os_type == "linux": - os_type = "unknown-linux-gnu" - elif os_type == "darwin": - os_type = "apple-darwin" - elif os_type == "android": - os_type = "linux-androideabi" - elif os_type == "windows": - # If we are in a Visual Studio environment, use msvc - if os.getenv("PLATFORM") is not None: - os_type = "pc-windows-msvc" - elif os.getenv("MSYSTEM") is not None: - os_type = "pc-windows-gnu" - else: - os_type = "unknown" - elif os_type.startswith("mingw64_nt-") or os_type.startswith("cygwin_nt-"): - os_type = "pc-windows-gnu" - elif os_type == "freebsd": - os_type = "unknown-freebsd" - else: - os_type = "unknown" - - cpu_type = platform.machine().lower() - if os_type.endswith("-msvc"): - # vcvars*.bat should set it properly - platform_env = os.environ.get("PLATFORM") - if platform_env == "X86": - cpu_type = "i686" - elif platform_env == "X64": - cpu_type = "x86_64" - else: - cpu_type = "unknown" - elif cpu_type in ["i386", "i486", "i686", "i768", "x86"]: - cpu_type = "i686" - elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]: - cpu_type = "x86_64" - elif cpu_type == "arm": - cpu_type = "arm" - else: - cpu_type = "unknown" - - return "%s-%s" % (cpu_type, os_type) - - def normalize_env(env): # There is a bug in subprocess where it doesn't like unicode types in # environment variables. Here, ensure all unicode are converted to @@ -428,14 +384,18 @@ class CommandBase(object): if "msvc" in (target or host_triple()): msvc_x64 = "64" if "x86_64" in (target or host_triple()) else "" msvc_deps_dir = path.join(self.context.sharedir, "msvc-dependencies") - extra_path += [path.join(msvc_deps_dir, "cmake", "bin")] - extra_path += [path.join(msvc_deps_dir, "ninja", "bin")] + + def package_dir(package): + return path.join(msvc_deps_dir, package, msvc_deps[package]) + + extra_path += [path.join(package_dir("cmake"), "bin")] + extra_path += [path.join(package_dir("ninja"), "bin")] # Link openssl - env["OPENSSL_INCLUDE_DIR"] = path.join(msvc_deps_dir, "openssl", "include") - env["OPENSSL_LIB_DIR"] = path.join(msvc_deps_dir, "openssl", "lib" + msvc_x64) + env["OPENSSL_INCLUDE_DIR"] = path.join(package_dir("openssl"), "include") + env["OPENSSL_LIB_DIR"] = path.join(package_dir("openssl"), "lib" + msvc_x64) env["OPENSSL_LIBS"] = "ssleay32MD:libeay32MD" # Link moztools - env["MOZTOOLS_PATH"] = path.join(msvc_deps_dir, "moztools", "bin") + env["MOZTOOLS_PATH"] = path.join(package_dir("moztools"), "bin") if is_windows(): if not os.environ.get("NATIVE_WIN32_PYTHON"): diff --git a/python/servo/bootstrapper/packages.py b/python/servo/packages.py index 7e9a7cd9c0e..9e4906dc5cf 100644 --- a/python/servo/bootstrapper/packages.py +++ b/python/servo/packages.py @@ -2,9 +2,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -# Listed all packages for different platforms in one file - -WINDOWS_GNU = [ +WINDOWS_GNU = set([ + "diffutils", + "make", "mingw-w64-x86_64-toolchain", "mingw-w64-x86_64-freetype", "mingw-w64-x86_64-icu", @@ -12,17 +12,15 @@ WINDOWS_GNU = [ "mingw-w64-x86_64-ca-certificates", "mingw-w64-x86_64-expat", "mingw-w64-x86_64-cmake", - "tar", - "diffutils", "patch", "patchutils", - "make", "python2-setuptools", -] + "tar", +]) -WINDOWS_MSVC = [ - "cmake-3.6.1", - "ninja-1.7.1", - "openssl-1.0.1t-vs2015", - "moztools-0.0.1-5", -] +WINDOWS_MSVC = { + "cmake": "3.6.1", + "moztools": "0.0.1-5", + "ninja": "1.7.1", + "openssl": "1.0.1t-vs2015", +} diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index 04b0542a079..185e7420a47 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -30,8 +30,9 @@ from mach.decorators import ( from servo.command_base import ( BuildNotFound, CommandBase, - call, cd, check_call, host_triple, set_osmesa_env, + call, cd, check_call, set_osmesa_env, ) +from servo.util import host_triple from wptrunner import wptcommandline from update import updatecommandline diff --git a/python/servo/util.py b/python/servo/util.py new file mode 100644 index 00000000000..03f993b3bf0 --- /dev/null +++ b/python/servo/util.py @@ -0,0 +1,151 @@ +# Copyright 2013 The Servo Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution. +# +# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import os.path as path +import platform +import sys +from socket import error as socket_error +import StringIO +import tarfile +import zipfile +import urllib2 + + +def host_triple(): + os_type = platform.system().lower() + if os_type == "linux": + os_type = "unknown-linux-gnu" + elif os_type == "darwin": + os_type = "apple-darwin" + elif os_type == "android": + os_type = "linux-androideabi" + elif os_type == "windows": + # If we are in a Visual Studio environment, use msvc + if os.getenv("PLATFORM") is not None: + os_type = "pc-windows-msvc" + elif os.getenv("MSYSTEM") is not None: + os_type = "pc-windows-gnu" + else: + os_type = "unknown" + elif os_type.startswith("mingw64_nt-") or os_type.startswith("cygwin_nt-"): + os_type = "pc-windows-gnu" + elif os_type == "freebsd": + os_type = "unknown-freebsd" + else: + os_type = "unknown" + + cpu_type = platform.machine().lower() + if os_type.endswith("-msvc"): + # vcvars*.bat should set it properly + platform_env = os.environ.get("PLATFORM") + if platform_env == "X86": + cpu_type = "i686" + elif platform_env == "X64": + cpu_type = "x86_64" + else: + cpu_type = "unknown" + elif cpu_type in ["i386", "i486", "i686", "i768", "x86"]: + cpu_type = "i686" + elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]: + cpu_type = "x86_64" + elif cpu_type == "arm": + cpu_type = "arm" + else: + cpu_type = "unknown" + + return "{}-{}".format(cpu_type, os_type) + + +def download(desc, src, writer, start_byte=0): + if start_byte: + print("Resuming download of {}...".format(desc)) + else: + print("Downloading {}...".format(desc)) + dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty()) + + try: + req = urllib2.Request(src) + if start_byte: + req = urllib2.Request(src, headers={'Range': 'bytes={}-'.format(start_byte)}) + resp = urllib2.urlopen(req) + + fsize = None + if resp.info().getheader('Content-Length'): + fsize = int(resp.info().getheader('Content-Length').strip()) + start_byte + + recved = start_byte + chunk_size = 8192 + + while True: + chunk = resp.read(chunk_size) + if not chunk: + break + recved += len(chunk) + if not dumb: + if fsize is not None: + pct = recved * 100.0 / fsize + print("\rDownloading %s: %5.1f%%" % (desc, pct), end="") + + sys.stdout.flush() + writer.write(chunk) + + if not dumb: + print() + except urllib2.HTTPError, e: + print("Download failed ({}): {} - {}".format(e.code, e.reason, src)) + if e.code == 403: + print("No Rust compiler binary available for this platform. " + "Please see https://github.com/servo/servo/#prerequisites") + sys.exit(1) + except urllib2.URLError, e: + print("Error downloading {}: {}. The failing URL was: {}".format(desc, e.reason, src)) + sys.exit(1) + except socket_error, e: + print("Looks like there's a connectivity issue, check your Internet connection. {}".format(e)) + sys.exit(1) + except KeyboardInterrupt: + writer.flush() + raise + + +def download_bytes(desc, src): + content_writer = StringIO.StringIO() + download(desc, src, content_writer) + return content_writer.getvalue() + + +def download_file(desc, src, dst): + tmp_path = dst + ".part" + try: + start_byte = path.getsize(tmp_path) + with open(tmp_path, 'ab') as fd: + download(desc, src, fd, start_byte=start_byte) + except os.error: + with open(tmp_path, 'wb') as fd: + download(desc, src, fd) + os.rename(tmp_path, dst) + + +def extract(src, dst, movedir=None): + if src.endswith(".zip"): + zipfile.ZipFile(src).extractall(dst) + else: + tarfile.open(src).extractall(dst) + + if movedir: + for f in os.listdir(movedir): + frm = path.join(movedir, f) + to = path.join(dst, f) + os.rename(frm, to) + os.rmdir(movedir) + + os.remove(src) |