diff options
Diffstat (limited to 'python/servo/bootstrap_commands.py')
| -rw-r--r-- | python/servo/bootstrap_commands.py | 343 |
1 files changed, 138 insertions, 205 deletions
diff --git a/python/servo/bootstrap_commands.py b/python/servo/bootstrap_commands.py index ba583735011..3be7a208815 100644 --- a/python/servo/bootstrap_commands.py +++ b/python/servo/bootstrap_commands.py @@ -13,10 +13,11 @@ import base64 import json import os import os.path as path +import platform import re -import shutil import subprocess import sys +import traceback import urllib2 import glob @@ -27,9 +28,8 @@ from mach.decorators import ( ) import servo.bootstrap as bootstrap -from servo.command_base import CommandBase, BIN_SUFFIX, cd -from servo.util import delete, download_bytes, download_file, extract, host_triple -from servo.util import STATIC_RUST_LANG_ORG_DIST +from servo.command_base import CommandBase, cd, check_call +from servo.util import delete, download_bytes, download_file, extract, check_hash @CommandProvider @@ -39,6 +39,7 @@ class MachCommands(CommandBase): category='bootstrap') def env(self): env = self.build_env() + print("export RUSTFLAGS=%s" % env["RUSTFLAGS"]) print("export PATH=%s" % env["PATH"]) if sys.platform == "darwin": print("export DYLD_LIBRARY_PATH=%s" % env["DYLD_LIBRARY_PATH"]) @@ -54,160 +55,107 @@ class MachCommands(CommandBase): def bootstrap(self, force=False): return bootstrap.bootstrap(self.context, force=force) - @Command('bootstrap-rust', - description='Download the Rust compiler', + @Command('bootstrap-android', + description='Install the Android SDK and NDK.', category='bootstrap') - @CommandArgument('--force', '-f', - action='store_true', - help='Force download even if a copy already exists') - @CommandArgument('--target', - action='append', - default=[], - help='Download rust stdlib for specified target') - @CommandArgument('--stable', - action='store_true', - help='Use stable rustc version') - def bootstrap_rustc(self, force=False, target=[], stable=False): - self.set_use_stable_rust(stable) - rust_dir = path.join(self.context.sharedir, "rust", self.rust_path()) - install_dir = path.join(self.context.sharedir, "rust", self.rust_install_dir()) - version = self.rust_stable_version() if stable else "nightly" - - nightly_dist = STATIC_RUST_LANG_ORG_DIST + "/" + self.rust_nightly_date() - - if not force and path.exists(path.join(rust_dir, "rustc", "bin", "rustc" + BIN_SUFFIX)): - print("Rust compiler already downloaded.", end=" ") - print("Use |bootstrap-rust --force| to download again.") - else: - if path.isdir(rust_dir): - shutil.rmtree(rust_dir) - os.makedirs(rust_dir) - - # The nightly Rust compiler is hosted on the nightly server under the date with a name - # rustc-nightly-HOST-TRIPLE.tar.gz, whereas the stable compiler is named - # rustc-VERSION-HOST-TRIPLE.tar.gz. We just need to pull down and extract it, - # giving a directory name that will be the same as the tarball name (rustc is - # in that directory). - if stable: - base_url = STATIC_RUST_LANG_ORG_DIST - else: - base_url = nightly_dist - - rustc_url = base_url + "/rustc-%s-%s.tar.gz" % (version, host_triple()) - tgz_file = rust_dir + '-rustc.tar.gz' - download_file("Rust compiler", rustc_url, tgz_file) - - print("Extracting Rust compiler...") - extract(tgz_file, install_dir) - print("Rust compiler ready.") - - # Each Rust stdlib has a name of the form `rust-std-nightly-TRIPLE.tar.gz` for the nightly - # releases, or rust-std-VERSION-TRIPLE.tar.gz for stable releases, with - # a directory of the name `rust-std-TRIPLE` inside and then a `lib` directory. - # This `lib` directory needs to be extracted and merged with the `rustc/lib` - # directory from the host compiler above. - lib_dir = path.join(install_dir, - "rustc-%s-%s" % (version, host_triple()), - "rustc", "lib", "rustlib") - - # ensure that the libs for the host's target is downloaded - host_target = host_triple() - if host_target not in target: - target.append(host_target) - - for target_triple in target: - target_lib_dir = path.join(lib_dir, target_triple) - if path.exists(target_lib_dir): - # No need to check for force. If --force the directory is already deleted - print("Rust lib for target {} already downloaded.".format(target_triple), end=" ") - print("Use |bootstrap-rust --force| to download again.") - continue - - tarball = "rust-std-%s-%s.tar.gz" % (version, target_triple) - tgz_file = path.join(install_dir, tarball) - if self.use_stable_rust(): - std_url = STATIC_RUST_LANG_ORG_DIST + "/" + tarball - else: - std_url = nightly_dist + "/" + tarball - - download_file("Host rust library for target %s" % target_triple, std_url, tgz_file) - print("Extracting Rust stdlib for target %s..." % target_triple) - extract(tgz_file, install_dir) - shutil.copytree(path.join(install_dir, - "rust-std-%s-%s" % (version, target_triple), - "rust-std-%s" % target_triple, - "lib", "rustlib", target_triple), - path.join(install_dir, - "rustc-%s-%s" % (version, host_triple()), - "rustc", - "lib", "rustlib", target_triple)) - shutil.rmtree(path.join(install_dir, "rust-std-%s-%s" % (version, target_triple))) - - print("Rust {} libs ready.".format(target_triple)) - - @Command('bootstrap-rust-docs', - description='Download the Rust documentation', - category='bootstrap') - @CommandArgument('--force', '-f', - action='store_true', - help='Force download even if docs already exist') - def bootstrap_rustc_docs(self, force=False): - self.ensure_bootstrapped() - rust_root = self.config["tools"]["rust-root"] - docs_dir = path.join(rust_root, "doc") - if not force and path.exists(docs_dir): - print("Rust docs already downloaded.", end=" ") - print("Use |bootstrap-rust-docs --force| to download again.") - return - - if path.isdir(docs_dir): - shutil.rmtree(docs_dir) - docs_name = self.rust_path().replace("rustc-", "rust-docs-") - docs_url = "%s/%s/rust-docs-nightly-%s.tar.gz" % ( - STATIC_RUST_LANG_ORG_DIST, self.rust_nightly_date(), host_triple() - ) - tgz_file = path.join(rust_root, 'doc.tar.gz') - - download_file("Rust docs", docs_url, tgz_file) - - print("Extracting Rust docs...") - temp_dir = path.join(rust_root, "temp_docs") - if path.isdir(temp_dir): - shutil.rmtree(temp_dir) - extract(tgz_file, temp_dir) - shutil.move(path.join(temp_dir, docs_name.split("/")[1], - "rust-docs", "share", "doc", "rust", "html"), - docs_dir) - shutil.rmtree(temp_dir) - print("Rust docs ready.") - - @Command('bootstrap-cargo', - description='Download the Cargo build tool', - category='bootstrap') - @CommandArgument('--force', '-f', - action='store_true', - help='Force download even if cargo already exists') - def bootstrap_cargo(self, force=False): - cargo_dir = path.join(self.context.sharedir, "cargo", self.rust_nightly_date()) - if not force and path.exists(path.join(cargo_dir, "cargo", "bin", "cargo" + BIN_SUFFIX)): - print("Cargo already downloaded.", end=" ") - print("Use |bootstrap-cargo --force| to download again.") - return - - if path.isdir(cargo_dir): - shutil.rmtree(cargo_dir) - os.makedirs(cargo_dir) - - tgz_file = "cargo-nightly-%s.tar.gz" % host_triple() - nightly_url = "%s/%s/%s" % (STATIC_RUST_LANG_ORG_DIST, self.rust_nightly_date(), tgz_file) - - download_file("Cargo nightly", nightly_url, tgz_file) + def bootstrap_android(self): + + ndk = "android-ndk-r12b-{system}-{arch}" + tools = "sdk-tools-{system}-4333796" + + sdk_build_tools = "25.0.2" + emulator_images = [ + ("servo-arm", "25", "google_apis;armeabi-v7a"), + ("servo-x86", "28", "google_apis;x86"), + ] + + known_sha1 = { + # https://dl.google.com/android/repository/repository2-1.xml + "sdk-tools-darwin-4333796.zip": "ed85ea7b59bc3483ce0af4c198523ba044e083ad", + "sdk-tools-linux-4333796.zip": "8c7c28554a32318461802c1291d76fccfafde054", + "sdk-tools-windows-4333796.zip": "aa298b5346ee0d63940d13609fe6bec621384510", + + # https://developer.android.com/ndk/downloads/older_releases + "android-ndk-r12b-windows-x86.zip": "8e6eef0091dac2f3c7a1ecbb7070d4fa22212c04", + "android-ndk-r12b-windows-x86_64.zip": "337746d8579a1c65e8a69bf9cbdc9849bcacf7f5", + "android-ndk-r12b-darwin-x86_64.zip": "e257fe12f8947be9f79c10c3fffe87fb9406118a", + "android-ndk-r12b-linux-x86_64.zip": "170a119bfa0f0ce5dc932405eaa3a7cc61b27694", + } - print("Extracting Cargo nightly...") - nightly_dir = path.join(cargo_dir, - path.basename(tgz_file).replace(".tar.gz", "")) - extract(tgz_file, cargo_dir, movedir=nightly_dir) - print("Cargo ready.") + toolchains = path.join(self.context.topdir, "android-toolchains") + if not path.isdir(toolchains): + os.makedirs(toolchains) + + def download(target_dir, name, flatten=False): + final = path.join(toolchains, target_dir) + if path.isdir(final): + return + + base_url = "https://dl.google.com/android/repository/" + filename = name + ".zip" + url = base_url + filename + archive = path.join(toolchains, filename) + + if not path.isfile(archive): + download_file(filename, url, archive) + check_hash(archive, known_sha1[filename], "sha1") + print("Extracting " + filename) + remove = True # Set to False to avoid repeated downloads while debugging this script + if flatten: + extracted = final + "_" + extract(archive, extracted, remove=remove) + contents = os.listdir(extracted) + assert len(contents) == 1 + os.rename(path.join(extracted, contents[0]), final) + os.rmdir(extracted) + else: + extract(archive, final, remove=remove) + + system = platform.system().lower() + machine = platform.machine().lower() + arch = {"i386": "x86"}.get(machine, machine) + download("ndk", ndk.format(system=system, arch=arch), flatten=True) + download("sdk", tools.format(system=system)) + + subprocess.check_call([ + path.join(toolchains, "sdk", "tools", "bin", "sdkmanager"), + "platform-tools", + "build-tools;" + sdk_build_tools, + "emulator", + ] + [ + arg + for avd_name, api_level, system_image in emulator_images + for arg in [ + "platforms;android-" + api_level, + "system-images;android-%s;%s" % (api_level, system_image), + ] + ]) + for avd_name, api_level, system_image in emulator_images: + process = subprocess.Popen(stdin=subprocess.PIPE, stdout=subprocess.PIPE, args=[ + path.join(toolchains, "sdk", "tools", "bin", "avdmanager"), + "create", "avd", + "--path", path.join(toolchains, "avd", avd_name), + "--name", avd_name, + "--package", "system-images;android-%s;%s" % (api_level, system_image), + "--force", + ]) + output = b"" + while 1: + # Read one byte at a time because in Python: + # * readline() blocks until "\n", which doesn't come before the prompt + # * read() blocks until EOF, which doesn't come before the prompt + # * read(n) keeps reading until it gets n bytes or EOF, + # but we don't know reliably how many bytes to read until the prompt + byte = process.stdout.read(1) + if len(byte) == 0: + break + output += byte + # There seems to be no way to disable this prompt: + if output.endswith(b"Do you wish to create a custom hardware profile? [no]"): + process.stdin.write("no\n") + assert process.wait() == 0 + with open(path.join(toolchains, "avd", avd_name, "config.ini"), "a") as f: + f.write("disk.dataPartition.size=2G\n") @Command('update-hsts-preload', description='Download the HSTS preload list', @@ -282,51 +230,28 @@ class MachCommands(CommandBase): default='1', help='Keep up to this many most recent nightlies') def clean_nightlies(self, force=False, keep=None): - rust_current_nightly = self.rust_nightly_date() - rust_current_stable = self.rust_stable_version() - print("Current Rust nightly version: {}".format(rust_current_nightly)) - print("Current Rust stable version: {}".format(rust_current_stable)) - to_keep = set() - if int(keep) == 1: - # Optimize keep=1 case to not invoke git - to_keep.add(rust_current_nightly) - to_keep.add(rust_current_stable) - else: - for version_file in ['rust-toolchain', 'rust-stable-version']: - cmd = subprocess.Popen( - ['git', 'log', '--oneline', '--no-color', '-n', keep, '--patch', version_file], - stdout=subprocess.PIPE, - universal_newlines=True - ) - stdout, _ = cmd.communicate() - for line in stdout.splitlines(): - if line.startswith(b"+") and not line.startswith(b"+++"): - line = line[len(b"+"):] - if line.startswith(b"nightly-"): - line = line[len(b"nightly-"):] - to_keep.add(line) + default_toolchain = self.default_toolchain() + print("Current Rust version for Servo: {}".format(default_toolchain)) + old_toolchains = [] + keep = int(keep) + stdout = subprocess.check_output(['git', 'log', '--format=%H', 'rust-toolchain']) + for i, commit_hash in enumerate(stdout.split(), 1): + if i > keep: + toolchain = subprocess.check_output( + ['git', 'show', '%s:rust-toolchain' % commit_hash]) + old_toolchains.append(toolchain.strip()) removing_anything = False - for tool in ["rust", "cargo"]: - base = path.join(self.context.sharedir, tool) - if not path.isdir(base): - continue - for name in os.listdir(base): - full_path = path.join(base, name) - if name.startswith("rust-"): - name = name[len("rust-"):] - if name.endswith("-alt"): - name = name[:-len("-alt")] - if name not in to_keep: + stdout = subprocess.check_output(['rustup', 'toolchain', 'list']) + for toolchain_with_host in stdout.split(): + for old in old_toolchains: + if toolchain_with_host.startswith(old): removing_anything = True if force: - print("Removing {}".format(full_path)) - try: - delete(full_path) - except OSError as e: - print("Removal failed with error {}".format(e)) + print("Removing {}".format(toolchain_with_host)) + check_call(["rustup", "uninstall", toolchain_with_host]) else: - print("Would remove {}".format(full_path)) + print("Would remove {}".format(toolchain_with_host)) if not removing_anything: print("Nothing to remove.") elif not force: @@ -345,10 +270,7 @@ class MachCommands(CommandBase): @CommandArgument('--keep', default='1', help='Keep up to this many most recent dependencies') - @CommandArgument('--custom-path', '-c', - action='store_true', - help='Get Cargo path from CARGO_HOME environment variable') - def clean_cargo_cache(self, force=False, show_size=False, keep=None, custom_path=False): + def clean_cargo_cache(self, force=False, show_size=False, keep=None): def get_size(path): if os.path.isfile(path): return os.path.getsize(path) / (1024 * 1024.0) @@ -365,10 +287,11 @@ class MachCommands(CommandBase): 'git': {}, } import toml - if os.environ.get("CARGO_HOME", "") and custom_path: + if os.environ.get("CARGO_HOME", ""): cargo_dir = os.environ.get("CARGO_HOME") else: - cargo_dir = path.join(self.context.topdir, ".cargo") + home_dir = os.path.expanduser("~") + cargo_dir = path.join(home_dir, ".cargo") if not os.path.isdir(cargo_dir): return cargo_file = open(path.join(self.context.topdir, "Cargo.lock")) @@ -414,8 +337,14 @@ class MachCommands(CommandBase): git_dir = path.join(cargo_dir, "git") git_db_dir = path.join(git_dir, "db") git_checkout_dir = path.join(git_dir, "checkouts") - git_db_list = filter(lambda f: not f.startswith('.'), os.listdir(git_db_dir)) - git_checkout_list = os.listdir(git_checkout_dir) + if os.path.isdir(git_db_dir): + git_db_list = filter(lambda f: not f.startswith('.'), os.listdir(git_db_dir)) + else: + git_db_list = [] + if os.path.isdir(git_checkout_dir): + git_checkout_list = os.listdir(git_checkout_dir) + else: + git_checkout_list = [] for d in list(set(git_db_list + git_checkout_list)): crate_name = d.replace("-{}".format(d.split("-")[-1]), "") @@ -493,7 +422,11 @@ class MachCommands(CommandBase): print("Removing `{}`{} package from {}".format(*print_msg)) for crate_path in crate_paths: if os.path.exists(crate_path): - delete(crate_path) + try: + delete(crate_path) + except: + print(traceback.format_exc()) + print("Delete %s failed!" % crate_path) else: print("Would remove `{}`{} package from {}".format(*print_msg)) |