Auto merge of #14974 - aneeshusa:add-mach-bootstrap, r=metajack

Use Salt for mach bootstrap

<!-- Please describe your changes on the following line: -->

This is currently WIP, but allows Salt for `mach bootstrap`. Not looking for review yet, just posting for visibility. You can run `./mach bootstrap` and Salt will run, letting you know what changes it would make for bootstrapping (doesn't actually run yet though).

Currently, this reads from saltfs using gitfs, meaning it always tracks the master branch. (Note that this is blocked on https://github.com/servo/saltfs/pull/577 landing; in the meantime, I've included a small workaround in this PR to pull from my updated saltfs branch, which will need to be removed.) In the future, the relevant Salt code may be merged in tree as part of Docker support for TC, and the bootstrapper should be updated to read from in tree.

Also, our Windows machines have not been Salted, so the existing Windows bootstrappers are retained.

TODO:
- [x] Hook into existing bootstrapping code less hackily
- [x] Actually bootstrap instead of always using `test=True` mode (includes sudo)
- [x] Default to interactive mode (test first, then ask), with a force flag
- [x] Don't require running from the repository root directory
- [x] Make it easy to add support for other distros

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: -->
- [ ] `./mach build -d` does not report any errors
- [x] `./mach test-tidy` does not report any errors
- [ ] These changes fix #__ (github issue number if applicable).

<!-- Either: -->
- [ ] There are tests for these changes OR
- [x] These changes do not require tests because they should be verified manually

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/14974)
<!-- Reviewable:end -->

12 files changed, 414 insertions, 439 deletions

diff --git a/python/servo/bootstrap.py b/python/servo/bootstrap.py
new file mode 100644
index 00000000000..54862a885bd
--- /dev/null
+++ b/python/servo/bootstrap.py
@@ -0,0 +1,229 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+from distutils.spawn import find_executable
+import json
+import os
+import platform
+import shutil
+import subprocess
+
+import servo.packages as packages
+from servo.util import extract, download_file, host_triple
+
+
+def salt(context, force=False):
+    # Ensure Salt is installed in the virtualenv
+    # It's not instaled globally because it's a large, non-required dependency,
+    # and the installation fails on Windows
+    print("Checking Salt installation...", end='')
+    reqs_path = os.path.join(context.topdir, 'python', 'requirements-salt.txt')
+    process = subprocess.Popen(
+        ["pip", "install", "-q", "-I", "-r", reqs_path],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE
+    )
+    process.wait()
+    if process.returncode:
+        out, err = process.communicate()
+        print('failed to install Salt via pip:')
+        print('Output: {}\nError: {}'.format(out, err))
+        return 1
+    print("done")
+
+    salt_root = os.path.join(context.sharedir, 'salt')
+    config_dir = os.path.join(salt_root, 'etc', 'salt')
+    pillar_dir = os.path.join(config_dir, 'pillars')
+
+    # In order to allow `mach bootstrap` to work from any CWD,
+    # the `root_dir` must be an absolute path.
+    # We place it under `context.sharedir` because
+    # Salt caches data (e.g. gitfs files) in its `var` subdirectory.
+    # Hence, dynamically generate the config with an appropriate `root_dir`
+    # and serialize it as JSON (which is valid YAML).
+    config = {
+        'fileserver_backend': ['git'],
+        'gitfs_env_whitelist': 'base',
+        'gitfs_provider': 'gitpython',
+        'gitfs_remotes': [
+            'https://github.com/servo/saltfs.git',
+        ],
+        'hash_type': 'sha384',
+        'master': 'localhost',
+        'root_dir': salt_root,
+        'state_output': 'changes',
+        'state_tabular': True,
+    }
+
+    if not os.path.exists(config_dir):
+        os.makedirs(config_dir, mode=0o700)
+    with open(os.path.join(config_dir, 'minion'), 'w') as config_file:
+        config_file.write(json.dumps(config) + '\n')
+
+    # Similarly, the pillar data is created dynamically
+    # and temporarily serialized to disk.
+    # This dynamism is not yet used, but will be in the future
+    # to enable Android bootstrapping by using
+    # context.sharedir as a location for Android packages.
+    pillar = {
+        'top.sls': {
+            'base': {
+                '*': ['bootstrap'],
+            },
+        },
+        'bootstrap.sls': {
+            'fully_managed': False,
+        },
+    }
+    if os.path.exists(pillar_dir):
+        shutil.rmtree(pillar_dir)
+    os.makedirs(pillar_dir, mode=0o700)
+    for filename in pillar:
+        with open(os.path.join(pillar_dir, filename), 'w') as pillar_file:
+            pillar_file.write(json.dumps(pillar[filename]) + '\n')
+
+    cmd = [
+        'sudo',
+        # sudo escapes from the venv, need to use full path
+        find_executable('salt-call'),
+        '--local',
+        '--config-dir={}'.format(config_dir),
+        '--pillar-root={}'.format(pillar_dir),
+        'state.apply',
+        'servo-build-dependencies',
+    ]
+
+    if not force:
+        print('Running bootstrap in dry-run mode to show changes')
+        # Because `test=True` mode runs each state individually without
+        # considering how required/previous states affect the system,
+        # it will often report states with requisites as failing due
+        # to the requisites not actually being run,
+        # even though these are spurious and will succeed during
+        # the actual highstate.
+        # Hence `--retcode-passthrough` is not helpful in dry-run mode,
+        # so only detect failures of the actual salt-call binary itself.
+        retcode = subprocess.call(cmd + ['test=True'])
+        if retcode != 0:
+            print('Something went wrong while bootstrapping')
+            return retcode
+
+        proceed = raw_input(
+            'Proposed changes are above, proceed with bootstrap? [y/N]: '
+        )
+        if proceed.lower() not in ['y', 'yes']:
+            return 0
+
+        print('')
+
+    print('Running Salt bootstrap')
+    retcode = subprocess.call(cmd + ['--retcode-passthrough'])
+    if retcode == 0:
+        print('Salt bootstrapping complete')
+    else:
+        print('Salt bootstrapping encountered errors')
+    return retcode
+
+
+def windows_gnu(context, force=False):
+    '''Bootstrapper for msys2 based environments for building in Windows.'''
+
+    if not find_executable('pacman'):
+        print(
+            'The Windows GNU bootstrapper only works with msys2 with pacman. '
+            'Get msys2 at http://msys2.github.io/'
+        )
+        return 1
+
+    # Ensure repositories are up to date
+    command = ['pacman', '--sync', '--refresh']
+    subprocess.check_call(command)
+
+    # Install packages
+    command = ['pacman', '--sync', '--needed']
+    if force:
+        command.append('--noconfirm')
+    subprocess.check_call(command + list(packages.WINDOWS_GNU))
+
+    # Downgrade GCC to 5.4.0-1
+    gcc_pkgs = ["gcc", "gcc-ada", "gcc-fortran", "gcc-libgfortran", "gcc-libs", "gcc-objc"]
+    gcc_version = "5.4.0-1"
+    mingw_url = "http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-{}-{}-any.pkg.tar.xz"
+    gcc_list = [mingw_url.format(gcc, gcc_version) for gcc in gcc_pkgs]
+
+    # Note: `--upgrade` also does downgrades
+    downgrade_command = ['pacman', '--upgrade']
+    if force:
+        downgrade_command.append('--noconfirm')
+    subprocess.check_call(downgrade_command + gcc_list)
+
+
+def windows_msvc(context, force=False):
+    '''Bootstrapper for MSVC building on Windows.'''
+
+    deps_dir = os.path.join(context.sharedir, "msvc-dependencies")
+    deps_url = "https://servo-rust.s3.amazonaws.com/msvc-deps/"
+
+    def version(package):
+        return packages.WINDOWS_MSVC[package]
+
+    def package_dir(package):
+        return os.path.join(deps_dir, package, version(package))
+
+    to_install = {}
+    for package in packages.WINDOWS_MSVC:
+        # Don't install CMake if it already exists in PATH
+        if package == "cmake" and find_executable(package):
+            continue
+
+        if not os.path.isdir(package_dir(package)):
+            to_install[package] = version(package)
+
+    if not to_install:
+        return 0
+
+    print("Installing missing MSVC dependencies...")
+    for package in to_install:
+        full_spec = '{}-{}'.format(package, version(package))
+
+        parent_dir = os.path.dirname(package_dir(package))
+        if not os.path.isdir(parent_dir):
+            os.makedirs(parent_dir)
+
+        zip_path = package_dir(package) + ".zip"
+        if not os.path.isfile(zip_path):
+            zip_url = "{}{}.zip".format(deps_url, full_spec)
+            download_file(full_spec, zip_url, zip_path)
+
+        print("Extracting {}...".format(full_spec), end='')
+        extract(zip_path, deps_dir)
+        print("done")
+
+        extracted_path = os.path.join(deps_dir, full_spec)
+        os.rename(extracted_path, package_dir(package))
+
+    return 0
+
+
+def bootstrap(context, force=False):
+    '''Dispatches to the right bootstrapping function for the OS.'''
+
+    bootstrapper = None
+
+    if "windows-gnu" in host_triple():
+        bootstrapper = windows_gnu
+    elif "windows-msvc" in host_triple():
+        bootstrapper = windows_msvc
+    elif "linux-gnu" in host_triple():
+        distro, version, _ = platform.linux_distribution()
+        if distro == 'Ubuntu' and version == '14.04':
+            bootstrapper = salt
+
+    if bootstrapper is None:
+        print('Bootstrap support is not yet available for your OS.')
+        return 1
+
+    return bootstrapper(context, force=force)
diff --git a/python/servo/bootstrap_commands.py b/python/servo/bootstrap_commands.py
index 9bc7c60fd4a..8975f174d84 100644
--- a/python/servo/bootstrap_commands.py
+++ b/python/servo/bootstrap_commands.py
@@ -7,8 +7,7 @@
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
-from __future__ import print_function, unicode_literals
-from socket import error as socket_error
+from __future__ import absolute_import, print_function, unicode_literals
 
 import base64
 import json
@@ -17,9 +16,6 @@ import os.path as path
 import re
 import shutil
 import sys
-import StringIO
-import tarfile
-import zipfile
 import urllib2
 
 from mach.decorators import (
@@ -28,93 +24,9 @@ from mach.decorators import (
     Command,
 )
 
-from servo.command_base import CommandBase, host_triple, BIN_SUFFIX
-
-
-def download(desc, src, writer, start_byte=0):
-    if start_byte:
-        print("Resuming download of %s..." % desc)
-    else:
-        print("Downloading %s..." % desc)
-    dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty())
-
-    try:
-        req = urllib2.Request(src)
-        if start_byte:
-            req = urllib2.Request(src, headers={'Range': 'bytes={}-'.format(start_byte)})
-        resp = urllib2.urlopen(req)
-
-        fsize = None
-        if resp.info().getheader('Content-Length'):
-            fsize = int(resp.info().getheader('Content-Length').strip()) + start_byte
-
-        recved = start_byte
-        chunk_size = 8192
-
-        while True:
-            chunk = resp.read(chunk_size)
-            if not chunk:
-                break
-            recved += len(chunk)
-            if not dumb:
-                if fsize is not None:
-                    pct = recved * 100.0 / fsize
-                    print("\rDownloading %s: %5.1f%%" % (desc, pct), end="")
-
-                sys.stdout.flush()
-            writer.write(chunk)
-
-        if not dumb:
-            print()
-    except urllib2.HTTPError, e:
-        print("Download failed (%d): %s - %s" % (e.code, e.reason, src))
-        if e.code == 403:
-            print("No Rust compiler binary available for this platform. "
-                  "Please see https://github.com/servo/servo/#prerequisites")
-        sys.exit(1)
-    except urllib2.URLError, e:
-        print("Error downloading Rust compiler: %s. The failing URL was: %s" % (e.reason, src))
-        sys.exit(1)
-    except socket_error, e:
-        print("Looks like there's a connectivity issue, check your Internet connection. %s" % (e))
-        sys.exit(1)
-    except KeyboardInterrupt:
-        writer.flush()
-        raise
-
-
-def download_file(desc, src, dst):
-    tmp_path = dst + ".part"
-    try:
-        start_byte = os.path.getsize(tmp_path)
-        with open(tmp_path, 'ab') as fd:
-            download(desc, src, fd, start_byte=start_byte)
-    except os.error:
-        with open(tmp_path, 'wb') as fd:
-            download(desc, src, fd)
-    os.rename(tmp_path, dst)
-
-
-def download_bytes(desc, src):
-    content_writer = StringIO.StringIO()
-    download(desc, src, content_writer)
-    return content_writer.getvalue()
-
-
-def extract(src, dst, movedir=None):
-    if src.endswith(".zip"):
-        zipfile.ZipFile(src).extractall(dst)
-    else:
-        tarfile.open(src).extractall(dst)
-
-    if movedir:
-        for f in os.listdir(movedir):
-            frm = path.join(movedir, f)
-            to = path.join(dst, f)
-            os.rename(frm, to)
-        os.rmdir(movedir)
-
-    os.remove(src)
+import servo.bootstrap as bootstrap
+from servo.command_base import CommandBase, BIN_SUFFIX
+from servo.util import download_bytes, download_file, extract, host_triple
 
 
 @CommandProvider
@@ -133,20 +45,11 @@ class MachCommands(CommandBase):
     @Command('bootstrap',
              description='Install required packages for building.',
              category='bootstrap')
-    @CommandArgument('--interactive', "-i",
-                     action='store_true',
-                     help='Need to answer any (Y/n) interactive prompts.')
-    @CommandArgument('--android',
-                     action='store_true',
-                     help='Install required packages for Android')
     @CommandArgument('--force', '-f',
                      action='store_true',
-                     help='Force reinstall packages')
-    def bootstrap(self, android=False, interactive=False, force=False):
-        from servo.bootstrapper.bootstrap import Bootstrapper
-
-        bootstrapper = Bootstrapper(self.context)
-        bootstrapper.bootstrap(android=android, interactive=interactive, force=force)
+                     help='Boostrap without confirmation')
+    def bootstrap(self, force=False):
+        return bootstrap.bootstrap(self.context, force=force)
 
     @Command('bootstrap-rust',
              description='Download the Rust compiler',
diff --git a/python/servo/bootstrapper/__init__.py b/python/servo/bootstrapper/__init__.py
deleted file mode 100644
index 6fbe8159b2d..00000000000
--- a/python/servo/bootstrapper/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
diff --git a/python/servo/bootstrapper/base.py b/python/servo/bootstrapper/base.py
deleted file mode 100644
index 40f3d1f5ebc..00000000000
--- a/python/servo/bootstrapper/base.py
+++ /dev/null
@@ -1,62 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import print_function, unicode_literals
-
-import distutils
-import subprocess
-
-
-class BaseBootstrapper(object):
-    """Base class for system bootstrappers."""
-
-    def __init__(self, interactive=False):
-        self.package_manager_updated = False
-        self.interactive = interactive
-
-    def ensure_system_packages(self):
-        '''
-        Check for missing packages.
-        '''
-        raise NotImplementedError('%s must implement ensure_system_packages()' %
-                                  __name__)
-
-    def install_system_packages(self):
-        '''
-        Install packages required to build Servo.
-        '''
-        raise NotImplementedError('%s must implement install_system_packages()' %
-                                  __name__)
-
-    def install_mobile_android_packages(self):
-        '''
-        Install packages required to build Servo for Android.
-        '''
-        raise NotImplementedError('Cannot bootstrap Servo for Android: '
-                                  '%s does not yet implement install_mobile_android_packages()'
-                                  % __name__)
-
-    def which(self, name):
-        """Python implementation of which.
-
-        It returns the path of an executable or None if it couldn't be found.
-        """
-        return distutils.spawn.find_executable(name)
-
-    def check_output(self, *args, **kwargs):
-        """Run subprocess.check_output."""
-        return subprocess.check_output(*args, **kwargs)
-
-    def _ensure_package_manager_updated(self):
-        if self.package_manager_updated:
-            return
-
-        self._update_package_manager()
-        self.package_manager_updated = True
-
-    def _update_package_manager(self):
-        """Updates the package manager's manifests/package list.
-
-        This should be defined in child classes.
-        """
diff --git a/python/servo/bootstrapper/bootstrap.py b/python/servo/bootstrapper/bootstrap.py
deleted file mode 100644
index d07dc12fc3c..00000000000
--- a/python/servo/bootstrapper/bootstrap.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import print_function
-
-import sys
-
-from windows_gnu import WindowsGnuBootstrapper
-from windows_msvc import WindowsMsvcBootstrapper
-
-
-class Bootstrapper(object):
-    """Main class that performs system bootstrap."""
-
-    def __init__(self, context):
-        self.instance = None
-        cls = None
-        args = {}
-
-        if sys.platform.startswith('msys'):
-            cls = WindowsGnuBootstrapper
-
-        elif sys.platform.startswith('win32'):
-            cls = WindowsMsvcBootstrapper
-
-        if cls is None:
-            sys.exit('Bootstrap support is not yet available for your OS.')
-
-        self.instance = cls(**args)
-        self.instance.context = context
-
-    def bootstrap(self, android=False, interactive=False, force=False):
-        self.instance.interactive = interactive
-        self.instance.force = force
-
-        if android:
-            self.instance.install_mobile_android_packages()
-        elif force:
-            self.instance.install_system_packages()
-        else:
-            self.instance.ensure_system_packages()
diff --git a/python/servo/bootstrapper/windows_gnu.py b/python/servo/bootstrapper/windows_gnu.py
deleted file mode 100644
index 794f1790cd1..00000000000
--- a/python/servo/bootstrapper/windows_gnu.py
+++ /dev/null
@@ -1,75 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import sys
-import subprocess
-
-from base import BaseBootstrapper
-from packages import WINDOWS_GNU as deps
-
-
-class WindowsGnuBootstrapper(BaseBootstrapper):
-    '''Bootstrapper for msys2 based environments for building in Windows.'''
-
-    def __init__(self, **kwargs):
-        BaseBootstrapper.__init__(self, **kwargs)
-
-        if not self.which('pacman'):
-            raise NotImplementedError('The Windows bootstrapper only works with msys2 with pacman. Get msys2 at '
-                                      'http://msys2.github.io/')
-
-    def ensure_system_packages(self):
-        install_packages = []
-        for p in deps:
-            command = ['pacman', '-Qs', p]
-            if self.run_check(command):
-                install_packages += [p]
-        if install_packages:
-            install_packages(install_packages)
-
-    def install_system_packages(self, packages=deps):
-        self._ensure_package_manager_updated()
-        self.pacman_install(*packages)
-
-    def install_mobile_android_packages(self):
-        sys.exit('We do not support building Android on Windows. Sorry!')
-
-    def _update_package_manager(self):
-        self.pacman_update()
-
-    def run(self, command):
-        subprocess.check_call(command, stdin=sys.stdin)
-
-    def run_check(self, command):
-        return subprocess.call(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
-    def pacman_update(self):
-        command = ['pacman', '--sync', '--refresh']
-        self.run(command)
-
-    def pacman_upgrade(self):
-        command = ['pacman', '--sync', '--refresh', '--sysupgrade']
-        self.run(command)
-
-    def pacman_install(self, *packages):
-        command = ['pacman', '--sync']
-        if not self.force:
-            command.append('--needed')
-        if not self.interactive:
-            command.append('--noconfirm')
-        command.extend(packages)
-        self.run(command)
-
-        # downgrade GCC to 5.4.0-1
-        gcc_type = ["gcc", "gcc-ada", "gcc-fortran", "gcc-libgfortran", "gcc-libs", "gcc-objc"]
-        gcc_version = "5.4.0-1"
-        mingw_url = "http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-{}-{}-any.pkg.tar.xz"
-        gcc_list = []
-        for gcc in gcc_type:
-            gcc_list += [mingw_url.format(gcc, gcc_version)]
-        downgrade_command = ['pacman', '-U']
-        if not self.interactive:
-            downgrade_command.append('--noconfirm')
-        downgrade_command.extend(gcc_list)
-        self.run(downgrade_command)
diff --git a/python/servo/bootstrapper/windows_msvc.py b/python/servo/bootstrapper/windows_msvc.py
deleted file mode 100644
index 11ff765c99d..00000000000
--- a/python/servo/bootstrapper/windows_msvc.py
+++ /dev/null
@@ -1,86 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import os
-import sys
-import shutil
-from distutils import spawn
-
-from base import BaseBootstrapper
-from packages import WINDOWS_MSVC as deps
-
-
-class WindowsMsvcBootstrapper(BaseBootstrapper):
-    '''Bootstrapper for MSVC building on Windows.'''
-
-    def __init__(self, **kwargs):
-        BaseBootstrapper.__init__(self, **kwargs)
-
-    def ensure_system_packages(self):
-        self.install_system_packages()
-
-    def install_system_packages(self, packages=deps):
-        from servo.bootstrap_commands import extract, download_file
-
-        deps_dir = os.path.join(self.context.sharedir, "msvc-dependencies")
-        deps_url = "https://servo-rust.s3.amazonaws.com/msvc-deps/"
-        first_run = True
-
-        if self.force:
-            if os.path.isdir(deps_dir):
-                shutil.rmtree(deps_dir)
-
-        if not os.path.isdir(deps_dir):
-            os.makedirs(deps_dir)
-
-        # Read file with installed dependencies, if exist
-        installed_deps_file = os.path.join(deps_dir, "installed-dependencies.txt")
-        if os.path.exists(installed_deps_file):
-            installed_deps = [l.strip() for l in open(installed_deps_file)]
-        else:
-            installed_deps = []
-
-        # list of dependencies that need to be updated
-        update_deps = list(set(packages) - set(installed_deps))
-
-        for dep in packages:
-            dep_name = dep.split("-")[0]
-
-            # Don't download CMake if already exists in PATH
-            if dep_name == "cmake":
-                if spawn.find_executable(dep_name):
-                    continue
-
-            dep_dir = os.path.join(deps_dir, dep_name)
-            # if not installed or need to be updated
-            if not os.path.exists(dep_dir) or dep in update_deps:
-                if first_run:
-                    print "Installing missing MSVC dependencies..."
-                    first_run = False
-
-                dep_version_dir = os.path.join(deps_dir, dep)
-
-                if os.path.exists(dep_version_dir):
-                    shutil.rmtree(dep_version_dir)
-
-                dep_zip = dep_version_dir + ".zip"
-                if not os.path.isfile(dep_zip):
-                    download_file(dep, "%s%s.zip" % (deps_url, dep), dep_zip)
-
-                print "Extracting %s..." % dep,
-                extract(dep_zip, deps_dir)
-                print "done"
-
-                # Delete directory if exist
-                if os.path.exists(dep_dir):
-                    shutil.rmtree(dep_dir)
-                os.rename(dep_version_dir, dep_dir)
-
-        # Write in installed-dependencies.txt file
-        with open(installed_deps_file, 'w') as installed_file:
-            for line in packages:
-                installed_file.write(line + "\n")
-
-    def install_mobile_android_packages(self):
-        sys.exit('We do not support building Android on Windows. Sorry!')
diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py
index 6e7ec8d24c5..5fbb7ba164d 100644
--- a/python/servo/build_commands.py
+++ b/python/servo/build_commands.py
@@ -24,7 +24,8 @@ from mach.decorators import (
     Command,
 )
 
-from servo.command_base import CommandBase, cd, call, BIN_SUFFIX, host_triple, find_dep_path_newest
+from servo.command_base import CommandBase, cd, call, BIN_SUFFIX, find_dep_path_newest
+from servo.util import host_triple
 
 
 def format_duration(seconds):
diff --git a/python/servo/command_base.py b/python/servo/command_base.py
index 80fe675da1c..e80c9cb018b 100644
--- a/python/servo/command_base.py
+++ b/python/servo/command_base.py
@@ -18,11 +18,12 @@ import subprocess
 from subprocess import PIPE
 import sys
 import tarfile
-import platform
 
+from mach.registrar import Registrar
 import toml
 
-from mach.registrar import Registrar
+from servo.packages import WINDOWS_MSVC as msvc_deps
+from servo.util import host_triple
 
 BIN_SUFFIX = ".exe" if sys.platform == "win32" else ""
 
@@ -107,51 +108,6 @@ def archive_deterministically(dir_to_archive, dest_archive, prepend_path=None):
         os.rename(temp_file, dest_archive)
 
 
-def host_triple():
-    os_type = platform.system().lower()
-    if os_type == "linux":
-        os_type = "unknown-linux-gnu"
-    elif os_type == "darwin":
-        os_type = "apple-darwin"
-    elif os_type == "android":
-        os_type = "linux-androideabi"
-    elif os_type == "windows":
-        # If we are in a Visual Studio environment, use msvc
-        if os.getenv("PLATFORM") is not None:
-            os_type = "pc-windows-msvc"
-        elif os.getenv("MSYSTEM") is not None:
-            os_type = "pc-windows-gnu"
-        else:
-            os_type = "unknown"
-    elif os_type.startswith("mingw64_nt-") or os_type.startswith("cygwin_nt-"):
-        os_type = "pc-windows-gnu"
-    elif os_type == "freebsd":
-        os_type = "unknown-freebsd"
-    else:
-        os_type = "unknown"
-
-    cpu_type = platform.machine().lower()
-    if os_type.endswith("-msvc"):
-        # vcvars*.bat should set it properly
-        platform_env = os.environ.get("PLATFORM")
-        if platform_env == "X86":
-            cpu_type = "i686"
-        elif platform_env == "X64":
-            cpu_type = "x86_64"
-        else:
-            cpu_type = "unknown"
-    elif cpu_type in ["i386", "i486", "i686", "i768", "x86"]:
-        cpu_type = "i686"
-    elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]:
-        cpu_type = "x86_64"
-    elif cpu_type == "arm":
-        cpu_type = "arm"
-    else:
-        cpu_type = "unknown"
-
-    return "%s-%s" % (cpu_type, os_type)
-
-
 def normalize_env(env):
     # There is a bug in subprocess where it doesn't like unicode types in
     # environment variables. Here, ensure all unicode are converted to
@@ -428,14 +384,18 @@ class CommandBase(object):
         if "msvc" in (target or host_triple()):
             msvc_x64 = "64" if "x86_64" in (target or host_triple()) else ""
             msvc_deps_dir = path.join(self.context.sharedir, "msvc-dependencies")
-            extra_path += [path.join(msvc_deps_dir, "cmake", "bin")]
-            extra_path += [path.join(msvc_deps_dir, "ninja", "bin")]
+
+            def package_dir(package):
+                return path.join(msvc_deps_dir, package, msvc_deps[package])
+
+            extra_path += [path.join(package_dir("cmake"), "bin")]
+            extra_path += [path.join(package_dir("ninja"), "bin")]
             # Link openssl
-            env["OPENSSL_INCLUDE_DIR"] = path.join(msvc_deps_dir, "openssl", "include")
-            env["OPENSSL_LIB_DIR"] = path.join(msvc_deps_dir, "openssl", "lib" + msvc_x64)
+            env["OPENSSL_INCLUDE_DIR"] = path.join(package_dir("openssl"), "include")
+            env["OPENSSL_LIB_DIR"] = path.join(package_dir("openssl"), "lib" + msvc_x64)
             env["OPENSSL_LIBS"] = "ssleay32MD:libeay32MD"
             # Link moztools
-            env["MOZTOOLS_PATH"] = path.join(msvc_deps_dir, "moztools", "bin")
+            env["MOZTOOLS_PATH"] = path.join(package_dir("moztools"), "bin")
 
         if is_windows():
             if not os.environ.get("NATIVE_WIN32_PYTHON"):
diff --git a/python/servo/bootstrapper/packages.py b/python/servo/packages.py
index 7e9a7cd9c0e..9e4906dc5cf 100644
--- a/python/servo/bootstrapper/packages.py
+++ b/python/servo/packages.py
@@ -2,9 +2,9 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# Listed all packages for different platforms in one file
-
-WINDOWS_GNU = [
+WINDOWS_GNU = set([
+    "diffutils",
+    "make",
     "mingw-w64-x86_64-toolchain",
     "mingw-w64-x86_64-freetype",
     "mingw-w64-x86_64-icu",
@@ -12,17 +12,15 @@ WINDOWS_GNU = [
     "mingw-w64-x86_64-ca-certificates",
     "mingw-w64-x86_64-expat",
     "mingw-w64-x86_64-cmake",
-    "tar",
-    "diffutils",
     "patch",
     "patchutils",
-    "make",
     "python2-setuptools",
-]
+    "tar",
+])
 
-WINDOWS_MSVC = [
-    "cmake-3.6.1",
-    "ninja-1.7.1",
-    "openssl-1.0.1t-vs2015",
-    "moztools-0.0.1-5",
-]
+WINDOWS_MSVC = {
+    "cmake": "3.6.1",
+    "moztools": "0.0.1-5",
+    "ninja": "1.7.1",
+    "openssl": "1.0.1t-vs2015",
+}
diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py
index 04b0542a079..185e7420a47 100644
--- a/python/servo/testing_commands.py
+++ b/python/servo/testing_commands.py
@@ -30,8 +30,9 @@ from mach.decorators import (
 
 from servo.command_base import (
     BuildNotFound, CommandBase,
-    call, cd, check_call, host_triple, set_osmesa_env,
+    call, cd, check_call, set_osmesa_env,
 )
+from servo.util import host_triple
 
 from wptrunner import wptcommandline
 from update import updatecommandline
diff --git a/python/servo/util.py b/python/servo/util.py
new file mode 100644
index 00000000000..03f993b3bf0
--- /dev/null
+++ b/python/servo/util.py
@@ -0,0 +1,151 @@
+# Copyright 2013 The Servo Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import os.path as path
+import platform
+import sys
+from socket import error as socket_error
+import StringIO
+import tarfile
+import zipfile
+import urllib2
+
+
+def host_triple():
+    os_type = platform.system().lower()
+    if os_type == "linux":
+        os_type = "unknown-linux-gnu"
+    elif os_type == "darwin":
+        os_type = "apple-darwin"
+    elif os_type == "android":
+        os_type = "linux-androideabi"
+    elif os_type == "windows":
+        # If we are in a Visual Studio environment, use msvc
+        if os.getenv("PLATFORM") is not None:
+            os_type = "pc-windows-msvc"
+        elif os.getenv("MSYSTEM") is not None:
+            os_type = "pc-windows-gnu"
+        else:
+            os_type = "unknown"
+    elif os_type.startswith("mingw64_nt-") or os_type.startswith("cygwin_nt-"):
+        os_type = "pc-windows-gnu"
+    elif os_type == "freebsd":
+        os_type = "unknown-freebsd"
+    else:
+        os_type = "unknown"
+
+    cpu_type = platform.machine().lower()
+    if os_type.endswith("-msvc"):
+        # vcvars*.bat should set it properly
+        platform_env = os.environ.get("PLATFORM")
+        if platform_env == "X86":
+            cpu_type = "i686"
+        elif platform_env == "X64":
+            cpu_type = "x86_64"
+        else:
+            cpu_type = "unknown"
+    elif cpu_type in ["i386", "i486", "i686", "i768", "x86"]:
+        cpu_type = "i686"
+    elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]:
+        cpu_type = "x86_64"
+    elif cpu_type == "arm":
+        cpu_type = "arm"
+    else:
+        cpu_type = "unknown"
+
+    return "{}-{}".format(cpu_type, os_type)
+
+
+def download(desc, src, writer, start_byte=0):
+    if start_byte:
+        print("Resuming download of {}...".format(desc))
+    else:
+        print("Downloading {}...".format(desc))
+    dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty())
+
+    try:
+        req = urllib2.Request(src)
+        if start_byte:
+            req = urllib2.Request(src, headers={'Range': 'bytes={}-'.format(start_byte)})
+        resp = urllib2.urlopen(req)
+
+        fsize = None
+        if resp.info().getheader('Content-Length'):
+            fsize = int(resp.info().getheader('Content-Length').strip()) + start_byte
+
+        recved = start_byte
+        chunk_size = 8192
+
+        while True:
+            chunk = resp.read(chunk_size)
+            if not chunk:
+                break
+            recved += len(chunk)
+            if not dumb:
+                if fsize is not None:
+                    pct = recved * 100.0 / fsize
+                    print("\rDownloading %s: %5.1f%%" % (desc, pct), end="")
+
+                sys.stdout.flush()
+            writer.write(chunk)
+
+        if not dumb:
+            print()
+    except urllib2.HTTPError, e:
+        print("Download failed ({}): {} - {}".format(e.code, e.reason, src))
+        if e.code == 403:
+            print("No Rust compiler binary available for this platform. "
+                  "Please see https://github.com/servo/servo/#prerequisites")
+        sys.exit(1)
+    except urllib2.URLError, e:
+        print("Error downloading {}: {}. The failing URL was: {}".format(desc, e.reason, src))
+        sys.exit(1)
+    except socket_error, e:
+        print("Looks like there's a connectivity issue, check your Internet connection. {}".format(e))
+        sys.exit(1)
+    except KeyboardInterrupt:
+        writer.flush()
+        raise
+
+
+def download_bytes(desc, src):
+    content_writer = StringIO.StringIO()
+    download(desc, src, content_writer)
+    return content_writer.getvalue()
+
+
+def download_file(desc, src, dst):
+    tmp_path = dst + ".part"
+    try:
+        start_byte = path.getsize(tmp_path)
+        with open(tmp_path, 'ab') as fd:
+            download(desc, src, fd, start_byte=start_byte)
+    except os.error:
+        with open(tmp_path, 'wb') as fd:
+            download(desc, src, fd)
+    os.rename(tmp_path, dst)
+
+
+def extract(src, dst, movedir=None):
+    if src.endswith(".zip"):
+        zipfile.ZipFile(src).extractall(dst)
+    else:
+        tarfile.open(src).extractall(dst)
+
+    if movedir:
+        for f in os.listdir(movedir):
+            frm = path.join(movedir, f)
+            to = path.join(dst, f)
+            os.rename(frm, to)
+        os.rmdir(movedir)
+
+    os.remove(src)