1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function
from distutils.spawn import find_executable
from distutils.version import LooseVersion
import json
import os
import platform
import shutil
import subprocess
from subprocess import PIPE
import servo.packages as packages
from servo.util import extract, download_file, host_triple
def run_as_root(command):
if os.geteuid() != 0:
command.insert(0, 'sudo')
return subprocess.call(command)
def install_salt_dependencies(context, force):
install = False
if context.distro == 'Ubuntu':
pkgs = ['build-essential', 'libssl-dev', 'libffi-dev', 'python-dev']
command = ['apt-get', 'install']
if subprocess.call(['dpkg', '-s'] + pkgs, stdout=PIPE, stderr=PIPE) != 0:
install = True
elif context.distro in ['CentOS', 'CentOS Linux', 'Fedora']:
installed_pkgs = str(subprocess.check_output(['rpm', '-qa'])).replace('\n', '|')
pkgs = ['gcc', 'libffi-devel', 'python-devel', 'openssl-devel']
for p in pkgs:
command = ['dnf', 'install']
if "|{}".format(p) not in installed_pkgs:
install = True
break
if install:
if force:
command.append('-y')
print("Installing missing Salt dependencies...")
run_as_root(command + pkgs)
def salt(context, force=False):
# Ensure Salt dependencies are installed
install_salt_dependencies(context, force)
# Ensure Salt is installed in the virtualenv
# It's not instaled globally because it's a large, non-required dependency,
# and the installation fails on Windows
print("Checking Salt installation...", end='')
reqs_path = os.path.join(context.topdir, 'python', 'requirements-salt.txt')
process = subprocess.Popen(
["pip", "install", "-q", "-I", "-r", reqs_path],
stdout=PIPE,
stderr=PIPE
)
process.wait()
if process.returncode:
out, err = process.communicate()
print('failed to install Salt via pip:')
print('Output: {}\nError: {}'.format(out, err))
return 1
print("done")
salt_root = os.path.join(context.sharedir, 'salt')
config_dir = os.path.join(salt_root, 'etc', 'salt')
pillar_dir = os.path.join(config_dir, 'pillars')
# In order to allow `mach bootstrap` to work from any CWD,
# the `root_dir` must be an absolute path.
# We place it under `context.sharedir` because
# Salt caches data (e.g. gitfs files) in its `var` subdirectory.
# Hence, dynamically generate the config with an appropriate `root_dir`
# and serialize it as JSON (which is valid YAML).
config = {
'hash_type': 'sha384',
'master': 'localhost',
'root_dir': salt_root,
'state_output': 'changes',
'state_tabular': True,
}
if 'SERVO_SALTFS_ROOT' in os.environ:
config.update({
'fileserver_backend': ['roots'],
'file_roots': {
'base': [os.path.abspath(os.environ['SERVO_SALTFS_ROOT'])],
},
})
else:
config.update({
'fileserver_backend': ['git'],
'gitfs_env_whitelist': 'base',
'gitfs_provider': 'gitpython',
'gitfs_remotes': [
'https://github.com/servo/saltfs.git',
],
})
if not os.path.exists(config_dir):
os.makedirs(config_dir, mode=0o700)
with open(os.path.join(config_dir, 'minion'), 'w') as config_file:
config_file.write(json.dumps(config) + '\n')
# Similarly, the pillar data is created dynamically
# and temporarily serialized to disk.
# This dynamism is not yet used, but will be in the future
# to enable Android bootstrapping by using
# context.sharedir as a location for Android packages.
pillar = {
'top.sls': {
'base': {
'*': ['bootstrap'],
},
},
'bootstrap.sls': {
'fully_managed': False,
},
}
if os.path.exists(pillar_dir):
shutil.rmtree(pillar_dir)
os.makedirs(pillar_dir, mode=0o700)
for filename in pillar:
with open(os.path.join(pillar_dir, filename), 'w') as pillar_file:
pillar_file.write(json.dumps(pillar[filename]) + '\n')
cmd = [
# sudo escapes from the venv, need to use full path
find_executable('salt-call'),
'--local',
'--config-dir={}'.format(config_dir),
'--pillar-root={}'.format(pillar_dir),
'state.apply',
'servo-build-dependencies',
]
if not force:
print('Running bootstrap in dry-run mode to show changes')
# Because `test=True` mode runs each state individually without
# considering how required/previous states affect the system,
# it will often report states with requisites as failing due
# to the requisites not actually being run,
# even though these are spurious and will succeed during
# the actual highstate.
# Hence `--retcode-passthrough` is not helpful in dry-run mode,
# so only detect failures of the actual salt-call binary itself.
retcode = run_as_root(cmd + ['test=True'])
if retcode != 0:
print('Something went wrong while bootstrapping')
return retcode
proceed = raw_input(
'Proposed changes are above, proceed with bootstrap? [y/N]: '
)
if proceed.lower() not in ['y', 'yes']:
return 0
print('')
print('Running Salt bootstrap')
retcode = run_as_root(cmd + ['--retcode-passthrough'])
if retcode == 0:
print('Salt bootstrapping complete')
else:
print('Salt bootstrapping encountered errors')
return retcode
def windows_msvc(context, force=False):
'''Bootstrapper for MSVC building on Windows.'''
deps_dir = os.path.join(context.sharedir, "msvc-dependencies")
deps_url = "https://servo-deps.s3.amazonaws.com/msvc-deps/"
def version(package):
return packages.WINDOWS_MSVC[package]
def package_dir(package):
return os.path.join(deps_dir, package, version(package))
def check_cmake(version):
cmake_path = find_executable("cmake")
if cmake_path:
cmake = subprocess.Popen([cmake_path, "--version"], stdout=PIPE)
cmake_version = cmake.stdout.read().splitlines()[0].replace("cmake version ", "")
if LooseVersion(cmake_version) >= LooseVersion(version):
return True
return False
to_install = {}
for package in packages.WINDOWS_MSVC:
# Don't install CMake if it already exists in PATH
if package == "cmake" and check_cmake(version("cmake")):
continue
if not os.path.isdir(package_dir(package)):
to_install[package] = version(package)
if not to_install:
return 0
print("Installing missing MSVC dependencies...")
for package in to_install:
full_spec = '{}-{}'.format(package, version(package))
parent_dir = os.path.dirname(package_dir(package))
if not os.path.isdir(parent_dir):
os.makedirs(parent_dir)
zip_path = package_dir(package) + ".zip"
if not os.path.isfile(zip_path):
zip_url = "{}{}.zip".format(deps_url, full_spec)
download_file(full_spec, zip_url, zip_path)
print("Extracting {}...".format(full_spec), end='')
extract(zip_path, deps_dir)
print("done")
extracted_path = os.path.join(deps_dir, full_spec)
os.rename(extracted_path, package_dir(package))
return 0
def bootstrap(context, force=False):
'''Dispatches to the right bootstrapping function for the OS.'''
bootstrapper = None
if "windows-msvc" in host_triple():
bootstrapper = windows_msvc
elif "linux-gnu" in host_triple():
distro, version, _ = platform.linux_distribution()
if distro.lower() in [
'centos',
'centos linux',
'debian',
'fedora',
'ubuntu',
]:
context.distro = distro
bootstrapper = salt
if bootstrapper is None:
print('Bootstrap support is not yet available for your OS.')
return 1
return bootstrapper(context, force=force)
|
