diff options
| -rw-r--r-- | etc/ci/buildbot_steps.yml | 17 | ||||
| -rw-r--r-- | python/servo/bootstrap_commands.py | 61 | ||||
| -rw-r--r-- | python/servo/package_commands.py | 8 | ||||
| -rw-r--r-- | python/servo/util.py | 18 |
4 files changed, 77 insertions, 27 deletions
diff --git a/etc/ci/buildbot_steps.yml b/etc/ci/buildbot_steps.yml index f3ff1077473..96ec70fc6a0 100644 --- a/etc/ci/buildbot_steps.yml +++ b/etc/ci/buildbot_steps.yml @@ -1,4 +1,5 @@ mac-rel-wpt1: + - ./mach clean-nightlies --keep 3 --force - ./mach build --release - ./mach test-wpt-failure - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 1 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed @@ -9,12 +10,14 @@ mac-rel-wpt1: - bash ./etc/ci/manifest_changed.sh mac-rel-wpt2: + - ./mach clean-nightlies --keep 3 --force - ./mach build --release - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 2 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker - ./mach build-geckolib --release mac-dev-unit: + - ./mach clean-nightlies --keep 3 --force - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --dev - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach test-unit - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach package --dev @@ -24,6 +27,7 @@ mac-dev-unit: - bash ./etc/ci/manifest_changed.sh mac-rel-css: + - ./mach clean-nightlies --keep 3 --force - ./mach build --release - ./mach test-css --release --processes 4 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed - ./mach filter-intermittents css-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-css-errorsummary.log --use-tracker @@ -31,20 +35,24 @@ mac-rel-css: - bash ./etc/ci/manifest_changed.sh mac-nightly: + - ./mach clean-nightlies --keep 3 --force - ./mach build --release - ./mach package --release - ./etc/ci/upload_nightly.sh mac - ./etc/ci/upload_nightly.sh macbrew linux-rel-intermittent: + - ./mach clean-nightlies --keep 3 --force - ./mach build --release - ./etc/ci/check_intermittents.sh --log-raw intermittents.log mac-rel-intermittent: + - ./mach clean-nightlies --keep 3 --force - ./mach build --release - ./etc/ci/check_intermittents.sh --log-raw intermittents.log linux-dev: + - ./mach clean-nightlies --keep 3 --force - ./mach test-tidy --no-progress --all - ./mach test-tidy --no-progress --self-test - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --dev @@ -59,6 +67,7 @@ linux-dev: - bash ./etc/ci/check_no_panic.sh linux-rel-wpt: + - ./mach clean-nightlies --keep 3 --force - ./mach build --release --with-debug-assertions - ./mach test-wpt-failure - ./mach test-wpt --release --processes 24 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed @@ -66,6 +75,7 @@ linux-rel-wpt: - ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource linux-rel-css: + - ./mach clean-nightlies --keep 3 --force - ./mach build --release --with-debug-assertions - ./mach test-css --release --processes 16 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed - ./mach filter-intermittents css-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-css-errorsummary.log --use-tracker @@ -76,11 +86,13 @@ linux-rel-css: - bash ./etc/ci/manifest_changed.sh linux-nightly: + - ./mach clean-nightlies --keep 3 --force - ./mach build --release - ./mach package --release - ./etc/ci/upload_nightly.sh linux android: + - ./mach clean-nightlies --keep 3 --force - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --android --dev - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach package --android --dev - bash ./etc/ci/lockfile_changed.sh @@ -88,27 +100,32 @@ android: - python ./etc/ci/check_dynamic_symbols.py android-nightly: + - ./mach clean-nightlies --keep 3 --force - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --android --release - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach package --android --release - ./etc/ci/upload_nightly.sh android arm32: + - ./mach clean-nightlies --keep 3 --force - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --rel --target=arm-unknown-linux-gnueabihf - bash ./etc/ci/lockfile_changed.sh - bash ./etc/ci/manifest_changed.sh arm64: + - ./mach clean-nightlies --keep 3 --force - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --rel --target=aarch64-unknown-linux-gnu - bash ./etc/ci/lockfile_changed.sh - bash ./etc/ci/manifest_changed.sh windows-msvc-dev: + - mach.bat clean-nightlies --keep 3 --force - mach.bat build --dev - mach.bat test-unit - mach.bat package --dev - mach.bat build-geckolib windows-msvc-nightly: + - mach.bat clean-nightlies --keep 3 --force - mach.bat build --release - mach.bat package --release - bash -l ./etc/ci/upload_nightly.sh windows-msvc diff --git a/python/servo/bootstrap_commands.py b/python/servo/bootstrap_commands.py index 998bb5a19b7..28b5d4fa673 100644 --- a/python/servo/bootstrap_commands.py +++ b/python/servo/bootstrap_commands.py @@ -15,6 +15,7 @@ import os import os.path as path import re import shutil +import subprocess import sys import urllib2 @@ -26,7 +27,7 @@ from mach.decorators import ( import servo.bootstrap as bootstrap from servo.command_base import CommandBase, BIN_SUFFIX -from servo.util import download_bytes, download_file, extract, host_triple +from servo.util import delete, download_bytes, download_file, extract, host_triple @CommandProvider @@ -285,26 +286,56 @@ class MachCommands(CommandBase): @CommandArgument('--force', '-f', action='store_true', help='Actually remove stuff') - def clean_nightlies(self, force=False): - rust_current = self.rust_path().split('/')[0] + @CommandArgument('--keep', + default='1', + help='Keep up to this many most recent nightlies') + def clean_nightlies(self, force=False, keep=None): + rust_current = self.rust_version() cargo_current = self.cargo_build_id() - print("Current Rust version: " + rust_current) - print("Current Cargo version: " + cargo_current) + print("Current Rust version: {}".format(rust_current)) + print("Current Cargo version: {}".format(cargo_current)) + to_keep = { + 'rust': set(), + 'cargo': set(), + } + if int(keep) == 1: + # Optimize keep=1 case to not invoke git + to_keep['rust'].add(rust_current) + to_keep['cargo'].add(cargo_current) + else: + for tool in ["rust", "cargo"]: + commit_file = '{}-commit-hash'.format(tool) + cmd = subprocess.Popen( + ['git', 'log', '--pretty=format:%H', '-n', keep, commit_file], + stdout=subprocess.PIPE, + universal_newlines=True + ) + stdout, _ = cmd.communicate() + for commit in stdout.splitlines(): + cmd = subprocess.Popen( + ['git', 'show', '{}:{}'.format(commit, commit_file)], + stdout=subprocess.PIPE, + universal_newlines=True + ) + commit_hash, _ = cmd.communicate() + to_keep[tool].add(commit_hash.rstrip()) + removing_anything = False - for current, base in [(rust_current, "rust"), (cargo_current, "cargo")]: - base = path.join(self.context.sharedir, base) + for tool in ["rust", "cargo"]: + base = path.join(self.context.sharedir, tool) for name in os.listdir(base): - if name != current: + # We append `-alt` if LLVM assertions aren't enabled, + # so use just the commit hash itself. + # This may occasionally leave an extra nightly behind + # but won't remove too many nightlies. + if name.partition('-')[0] not in to_keep[tool]: removing_anything = True - name = path.join(base, name) + full_path = path.join(base, name) if force: - print("Removing " + name) - if os.path.isdir(name): - shutil.rmtree(name) - else: - os.remove(name) + print("Removing {}".format(full_path)) + delete(full_path) else: - print("Would remove " + name) + print("Would remove {}".format(full_path)) if not removing_anything: print("Nothing to remove.") elif not force: diff --git a/python/servo/package_commands.py b/python/servo/package_commands.py index 68569838c4c..9abffc603e5 100644 --- a/python/servo/package_commands.py +++ b/python/servo/package_commands.py @@ -33,13 +33,7 @@ from servo.command_base import ( is_windows, get_browserhtml_path, ) - - -def delete(path): - try: - os.remove(path) # Succeeds if path was a file - except OSError: # Or, if path was a directory... - shutil.rmtree(path) # Remove it and all its contents. +from servo.util import delete def otool(s): diff --git a/python/servo/util.py b/python/servo/util.py index c3217df8bfd..ad55d6b3f79 100644 --- a/python/servo/util.py +++ b/python/servo/util.py @@ -10,16 +10,24 @@ from __future__ import absolute_import, print_function, unicode_literals import os -import os.path as path +import os.path import platform -import sys +import shutil from socket import error as socket_error import StringIO +import sys import tarfile import zipfile import urllib2 +def delete(path): + if os.path.isdir(path) and not os.path.islink(path): + shutil.rmtree(path) + else: + os.remove(path) + + def host_platform(): os_type = platform.system().lower() if os_type == "linux": @@ -126,7 +134,7 @@ def download_bytes(desc, src): def download_file(desc, src, dst): tmp_path = dst + ".part" try: - start_byte = path.getsize(tmp_path) + start_byte = os.path.getsize(tmp_path) with open(tmp_path, 'ab') as fd: download(desc, src, fd, start_byte=start_byte) except os.error: @@ -143,8 +151,8 @@ def extract(src, dst, movedir=None): if movedir: for f in os.listdir(movedir): - frm = path.join(movedir, f) - to = path.join(dst, f) + frm = os.path.join(movedir, f) + to = os.path.join(dst, f) os.rename(frm, to) os.rmdir(movedir) |