diff options
| -rw-r--r-- | etc/ci/buildbot_steps.yml | 10 | ||||
| -rwxr-xr-x | etc/ci/upload_nightly.sh | 113 | ||||
| -rw-r--r-- | python/requirements.txt | 3 | ||||
| -rw-r--r-- | python/servo/package_commands.py | 122 |
4 files changed, 129 insertions, 119 deletions
diff --git a/etc/ci/buildbot_steps.yml b/etc/ci/buildbot_steps.yml index c3b413742f3..eb48d4ae3cd 100644 --- a/etc/ci/buildbot_steps.yml +++ b/etc/ci/buildbot_steps.yml @@ -38,8 +38,8 @@ mac-nightly: - ./mach clean-nightlies --keep 3 --force - ./mach build --release - ./mach package --release - - ./etc/ci/upload_nightly.sh mac - - ./etc/ci/upload_nightly.sh macbrew + - ./mach upload-nightly mac + - ./mach upload-nightly macbrew linux-rel-intermittent: - ./mach clean-nightlies --keep 3 --force @@ -89,7 +89,7 @@ linux-nightly: - ./mach clean-nightlies --keep 3 --force - ./mach build --release - ./mach package --release - - ./etc/ci/upload_nightly.sh linux + - ./mach upload-nightly linux android: - ./mach clean-nightlies --keep 3 --force @@ -103,7 +103,7 @@ android-nightly: - ./mach clean-nightlies --keep 3 --force - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --android --release - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ANDROID_SDK=/home/servo/android/sdk/r25.2.3 ./mach package --android --release - - ./etc/ci/upload_nightly.sh android + - ./mach upload-nightly android arm32: - ./mach clean-nightlies --keep 3 --force @@ -128,4 +128,4 @@ windows-msvc-nightly: - mach.bat clean-nightlies --keep 3 --force - mach.bat build --release - mach.bat package --release - - bash -l ./etc/ci/upload_nightly.sh windows-msvc + - mach.bat upload-nightly windows-msvc diff --git a/etc/ci/upload_nightly.sh b/etc/ci/upload_nightly.sh deleted file mode 100755 index fbdb7ba99ad..00000000000 --- a/etc/ci/upload_nightly.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env bash - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -set -o errexit -set -o nounset -set -o pipefail -shopt -s failglob - - -usage() { - printf \ - "usage: %s android|linux|mac|macbrew|windows-msvc\n" \ - "${0}" -} - - -upload() { - local nightly_filename - nightly_filename="${4}-$(basename "${2}")" - local -r nightly_upload_dir="s3://servo-builds/nightly/${1}" - local -r package_upload_path="${nightly_upload_dir}/${nightly_filename}" - s3cmd --mime-type="application/octet-stream" \ - put "${2}" "${package_upload_path}" - s3cmd cp "${package_upload_path}" "${nightly_upload_dir}/servo-latest.${3}" -} - -update_brew() { - echo "Updating brew formula" - - local package_url sha version script_dir tmp_dir nightly_filename - - nightly_filename="${2}-$(basename "${1}")" - package_url="https://download.servo.org/nightly/macbrew/${nightly_filename}" - sha="$(shasum -a 256 "${1}" | sed -e 's/ .*//')" - - # This will transform a timestamp (2016-12-13T08-01-10Z for example) - # into a valid brew version number (2016.12.13). - version="$(echo "${2}" | \ - sed -n 's/\([0-9]\{4\}\)-\([0-9]\{2\}\)-\([0-9]\{2\}\).*/\1.\2.\3/p')" - - script_dir="${PWD}/$(dirname "${0}")" - tmp_dir="$(mktemp -d -t homebrew-servo.XXXXX)" - - git -C "${tmp_dir}" clone https://github.com/servo/homebrew-servo.git . - - # Not using "/" as it's used in PACKAGEURL - sed "s|PACKAGEURL|${package_url}|g - s|SHA|${sha}|g - s|VERSION|${version}|g" \ - < "${script_dir}/servo-binary-formula.rb.in" \ - > "${tmp_dir}/Formula/servo-bin.rb" - - git -C "${tmp_dir}" add ./Formula/servo-bin.rb - git -C "${tmp_dir}" \ - -c user.name="Tom Servo" \ - -c user.email="servo@servo.org" \ - commit \ - --message="Version bump: ${version}" - - git -C "${tmp_dir}" push -qf \ - "https://${GITHUB_HOMEBREW_TOKEN}@github.com/servo/homebrew-servo.git" \ - master >/dev/null 2>&1 - - rm -rf "${tmp_dir}" -} - -main() { - if (( "${#}" != 1 )); then - usage >&2 - return 1 - fi - - local platform package extension nightly_timestamp - platform="${1}" - nightly_timestamp="$(date -u +"%Y-%m-%dT%H-%M-%SZ")" - - if [[ "${platform}" == "android" ]]; then - extension=apk - package=target/arm-linux-androideabi/release/*."${extension}" - elif [[ "${platform}" == "linux" ]]; then - extension=tar.gz - package=target/release/*."${extension}" - elif [[ "${platform}" == "mac" ]]; then - extension=dmg - package=target/release/*."${extension}" - elif [[ "${platform}" == "macbrew" ]]; then - extension=tar.gz - package=target/release/brew/*."${extension}" - elif [[ "${platform}" == "windows-msvc" ]]; then - extension=msi - package=target/release/msi/*.msi - upload "${platform}" ${package} "${extension}" "${nightly_timestamp}" - extension=zip - package=target/release/msi/*.zip - else - usage >&2 - return 1 - fi - - # Lack of quotes on package allows glob expansion - # Note that this is not robust in the case of embedded spaces - # TODO(aneeshusa): make this glob robust using e.g. arrays or Python - upload "${platform}" ${package} "${extension}" "${nightly_timestamp}" - - if [[ "${platform}" == "macbrew" ]]; then - update_brew ${package} "${nightly_timestamp}" - fi -} - -main "${@}" diff --git a/python/requirements.txt b/python/requirements.txt index 41b25ccdb5a..b96c4741b05 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -24,4 +24,7 @@ ply == 3.8 # For Cross-platform colored terminal text colorama == 0.3.7 +# For package uploading +boto3 == 1.4.4 + -e python/tidy diff --git a/python/servo/package_commands.py b/python/servo/package_commands.py index 63921037c25..8b3d9d32418 100644 --- a/python/servo/package_commands.py +++ b/python/servo/package_commands.py @@ -7,13 +7,17 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -from __future__ import print_function, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals +from datetime import datetime +import hashlib import json import os import os.path as path import shutil import subprocess +import sys +import tempfile from mach.decorators import ( CommandArgument, @@ -35,6 +39,26 @@ from servo.command_base import ( from servo.util import delete +PACKAGES = { + 'android': [ + 'target/arm-linux-androideabi/release/servo.apk', + ], + 'linux': [ + 'target/release/servo-tech-demo.tar.gz', + ], + 'mac': [ + 'target/release/servo-tech-demo.dmg', + ], + 'macbrew': [ + 'target/release/brew/servo.tar.gz', + ], + 'windows-msvc': [ + r'target\release\msi\Servo.msi', + r'target\release\msi\Servo.zip', + ], +} + + def otool(s): o = subprocess.Popen(['/usr/bin/otool', '-L', s], stdout=subprocess.PIPE) for l in o.stdout: @@ -364,3 +388,99 @@ class PackageCommands(CommandBase): print(" ".join(exec_command)) return subprocess.call(exec_command, env=self.build_env()) + + @Command('upload-nightly', + description='Upload Servo nightly to S3', + category='package') + @CommandArgument('platform', + choices=PACKAGES.keys(), + help='Package platform type to upload') + def upload_nightly(self, platform): + import boto3 + + def nightly_filename(package, timestamp): + return '{}-{}'.format( + timestamp.isoformat() + 'Z', # The `Z` denotes UTC + path.basename(package) + ) + + def upload_to_s3(platform, package, timestamp): + s3 = boto3.client('s3') + BUCKET = 'servo-builds' + + nightly_dir = 'nightly/{}'.format(platform) + filename = nightly_filename(package, timestamp) + package_upload_key = '{}/{}'.format(nightly_dir, filename) + extension = path.basename(package).partition('.')[2] + latest_upload_key = '{}/servo-latest.{}'.format(nightly_dir, extension) + + s3.upload_file(package, BUCKET, package_upload_key) + copy_source = { + 'Bucket': BUCKET, + 'Key': package_upload_key, + } + s3.copy(copy_source, BUCKET, latest_upload_key) + + def update_brew(package, timestamp): + print("Updating brew formula") + + package_url = 'https://download.servo.org/nightly/macbrew/{}'.format( + nightly_filename(package, timestamp) + ) + with open(package) as p: + digest = hashlib.sha256(p.read()).hexdigest() + + brew_version = timestamp.strftime('%Y.%m.%d') + + with tempfile.TemporaryDirectory(prefix='homebrew-servo') as tmp_dir: + def call_git(cmd, **kwargs): + subprocess.check_call( + ['git', '-C', tmp_dir] + cmd, + **kwargs + ) + + call_git([ + 'clone', + 'https://github.com/servo/homebrew-servo.git', + '.', + ]) + + script_dir = path.dirname(path.realpath(__file__)) + with open(path.join(script_dir, 'servo-binary-formula.rb.in')) as f: + formula = f.read() + formula = formula.replace('PACKAGEURL', package_url) + formula = formula.replace('SHA', digest) + formula = formula.replace('VERSION', brew_version) + with open(path.join(tmp_dir, 'Formula', 'servo-bin.rb')) as f: + f.write(formula) + + call_git(['add', path.join('.', 'Formula', 'servo-bin.rb')]) + call_git([ + '-c', 'user.name=Tom Servo', + '-c', 'user.email=servo@servo.org', + 'commit', + '--message=Version Bump: {}'.format(brew_version), + ]) + + token = os.environ['GITHUB_HOMEBREW_TOKEN'] + call_git([ + 'push', + '-qf', + 'https://{}@github.com/servo/homebrew-servo.git'.format(token), + 'master', + ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + timestamp = datetime.utcnow().replace(microsecond=0) + for package in PACKAGES[platform]: + if not path.isfile(package): + print("Could not find package for {} at {}".format( + platform, + package + ), file=sys.stderr) + return 1 + upload_to_s3(platform, package, timestamp) + + if platform == 'macbrew': + update_brew(package, timestamp) + + return 0 |