From 043f78db53818431ef20d24107e445128e856822 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Fri, 21 Apr 2017 19:17:22 -0400 Subject: Move nightly upload script into Mach Now that MinGW and MSYS have been removed from the Windows builders, bash is not available to run the previous upload_nightlies.sh script. Convert the script to Python 2 for cross-platform support. Additionally, switch to the `boto3` library for easy uploading without needing to install `s3cmd`, and move the code into mach for easy `boto3` installation as the new `./mach upload-nightly` command. Also, hard-code the paths to the packages instead of using globs to look for them, as the paths are static. (The paths used to contain timestamps, but we now insert timestamps when uploading to S3 to improve reproducibility.) --- python/servo/package_commands.py | 122 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 121 insertions(+), 1 deletion(-) (limited to 'python/servo/package_commands.py') diff --git a/python/servo/package_commands.py b/python/servo/package_commands.py index 63921037c25..8b3d9d32418 100644 --- a/python/servo/package_commands.py +++ b/python/servo/package_commands.py @@ -7,13 +7,17 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -from __future__ import print_function, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals +from datetime import datetime +import hashlib import json import os import os.path as path import shutil import subprocess +import sys +import tempfile from mach.decorators import ( CommandArgument, @@ -35,6 +39,26 @@ from servo.command_base import ( from servo.util import delete +PACKAGES = { + 'android': [ + 'target/arm-linux-androideabi/release/servo.apk', + ], + 'linux': [ + 'target/release/servo-tech-demo.tar.gz', + ], + 'mac': [ + 'target/release/servo-tech-demo.dmg', + ], + 'macbrew': [ + 'target/release/brew/servo.tar.gz', + ], + 'windows-msvc': [ + r'target\release\msi\Servo.msi', + r'target\release\msi\Servo.zip', + ], +} + + def otool(s): o = subprocess.Popen(['/usr/bin/otool', '-L', s], stdout=subprocess.PIPE) for l in o.stdout: @@ -364,3 +388,99 @@ class PackageCommands(CommandBase): print(" ".join(exec_command)) return subprocess.call(exec_command, env=self.build_env()) + + @Command('upload-nightly', + description='Upload Servo nightly to S3', + category='package') + @CommandArgument('platform', + choices=PACKAGES.keys(), + help='Package platform type to upload') + def upload_nightly(self, platform): + import boto3 + + def nightly_filename(package, timestamp): + return '{}-{}'.format( + timestamp.isoformat() + 'Z', # The `Z` denotes UTC + path.basename(package) + ) + + def upload_to_s3(platform, package, timestamp): + s3 = boto3.client('s3') + BUCKET = 'servo-builds' + + nightly_dir = 'nightly/{}'.format(platform) + filename = nightly_filename(package, timestamp) + package_upload_key = '{}/{}'.format(nightly_dir, filename) + extension = path.basename(package).partition('.')[2] + latest_upload_key = '{}/servo-latest.{}'.format(nightly_dir, extension) + + s3.upload_file(package, BUCKET, package_upload_key) + copy_source = { + 'Bucket': BUCKET, + 'Key': package_upload_key, + } + s3.copy(copy_source, BUCKET, latest_upload_key) + + def update_brew(package, timestamp): + print("Updating brew formula") + + package_url = 'https://download.servo.org/nightly/macbrew/{}'.format( + nightly_filename(package, timestamp) + ) + with open(package) as p: + digest = hashlib.sha256(p.read()).hexdigest() + + brew_version = timestamp.strftime('%Y.%m.%d') + + with tempfile.TemporaryDirectory(prefix='homebrew-servo') as tmp_dir: + def call_git(cmd, **kwargs): + subprocess.check_call( + ['git', '-C', tmp_dir] + cmd, + **kwargs + ) + + call_git([ + 'clone', + 'https://github.com/servo/homebrew-servo.git', + '.', + ]) + + script_dir = path.dirname(path.realpath(__file__)) + with open(path.join(script_dir, 'servo-binary-formula.rb.in')) as f: + formula = f.read() + formula = formula.replace('PACKAGEURL', package_url) + formula = formula.replace('SHA', digest) + formula = formula.replace('VERSION', brew_version) + with open(path.join(tmp_dir, 'Formula', 'servo-bin.rb')) as f: + f.write(formula) + + call_git(['add', path.join('.', 'Formula', 'servo-bin.rb')]) + call_git([ + '-c', 'user.name=Tom Servo', + '-c', 'user.email=servo@servo.org', + 'commit', + '--message=Version Bump: {}'.format(brew_version), + ]) + + token = os.environ['GITHUB_HOMEBREW_TOKEN'] + call_git([ + 'push', + '-qf', + 'https://{}@github.com/servo/homebrew-servo.git'.format(token), + 'master', + ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + timestamp = datetime.utcnow().replace(microsecond=0) + for package in PACKAGES[platform]: + if not path.isfile(package): + print("Could not find package for {} at {}".format( + platform, + package + ), file=sys.stderr) + return 1 + upload_to_s3(platform, package, timestamp) + + if platform == 'macbrew': + update_brew(package, timestamp) + + return 0 -- cgit v1.2.3