aboutsummaryrefslogtreecommitdiffstats
path: root/python
diff options
context:
space:
mode:
Diffstat (limited to 'python')
-rw-r--r--python/servo/build_commands.py6
-rw-r--r--python/servo/command_base.py57
-rw-r--r--python/servo/package_commands.py25
-rw-r--r--python/servo/post_build_commands.py4
-rw-r--r--python/servo/testing_commands.py11
-rw-r--r--python/tidy/servo_tidy/licenseck.py8
-rw-r--r--python/tidy/servo_tidy/tidy.py59
-rw-r--r--python/tidy/servo_tidy_tests/shell_tidy.sh14
-rw-r--r--python/tidy/servo_tidy_tests/test_tidy.py10
9 files changed, 178 insertions, 16 deletions
diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py
index df727e2776f..275b2309f2e 100644
--- a/python/servo/build_commands.py
+++ b/python/servo/build_commands.py
@@ -209,7 +209,7 @@ class MachCommands(CommandBase):
opts += ["--features", "%s" % ' '.join(features)]
build_start = time()
- env = self.build_env(target=target)
+ env = self.build_env(target=target, is_build=True)
if android:
# Build OpenSSL for android
@@ -299,7 +299,7 @@ class MachCommands(CommandBase):
build_start = time()
with cd(path.join("ports", "cef")):
ret = call(["cargo", "build"] + opts,
- env=self.build_env(), verbose=verbose)
+ env=self.build_env(is_build=True), verbose=verbose)
elapsed = time() - build_start
# Generate Desktop Notification if elapsed-time > some threshold value
@@ -334,7 +334,7 @@ class MachCommands(CommandBase):
if release:
opts += ["--release"]
- env = self.build_env()
+ env = self.build_env(is_build=True)
env["CARGO_TARGET_DIR"] = path.join(self.context.topdir, "target", "geckolib").encode("UTF-8")
build_start = time()
diff --git a/python/servo/command_base.py b/python/servo/command_base.py
index 328d0965b65..99dc78235ed 100644
--- a/python/servo/command_base.py
+++ b/python/servo/command_base.py
@@ -7,12 +7,16 @@
# option. This file may not be copied, modified, or distributed
# except according to those terms.
+import gzip
+import itertools
+import locale
import os
from os import path
import contextlib
import subprocess
from subprocess import PIPE
import sys
+import tarfile
import platform
import toml
@@ -33,6 +37,55 @@ def cd(new_path):
os.chdir(previous_path)
+@contextlib.contextmanager
+def setlocale(name):
+ """Context manager for changing the current locale"""
+ saved_locale = locale.setlocale(locale.LC_ALL)
+ try:
+ yield locale.setlocale(locale.LC_ALL, name)
+ finally:
+ locale.setlocale(locale.LC_ALL, saved_locale)
+
+
+def archive_deterministically(dir_to_archive, dest_archive, prepend_path=None):
+ """Create a .tar.gz archive in a deterministic (reproducible) manner.
+
+ See https://reproducible-builds.org/docs/archives/ for more details."""
+
+ def reset(tarinfo):
+ """Helper to reset owner/group and modification time for tar entries"""
+ tarinfo.uid = tarinfo.gid = 0
+ tarinfo.uname = tarinfo.gname = "root"
+ tarinfo.mtime = 0
+ return tarinfo
+
+ dest_archive = os.path.abspath(dest_archive)
+ with cd(dir_to_archive):
+ current_dir = "."
+ file_list = [current_dir]
+ for root, dirs, files in os.walk(current_dir):
+ for name in itertools.chain(dirs, files):
+ file_list.append(os.path.join(root, name))
+
+ # Sort file entries with the fixed locale
+ with setlocale('C'):
+ file_list.sort(cmp=locale.strcoll)
+
+ # Use a temporary file and atomic rename to avoid partially-formed
+ # packaging (in case of exceptional situations like running out of disk space).
+ # TODO do this in a temporary folder after #11983 is fixed
+ temp_file = '{}.temp~'.format(dest_archive)
+ with os.fdopen(os.open(temp_file, os.O_WRONLY | os.O_CREAT, 0644), 'w') as out_file:
+ with gzip.GzipFile('wb', fileobj=out_file, mtime=0) as gzip_file:
+ with tarfile.open(fileobj=gzip_file, mode='w:') as tar_file:
+ for entry in file_list:
+ arcname = entry
+ if prepend_path is not None:
+ arcname = os.path.normpath(os.path.join(prepend_path, arcname))
+ tar_file.add(entry, filter=reset, recursive=False, arcname=arcname)
+ os.rename(temp_file, dest_archive)
+
+
def host_triple():
os_type = platform.system().lower()
if os_type == "linux":
@@ -295,7 +348,7 @@ class CommandBase(object):
" --release" if release else ""))
sys.exit()
- def build_env(self, hosts_file_path=None, target=None):
+ def build_env(self, hosts_file_path=None, target=None, is_build=False):
"""Return an extended environment dictionary."""
env = os.environ.copy()
if sys.platform == "win32" and type(env['PATH']) == unicode:
@@ -392,7 +445,7 @@ class CommandBase(object):
env['RUSTFLAGS'] = env.get('RUSTFLAGS', "") + " -W unused-extern-crates"
git_info = []
- if os.path.isdir('.git'):
+ if os.path.isdir('.git') and is_build:
git_sha = subprocess.check_output([
'git', 'rev-parse', '--short', 'HEAD'
]).strip()
diff --git a/python/servo/package_commands.py b/python/servo/package_commands.py
index 1bd91e6eae1..fd7520fb10b 100644
--- a/python/servo/package_commands.py
+++ b/python/servo/package_commands.py
@@ -16,7 +16,6 @@ sys.path.append(path.join(path.dirname(sys.argv[0]), "components", "style", "pro
import os
import shutil
import subprocess
-import tarfile
from mach.registrar import Registrar
from datetime import datetime
@@ -29,7 +28,14 @@ from mach.decorators import (
from mako.template import Template
-from servo.command_base import CommandBase, cd, BuildNotFound, is_macosx, is_windows
+from servo.command_base import (
+ archive_deterministically,
+ BuildNotFound,
+ cd,
+ CommandBase,
+ is_macosx,
+ is_windows,
+)
from servo.post_build_commands import find_dep_path_newest
@@ -146,7 +152,7 @@ class PackageCommands(CommandBase):
print("Writing run-servo")
bhtml_path = path.join('${0%/*}/../Resources', browserhtml_path.split('/')[-1], 'out', 'index.html')
runservo = os.open(dir_to_app + '/Contents/MacOS/run-servo', os.O_WRONLY | os.O_CREAT, int("0755", 8))
- os.write(runservo, '#!/bin/bash\nexec ${0%/*}/servo ' + bhtml_path)
+ os.write(runservo, '#!/bin/bash\nexec ${0%/*}/servo -M -S ' + bhtml_path)
os.close(runservo)
print("Creating dmg")
@@ -205,7 +211,10 @@ class PackageCommands(CommandBase):
else:
dir_to_package = '/'.join(binary_path.split('/')[:-1])
dir_to_root = '/'.join(binary_path.split('/')[:-3])
- shutil.copytree(dir_to_root + '/resources', dir_to_package + '/resources')
+ resources_dir = dir_to_package + '/resources'
+ if os.path.exists(resources_dir):
+ delete(resources_dir)
+ shutil.copytree(dir_to_root + '/resources', resources_dir)
browserhtml_path = find_dep_path_newest('browserhtml', binary_path)
if browserhtml_path is None:
print("Could not find browserhtml package; perhaps you haven't built Servo.")
@@ -220,7 +229,7 @@ class PackageCommands(CommandBase):
delete(dir_to_package + '/build/' + f)
print("Writing runservo.sh")
# TODO: deduplicate this arg list from post_build_commands
- servo_args = ['-w', '-b',
+ servo_args = ['-w', '-b', '-M', '-S',
'--pref', 'dom.mozbrowser.enabled',
'--pref', 'dom.forcetouch.enabled',
'--pref', 'shell.builtin-key-shortcuts.enabled=false',
@@ -234,9 +243,9 @@ class PackageCommands(CommandBase):
time = datetime.utcnow().replace(microsecond=0).isoformat()
time = time.replace(':', "-")
tar_path += time + "-servo-tech-demo.tar.gz"
- with tarfile.open(tar_path, "w:gz") as tar:
- # arcname is to add by relative rather than absolute path
- tar.add(dir_to_package, arcname='servo/')
+
+ archive_deterministically(dir_to_package, tar_path, prepend_path='servo/')
+
print("Packaged Servo into " + tar_path)
@Command('install',
diff --git a/python/servo/post_build_commands.py b/python/servo/post_build_commands.py
index 733ec665cd6..0ce67a972cf 100644
--- a/python/servo/post_build_commands.py
+++ b/python/servo/post_build_commands.py
@@ -111,6 +111,10 @@ class PostBuildCommands(CommandBase):
# Convert to a relative path to avoid mingw -> Windows path conversions
browserhtml_path = path.relpath(browserhtml_path, os.getcwd())
+ if not is_windows():
+ # multiprocess + sandbox
+ args = args + ['-M', '-S']
+
args = args + ['-w',
'--pref', 'dom.mozbrowser.enabled',
'--pref', 'dom.forcetouch.enabled',
diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py
index 4776a9209d1..5eaad0ba7b9 100644
--- a/python/servo/testing_commands.py
+++ b/python/servo/testing_commands.py
@@ -158,6 +158,17 @@ class MachCommands(CommandBase):
return suite
return None
+ @Command('test-geckolib',
+ description='Test geckolib sanity checks',
+ category='testing')
+ def test_geckolib(self):
+ self.ensure_bootstrapped()
+
+ env = self.build_env()
+ env["RUST_BACKTRACE"] = "1"
+
+ return call(["cargo", "test"], env=env, cwd=path.join("ports", "geckolib"))
+
@Command('test-unit',
description='Run unit tests',
category='testing')
diff --git a/python/tidy/servo_tidy/licenseck.py b/python/tidy/servo_tidy/licenseck.py
index 75819bcefd2..3a17f1fdf04 100644
--- a/python/tidy/servo_tidy/licenseck.py
+++ b/python/tidy/servo_tidy/licenseck.py
@@ -24,6 +24,14 @@ licenses = [
""",
"""\
+#!/usr/bin/env bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+""",
+
+"""\
#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
diff --git a/python/tidy/servo_tidy/tidy.py b/python/tidy/servo_tidy/tidy.py
index 873af6ca22b..3433e8265ed 100644
--- a/python/tidy/servo_tidy/tidy.py
+++ b/python/tidy/servo_tidy/tidy.py
@@ -26,7 +26,7 @@ MAX_LICENSE_LINESPAN = max(len(license.splitlines()) for license in licenses)
# File patterns to include in the non-WPT tidy check.
FILE_PATTERNS_TO_CHECK = ["*.rs", "*.rc", "*.cpp", "*.c",
- "*.h", "Cargo.lock", "*.py",
+ "*.h", "Cargo.lock", "*.py", "*.sh",
"*.toml", "*.webidl", "*.json"]
# File patterns that are ignored for all tidy and lint checks.
@@ -43,6 +43,7 @@ IGNORED_FILES = [
os.path.join(".", "tests", "wpt", "metadata", "MANIFEST.json"),
os.path.join(".", "tests", "wpt", "metadata-css", "MANIFEST.json"),
os.path.join(".", "components", "script", "dom", "webidls", "ForceTouchEvent.webidl"),
+ os.path.join(".", "support", "android", "openssl.sh"),
# FIXME(pcwalton, #11679): This is a workaround for a tidy error on the quoted string
# `"__TEXT,_info_plist"` inside an attribute.
os.path.join(".", "components", "servo", "platform", "macos", "mod.rs"),
@@ -169,7 +170,11 @@ def check_modeline(file_name, lines):
def check_length(file_name, idx, line):
if file_name.endswith(".lock") or file_name.endswith(".json"):
raise StopIteration
- max_length = 120
+ # Prefer shorter lines when shell scripting.
+ if file_name.endswith(".sh"):
+ max_length = 80
+ else:
+ max_length = 120
if len(line.rstrip('\n')) > max_length:
yield (idx + 1, "Line is longer than %d characters" % max_length)
@@ -306,6 +311,53 @@ def check_toml(file_name, lines):
yield (0, ".toml file should contain a valid license.")
+def check_shell(file_name, lines):
+ if not file_name.endswith(".sh"):
+ raise StopIteration
+
+ shebang = "#!/usr/bin/env bash"
+ required_options = {"set -o errexit", "set -o nounset", "set -o pipefail"}
+
+ did_shebang_check = False
+
+ if len(lines) == 0:
+ yield (0, 'script is an empty file')
+ return
+
+ if lines[0].rstrip() != shebang:
+ yield (1, 'script does not have shebang "{}"'.format(shebang))
+
+ for idx in range(1, len(lines)):
+ stripped = lines[idx].rstrip()
+ # Comments or blank lines are ignored. (Trailing whitespace is caught with a separate linter.)
+ if lines[idx].startswith("#") or stripped == "":
+ continue
+
+ if not did_shebang_check:
+ if stripped in required_options:
+ required_options.remove(stripped)
+ else:
+ # The first non-comment, non-whitespace, non-option line is the first "real" line of the script.
+ # The shebang, options, etc. must come before this.
+ if len(required_options) != 0:
+ formatted = ['"{}"'.format(opt) for opt in required_options]
+ yield (idx + 1, "script is missing options {}".format(", ".join(formatted)))
+ did_shebang_check = True
+
+ if "`" in stripped:
+ yield (idx + 1, "script should not use backticks for command substitution")
+
+ if " [ " in stripped or stripped.startswith("[ "):
+ yield (idx + 1, "script should use `[[` instead of `[` for conditional testing")
+
+ for dollar in re.finditer('\$', stripped):
+ next_idx = dollar.end()
+ if next_idx < len(stripped):
+ next_char = stripped[next_idx]
+ if not (next_char == '{' or next_char == '('):
+ yield(idx + 1, "variable substitutions should use the full \"${VAR}\" form")
+
+
def check_rust(file_name, lines):
if not file_name.endswith(".rs") or \
file_name.endswith(".mako.rs") or \
@@ -694,7 +746,8 @@ def scan(only_changed_files=False, progress=True):
# standard checks
files_to_check = filter_files('.', only_changed_files, progress)
checking_functions = (check_flake8, check_lock, check_webidl_spec, check_json)
- line_checking_functions = (check_license, check_by_line, check_toml, check_rust, check_spec, check_modeline)
+ line_checking_functions = (check_license, check_by_line, check_toml, check_shell,
+ check_rust, check_spec, check_modeline)
errors = collect_errors_for_files(files_to_check, checking_functions, line_checking_functions)
# check dependecy licenses
dep_license_errors = check_dep_license_errors(get_dep_toml_files(only_changed_files), progress)
diff --git a/python/tidy/servo_tidy_tests/shell_tidy.sh b/python/tidy/servo_tidy_tests/shell_tidy.sh
new file mode 100644
index 00000000000..e38358fc3b6
--- /dev/null
+++ b/python/tidy/servo_tidy_tests/shell_tidy.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+#
+# Tests tidy for shell scripts.
+
+set -o nounset
+
+# Talking about some `concept in backticks` # shouldn't trigger
+echo "hello world"
+some_var=`echo "command substitution"`
+another_var="$some_var"
+if [ -z "${some_var}" ]; then
+ echo "should have used [["
+fi
+[ -z "${another_var}" ]
diff --git a/python/tidy/servo_tidy_tests/test_tidy.py b/python/tidy/servo_tidy_tests/test_tidy.py
index 6497fa016b7..92b304a5c1a 100644
--- a/python/tidy/servo_tidy_tests/test_tidy.py
+++ b/python/tidy/servo_tidy_tests/test_tidy.py
@@ -48,6 +48,16 @@ class CheckTidiness(unittest.TestCase):
self.assertEqual('incorrect license', errors.next()[2])
self.assertNoMoreErrors(errors)
+ def test_shell(self):
+ errors = tidy.collect_errors_for_files(iterFile('shell_tidy.sh'), [], [tidy.check_shell], print_text=False)
+ self.assertEqual('script does not have shebang "#!/usr/bin/env bash"', errors.next()[2])
+ self.assertEqual('script is missing options "set -o errexit", "set -o pipefail"', errors.next()[2])
+ self.assertEqual('script should not use backticks for command substitution', errors.next()[2])
+ self.assertEqual('variable substitutions should use the full \"${VAR}\" form', errors.next()[2])
+ self.assertEqual('script should use `[[` instead of `[` for conditional testing', errors.next()[2])
+ self.assertEqual('script should use `[[` instead of `[` for conditional testing', errors.next()[2])
+ self.assertNoMoreErrors(errors)
+
def test_rust(self):
errors = tidy.collect_errors_for_files(iterFile('rust_tidy.rs'), [], [tidy.check_rust], print_text=False)
self.assertEqual('use statement spans multiple lines', errors.next()[2])
generated by cgit v1.2.3 (git 2.25.1) at 2025-05-17 16:45:09 +0000