diff options
author | Mukilan Thiyagarajan <mukilan@igalia.com> | 2024-12-16 14:50:37 +0530 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-12-16 09:20:37 +0000 |
commit | 88a35b3cc99b6bf47099f1551b6a2349bba76c15 (patch) | |
tree | 67ad14a3a325e09c614b2ad53da50b846d023661 | |
parent | f757fa46acc865f42205d152dd06e381a674f788 (diff) | |
download | servo-88a35b3cc99b6bf47099f1551b6a2349bba76c15.tar.gz servo-88a35b3cc99b6bf47099f1551b6a2349bba76c15.zip |
mach: adopt `uv` and avoid system python (#34632)
This allows us to use `uv` for:
1. Installing a pinned Python version
2. Installing the dependency packages using `uv`'s pip compatible interface.
4. Bootstrapping `mach` without a Python installion on the host, using `uv
run`
This change also introduces a new 'composite' GitHub action to setup
python in the different CI workflows. There is no support for externally
managed python installations and virtual environments. These could be
added in the future.
Fixes #34095, #34547
Signed-off-by: Mukilan Thiyagarajan <mukilan@igalia.com>
-rw-r--r-- | .github/actions/setup-python/action.yml | 20 | ||||
-rw-r--r-- | .github/workflows/android.yml | 4 | ||||
-rw-r--r-- | .github/workflows/docs.yml | 3 | ||||
-rw-r--r-- | .github/workflows/lint.yml | 9 | ||||
-rw-r--r-- | .github/workflows/linux-bencher.yml | 3 | ||||
-rw-r--r-- | .github/workflows/linux-wpt.yml | 3 | ||||
-rw-r--r-- | .github/workflows/linux.yml | 9 | ||||
-rw-r--r-- | .github/workflows/mac-wpt.yml | 8 | ||||
-rw-r--r-- | .github/workflows/mac.yml | 8 | ||||
-rw-r--r-- | .github/workflows/main.yml | 7 | ||||
-rw-r--r-- | .github/workflows/ohos.yml | 4 | ||||
-rw-r--r-- | .github/workflows/pull-request-wpt-export.yml | 10 | ||||
-rw-r--r-- | .github/workflows/scheduled-wpt-import.yml | 3 | ||||
-rw-r--r-- | .github/workflows/try-label.yml | 7 | ||||
-rw-r--r-- | .github/workflows/try.yml | 7 | ||||
-rw-r--r-- | .github/workflows/windows.yml | 6 | ||||
-rw-r--r-- | .python-version | 1 | ||||
-rw-r--r-- | README.md | 16 | ||||
-rwxr-xr-x | mach | 52 | ||||
-rw-r--r-- | mach.bat | 9 | ||||
-rw-r--r-- | python/mach_bootstrap.py | 74 | ||||
-rw-r--r-- | python/servo/build_commands.py | 5 | ||||
-rw-r--r-- | python/servo/post_build_commands.py | 3 | ||||
-rw-r--r-- | python/servo/testing_commands.py | 2 | ||||
-rw-r--r-- | servo-tidy.toml | 1 | ||||
-rw-r--r-- | shell.nix | 10 |
26 files changed, 131 insertions, 153 deletions
diff --git a/.github/actions/setup-python/action.yml b/.github/actions/setup-python/action.yml new file mode 100644 index 00000000000..9889c517e6b --- /dev/null +++ b/.github/actions/setup-python/action.yml @@ -0,0 +1,20 @@ +name: Setup Python and uv +inputs: + skip-python-setup: + required: false + description: "Whether to skip installing python using Github's `setup-python` action" + default: false +runs: + using: "composite" + steps: + # Use the setup-python action to take advantage of the cache. uv will + # symlink to this version. + - name: Setup system python + if: ${{ inputs.skip-python-setup != 'true' }} + uses: actions/setup-python@v5 + with: + python-version-file: '.python-version' + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "0.5.6" diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index dfa1b057800..2cfaf5745c6 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -61,8 +61,8 @@ jobs: uses: mozilla-actions/sccache-action@v0.0.7 - name: Install crown run: cargo install --path support/crown - - name: Bootstrap Python - run: python3 -m pip install --upgrade pip virtualenv + - name: Setup Python + uses: ./.github/actions/setup-python - name: Bootstrap dependencies run: sudo apt update && python3 ./mach bootstrap --skip-lints - name: Set up JDK 17 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index f546469f962..74530c40e1f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -13,9 +13,10 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 + - name: Setup Python + uses: ./.github/actions/setup-python - name: Bootstrap run: | - python3 -m pip install --upgrade pip sudo apt update python3 ./mach bootstrap --skip-lints - name: Set LIBCLANG_PATH # This is needed for bindgen in mozangle. diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 33b128de9d4..59a5d8fe615 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -33,9 +33,8 @@ jobs: uses: mozilla-actions/sccache-action@v0.0.6 - name: Set LIBCLANG_PATH env # needed for bindgen in mozangle run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV - - uses: actions/setup-python@v5 - with: - python-version: '3.10' + - name: Setup Python + uses: ./.github/actions/setup-python - name: Install taplo uses: baptiste0928/cargo-install@v3 with: @@ -46,8 +45,6 @@ jobs: with: crate: cargo-deny locked: true - - name: Bootstrap Python - run: python3 -m pip install --upgrade pip - name: Bootstrap dependencies run: | sudo apt update @@ -57,4 +54,4 @@ jobs: run: | python3 ./mach clippy --use-crown --locked -- -- --deny warnings - name: Tidy - run: python3 ./mach test-tidy --no-progress --all
\ No newline at end of file + run: python3 ./mach test-tidy --no-progress --all diff --git a/.github/workflows/linux-bencher.yml b/.github/workflows/linux-bencher.yml index e7b98608bb9..eb4bc9c4218 100644 --- a/.github/workflows/linux-bencher.yml +++ b/.github/workflows/linux-bencher.yml @@ -35,9 +35,10 @@ jobs: path: release-binary-linux - name: unPackage binary run: tar -xzf release-binary-linux/target.tar.gz + - name: Setup Python + uses: ./.github/actions/setup-python - name: Bootstrap dependencies run: | - python3 -m pip install --upgrade pip sudo apt update sudo apt install -qy --no-install-recommends mesa-vulkan-drivers python3 ./mach bootstrap --skip-lints diff --git a/.github/workflows/linux-wpt.yml b/.github/workflows/linux-wpt.yml index 34857bdc9d0..f710518b462 100644 --- a/.github/workflows/linux-wpt.yml +++ b/.github/workflows/linux-wpt.yml @@ -52,9 +52,10 @@ jobs: path: ${{ inputs.profile }}-binary-linux - name: unPackage binary run: tar -xzf ${{ inputs.profile }}-binary-linux/target.tar.gz + - name: Setup Python + uses: ./.github/actions/setup-python - name: Bootstrap dependencies run: | - python3 -m pip install --upgrade pip sudo apt update sudo apt install -qy --no-install-recommends mesa-vulkan-drivers python3 ./mach bootstrap --skip-lints diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 3d004e44974..b1bfe234514 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -141,13 +141,10 @@ jobs: - name: Set LIBCLANG_PATH env # needed for bindgen in mozangle if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) && !inputs.upload }} # not needed on ubuntu 20.04 used for nightly run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV - - uses: actions/setup-python@v5 - if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} + - name: Setup Python + uses: ./.github/actions/setup-python with: - python-version: '3.10' - - name: Bootstrap Python - if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} - run: python3 -m pip install --upgrade pip + skip-python-setup: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) }} - name: Bootstrap dependencies if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} run: | diff --git a/.github/workflows/mac-wpt.yml b/.github/workflows/mac-wpt.yml index 74a1e1439fe..610da047a5f 100644 --- a/.github/workflows/mac-wpt.yml +++ b/.github/workflows/mac-wpt.yml @@ -41,15 +41,11 @@ jobs: - uses: actions/download-artifact@v4 with: name: ${{ inputs.profile }}-binary-macos - # Python 3.13 breaks wptrunner, so pin the version until - # web-platform-tests/wpt#48585 is fixed. - - uses: actions/setup-python@v5 - with: - python-version: '3.12' + - name: Setup Python + uses: ./.github/actions/setup-python - name: Prep test environment run: | gtar -xzf target.tar.gz - python3 -m pip install --upgrade pip python3 ./mach bootstrap --skip-lints - name: Smoketest run: python3 ./mach smoketest --${{ inputs.profile }} diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index b201b4c2584..05882721657 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -79,18 +79,14 @@ jobs: if: github.event_name == 'pull_request_target' with: ref: ${{ github.event.pull_request.head.sha }} - # Python 3.13 breaks wptrunner, so pin the version until - # web-platform-tests/wpt#48585 is fixed. - - uses: actions/setup-python@v5 - with: - python-version: '3.12' + - name: Setup Python + uses: ./.github/actions/setup-python - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.6 - name: Install crown run: cargo install --path support/crown - name: Bootstrap run: | - python3 -m pip install --upgrade pip python3 ./mach bootstrap --skip-lints brew install gnu-tar - name: Build (${{ inputs.profile }}) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 83e09cd3977..2d043312f0a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,15 +20,16 @@ jobs: outputs: configuration: ${{ steps.configuration.outputs.result }} steps: - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - uses: actions/checkout@v4 with: fetch-depth: 1 sparse-checkout: | python/servo/try_parser.py + .github/actions/setup-python + .python-version sparse-checkout-cone-mode: false + - name: Setup Python + uses: ./.github/actions/setup-python - name: Get Configuration id: configuration run: | diff --git a/.github/workflows/ohos.yml b/.github/workflows/ohos.yml index 409f1f5d627..0a64c5d8f64 100644 --- a/.github/workflows/ohos.yml +++ b/.github/workflows/ohos.yml @@ -55,8 +55,8 @@ jobs: uses: mozilla-actions/sccache-action@v0.0.6 - name: Install crown run: cargo install --path support/crown - - name: Bootstrap Python - run: python3 -m pip install --upgrade pip virtualenv + - name: Setup Python + uses: ./.github/actions/setup-python - name: Bootstrap dependencies run: sudo apt update && python3 ./mach bootstrap --skip-lints - name: Setup OpenHarmony SDK diff --git a/.github/workflows/pull-request-wpt-export.yml b/.github/workflows/pull-request-wpt-export.yml index 2942fe9f2cd..8f78ca74f7e 100644 --- a/.github/workflows/pull-request-wpt-export.yml +++ b/.github/workflows/pull-request-wpt-export.yml @@ -28,10 +28,16 @@ jobs: # using the token specified here. # See https://github.com/actions/checkout/issues/162. token: ${{ secrets.WPT_SYNC_TOKEN }} + - name: Setup Python + uses: ./servo/.github/actions/setup-python - name: Install requirements - run: pip install -r servo/python/requirements.txt + run: | + uv venv + uv pip install -r servo/python/requirements.txt - name: Process pull request - run: servo/python/wpt/export.py + run: | + source .venv/bin/activate + servo/python/wpt/export.py env: GITHUB_CONTEXT: ${{ toJson(github) }} WPT_SYNC_TOKEN: ${{ secrets.WPT_SYNC_TOKEN }} diff --git a/.github/workflows/scheduled-wpt-import.yml b/.github/workflows/scheduled-wpt-import.yml index 1ed65b12c32..145133b817c 100644 --- a/.github/workflows/scheduled-wpt-import.yml +++ b/.github/workflows/scheduled-wpt-import.yml @@ -35,9 +35,10 @@ jobs: - uses: actions/download-artifact@v4 with: name: wpt-full-logs-linux-layout-2020 + - name: Setup Python + uses: ./.github/actions/setup-python - name: Prep environment run: | - python3 -m pip install --upgrade pip sudo apt update python3 ./mach bootstrap - name: Add upstream remote diff --git a/.github/workflows/try-label.yml b/.github/workflows/try-label.yml index 610aeb883c9..1a95432cd7d 100644 --- a/.github/workflows/try-label.yml +++ b/.github/workflows/try-label.yml @@ -77,14 +77,15 @@ jobs: } return try_string; - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - uses: actions/checkout@v4 with: sparse-checkout: | python/servo/try_parser.py + .github/actions/setup-python + .python-version sparse-checkout-cone-mode: false + - name: Setup Python + uses: ./.github/actions/setup-python - name: Parse Labels if: ${{ steps.try_string.outputs.result }} id: configuration diff --git a/.github/workflows/try.yml b/.github/workflows/try.yml index ca81a8f91cd..d81715fe92d 100644 --- a/.github/workflows/try.yml +++ b/.github/workflows/try.yml @@ -32,15 +32,16 @@ jobs: outputs: configuration: ${{ steps.configuration.outputs.result }} steps: - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - uses: actions/checkout@v4 with: fetch-depth: 1 sparse-checkout: | python/servo/try_parser.py + .github/actions/setup-python + .python-version sparse-checkout-cone-mode: false + - name: Setup Python + uses: ./.github/actions/setup-python - name: Get Full Configuration id: full_config run: | diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index a59c23dc919..a9c0805c80c 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -114,13 +114,13 @@ jobs: choco install wixtoolset echo "C:\\Program Files (x86)\\WiX Toolset v3.11\\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - uses: actions/setup-python@v5 - if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} + - name: Setup Python + uses: ./.github/actions/setup-python with: - python-version: "3.10" + skip-python-setup: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) }} - name: Bootstrap if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} run: | - python -m pip install --upgrade pip python mach fetch python mach bootstrap-gstreamer # For some reason WiX isn't currently on the GitHub runner path. This is a diff --git a/.python-version b/.python-version new file mode 100644 index 00000000000..e4fba218358 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/README.md b/README.md index 0c83a837d29..8147702f92d 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,8 @@ For more detailed build instructions, see the Servo book under [Setting up your ### macOS -- Download and install [`python`](https://www.python.org/downloads/macos/) (version 3.10 to 3.12), [Xcode](https://developer.apple.com/xcode/), and [`brew`](https://brew.sh/). +- Download and install [Xcode](https://developer.apple.com/xcode/) and [`brew`](https://brew.sh/). +- Install `uv`: `curl -LsSf https://astral.sh/uv/install.sh | sh` - Install `rustup`: `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` - Restart your shell to make sure `cargo` is available - Install the other dependencies: `./mach bootstrap` @@ -25,11 +26,12 @@ For more detailed build instructions, see the Servo book under [Setting up your ### Linux -- Install `curl` and `python` (version 3.10 to 3.12): - - Arch: `sudo pacman -S --needed curl python python-pip` - - Debian, Ubuntu: `sudo apt install curl python3-pip python3-venv python3-setuptools` - - Fedora: `sudo dnf install curl python3 python3-pip python3-devel` - - Gentoo: `sudo emerge net-misc/curl dev-python/pip` +- Install `curl`: + - Arch: `sudo pacman -S --needed curl` + - Debian, Ubuntu: `sudo apt install curl` + - Fedora: `sudo dnf install curl` + - Gentoo: `sudo emerge net-misc/curl` +- Install `uv`: `curl -LsSf https://astral.sh/uv/install.sh | sh` - Install `rustup`: `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` - Restart your shell to make sure `cargo` is available - Install the other dependencies: `./mach bootstrap` @@ -37,7 +39,7 @@ For more detailed build instructions, see the Servo book under [Setting up your ### Windows -- Download and install [`python`](https://www.python.org/downloads/windows/) (version 3.10 to 3.12), [`choco`](https://chocolatey.org/install#individual), and [`rustup`](https://win.rustup.rs/) +- Download [`uv`](https://docs.astral.sh/uv/getting-started/installation/#standalone-installer), [`choco`](https://chocolatey.org/install#individual), and [`rustup`](https://win.rustup.rs/) - Be sure to select *Quick install via the Visual Studio Community installer* - In the Visual Studio Installer, ensure the following components are installed: - **Windows 10 SDK (10.0.19041.0)** (`Microsoft.VisualStudio.Component.Windows10SDK.19041`) @@ -1,20 +1,34 @@ -#!/usr/bin/env python3 +#!/bin/sh # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +# The beginning of this script is both valid shell and valid python, such that +# the script starts with the shell and is reexecuted with `uv run`. This +# ensures that the Python provided by the virtual environment (in the .venv +# directory) is used. If the virtual environment does not exist, `uv run` will +# still use the correct version of Python given in `.python-version` and +# python/mach_bootstrap.py will provision a new environment that will be used +# for the subsequent runs. +''':' && { + MACH_DIR=$(dirname "$0"); + run_in_nix_if_needed() { + if { [ -f /etc/NIXOS ] || [ -n "${MACH_USE_NIX}" ]; } && [ -z "${IN_NIX_SHELL}" ]; then + EXTRA_NIX_ARGS=${SERVO_ANDROID_BUILD:+'--arg buildAndroid true'} + echo "NOTE: Entering nix-shell ${MACH_DIR}/shell.nix" + exec nix-shell "${MACH_DIR}/shell.nix" $EXTRA_NIX_ARGS --run "$*" + else + exec "$@" + fi + } + + run_in_nix_if_needed uv run python ${MACH_DIR}/mach "$@" +} +''' + import os import sys -# Destructure because version_info > max_ver is true when running the same version. -ver = (sys.version_info[0], sys.version_info[1]) -min_ver = (3, 10) -max_ver = (3, 12) # WPT does not support Python 3.13. See issue #34095. -if ver < min_ver or ver > max_ver: - print("mach does not support python {0}.{1}, please install 3.{2} <= python <= 3.{3}" \ - .format(ver[0], ver[1], min_ver[1], max_ver[1])) - sys.exit(1) - def main(args): topdir = os.path.abspath(os.path.dirname(sys.argv[0])) sys.path.insert(0, os.path.join(topdir, "python")) @@ -27,20 +41,4 @@ def main(args): if __name__ == '__main__': - sys.dont_write_bytecode = True - - need_nix_shell = os.path.exists('/etc/NIXOS') or 'MACH_USE_NIX' in os.environ - if need_nix_shell and not 'IN_NIX_SHELL' in os.environ: - import subprocess - from shlex import quote - mach_dir = os.path.abspath(os.path.dirname(__file__)) - build_android_args = ['--arg', 'buildAndroid', 'true'] if 'SERVO_ANDROID_BUILD' in os.environ else [] - print(f'NOTE: Entering nix-shell {mach_dir}/shell.nix') - try: - # sys argv already contains the ./mach part, so we just need to pass it as-is - result = subprocess.run(['nix-shell', f'{mach_dir}/shell.nix'] + build_android_args + ['--run', ' '.join(map(quote, sys.argv))]) - sys.exit(result.returncode) - except KeyboardInterrupt: - sys.exit(0) - else: - main(sys.argv) + main(sys.argv) @@ -1,9 +1,4 @@ @echo off -set workdir=%~dp0 -where /Q py.exe -IF %ERRORLEVEL% NEQ 0 ( - python %workdir%mach %* -) ELSE ( - py -3 %workdir%mach %* -) +set workdir=%~dp0 +uv run python %workdir%mach %* diff --git a/python/mach_bootstrap.py b/python/mach_bootstrap.py index 43690c3d951..14f28f512ce 100644 --- a/python/mach_bootstrap.py +++ b/python/mach_bootstrap.py @@ -4,10 +4,9 @@ import hashlib import os -import platform -import site import subprocess import sys +import runpy SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__)) TOP_DIR = os.path.abspath(os.path.join(SCRIPT_PATH, "..")) @@ -81,35 +80,16 @@ CATEGORIES = { } -# venv calls its scripts folder "bin" on non-Windows and "Scripts" on Windows. -def _get_virtualenv_script_dir(): - if os.name == "nt" and os.sep != "/": - return "Scripts" - return "bin" - - -# venv names its lib folder something like "lib/python3.11/site-packages" on -# non-Windows and "Lib\site-packages" on Window. -def _get_virtualenv_lib_dir(): - if os.name == "nt" and os.sep != "/": - return os.path.join("Lib", "site-packages") - return os.path.join( - "lib", - f"python{sys.version_info[0]}.{sys.version_info[1]}", - "site-packages" - ) - - -def _process_exec(args): +def _process_exec(args, cwd): try: - subprocess.check_output(args, stderr=subprocess.STDOUT) + subprocess.check_output(args, stderr=subprocess.STDOUT, cwd=cwd) except subprocess.CalledProcessError as exception: print(exception.output.decode(sys.stdout.encoding)) print(f"Process failed with return code: {exception.returncode}") sys.exit(1) -def install_virtual_env_requirements(project_path: str, python: str, virtualenv_path: str): +def install_virtual_env_requirements(project_path: str, virtualenv_path: str): requirements_paths = [ os.path.join(project_path, "python", "requirements.txt"), os.path.join(project_path, WPT_TOOLS_PATH, "requirements_tests.txt",), @@ -131,45 +111,32 @@ def install_virtual_env_requirements(project_path: str, python: str, virtualenv_ requirements_hash = requirements_hasher.hexdigest() if marker_hash != requirements_hash: - print(" * Upgrading pip...") - _process_exec([python, "-m", "pip", "install", "--upgrade", "pip"]) - print(" * Installing Python requirements...") - _process_exec([python, "-m", "pip", "install", "-I", - "-r", requirements_paths[0], - "-r", requirements_paths[1], - "-r", requirements_paths[2]]) + pip_install_command = ["uv", "pip", "install"] + for requirements in requirements_paths: + pip_install_command.extend(["-r", requirements]) + _process_exec(pip_install_command, cwd=project_path) with open(marker_path, "w") as marker_file: marker_file.write(requirements_hash) def _activate_virtualenv(topdir): - virtualenv_path = os.path.join(topdir, "python", "_venv%d.%d" % (sys.version_info[0], sys.version_info[1])) - python = sys.executable + virtualenv_path = os.path.join(topdir, ".venv") if os.environ.get("VIRTUAL_ENV") != virtualenv_path: - venv_script_path = os.path.join(virtualenv_path, _get_virtualenv_script_dir()) if not os.path.exists(virtualenv_path): print(" * Setting up virtual environment...") - _process_exec([python, "-m", "venv", "--system-site-packages", virtualenv_path]) + _process_exec(["uv", "venv"], cwd=topdir) - # This general approach is taken from virtualenv's `activate_this.py`. - os.environ["PATH"] = os.pathsep.join([venv_script_path, *os.environ.get("PATH", "").split(os.pathsep)]) - os.environ["VIRTUAL_ENV"] = virtualenv_path + script_dir = "Scripts" if _is_windows() else "bin" + runpy.run_path(os.path.join(virtualenv_path, script_dir, 'activate_this.py')) - prev_length = len(sys.path) - lib_path = os.path.realpath(os.path.join(virtualenv_path, _get_virtualenv_lib_dir())) - site.addsitedir(lib_path) - sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length] + install_virtual_env_requirements(topdir, virtualenv_path) - sys.real_prefix = sys.prefix - sys.prefix = virtualenv_path - - # Use the python in our venv for subprocesses, not the python we were originally run with. - # Otherwise pip may still try to write to the wrong site-packages directory. - python = os.path.join(venv_script_path, "python") - - install_virtual_env_requirements(topdir, python, virtualenv_path) + # Turn off warnings about deprecated syntax in our indirect dependencies. + # TODO: Find a better approach for doing this. + import warnings + warnings.filterwarnings('ignore', category=SyntaxWarning, module=r'.*.venv') def _ensure_case_insensitive_if_windows(): @@ -221,13 +188,6 @@ def bootstrap(topdir): print('Current path:', topdir) sys.exit(1) - # Ensure we are running Python 3.10+. We put this check here so we generate a - # user-friendly error message rather than a cryptic stack trace on module import. - if sys.version_info < (3, 10): - print('Python3 (>=3.10) is required to run mach.') - print('You are running Python', platform.python_version()) - sys.exit(1) - _activate_virtualenv(topdir) def populate_context(context, key=None): diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py index c48999f99f9..b21f89980c0 100644 --- a/python/servo/build_commands.py +++ b/python/servo/build_commands.py @@ -175,7 +175,7 @@ class MachCommands(CommandBase): return status @Command('clean', - description='Clean the target/ and python/_venv[version]/ directories', + description='Clean the target/ and Python virtual environment directories', category='build') @CommandArgument('--manifest-path', default=None, @@ -188,8 +188,7 @@ class MachCommands(CommandBase): def clean(self, manifest_path=None, params=[], verbose=False): self.ensure_bootstrapped() - virtualenv_fname = '_venv%d.%d' % (sys.version_info[0], sys.version_info[1]) - virtualenv_path = path.join(self.get_top_dir(), 'python', virtualenv_fname) + virtualenv_path = path.join(self.get_top_dir(), '.venv') if path.exists(virtualenv_path): print('Removing virtualenv directory: %s' % virtualenv_path) shutil.rmtree(virtualenv_path) diff --git a/python/servo/post_build_commands.py b/python/servo/post_build_commands.py index dfe775d83be..99c5424baf3 100644 --- a/python/servo/post_build_commands.py +++ b/python/servo/post_build_commands.py @@ -80,8 +80,7 @@ class PostBuildCommands(CommandBase): @CommandBase.allow_target_configuration def run(self, servo_binary: str, params, debugger=False, debugger_cmd=None, headless=False, software=False, emulator=False, usb=False): - self._run(servo_binary, params, debugger, debugger_cmd, - headless, software, emulator, usb) + return self._run(servo_binary, params, debugger, debugger_cmd, headless, software, emulator, usb) def _run(self, servo_binary: str, params, debugger=False, debugger_cmd=None, headless=False, software=False, emulator=False, usb=False): diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index de421068050..ff7ad897723 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -18,7 +18,7 @@ import subprocess import textwrap import json -from python.servo.post_build_commands import PostBuildCommands +from servo.post_build_commands import PostBuildCommands import wpt import wpt.manifestupdate import wpt.run diff --git a/servo-tidy.toml b/servo-tidy.toml index 6e759d7174c..9a215213e98 100644 --- a/servo-tidy.toml +++ b/servo-tidy.toml @@ -41,7 +41,6 @@ directories = [ "./tests/wpt/mozilla/tests/mozilla/referrer-policy", "./tests/wpt/mozilla/tests/webgl", "./python/tidy/tests", - "./python/_v*", "./python/mach", # Generated and upstream code combined with our own. Could use cleanup "./target", diff --git a/shell.nix b/shell.nix index 1a5ffa33bd4..375a22c48f7 100644 --- a/shell.nix +++ b/shell.nix @@ -5,7 +5,7 @@ buildAndroid ? false }: with import (builtins.fetchTarball { - url = "https://github.com/NixOS/nixpkgs/archive/d04953086551086b44b6f3c6b7eeb26294f207da.tar.gz"; + url = "https://github.com/NixOS/nixpkgs/archive/5d67ea6b4b63378b9c13be21e2ec9d1afc921713.tar.gz"; }) { overlays = [ (import (builtins.fetchTarball { @@ -84,7 +84,13 @@ stdenv.mkDerivation (androidEnvironment // { # Build utilities cmake dbus gcc git pkg-config which llvm perl yasm m4 - (python3.withPackages (ps: with ps; [virtualenv pip dbus])) + + # Ensure the Python version is same as the one in `.python-version` file so + # that `uv` will just symlink to the one in nix store. Otherwise `uv` will + # download a pre-built binary that won't work on nix. + # FIXME: dbus python module needs to be installed into the virtual environment. + python312 + uv # This pins gnumake to 4.3 since 4.4 breaks jobserver # functionality in mozjs and causes builds to be extremely |