Allow building with Nix on Linux distros other than NixOS (#31001)

* remove NixOS check from mach wrapper

* pin nixpkgs to fix jemalloc-sys + mozangle build scripts

* add stdenv.cc.cc to LD_LIBRARY_PATH to fix webrender build.rs

* add all buildInputs to LD_LIBRARY_PATH to fix ldd servo

* add glib + zlib to buildInputs to fix ldd servo

* working! patchelf --remove-rpath --set-interpreter

* undo changes that were ultimately unhelpful

* make compiled programs independent of Nix

* avoid installing patchelf’d toolchains in ~/.rustup

* remove another spurious change

* fix rustup pollution

* only enable on other distros when MACH_USE_NIX is set

3 files changed, 53 insertions, 6 deletions

diff --git a/.gitignore b/.gitignore
index b9785f0beaf..3c5d4915e2f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 /.servo
 /.cargo/*
 !/.cargo/config.*
+/.rustup
 /.servobuild
 /.wpt
 /.vs
diff --git a/etc/shell.nix b/etc/shell.nix
index 0eebfcf2692..306b25acb9c 100644
--- a/etc/shell.nix
+++ b/etc/shell.nix
@@ -1,7 +1,9 @@
 # This provides a shell with all the necesarry packages required to run mach and build servo
 # NOTE: This does not work offline or for nix-build
 
-with import <nixpkgs> {
+with import (builtins.fetchTarball {
+  url = "https://github.com/NixOS/nixpkgs/archive/6adf48f53d819a7b6e15672817fa1e78e5f4e84f.tar.gz";
+}) {
   overlays = [
     (import (builtins.fetchTarball {
       # Bumped the channel in rust-toolchain.toml? Bump this commit too!
@@ -10,9 +12,6 @@ with import <nixpkgs> {
   ];
 };
 let
-    pinnedNixpkgs = import (builtins.fetchTarball {
-      url = "https://github.com/NixOS/nixpkgs/archive/6adf48f53d819a7b6e15672817fa1e78e5f4e84f.tar.gz";
-    }) {};
     rustToolchain = rust-bin.fromRustupToolchainFile ../rust-toolchain.toml;
     rustPlatform = makeRustPlatform {
       cargo = rustToolchain;
@@ -43,7 +42,7 @@ clangStdenv.mkDerivation rec {
     # functionality in mozjs and causes builds to be extremely
     # slow as it behaves as if -j1 was passed.
     # See https://github.com/servo/mozjs/issues/375
-    pinnedNixpkgs.gnumake
+    gnumake
 
     # crown needs to be in our Cargo workspace so we can test it with `mach test`. This means its
     # dependency tree is listed in the main Cargo.lock, making it awkward to build with Nix because
@@ -73,6 +72,7 @@ clangStdenv.mkDerivation rec {
           cp -- ${vendorConfig} .cargo/config.toml
           > $out cargo run --offline -- ${../Cargo.lock} crown
         '';
+        dontInstall = true;
       });
     in (rustPlatform.buildRustPackage rec {
       name = "crown";
@@ -126,5 +126,50 @@ clangStdenv.mkDerivation rec {
     # Fix invalid option errors during linking
     # https://github.com/mozilla/nixpkgs-mozilla/commit/c72ff151a3e25f14182569679ed4cd22ef352328
     unset AS
+
+    # Compiling programs under Nix sets the interpreter (ELF INTERP) and rpath (ELF DT_RUNPATH [1])
+    # to ensure that it can find the needed (ELF DT_NEEDED) libraries in the Nix store.
+    #
+    # This is good on NixOS, but bad everywhere else. Using the Nix interpreter makes the programs
+    # dependent on the Nix store, making them impossible to distribute and run on other machines
+    # without `nix bundle`. Even on the same machine, the program will crash in a variety of ways
+    # because of the “OpenGL problem” [2] and other mismatches in X11 and Wayland libraries. Worse
+    # still, it makes everyone else suffer the problems NixOS has, like needing $LD_LIBRARY_PATH
+    # (see above) and needing to disable rust-lld (servo#30123).
+    #
+    # We can make the programs independent of Nix by resetting $NIX_DYNAMIC_LINKER to the system
+    # interpreter, setting $NIX_DONT_SET_RPATH to prevent the clang and ld wrappers from adding
+    # -rpath options to $NIX_LDFLAGS [3][4], and removing any -rpath options that get added by
+    # clangStdenv despite $NIX_DONT_SET_RPATH.
+    #
+    # This is comparable to fixing target/*/servo after the fact with:
+    #
+    #     patchelf --remove-rpath --set-interpreter $(patchelf --print-interpreter /usr/bin/env)
+    #
+    # [1] DT_RPATH breaks LD_LIBRARY_PATH and is no longer used
+    #     https://medium.com/obscure-system/rpath-vs-runpath-883029b17c45
+    # [2] “Using Nix on non-NixOS distros, it’s common to see GL application errors:”
+    #     https://github.com/nix-community/nixGL
+    # [3] https://ryantm.github.io/nixpkgs/stdenv/stdenv/#bintools-wrapper
+    # [4] https://matklad.github.io/2022/03/14/rpath-or-why-lld-doesnt-work-on-nixos.html
+    if ! [ -e /etc/NIXOS ]; then
+      set -- $NIX_LDFLAGS
+      for i; do
+        shift
+        if [ "$i" = -rpath ]; then
+          shift
+        else
+          set -- "$@" "$i"
+        fi
+      done
+      export NIX_DYNAMIC_LINKER=$(patchelf --print-interpreter /usr/bin/env)
+      export NIX_DONT_SET_RPATH=1
+      export NIX_LDFLAGS="$@"
+
+      # Don’t pollute ~/.rustup with toolchains installed by nixpkgs rustup, because they
+      # get patched in a way that makes them dependent on the Nix store.
+      repo_root=$(git rev-parse --show-toplevel)
+      export RUSTUP_HOME=$repo_root/.rustup
+    fi
   '';
 }
diff --git a/mach b/mach
index 4e768cfc650..57259217ae8 100755
--- a/mach
+++ b/mach
@@ -31,7 +31,8 @@ def main(args):
 if __name__ == '__main__':
     sys.dont_write_bytecode = True
 
-    if os.path.exists('/etc/NIXOS') and not 'IN_NIX_SHELL' in os.environ: # we're on a nixOS system, need to run mach in nix-shell
+    need_nix_shell = os.path.exists('/etc/NIXOS') or 'MACH_USE_NIX' in os.environ
+    if need_nix_shell and not 'IN_NIX_SHELL' in os.environ:
         import subprocess
         from shlex import quote
         mach_dir = os.path.abspath(os.path.dirname(__file__))