Rollup merge of #99992 - jyn514:shell-scripts, r=Mark-Simulacrum

Add `x.sh` and `x.ps1` shell scripts

This is a more ambitious version of #98716.
It still changes the x.py shebang back to python3, for compatibility with non-Unix systems,
but also adds alternative entrypoints for systems without `python3` installed.

These scripts will be necessary for the rust entrypoint (#94829), so I see
little downside in adding them early.

I'll update the dev-guide to suggest using these instead of x.py once this is merged.

Fixes #98650

r? `@Mark-Simulacrum` cc `@dtolnay` `@CAD97` `@yoshuawuyts`

Author: matthiaskrgr

src/bootstrap/mk/Makefile.in

@@ -66,16 +66,21 @@ TESTS_IN_2 := \
 	src/test/ui \
 	src/tools/linkchecker
 
+## MSVC native builders
+
+# these intentionally don't use `$(BOOTSTRAP)` so we can test the shebang on Windows
 ci-subset-1:
-	$(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_2:%=--exclude %)
+	$(Q)$(CFG_SRC_DIR)/x.py test --stage 2 $(TESTS_IN_2:%=--exclude %)
 ci-subset-2:
-	$(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_2)
+	$(Q)$(CFG_SRC_DIR)/x.ps1 test --stage 2 $(TESTS_IN_2)
+
+## MingW native builders
 
 TESTS_IN_MINGW_2 := \
 	src/test/ui
 
 ci-mingw-subset-1:
-	$(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_MINGW_2:%=--exclude %)
+	$(Q)$(CFG_SRC_DIR)/x.sh test --stage 2 $(TESTS_IN_MINGW_2:%=--exclude %)
 ci-mingw-subset-2:
 	$(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_MINGW_2)
 

src/ci/docker/host-x86_64/x86_64-gnu-llvm-12/Dockerfile

@@ -1,6 +1,8 @@
 FROM ubuntu:20.04
 
 ARG DEBIAN_FRONTEND=noninteractive
+
+# NOTE: intentionally installs both python2 and python3 so we can test support for both.
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   gcc-multilib \
@@ -10,6 +12,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   curl \
   ca-certificates \
   python2.7 \
+  python3.9 \
   git \
   cmake \
   sudo \
@@ -23,6 +26,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   xz-utils \
   nodejs
 
+# Install powershell so we can test x.ps1 on Linux
+RUN apt-get update && \
+    apt-get install -y apt-transport-https software-properties-common && \
+    curl -s "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb" > packages-microsoft-prod.deb && \
+    dpkg -i packages-microsoft-prod.deb && \
+    apt-get update && \
+    apt-get install -y powershell
+
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
@@ -33,21 +44,22 @@ ENV RUST_CONFIGURE_ARGS \
       --enable-llvm-link-shared \
       --set rust.thin-lto-import-instr-limit=10
 
-ENV SCRIPT python2.7 ../x.py --stage 2 test --exclude src/tools/tidy && \
+# NOTE: intentionally uses all of `x.py`, `x.sh`, and `x.ps1` to make sure they all work on Linux.
+ENV SCRIPT ../x.py --stage 2 test --exclude src/tools/tidy && \
            # Run the `mir-opt` tests again but this time for a 32-bit target.
            # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have
            # both 32-bit and 64-bit outputs updated by the PR author, before
            # the PR is approved and tested for merging.
            # It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`,
            # despite having different output on 32-bit vs 64-bit targets.
-           python2.7 ../x.py --stage 2 test src/test/mir-opt \
+           ../x.sh --stage 2 test src/test/mir-opt \
                              --host='' --target=i686-unknown-linux-gnu && \
            # Run the UI test suite again, but in `--pass=check` mode
            #
            # This is intended to make sure that both `--pass=check` continues to
            # work.
            #
-           python2.7 ../x.py --stage 2 test src/test/ui --pass=check \
+           ../x.ps1 --stage 2 test src/test/ui --pass=check \
                              --host='' --target=i686-unknown-linux-gnu && \
            # Run tidy at the very end, after all the other tests.
            python2.7 ../x.py --stage 2 test src/tools/tidy

src/tools/tidy/src/bins.rs

@@ -96,6 +96,8 @@ mod os_impl {
 
     #[cfg(unix)]
     pub fn check(path: &Path, bad: &mut bool) {
+        use std::ffi::OsStr;
+
         const ALLOWED: &[&str] = &["configure"];
 
         crate::walk_no_read(
@@ -117,9 +119,9 @@ mod os_impl {
             },
             &mut |entry| {
                 let file = entry.path();
-                let filename = file.file_name().unwrap().to_string_lossy();
-                let extensions = [".py", ".sh"];
-                if extensions.iter().any(|e| filename.ends_with(e)) {
+                let extension = file.extension();
+                let scripts = ["py", "sh", "ps1"];
+                if scripts.into_iter().any(|e| extension == Some(OsStr::new(e))) {
                     return;
                 }
 

x.ps1

@@ -0,0 +1,28 @@
+#!/usr/bin/env pwsh
+
+# See x.sh for why these scripts exist.
+
+$xpy = Join-Path $PSScriptRoot x.py
+# Start-Process for some reason splits arguments on spaces. (Isn't powershell supposed to be simpler than bash?)
+# Double-quote all the arguments so it doesn't do that.
+$xpy_args = @("""$xpy""")
+foreach ($arg in $args) {
+    $xpy_args += """$arg"""
+}
+
+foreach ($python in "py", "python3", "python", "python2") {
+    # NOTE: this only tests that the command exists in PATH, not that it's actually
+    # executable. The latter is not possible in a portable way, see
+    # https://github.com/PowerShell/PowerShell/issues/12625.
+    if (Get-Command $python -ErrorAction SilentlyContinue) {
+        if ($python -eq "py") {
+            # Use python3, not python2
+            $xpy_args = @("-3") + $xpy_args
+        }
+        $process = Start-Process -NoNewWindow -Wait -PassThru $python $xpy_args
+        Exit $process.ExitCode
+    }
+}
+
+Write-Error "${PSCommandPath}: error: did not find python installed"
+Exit 1

x.py

@@ -1,36 +1,16 @@
-#!/usr/bin/env bash
+#!/usr/bin/env python3
+# Some systems don't have `python3` in their PATH. This isn't supported by x.py directly;
+# they should use `x.sh` or `x.ps1` instead.
 
-# Modern Linux and macOS systems commonly only have a thing called `python3` and
-# not `python`, while Windows commonly does not have `python3`, so we cannot
-# directly use python in the shebang and have it consistently work. Instead we
-# embed some bash to look for a python to run the rest of the script.
-#
-# On Windows, `py -3` sometimes works. We need to try it first because `python3`
-# sometimes tries to launch the app store on Windows.
-'''':
-for PYTHON in "py -3" python3 python python2; do
-    if command -v $PYTHON >/dev/null; then
-        exec $PYTHON "$0" "$@"
-        break
-    fi
-done
-echo "$0: error: did not find python installed" >&2
-exit 1
-'''
-
-# The rest of this file is Python.
-#
 # This file is only a "symlink" to bootstrap.py, all logic should go there.
 
 import os
 import sys
 
 # If this is python2, check if python3 is available and re-execute with that
-# interpreter.
+# interpreter. Only python3 allows downloading CI LLVM.
 #
-# `./x.py` would not normally benefit from this because the bash above tries
-# python3 before 2, but this matters if someone ran `python x.py` and their
-# system's `python` is python2.
+# This matters if someone's system `python` is python2.
 if sys.version_info.major < 3:
     try:
         os.execvp("py", ["py", "-3"] + sys.argv)

x.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+# Modern Linux and macOS systems commonly only have a thing called `python3` and
+# not `python`, while Windows commonly does not have `python3`, so we cannot
+# directly use python in the x.py shebang and have it consistently work. Instead we
+# have a shell script to look for a python to run x.py.
+
+set -eu
+
+realpath() {
+    if [ -d "$1" ]; then
+        CDPATH='' command cd "$1" && pwd -P   
+    else
+        echo "$(realpath "$(dirname "$1")")/$(basename "$1")"
+    fi
+}
+
+xpy=$(dirname "$(realpath "$0")")/x.py
+
+# On Windows, `py -3` sometimes works. We need to try it first because `python3`
+# sometimes tries to launch the app store on Windows.
+for SEARCH_PYTHON in py python3 python python2; do
+    if python=$(command -v $SEARCH_PYTHON) && [ -x "$python" ]; then
+        if [ $SEARCH_PYTHON = py ]; then
+            extra_arg="-3"
+        else
+            extra_arg=""
+        fi
+        exec "$python" $extra_arg "$xpy" "$@"
+    fi
+done
+echo "$0: error: did not find python installed" >&2
+exit 1