Rollup merge of #103969 - ferrocene:pa-download-rustc-ui-tests, r=jyn514

Partial support for running UI tests with `download-rustc`

Right now trying to run UI tests with `download-rustc` results in a bunch of test failures, requiring someone who wants to only work on tests to also build the full compiler. This PR **partially** addresses the problem by solving a lot of the errors (but not all).

* This installs the `rust-src` component on CI toolchains, since the test output depends on whether the standard library source code is installed; We can't just symlink the current source because the `rustc-dev` component already includes the compiler sources, but not the library sources, and mixing things is worse IMO.
* This ensures the `$SRC_DIR` normalization done by compiletest correctly handles the source code added above.
* This unconditionally sets `remap-prefix` to the prefix used in the downloaded toolchain, to ensure compiletest normalizes it.

Author: matthiaskrgr

src/bootstrap/config.rs

@@ -1511,19 +1511,25 @@ impl Config {
 
     /// Return whether we will use a downloaded, pre-compiled version of rustc, or just build from source.
     pub(crate) fn download_rustc(&self) -> bool {
-        static DOWNLOAD_RUSTC: OnceCell<bool> = OnceCell::new();
+        self.download_rustc_commit().is_some()
+    }
+
+    pub(crate) fn download_rustc_commit(&self) -> Option<&'static str> {
+        static DOWNLOAD_RUSTC: OnceCell<Option<String>> = OnceCell::new();
         if self.dry_run() && DOWNLOAD_RUSTC.get().is_none() {
             // avoid trying to actually download the commit
-            return false;
+            return None;
         }
 
-        *DOWNLOAD_RUSTC.get_or_init(|| match &self.download_rustc_commit {
-            None => false,
-            Some(commit) => {
-                self.download_ci_rustc(commit);
-                true
-            }
-        })
+        DOWNLOAD_RUSTC
+            .get_or_init(|| match &self.download_rustc_commit {
+                None => None,
+                Some(commit) => {
+                    self.download_ci_rustc(commit);
+                    Some(commit.clone())
+                }
+            })
+            .as_deref()
     }
 
     pub(crate) fn initial_rustfmt(&self) -> Option<PathBuf> {

src/bootstrap/test.rs

@@ -1401,6 +1401,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the
 
         cmd.arg("--src-base").arg(builder.src.join("src/test").join(suite));
         cmd.arg("--build-base").arg(testdir(builder, compiler.host).join(suite));
+        cmd.arg("--sysroot-base").arg(builder.sysroot(compiler));
         cmd.arg("--stage-id").arg(format!("stage{}-{}", compiler.stage, target));
         cmd.arg("--suite").arg(suite);
         cmd.arg("--mode").arg(mode);
@@ -1670,6 +1671,10 @@ note: if you're sure you want to do this, please open an issue as to why. In the
 
         cmd.arg("--channel").arg(&builder.config.channel);
 
+        if let Some(commit) = builder.config.download_rustc_commit() {
+            cmd.env("FAKE_DOWNLOAD_RUSTC_PREFIX", format!("/rustc/{commit}"));
+        }
+
         builder.ci_env.force_coloring_in_ci(&mut cmd);
 
         builder.info(&format!(

src/tools/compiletest/src/common.rs

@@ -230,6 +230,9 @@ pub struct Config {
     /// The directory where programs should be built
     pub build_base: PathBuf,
 
+    /// The directory containing the compiler sysroot
+    pub sysroot_base: PathBuf,
+
     /// The name of the stage being built (stage1, etc)
     pub stage_id: String,
 

src/tools/compiletest/src/header/tests.rs

@@ -46,6 +46,7 @@ fn config() -> Config {
         "--jsondocck-path=",
         "--src-base=",
         "--build-base=",
+        "--sysroot-base=",
         "--stage-id=stage2",
         "--cc=c",
         "--cxx=c++",

src/tools/compiletest/src/main.rs

@@ -69,6 +69,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         .optopt("", "llvm-filecheck", "path to LLVM's FileCheck binary", "DIR")
         .reqopt("", "src-base", "directory to scan for test files", "PATH")
         .reqopt("", "build-base", "directory to deposit test outputs", "PATH")
+        .reqopt("", "sysroot-base", "directory containing the compiler sysroot", "PATH")
         .reqopt("", "stage-id", "the target-stage identifier", "stageN-TARGET")
         .reqopt(
             "",
@@ -234,6 +235,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         llvm_bin_dir: matches.opt_str("llvm-bin-dir").map(PathBuf::from),
         src_base,
         build_base: opt_path(matches, "build-base"),
+        sysroot_base: opt_path(matches, "sysroot-base"),
         stage_id: matches.opt_str("stage-id").unwrap(),
         mode,
         suite: matches.opt_str("suite").unwrap(),

src/tools/compiletest/src/runtest.rs

@@ -3533,22 +3533,25 @@ impl<'test> TestCx<'test> {
         let parent_dir = self.testpaths.file.parent().unwrap();
         normalize_path(parent_dir, "$DIR");
 
-        // Paths into the libstd/libcore
-        let base_dir = self.config.src_base.parent().unwrap().parent().unwrap().parent().unwrap();
-        let src_dir = base_dir.join("library");
-        normalize_path(&src_dir, "$SRC_DIR");
-
-        // `ui-fulldeps` tests can show paths to the compiler source when testing macros from
-        // `rustc_macros`
-        // eg. /home/user/rust/compiler
-        let compiler_src_dir = base_dir.join("compiler");
-        normalize_path(&compiler_src_dir, "$COMPILER_DIR");
-
-        if let Some(virtual_rust_source_base_dir) =
-            option_env!("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR").map(PathBuf::from)
-        {
-            normalize_path(&virtual_rust_source_base_dir.join("library"), "$SRC_DIR");
-            normalize_path(&virtual_rust_source_base_dir.join("compiler"), "$COMPILER_DIR");
+        let source_bases = &[
+            // Source base on the current filesystem (calculated as parent of `src/test/$suite`):
+            Some(self.config.src_base.parent().unwrap().parent().unwrap().parent().unwrap().into()),
+            // Source base on the sysroot (from the src components downloaded by `download-rustc`):
+            Some(self.config.sysroot_base.join("lib").join("rustlib").join("src").join("rust")),
+            // Virtual `/rustc/$sha` remapped paths (if `remap-debuginfo` is enabled):
+            option_env!("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR").map(PathBuf::from),
+            // Virtual `/rustc/$sha` coming from download-rustc:
+            std::env::var_os("FAKE_DOWNLOAD_RUSTC_PREFIX").map(PathBuf::from),
+        ];
+        for base_dir in source_bases {
+            if let Some(base_dir) = base_dir {
+                // Paths into the libstd/libcore
+                normalize_path(&base_dir.join("library"), "$SRC_DIR");
+                // `ui-fulldeps` tests can show paths to the compiler source when testing macros from
+                // `rustc_macros`
+                // eg. /home/user/rust/compiler
+                normalize_path(&base_dir.join("compiler"), "$COMPILER_DIR");
+            }
         }
 
         // Paths into the build directory