Rollup merge of #80932 - jyn514:download-windows-llvm, r=Mark-Simulacrum

Allow downloading LLVM on Windows and MacOS

- Don't ignore packaging `llvm/lib/` for `rust-dev` when LLVM is linked
statically
- Add `link-type.txt` so bootstrap knows whether llvm was linked
  statically or dynamically
- Don't assume CI LLVM is linked dynamically in `bootstrap::config`
- Fall back to dynamic linking if `link-type.txt` doesn't exist
- Fix existing bug that split the output of `llvm-config` on lines, not spaces
- Only special case MacOS when dynamic linking. Static linking works fine.
- Enable building LLVM tests

  This works around the following llvm bug:

  ```
  llvm-config: error: component libraries and shared library

  llvm-config: error: missing: /home/joshua/rustc2/build/x86_64-unknown-linux-gnu/llvm/build/lib/libgtest.a
  llvm-config: error: missing: /home/joshua/rustc2/build/x86_64-unknown-linux-gnu/llvm/build/lib/libgtest_main.a
  llvm-config: error: missing: /home/joshua/rustc2/build/x86_64-unknown-linux-gnu/llvm/build/lib/libLLVMTestingSupport.a
  thread 'main' panicked at 'command did not execute successfully: "/home/joshua/rustc2/build/x86_64-unknown-linux-gnu/llvm/build/bin/llvm-config" "--libfiles"
  ```

  I'm not sure why llvm-config thinks these are required, but to avoid
  the error, this builds them anyway.

- Bump version of `download-ci-llvm-stamp`

  `src/llvm-project` hasn't changed, but the generated tarball has.

Fixes #77084.

# Current Status

This works on both MacOS and Windows! 🎉 🎉 Thanks to ```@nagisa,``` ```@halkcyon,``` ```@Lokathor,``` ```@jryans,``` and ```@poliorcetics``` for helping me test!

The `if-available` check now supports all tier 1 platforms. Although only x64 apple and x64 msvc have been tested, none of the changes here are Windows or Mac specific, and I expect this to work anywhere that LLVM artifacts are uploaded to CI (i.e. the `rust-dev` component exists).

## Windows

Note that if you have an old version of MSVC build tools you'll need to update them. VS Build Tools 2019 14.28 and later are known to work. With old tools, you may see an error like the following:

```
error LNK2001: unresolved external symbol __imp___std_init_once_complete
```

Author: m-ou-se

config.toml.example

@@ -35,9 +35,11 @@ changelog-seen = 2
 # Unless you're developing for a target where Rust CI doesn't build a compiler
 # toolchain or changing LLVM locally, you probably want to set this to true.
 #
-# It's currently false by default due to being newly added; please file bugs if
-# enabling this did not work for you on x86_64-unknown-linux-gnu.
-# Other target triples are currently not supported; see #77084.
+# This is false by default so that distributions don't unexpectedly download
+# LLVM from the internet.
+#
+# All tier 1 targets are currently supported; set this to `"if-supported"` if
+# you are not sure whether you're on a tier 1 target.
 #
 # We also currently only support this when building LLVM for the build triple.
 #

src/bootstrap/bootstrap.py

@@ -465,8 +465,21 @@ def download_stage0(self):
 
     def downloading_llvm(self):
         opt = self.get_toml('download-ci-llvm', 'llvm')
+        # This is currently all tier 1 targets (since others may not have CI
+        # artifacts)
+        # https://doc.rust-lang.org/rustc/platform-support.html#tier-1
+        supported_platforms = [
+            "aarch64-unknown-linux-gnu",
+            "i686-pc-windows-gnu",
+            "i686-pc-windows-msvc",
+            "i686-unknown-linux-gnu",
+            "x86_64-unknown-linux-gnu",
+            "x86_64-apple-darwin",
+            "x86_64-pc-windows-gnu",
+            "x86_64-pc-windows-msvc",
+        ]
         return opt == "true" \
-            or (opt == "if-available" and self.build == "x86_64-unknown-linux-gnu")
+            or (opt == "if-available" and self.build in supported_platforms)
 
     def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
         if date is None:

src/bootstrap/config.rs

@@ -816,8 +816,10 @@ impl Config {
                 check_ci_llvm!(llvm.allow_old_toolchain);
                 check_ci_llvm!(llvm.polly);
 
-                // CI-built LLVM is shared
-                config.llvm_link_shared = true;
+                // CI-built LLVM can be either dynamic or static.
+                let ci_llvm = config.out.join(&*config.build.triple).join("ci-llvm");
+                let link_type = t!(std::fs::read_to_string(ci_llvm.join("link-type.txt")));
+                config.llvm_link_shared = link_type == "dynamic";
             }
 
             if config.llvm_thin_lto {

src/bootstrap/dist.rs

@@ -1800,19 +1800,11 @@ fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) {
     }
 }
 
-/// Maybe add libLLVM.so to the given destination lib-dir. It will only have
-/// been built if LLVM tools are linked dynamically.
+/// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking.
 ///
-/// Note: This function does not yet support Windows, but we also don't support
-///       linking LLVM tools dynamically on Windows yet.
-fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) {
-    if !builder.config.llvm_link_shared {
-        // We do not need to copy LLVM files into the sysroot if it is not
-        // dynamically linked; it is already included into librustc_llvm
-        // statically.
-        return;
-    }
 
+/// Returns whether the files were actually copied.
+fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool {
     if let Some(config) = builder.config.target_config.get(&target) {
         if config.llvm_config.is_some() && !builder.config.llvm_from_ci {
             // If the LLVM was externally provided, then we don't currently copy
@@ -1828,7 +1820,7 @@ fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir
             //
             // If the LLVM is coming from ourselves (just from CI) though, we
             // still want to install it, as it otherwise won't be available.
-            return;
+            return false;
         }
     }
 
@@ -1837,31 +1829,48 @@ fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir
     // clear why this is the case, though. llvm-config will emit the versioned
     // paths and we don't want those in the sysroot (as we're expecting
     // unversioned paths).
-    if target.contains("apple-darwin") {
+    if target.contains("apple-darwin") && builder.config.llvm_link_shared {
         let src_libdir = builder.llvm_out(target).join("lib");
         let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
         if llvm_dylib_path.exists() {
             builder.install(&llvm_dylib_path, dst_libdir, 0o644);
         }
+        !builder.config.dry_run
     } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) {
-        let files = output(Command::new(llvm_config).arg("--libfiles"));
-        for file in files.lines() {
+        let mut cmd = Command::new(llvm_config);
+        cmd.arg("--libfiles");
+        builder.verbose(&format!("running {:?}", cmd));
+        let files = output(&mut cmd);
+        for file in files.trim_end().split(' ') {
             builder.install(Path::new(file), dst_libdir, 0o644);
         }
+        !builder.config.dry_run
+    } else {
+        false
     }
 }
 
 /// Maybe add libLLVM.so to the target lib-dir for linking.
 pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
     let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib");
-    maybe_install_llvm(builder, target, &dst_libdir);
+    // We do not need to copy LLVM files into the sysroot if it is not
+    // dynamically linked; it is already included into librustc_llvm
+    // statically.
+    if builder.config.llvm_link_shared {
+        maybe_install_llvm(builder, target, &dst_libdir);
+    }
 }
 
 /// Maybe add libLLVM.so to the runtime lib-dir for rustc itself.
 pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) {
     let dst_libdir =
         sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target }));
-    maybe_install_llvm(builder, target, &dst_libdir);
+    // We do not need to copy LLVM files into the sysroot if it is not
+    // dynamically linked; it is already included into librustc_llvm
+    // statically.
+    if builder.config.llvm_link_shared {
+        maybe_install_llvm(builder, target, &dst_libdir);
+    }
 }
 
 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
@@ -1973,7 +1982,10 @@ impl Step for RustDev {
         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
         // of `rustc-dev` to support the inherited `-lLLVM` when using the
         // compiler libraries.
-        maybe_install_llvm(builder, target, &tarball.image_dir().join("lib"));
+        let dst_libdir = tarball.image_dir().join("lib");
+        maybe_install_llvm(builder, target, &dst_libdir);
+        let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
+        t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
 
         Some(tarball.generate())
     }

src/bootstrap/download-ci-llvm-stamp

@@ -1,4 +1,4 @@
 Change this file to make users of the `download-ci-llvm` configuration download
 a new version of LLVM from CI, even if the LLVM submodule hasn’t changed.
 
-Last change is for: https://github.com/rust-lang/rust/pull/80087
+Last change is for: https://github.com/rust-lang/rust/pull/80932

src/bootstrap/native.rs

@@ -171,7 +171,6 @@ impl Step for Llvm {
             .define("LLVM_TARGETS_TO_BUILD", llvm_targets)
             .define("LLVM_EXPERIMENTAL_TARGETS_TO_BUILD", llvm_exp_targets)
             .define("LLVM_INCLUDE_EXAMPLES", "OFF")
-            .define("LLVM_INCLUDE_TESTS", "OFF")
             .define("LLVM_INCLUDE_DOCS", "OFF")
             .define("LLVM_INCLUDE_BENCHMARKS", "OFF")
             .define("LLVM_ENABLE_TERMINFO", "OFF")