Auto merge of rust-lang#125642 - khuey:zstd, r=<try>

Enable zstd for debug compression.

Set LLVM_ENABLE_ZSTD alongside LLVM_ENABLE_ZLIB so that --compress-debug-sections=zstd is an option.

See rust-lang#120953

Author: bors

compiler/rustc_llvm/build.rs

@@ -259,15 +259,32 @@ fn main() {
     cmd.args(&components);
 
     for lib in output(&mut cmd).split_whitespace() {
+        let mut is_static = false;
         let name = if let Some(stripped) = lib.strip_prefix("-l") {
             stripped
         } else if let Some(stripped) = lib.strip_prefix('-') {
             stripped
         } else if Path::new(lib).exists() {
             // On MSVC llvm-config will print the full name to libraries, but
             // we're only interested in the name part
-            let name = Path::new(lib).file_name().unwrap().to_str().unwrap();
-            name.trim_end_matches(".lib")
+            // On Unix when we get a static library llvm-config will print the
+            // full name and we *are* interested in the path, but we need to
+            // handle it separately. For example, when statically linking to
+            // libzstd llvm-config will output something like
+            //   -lrt -ldl -lm -lz /usr/local/lib/libzstd.a -lxml2
+            // and we transform the zstd part into
+            //   cargo:rustc-link-search-native=/usr/local/lib
+            //   cargo:rustc-link-lib=static=zstd
+            let path = Path::new(lib);
+            if lib.ends_with(".a") {
+                is_static = true;
+                println!("cargo:rustc-link-search=native={}", path.parent().unwrap().display());
+                let name = path.file_stem().unwrap().to_str().unwrap();
+                name.trim_start_matches("lib")
+            } else {
+                let name = path.file_name().unwrap().to_str().unwrap();
+                name.trim_end_matches(".lib")
+            }
         } else if lib.ends_with(".lib") {
             // Some MSVC libraries just come up with `.lib` tacked on, so chop
             // that off
@@ -285,7 +302,13 @@ fn main() {
             continue;
         }
 
-        let kind = if name.starts_with("LLVM") { llvm_kind } else { "dylib" };
+        let kind = if name.starts_with("LLVM") {
+            llvm_kind
+        } else if is_static {
+            "static"
+        } else {
+            "dylib"
+        };
         println!("cargo:rustc-link-lib={kind}={name}");
     }
 

config.example.toml

@@ -713,6 +713,11 @@
 # option will override this if set.
 #llvm-libunwind = 'no'
 
+# Global default for llvm-libzstd for all targets. See the target-specific
+# documentation for llvm-libzstd below. Note that the target-specific
+# option will override this if set.
+#llvm-libzstd = false
+
 # Enable Windows Control Flow Guard checks in the standard library.
 # This only applies from stage 1 onwards, and only for Windows targets.
 #control-flow-guard = false
@@ -816,6 +821,9 @@
 # it must link to `libgcc_eh.a` to get a working output, and this option have no effect.
 #llvm-libunwind = 'no' if Linux, 'in-tree' if Fuchsia
 
+# Enable LLVM to use zstd for compression.
+#llvm-libzstd = if linux { true } else { false }
+
 # Build the sanitizer runtimes for this target.
 # This option will override the same option under [build] section.
 #sanitizers = build.sanitizers (bool)

src/bootstrap/download-ci-llvm-stamp

@@ -1,4 +1,4 @@
 Change this file to make users of the `download-ci-llvm` configuration download
 a new version of LLVM from CI, even if the LLVM submodule hasn’t changed.
 
-Last change is for: https://github.com/rust-lang/rust/pull/126298
+Last change is for: https://github.com/rust-lang/rust/pull/125642

src/bootstrap/src/core/build_steps/llvm.rs

@@ -368,15 +368,6 @@ impl Step for Llvm {
             cfg.define("LLVM_PROFDATA_FILE", path);
         }
 
-        // Disable zstd to avoid a dependency on libzstd.so.
-        cfg.define("LLVM_ENABLE_ZSTD", "OFF");
-
-        if !target.is_windows() {
-            cfg.define("LLVM_ENABLE_ZLIB", "ON");
-        } else {
-            cfg.define("LLVM_ENABLE_ZLIB", "OFF");
-        }
-
         // Are we compiling for iOS/tvOS/watchOS/visionOS?
         if target.contains("apple-ios")
             || target.contains("apple-tvos")
@@ -824,6 +815,18 @@ fn configure_llvm(builder: &Builder<'_>, target: TargetSelection, cfg: &mut cmak
         }
     }
 
+    // Libraries for ELF section compression.
+    if !target.is_windows() {
+        cfg.define("LLVM_ENABLE_ZLIB", "ON");
+    } else {
+        cfg.define("LLVM_ENABLE_ZLIB", "OFF");
+    }
+
+    if builder.config.llvm_libzstd(target) {
+        cfg.define("LLVM_ENABLE_ZSTD", "FORCE_ON");
+        cfg.define("LLVM_USE_STATIC_ZSTD", "TRUE");
+    }
+
     if let Some(ref linker) = builder.config.llvm_use_linker {
         cfg.define("LLVM_USE_LINKER", linker);
     }

src/bootstrap/src/core/config/config.rs

@@ -280,6 +280,7 @@ pub struct Config {
     pub llvm_profile_use: Option<String>,
     pub llvm_profile_generate: bool,
     pub llvm_libunwind_default: Option<LlvmLibunwind>,
+    pub llvm_libzstd_default: Option<bool>,
     pub enable_bolt_settings: bool,
 
     pub reproducible_artifacts: Vec<String>,
@@ -545,6 +546,7 @@ pub struct Target {
     /// Some(path to FileCheck) if one was specified.
     pub llvm_filecheck: Option<PathBuf>,
     pub llvm_libunwind: Option<LlvmLibunwind>,
+    pub llvm_libzstd: Option<bool>,
     pub cc: Option<PathBuf>,
     pub cxx: Option<PathBuf>,
     pub ar: Option<PathBuf>,
@@ -1101,6 +1103,7 @@ define_config! {
         jemalloc: Option<bool> = "jemalloc",
         test_compare_mode: Option<bool> = "test-compare-mode",
         llvm_libunwind: Option<String> = "llvm-libunwind",
+        llvm_libzstd: Option<bool> = "llvm-libzstd",
         control_flow_guard: Option<bool> = "control-flow-guard",
         ehcont_guard: Option<bool> = "ehcont-guard",
         new_symbol_mangling: Option<bool> = "new-symbol-mangling",
@@ -1127,6 +1130,7 @@ define_config! {
         llvm_has_rust_patches: Option<bool> = "llvm-has-rust-patches",
         llvm_filecheck: Option<String> = "llvm-filecheck",
         llvm_libunwind: Option<String> = "llvm-libunwind",
+        llvm_libzstd: Option<bool> = "llvm-libzstd",
         sanitizers: Option<bool> = "sanitizers",
         profiler: Option<StringOrBool> = "profiler",
         rpath: Option<bool> = "rpath",
@@ -1628,6 +1632,7 @@ impl Config {
                 jemalloc,
                 test_compare_mode,
                 llvm_libunwind,
+                llvm_libzstd,
                 control_flow_guard,
                 ehcont_guard,
                 new_symbol_mangling,
@@ -1714,6 +1719,7 @@ impl Config {
             set(&mut config.ehcont_guard, ehcont_guard);
             config.llvm_libunwind_default =
                 llvm_libunwind.map(|v| v.parse().expect("failed to parse rust.llvm-libunwind"));
+            config.llvm_libzstd_default = llvm_libzstd;
 
             if let Some(ref backends) = codegen_backends {
                 let available_backends = ["llvm", "cranelift", "gcc"];
@@ -1906,6 +1912,7 @@ impl Config {
                         panic!("failed to parse target.{triple}.llvm-libunwind")
                     })
                 });
+                target.llvm_libzstd = cfg.llvm_libzstd;
                 if let Some(s) = cfg.no_std {
                     target.no_std = s;
                 }
@@ -2392,6 +2399,14 @@ impl Config {
             })
     }
 
+    pub fn llvm_libzstd(&self, target: TargetSelection) -> bool {
+        self.target_config
+            .get(&target)
+            .and_then(|t| t.llvm_libzstd)
+            .or(self.llvm_libzstd_default)
+            .unwrap_or(target.contains("linux"))
+    }
+
     pub fn split_debuginfo(&self, target: TargetSelection) -> SplitDebuginfo {
         self.target_config
             .get(&target)
@@ -2696,6 +2711,7 @@ fn check_incompatible_options_for_ci_rustc(rust: &Rust) -> Result<(), String> {
         remap_debuginfo: _,
         test_compare_mode: _,
         llvm_libunwind: _,
+        llvm_libzstd: _,
         control_flow_guard: _,
         ehcont_guard: _,
         new_symbol_mangling: _,

src/bootstrap/src/utils/change_tracker.rs

@@ -220,4 +220,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[
         severity: ChangeSeverity::Warning,
         summary: "For tarball sources, default value for `rust.channel` will be taken from `src/ci/channel` file.",
     },
+    ChangeInfo {
+        change_id: 125642,
+        severity: ChangeSeverity::Info,
+        summary: "New option `llvm-libzstd` to control whether llvm is built with zstd support.",
+    },
 ];

src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile

@@ -62,6 +62,10 @@ COPY host-x86_64/dist-x86_64-linux/build-clang.sh /tmp/
 RUN ./build-clang.sh
 ENV CC=clang CXX=clang++
 
+# rustc's LLVM needs zstd.
+COPY scripts/zstd.sh /tmp/
+RUN ./zstd.sh
+
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 

src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile

@@ -17,6 +17,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   pkg-config \
   xz-utils \
   mingw-w64 \
+  zlib1g-dev \
+  libzstd-dev \
   && rm -rf /var/lib/apt/lists/*
 
 COPY scripts/sccache.sh /scripts/

src/ci/docker/scripts/zstd.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -ex
+
+hide_output() {
+  set +x
+  on_err="
+echo ERROR: An error was encountered with the build.
+cat /tmp/zstd_build.log
+exit 1
+"
+  trap "$on_err" ERR
+  bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
+  PING_LOOP_PID=$!
+  "$@" &> /tmp/zstd_build.log
+  trap - ERR
+  kill $PING_LOOP_PID
+  rm /tmp/zstd_build.log
+  set -x
+}
+
+ZSTD=1.5.6
+curl -L https://github.com/facebook/zstd/releases/download/v$ZSTD/zstd-$ZSTD.tar.gz | tar xzf -
+
+cd zstd-$ZSTD
+CFLAGS=-fPIC hide_output make -j$(nproc) VERBOSE=1
+hide_output make install
+
+cd ..
+rm -rf zstd-$ZSTD

tests/run-make/rust-lld-compress-debug-sections/main.rs

@@ -0,0 +1 @@
+fn main() {}

tests/run-make/rust-lld-compress-debug-sections/rmake.rs

@@ -0,0 +1,39 @@
+// Checks the `compress-debug-sections` option on rust-lld.
+
+//@ needs-rust-lld
+//@ only-linux
+//@ ignore-cross-compile
+
+// FIXME: This test isn't comprehensive and isn't covering all possible combinations.
+
+use run_make_support::{assert_contains, cmd, llvm_readobj, run_in_tmpdir, rustc};
+
+fn check_compression(compression: &str, to_find: &str) {
+    run_in_tmpdir(|| {
+        let out = rustc()
+            .arg("-Zlinker-features=+lld")
+            .arg("-Clink-self-contained=+linker")
+            .arg("-Zunstable-options")
+            .arg("-Cdebuginfo=full")
+            .link_arg(&format!("-Wl,--compress-debug-sections={compression}"))
+            .input("main.rs")
+            .run_unchecked();
+        let stderr = out.stderr_utf8();
+        if stderr.is_empty() {
+            llvm_readobj().arg("-t").arg("main").run().assert_stdout_contains(to_find);
+        } else {
+            assert_contains(
+                stderr,
+                format!(
+                    "LLVM was not built with LLVM_ENABLE_{to_find} \
+                     or did not find {compression} at build time"
+                ),
+            );
+        }
+    });
+}
+
+fn main() {
+    check_compression("zlib", "ZLIB");
+    check_compression("zstd", "ZSTD");
+}