Auto merge of #89819 - davidtwco:issue-81024-multiple-crates-multiple-dwarves, r=nagisa

cg: split dwarf for crate dependencies

Fixes #81024.

- In #79570, `-Z split-dwarf-kind={none,single,split}` was replaced by `-C split-debuginfo={off,packed,unpacked}`. `-C split-debuginfo`'s packed and unpacked aren't exact parallels to single and split, respectively.

  On Unix, `-C split-debuginfo=packed` will put debuginfo in object files and package debuginfo into a DWARF package file (`.dwp`) and `-C split-debuginfo=unpacked` will put debuginfo in dwarf object files and won't package it.

  In the initial implementation of Split DWARF, split mode wrote sections which did not require relocation into a DWARF object (`.dwo`) file which was ignored by the linker and then packaged those DWARF objects into DWARF packages (`.dwp`). In single mode, sections which did not require relocation were written into object files but ignored by the linker and were not packaged. However, both split and single modes could be packaged or not, the primary difference in behaviour was where the debuginfo sections that did not require link-time relocation were written (in a DWARF object or the object file).

  In the first commit of this PR, I re-introduce a `-Z split-dwarf-kind` flag, which can be used to pick between split and single modes when `-C split-debuginfo` is used to enable Split DWARF (either packed or unpacked).
- Split DWARF packaging requires all of the object files to exist, including those in dependencies. ~~Therefore, the second commit of this PR makes rustc keep all objects or dwarf objects for unpacked mode and if the crate is a dependency in packed mode (determined by heuristic: if no linking is taking place), then objects or dwarf objects are kept. Objects are kept if `-Z split-dwarf-kind` is `SplitDwarfKind::Single`, and dwarf objects if `SplitDwarfKind::Split`.~~

  ~~There are other approaches that could be taken to supporting packed Split DWARF with crate dependencies but this seemed like the least complicated and was contained to only rustc. Other potential approaches are described in #81024 (comment), I'm happy to change the approach I've taken here if it isn't what we're looking for.~~ See #89819 (comment) for the current approach.
- ~~There's still a dependency on `llvm-dwp` after this change, which [we probably want to move away from](https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/llvm-dwp.20is.20not.20recommended) but that seems out-of-scope for this PR. Ideally, Split DWARF (in packed or unpacked modes) will be usable on nightly after this lands. If there aren't any bugs reported then it's possible we could allow Split DWARF to be used on stable after this change, it depends whether or not switching away from `llvm-dwp` later would break any guarantees, or whether we'd want to change how we handle this cross-crate case in future.~~ See #89819 (comment).

r? `@nagisa`
cc `@alexcrichton`

Author: bors

Cargo.lock

@@ -9,7 +9,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3e61f2b7f93d2c7d2b08263acaa4a363b3e276806c68af6134c44f523bf1aacd"
 dependencies = [
  "compiler_builtins",
- "gimli",
+ "gimli 0.25.0",
  "rustc-std-workspace-alloc",
  "rustc-std-workspace-core",
 ]
@@ -87,9 +87,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.34"
+version = "1.0.51"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf8dcb5b4bbaa28653b647d8c77bd4ed40183b48882e130c1f1ffb73de069fd7"
+checksum = "8b26702f315f53b6071259e15dd9d64528213b44d61de1ec926eca7715d62203"
 
 [[package]]
 name = "array_tool"
@@ -1158,6 +1158,12 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
 
+[[package]]
+name = "fallible-iterator"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
+
 [[package]]
 name = "filetime"
 version = "0.2.14"
@@ -1446,6 +1452,17 @@ dependencies = [
  "rustc-std-workspace-core",
 ]
 
+[[package]]
+name = "gimli"
+version = "0.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4"
+dependencies = [
+ "fallible-iterator",
+ "indexmap",
+ "stable_deref_trait",
+]
+
 [[package]]
 name = "git2"
 version = "0.13.23"
@@ -2339,6 +2356,18 @@ dependencies = [
  "rustc-std-workspace-core",
 ]
 
+[[package]]
+name = "object"
+version = "0.27.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9"
+dependencies = [
+ "crc32fast",
+ "flate2",
+ "indexmap",
+ "memchr",
+]
+
 [[package]]
 name = "odht"
 version = "0.3.1"
@@ -3725,10 +3754,11 @@ dependencies = [
  "itertools 0.9.0",
  "jobserver",
  "libc",
- "object",
+ "object 0.26.2",
  "pathdiff",
  "regex",
  "rustc_apfloat",
+ "rustc_arena",
  "rustc_ast",
  "rustc_attr",
  "rustc_data_structures",
@@ -3749,6 +3779,7 @@ dependencies = [
  "smallvec",
  "snap",
  "tempfile",
+ "thorin-dwp",
  "tracing",
 ]
 
@@ -4993,7 +5024,7 @@ dependencies = [
  "hermit-abi",
  "libc",
  "miniz_oxide",
- "object",
+ "object 0.26.2",
  "panic_abort",
  "panic_unwind",
  "profiler_builtins",
@@ -5057,9 +5088,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
 
 [[package]]
 name = "structopt"
-version = "0.3.16"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de5472fb24d7e80ae84a7801b7978f95a19ec32cb1876faea59ab711eb901976"
+checksum = "40b9788f4202aa75c240ecc9c15c65185e6a39ccdeb0fd5d008b98825464c87c"
 dependencies = [
  "clap",
  "lazy_static",
@@ -5068,9 +5099,9 @@ dependencies = [
 
 [[package]]
 name = "structopt-derive"
-version = "0.4.9"
+version = "0.4.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e0eb37335aeeebe51be42e2dc07f031163fbabfa6ac67d7ea68b5c2f68d5f99"
+checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"
 dependencies = [
  "heck",
  "proc-macro-error",
@@ -5249,24 +5280,36 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.20"
+version = "1.0.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7dfdd070ccd8ccb78f4ad66bf1982dc37f620ef696c6b5028fe2ed83dd3d0d08"
+checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.20"
+version = "1.0.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd80fc12f73063ac132ac92aceea36734f04a1d93c1240c6944e23a3b8841793"
+checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b"
 dependencies = [
  "proc-macro2",
  "quote",
  "syn",
 ]
 
+[[package]]
+name = "thorin-dwp"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "039d1fc0bfdb73910c2702893515580e38c192f47a987bc98ddd38a36f2d953a"
+dependencies = [
+ "gimli 0.26.1",
+ "indexmap",
+ "object 0.27.1",
+ "tracing",
+]
+
 [[package]]
 name = "thread_local"
 version = "1.0.1"
@@ -5394,9 +5437,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6"
 
 [[package]]
 name = "tracing"
-version = "0.1.28"
+version = "0.1.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84f96e095c0c82419687c20ddf5cb3eadb61f4e1405923c9dc8e53a1adacbda8"
+checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105"
 dependencies = [
  "cfg-if 1.0.0",
  "pin-project-lite",
@@ -5406,9 +5449,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-attributes"
-version = "0.1.17"
+version = "0.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4f915eb6abf914599c200260efced9203504c4c37380af10cdf3b7d36970650"
+checksum = "f4f480b8f81512e825f337ad51e94c1eb5d3bbdf2b363dcd01e2b19a9ffe3f8e"
 dependencies = [
  "proc-macro2",
  "quote",

compiler/rustc_codegen_llvm/src/back/write.rs

@@ -23,7 +23,7 @@ use rustc_errors::{FatalError, Handler, Level};
 use rustc_fs_util::{link_or_copy, path_to_c_string};
 use rustc_middle::bug;
 use rustc_middle::ty::TyCtxt;
-use rustc_session::config::{self, Lto, OutputType, Passes, SwitchWithOptPath};
+use rustc_session::config::{self, Lto, OutputType, Passes, SplitDwarfKind, SwitchWithOptPath};
 use rustc_session::Session;
 use rustc_span::symbol::sym;
 use rustc_span::InnerSpan;
@@ -106,7 +106,11 @@ pub fn create_informational_target_machine(sess: &Session) -> &'static mut llvm:
 
 pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut llvm::TargetMachine {
     let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() {
-        tcx.output_filenames(()).split_dwarf_path(tcx.sess.split_debuginfo(), Some(mod_name))
+        tcx.output_filenames(()).split_dwarf_path(
+            tcx.sess.split_debuginfo(),
+            tcx.sess.opts.debugging_opts.split_dwarf_kind,
+            Some(mod_name),
+        )
     } else {
         None
     };
@@ -892,17 +896,18 @@ pub(crate) unsafe fn codegen(
                     .generic_activity_with_arg("LLVM_module_codegen_emit_obj", &*module.name);
 
                 let dwo_out = cgcx.output_filenames.temp_path_dwo(module_name);
-                let dwo_out = match cgcx.split_debuginfo {
-                    // Don't change how DWARF is emitted in single mode (or when disabled).
-                    SplitDebuginfo::Off | SplitDebuginfo::Packed => None,
-                    // Emit (a subset of the) DWARF into a separate file in split mode.
-                    SplitDebuginfo::Unpacked => {
-                        if cgcx.target_can_use_split_dwarf {
-                            Some(dwo_out.as_path())
-                        } else {
-                            None
-                        }
-                    }
+                let dwo_out = match (cgcx.split_debuginfo, cgcx.split_dwarf_kind) {
+                    // Don't change how DWARF is emitted when disabled.
+                    (SplitDebuginfo::Off, _) => None,
+                    // Don't provide a DWARF object path if split debuginfo is enabled but this is
+                    // a platform that doesn't support Split DWARF.
+                    _ if !cgcx.target_can_use_split_dwarf => None,
+                    // Don't provide a DWARF object path in single mode, sections will be written
+                    // into the object as normal but ignored by linker.
+                    (_, SplitDwarfKind::Single) => None,
+                    // Emit (a subset of the) DWARF into a separate dwarf object file in split
+                    // mode.
+                    (_, SplitDwarfKind::Split) => Some(dwo_out.as_path()),
                 };
 
                 with_codegen(tm, llmod, config.no_builtins, |cpm| {
@@ -939,7 +944,9 @@ pub(crate) unsafe fn codegen(
 
     Ok(module.into_compiled_module(
         config.emit_obj != EmitObj::None,
-        cgcx.target_can_use_split_dwarf && cgcx.split_debuginfo == SplitDebuginfo::Unpacked,
+        cgcx.target_can_use_split_dwarf
+            && cgcx.split_debuginfo != SplitDebuginfo::Off
+            && cgcx.split_dwarf_kind == SplitDwarfKind::Split,
         config.emit_bc,
         &cgcx.output_filenames,
     ))

compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs

@@ -1072,7 +1072,11 @@ pub fn compile_unit_metadata<'ll, 'tcx>(
     let output_filenames = tcx.output_filenames(());
     let split_name = if tcx.sess.target_can_use_split_dwarf() {
         output_filenames
-            .split_dwarf_path(tcx.sess.split_debuginfo(), Some(codegen_unit_name))
+            .split_dwarf_path(
+                tcx.sess.split_debuginfo(),
+                tcx.sess.opts.debugging_opts.split_dwarf_kind,
+                Some(codegen_unit_name),
+            )
             // We get a path relative to the working directory from split_dwarf_path
             .map(|f| tcx.sess.source_map().path_mapping().map_prefix(f).0)
     } else {

compiler/rustc_codegen_ssa/Cargo.toml

@@ -14,12 +14,14 @@ tracing = "0.1"
 libc = "0.2.50"
 jobserver = "0.1.22"
 tempfile = "3.2"
+thorin-dwp = "0.1.1"
 pathdiff = "0.2.0"
 snap = "1"
 smallvec = { version = "1.6.1", features = ["union", "may_dangle"] }
 regex = "1.4"
 
 rustc_serialize = { path = "../rustc_serialize" }
+rustc_arena = { path = "../rustc_arena" }
 rustc_ast = { path = "../rustc_ast" }
 rustc_span = { path = "../rustc_span" }
 rustc_middle = { path = "../rustc_middle" }