Rollup merge of rust-lang#79570 - alexcrichton:split-debuginfo, r=bjorn3

rustc: Stabilize `-Zrun-dsymutil` as `-Csplit-debuginfo`

This commit adds a new stable codegen option to rustc,
`-Csplit-debuginfo`. The old `-Zrun-dsymutil` flag is deleted and now
subsumed by this stable flag. Additionally `-Zsplit-dwarf` is also
subsumed by this flag but still requires `-Zunstable-options` to
actually activate. The `-Csplit-debuginfo` flag takes one of
three values:

* `off` - This indicates that split-debuginfo from the final artifact is
  not desired. This is not supported on Windows and is the default on
  Unix platforms except macOS. On macOS this means that `dsymutil` is
  not executed.

* `packed` - This means that debuginfo is desired in one location
  separate from the main executable. This is the default on Windows
  (`*.pdb`) and macOS (`*.dSYM`). On other Unix platforms this subsumes
  `-Zsplit-dwarf=single` and produces a `*.dwp` file.

* `unpacked` - This means that debuginfo will be roughly equivalent to
  object files, meaning that it's throughout the build directory
  rather than in one location (often the fastest for local development).
  This is not the default on any platform and is not supported on Windows.

Each target can indicate its own default preference for how debuginfo is
handled. Almost all platforms default to `off` except for Windows and
macOS which default to `packed` for historical reasons.

Some equivalencies for previous unstable flags with the new flags are:

* `-Zrun-dsymutil=yes` -> `-Csplit-debuginfo=packed`
* `-Zrun-dsymutil=no` -> `-Csplit-debuginfo=unpacked`
* `-Zsplit-dwarf=single` -> `-Csplit-debuginfo=packed`
* `-Zsplit-dwarf=split` -> `-Csplit-debuginfo=unpacked`

Note that `-Csplit-debuginfo` still requires `-Zunstable-options` for
non-macOS platforms since split-dwarf support was *just* implemented in
rustc.

There's some more rationale listed on rust-lang#79361, but the main gist of the
motivation for this commit is that `dsymutil` can take quite a long time
to execute in debug builds and provides little benefit. This means that
incremental compile times appear that much worse on macOS because the
compiler is constantly running `dsymutil` over every single binary it
produces during `cargo build` (even build scripts!). Ideally rustc would
switch to not running `dsymutil` by default, but that's a problem left
to get tackled another day.

Closes rust-lang#79361

Author: JohnTitor

compiler/rustc_codegen_llvm/src/back/lto.rs

@@ -732,10 +732,7 @@ pub unsafe fn optimize_thin_module(
     let diag_handler = cgcx.create_diag_handler();
 
     let module_name = &thin_module.shared.module_names[thin_module.idx];
-    let split_dwarf_file = cgcx
-        .output_filenames
-        .split_dwarf_filename(cgcx.split_dwarf_kind, Some(module_name.to_str().unwrap()));
-    let tm_factory_config = TargetMachineFactoryConfig { split_dwarf_file };
+    let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, module_name.to_str().unwrap());
     let tm =
         (cgcx.tm_factory)(tm_factory_config).map_err(|e| write::llvm_err(&diag_handler, &e))?;
 

compiler/rustc_codegen_llvm/src/back/write.rs

@@ -23,13 +23,11 @@ use rustc_fs_util::{link_or_copy, path_to_c_string};
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::bug;
 use rustc_middle::ty::TyCtxt;
-use rustc_session::config::{
-    self, Lto, OutputType, Passes, SanitizerSet, SplitDwarfKind, SwitchWithOptPath,
-};
+use rustc_session::config::{self, Lto, OutputType, Passes, SanitizerSet, SwitchWithOptPath};
 use rustc_session::Session;
 use rustc_span::symbol::sym;
 use rustc_span::InnerSpan;
-use rustc_target::spec::{CodeModel, RelocModel};
+use rustc_target::spec::{CodeModel, RelocModel, SplitDebuginfo};
 use tracing::debug;
 
 use libc::{c_char, c_int, c_uint, c_void, size_t};
@@ -93,9 +91,12 @@ pub fn create_informational_target_machine(sess: &Session) -> &'static mut llvm:
 }
 
 pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut llvm::TargetMachine {
-    let split_dwarf_file = tcx
-        .output_filenames(LOCAL_CRATE)
-        .split_dwarf_filename(tcx.sess.opts.debugging_opts.split_dwarf, Some(mod_name));
+    let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() {
+        tcx.output_filenames(LOCAL_CRATE)
+            .split_dwarf_filename(tcx.sess.split_debuginfo(), Some(mod_name))
+    } else {
+        None
+    };
     let config = TargetMachineFactoryConfig { split_dwarf_file };
     target_machine_factory(&tcx.sess, tcx.backend_optimization_level(LOCAL_CRATE))(config)
         .unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), &err).raise())
@@ -838,11 +839,17 @@ pub(crate) unsafe fn codegen(
                     .generic_activity_with_arg("LLVM_module_codegen_emit_obj", &module.name[..]);
 
                 let dwo_out = cgcx.output_filenames.temp_path_dwo(module_name);
-                let dwo_out = match cgcx.split_dwarf_kind {
+                let dwo_out = match cgcx.split_debuginfo {
                     // Don't change how DWARF is emitted in single mode (or when disabled).
-                    SplitDwarfKind::None | SplitDwarfKind::Single => None,
+                    SplitDebuginfo::Off | SplitDebuginfo::Packed => None,
                     // Emit (a subset of the) DWARF into a separate file in split mode.
-                    SplitDwarfKind::Split => Some(dwo_out.as_path()),
+                    SplitDebuginfo::Unpacked => {
+                        if cgcx.target_can_use_split_dwarf {
+                            Some(dwo_out.as_path())
+                        } else {
+                            None
+                        }
+                    }
                 };
 
                 with_codegen(tm, llmod, config.no_builtins, |cpm| {
@@ -880,7 +887,7 @@ pub(crate) unsafe fn codegen(
 
     Ok(module.into_compiled_module(
         config.emit_obj != EmitObj::None,
-        cgcx.split_dwarf_kind == SplitDwarfKind::Split,
+        cgcx.target_can_use_split_dwarf && cgcx.split_debuginfo == SplitDebuginfo::Unpacked,
         config.emit_bc,
         &cgcx.output_filenames,
     ))

compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs

@@ -995,10 +995,13 @@ pub fn compile_unit_metadata(
     let flags = "\0";
 
     let out_dir = &tcx.output_filenames(LOCAL_CRATE).out_directory;
-    let split_name = tcx
-        .output_filenames(LOCAL_CRATE)
-        .split_dwarf_filename(tcx.sess.opts.debugging_opts.split_dwarf, Some(codegen_unit_name))
-        .unwrap_or_default();
+    let split_name = if tcx.sess.target_can_use_split_dwarf() {
+        tcx.output_filenames(LOCAL_CRATE)
+            .split_dwarf_filename(tcx.sess.split_debuginfo(), Some(codegen_unit_name))
+    } else {
+        None
+    }
+    .unwrap_or_default();
     let out_dir = out_dir.to_str().unwrap();
     let split_name = split_name.to_str().unwrap();
 

compiler/rustc_codegen_llvm/src/lib.rs

@@ -351,12 +351,7 @@ impl ModuleLlvm {
         unsafe {
             let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names);
             let llmod_raw = back::lto::parse_module(llcx, name, buffer, handler)?;
-
-            let split_dwarf_file = cgcx
-                .output_filenames
-                .split_dwarf_filename(cgcx.split_dwarf_kind, Some(name.to_str().unwrap()));
-            let tm_factory_config = TargetMachineFactoryConfig { split_dwarf_file };
-
+            let tm_factory_config = TargetMachineFactoryConfig::new(&cgcx, name.to_str().unwrap());
             let tm = match (cgcx.tm_factory)(tm_factory_config) {
                 Ok(m) => m,
                 Err(e) => {

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -14,7 +14,7 @@ use rustc_session::utils::NativeLibKind;
 use rustc_session::{filesearch, Session};
 use rustc_span::symbol::Symbol;
 use rustc_target::spec::crt_objects::{CrtObjects, CrtObjectsFallback};
-use rustc_target::spec::{LinkOutputKind, LinkerFlavor, LldFlavor};
+use rustc_target::spec::{LinkOutputKind, LinkerFlavor, LldFlavor, SplitDebuginfo};
 use rustc_target::spec::{PanicStrategy, RelocModel, RelroLevel, Target};
 
 use super::archive::ArchiveBuilder;
@@ -99,9 +99,6 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>(
                         path.as_ref(),
                         target_cpu,
                     );
-                    if sess.opts.debugging_opts.split_dwarf == config::SplitDwarfKind::Split {
-                        link_dwarf_object(sess, &out_filename);
-                    }
                 }
             }
             if sess.opts.json_artifact_notifications {
@@ -828,29 +825,43 @@ fn link_natively<'a, B: ArchiveBuilder<'a>>(
         }
     }
 
-    // On macOS, debuggers need this utility to get run to do some munging of
-    // the symbols. Note, though, that if the object files are being preserved
-    // for their debug information there's no need for us to run dsymutil.
-    if sess.target.is_like_osx
-        && sess.opts.debuginfo != DebugInfo::None
-        && !preserve_objects_for_their_debuginfo(sess)
-    {
-        let prog = Command::new("dsymutil").arg(out_filename).output();
-        match prog {
-            Ok(prog) => {
-                if !prog.status.success() {
-                    let mut output = prog.stderr.clone();
-                    output.extend_from_slice(&prog.stdout);
-                    sess.struct_warn(&format!(
-                        "processing debug info with `dsymutil` failed: {}",
-                        prog.status
-                    ))
-                    .note(&escape_string(&output))
-                    .emit();
+    match sess.split_debuginfo() {
+        // If split debug information is disabled or located in individual files
+        // there's nothing to do here.
+        SplitDebuginfo::Off | SplitDebuginfo::Unpacked => {}
+
+        // If packed split-debuginfo is requested, but the final compilation
+        // doesn't actually have any debug information, then we skip this step.
+        SplitDebuginfo::Packed if sess.opts.debuginfo == DebugInfo::None => {}
+
+        // On macOS the external `dsymutil` tool is used to create the packed
+        // debug information. Note that this will read debug information from
+        // the objects on the filesystem which we'll clean up later.
+        SplitDebuginfo::Packed if sess.target.is_like_osx => {
+            let prog = Command::new("dsymutil").arg(out_filename).output();
+            match prog {
+                Ok(prog) => {
+                    if !prog.status.success() {
+                        let mut output = prog.stderr.clone();
+                        output.extend_from_slice(&prog.stdout);
+                        sess.struct_warn(&format!(
+                            "processing debug info with `dsymutil` failed: {}",
+                            prog.status
+                        ))
+                        .note(&escape_string(&output))
+                        .emit();
+                    }
                 }
+                Err(e) => sess.fatal(&format!("unable to run `dsymutil`: {}", e)),
             }
-            Err(e) => sess.fatal(&format!("unable to run `dsymutil`: {}", e)),
         }
+
+        // On MSVC packed debug information is produced by the linker itself so
+        // there's no need to do anything else here.
+        SplitDebuginfo::Packed if sess.target.is_like_msvc => {}
+
+        // ... and otherwise we're processing a `*.dwp` packed dwarf file.
+        SplitDebuginfo::Packed => link_dwarf_object(sess, &out_filename),
     }
 }
 
@@ -1050,28 +1061,9 @@ fn preserve_objects_for_their_debuginfo(sess: &Session) -> bool {
         return false;
     }
 
-    // Single mode keeps debuginfo in the same object file, but in such a way that it it skipped
-    // by the linker - so it's expected that when codegen units are linked together that this
-    // debuginfo would be lost without keeping around the temps.
-    if sess.opts.debugging_opts.split_dwarf == config::SplitDwarfKind::Single {
-        return true;
-    }
-
-    // If we're on OSX then the equivalent of split dwarf is turned on by
-    // default. The final executable won't actually have any debug information
-    // except it'll have pointers to elsewhere. Historically we've always run
-    // `dsymutil` to "link all the dwarf together" but this is actually sort of
-    // a bummer for incremental compilation! (the whole point of split dwarf is
-    // that you don't do this sort of dwarf link).
-    //
-    // Basically as a result this just means that if we're on OSX and we're
-    // *not* running dsymutil then the object files are the only source of truth
-    // for debug information, so we must preserve them.
-    if sess.target.is_like_osx {
-        return !sess.opts.debugging_opts.run_dsymutil;
-    }
-
-    false
+    // "unpacked" split debuginfo means that we leave object files as the
+    // debuginfo is found in the original object files themselves
+    sess.split_debuginfo() == SplitDebuginfo::Unpacked
 }
 
 pub fn archive_search_paths(sess: &Session) -> Vec<PathBuf> {

compiler/rustc_codegen_ssa/src/back/write.rs

@@ -282,6 +282,20 @@ pub struct TargetMachineFactoryConfig {
     pub split_dwarf_file: Option<PathBuf>,
 }
 
+impl TargetMachineFactoryConfig {
+    pub fn new(
+        cgcx: &CodegenContext<impl WriteBackendMethods>,
+        module_name: &str,
+    ) -> TargetMachineFactoryConfig {
+        let split_dwarf_file = if cgcx.target_can_use_split_dwarf {
+            cgcx.output_filenames.split_dwarf_filename(cgcx.split_debuginfo, Some(module_name))
+        } else {
+            None
+        };
+        TargetMachineFactoryConfig { split_dwarf_file }
+    }
+}
+
 pub type TargetMachineFactoryFn<B> = Arc<
     dyn Fn(TargetMachineFactoryConfig) -> Result<<B as WriteBackendMethods>::TargetMachine, String>
         + Send
@@ -311,10 +325,11 @@ pub struct CodegenContext<B: WriteBackendMethods> {
     pub tm_factory: TargetMachineFactoryFn<B>,
     pub msvc_imps_needed: bool,
     pub is_pe_coff: bool,
+    pub target_can_use_split_dwarf: bool,
     pub target_pointer_width: u32,
     pub target_arch: String,
     pub debuginfo: config::DebugInfo,
-    pub split_dwarf_kind: config::SplitDwarfKind,
+    pub split_debuginfo: rustc_target::spec::SplitDebuginfo,
 
     // Number of cgus excluding the allocator/metadata modules
     pub total_cgus: usize,
@@ -1035,10 +1050,11 @@ fn start_executing_work<B: ExtraBackendMethods>(
         total_cgus,
         msvc_imps_needed: msvc_imps_needed(tcx),
         is_pe_coff: tcx.sess.target.is_like_windows,
+        target_can_use_split_dwarf: tcx.sess.target_can_use_split_dwarf(),
         target_pointer_width: tcx.sess.target.pointer_width,
         target_arch: tcx.sess.target.arch.clone(),
         debuginfo: tcx.sess.opts.debuginfo,
-        split_dwarf_kind: tcx.sess.opts.debugging_opts.split_dwarf,
+        split_debuginfo: tcx.sess.split_debuginfo(),
     };
 
     // This is the "main loop" of parallel work happening for parallel codegen.

compiler/rustc_interface/src/tests.rs

@@ -17,7 +17,7 @@ use rustc_span::edition::{Edition, DEFAULT_EDITION};
 use rustc_span::symbol::sym;
 use rustc_span::SourceFileHashAlgorithm;
 use rustc_target::spec::{CodeModel, LinkerFlavor, MergeFunctions, PanicStrategy};
-use rustc_target::spec::{RelocModel, RelroLevel, TlsModel};
+use rustc_target::spec::{RelocModel, RelroLevel, SplitDebuginfo, TlsModel};
 use std::collections::{BTreeMap, BTreeSet};
 use std::iter::FromIterator;
 use std::path::PathBuf;
@@ -446,6 +446,7 @@ fn test_codegen_options_tracking_hash() {
     tracked!(profile_use, Some(PathBuf::from("abc")));
     tracked!(relocation_model, Some(RelocModel::Pic));
     tracked!(soft_float, true);
+    tracked!(split_debuginfo, Some(SplitDebuginfo::Packed));
     tracked!(target_cpu, Some(String::from("abc")));
     tracked!(target_feature, String::from("all the features, all of them"));
 }
@@ -579,7 +580,6 @@ fn test_debugging_options_tracking_hash() {
     tracked!(relax_elf_relocations, Some(true));
     tracked!(relro_level, Some(RelroLevel::Full));
     tracked!(report_delayed_bugs, true);
-    tracked!(run_dsymutil, false);
     tracked!(sanitizer, SanitizerSet::ADDRESS);
     tracked!(sanitizer_memory_track_origins, 2);
     tracked!(sanitizer_recover, SanitizerSet::ADDRESS);

compiler/rustc_session/src/config.rs

@@ -13,7 +13,7 @@ use rustc_data_structures::impl_stable_hash_via_hash;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 
 use rustc_target::abi::{Align, TargetDataLayout};
-use rustc_target::spec::{Target, TargetTriple};
+use rustc_target::spec::{SplitDebuginfo, Target, TargetTriple};
 
 use crate::parse::CrateConfig;
 use rustc_feature::UnstableFeatures;
@@ -221,23 +221,6 @@ pub enum DebugInfo {
     Full,
 }
 
-/// Some debuginfo requires link-time relocation and some does not. LLVM can partition the debuginfo
-/// into sections depending on whether or not it requires link-time relocation. Split DWARF
-/// provides a mechanism which allows the linker to skip the sections which don't require link-time
-/// relocation - either by putting those sections into DWARF object files, or keeping them in the
-/// object file in such a way that the linker will skip them.
-#[derive(Clone, Copy, Debug, PartialEq, Hash)]
-pub enum SplitDwarfKind {
-    /// Disabled.
-    None,
-    /// Sections which do not require relocation are written into the object file but ignored
-    /// by the linker.
-    Single,
-    /// Sections which do not require relocation are written into a DWARF object (`.dwo`) file,
-    /// which is skipped by the linker by virtue of being a different file.
-    Split,
-}
-
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, PartialOrd, Ord)]
 #[derive(Encodable, Decodable)]
 pub enum OutputType {
@@ -635,30 +618,30 @@ impl OutputFilenames {
     /// mode is being used, which is the logic that this function is intended to encapsulate.
     pub fn split_dwarf_filename(
         &self,
-        split_dwarf_kind: SplitDwarfKind,
+        split_debuginfo_kind: SplitDebuginfo,
         cgu_name: Option<&str>,
     ) -> Option<PathBuf> {
-        self.split_dwarf_path(split_dwarf_kind, cgu_name)
+        self.split_dwarf_path(split_debuginfo_kind, cgu_name)
             .map(|path| path.strip_prefix(&self.out_directory).unwrap_or(&path).to_path_buf())
     }
 
     /// Returns the path for the Split DWARF file - this can differ depending on which Split DWARF
     /// mode is being used, which is the logic that this function is intended to encapsulate.
     pub fn split_dwarf_path(
         &self,
-        split_dwarf_kind: SplitDwarfKind,
+        split_debuginfo_kind: SplitDebuginfo,
         cgu_name: Option<&str>,
     ) -> Option<PathBuf> {
         let obj_out = self.temp_path(OutputType::Object, cgu_name);
         let dwo_out = self.temp_path_dwo(cgu_name);
-        match split_dwarf_kind {
-            SplitDwarfKind::None => None,
+        match split_debuginfo_kind {
+            SplitDebuginfo::Off => None,
             // Single mode doesn't change how DWARF is emitted, but does add Split DWARF attributes
             // (pointing at the path which is being determined here). Use the path to the current
             // object file.
-            SplitDwarfKind::Single => Some(obj_out),
+            SplitDebuginfo::Packed => Some(obj_out),
             // Split mode emits the DWARF into a different file, use that path.
-            SplitDwarfKind::Split => Some(dwo_out),
+            SplitDebuginfo::Unpacked => Some(dwo_out),
         }
     }
 }
@@ -1910,6 +1893,15 @@ pub fn build_session_options(matches: &getopts::Matches) -> Options {
 
     let pretty = parse_pretty(matches, &debugging_opts, error_format);
 
+    if !debugging_opts.unstable_options
+        && !target_triple.triple().contains("apple")
+        && cg.split_debuginfo.is_some()
+    {
+        {
+            early_error(error_format, "`-Csplit-debuginfo` is unstable on this platform");
+        }
+    }
+
     Options {
         crate_types,
         optimize: opt_level,
@@ -2191,7 +2183,7 @@ crate mod dep_tracking {
     use rustc_feature::UnstableFeatures;
     use rustc_span::edition::Edition;
     use rustc_target::spec::{CodeModel, MergeFunctions, PanicStrategy, RelocModel};
-    use rustc_target::spec::{RelroLevel, TargetTriple, TlsModel};
+    use rustc_target::spec::{RelroLevel, SplitDebuginfo, TargetTriple, TlsModel};
     use std::collections::hash_map::DefaultHasher;
     use std::collections::BTreeMap;
     use std::hash::Hash;
@@ -2263,6 +2255,7 @@ crate mod dep_tracking {
     impl_dep_tracking_hash_via_hash!(TargetTriple);
     impl_dep_tracking_hash_via_hash!(Edition);
     impl_dep_tracking_hash_via_hash!(LinkerPluginLto);
+    impl_dep_tracking_hash_via_hash!(Option<SplitDebuginfo>);
     impl_dep_tracking_hash_via_hash!(SwitchWithOptPath);
     impl_dep_tracking_hash_via_hash!(Option<SymbolManglingVersion>);
     impl_dep_tracking_hash_via_hash!(Option<SourceFileHashAlgorithm>);