Auto merge of rust-lang#117574 - onur-ozkan:fix-compiler-crate-linking, r=<try>

re-enable cranelift on CI

First commit addresses the linking issue with compiler crates. Second one ensures that compiler crates are linked correctly, allowing us to detect these hard-to-catch bugs on CI. The remaining three commits cherry-picked from rust-lang#117328 to re-enable the Cranelift backend on CI.

More info: rust-lang#117430

cc `@bjorn3` `@RalfJung`

Author: bors

.github/workflows/ci.yml

@@ -543,7 +543,6 @@ jobs:
   try:
     name: "try - ${{ matrix.name }}"
     env:
-      DIST_TRY_BUILD: 1
       CI_JOB_NAME: "${{ matrix.name }}"
       CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
       HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"

src/bootstrap/src/core/build_steps/compile.rs

@@ -413,11 +413,6 @@ pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, car
 
     let mut features = String::new();
 
-    // Cranelift doesn't support `asm`.
-    if stage != 0 && builder.config.default_codegen_backend().unwrap_or_default() == "cranelift" {
-        features += " compiler-builtins-no-asm";
-    }
-
     if builder.no_std(target) == Some(true) {
         features += " compiler-builtins-mem";
         if !target.starts_with("bpf") {

src/bootstrap/src/core/build_steps/dist.rs

@@ -1298,13 +1298,21 @@ impl Step for CodegenBackend {
     }
 
     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+        if builder.config.dry_run() {
+            return None;
+        }
+
         // This prevents rustc_codegen_cranelift from being built for "dist"
         // or "install" on the stable/beta channels. It is not yet stable and
         // should not be included.
         if !builder.build.unstable_features() {
             return None;
         }
 
+        if !builder.config.rust_codegen_backends.contains(&self.backend) {
+            return None;
+        }
+
         if self.backend == "cranelift" {
             if !target_supports_cranelift_backend(self.compiler.host) {
                 builder.info("target not supported by rustc_codegen_cranelift. skipping");
@@ -1343,12 +1351,15 @@ impl Step for CodegenBackend {
         let backends_dst = PathBuf::from("lib").join(&backends_rel);
 
         let backend_name = format!("rustc_codegen_{}", backend);
+        let mut found_backend = false;
         for backend in fs::read_dir(&backends_src).unwrap() {
             let file_name = backend.unwrap().file_name();
             if file_name.to_str().unwrap().contains(&backend_name) {
                 tarball.add_file(backends_src.join(file_name), &backends_dst, 0o644);
+                found_backend = true;
             }
         }
+        assert!(found_backend);
 
         Some(tarball.generate())
     }

src/bootstrap/src/core/build_steps/doc.rs

@@ -685,18 +685,7 @@ impl Step for Rustc {
             target,
         );
 
-        // This uses a shared directory so that librustdoc documentation gets
-        // correctly built and merged with the rustc documentation. This is
-        // needed because rustdoc is built in a different directory from
-        // rustc. rustdoc needs to be able to see everything, for example when
-        // merging the search index, or generating local (relative) links.
         let out_dir = builder.stage_out(compiler, Mode::Rustc).join(target.triple).join("doc");
-        t!(fs::create_dir_all(out_dir.parent().unwrap()));
-        symlink_dir_force(&builder.config, &out, &out_dir);
-        // Cargo puts proc macros in `target/doc` even if you pass `--target`
-        // explicitly (https://github.com/rust-lang/cargo/issues/7677).
-        let proc_macro_out_dir = builder.stage_out(compiler, Mode::Rustc).join("doc");
-        symlink_dir_force(&builder.config, &out, &proc_macro_out_dir);
 
         // Build cargo command.
         let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "doc");
@@ -736,8 +725,29 @@ impl Step for Rustc {
             }
         }
 
+        // This uses a shared directory so that librustdoc documentation gets
+        // correctly built and merged with the rustc documentation.
+        //
+        // This is needed because rustdoc is built in a different directory from
+        // rustc. rustdoc needs to be able to see everything, for example when
+        // merging the search index, or generating local (relative) links.
+        symlink_dir_force(&builder.config, &out, &out_dir);
+        // Cargo puts proc macros in `target/doc` even if you pass `--target`
+        // explicitly (https://github.com/rust-lang/cargo/issues/7677).
+        let proc_macro_out_dir = builder.stage_out(compiler, Mode::Rustc).join("doc");
+        symlink_dir_force(&builder.config, &out, &proc_macro_out_dir);
+
         builder.run(&mut cargo.into());
 
+        if !builder.config.dry_run() {
+            // Sanity check on linked compiler crates
+            for krate in &*self.crates {
+                let dir_name = krate.replace("-", "_");
+                // Making sure the directory exists and is not empty.
+                assert!(out.join(&*dir_name).read_dir().unwrap().next().is_some());
+            }
+        }
+
         if builder.paths.iter().any(|path| path.ends_with("compiler")) {
             // For `x.py doc compiler --open`, open `rustc_middle` by default.
             let index = out.join("rustc_middle").join("index.html");
@@ -756,10 +766,10 @@ macro_rules! tool_doc {
         $should_run: literal,
         $path: literal,
         $(rustc_tool = $rustc_tool:literal, )?
-        $(in_tree = $in_tree:literal, )?
-        [$($extra_arg: literal),+ $(,)?]
-        $(,)?
-    ) => {
+        $(in_tree = $in_tree:literal ,)?
+        $(is_library = $is_library:expr,)?
+        $(crates = $crates:expr)?
+       ) => {
         #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
         pub struct $tool {
             target: TargetSelection,
@@ -818,9 +828,15 @@ macro_rules! tool_doc {
                     // Cargo uses a different directory for proc macros.
                     builder.stage_out(compiler, Mode::ToolRustc).join("doc"),
                 ];
+
                 for out_dir in out_dirs {
                     t!(fs::create_dir_all(&out_dir));
                     symlink_dir_force(&builder.config, &out, &out_dir);
+
+                    $(for krate in $crates {
+                        let dir_name = krate.replace("-", "_");
+                        t!(fs::create_dir_all(&out_dir.join(&*dir_name)));
+                    })?
                 }
 
                 // Build cargo command.
@@ -839,9 +855,13 @@ macro_rules! tool_doc {
                 // Only include compiler crates, no dependencies of those, such as `libc`.
                 cargo.arg("--no-deps");
 
-                $(
-                    cargo.arg($extra_arg);
-                )+
+                if false $(|| $is_library)? {
+                    cargo.arg("--lib");
+                }
+
+                $(for krate in $crates {
+                    cargo.arg("-p").arg(krate);
+                })?
 
                 cargo.rustdocflag("--document-private-items");
                 // Since we always pass --document-private-items, there's no need to warn about linking to private items.
@@ -853,60 +873,56 @@ macro_rules! tool_doc {
 
                 let _guard = builder.msg_doc(compiler, stringify!($tool).to_lowercase(), target);
                 builder.run(&mut cargo.into());
+
+                if !builder.config.dry_run() {
+                    // Sanity check on linked doc directories
+                    $(for krate in $crates {
+                        let dir_name = krate.replace("-", "_");
+                        // Making sure the directory exists and is not empty.
+                        assert!(out.join(&*dir_name).read_dir().unwrap().next().is_some());
+                    })?
+                }
             }
         }
     }
 }
 
-tool_doc!(
-    Rustdoc,
-    "rustdoc-tool",
-    "src/tools/rustdoc",
-    ["-p", "rustdoc", "-p", "rustdoc-json-types"]
-);
+tool_doc!(Rustdoc, "rustdoc-tool", "src/tools/rustdoc", crates = ["rustdoc", "rustdoc-json-types"]);
 tool_doc!(
     Rustfmt,
     "rustfmt-nightly",
     "src/tools/rustfmt",
-    ["-p", "rustfmt-nightly", "-p", "rustfmt-config_proc_macro"],
+    crates = ["rustfmt-nightly", "rustfmt-config_proc_macro"]
 );
-tool_doc!(Clippy, "clippy", "src/tools/clippy", ["-p", "clippy_utils"]);
-tool_doc!(Miri, "miri", "src/tools/miri", ["-p", "miri"]);
+tool_doc!(Clippy, "clippy", "src/tools/clippy", crates = ["clippy_utils"]);
+tool_doc!(Miri, "miri", "src/tools/miri", crates = ["miri"]);
 tool_doc!(
     Cargo,
     "cargo",
     "src/tools/cargo",
     rustc_tool = false,
     in_tree = false,
-    [
-        "-p",
+    crates = [
         "cargo",
-        "-p",
         "cargo-platform",
-        "-p",
         "cargo-util",
-        "-p",
         "crates-io",
-        "-p",
         "cargo-test-macro",
-        "-p",
         "cargo-test-support",
-        "-p",
         "cargo-credential",
-        "-p",
         "mdman",
         // FIXME: this trips a license check in tidy.
-        // "-p",
         // "resolver-tests",
     ]
 );
-tool_doc!(Tidy, "tidy", "src/tools/tidy", rustc_tool = false, ["-p", "tidy"]);
+tool_doc!(Tidy, "tidy", "src/tools/tidy", rustc_tool = false, crates = ["tidy"]);
 tool_doc!(
     Bootstrap,
     "bootstrap",
     "src/bootstrap",
     rustc_tool = false,
-    ["--lib", "-p", "bootstrap"]
+    is_library = true,
+    crates = ["bootstrap"]
 );
 
 #[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)]

src/ci/docker/run.sh

@@ -302,6 +302,7 @@ docker \
   --env DIST_TRY_BUILD \
   --env PR_CI_JOB \
   --env OBJDIR_ON_HOST="$objdir" \
+  --env CODEGEN_BACKENDS \
   --init \
   --rm \
   rust-ci \

src/ci/github-actions/ci.yml

@@ -700,7 +700,6 @@ jobs:
     <<: *base-ci-job
     name: try - ${{ matrix.name }}
     env:
-      DIST_TRY_BUILD: 1
       <<: [*shared-ci-variables, *prod-variables]
     if: github.event_name == 'push' && (((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust') || ((github.ref == 'refs/heads/automation/bors/try') && github.repository == 'rust-lang/rust'))
     strategy: