Unrolled build for rust-lang#123942

Rollup merge of rust-lang#123942 - onur-ozkan:x-vendor, r=Mark-Simulacrum

`x vendor`

This PR implements `x vendor` on bootstrap; enabling dependency vendoring without the need for developers to have `cargo` installed on their system (previously, we suggested running `cargo vendor ...` but now we can accomplish the same task with `x vendor`).

In addition, fixes rust-lang#112391 problem.

Author: rust-timer

src/bootstrap/bootstrap.py

@@ -1035,14 +1035,8 @@ def check_vendored_status(self):
         if self.use_vendored_sources:
             vendor_dir = os.path.join(self.rust_root, 'vendor')
             if not os.path.exists(vendor_dir):
-                sync_dirs = "--sync ./src/tools/cargo/Cargo.toml " \
-                            "--sync ./src/tools/rust-analyzer/Cargo.toml " \
-                            "--sync ./compiler/rustc_codegen_cranelift/Cargo.toml " \
-                            "--sync ./compiler/rustc_codegen_gcc/Cargo.toml " \
-                            "--sync ./src/bootstrap/Cargo.toml "
                 eprint('ERROR: vendoring required, but vendor directory does not exist.')
-                eprint('       Run `cargo vendor {}` to initialize the '
-                      'vendor directory.'.format(sync_dirs))
+                eprint('       Run `x.py vendor` to initialize the vendor directory.')
                 eprint('       Alternatively, use the pre-vendored `rustc-src` dist component.')
                 eprint('       To get a stable/beta/nightly version, download it from: ')
                 eprint('       '

src/bootstrap/src/core/build_steps/mod.rs

@@ -14,3 +14,4 @@ pub(crate) mod synthetic_targets;
 pub(crate) mod test;
 pub(crate) mod tool;
 pub(crate) mod toolstate;
+pub(crate) mod vendor;

src/bootstrap/src/core/build_steps/vendor.rs

@@ -0,0 +1,61 @@
+use crate::core::builder::{Builder, RunConfig, ShouldRun, Step};
+use std::path::PathBuf;
+use std::process::Command;
+
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
+pub(crate) struct Vendor {
+    sync_args: Vec<PathBuf>,
+    versioned_dirs: bool,
+    root_dir: PathBuf,
+}
+
+impl Step for Vendor {
+    type Output = ();
+    const DEFAULT: bool = true;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.alias("placeholder").default_condition(true)
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Vendor {
+            sync_args: run.builder.config.cmd.vendor_sync_args(),
+            versioned_dirs: run.builder.config.cmd.vendor_versioned_dirs(),
+            root_dir: run.builder.src.clone(),
+        });
+    }
+
+    fn run(self, builder: &Builder<'_>) -> Self::Output {
+        let mut cmd = Command::new(&builder.initial_cargo);
+        cmd.arg("vendor");
+
+        if self.versioned_dirs {
+            cmd.arg("--versioned-dirs");
+        }
+
+        // Sync these paths by default.
+        for p in [
+            "src/tools/cargo/Cargo.toml",
+            "src/tools/rust-analyzer/Cargo.toml",
+            "compiler/rustc_codegen_cranelift/Cargo.toml",
+            "compiler/rustc_codegen_gcc/Cargo.toml",
+            "src/bootstrap/Cargo.toml",
+        ] {
+            cmd.arg("--sync").arg(builder.src.join(p));
+        }
+
+        // Also sync explicitly requested paths.
+        for sync_arg in self.sync_args {
+            cmd.arg("--sync").arg(sync_arg);
+        }
+
+        // Will read the libstd Cargo.toml
+        // which uses the unstable `public-dependency` feature.
+        cmd.env("RUSTC_BOOTSTRAP", "1");
+
+        cmd.current_dir(self.root_dir);
+
+        builder.run(&mut cmd);
+    }
+}

src/bootstrap/src/core/builder.rs

@@ -14,8 +14,9 @@ use std::sync::OnceLock;
 use std::time::{Duration, Instant};
 
 use crate::core::build_steps::tool::{self, SourceType};
-use crate::core::build_steps::{check, clean, compile, dist, doc, install, run, setup, test};
-use crate::core::build_steps::{clippy, llvm};
+use crate::core::build_steps::{
+    check, clean, clippy, compile, dist, doc, install, llvm, run, setup, test, vendor,
+};
 use crate::core::config::flags::{Color, Subcommand};
 use crate::core::config::{DryRun, SplitDebuginfo, TargetSelection};
 use crate::prepare_behaviour_dump_dir;
@@ -638,6 +639,7 @@ pub enum Kind {
     Run,
     Setup,
     Suggest,
+    Vendor,
 }
 
 impl Kind {
@@ -658,6 +660,7 @@ impl Kind {
             Kind::Run => "run",
             Kind::Setup => "setup",
             Kind::Suggest => "suggest",
+            Kind::Vendor => "vendor",
         }
     }
 
@@ -921,6 +924,7 @@ impl<'a> Builder<'a> {
             ),
             Kind::Setup => describe!(setup::Profile, setup::Hook, setup::Link, setup::Vscode),
             Kind::Clean => describe!(clean::CleanAll, clean::Rustc, clean::Std),
+            Kind::Vendor => describe!(vendor::Vendor),
             // special-cased in Build::build()
             Kind::Format | Kind::Suggest => vec![],
         }
@@ -993,6 +997,7 @@ impl<'a> Builder<'a> {
                 Kind::Setup,
                 path.as_ref().map_or([].as_slice(), |path| std::slice::from_ref(path)),
             ),
+            Subcommand::Vendor { .. } => (Kind::Vendor, &paths[..]),
         };
 
         Self::new_internal(build, kind, paths.to_owned())

src/bootstrap/src/core/config/config.rs

@@ -2027,7 +2027,8 @@ impl Config {
             | Subcommand::Run { .. }
             | Subcommand::Setup { .. }
             | Subcommand::Format { .. }
-            | Subcommand::Suggest { .. } => flags.stage.unwrap_or(0),
+            | Subcommand::Suggest { .. }
+            | Subcommand::Vendor { .. } => flags.stage.unwrap_or(0),
         };
 
         // CI should always run stage 2 builds, unless it specifically states otherwise
@@ -2054,7 +2055,8 @@ impl Config {
                 | Subcommand::Run { .. }
                 | Subcommand::Setup { .. }
                 | Subcommand::Format { .. }
-                | Subcommand::Suggest { .. } => {}
+                | Subcommand::Suggest { .. }
+                | Subcommand::Vendor { .. } => {}
             }
         }
 

src/bootstrap/src/core/config/flags.rs

@@ -456,6 +456,15 @@ Arguments:
         #[arg(long)]
         run: bool,
     },
+    /// Vendor dependencies
+    Vendor {
+        /// Additional `Cargo.toml` to sync and vendor
+        #[arg(long)]
+        sync: Vec<PathBuf>,
+        /// Always include version in subdir name
+        #[arg(long)]
+        versioned_dirs: bool,
+    },
 }
 
 impl Subcommand {
@@ -476,6 +485,7 @@ impl Subcommand {
             Subcommand::Run { .. } => Kind::Run,
             Subcommand::Setup { .. } => Kind::Setup,
             Subcommand::Suggest { .. } => Kind::Suggest,
+            Subcommand::Vendor { .. } => Kind::Vendor,
         }
     }
 
@@ -581,6 +591,20 @@ impl Subcommand {
             _ => false,
         }
     }
+
+    pub fn vendor_versioned_dirs(&self) -> bool {
+        match *self {
+            Subcommand::Vendor { versioned_dirs, .. } => versioned_dirs,
+            _ => false,
+        }
+    }
+
+    pub fn vendor_sync_args(&self) -> Vec<PathBuf> {
+        match self {
+            Subcommand::Vendor { sync, .. } => sync.clone(),
+            _ => vec![],
+        }
+    }
 }
 
 /// Returns the shell completion for a given shell, if the result differs from the current

src/etc/completions/x.py.fish

@@ -47,6 +47,7 @@ complete -c x.py -n "__fish_use_subcommand" -f -a "install" -d 'Install distribu
 complete -c x.py -n "__fish_use_subcommand" -f -a "run" -d 'Run tools contained in this repository'
 complete -c x.py -n "__fish_use_subcommand" -f -a "setup" -d 'Set up the environment for development'
 complete -c x.py -n "__fish_use_subcommand" -f -a "suggest" -d 'Suggest a subset of tests to run, based on modified files'
+complete -c x.py -n "__fish_use_subcommand" -f -a "vendor" -d 'Vendor dependencies'
 complete -c x.py -n "__fish_seen_subcommand_from build" -l config -d 'TOML configuration file for build' -r -F
 complete -c x.py -n "__fish_seen_subcommand_from build" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)"
 complete -c x.py -n "__fish_seen_subcommand_from build" -l build -d 'build target of the stage0 compiler' -r -f
@@ -590,3 +591,39 @@ complete -c x.py -n "__fish_seen_subcommand_from suggest" -l llvm-profile-genera
 complete -c x.py -n "__fish_seen_subcommand_from suggest" -l enable-bolt-settings -d 'Enable BOLT link flags'
 complete -c x.py -n "__fish_seen_subcommand_from suggest" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
 complete -c x.py -n "__fish_seen_subcommand_from suggest" -s h -l help -d 'Print help (see more with \'--help\')'
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l sync -d 'Additional `Cargo.toml` to sync and vendor' -r -F
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l config -d 'TOML configuration file for build' -r -F
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)"
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l build -d 'build target of the stage0 compiler' -r -f
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l host -d 'host targets to build' -r -f
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l target -d 'target targets to build' -r -f
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l exclude -d 'build paths to exclude' -r -F
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l skip -d 'build paths to skip' -r -F
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l rustc-error-format -r -f
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l on-fail -d 'command to run on failure' -r -f -a "(__fish_complete_command)"
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l stage -d 'stage to build (indicates compiler to use/test, e.g., stage 0 uses the bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)' -r -f
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l keep-stage -d 'stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l keep-stage-std -d 'stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l src -d 'path to the root of the rust checkout' -r -f -a "(__fish_complete_directories)"
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -s j -l jobs -d 'number of jobs to run in parallel' -r -f
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny	'',warn	'',default	''}"
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l error-format -d 'rustc error format' -r -f
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always	'',never	'',auto	''}"
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true	'',false	''}"
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l rust-profile-generate -d 'generate PGO profile with rustc build' -r -F
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l rust-profile-use -d 'use PGO profile for rustc build' -r -F
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l llvm-profile-use -d 'use PGO profile for LLVM build' -r -F
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l reproducible-artifact -d 'Additional reproducible artifacts that should be added to the reproducible artifacts archive' -r
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l set -d 'override options in config.toml' -r -f
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l versioned-dirs -d 'Always include version in subdir name'
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -s v -l verbose -d 'use verbose output (-vv for very verbose)'
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -s i -l incremental -d 'use incremental compilation'
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l include-default-paths -d 'include default paths in addition to the provided ones'
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l dry-run -d 'dry run; don\'t build anything'
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l dump-bootstrap-shims -d 'Indicates whether to dump the work done from bootstrap shims'
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l json-output -d 'use message-format=json'
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l bypass-bootstrap-lock -d 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)'
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l enable-bolt-settings -d 'Enable BOLT link flags'
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
+complete -c x.py -n "__fish_seen_subcommand_from vendor" -s h -l help -d 'Print help (see more with \'--help\')'

src/etc/completions/x.py.ps1

@@ -74,6 +74,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('run', 'run', [CompletionResultType]::ParameterValue, 'Run tools contained in this repository')
             [CompletionResult]::new('setup', 'setup', [CompletionResultType]::ParameterValue, 'Set up the environment for development')
             [CompletionResult]::new('suggest', 'suggest', [CompletionResultType]::ParameterValue, 'Suggest a subset of tests to run, based on modified files')
+            [CompletionResult]::new('vendor', 'vendor', [CompletionResultType]::ParameterValue, 'Vendor dependencies')
             break
         }
         'x.py;build' {
@@ -724,6 +725,49 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             break
         }
+        'x.py;vendor' {
+            [CompletionResult]::new('--sync', 'sync', [CompletionResultType]::ParameterName, 'Additional `Cargo.toml` to sync and vendor')
+            [CompletionResult]::new('--config', 'config', [CompletionResultType]::ParameterName, 'TOML configuration file for build')
+            [CompletionResult]::new('--build-dir', 'build-dir', [CompletionResultType]::ParameterName, 'Build directory, overrides `build.build-dir` in `config.toml`')
+            [CompletionResult]::new('--build', 'build', [CompletionResultType]::ParameterName, 'build target of the stage0 compiler')
+            [CompletionResult]::new('--host', 'host', [CompletionResultType]::ParameterName, 'host targets to build')
+            [CompletionResult]::new('--target', 'target', [CompletionResultType]::ParameterName, 'target targets to build')
+            [CompletionResult]::new('--exclude', 'exclude', [CompletionResultType]::ParameterName, 'build paths to exclude')
+            [CompletionResult]::new('--skip', 'skip', [CompletionResultType]::ParameterName, 'build paths to skip')
+            [CompletionResult]::new('--rustc-error-format', 'rustc-error-format', [CompletionResultType]::ParameterName, 'rustc-error-format')
+            [CompletionResult]::new('--on-fail', 'on-fail', [CompletionResultType]::ParameterName, 'command to run on failure')
+            [CompletionResult]::new('--stage', 'stage', [CompletionResultType]::ParameterName, 'stage to build (indicates compiler to use/test, e.g., stage 0 uses the bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)')
+            [CompletionResult]::new('--keep-stage', 'keep-stage', [CompletionResultType]::ParameterName, 'stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)')
+            [CompletionResult]::new('--keep-stage-std', 'keep-stage-std', [CompletionResultType]::ParameterName, 'stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)')
+            [CompletionResult]::new('--src', 'src', [CompletionResultType]::ParameterName, 'path to the root of the rust checkout')
+            [CompletionResult]::new('-j', 'j', [CompletionResultType]::ParameterName, 'number of jobs to run in parallel')
+            [CompletionResult]::new('--jobs', 'jobs', [CompletionResultType]::ParameterName, 'number of jobs to run in parallel')
+            [CompletionResult]::new('--warnings', 'warnings', [CompletionResultType]::ParameterName, 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour')
+            [CompletionResult]::new('--error-format', 'error-format', [CompletionResultType]::ParameterName, 'rustc error format')
+            [CompletionResult]::new('--color', 'color', [CompletionResultType]::ParameterName, 'whether to use color in cargo and rustc output')
+            [CompletionResult]::new('--llvm-skip-rebuild', 'llvm-skip-rebuild', [CompletionResultType]::ParameterName, 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml')
+            [CompletionResult]::new('--rust-profile-generate', 'rust-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with rustc build')
+            [CompletionResult]::new('--rust-profile-use', 'rust-profile-use', [CompletionResultType]::ParameterName, 'use PGO profile for rustc build')
+            [CompletionResult]::new('--llvm-profile-use', 'llvm-profile-use', [CompletionResultType]::ParameterName, 'use PGO profile for LLVM build')
+            [CompletionResult]::new('--reproducible-artifact', 'reproducible-artifact', [CompletionResultType]::ParameterName, 'Additional reproducible artifacts that should be added to the reproducible artifacts archive')
+            [CompletionResult]::new('--set', 'set', [CompletionResultType]::ParameterName, 'override options in config.toml')
+            [CompletionResult]::new('--versioned-dirs', 'versioned-dirs', [CompletionResultType]::ParameterName, 'Always include version in subdir name')
+            [CompletionResult]::new('-v', 'v', [CompletionResultType]::ParameterName, 'use verbose output (-vv for very verbose)')
+            [CompletionResult]::new('--verbose', 'verbose', [CompletionResultType]::ParameterName, 'use verbose output (-vv for very verbose)')
+            [CompletionResult]::new('-i', 'i', [CompletionResultType]::ParameterName, 'use incremental compilation')
+            [CompletionResult]::new('--incremental', 'incremental', [CompletionResultType]::ParameterName, 'use incremental compilation')
+            [CompletionResult]::new('--include-default-paths', 'include-default-paths', [CompletionResultType]::ParameterName, 'include default paths in addition to the provided ones')
+            [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything')
+            [CompletionResult]::new('--dump-bootstrap-shims', 'dump-bootstrap-shims', [CompletionResultType]::ParameterName, 'Indicates whether to dump the work done from bootstrap shims')
+            [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
+            [CompletionResult]::new('--bypass-bootstrap-lock', 'bypass-bootstrap-lock', [CompletionResultType]::ParameterName, 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)')
+            [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
+            [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
+            [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
+            [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
+            [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
+            break
+        }
     })
 
     $completions.Where{ $_.CompletionText -like "$wordToComplete*" } |