Auto merge of rust-lang#102565 - jyn514:refactor-build-manifest, r=Mark-Simulacrum

Refactor build-manifest to minimize the number of changes needed to add a new component

- Add all components to `PkgType`
- Automate functionality wherever possible, so functions often don't have to be manually edited
- Where that's not possible, use exhaustive matches on `PkgType` instead of adding individual strings.
- Add documentation for how to add a component. Improve the existing documentation for how to test changes.

I tested locally that this generates an identical manifest before and after my change, as follows:
```sh
git checkout d44e142
cargo +nightly run --manifest-path src/tools/build-manifest/Cargo.toml build/dist build/manifest-before 1970-01-01 http://example.com nightly
git checkout refactor-build-manifest
cargo +nightly run --manifest-path src/tools/build-manifest/Cargo.toml build/dist build/manifest-before 1970-01-01 http://example.com nightly
sort -u build/manifest-before/channel-rust-nightly.toml | diff - <(sort -u build/manifest-after/channel-rust-nightly.toml)
```
I then verified by hand that the differences before sorting are inconsequential (mostly targets being slightly reordered).

The only change in behavior is that `llvm-tools` is now properly renamed to `llvm-tools-preview`:
```
; sort -u build/manifest-before/channel-rust-nightly.toml | diff - <(sort -u build/manifest-after/channel-rust-nightly.toml)
784a785
> [renames.llvm-tools]
894a896
> to = "llvm-tools-preview"
```

This is based on rust-lang#102241 and should not be merged before.

Author: bors

src/tools/build-manifest/README.md

@@ -1,27 +1,33 @@
 # build-manifest
 
-This tool generates the manifests uploaded to static.rust-lang.org and used by
-rustup. The tool is invoked by the bootstrap tool.
+This tool generates the manifests uploaded to static.rust-lang.org and used by rustup.
+You can see a full list of all manifests at <https://static.rust-lang.org/manifests.txt>.
+This listing is updated by <https://github.com/rust-lang/generate-manifest-list> every 7 days.
+
+This gets called by `promote-release` <https://github.com/rust-lang/promote-release> via `x.py dist hash-and-sign`.
+
+## Adding a new component
+
+1. Add a new `Step` to `dist.rs`. This should usually be named after the filename of the uploaded tarball. See https://github.com/rust-lang/rust/pull/101799/files#diff-2c56335faa24486df09ba392d8900c57e2fac4633e1f7038469bcf9ed3feb871 for an example.
+    a. If appropriate, call `tarball.is_preview(true)` for the component.
+2. Add a new `PkgType` to build-manifest. Fix all the compile errors as appropriate.
 
 ## Testing changes locally
 
 In order to test the changes locally you need to have a valid dist directory
 available locally. If you don't want to build all the compiler, you can easily
 create one from the nightly artifacts with:
 
-```
-#!/bin/bash
-for cmpn in rust rustc rust-std rust-docs cargo; do
-    wget https://static.rust-lang.org/dist/${cmpn}-nightly-x86_64-unknown-linux-gnu.tar.gz
+```sh
+for component in rust rustc rust-std rust-docs cargo; do
+    wget -P build/dist https://static.rust-lang.org/dist/${component}-nightly-x86_64-unknown-linux-gnu.tar.gz
 done
 ```
 
-Then, you can generate the manifest and all the packages from `path/to/dist` to
-`path/to/output` with:
+Then, you can generate the manifest and all the packages from `build/dist` to
+`build/manifest` with:
 
+```sh
+mkdir -p build/manifest
+cargo +nightly run --release -p build-manifest build/dist build/manifest 1970-01-01 http://example.com nightly
 ```
-$ cargo +nightly run path/to/dist path/to/output 1970-01-01 http://example.com CHANNEL
-```
-
-Remember to replace `CHANNEL` with the channel you produced dist artifacts of
-and `VERSION` with the current Rust version.

src/tools/build-manifest/src/main.rs

@@ -1,8 +1,4 @@
-//! Build a dist manifest, hash and sign everything.
-//! This gets called by `promote-release`
-//! (https://github.com/rust-lang/rust-central-station/tree/master/promote-release)
-//! via `x.py dist hash-and-sign`; the cmdline arguments are set up
-//! by rustbuild (in `src/bootstrap/dist.rs`).
+#![doc = include_str!("../README.md")]
 
 mod checksum;
 mod manifest;
@@ -184,7 +180,7 @@ static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin"
 
 static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"];
 
-static NIGHTLY_ONLY_COMPONENTS: &[&str] = &["miri-preview", "rust-docs-json-preview"];
+static NIGHTLY_ONLY_COMPONENTS: &[PkgType] = &[PkgType::Miri, PkgType::JsonDocs];
 
 macro_rules! t {
     ($e:expr) => {
@@ -287,28 +283,9 @@ impl Builder {
     }
 
     fn add_packages_to(&mut self, manifest: &mut Manifest) {
-        macro_rules! package {
-            ($name:expr, $targets:expr) => {
-                self.package($name, &mut manifest.pkg, $targets, &[])
-            };
+        for pkg in PkgType::all() {
+            self.package(pkg, &mut manifest.pkg);
         }
-        package!("rustc", HOSTS);
-        package!("rustc-dev", HOSTS);
-        package!("reproducible-artifacts", HOSTS);
-        package!("rustc-docs", HOSTS);
-        package!("cargo", HOSTS);
-        package!("rust-mingw", MINGW);
-        package!("rust-std", TARGETS);
-        self.package("rust-docs", &mut manifest.pkg, HOSTS, DOCS_FALLBACK);
-        self.package("rust-docs-json-preview", &mut manifest.pkg, HOSTS, DOCS_FALLBACK);
-        package!("rust-src", &["*"]);
-        package!("rls-preview", HOSTS);
-        package!("rust-analyzer-preview", HOSTS);
-        package!("clippy-preview", HOSTS);
-        package!("miri-preview", HOSTS);
-        package!("rustfmt-preview", HOSTS);
-        package!("rust-analysis", TARGETS);
-        package!("llvm-tools-preview", TARGETS);
     }
 
     fn add_artifacts_to(&mut self, manifest: &mut Manifest) {
@@ -333,44 +310,28 @@ impl Builder {
     }
 
     fn add_profiles_to(&mut self, manifest: &mut Manifest) {
-        let mut profile = |name, pkgs| self.profile(name, &mut manifest.profiles, pkgs);
-        profile("minimal", &["rustc", "cargo", "rust-std", "rust-mingw"]);
-        profile(
-            "default",
-            &[
-                "rustc",
-                "cargo",
-                "rust-std",
-                "rust-mingw",
-                "rust-docs",
-                "rustfmt-preview",
-                "clippy-preview",
-            ],
-        );
-        profile(
-            "complete",
-            &[
-                "rustc",
-                "cargo",
-                "rust-std",
-                "rust-mingw",
-                "rust-docs",
-                "rustfmt-preview",
-                "clippy-preview",
-                "rls-preview",
-                "rust-analyzer-preview",
-                "rust-src",
-                "llvm-tools-preview",
-                "rust-analysis",
-                "miri-preview",
-            ],
-        );
+        use PkgType::*;
+
+        let mut profile = |name, pkgs: &_| self.profile(name, &mut manifest.profiles, pkgs);
+
+        // Use a Vec here to make sure we don't exclude any components in an earlier profile.
+        let minimal = vec![Rustc, Cargo, RustStd, RustMingw];
+        profile("minimal", &minimal);
+
+        let mut default = minimal;
+        default.extend([HtmlDocs, Rustfmt, Clippy]);
+        profile("default", &default);
+
+        // NOTE: this profile is effectively deprecated; do not add new components to it.
+        let mut complete = default;
+        complete.extend([Rls, RustAnalyzer, RustSrc, LlvmTools, RustAnalysis, Miri]);
+        profile("complete", &complete);
 
         // The compiler libraries are not stable for end users, and they're also huge, so we only
         // `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible
         // for users to install the additional component manually, if needed.
         if self.versions.channel() == "nightly" {
-            self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]);
+            self.extend_profile("complete", &mut manifest.profiles, &[RustcDev]);
             // Do not include the rustc-docs component for now, as it causes
             // conflicts with the rust-docs component when installed. See
             // #75833.
@@ -382,12 +343,11 @@ impl Builder {
         let mut rename = |from: &str, to: &str| {
             manifest.renames.insert(from.to_owned(), Rename { to: to.to_owned() })
         };
-        rename("rls", "rls-preview");
-        rename("rustfmt", "rustfmt-preview");
-        rename("clippy", "clippy-preview");
-        rename("miri", "miri-preview");
-        rename("rust-docs-json", "rust-docs-json-preview");
-        rename("rust-analyzer", "rust-analyzer-preview");
+        for pkg in PkgType::all() {
+            if pkg.is_preview() {
+                rename(pkg.tarball_component_name(), &pkg.manifest_component_name());
+            }
+        }
     }
 
     fn rust_package(&mut self, manifest: &Manifest) -> Package {
@@ -419,42 +379,52 @@ impl Builder {
         let mut components = Vec::new();
         let mut extensions = Vec::new();
 
-        let host_component = |pkg| Component::from_str(pkg, host);
-
-        // rustc/rust-std/cargo/docs are all required,
-        // and so is rust-mingw if it's available for the target.
-        components.extend(vec![
-            host_component("rustc"),
-            host_component("rust-std"),
-            host_component("cargo"),
-            host_component("rust-docs"),
-        ]);
-        if host.contains("pc-windows-gnu") {
-            components.push(host_component("rust-mingw"));
-        }
+        let host_component = |pkg: &_| Component::from_pkg(pkg, host);
 
-        // Tools are always present in the manifest,
-        // but might be marked as unavailable if they weren't built.
-        extensions.extend(vec![
-            host_component("clippy-preview"),
-            host_component("miri-preview"),
-            host_component("rls-preview"),
-            host_component("rust-analyzer-preview"),
-            host_component("rustfmt-preview"),
-            host_component("llvm-tools-preview"),
-            host_component("rust-analysis"),
-            host_component("rust-docs-json-preview"),
-        ]);
-
-        extensions.extend(
-            TARGETS
-                .iter()
-                .filter(|&&target| target != host)
-                .map(|target| Component::from_str("rust-std", target)),
-        );
-        extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-dev", target)));
-        extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-docs", target)));
-        extensions.push(Component::from_str("rust-src", "*"));
+        for pkg in PkgType::all() {
+            match pkg {
+                // rustc/rust-std/cargo/docs are all required
+                PkgType::Rustc | PkgType::Cargo | PkgType::HtmlDocs => {
+                    components.push(host_component(pkg));
+                }
+                PkgType::RustStd => {
+                    components.push(host_component(pkg));
+                    extensions.extend(
+                        TARGETS
+                            .iter()
+                            .filter(|&&target| target != host)
+                            .map(|target| Component::from_pkg(pkg, target)),
+                    );
+                }
+                // so is rust-mingw if it's available for the target
+                PkgType::RustMingw => {
+                    if host.contains("pc-windows-gnu") {
+                        components.push(host_component(pkg));
+                    }
+                }
+                // Tools are always present in the manifest,
+                // but might be marked as unavailable if they weren't built.
+                PkgType::Clippy
+                | PkgType::Miri
+                | PkgType::Rls
+                | PkgType::RustAnalyzer
+                | PkgType::Rustfmt
+                | PkgType::LlvmTools
+                | PkgType::RustAnalysis
+                | PkgType::JsonDocs => {
+                    extensions.push(host_component(pkg));
+                }
+                PkgType::RustcDev | PkgType::RustcDocs => {
+                    extensions.extend(HOSTS.iter().map(|target| Component::from_pkg(pkg, target)));
+                }
+                PkgType::RustSrc => {
+                    extensions.push(Component::from_pkg(pkg, "*"));
+                }
+                PkgType::Rust => {}
+                // NOTE: this is intentional, these artifacts aren't intended to be used with rustup
+                PkgType::ReproducibleArtifacts => {}
+            }
+        }
 
         // If the components/extensions don't actually exist for this
         // particular host/target combination then nix it entirely from our
@@ -481,43 +451,44 @@ impl Builder {
         &mut self,
         profile_name: &str,
         dst: &mut BTreeMap<String, Vec<String>>,
-        pkgs: &[&str],
+        pkgs: &[PkgType],
     ) {
-        dst.insert(profile_name.to_owned(), pkgs.iter().map(|s| (*s).to_owned()).collect());
+        dst.insert(
+            profile_name.to_owned(),
+            pkgs.iter().map(|s| s.manifest_component_name()).collect(),
+        );
     }
 
     fn extend_profile(
         &mut self,
         profile_name: &str,
         dst: &mut BTreeMap<String, Vec<String>>,
-        pkgs: &[&str],
+        pkgs: &[PkgType],
     ) {
         dst.get_mut(profile_name)
             .expect("existing profile")
-            .extend(pkgs.iter().map(|s| (*s).to_owned()));
+            .extend(pkgs.iter().map(|s| s.manifest_component_name()));
     }
 
-    fn package(
-        &mut self,
-        pkgname: &str,
-        dst: &mut BTreeMap<String, Package>,
-        targets: &[&str],
-        fallback: &[(&str, &str)],
-    ) {
-        let version_info = self
-            .versions
-            .version(&PkgType::from_component(pkgname))
-            .expect("failed to load package version");
+    fn package(&mut self, pkg: &PkgType, dst: &mut BTreeMap<String, Package>) {
+        if *pkg == PkgType::Rust {
+            // This is handled specially by `rust_package` later.
+            // Order is important, so don't call `rust_package` here.
+            return;
+        }
+
+        let fallback = if pkg.use_docs_fallback() { DOCS_FALLBACK } else { &[] };
+        let version_info = self.versions.version(&pkg).expect("failed to load package version");
         let mut is_present = version_info.present;
 
         // Never ship nightly-only components for other trains.
-        if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) {
+        if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkg) {
             is_present = false; // Pretend the component is entirely missing.
         }
 
         macro_rules! tarball_name {
             ($target_name:expr) => {
-                self.versions.tarball_name(&PkgType::from_component(pkgname), $target_name).unwrap()
+                self.versions.tarball_name(pkg, $target_name).unwrap()
             };
         }
         let mut target_from_compressed_tar = |target_name| {
@@ -546,7 +517,8 @@ impl Builder {
             Target::unavailable()
         };
 
-        let targets = targets
+        let targets = pkg
+            .targets()
             .iter()
             .map(|name| {
                 let target = if is_present {
@@ -561,7 +533,7 @@ impl Builder {
             .collect();
 
         dst.insert(
-            pkgname.to_string(),
+            pkg.manifest_component_name(),
             Package {
                 version: version_info.version.unwrap_or_default(),
                 git_commit_hash: version_info.git_commit,

src/tools/build-manifest/src/manifest.rs

@@ -1,3 +1,4 @@
+use crate::versions::PkgType;
 use crate::Builder;
 use serde::{Serialize, Serializer};
 use std::collections::BTreeMap;
@@ -116,8 +117,8 @@ pub(crate) struct Component {
 }
 
 impl Component {
-    pub(crate) fn from_str(pkg: &str, target: &str) -> Self {
-        Self { pkg: pkg.to_string(), target: target.to_string() }
+    pub(crate) fn from_pkg(pkg: &PkgType, target: &str) -> Self {
+        Self { pkg: pkg.manifest_component_name(), target: target.to_string() }
     }
 }