Rollup merge of #112281 - jyn514:test-bootstrap-py, r=albertlarsan68

Test the cargo args generated by bootstrap.py

I recommend reviewing this commit-by-commit using the instructions in https://rustc-dev-guide.rust-lang.org/git.html#moving-large-sections-of-code.

- Test cargo arguments passed by bootstrap.py

  This moves a lot of code around, but the logic itself is not too terribly complicated.

    - Move almost all logic in `def bootstrap` to the `RustBuild` class, to avoid mixing setting configuration with running commands
    - Update various doctests to the new (more complete) RustBuild config. In particular, don't pretend that `bin_root` supports `build` being unset.
    - Change `parse_args` not to use a global, to allow testing it
    - Set BUILD_DIR appropriately so bootstrap.py doesn't panic because cargo isn't found

- Allow passing arguments to `bootstrap_test.py`

    Previous, it used the built-in test runner, which doesn't support options unless they're manually passed in the script.

- Fix progress messages for configure in bootstrap_test.py

    Before it would unconditionally print `configure-args = []`.

r? `@albertlarsan68` cc #112089 #111979 (comment)

Author: TaKO8Ki

src/bootstrap/bootstrap.py

@@ -458,23 +458,51 @@ def unpack_component(download_info):
         verbose=download_info.verbose,
     )
 
-class RustBuild(object):
-    """Provide all the methods required to build Rust"""
+class FakeArgs:
+    """Used for unit tests to avoid updating all call sites"""
     def __init__(self):
-        self.checksums_sha256 = {}
-        self.stage0_compiler = None
-        self.download_url = ''
         self.build = ''
         self.build_dir = ''
         self.clean = False
-        self.config_toml = ''
-        self.rust_root = ''
-        self.use_locked_deps = False
-        self.use_vendored_sources = False
         self.verbose = False
+        self.json_output = False
+        self.color = 'auto'
+        self.warnings = 'default'
+
+class RustBuild(object):
+    """Provide all the methods required to build Rust"""
+    def __init__(self, config_toml="", args=FakeArgs()):
         self.git_version = None
         self.nix_deps_dir = None
         self._should_fix_bins_and_dylibs = None
+        self.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
+
+        self.config_toml = config_toml
+
+        self.clean = args.clean
+        self.json_output = args.json_output
+        self.verbose = args.verbose
+        self.color = args.color
+        self.warnings = args.warnings
+
+        config_verbose_count = self.get_toml('verbose', 'build')
+        if config_verbose_count is not None:
+            self.verbose = max(self.verbose, int(config_verbose_count))
+
+        self.use_vendored_sources = self.get_toml('vendor', 'build') == 'true'
+        self.use_locked_deps = self.get_toml('locked-deps', 'build') == 'true'
+
+        build_dir = args.build_dir or self.get_toml('build-dir', 'build') or 'build'
+        self.build_dir = os.path.abspath(build_dir)
+
+        with open(os.path.join(self.rust_root, "src", "stage0.json")) as f:
+            data = json.load(f)
+        self.checksums_sha256 = data["checksums_sha256"]
+        self.stage0_compiler = Stage0Toolchain(data["compiler"])
+        self.download_url = os.getenv("RUSTUP_DIST_SERVER") or data["config"]["dist_server"]
+
+        self.build = args.build or self.build_triple()
+
 
     def download_toolchain(self):
         """Fetch the build system for Rust, written in Rust
@@ -704,9 +732,10 @@ def rustc_stamp(self):
         """Return the path for .rustc-stamp at the given stage
 
         >>> rb = RustBuild()
+        >>> rb.build = "host"
         >>> rb.build_dir = "build"
-        >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
-        True
+        >>> expected = os.path.join("build", "host", "stage0", ".rustc-stamp")
+        >>> assert rb.rustc_stamp() == expected, rb.rustc_stamp()
         """
         return os.path.join(self.bin_root(), '.rustc-stamp')
 
@@ -721,15 +750,9 @@ def bin_root(self):
         """Return the binary root directory for the given stage
 
         >>> rb = RustBuild()
-        >>> rb.build_dir = "build"
-        >>> rb.bin_root() == os.path.join("build", "stage0")
-        True
-
-        When the 'build' property is given should be a nested directory:
-
         >>> rb.build = "devel"
-        >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
-        True
+        >>> expected = os.path.abspath(os.path.join("build", "devel", "stage0"))
+        >>> assert rb.bin_root() == expected, rb.bin_root()
         """
         subdir = "stage0"
         return os.path.join(self.build_dir, self.build, subdir)
@@ -761,9 +784,12 @@ def get_toml(self, key, section=None):
         >>> rb.get_toml("key1")
         'true'
         """
+        return RustBuild.get_toml_static(self.config_toml, key, section)
 
+    @staticmethod
+    def get_toml_static(config_toml, key, section=None):
         cur_section = None
-        for line in self.config_toml.splitlines():
+        for line in config_toml.splitlines():
             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
             if section_match is not None:
                 cur_section = section_match.group(1)
@@ -772,7 +798,7 @@ def get_toml(self, key, section=None):
             if match is not None:
                 value = match.group(1)
                 if section is None or section == cur_section:
-                    return self.get_string(value) or value.strip()
+                    return RustBuild.get_string(value) or value.strip()
         return None
 
     def cargo(self):
@@ -835,13 +861,23 @@ def bootstrap_binary(self):
         """
         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 
-    def build_bootstrap(self, color, warnings, verbose_count):
+    def build_bootstrap(self):
         """Build bootstrap"""
         env = os.environ.copy()
         if "GITHUB_ACTIONS" in env:
             print("::group::Building bootstrap")
         else:
             print("Building bootstrap", file=sys.stderr)
+
+        args = self.build_bootstrap_cmd(env)
+        # Run this from the source directory so cargo finds .cargo/config
+        run(args, env=env, verbose=self.verbose, cwd=self.rust_root)
+
+        if "GITHUB_ACTIONS" in env:
+            print("::endgroup::")
+
+    def build_bootstrap_cmd(self, env):
+        """For tests."""
         build_dir = os.path.join(self.build_dir, "bootstrap")
         if self.clean and os.path.exists(build_dir):
             shutil.rmtree(build_dir)
@@ -894,10 +930,10 @@ def build_bootstrap(self, color, warnings, verbose_count):
         if target_linker is not None:
             env["RUSTFLAGS"] += " -C linker=" + target_linker
         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
-        if warnings == "default":
+        if self.warnings == "default":
             deny_warnings = self.get_toml("deny-warnings", "rust") != "false"
         else:
-            deny_warnings = warnings == "deny"
+            deny_warnings = self.warnings == "deny"
         if deny_warnings:
             env["RUSTFLAGS"] += " -Dwarnings"
 
@@ -908,7 +944,7 @@ def build_bootstrap(self, color, warnings, verbose_count):
                 self.cargo()))
         args = [self.cargo(), "build", "--manifest-path",
                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
-        args.extend("--verbose" for _ in range(verbose_count))
+        args.extend("--verbose" for _ in range(self.verbose))
         if self.use_locked_deps:
             args.append("--locked")
         if self.use_vendored_sources:
@@ -918,20 +954,16 @@ def build_bootstrap(self, color, warnings, verbose_count):
             args.append("build-metrics")
         if self.json_output:
             args.append("--message-format=json")
-        if color == "always":
+        if self.color == "always":
             args.append("--color=always")
-        elif color == "never":
+        elif self.color == "never":
             args.append("--color=never")
         try:
             args += env["CARGOFLAGS"].split()
         except KeyError:
             pass
 
-        # Run this from the source directory so cargo finds .cargo/config
-        run(args, env=env, verbose=self.verbose, cwd=self.rust_root)
-
-        if "GITHUB_ACTIONS" in env:
-            print("::endgroup::")
+        return args
 
     def build_triple(self):
         """Build triple as in LLVM
@@ -981,7 +1013,7 @@ def check_vendored_status(self):
             if os.path.exists(cargo_dir):
                 shutil.rmtree(cargo_dir)
 
-def parse_args():
+def parse_args(args):
     """Parse the command line arguments that the python script needs."""
     parser = argparse.ArgumentParser(add_help=False)
     parser.add_argument('-h', '--help', action='store_true')
@@ -994,16 +1026,11 @@ def parse_args():
     parser.add_argument('--warnings', choices=['deny', 'warn', 'default'], default='default')
     parser.add_argument('-v', '--verbose', action='count', default=0)
 
-    return parser.parse_known_args(sys.argv)[0]
+    return parser.parse_known_args(args)[0]
 
 def bootstrap(args):
     """Configure, fetch, build and run the initial bootstrap"""
-    # Configure initial bootstrap
-    build = RustBuild()
-    build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
-    build.verbose = args.verbose != 0
-    build.clean = args.clean
-    build.json_output = args.json_output
+    rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
 
     # Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`,
     # then `config.toml` in the root directory.
@@ -1012,52 +1039,37 @@ def bootstrap(args):
     if using_default_path:
         toml_path = 'config.toml'
         if not os.path.exists(toml_path):
-            toml_path = os.path.join(build.rust_root, toml_path)
+            toml_path = os.path.join(rust_root, toml_path)
 
     # Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path,
     # but not if `config.toml` hasn't been created.
     if not using_default_path or os.path.exists(toml_path):
         with open(toml_path) as config:
-            build.config_toml = config.read()
+            config_toml = config.read()
+    else:
+        config_toml = ''
 
-    profile = build.get_toml('profile')
+    profile = RustBuild.get_toml_static(config_toml, 'profile')
     if profile is not None:
         include_file = 'config.{}.toml'.format(profile)
-        include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults')
+        include_dir = os.path.join(rust_root, 'src', 'bootstrap', 'defaults')
         include_path = os.path.join(include_dir, include_file)
-        # HACK: This works because `build.get_toml()` returns the first match it finds for a
+        # HACK: This works because `self.get_toml()` returns the first match it finds for a
         # specific key, so appending our defaults at the end allows the user to override them
         with open(include_path) as included_toml:
-            build.config_toml += os.linesep + included_toml.read()
-
-    verbose_count = args.verbose
-    config_verbose_count = build.get_toml('verbose', 'build')
-    if config_verbose_count is not None:
-        verbose_count = max(args.verbose, int(config_verbose_count))
-
-    build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
-    build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
+            config_toml += os.linesep + included_toml.read()
 
+    # Configure initial bootstrap
+    build = RustBuild(config_toml, args)
     build.check_vendored_status()
 
-    build_dir = args.build_dir or build.get_toml('build-dir', 'build') or 'build'
-    build.build_dir = os.path.abspath(build_dir)
-
-    with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
-        data = json.load(f)
-    build.checksums_sha256 = data["checksums_sha256"]
-    build.stage0_compiler = Stage0Toolchain(data["compiler"])
-    build.download_url = os.getenv("RUSTUP_DIST_SERVER") or data["config"]["dist_server"]
-
-    build.build = args.build or build.build_triple()
-
     if not os.path.exists(build.build_dir):
         os.makedirs(build.build_dir)
 
     # Fetch/build the bootstrap
     build.download_toolchain()
     sys.stdout.flush()
-    build.build_bootstrap(args.color, args.warnings, verbose_count)
+    build.build_bootstrap()
     sys.stdout.flush()
 
     # Run the bootstrap
@@ -1077,7 +1089,7 @@ def main():
     if len(sys.argv) > 1 and sys.argv[1] == 'help':
         sys.argv[1] = '-h'
 
-    args = parse_args()
+    args = parse_args(sys.argv)
     help_triggered = args.help or len(sys.argv) == 1
 
     # If the user is asking for help, let them know that the whole download-and-build