Download pre-compiled bootstrap from CI

src/bootstrap/bootstrap.py

@@ -11,6 +11,7 @@
 import sys
 import tarfile
 import tempfile
+import pathlib
 
 from time import time
 from multiprocessing import Pool, cpu_count
@@ -42,30 +43,31 @@ def get_cpus():
 
 
 
-def get(base, url, path, checksums, verbose=False):
+def get(base, url, path, checksums, verbose=False, verify_checksum=True):
     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
         temp_path = temp_file.name
 
     try:
-        if url not in checksums:
+        if url not in checksums and verify_checksum:
             raise RuntimeError(("src/stage0.json doesn't contain a checksum for {}. "
                                 "Pre-built artifacts might not be available for this "
                                 "target at this time, see https://doc.rust-lang.org/nightly"
                                 "/rustc/platform-support.html for more information.")
                                 .format(url))
-        sha256 = checksums[url]
-        if os.path.exists(path):
-            if verify(path, sha256, False):
-                if verbose:
-                    print("using already-download file", path, file=sys.stderr)
-                return
-            else:
-                if verbose:
-                    print("ignoring already-download file",
-                        path, "due to failed verification", file=sys.stderr)
-                os.unlink(path)
+        if verify_checksum:
+            sha256 = checksums[url]
+            if os.path.exists(path):
+                if verify(path, sha256, False):
+                    if verbose:
+                        print("using already-download file", path, file=sys.stderr)
+                    return
+                else:
+                    if verbose:
+                        print("ignoring already-download file",
+                            path, "due to failed verification", file=sys.stderr)
+                    os.unlink(path)
         download(temp_path, "{}/{}".format(base, url), True, verbose)
-        if not verify(temp_path, sha256, verbose):
+        if verify_checksum and not verify(temp_path, checksums[url], verbose):
             raise RuntimeError("failed verification")
         if verbose:
             print("moving {} to {}".format(temp_path, path), file=sys.stderr)
@@ -213,7 +215,11 @@ def require(cmd, exit=True, exception=False):
             sys.exit(1)
         return None
 
+def output_cmd(cmd):
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, text=True)
+    output = p.communicate()[0].strip('"').strip()
 
+    return output
 
 def format_build_time(duration):
     """Return a nicer format for build time
@@ -424,6 +430,7 @@ class DownloadInfo:
     def __init__(
         self,
         base_download_url,
+        artifacts_server,
         download_path,
         bin_root,
         tarball_path,
@@ -434,6 +441,7 @@ def __init__(
     ):
         self.base_download_url = base_download_url
         self.download_path = download_path
+        self.artifacts_server = artifacts_server
         self.bin_root = bin_root
         self.tarball_path = tarball_path
         self.tarball_suffix = tarball_suffix
@@ -463,6 +471,11 @@ def unpack_component(download_info):
 class FakeArgs:
     """Used for unit tests to avoid updating all call sites"""
     def __init__(self):
+        self.checksums_sha256 = {}
+        self.stage0_compiler = None
+        self.download_url = ''
+        self.artifacts_server = ''
+        self.is_precompiled_bootstrap = False
         self.build = ''
         self.build_dir = ''
         self.clean = False
@@ -488,6 +501,7 @@ def __init__(self, config_toml="", args=None):
         self.verbose = args.verbose
         self.color = args.color
         self.warnings = args.warnings
+        self.is_precompiled_bootstrap = False
 
         config_verbose_count = self.get_toml('verbose', 'build')
         if config_verbose_count is not None:
@@ -507,6 +521,7 @@ def __init__(self, config_toml="", args=None):
 
         self.build = args.build or self.build_triple()
 
+        self.git_merge_commit_mail = None
 
     def download_toolchain(self):
         """Fetch the build system for Rust, written in Rust
@@ -565,6 +580,7 @@ def download_toolchain(self):
                 DownloadInfo(
                     base_download_url=self.download_url,
                     download_path="dist/{}/{}".format(self.stage0_compiler.date, filename),
+                    artifacts_server=self.artifacts_server,
                     bin_root=self.bin_root(),
                     tarball_path=os.path.join(rustc_cache, filename),
                     tarball_suffix=tarball_suffix,
@@ -606,20 +622,59 @@ def download_toolchain(self):
                 rust_stamp.write(key)
 
     def _download_component_helper(
-        self, filename, pattern, tarball_suffix, rustc_cache,
+        self, key, filename, pattern, tarball_suffix,
     ):
-        key = self.stage0_compiler.date
-
+        cache_dst = os.path.join(self.build_dir, "cache")
+        rustc_cache = os.path.join(cache_dst)
         tarball = os.path.join(rustc_cache, filename)
         if not os.path.exists(tarball):
-            get(
-                self.download_url,
-                "dist/{}/{}".format(key, filename),
+            try:
+                get(
+                self.artifacts_server,
+                "{}/{}".format(key, filename),
                 tarball,
                 self.checksums_sha256,
                 verbose=self.verbose,
+                verify_checksum=False,
             )
+            except:
+                return False
         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
+        return True
+
+
+
+    def download_bootstrap(self, commit_hash):
+        filename = "{}-{}.tar.xz".format("bootstrap-nightly", self.build_triple())
+        success = self._download_component_helper(commit_hash, filename, "bootstrap", "tar.xz")
+
+        return success
+
+    def last_bootstrap_commit(self):
+        top_level = output_cmd(["git", "rev-parse", "--show-toplevel"])
+
+        cmd = ["git", "rev-list", "--author={}".format(self.git_merge_commit_mail),
+               "-n1", "--first-parent", "HEAD"]
+        merge_base = output_cmd(cmd)
+
+        path = "{}/src/bootstrap".format(top_level)
+        cmd = ["git", "diff-index", "--quiet", "HEAD", "--", path]
+
+        ret = subprocess.Popen(cmd, cwd=self.rust_root)
+        code = ret.wait()
+        if code != 0:
+            return None
+        else:
+            return merge_base
+
+    def bootstrap_out_of_date(self, commit):
+        stamp_path = pathlib.Path(self.bin_root()).joinpath("bootstrap/.bootstrap-stamp")
+        if not stamp_path.exists():
+            return True
+
+        return stamp_path.read_text != commit
+
+
 
     def should_fix_bins_and_dylibs(self):
         """Whether or not `fix_bin_or_dylib` needs to be run; can only be True
@@ -863,7 +918,14 @@ def bootstrap_binary(self):
         ... "debug", "bootstrap")
         True
         """
-        return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
+        if self.is_precompiled_bootstrap:
+            root = self.bin_root()
+            subfolder = "nightly-{}".format(self.build_triple())
+            return os.path.join(root, subfolder, "bootstrap", "bootstrap", "bin", "bootstrap")
+
+        else:
+            return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
+
 
     def build_bootstrap(self):
         """Build bootstrap"""
@@ -1068,13 +1130,45 @@ def bootstrap(args):
     build = RustBuild(config_toml, args)
     build.check_vendored_status()
 
+    build_dir = args.build_dir or build.get_toml('build-dir', 'build') or 'build'
+    build.build_dir = os.path.abspath(build_dir)
+
+    with open(os.path.join(build.rust_root, "src", "stage0.json")) as f:
+        data = json.load(f)
+    build.checksums_sha256 = data["checksums_sha256"]
+    build.stage0_compiler = Stage0Toolchain(data["compiler"])
+    build.download_url = os.getenv("RUSTUP_DIST_SERVER") or data["config"]["dist_server"]
+    build.artifacts_server = data["config"]["artifacts_server"]
+    build.git_merge_commit_mail = data["config"]["git_merge_commit_email"]
+
+    build.build = args.build or build.build_triple()
+
     if not os.path.exists(build.build_dir):
         os.makedirs(build.build_dir)
 
     # Fetch/build the bootstrap
     build.download_toolchain()
     sys.stdout.flush()
-    build.build_bootstrap()
+
+    bootstrap_bin = pathlib.Path(build.build_dir).joinpath("bootstrap/debug/bootstrap")
+
+    try:
+        last_commit = build.last_bootstrap_commit()
+        if last_commit is None:
+            build.build_bootstrap()
+        else:
+            if build.bootstrap_out_of_date(last_commit):
+                success = build.download_bootstrap(last_commit)
+                if success:
+                    stamp = pathlib.Path(build.build_dir).joinpath("bootstrap/.bootstrap-stamp")
+                    stamp.write_text(last_commit)
+                    build.is_precompiled_bootstrap = True
+
+    except Exception as e:
+        print(str(e))
+
+    if build.is_precompiled_bootstrap:
+        build.build_bootstrap()
     sys.stdout.flush()
 
     # Run the bootstrap
@@ -1101,8 +1195,8 @@ def main():
     # process has to happen before anything is printed out.
     if help_triggered:
         print(
-            "info: Downloading and building bootstrap before processing --help command.\n"
-            "      See src/bootstrap/README.md for help with common commands."
+            "info: Checking if bootstrap needs to be downloaded or built before processing"
+            " --help command.\n      See src/bootstrap/README.md for help with common commands."
         , file=sys.stderr)
 
     exit_code = 0