rustc: More stable hashes of command line arguments

Currently rustc isn't always the best at producing deterministic builds of a
crate when the source directory of a crate is changed. This is happening due to
what appears two different sources:

* First the `-L` paths passed to rustc are hashed into the crate hash. These
  paths through Cargo are typically absolute paths that can vary if the build
  directory changes.

* Next the paths passed to `--extern` are also hashed which like `-L` can change
  if the build directory changes.

This commit fixes these two sources of nondeterminism by ensuring that avoiding
tracking the hashes of these arguments on the command line. For `-L` paths
they're either related to loading crates (whose hashes are tracked elsewhere) or
native librarise used in the linking phase (which isn't incremental). The
`--extern` paths are similar in that they're related to crate resolution which
is already tracked independently of the command line arguments.

Closes #48019

Author: alexcrichton

src/librustc/session/config.rs

@@ -420,10 +420,7 @@ top_level_options!(
         lint_cap: Option<lint::Level> [TRACKED],
         describe_lints: bool [UNTRACKED],
         output_types: OutputTypes [TRACKED],
-        // FIXME(mw): We track this for now but it actually doesn't make too
-        //            much sense: The search path can stay the same while the
-        //            things discovered there might have changed on disk.
-        search_paths: SearchPaths [TRACKED],
+        search_paths: SearchPaths [UNTRACKED],
         libs: Vec<(String, Option<String>, Option<cstore::NativeLibraryKind>)> [TRACKED],
         maybe_sysroot: Option<PathBuf> [TRACKED],
 
@@ -442,10 +439,7 @@ top_level_options!(
         // version of `debugging_opts.borrowck`, which is just a plain string.
         borrowck_mode: BorrowckMode [UNTRACKED],
         cg: CodegenOptions [TRACKED],
-        // FIXME(mw): We track this for now but it actually doesn't make too
-        //            much sense: The value of this option can stay the same
-        //            while the files they refer to might have changed on disk.
-        externs: Externs [TRACKED],
+        externs: Externs [UNTRACKED],
         crate_name: Option<String> [TRACKED],
         // An optional name to use as the crate for std during std injection,
         // written `extern crate std = "name"`. Default to "std". Used by
@@ -2141,13 +2135,12 @@ impl fmt::Display for CrateType {
 mod dep_tracking {
     use lint;
     use middle::cstore;
-    use session::search_paths::{PathKind, SearchPaths};
     use std::collections::BTreeMap;
     use std::hash::Hash;
     use std::path::PathBuf;
     use std::collections::hash_map::DefaultHasher;
     use super::{Passes, CrateType, OptLevel, DebugInfoLevel, Lto,
-                OutputTypes, Externs, ErrorOutputType, Sanitizer, Epoch};
+                OutputTypes, ErrorOutputType, Sanitizer, Epoch};
     use syntax::feature_gate::UnstableFeatures;
     use rustc_back::{PanicStrategy, RelroLevel};
 
@@ -2204,7 +2197,6 @@ mod dep_tracking {
     impl_dep_tracking_hash_via_hash!(Lto);
     impl_dep_tracking_hash_via_hash!(DebugInfoLevel);
     impl_dep_tracking_hash_via_hash!(UnstableFeatures);
-    impl_dep_tracking_hash_via_hash!(Externs);
     impl_dep_tracking_hash_via_hash!(OutputTypes);
     impl_dep_tracking_hash_via_hash!(cstore::NativeLibraryKind);
     impl_dep_tracking_hash_via_hash!(Sanitizer);
@@ -2218,15 +2210,6 @@ mod dep_tracking {
     impl_dep_tracking_hash_for_sortable_vec_of!((String, Option<String>,
                                                  Option<cstore::NativeLibraryKind>));
     impl_dep_tracking_hash_for_sortable_vec_of!((String, u64));
-    impl DepTrackingHash for SearchPaths {
-        fn hash(&self, hasher: &mut DefaultHasher, _: ErrorOutputType) {
-            let mut elems: Vec<_> = self
-                .iter(PathKind::All)
-                .collect();
-            elems.sort();
-            Hash::hash(&elems, hasher);
-        }
-    }
 
     impl<T1, T2> DepTrackingHash for (T1, T2)
         where T1: DepTrackingHash,
@@ -2413,43 +2396,6 @@ mod tests {
         assert_eq!(v1.dep_tracking_hash(), v1.clone().dep_tracking_hash());
     }
 
-    #[test]
-    fn test_externs_tracking_hash_different_values() {
-        let mut v1 = super::basic_options();
-        let mut v2 = super::basic_options();
-        let mut v3 = super::basic_options();
-
-        v1.externs = Externs::new(mk_map(vec![
-            (String::from("a"), mk_set(vec![String::from("b"),
-                                            String::from("c")])),
-            (String::from("d"), mk_set(vec![String::from("e"),
-                                            String::from("f")])),
-        ]));
-
-        v2.externs = Externs::new(mk_map(vec![
-            (String::from("a"), mk_set(vec![String::from("b"),
-                                            String::from("c")])),
-            (String::from("X"), mk_set(vec![String::from("e"),
-                                            String::from("f")])),
-        ]));
-
-        v3.externs = Externs::new(mk_map(vec![
-            (String::from("a"), mk_set(vec![String::from("b"),
-                                            String::from("c")])),
-            (String::from("d"), mk_set(vec![String::from("X"),
-                                            String::from("f")])),
-        ]));
-
-        assert!(v1.dep_tracking_hash() != v2.dep_tracking_hash());
-        assert!(v1.dep_tracking_hash() != v3.dep_tracking_hash());
-        assert!(v2.dep_tracking_hash() != v3.dep_tracking_hash());
-
-        // Check clone
-        assert_eq!(v1.dep_tracking_hash(), v1.clone().dep_tracking_hash());
-        assert_eq!(v2.dep_tracking_hash(), v2.clone().dep_tracking_hash());
-        assert_eq!(v3.dep_tracking_hash(), v3.clone().dep_tracking_hash());
-    }
-
     #[test]
     fn test_externs_tracking_hash_different_construction_order() {
         let mut v1 = super::basic_options();
@@ -2540,69 +2486,6 @@ mod tests {
         assert_eq!(v2.dep_tracking_hash(), v2.clone().dep_tracking_hash());
     }
 
-    #[test]
-    fn test_search_paths_tracking_hash_different_values() {
-        let mut v1 = super::basic_options();
-        let mut v2 = super::basic_options();
-        let mut v3 = super::basic_options();
-        let mut v4 = super::basic_options();
-        let mut v5 = super::basic_options();
-
-        // Reference
-        v1.search_paths.add_path("native=abc", super::ErrorOutputType::Json(false));
-        v1.search_paths.add_path("crate=def", super::ErrorOutputType::Json(false));
-        v1.search_paths.add_path("dependency=ghi", super::ErrorOutputType::Json(false));
-        v1.search_paths.add_path("framework=jkl", super::ErrorOutputType::Json(false));
-        v1.search_paths.add_path("all=mno", super::ErrorOutputType::Json(false));
-
-        // Native changed
-        v2.search_paths.add_path("native=XXX", super::ErrorOutputType::Json(false));
-        v2.search_paths.add_path("crate=def", super::ErrorOutputType::Json(false));
-        v2.search_paths.add_path("dependency=ghi", super::ErrorOutputType::Json(false));
-        v2.search_paths.add_path("framework=jkl", super::ErrorOutputType::Json(false));
-        v2.search_paths.add_path("all=mno", super::ErrorOutputType::Json(false));
-
-        // Crate changed
-        v2.search_paths.add_path("native=abc", super::ErrorOutputType::Json(false));
-        v2.search_paths.add_path("crate=XXX", super::ErrorOutputType::Json(false));
-        v2.search_paths.add_path("dependency=ghi", super::ErrorOutputType::Json(false));
-        v2.search_paths.add_path("framework=jkl", super::ErrorOutputType::Json(false));
-        v2.search_paths.add_path("all=mno", super::ErrorOutputType::Json(false));
-
-        // Dependency changed
-        v3.search_paths.add_path("native=abc", super::ErrorOutputType::Json(false));
-        v3.search_paths.add_path("crate=def", super::ErrorOutputType::Json(false));
-        v3.search_paths.add_path("dependency=XXX", super::ErrorOutputType::Json(false));
-        v3.search_paths.add_path("framework=jkl", super::ErrorOutputType::Json(false));
-        v3.search_paths.add_path("all=mno", super::ErrorOutputType::Json(false));
-
-        // Framework changed
-        v4.search_paths.add_path("native=abc", super::ErrorOutputType::Json(false));
-        v4.search_paths.add_path("crate=def", super::ErrorOutputType::Json(false));
-        v4.search_paths.add_path("dependency=ghi", super::ErrorOutputType::Json(false));
-        v4.search_paths.add_path("framework=XXX", super::ErrorOutputType::Json(false));
-        v4.search_paths.add_path("all=mno", super::ErrorOutputType::Json(false));
-
-        // All changed
-        v5.search_paths.add_path("native=abc", super::ErrorOutputType::Json(false));
-        v5.search_paths.add_path("crate=def", super::ErrorOutputType::Json(false));
-        v5.search_paths.add_path("dependency=ghi", super::ErrorOutputType::Json(false));
-        v5.search_paths.add_path("framework=jkl", super::ErrorOutputType::Json(false));
-        v5.search_paths.add_path("all=XXX", super::ErrorOutputType::Json(false));
-
-        assert!(v1.dep_tracking_hash() != v2.dep_tracking_hash());
-        assert!(v1.dep_tracking_hash() != v3.dep_tracking_hash());
-        assert!(v1.dep_tracking_hash() != v4.dep_tracking_hash());
-        assert!(v1.dep_tracking_hash() != v5.dep_tracking_hash());
-
-        // Check clone
-        assert_eq!(v1.dep_tracking_hash(), v1.clone().dep_tracking_hash());
-        assert_eq!(v2.dep_tracking_hash(), v2.clone().dep_tracking_hash());
-        assert_eq!(v3.dep_tracking_hash(), v3.clone().dep_tracking_hash());
-        assert_eq!(v4.dep_tracking_hash(), v4.clone().dep_tracking_hash());
-        assert_eq!(v5.dep_tracking_hash(), v5.clone().dep_tracking_hash());
-    }
-
     #[test]
     fn test_search_paths_tracking_hash_different_order() {
         let mut v1 = super::basic_options();

src/test/run-make/reproducible-build/Makefile

@@ -1,20 +1,78 @@
 -include ../tools.mk
-all:
+all:  \
+	smoke \
+	debug \
+	opt \
+	link_paths \
+	remap_paths \
+	different_source_dirs \
+	extern_flags
+
+smoke:
+	rm -rf $(TMPDIR) && mkdir $(TMPDIR)
 	$(RUSTC) reproducible-build-aux.rs
 	$(RUSTC) reproducible-build.rs -o"$(TMPDIR)/reproducible-build1"
 	$(RUSTC) reproducible-build.rs -o"$(TMPDIR)/reproducible-build2"
+	$(B2)
 	nm "$(TMPDIR)/reproducible-build1" | sort > "$(TMPDIR)/reproducible-build1.nm"
 	nm "$(TMPDIR)/reproducible-build2" | sort > "$(TMPDIR)/reproducible-build2.nm"
 	cmp "$(TMPDIR)/reproducible-build1.nm" "$(TMPDIR)/reproducible-build2.nm" || exit 1
+
+debug:
+	rm -rf $(TMPDIR) && mkdir $(TMPDIR)
 	$(RUSTC) reproducible-build-aux.rs -g
-	$(RUSTC) reproducible-build.rs -g -o"$(TMPDIR)/reproducible-build1-debug"
-	$(RUSTC) reproducible-build.rs -g -o"$(TMPDIR)/reproducible-build2-debug"
-	nm "$(TMPDIR)/reproducible-build1-debug" | sort > "$(TMPDIR)/reproducible-build1-debug.nm"
-	nm "$(TMPDIR)/reproducible-build2-debug" | sort > "$(TMPDIR)/reproducible-build2-debug.nm"
+	$(RUSTC) reproducible-build.rs -o"$(TMPDIR)/reproducible-build1" -g
+	$(RUSTC) reproducible-build.rs -o"$(TMPDIR)/reproducible-build2" -g
+	nm "$(TMPDIR)/reproducible-build1" | sort > "$(TMPDIR)/reproducible-build1-debug.nm"
+	nm "$(TMPDIR)/reproducible-build2" | sort > "$(TMPDIR)/reproducible-build2-debug.nm"
 	cmp "$(TMPDIR)/reproducible-build1-debug.nm" "$(TMPDIR)/reproducible-build2-debug.nm" || exit 1
+
+opt:
+	rm -rf $(TMPDIR) && mkdir $(TMPDIR)
 	$(RUSTC) reproducible-build-aux.rs -O
-	$(RUSTC) reproducible-build.rs -O -o"$(TMPDIR)/reproducible-build1-opt"
-	$(RUSTC) reproducible-build.rs -O -o"$(TMPDIR)/reproducible-build2-opt"
-	nm "$(TMPDIR)/reproducible-build1-opt" | sort > "$(TMPDIR)/reproducible-build1-opt.nm"
-	nm "$(TMPDIR)/reproducible-build2-opt" | sort > "$(TMPDIR)/reproducible-build2-opt.nm"
+	$(RUSTC) reproducible-build.rs -o"$(TMPDIR)/reproducible-build1" -O
+	$(RUSTC) reproducible-build.rs -o"$(TMPDIR)/reproducible-build2" -O
+	nm "$(TMPDIR)/reproducible-build1" | sort > "$(TMPDIR)/reproducible-build1-opt.nm"
+	nm "$(TMPDIR)/reproducible-build2" | sort > "$(TMPDIR)/reproducible-build2-opt.nm"
 	cmp "$(TMPDIR)/reproducible-build1-opt.nm" "$(TMPDIR)/reproducible-build2-opt.nm" || exit 1
+
+link_paths:
+	rm -rf $(TMPDIR) && mkdir $(TMPDIR)
+	$(RUSTC) reproducible-build-aux.rs
+	$(RUSTC) reproducible-build.rs --crate-type rlib -L /b
+	cp $(TMPDIR)/libreproducible_build.rlib $(TMPDIR)/libfoo.rlib
+	$(RUSTC) reproducible-build.rs --crate-type rlib -L /a
+	cmp "$(TMPDIR)/libreproducible_build.rlib" "$(TMPDIR)/libfoo.rlib" || exit 1
+
+remap_paths:
+	rm -rf $(TMPDIR) && mkdir $(TMPDIR)
+	$(RUSTC) reproducible-build-aux.rs
+	$(RUSTC) reproducible-build.rs --crate-type rlib --remap-path-prefix=/a=/c
+	cp $(TMPDIR)/libreproducible_build.rlib $(TMPDIR)/libfoo.rlib
+	$(RUSTC) reproducible-build.rs --crate-type rlib --remap-path-prefix=/b=/c
+	cmp "$(TMPDIR)/libreproducible_build.rlib" "$(TMPDIR)/libfoo.rlib" || exit 1
+
+different_source_dirs:
+	rm -rf $(TMPDIR) && mkdir $(TMPDIR)
+	$(RUSTC) reproducible-build-aux.rs
+	mkdir $(TMPDIR)/test
+	cp reproducible-build.rs $(TMPDIR)/test
+	$(RUSTC) reproducible-build.rs --crate-type rlib --remap-path-prefix=$$PWD=/b
+	cp $(TMPDIR)/libreproducible_build.rlib $(TMPDIR)/libfoo.rlib
+	(cd $(TMPDIR)/test && $(RUSTC) reproducible-build.rs \
+		--remap-path-prefix=$(TMPDIR)/test=/b \
+		--crate-type rlib)
+	cmp "$(TMPDIR)/libreproducible_build.rlib" "$(TMPDIR)/libfoo.rlib" || exit 1
+
+extern_flags:
+	rm -rf $(TMPDIR) && mkdir $(TMPDIR)
+	$(RUSTC) reproducible-build-aux.rs
+	$(RUSTC) reproducible-build.rs \
+		--extern reproducible_build_aux=$(TMPDIR)/libreproducible_build_aux.rlib \
+		--crate-type rlib
+	cp $(TMPDIR)/libreproducible_build_aux.rlib $(TMPDIR)/libbar.rlib
+	cp $(TMPDIR)/libreproducible_build.rlib $(TMPDIR)/libfoo.rlib
+	$(RUSTC) reproducible-build.rs \
+		--extern reproducible_build_aux=$(TMPDIR)/libbar.rlib \
+		--crate-type rlib
+	cmp "$(TMPDIR)/libreproducible_build.rlib" "$(TMPDIR)/libfoo.rlib" || exit 1

src/test/run-make/reproducible-build/reproducible-build.rs

@@ -28,7 +28,7 @@
 // - Trait object shims
 // - Fn Pointer shims
 
-#![allow(dead_code)]
+#![allow(dead_code, warnings)]
 
 extern crate reproducible_build_aux;