diff --git a/Cargo.lock b/Cargo.lock index 3237b90a7c06f..30ff11915258d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3802,6 +3802,7 @@ dependencies = [ "rustc_data_structures", "rustc_fs_util", "rustc_hir", + "rustc_query_system", "rustc_session", "rustc_span", "serialize", diff --git a/src/librustc/dep_graph/mod.rs b/src/librustc/dep_graph/mod.rs index f56df19bfb061..5653af6b2bfb3 100644 --- a/src/librustc/dep_graph/mod.rs +++ b/src/librustc/dep_graph/mod.rs @@ -11,8 +11,8 @@ mod dep_node; pub(crate) use rustc_query_system::dep_graph::DepNodeParams; pub use rustc_query_system::dep_graph::{ - debug, hash_result, DepContext, DepNodeColor, DepNodeIndex, SerializedDepNodeIndex, - WorkProduct, WorkProductFileKind, WorkProductId, + debug, hash_result, DepContext, DepNodeColor, DepNodeIndex, WorkProduct, WorkProductFileKind, + WorkProductId, }; pub use dep_node::{label_strs, DepConstructor, DepKind, DepNode, DepNodeExt}; @@ -159,8 +159,8 @@ impl<'tcx> DepContext for TyCtxt<'tcx> { try_load_from_on_disk_cache(*self, dep_node) } - fn load_diagnostics(&self, prev_dep_node_index: SerializedDepNodeIndex) -> Vec { - self.queries.on_disk_cache.load_diagnostics(*self, prev_dep_node_index) + fn load_diagnostics(&self, dep_node_index: DepNodeIndex) -> Vec { + self.queries.on_disk_cache.load_diagnostics(*self, dep_node_index) } fn store_diagnostics(&self, dep_node_index: DepNodeIndex, diagnostics: ThinVec) { diff --git a/src/librustc/query/mod.rs b/src/librustc/query/mod.rs index c1ece6275092d..0ab133f513a53 100644 --- a/src/librustc/query/mod.rs +++ b/src/librustc/query/mod.rs @@ -1,4 +1,4 @@ -use crate::dep_graph::SerializedDepNodeIndex; +use crate::dep_graph::DepNodeIndex; use crate::mir; use crate::mir::interpret::{GlobalId, LitToConstInput}; use crate::traits; diff --git a/src/librustc/ty/query/on_disk_cache.rs b/src/librustc/ty/query/on_disk_cache.rs index 8aecc0e698a8e..967e3a130f3b6 100644 --- a/src/librustc/ty/query/on_disk_cache.rs +++ b/src/librustc/ty/query/on_disk_cache.rs @@ -1,4 +1,4 @@ -use crate::dep_graph::{DepNodeIndex, SerializedDepNodeIndex}; +use crate::dep_graph::DepNodeIndex; use crate::mir::interpret::{AllocDecodingSession, AllocDecodingState}; use crate::mir::{self, interpret}; use crate::ty::codec::{self as ty_codec, TyDecoder, TyEncoder}; @@ -12,7 +12,7 @@ use rustc_data_structures::thin_vec::ThinVec; use rustc_errors::Diagnostic; use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, LOCAL_CRATE}; use rustc_hir::definitions::DefPathHash; -use rustc_index::vec::{Idx, IndexVec}; +use rustc_index::vec::IndexVec; use rustc_serialize::{ opaque, Decodable, Decoder, Encodable, Encoder, SpecializedDecoder, SpecializedEncoder, UseSpecializedDecodable, UseSpecializedEncodable, @@ -60,11 +60,11 @@ pub struct OnDiskCache<'sess> { // A map from dep-node to the position of the cached query result in // `serialized_data`. - query_result_index: FxHashMap, + query_result_index: FxHashMap, // A map from dep-node to the position of any associated diagnostics in // `serialized_data`. - prev_diagnostics_index: FxHashMap, + prev_diagnostics_index: FxHashMap, alloc_decoding_state: AllocDecodingState, } @@ -80,8 +80,8 @@ struct Footer { interpret_alloc_index: Vec, } -type EncodedQueryResultIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>; -type EncodedDiagnosticsIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>; +type EncodedQueryResultIndex = Vec<(DepNodeIndex, AbsoluteBytePos)>; +type EncodedDiagnosticsIndex = Vec<(DepNodeIndex, AbsoluteBytePos)>; type EncodedDiagnostics = Vec; #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, RustcEncodable, RustcDecodable)] @@ -224,11 +224,10 @@ impl<'sess> OnDiskCache<'sess> { .current_diagnostics .borrow() .iter() - .map(|(dep_node_index, diagnostics)| { + .map(|(&dep_node_index, diagnostics)| { let pos = AbsoluteBytePos::new(encoder.position()); // Let's make sure we get the expected type here. let diagnostics: &EncodedDiagnostics = diagnostics; - let dep_node_index = SerializedDepNodeIndex::new(dep_node_index.index()); encoder.encode_tagged(dep_node_index, diagnostics)?; Ok((dep_node_index, pos)) @@ -304,7 +303,7 @@ impl<'sess> OnDiskCache<'sess> { pub fn load_diagnostics( &self, tcx: TyCtxt<'_>, - dep_node_index: SerializedDepNodeIndex, + dep_node_index: DepNodeIndex, ) -> Vec { let diagnostics: Option = self.load_indexed(tcx, dep_node_index, &self.prev_diagnostics_index, "diagnostics"); @@ -328,11 +327,11 @@ impl<'sess> OnDiskCache<'sess> { } /// Returns the cached query result if there is something in the cache for - /// the given `SerializedDepNodeIndex`; otherwise returns `None`. + /// the given `DepNodeIndex`; otherwise returns `None`. pub fn try_load_query_result( &self, tcx: TyCtxt<'_>, - dep_node_index: SerializedDepNodeIndex, + dep_node_index: DepNodeIndex, ) -> Option where T: Decodable, @@ -361,8 +360,8 @@ impl<'sess> OnDiskCache<'sess> { fn load_indexed<'tcx, T>( &self, tcx: TyCtxt<'tcx>, - dep_node_index: SerializedDepNodeIndex, - index: &FxHashMap, + dep_node_index: DepNodeIndex, + index: &FxHashMap, debug_tag: &'static str, ) -> Option where @@ -1009,12 +1008,10 @@ where state.iter_results(|results| { for (key, value, dep_node) in results { if Q::cache_on_disk(tcx, key.clone(), Some(&value)) { - let dep_node = SerializedDepNodeIndex::new(dep_node.index()); - // Record position of the cache entry. query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position()))); - // Encode the type check tables with the `SerializedDepNodeIndex` + // Encode the type check tables with the `DepNodeIndex` // as tag. encoder.encode_tagged(dep_node, &value)?; } diff --git a/src/librustc_data_structures/sharded.rs b/src/librustc_data_structures/sharded.rs index 485719c517564..0cc0b5eb8399b 100644 --- a/src/librustc_data_structures/sharded.rs +++ b/src/librustc_data_structures/sharded.rs @@ -30,16 +30,16 @@ pub struct Sharded { impl Default for Sharded { #[inline] fn default() -> Self { - Self::new(T::default) + Self::new(|_| T::default()) } } impl Sharded { #[inline] - pub fn new(mut value: impl FnMut() -> T) -> Self { + pub fn new(mut value: impl FnMut(usize) -> T) -> Self { // Create a vector of the values we want let mut values: SmallVec<[_; SHARDS]> = - (0..SHARDS).map(|_| CacheAligned(Lock::new(value()))).collect(); + (0..SHARDS).map(|i| CacheAligned(Lock::new(value(i)))).collect(); // Create an uninitialized array let mut shards: mem::MaybeUninit<[CacheAligned>; SHARDS]> = @@ -63,6 +63,15 @@ impl Sharded { if SHARDS == 1 { &self.shards[0].0 } else { self.get_shard_by_hash(make_hash(val)) } } + /// The shard is selected by hashing `val` with `FxHasher`. + pub fn get_shard_by_value_mut(&mut self, val: &K) -> &mut T { + if SHARDS == 1 { + self.shards[0].0.get_mut() + } else { + self.shards[self.get_shard_index_by_hash(make_hash(val))].0.get_mut() + } + } + /// Get a shard with a pre-computed hash value. If `get_shard_by_value` is /// ever used in combination with `get_shard_by_hash` on a single `Sharded` /// instance, then `hash` must be computed with `FxHasher`. Otherwise, diff --git a/src/librustc_incremental/Cargo.toml b/src/librustc_incremental/Cargo.toml index eddfb81ea8b0a..c930b2b7e2592 100644 --- a/src/librustc_incremental/Cargo.toml +++ b/src/librustc_incremental/Cargo.toml @@ -21,3 +21,4 @@ rustc_ast = { path = "../librustc_ast" } rustc_span = { path = "../librustc_span" } rustc_fs_util = { path = "../librustc_fs_util" } rustc_session = { path = "../librustc_session" } +rustc_query_system = { path = "../librustc_query_system" } diff --git a/src/librustc_incremental/persist/load.rs b/src/librustc_incremental/persist/load.rs index 537906eb87189..7e962d72c267a 100644 --- a/src/librustc_incremental/persist/load.rs +++ b/src/librustc_incremental/persist/load.rs @@ -1,9 +1,10 @@ //! Code to save/load the dep-graph from files. -use rustc::dep_graph::{PreviousDepGraph, SerializedDepGraph, WorkProduct, WorkProductId}; +use rustc::dep_graph::{DepKind, PreviousDepGraph, SerializedDepGraph}; use rustc::ty::query::OnDiskCache; use rustc::ty::TyCtxt; use rustc_data_structures::fx::FxHashMap; +use rustc_query_system::dep_graph::{CurrentDepGraph, DepGraphArgs}; use rustc_serialize::opaque::Decoder; use rustc_serialize::Decodable as RustcDecodable; use rustc_session::Session; @@ -22,16 +23,14 @@ pub fn dep_graph_tcx_init(tcx: TyCtxt<'_>) { tcx.allocate_metadata_dep_nodes(); } -type WorkProductMap = FxHashMap; - pub enum LoadResult { Ok { data: T }, DataOutOfDate, Error { message: String }, } -impl LoadResult<(PreviousDepGraph, WorkProductMap)> { - pub fn open(self, sess: &Session) -> (PreviousDepGraph, WorkProductMap) { +impl LoadResult> { + pub fn open(self, sess: &Session) -> DepGraphArgs { match self { LoadResult::Error { message } => { sess.warn(&message); @@ -88,7 +87,7 @@ impl MaybeAsync { } } -pub type DepGraphFuture = MaybeAsync>; +pub type DepGraphFuture = MaybeAsync>>; /// Launch a thread and load the dependency graph in the background. pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { @@ -188,7 +187,13 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { let dep_graph = SerializedDepGraph::decode(&mut decoder) .expect("Error reading cached dep-graph"); - LoadResult::Ok { data: (PreviousDepGraph::new(dep_graph), prev_work_products) } + let (prev_graph, state) = PreviousDepGraph::new_and_state(dep_graph); + let current = prof + .generic_activity("incr_comp_load_setup_dep_graph") + .run(|| CurrentDepGraph::new(&prev_graph)); + LoadResult::Ok { + data: DepGraphArgs { state, prev_graph, prev_work_products, current }, + } } } })) diff --git a/src/librustc_interface/queries.rs b/src/librustc_interface/queries.rs index b0eeb57173fa3..6b13704e83373 100644 --- a/src/librustc_interface/queries.rs +++ b/src/librustc_interface/queries.rs @@ -191,16 +191,15 @@ impl<'tcx> Queries<'tcx> { Ok(match self.dep_graph_future()?.take() { None => DepGraph::new_disabled(), Some(future) => { - let (prev_graph, prev_work_products) = - self.session().time("blocked_on_dep_graph_loading", || { - future - .open() - .unwrap_or_else(|e| rustc_incremental::LoadResult::Error { - message: format!("could not decode incremental cache: {:?}", e), - }) - .open(self.session()) - }); - DepGraph::new(prev_graph, prev_work_products) + let args = self.session().time("blocked_on_dep_graph_loading", || { + future + .open() + .unwrap_or_else(|e| rustc_incremental::LoadResult::Error { + message: format!("could not decode incremental cache: {:?}", e), + }) + .open(self.session()) + }); + DepGraph::new(args) } }) }) diff --git a/src/librustc_macros/src/query.rs b/src/librustc_macros/src/query.rs index 26c3bce4a9a02..ba33accbd9104 100644 --- a/src/librustc_macros/src/query.rs +++ b/src/librustc_macros/src/query.rs @@ -324,7 +324,7 @@ fn add_query_description_impl( #[inline] fn try_load_from_disk( #tcx: TyCtxt<'tcx>, - #id: SerializedDepNodeIndex + #id: DepNodeIndex ) -> Option { #block } @@ -335,7 +335,7 @@ fn add_query_description_impl( #[inline] fn try_load_from_disk( tcx: TyCtxt<'tcx>, - id: SerializedDepNodeIndex + id: DepNodeIndex ) -> Option { tcx.queries.on_disk_cache.try_load_query_result(tcx, id) } diff --git a/src/librustc_query_system/dep_graph/graph.rs b/src/librustc_query_system/dep_graph/graph.rs index 73983e1644cb0..8871c37b73aba 100644 --- a/src/librustc_query_system/dep_graph/graph.rs +++ b/src/librustc_query_system/dep_graph/graph.rs @@ -3,13 +3,13 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::profiling::QueryInvocationId; use rustc_data_structures::sharded::{self, Sharded}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; -use rustc_data_structures::sync::{AtomicU32, AtomicU64, Lock, Lrc, Ordering}; +use rustc_data_structures::sync::{AtomicCell, AtomicU32, AtomicU64, Lock, Lrc}; use rustc_data_structures::unlikely; use rustc_errors::Diagnostic; use rustc_index::vec::{Idx, IndexVec}; use parking_lot::{Condvar, Mutex}; -use smallvec::{smallvec, SmallVec}; +use smallvec::SmallVec; use std::collections::hash_map::Entry; use std::env; use std::hash::Hash; @@ -20,7 +20,7 @@ use std::sync::atomic::Ordering::Relaxed; use super::debug::EdgeFilter; use super::prev::PreviousDepGraph; use super::query::DepGraphQuery; -use super::serialized::{SerializedDepGraph, SerializedDepNodeIndex}; +use super::serialized::SerializedDepGraph; use super::{DepContext, DepKind, DepNode, WorkProductId}; #[derive(Clone)] @@ -49,17 +49,41 @@ impl std::convert::From for QueryInvocationId { } } -#[derive(PartialEq)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] pub enum DepNodeColor { Red, - Green(DepNodeIndex), + Green, } impl DepNodeColor { pub fn is_green(self) -> bool { match self { DepNodeColor::Red => false, - DepNodeColor::Green(_) => true, + DepNodeColor::Green => true, + } + } +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub enum DepNodeState { + /// The dep node index is invalid and does not refer to any dep node. + Invalid, + + /// The node is from the previous session, but its state is unknown + Unknown, + + Red, + + Green, +} + +impl DepNodeState { + pub fn color(self) -> Option { + match self { + DepNodeState::Invalid => panic!(), + DepNodeState::Unknown => None, + DepNodeState::Red => Some(DepNodeColor::Red), + DepNodeState::Green => Some(DepNodeColor::Green), } } } @@ -102,22 +126,37 @@ where Some(stable_hasher.finish()) } -impl DepGraph { - pub fn new( - prev_graph: PreviousDepGraph, - prev_work_products: FxHashMap, - ) -> DepGraph { - let prev_graph_node_count = prev_graph.node_count(); +pub struct DepGraphArgs { + pub prev_graph: PreviousDepGraph, + pub prev_work_products: FxHashMap, + pub state: IndexVec>, + pub current: CurrentDepGraph, +} + +impl Default for DepGraphArgs { + fn default() -> Self { + let prev_graph = Default::default(); + Self { + current: CurrentDepGraph::new(&prev_graph), + prev_work_products: Default::default(), + state: Default::default(), + prev_graph, + } + } +} +impl DepGraph { + pub fn new(args: DepGraphArgs) -> Self { + let colors = DepNodeColorMap { values: args.state }; DepGraph { data: Some(Lrc::new(DepGraphData { - previous_work_products: prev_work_products, + previous_work_products: args.prev_work_products, dep_node_debug: Default::default(), - current: CurrentDepGraph::new(prev_graph_node_count), + current: args.current, emitting_diagnostics: Default::default(), emitting_diagnostics_cond_var: Condvar::new(), - previous: prev_graph, - colors: DepNodeColorMap::new(prev_graph_node_count), + colors, + previous: args.prev_graph, })), virtual_dep_node_index: Lrc::new(AtomicU32::new(0)), } @@ -134,12 +173,18 @@ impl DepGraph { } pub fn query(&self) -> DepGraphQuery { - let data = self.data.as_ref().unwrap().current.data.lock(); - let nodes: Vec<_> = data.iter().map(|n| n.node).collect(); + let data = self.data.as_ref().unwrap(); + let node_data = data.current.data.lock(); + + // Recreate the dep graph, but ignore nodes from the previous dep graph + let current_nodes = + || node_data.iter_enumerated().filter(|&(i, _)| data.current(i)).map(|(_, n)| n); + let nodes: Vec<_> = current_nodes().map(|n| n.node).collect(); let mut edges = Vec::new(); - for (from, edge_targets) in data.iter().map(|d| (d.node, &d.edges)) { + for (from, edge_targets) in current_nodes().map(|d| (d.node, &d.edges)) { for &edge_target in edge_targets.iter() { - let to = data[edge_target].node; + assert!(data.current(edge_target)); + let to = node_data[edge_target].node; edges.push((from, to)); } } @@ -212,7 +257,6 @@ impl DepGraph { phantom_data: PhantomData, }) }, - |data, key, fingerprint, task| data.complete_task(key, task.unwrap(), fingerprint), hash_result, ) } @@ -225,12 +269,6 @@ impl DepGraph { no_tcx: bool, task: fn(Ctxt, A) -> R, create_task: fn(DepNode) -> Option>, - finish_task_and_alloc_depnode: fn( - &CurrentDepGraph, - DepNode, - Fingerprint, - Option>, - ) -> DepNodeIndex, hash_result: impl FnOnce(&mut Ctxt::StableHashingContext, &R) -> Option, ) -> (R, DepNodeIndex) { if let Some(ref data) = self.data { @@ -251,52 +289,54 @@ impl DepGraph { let current_fingerprint = hash_result(&mut hcx, &result); - let dep_node_index = finish_task_and_alloc_depnode( - &data.current, - key, - current_fingerprint.unwrap_or(Fingerprint::ZERO), - task_deps.map(|lock| lock.into_inner()), - ); + let deps = task_deps.map(|lock| lock.into_inner().reads).unwrap_or(SmallVec::new()); + let fingerprint = current_fingerprint.unwrap_or(Fingerprint::ZERO); let print_status = cfg!(debug_assertions) && cx.debug_dep_tasks(); - // Determine the color of the new DepNode. - if let Some(prev_index) = data.previous.node_to_index_opt(&key) { + let dep_node_index = if let Some(prev_index) = data.previous.node_to_index_opt(&key) { let prev_fingerprint = data.previous.fingerprint_by_index(prev_index); + // Determine the color of the DepNode. let color = if let Some(current_fingerprint) = current_fingerprint { if current_fingerprint == prev_fingerprint { if print_status { eprintln!("[task::green] {:?}", key); } - DepNodeColor::Green(dep_node_index) + DepNodeState::Green } else { if print_status { eprintln!("[task::red] {:?}", key); } - DepNodeColor::Red + DepNodeState::Red } } else { if print_status { eprintln!("[task::unknown] {:?}", key); } // Mark the node as Red if we can't hash the result - DepNodeColor::Red + DepNodeState::Red }; - debug_assert!( - data.colors.get(prev_index).is_none(), - "DepGraph::with_task() - Duplicate DepNodeColor \ - insertion for {:?}", + debug_assert_eq!( + data.colors.get(prev_index).color(), + None, + "DepGraph::with_task() - Duplicate DepNodeState insertion for {:?}", key ); data.colors.insert(prev_index, color); + + data.current.update_node(prev_index, key, deps, fingerprint); + + prev_index } else { if print_status { eprintln!("[task::new] {:?}", key); } - } + + data.current.new_node(key, deps, fingerprint, &data.previous) + }; (result, dep_node_index) } else { @@ -333,16 +373,7 @@ impl DepGraph { task: fn(Ctxt, A) -> R, hash_result: impl FnOnce(&mut Ctxt::StableHashingContext, &R) -> Option, ) -> (R, DepNodeIndex) { - self.with_task_impl( - key, - cx, - arg, - false, - task, - |_| None, - |data, key, fingerprint, _| data.alloc_node(key, smallvec![], fingerprint), - hash_result, - ) + self.with_task_impl(key, cx, arg, false, task, |_| None, hash_result) } #[inline] @@ -386,7 +417,8 @@ impl DepGraph { .node_to_node_index .get_shard_by_value(&dep_node) .lock() - .contains_key(dep_node) + .get(dep_node) + .map_or(false, |index| data.current(*index)) } else { false } @@ -394,19 +426,15 @@ impl DepGraph { #[inline] pub fn fingerprint_of(&self, dep_node_index: DepNodeIndex) -> Fingerprint { - let data = self.data.as_ref().expect("dep graph enabled").current.data.lock(); - data[dep_node_index].fingerprint + let data = self.data.as_ref().expect("dep graph enabled"); + assert!(data.current(dep_node_index)); + data.current.data.lock()[dep_node_index].fingerprint } pub fn prev_fingerprint_of(&self, dep_node: &DepNode) -> Option { self.data.as_ref().unwrap().previous.fingerprint_of(dep_node) } - #[inline] - pub fn prev_dep_node_index_of(&self, dep_node: &DepNode) -> SerializedDepNodeIndex { - self.data.as_ref().unwrap().previous.node_to_index(dep_node) - } - /// Checks whether a previous work product exists for `v` and, if /// so, return the path that leads to it. Used to skip doing work. pub fn previous_work_product(&self, v: &WorkProductId) -> Option { @@ -451,11 +479,39 @@ impl DepGraph { } pub fn serialize(&self) -> SerializedDepGraph { - let data = self.data.as_ref().unwrap().current.data.lock(); + let data = self.data.as_ref().unwrap(); + let colors = &data.colors; + let data = data.current.data.lock(); + + let fingerprints: IndexVec = data.iter().map(|d| d.fingerprint).collect(); + let mut nodes: IndexVec = data.iter().map(|d| d.node).collect(); + + // Invalidate dep nodes with unknown state as these cannot safely + // be marked green in the next session. One of the dependencies of the + // unknown node may have changed in this session (and is currently marked red), + // but might be green again in the next session, which may cause the unknown node + // to incorrectly be marked green in the next session, even though one of its dependencies + // did actually change. + for index in colors.values.indices() { + match colors.get(index) { + // In order to this invalidation to be safe, none of the valid nodes can + // point to unknown nodes. + DepNodeState::Unknown => { + // Change the node kind to Null so we know this node is invalidated when + // we load the dep graph + nodes[index] = DepNode::new_no_params(DepKind::NULL); + // FIXME: Also clear edges + } - let fingerprints: IndexVec = - data.iter().map(|d| d.fingerprint).collect(); - let nodes: IndexVec = data.iter().map(|d| d.node).collect(); + // For green nodes, we either executed the query (which always uses valid nodes) + // or we marked it as green because all its dependencies are green and valid. + DepNodeState::Green | + // Red nodes were always exexuted. + DepNodeState::Red | + // We don't need to invalidate already invalid nodes + DepNodeState::Invalid => {}, + } + } let total_edge_count: usize = data.iter().map(|d| d.edges.len()).sum(); @@ -465,7 +521,7 @@ impl DepGraph { for (current_dep_node_index, edges) in data.iter_enumerated().map(|(i, d)| (i, &d.edges)) { let start = edge_list_data.len() as u32; // This should really just be a memcpy :/ - edge_list_data.extend(edges.iter().map(|i| SerializedDepNodeIndex::new(i.index()))); + edge_list_data.extend(edges.iter().map(|i| DepNodeIndex::new(i.index()))); let end = edge_list_data.len() as u32; debug_assert_eq!(current_dep_node_index.index(), edge_list_indices.len()); @@ -481,7 +537,7 @@ impl DepGraph { pub fn node_color(&self, dep_node: &DepNode) -> Option { if let Some(ref data) = self.data { if let Some(prev_index) = data.previous.node_to_index_opt(dep_node) { - return data.colors.get(prev_index); + return data.colors.get(prev_index).color(); } else { // This is a node that did not exist in the previous compilation // session, so we consider it to be red. @@ -500,11 +556,11 @@ impl DepGraph { &self, tcx: Ctxt, dep_node: &DepNode, - ) -> Option<(SerializedDepNodeIndex, DepNodeIndex)> { - self.try_mark_green(tcx, dep_node).map(|(prev_index, dep_node_index)| { + ) -> Option { + self.try_mark_green(tcx, dep_node).map(|prev_index| { debug_assert!(self.is_green(&dep_node)); - self.read_index(dep_node_index); - (prev_index, dep_node_index) + self.read_index(prev_index); + prev_index }) } @@ -512,7 +568,7 @@ impl DepGraph { &self, tcx: Ctxt, dep_node: &DepNode, - ) -> Option<(SerializedDepNodeIndex, DepNodeIndex)> { + ) -> Option { debug_assert!(!dep_node.kind.is_eval_always()); // Return None if the dep graph is disabled @@ -522,16 +578,80 @@ impl DepGraph { let prev_index = data.previous.node_to_index_opt(dep_node)?; match data.colors.get(prev_index) { - Some(DepNodeColor::Green(dep_node_index)) => Some((prev_index, dep_node_index)), - Some(DepNodeColor::Red) => None, - None => { + DepNodeState::Invalid => panic!(), + DepNodeState::Green => Some(prev_index), + DepNodeState::Red => None, + DepNodeState::Unknown => { // This DepNode and the corresponding query invocation existed // in the previous compilation session too, so we can try to // mark it as green by recursively marking all of its // dependencies green. - self.try_mark_previous_green(tcx, data, prev_index, &dep_node) - .map(|dep_node_index| (prev_index, dep_node_index)) + if self.try_mark_previous_green(tcx, data, prev_index, &dep_node) { + Some(prev_index) + } else { + None + } + } + } + } + + /// Try to force a dep node to execute and see if it's green + fn try_force_previous_green>( + &self, + tcx: Ctxt, + data: &DepGraphData, + dep_node_index: DepNodeIndex, + ) -> bool { + let dep_node = &data.previous.index_to_node(dep_node_index); + + debug!("try_force_previous_green({:?}) --- trying to force", dep_node); + if tcx.try_force_from_dep_node(dep_node) { + match data.colors.get(dep_node_index) { + DepNodeState::Green => { + debug!( + "try_force_previous_green({:?}) --- managed to \ + FORCE to green", + dep_node + ); + true + } + DepNodeState::Red => { + debug!( + "try_force_previous_green({:?}) - END - was red after forcing", + dep_node + ); + false + } + DepNodeState::Invalid => panic!(), + DepNodeState::Unknown => { + if !tcx.has_errors_or_delayed_span_bugs() { + panic!( + "try_force_previous_green() - Forcing the DepNode \ + should have set its color" + ) + } else { + // If the query we just forced has resulted in + // some kind of compilation error, we cannot rely on + // the dep-node color having been properly updated. + // This means that the query system has reached an + // invalid state. We let the compiler continue (by + // returning `None`) so it can emit error messages + // and wind down, but rely on the fact that this + // invalid state will not be persisted to the + // incremental compilation cache because of + // compilation errors being present. + debug!( + "try_force_previous_green({:?}) resulted in compilation error", + dep_node + ); + false + } + } } + } else { + // The DepNode could not be forced. + debug!("try_force_previous_green({:?}) - END - could not be forced", dep_node); + false } } @@ -540,38 +660,28 @@ impl DepGraph { &self, tcx: Ctxt, data: &DepGraphData, - prev_dep_node_index: SerializedDepNodeIndex, + dep_node_index: DepNodeIndex, + // FIXME: Remove this, only used in debug statements dep_node: &DepNode, - ) -> Option { + ) -> bool { debug!("try_mark_previous_green({:?}) - BEGIN", dep_node); #[cfg(not(parallel_compiler))] - { - debug_assert!( - !data - .current - .node_to_node_index - .get_shard_by_value(dep_node) - .lock() - .contains_key(dep_node) - ); - debug_assert!(data.colors.get(prev_dep_node_index).is_none()); - } + debug_assert!(data.colors.get(dep_node_index).color().is_none()); // We never try to mark eval_always nodes as green debug_assert!(!dep_node.kind.is_eval_always()); - debug_assert_eq!(data.previous.index_to_node(prev_dep_node_index), *dep_node); + debug_assert_eq!(data.previous.index_to_node(dep_node_index), *dep_node); - let prev_deps = data.previous.edge_targets_from(prev_dep_node_index); - - let mut current_deps = SmallVec::new(); + let prev_deps = data.previous.edge_targets_from(dep_node_index); for &dep_dep_node_index in prev_deps { let dep_dep_node_color = data.colors.get(dep_dep_node_index); match dep_dep_node_color { - Some(DepNodeColor::Green(node_index)) => { + DepNodeState::Invalid => panic!(), + DepNodeState::Green => { // This dependency has been marked as green before, we are // still fine and can continue with checking the other // dependencies. @@ -581,9 +691,8 @@ impl DepGraph { dep_node, data.previous.index_to_node(dep_dep_node_index) ); - current_deps.push(node_index); } - Some(DepNodeColor::Red) => { + DepNodeState::Red => { // We found a dependency the value of which has changed // compared to the previous compilation session. We cannot // mark the DepNode as green and also don't need to bother @@ -594,97 +703,31 @@ impl DepGraph { dep_node, data.previous.index_to_node(dep_dep_node_index) ); - return None; + return false; } - None => { + DepNodeState::Unknown => { let dep_dep_node = &data.previous.index_to_node(dep_dep_node_index); // We don't know the state of this dependency. If it isn't // an eval_always node, let's try to mark it green recursively. if !dep_dep_node.kind.is_eval_always() { - debug!( - "try_mark_previous_green({:?}) --- state of dependency {:?} \ - is unknown, trying to mark it green", - dep_node, dep_dep_node - ); - - let node_index = self.try_mark_previous_green( - tcx, - data, - dep_dep_node_index, - dep_dep_node, - ); - if let Some(node_index) = node_index { + // We don't know the state of this dependency. + // We known it is not an eval_always node, since those get marked as `Invalid`. + // Let's try to mark it green recursively. + if self.try_mark_previous_green(tcx, data, dep_dep_node_index, dep_dep_node) + { debug!( "try_mark_previous_green({:?}) --- managed to MARK \ - dependency {:?} as green", + dependency {:?} as green", dep_node, dep_dep_node ); - current_deps.push(node_index); continue; } } // We failed to mark it green, so we try to force the query. - debug!( - "try_mark_previous_green({:?}) --- trying to force \ - dependency {:?}", - dep_node, dep_dep_node - ); - if tcx.try_force_from_dep_node(dep_dep_node) { - let dep_dep_node_color = data.colors.get(dep_dep_node_index); - - match dep_dep_node_color { - Some(DepNodeColor::Green(node_index)) => { - debug!( - "try_mark_previous_green({:?}) --- managed to \ - FORCE dependency {:?} to green", - dep_node, dep_dep_node - ); - current_deps.push(node_index); - } - Some(DepNodeColor::Red) => { - debug!( - "try_mark_previous_green({:?}) - END - \ - dependency {:?} was red after forcing", - dep_node, dep_dep_node - ); - return None; - } - None => { - if !tcx.has_errors_or_delayed_span_bugs() { - panic!( - "try_mark_previous_green() - Forcing the DepNode \ - should have set its color" - ) - } else { - // If the query we just forced has resulted in - // some kind of compilation error, we cannot rely on - // the dep-node color having been properly updated. - // This means that the query system has reached an - // invalid state. We let the compiler continue (by - // returning `None`) so it can emit error messages - // and wind down, but rely on the fact that this - // invalid state will not be persisted to the - // incremental compilation cache because of - // compilation errors being present. - debug!( - "try_mark_previous_green({:?}) - END - \ - dependency {:?} resulted in compilation error", - dep_node, dep_dep_node - ); - return None; - } - } - } - } else { - // The DepNode could not be forced. - debug!( - "try_mark_previous_green({:?}) - END - dependency {:?} \ - could not be forced", - dep_node, dep_dep_node - ); - return None; + if !self.try_force_previous_green(tcx, data, dep_dep_node_index) { + return false; } } } @@ -696,40 +739,42 @@ impl DepGraph { // There may be multiple threads trying to mark the same dep node green concurrently - let dep_node_index = { - // Copy the fingerprint from the previous graph, - // so we don't have to recompute it - let fingerprint = data.previous.fingerprint_by_index(prev_dep_node_index); - - // We allocating an entry for the node in the current dependency graph and - // adding all the appropriate edges imported from the previous graph - data.current.intern_node(*dep_node, current_deps, fingerprint) - }; + #[cfg(not(parallel_compiler))] + debug_assert_eq!( + data.colors.get(dep_node_index).color(), + None, + "DepGraph::try_mark_previous_green() - Duplicate DepNodeState \ + insertion for {:?}", + dep_node + ); // ... emitting any stored diagnostic ... // FIXME: Store the fact that a node has diagnostics in a bit in the dep graph somewhere // Maybe store a list on disk and encode this fact in the DepNodeState - let diagnostics = tcx.load_diagnostics(prev_dep_node_index); + let diagnostics = tcx.load_diagnostics(dep_node_index); #[cfg(not(parallel_compiler))] - debug_assert!( - data.colors.get(prev_dep_node_index).is_none(), + debug_assert_eq!( + data.colors.get(dep_node_index).color(), + None, "DepGraph::try_mark_previous_green() - Duplicate DepNodeColor \ - insertion for {:?}", + insertion for {:?}", dep_node ); if unlikely!(!diagnostics.is_empty()) { - self.emit_diagnostics(tcx, data, dep_node_index, prev_dep_node_index, diagnostics); + self.emit_diagnostics(tcx, data, dep_node_index, diagnostics); } // ... and finally storing a "Green" entry in the color map. // Multiple threads can all write the same color here - data.colors.insert(prev_dep_node_index, DepNodeColor::Green(dep_node_index)); + + data.colors.insert(dep_node_index, DepNodeState::Green); debug!("try_mark_previous_green({:?}) - END - successfully marked as green", dep_node); - Some(dep_node_index) + + true } /// Atomically emits some loaded diagnostics. @@ -741,12 +786,11 @@ impl DepGraph { tcx: Ctxt, data: &DepGraphData, dep_node_index: DepNodeIndex, - prev_dep_node_index: SerializedDepNodeIndex, diagnostics: Vec, ) { let mut emitting = data.emitting_diagnostics.lock(); - if data.colors.get(prev_dep_node_index) == Some(DepNodeColor::Green(dep_node_index)) { + if data.colors.get(dep_node_index) == DepNodeState::Green { // The node is already green so diagnostics must have been emitted already return; } @@ -767,7 +811,7 @@ impl DepGraph { } // Mark the node as green now that diagnostics are emitted - data.colors.insert(prev_dep_node_index, DepNodeColor::Green(dep_node_index)); + data.colors.insert(dep_node_index, DepNodeState::Green); // Remove the node from the set data.emitting_diagnostics.lock().remove(&dep_node_index); @@ -779,8 +823,7 @@ impl DepGraph { loop { data.emitting_diagnostics_cond_var.wait(&mut emitting); - if data.colors.get(prev_dep_node_index) == Some(DepNodeColor::Green(dep_node_index)) - { + if data.colors.get(dep_node_index) == DepNodeState::Green { break; } } @@ -807,11 +850,19 @@ impl DepGraph { let data = self.data.as_ref().unwrap(); for prev_index in data.colors.values.indices() { match data.colors.get(prev_index) { - Some(DepNodeColor::Green(_)) => { + DepNodeState::Green => { let dep_node = data.previous.index_to_node(prev_index); tcx.try_load_from_on_disk_cache(&dep_node); } - None | Some(DepNodeColor::Red) => { + + // There cannot be results stored for invalid indices. + DepNodeState::Invalid | + + // Unknown nodes are unused, so we don't want to promote these and we would + // not to mark their colors in order to do so anyway. + DepNodeState::Unknown | + + DepNodeState::Red => { // We can skip red nodes because a node can only be marked // as red if the query result was recomputed and thus is // already in memory. @@ -893,7 +944,7 @@ struct DepNodeData { /// The only operation that must manipulate both locks is adding new nodes, in which case /// we first acquire the `node_to_node_index` lock and then, once a new node is to be inserted, /// acquire the lock on `data.` -pub(super) struct CurrentDepGraph { +pub struct CurrentDepGraph { data: Lock>>, node_to_node_index: Sharded, DepNodeIndex>>, @@ -922,7 +973,7 @@ pub(super) struct CurrentDepGraph { } impl CurrentDepGraph { - fn new(prev_graph_node_count: usize) -> CurrentDepGraph { + pub fn new(prev_graph: &PreviousDepGraph) -> Self { use std::time::{SystemTime, UNIX_EPOCH}; let duration = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); @@ -946,11 +997,11 @@ impl CurrentDepGraph { // that we hopefully don't have to re-allocate during this compilation // session. The over-allocation is 2% plus a small constant to account // for the fact that in very small crates 2% might not be enough. - let new_node_count_estimate = (prev_graph_node_count * 102) / 100 + 200; + let new_node_count_estimate = (prev_graph.node_count() * 102) / 100 + 200; - CurrentDepGraph { + let mut current = CurrentDepGraph { data: Lock::new(IndexVec::with_capacity(new_node_count_estimate)), - node_to_node_index: Sharded::new(|| { + node_to_node_index: Sharded::new(|_| { FxHashMap::with_capacity_and_hasher( new_node_count_estimate / sharded::SHARDS, Default::default(), @@ -960,16 +1011,23 @@ impl CurrentDepGraph { forbidden_edge, total_read_count: AtomicU64::new(0), total_duplicate_read_count: AtomicU64::new(0), + }; + + // Fill in with the nodes from the previous session + for (idx, &node) in prev_graph.data.nodes.iter_enumerated() { + current.data.get_mut().push(DepNodeData { + node, + fingerprint: prev_graph.fingerprint_by_index(idx), + edges: prev_graph.edge_targets_from(idx).iter().copied().collect(), + }); } - } - fn complete_task( - &self, - node: DepNode, - task_deps: TaskDeps, - fingerprint: Fingerprint, - ) -> DepNodeIndex { - self.alloc_node(node, task_deps.reads, fingerprint) + // Fill in with the node to index map from the previous session + for (node, idx) in prev_graph.index.iter() { + current.node_to_node_index.get_shard_by_value_mut(node).insert(*node, *idx); + } + + current } fn complete_anon_task(&self, kind: K, task_deps: TaskDeps) -> DepNodeIndex { @@ -996,18 +1054,31 @@ impl CurrentDepGraph { self.intern_node(target_dep_node, task_deps.reads, Fingerprint::ZERO) } - fn alloc_node( + fn new_node( &self, dep_node: DepNode, edges: EdgesVec, fingerprint: Fingerprint, + previous: &PreviousDepGraph, ) -> DepNodeIndex { + debug_assert!(previous.node_to_index_opt(&dep_node).is_none()); debug_assert!( !self.node_to_node_index.get_shard_by_value(&dep_node).lock().contains_key(&dep_node) ); self.intern_node(dep_node, edges, fingerprint) } + fn update_node( + &self, + index: DepNodeIndex, + dep_node: DepNode, + edges: SmallVec<[DepNodeIndex; 8]>, + fingerprint: Fingerprint, + ) { + // Update the edges and fingerprint for this dep node + self.data.lock()[index] = DepNodeData { node: dep_node, edges, fingerprint }; + } + fn intern_node( &self, dep_node: DepNode, @@ -1028,6 +1099,16 @@ impl CurrentDepGraph { } impl DepGraphData { + fn current(&self, dep_node_index: DepNodeIndex) -> bool { + if dep_node_index.as_usize() < self.previous.node_count() { + // From this session if it's colored + self.colors.get(dep_node_index).color().is_some() + } else { + // From this session + true + } + } + #[inline(never)] fn read_index(&self, source: DepNodeIndex) { K::read_deps(|task_deps| { @@ -1097,38 +1178,18 @@ impl Default for TaskDeps { } } -// A data structure that stores Option values as a contiguous -// array, using one u32 per entry. struct DepNodeColorMap { - values: IndexVec, + values: IndexVec>, } -const COMPRESSED_NONE: u32 = 0; -const COMPRESSED_RED: u32 = 1; -const COMPRESSED_FIRST_GREEN: u32 = 2; - impl DepNodeColorMap { - fn new(size: usize) -> DepNodeColorMap { - DepNodeColorMap { values: (0..size).map(|_| AtomicU32::new(COMPRESSED_NONE)).collect() } - } - - fn get(&self, index: SerializedDepNodeIndex) -> Option { - match self.values[index].load(Ordering::Acquire) { - COMPRESSED_NONE => None, - COMPRESSED_RED => Some(DepNodeColor::Red), - value => { - Some(DepNodeColor::Green(DepNodeIndex::from_u32(value - COMPRESSED_FIRST_GREEN))) - } - } + #[inline] + fn get(&self, index: DepNodeIndex) -> DepNodeState { + self.values[index].load() } - fn insert(&self, index: SerializedDepNodeIndex, color: DepNodeColor) { - self.values[index].store( - match color { - DepNodeColor::Red => COMPRESSED_RED, - DepNodeColor::Green(index) => index.as_u32() + COMPRESSED_FIRST_GREEN, - }, - Ordering::Release, - ) + #[inline] + fn insert(&self, index: DepNodeIndex, state: DepNodeState) { + self.values[index].store(state) } } diff --git a/src/librustc_query_system/dep_graph/mod.rs b/src/librustc_query_system/dep_graph/mod.rs index fbc91575ede41..0dff6bd62375e 100644 --- a/src/librustc_query_system/dep_graph/mod.rs +++ b/src/librustc_query_system/dep_graph/mod.rs @@ -7,10 +7,13 @@ mod serialized; pub use dep_node::{DepNode, DepNodeParams, WorkProductId}; pub use graph::WorkProductFileKind; -pub use graph::{hash_result, DepGraph, DepNodeColor, DepNodeIndex, TaskDeps, WorkProduct}; +pub use graph::{ + hash_result, CurrentDepGraph, DepGraph, DepGraphArgs, DepNodeColor, DepNodeIndex, DepNodeState, + TaskDeps, WorkProduct, +}; pub use prev::PreviousDepGraph; pub use query::DepGraphQuery; -pub use serialized::{SerializedDepGraph, SerializedDepNodeIndex}; +pub use serialized::SerializedDepGraph; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::sync::Lock; @@ -42,7 +45,7 @@ pub trait DepContext: Copy { fn try_load_from_on_disk_cache(&self, dep_node: &DepNode); /// Load diagnostics associated to the node in the previous session. - fn load_diagnostics(&self, prev_dep_node_index: SerializedDepNodeIndex) -> Vec; + fn load_diagnostics(&self, dep_node_index: DepNodeIndex) -> Vec; /// Register diagnostics for the given node, for use in next session. fn store_diagnostics(&self, dep_node_index: DepNodeIndex, diagnostics: ThinVec); @@ -60,6 +63,7 @@ pub trait DepContext: Copy { /// Describe the different families of dependency nodes. pub trait DepKind: Copy + fmt::Debug + Eq + Ord + Hash { + /// An unused dep kind. const NULL: Self; /// Return whether this kind always require evaluation. diff --git a/src/librustc_query_system/dep_graph/prev.rs b/src/librustc_query_system/dep_graph/prev.rs index 5cba64cac4b34..9a6ec3ee48470 100644 --- a/src/librustc_query_system/dep_graph/prev.rs +++ b/src/librustc_query_system/dep_graph/prev.rs @@ -1,47 +1,73 @@ -use super::serialized::{SerializedDepGraph, SerializedDepNodeIndex}; -use super::{DepKind, DepNode}; +use super::serialized::SerializedDepGraph; +use super::{DepKind, DepNode, DepNodeIndex, DepNodeState}; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::AtomicCell; +use rustc_index::vec::IndexVec; -#[derive(Debug, RustcEncodable, RustcDecodable)] +#[derive(Debug)] pub struct PreviousDepGraph { - data: SerializedDepGraph, - index: FxHashMap, SerializedDepNodeIndex>, + pub(super) data: SerializedDepGraph, + pub(super) index: FxHashMap, DepNodeIndex>, + pub(super) unused: Vec, } impl Default for PreviousDepGraph { fn default() -> Self { - PreviousDepGraph { data: Default::default(), index: Default::default() } + Self { data: Default::default(), index: Default::default(), unused: Default::default() } } } impl PreviousDepGraph { - pub fn new(data: SerializedDepGraph) -> PreviousDepGraph { + pub fn new_and_state( + data: SerializedDepGraph, + ) -> (Self, IndexVec>) { + let mut unused = Vec::new(); + + let state: IndexVec<_, _> = data + .nodes + .iter_enumerated() + .map(|(index, node)| { + if node.kind == DepKind::NULL { + // There might be `DepKind::NULL` nodes due to thread-local dep node indices + // that didn't get assigned anything. + // We also changed outdated nodes to `DepKind::NULL`. + unused.push(index); + AtomicCell::new(DepNodeState::Invalid) + } else { + AtomicCell::new(DepNodeState::Unknown) + } + }) + .collect(); + let index: FxHashMap<_, _> = - data.nodes.iter_enumerated().map(|(idx, &dep_node)| (dep_node, idx)).collect(); - PreviousDepGraph { data, index } + data.nodes + .iter_enumerated() + .filter_map(|(idx, &dep_node)| { + if dep_node.kind == DepKind::NULL { None } else { Some((dep_node, idx)) } + }) + .collect(); + + (PreviousDepGraph { data, index, unused }, state) } #[inline] - pub fn edge_targets_from( - &self, - dep_node_index: SerializedDepNodeIndex, - ) -> &[SerializedDepNodeIndex] { + pub fn edge_targets_from(&self, dep_node_index: DepNodeIndex) -> &[DepNodeIndex] { self.data.edge_targets_from(dep_node_index) } #[inline] - pub fn index_to_node(&self, dep_node_index: SerializedDepNodeIndex) -> DepNode { + pub fn index_to_node(&self, dep_node_index: DepNodeIndex) -> DepNode { self.data.nodes[dep_node_index] } #[inline] - pub fn node_to_index(&self, dep_node: &DepNode) -> SerializedDepNodeIndex { + pub fn node_to_index(&self, dep_node: &DepNode) -> DepNodeIndex { self.index[dep_node] } #[inline] - pub fn node_to_index_opt(&self, dep_node: &DepNode) -> Option { + pub fn node_to_index_opt(&self, dep_node: &DepNode) -> Option { self.index.get(dep_node).cloned() } @@ -51,11 +77,11 @@ impl PreviousDepGraph { } #[inline] - pub fn fingerprint_by_index(&self, dep_node_index: SerializedDepNodeIndex) -> Fingerprint { + pub fn fingerprint_by_index(&self, dep_node_index: DepNodeIndex) -> Fingerprint { self.data.fingerprints[dep_node_index] } pub fn node_count(&self) -> usize { - self.index.len() + self.data.nodes.len() } } diff --git a/src/librustc_query_system/dep_graph/serialized.rs b/src/librustc_query_system/dep_graph/serialized.rs index 4a89da23ea6a5..ffe11230ae6b2 100644 --- a/src/librustc_query_system/dep_graph/serialized.rs +++ b/src/librustc_query_system/dep_graph/serialized.rs @@ -1,28 +1,24 @@ //! The data that we will serialize and deserialize. -use super::{DepKind, DepNode}; +use super::{DepKind, DepNode, DepNodeIndex}; use rustc_data_structures::fingerprint::Fingerprint; use rustc_index::vec::IndexVec; -rustc_index::newtype_index! { - pub struct SerializedDepNodeIndex { .. } -} - /// Data for use when recompiling the **current crate**. #[derive(Debug, RustcEncodable, RustcDecodable)] pub struct SerializedDepGraph { /// The set of all DepNodes in the graph - pub nodes: IndexVec>, + pub nodes: IndexVec>, /// The set of all Fingerprints in the graph. Each Fingerprint corresponds to /// the DepNode at the same index in the nodes vector. - pub fingerprints: IndexVec, + pub fingerprints: IndexVec, /// For each DepNode, stores the list of edges originating from that /// DepNode. Encoded as a [start, end) pair indexing into edge_list_data, /// which holds the actual DepNodeIndices of the target nodes. - pub edge_list_indices: IndexVec, + pub edge_list_indices: IndexVec, /// A flattened list of all edge targets in the graph. Edge sources are /// implicit in edge_list_indices. - pub edge_list_data: Vec, + pub edge_list_data: Vec, } impl Default for SerializedDepGraph { @@ -38,7 +34,7 @@ impl Default for SerializedDepGraph { impl SerializedDepGraph { #[inline] - pub fn edge_targets_from(&self, source: SerializedDepNodeIndex) -> &[SerializedDepNodeIndex] { + pub fn edge_targets_from(&self, source: DepNodeIndex) -> &[DepNodeIndex] { let targets = self.edge_list_indices[source]; &self.edge_list_data[targets.0 as usize..targets.1 as usize] } diff --git a/src/librustc_query_system/query/config.rs b/src/librustc_query_system/query/config.rs index 20dad0bd47ebc..99373035f4c65 100644 --- a/src/librustc_query_system/query/config.rs +++ b/src/librustc_query_system/query/config.rs @@ -1,7 +1,7 @@ //! Query configuration and description traits. use crate::dep_graph::DepNode; -use crate::dep_graph::SerializedDepNodeIndex; +use crate::dep_graph::DepNodeIndex; use crate::query::caches::QueryCache; use crate::query::plumbing::CycleError; use crate::query::{QueryContext, QueryState}; @@ -54,7 +54,7 @@ pub trait QueryDescription: QueryAccessors { false } - fn try_load_from_disk(_: CTX, _: SerializedDepNodeIndex) -> Option { + fn try_load_from_disk(_: CTX, _: DepNodeIndex) -> Option { panic!("QueryDescription::load_from_disk() called for an unsupported query.") } } @@ -76,7 +76,7 @@ where false } - default fn try_load_from_disk(_: CTX, _: SerializedDepNodeIndex) -> Option { + default fn try_load_from_disk(_: CTX, _: DepNodeIndex) -> Option { panic!("QueryDescription::load_from_disk() called for an unsupported query.") } } diff --git a/src/librustc_query_system/query/plumbing.rs b/src/librustc_query_system/query/plumbing.rs index b371a914c6fce..c35f59766d880 100644 --- a/src/librustc_query_system/query/plumbing.rs +++ b/src/librustc_query_system/query/plumbing.rs @@ -2,8 +2,7 @@ //! generate the actual methods on tcx which find and execute the provider, //! manage the caches, and so forth. -use crate::dep_graph::{DepKind, DepNode}; -use crate::dep_graph::{DepNodeIndex, SerializedDepNodeIndex}; +use crate::dep_graph::{DepKind, DepNode, DepNodeIndex}; use crate::query::caches::QueryCache; use crate::query::config::QueryDescription; use crate::query::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryShardJobId}; @@ -439,12 +438,11 @@ where // `try_mark_green()`, so we can ignore them here. let loaded = tcx.start_query(job.id, None, |tcx| { let marked = tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node); - marked.map(|(prev_dep_node_index, dep_node_index)| { + marked.map(|dep_node_index| { ( load_from_disk_and_cache_in_memory::( tcx, key.clone(), - prev_dep_node_index, dep_node_index, &dep_node, ), @@ -466,7 +464,6 @@ where fn load_from_disk_and_cache_in_memory( tcx: CTX, key: Q::Key, - prev_dep_node_index: SerializedDepNodeIndex, dep_node_index: DepNodeIndex, dep_node: &DepNode, ) -> Q::Value @@ -482,7 +479,7 @@ where // First we try to load the result from the on-disk cache. let result = if Q::cache_on_disk(tcx, key.clone(), None) { let prof_timer = tcx.profiler().incr_cache_loading(); - let result = Q::try_load_from_disk(tcx, prev_dep_node_index); + let result = Q::try_load_from_disk(tcx, dep_node_index); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); // We always expect to find a cached result for things that @@ -516,7 +513,7 @@ where // If `-Zincremental-verify-ich` is specified, re-hash results from // the cache and make sure that they have the expected fingerprint. if unlikely!(tcx.incremental_verify_ich()) { - incremental_verify_ich::(tcx, &result, dep_node, dep_node_index); + incremental_verify_ich::(tcx, &result, dep_node); } result @@ -524,31 +521,20 @@ where #[inline(never)] #[cold] -fn incremental_verify_ich( - tcx: CTX, - result: &Q::Value, - dep_node: &DepNode, - dep_node_index: DepNodeIndex, -) where +fn incremental_verify_ich(tcx: CTX, result: &Q::Value, dep_node: &DepNode) +where CTX: QueryContext, Q: QueryDescription, { - assert!( - Some(tcx.dep_graph().fingerprint_of(dep_node_index)) - == tcx.dep_graph().prev_fingerprint_of(dep_node), - "fingerprint for green query instance not loaded from cache: {:?}", - dep_node, - ); - debug!("BEGIN verify_ich({:?})", dep_node); let mut hcx = tcx.create_stable_hashing_context(); let new_hash = Q::hash_result(&mut hcx, result).unwrap_or(Fingerprint::ZERO); debug!("END verify_ich({:?})", dep_node); - let old_hash = tcx.dep_graph().fingerprint_of(dep_node_index); + let old_hash = tcx.dep_graph().prev_fingerprint_of(dep_node); - assert!(new_hash == old_hash, "found unstable fingerprints for {:?}", dep_node,); + assert!(Some(new_hash) == old_hash, "found unstable fingerprints for {:?}", dep_node,); } #[inline(always)] @@ -659,7 +645,7 @@ where // in-memory cache, or another query down the line will. let _ = get_query::(tcx, DUMMY_SP, key); } - Some((_, dep_node_index)) => { + Some(dep_node_index) => { tcx.profiler().query_cache_hit(dep_node_index.into()); } }