From 481bebaf5f1204ce1582bf1a31bf170a72b4d557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Fri, 12 Apr 2019 09:22:05 +0200 Subject: [PATCH 01/17] Add a Worker type which allows executing code in the background --- src/librustc_data_structures/sync.rs | 2 + src/librustc_data_structures/sync/worker.rs | 169 ++++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 src/librustc_data_structures/sync/worker.rs diff --git a/src/librustc_data_structures/sync.rs b/src/librustc_data_structures/sync.rs index 73247c1469efd..be5ae47d794d7 100644 --- a/src/librustc_data_structures/sync.rs +++ b/src/librustc_data_structures/sync.rs @@ -23,6 +23,8 @@ use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; use crate::owning_ref::{Erased, OwningRef}; +pub mod worker; + pub fn serial_join(oper_a: A, oper_b: B) -> (RA, RB) where A: FnOnce() -> RA, B: FnOnce() -> RB diff --git a/src/librustc_data_structures/sync/worker.rs b/src/librustc_data_structures/sync/worker.rs new file mode 100644 index 0000000000000..8e4ae3e4de2fe --- /dev/null +++ b/src/librustc_data_structures/sync/worker.rs @@ -0,0 +1,169 @@ +use super::{Lrc, Lock}; + +pub trait Worker: super::Send { + type Message: super::Send; + type Result: super::Send; + + fn message(&mut self, msg: Self::Message); + fn complete(self) -> Self::Result; +} + +pub use executor::WorkerExecutor; + +#[cfg(parallel_compiler)] +mod executor { + use super::*; + use crate::jobserver; + use parking_lot::Condvar; + use std::mem; + + struct WorkerQueue { + scheduled: bool, + complete: bool, + messages: Vec, + result: Option, + } + + /// Allows executing a worker on any Rayon thread, + /// sending it messages and waiting for it to complete its computation. + pub struct WorkerExecutor { + queue: Lock>, + worker: Lock>, + #[cfg(parallel_compiler)] + cond_var: Condvar, + } + + impl WorkerExecutor { + pub fn new(worker: T) -> Self { + WorkerExecutor { + queue: Lock::new(WorkerQueue { + scheduled: false, + complete: false, + messages: Vec::new(), + result: None, + }), + worker: Lock::new(Some(worker)), + #[cfg(parallel_compiler)] + cond_var: Condvar::new(), + } + } + + fn try_run_worker(&self) { + if let Some(mut worker) = self.worker.try_lock() { + self.run_worker(&mut *worker); + } + } + + fn run_worker(&self, worker: &mut Option) { + let worker_ref = if let Some(worker_ref) = worker.as_mut() { + worker_ref + } else { + return + }; + loop { + let msgs = { + let mut queue = self.queue.lock(); + let msgs = mem::replace(&mut queue.messages, Vec::new()); + if msgs.is_empty() { + queue.scheduled = false; + if queue.complete { + queue.result = Some(worker.take().unwrap().complete()); + self.cond_var.notify_all(); + } + break; + } + msgs + }; + for msg in msgs { + worker_ref.message(msg); + } + } + } + + pub fn complete(&self) -> T::Result { + let mut queue = self.queue.lock(); + assert!(!queue.complete); + queue.complete = true; + if !queue.scheduled { + // The worker is not scheduled to run, just run it on the current thread. + queue.scheduled = true; + mem::drop(queue); + self.run_worker(&mut *self.worker.lock()); + queue = self.queue.lock(); + } else if let Some(mut worker) = self.worker.try_lock() { + // Try to run the worker on the current thread. + // It was scheduled to run, but it may not have started yet. + // If we are using a single thread, it may never start at all. + mem::drop(queue); + self.run_worker(&mut *worker); + queue = self.queue.lock(); + } else { + // The worker must be running on some other thread, + // and will eventually look at the queue again, since queue.scheduled is true. + // Wait for it. + + #[cfg(parallel_compiler)] + { + // Wait for the result + jobserver::release_thread(); + self.cond_var.wait(&mut queue); + jobserver::acquire_thread(); + } + } + queue.result.take().unwrap() + } + + fn queue_message(&self, msg: T::Message) -> bool { + let mut queue = self.queue.lock(); + queue.messages.push(msg); + let was_scheduled = queue.scheduled; + if !was_scheduled { + queue.scheduled = true; + } + was_scheduled + } + + pub fn message_in_pool(self: &Lrc, msg: T::Message) + where + T: 'static + { + if !self.queue_message(msg) { + let this = self.clone(); + #[cfg(parallel_compiler)] + rayon::spawn(move || this.try_run_worker()); + #[cfg(not(parallel_compiler))] + this.try_run_worker(); + } + } + } +} + +#[cfg(not(parallel_compiler))] +mod executor { + use super::*; + + pub struct WorkerExecutor { + worker: Lock>, + } + + impl WorkerExecutor { + pub fn new(worker: T) -> Self { + WorkerExecutor { + worker: Lock::new(Some(worker)), + } + } + + #[inline] + pub fn complete(&self) -> T::Result { + self.worker.lock().take().unwrap().complete() + } + + #[inline] + pub fn message_in_pool(self: &Lrc, msg: T::Message) + where + T: 'static + { + self.worker.lock().as_mut().unwrap().message(msg); + } + } +} \ No newline at end of file From ed3072d57ab837766d8e2b3f0f01faf0024c2503 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Tue, 16 Apr 2019 10:45:01 +0200 Subject: [PATCH 02/17] Fix inherent_impls --- .../coherence/inherent_impls.rs | 33 +++---------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/src/librustc_typeck/coherence/inherent_impls.rs b/src/librustc_typeck/coherence/inherent_impls.rs index 6088c03fc0681..0b04160522047 100644 --- a/src/librustc_typeck/coherence/inherent_impls.rs +++ b/src/librustc_typeck/coherence/inherent_impls.rs @@ -7,7 +7,6 @@ //! `tcx.inherent_impls(def_id)`). That value, however, //! is computed by selecting an idea from this table. -use rustc::dep_graph::DepKind; use rustc::hir::def_id::{CrateNum, DefId, LOCAL_CRATE}; use rustc::hir; use rustc::hir::itemlikevisit::ItemLikeVisitor; @@ -36,35 +35,11 @@ pub fn crate_inherent_impls<'tcx>( pub fn inherent_impls<'tcx>(tcx: TyCtxt<'tcx>, ty_def_id: DefId) -> &'tcx [DefId] { assert!(ty_def_id.is_local()); - // NB. Until we adopt the red-green dep-tracking algorithm (see - // [the plan] for details on that), we do some hackery here to get - // the dependencies correct. Basically, we use a `with_ignore` to - // read the result we want. If we didn't have the `with_ignore`, - // we would wind up with a dependency on the entire crate, which - // we don't want. Then we go and add dependencies on all the impls - // in the result (which is what we wanted). - // - // The result is a graph with an edge from `Hir(I)` for every impl - // `I` defined on some type `T` to `CoherentInherentImpls(T)`, - // thus ensuring that if any of those impls change, the set of - // inherent impls is considered dirty. - // - // [the plan]: https://github.com/rust-lang/rust-roadmap/issues/4 - - let result = tcx.dep_graph.with_ignore(|| { - let crate_map = tcx.crate_inherent_impls(ty_def_id.krate); - match crate_map.inherent_impls.get(&ty_def_id) { - Some(v) => &v[..], - None => &[], - } - }); - - for &impl_def_id in &result[..] { - let def_path_hash = tcx.def_path_hash(impl_def_id); - tcx.dep_graph.read(def_path_hash.to_dep_node(DepKind::Hir)); + let crate_map = tcx.crate_inherent_impls(ty_def_id.krate); + match crate_map.inherent_impls.get(&ty_def_id) { + Some(v) => &v[..], + None => &[], } - - result } struct InherentCollect<'tcx> { From 5604fe903adb945597641e25dc9398afef69d9cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Tue, 16 Apr 2019 09:59:55 +0200 Subject: [PATCH 03/17] Serialize dep nodes in the background --- src/librustc/dep_graph/graph.rs | 106 +++++++--------- src/librustc/dep_graph/prev.rs | 62 +++++++-- src/librustc/dep_graph/serialized.rs | 132 ++++++++++++++++---- src/librustc/hir/map/collector.rs | 6 +- src/librustc/lib.rs | 1 + src/librustc/ty/context.rs | 11 +- src/librustc/ty/query/plumbing.rs | 13 +- src/librustc_data_structures/lib.rs | 1 + src/librustc_data_structures/sync/worker.rs | 2 +- src/librustc_incremental/lib.rs | 2 +- src/librustc_incremental/persist/fs.rs | 9 +- src/librustc_incremental/persist/load.rs | 24 +++- src/librustc_incremental/persist/mod.rs | 2 +- src/librustc_incremental/persist/save.rs | 83 +++++++----- src/librustc_interface/queries.rs | 12 +- 15 files changed, 305 insertions(+), 161 deletions(-) diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index c2e3c12cea8ed..058d3fb9e830a 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -5,6 +5,7 @@ use rustc_data_structures::indexed_vec::{Idx, IndexVec}; use smallvec::SmallVec; use rustc_data_structures::sync::{Lrc, Lock, AtomicU32, Ordering}; use std::env; +use std::fs::File; use std::hash::Hash; use std::collections::hash_map::Entry; use crate::ty::{self, TyCtxt}; @@ -17,7 +18,7 @@ use super::debug::EdgeFilter; use super::dep_node::{DepNode, DepKind, WorkProductId}; use super::query::DepGraphQuery; use super::safe::DepGraphSafe; -use super::serialized::{SerializedDepGraph, SerializedDepNodeIndex}; +use super::serialized::{SerializedDepNodeIndex, Serializer}; use super::prev::PreviousDepGraph; #[derive(Clone)] @@ -30,7 +31,7 @@ newtype_index! { } impl DepNodeIndex { - const INVALID: DepNodeIndex = DepNodeIndex::MAX; + pub(super) const INVALID: DepNodeIndex = DepNodeIndex::MAX; } #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] @@ -76,7 +77,7 @@ struct DepGraphData { dep_node_debug: Lock>, // Used for testing, only populated when -Zquery-dep-graph is specified. - loaded_from_cache: Lock>, + loaded_from_cache: Lock>, } pub fn hash_result(hcx: &mut StableHashingContext<'_>, result: &R) -> Option @@ -90,15 +91,18 @@ where } impl DepGraph { - pub fn new(prev_graph: PreviousDepGraph, - prev_work_products: FxHashMap) -> DepGraph { + pub fn new( + prev_graph: PreviousDepGraph, + prev_work_products: FxHashMap, + file: File, + ) -> DepGraph { let prev_graph_node_count = prev_graph.node_count(); DepGraph { data: Some(Lrc::new(DepGraphData { previous_work_products: prev_work_products, dep_node_debug: Default::default(), - current: Lock::new(CurrentDepGraph::new(prev_graph_node_count)), + current: Lock::new(CurrentDepGraph::new(prev_graph_node_count, file)), emitted_diagnostics: Default::default(), emitted_diagnostics_cond_var: Condvar::new(), previous: prev_graph, @@ -121,6 +125,8 @@ impl DepGraph { } pub fn query(&self) -> DepGraphQuery { + // FIXME + panic!()/* let current_dep_graph = self.data.as_ref().unwrap().current.borrow(); let nodes: Vec<_> = current_dep_graph.data.iter().map(|n| n.node).collect(); let mut edges = Vec::new(); @@ -132,7 +138,7 @@ impl DepGraph { } } - DepGraphQuery::new(&nodes[..], &edges[..]) + DepGraphQuery::new(&nodes[..], &edges[..])*/ } pub fn assert_ignored(&self) @@ -435,9 +441,10 @@ impl DepGraph { } #[inline] - pub fn fingerprint_of(&self, dep_node_index: DepNodeIndex) -> Fingerprint { + pub fn fingerprint_of(&self, _dep_node_index: DepNodeIndex) -> Fingerprint { + panic!()/* let current = self.data.as_ref().expect("dep graph enabled").current.borrow_mut(); - current.data[dep_node_index].fingerprint + current.data[dep_node_index].fingerprint*/ } pub fn prev_fingerprint_of(&self, dep_node: &DepNode) -> Option { @@ -500,41 +507,9 @@ impl DepGraph { } } - pub fn serialize(&self) -> SerializedDepGraph { - let current_dep_graph = self.data.as_ref().unwrap().current.borrow(); - - let fingerprints: IndexVec = - current_dep_graph.data.iter().map(|d| d.fingerprint).collect(); - let nodes: IndexVec = - current_dep_graph.data.iter().map(|d| d.node).collect(); - - let total_edge_count: usize = current_dep_graph.data.iter() - .map(|d| d.edges.len()) - .sum(); - - let mut edge_list_indices = IndexVec::with_capacity(nodes.len()); - let mut edge_list_data = Vec::with_capacity(total_edge_count); - - for (current_dep_node_index, edges) in current_dep_graph.data.iter_enumerated() - .map(|(i, d)| (i, &d.edges)) { - let start = edge_list_data.len() as u32; - // This should really just be a memcpy :/ - edge_list_data.extend(edges.iter().map(|i| SerializedDepNodeIndex::new(i.index()))); - let end = edge_list_data.len() as u32; - - debug_assert_eq!(current_dep_node_index.index(), edge_list_indices.len()); - edge_list_indices.push((start, end)); - } - - debug_assert!(edge_list_data.len() <= ::std::u32::MAX as usize); - debug_assert_eq!(edge_list_data.len(), total_edge_count); - - SerializedDepGraph { - nodes, - fingerprints, - edge_list_indices, - edge_list_data, - } + pub fn serialize(&self) { + // FIXME: Can this deadlock? + self.data.as_ref().unwrap().current.lock().serializer.complete() } pub fn node_color(&self, dep_node: &DepNode) -> Option { @@ -870,23 +845,20 @@ impl DepGraph { } } - pub fn mark_loaded_from_cache(&self, dep_node_index: DepNodeIndex, state: bool) { - debug!("mark_loaded_from_cache({:?}, {})", - self.data.as_ref().unwrap().current.borrow().data[dep_node_index].node, - state); + pub fn mark_loaded_from_cache(&self, dep_node: DepNode, state: bool) { + debug!("mark_loaded_from_cache({:?}, {})", dep_node, state); self.data .as_ref() .unwrap() .loaded_from_cache .borrow_mut() - .insert(dep_node_index, state); + .insert(dep_node, state); } pub fn was_loaded_from_cache(&self, dep_node: &DepNode) -> Option { let data = self.data.as_ref().unwrap(); - let dep_node_index = data.current.borrow().node_to_node_index[dep_node]; - data.loaded_from_cache.borrow().get(&dep_node_index).cloned() + data.loaded_from_cache.borrow().get(&dep_node).cloned() } } @@ -935,15 +907,15 @@ pub enum WorkProductFileKind { BytecodeCompressed, } -#[derive(Clone)] -struct DepNodeData { - node: DepNode, - edges: SmallVec<[DepNodeIndex; 8]>, - fingerprint: Fingerprint, +#[derive(Clone, Debug)] +pub(super) struct DepNodeData { + pub(super) node: DepNode, + pub(super) edges: SmallVec<[DepNodeIndex; 8]>, + pub(super) fingerprint: Fingerprint, } pub(super) struct CurrentDepGraph { - data: IndexVec, + nodes: usize, node_to_node_index: FxHashMap, #[allow(dead_code)] forbidden_edge: Option, @@ -963,10 +935,13 @@ pub(super) struct CurrentDepGraph { total_read_count: u64, total_duplicate_read_count: u64, + + /// Produces the serialized dep graph for the next session, + serializer: Serializer, } impl CurrentDepGraph { - fn new(prev_graph_node_count: usize) -> CurrentDepGraph { + fn new(prev_graph_node_count: usize, file: File) -> CurrentDepGraph { use std::time::{SystemTime, UNIX_EPOCH}; let duration = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); @@ -996,7 +971,7 @@ impl CurrentDepGraph { let new_node_count_estimate = (prev_graph_node_count * 102) / 100 + 200; CurrentDepGraph { - data: IndexVec::with_capacity(new_node_count_estimate), + nodes: 0, node_to_node_index: FxHashMap::with_capacity_and_hasher( new_node_count_estimate, Default::default(), @@ -1005,6 +980,7 @@ impl CurrentDepGraph { forbidden_edge, total_read_count: 0, total_duplicate_read_count: 0, + serializer: Serializer::new(file), } } @@ -1057,13 +1033,14 @@ impl CurrentDepGraph { edges: SmallVec<[DepNodeIndex; 8]>, fingerprint: Fingerprint ) -> (DepNodeIndex, bool) { - debug_assert_eq!(self.node_to_node_index.len(), self.data.len()); + debug_assert_eq!(self.node_to_node_index.len(), self.nodes); match self.node_to_node_index.entry(dep_node) { Entry::Occupied(entry) => (*entry.get(), false), Entry::Vacant(entry) => { - let dep_node_index = DepNodeIndex::new(self.data.len()); - self.data.push(DepNodeData { + let dep_node_index = DepNodeIndex::new(self.nodes); + self.nodes += 1; + self.serializer.serialize(DepNodeData { node: dep_node, edges, fingerprint @@ -1087,7 +1064,7 @@ impl DepGraphData { if task_deps.read_set.insert(source) { task_deps.reads.push(source); - #[cfg(debug_assertions)] + /*#[cfg(debug_assertions)] { if let Some(target) = task_deps.node { let graph = self.current.lock(); @@ -1100,7 +1077,7 @@ impl DepGraphData { } } } - } + }*/ } else if cfg!(debug_assertions) { self.current.lock().total_duplicate_read_count += 1; } @@ -1111,6 +1088,7 @@ impl DepGraphData { pub struct TaskDeps { #[cfg(debug_assertions)] + #[allow(dead_code)] node: Option, reads: SmallVec<[DepNodeIndex; 8]>, read_set: FxHashSet, diff --git a/src/librustc/dep_graph/prev.rs b/src/librustc/dep_graph/prev.rs index d971690bbe317..55f210b01903b 100644 --- a/src/librustc/dep_graph/prev.rs +++ b/src/librustc/dep_graph/prev.rs @@ -1,21 +1,64 @@ use crate::ich::Fingerprint; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::indexed_vec::IndexVec; use super::dep_node::DepNode; use super::serialized::{SerializedDepGraph, SerializedDepNodeIndex}; #[derive(Debug, RustcEncodable, RustcDecodable, Default)] pub struct PreviousDepGraph { - data: SerializedDepGraph, + /// Maps from dep nodes to their previous index, if any. index: FxHashMap, + /// The set of all DepNodes in the graph + nodes: IndexVec, + /// The set of all Fingerprints in the graph. Each Fingerprint corresponds to + /// the DepNode at the same index in the nodes vector. + fingerprints: IndexVec, + /// For each DepNode, stores the list of edges originating from that + /// DepNode. Encoded as a [start, end) pair indexing into edge_list_data, + /// which holds the actual DepNodeIndices of the target nodes. + edge_list_indices: IndexVec, + /// A flattened list of all edge targets in the graph. Edge sources are + /// implicit in edge_list_indices. + edge_list_data: Vec, } impl PreviousDepGraph { - pub fn new(data: SerializedDepGraph) -> PreviousDepGraph { - let index: FxHashMap<_, _> = data.nodes + pub fn new(graph: SerializedDepGraph) -> PreviousDepGraph { + let index: FxHashMap<_, _> = graph.nodes .iter_enumerated() - .map(|(idx, &dep_node)| (dep_node, idx)) + .map(|(idx, dep_node)| (dep_node.node, idx)) .collect(); - PreviousDepGraph { data, index } + + let fingerprints: IndexVec = + graph.nodes.iter().map(|d| d.fingerprint).collect(); + let nodes: IndexVec = + graph.nodes.iter().map(|d| d.node).collect(); + + let total_edge_count: usize = graph.nodes.iter().map(|d| d.deps.len()).sum(); + + let mut edge_list_indices = IndexVec::with_capacity(nodes.len()); + let mut edge_list_data = Vec::with_capacity(total_edge_count); + + for (current_dep_node_index, edges) in graph.nodes.iter_enumerated() + .map(|(i, d)| (i, &d.deps)) { + let start = edge_list_data.len() as u32; + edge_list_data.extend(edges.iter().cloned()); + let end = edge_list_data.len() as u32; + + debug_assert_eq!(current_dep_node_index.index(), edge_list_indices.len()); + edge_list_indices.push((start, end)); + } + + debug_assert!(edge_list_data.len() <= ::std::u32::MAX as usize); + debug_assert_eq!(edge_list_data.len(), total_edge_count); + + PreviousDepGraph { + fingerprints, + nodes, + edge_list_indices, + edge_list_data, + index, + } } #[inline] @@ -23,12 +66,13 @@ impl PreviousDepGraph { &self, dep_node_index: SerializedDepNodeIndex ) -> &[SerializedDepNodeIndex] { - self.data.edge_targets_from(dep_node_index) + let targets = self.edge_list_indices[dep_node_index]; + &self.edge_list_data[targets.0 as usize..targets.1 as usize] } #[inline] pub fn index_to_node(&self, dep_node_index: SerializedDepNodeIndex) -> DepNode { - self.data.nodes[dep_node_index] + self.nodes[dep_node_index] } #[inline] @@ -45,14 +89,14 @@ impl PreviousDepGraph { pub fn fingerprint_of(&self, dep_node: &DepNode) -> Option { self.index .get(dep_node) - .map(|&node_index| self.data.fingerprints[node_index]) + .map(|&node_index| self.fingerprints[node_index]) } #[inline] pub fn fingerprint_by_index(&self, dep_node_index: SerializedDepNodeIndex) -> Fingerprint { - self.data.fingerprints[dep_node_index] + self.fingerprints[dep_node_index] } pub fn node_count(&self) -> usize { diff --git a/src/librustc/dep_graph/serialized.rs b/src/librustc/dep_graph/serialized.rs index b64f71ed908d8..2688d2a9e2c20 100644 --- a/src/librustc/dep_graph/serialized.rs +++ b/src/librustc/dep_graph/serialized.rs @@ -1,36 +1,120 @@ -//! The data that we will serialize and deserialize. - +use rustc_data_structures::sync::worker::{Worker, WorkerExecutor}; +use rustc_data_structures::sync::Lrc; +use rustc_data_structures::{unlikely, cold_path}; +use rustc_data_structures::indexed_vec::{IndexVec, Idx}; +use rustc_serialize::opaque; +use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; +use std::mem; +use std::fs::File; +use std::io::Write; +use super::graph::DepNodeData; use crate::dep_graph::DepNode; use crate::ich::Fingerprint; -use rustc_data_structures::indexed_vec::{IndexVec, Idx}; newtype_index! { pub struct SerializedDepNodeIndex { .. } } /// Data for use when recompiling the **current crate**. -#[derive(Debug, RustcEncodable, RustcDecodable, Default)] +#[derive(Debug, Default)] pub struct SerializedDepGraph { - /// The set of all DepNodes in the graph - pub nodes: IndexVec, - /// The set of all Fingerprints in the graph. Each Fingerprint corresponds to - /// the DepNode at the same index in the nodes vector. - pub fingerprints: IndexVec, - /// For each DepNode, stores the list of edges originating from that - /// DepNode. Encoded as a [start, end) pair indexing into edge_list_data, - /// which holds the actual DepNodeIndices of the target nodes. - pub edge_list_indices: IndexVec, - /// A flattened list of all edge targets in the graph. Edge sources are - /// implicit in edge_list_indices. - pub edge_list_data: Vec, -} - -impl SerializedDepGraph { + pub nodes: IndexVec, +} + +impl Decodable for SerializedDepGraph { + fn decode(d: &mut D) -> Result { + let mut nodes = IndexVec::new(); + loop { + let count = d.read_usize()?; + if count == 0 { + break; + } + for _ in 0..count { + nodes.push(SerializedNode::decode(d)?); + } + } + Ok(SerializedDepGraph { + nodes, + }) + } +} + +#[derive(Debug, RustcDecodable)] +pub struct SerializedNode { + pub node: DepNode, + pub deps: Vec, + pub fingerprint: Fingerprint, +} + +struct SerializerWorker { + file: File, +} + +impl Worker for SerializerWorker { + type Message = (usize, Vec); + type Result = (); + + fn message(&mut self, (buffer_size_est, nodes): (usize, Vec)) { + let mut encoder = opaque::Encoder::new(Vec::with_capacity(buffer_size_est * 4)); + assert!(!nodes.is_empty()); + encoder.emit_usize(nodes.len()).ok(); + for data in nodes { + data.node.encode(&mut encoder).ok(); + data.edges.encode(&mut encoder).ok(); + data.fingerprint.encode(&mut encoder).ok(); + } + self.file.write_all(&encoder.into_inner()).expect("unable to write to temp dep graph"); + } + + fn complete(mut self) { + let mut encoder = opaque::Encoder::new(Vec::with_capacity(16)); + encoder.emit_usize(0).ok(); + self.file.write_all(&encoder.into_inner()).expect("unable to write to temp dep graph"); + } +} + +const BUFFER_SIZE: usize = 800000; + +pub struct Serializer { + worker: Lrc>, + buffer: Vec, + buffer_size: usize, +} + +impl Serializer { + pub fn new(file: File) -> Self { + Serializer { + worker: Lrc::new(WorkerExecutor::new(SerializerWorker { + file, + })), + buffer: Vec::with_capacity(BUFFER_SIZE), + buffer_size: 0, + } + } + + fn flush(&mut self) { + let msgs = mem::replace(&mut self.buffer, Vec::with_capacity(BUFFER_SIZE)); + let buffer_size = self.buffer_size; + self.buffer_size = 0; + self.worker.message_in_pool((buffer_size, msgs)); + } + #[inline] - pub fn edge_targets_from(&self, - source: SerializedDepNodeIndex) - -> &[SerializedDepNodeIndex] { - let targets = self.edge_list_indices[source]; - &self.edge_list_data[targets.0 as usize..targets.1 as usize] + pub(super) fn serialize(&mut self, data: DepNodeData) { + let edges = data.edges.len(); + self.buffer.push(data); + self.buffer_size += 8 + edges; + if unlikely!(self.buffer_size >= BUFFER_SIZE) { + cold_path(|| { + self.flush(); + }) + } + } + + pub fn complete(&mut self) { + if self.buffer.len() > 0 { + self.flush(); + } + self.worker.complete() } } diff --git a/src/librustc/hir/map/collector.rs b/src/librustc/hir/map/collector.rs index 8a59f6b69bcd6..704d44dc8ab6b 100644 --- a/src/librustc/hir/map/collector.rs +++ b/src/librustc/hir/map/collector.rs @@ -62,7 +62,11 @@ where let dep_node_index = dep_graph.input_task(dep_node, &mut *hcx, &input).1; let hash = if dep_graph.is_fully_enabled() { - dep_graph.fingerprint_of(dep_node_index) + let mut stable_hasher = StableHasher::new(); + input.hash_stable(hcx, &mut stable_hasher); + stable_hasher.finish() + // FIXME + //dep_graph.fingerprint_of(dep_node_index) } else { let mut stable_hasher = StableHasher::new(); input.hash_stable(hcx, &mut stable_hasher); diff --git a/src/librustc/lib.rs b/src/librustc/lib.rs index e2f2799d9634d..01618462235db 100644 --- a/src/librustc/lib.rs +++ b/src/librustc/lib.rs @@ -39,6 +39,7 @@ #![feature(core_intrinsics)] #![feature(drain_filter)] #![feature(inner_deref)] +#![feature(duration_float)] #![cfg_attr(windows, feature(libc))] #![feature(never_type)] #![feature(exhaustive_patterns)] diff --git a/src/librustc/ty/context.rs b/src/librustc/ty/context.rs index e5d06532b3a16..3a98f3ba00766 100644 --- a/src/librustc/ty/context.rs +++ b/src/librustc/ty/context.rs @@ -4,7 +4,7 @@ use crate::arena::Arena; use crate::dep_graph::DepGraph; -use crate::dep_graph::{self, DepNode, DepConstructor}; +use crate::dep_graph::{DepNode, DepConstructor}; use crate::session::Session; use crate::session::config::{BorrowckMode, OutputFilenames}; use crate::session::config::CrateType; @@ -1489,12 +1489,7 @@ impl<'tcx> TyCtxt<'tcx> { for cnum in self.cstore.crates_untracked() { let dep_node = DepNode::new(self, DepConstructor::CrateMetadata(cnum)); let crate_hash = self.cstore.crate_hash_untracked(cnum); - self.dep_graph.with_task(dep_node, - self, - crate_hash, - |_, x| x, // No transformation needed - dep_graph::hash_result, - ); + self.dep_graph.input_task(dep_node, self, crate_hash); } } @@ -1842,7 +1837,7 @@ pub mod tls { /// This is used to get the pointer to the current ImplicitCtxt. #[cfg(parallel_compiler)] #[inline] - fn get_tlv() -> usize { + pub fn get_tlv() -> usize { rayon_core::tlv::get() } diff --git a/src/librustc/ty/query/plumbing.rs b/src/librustc/ty/query/plumbing.rs index 48e68167f824c..d67503f65e690 100644 --- a/src/librustc/ty/query/plumbing.rs +++ b/src/librustc/ty/query/plumbing.rs @@ -490,7 +490,7 @@ impl<'tcx> TyCtxt<'tcx> { } if unlikely!(self.sess.opts.debugging_opts.query_dep_graph) { - self.dep_graph.mark_loaded_from_cache(dep_node_index, true); + self.dep_graph.mark_loaded_from_cache(*dep_node, true); } result @@ -500,10 +500,11 @@ impl<'tcx> TyCtxt<'tcx> { #[cold] fn incremental_verify_ich>( self, - result: &Q::Value, - dep_node: &DepNode, - dep_node_index: DepNodeIndex, + _result: &Q::Value, + _dep_node: &DepNode, + _dep_node_index: DepNodeIndex, ) { + panic!()/* use crate::ich::Fingerprint; assert!(Some(self.dep_graph.fingerprint_of(dep_node_index)) == @@ -520,7 +521,7 @@ impl<'tcx> TyCtxt<'tcx> { let old_hash = self.dep_graph.fingerprint_of(dep_node_index); assert!(new_hash == old_hash, "Found unstable fingerprints \ - for {:?}", dep_node); + for {:?}", dep_node);*/ } #[inline(always)] @@ -566,7 +567,7 @@ impl<'tcx> TyCtxt<'tcx> { profq_msg!(self, ProfileQueriesMsg::ProviderEnd); if unlikely!(self.sess.opts.debugging_opts.query_dep_graph) { - self.dep_graph.mark_loaded_from_cache(dep_node_index, false); + self.dep_graph.mark_loaded_from_cache(dep_node, false); } if dep_node.kind != crate::dep_graph::DepKind::Null { diff --git a/src/librustc_data_structures/lib.rs b/src/librustc_data_structures/lib.rs index a1d7ab8856daa..18c7be71fc4f2 100644 --- a/src/librustc_data_structures/lib.rs +++ b/src/librustc_data_structures/lib.rs @@ -22,6 +22,7 @@ #![feature(stmt_expr_attributes)] #![feature(core_intrinsics)] #![feature(integer_atomics)] +#![feature(arbitrary_self_types)] #![cfg_attr(unix, feature(libc))] #![cfg_attr(test, feature(test))] diff --git a/src/librustc_data_structures/sync/worker.rs b/src/librustc_data_structures/sync/worker.rs index 8e4ae3e4de2fe..7ce19ad273cb4 100644 --- a/src/librustc_data_structures/sync/worker.rs +++ b/src/librustc_data_structures/sync/worker.rs @@ -166,4 +166,4 @@ mod executor { self.worker.lock().as_mut().unwrap().message(msg); } } -} \ No newline at end of file +} diff --git a/src/librustc_incremental/lib.rs b/src/librustc_incremental/lib.rs index ffea495d3ebdb..508711d6bfcb5 100644 --- a/src/librustc_incremental/lib.rs +++ b/src/librustc_incremental/lib.rs @@ -24,7 +24,7 @@ mod persist; pub use assert_dep_graph::assert_dep_graph; pub use persist::dep_graph_tcx_init; -pub use persist::{DepGraphFuture, load_dep_graph}; +pub use persist::{DepGraphFuture, load_dep_graph, dep_graph_from_future}; pub use persist::load_query_result_cache; pub use persist::LoadResult; pub use persist::copy_cgu_workproducts_to_incr_comp_cache_dir; diff --git a/src/librustc_incremental/persist/fs.rs b/src/librustc_incremental/persist/fs.rs index 7f697b5448464..c2010ef340939 100644 --- a/src/librustc_incremental/persist/fs.rs +++ b/src/librustc_incremental/persist/fs.rs @@ -119,6 +119,7 @@ use rand::{RngCore, thread_rng}; const LOCK_FILE_EXT: &str = ".lock"; const DEP_GRAPH_FILENAME: &str = "dep-graph.bin"; +const TEMP_DEP_GRAPH_FILENAME: &str = "dep-graph.bin.tmp"; const WORK_PRODUCTS_FILENAME: &str = "work-products.bin"; const QUERY_CACHE_FILENAME: &str = "query-cache.bin"; @@ -134,6 +135,9 @@ pub fn dep_graph_path(sess: &Session) -> PathBuf { pub fn dep_graph_path_from(incr_comp_session_dir: &Path) -> PathBuf { in_incr_comp_dir(incr_comp_session_dir, DEP_GRAPH_FILENAME) } +pub fn temp_dep_graph_path_from(incr_comp_session_dir: &Path) -> PathBuf { + in_incr_comp_dir(incr_comp_session_dir, TEMP_DEP_GRAPH_FILENAME) +} pub fn work_products_path(sess: &Session) -> PathBuf { in_incr_comp_dir_sess(sess, WORK_PRODUCTS_FILENAME) @@ -379,7 +383,10 @@ pub fn delete_all_session_dir_contents(sess: &Session) -> io::Result<()> { let sess_dir_iterator = sess.incr_comp_session_dir().read_dir()?; for entry in sess_dir_iterator { let entry = entry?; - safe_remove_file(&entry.path())? + let path = entry.path(); + if path != temp_dep_graph_path_from(&sess.incr_comp_session_dir()) { + safe_remove_file(&path)? + } } Ok(()) } diff --git a/src/librustc_incremental/persist/load.rs b/src/librustc_incremental/persist/load.rs index d9bcc0b2a83c7..18eb84f7c622d 100644 --- a/src/librustc_incremental/persist/load.rs +++ b/src/librustc_incremental/persist/load.rs @@ -1,19 +1,21 @@ //! Code to save/load the dep-graph from files. use rustc_data_structures::fx::FxHashMap; -use rustc::dep_graph::{PreviousDepGraph, SerializedDepGraph, WorkProduct, WorkProductId}; +use rustc::dep_graph::{DepGraph, PreviousDepGraph, SerializedDepGraph, WorkProduct, WorkProductId}; use rustc::session::Session; use rustc::ty::TyCtxt; use rustc::ty::query::OnDiskCache; -use rustc::util::common::time_ext; +use rustc::util::common::{time, time_ext}; use rustc_serialize::Decodable as RustcDecodable; use rustc_serialize::opaque::Decoder; +use rustc_serialize::Encodable; use std::path::Path; use super::data::*; use super::fs::*; use super::file_format; use super::work_product; +use super::save::save_in; pub fn dep_graph_tcx_init<'tcx>(tcx: TyCtxt<'tcx>) { if !tcx.dep_graph.is_fully_enabled() { @@ -23,6 +25,24 @@ pub fn dep_graph_tcx_init<'tcx>(tcx: TyCtxt<'tcx>) { tcx.allocate_metadata_dep_nodes(); } +pub fn dep_graph_from_future(sess: &Session, future: DepGraphFuture) -> DepGraph { + let (prev_graph, prev_work_products) = + time(sess, "blocked while dep-graph loading finishes", || { + future.open().unwrap_or_else(|e| LoadResult::Error { + message: format!("could not decode incremental cache: {:?}", e), + }).open(sess) + }); + let temp_path = temp_dep_graph_path_from(&sess.incr_comp_session_dir()); + + // Write the file header to the temp file + let file = save_in(sess, temp_path, |encoder| { + // Encode the commandline arguments hash + sess.opts.dep_tracking_hash().encode(encoder).unwrap(); + }).unwrap(); + + DepGraph::new(prev_graph, prev_work_products, file) +} + type WorkProductMap = FxHashMap; pub enum LoadResult { diff --git a/src/librustc_incremental/persist/mod.rs b/src/librustc_incremental/persist/mod.rs index bf404140f18d5..d72e7d1477f44 100644 --- a/src/librustc_incremental/persist/mod.rs +++ b/src/librustc_incremental/persist/mod.rs @@ -16,7 +16,7 @@ pub use fs::in_incr_comp_dir; pub use fs::in_incr_comp_dir_sess; pub use fs::prepare_session_directory; pub use load::dep_graph_tcx_init; -pub use load::{DepGraphFuture, load_dep_graph}; +pub use load::{DepGraphFuture, load_dep_graph, dep_graph_from_future}; pub use load::load_query_result_cache; pub use load::LoadResult; pub use save::save_dep_graph; diff --git a/src/librustc_incremental/persist/save.rs b/src/librustc_incremental/persist/save.rs index 49c79ec09f5e2..e35db3b84f534 100644 --- a/src/librustc_incremental/persist/save.rs +++ b/src/librustc_incremental/persist/save.rs @@ -1,4 +1,4 @@ -use rustc::dep_graph::{DepGraph, DepKind, WorkProduct, WorkProductId}; +use rustc::dep_graph::{DepGraph, /*DepKind,*/ WorkProduct, WorkProductId}; use rustc::session::Session; use rustc::ty::TyCtxt; use rustc::util::common::time; @@ -6,7 +6,8 @@ use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::join; use rustc_serialize::Encodable as RustcEncodable; use rustc_serialize::opaque::Encoder; -use std::fs; +use std::fs::{self, File}; +use std::io::{self, Write}; use std::path::PathBuf; use super::data::*; @@ -24,6 +25,7 @@ pub fn save_dep_graph<'tcx>(tcx: TyCtxt<'tcx>) { } let query_cache_path = query_cache_path(sess); + let temp_dep_graph_path = temp_dep_graph_path_from(&sess.incr_comp_session_dir()); let dep_graph_path = dep_graph_path(sess); join(move || { @@ -31,18 +33,12 @@ pub fn save_dep_graph<'tcx>(tcx: TyCtxt<'tcx>) { time(sess, "persist query result cache", || { save_in(sess, query_cache_path, - |e| encode_query_cache(tcx, e)); + |e| encode_query_cache(tcx, e)).unwrap(); }); } }, || { - time(sess, "persist dep-graph", || { - save_in(sess, - dep_graph_path, - |e| { - time(sess, "encode dep-graph", || { - encode_dep_graph(tcx, e) - }) - }); + time(sess, "swap dep-graph", || { + swap_dep_graph(tcx, temp_dep_graph_path, dep_graph_path) }); }); @@ -60,7 +56,7 @@ pub fn save_work_product_index(sess: &Session, debug!("save_work_product_index()"); dep_graph.assert_ignored(); let path = work_products_path(sess); - save_in(sess, path, |e| encode_work_product_index(&new_work_products, e)); + save_in(sess, path, |e| encode_work_product_index(&new_work_products, e)).unwrap(); // We also need to clean out old work-products, as not all of them are // deleted during invalidation. Some object files don't change their @@ -86,8 +82,9 @@ pub fn save_work_product_index(sess: &Session, }); } -fn save_in(sess: &Session, path_buf: PathBuf, encode: F) - where F: FnOnce(&mut Encoder) +pub(super) fn save_in(sess: &Session, path_buf: PathBuf, encode: F) -> io::Result +where + F: FnOnce(&mut Encoder) { debug!("save: storing data in {}", path_buf.display()); @@ -104,7 +101,7 @@ fn save_in(sess: &Session, path_buf: PathBuf, encode: F) sess.err(&format!("unable to delete old dep-graph at `{}`: {}", path_buf.display(), err)); - return; + return Err(err); } } } @@ -116,7 +113,9 @@ fn save_in(sess: &Session, path_buf: PathBuf, encode: F) // write the data out let data = encoder.into_inner(); - match fs::write(&path_buf, data) { + let mut file = File::create(&path_buf)?; + + match file.write_all(&data) { Ok(_) => { debug!("save: data written to disk successfully"); } @@ -124,20 +123,42 @@ fn save_in(sess: &Session, path_buf: PathBuf, encode: F) sess.err(&format!("failed to write dep-graph to `{}`: {}", path_buf.display(), err)); - return; + return Err(err); } } -} -fn encode_dep_graph(tcx: TyCtxt<'_>, encoder: &mut Encoder) { - // First encode the commandline arguments hash - tcx.sess.opts.dep_tracking_hash().encode(encoder).unwrap(); + Ok(file) +} +fn swap_dep_graph(tcx: TyCtxt<'_, '_>, temp: PathBuf, old: PathBuf) { + let sess = tcx.sess; // Encode the graph data. - let serialized_graph = time(tcx.sess, "getting serialized graph", || { - tcx.dep_graph.serialize() + time(tcx.sess, "finish graph serialization", || { + tcx.dep_graph.serialize(); }); + // delete the old dep-graph, if any + // Note: It's important that we actually delete the old file and not just + // truncate and overwrite it, since it might be a shared hard-link, the + // underlying data of which we don't want to modify + if old.exists() { + match fs::remove_file(&old) { + Ok(()) => { + debug!("save: remove old file"); + } + Err(err) => { + sess.err(&format!("unable to delete old dep-graph at `{}`: {}", + old.display(), + err)); + return; + } + } + } + + fs::rename(temp, old).expect("unable to rename temp dep graph"); + + +/* if tcx.sess.opts.debugging_opts.incremental_info { #[derive(Clone)] struct Stat { @@ -147,20 +168,19 @@ fn encode_dep_graph(tcx: TyCtxt<'_>, encoder: &mut Encoder) { } let total_node_count = serialized_graph.nodes.len(); - let total_edge_count = serialized_graph.edge_list_data.len(); + let total_edge_count: usize = serialized_graph.nodes.iter().map(|d| d.deps.len()).sum(); let mut counts: FxHashMap<_, Stat> = FxHashMap::default(); - for (i, &node) in serialized_graph.nodes.iter_enumerated() { - let stat = counts.entry(node.kind).or_insert(Stat { - kind: node.kind, + for node in serialized_graph.nodes.iter() { + let stat = counts.entry(node.node.kind).or_insert(Stat { + kind: node.node.kind, node_counter: 0, edge_counter: 0, }); stat.node_counter += 1; - let (edge_start, edge_end) = serialized_graph.edge_list_indices[i]; - stat.edge_counter += (edge_end - edge_start) as u64; + stat.edge_counter += node.deps.len() as u64; } let mut counts: Vec<_> = counts.values().cloned().collect(); @@ -212,10 +232,7 @@ fn encode_dep_graph(tcx: TyCtxt<'_>, encoder: &mut Encoder) { println!("{}", SEPARATOR); println!("[incremental]"); } - - time(tcx.sess, "encoding serialized graph", || { - serialized_graph.encode(encoder).unwrap(); - }); +*/ } fn encode_work_product_index(work_products: &FxHashMap, diff --git a/src/librustc_interface/queries.rs b/src/librustc_interface/queries.rs index 570509ffb2b8c..ca3128c9ef9b6 100644 --- a/src/librustc_interface/queries.rs +++ b/src/librustc_interface/queries.rs @@ -1,7 +1,7 @@ use crate::interface::{Compiler, Result}; use crate::passes::{self, BoxedResolver, ExpansionResult, BoxedGlobalCtxt, PluginInfo}; -use rustc_incremental::DepGraphFuture; +use rustc_incremental::{DepGraphFuture, dep_graph_from_future}; use rustc_data_structures::sync::Lrc; use rustc::session::config::{Input, OutputFilenames, OutputType}; use rustc::session::Session; @@ -172,15 +172,7 @@ impl Compiler { self.queries.dep_graph.compute(|| { Ok(match self.dep_graph_future()?.take() { None => DepGraph::new_disabled(), - Some(future) => { - let (prev_graph, prev_work_products) = - time(self.session(), "blocked while dep-graph loading finishes", || { - future.open().unwrap_or_else(|e| rustc_incremental::LoadResult::Error { - message: format!("could not decode incremental cache: {:?}", e), - }).open(self.session()) - }); - DepGraph::new(prev_graph, prev_work_products) - } + Some(future) => dep_graph_from_future(self.session(), future), }) }) } From 69c78625379c4c969a23a3fbcbc8828fa7fb3dab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Fri, 15 Mar 2019 12:17:11 +0100 Subject: [PATCH 04/17] Add an AtomicCell abstraction --- src/librustc_data_structures/Cargo.toml | 1 + src/librustc_data_structures/sync.rs | 59 ++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/src/librustc_data_structures/Cargo.toml b/src/librustc_data_structures/Cargo.toml index cd792d31187bd..fc58da4335025 100644 --- a/src/librustc_data_structures/Cargo.toml +++ b/src/librustc_data_structures/Cargo.toml @@ -19,6 +19,7 @@ rustc_cratesio_shim = { path = "../librustc_cratesio_shim" } serialize = { path = "../libserialize" } graphviz = { path = "../libgraphviz" } cfg-if = "0.1.2" +crossbeam-utils = { version = "0.6.5", features = ["nightly"] } stable_deref_trait = "1.0.0" rayon = { version = "0.2.0", package = "rustc-rayon" } rayon-core = { version = "0.2.0", package = "rustc-rayon-core" } diff --git a/src/librustc_data_structures/sync.rs b/src/librustc_data_structures/sync.rs index be5ae47d794d7..60c4c767e0016 100644 --- a/src/librustc_data_structures/sync.rs +++ b/src/librustc_data_structures/sync.rs @@ -69,6 +69,59 @@ cfg_if! { use std::ops::Add; use std::panic::{resume_unwind, catch_unwind, AssertUnwindSafe}; + #[derive(Debug)] + pub struct AtomicCell(Cell); + + impl AtomicCell { + #[inline] + pub fn new(v: T) -> Self { + AtomicCell(Cell::new(v)) + } + } + + impl AtomicCell { + pub fn into_inner(self) -> T { + self.0.into_inner() + } + + #[inline] + pub fn load(&self) -> T { + self.0.get() + } + + #[inline] + pub fn store(&self, val: T) { + self.0.set(val) + } + + pub fn swap(&self, val: T) -> T { + self.0.replace(val) + } + } + + impl AtomicCell { + pub fn compare_exchange(&self, + current: T, + new: T) + -> Result { + let read = self.0.get(); + if read == current { + self.0.set(new); + Ok(read) + } else { + Err(read) + } + } + } + + impl + Copy> AtomicCell { + pub fn fetch_add(&self, val: T) -> T { + let old = self.0.get(); + self.0.set(old + val); + old + } + } + #[derive(Debug)] pub struct Atomic(Cell); @@ -79,7 +132,7 @@ cfg_if! { } } - impl Atomic { + impl Atomic { pub fn into_inner(self) -> T { self.0.into_inner() } @@ -97,7 +150,9 @@ cfg_if! { pub fn swap(&self, val: T, _: Ordering) -> T { self.0.replace(val) } + } + impl Atomic { pub fn compare_exchange(&self, current: T, new: T, @@ -273,6 +328,8 @@ cfg_if! { pub use std::sync::atomic::{AtomicBool, AtomicUsize, AtomicU32, AtomicU64}; + pub use crossbeam_utils::atomic::AtomicCell; + pub use std::sync::Arc as Lrc; pub use std::sync::Weak as Weak; From 84359512bd5990454d0f024b3d4dbd82f45b44a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Fri, 19 Apr 2019 21:38:27 +0200 Subject: [PATCH 05/17] Add compare_and_swap --- src/librustc_data_structures/sync.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/librustc_data_structures/sync.rs b/src/librustc_data_structures/sync.rs index 60c4c767e0016..224f2a1d566f0 100644 --- a/src/librustc_data_structures/sync.rs +++ b/src/librustc_data_structures/sync.rs @@ -100,6 +100,13 @@ cfg_if! { } impl AtomicCell { + pub fn compare_and_swap(&self, current: T, new: T) -> T { + match self.compare_exchange(current, new) { + Ok(v) => v, + Err(v) => v, + } + } + pub fn compare_exchange(&self, current: T, new: T) From 137f61b2dd276c3c4d68619fb332512c777552cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Fri, 19 Apr 2019 18:49:15 +0200 Subject: [PATCH 06/17] Only store changed dep nodes --- src/librustc/dep_graph/graph.rs | 443 ++++++++++-------- src/librustc/dep_graph/prev.rs | 18 +- src/librustc/dep_graph/serialized.rs | 148 ++++-- src/librustc/ty/query/plumbing.rs | 13 +- src/librustc_codegen_ssa/base.rs | 4 +- .../persist/dirty_clean.rs | 18 +- .../persist/file_format.rs | 11 +- src/librustc_incremental/persist/load.rs | 59 +-- 8 files changed, 414 insertions(+), 300 deletions(-) diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index 058d3fb9e830a..a9361fa15de87 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -3,7 +3,8 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::indexed_vec::{Idx, IndexVec}; use smallvec::SmallVec; -use rustc_data_structures::sync::{Lrc, Lock, AtomicU32, Ordering}; +use rustc_data_structures::sync::{Lrc, Lock, AtomicCell, AtomicUsize, AtomicU64}; +use std::sync::atomic::Ordering::{Acquire, SeqCst}; use std::env; use std::fs::File; use std::hash::Hash; @@ -37,14 +38,43 @@ impl DepNodeIndex { #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] pub enum DepNodeColor { Red, - Green(DepNodeIndex) + Green, } impl DepNodeColor { pub fn is_green(self) -> bool { match self { DepNodeColor::Red => false, - DepNodeColor::Green(_) => true, + DepNodeColor::Green => true, + } + } +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub enum DepNodeState { + /// The node is invalid since its result is older than the previous session. + Invalidated, + + /// The node is from the previous session, but its state is unknown + Unknown, + + /// The node will eventually be green, + /// but its side effects (like error messages) have not yet happened. + WillBeGreen, + + Red, + + Green, +} + +impl DepNodeState { + pub fn color(self) -> Option { + match self { + DepNodeState::Invalidated | + DepNodeState::Unknown | + DepNodeState::WillBeGreen => None, + DepNodeState::Red => Some(DepNodeColor::Red), + DepNodeState::Green => Some(DepNodeColor::Green), } } } @@ -54,7 +84,7 @@ struct DepGraphData { /// tracking. The `current` field is the dependency graph of only the /// current compilation session: We don't merge the previous dep-graph into /// current one anymore. - current: Lock, + current: CurrentDepGraph, /// The dep-graph from the previous compilation session. It contains all /// nodes and edges as well as all fingerprints of nodes that have them. @@ -97,16 +127,19 @@ impl DepGraph { file: File, ) -> DepGraph { let prev_graph_node_count = prev_graph.node_count(); + let colors = DepNodeColorMap { + values: prev_graph.state.iter().map(|s| AtomicCell::new(*s)).collect(), + }; DepGraph { data: Some(Lrc::new(DepGraphData { previous_work_products: prev_work_products, dep_node_debug: Default::default(), - current: Lock::new(CurrentDepGraph::new(prev_graph_node_count, file)), + current: CurrentDepGraph::new(prev_graph_node_count, file), emitted_diagnostics: Default::default(), emitted_diagnostics_cond_var: Condvar::new(), + colors, previous: prev_graph, - colors: DepNodeColorMap::new(prev_graph_node_count), loaded_from_cache: Default::default(), })), } @@ -211,13 +244,11 @@ impl DepGraph { reads: SmallVec::new(), read_set: Default::default(), }), - |data, key, fingerprint, task| { - data.borrow_mut().complete_task(key, task.unwrap(), fingerprint) - }, hash_result) } - /// Creates a new dep-graph input with value `input` + /// Creates a new dep-graph input with value `input`. + /// Dep nodes created by this function can be used by the `read` method. pub fn input_task<'a, C, R>(&self, key: DepNode, cx: C, @@ -230,12 +261,15 @@ impl DepGraph { arg } - self.with_task_impl(key, cx, input, true, identity_fn, + let (r, index) = self.with_task_impl(key, cx, input, true, identity_fn, |_| None, - |data, key, fingerprint, _| { - data.borrow_mut().alloc_node(key, SmallVec::new(), fingerprint) - }, - hash_result::) + hash_result::); + + self.data.as_ref().map(|data| { + assert!(data.current.input_node_to_node_index.lock().insert(key, index).is_none()); + }); + + (r, index) } fn with_task_impl<'a, C, A, R>( @@ -246,10 +280,6 @@ impl DepGraph { no_tcx: bool, task: fn(C, A) -> R, create_task: fn(DepNode) -> Option, - finish_task_and_alloc_depnode: fn(&Lock, - DepNode, - Fingerprint, - Option) -> DepNodeIndex, hash_result: impl FnOnce(&mut StableHashingContext<'_>, &R) -> Option, ) -> (R, DepNodeIndex) where @@ -290,49 +320,64 @@ impl DepGraph { let current_fingerprint = hash_result(&mut hcx, &result); - let dep_node_index = finish_task_and_alloc_depnode( - &data.current, - key, - current_fingerprint.unwrap_or(Fingerprint::ZERO), - task_deps.map(|lock| lock.into_inner()), - ); + let deps = task_deps.map(|lock| lock.into_inner().reads).unwrap_or(SmallVec::new()); + let fingerprint = current_fingerprint.unwrap_or(Fingerprint::ZERO); let print_status = cfg!(debug_assertions) && hcx.sess().opts.debugging_opts.dep_tasks; - // Determine the color of the new DepNode. - if let Some(prev_index) = data.previous.node_to_index_opt(&key) { + let dep_node_index = if let Some(prev_index) = data.previous.node_to_index_opt(&key) { let prev_fingerprint = data.previous.fingerprint_by_index(prev_index); + // Determine the color of the previous DepNode. let color = if let Some(current_fingerprint) = current_fingerprint { if current_fingerprint == prev_fingerprint { if print_status { eprintln!("[task::green] {:?}", key); } - DepNodeColor::Green(dep_node_index) + DepNodeState::Green } else { if print_status { eprintln!("[task::red] {:?}", key); } - DepNodeColor::Red + DepNodeState::Red } } else { if print_status { eprintln!("[task::unknown] {:?}", key); } // Mark the node as Red if we can't hash the result - DepNodeColor::Red + DepNodeState::Red }; - debug_assert!(data.colors.get(prev_index).is_none(), - "DepGraph::with_task() - Duplicate DepNodeColor \ - insertion for {:?}", key); + debug_assert_eq!( + data.colors.get(prev_index), + DepNodeState::Unknown, + "DepGraph::with_task() - Duplicate DepNodeState insertion for {:?}", + key + ); data.colors.insert(prev_index, color); + + data.current.update_node( + prev_index.current(), + key, + deps, + fingerprint + ); + + prev_index.current() } else { if print_status { eprintln!("[task::new] {:?}", key); } - } + + data.current.new_node( + key, + deps, + fingerprint, + &data.previous, + ) + }; (result, dep_node_index) } else { @@ -368,8 +413,7 @@ impl DepGraph { (r, task_deps.into_inner()) }); let dep_node_index = data.current - .borrow_mut() - .complete_anon_task(dep_kind, task_deps); + .complete_anon_task(dep_kind, task_deps, &data.previous); (result, dep_node_index) } else { (op(), DepNodeIndex::INVALID) @@ -391,19 +435,15 @@ impl DepGraph { { self.with_task_impl(key, cx, arg, false, task, |_| None, - |data, key, fingerprint, _| { - let mut current = data.borrow_mut(); - current.alloc_node(key, smallvec![], fingerprint) - }, hash_result) } #[inline] pub fn read(&self, v: DepNode) { if let Some(ref data) = self.data { - let current = data.current.borrow_mut(); - if let Some(&dep_node_index) = current.node_to_node_index.get(&v) { - std::mem::drop(current); + let map = data.current.input_node_to_node_index.lock(); + if let Some(dep_node_index) = map.get(&v).cloned() { + std::mem::drop(map); data.read_index(dep_node_index); } else { bug!("DepKind {:?} should be pre-allocated but isn't.", v.kind) @@ -417,7 +457,7 @@ impl DepGraph { data.read_index(dep_node_index); } } - +/* #[inline] pub fn dep_node_index_of(&self, dep_node: &DepNode) -> DepNodeIndex { self.data @@ -439,7 +479,7 @@ impl DepGraph { false } } - +*/ #[inline] pub fn fingerprint_of(&self, _dep_node_index: DepNodeIndex) -> Fingerprint { panic!()/* @@ -498,24 +538,35 @@ impl DepGraph { pub fn edge_deduplication_data(&self) -> Option<(u64, u64)> { if cfg!(debug_assertions) { - let current_dep_graph = self.data.as_ref().unwrap().current.borrow(); + let current_dep_graph = &self.data.as_ref().unwrap().current; - Some((current_dep_graph.total_read_count, - current_dep_graph.total_duplicate_read_count)) + Some((current_dep_graph.total_read_count.load(Acquire), + current_dep_graph.total_duplicate_read_count.load(Acquire))) } else { None } } pub fn serialize(&self) { + let data = self.data.as_ref().unwrap(); + // Invalidate dep nodes with unknown state as these cannot safely + // be marked green in the next session. + let invalidate = data.colors.values.indices().filter_map(|prev_index| { + match data.colors.get(prev_index) { + DepNodeState::Unknown => { + Some(prev_index.current()) + } + _ => None, + } + }).collect(); // FIXME: Can this deadlock? - self.data.as_ref().unwrap().current.lock().serializer.complete() + data.current.serializer.lock().complete(invalidate) } pub fn node_color(&self, dep_node: &DepNode) -> Option { if let Some(ref data) = self.data { if let Some(prev_index) = data.previous.node_to_index_opt(dep_node) { - return data.colors.get(prev_index) + return data.colors.get(prev_index).color() } else { // This is a node that did not exist in the previous compilation // session, so we consider it to be red. @@ -556,21 +607,25 @@ impl DepGraph { let prev_index = data.previous.node_to_index_opt(dep_node)?; match data.colors.get(prev_index) { - Some(DepNodeColor::Green(dep_node_index)) => Some((prev_index, dep_node_index)), - Some(DepNodeColor::Red) => None, - None => { + DepNodeState::Green => Some((prev_index, prev_index.current())), + DepNodeState::Invalidated | + DepNodeState::Red => None, + DepNodeState::Unknown | + DepNodeState::WillBeGreen => { // This DepNode and the corresponding query invocation existed // in the previous compilation session too, so we can try to // mark it as green by recursively marking all of its // dependencies green. - self.try_mark_previous_green( + if self.try_mark_previous_green( tcx.global_tcx(), data, prev_index, &dep_node - ).map(|dep_node_index| { - (prev_index, dep_node_index) - }) + ) { + Some((prev_index, prev_index.current())) + } else { + None + } } } } @@ -580,31 +635,29 @@ impl DepGraph { &self, tcx: TyCtxt<'tcx>, data: &DepGraphData, - prev_dep_node_index: SerializedDepNodeIndex, + dep_node_index: SerializedDepNodeIndex, + // FIXME: Remove this, only used in debug statements dep_node: &DepNode, - ) -> Option { + ) -> bool { debug!("try_mark_previous_green({:?}) - BEGIN", dep_node); #[cfg(not(parallel_compiler))] { - debug_assert!(!data.current.borrow().node_to_node_index.contains_key(dep_node)); - debug_assert!(data.colors.get(prev_dep_node_index).is_none()); + debug_assert!(data.colors.get(dep_node_index).color().is_none()); } // We never try to mark eval_always nodes as green debug_assert!(!dep_node.kind.is_eval_always()); - debug_assert_eq!(data.previous.index_to_node(prev_dep_node_index), *dep_node); + debug_assert_eq!(data.previous.index_to_node(dep_node_index), *dep_node); - let prev_deps = data.previous.edge_targets_from(prev_dep_node_index); - - let mut current_deps = SmallVec::new(); + let prev_deps = data.previous.edge_targets_from(dep_node_index); for &dep_dep_node_index in prev_deps { let dep_dep_node_color = data.colors.get(dep_dep_node_index); match dep_dep_node_color { - Some(DepNodeColor::Green(node_index)) => { + DepNodeState::Green => { // This dependency has been marked as green before, we are // still fine and can continue with checking the other // dependencies. @@ -612,9 +665,8 @@ impl DepGraph { be immediately green", dep_node, data.previous.index_to_node(dep_dep_node_index)); - current_deps.push(node_index); } - Some(DepNodeColor::Red) => { + DepNodeState::Red => { // We found a dependency the value of which has changed // compared to the previous compilation session. We cannot // mark the DepNode as green and also don't need to bother @@ -623,9 +675,11 @@ impl DepGraph { immediately red", dep_node, data.previous.index_to_node(dep_dep_node_index)); - return None + return false } - None => { + DepNodeState::Invalidated => panic!("can this happen?"), + DepNodeState::WillBeGreen | + DepNodeState::Unknown => { let dep_dep_node = &data.previous.index_to_node(dep_dep_node_index); // We don't know the state of this dependency. If it isn't @@ -635,16 +689,14 @@ impl DepGraph { is unknown, trying to mark it green", dep_node, dep_dep_node); - let node_index = self.try_mark_previous_green( + if self.try_mark_previous_green( tcx, data, dep_dep_node_index, dep_dep_node - ); - if let Some(node_index) = node_index { + ) { debug!("try_mark_previous_green({:?}) --- managed to MARK \ dependency {:?} as green", dep_node, dep_dep_node); - current_deps.push(node_index); continue; } } else { @@ -655,7 +707,7 @@ impl DepGraph { if dep_dep_node.extract_def_id(tcx).is_none() { // If the node does not exist anymore, we // just fail to mark green. - return None + return false } else { // If the node does exist, it should have // been pre-allocated. @@ -678,36 +730,30 @@ impl DepGraph { let dep_dep_node_color = data.colors.get(dep_dep_node_index); match dep_dep_node_color { - Some(DepNodeColor::Green(node_index)) => { + DepNodeState::Green => { debug!("try_mark_previous_green({:?}) --- managed to \ FORCE dependency {:?} to green", dep_node, dep_dep_node); - current_deps.push(node_index); } - Some(DepNodeColor::Red) => { + DepNodeState::Red => { debug!("try_mark_previous_green({:?}) - END - \ dependency {:?} was red after forcing", dep_node, dep_dep_node); - return None + return false } - None => { - if !tcx.sess.has_errors() { - bug!("try_mark_previous_green() - Forcing the DepNode \ - should have set its color") - } else { - // If the query we just forced has resulted - // in some kind of compilation error, we - // don't expect that the corresponding - // dep-node color has been updated. - } + DepNodeState::Invalidated | + DepNodeState::Unknown | + DepNodeState::WillBeGreen => { + bug!("try_mark_previous_green() - Forcing the DepNode \ + should have set its color") } } } else { // The DepNode could not be forced. debug!("try_mark_previous_green({:?}) - END - dependency {:?} \ could not be forced", dep_node, dep_dep_node); - return None + return false } } } @@ -719,48 +765,40 @@ impl DepGraph { // There may be multiple threads trying to mark the same dep node green concurrently - let (dep_node_index, did_allocation) = { - let mut current = data.current.borrow_mut(); - - // Copy the fingerprint from the previous graph, - // so we don't have to recompute it - let fingerprint = data.previous.fingerprint_by_index(prev_dep_node_index); + #[cfg(not(parallel_compiler))] + debug_assert_eq!(data.colors.get(dep_node_index), DepNodeState::Unknown, + "DepGraph::try_mark_previous_green() - Duplicate DepNodeState \ + insertion for {:?}", dep_node); - // We allocating an entry for the node in the current dependency graph and - // adding all the appropriate edges imported from the previous graph - current.intern_node(*dep_node, current_deps, fingerprint) - }; + let did_mark = data.colors.mark_as_will_be_green(dep_node_index); // ... emitting any stored diagnostic ... let diagnostics = tcx.queries.on_disk_cache - .load_diagnostics(tcx, prev_dep_node_index); + .load_diagnostics(tcx, dep_node_index); if unlikely!(diagnostics.len() > 0) { self.emit_diagnostics( tcx, data, - dep_node_index, - did_allocation, + dep_node_index.current(), + did_mark, diagnostics ); } // ... and finally storing a "Green" entry in the color map. // Multiple threads can all write the same color here - #[cfg(not(parallel_compiler))] - debug_assert!(data.colors.get(prev_dep_node_index).is_none(), - "DepGraph::try_mark_previous_green() - Duplicate DepNodeColor \ - insertion for {:?}", dep_node); - - data.colors.insert(prev_dep_node_index, DepNodeColor::Green(dep_node_index)); + // FIXME: Mark as green-promoted? + data.colors.insert(dep_node_index, DepNodeState::Green); debug!("try_mark_previous_green({:?}) - END - successfully marked as green", dep_node); - Some(dep_node_index) + + true } /// Atomically emits some loaded diagnotics, assuming that this only gets called with - /// `did_allocation` set to `true` on a single thread. + /// `did_mark` set to `true` on a single thread. #[cold] #[inline(never)] fn emit_diagnostics<'tcx>( @@ -768,10 +806,10 @@ impl DepGraph { tcx: TyCtxt<'tcx>, data: &DepGraphData, dep_node_index: DepNodeIndex, - did_allocation: bool, + did_mark: bool, diagnostics: Vec, ) { - if did_allocation || !cfg!(parallel_compiler) { + if did_mark || !cfg!(parallel_compiler) { // Only the thread which did the allocation emits the error messages let handle = tcx.sess.diagnostic(); @@ -785,7 +823,7 @@ impl DepGraph { #[cfg(parallel_compiler)] { - // Mark the diagnostics and emitted and wake up waiters + // Mark the diagnostics aa emitted and wake up waiters data.emitted_diagnostics.lock().insert(dep_node_index); data.emitted_diagnostics_cond_var.notify_all(); } @@ -821,7 +859,7 @@ impl DepGraph { let data = self.data.as_ref().unwrap(); data.colors.values.indices().filter_map(|prev_index| { match data.colors.get(prev_index) { - Some(DepNodeColor::Green(_)) => { + DepNodeState::Green => { let dep_node = data.previous.index_to_node(prev_index); if dep_node.cache_on_disk(tcx) { Some(dep_node) @@ -829,8 +867,10 @@ impl DepGraph { None } } - None | - Some(DepNodeColor::Red) => { + DepNodeState::WillBeGreen => bug!("no tasks should be in progress"), + DepNodeState::Invalidated | + DepNodeState::Unknown | + DepNodeState::Red => { // We can skip red nodes because a node can only be marked // as red if the query result was recomputed and thus is // already in memory. @@ -907,16 +947,23 @@ pub enum WorkProductFileKind { BytecodeCompressed, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, RustcDecodable, RustcEncodable)] pub(super) struct DepNodeData { pub(super) node: DepNode, - pub(super) edges: SmallVec<[DepNodeIndex; 8]>, + pub(super) edges: TaskReads, pub(super) fingerprint: Fingerprint, } pub(super) struct CurrentDepGraph { - nodes: usize, - node_to_node_index: FxHashMap, + //data: FxHashMap, + node_count: AtomicUsize, + + /// Used to map input nodes to a node index. Used by the `read` method. + input_node_to_node_index: Lock>, + + /// Used to map input nodes to a node index + anon_node_to_node_index: Lock>, + #[allow(dead_code)] forbidden_edge: Option, @@ -933,11 +980,11 @@ pub(super) struct CurrentDepGraph { /// the `DepGraph` is created. anon_id_seed: Fingerprint, - total_read_count: u64, - total_duplicate_read_count: u64, + total_read_count: AtomicU64, + total_duplicate_read_count: AtomicU64, /// Produces the serialized dep graph for the next session, - serializer: Serializer, + serializer: Lock, } impl CurrentDepGraph { @@ -968,32 +1015,27 @@ impl CurrentDepGraph { // that we hopefully don't have to re-allocate during this compilation // session. The over-allocation is 2% plus a small constant to account // for the fact that in very small crates 2% might not be enough. - let new_node_count_estimate = (prev_graph_node_count * 102) / 100 + 200; + //let new_node_count_estimate = (prev_graph_node_count * 102) / 100 + 200; CurrentDepGraph { - nodes: 0, - node_to_node_index: FxHashMap::with_capacity_and_hasher( - new_node_count_estimate, - Default::default(), - ), + //data: FxHashMap::default(), + node_count: AtomicUsize::new(prev_graph_node_count), + anon_node_to_node_index: Default::default(), + input_node_to_node_index: Default::default(), anon_id_seed: stable_hasher.finish(), forbidden_edge, - total_read_count: 0, - total_duplicate_read_count: 0, - serializer: Serializer::new(file), + total_read_count: AtomicU64::new(0), + total_duplicate_read_count: AtomicU64::new(0), + serializer: Lock::new(Serializer::new(file)), } } - fn complete_task( - &mut self, - node: DepNode, + fn complete_anon_task( + &self, + kind: DepKind, task_deps: TaskDeps, - fingerprint: Fingerprint + previous: &PreviousDepGraph, ) -> DepNodeIndex { - self.alloc_node(node, task_deps.reads, fingerprint) - } - - fn complete_anon_task(&mut self, kind: DepKind, task_deps: TaskDeps) -> DepNodeIndex { debug_assert!(!kind.is_eval_always()); let mut hasher = StableHasher::new(); @@ -1005,7 +1047,7 @@ impl CurrentDepGraph { // to map the dependencies to a single value on a per session basis. task_deps.reads.hash(&mut hasher); - let target_dep_node = DepNode { + let dep_node = DepNode { kind, // Fingerprint::combine() is faster than sending Fingerprint @@ -1014,41 +1056,53 @@ impl CurrentDepGraph { hash: self.anon_id_seed.combine(hasher.finish()), }; - self.intern_node(target_dep_node, task_deps.reads, Fingerprint::ZERO).0 + match self.anon_node_to_node_index.lock().entry(dep_node) { + Entry::Occupied(entry) => *entry.get(), + Entry::Vacant(entry) => { + // Make sure there's no collision with a previous dep node + // FIXME: Ensure this by always allocating a new index for each + // anon task instead of hashing a random seed? + let dep_node_index = self.new_node( + dep_node, + task_deps.reads, + Fingerprint::ZERO, + previous, + ); + entry.insert(dep_node_index); + dep_node_index + } + } } - fn alloc_node( - &mut self, + fn new_node( + &self, dep_node: DepNode, - edges: SmallVec<[DepNodeIndex; 8]>, - fingerprint: Fingerprint + edges: TaskReads, + fingerprint: Fingerprint, + previous: &PreviousDepGraph, ) -> DepNodeIndex { - debug_assert!(!self.node_to_node_index.contains_key(&dep_node)); - self.intern_node(dep_node, edges, fingerprint).0 + debug_assert!(previous.node_to_index_opt(&dep_node).is_none()); + let dep_node_index = DepNodeIndex::new(self.node_count.fetch_add(1, SeqCst)); + self.serializer.lock().serialize_new(DepNodeData { + node: dep_node, + edges, + fingerprint + }); + dep_node_index } - fn intern_node( - &mut self, + fn update_node( + &self, + index: DepNodeIndex, dep_node: DepNode, - edges: SmallVec<[DepNodeIndex; 8]>, - fingerprint: Fingerprint - ) -> (DepNodeIndex, bool) { - debug_assert_eq!(self.node_to_node_index.len(), self.nodes); - - match self.node_to_node_index.entry(dep_node) { - Entry::Occupied(entry) => (*entry.get(), false), - Entry::Vacant(entry) => { - let dep_node_index = DepNodeIndex::new(self.nodes); - self.nodes += 1; - self.serializer.serialize(DepNodeData { - node: dep_node, - edges, - fingerprint - }); - entry.insert(dep_node_index); - (dep_node_index, true) - } - } + edges: TaskReads, + fingerprint: Fingerprint, + ) { + self.serializer.lock().serialize_updated(index, DepNodeData { + node: dep_node, + edges, + fingerprint + }); } } @@ -1059,7 +1113,7 @@ impl DepGraphData { if let Some(task_deps) = icx.task_deps { let mut task_deps = task_deps.lock(); if cfg!(debug_assertions) { - self.current.lock().total_read_count += 1; + self.current.total_read_count.fetch_add(1, SeqCst); } if task_deps.read_set.insert(source) { task_deps.reads.push(source); @@ -1069,7 +1123,7 @@ impl DepGraphData { if let Some(target) = task_deps.node { let graph = self.current.lock(); if let Some(ref forbidden_edge) = graph.forbidden_edge { - let source = graph.data[source].node; + let source = graph.data[&source].node; if forbidden_edge.test(&source, &target) { bug!("forbidden edge {:?} -> {:?} created", source, @@ -1079,52 +1133,45 @@ impl DepGraphData { } }*/ } else if cfg!(debug_assertions) { - self.current.lock().total_duplicate_read_count += 1; + self.current.total_duplicate_read_count.fetch_add(1, SeqCst); } } }) } } +type TaskReads = SmallVec<[DepNodeIndex; 8]>; + pub struct TaskDeps { #[cfg(debug_assertions)] #[allow(dead_code)] node: Option, - reads: SmallVec<[DepNodeIndex; 8]>, + reads: TaskReads, read_set: FxHashSet, } -// A data structure that stores Option values as a contiguous -// array, using one u32 per entry. struct DepNodeColorMap { - values: IndexVec, + values: IndexVec>, } -const COMPRESSED_NONE: u32 = 0; -const COMPRESSED_RED: u32 = 1; -const COMPRESSED_FIRST_GREEN: u32 = 2; - impl DepNodeColorMap { - fn new(size: usize) -> DepNodeColorMap { - DepNodeColorMap { - values: (0..size).map(|_| AtomicU32::new(COMPRESSED_NONE)).collect(), - } + /// Tries to mark the node as WillBeGreen. Returns false if another thread did it before us. + #[inline] + fn mark_as_will_be_green(&self, index: SerializedDepNodeIndex) -> bool { + let prev = self.values[index].compare_and_swap( + DepNodeState::Unknown, + DepNodeState::WillBeGreen + ); + prev == DepNodeState::Unknown } - fn get(&self, index: SerializedDepNodeIndex) -> Option { - match self.values[index].load(Ordering::Acquire) { - COMPRESSED_NONE => None, - COMPRESSED_RED => Some(DepNodeColor::Red), - value => Some(DepNodeColor::Green(DepNodeIndex::from_u32( - value - COMPRESSED_FIRST_GREEN - ))) - } + #[inline] + fn get(&self, index: SerializedDepNodeIndex) -> DepNodeState { + self.values[index].load() } - fn insert(&self, index: SerializedDepNodeIndex, color: DepNodeColor) { - self.values[index].store(match color { - DepNodeColor::Red => COMPRESSED_RED, - DepNodeColor::Green(index) => index.as_u32() + COMPRESSED_FIRST_GREEN, - }, Ordering::Release) + #[inline] + fn insert(&self, index: SerializedDepNodeIndex, state: DepNodeState) { + self.values[index].store(state) } } diff --git a/src/librustc/dep_graph/prev.rs b/src/librustc/dep_graph/prev.rs index 55f210b01903b..4c7cf06f005d6 100644 --- a/src/librustc/dep_graph/prev.rs +++ b/src/librustc/dep_graph/prev.rs @@ -2,9 +2,10 @@ use crate::ich::Fingerprint; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::indexed_vec::IndexVec; use super::dep_node::DepNode; +use super::graph::DepNodeState; use super::serialized::{SerializedDepGraph, SerializedDepNodeIndex}; -#[derive(Debug, RustcEncodable, RustcDecodable, Default)] +#[derive(Debug, /*RustcEncodable, RustcDecodable,*/ Default)] pub struct PreviousDepGraph { /// Maps from dep nodes to their previous index, if any. index: FxHashMap, @@ -20,13 +21,17 @@ pub struct PreviousDepGraph { /// A flattened list of all edge targets in the graph. Edge sources are /// implicit in edge_list_indices. edge_list_data: Vec, + /// A set of nodes which are no longer valid. + pub(super) state: IndexVec, } impl PreviousDepGraph { pub fn new(graph: SerializedDepGraph) -> PreviousDepGraph { let index: FxHashMap<_, _> = graph.nodes .iter_enumerated() - .map(|(idx, dep_node)| (dep_node.node, idx)) + .map(|(idx, dep_node)| { + (dep_node.node, SerializedDepNodeIndex::from_u32(idx.as_u32())) + }) .collect(); let fingerprints: IndexVec = @@ -34,15 +39,17 @@ impl PreviousDepGraph { let nodes: IndexVec = graph.nodes.iter().map(|d| d.node).collect(); - let total_edge_count: usize = graph.nodes.iter().map(|d| d.deps.len()).sum(); + let total_edge_count: usize = graph.nodes.iter().map(|d| d.edges.len()).sum(); let mut edge_list_indices = IndexVec::with_capacity(nodes.len()); let mut edge_list_data = Vec::with_capacity(total_edge_count); for (current_dep_node_index, edges) in graph.nodes.iter_enumerated() - .map(|(i, d)| (i, &d.deps)) { + .map(|(i, d)| (i, &d.edges)) { let start = edge_list_data.len() as u32; - edge_list_data.extend(edges.iter().cloned()); + edge_list_data.extend(edges.iter().map(|i| { + SerializedDepNodeIndex::from_u32(i.as_u32()) + })); let end = edge_list_data.len() as u32; debug_assert_eq!(current_dep_node_index.index(), edge_list_indices.len()); @@ -58,6 +65,7 @@ impl PreviousDepGraph { edge_list_indices, edge_list_data, index, + state: graph.state.convert_index_type(), } } diff --git a/src/librustc/dep_graph/serialized.rs b/src/librustc/dep_graph/serialized.rs index 2688d2a9e2c20..0d7494728dbf4 100644 --- a/src/librustc/dep_graph/serialized.rs +++ b/src/librustc/dep_graph/serialized.rs @@ -3,47 +3,73 @@ use rustc_data_structures::sync::Lrc; use rustc_data_structures::{unlikely, cold_path}; use rustc_data_structures::indexed_vec::{IndexVec, Idx}; use rustc_serialize::opaque; -use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; +use rustc_serialize::{Decodable, Encodable}; use std::mem; use std::fs::File; use std::io::Write; -use super::graph::DepNodeData; -use crate::dep_graph::DepNode; -use crate::ich::Fingerprint; +use super::graph::{DepNodeData, DepNodeIndex, DepNodeState}; newtype_index! { pub struct SerializedDepNodeIndex { .. } } -/// Data for use when recompiling the **current crate**. +impl SerializedDepNodeIndex { + pub fn current(self) -> DepNodeIndex { + DepNodeIndex::from_u32(self.as_u32()) + } +} + #[derive(Debug, Default)] pub struct SerializedDepGraph { - pub nodes: IndexVec, + pub(super) nodes: IndexVec, + pub(super) state: IndexVec, } -impl Decodable for SerializedDepGraph { - fn decode(d: &mut D) -> Result { +impl SerializedDepGraph { + pub fn decode(d: &mut opaque::Decoder<'_>) -> Result { let mut nodes = IndexVec::new(); + let mut invalidated_list = Vec::new(); loop { - let count = d.read_usize()?; - if count == 0 { + if d.position() == d.data.len() { break; } - for _ in 0..count { - nodes.push(SerializedNode::decode(d)?); + match Action::decode(d)? { + Action::NewNodes(new_nodes) => { + nodes.extend(new_nodes); + } + Action::UpdateNodes(changed) => { + for (i, data) in changed { + nodes[i] = data; + } + } + Action::InvalidateNodes(nodes) => { + invalidated_list.extend(nodes); + } } } + let mut state: IndexVec<_, _> = (0..nodes.len()).map(|_| { + DepNodeState::Unknown + }).collect(); + for i in invalidated_list { + state[i] = DepNodeState::Invalidated; + } Ok(SerializedDepGraph { nodes, + state, }) } } -#[derive(Debug, RustcDecodable)] -pub struct SerializedNode { - pub node: DepNode, - pub deps: Vec, - pub fingerprint: Fingerprint, +#[derive(Debug, RustcEncodable, RustcDecodable)] +enum Action { + NewNodes(Vec), + UpdateNodes(Vec<(DepNodeIndex, DepNodeData)>), + // FIXME: Is this redundant since these nodes will be also be updated? + // Could one of the indirect dependencies of a dep node change its result and + // cause a red node to be incorrectly green again? + // What about nodes which are in an unknown state? + // We must invalidate unknown nodes. Red nodes will have an entry in UpdateNodes + InvalidateNodes(Vec) } struct SerializerWorker { @@ -51,34 +77,26 @@ struct SerializerWorker { } impl Worker for SerializerWorker { - type Message = (usize, Vec); + type Message = (usize, Action); type Result = (); - fn message(&mut self, (buffer_size_est, nodes): (usize, Vec)) { - let mut encoder = opaque::Encoder::new(Vec::with_capacity(buffer_size_est * 4)); - assert!(!nodes.is_empty()); - encoder.emit_usize(nodes.len()).ok(); - for data in nodes { - data.node.encode(&mut encoder).ok(); - data.edges.encode(&mut encoder).ok(); - data.fingerprint.encode(&mut encoder).ok(); - } + fn message(&mut self, (buffer_size_est, action): (usize, Action)) { + let mut encoder = opaque::Encoder::new(Vec::with_capacity(buffer_size_est * 5)); + action.encode(&mut encoder).ok(); self.file.write_all(&encoder.into_inner()).expect("unable to write to temp dep graph"); } - fn complete(mut self) { - let mut encoder = opaque::Encoder::new(Vec::with_capacity(16)); - encoder.emit_usize(0).ok(); - self.file.write_all(&encoder.into_inner()).expect("unable to write to temp dep graph"); - } + fn complete(self) {} } const BUFFER_SIZE: usize = 800000; pub struct Serializer { worker: Lrc>, - buffer: Vec, - buffer_size: usize, + new_buffer: Vec, + new_buffer_size: usize, + updated_buffer: Vec<(DepNodeIndex, DepNodeData)>, + updated_buffer_size: usize, } impl Serializer { @@ -87,33 +105,65 @@ impl Serializer { worker: Lrc::new(WorkerExecutor::new(SerializerWorker { file, })), - buffer: Vec::with_capacity(BUFFER_SIZE), - buffer_size: 0, + new_buffer: Vec::with_capacity(BUFFER_SIZE), + new_buffer_size: 0, + updated_buffer: Vec::with_capacity(BUFFER_SIZE), + updated_buffer_size: 0, + } + } + + fn flush_new(&mut self) { + let msgs = mem::replace(&mut self.new_buffer, Vec::with_capacity(BUFFER_SIZE)); + let buffer_size = self.new_buffer_size; + self.new_buffer_size = 0; + self.worker.message_in_pool((buffer_size, Action::NewNodes(msgs))); + } + + #[inline] + pub(super) fn serialize_new(&mut self, data: DepNodeData) { + let edges = data.edges.len(); + self.new_buffer.push(data); + self.new_buffer_size += 8 + edges; + if unlikely!(self.new_buffer_size >= BUFFER_SIZE) { + cold_path(|| { + self.flush_new(); + }) } } - fn flush(&mut self) { - let msgs = mem::replace(&mut self.buffer, Vec::with_capacity(BUFFER_SIZE)); - let buffer_size = self.buffer_size; - self.buffer_size = 0; - self.worker.message_in_pool((buffer_size, msgs)); + fn flush_updated(&mut self) { + let msgs = mem::replace(&mut self.updated_buffer, Vec::with_capacity(BUFFER_SIZE)); + let buffer_size = self.updated_buffer_size; + self.updated_buffer_size = 0; + self.worker.message_in_pool((buffer_size, Action::UpdateNodes(msgs))); } #[inline] - pub(super) fn serialize(&mut self, data: DepNodeData) { + pub(super) fn serialize_updated(&mut self, index: DepNodeIndex, data: DepNodeData) { let edges = data.edges.len(); - self.buffer.push(data); - self.buffer_size += 8 + edges; - if unlikely!(self.buffer_size >= BUFFER_SIZE) { + self.updated_buffer.push((index, data)); + self.updated_buffer_size += 9 + edges; + if unlikely!(self.updated_buffer_size >= BUFFER_SIZE) { cold_path(|| { - self.flush(); + self.flush_updated(); }) } } - pub fn complete(&mut self) { - if self.buffer.len() > 0 { - self.flush(); + pub fn complete( + &mut self, + invalidate: Vec, + ) { + if self.new_buffer.len() > 0 { + self.flush_new(); + } + if self.updated_buffer.len() > 0 { + self.flush_updated(); + } + if !invalidate.is_empty() { + self.worker.message_in_pool( + (invalidate.len(), Action::InvalidateNodes(invalidate)) + ); } self.worker.complete() } diff --git a/src/librustc/ty/query/plumbing.rs b/src/librustc/ty/query/plumbing.rs index d67503f65e690..37787635275c0 100644 --- a/src/librustc/ty/query/plumbing.rs +++ b/src/librustc/ty/query/plumbing.rs @@ -536,11 +536,14 @@ impl<'tcx> TyCtxt<'tcx> { // in DepGraph::try_mark_green() // 2. Two distinct query keys get mapped to the same DepNode // (see for example #48923) - assert!(!self.dep_graph.dep_node_exists(&dep_node), - "Forcing query with already existing DepNode.\n\ - - query-key: {:?}\n\ - - dep-node: {:?}", - key, dep_node); + /*debug_assert!( + !self.dep_graph.dep_node_exists(&dep_node), + "Forcing query with already existing DepNode.\n\ + - query-key: {:?}\n\ + - dep-node: {:?}", + key, + dep_node, + );*/ profq_msg!(self, ProfileQueriesMsg::ProviderBegin); self.sess.profiler(|p| p.start_query(Q::NAME)); diff --git a/src/librustc_codegen_ssa/base.rs b/src/librustc_codegen_ssa/base.rs index ca686453b6d4e..04a44caa049e1 100644 --- a/src/librustc_codegen_ssa/base.rs +++ b/src/librustc_codegen_ssa/base.rs @@ -868,9 +868,9 @@ fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguR // know that later). If we are not doing LTO, there is only one optimized // version of each module, so we re-use that. let dep_node = cgu.codegen_dep_node(tcx); - assert!(!tcx.dep_graph.dep_node_exists(&dep_node), + /*debug_assert!(!tcx.dep_graph.dep_node_exists(&dep_node), "CompileCodegenUnit dep-node for CGU `{}` already exists before marking.", - cgu.name()); + cgu.name());*/ if tcx.dep_graph.try_mark_green(tcx, &dep_node).is_some() { // We can re-use either the pre- or the post-thinlto state diff --git a/src/librustc_incremental/persist/dirty_clean.rs b/src/librustc_incremental/persist/dirty_clean.rs index e2e4a4ebcb056..935b5c49740ac 100644 --- a/src/librustc_incremental/persist/dirty_clean.rs +++ b/src/librustc_incremental/persist/dirty_clean.rs @@ -459,7 +459,7 @@ impl DirtyCleanVisitor<'tcx> { }) } - fn dep_node_str(&self, dep_node: &DepNode) -> String { + fn _dep_node_str(&self, dep_node: &DepNode) -> String { if let Some(def_id) = dep_node.extract_def_id(self.tcx) { format!("{:?}({})", dep_node.kind, @@ -469,10 +469,10 @@ impl DirtyCleanVisitor<'tcx> { } } - fn assert_dirty(&self, item_span: Span, dep_node: DepNode) { + fn assert_dirty(&self, _item_span: Span, dep_node: DepNode) { debug!("assert_dirty({:?})", dep_node); - - let dep_node_index = self.tcx.dep_graph.dep_node_index_of(&dep_node); + panic!() + /*let dep_node_index = self.tcx.dep_graph.dep_node_index_of(&dep_node); let current_fingerprint = self.tcx.dep_graph.fingerprint_of(dep_node_index); let prev_fingerprint = self.tcx.dep_graph.prev_fingerprint_of(&dep_node); @@ -481,13 +481,13 @@ impl DirtyCleanVisitor<'tcx> { self.tcx.sess.span_err( item_span, &format!("`{}` should be dirty but is not", dep_node_str)); - } + }*/ } - fn assert_clean(&self, item_span: Span, dep_node: DepNode) { + fn assert_clean(&self, _item_span: Span, dep_node: DepNode) { debug!("assert_clean({:?})", dep_node); - - let dep_node_index = self.tcx.dep_graph.dep_node_index_of(&dep_node); + panic!() + /*let dep_node_index = self.tcx.dep_graph.dep_node_index_of(&dep_node); let current_fingerprint = self.tcx.dep_graph.fingerprint_of(dep_node_index); let prev_fingerprint = self.tcx.dep_graph.prev_fingerprint_of(&dep_node); @@ -496,7 +496,7 @@ impl DirtyCleanVisitor<'tcx> { self.tcx.sess.span_err( item_span, &format!("`{}` should be clean but is not", dep_node_str)); - } + }*/ } fn check_item(&mut self, item_id: hir::HirId, item_span: Span) { diff --git a/src/librustc_incremental/persist/file_format.rs b/src/librustc_incremental/persist/file_format.rs index f363f718496fa..2a648807bbfc0 100644 --- a/src/librustc_incremental/persist/file_format.rs +++ b/src/librustc_incremental/persist/file_format.rs @@ -11,7 +11,7 @@ use std::io::{self, Read}; use std::path::Path; -use std::fs; +use std::fs::{OpenOptions, File}; use std::env; use rustc::session::config::nightly_options; @@ -49,13 +49,16 @@ pub fn write_file_header(stream: &mut Encoder) { /// - Returns `Err(..)` if some kind of IO error occurred while reading the /// file. pub fn read_file(report_incremental_info: bool, path: &Path) - -> io::Result, usize)>> + -> io::Result, usize, File)>> { if !path.exists() { return Ok(None); } - let data = fs::read(path)?; + let mut real_file = OpenOptions::new().read(true).write(true).open(path)?; + let len = real_file.metadata().map(|m| m.len() as usize + 1).unwrap_or(0); + let mut data = Vec::with_capacity(len); + real_file.read_to_end(&mut data)?; let mut file = io::Cursor::new(data); @@ -100,7 +103,7 @@ pub fn read_file(report_incremental_info: bool, path: &Path) } let post_header_start_pos = file.position() as usize; - Ok(Some((file.into_inner(), post_header_start_pos))) + Ok(Some((file.into_inner(), post_header_start_pos, real_file))) } fn report_format_mismatch(report_incremental_info: bool, file: &Path, message: &str) { diff --git a/src/librustc_incremental/persist/load.rs b/src/librustc_incremental/persist/load.rs index 18eb84f7c622d..802a97c0eaa4e 100644 --- a/src/librustc_incremental/persist/load.rs +++ b/src/librustc_incremental/persist/load.rs @@ -10,6 +10,7 @@ use rustc_serialize::Decodable as RustcDecodable; use rustc_serialize::opaque::Decoder; use rustc_serialize::Encodable; use std::path::Path; +use std::fs::{self, File}; use super::data::*; use super::fs::*; @@ -26,19 +27,21 @@ pub fn dep_graph_tcx_init<'tcx>(tcx: TyCtxt<'tcx>) { } pub fn dep_graph_from_future(sess: &Session, future: DepGraphFuture) -> DepGraph { - let (prev_graph, prev_work_products) = + let (prev_graph, prev_work_products, file) = time(sess, "blocked while dep-graph loading finishes", || { future.open().unwrap_or_else(|e| LoadResult::Error { message: format!("could not decode incremental cache: {:?}", e), - }).open(sess) + }).open(sess).unwrap_or_else(|| { + let temp_path = temp_dep_graph_path_from(&sess.incr_comp_session_dir()); + // Write the file header to the temp file + let file = save_in(sess, temp_path, |encoder| { + // Encode the commandline arguments hash + sess.opts.dep_tracking_hash().encode(encoder).unwrap(); + }).unwrap(); + + (Default::default(), Default::default(), file) + }) }); - let temp_path = temp_dep_graph_path_from(&sess.incr_comp_session_dir()); - - // Write the file header to the temp file - let file = save_in(sess, temp_path, |encoder| { - // Encode the commandline arguments hash - sess.opts.dep_tracking_hash().encode(encoder).unwrap(); - }).unwrap(); DepGraph::new(prev_graph, prev_work_products, file) } @@ -51,12 +54,12 @@ pub enum LoadResult { Error { message: String }, } -impl LoadResult<(PreviousDepGraph, WorkProductMap)> { - pub fn open(self, sess: &Session) -> (PreviousDepGraph, WorkProductMap) { +impl LoadResult<(PreviousDepGraph, WorkProductMap, File)> { + pub fn open(self, sess: &Session) -> Option<(PreviousDepGraph, WorkProductMap, File)> { match self { LoadResult::Error { message } => { sess.warn(&message); - Default::default() + None }, LoadResult::DataOutOfDate => { if let Err(err) = delete_all_session_dir_contents(sess) { @@ -64,15 +67,15 @@ impl LoadResult<(PreviousDepGraph, WorkProductMap)> { incremental compilation session directory contents `{}`: {}.", dep_graph_path(sess).display(), err)); } - Default::default() + None } - LoadResult::Ok { data } => data + LoadResult::Ok { data } => Some(data) } } } -fn load_data(report_incremental_info: bool, path: &Path) -> LoadResult<(Vec, usize)> { +fn load_data(report_incremental_info: bool, path: &Path) -> LoadResult<(Vec, usize, File)> { match file_format::read_file(report_incremental_info, path) { Ok(Some(data_and_pos)) => LoadResult::Ok { data: data_and_pos @@ -113,7 +116,7 @@ impl MaybeAsync { } } -pub type DepGraphFuture = MaybeAsync>; +pub type DepGraphFuture = MaybeAsync>; /// Launch a thread and load the dependency graph in the background. pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { @@ -122,16 +125,14 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { let time_passes = sess.time_passes(); - if sess.opts.incremental.is_none() { - // No incremental compilation. - return MaybeAsync::Sync(LoadResult::Ok { - data: Default::default(), - }); - } - // Calling `sess.incr_comp_session_dir()` will panic if `sess.opts.incremental.is_none()`. // Fortunately, we just checked that this isn't the case. let path = dep_graph_path_from(&sess.incr_comp_session_dir()); + let temp_path = temp_dep_graph_path_from(&sess.incr_comp_session_dir()); + if path.exists() { + fs::copy(&path, &temp_path).unwrap(); + } + let report_incremental_info = sess.opts.debugging_opts.incremental_info; let expected_hash = sess.opts.dep_tracking_hash(); @@ -144,7 +145,7 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { let work_products_path = work_products_path(sess); let load_result = load_data(report_incremental_info, &work_products_path); - if let LoadResult::Ok { data: (work_products_data, start_pos) } = load_result { + if let LoadResult::Ok { data: (work_products_data, start_pos, _) } = load_result { // Decode the list of work_products let mut work_product_decoder = Decoder::new(&work_products_data[..], start_pos); let work_products: Vec = @@ -181,10 +182,10 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { MaybeAsync::Async(std::thread::spawn(move || { time_ext(time_passes, None, "background load prev dep-graph", move || { - match load_data(report_incremental_info, &path) { + match load_data(report_incremental_info, &temp_path) { LoadResult::DataOutOfDate => LoadResult::DataOutOfDate, LoadResult::Error { message } => LoadResult::Error { message }, - LoadResult::Ok { data: (bytes, start_pos) } => { + LoadResult::Ok { data: (bytes, start_pos, file) } => { let mut decoder = Decoder::new(&bytes, start_pos); let prev_commandline_args_hash = u64::decode(&mut decoder) @@ -205,7 +206,9 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { let dep_graph = SerializedDepGraph::decode(&mut decoder) .expect("Error reading cached dep-graph"); - LoadResult::Ok { data: (PreviousDepGraph::new(dep_graph), prev_work_products) } + LoadResult::Ok { + data: (PreviousDepGraph::new(dep_graph), prev_work_products, file) + } } } }) @@ -219,7 +222,7 @@ pub fn load_query_result_cache<'sess>(sess: &'sess Session) -> OnDiskCache<'sess } match load_data(sess.opts.debugging_opts.incremental_info, &query_cache_path(sess)) { - LoadResult::Ok{ data: (bytes, start_pos) } => OnDiskCache::new(sess, bytes, start_pos), + LoadResult::Ok{ data: (bytes, start_pos, _) } => OnDiskCache::new(sess, bytes, start_pos), _ => OnDiskCache::new_empty(sess.source_map()) } } From 1a4abcc57e0d6bc800aa4ae691076ffc6f0d19a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Sat, 20 Apr 2019 13:09:38 +0200 Subject: [PATCH 07/17] Store dep node result fingerprints in a separate file --- src/librustc/dep_graph/graph.rs | 273 ++++++++++++----------- src/librustc/dep_graph/mod.rs | 4 +- src/librustc/dep_graph/prev.rs | 45 ++-- src/librustc/dep_graph/serialized.rs | 135 +++++++---- src/librustc/query/mod.rs | 2 +- src/librustc/ty/query/config.rs | 8 +- src/librustc/ty/query/on_disk_cache.rs | 30 +-- src/librustc/ty/query/plumbing.rs | 8 +- src/librustc_incremental/persist/fs.rs | 9 +- src/librustc_incremental/persist/load.rs | 156 ++++++++----- src/librustc_incremental/persist/save.rs | 64 +++--- src/librustc_macros/src/query.rs | 4 +- 12 files changed, 414 insertions(+), 324 deletions(-) diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index a9361fa15de87..d340e0dd9f2d7 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -3,12 +3,13 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::indexed_vec::{Idx, IndexVec}; use smallvec::SmallVec; -use rustc_data_structures::sync::{Lrc, Lock, AtomicCell, AtomicUsize, AtomicU64}; +use rustc_data_structures::sync::{Lrc, Lock, AtomicCell, AtomicU64}; use std::sync::atomic::Ordering::{Acquire, SeqCst}; use std::env; use std::fs::File; use std::hash::Hash; use std::collections::hash_map::Entry; +use std::mem; use crate::ty::{self, TyCtxt}; use crate::util::common::{ProfileQueriesMsg, profq_msg}; use parking_lot::{Mutex, Condvar}; @@ -19,7 +20,7 @@ use super::debug::EdgeFilter; use super::dep_node::{DepNode, DepKind, WorkProductId}; use super::query::DepGraphQuery; use super::safe::DepGraphSafe; -use super::serialized::{SerializedDepNodeIndex, Serializer}; +use super::serialized::Serializer; use super::prev::PreviousDepGraph; #[derive(Clone)] @@ -53,14 +54,16 @@ impl DepNodeColor { #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] pub enum DepNodeState { /// The node is invalid since its result is older than the previous session. - Invalidated, + Invalid, /// The node is from the previous session, but its state is unknown Unknown, - /// The node will eventually be green, + /// The node will eventually be green (it was previously Unknown), /// but its side effects (like error messages) have not yet happened. - WillBeGreen, + /// This only exist so we can know which thread + /// marked a node as green in `try_mark_previous_green`. + WasUnknownWillBeGreen, Red, @@ -70,9 +73,9 @@ pub enum DepNodeState { impl DepNodeState { pub fn color(self) -> Option { match self { - DepNodeState::Invalidated | + DepNodeState::Invalid | DepNodeState::Unknown | - DepNodeState::WillBeGreen => None, + DepNodeState::WasUnknownWillBeGreen => None, DepNodeState::Red => Some(DepNodeColor::Red), DepNodeState::Green => Some(DepNodeColor::Green), } @@ -88,7 +91,7 @@ struct DepGraphData { /// The dep-graph from the previous compilation session. It contains all /// nodes and edges as well as all fingerprints of nodes that have them. - previous: PreviousDepGraph, + previous: Lrc, colors: DepNodeColorMap, @@ -120,22 +123,25 @@ where Some(stable_hasher.finish()) } +pub struct DepGraphArgs { + pub prev_graph: PreviousDepGraph, + pub prev_work_products: FxHashMap, + pub file: File, + pub state: IndexVec>, +} + impl DepGraph { - pub fn new( - prev_graph: PreviousDepGraph, - prev_work_products: FxHashMap, - file: File, - ) -> DepGraph { - let prev_graph_node_count = prev_graph.node_count(); + pub fn new(args: DepGraphArgs) -> DepGraph { let colors = DepNodeColorMap { - values: prev_graph.state.iter().map(|s| AtomicCell::new(*s)).collect(), + values: args.state, }; + let prev_graph = Lrc::new(args.prev_graph); DepGraph { data: Some(Lrc::new(DepGraphData { - previous_work_products: prev_work_products, + previous_work_products: args.prev_work_products, dep_node_debug: Default::default(), - current: CurrentDepGraph::new(prev_graph_node_count, file), + current: CurrentDepGraph::new(prev_graph.clone(), args.file), emitted_diagnostics: Default::default(), emitted_diagnostics_cond_var: Condvar::new(), colors, @@ -359,13 +365,13 @@ impl DepGraph { data.colors.insert(prev_index, color); data.current.update_node( - prev_index.current(), + prev_index, key, deps, fingerprint ); - prev_index.current() + prev_index } else { if print_status { eprintln!("[task::new] {:?}", key); @@ -491,11 +497,6 @@ impl DepGraph { self.data.as_ref().unwrap().previous.fingerprint_of(dep_node) } - #[inline] - pub fn prev_dep_node_index_of(&self, dep_node: &DepNode) -> SerializedDepNodeIndex { - self.data.as_ref().unwrap().previous.node_to_index(dep_node) - } - /// Checks whether a previous work product exists for `v` and, if /// so, return the path that leads to it. Used to skip doing work. pub fn previous_work_product(&self, v: &WorkProductId) -> Option { @@ -547,15 +548,13 @@ impl DepGraph { } } - pub fn serialize(&self) { + pub fn serialize(&self) -> IndexVec { let data = self.data.as_ref().unwrap(); // Invalidate dep nodes with unknown state as these cannot safely // be marked green in the next session. let invalidate = data.colors.values.indices().filter_map(|prev_index| { match data.colors.get(prev_index) { - DepNodeState::Unknown => { - Some(prev_index.current()) - } + DepNodeState::Unknown => Some(prev_index), _ => None, } }).collect(); @@ -585,11 +584,11 @@ impl DepGraph { &self, tcx: TyCtxt<'_>, dep_node: &DepNode, - ) -> Option<(SerializedDepNodeIndex, DepNodeIndex)> { - self.try_mark_green(tcx, dep_node).map(|(prev_index, dep_node_index)| { + ) -> Option { + self.try_mark_green(tcx, dep_node).map(|prev_index| { debug_assert!(self.is_green(&dep_node)); - self.read_index(dep_node_index); - (prev_index, dep_node_index) + self.read_index(prev_index); + prev_index }) } @@ -597,7 +596,7 @@ impl DepGraph { &self, tcx: TyCtxt<'_>, dep_node: &DepNode, - ) -> Option<(SerializedDepNodeIndex, DepNodeIndex)> { + ) -> Option { debug_assert!(!dep_node.kind.is_eval_always()); // Return None if the dep graph is disabled @@ -607,11 +606,11 @@ impl DepGraph { let prev_index = data.previous.node_to_index_opt(dep_node)?; match data.colors.get(prev_index) { - DepNodeState::Green => Some((prev_index, prev_index.current())), - DepNodeState::Invalidated | + DepNodeState::Green => Some(prev_index), + DepNodeState::Invalid | DepNodeState::Red => None, DepNodeState::Unknown | - DepNodeState::WillBeGreen => { + DepNodeState::WasUnknownWillBeGreen => { // This DepNode and the corresponding query invocation existed // in the previous compilation session too, so we can try to // mark it as green by recursively marking all of its @@ -622,7 +621,7 @@ impl DepGraph { prev_index, &dep_node ) { - Some((prev_index, prev_index.current())) + Some(prev_index) } else { None } @@ -630,12 +629,73 @@ impl DepGraph { } } + /// Try to force a dep node to execute and see if it's green + fn try_force_previous_green<'tcx>( + &self, + tcx: TyCtxt<'_, 'tcx, 'tcx>, + data: &DepGraphData, + dep_node_index: DepNodeIndex, + ) -> bool { + let dep_node = &data.previous.index_to_node(dep_node_index); + + match dep_node.kind { + DepKind::Hir | + DepKind::HirBody | + DepKind::CrateMetadata => { + if dep_node.extract_def_id(tcx).is_none() { + // If the node does not exist anymore, we + // just fail to mark green. + return false + } else { + // If the node does exist, it should have + // been pre-allocated. + bug!("DepNode {:?} should have been \ + pre-allocated but wasn't.", + dep_node) + } + } + _ => { + // For other kinds of nodes it's OK to be + // forced. + } + } + + debug!("try_force_previous_green({:?}) --- trying to force", dep_node); + if crate::ty::query::force_from_dep_node(tcx, dep_node) { + match data.colors.get(dep_node_index) { + DepNodeState::Green => { + debug!("try_force_previous_green({:?}) --- managed to \ + FORCE to green", + dep_node); + true + } + DepNodeState::Red => { + debug!( + "try_force_previous_green({:?}) - END - was red after forcing", + dep_node + ); + false + } + DepNodeState::Invalid | + DepNodeState::Unknown | + DepNodeState::WasUnknownWillBeGreen => { + bug!("try_force_previous_green() - Forcing the DepNode \ + should have set its color") + } + } + } else { + // The DepNode could not be forced. + debug!("try_force_previous_green({:?}) - END - could not be forced", dep_node); + false + } + } + /// Try to mark a dep-node which existed in the previous compilation session as green. fn try_mark_previous_green<'tcx>( &self, tcx: TyCtxt<'tcx>, data: &DepGraphData, - dep_node_index: SerializedDepNodeIndex, + dep_node_index: DepNodeIndex, // FIXME: Remove this, only used in debug statements dep_node: &DepNode, ) -> bool { @@ -646,6 +706,9 @@ impl DepGraph { debug_assert!(data.colors.get(dep_node_index).color().is_none()); } + // We cannot mark invalid results as green + debug_assert_ne!(data.colors.get(dep_node_index), DepNodeState::Invalid); + // We never try to mark eval_always nodes as green debug_assert!(!dep_node.kind.is_eval_always()); @@ -677,83 +740,34 @@ impl DepGraph { data.previous.index_to_node(dep_dep_node_index)); return false } - DepNodeState::Invalidated => panic!("can this happen?"), - DepNodeState::WillBeGreen | + // Either the previous result is too old or + // this is a eval_always node. Try to force the node + DepNodeState::Invalid => { + if !self.try_force_previous_green(tcx, data, dep_dep_node_index) { + return false; + } + } + DepNodeState::WasUnknownWillBeGreen | DepNodeState::Unknown => { let dep_dep_node = &data.previous.index_to_node(dep_dep_node_index); - // We don't know the state of this dependency. If it isn't - // an eval_always node, let's try to mark it green recursively. - if !dep_dep_node.kind.is_eval_always() { - debug!("try_mark_previous_green({:?}) --- state of dependency {:?} \ - is unknown, trying to mark it green", dep_node, - dep_dep_node); - - if self.try_mark_previous_green( - tcx, - data, - dep_dep_node_index, - dep_dep_node - ) { - debug!("try_mark_previous_green({:?}) --- managed to MARK \ - dependency {:?} as green", dep_node, dep_dep_node); - continue; - } - } else { - match dep_dep_node.kind { - DepKind::Hir | - DepKind::HirBody | - DepKind::CrateMetadata => { - if dep_dep_node.extract_def_id(tcx).is_none() { - // If the node does not exist anymore, we - // just fail to mark green. - return false - } else { - // If the node does exist, it should have - // been pre-allocated. - bug!("DepNode {:?} should have been \ - pre-allocated but wasn't.", - dep_dep_node) - } - } - _ => { - // For other kinds of nodes it's OK to be - // forced. - } - } + // We don't know the state of this dependency. + // We known it is not an eval_always node, since those get marked as `Invalid`. + // Let's try to mark it green recursively. + if self.try_mark_previous_green( + tcx, + data, + dep_dep_node_index, + dep_dep_node + ) { + debug!("try_mark_previous_green({:?}) --- managed to MARK \ + dependency {:?} as green", dep_node, dep_dep_node); + continue; } // We failed to mark it green, so we try to force the query. - debug!("try_mark_previous_green({:?}) --- trying to force \ - dependency {:?}", dep_node, dep_dep_node); - if crate::ty::query::force_from_dep_node(tcx, dep_dep_node) { - let dep_dep_node_color = data.colors.get(dep_dep_node_index); - - match dep_dep_node_color { - DepNodeState::Green => { - debug!("try_mark_previous_green({:?}) --- managed to \ - FORCE dependency {:?} to green", - dep_node, dep_dep_node); - } - DepNodeState::Red => { - debug!("try_mark_previous_green({:?}) - END - \ - dependency {:?} was red after forcing", - dep_node, - dep_dep_node); - return false - } - DepNodeState::Invalidated | - DepNodeState::Unknown | - DepNodeState::WillBeGreen => { - bug!("try_mark_previous_green() - Forcing the DepNode \ - should have set its color") - } - } - } else { - // The DepNode could not be forced. - debug!("try_mark_previous_green({:?}) - END - dependency {:?} \ - could not be forced", dep_node, dep_dep_node); - return false + if !self.try_force_previous_green(tcx, data, dep_dep_node_index) { + return false; } } } @@ -781,10 +795,13 @@ impl DepGraph { self.emit_diagnostics( tcx, data, - dep_node_index.current(), + dep_node_index, did_mark, diagnostics ); + } else { + // Avoid calling the destructor, since LLVM fails to optimize it away + mem::forget(diagnostics); } // ... and finally storing a "Green" entry in the color map. @@ -867,8 +884,8 @@ impl DepGraph { None } } - DepNodeState::WillBeGreen => bug!("no tasks should be in progress"), - DepNodeState::Invalidated | + DepNodeState::WasUnknownWillBeGreen => bug!("no tasks should be in progress"), + DepNodeState::Invalid | DepNodeState::Unknown | DepNodeState::Red => { // We can skip red nodes because a node can only be marked @@ -947,7 +964,7 @@ pub enum WorkProductFileKind { BytecodeCompressed, } -#[derive(Clone, Debug, RustcDecodable, RustcEncodable)] +#[derive(Clone, Debug)] pub(super) struct DepNodeData { pub(super) node: DepNode, pub(super) edges: TaskReads, @@ -956,8 +973,6 @@ pub(super) struct DepNodeData { pub(super) struct CurrentDepGraph { //data: FxHashMap, - node_count: AtomicUsize, - /// Used to map input nodes to a node index. Used by the `read` method. input_node_to_node_index: Lock>, @@ -988,7 +1003,7 @@ pub(super) struct CurrentDepGraph { } impl CurrentDepGraph { - fn new(prev_graph_node_count: usize, file: File) -> CurrentDepGraph { + fn new(prev_graph: Lrc, file: File) -> CurrentDepGraph { use std::time::{SystemTime, UNIX_EPOCH}; let duration = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); @@ -1015,18 +1030,17 @@ impl CurrentDepGraph { // that we hopefully don't have to re-allocate during this compilation // session. The over-allocation is 2% plus a small constant to account // for the fact that in very small crates 2% might not be enough. - //let new_node_count_estimate = (prev_graph_node_count * 102) / 100 + 200; + //let new_node_count_estimate = (prev_graph.node_count() * 102) / 100 + 200; CurrentDepGraph { //data: FxHashMap::default(), - node_count: AtomicUsize::new(prev_graph_node_count), anon_node_to_node_index: Default::default(), input_node_to_node_index: Default::default(), anon_id_seed: stable_hasher.finish(), forbidden_edge, total_read_count: AtomicU64::new(0), total_duplicate_read_count: AtomicU64::new(0), - serializer: Lock::new(Serializer::new(file)), + serializer: Lock::new(Serializer::new(file, prev_graph)), } } @@ -1082,13 +1096,11 @@ impl CurrentDepGraph { previous: &PreviousDepGraph, ) -> DepNodeIndex { debug_assert!(previous.node_to_index_opt(&dep_node).is_none()); - let dep_node_index = DepNodeIndex::new(self.node_count.fetch_add(1, SeqCst)); self.serializer.lock().serialize_new(DepNodeData { node: dep_node, edges, fingerprint - }); - dep_node_index + }) } fn update_node( @@ -1151,27 +1163,28 @@ pub struct TaskDeps { } struct DepNodeColorMap { - values: IndexVec>, + values: IndexVec>, } impl DepNodeColorMap { - /// Tries to mark the node as WillBeGreen. Returns false if another thread did it before us. + /// Tries to mark the node as WasUnknownWillBeGreen. + /// Returns false if another thread did it before us. #[inline] - fn mark_as_will_be_green(&self, index: SerializedDepNodeIndex) -> bool { - let prev = self.values[index].compare_and_swap( + fn mark_as_will_be_green(&self, index: DepNodeIndex) -> bool { + let actual_prev = self.values[index].compare_and_swap( DepNodeState::Unknown, - DepNodeState::WillBeGreen + DepNodeState::WasUnknownWillBeGreen, ); - prev == DepNodeState::Unknown + actual_prev == DepNodeState::Unknown } #[inline] - fn get(&self, index: SerializedDepNodeIndex) -> DepNodeState { + fn get(&self, index: DepNodeIndex) -> DepNodeState { self.values[index].load() } #[inline] - fn insert(&self, index: SerializedDepNodeIndex, state: DepNodeState) { + fn insert(&self, index: DepNodeIndex, state: DepNodeState) { self.values[index].store(state) } } diff --git a/src/librustc/dep_graph/mod.rs b/src/librustc/dep_graph/mod.rs index 1535e6d349cf1..1af433cb06210 100644 --- a/src/librustc/dep_graph/mod.rs +++ b/src/librustc/dep_graph/mod.rs @@ -11,9 +11,9 @@ pub mod cgu_reuse_tracker; pub use self::dep_tracking_map::{DepTrackingMap, DepTrackingMapConfig}; pub use self::dep_node::{DepNode, DepKind, DepConstructor, WorkProductId, RecoverKey, label_strs}; pub use self::graph::{DepGraph, WorkProduct, DepNodeIndex, DepNodeColor, TaskDeps, hash_result}; -pub use self::graph::WorkProductFileKind; +pub use self::graph::{DepGraphArgs, WorkProductFileKind}; pub use self::prev::PreviousDepGraph; pub use self::query::DepGraphQuery; pub use self::safe::AssertDepGraphSafe; pub use self::safe::DepGraphSafe; -pub use self::serialized::{SerializedDepGraph, SerializedDepNodeIndex}; +pub use self::serialized::SerializedDepGraph; diff --git a/src/librustc/dep_graph/prev.rs b/src/librustc/dep_graph/prev.rs index 4c7cf06f005d6..463abdb5a7f38 100644 --- a/src/librustc/dep_graph/prev.rs +++ b/src/librustc/dep_graph/prev.rs @@ -2,41 +2,35 @@ use crate::ich::Fingerprint; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::indexed_vec::IndexVec; use super::dep_node::DepNode; -use super::graph::DepNodeState; -use super::serialized::{SerializedDepGraph, SerializedDepNodeIndex}; +use super::graph::DepNodeIndex; +use super::serialized::SerializedDepGraph; -#[derive(Debug, /*RustcEncodable, RustcDecodable,*/ Default)] +#[derive(Debug, Default)] pub struct PreviousDepGraph { /// Maps from dep nodes to their previous index, if any. - index: FxHashMap, + index: FxHashMap, /// The set of all DepNodes in the graph - nodes: IndexVec, + nodes: IndexVec, /// The set of all Fingerprints in the graph. Each Fingerprint corresponds to /// the DepNode at the same index in the nodes vector. - fingerprints: IndexVec, + pub(super) fingerprints: IndexVec, /// For each DepNode, stores the list of edges originating from that /// DepNode. Encoded as a [start, end) pair indexing into edge_list_data, /// which holds the actual DepNodeIndices of the target nodes. - edge_list_indices: IndexVec, + edge_list_indices: IndexVec, /// A flattened list of all edge targets in the graph. Edge sources are /// implicit in edge_list_indices. - edge_list_data: Vec, - /// A set of nodes which are no longer valid. - pub(super) state: IndexVec, + edge_list_data: Vec, } impl PreviousDepGraph { pub fn new(graph: SerializedDepGraph) -> PreviousDepGraph { let index: FxHashMap<_, _> = graph.nodes .iter_enumerated() - .map(|(idx, dep_node)| { - (dep_node.node, SerializedDepNodeIndex::from_u32(idx.as_u32())) - }) + .map(|(idx, dep_node)| (dep_node.node, idx)) .collect(); - let fingerprints: IndexVec = - graph.nodes.iter().map(|d| d.fingerprint).collect(); - let nodes: IndexVec = + let nodes: IndexVec = graph.nodes.iter().map(|d| d.node).collect(); let total_edge_count: usize = graph.nodes.iter().map(|d| d.edges.len()).sum(); @@ -47,9 +41,7 @@ impl PreviousDepGraph { for (current_dep_node_index, edges) in graph.nodes.iter_enumerated() .map(|(i, d)| (i, &d.edges)) { let start = edge_list_data.len() as u32; - edge_list_data.extend(edges.iter().map(|i| { - SerializedDepNodeIndex::from_u32(i.as_u32()) - })); + edge_list_data.extend(edges.iter().cloned()); let end = edge_list_data.len() as u32; debug_assert_eq!(current_dep_node_index.index(), edge_list_indices.len()); @@ -60,36 +52,35 @@ impl PreviousDepGraph { debug_assert_eq!(edge_list_data.len(), total_edge_count); PreviousDepGraph { - fingerprints, + fingerprints: graph.fingerprints, nodes, edge_list_indices, edge_list_data, index, - state: graph.state.convert_index_type(), } } #[inline] pub fn edge_targets_from( &self, - dep_node_index: SerializedDepNodeIndex - ) -> &[SerializedDepNodeIndex] { + dep_node_index: DepNodeIndex + ) -> &[DepNodeIndex] { let targets = self.edge_list_indices[dep_node_index]; &self.edge_list_data[targets.0 as usize..targets.1 as usize] } #[inline] - pub fn index_to_node(&self, dep_node_index: SerializedDepNodeIndex) -> DepNode { + pub fn index_to_node(&self, dep_node_index: DepNodeIndex) -> DepNode { self.nodes[dep_node_index] } #[inline] - pub fn node_to_index(&self, dep_node: &DepNode) -> SerializedDepNodeIndex { + pub fn node_to_index(&self, dep_node: &DepNode) -> DepNodeIndex { self.index[dep_node] } #[inline] - pub fn node_to_index_opt(&self, dep_node: &DepNode) -> Option { + pub fn node_to_index_opt(&self, dep_node: &DepNode) -> Option { self.index.get(dep_node).cloned() } @@ -102,7 +93,7 @@ impl PreviousDepGraph { #[inline] pub fn fingerprint_by_index(&self, - dep_node_index: SerializedDepNodeIndex) + dep_node_index: DepNodeIndex) -> Fingerprint { self.fingerprints[dep_node_index] } diff --git a/src/librustc/dep_graph/serialized.rs b/src/librustc/dep_graph/serialized.rs index 0d7494728dbf4..4623857ab0fa2 100644 --- a/src/librustc/dep_graph/serialized.rs +++ b/src/librustc/dep_graph/serialized.rs @@ -1,66 +1,86 @@ use rustc_data_structures::sync::worker::{Worker, WorkerExecutor}; -use rustc_data_structures::sync::Lrc; +use rustc_data_structures::sync::{Lrc, AtomicCell}; use rustc_data_structures::{unlikely, cold_path}; -use rustc_data_structures::indexed_vec::{IndexVec, Idx}; -use rustc_serialize::opaque; -use rustc_serialize::{Decodable, Encodable}; +use rustc_data_structures::indexed_vec::{Idx, IndexVec}; +use rustc_data_structures::fingerprint::Fingerprint; +use rustc_serialize::{Decodable, Encodable, opaque}; use std::mem; use std::fs::File; use std::io::Write; +use crate::dep_graph::dep_node::DepNode; +use super::prev::PreviousDepGraph; use super::graph::{DepNodeData, DepNodeIndex, DepNodeState}; -newtype_index! { - pub struct SerializedDepNodeIndex { .. } -} - -impl SerializedDepNodeIndex { - pub fn current(self) -> DepNodeIndex { - DepNodeIndex::from_u32(self.as_u32()) - } -} - #[derive(Debug, Default)] pub struct SerializedDepGraph { - pub(super) nodes: IndexVec, - pub(super) state: IndexVec, + pub(super) nodes: IndexVec, + pub(super) fingerprints: IndexVec, +} + +#[derive(Clone, Debug, RustcDecodable, RustcEncodable)] +pub(super) struct SerializedDepNodeData { + pub(super) node: DepNode, + pub(super) edges: Vec, } impl SerializedDepGraph { - pub fn decode(d: &mut opaque::Decoder<'_>) -> Result { - let mut nodes = IndexVec::new(); - let mut invalidated_list = Vec::new(); + pub fn decode( + d: &mut opaque::Decoder<'_>, + results_d: &mut opaque::Decoder<'_>, + ) -> Result<(Self, IndexVec>), String> { + let fingerprints: IndexVec = IndexVec::decode(results_d)?; + let mut nodes = IndexVec::with_capacity(fingerprints.len()); + let mut state: IndexVec<_, _> = (0..fingerprints.len()).map(|_| { + AtomicCell::new(DepNodeState::Unknown) + }).collect(); loop { if d.position() == d.data.len() { break; } - match Action::decode(d)? { - Action::NewNodes(new_nodes) => { + match SerializedAction::decode(d)? { + SerializedAction::NewNodes(new_nodes) => { + for (i, data) in new_nodes.iter().enumerate() { + // Mark the result of eval_always nodes as invalid so they will + // get executed again. + if unlikely!(data.node.kind.is_eval_always()) { + let idx = DepNodeIndex::new(nodes.len() + i); + state[idx] = AtomicCell::new(DepNodeState::Invalid); + } + } nodes.extend(new_nodes); } - Action::UpdateNodes(changed) => { - for (i, data) in changed { - nodes[i] = data; + SerializedAction::UpdateEdges(changed) => { + for (i, edges) in changed { + // Updated results are valid again, except for eval_always nodes + // which always start out invalid. + if likely!(!nodes[i].node.kind.is_eval_always()) { + state[i] = AtomicCell::new(DepNodeState::Unknown); + } + nodes[i].edges = edges; } } - Action::InvalidateNodes(nodes) => { - invalidated_list.extend(nodes); + SerializedAction::InvalidateNodes(nodes) => { + for i in nodes { + state[i] = AtomicCell::new(DepNodeState::Invalid); + } } } } - let mut state: IndexVec<_, _> = (0..nodes.len()).map(|_| { - DepNodeState::Unknown - }).collect(); - for i in invalidated_list { - state[i] = DepNodeState::Invalidated; - } - Ok(SerializedDepGraph { + Ok((SerializedDepGraph { nodes, - state, - }) + fingerprints, + }, state)) } } -#[derive(Debug, RustcEncodable, RustcDecodable)] +#[derive(Debug, RustcDecodable, RustcEncodable)] +enum SerializedAction { + NewNodes(Vec), + UpdateEdges(Vec<(DepNodeIndex, Vec)>), + InvalidateNodes(Vec) +} + +#[derive(Debug)] enum Action { NewNodes(Vec), UpdateNodes(Vec<(DepNodeIndex, DepNodeData)>), @@ -73,26 +93,50 @@ enum Action { } struct SerializerWorker { + fingerprints: IndexVec, + previous: Lrc, file: File, } impl Worker for SerializerWorker { type Message = (usize, Action); - type Result = (); + type Result = IndexVec; fn message(&mut self, (buffer_size_est, action): (usize, Action)) { let mut encoder = opaque::Encoder::new(Vec::with_capacity(buffer_size_est * 5)); + let action = match action { + Action::UpdateNodes(nodes) => { + SerializedAction::UpdateEdges(nodes.into_iter().filter(|&(i, ref data)| { + self.fingerprints[i] = data.fingerprint; + // Only emit nodes which actually changed + &*data.edges != self.previous.edge_targets_from(i) + }).map(|(i, data)| (i, data.edges.into_iter().collect::>())).collect()) + } + Action::NewNodes(nodes) => { + SerializedAction::NewNodes(nodes.into_iter().map(|data| { + self.fingerprints.push(data.fingerprint); + SerializedDepNodeData { + node: data.node, + edges: data.edges.into_iter().collect(), + } + }).collect()) + } + Action::InvalidateNodes(nodes) => SerializedAction::InvalidateNodes(nodes), + }; action.encode(&mut encoder).ok(); self.file.write_all(&encoder.into_inner()).expect("unable to write to temp dep graph"); } - fn complete(self) {} + fn complete(self) -> IndexVec { + self.fingerprints + } } const BUFFER_SIZE: usize = 800000; pub struct Serializer { worker: Lrc>, + node_count: u32, new_buffer: Vec, new_buffer_size: usize, updated_buffer: Vec<(DepNodeIndex, DepNodeData)>, @@ -100,9 +144,15 @@ pub struct Serializer { } impl Serializer { - pub fn new(file: File) -> Self { + pub fn new( + file: File, + previous: Lrc, + ) -> Self { Serializer { + node_count: previous.node_count() as u32, worker: Lrc::new(WorkerExecutor::new(SerializerWorker { + fingerprints: previous.fingerprints.clone(), + previous, file, })), new_buffer: Vec::with_capacity(BUFFER_SIZE), @@ -120,7 +170,7 @@ impl Serializer { } #[inline] - pub(super) fn serialize_new(&mut self, data: DepNodeData) { + pub(super) fn serialize_new(&mut self, data: DepNodeData) -> DepNodeIndex { let edges = data.edges.len(); self.new_buffer.push(data); self.new_buffer_size += 8 + edges; @@ -129,6 +179,9 @@ impl Serializer { self.flush_new(); }) } + let index = self.node_count; + self.node_count += 1; + DepNodeIndex::from_u32(index) } fn flush_updated(&mut self) { @@ -153,7 +206,7 @@ impl Serializer { pub fn complete( &mut self, invalidate: Vec, - ) { + ) -> IndexVec { if self.new_buffer.len() > 0 { self.flush_new(); } diff --git a/src/librustc/query/mod.rs b/src/librustc/query/mod.rs index 10efef54526a6..bcaf351542882 100644 --- a/src/librustc/query/mod.rs +++ b/src/librustc/query/mod.rs @@ -2,7 +2,7 @@ use crate::ty::query::QueryDescription; use crate::ty::query::queries; use crate::ty::{self, ParamEnvAnd, Ty, TyCtxt}; use crate::ty::subst::SubstsRef; -use crate::dep_graph::SerializedDepNodeIndex; +use crate::dep_graph::DepNodeIndex; use crate::hir::def_id::{CrateNum, DefId, DefIndex}; use crate::mir; use crate::mir::interpret::GlobalId; diff --git a/src/librustc/ty/query/config.rs b/src/librustc/ty/query/config.rs index 13d93f173e845..9b4e9108087d0 100644 --- a/src/librustc/ty/query/config.rs +++ b/src/librustc/ty/query/config.rs @@ -1,4 +1,4 @@ -use crate::dep_graph::SerializedDepNodeIndex; +use crate::dep_graph::DepNodeIndex; use crate::dep_graph::DepNode; use crate::hir::def_id::{CrateNum, DefId}; use crate::ty::TyCtxt; @@ -54,7 +54,9 @@ pub(crate) trait QueryDescription<'tcx>: QueryAccessors<'tcx> { false } - fn try_load_from_disk(_: TyCtxt<'tcx>, _: SerializedDepNodeIndex) -> Option { + fn try_load_from_disk(_: TyCtxt<'tcx>, + _: DepNodeIndex) + -> Option { bug!("QueryDescription::load_from_disk() called for an unsupported query.") } } @@ -86,7 +88,7 @@ macro_rules! impl_disk_cacheable_query( #[inline] fn try_load_from_disk(tcx: TyCtxt<'tcx>, - id: SerializedDepNodeIndex) + id: DepNodeIndex) -> Option { tcx.queries.on_disk_cache.try_load_query_result(tcx, id) } diff --git a/src/librustc/ty/query/on_disk_cache.rs b/src/librustc/ty/query/on_disk_cache.rs index 6f83991a2daa2..5924f341bfbbe 100644 --- a/src/librustc/ty/query/on_disk_cache.rs +++ b/src/librustc/ty/query/on_disk_cache.rs @@ -1,4 +1,4 @@ -use crate::dep_graph::{DepNodeIndex, SerializedDepNodeIndex}; +use crate::dep_graph::DepNodeIndex; use crate::hir; use crate::hir::def_id::{CrateNum, DefIndex, DefId, LocalDefId, LOCAL_CRATE}; use crate::hir::map::definitions::DefPathHash; @@ -62,11 +62,11 @@ pub struct OnDiskCache<'sess> { // A map from dep-node to the position of the cached query result in // `serialized_data`. - query_result_index: FxHashMap, + query_result_index: FxHashMap, // A map from dep-node to the position of any associated diagnostics in // `serialized_data`. - prev_diagnostics_index: FxHashMap, + prev_diagnostics_index: FxHashMap, alloc_decoding_state: AllocDecodingState, } @@ -82,8 +82,8 @@ struct Footer { interpret_alloc_index: Vec, } -type EncodedQueryResultIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>; -type EncodedDiagnosticsIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>; +type EncodedQueryResultIndex = Vec<(DepNodeIndex, AbsoluteBytePos)>; +type EncodedDiagnosticsIndex = Vec<(DepNodeIndex, AbsoluteBytePos)>; type EncodedDiagnostics = Vec; #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, RustcEncodable, RustcDecodable)] @@ -230,12 +230,12 @@ impl<'sess> OnDiskCache<'sess> { use crate::ty::query::config::QueryDescription; if const_eval::cache_on_disk(tcx, key.clone()) { if let Ok(ref value) = entry.value { - let dep_node = SerializedDepNodeIndex::new(entry.index.index()); + let dep_node = DepNodeIndex::new(entry.index.index()); // Record position of the cache entry qri.push((dep_node, AbsoluteBytePos::new(enc.position()))); - // Encode the type check tables with the SerializedDepNodeIndex + // Encode the type check tables with the DepNodeIndex // as tag. enc.encode_tagged(dep_node, value)?; } @@ -253,7 +253,7 @@ impl<'sess> OnDiskCache<'sess> { let pos = AbsoluteBytePos::new(encoder.position()); // Let's make sure we get the expected type here: let diagnostics: &EncodedDiagnostics = diagnostics; - let dep_node_index = SerializedDepNodeIndex::new(dep_node_index.index()); + let dep_node_index = DepNodeIndex::new(dep_node_index.index()); encoder.encode_tagged(dep_node_index, diagnostics)?; Ok((dep_node_index, pos)) @@ -327,7 +327,7 @@ impl<'sess> OnDiskCache<'sess> { pub fn load_diagnostics<'tcx>( &self, tcx: TyCtxt<'tcx>, - dep_node_index: SerializedDepNodeIndex, + dep_node_index: DepNodeIndex, ) -> Vec { let diagnostics: Option = self.load_indexed( tcx, @@ -352,11 +352,11 @@ impl<'sess> OnDiskCache<'sess> { } /// Returns the cached query result if there is something in the cache for - /// the given `SerializedDepNodeIndex`; otherwise returns `None`. + /// the given `DepNodeIndex`; otherwise returns `None`. pub fn try_load_query_result<'tcx, T>( &self, tcx: TyCtxt<'tcx>, - dep_node_index: SerializedDepNodeIndex, + dep_node_index: DepNodeIndex, ) -> Option where T: Decodable, @@ -386,8 +386,8 @@ impl<'sess> OnDiskCache<'sess> { fn load_indexed<'tcx, T>( &self, tcx: TyCtxt<'tcx>, - dep_node_index: SerializedDepNodeIndex, - index: &FxHashMap, + dep_node_index: DepNodeIndex, + index: &FxHashMap, debug_tag: &'static str, ) -> Option where @@ -1093,12 +1093,12 @@ where assert!(map.active.is_empty()); for (key, entry) in map.results.iter() { if Q::cache_on_disk(tcx, key.clone()) { - let dep_node = SerializedDepNodeIndex::new(entry.index.index()); + let dep_node = DepNodeIndex::new(entry.index.index()); // Record position of the cache entry query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position()))); - // Encode the type check tables with the SerializedDepNodeIndex + // Encode the type check tables with the DepNodeIndex // as tag. encoder.encode_tagged(dep_node, &entry.value)?; } diff --git a/src/librustc/ty/query/plumbing.rs b/src/librustc/ty/query/plumbing.rs index 37787635275c0..bb25e1017e59d 100644 --- a/src/librustc/ty/query/plumbing.rs +++ b/src/librustc/ty/query/plumbing.rs @@ -2,7 +2,7 @@ //! generate the actual methods on tcx which find and execute the provider, //! manage the caches, and so forth. -use crate::dep_graph::{DepNodeIndex, DepNode, DepKind, SerializedDepNodeIndex}; +use crate::dep_graph::{DepNodeIndex, DepNode, DepKind}; use crate::ty::tls; use crate::ty::{self, TyCtxt}; use crate::ty::query::Query; @@ -411,10 +411,9 @@ impl<'tcx> TyCtxt<'tcx> { // try_mark_green(), so we can ignore them here. let loaded = self.start_query(job.job.clone(), None, |tcx| { let marked = tcx.dep_graph.try_mark_green_and_read(tcx, &dep_node); - marked.map(|(prev_dep_node_index, dep_node_index)| { + marked.map(|dep_node_index| { (tcx.load_from_disk_and_cache_in_memory::( key.clone(), - prev_dep_node_index, dep_node_index, &dep_node ), dep_node_index) @@ -434,7 +433,6 @@ impl<'tcx> TyCtxt<'tcx> { fn load_from_disk_and_cache_in_memory>( self, key: Q::Key, - prev_dep_node_index: SerializedDepNodeIndex, dep_node_index: DepNodeIndex, dep_node: &DepNode, ) -> Q::Value { @@ -447,7 +445,7 @@ impl<'tcx> TyCtxt<'tcx> { let result = if Q::cache_on_disk(self.global_tcx(), key.clone()) && self.sess.opts.debugging_opts.incremental_queries { self.sess.profiler(|p| p.incremental_load_result_start(Q::NAME)); - let result = Q::try_load_from_disk(self.global_tcx(), prev_dep_node_index); + let result = Q::try_load_from_disk(self.global_tcx(), dep_node_index); self.sess.profiler(|p| p.incremental_load_result_end(Q::NAME)); // We always expect to find a cached result for things that diff --git a/src/librustc_incremental/persist/fs.rs b/src/librustc_incremental/persist/fs.rs index c2010ef340939..e9b0ad115f311 100644 --- a/src/librustc_incremental/persist/fs.rs +++ b/src/librustc_incremental/persist/fs.rs @@ -119,7 +119,7 @@ use rand::{RngCore, thread_rng}; const LOCK_FILE_EXT: &str = ".lock"; const DEP_GRAPH_FILENAME: &str = "dep-graph.bin"; -const TEMP_DEP_GRAPH_FILENAME: &str = "dep-graph.bin.tmp"; +pub const DEP_GRAPH_RESULTS_FILENAME: &str = "dep-graph-results.bin"; const WORK_PRODUCTS_FILENAME: &str = "work-products.bin"; const QUERY_CACHE_FILENAME: &str = "query-cache.bin"; @@ -135,9 +135,6 @@ pub fn dep_graph_path(sess: &Session) -> PathBuf { pub fn dep_graph_path_from(incr_comp_session_dir: &Path) -> PathBuf { in_incr_comp_dir(incr_comp_session_dir, DEP_GRAPH_FILENAME) } -pub fn temp_dep_graph_path_from(incr_comp_session_dir: &Path) -> PathBuf { - in_incr_comp_dir(incr_comp_session_dir, TEMP_DEP_GRAPH_FILENAME) -} pub fn work_products_path(sess: &Session) -> PathBuf { in_incr_comp_dir_sess(sess, WORK_PRODUCTS_FILENAME) @@ -384,9 +381,7 @@ pub fn delete_all_session_dir_contents(sess: &Session) -> io::Result<()> { for entry in sess_dir_iterator { let entry = entry?; let path = entry.path(); - if path != temp_dep_graph_path_from(&sess.incr_comp_session_dir()) { - safe_remove_file(&path)? - } + safe_remove_file(&path)? } Ok(()) } diff --git a/src/librustc_incremental/persist/load.rs b/src/librustc_incremental/persist/load.rs index 802a97c0eaa4e..669b28a475b90 100644 --- a/src/librustc_incremental/persist/load.rs +++ b/src/librustc_incremental/persist/load.rs @@ -1,7 +1,7 @@ //! Code to save/load the dep-graph from files. use rustc_data_structures::fx::FxHashMap; -use rustc::dep_graph::{DepGraph, PreviousDepGraph, SerializedDepGraph, WorkProduct, WorkProductId}; +use rustc::dep_graph::{DepGraph, DepGraphArgs, PreviousDepGraph, SerializedDepGraph}; use rustc::session::Session; use rustc::ty::TyCtxt; use rustc::ty::query::OnDiskCache; @@ -27,26 +27,28 @@ pub fn dep_graph_tcx_init<'tcx>(tcx: TyCtxt<'tcx>) { } pub fn dep_graph_from_future(sess: &Session, future: DepGraphFuture) -> DepGraph { - let (prev_graph, prev_work_products, file) = - time(sess, "blocked while dep-graph loading finishes", || { - future.open().unwrap_or_else(|e| LoadResult::Error { - message: format!("could not decode incremental cache: {:?}", e), - }).open(sess).unwrap_or_else(|| { - let temp_path = temp_dep_graph_path_from(&sess.incr_comp_session_dir()); - // Write the file header to the temp file - let file = save_in(sess, temp_path, |encoder| { - // Encode the commandline arguments hash - sess.opts.dep_tracking_hash().encode(encoder).unwrap(); - }).unwrap(); - - (Default::default(), Default::default(), file) - }) - }); - - DepGraph::new(prev_graph, prev_work_products, file) -} + let args = time(sess, "blocked while dep-graph loading finishes", || { + future.open().unwrap_or_else(|e| LoadResult::Error { + message: format!("could not decode incremental cache: {:?}", e), + }).open(sess).unwrap_or_else(|| { + let path = dep_graph_path_from(&sess.incr_comp_session_dir()); + // Write the file header to the temp file + let file = save_in(sess, &path, |encoder| { + // Encode the commandline arguments hash + sess.opts.dep_tracking_hash().encode(encoder).unwrap(); + }).unwrap(); + + DepGraphArgs { + prev_graph: Default::default(), + prev_work_products: Default::default(), + file, + state: Default::default(), + } + }) + }); -type WorkProductMap = FxHashMap; + DepGraph::new(args) +} pub enum LoadResult { Ok { data: T }, @@ -54,8 +56,8 @@ pub enum LoadResult { Error { message: String }, } -impl LoadResult<(PreviousDepGraph, WorkProductMap, File)> { - pub fn open(self, sess: &Session) -> Option<(PreviousDepGraph, WorkProductMap, File)> { +impl LoadResult { + pub fn open(self, sess: &Session) -> Option { match self { LoadResult::Error { message } => { sess.warn(&message); @@ -116,7 +118,39 @@ impl MaybeAsync { } } -pub type DepGraphFuture = MaybeAsync>; +pub type DepGraphFuture = MaybeAsync>; + +fn load_graph_file( + report_incremental_info: bool, + path: &Path, + expected_hash: u64, +) -> LoadResult<(Vec, usize, File)> { + match load_data(report_incremental_info, &path) { + LoadResult::DataOutOfDate => LoadResult::DataOutOfDate, + LoadResult::Error { message } => LoadResult::Error { message }, + LoadResult::Ok { data: (bytes, start_pos, file) } => { + let mut decoder = Decoder::new(&bytes, start_pos); + let prev_commandline_args_hash = u64::decode(&mut decoder) + .expect("Error reading commandline arg hash from cached dep-graph"); + + if prev_commandline_args_hash != expected_hash { + if report_incremental_info { + println!("[incremental] completely ignoring cache because of \ + differing commandline arguments"); + } + // We can't reuse the cache, purge it. + debug!("load_dep_graph_new: differing commandline arg hashes"); + + // No need to do any further work + return LoadResult::DataOutOfDate; + } + let pos = decoder.position(); + LoadResult::Ok { + data: (bytes, pos, file) + } + } + } +} /// Launch a thread and load the dependency graph in the background. pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { @@ -127,12 +161,20 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { // Calling `sess.incr_comp_session_dir()` will panic if `sess.opts.incremental.is_none()`. // Fortunately, we just checked that this isn't the case. - let path = dep_graph_path_from(&sess.incr_comp_session_dir()); - let temp_path = temp_dep_graph_path_from(&sess.incr_comp_session_dir()); - if path.exists() { - fs::copy(&path, &temp_path).unwrap(); + let dir = &sess.incr_comp_session_dir(); + + let path = dep_graph_path_from(dir); + { + let temp_path = path.with_extension("tmp"); + if path.exists() { + fs::copy(&path, &temp_path).unwrap(); + fs::remove_file(&path).unwrap(); + fs::rename(&temp_path, &path).unwrap(); + } } + let results_path = dir.join(DEP_GRAPH_RESULTS_FILENAME); + let report_incremental_info = sess.opts.debugging_opts.incremental_info; let expected_hash = sess.opts.dep_tracking_hash(); @@ -182,33 +224,39 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { MaybeAsync::Async(std::thread::spawn(move || { time_ext(time_passes, None, "background load prev dep-graph", move || { - match load_data(report_incremental_info, &temp_path) { - LoadResult::DataOutOfDate => LoadResult::DataOutOfDate, - LoadResult::Error { message } => LoadResult::Error { message }, - LoadResult::Ok { data: (bytes, start_pos, file) } => { - - let mut decoder = Decoder::new(&bytes, start_pos); - let prev_commandline_args_hash = u64::decode(&mut decoder) - .expect("Error reading commandline arg hash from cached dep-graph"); - - if prev_commandline_args_hash != expected_hash { - if report_incremental_info { - println!("[incremental] completely ignoring cache because of \ - differing commandline arguments"); - } - // We can't reuse the cache, purge it. - debug!("load_dep_graph_new: differing commandline arg hashes"); - - // No need to do any further work - return LoadResult::DataOutOfDate; - } - - let dep_graph = SerializedDepGraph::decode(&mut decoder) - .expect("Error reading cached dep-graph"); - - LoadResult::Ok { - data: (PreviousDepGraph::new(dep_graph), prev_work_products, file) - } + let (bytes, pos, file) = match load_graph_file( + report_incremental_info, + &path, + expected_hash + ) { + LoadResult::DataOutOfDate => return LoadResult::DataOutOfDate, + LoadResult::Error { message } => return LoadResult::Error { message }, + LoadResult::Ok { data } => data, + }; + + let (results_bytes, results_pos, _) = match load_graph_file( + report_incremental_info, + &results_path, + expected_hash, + ) { + LoadResult::DataOutOfDate => return LoadResult::DataOutOfDate, + LoadResult::Error { message } => return LoadResult::Error { message }, + LoadResult::Ok { data } => data, + }; + + let mut decoder = Decoder::new(&bytes, pos); + let mut results_decoder = Decoder::new(&results_bytes, results_pos); + let (dep_graph, state) = SerializedDepGraph::decode( + &mut decoder, + &mut results_decoder, + ).expect("Error reading cached dep-graph"); + + LoadResult::Ok { + data: DepGraphArgs { + prev_graph: PreviousDepGraph::new(dep_graph), + prev_work_products, + file, + state, } } }) diff --git a/src/librustc_incremental/persist/save.rs b/src/librustc_incremental/persist/save.rs index e35db3b84f534..05d7e4fd608c6 100644 --- a/src/librustc_incremental/persist/save.rs +++ b/src/librustc_incremental/persist/save.rs @@ -8,7 +8,7 @@ use rustc_serialize::Encodable as RustcEncodable; use rustc_serialize::opaque::Encoder; use std::fs::{self, File}; use std::io::{self, Write}; -use std::path::PathBuf; +use std::path::Path; use super::data::*; use super::fs::*; @@ -24,21 +24,21 @@ pub fn save_dep_graph<'tcx>(tcx: TyCtxt<'tcx>) { return; } + let dir = &sess.incr_comp_session_dir(); let query_cache_path = query_cache_path(sess); - let temp_dep_graph_path = temp_dep_graph_path_from(&sess.incr_comp_session_dir()); - let dep_graph_path = dep_graph_path(sess); + let results_path = dir.join(DEP_GRAPH_RESULTS_FILENAME); join(move || { if tcx.sess.opts.debugging_opts.incremental_queries { time(sess, "persist query result cache", || { save_in(sess, - query_cache_path, + &query_cache_path, |e| encode_query_cache(tcx, e)).unwrap(); }); } }, || { - time(sess, "swap dep-graph", || { - swap_dep_graph(tcx, temp_dep_graph_path, dep_graph_path) + time(sess, "save dep-graph", || { + finish_dep_graph(tcx, &results_path) }); }); @@ -56,7 +56,7 @@ pub fn save_work_product_index(sess: &Session, debug!("save_work_product_index()"); dep_graph.assert_ignored(); let path = work_products_path(sess); - save_in(sess, path, |e| encode_work_product_index(&new_work_products, e)).unwrap(); + save_in(sess, &path, |e| encode_work_product_index(&new_work_products, e)).unwrap(); // We also need to clean out old work-products, as not all of them are // deleted during invalidation. Some object files don't change their @@ -82,24 +82,24 @@ pub fn save_work_product_index(sess: &Session, }); } -pub(super) fn save_in(sess: &Session, path_buf: PathBuf, encode: F) -> io::Result +pub(super) fn save_in(sess: &Session, path: &Path, encode: F) -> io::Result where F: FnOnce(&mut Encoder) { - debug!("save: storing data in {}", path_buf.display()); + debug!("save: storing data in {}", path.display()); // delete the old dep-graph, if any // Note: It's important that we actually delete the old file and not just // truncate and overwrite it, since it might be a shared hard-link, the // underlying data of which we don't want to modify - if path_buf.exists() { - match fs::remove_file(&path_buf) { + if path.exists() { + match fs::remove_file(path) { Ok(()) => { debug!("save: remove old file"); } Err(err) => { sess.err(&format!("unable to delete old dep-graph at `{}`: {}", - path_buf.display(), + path.display(), err)); return Err(err); } @@ -113,7 +113,7 @@ where // write the data out let data = encoder.into_inner(); - let mut file = File::create(&path_buf)?; + let mut file = File::create(path)?; match file.write_all(&data) { Ok(_) => { @@ -121,7 +121,7 @@ where } Err(err) => { sess.err(&format!("failed to write dep-graph to `{}`: {}", - path_buf.display(), + path.display(), err)); return Err(err); } @@ -130,33 +130,23 @@ where Ok(file) } -fn swap_dep_graph(tcx: TyCtxt<'_, '_>, temp: PathBuf, old: PathBuf) { +fn finish_dep_graph(tcx: TyCtxt<'_>, results_path: &Path) { let sess = tcx.sess; // Encode the graph data. - time(tcx.sess, "finish graph serialization", || { - tcx.dep_graph.serialize(); + let results = time(tcx.sess, "finish graph serialization", || { + tcx.dep_graph.serialize() }); - // delete the old dep-graph, if any - // Note: It's important that we actually delete the old file and not just - // truncate and overwrite it, since it might be a shared hard-link, the - // underlying data of which we don't want to modify - if old.exists() { - match fs::remove_file(&old) { - Ok(()) => { - debug!("save: remove old file"); - } - Err(err) => { - sess.err(&format!("unable to delete old dep-graph at `{}`: {}", - old.display(), - err)); - return; - } - } - } - - fs::rename(temp, old).expect("unable to rename temp dep graph"); - + time(sess, "save dep-graph results", || { + save_in(sess, results_path, |e| { + // First encode the commandline arguments hash + sess.opts.dep_tracking_hash().encode(e).unwrap(); + time(sess, "encode dep-graph results", || { + // Save the result vector + results.encode(e).unwrap(); + }); + }).unwrap(); + }); /* if tcx.sess.opts.debugging_opts.incremental_info { diff --git a/src/librustc_macros/src/query.rs b/src/librustc_macros/src/query.rs index 0474d2a2e3b3a..79696f0e8785d 100644 --- a/src/librustc_macros/src/query.rs +++ b/src/librustc_macros/src/query.rs @@ -328,7 +328,7 @@ fn add_query_description_impl( #[inline] fn try_load_from_disk( #tcx: TyCtxt<'tcx>, - #id: SerializedDepNodeIndex + #id: DepNodeIndex ) -> Option { #block } @@ -339,7 +339,7 @@ fn add_query_description_impl( #[inline] fn try_load_from_disk( tcx: TyCtxt<'tcx>, - id: SerializedDepNodeIndex + id: DepNodeIndex ) -> Option { tcx.queries.on_disk_cache.try_load_query_result(tcx, id) } From 62ca540b83b01dc835e7779612b0a85476166adf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Sat, 27 Apr 2019 06:45:54 +0200 Subject: [PATCH 08/17] wip --- src/librustc/dep_graph/graph.rs | 62 ++++- src/librustc/dep_graph/mod.rs | 2 +- src/librustc/dep_graph/prev.rs | 52 +--- src/librustc/dep_graph/serialized.rs | 299 +++++++++++++++++------ src/librustc_incremental/persist/load.rs | 31 ++- 5 files changed, 294 insertions(+), 152 deletions(-) diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index d340e0dd9f2d7..2f70764fa9642 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -53,9 +53,13 @@ impl DepNodeColor { #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] pub enum DepNodeState { - /// The node is invalid since its result is older than the previous session. + /// The dep node index is invalid and does not refer to any dep node. Invalid, + /// The node is from the previous session and it is a eval_always node, + // but its state is unknown. + UnknownEvalAlways, + /// The node is from the previous session, but its state is unknown Unknown, @@ -73,7 +77,8 @@ pub enum DepNodeState { impl DepNodeState { pub fn color(self) -> Option { match self { - DepNodeState::Invalid | + DepNodeState::Invalid => bug!(), + DepNodeState::UnknownEvalAlways | DepNodeState::Unknown | DepNodeState::WasUnknownWillBeGreen => None, DepNodeState::Red => Some(DepNodeColor::Red), @@ -128,6 +133,7 @@ pub struct DepGraphArgs { pub prev_work_products: FxHashMap, pub file: File, pub state: IndexVec>, + pub invalidated: Vec, } impl DepGraph { @@ -141,7 +147,7 @@ impl DepGraph { data: Some(Lrc::new(DepGraphData { previous_work_products: args.prev_work_products, dep_node_debug: Default::default(), - current: CurrentDepGraph::new(prev_graph.clone(), args.file), + current: CurrentDepGraph::new(prev_graph.clone(), args.file, args.invalidated), emitted_diagnostics: Default::default(), emitted_diagnostics_cond_var: Condvar::new(), colors, @@ -551,11 +557,28 @@ impl DepGraph { pub fn serialize(&self) -> IndexVec { let data = self.data.as_ref().unwrap(); // Invalidate dep nodes with unknown state as these cannot safely - // be marked green in the next session. + // be marked green in the next session. One of the dependencies of the + // unknown node may have changed in this session (and is currently marked red), + // but might be green again in the next session, which may cause the unknown node + // to incorrectly be marked green in the next session, even though one of its dependencies + // did actually change. + let invalidate = data.colors.values.indices().filter_map(|prev_index| { match data.colors.get(prev_index) { - DepNodeState::Unknown => Some(prev_index), - _ => None, + // In order to this invalidation to be safe, none of the valid nodes can + // point to unknown nodes. + DepNodeState::Unknown | + DepNodeState::UnknownEvalAlways => Some(prev_index), + + DepNodeState::WasUnknownWillBeGreen => bug!(), + + // For green nodes, we either executed the query (which always uses valid nodes) + // or we marked it as green because all its dependencies are green and valid. + DepNodeState::Green | + // Red nodes were always exexuted. + DepNodeState::Red | + // We don't need to invalidate already invalid nodes + DepNodeState::Invalid => None, } }).collect(); // FIXME: Can this deadlock? @@ -606,8 +629,11 @@ impl DepGraph { let prev_index = data.previous.node_to_index_opt(dep_node)?; match data.colors.get(prev_index) { + DepNodeState::Invalid => bug!(), DepNodeState::Green => Some(prev_index), - DepNodeState::Invalid | + // We don't need to mark eval_always nodes as green here, since we'll just be executing + // the query after anyway. + DepNodeState::UnknownEvalAlways | DepNodeState::Red => None, DepNodeState::Unknown | DepNodeState::WasUnknownWillBeGreen => { @@ -677,6 +703,7 @@ impl DepGraph { false } DepNodeState::Invalid | + DepNodeState::UnknownEvalAlways | DepNodeState::Unknown | DepNodeState::WasUnknownWillBeGreen => { bug!("try_force_previous_green() - Forcing the DepNode \ @@ -720,6 +747,7 @@ impl DepGraph { let dep_dep_node_color = data.colors.get(dep_dep_node_index); match dep_dep_node_color { + DepNodeState::Invalid => bug!(), DepNodeState::Green => { // This dependency has been marked as green before, we are // still fine and can continue with checking the other @@ -740,9 +768,8 @@ impl DepGraph { data.previous.index_to_node(dep_dep_node_index)); return false } - // Either the previous result is too old or - // this is a eval_always node. Try to force the node - DepNodeState::Invalid => { + // This is a eval_always node. Try to force the node + DepNodeState::UnknownEvalAlways => { if !self.try_force_previous_green(tcx, data, dep_dep_node_index) { return false; } @@ -885,8 +912,15 @@ impl DepGraph { } } DepNodeState::WasUnknownWillBeGreen => bug!("no tasks should be in progress"), + + // There cannot be results stored for invalid indices. DepNodeState::Invalid | + + // Unknown nodes are unused, so we don't want to promote these and we would + // not to mark their colors in order to do so anyway. + DepNodeState::UnknownEvalAlways | DepNodeState::Unknown | + DepNodeState::Red => { // We can skip red nodes because a node can only be marked // as red if the query result was recomputed and thus is @@ -1003,7 +1037,11 @@ pub(super) struct CurrentDepGraph { } impl CurrentDepGraph { - fn new(prev_graph: Lrc, file: File) -> CurrentDepGraph { + fn new( + prev_graph: Lrc, + file: File, + invalidated: Vec, + ) -> CurrentDepGraph { use std::time::{SystemTime, UNIX_EPOCH}; let duration = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); @@ -1040,7 +1078,7 @@ impl CurrentDepGraph { forbidden_edge, total_read_count: AtomicU64::new(0), total_duplicate_read_count: AtomicU64::new(0), - serializer: Lock::new(Serializer::new(file, prev_graph)), + serializer: Lock::new(Serializer::new(file, prev_graph, invalidated)), } } diff --git a/src/librustc/dep_graph/mod.rs b/src/librustc/dep_graph/mod.rs index 1af433cb06210..cdbe289721c11 100644 --- a/src/librustc/dep_graph/mod.rs +++ b/src/librustc/dep_graph/mod.rs @@ -16,4 +16,4 @@ pub use self::prev::PreviousDepGraph; pub use self::query::DepGraphQuery; pub use self::safe::AssertDepGraphSafe; pub use self::safe::DepGraphSafe; -pub use self::serialized::SerializedDepGraph; +pub use self::serialized::decode_dep_graph; diff --git a/src/librustc/dep_graph/prev.rs b/src/librustc/dep_graph/prev.rs index 463abdb5a7f38..8bd8d5bf6afc1 100644 --- a/src/librustc/dep_graph/prev.rs +++ b/src/librustc/dep_graph/prev.rs @@ -3,70 +3,28 @@ use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::indexed_vec::IndexVec; use super::dep_node::DepNode; use super::graph::DepNodeIndex; -use super::serialized::SerializedDepGraph; #[derive(Debug, Default)] pub struct PreviousDepGraph { /// Maps from dep nodes to their previous index, if any. - index: FxHashMap, + pub(super) index: FxHashMap, /// The set of all DepNodes in the graph - nodes: IndexVec, + pub(super) nodes: IndexVec, /// The set of all Fingerprints in the graph. Each Fingerprint corresponds to /// the DepNode at the same index in the nodes vector. pub(super) fingerprints: IndexVec, /// For each DepNode, stores the list of edges originating from that - /// DepNode. Encoded as a [start, end) pair indexing into edge_list_data, - /// which holds the actual DepNodeIndices of the target nodes. - edge_list_indices: IndexVec, - /// A flattened list of all edge targets in the graph. Edge sources are - /// implicit in edge_list_indices. - edge_list_data: Vec, + /// DepNode. + pub(super) edges: IndexVec>>, } impl PreviousDepGraph { - pub fn new(graph: SerializedDepGraph) -> PreviousDepGraph { - let index: FxHashMap<_, _> = graph.nodes - .iter_enumerated() - .map(|(idx, dep_node)| (dep_node.node, idx)) - .collect(); - - let nodes: IndexVec = - graph.nodes.iter().map(|d| d.node).collect(); - - let total_edge_count: usize = graph.nodes.iter().map(|d| d.edges.len()).sum(); - - let mut edge_list_indices = IndexVec::with_capacity(nodes.len()); - let mut edge_list_data = Vec::with_capacity(total_edge_count); - - for (current_dep_node_index, edges) in graph.nodes.iter_enumerated() - .map(|(i, d)| (i, &d.edges)) { - let start = edge_list_data.len() as u32; - edge_list_data.extend(edges.iter().cloned()); - let end = edge_list_data.len() as u32; - - debug_assert_eq!(current_dep_node_index.index(), edge_list_indices.len()); - edge_list_indices.push((start, end)); - } - - debug_assert!(edge_list_data.len() <= ::std::u32::MAX as usize); - debug_assert_eq!(edge_list_data.len(), total_edge_count); - - PreviousDepGraph { - fingerprints: graph.fingerprints, - nodes, - edge_list_indices, - edge_list_data, - index, - } - } - #[inline] pub fn edge_targets_from( &self, dep_node_index: DepNodeIndex ) -> &[DepNodeIndex] { - let targets = self.edge_list_indices[dep_node_index]; - &self.edge_list_data[targets.0 as usize..targets.1 as usize] + self.edges[dep_node_index].as_ref().unwrap() } #[inline] diff --git a/src/librustc/dep_graph/serialized.rs b/src/librustc/dep_graph/serialized.rs index 4623857ab0fa2..16fdd5427c27e 100644 --- a/src/librustc/dep_graph/serialized.rs +++ b/src/librustc/dep_graph/serialized.rs @@ -1,95 +1,111 @@ use rustc_data_structures::sync::worker::{Worker, WorkerExecutor}; use rustc_data_structures::sync::{Lrc, AtomicCell}; use rustc_data_structures::{unlikely, cold_path}; -use rustc_data_structures::indexed_vec::{Idx, IndexVec}; +use rustc_data_structures::indexed_vec::IndexVec; use rustc_data_structures::fingerprint::Fingerprint; -use rustc_serialize::{Decodable, Encodable, opaque}; +use rustc_data_structures::fx::FxHashMap; +use rustc_serialize::{Decodable, Encodable, Encoder, Decoder, opaque}; use std::mem; use std::fs::File; use std::io::Write; -use crate::dep_graph::dep_node::DepNode; +use std::iter::repeat; +use crate::util::common::time_ext; +use crate::dep_graph::dep_node::{DepKind, DepNode}; use super::prev::PreviousDepGraph; use super::graph::{DepNodeData, DepNodeIndex, DepNodeState}; -#[derive(Debug, Default)] -pub struct SerializedDepGraph { - pub(super) nodes: IndexVec, - pub(super) fingerprints: IndexVec, -} - -#[derive(Clone, Debug, RustcDecodable, RustcEncodable)] -pub(super) struct SerializedDepNodeData { - pub(super) node: DepNode, - pub(super) edges: Vec, -} - -impl SerializedDepGraph { - pub fn decode( - d: &mut opaque::Decoder<'_>, - results_d: &mut opaque::Decoder<'_>, - ) -> Result<(Self, IndexVec>), String> { - let fingerprints: IndexVec = IndexVec::decode(results_d)?; - let mut nodes = IndexVec::with_capacity(fingerprints.len()); - let mut state: IndexVec<_, _> = (0..fingerprints.len()).map(|_| { - AtomicCell::new(DepNodeState::Unknown) +pub fn decode_dep_graph( + time_passes: bool, + d: &mut opaque::Decoder<'_>, + results_d: &mut opaque::Decoder<'_>, +) -> Result<( + PreviousDepGraph, + IndexVec>, + Vec, + ), String> { + let fingerprints: IndexVec = + time_ext(time_passes, None, "decode prev result fingerprints", || { + IndexVec::decode(results_d) + })?; + let mut nodes: IndexVec<_, _> = repeat(DepNode { + kind: DepKind::Null, + hash: Fingerprint::ZERO, + }).take(fingerprints.len()).collect(); + let mut edges: IndexVec<_, _> = repeat(None).take(fingerprints.len()).collect(); + let mut state: IndexVec<_, _> = (0..fingerprints.len()).map(|_| { + AtomicCell::new(DepNodeState::Invalid) }).collect(); - loop { - if d.position() == d.data.len() { - break; - } - match SerializedAction::decode(d)? { - SerializedAction::NewNodes(new_nodes) => { - for (i, data) in new_nodes.iter().enumerate() { - // Mark the result of eval_always nodes as invalid so they will - // get executed again. - if unlikely!(data.node.kind.is_eval_always()) { - let idx = DepNodeIndex::new(nodes.len() + i); - state[idx] = AtomicCell::new(DepNodeState::Invalid); - } + let mut invalid = Vec::new(); + loop { + if d.position() == d.data.len() { + break; + } + match SerializedAction::decode(d)? { + SerializedAction::AllocateNodes => { + let len = d.read_u32()?; + let start = DepNodeIndex::decode(d)?.as_u32(); + for i in 0..len { + let i = DepNodeIndex::from_u32(start + i); + let node = DepNode::decode(d)?; + nodes[i] = node; + edges[i] = Some(Box::<[DepNodeIndex]>::decode(d)?); + + if likely!(node.kind.is_eval_always()) { + state[i] = AtomicCell::new(DepNodeState::Unknown); + } else { + state[i] = AtomicCell::new(DepNodeState::UnknownEvalAlways); } - nodes.extend(new_nodes); } - SerializedAction::UpdateEdges(changed) => { - for (i, edges) in changed { - // Updated results are valid again, except for eval_always nodes - // which always start out invalid. - if likely!(!nodes[i].node.kind.is_eval_always()) { - state[i] = AtomicCell::new(DepNodeState::Unknown); - } - nodes[i].edges = edges; - } + } + SerializedAction::UpdateEdges => { + let len = d.read_u32()?; + for _ in 0..len { + let i = DepNodeIndex::decode(d)?; + edges[i] = Some(Box::<[DepNodeIndex]>::decode(d)?); } - SerializedAction::InvalidateNodes(nodes) => { - for i in nodes { - state[i] = AtomicCell::new(DepNodeState::Invalid); - } + } + SerializedAction::InvalidateNodes => { + let len = d.read_u32()?; + for _ in 0..len { + let i = DepNodeIndex::decode(d)?; + state[i] = AtomicCell::new(DepNodeState::Invalid); + invalid.push(i); } } } - Ok((SerializedDepGraph { - nodes, - fingerprints, - }, state)) } + let index: FxHashMap<_, _> = time_ext(time_passes, None, "create dep node index", || { + nodes + .iter_enumerated() + .filter(|(idx, _)| *state[*idx].get_mut() != DepNodeState::Invalid) + .map(|(idx, dep_node)| (*dep_node, idx)) + .collect() + }); + Ok((PreviousDepGraph { + index, + nodes, + fingerprints, + edges, + }, state, invalid)) } #[derive(Debug, RustcDecodable, RustcEncodable)] enum SerializedAction { - NewNodes(Vec), - UpdateEdges(Vec<(DepNodeIndex, Vec)>), - InvalidateNodes(Vec) + AllocateNodes, + UpdateEdges, + InvalidateNodes, } #[derive(Debug)] enum Action { - NewNodes(Vec), + NewNodes(Vec<(DepNodeIndex, DepNodeData)>), UpdateNodes(Vec<(DepNodeIndex, DepNodeData)>), // FIXME: Is this redundant since these nodes will be also be updated? // Could one of the indirect dependencies of a dep node change its result and // cause a red node to be incorrectly green again? // What about nodes which are in an unknown state? // We must invalidate unknown nodes. Red nodes will have an entry in UpdateNodes - InvalidateNodes(Vec) + InvalidateNodes(Vec), } struct SerializerWorker { @@ -98,6 +114,120 @@ struct SerializerWorker { file: File, } +impl SerializerWorker { + fn encode_deps( + &mut self, + encoder: &mut opaque::Encoder, + data: &DepNodeData, + ) { + // Encode dependencies + encoder.emit_u32(data.edges.len() as u32).ok(); + for edge in &data.edges { + edge.encode(encoder).ok(); + } + } + + fn encode_new_nodes( + &mut self, + encoder: &mut opaque::Encoder, + nodes: Vec<(DepNodeIndex, DepNodeData)> + ) { + // Calculates the number of nodes with indices consecutively increasing by one + // starting at `i`. + let run_length = |i: usize| { + let start = nodes[i].0.as_u32() as usize; + let mut l = 1; + loop { + if i + l >= nodes.len() { + return l; + } + if nodes[i + l].0.as_u32() as usize != start + l { + return l; + } + l += 1; + } + }; + + let mut i = 0; + + loop { + if i >= nodes.len() { + break; + } + + SerializedAction::AllocateNodes.encode(encoder).ok(); + + let len = run_length(i); + + // Emit the number of nodes we're emitting + encoder.emit_u32(len as u32).ok(); + + // Emit the dep node index of the first node + nodes[i].0.encode(encoder).ok(); + + for data in &nodes[i..(i+len)] { + // Update the result fingerprint + self.set_fingerprint(data.0, data.1.fingerprint); + + // Encode the node + data.1.node.encode(encoder).ok(); + + // Encode dependencies + self.encode_deps(encoder, &data.1); + } + + i += len; + } + } + + fn encode_updated_nodes( + &mut self, + encoder: &mut opaque::Encoder, + mut nodes: Vec<(DepNodeIndex, DepNodeData)> + ) { + // Figure out how many nodes actually changed + let mut count = 0u32; + for &mut (ref mut i, ref data) in &mut nodes { + self.fingerprints[*i] = data.fingerprint; + + if &*data.edges != self.previous.edge_targets_from(*i) { + count += 1; + } else { + // Mark this node as unchanged + *i = DepNodeIndex::INVALID; + } + } + if count == 0 { + return; + } + + SerializedAction::UpdateEdges.encode(encoder).ok(); + + encoder.emit_u32(count).ok(); + + for (i, data) in nodes { + if i == DepNodeIndex::INVALID { + continue; + } + + // Encode index + i.encode(encoder).ok(); + + // Encode dependencies + self.encode_deps(encoder, &data); + } + } + + fn set_fingerprint(&mut self, i: DepNodeIndex, fingerprint: Fingerprint) { + let ii = i.as_u32() as usize; + let len = self.fingerprints.len(); + if ii >= len { + self.fingerprints.extend(repeat(Fingerprint::ZERO).take(ii - len + 1)); + } + self.fingerprints[i] = fingerprint; + } +} + impl Worker for SerializerWorker { type Message = (usize, Action); type Result = IndexVec; @@ -106,22 +236,18 @@ impl Worker for SerializerWorker { let mut encoder = opaque::Encoder::new(Vec::with_capacity(buffer_size_est * 5)); let action = match action { Action::UpdateNodes(nodes) => { - SerializedAction::UpdateEdges(nodes.into_iter().filter(|&(i, ref data)| { - self.fingerprints[i] = data.fingerprint; - // Only emit nodes which actually changed - &*data.edges != self.previous.edge_targets_from(i) - }).map(|(i, data)| (i, data.edges.into_iter().collect::>())).collect()) + self.encode_updated_nodes(&mut encoder, nodes) } Action::NewNodes(nodes) => { - SerializedAction::NewNodes(nodes.into_iter().map(|data| { - self.fingerprints.push(data.fingerprint); - SerializedDepNodeData { - node: data.node, - edges: data.edges.into_iter().collect(), - } - }).collect()) + self.encode_new_nodes(&mut encoder, nodes); } - Action::InvalidateNodes(nodes) => SerializedAction::InvalidateNodes(nodes), + Action::InvalidateNodes(nodes) => { + SerializedAction::InvalidateNodes.encode(&mut encoder).ok(); + encoder.emit_u32(nodes.len() as u32).ok(); + for node in nodes { + node.encode(&mut encoder).ok(); + } + }, }; action.encode(&mut encoder).ok(); self.file.write_all(&encoder.into_inner()).expect("unable to write to temp dep graph"); @@ -137,7 +263,8 @@ const BUFFER_SIZE: usize = 800000; pub struct Serializer { worker: Lrc>, node_count: u32, - new_buffer: Vec, + invalids: Vec, + new_buffer: Vec<(DepNodeIndex, DepNodeData)>, new_buffer_size: usize, updated_buffer: Vec<(DepNodeIndex, DepNodeData)>, updated_buffer_size: usize, @@ -147,8 +274,10 @@ impl Serializer { pub fn new( file: File, previous: Lrc, + invalids: Vec, ) -> Self { Serializer { + invalids, node_count: previous.node_count() as u32, worker: Lrc::new(WorkerExecutor::new(SerializerWorker { fingerprints: previous.fingerprints.clone(), @@ -169,19 +298,31 @@ impl Serializer { self.worker.message_in_pool((buffer_size, Action::NewNodes(msgs))); } + #[inline] + fn alloc_index(&mut self) -> DepNodeIndex { + if let Some(invalid) = self.invalids.pop() { + // Reuse an invalided index + invalid + } else { + // Create a new index + let index = self.node_count; + self.node_count += 1; + DepNodeIndex::from_u32(index) + } + } + #[inline] pub(super) fn serialize_new(&mut self, data: DepNodeData) -> DepNodeIndex { + let index = self.alloc_index(); let edges = data.edges.len(); - self.new_buffer.push(data); - self.new_buffer_size += 8 + edges; + self.new_buffer.push((index, data)); + self.new_buffer_size += 9 + edges; if unlikely!(self.new_buffer_size >= BUFFER_SIZE) { cold_path(|| { self.flush_new(); }) } - let index = self.node_count; - self.node_count += 1; - DepNodeIndex::from_u32(index) + index } fn flush_updated(&mut self) { diff --git a/src/librustc_incremental/persist/load.rs b/src/librustc_incremental/persist/load.rs index 669b28a475b90..51708df34ee40 100644 --- a/src/librustc_incremental/persist/load.rs +++ b/src/librustc_incremental/persist/load.rs @@ -1,7 +1,7 @@ //! Code to save/load the dep-graph from files. use rustc_data_structures::fx::FxHashMap; -use rustc::dep_graph::{DepGraph, DepGraphArgs, PreviousDepGraph, SerializedDepGraph}; +use rustc::dep_graph::{DepGraph, DepGraphArgs, decode_dep_graph}; use rustc::session::Session; use rustc::ty::TyCtxt; use rustc::ty::query::OnDiskCache; @@ -43,6 +43,7 @@ pub fn dep_graph_from_future(sess: &Session, future: DepGraphFuture) -> DepGraph prev_work_products: Default::default(), file, state: Default::default(), + invalidated: Vec::new(), } }) }); @@ -246,19 +247,23 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { let mut decoder = Decoder::new(&bytes, pos); let mut results_decoder = Decoder::new(&results_bytes, results_pos); - let (dep_graph, state) = SerializedDepGraph::decode( - &mut decoder, - &mut results_decoder, - ).expect("Error reading cached dep-graph"); - - LoadResult::Ok { - data: DepGraphArgs { - prev_graph: PreviousDepGraph::new(dep_graph), - prev_work_products, - file, - state, + time_ext(time_passes, None, "decode prev dep-graph", || { + let (prev_graph, state, invalidated) = decode_dep_graph( + time_passes, + &mut decoder, + &mut results_decoder, + ).expect("Error reading cached dep-graph"); + + LoadResult::Ok { + data: DepGraphArgs { + prev_graph, + prev_work_products, + file, + state, + invalidated, + } } - } + }) }) })) } From ce85ac2d573644b6af0487ed31eb7fe775868870 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Mon, 15 Apr 2019 16:06:37 +0200 Subject: [PATCH 09/17] Encode DepNodeKind in the lowest bit of DepNodeKind --- src/librustc/dep_graph/dep_node.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/librustc/dep_graph/dep_node.rs b/src/librustc/dep_graph/dep_node.rs index f7647167a7578..486fcb82266fd 100644 --- a/src/librustc/dep_graph/dep_node.rs +++ b/src/librustc/dep_graph/dep_node.rs @@ -106,10 +106,23 @@ macro_rules! define_dep_nodes { $({ $($struct_arg_name:ident : $struct_arg_ty:ty),* })* ,)* ) => ( + // Used to get an unique integer per dep kind + enum DepKindCounter { + $($variant),* + } + + /// Encodes `eval_always` in the lowest bit of the discriminant + const fn dep_kind_discriminant(kind: DepKindCounter, eval_always: bool) -> isize { + (((kind as u16) << 1) | (eval_always as u16)) as isize + } + #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, RustcEncodable, RustcDecodable)] pub enum DepKind { - $($variant),* + $($variant = dep_kind_discriminant( + DepKindCounter::$variant, + contains_eval_always_attr!($($attr),*), + )),* } impl DepKind { @@ -155,11 +168,8 @@ macro_rules! define_dep_nodes { #[inline(always)] pub fn is_eval_always(&self) -> bool { - match *self { - $( - DepKind :: $variant => { contains_eval_always_attr!($($attr), *) } - )* - } + // `eval_always` is encoded in the lowest bit of DepNodeKind + *self as u16 & 1 == 1 } #[allow(unreachable_code)] From 9a6ef66db4324cf7a48e422c1cc60df76e86f0ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Sun, 28 Apr 2019 08:41:47 +0200 Subject: [PATCH 10/17] wip --- src/librustc/dep_graph/graph.rs | 6 +++--- src/librustc/dep_graph/serialized.rs | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index 2f70764fa9642..fbc23f13d2a8a 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -362,8 +362,8 @@ impl DepGraph { }; debug_assert_eq!( - data.colors.get(prev_index), - DepNodeState::Unknown, + data.colors.get(prev_index).color(), + None, "DepGraph::with_task() - Duplicate DepNodeState insertion for {:?}", key ); @@ -807,7 +807,7 @@ impl DepGraph { // There may be multiple threads trying to mark the same dep node green concurrently #[cfg(not(parallel_compiler))] - debug_assert_eq!(data.colors.get(dep_node_index), DepNodeState::Unknown, + debug_assert_eq!(data.colors.get(dep_node_index).color(), None, "DepGraph::try_mark_previous_green() - Duplicate DepNodeState \ insertion for {:?}", dep_node); diff --git a/src/librustc/dep_graph/serialized.rs b/src/librustc/dep_graph/serialized.rs index 16fdd5427c27e..eb89be64990cd 100644 --- a/src/librustc/dep_graph/serialized.rs +++ b/src/librustc/dep_graph/serialized.rs @@ -50,10 +50,10 @@ pub fn decode_dep_graph( nodes[i] = node; edges[i] = Some(Box::<[DepNodeIndex]>::decode(d)?); - if likely!(node.kind.is_eval_always()) { - state[i] = AtomicCell::new(DepNodeState::Unknown); - } else { + if unlikely!(node.kind.is_eval_always()) { state[i] = AtomicCell::new(DepNodeState::UnknownEvalAlways); + } else { + state[i] = AtomicCell::new(DepNodeState::Unknown); } } } From 4bb20a0bda37506b98617fd34844f0294c6b7e43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Mon, 29 Apr 2019 05:17:43 +0200 Subject: [PATCH 11/17] Garbage collect dep graph --- src/librustc/dep_graph/mod.rs | 2 +- src/librustc/dep_graph/serialized.rs | 199 +++++++++++++++++++++-- src/librustc_incremental/persist/load.rs | 43 +++-- 3 files changed, 212 insertions(+), 32 deletions(-) diff --git a/src/librustc/dep_graph/mod.rs b/src/librustc/dep_graph/mod.rs index cdbe289721c11..c449844265b00 100644 --- a/src/librustc/dep_graph/mod.rs +++ b/src/librustc/dep_graph/mod.rs @@ -16,4 +16,4 @@ pub use self::prev::PreviousDepGraph; pub use self::query::DepGraphQuery; pub use self::safe::AssertDepGraphSafe; pub use self::safe::DepGraphSafe; -pub use self::serialized::decode_dep_graph; +pub use self::serialized::{DecodedDepGraph, decode_dep_graph, gc_dep_graph}; diff --git a/src/librustc/dep_graph/serialized.rs b/src/librustc/dep_graph/serialized.rs index eb89be64990cd..ec3ed65c56af8 100644 --- a/src/librustc/dep_graph/serialized.rs +++ b/src/librustc/dep_graph/serialized.rs @@ -1,7 +1,7 @@ use rustc_data_structures::sync::worker::{Worker, WorkerExecutor}; use rustc_data_structures::sync::{Lrc, AtomicCell}; use rustc_data_structures::{unlikely, cold_path}; -use rustc_data_structures::indexed_vec::IndexVec; +use rustc_data_structures::indexed_vec::{Idx, IndexVec}; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxHashMap; use rustc_serialize::{Decodable, Encodable, Encoder, Decoder, opaque}; @@ -14,15 +14,158 @@ use crate::dep_graph::dep_node::{DepKind, DepNode}; use super::prev::PreviousDepGraph; use super::graph::{DepNodeData, DepNodeIndex, DepNodeState}; +// calcutale a list of bytes to copy from the previous graph + +fn decode_bounds( + d: &mut opaque::Decoder<'_>, + f: impl FnOnce(&mut opaque::Decoder<'_>) -> Result<(), String> +) -> Result<(usize, usize), String> { + let start = d.position(); + f(d)?; + Ok((start, d.position())) +} + +type NodePoisitions = Vec>; + +fn read_dep_graph_positions( + d: &mut opaque::Decoder<'_>, + result: &DecodedDepGraph, +) -> Result<(NodePoisitions, NodePoisitions), String> { + let node_count = result.prev_graph.nodes.len(); + let mut nodes: NodePoisitions = repeat(None).take(node_count).collect(); + let mut edges: NodePoisitions = repeat(None).take(node_count).collect(); + + loop { + if d.position() == d.data.len() { + break; + } + match SerializedAction::decode(d)? { + SerializedAction::AllocateNodes => { + let len = d.read_u32()?; + let start = DepNodeIndex::decode(d)?.as_u32(); + for i in 0..len { + let i = (start + i) as usize; + nodes[i] = Some(decode_bounds(d, |d| DepNode::decode(d).map(|_| ()))?); + edges[i] = Some(decode_bounds(d, |d| { + let len = d.read_u32()?; + for _ in 0..len { + DepNodeIndex::decode(d)?; + } + Ok(()) + })?); + } + } + SerializedAction::UpdateEdges => { + let len = d.read_u32()?; + for _ in 0..len { + let i = DepNodeIndex::decode(d)?.as_u32() as usize; + edges[i] = Some(decode_bounds(d, |d| { + let len = d.read_u32()?; + for _ in 0..len { + DepNodeIndex::decode(d)?; + } + Ok(()) + })?); + } + } + SerializedAction::InvalidateNodes => { + let len = d.read_u32()?; + for _ in 0..len { + let i = DepNodeIndex::decode(d)?.as_u32() as usize; + nodes[i] = None; + edges[i] = None; + } + } + } + } + + Ok((nodes, edges)) +} + +pub fn gc_dep_graph( + time_passes: bool, + d: &mut opaque::Decoder<'_>, + result: &DecodedDepGraph, + file: &mut File, +) { + let (nodes, edges) = time_ext(time_passes, None, "read dep-graph positions", || { + read_dep_graph_positions(d, result).unwrap() + }); + + let mut i = 0; + + loop { + // Skip empty nodes + while i < nodes.len() && nodes[i].is_none() { + i += 1; + } + + // Break if we are done + if i >= nodes.len() { + break; + } + + // Find out how many consecutive nodes we will emit + let mut len = 1; + while i + len < nodes.len() && nodes[i + len].is_some() { + len += 1; + } + + let mut encoder = opaque::Encoder::new(Vec::with_capacity(11)); + SerializedAction::AllocateNodes.encode(&mut encoder).ok(); + // Emit the number of nodes we're emitting + encoder.emit_u32(len as u32).ok(); + + // Emit the dep node index of the first node + DepNodeIndex::new(i).encode(&mut encoder).ok(); + + file.write_all(&encoder.into_inner()).unwrap(); + + let mut buffers = Vec::with_capacity(nodes.len() * 2); + + let push_buffer = |buffers: &mut Vec<(usize, usize)>, range: (usize, usize)| { + if let Some(last) = buffers.last_mut() { + if last.1 == range.0 { + // Extend the last range + last.1 = range.1; + return; + } + } + buffers.push(range); + }; + + for i in i..(i + len) { + // Encode the node + push_buffer(&mut buffers, nodes[i].unwrap()); + + // Encode dependencies + push_buffer(&mut buffers, edges[i].unwrap()); + } + + for buffer in buffers { + file.write_all(&d.data[buffer.0..buffer.1]).unwrap(); + } + + i += len; + } +} + +pub struct DecodedDepGraph { + pub prev_graph: PreviousDepGraph, + pub state: IndexVec>, + pub invalidated: Vec, + pub needs_gc: bool, +} + pub fn decode_dep_graph( time_passes: bool, d: &mut opaque::Decoder<'_>, results_d: &mut opaque::Decoder<'_>, -) -> Result<( - PreviousDepGraph, - IndexVec>, - Vec, - ), String> { +) -> Result { + // Metrics used to decided when to GC + let mut valid_data = 0; + let mut total_data = 0; + let fingerprints: IndexVec = time_ext(time_passes, None, "decode prev result fingerprints", || { IndexVec::decode(results_d) @@ -35,7 +178,6 @@ pub fn decode_dep_graph( let mut state: IndexVec<_, _> = (0..fingerprints.len()).map(|_| { AtomicCell::new(DepNodeState::Invalid) }).collect(); - let mut invalid = Vec::new(); loop { if d.position() == d.data.len() { break; @@ -48,7 +190,10 @@ pub fn decode_dep_graph( let i = DepNodeIndex::from_u32(start + i); let node = DepNode::decode(d)?; nodes[i] = node; - edges[i] = Some(Box::<[DepNodeIndex]>::decode(d)?); + let node_edges = Box::<[DepNodeIndex]>::decode(d)?; + valid_data += node_edges.len(); + total_data += node_edges.len(); + edges[i] = Some(node_edges); if unlikely!(node.kind.is_eval_always()) { state[i] = AtomicCell::new(DepNodeState::UnknownEvalAlways); @@ -56,21 +201,28 @@ pub fn decode_dep_graph( state[i] = AtomicCell::new(DepNodeState::Unknown); } } + valid_data += len as usize * 8; + total_data += len as usize * 8; } SerializedAction::UpdateEdges => { let len = d.read_u32()?; for _ in 0..len { let i = DepNodeIndex::decode(d)?; - edges[i] = Some(Box::<[DepNodeIndex]>::decode(d)?); + valid_data -= edges[i].as_ref().map_or(0, |edges| edges.len()); + let node_edges = Box::<[DepNodeIndex]>::decode(d)?; + valid_data += node_edges.len(); + total_data += node_edges.len(); + edges[i] = Some(node_edges); } } SerializedAction::InvalidateNodes => { let len = d.read_u32()?; for _ in 0..len { let i = DepNodeIndex::decode(d)?; + valid_data -= edges[i].as_ref().map_or(0, |edges| edges.len()); state[i] = AtomicCell::new(DepNodeState::Invalid); - invalid.push(i); } + valid_data -= len as usize * 8; } } } @@ -81,12 +233,27 @@ pub fn decode_dep_graph( .map(|(idx, dep_node)| (*dep_node, idx)) .collect() }); - Ok((PreviousDepGraph { - index, - nodes, - fingerprints, - edges, - }, state, invalid)) + + debug!( + "valid bytes {} total bytes {} ratio {}", + valid_data, + total_data, + valid_data as f32 / total_data as f32 + ); + + Ok(DecodedDepGraph { + prev_graph: PreviousDepGraph { + index, + nodes, + fingerprints, + edges, + }, + invalidated: state.indices() + .filter(|&i| *state[i].get_mut() == DepNodeState::Invalid) + .collect(), + state, + needs_gc: valid_data + valid_data / 3 < total_data, + }) } #[derive(Debug, RustcDecodable, RustcEncodable)] diff --git a/src/librustc_incremental/persist/load.rs b/src/librustc_incremental/persist/load.rs index 51708df34ee40..5ff62b9b8f124 100644 --- a/src/librustc_incremental/persist/load.rs +++ b/src/librustc_incremental/persist/load.rs @@ -1,7 +1,7 @@ //! Code to save/load the dep-graph from files. use rustc_data_structures::fx::FxHashMap; -use rustc::dep_graph::{DepGraph, DepGraphArgs, decode_dep_graph}; +use rustc::dep_graph::{DepGraph, DepGraphArgs, decode_dep_graph, gc_dep_graph}; use rustc::session::Session; use rustc::ty::TyCtxt; use rustc::ty::query::OnDiskCache; @@ -11,6 +11,7 @@ use rustc_serialize::opaque::Decoder; use rustc_serialize::Encodable; use std::path::Path; use std::fs::{self, File}; +use std::io::{Seek, SeekFrom}; use super::data::*; use super::fs::*; @@ -225,7 +226,7 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { MaybeAsync::Async(std::thread::spawn(move || { time_ext(time_passes, None, "background load prev dep-graph", move || { - let (bytes, pos, file) = match load_graph_file( + let (bytes, pos, mut file) = match load_graph_file( report_incremental_info, &path, expected_hash @@ -247,23 +248,35 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { let mut decoder = Decoder::new(&bytes, pos); let mut results_decoder = Decoder::new(&results_bytes, results_pos); - time_ext(time_passes, None, "decode prev dep-graph", || { - let (prev_graph, state, invalidated) = decode_dep_graph( + + let result = time_ext(time_passes, None, "decode prev dep-graph", || { + decode_dep_graph( time_passes, &mut decoder, &mut results_decoder, - ).expect("Error reading cached dep-graph"); - - LoadResult::Ok { - data: DepGraphArgs { - prev_graph, - prev_work_products, - file, - state, - invalidated, - } + ).expect("Error reading cached dep-graph") + }); + + if result.needs_gc { + // Reset the file to just the header + file.seek(SeekFrom::Start(pos as u64)).unwrap(); + file.set_len(pos as u64).unwrap(); + + time_ext(time_passes, None, "garbage collect prev dep-graph", || { + let mut decoder = Decoder::new(&bytes, pos); + gc_dep_graph(time_passes, &mut decoder, &result, &mut file); + }); + } + + LoadResult::Ok { + data: DepGraphArgs { + prev_graph: result.prev_graph, + prev_work_products, + file, + state: result.state, + invalidated: result.invalidated, } - }) + } }) })) } From 18a6d64a07b495fe4fd40531026bbb60b558de22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Tue, 30 Apr 2019 08:17:40 +0200 Subject: [PATCH 12/17] wip --- src/librustc/dep_graph/graph.rs | 245 ++++++++++-------- src/librustc/dep_graph/mod.rs | 4 +- src/librustc/dep_graph/query.rs | 15 +- src/librustc/dep_graph/serialized.rs | 129 +++++++-- src/librustc/session/mod.rs | 5 + src/librustc/ty/query/plumbing.rs | 4 +- src/librustc_codegen_ssa/base.rs | 8 +- src/librustc_incremental/assert_dep_graph.rs | 24 +- src/librustc_incremental/lib.rs | 1 - .../persist/dirty_clean.rs | 4 +- src/librustc_incremental/persist/load.rs | 20 +- src/librustc_incremental/persist/save.rs | 48 ++-- 12 files changed, 323 insertions(+), 184 deletions(-) diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index fbc23f13d2a8a..0253b5d02cc60 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -20,9 +20,49 @@ use super::debug::EdgeFilter; use super::dep_node::{DepNode, DepKind, WorkProductId}; use super::query::DepGraphQuery; use super::safe::DepGraphSafe; -use super::serialized::Serializer; +use super::serialized::{Serializer, DepGraphModel, CompletedDepGraph}; use super::prev::PreviousDepGraph; +/// Represents a final dep graph with all the changed from this session applied. +/// This is equivalent to the graph stored on disk. +pub struct ReconstructedDepGraph { + /// Hashes of the results of dep nodes + results: IndexVec, + data: FxHashMap, + /// Maps from dep nodes to their index + index: FxHashMap, +} + +impl ReconstructedDepGraph { + pub fn new(completed: &CompletedDepGraph) -> Self { + let data = completed.model.clone().unwrap().data; + let index = data.iter().map(|(idx, dep_node)| (dep_node.node, *idx)).collect(); + ReconstructedDepGraph { + results: completed.results.clone(), + data, + index, + } + } + + #[inline] + pub fn dep_node_index_of(&self, dep_node: &DepNode) -> DepNodeIndex { + self.index.get(dep_node).cloned().unwrap() + } + + #[inline] + pub fn fingerprint_of(&self, dep_node_index: DepNodeIndex) -> Fingerprint { + self.results[dep_node_index] + } + + pub fn query(&self) -> DepGraphQuery { + let nodes = self.data.values().map(|n| n.node); + let edges = self.data.iter().flat_map(|(_, n)| { + n.edges.iter().map(move |edge| (n.node, self.data.get(&edge).unwrap().node)) + }); + DepGraphQuery::new(nodes, edges) + } +} + #[derive(Clone)] pub struct DepGraph { data: Option>, @@ -63,12 +103,6 @@ pub enum DepNodeState { /// The node is from the previous session, but its state is unknown Unknown, - /// The node will eventually be green (it was previously Unknown), - /// but its side effects (like error messages) have not yet happened. - /// This only exist so we can know which thread - /// marked a node as green in `try_mark_previous_green`. - WasUnknownWillBeGreen, - Red, Green, @@ -79,8 +113,7 @@ impl DepNodeState { match self { DepNodeState::Invalid => bug!(), DepNodeState::UnknownEvalAlways | - DepNodeState::Unknown | - DepNodeState::WasUnknownWillBeGreen => None, + DepNodeState::Unknown => None, DepNodeState::Red => Some(DepNodeColor::Red), DepNodeState::Green => Some(DepNodeColor::Green), } @@ -100,11 +133,11 @@ struct DepGraphData { colors: DepNodeColorMap, - /// A set of loaded diagnostics that have been emitted. - emitted_diagnostics: Mutex>, + /// A set of loaded diagnostics that is in the progress of being emitted. + emitting_diagnostics: Mutex>, /// Used to wait for diagnostics to be emitted. - emitted_diagnostics_cond_var: Condvar, + emitting_diagnostics_cond_var: Condvar, /// When we load, there may be `.o` files, cached MIR, or other such /// things available to us. If we find that they are not dirty, we @@ -134,6 +167,7 @@ pub struct DepGraphArgs { pub file: File, pub state: IndexVec>, pub invalidated: Vec, + pub model: Option, } impl DepGraph { @@ -147,9 +181,14 @@ impl DepGraph { data: Some(Lrc::new(DepGraphData { previous_work_products: args.prev_work_products, dep_node_debug: Default::default(), - current: CurrentDepGraph::new(prev_graph.clone(), args.file, args.invalidated), - emitted_diagnostics: Default::default(), - emitted_diagnostics_cond_var: Condvar::new(), + current: CurrentDepGraph::new( + prev_graph.clone(), + args.file, + args.invalidated, + args.model + ), + emitting_diagnostics: Default::default(), + emitting_diagnostics_cond_var: Condvar::new(), colors, previous: prev_graph, loaded_from_cache: Default::default(), @@ -169,23 +208,6 @@ impl DepGraph { self.data.is_some() } - pub fn query(&self) -> DepGraphQuery { - // FIXME - panic!()/* - let current_dep_graph = self.data.as_ref().unwrap().current.borrow(); - let nodes: Vec<_> = current_dep_graph.data.iter().map(|n| n.node).collect(); - let mut edges = Vec::new(); - for (from, edge_targets) in current_dep_graph.data.iter() - .map(|d| (d.node, &d.edges)) { - for &edge_target in edge_targets.iter() { - let to = current_dep_graph.data[edge_target].node; - edges.push((from, to)); - } - } - - DepGraphQuery::new(&nodes[..], &edges[..])*/ - } - pub fn assert_ignored(&self) { if let Some(..) = self.data { @@ -469,36 +491,24 @@ impl DepGraph { data.read_index(dep_node_index); } } -/* - #[inline] - pub fn dep_node_index_of(&self, dep_node: &DepNode) -> DepNodeIndex { - self.data - .as_ref() - .unwrap() - .current - .borrow_mut() - .node_to_node_index - .get(dep_node) - .cloned() - .unwrap() - } #[inline] - pub fn dep_node_exists(&self, dep_node: &DepNode) -> bool { - if let Some(ref data) = self.data { - data.current.borrow_mut().node_to_node_index.contains_key(dep_node) - } else { - false + pub fn dep_node_exists(&self, _dep_node: &DepNode) -> bool { + #[cfg(debug_assertions)] + { + if let Some(ref data) = self.data { + let index = data.current.node_to_node_index.lock().get(_dep_node).cloned(); + index.map_or(false, |index| { + data.colors.get(index).color().is_some() + }) + } else { + false + } } - } -*/ - #[inline] - pub fn fingerprint_of(&self, _dep_node_index: DepNodeIndex) -> Fingerprint { - panic!()/* - let current = self.data.as_ref().expect("dep graph enabled").current.borrow_mut(); - current.data[dep_node_index].fingerprint*/ - } + #[cfg(not(debug_assertions))] + panic!() + } pub fn prev_fingerprint_of(&self, dep_node: &DepNode) -> Option { self.data.as_ref().unwrap().previous.fingerprint_of(dep_node) } @@ -554,7 +564,7 @@ impl DepGraph { } } - pub fn serialize(&self) -> IndexVec { + pub fn complete(&self) -> CompletedDepGraph { let data = self.data.as_ref().unwrap(); // Invalidate dep nodes with unknown state as these cannot safely // be marked green in the next session. One of the dependencies of the @@ -570,8 +580,6 @@ impl DepGraph { DepNodeState::Unknown | DepNodeState::UnknownEvalAlways => Some(prev_index), - DepNodeState::WasUnknownWillBeGreen => bug!(), - // For green nodes, we either executed the query (which always uses valid nodes) // or we marked it as green because all its dependencies are green and valid. DepNodeState::Green | @@ -635,8 +643,7 @@ impl DepGraph { // the query after anyway. DepNodeState::UnknownEvalAlways | DepNodeState::Red => None, - DepNodeState::Unknown | - DepNodeState::WasUnknownWillBeGreen => { + DepNodeState::Unknown => { // This DepNode and the corresponding query invocation existed // in the previous compilation session too, so we can try to // mark it as green by recursively marking all of its @@ -704,8 +711,7 @@ impl DepGraph { } DepNodeState::Invalid | DepNodeState::UnknownEvalAlways | - DepNodeState::Unknown | - DepNodeState::WasUnknownWillBeGreen => { + DepNodeState::Unknown => { bug!("try_force_previous_green() - Forcing the DepNode \ should have set its color") } @@ -774,7 +780,6 @@ impl DepGraph { return false; } } - DepNodeState::WasUnknownWillBeGreen | DepNodeState::Unknown => { let dep_dep_node = &data.previous.index_to_node(dep_dep_node_index); @@ -811,10 +816,10 @@ impl DepGraph { "DepGraph::try_mark_previous_green() - Duplicate DepNodeState \ insertion for {:?}", dep_node); - let did_mark = data.colors.mark_as_will_be_green(dep_node_index); - // ... emitting any stored diagnostic ... + // FIXME: Store the fact that a node has diagnostics in a bit in the dep graph somewhere + // Maybe store a list on disk and encode this fact in the DepNodeState let diagnostics = tcx.queries.on_disk_cache .load_diagnostics(tcx, dep_node_index); @@ -823,7 +828,6 @@ impl DepGraph { tcx, data, dep_node_index, - did_mark, diagnostics ); } else { @@ -841,8 +845,8 @@ impl DepGraph { true } - /// Atomically emits some loaded diagnotics, assuming that this only gets called with - /// `did_mark` set to `true` on a single thread. + /// Atomically emits some loaded diagnotics. + /// This may be called concurrently on multiple threads for the same dep node. #[cold] #[inline(never)] fn emit_diagnostics<'tcx>( @@ -850,36 +854,47 @@ impl DepGraph { tcx: TyCtxt<'tcx>, data: &DepGraphData, dep_node_index: DepNodeIndex, - did_mark: bool, diagnostics: Vec, ) { - if did_mark || !cfg!(parallel_compiler) { - // Only the thread which did the allocation emits the error messages - let handle = tcx.sess.diagnostic(); + let mut emitting = data.emitting_diagnostics.lock(); + + if data.colors.get(dep_node_index) == DepNodeState::Green { + // The node is already green so diagnostics must have been emitted already + return; + } + + if emitting.insert(dep_node_index) { + // We were the first to insert the node in the set so this thread + // must emit the diagnostics and signal other potentially waiting + // threads after. + mem::drop(emitting); // Promote the previous diagnostics to the current session. tcx.queries.on_disk_cache .store_diagnostics(dep_node_index, diagnostics.clone().into()); + let handle = tcx.sess.diagnostic(); + for diagnostic in diagnostics { DiagnosticBuilder::new_diagnostic(handle, diagnostic).emit(); } - #[cfg(parallel_compiler)] - { - // Mark the diagnostics aa emitted and wake up waiters - data.emitted_diagnostics.lock().insert(dep_node_index); - data.emitted_diagnostics_cond_var.notify_all(); - } + // Mark the node as green now that diagnostics are emitted + data.colors.insert(dep_node_index, DepNodeState::Green); + + // Remove the node from the set + data.emitting_diagnostics.lock().remove(&dep_node_index); + + // Wake up waiters + data.emitting_diagnostics_cond_var.notify_all(); } else { - // The other threads will wait for the diagnostics to be emitted + // We must wait for the other thread to finish emitting the diagnostic - let mut emitted_diagnostics = data.emitted_diagnostics.lock(); loop { - if emitted_diagnostics.contains(&dep_node_index) { + data.emitting_diagnostics_cond_var.wait(&mut emitting); + if data.colors.get(dep_node_index) == DepNodeState::Green { break; } - data.emitted_diagnostics_cond_var.wait(&mut emitted_diagnostics); } } } @@ -911,7 +926,6 @@ impl DepGraph { None } } - DepNodeState::WasUnknownWillBeGreen => bug!("no tasks should be in progress"), // There cannot be results stored for invalid indices. DepNodeState::Invalid | @@ -998,15 +1012,23 @@ pub enum WorkProductFileKind { BytecodeCompressed, } -#[derive(Clone, Debug)] -pub(super) struct DepNodeData { - pub(super) node: DepNode, - pub(super) edges: TaskReads, +#[derive(Clone, Debug, RustcEncodable, RustcDecodable)] +pub struct DepNodeData { + pub node: DepNode, + pub edges: TaskReads, pub(super) fingerprint: Fingerprint, } pub(super) struct CurrentDepGraph { - //data: FxHashMap, + /// The dep nodes associated with each index. + // Used to detect forbidden edges for debugging purposes + #[cfg(debug_assertions)] + nodes: Lock>, + + /// Used to map nodes to a node index. + #[cfg(debug_assertions)] + node_to_node_index: Lock>, + /// Used to map input nodes to a node index. Used by the `read` method. input_node_to_node_index: Lock>, @@ -1041,6 +1063,7 @@ impl CurrentDepGraph { prev_graph: Lrc, file: File, invalidated: Vec, + model: Option, ) -> CurrentDepGraph { use std::time::{SystemTime, UNIX_EPOCH}; @@ -1071,14 +1094,17 @@ impl CurrentDepGraph { //let new_node_count_estimate = (prev_graph.node_count() * 102) / 100 + 200; CurrentDepGraph { - //data: FxHashMap::default(), + #[cfg(debug_assertions)] + nodes: Lock::new(prev_graph.index.iter().map(|(&n, &i)| (i, n)).collect()), + #[cfg(debug_assertions)] + node_to_node_index: Lock::new(prev_graph.index.clone()), anon_node_to_node_index: Default::default(), input_node_to_node_index: Default::default(), anon_id_seed: stable_hasher.finish(), forbidden_edge, total_read_count: AtomicU64::new(0), total_duplicate_read_count: AtomicU64::new(0), - serializer: Lock::new(Serializer::new(file, prev_graph, invalidated)), + serializer: Lock::new(Serializer::new(file, prev_graph, invalidated, model)), } } @@ -1134,11 +1160,19 @@ impl CurrentDepGraph { previous: &PreviousDepGraph, ) -> DepNodeIndex { debug_assert!(previous.node_to_index_opt(&dep_node).is_none()); - self.serializer.lock().serialize_new(DepNodeData { + let index = self.serializer.lock().serialize_new(DepNodeData { node: dep_node, edges, fingerprint - }) + }); + + #[cfg(debug_assertions)] + { + assert!(self.node_to_node_index.lock().insert(dep_node, index).is_none()); + assert!(self.nodes.lock().insert(index, dep_node).is_none()); + } + + index } fn update_node( @@ -1168,12 +1202,12 @@ impl DepGraphData { if task_deps.read_set.insert(source) { task_deps.reads.push(source); - /*#[cfg(debug_assertions)] + #[cfg(debug_assertions)] { if let Some(target) = task_deps.node { - let graph = self.current.lock(); - if let Some(ref forbidden_edge) = graph.forbidden_edge { - let source = graph.data[&source].node; + let nodes = self.current.nodes.lock(); + if let Some(ref forbidden_edge) = self.current.forbidden_edge { + let source = nodes[&source]; if forbidden_edge.test(&source, &target) { bug!("forbidden edge {:?} -> {:?} created", source, @@ -1181,7 +1215,7 @@ impl DepGraphData { } } } - }*/ + } } else if cfg!(debug_assertions) { self.current.total_duplicate_read_count.fetch_add(1, SeqCst); } @@ -1205,17 +1239,6 @@ struct DepNodeColorMap { } impl DepNodeColorMap { - /// Tries to mark the node as WasUnknownWillBeGreen. - /// Returns false if another thread did it before us. - #[inline] - fn mark_as_will_be_green(&self, index: DepNodeIndex) -> bool { - let actual_prev = self.values[index].compare_and_swap( - DepNodeState::Unknown, - DepNodeState::WasUnknownWillBeGreen, - ); - actual_prev == DepNodeState::Unknown - } - #[inline] fn get(&self, index: DepNodeIndex) -> DepNodeState { self.values[index].load() diff --git a/src/librustc/dep_graph/mod.rs b/src/librustc/dep_graph/mod.rs index c449844265b00..4a89b936f2292 100644 --- a/src/librustc/dep_graph/mod.rs +++ b/src/librustc/dep_graph/mod.rs @@ -11,9 +11,9 @@ pub mod cgu_reuse_tracker; pub use self::dep_tracking_map::{DepTrackingMap, DepTrackingMapConfig}; pub use self::dep_node::{DepNode, DepKind, DepConstructor, WorkProductId, RecoverKey, label_strs}; pub use self::graph::{DepGraph, WorkProduct, DepNodeIndex, DepNodeColor, TaskDeps, hash_result}; -pub use self::graph::{DepGraphArgs, WorkProductFileKind}; +pub use self::graph::{DepGraphArgs, WorkProductFileKind, ReconstructedDepGraph}; pub use self::prev::PreviousDepGraph; pub use self::query::DepGraphQuery; pub use self::safe::AssertDepGraphSafe; pub use self::safe::DepGraphSafe; -pub use self::serialized::{DecodedDepGraph, decode_dep_graph, gc_dep_graph}; +pub use self::serialized::{CompletedDepGraph, DecodedDepGraph, decode_dep_graph, gc_dep_graph}; diff --git a/src/librustc/dep_graph/query.rs b/src/librustc/dep_graph/query.rs index cd4ced238d360..9e8af35d6c8cb 100644 --- a/src/librustc/dep_graph/query.rs +++ b/src/librustc/dep_graph/query.rs @@ -11,18 +11,19 @@ pub struct DepGraphQuery { } impl DepGraphQuery { - pub fn new(nodes: &[DepNode], - edges: &[(DepNode, DepNode)]) - -> DepGraphQuery { - let mut graph = Graph::with_capacity(nodes.len(), edges.len()); + pub fn new( + nodes: impl Iterator, + edges: impl Iterator, + ) -> DepGraphQuery { + let mut graph = Graph::with_capacity(nodes.size_hint().0, edges.size_hint().0); let mut indices = FxHashMap::default(); for node in nodes { indices.insert(node.clone(), graph.add_node(node.clone())); } - for &(ref source, ref target) in edges { - let source = indices[source]; - let target = indices[target]; + for (source, target) in edges { + let source = indices[&source]; + let target = indices[&target]; graph.add_edge(source, target, ()); } diff --git a/src/librustc/dep_graph/serialized.rs b/src/librustc/dep_graph/serialized.rs index ec3ed65c56af8..c628cd9fc6c67 100644 --- a/src/librustc/dep_graph/serialized.rs +++ b/src/librustc/dep_graph/serialized.rs @@ -88,7 +88,7 @@ pub fn gc_dep_graph( result: &DecodedDepGraph, file: &mut File, ) { - let (nodes, edges) = time_ext(time_passes, None, "read dep-graph positions", || { + let (nodes, edges) = time_ext(time_passes, None, "read dep-graph positions", || { read_dep_graph_positions(d, result).unwrap() }); @@ -150,11 +150,78 @@ pub fn gc_dep_graph( } } +/// A simpler dep graph used for debugging and testing purposes. +#[derive(Clone, Debug, RustcEncodable, RustcDecodable, Default)] +pub struct DepGraphModel { + pub data: FxHashMap, +} + +impl DepGraphModel { + fn apply(&mut self, action: &Action) { + match action { + Action::UpdateNodes(nodes) => { + for n in nodes { + self.data + .entry(n.0) + .or_insert_with(|| panic!()).edges = n.1.edges.clone(); + } + } + Action::NewNodes(nodes) => { + for n in nodes { + assert!(self.data.insert(n.0, n.1.clone()).is_none()); + } + } + Action::InvalidateNodes(nodes) => { + for n in nodes { + assert!(self.data.remove(&n).is_some()); + } + }, + } + } +} + +#[derive(Clone, Debug, RustcEncodable, RustcDecodable, Default)] +pub struct CompletedDepGraph { + /// Hashes of the results of dep nodes + pub(super) results: IndexVec, + /// A simpler dep graph stored alongside the result for debugging purposes. + /// This is also constructed when we want to query the dep graph. + pub model: Option, +} + pub struct DecodedDepGraph { pub prev_graph: PreviousDepGraph, pub state: IndexVec>, pub invalidated: Vec, pub needs_gc: bool, + pub model: Option, +} + +impl DecodedDepGraph { + /// Asserts that the model matches the real dep graph we decoded + fn validate_model(&mut self) { + let model = if let Some(ref model) = self.model { + model + } else { + return + }; + + for i in self.state.indices() { + if *self.state[i].get_mut() == DepNodeState::Invalid { + assert!(!model.data.contains_key(&i)); + assert_eq!(self.prev_graph.edges[i], None); + } else { + let data = model.data.get(&i).unwrap(); + assert_eq!(self.prev_graph.nodes[i], data.node); + assert_eq!(&self.prev_graph.edges[i].as_ref().unwrap()[..], &data.edges[..]); + } + } + + for k in model.data.keys() { + assert!((k.as_u32() as usize) < self.state.len()); + } + + } } pub fn decode_dep_graph( @@ -162,20 +229,21 @@ pub fn decode_dep_graph( d: &mut opaque::Decoder<'_>, results_d: &mut opaque::Decoder<'_>, ) -> Result { - // Metrics used to decided when to GC + // Metrics used to decide when to GC let mut valid_data = 0; let mut total_data = 0; - let fingerprints: IndexVec = - time_ext(time_passes, None, "decode prev result fingerprints", || { - IndexVec::decode(results_d) - })?; + let result_format = time_ext(time_passes, None, "decode prev result fingerprints", || { + CompletedDepGraph::decode(results_d) + })?; + + let node_count = result_format.results.len(); let mut nodes: IndexVec<_, _> = repeat(DepNode { kind: DepKind::Null, hash: Fingerprint::ZERO, - }).take(fingerprints.len()).collect(); - let mut edges: IndexVec<_, _> = repeat(None).take(fingerprints.len()).collect(); - let mut state: IndexVec<_, _> = (0..fingerprints.len()).map(|_| { + }).take(node_count).collect(); + let mut edges: IndexVec<_, _> = repeat(None).take(node_count).collect(); + let mut state: IndexVec<_, _> = (0..node_count).map(|_| { AtomicCell::new(DepNodeState::Invalid) }).collect(); loop { @@ -221,6 +289,7 @@ pub fn decode_dep_graph( let i = DepNodeIndex::decode(d)?; valid_data -= edges[i].as_ref().map_or(0, |edges| edges.len()); state[i] = AtomicCell::new(DepNodeState::Invalid); + edges[i] = None; } valid_data -= len as usize * 8; } @@ -241,11 +310,11 @@ pub fn decode_dep_graph( valid_data as f32 / total_data as f32 ); - Ok(DecodedDepGraph { + let mut graph = DecodedDepGraph { prev_graph: PreviousDepGraph { index, nodes, - fingerprints, + fingerprints: result_format.results, edges, }, invalidated: state.indices() @@ -253,7 +322,12 @@ pub fn decode_dep_graph( .collect(), state, needs_gc: valid_data + valid_data / 3 < total_data, - }) + model: result_format.model, + }; + + graph.validate_model(); + + Ok(graph) } #[derive(Debug, RustcDecodable, RustcEncodable)] @@ -279,6 +353,7 @@ struct SerializerWorker { fingerprints: IndexVec, previous: Lrc, file: File, + model: Option, } impl SerializerWorker { @@ -397,9 +472,12 @@ impl SerializerWorker { impl Worker for SerializerWorker { type Message = (usize, Action); - type Result = IndexVec; + type Result = CompletedDepGraph; fn message(&mut self, (buffer_size_est, action): (usize, Action)) { + // Apply the action to the model if present + self.model.as_mut().map(|model| model.apply(&action)); + let mut encoder = opaque::Encoder::new(Vec::with_capacity(buffer_size_est * 5)); let action = match action { Action::UpdateNodes(nodes) => { @@ -420,8 +498,11 @@ impl Worker for SerializerWorker { self.file.write_all(&encoder.into_inner()).expect("unable to write to temp dep graph"); } - fn complete(self) -> IndexVec { - self.fingerprints + fn complete(self) -> CompletedDepGraph { + CompletedDepGraph { + results: self.fingerprints, + model: self.model + } } } @@ -430,7 +511,7 @@ const BUFFER_SIZE: usize = 800000; pub struct Serializer { worker: Lrc>, node_count: u32, - invalids: Vec, + invalidated: Vec, new_buffer: Vec<(DepNodeIndex, DepNodeData)>, new_buffer_size: usize, updated_buffer: Vec<(DepNodeIndex, DepNodeData)>, @@ -441,15 +522,17 @@ impl Serializer { pub fn new( file: File, previous: Lrc, - invalids: Vec, + invalidated: Vec, + model: Option, ) -> Self { Serializer { - invalids, - node_count: previous.node_count() as u32, + invalidated, + node_count: previous.nodes.len() as u32, worker: Lrc::new(WorkerExecutor::new(SerializerWorker { fingerprints: previous.fingerprints.clone(), previous, file, + model, })), new_buffer: Vec::with_capacity(BUFFER_SIZE), new_buffer_size: 0, @@ -467,9 +550,9 @@ impl Serializer { #[inline] fn alloc_index(&mut self) -> DepNodeIndex { - if let Some(invalid) = self.invalids.pop() { + if let Some(invalidated) = self.invalidated.pop() { // Reuse an invalided index - invalid + invalidated } else { // Create a new index let index = self.node_count; @@ -511,10 +594,10 @@ impl Serializer { } } - pub fn complete( + pub(super) fn complete( &mut self, invalidate: Vec, - ) -> IndexVec { + ) -> CompletedDepGraph { if self.new_buffer.len() > 0 { self.flush_new(); } diff --git a/src/librustc/session/mod.rs b/src/librustc/session/mod.rs index d04b9ac083ce5..9a792f876e897 100644 --- a/src/librustc/session/mod.rs +++ b/src/librustc/session/mod.rs @@ -726,6 +726,11 @@ impl Session { ) } + pub fn reconstruct_dep_graph(&self) -> bool { + self.opts.debugging_opts.dump_dep_graph || + self.opts.debugging_opts.query_dep_graph + } + pub fn set_incr_session_load_dep_graph(&self, load: bool) { let mut incr_comp_session = self.incr_comp_session.borrow_mut(); diff --git a/src/librustc/ty/query/plumbing.rs b/src/librustc/ty/query/plumbing.rs index bb25e1017e59d..480b0f36789f3 100644 --- a/src/librustc/ty/query/plumbing.rs +++ b/src/librustc/ty/query/plumbing.rs @@ -534,14 +534,14 @@ impl<'tcx> TyCtxt<'tcx> { // in DepGraph::try_mark_green() // 2. Two distinct query keys get mapped to the same DepNode // (see for example #48923) - /*debug_assert!( + debug_assert!( !self.dep_graph.dep_node_exists(&dep_node), "Forcing query with already existing DepNode.\n\ - query-key: {:?}\n\ - dep-node: {:?}", key, dep_node, - );*/ + ); profq_msg!(self, ProfileQueriesMsg::ProviderBegin); self.sess.profiler(|p| p.start_query(Q::NAME)); diff --git a/src/librustc_codegen_ssa/base.rs b/src/librustc_codegen_ssa/base.rs index 04a44caa049e1..b5d5ec749d93a 100644 --- a/src/librustc_codegen_ssa/base.rs +++ b/src/librustc_codegen_ssa/base.rs @@ -703,10 +703,6 @@ impl Drop for AbortCodegenOnDrop { } fn assert_and_save_dep_graph<'tcx>(tcx: TyCtxt<'tcx>) { - time(tcx.sess, - "assert dep graph", - || ::rustc_incremental::assert_dep_graph(tcx)); - time(tcx.sess, "serialize dep graph", || ::rustc_incremental::save_dep_graph(tcx)); @@ -868,9 +864,9 @@ fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguR // know that later). If we are not doing LTO, there is only one optimized // version of each module, so we re-use that. let dep_node = cgu.codegen_dep_node(tcx); - /*debug_assert!(!tcx.dep_graph.dep_node_exists(&dep_node), + debug_assert!(!tcx.dep_graph.dep_node_exists(&dep_node), "CompileCodegenUnit dep-node for CGU `{}` already exists before marking.", - cgu.name());*/ + cgu.name()); if tcx.dep_graph.try_mark_green(tcx, &dep_node).is_some() { // We can re-use either the pre- or the post-thinlto state diff --git a/src/librustc_incremental/assert_dep_graph.rs b/src/librustc_incremental/assert_dep_graph.rs index a43347a2197c3..eab8861c0f09c 100644 --- a/src/librustc_incremental/assert_dep_graph.rs +++ b/src/librustc_incremental/assert_dep_graph.rs @@ -34,7 +34,7 @@ //! ``` use graphviz as dot; -use rustc::dep_graph::{DepGraphQuery, DepNode, DepKind}; +use rustc::dep_graph::{ReconstructedDepGraph, CompletedDepGraph, DepGraphQuery, DepNode, DepKind}; use rustc::dep_graph::debug::{DepNodeFilter, EdgeFilter}; use rustc::hir::def_id::DefId; use rustc::ty::TyCtxt; @@ -51,10 +51,13 @@ use std::io::Write; use syntax::ast; use syntax_pos::Span; -pub fn assert_dep_graph<'tcx>(tcx: TyCtxt<'tcx>) { +pub(crate) fn assert_dep_graph<'tcx>( + tcx: TyCtxt<'tcx>, + dep_graph: &CompletedDepGraph, +) { tcx.dep_graph.with_ignore(|| { if tcx.sess.opts.debugging_opts.dump_dep_graph { - dump_graph(tcx); + dump_graph(&ReconstructedDepGraph::new(dep_graph)); } // if the `rustc_attrs` feature is not enabled, then the @@ -82,7 +85,7 @@ pub fn assert_dep_graph<'tcx>(tcx: TyCtxt<'tcx>) { } // Check paths. - check_paths(tcx, &if_this_changed, &then_this_would_need); + check_paths(tcx, dep_graph, &if_this_changed, &then_this_would_need); }) } @@ -184,7 +187,12 @@ impl Visitor<'tcx> for IfThisChanged<'tcx> { } } -fn check_paths<'tcx>(tcx: TyCtxt<'tcx>, if_this_changed: &Sources, then_this_would_need: &Targets) { +fn check_paths( + tcx: TyCtxt<'_>, + dep_graph: &CompletedDepGraph, + if_this_changed: &Sources, + then_this_would_need: &Targets, +) { // Return early here so as not to construct the query, which is not cheap. if if_this_changed.is_empty() { for &(target_span, _, _, _) in then_this_would_need { @@ -195,7 +203,7 @@ fn check_paths<'tcx>(tcx: TyCtxt<'tcx>, if_this_changed: &Sources, then_this_wou } return; } - let query = tcx.dep_graph.query(); + let query = ReconstructedDepGraph::new(dep_graph).query(); for &(_, source_def_id, ref source_dep_node) in if_this_changed { let dependents = query.transitive_predecessors(source_dep_node); for &(target_span, ref target_pass, _, ref target_dep_node) in then_this_would_need { @@ -214,9 +222,9 @@ fn check_paths<'tcx>(tcx: TyCtxt<'tcx>, if_this_changed: &Sources, then_this_wou } } -fn dump_graph(tcx: TyCtxt<'_>) { +fn dump_graph(dep_graph: &ReconstructedDepGraph) { let path: String = env::var("RUST_DEP_GRAPH").unwrap_or_else(|_| "dep_graph".to_string()); - let query = tcx.dep_graph.query(); + let query = dep_graph.query(); let nodes = match env::var("RUST_DEP_GRAPH_FILTER") { Ok(string) => { diff --git a/src/librustc_incremental/lib.rs b/src/librustc_incremental/lib.rs index 508711d6bfcb5..fd550c49d9f7d 100644 --- a/src/librustc_incremental/lib.rs +++ b/src/librustc_incremental/lib.rs @@ -22,7 +22,6 @@ mod assert_dep_graph; pub mod assert_module_sources; mod persist; -pub use assert_dep_graph::assert_dep_graph; pub use persist::dep_graph_tcx_init; pub use persist::{DepGraphFuture, load_dep_graph, dep_graph_from_future}; pub use persist::load_query_result_cache; diff --git a/src/librustc_incremental/persist/dirty_clean.rs b/src/librustc_incremental/persist/dirty_clean.rs index 935b5c49740ac..46202b574de3e 100644 --- a/src/librustc_incremental/persist/dirty_clean.rs +++ b/src/librustc_incremental/persist/dirty_clean.rs @@ -234,7 +234,7 @@ pub fn check_dirty_clean_annotations<'tcx>(tcx: TyCtxt<'tcx>) { }) } -pub struct DirtyCleanVisitor<'tcx> { +struct DirtyCleanVisitor<'tcx> { tcx: TyCtxt<'tcx>, checked_attrs: FxHashSet, } @@ -589,7 +589,7 @@ fn expect_associated_value(tcx: TyCtxt<'_>, item: &NestedMetaItem) -> ast::Name // A visitor that collects all #[rustc_dirty]/#[rustc_clean] attributes from // the HIR. It is used to verfiy that we really ran checks for all annotated // nodes. -pub struct FindAllAttrs<'tcx> { +struct FindAllAttrs<'tcx> { tcx: TyCtxt<'tcx>, attr_names: Vec, found_attrs: Vec<&'tcx Attribute>, diff --git a/src/librustc_incremental/persist/load.rs b/src/librustc_incremental/persist/load.rs index 5ff62b9b8f124..6f8df9badac50 100644 --- a/src/librustc_incremental/persist/load.rs +++ b/src/librustc_incremental/persist/load.rs @@ -39,12 +39,20 @@ pub fn dep_graph_from_future(sess: &Session, future: DepGraphFuture) -> DepGraph sess.opts.dep_tracking_hash().encode(encoder).unwrap(); }).unwrap(); + let use_model = sess.reconstruct_dep_graph() || + cfg!(debug_assertions); + DepGraphArgs { prev_graph: Default::default(), prev_work_products: Default::default(), file, state: Default::default(), invalidated: Vec::new(), + model: if use_model { + Some(Default::default()) + } else { + None + }, } }) }); @@ -179,6 +187,7 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { let report_incremental_info = sess.opts.debugging_opts.incremental_info; let expected_hash = sess.opts.dep_tracking_hash(); + let use_model = sess.reconstruct_dep_graph() || cfg!(debug_assertions); let mut prev_work_products = FxHashMap::default(); @@ -248,8 +257,8 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { let mut decoder = Decoder::new(&bytes, pos); let mut results_decoder = Decoder::new(&results_bytes, results_pos); - - let result = time_ext(time_passes, None, "decode prev dep-graph", || { + + let mut result = time_ext(time_passes, None, "decode prev dep-graph", || { decode_dep_graph( time_passes, &mut decoder, @@ -262,12 +271,16 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { file.seek(SeekFrom::Start(pos as u64)).unwrap(); file.set_len(pos as u64).unwrap(); - time_ext(time_passes, None, "garbage collect prev dep-graph", || { + time_ext(time_passes, None, "garbage collect prev dep-graph", || { let mut decoder = Decoder::new(&bytes, pos); gc_dep_graph(time_passes, &mut decoder, &result, &mut file); }); } + if use_model && result.model.is_none() { + result.model = Some(Default::default()); + } + LoadResult::Ok { data: DepGraphArgs { prev_graph: result.prev_graph, @@ -275,6 +288,7 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { file, state: result.state, invalidated: result.invalidated, + model: result.model, } } }) diff --git a/src/librustc_incremental/persist/save.rs b/src/librustc_incremental/persist/save.rs index 05d7e4fd608c6..e1ab9fb69866b 100644 --- a/src/librustc_incremental/persist/save.rs +++ b/src/librustc_incremental/persist/save.rs @@ -1,4 +1,4 @@ -use rustc::dep_graph::{DepGraph, /*DepKind,*/ WorkProduct, WorkProductId}; +use rustc::dep_graph::{DepGraph, DepKind, WorkProduct, WorkProductId}; use rustc::session::Session; use rustc::ty::TyCtxt; use rustc::util::common::time; @@ -10,6 +10,7 @@ use std::fs::{self, File}; use std::io::{self, Write}; use std::path::Path; +use crate::assert_dep_graph::assert_dep_graph; use super::data::*; use super::fs::*; use super::dirty_clean; @@ -133,23 +134,17 @@ where fn finish_dep_graph(tcx: TyCtxt<'_>, results_path: &Path) { let sess = tcx.sess; // Encode the graph data. - let results = time(tcx.sess, "finish graph serialization", || { - tcx.dep_graph.serialize() + let mut results = time(tcx.sess, "finish graph serialization", || { + tcx.dep_graph.complete() }); - time(sess, "save dep-graph results", || { - save_in(sess, results_path, |e| { - // First encode the commandline arguments hash - sess.opts.dep_tracking_hash().encode(e).unwrap(); - time(sess, "encode dep-graph results", || { - // Save the result vector - results.encode(e).unwrap(); - }); - }).unwrap(); - }); + time(tcx.sess, + "assert dep graph", + || assert_dep_graph(tcx, &results)); -/* if tcx.sess.opts.debugging_opts.incremental_info { + let data = &results.model.as_ref().unwrap().data; + #[derive(Clone)] struct Stat { kind: DepKind, @@ -157,12 +152,12 @@ fn finish_dep_graph(tcx: TyCtxt<'_>, results_path: &Path) { edge_counter: u64, } - let total_node_count = serialized_graph.nodes.len(); - let total_edge_count: usize = serialized_graph.nodes.iter().map(|d| d.deps.len()).sum(); + let total_node_count = data.len(); + let total_edge_count: usize = data.values().map(|d| d.edges.len()).sum(); let mut counts: FxHashMap<_, Stat> = FxHashMap::default(); - for node in serialized_graph.nodes.iter() { + for node in data.values() { let stat = counts.entry(node.node.kind).or_insert(Stat { kind: node.node.kind, node_counter: 0, @@ -170,7 +165,7 @@ fn finish_dep_graph(tcx: TyCtxt<'_>, results_path: &Path) { }); stat.node_counter += 1; - stat.edge_counter += node.deps.len() as u64; + stat.edge_counter += node.edges.len() as u64; } let mut counts: Vec<_> = counts.values().cloned().collect(); @@ -222,7 +217,22 @@ fn finish_dep_graph(tcx: TyCtxt<'_>, results_path: &Path) { println!("{}", SEPARATOR); println!("[incremental]"); } -*/ + + if !cfg!(debug_assertions) { + // Only save the model in debug builds + results.model = None; + } + + time(sess, "save dep-graph results", || { + save_in(sess, results_path, |e| { + // First encode the commandline arguments hash + sess.opts.dep_tracking_hash().encode(e).unwrap(); + time(sess, "encode dep-graph results", || { + // Save the result vector + results.encode(e).unwrap(); + }); + }).unwrap(); + }); } fn encode_work_product_index(work_products: &FxHashMap, From edc0107e073b21a427b4962d4281ae3b20db6a24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Mon, 6 May 2019 09:32:50 +0200 Subject: [PATCH 13/17] Add get_mut --- src/librustc_data_structures/sync.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/librustc_data_structures/sync.rs b/src/librustc_data_structures/sync.rs index 224f2a1d566f0..1b62bd5633c2e 100644 --- a/src/librustc_data_structures/sync.rs +++ b/src/librustc_data_structures/sync.rs @@ -77,6 +77,11 @@ cfg_if! { pub fn new(v: T) -> Self { AtomicCell(Cell::new(v)) } + + #[inline] + pub fn get_mut(&mut self) -> &mut T { + self.0.get_mut() + } } impl AtomicCell { From 8877f0231e92e23da868a11879becfac0cd23d91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Mon, 6 May 2019 10:42:23 +0200 Subject: [PATCH 14/17] wip --- src/librustc/dep_graph/graph.rs | 16 ++++++++-- src/librustc/dep_graph/serialized.rs | 8 +++-- src/librustc/ty/query/plumbing.rs | 19 ++++-------- .../persist/dirty_clean.rs | 30 +++++++++++-------- src/librustc_incremental/persist/load.rs | 27 +++++++++++++++-- src/librustc_incremental/persist/save.rs | 18 +++++++++-- 6 files changed, 81 insertions(+), 37 deletions(-) diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index 0253b5d02cc60..d927af7ec0cfd 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -164,7 +164,7 @@ where pub struct DepGraphArgs { pub prev_graph: PreviousDepGraph, pub prev_work_products: FxHashMap, - pub file: File, + pub file: Option, pub state: IndexVec>, pub invalidated: Vec, pub model: Option, @@ -712,8 +712,18 @@ impl DepGraph { DepNodeState::Invalid | DepNodeState::UnknownEvalAlways | DepNodeState::Unknown => { - bug!("try_force_previous_green() - Forcing the DepNode \ + if !tcx.sess.has_errors() { + bug!("try_force_previous_green() - Forcing the DepNode \ should have set its color") + } else { + // If the query we just forced has resulted + // in some kind of compilation error, we + // don't expect that the corresponding + // dep-node color has been updated. + // A query cycle which does not panic is one + // such error. + false + } } } } else { @@ -1061,7 +1071,7 @@ pub(super) struct CurrentDepGraph { impl CurrentDepGraph { fn new( prev_graph: Lrc, - file: File, + file: Option, invalidated: Vec, model: Option, ) -> CurrentDepGraph { diff --git a/src/librustc/dep_graph/serialized.rs b/src/librustc/dep_graph/serialized.rs index c628cd9fc6c67..1d9101a1f6aff 100644 --- a/src/librustc/dep_graph/serialized.rs +++ b/src/librustc/dep_graph/serialized.rs @@ -352,7 +352,7 @@ enum Action { struct SerializerWorker { fingerprints: IndexVec, previous: Lrc, - file: File, + file: Option, model: Option, } @@ -495,7 +495,9 @@ impl Worker for SerializerWorker { }, }; action.encode(&mut encoder).ok(); - self.file.write_all(&encoder.into_inner()).expect("unable to write to temp dep graph"); + self.file.as_mut().map(|file| { + file.write_all(&encoder.into_inner()).expect("unable to write to temp dep graph"); + }); } fn complete(self) -> CompletedDepGraph { @@ -520,7 +522,7 @@ pub struct Serializer { impl Serializer { pub fn new( - file: File, + file: Option, previous: Lrc, invalidated: Vec, model: Option, diff --git a/src/librustc/ty/query/plumbing.rs b/src/librustc/ty/query/plumbing.rs index 480b0f36789f3..1f3b741c5bda2 100644 --- a/src/librustc/ty/query/plumbing.rs +++ b/src/librustc/ty/query/plumbing.rs @@ -484,7 +484,7 @@ impl<'tcx> TyCtxt<'tcx> { // If -Zincremental-verify-ich is specified, re-hash results from // the cache and make sure that they have the expected fingerprint. if unlikely!(self.sess.opts.debugging_opts.incremental_verify_ich) { - self.incremental_verify_ich::(&result, dep_node, dep_node_index); + self.incremental_verify_ich::(&result, dep_node); } if unlikely!(self.sess.opts.debugging_opts.query_dep_graph) { @@ -498,28 +498,21 @@ impl<'tcx> TyCtxt<'tcx> { #[cold] fn incremental_verify_ich>( self, - _result: &Q::Value, - _dep_node: &DepNode, - _dep_node_index: DepNodeIndex, + result: &Q::Value, + dep_node: &DepNode, ) { - panic!()/* use crate::ich::Fingerprint; - assert!(Some(self.dep_graph.fingerprint_of(dep_node_index)) == - self.dep_graph.prev_fingerprint_of(dep_node), - "Fingerprint for green query instance not loaded \ - from cache: {:?}", dep_node); - debug!("BEGIN verify_ich({:?})", dep_node); let mut hcx = self.create_stable_hashing_context(); let new_hash = Q::hash_result(&mut hcx, result).unwrap_or(Fingerprint::ZERO); debug!("END verify_ich({:?})", dep_node); - let old_hash = self.dep_graph.fingerprint_of(dep_node_index); + let old_hash = self.dep_graph.prev_fingerprint_of(dep_node); - assert!(new_hash == old_hash, "Found unstable fingerprints \ - for {:?}", dep_node);*/ + assert!(Some(new_hash) == old_hash, "Found unstable fingerprints \ + for {:?}", dep_node); } #[inline(always)] diff --git a/src/librustc_incremental/persist/dirty_clean.rs b/src/librustc_incremental/persist/dirty_clean.rs index 46202b574de3e..aa8d71248b5ea 100644 --- a/src/librustc_incremental/persist/dirty_clean.rs +++ b/src/librustc_incremental/persist/dirty_clean.rs @@ -15,7 +15,7 @@ use std::iter::FromIterator; use std::vec::Vec; -use rustc::dep_graph::{DepNode, label_strs}; +use rustc::dep_graph::{CompletedDepGraph, ReconstructedDepGraph, DepNode, label_strs}; use rustc::hir; use rustc::hir::{ItemKind as HirItem, ImplItemKind, TraitItemKind}; use rustc::hir::Node as HirNode; @@ -206,16 +206,21 @@ impl Assertion { } } -pub fn check_dirty_clean_annotations<'tcx>(tcx: TyCtxt<'tcx>) { +pub fn check_dirty_clean_annotations( + tcx: TyCtxt<'_>, + dep_graph: &CompletedDepGraph, +) { // can't add `#[rustc_dirty]` etc without opting in to this feature if !tcx.features().rustc_attrs { return; } tcx.dep_graph.with_ignore(|| { + let graph = ReconstructedDepGraph::new(dep_graph); let krate = tcx.hir().krate(); let mut dirty_clean_visitor = DirtyCleanVisitor { tcx, + graph, checked_attrs: Default::default(), }; krate.visit_all_item_likes(&mut dirty_clean_visitor); @@ -236,6 +241,7 @@ pub fn check_dirty_clean_annotations<'tcx>(tcx: TyCtxt<'tcx>) { struct DirtyCleanVisitor<'tcx> { tcx: TyCtxt<'tcx>, + graph: ReconstructedDepGraph, checked_attrs: FxHashSet, } @@ -459,7 +465,7 @@ impl DirtyCleanVisitor<'tcx> { }) } - fn _dep_node_str(&self, dep_node: &DepNode) -> String { + fn dep_node_str(&self, dep_node: &DepNode) -> String { if let Some(def_id) = dep_node.extract_def_id(self.tcx) { format!("{:?}({})", dep_node.kind, @@ -469,11 +475,10 @@ impl DirtyCleanVisitor<'tcx> { } } - fn assert_dirty(&self, _item_span: Span, dep_node: DepNode) { + fn assert_dirty(&self, item_span: Span, dep_node: DepNode) { debug!("assert_dirty({:?})", dep_node); - panic!() - /*let dep_node_index = self.tcx.dep_graph.dep_node_index_of(&dep_node); - let current_fingerprint = self.tcx.dep_graph.fingerprint_of(dep_node_index); + let dep_node_index = self.graph.dep_node_index_of(&dep_node); + let current_fingerprint = self.graph.fingerprint_of(dep_node_index); let prev_fingerprint = self.tcx.dep_graph.prev_fingerprint_of(&dep_node); if Some(current_fingerprint) == prev_fingerprint { @@ -481,14 +486,13 @@ impl DirtyCleanVisitor<'tcx> { self.tcx.sess.span_err( item_span, &format!("`{}` should be dirty but is not", dep_node_str)); - }*/ + } } - fn assert_clean(&self, _item_span: Span, dep_node: DepNode) { + fn assert_clean(&self, item_span: Span, dep_node: DepNode) { debug!("assert_clean({:?})", dep_node); - panic!() - /*let dep_node_index = self.tcx.dep_graph.dep_node_index_of(&dep_node); - let current_fingerprint = self.tcx.dep_graph.fingerprint_of(dep_node_index); + let dep_node_index = self.graph.dep_node_index_of(&dep_node); + let current_fingerprint = self.graph.fingerprint_of(dep_node_index); let prev_fingerprint = self.tcx.dep_graph.prev_fingerprint_of(&dep_node); if Some(current_fingerprint) != prev_fingerprint { @@ -496,7 +500,7 @@ impl DirtyCleanVisitor<'tcx> { self.tcx.sess.span_err( item_span, &format!("`{}` should be clean but is not", dep_node_str)); - }*/ + } } fn check_item(&mut self, item_id: hir::HirId, item_span: Span) { diff --git a/src/librustc_incremental/persist/load.rs b/src/librustc_incremental/persist/load.rs index 6f8df9badac50..fa21186929c3c 100644 --- a/src/librustc_incremental/persist/load.rs +++ b/src/librustc_incremental/persist/load.rs @@ -45,7 +45,7 @@ pub fn dep_graph_from_future(sess: &Session, future: DepGraphFuture) -> DepGraph DepGraphArgs { prev_graph: Default::default(), prev_work_products: Default::default(), - file, + file: Some(file), state: Default::default(), invalidated: Vec::new(), model: if use_model { @@ -169,6 +169,28 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { let time_passes = sess.time_passes(); + let use_model = sess.reconstruct_dep_graph() || cfg!(debug_assertions); + + if sess.opts.incremental.is_none() { + let args = DepGraphArgs { + prev_graph: Default::default(), + prev_work_products: Default::default(), + file: None, + state: Default::default(), + invalidated: Vec::new(), + model: if use_model { + Some(Default::default()) + } else { + None + }, + }; + + // No incremental compilation. + return MaybeAsync::Sync(LoadResult::Ok { + data: args, + }); + } + // Calling `sess.incr_comp_session_dir()` will panic if `sess.opts.incremental.is_none()`. // Fortunately, we just checked that this isn't the case. let dir = &sess.incr_comp_session_dir(); @@ -187,7 +209,6 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { let report_incremental_info = sess.opts.debugging_opts.incremental_info; let expected_hash = sess.opts.dep_tracking_hash(); - let use_model = sess.reconstruct_dep_graph() || cfg!(debug_assertions); let mut prev_work_products = FxHashMap::default(); @@ -285,7 +306,7 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { data: DepGraphArgs { prev_graph: result.prev_graph, prev_work_products, - file, + file: Some(file), state: result.state, invalidated: result.invalidated, model: result.model, diff --git a/src/librustc_incremental/persist/save.rs b/src/librustc_incremental/persist/save.rs index e1ab9fb69866b..76a522d592d9a 100644 --- a/src/librustc_incremental/persist/save.rs +++ b/src/librustc_incremental/persist/save.rs @@ -21,7 +21,20 @@ pub fn save_dep_graph<'tcx>(tcx: TyCtxt<'tcx>) { debug!("save_dep_graph()"); tcx.dep_graph.with_ignore(|| { let sess = tcx.sess; + if sess.opts.incremental.is_none() { + if !tcx.sess.opts.build_dep_graph() { + return; + } + + let results = time(tcx.sess, "finish dep graph", || { + tcx.dep_graph.complete() + }); + + time(tcx.sess, + "assert dep graph", + || assert_dep_graph(tcx, &results)); + return; } @@ -42,8 +55,6 @@ pub fn save_dep_graph<'tcx>(tcx: TyCtxt<'tcx>) { finish_dep_graph(tcx, &results_path) }); }); - - dirty_clean::check_dirty_clean_annotations(tcx); }) } @@ -142,6 +153,9 @@ fn finish_dep_graph(tcx: TyCtxt<'_>, results_path: &Path) { "assert dep graph", || assert_dep_graph(tcx, &results)); + + dirty_clean::check_dirty_clean_annotations(tcx, &results); + if tcx.sess.opts.debugging_opts.incremental_info { let data = &results.model.as_ref().unwrap().data; From fb706ba68cf4be22c3dee3525880abfae66007fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Thu, 13 Dec 2018 06:40:22 +0100 Subject: [PATCH 15/17] test --- src/bootstrap/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 66f504ea924e9..8d3ef8401e50b 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -567,7 +567,7 @@ impl Config { set(&mut config.lld_enabled, rust.lld); set(&mut config.lldb_enabled, rust.lldb); set(&mut config.llvm_tools_enabled, rust.llvm_tools); - config.rustc_parallel = rust.parallel_compiler.unwrap_or(false); + config.rustc_parallel = rust.parallel_compiler.unwrap_or(true); config.rustc_default_linker = rust.default_linker.clone(); config.musl_root = rust.musl_root.clone().map(PathBuf::from); config.save_toolstates = rust.save_toolstates.clone().map(PathBuf::from); From 1bc17c2262d312455b732d020bfedd1795ff5f3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Tue, 18 Jun 2019 03:25:12 +0200 Subject: [PATCH 16/17] rebase fix --- src/librustc/dep_graph/graph.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index d927af7ec0cfd..a1b6cd3b17091 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -663,9 +663,9 @@ impl DepGraph { } /// Try to force a dep node to execute and see if it's green - fn try_force_previous_green<'tcx>( + fn try_force_previous_green( &self, - tcx: TyCtxt<'_, 'tcx, 'tcx>, + tcx: TyCtxt<'_>, data: &DepGraphData, dep_node_index: DepNodeIndex, ) -> bool { From 9f268ad2f5009213d5f33fea02d359536cc89cec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Tue, 18 Jun 2019 03:25:29 +0200 Subject: [PATCH 17/17] Update Cargo.lock --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index d3c6be59b75df..63a84f1fc2adc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2780,6 +2780,7 @@ name = "rustc_data_structures" version = "0.0.0" dependencies = [ "cfg-if 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "ena 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "graphviz 0.0.0", "indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",