Auto merge of #71044 - ecstatic-morse:body-predecessor-cache, r=oli-obk

Remove `BodyAndCache`

...returning to the original approach using interior mutability within `Body`. This simplifies the API at the cost of some uncontended mutex locks when the parallel compiler is enabled.

The current API requires you to either have a mutable reference to `Body` (`&mut BodyAndCache`), or to compute the predecessor graph ahead of time by creating a `ReadOnlyBodyAndCache`. This is not a good fit for, e.g., the dataflow framework, which
1. does not mutate the MIR
2. only sometimes needs the predecessor graph (for backward dataflow problems)

Author: bors

src/librustc_codegen_ssa/mir/analyze.rs

@@ -120,7 +120,7 @@ impl<Bx: BuilderMethods<'a, 'tcx>> LocalAnalyzer<'mir, 'a, 'tcx, Bx> {
             };
             if is_consume {
                 let base_ty =
-                    mir::Place::ty_from(place_ref.local, proj_base, *self.fx.mir, cx.tcx());
+                    mir::Place::ty_from(place_ref.local, proj_base, self.fx.mir, cx.tcx());
                 let base_ty = self.fx.monomorphize(&base_ty);
 
                 // ZSTs don't require any actual memory access.

src/librustc_codegen_ssa/mir/block.rs

@@ -152,7 +152,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
     // a loop.
     fn maybe_sideeffect<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        mir: mir::ReadOnlyBodyAndCache<'tcx, 'tcx>,
+        mir: &'tcx mir::Body<'tcx>,
         bx: &mut Bx,
         targets: &[mir::BasicBlock],
     ) {
@@ -306,7 +306,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         target: mir::BasicBlock,
         unwind: Option<mir::BasicBlock>,
     ) {
-        let ty = location.ty(*self.mir, bx.tcx()).ty;
+        let ty = location.ty(self.mir, bx.tcx()).ty;
         let ty = self.monomorphize(&ty);
         let drop_fn = Instance::resolve_drop_in_place(bx.tcx(), ty);
 
@@ -572,7 +572,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let extra_args = extra_args
             .iter()
             .map(|op_arg| {
-                let op_ty = op_arg.ty(*self.mir, bx.tcx());
+                let op_ty = op_arg.ty(self.mir, bx.tcx());
                 self.monomorphize(&op_ty)
             })
             .collect::<Vec<_>>();

src/librustc_codegen_ssa/mir/mod.rs

@@ -21,7 +21,7 @@ use self::operand::{OperandRef, OperandValue};
 pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
     instance: Instance<'tcx>,
 
-    mir: mir::ReadOnlyBodyAndCache<'tcx, 'tcx>,
+    mir: &'tcx mir::Body<'tcx>,
 
     debug_context: Option<FunctionDebugContext<Bx::DIScope>>,
 
@@ -169,7 +169,6 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         .collect();
 
     let (landing_pads, funclets) = create_funclets(&mir, &mut bx, &cleanup_kinds, &block_bxs);
-    let mir_body: &mir::Body<'_> = *mir;
     let mut fx = FunctionCx {
         instance,
         mir,
@@ -197,7 +196,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         let args = arg_local_refs(&mut bx, &mut fx, &memory_locals);
 
         let mut allocate_local = |local| {
-            let decl = &mir_body.local_decls[local];
+            let decl = &mir.local_decls[local];
             let layout = bx.layout_of(fx.monomorphize(&decl.ty));
             assert!(!layout.ty.has_erasable_regions());
 
@@ -223,7 +222,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         let retptr = allocate_local(mir::RETURN_PLACE);
         iter::once(retptr)
             .chain(args.into_iter())
-            .chain(mir_body.vars_and_temps_iter().map(allocate_local))
+            .chain(mir.vars_and_temps_iter().map(allocate_local))
             .collect()
     };
 
@@ -235,8 +234,8 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         bx.br(fx.blocks[mir::START_BLOCK]);
     }
 
-    let rpo = traversal::reverse_postorder(&mir_body);
-    let mut visited = BitSet::new_empty(mir_body.basic_blocks().len());
+    let rpo = traversal::reverse_postorder(&mir);
+    let mut visited = BitSet::new_empty(mir.basic_blocks().len());
 
     // Codegen the body of each block using reverse postorder
     for (bb, _) in rpo {
@@ -246,7 +245,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 
     // Remove blocks that haven't been visited, or have no
     // predecessors.
-    for bb in mir_body.basic_blocks().indices() {
+    for bb in mir.basic_blocks().indices() {
         // Unreachable block
         if !visited.contains(bb.index()) {
             debug!("codegen_mir: block {:?} was not visited", bb);

src/librustc_codegen_ssa/mir/place.rs

@@ -497,7 +497,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
     pub fn monomorphized_place_ty(&self, place_ref: mir::PlaceRef<'tcx>) -> Ty<'tcx> {
         let tcx = self.cx.tcx();
-        let place_ty = mir::Place::ty_from(place_ref.local, place_ref.projection, *self.mir, tcx);
+        let place_ty = mir::Place::ty_from(place_ref.local, place_ref.projection, self.mir, tcx);
         self.monomorphize(&place_ty.ty)
     }
 }

src/librustc_codegen_ssa/mir/rvalue.rs

@@ -473,7 +473,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             }
 
             mir::Rvalue::Discriminant(ref place) => {
-                let discr_ty = rvalue.ty(*self.mir, bx.tcx());
+                let discr_ty = rvalue.ty(self.mir, bx.tcx());
                 let discr = self
                     .codegen_place(&mut bx, place.as_ref())
                     .codegen_get_discr(&mut bx, discr_ty);
@@ -529,7 +529,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             mir::Rvalue::Repeat(..) | mir::Rvalue::Aggregate(..) => {
                 // According to `rvalue_creates_operand`, only ZST
                 // aggregate rvalues are allowed to be operands.
-                let ty = rvalue.ty(*self.mir, self.cx.tcx());
+                let ty = rvalue.ty(self.mir, self.cx.tcx());
                 let operand =
                     OperandRef::new_zst(&mut bx, self.cx.layout_of(self.monomorphize(&ty)));
                 (bx, operand)
@@ -749,7 +749,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 true,
             mir::Rvalue::Repeat(..) |
             mir::Rvalue::Aggregate(..) => {
-                let ty = rvalue.ty(*self.mir, self.cx.tcx());
+                let ty = rvalue.ty(self.mir, self.cx.tcx());
                 let ty = self.monomorphize(&ty);
                 self.cx.spanned_layout_of(ty, span).is_zst()
             }

src/librustc_metadata/rmeta/decoder.rs

@@ -26,7 +26,7 @@ use rustc_middle::middle::cstore::{CrateSource, ExternCrate};
 use rustc_middle::middle::cstore::{ForeignModule, LinkagePreference, NativeLibrary};
 use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportLevel};
 use rustc_middle::mir::interpret::{AllocDecodingSession, AllocDecodingState};
-use rustc_middle::mir::{self, interpret, BodyAndCache, Promoted};
+use rustc_middle::mir::{self, interpret, Body, Promoted};
 use rustc_middle::ty::codec::TyDecoder;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_middle::util::common::record_time;
@@ -1099,40 +1099,28 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
         !self.is_proc_macro(id) && self.root.tables.mir.get(self, id).is_some()
     }
 
-    fn get_optimized_mir(&self, tcx: TyCtxt<'tcx>, id: DefIndex) -> BodyAndCache<'tcx> {
-        let mut cache = self
-            .root
+    fn get_optimized_mir(&self, tcx: TyCtxt<'tcx>, id: DefIndex) -> Body<'tcx> {
+        self.root
             .tables
             .mir
             .get(self, id)
             .filter(|_| !self.is_proc_macro(id))
             .unwrap_or_else(|| {
                 bug!("get_optimized_mir: missing MIR for `{:?}`", self.local_def_id(id))
             })
-            .decode((self, tcx));
-        cache.ensure_predecessors();
-        cache
+            .decode((self, tcx))
     }
 
-    fn get_promoted_mir(
-        &self,
-        tcx: TyCtxt<'tcx>,
-        id: DefIndex,
-    ) -> IndexVec<Promoted, BodyAndCache<'tcx>> {
-        let mut cache = self
-            .root
+    fn get_promoted_mir(&self, tcx: TyCtxt<'tcx>, id: DefIndex) -> IndexVec<Promoted, Body<'tcx>> {
+        self.root
             .tables
             .promoted_mir
             .get(self, id)
             .filter(|_| !self.is_proc_macro(id))
             .unwrap_or_else(|| {
                 bug!("get_promoted_mir: missing MIR for `{:?}`", self.local_def_id(id))
             })
-            .decode((self, tcx));
-        for body in cache.iter_mut() {
-            body.ensure_predecessors();
-        }
-        cache
+            .decode((self, tcx))
     }
 
     fn mir_const_qualif(&self, id: DefIndex) -> mir::ConstQualifs {

src/librustc_metadata/rmeta/encoder.rs

@@ -68,6 +68,7 @@ macro_rules! encoder_methods {
 impl<'tcx> Encoder for EncodeContext<'tcx> {
     type Error = <opaque::Encoder as Encoder>::Error;
 
+    #[inline]
     fn emit_unit(&mut self) -> Result<(), Self::Error> {
         Ok(())
     }

src/librustc_metadata/rmeta/mod.rs

@@ -275,8 +275,8 @@ define_tables! {
     // Also, as an optimization, a missing entry indicates an empty `&[]`.
     inferred_outlives: Table<DefIndex, Lazy!(&'tcx [(ty::Predicate<'tcx>, Span)])>,
     super_predicates: Table<DefIndex, Lazy!(ty::GenericPredicates<'tcx>)>,
-    mir: Table<DefIndex, Lazy!(mir::BodyAndCache<'tcx>)>,
-    promoted_mir: Table<DefIndex, Lazy!(IndexVec<mir::Promoted, mir::BodyAndCache<'tcx>>)>,
+    mir: Table<DefIndex, Lazy!(mir::Body<'tcx>)>,
+    promoted_mir: Table<DefIndex, Lazy!(IndexVec<mir::Promoted, mir::Body<'tcx>>)>,
 }
 
 #[derive(Copy, Clone, RustcEncodable, RustcDecodable)]

src/librustc_middle/arena.rs

@@ -15,17 +15,17 @@ macro_rules! arena_types {
             [] generics: rustc_middle::ty::Generics,
             [] trait_def: rustc_middle::ty::TraitDef,
             [] adt_def: rustc_middle::ty::AdtDef,
-            [] steal_mir: rustc_middle::ty::steal::Steal<rustc_middle::mir::BodyAndCache<$tcx>>,
-            [] mir: rustc_middle::mir::BodyAndCache<$tcx>,
+            [] steal_mir: rustc_middle::ty::steal::Steal<rustc_middle::mir::Body<$tcx>>,
+            [] mir: rustc_middle::mir::Body<$tcx>,
             [] steal_promoted: rustc_middle::ty::steal::Steal<
                 rustc_index::vec::IndexVec<
                     rustc_middle::mir::Promoted,
-                    rustc_middle::mir::BodyAndCache<$tcx>
+                    rustc_middle::mir::Body<$tcx>
                 >
             >,
             [] promoted: rustc_index::vec::IndexVec<
                 rustc_middle::mir::Promoted,
-                rustc_middle::mir::BodyAndCache<$tcx>
+                rustc_middle::mir::Body<$tcx>
             >,
             [decode] tables: rustc_middle::ty::TypeckTables<$tcx>,
             [decode] borrowck_result: rustc_middle::mir::BorrowCheckResult<$tcx>,