perf: Add inline on commonly used methods added in 69464

Reclaims most of the regression in inflate

Author: Markus Westerlind

src/librustc_data_structures/snapshot_map/mod.rs

@@ -37,6 +37,7 @@ pub enum UndoLog<K, V> {
 }
 
 impl<K, V, M, L> SnapshotMap<K, V, M, L> {
+    #[inline]
     pub fn with_log<L2>(&mut self, undo_log: L2) -> SnapshotMap<K, V, &mut M, L2> {
         SnapshotMap { map: &mut self.map, undo_log, _marker: PhantomData }
     }

src/librustc_infer/infer/mod.rs

@@ -219,18 +219,22 @@ impl<'tcx> InferCtxtInner<'tcx> {
         }
     }
 
+    #[inline]
     pub fn region_obligations(&self) -> &[(hir::HirId, RegionObligation<'tcx>)] {
         &self.region_obligations
     }
 
+    #[inline]
     pub fn projection_cache(&mut self) -> traits::ProjectionCache<'_, 'tcx> {
         self.projection_cache.with_log(&mut self.undo_log)
     }
 
+    #[inline]
     fn type_variables(&mut self) -> type_variable::TypeVariableTable<'_, 'tcx> {
         self.type_variable_storage.with_log(&mut self.undo_log)
     }
 
+    #[inline]
     fn int_unification_table(
         &mut self,
     ) -> ut::UnificationTable<
@@ -243,6 +247,7 @@ impl<'tcx> InferCtxtInner<'tcx> {
         self.int_unification_storage.with_log(&mut self.undo_log)
     }
 
+    #[inline]
     fn float_unification_table(
         &mut self,
     ) -> ut::UnificationTable<
@@ -255,6 +260,7 @@ impl<'tcx> InferCtxtInner<'tcx> {
         self.float_unification_storage.with_log(&mut self.undo_log)
     }
 
+    #[inline]
     fn const_unification_table(
         &mut self,
     ) -> ut::UnificationTable<
@@ -267,6 +273,7 @@ impl<'tcx> InferCtxtInner<'tcx> {
         self.const_unification_storage.with_log(&mut self.undo_log)
     }
 
+    #[inline]
     pub fn unwrap_region_constraints(&mut self) -> RegionConstraintCollector<'_, 'tcx> {
         self.region_constraint_storage
             .as_mut()
@@ -1645,14 +1652,13 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> {
     /// having to resort to storing full `GenericArg`s in `stalled_on`.
     #[inline(always)]
     pub fn ty_or_const_infer_var_changed(&self, infer_var: TyOrConstInferVar<'tcx>) -> bool {
-        let mut inner = self.inner.borrow_mut();
         match infer_var {
             TyOrConstInferVar::Ty(v) => {
                 use self::type_variable::TypeVariableValue;
 
                 // If `inlined_probe` returns a `Known` value, it never equals
                 // `ty::Infer(ty::TyVar(v))`.
-                match inner.type_variables().inlined_probe(v) {
+                match self.inner.borrow_mut().type_variables().inlined_probe(v) {
                     TypeVariableValue::Unknown { .. } => false,
                     TypeVariableValue::Known { .. } => true,
                 }
@@ -1662,23 +1668,23 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> {
                 // If `inlined_probe_value` returns a value it's always a
                 // `ty::Int(_)` or `ty::UInt(_)`, which never matches a
                 // `ty::Infer(_)`.
-                inner.int_unification_table().inlined_probe_value(v).is_some()
+                self.inner.borrow_mut().int_unification_table().inlined_probe_value(v).is_some()
             }
 
             TyOrConstInferVar::TyFloat(v) => {
                 // If `probe_value` returns a value it's always a
                 // `ty::Float(_)`, which never matches a `ty::Infer(_)`.
                 //
                 // Not `inlined_probe_value(v)` because this call site is colder.
-                inner.float_unification_table().probe_value(v).is_some()
+                self.inner.borrow_mut().float_unification_table().probe_value(v).is_some()
             }
 
             TyOrConstInferVar::Const(v) => {
                 // If `probe_value` returns a `Known` value, it never equals
                 // `ty::ConstKind::Infer(ty::InferConst::Var(v))`.
                 //
                 // Not `inlined_probe_value(v)` because this call site is colder.
-                match inner.const_unification_table().probe_value(v).val {
+                match self.inner.borrow_mut().const_unification_table().probe_value(v).val {
                     ConstVariableValue::Unknown { .. } => false,
                     ConstVariableValue::Known { .. } => true,
                 }

src/librustc_infer/infer/region_constraints/mod.rs

@@ -68,12 +68,14 @@ pub struct RegionConstraintCollector<'a, 'tcx> {
 
 impl std::ops::Deref for RegionConstraintCollector<'_, 'tcx> {
     type Target = RegionConstraintStorage<'tcx>;
+    #[inline]
     fn deref(&self) -> &RegionConstraintStorage<'tcx> {
         self.storage
     }
 }
 
 impl std::ops::DerefMut for RegionConstraintCollector<'_, 'tcx> {
+    #[inline]
     fn deref_mut(&mut self) -> &mut RegionConstraintStorage<'tcx> {
         self.storage
     }
@@ -345,6 +347,7 @@ impl<'tcx> RegionConstraintStorage<'tcx> {
         Self::default()
     }
 
+    #[inline]
     pub(crate) fn with_log<'a>(
         &'a mut self,
         undo_log: &'a mut InferCtxtUndoLogs<'tcx>,
@@ -796,6 +799,7 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> {
             .unwrap_or(None)
     }
 
+    #[inline]
     fn unification_table(&mut self) -> super::UnificationTable<'_, 'tcx, ty::RegionVid> {
         ut::UnificationTable::with_log(&mut self.storage.unification_table, self.undo_log)
     }

src/librustc_infer/infer/type_variable.rs

@@ -87,11 +87,7 @@ pub struct TypeVariableStorage<'tcx> {
 }
 
 pub struct TypeVariableTable<'a, 'tcx> {
-    values: &'a mut sv::SnapshotVecStorage<Delegate>,
-
-    eq_relations: &'a mut ut::UnificationTableStorage<TyVidEqKey<'tcx>>,
-
-    sub_relations: &'a mut ut::UnificationTableStorage<ty::TyVid>,
+    storage: &'a mut TypeVariableStorage<'tcx>,
 
     undo_log: &'a mut InferCtxtUndoLogs<'tcx>,
 }
@@ -165,12 +161,12 @@ impl<'tcx> TypeVariableStorage<'tcx> {
         }
     }
 
+    #[inline]
     pub(crate) fn with_log<'a>(
         &'a mut self,
         undo_log: &'a mut InferCtxtUndoLogs<'tcx>,
     ) -> TypeVariableTable<'a, 'tcx> {
-        let TypeVariableStorage { values, eq_relations, sub_relations } = self;
-        TypeVariableTable { values, eq_relations, sub_relations, undo_log }
+        TypeVariableTable { storage: self, undo_log }
     }
 }
 
@@ -180,15 +176,15 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> {
     /// Note that this function does not return care whether
     /// `vid` has been unified with something else or not.
     pub fn var_diverges(&self, vid: ty::TyVid) -> bool {
-        self.values.get(vid.index as usize).diverging
+        self.storage.values.get(vid.index as usize).diverging
     }
 
     /// Returns the origin that was given when `vid` was created.
     ///
     /// Note that this function does not return care whether
     /// `vid` has been unified with something else or not.
     pub fn var_origin(&self, vid: ty::TyVid) -> &TypeVariableOrigin {
-        &self.values.get(vid.index as usize).origin
+        &self.storage.values.get(vid.index as usize).origin
     }
 
     /// Records that `a == b`, depending on `dir`.
@@ -265,7 +261,7 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> {
 
     /// Returns the number of type variables created thus far.
     pub fn num_vars(&self) -> usize {
-        self.values.len()
+        self.storage.values.len()
     }
 
     /// Returns the "root" variable of `vid` in the `eq_relations`
@@ -319,18 +315,21 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> {
         }
     }
 
+    #[inline]
     fn values(
         &mut self,
     ) -> sv::SnapshotVec<Delegate, &mut Vec<TypeVariableData>, &mut InferCtxtUndoLogs<'tcx>> {
-        self.values.with_log(self.undo_log)
+        self.storage.values.with_log(self.undo_log)
     }
 
+    #[inline]
     fn eq_relations(&mut self) -> super::UnificationTable<'_, 'tcx, TyVidEqKey<'tcx>> {
-        self.eq_relations.with_log(self.undo_log)
+        self.storage.eq_relations.with_log(self.undo_log)
     }
 
+    #[inline]
     fn sub_relations(&mut self) -> super::UnificationTable<'_, 'tcx, ty::TyVid> {
-        self.sub_relations.with_log(self.undo_log)
+        self.storage.sub_relations.with_log(self.undo_log)
     }
 
     /// Returns a range of the type variables created during the snapshot.
@@ -342,7 +341,7 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> {
         (
             range.start..range.end,
             (range.start.index..range.end.index)
-                .map(|index| self.values.get(index as usize).origin)
+                .map(|index| self.storage.values.get(index as usize).origin)
                 .collect(),
         )
     }
@@ -378,7 +377,7 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> {
                         // quick check to see if this variable was
                         // created since the snapshot started or not.
                         let mut eq_relations = ut::UnificationTable::with_log(
-                            &mut *self.eq_relations,
+                            &mut self.storage.eq_relations,
                             &mut *self.undo_log,
                         );
                         let escaping_type = match eq_relations.probe_value(vid) {
@@ -400,7 +399,7 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> {
     /// Returns indices of all variables that are not yet
     /// instantiated.
     pub fn unsolved_variables(&mut self) -> Vec<ty::TyVid> {
-        (0..self.values.len())
+        (0..self.storage.values.len())
             .filter_map(|i| {
                 let vid = ty::TyVid { index: i as u32 };
                 match self.probe(vid) {

src/librustc_infer/infer/undo_log.rs

@@ -100,10 +100,12 @@ impl<'tcx, T> UndoLogs<T> for InferCtxtUndoLogs<'tcx>
 where
     UndoLog<'tcx>: From<T>,
 {
+    #[inline]
     fn num_open_snapshots(&self) -> usize {
         self.num_open_snapshots
     }
 
+    #[inline]
     fn push(&mut self, undo: T) {
         if self.in_snapshot() {
             self.logs.push(undo.into())

src/librustc_infer/traits/project.rs

@@ -95,6 +95,7 @@ pub enum ProjectionCacheEntry<'tcx> {
 }
 
 impl<'tcx> ProjectionCacheStorage<'tcx> {
+    #[inline]
     pub(crate) fn with_log<'a>(
         &'a mut self,
         undo_log: &'a mut InferCtxtUndoLogs<'tcx>,
@@ -104,6 +105,7 @@ impl<'tcx> ProjectionCacheStorage<'tcx> {
 }
 
 impl<'tcx> ProjectionCache<'_, 'tcx> {
+    #[inline]
     fn map(
         &mut self,
     ) -> SnapshotMapRef<