Auto merge of #89239 - petrochenkov:modcache, r=cjgillot

resolve: Cache module loading for all foreign modules

It was previously cached for modules loaded from `fn get_module`, but not for modules loaded from `fn build_reduced_graph_for_external_crate_res`.
This also makes all foreign modules use their real parent, span and expansion instead of possibly a parent/span/expansion of their reexport.
Modules are also often compared using referential equality (`ptr::eq`), this change makes such comparisons correct in all cases.

An ICE happening on attempt to decode expansions for foreign enums and traits is avoided.

Also local enums and traits are now added to the module map.

Follow up to #88872.
r? `@cjgillot`

Author: bors

compiler/rustc_resolve/src/build_reduced_graph.rs

@@ -127,7 +127,6 @@ impl<'a> Resolver<'a> {
     /// If `def_id` refers to a module (in resolver's sense, i.e. a module item, crate root, enum,
     /// or trait), then this function returns that module's resolver representation, otherwise it
     /// returns `None`.
-    /// FIXME: `Module`s for local enums and traits are not currently found.
     crate fn get_module(&mut self, def_id: DefId) -> Option<Module<'a>> {
         if let module @ Some(..) = self.module_map.get(&def_id) {
             return module.copied();
@@ -146,17 +145,21 @@ impl<'a> Resolver<'a> {
                     } else {
                         def_key.disambiguated_data.data.get_opt_name().expect("module without name")
                     };
+                    let expn_id = if def_kind == DefKind::Mod {
+                        self.cstore().module_expansion_untracked(def_id, &self.session)
+                    } else {
+                        // FIXME: Parent expansions for enums and traits are not kept in metadata.
+                        ExpnId::root()
+                    };
 
-                    let module = self.arenas.new_module(
+                    Some(self.new_module(
                         parent,
                         ModuleKind::Def(def_kind, def_id, name),
-                        self.cstore().module_expansion_untracked(def_id, &self.session),
+                        expn_id,
                         self.cstore().get_span_untracked(def_id, &self.session),
                         // FIXME: Account for `#[no_implicit_prelude]` attributes.
                         parent.map_or(false, |module| module.no_implicit_prelude),
-                    );
-                    self.module_map.insert(def_id, module);
-                    Some(module)
+                    ))
                 }
                 _ => None,
             }
@@ -217,8 +220,7 @@ impl<'a> Resolver<'a> {
     }
 
     crate fn build_reduced_graph_external(&mut self, module: Module<'a>) {
-        let def_id = module.def_id().expect("unpopulated module without a def-id");
-        for child in self.cstore().item_children_untracked(def_id, self.session) {
+        for child in self.cstore().item_children_untracked(module.def_id(), self.session) {
             let parent_scope = ParentScope::module(module, self);
             BuildReducedGraphVisitor { r: self, parent_scope }
                 .build_reduced_graph_for_external_crate_res(child);
@@ -759,7 +761,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
             }
 
             ItemKind::Mod(..) => {
-                let module = self.r.arenas.new_module(
+                let module = self.r.new_module(
                     Some(parent),
                     ModuleKind::Def(DefKind::Mod, def_id, ident.name),
                     expansion.to_expn_id(),
@@ -768,7 +770,6 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
                         || self.r.session.contains_name(&item.attrs, sym::no_implicit_prelude),
                 );
                 self.r.define(parent, ident, TypeNS, (module, vis, sp, expansion));
-                self.r.module_map.insert(def_id, module);
 
                 // Descend into the module.
                 self.parent_scope.module = module;
@@ -799,7 +800,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
             }
 
             ItemKind::Enum(_, _) => {
-                let module = self.r.arenas.new_module(
+                let module = self.r.new_module(
                     Some(parent),
                     ModuleKind::Def(DefKind::Enum, def_id, ident.name),
                     expansion.to_expn_id(),
@@ -873,7 +874,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
 
             ItemKind::Trait(..) => {
                 // Add all the items within to a new module.
-                let module = self.r.arenas.new_module(
+                let module = self.r.new_module(
                     Some(parent),
                     ModuleKind::Def(DefKind::Trait, def_id, ident.name),
                     expansion.to_expn_id(),
@@ -916,7 +917,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
         let parent = self.parent_scope.module;
         let expansion = self.parent_scope.expansion;
         if self.block_needs_anonymous_module(block) {
-            let module = self.r.arenas.new_module(
+            let module = self.r.new_module(
                 Some(parent),
                 ModuleKind::Block(block.id),
                 expansion.to_expn_id(),
@@ -936,15 +937,8 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
         let expansion = self.parent_scope.expansion;
         // Record primary definitions.
         match res {
-            Res::Def(kind @ (DefKind::Mod | DefKind::Enum | DefKind::Trait), def_id) => {
-                let module = self.r.arenas.new_module(
-                    Some(parent),
-                    ModuleKind::Def(kind, def_id, ident.name),
-                    expansion.to_expn_id(),
-                    span,
-                    // FIXME: Account for `#[no_implicit_prelude]` attributes.
-                    parent.no_implicit_prelude,
-                );
+            Res::Def(DefKind::Mod | DefKind::Enum | DefKind::Trait, def_id) => {
+                let module = self.r.expect_module(def_id);
                 self.r.define(parent, ident, TypeNS, (module, vis, span, expansion));
             }
             Res::Def(

compiler/rustc_resolve/src/diagnostics.rs

@@ -801,7 +801,7 @@ impl<'a> Resolver<'a> {
             None => worklist_via_import.pop(),
             Some(x) => Some(x),
         } {
-            let in_module_is_extern = !in_module.def_id().unwrap().is_local();
+            let in_module_is_extern = !in_module.def_id().is_local();
             // We have to visit module children in deterministic order to avoid
             // instabilities in reported imports (#43552).
             in_module.for_each_child(self, |this, ident, ns, name_binding| {
@@ -884,7 +884,7 @@ impl<'a> Resolver<'a> {
 
                     if !is_extern_crate_that_also_appears_in_prelude {
                         // add the module to the lookup
-                        if seen_modules.insert(module.def_id().unwrap()) {
+                        if seen_modules.insert(module.def_id()) {
                             if via_import { &mut worklist_via_import } else { &mut worklist }
                                 .push((module, path_segments, child_accessible));
                         }

compiler/rustc_resolve/src/imports.rs

@@ -989,7 +989,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
                 }
 
                 if let ModuleOrUniformRoot::Module(module) = module {
-                    if module.def_id() == import.parent_scope.module.def_id() {
+                    if ptr::eq(module, import.parent_scope.module) {
                         // Importing a module into itself is not allowed.
                         return Some(UnresolvedImportError {
                             span: import.span,
@@ -1341,7 +1341,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
         if module.is_trait() {
             self.r.session.span_err(import.span, "items in traits are not importable.");
             return;
-        } else if module.def_id() == import.parent_scope.module.def_id() {
+        } else if ptr::eq(module, import.parent_scope.module) {
             return;
         } else if let ImportKind::Glob { is_prelude: true, .. } = import.kind {
             self.r.prelude = Some(module);
@@ -1400,7 +1400,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
         });
 
         if !reexports.is_empty() {
-            if let Some(def_id) = module.def_id() {
+            if let Some(def_id) = module.opt_def_id() {
                 // Call to `expect_local` should be fine because current
                 // code is only called for local modules.
                 self.r.export_map.insert(def_id.expect_local(), reexports);

compiler/rustc_resolve/src/late/diagnostics.rs

@@ -1491,7 +1491,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                     // form the path
                     let mut path_segments = path_segments.clone();
                     path_segments.push(ast::PathSegment::from_ident(ident));
-                    let module_def_id = module.def_id().unwrap();
+                    let module_def_id = module.def_id();
                     if module_def_id == def_id {
                         let path =
                             Path { span: name_binding.span, segments: path_segments, tokens: None };

compiler/rustc_resolve/src/lib.rs

@@ -413,7 +413,7 @@ impl ModuleOrUniformRoot<'_> {
     fn same_def(lhs: Self, rhs: Self) -> bool {
         match (lhs, rhs) {
             (ModuleOrUniformRoot::Module(lhs), ModuleOrUniformRoot::Module(rhs)) => {
-                lhs.def_id() == rhs.def_id()
+                ptr::eq(lhs, rhs)
             }
             (
                 ModuleOrUniformRoot::CrateRootAndExternPrelude,
@@ -602,7 +602,11 @@ impl<'a> ModuleData<'a> {
         }
     }
 
-    fn def_id(&self) -> Option<DefId> {
+    fn def_id(&self) -> DefId {
+        self.opt_def_id().expect("`ModuleData::def_id` is called on a block module")
+    }
+
+    fn opt_def_id(&self) -> Option<DefId> {
         match self.kind {
             ModuleKind::Def(_, def_id, _) => Some(def_id),
             _ => None,
@@ -1071,12 +1075,17 @@ impl<'a> ResolverArenas<'a> {
         expn_id: ExpnId,
         span: Span,
         no_implicit_prelude: bool,
+        module_map: &mut FxHashMap<DefId, Module<'a>>,
     ) -> Module<'a> {
         let module =
             self.modules.alloc(ModuleData::new(parent, kind, expn_id, span, no_implicit_prelude));
-        if module.def_id().map_or(true, |def_id| def_id.is_local()) {
+        let def_id = module.opt_def_id();
+        if def_id.map_or(true, |def_id| def_id.is_local()) {
             self.local_modules.borrow_mut().push(module);
         }
+        if let Some(def_id) = def_id {
+            module_map.insert(def_id, module);
+        }
         module
     }
     fn local_modules(&'a self) -> std::cell::Ref<'a, Vec<Module<'a>>> {
@@ -1276,22 +1285,23 @@ impl<'a> Resolver<'a> {
         arenas: &'a ResolverArenas<'a>,
     ) -> Resolver<'a> {
         let root_def_id = CRATE_DEF_ID.to_def_id();
+        let mut module_map = FxHashMap::default();
         let graph_root = arenas.new_module(
             None,
             ModuleKind::Def(DefKind::Mod, root_def_id, kw::Empty),
             ExpnId::root(),
             krate.span,
             session.contains_name(&krate.attrs, sym::no_implicit_prelude),
+            &mut module_map,
         );
         let empty_module = arenas.new_module(
             None,
             ModuleKind::Def(DefKind::Mod, root_def_id, kw::Empty),
             ExpnId::root(),
             DUMMY_SP,
             true,
+            &mut FxHashMap::default(),
         );
-        let mut module_map = FxHashMap::default();
-        module_map.insert(root_def_id, graph_root);
 
         let definitions = Definitions::new(session.local_stable_crate_id(), krate.span);
         let root = definitions.get_root_def();
@@ -1434,6 +1444,18 @@ impl<'a> Resolver<'a> {
         resolver
     }
 
+    fn new_module(
+        &mut self,
+        parent: Option<Module<'a>>,
+        kind: ModuleKind,
+        expn_id: ExpnId,
+        span: Span,
+        no_implicit_prelude: bool,
+    ) -> Module<'a> {
+        let module_map = &mut self.module_map;
+        self.arenas.new_module(parent, kind, expn_id, span, no_implicit_prelude, module_map)
+    }
+
     fn create_stable_hashing_context(&self) -> ExpandHasher<'_, 'a> {
         ExpandHasher {
             source_map: CachingSourceMapView::new(self.session.source_map()),
@@ -1570,7 +1592,7 @@ impl<'a> Resolver<'a> {
 
         if let Some(module) = current_trait {
             if self.trait_may_have_item(Some(module), assoc_item) {
-                let def_id = module.def_id().unwrap();
+                let def_id = module.def_id();
                 found_traits.push(TraitCandidate { def_id, import_ids: smallvec![] });
             }
         }
@@ -2171,8 +2193,9 @@ impl<'a> Resolver<'a> {
                 return self.graph_root;
             }
         };
-        let module = self
-            .expect_module(module.def_id().map_or(LOCAL_CRATE, |def_id| def_id.krate).as_def_id());
+        let module = self.expect_module(
+            module.opt_def_id().map_or(LOCAL_CRATE, |def_id| def_id.krate).as_def_id(),
+        );
         debug!(
             "resolve_crate_root({:?}): got module {:?} ({:?}) (ident.span = {:?})",
             ident,
@@ -2999,7 +3022,7 @@ impl<'a> Resolver<'a> {
         }
 
         let container = match parent.kind {
-            ModuleKind::Def(kind, _, _) => kind.descr(parent.def_id().unwrap()),
+            ModuleKind::Def(kind, _, _) => kind.descr(parent.def_id()),
             ModuleKind::Block(..) => "block",
         };
 

compiler/rustc_span/src/hygiene.rs

@@ -601,7 +601,10 @@ pub fn debug_hygiene_data(verbose: bool) -> String {
                 let expn_data = expn_data.as_ref().expect("no expansion data for an expansion ID");
                 debug_expn_data((&id.to_expn_id(), expn_data))
             });
-            data.foreign_expn_data.iter().for_each(debug_expn_data);
+            // Sort the hash map for more reproducible output.
+            let mut foreign_expn_data: Vec<_> = data.foreign_expn_data.iter().collect();
+            foreign_expn_data.sort_by_key(|(id, _)| (id.krate, id.local_id));
+            foreign_expn_data.into_iter().for_each(debug_expn_data);
             s.push_str("\n\nSyntaxContexts:");
             data.syntax_context_data.iter().enumerate().for_each(|(id, ctxt)| {
                 s.push_str(&format!(

src/test/ui/macros/auxiliary/macro-def-site-super.rs

@@ -0,0 +1,13 @@
+#![feature(decl_macro)]
+
+mod inner1 {
+    pub struct Struct {}
+
+    pub mod inner2 {
+        pub macro mac() {
+            super::Struct
+        }
+    }
+}
+
+pub use inner1::inner2 as public;

src/test/ui/macros/macro-def-site-super.rs

@@ -0,0 +1,10 @@
+// `super` in a `macro` refers to the parent module of the macro itself and not its reexport.
+
+// check-pass
+// aux-build:macro-def-site-super.rs
+
+extern crate macro_def_site_super;
+
+type A = macro_def_site_super::public::mac!();
+
+fn main() {}

src/test/ui/proc-macro/meta-macro-hygiene.rs

@@ -4,6 +4,7 @@
 // compile-flags: -Z span-debug -Z macro-backtrace -Z unpretty=expanded,hygiene -Z trim-diagnostic-paths=no
 // check-pass
 // normalize-stdout-test "\d+#" -> "0#"
+// normalize-stdout-test "expn\d{3,}" -> "expnNNN"
 //
 // We don't care about symbol ids, so we set them all to 0
 // in the stdout

src/test/ui/proc-macro/meta-macro-hygiene.stdout

@@ -1,5 +1,5 @@
 Def site: $DIR/auxiliary/make-macro.rs:7:9: 7:56 (#5)
-Input: TokenStream [Ident { ident: "$crate", span: $DIR/meta-macro-hygiene.rs:23:37: 23:43 (#4) }, Punct { ch: ':', spacing: Joint, span: $DIR/meta-macro-hygiene.rs:23:43: 23:45 (#4) }, Punct { ch: ':', spacing: Alone, span: $DIR/meta-macro-hygiene.rs:23:43: 23:45 (#4) }, Ident { ident: "dummy", span: $DIR/meta-macro-hygiene.rs:23:45: 23:50 (#4) }, Punct { ch: '!', spacing: Alone, span: $DIR/meta-macro-hygiene.rs:23:50: 23:51 (#4) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: $DIR/meta-macro-hygiene.rs:23:51: 23:53 (#4) }]
+Input: TokenStream [Ident { ident: "$crate", span: $DIR/meta-macro-hygiene.rs:24:37: 24:43 (#4) }, Punct { ch: ':', spacing: Joint, span: $DIR/meta-macro-hygiene.rs:24:43: 24:45 (#4) }, Punct { ch: ':', spacing: Alone, span: $DIR/meta-macro-hygiene.rs:24:43: 24:45 (#4) }, Ident { ident: "dummy", span: $DIR/meta-macro-hygiene.rs:24:45: 24:50 (#4) }, Punct { ch: '!', spacing: Alone, span: $DIR/meta-macro-hygiene.rs:24:50: 24:51 (#4) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: $DIR/meta-macro-hygiene.rs:24:51: 24:53 (#4) }]
 Respanned: TokenStream [Ident { ident: "$crate", span: $DIR/auxiliary/make-macro.rs:7:9: 7:56 (#5) }, Punct { ch: ':', spacing: Joint, span: $DIR/auxiliary/make-macro.rs:7:9: 7:56 (#5) }, Punct { ch: ':', spacing: Alone, span: $DIR/auxiliary/make-macro.rs:7:9: 7:56 (#5) }, Ident { ident: "dummy", span: $DIR/auxiliary/make-macro.rs:7:9: 7:56 (#5) }, Punct { ch: '!', spacing: Alone, span: $DIR/auxiliary/make-macro.rs:7:9: 7:56 (#5) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: $DIR/auxiliary/make-macro.rs:7:9: 7:56 (#5) }]
 #![feature /* 0#0 */(prelude_import)]
 // aux-build:make-macro.rs
@@ -8,6 +8,7 @@ Respanned: TokenStream [Ident { ident: "$crate", span: $DIR/auxiliary/make-macro
 // compile-flags: -Z span-debug -Z macro-backtrace -Z unpretty=expanded,hygiene -Z trim-diagnostic-paths=no
 // check-pass
 // normalize-stdout-test "\d+#" -> "0#"
+// normalize-stdout-test "expn\d{3,}" -> "expnNNN"
 //
 // We don't care about symbol ids, so we set them all to 0
 // in the stdout
@@ -48,6 +49,7 @@ crate0::{{expn1}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt:
 crate0::{{expn2}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt: #0, kind: Macro(Bang, "produce_it")
 crate0::{{expn3}}: parent: crate0::{{expn2}}, call_site_ctxt: #4, def_site_ctxt: #0, kind: Macro(Bang, "meta_macro::print_def_site")
 crate0::{{expn4}}: parent: crate0::{{expn3}}, call_site_ctxt: #5, def_site_ctxt: #0, kind: Macro(Bang, "$crate::dummy")
+crate1::{{expnNNN}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt: #0, kind: Macro(Bang, "include")
 crate2::{{expn1}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt: #0, kind: AstPass(StdImports)
 
 SyntaxContexts:

src/test/ui/proc-macro/nonterminal-token-hygiene.rs

@@ -4,6 +4,7 @@
 // compile-flags: -Z span-debug -Z macro-backtrace -Z unpretty=expanded,hygiene
 // compile-flags: -Z trim-diagnostic-paths=no
 // normalize-stdout-test "\d+#" -> "0#"
+// normalize-stdout-test "expn\d{3,}" -> "expnNNN"
 // aux-build:test-macros.rs
 
 #![feature(decl_macro)]

src/test/ui/proc-macro/nonterminal-token-hygiene.stdout

@@ -6,19 +6,19 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
         stream: TokenStream [
             Ident {
                 ident: "struct",
-                span: $DIR/nonterminal-token-hygiene.rs:30:5: 30:11 (#5),
+                span: $DIR/nonterminal-token-hygiene.rs:31:5: 31:11 (#5),
             },
             Ident {
                 ident: "S",
-                span: $DIR/nonterminal-token-hygiene.rs:30:12: 30:13 (#5),
+                span: $DIR/nonterminal-token-hygiene.rs:31:12: 31:13 (#5),
             },
             Punct {
                 ch: ';',
                 spacing: Alone,
-                span: $DIR/nonterminal-token-hygiene.rs:30:13: 30:14 (#5),
+                span: $DIR/nonterminal-token-hygiene.rs:31:13: 31:14 (#5),
             },
         ],
-        span: $DIR/nonterminal-token-hygiene.rs:20:27: 20:32 (#6),
+        span: $DIR/nonterminal-token-hygiene.rs:21:27: 21:32 (#6),
     },
 ]
 #![feature /* 0#0 */(prelude_import)]
@@ -29,6 +29,7 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
 // compile-flags: -Z span-debug -Z macro-backtrace -Z unpretty=expanded,hygiene
 // compile-flags: -Z trim-diagnostic-paths=no
 // normalize-stdout-test "\d+#" -> "0#"
+// normalize-stdout-test "expn\d{3,}" -> "expnNNN"
 // aux-build:test-macros.rs
 
 #![feature /* 0#0 */(decl_macro)]
@@ -72,6 +73,7 @@ crate0::{{expn1}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt:
 crate0::{{expn2}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt: #0, kind: Macro(Bang, "outer")
 crate0::{{expn3}}: parent: crate0::{{expn2}}, call_site_ctxt: #4, def_site_ctxt: #4, kind: Macro(Bang, "inner")
 crate0::{{expn4}}: parent: crate0::{{expn3}}, call_site_ctxt: #6, def_site_ctxt: #0, kind: Macro(Bang, "print_bang")
+crate1::{{expnNNN}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt: #0, kind: Macro(Bang, "include")
 crate2::{{expn1}}: parent: crate0::{{expn0}}, call_site_ctxt: #0, def_site_ctxt: #0, kind: AstPass(StdImports)
 
 SyntaxContexts: