Rollup merge of rust-lang#63537 - petrochenkov:novisit, r=alexcrichton

expand: Unimplement `MutVisitor` on `MacroExpander`

Each call to `fully_expand_fragment` is something unique, interesting, and requiring attention.
It represents a "root" of expansion and its use means that something unusual is happening, like eager expansion or expansion performed outside of the primary expansion pass.
So, it shouldn't hide under a generic visitor call.

Also, from all the implemented visitor methods only two were actually used.

cc rust-lang#63468 (comment)

Author: Centril

src/libsyntax/ext/base.rs

@@ -920,8 +920,10 @@ pub fn expr_to_spanned_string<'a>(
     // Update `expr.span`'s ctxt now in case expr is an `include!` macro invocation.
     expr.span = expr.span.apply_mark(cx.current_expansion.id);
 
-    // we want to be able to handle e.g., `concat!("foo", "bar")`
-    cx.expander().visit_expr(&mut expr);
+    // Perform eager expansion on the expression.
+    // We want to be able to handle e.g., `concat!("foo", "bar")`.
+    let expr = cx.expander().fully_expand_fragment(AstFragment::Expr(expr)).make_expr();
+
     Err(match expr.node {
         ast::ExprKind::Lit(ref l) => match l.node {
             ast::LitKind::Str(s, style) => return Ok(respan(expr.span, (s, style))),
@@ -986,8 +988,12 @@ pub fn get_exprs_from_tts(cx: &mut ExtCtxt<'_>,
     let mut p = cx.new_parser_from_tts(tts);
     let mut es = Vec::new();
     while p.token != token::Eof {
-        let mut expr = panictry!(p.parse_expr());
-        cx.expander().visit_expr(&mut expr);
+        let expr = panictry!(p.parse_expr());
+
+        // Perform eager expansion on the expression.
+        // We want to be able to handle e.g., `concat!("foo", "bar")`.
+        let expr = cx.expander().fully_expand_fragment(AstFragment::Expr(expr)).make_expr();
+
         es.push(expr);
         if p.eat(&token::Comma) {
             continue;

src/libsyntax/ext/expand.rs

@@ -116,18 +116,6 @@ macro_rules! ast_fragments {
             }
         }
 
-        impl<'a, 'b> MutVisitor for MacroExpander<'a, 'b> {
-            fn filter_map_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> {
-                self.expand_fragment(AstFragment::OptExpr(Some(expr))).make_opt_expr()
-            }
-            $($(fn $mut_visit_ast(&mut self, ast: &mut $AstTy) {
-                visit_clobber(ast, |ast| self.expand_fragment(AstFragment::$Kind(ast)).$make_ast());
-            })?)*
-            $($(fn $flat_map_ast_elt(&mut self, ast_elt: <$AstTy as IntoIterator>::Item) -> $AstTy {
-                self.expand_fragment(AstFragment::$Kind(smallvec![ast_elt])).$make_ast()
-            })?)*
-        }
-
         impl<'a> MacResult for crate::ext::tt::macro_rules::ParserAnyMacro<'a> {
             $(fn $make_ast(self: Box<crate::ext::tt::macro_rules::ParserAnyMacro<'a>>)
                            -> Option<$AstTy> {
@@ -265,7 +253,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             tokens: None,
         })]);
 
-        match self.expand_fragment(krate_item).make_items().pop().map(P::into_inner) {
+        match self.fully_expand_fragment(krate_item).make_items().pop().map(P::into_inner) {
             Some(ast::Item { attrs, node: ast::ItemKind::Mod(module), .. }) => {
                 krate.attrs = attrs;
                 krate.module = module;
@@ -285,8 +273,8 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
         krate
     }
 
-    // Fully expand all macro invocations in this AST fragment.
-    fn expand_fragment(&mut self, input_fragment: AstFragment) -> AstFragment {
+    // Recursively expand all macro invocations in this AST fragment.
+    pub fn fully_expand_fragment(&mut self, input_fragment: AstFragment) -> AstFragment {
         let orig_expansion_data = self.cx.current_expansion.clone();
         self.cx.current_expansion.depth = 0;
 

src/libsyntax_ext/proc_macro_harness.rs

@@ -1,18 +1,17 @@
 use std::mem;
 
+use smallvec::smallvec;
 use syntax::ast::{self, Ident};
 use syntax::attr;
 use syntax::source_map::{ExpnInfo, ExpnKind, respan};
 use syntax::ext::base::{ExtCtxt, MacroKind};
-use syntax::ext::expand::ExpansionConfig;
+use syntax::ext::expand::{AstFragment, ExpansionConfig};
 use syntax::ext::hygiene::ExpnId;
 use syntax::ext::proc_macro::is_proc_macro_attr;
-use syntax::mut_visit::MutVisitor;
 use syntax::parse::ParseSess;
 use syntax::ptr::P;
 use syntax::symbol::{kw, sym};
 use syntax::visit::{self, Visitor};
-
 use syntax_pos::{Span, DUMMY_SP};
 
 struct ProcMacroDerive {
@@ -409,5 +408,7 @@ fn mk_decls(
         i
     });
 
-    cx.monotonic_expander().flat_map_item(module).pop().unwrap()
+    // Integrate the new module into existing module structures.
+    let module = AstFragment::Items(smallvec![module]);
+    cx.monotonic_expander().fully_expand_fragment(module).make_items().pop().unwrap()
 }

src/libsyntax_ext/test_harness.rs

@@ -6,7 +6,7 @@ use syntax::ast::{self, Ident};
 use syntax::attr;
 use syntax::entry::{self, EntryPointType};
 use syntax::ext::base::{ExtCtxt, Resolver};
-use syntax::ext::expand::ExpansionConfig;
+use syntax::ext::expand::{AstFragment, ExpansionConfig};
 use syntax::ext::hygiene::{ExpnId, MacroKind};
 use syntax::feature_gate::Features;
 use syntax::mut_visit::{*, ExpectOne};
@@ -74,12 +74,7 @@ impl<'a> MutVisitor for TestHarnessGenerator<'a> {
         noop_visit_crate(c, self);
 
         // Create a main function to run our tests
-        let test_main = {
-            let unresolved = mk_main(&mut self.cx);
-            self.cx.ext_cx.monotonic_expander().flat_map_item(unresolved).pop().unwrap()
-        };
-
-        c.module.items.push(test_main);
+        c.module.items.push(mk_main(&mut self.cx));
     }
 
     fn flat_map_item(&mut self, i: P<ast::Item>) -> SmallVec<[P<ast::Item>; 1]> {
@@ -216,17 +211,22 @@ fn mk_reexport_mod(cx: &mut TestCtxt<'_>,
     let name = Ident::from_str("__test_reexports").gensym();
     let parent = if parent == ast::DUMMY_NODE_ID { ast::CRATE_NODE_ID } else { parent };
     cx.ext_cx.current_expansion.id = cx.ext_cx.resolver.get_module_scope(parent);
-    let it = cx.ext_cx.monotonic_expander().flat_map_item(P(ast::Item {
+    let module = P(ast::Item {
         ident: name,
         attrs: Vec::new(),
         id: ast::DUMMY_NODE_ID,
         node: ast::ItemKind::Mod(reexport_mod),
         vis: dummy_spanned(ast::VisibilityKind::Public),
         span: DUMMY_SP,
         tokens: None,
-    })).pop().unwrap();
+    });
 
-    (it, name)
+    // Integrate the new module into existing module structures.
+    let module = AstFragment::Items(smallvec![module]);
+    let module =
+        cx.ext_cx.monotonic_expander().fully_expand_fragment(module).make_items().pop().unwrap();
+
+    (module, name)
 }
 
 /// Crawl over the crate, inserting test reexports and the test main function
@@ -321,16 +321,19 @@ fn mk_main(cx: &mut TestCtxt<'_>) -> P<ast::Item> {
         None => Ident::from_str_and_span("main", sp).gensym(),
     };
 
-    P(ast::Item {
+    let main = P(ast::Item {
         ident: main_id,
         attrs: vec![main_attr],
         id: ast::DUMMY_NODE_ID,
         node: main,
         vis: dummy_spanned(ast::VisibilityKind::Public),
         span: sp,
         tokens: None,
-    })
+    });
 
+    // Integrate the new item into existing module structures.
+    let main = AstFragment::Items(smallvec![main]);
+    cx.ext_cx.monotonic_expander().fully_expand_fragment(main).make_items().pop().unwrap()
 }
 
 fn path_name_i(idents: &[Ident]) -> String {