Rollup merge of rust-lang#58476 - nnethercote:rm-LazyTokenStream, r=petrochenkov

Remove `LazyTokenStream`.

`LazyTokenStream` was added in rust-lang#40939. Perhaps it was an effective optimization then, but no longer. This PR removes it, making the code both simpler and faster.

r? @alexcrichton

Author: Centril

src/librustc/hir/lowering.rs

@@ -1124,19 +1124,19 @@ impl<'a> LoweringContext<'a> {
             TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
                 span,
                 delim,
-                self.lower_token_stream(tts.into()).into(),
+                self.lower_token_stream(tts),
             ).into(),
         }
     }
 
     fn lower_token(&mut self, token: Token, span: Span) -> TokenStream {
         match token {
-            Token::Interpolated(_) => {}
-            other => return TokenTree::Token(span, other).into(),
+            Token::Interpolated(nt) => {
+                let tts = nt.to_tokenstream(&self.sess.parse_sess, span);
+                self.lower_token_stream(tts)
+            }
+            other => TokenTree::Token(span, other).into(),
         }
-
-        let tts = token.interpolated_to_tokenstream(&self.sess.parse_sess, span);
-        self.lower_token_stream(tts)
     }
 
     fn lower_arm(&mut self, arm: &Arm) -> hir::Arm {

src/librustc/hir/map/def_collector.rs

@@ -339,7 +339,7 @@ impl<'a> visit::Visitor<'a> for DefCollector<'a> {
 
     fn visit_token(&mut self, t: Token) {
         if let Token::Interpolated(nt) = t {
-            if let token::NtExpr(ref expr) = nt.0 {
+            if let token::NtExpr(ref expr) = *nt {
                 if let ExprKind::Mac(..) = expr.node {
                     self.visit_macro_invoc(expr.id);
                 }

src/librustc_resolve/build_reduced_graph.rs

@@ -1025,7 +1025,7 @@ impl<'a, 'b> Visitor<'a> for BuildReducedGraphVisitor<'a, 'b> {
 
     fn visit_token(&mut self, t: Token) {
         if let Token::Interpolated(nt) = t {
-            if let token::NtExpr(ref expr) = nt.0 {
+            if let token::NtExpr(ref expr) = *nt {
                 if let ast::ExprKind::Mac(..) = expr.node {
                     self.visit_invoc(expr.id);
                 }

src/libsyntax/attr/mod.rs

@@ -517,7 +517,7 @@ impl MetaItem {
                 let span = span.with_hi(segments.last().unwrap().ident.span.hi());
                 Path { span, segments }
             }
-            Some(TokenTree::Token(_, Token::Interpolated(ref nt))) => match nt.0 {
+            Some(TokenTree::Token(_, Token::Interpolated(nt))) => match *nt {
                 token::Nonterminal::NtIdent(ident, _) => Path::from_ident(ident),
                 token::Nonterminal::NtMeta(ref meta) => return Some(meta.clone()),
                 token::Nonterminal::NtPath(ref path) => path.clone(),
@@ -682,7 +682,7 @@ impl LitKind {
         match token {
             Token::Ident(ident, false) if ident.name == "true" => Some(LitKind::Bool(true)),
             Token::Ident(ident, false) if ident.name == "false" => Some(LitKind::Bool(false)),
-            Token::Interpolated(ref nt) => match nt.0 {
+            Token::Interpolated(nt) => match *nt {
                 token::NtExpr(ref v) | token::NtLiteral(ref v) => match v.node {
                     ExprKind::Lit(ref lit) => Some(lit.node.clone()),
                     _ => None,

src/libsyntax/ext/base.rs

@@ -266,7 +266,7 @@ impl<F> TTMacroExpander for F
         impl MutVisitor for AvoidInterpolatedIdents {
             fn visit_tt(&mut self, tt: &mut tokenstream::TokenTree) {
                 if let tokenstream::TokenTree::Token(_, token::Interpolated(nt)) = tt {
-                    if let token::NtIdent(ident, is_raw) = nt.0 {
+                    if let token::NtIdent(ident, is_raw) = **nt {
                         *tt = tokenstream::TokenTree::Token(ident.span,
                                                             token::Ident(ident, is_raw));
                     }

src/libsyntax/ext/expand.rs

@@ -25,6 +25,7 @@ use syntax_pos::{Span, DUMMY_SP, FileName};
 use syntax_pos::hygiene::ExpnFormat;
 
 use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::sync::Lrc;
 use std::fs;
 use std::io::ErrorKind;
 use std::{iter, mem};
@@ -584,14 +585,14 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             }
             AttrProcMacro(ref mac, ..) => {
                 self.gate_proc_macro_attr_item(attr.span, &item);
-                let item_tok = TokenTree::Token(DUMMY_SP, Token::interpolated(match item {
+                let item_tok = TokenTree::Token(DUMMY_SP, Token::Interpolated(Lrc::new(match item {
                     Annotatable::Item(item) => token::NtItem(item),
                     Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()),
                     Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()),
                     Annotatable::ForeignItem(item) => token::NtForeignItem(item.into_inner()),
                     Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()),
                     Annotatable::Expr(expr) => token::NtExpr(expr),
-                })).into();
+                }))).into();
                 let input = self.extract_proc_macro_attr_input(attr.tokens, attr.span);
                 let tok_result = mac.expand(self.cx, attr.span, input, item_tok);
                 let res = self.parse_ast_fragment(tok_result, invoc.fragment_kind,

src/libsyntax/ext/tt/macro_parser.rs

@@ -88,6 +88,7 @@ use smallvec::{smallvec, SmallVec};
 use syntax_pos::Span;
 
 use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::sync::Lrc;
 use std::collections::hash_map::Entry::{Occupied, Vacant};
 use std::mem;
 use std::ops::{Deref, DerefMut};
@@ -179,7 +180,7 @@ struct MatcherPos<'root, 'tt: 'root> {
     /// all bound matches from the submatcher into the shared top-level `matches` vector. If `sep`
     /// and `up` are `Some`, then `matches` is _not_ the shared top-level list. Instead, if one
     /// wants the shared `matches`, one should use `up.matches`.
-    matches: Box<[Rc<NamedMatchVec>]>,
+    matches: Box<[Lrc<NamedMatchVec>]>,
     /// The position in `matches` corresponding to the first metavar in this matcher's sequence of
     /// token trees. In other words, the first metavar in the first token of `top_elts` corresponds
     /// to `matches[match_lo]`.
@@ -218,7 +219,7 @@ struct MatcherPos<'root, 'tt: 'root> {
 impl<'root, 'tt> MatcherPos<'root, 'tt> {
     /// Adds `m` as a named match for the `idx`-th metavar.
     fn push_match(&mut self, idx: usize, m: NamedMatch) {
-        let matches = Rc::make_mut(&mut self.matches[idx]);
+        let matches = Lrc::make_mut(&mut self.matches[idx]);
         matches.push(m);
     }
 }
@@ -295,11 +296,11 @@ pub fn count_names(ms: &[TokenTree]) -> usize {
 }
 
 /// `len` `Vec`s (initially shared and empty) that will store matches of metavars.
-fn create_matches(len: usize) -> Box<[Rc<NamedMatchVec>]> {
+fn create_matches(len: usize) -> Box<[Lrc<NamedMatchVec>]> {
     if len == 0 {
         vec![]
     } else {
-        let empty_matches = Rc::new(SmallVec::new());
+        let empty_matches = Lrc::new(SmallVec::new());
         vec![empty_matches; len]
     }.into_boxed_slice()
 }
@@ -353,8 +354,8 @@ fn initial_matcher_pos<'root, 'tt>(ms: &'tt [TokenTree], open: Span) -> MatcherP
 /// token tree it was derived from.
 #[derive(Debug, Clone)]
 pub enum NamedMatch {
-    MatchedSeq(Rc<NamedMatchVec>, DelimSpan),
-    MatchedNonterminal(Rc<Nonterminal>),
+    MatchedSeq(Lrc<NamedMatchVec>, DelimSpan),
+    MatchedNonterminal(Lrc<Nonterminal>),
 }
 
 /// Takes a sequence of token trees `ms` representing a matcher which successfully matched input
@@ -561,7 +562,7 @@ fn inner_parse_loop<'root, 'tt>(
                         new_item.match_cur += seq.num_captures;
                         new_item.idx += 1;
                         for idx in item.match_cur..item.match_cur + seq.num_captures {
-                            new_item.push_match(idx, MatchedSeq(Rc::new(smallvec![]), sp));
+                            new_item.push_match(idx, MatchedSeq(Lrc::new(smallvec![]), sp));
                         }
                         cur_items.push(new_item);
                     }
@@ -707,7 +708,7 @@ pub fn parse(
                 let matches = eof_items[0]
                     .matches
                     .iter_mut()
-                    .map(|dv| Rc::make_mut(dv).pop().unwrap());
+                    .map(|dv| Lrc::make_mut(dv).pop().unwrap());
                 return nameize(sess, ms, matches);
             } else if eof_items.len() > 1 {
                 return Error(
@@ -780,7 +781,7 @@ pub fn parse(
                 let match_cur = item.match_cur;
                 item.push_match(
                     match_cur,
-                    MatchedNonterminal(Rc::new(parse_nt(&mut parser, span, &ident.as_str()))),
+                    MatchedNonterminal(Lrc::new(parse_nt(&mut parser, span, &ident.as_str()))),
                 );
                 item.idx += 1;
                 item.match_cur += 1;
@@ -829,7 +830,7 @@ fn may_begin_with(name: &str, token: &Token) -> bool {
         },
         "block" => match *token {
             Token::OpenDelim(token::Brace) => true,
-            Token::Interpolated(ref nt) => match nt.0 {
+            Token::Interpolated(ref nt) => match **nt {
                 token::NtItem(_)
                 | token::NtPat(_)
                 | token::NtTy(_)
@@ -843,9 +844,9 @@ fn may_begin_with(name: &str, token: &Token) -> bool {
         },
         "path" | "meta" => match *token {
             Token::ModSep | Token::Ident(..) => true,
-            Token::Interpolated(ref nt) => match nt.0 {
+            Token::Interpolated(ref nt) => match **nt {
                 token::NtPath(_) | token::NtMeta(_) => true,
-                _ => may_be_ident(&nt.0),
+                _ => may_be_ident(&nt),
             },
             _ => false,
         },
@@ -862,12 +863,12 @@ fn may_begin_with(name: &str, token: &Token) -> bool {
             Token::ModSep |                     // path
             Token::Lt |                         // path (UFCS constant)
             Token::BinOp(token::Shl) => true,   // path (double UFCS)
-            Token::Interpolated(ref nt) => may_be_ident(&nt.0),
+            Token::Interpolated(ref nt) => may_be_ident(nt),
             _ => false,
         },
         "lifetime" => match *token {
             Token::Lifetime(_) => true,
-            Token::Interpolated(ref nt) => match nt.0 {
+            Token::Interpolated(ref nt) => match **nt {
                 token::NtLifetime(_) | token::NtTT(_) => true,
                 _ => false,
             },

src/libsyntax/ext/tt/transcribe.rs

@@ -149,7 +149,7 @@ pub fn transcribe(cx: &ExtCtxt<'_>,
                             result.push(tt.clone().into());
                         } else {
                             sp = sp.apply_mark(cx.current_expansion.mark);
-                            let token = TokenTree::Token(sp, Token::interpolated((**nt).clone()));
+                            let token = TokenTree::Token(sp, Token::Interpolated(nt.clone()));
                             result.push(token.into());
                         }
                     } else {

src/libsyntax/mut_visit.rs

@@ -581,9 +581,8 @@ pub fn noop_visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
         token::Ident(id, _is_raw) => vis.visit_ident(id),
         token::Lifetime(id) => vis.visit_ident(id),
         token::Interpolated(nt) => {
-            let nt = Lrc::make_mut(nt);
-            vis.visit_interpolated(&mut nt.0);
-            nt.1 = token::LazyTokenStream::new();
+            let mut nt = Lrc::make_mut(nt);
+            vis.visit_interpolated(&mut nt);
         }
         _ => {}
     }

src/libsyntax/parse/attr.rs

@@ -141,7 +141,7 @@ impl<'a> Parser<'a> {
     /// The delimiters or `=` are still put into the resulting token stream.
     crate fn parse_meta_item_unrestricted(&mut self) -> PResult<'a, (ast::Path, TokenStream)> {
         let meta = match self.token {
-            token::Interpolated(ref nt) => match nt.0 {
+            token::Interpolated(ref nt) => match **nt {
                 Nonterminal::NtMeta(ref meta) => Some(meta.clone()),
                 _ => None,
             },
@@ -227,7 +227,7 @@ impl<'a> Parser<'a> {
     /// meta_item_inner : (meta_item | UNSUFFIXED_LIT) (',' meta_item_inner)? ;
     pub fn parse_meta_item(&mut self) -> PResult<'a, ast::MetaItem> {
         let nt_meta = match self.token {
-            token::Interpolated(ref nt) => match nt.0 {
+            token::Interpolated(ref nt) => match **nt {
                 token::NtMeta(ref e) => Some(e.clone()),
                 _ => None,
             },

src/libsyntax/parse/parser.rs

@@ -119,7 +119,7 @@ enum BlockMode {
 macro_rules! maybe_whole_expr {
     ($p:expr) => {
         if let token::Interpolated(nt) = $p.token.clone() {
-            match nt.0 {
+            match *nt {
                 token::NtExpr(ref e) | token::NtLiteral(ref e) => {
                     $p.bump();
                     return Ok((*e).clone());
@@ -146,7 +146,7 @@ macro_rules! maybe_whole_expr {
 macro_rules! maybe_whole {
     ($p:expr, $constructor:ident, |$x:ident| $e:expr) => {
         if let token::Interpolated(nt) = $p.token.clone() {
-            if let token::$constructor($x) = nt.0.clone() {
+            if let token::$constructor($x) = (*nt).clone() {
                 $p.bump();
                 return Ok($e);
             }
@@ -1570,7 +1570,7 @@ impl<'a> Parser<'a> {
                     Some(body)
                 }
                 token::Interpolated(ref nt) => {
-                    match &nt.0 {
+                    match **nt {
                         token::NtBlock(..) => {
                             *at_end = true;
                             let (inner_attrs, body) = self.parse_inner_attrs_and_block()?;
@@ -1913,7 +1913,7 @@ impl<'a> Parser<'a> {
 
     fn is_named_argument(&mut self) -> bool {
         let offset = match self.token {
-            token::Interpolated(ref nt) => match nt.0 {
+            token::Interpolated(ref nt) => match **nt {
                 token::NtPat(..) => return self.look_ahead(1, |t| t == &token::Colon),
                 _ => 0,
             }
@@ -2099,7 +2099,7 @@ impl<'a> Parser<'a> {
     /// Matches `token_lit = LIT_INTEGER | ...`.
     fn parse_lit_token(&mut self) -> PResult<'a, LitKind> {
         let out = match self.token {
-            token::Interpolated(ref nt) => match nt.0 {
+            token::Interpolated(ref nt) => match **nt {
                 token::NtExpr(ref v) | token::NtLiteral(ref v) => match v.node {
                     ExprKind::Lit(ref lit) => { lit.node.clone() }
                     _ => { return self.unexpected_last(&self.token); }
@@ -2299,7 +2299,7 @@ impl<'a> Parser<'a> {
     /// attributes.
     pub fn parse_path_allowing_meta(&mut self, style: PathStyle) -> PResult<'a, ast::Path> {
         let meta_ident = match self.token {
-            token::Interpolated(ref nt) => match nt.0 {
+            token::Interpolated(ref nt) => match **nt {
                 token::NtMeta(ref meta) => match meta.node {
                     ast::MetaItemKind::Word => Some(meta.ident.clone()),
                     _ => None,
@@ -3271,7 +3271,7 @@ impl<'a> Parser<'a> {
                 self.meta_var_span = Some(self.span);
                 // Interpolated identifier and lifetime tokens are replaced with usual identifier
                 // and lifetime tokens, so the former are never encountered during normal parsing.
-                match nt.0 {
+                match **nt {
                     token::NtIdent(ident, is_raw) => (token::Ident(ident, is_raw), ident.span),
                     token::NtLifetime(ident) => (token::Lifetime(ident), ident.span),
                     _ => return,
@@ -3403,7 +3403,7 @@ impl<'a> Parser<'a> {
                     // can't continue an expression after an ident
                     token::Ident(ident, is_raw) => token::ident_can_begin_expr(ident, is_raw),
                     token::Literal(..) | token::Pound => true,
-                    token::Interpolated(ref nt) => match nt.0 {
+                    token::Interpolated(ref nt) => match **nt {
                         token::NtIdent(..) | token::NtExpr(..) |
                         token::NtBlock(..) | token::NtPath(..) => true,
                         _ => false,