Skip to content

Commit 717f5df

Browse files
borsbors
committedNov 21, 2024
Auto merge of #132629 - nnethercote:124141-preliminaries, r=petrochenkov
#124141 preliminaries Preliminary changes required to start removing `Nonterminal` (#124141). r? `@petrochenkov`
·
1.86.01.84.0
2 parents 0b1bf71 + 03159d4 commit 717f5df

File tree

16 files changed

+307
-69
lines changed

16 files changed

+307
-69
lines changed
 

‎compiler/rustc_ast/src/attr/mod.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -457,7 +457,7 @@ impl MetaItemKind {
457457
tokens: &mut impl Iterator<Item = &'a TokenTree>,
458458
) -> Option<MetaItemKind> {
459459
match tokens.next() {
460-
Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {
460+
Some(TokenTree::Delimited(.., Delimiter::Invisible(_), inner_tokens)) => {
461461
MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
462462
}
463463
Some(TokenTree::Token(token, _)) => {
@@ -605,7 +605,7 @@ impl MetaItemInner {
605605
tokens.next();
606606
return Some(MetaItemInner::Lit(lit));
607607
}
608-
Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {
608+
Some(TokenTree::Delimited(.., Delimiter::Invisible(_), inner_tokens)) => {
609609
tokens.next();
610610
return MetaItemInner::from_tokens(&mut inner_tokens.trees().peekable());
611611
}

‎compiler/rustc_ast/src/token.rs

+144-8
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,86 @@ pub enum BinOpToken {
4242
Shr,
4343
}
4444

45+
// This type must not implement `Hash` due to the unusual `PartialEq` impl below.
46+
#[derive(Copy, Clone, Debug, Encodable, Decodable, HashStable_Generic)]
47+
pub enum InvisibleOrigin {
48+
// From the expansion of a metavariable in a declarative macro.
49+
MetaVar(MetaVarKind),
50+
51+
// Converted from `proc_macro::Delimiter` in
52+
// `proc_macro::Delimiter::to_internal`, i.e. returned by a proc macro.
53+
ProcMacro,
54+
55+
// Converted from `TokenKind::Interpolated` in
56+
// `TokenStream::flatten_token`. Treated similarly to `ProcMacro`.
57+
FlattenToken,
58+
}
59+
60+
impl PartialEq for InvisibleOrigin {
61+
#[inline]
62+
fn eq(&self, _other: &InvisibleOrigin) -> bool {
63+
// When we had AST-based nonterminals we couldn't compare them, and the
64+
// old `Nonterminal` type had an `eq` that always returned false,
65+
// resulting in this restriction:
66+
// https://doc.rust-lang.org/nightly/reference/macros-by-example.html#forwarding-a-matched-fragment
67+
// This `eq` emulates that behaviour. We could consider lifting this
68+
// restriction now but there are still cases involving invisible
69+
// delimiters that make it harder than it first appears.
70+
false
71+
}
72+
}
73+
74+
/// Annoyingly similar to `NonterminalKind`, but the slight differences are important.
75+
#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
76+
pub enum MetaVarKind {
77+
Item,
78+
Block,
79+
Stmt,
80+
Pat(NtPatKind),
81+
Expr {
82+
kind: NtExprKind,
83+
// This field is needed for `Token::can_begin_literal_maybe_minus`.
84+
can_begin_literal_maybe_minus: bool,
85+
// This field is needed for `Token::can_begin_string_literal`.
86+
can_begin_string_literal: bool,
87+
},
88+
Ty,
89+
Ident,
90+
Lifetime,
91+
Literal,
92+
Meta,
93+
Path,
94+
Vis,
95+
TT,
96+
}
97+
98+
impl fmt::Display for MetaVarKind {
99+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
100+
let sym = match self {
101+
MetaVarKind::Item => sym::item,
102+
MetaVarKind::Block => sym::block,
103+
MetaVarKind::Stmt => sym::stmt,
104+
MetaVarKind::Pat(PatParam { inferred: true } | PatWithOr) => sym::pat,
105+
MetaVarKind::Pat(PatParam { inferred: false }) => sym::pat_param,
106+
MetaVarKind::Expr { kind: Expr2021 { inferred: true } | Expr, .. } => sym::expr,
107+
MetaVarKind::Expr { kind: Expr2021 { inferred: false }, .. } => sym::expr_2021,
108+
MetaVarKind::Ty => sym::ty,
109+
MetaVarKind::Ident => sym::ident,
110+
MetaVarKind::Lifetime => sym::lifetime,
111+
MetaVarKind::Literal => sym::literal,
112+
MetaVarKind::Meta => sym::meta,
113+
MetaVarKind::Path => sym::path,
114+
MetaVarKind::Vis => sym::vis,
115+
MetaVarKind::TT => sym::tt,
116+
};
117+
write!(f, "{sym}")
118+
}
119+
}
120+
45121
/// Describes how a sequence of token trees is delimited.
46122
/// Cannot use `proc_macro::Delimiter` directly because this
47123
/// structure should implement some additional traits.
48-
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
49-
#[derive(Encodable, Decodable, Hash, HashStable_Generic)]
124+
#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
50125
pub enum Delimiter {
51126
/// `( ... )`
52127
Parenthesis,
@@ -59,7 +134,34 @@ pub enum Delimiter {
59134
/// "macro variable" `$var`. It is important to preserve operator priorities in cases like
60135
/// `$var * 3` where `$var` is `1 + 2`.
61136
/// Invisible delimiters might not survive roundtrip of a token stream through a string.
62-
Invisible,
137+
Invisible(InvisibleOrigin),
138+
}
139+
140+
impl Delimiter {
141+
// Should the parser skip these delimiters? Only happens for certain kinds
142+
// of invisible delimiters. Ideally this function will eventually disappear
143+
// and no invisible delimiters will be skipped.
144+
#[inline]
145+
pub fn skip(&self) -> bool {
146+
match self {
147+
Delimiter::Parenthesis | Delimiter::Bracket | Delimiter::Brace => false,
148+
Delimiter::Invisible(InvisibleOrigin::MetaVar(_)) => false,
149+
Delimiter::Invisible(InvisibleOrigin::FlattenToken | InvisibleOrigin::ProcMacro) => {
150+
true
151+
}
152+
}
153+
}
154+
155+
// This exists because `InvisibleOrigin`s should be compared. It is only used for assertions.
156+
pub fn eq_ignoring_invisible_origin(&self, other: &Delimiter) -> bool {
157+
match (self, other) {
158+
(Delimiter::Parenthesis, Delimiter::Parenthesis) => true,
159+
(Delimiter::Brace, Delimiter::Brace) => true,
160+
(Delimiter::Bracket, Delimiter::Bracket) => true,
161+
(Delimiter::Invisible(_), Delimiter::Invisible(_)) => true,
162+
_ => false,
163+
}
164+
}
63165
}
64166

65167
// Note that the suffix is *not* considered when deciding the `LitKind` in this
@@ -496,10 +598,11 @@ impl Token {
496598
/// **NB**: Take care when modifying this function, since it will change
497599
/// the stable set of tokens that are allowed to match an expr nonterminal.
498600
pub fn can_begin_expr(&self) -> bool {
601+
use Delimiter::*;
499602
match self.uninterpolate().kind {
500603
Ident(name, is_raw) =>
501604
ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
502-
OpenDelim(..) | // tuple, array or block
605+
OpenDelim(Parenthesis | Brace | Bracket) | // tuple, array or block
503606
Literal(..) | // literal
504607
Not | // operator not
505608
BinOp(Minus) | // unary minus
@@ -510,7 +613,7 @@ impl Token {
510613
// DotDotDot is no longer supported, but we need some way to display the error
511614
DotDot | DotDotDot | DotDotEq | // range notation
512615
Lt | BinOp(Shl) | // associated path
513-
PathSep | // global path
616+
PathSep | // global path
514617
Lifetime(..) | // labeled loop
515618
Pound => true, // expression attributes
516619
Interpolated(ref nt) =>
@@ -520,6 +623,12 @@ impl Token {
520623
NtLiteral(..) |
521624
NtPath(..)
522625
),
626+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
627+
MetaVarKind::Block |
628+
MetaVarKind::Expr { .. } |
629+
MetaVarKind::Literal |
630+
MetaVarKind::Path
631+
))) => true,
523632
_ => false,
524633
}
525634
}
@@ -553,6 +662,14 @@ impl Token {
553662
| NtPath(..)
554663
| NtTy(..)
555664
),
665+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
666+
MetaVarKind::Expr { .. } |
667+
MetaVarKind::Literal |
668+
MetaVarKind::Meta |
669+
MetaVarKind::Pat(_) |
670+
MetaVarKind::Path |
671+
MetaVarKind::Ty
672+
))) => true,
556673
_ => false,
557674
}
558675
}
@@ -573,6 +690,10 @@ impl Token {
573690
Lt | BinOp(Shl) | // associated path
574691
PathSep => true, // global path
575692
Interpolated(ref nt) => matches!(&**nt, NtTy(..) | NtPath(..)),
693+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
694+
MetaVarKind::Ty |
695+
MetaVarKind::Path
696+
))) => true,
576697
// For anonymous structs or unions, which only appear in specific positions
577698
// (type of struct fields or union fields), we don't consider them as regular types
578699
_ => false,
@@ -585,6 +706,9 @@ impl Token {
585706
OpenDelim(Delimiter::Brace) | Literal(..) | BinOp(Minus) => true,
586707
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
587708
Interpolated(ref nt) => matches!(&**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
709+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
710+
MetaVarKind::Expr { .. } | MetaVarKind::Block | MetaVarKind::Literal,
711+
))) => true,
588712
_ => false,
589713
}
590714
}
@@ -641,6 +765,13 @@ impl Token {
641765
},
642766
_ => false,
643767
},
768+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind {
769+
MetaVarKind::Literal => true,
770+
MetaVarKind::Expr { can_begin_literal_maybe_minus, .. } => {
771+
can_begin_literal_maybe_minus
772+
}
773+
_ => false,
774+
},
644775
_ => false,
645776
}
646777
}
@@ -656,6 +787,11 @@ impl Token {
656787
},
657788
_ => false,
658789
},
790+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind {
791+
MetaVarKind::Literal => true,
792+
MetaVarKind::Expr { can_begin_string_literal, .. } => can_begin_string_literal,
793+
_ => false,
794+
},
659795
_ => false,
660796
}
661797
}
@@ -896,7 +1032,7 @@ impl PartialEq<TokenKind> for Token {
8961032
}
8971033
}
8981034

899-
#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable)]
1035+
#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
9001036
pub enum NtPatKind {
9011037
// Matches or-patterns. Was written using `pat` in edition 2021 or later.
9021038
PatWithOr,
@@ -906,7 +1042,7 @@ pub enum NtPatKind {
9061042
PatParam { inferred: bool },
9071043
}
9081044

909-
#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable)]
1045+
#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
9101046
pub enum NtExprKind {
9111047
// Matches expressions using the post-edition 2024. Was written using
9121048
// `expr` in edition 2024 or later.
@@ -933,7 +1069,7 @@ pub enum Nonterminal {
9331069
NtVis(P<ast::Visibility>),
9341070
}
9351071

936-
#[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable)]
1072+
#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
9371073
pub enum NonterminalKind {
9381074
Item,
9391075
Block,

‎compiler/rustc_ast/src/tokenstream.rs

+3-3
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ use rustc_span::{DUMMY_SP, Span, SpanDecoder, SpanEncoder, Symbol, sym};
2424

2525
use crate::ast::{AttrStyle, StmtKind};
2626
use crate::ast_traits::{HasAttrs, HasTokens};
27-
use crate::token::{self, Delimiter, Nonterminal, Token, TokenKind};
27+
use crate::token::{self, Delimiter, InvisibleOrigin, Nonterminal, Token, TokenKind};
2828
use crate::{AttrVec, Attribute};
2929

3030
/// Part of a `TokenStream`.
@@ -484,13 +484,13 @@ impl TokenStream {
484484
token::NtLifetime(ident, is_raw) => TokenTree::Delimited(
485485
DelimSpan::from_single(token.span),
486486
DelimSpacing::new(Spacing::JointHidden, spacing),
487-
Delimiter::Invisible,
487+
Delimiter::Invisible(InvisibleOrigin::FlattenToken),
488488
TokenStream::token_alone(token::Lifetime(ident.name, is_raw), ident.span),
489489
),
490490
token::Interpolated(ref nt) => TokenTree::Delimited(
491491
DelimSpan::from_single(token.span),
492492
DelimSpacing::new(Spacing::JointHidden, spacing),
493-
Delimiter::Invisible,
493+
Delimiter::Invisible(InvisibleOrigin::FlattenToken),
494494
TokenStream::from_nonterminal_ast(&nt).flattened(),
495495
),
496496
_ => TokenTree::Token(token.clone(), spacing),

‎compiler/rustc_ast_pretty/src/pprust/state.rs

+2-3
Original file line numberDiff line numberDiff line change
@@ -942,9 +942,8 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
942942
token::CloseDelim(Delimiter::Bracket) => "]".into(),
943943
token::OpenDelim(Delimiter::Brace) => "{".into(),
944944
token::CloseDelim(Delimiter::Brace) => "}".into(),
945-
token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible) => {
946-
"".into()
947-
}
945+
token::OpenDelim(Delimiter::Invisible(_))
946+
| token::CloseDelim(Delimiter::Invisible(_)) => "".into(),
948947
token::Pound => "#".into(),
949948
token::Dollar => "$".into(),
950949
token::Question => "?".into(),

‎compiler/rustc_expand/src/mbe/diagnostics.rs

+4-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use std::borrow::Cow;
22

3-
use rustc_ast::token::{self, Token, TokenKind};
3+
use rustc_ast::token::{self, Delimiter, Token, TokenKind};
44
use rustc_ast::tokenstream::TokenStream;
55
use rustc_errors::{Applicability, Diag, DiagCtxtHandle, DiagMessage};
66
use rustc_macros::Subdiagnostic;
@@ -68,7 +68,9 @@ pub(super) fn failed_to_match_macro(
6868

6969
if let MatcherLoc::Token { token: expected_token } = &remaining_matcher
7070
&& (matches!(expected_token.kind, TokenKind::Interpolated(_))
71-
|| matches!(token.kind, TokenKind::Interpolated(_)))
71+
|| matches!(token.kind, TokenKind::Interpolated(_))
72+
|| matches!(expected_token.kind, TokenKind::OpenDelim(Delimiter::Invisible(_)))
73+
|| matches!(token.kind, TokenKind::OpenDelim(Delimiter::Invisible(_))))
7274
{
7375
err.note("captured metavariables except for `:tt`, `:ident` and `:lifetime` cannot be compared to other tokens");
7476
err.note("see <https://doc.rust-lang.org/nightly/reference/macros-by-example.html#forwarding-a-matched-fragment> for more information");

‎compiler/rustc_expand/src/mbe/macro_rules.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -693,7 +693,7 @@ fn has_compile_error_macro(rhs: &mbe::TokenTree) -> bool {
693693
&& let mbe::TokenTree::Token(bang) = bang
694694
&& let TokenKind::Not = bang.kind
695695
&& let mbe::TokenTree::Delimited(.., del) = args
696-
&& del.delim != Delimiter::Invisible
696+
&& !del.delim.skip()
697697
{
698698
true
699699
} else {

‎compiler/rustc_expand/src/mbe/quoted.rs

+6-5
Original file line numberDiff line numberDiff line change
@@ -165,11 +165,12 @@ fn parse_tree<'a>(
165165
// during parsing.
166166
let mut next = outer_trees.next();
167167
let mut trees: Box<dyn Iterator<Item = &tokenstream::TokenTree>>;
168-
if let Some(tokenstream::TokenTree::Delimited(.., Delimiter::Invisible, tts)) = next {
169-
trees = Box::new(tts.trees());
170-
next = trees.next();
171-
} else {
172-
trees = Box::new(outer_trees);
168+
match next {
169+
Some(tokenstream::TokenTree::Delimited(.., delim, tts)) if delim.skip() => {
170+
trees = Box::new(tts.trees());
171+
next = trees.next();
172+
}
173+
_ => trees = Box::new(outer_trees),
173174
}
174175

175176
match next {

‎compiler/rustc_expand/src/proc_macro_server.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ impl FromInternal<token::Delimiter> for Delimiter {
3838
token::Delimiter::Parenthesis => Delimiter::Parenthesis,
3939
token::Delimiter::Brace => Delimiter::Brace,
4040
token::Delimiter::Bracket => Delimiter::Bracket,
41-
token::Delimiter::Invisible => Delimiter::None,
41+
token::Delimiter::Invisible(_) => Delimiter::None,
4242
}
4343
}
4444
}
@@ -49,7 +49,7 @@ impl ToInternal<token::Delimiter> for Delimiter {
4949
Delimiter::Parenthesis => token::Delimiter::Parenthesis,
5050
Delimiter::Brace => token::Delimiter::Brace,
5151
Delimiter::Bracket => token::Delimiter::Bracket,
52-
Delimiter::None => token::Delimiter::Invisible,
52+
Delimiter::None => token::Delimiter::Invisible(token::InvisibleOrigin::ProcMacro),
5353
}
5454
}
5555
}

‎compiler/rustc_parse/messages.ftl

+7
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,9 @@ parse_expected_identifier_found_doc_comment = expected identifier, found doc com
216216
parse_expected_identifier_found_doc_comment_str = expected identifier, found doc comment `{$token}`
217217
parse_expected_identifier_found_keyword = expected identifier, found keyword
218218
parse_expected_identifier_found_keyword_str = expected identifier, found keyword `{$token}`
219+
parse_expected_identifier_found_metavar = expected identifier, found metavariable
220+
# This one deliberately doesn't print a token.
221+
parse_expected_identifier_found_metavar_str = expected identifier, found metavariable
219222
parse_expected_identifier_found_reserved_identifier = expected identifier, found reserved identifier
220223
parse_expected_identifier_found_reserved_identifier_str = expected identifier, found reserved identifier `{$token}`
221224
parse_expected_identifier_found_reserved_keyword = expected identifier, found reserved keyword
@@ -227,6 +230,8 @@ parse_expected_mut_or_const_in_raw_pointer_type = expected `mut` or `const` keyw
227230
228231
parse_expected_semi_found_doc_comment_str = expected `;`, found doc comment `{$token}`
229232
parse_expected_semi_found_keyword_str = expected `;`, found keyword `{$token}`
233+
# This one deliberately doesn't print a token.
234+
parse_expected_semi_found_metavar_str = expected `;`, found metavariable
230235
parse_expected_semi_found_reserved_identifier_str = expected `;`, found reserved identifier `{$token}`
231236
parse_expected_semi_found_reserved_keyword_str = expected `;`, found reserved keyword `{$token}`
232237
parse_expected_semi_found_str = expected `;`, found `{$token}`
@@ -864,6 +869,8 @@ parse_unexpected_token_after_not_logical = use `!` to perform logical negation
864869
parse_unexpected_token_after_struct_name = expected `where`, `{"{"}`, `(`, or `;` after struct name
865870
parse_unexpected_token_after_struct_name_found_doc_comment = expected `where`, `{"{"}`, `(`, or `;` after struct name, found doc comment `{$token}`
866871
parse_unexpected_token_after_struct_name_found_keyword = expected `where`, `{"{"}`, `(`, or `;` after struct name, found keyword `{$token}`
872+
# This one deliberately doesn't print a token.
873+
parse_unexpected_token_after_struct_name_found_metavar = expected `where`, `{"{"}`, `(`, or `;` after struct name, found metavar
867874
parse_unexpected_token_after_struct_name_found_other = expected `where`, `{"{"}`, `(`, or `;` after struct name, found `{$token}`
868875
869876
parse_unexpected_token_after_struct_name_found_reserved_identifier = expected `where`, `{"{"}`, `(`, or `;` after struct name, found reserved identifier `{$token}`

0 commit comments

Comments
 (0)
Please sign in to comment.