Skip to content

Commit 530a629

Browse files
Aaron1011Aaron1011
committed
Remove pretty-print/reparse hack, and add derive-specific hack
1 parent 2987785 commit 530a629

File tree

8 files changed

+184
-408
lines changed

8 files changed

+184
-408
lines changed

‎compiler/rustc_ast/src/tokenstream.rs

+6
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,12 @@ pub enum TokenTree {
4444
Delimited(DelimSpan, DelimToken, TokenStream),
4545
}
4646

47+
#[derive(Copy, Clone)]
48+
pub enum CanSynthesizeMissingTokens {
49+
Yes,
50+
No,
51+
}
52+
4753
// Ensure all fields of `TokenTree` is `Send` and `Sync`.
4854
#[cfg(parallel_compiler)]
4955
fn _dummy()

‎compiler/rustc_ast_lowering/src/lib.rs

+82-31
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737

3838
use rustc_ast::node_id::NodeMap;
3939
use rustc_ast::token::{self, DelimToken, Nonterminal, Token};
40-
use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree};
40+
use rustc_ast::tokenstream::{CanSynthesizeMissingTokens, DelimSpan, TokenStream, TokenTree};
4141
use rustc_ast::visit::{self, AssocCtxt, Visitor};
4242
use rustc_ast::walk_list;
4343
use rustc_ast::{self as ast, *};
@@ -206,7 +206,8 @@ pub trait ResolverAstLowering {
206206
) -> LocalDefId;
207207
}
208208

209-
type NtToTokenstream = fn(&Nonterminal, &ParseSess, Span) -> TokenStream;
209+
type NtToTokenstream =
210+
fn(&Nonterminal, &ParseSess, Span, CanSynthesizeMissingTokens) -> TokenStream;
210211

211212
/// Context of `impl Trait` in code, which determines whether it is allowed in an HIR subtree,
212213
/// and if so, what meaning it has.
@@ -393,6 +394,47 @@ enum AnonymousLifetimeMode {
393394
PassThrough,
394395
}
395396

397+
struct TokenStreamLowering<'a> {
398+
parse_sess: &'a ParseSess,
399+
synthesize_tokens: CanSynthesizeMissingTokens,
400+
nt_to_tokenstream: NtToTokenstream,
401+
}
402+
403+
impl<'a> TokenStreamLowering<'a> {
404+
fn lower_token_stream(&mut self, tokens: TokenStream) -> TokenStream {
405+
tokens.into_trees().flat_map(|tree| self.lower_token_tree(tree).into_trees()).collect()
406+
}
407+
408+
fn lower_token_tree(&mut self, tree: TokenTree) -> TokenStream {
409+
match tree {
410+
TokenTree::Token(token) => self.lower_token(token),
411+
TokenTree::Delimited(span, delim, tts) => {
412+
TokenTree::Delimited(span, delim, self.lower_token_stream(tts)).into()
413+
}
414+
}
415+
}
416+
417+
fn lower_token(&mut self, token: Token) -> TokenStream {
418+
match token.kind {
419+
token::Interpolated(nt) => {
420+
let tts = (self.nt_to_tokenstream)(
421+
&nt,
422+
self.parse_sess,
423+
token.span,
424+
self.synthesize_tokens,
425+
);
426+
TokenTree::Delimited(
427+
DelimSpan::from_single(token.span),
428+
DelimToken::NoDelim,
429+
self.lower_token_stream(tts),
430+
)
431+
.into()
432+
}
433+
_ => TokenTree::Token(token).into(),
434+
}
435+
}
436+
}
437+
396438
struct ImplTraitTypeIdVisitor<'a> {
397439
ids: &'a mut SmallVec<[NodeId; 1]>,
398440
}
@@ -955,40 +997,49 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
955997
match *args {
956998
MacArgs::Empty => MacArgs::Empty,
957999
MacArgs::Delimited(dspan, delim, ref tokens) => {
958-
MacArgs::Delimited(dspan, delim, self.lower_token_stream(tokens.clone()))
959-
}
960-
MacArgs::Eq(eq_span, ref tokens) => {
961-
MacArgs::Eq(eq_span, self.lower_token_stream(tokens.clone()))
962-
}
963-
}
964-
}
965-
966-
fn lower_token_stream(&mut self, tokens: TokenStream) -> TokenStream {
967-
tokens.into_trees().flat_map(|tree| self.lower_token_tree(tree).into_trees()).collect()
968-
}
969-
970-
fn lower_token_tree(&mut self, tree: TokenTree) -> TokenStream {
971-
match tree {
972-
TokenTree::Token(token) => self.lower_token(token),
973-
TokenTree::Delimited(span, delim, tts) => {
974-
TokenTree::Delimited(span, delim, self.lower_token_stream(tts)).into()
1000+
// This is either a non-key-value attribute, or a `macro_rules!` body.
1001+
// We either not have any nonterminals present (in the case of an attribute),
1002+
// or have tokens available for all nonterminals in the case of a nested
1003+
// `macro_rules`: e.g:
1004+
//
1005+
// ```rust
1006+
// macro_rules! outer {
1007+
// ($e:expr) => {
1008+
// macro_rules! inner {
1009+
// () => { $e }
1010+
// }
1011+
// }
1012+
// }
1013+
// ```
1014+
//
1015+
// In both cases, we don't want to synthesize any tokens
1016+
MacArgs::Delimited(
1017+
dspan,
1018+
delim,
1019+
self.lower_token_stream(tokens.clone(), CanSynthesizeMissingTokens::No),
1020+
)
9751021
}
1022+
// This is an inert key-value attribute - it will never be visible to macros
1023+
// after it gets lowered to HIR. Therefore, we can synthesize tokens with fake
1024+
// spans to handle nonterminals in `#[doc]` (e.g. `#[doc = $e]`).
1025+
MacArgs::Eq(eq_span, ref tokens) => MacArgs::Eq(
1026+
eq_span,
1027+
self.lower_token_stream(tokens.clone(), CanSynthesizeMissingTokens::Yes),
1028+
),
9761029
}
9771030
}
9781031

979-
fn lower_token(&mut self, token: Token) -> TokenStream {
980-
match token.kind {
981-
token::Interpolated(nt) => {
982-
let tts = (self.nt_to_tokenstream)(&nt, &self.sess.parse_sess, token.span);
983-
TokenTree::Delimited(
984-
DelimSpan::from_single(token.span),
985-
DelimToken::NoDelim,
986-
self.lower_token_stream(tts),
987-
)
988-
.into()
989-
}
990-
_ => TokenTree::Token(token).into(),
1032+
fn lower_token_stream(
1033+
&self,
1034+
tokens: TokenStream,
1035+
synthesize_tokens: CanSynthesizeMissingTokens,
1036+
) -> TokenStream {
1037+
TokenStreamLowering {
1038+
parse_sess: &self.sess.parse_sess,
1039+
synthesize_tokens,
1040+
nt_to_tokenstream: self.nt_to_tokenstream,
9911041
}
1042+
.lower_token_stream(tokens)
9921043
}
9931044

9941045
/// Given an associated type constraint like one of these:

‎compiler/rustc_expand/src/base.rs

+9-6
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ use crate::expand::{self, AstFragment, Invocation};
22
use crate::module::DirectoryOwnership;
33

44
use rustc_ast::ptr::P;
5-
use rustc_ast::token;
6-
use rustc_ast::tokenstream::TokenStream;
5+
use rustc_ast::token::{self, Nonterminal};
6+
use rustc_ast::tokenstream::{CanSynthesizeMissingTokens, TokenStream};
77
use rustc_ast::visit::{AssocCtxt, Visitor};
88
use rustc_ast::{self as ast, Attribute, NodeId, PatKind};
99
use rustc_attr::{self as attr, Deprecation, HasAttrs, Stability};
@@ -119,8 +119,8 @@ impl Annotatable {
119119
}
120120
}
121121

122-
crate fn into_tokens(self, sess: &ParseSess) -> TokenStream {
123-
let nt = match self {
122+
crate fn into_nonterminal(self) -> Nonterminal {
123+
match self {
124124
Annotatable::Item(item) => token::NtItem(item),
125125
Annotatable::TraitItem(item) | Annotatable::ImplItem(item) => {
126126
token::NtItem(P(item.and_then(ast::AssocItem::into_item)))
@@ -137,8 +137,11 @@ impl Annotatable {
137137
| Annotatable::Param(..)
138138
| Annotatable::StructField(..)
139139
| Annotatable::Variant(..) => panic!("unexpected annotatable"),
140-
};
141-
nt_to_tokenstream(&nt, sess, DUMMY_SP)
140+
}
141+
}
142+
143+
crate fn into_tokens(self, sess: &ParseSess) -> TokenStream {
144+
nt_to_tokenstream(&self.into_nonterminal(), sess, DUMMY_SP, CanSynthesizeMissingTokens::No)
142145
}
143146

144147
pub fn expect_item(self) -> P<ast::Item> {

‎compiler/rustc_expand/src/config.rs

+11-2
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ use smallvec::SmallVec;
2929
pub struct StripUnconfigured<'a> {
3030
pub sess: &'a Session,
3131
pub features: Option<&'a Features>,
32+
pub modified: bool,
3233
}
3334

3435
fn get_features(
@@ -199,7 +200,7 @@ fn get_features(
199200

200201
// `cfg_attr`-process the crate's attributes and compute the crate's features.
201202
pub fn features(sess: &Session, mut krate: ast::Crate) -> (ast::Crate, Features) {
202-
let mut strip_unconfigured = StripUnconfigured { sess, features: None };
203+
let mut strip_unconfigured = StripUnconfigured { sess, features: None, modified: false };
203204

204205
let unconfigured_attrs = krate.attrs.clone();
205206
let diag = &sess.parse_sess.span_diagnostic;
@@ -243,7 +244,12 @@ const CFG_ATTR_NOTE_REF: &str = "for more information, visit \
243244
impl<'a> StripUnconfigured<'a> {
244245
pub fn configure<T: HasAttrs>(&mut self, mut node: T) -> Option<T> {
245246
self.process_cfg_attrs(&mut node);
246-
self.in_cfg(node.attrs()).then_some(node)
247+
if self.in_cfg(node.attrs()) {
248+
Some(node)
249+
} else {
250+
self.modified = true;
251+
None
252+
}
247253
}
248254

249255
/// Parse and expand all `cfg_attr` attributes into a list of attributes
@@ -270,6 +276,9 @@ impl<'a> StripUnconfigured<'a> {
270276
return vec![attr];
271277
}
272278

279+
// A `#[cfg_attr]` either gets removed, or replaced with a new attribute
280+
self.modified = true;
281+
273282
let (cfg_predicate, expanded_attrs) = match self.parse_cfg_attr(&attr) {
274283
None => return vec![],
275284
Some(r) => r,

‎compiler/rustc_expand/src/expand.rs

+35-6
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ use rustc_ast::ptr::P;
1212
use rustc_ast::token;
1313
use rustc_ast::tokenstream::TokenStream;
1414
use rustc_ast::visit::{self, AssocCtxt, Visitor};
15-
use rustc_ast::{self as ast, AttrItem, Block, LitKind, NodeId, PatKind, Path};
15+
use rustc_ast::{self as ast, AttrItem, AttrStyle, Block, LitKind, NodeId, PatKind, Path};
1616
use rustc_ast::{ItemKind, MacArgs, MacCallStmt, MacStmtStyle, StmtKind, Unsafe};
1717
use rustc_ast_pretty::pprust;
1818
use rustc_attr::{self as attr, is_builtin_attr, HasAttrs};
@@ -522,12 +522,29 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
522522
item.visit_attrs(|attrs| attrs.retain(|a| !a.has_name(sym::derive)));
523523
(item, Vec::new())
524524
} else {
525-
let mut item = StripUnconfigured {
525+
let mut visitor = StripUnconfigured {
526526
sess: self.cx.sess,
527527
features: self.cx.ecfg.features,
528-
}
529-
.fully_configure(item);
528+
modified: false,
529+
};
530+
let mut item = visitor.fully_configure(item);
530531
item.visit_attrs(|attrs| attrs.retain(|a| !a.has_name(sym::derive)));
532+
if visitor.modified && !derives.is_empty() {
533+
// Erase the tokens if cfg-stripping modified the item
534+
// This will cause us to synthesize fake tokens
535+
// when `nt_to_tokenstream` is called on this item.
536+
match &mut item {
537+
Annotatable::Item(item) => item.tokens = None,
538+
Annotatable::Stmt(stmt) => {
539+
if let StmtKind::Item(item) = &mut stmt.kind {
540+
item.tokens = None
541+
} else {
542+
panic!("Unexpected stmt {:?}", stmt);
543+
}
544+
}
545+
_ => panic!("Unexpected annotatable {:?}", item),
546+
}
547+
}
531548

532549
invocations.reserve(derives.len());
533550
let derive_placeholders = derives
@@ -622,7 +639,11 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
622639

623640
let invocations = {
624641
let mut collector = InvocationCollector {
625-
cfg: StripUnconfigured { sess: &self.cx.sess, features: self.cx.ecfg.features },
642+
cfg: StripUnconfigured {
643+
sess: &self.cx.sess,
644+
features: self.cx.ecfg.features,
645+
modified: false,
646+
},
626647
cx: self.cx,
627648
invocations: Vec::new(),
628649
monotonic: self.monotonic,
@@ -716,7 +737,15 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
716737
SyntaxExtensionKind::Attr(expander) => {
717738
self.gate_proc_macro_input(&item);
718739
self.gate_proc_macro_attr_item(span, &item);
719-
let tokens = item.into_tokens(&self.cx.sess.parse_sess);
740+
let tokens = match attr.style {
741+
AttrStyle::Outer => item.into_tokens(&self.cx.sess.parse_sess),
742+
// FIXME: Properly collect tokens for inner attributes
743+
AttrStyle::Inner => rustc_parse::fake_token_stream(
744+
&self.cx.sess.parse_sess,
745+
&item.into_nonterminal(),
746+
span,
747+
),
748+
};
720749
let attr_item = attr.unwrap_normal_item();
721750
if let MacArgs::Eq(..) = attr_item.args {
722751
self.cx.span_err(span, "key-value macro attributes are not supported");

‎compiler/rustc_expand/src/proc_macro.rs

+7-2
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ use crate::proc_macro_server;
33

44
use rustc_ast::ptr::P;
55
use rustc_ast::token;
6-
use rustc_ast::tokenstream::{TokenStream, TokenTree};
6+
use rustc_ast::tokenstream::{CanSynthesizeMissingTokens, TokenStream, TokenTree};
77
use rustc_ast::{self as ast, *};
88
use rustc_data_structures::sync::Lrc;
99
use rustc_errors::{struct_span_err, Applicability, ErrorReported};
@@ -94,7 +94,12 @@ impl MultiItemModifier for ProcMacroDerive {
9494
let input = if item.pretty_printing_compatibility_hack() {
9595
TokenTree::token(token::Interpolated(Lrc::new(item)), DUMMY_SP).into()
9696
} else {
97-
nt_to_tokenstream(&item, &ecx.sess.parse_sess, DUMMY_SP)
97+
nt_to_tokenstream(
98+
&item,
99+
&ecx.sess.parse_sess,
100+
DUMMY_SP,
101+
CanSynthesizeMissingTokens::Yes,
102+
)
98103
};
99104

100105
let server = proc_macro_server::Rustc::new(ecx);

‎compiler/rustc_expand/src/proc_macro_server.rs

+3-2
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ use crate::base::ExtCtxt;
22

33
use rustc_ast as ast;
44
use rustc_ast::token;
5-
use rustc_ast::tokenstream::{self, DelimSpan, Spacing::*, TokenStream, TreeAndSpacing};
5+
use rustc_ast::tokenstream::{self, CanSynthesizeMissingTokens};
6+
use rustc_ast::tokenstream::{DelimSpan, Spacing::*, TokenStream, TreeAndSpacing};
67
use rustc_ast_pretty::pprust;
78
use rustc_data_structures::sync::Lrc;
89
use rustc_errors::Diagnostic;
@@ -178,7 +179,7 @@ impl FromInternal<(TreeAndSpacing, &'_ ParseSess, &'_ mut Vec<Self>)>
178179
{
179180
TokenTree::Ident(Ident::new(sess, name.name, is_raw, name.span))
180181
} else {
181-
let stream = nt_to_tokenstream(&nt, sess, span);
182+
let stream = nt_to_tokenstream(&nt, sess, span, CanSynthesizeMissingTokens::No);
182183
TokenTree::Group(Group {
183184
delimiter: Delimiter::None,
184185
stream,

0 commit comments

Comments
 (0)