syntax: Add tokens: Option<TokenStream> to Item

This commit adds a new field to the `Item` AST node in libsyntax to optionally
contain the original token stream that the item itself was parsed from. This is
currently `None` everywhere but is intended for use later with procedural
macros.

src/librustc_metadata/cstore_impl.rs

@@ -389,6 +389,7 @@ impl CrateStore for cstore::CStore {
                 legacy: def.legacy,
             }),
             vis: ast::Visibility::Inherited,
+            tokens: None,
         })
     }
 

src/libsyntax/ast.rs

@@ -1812,6 +1812,12 @@ pub struct Item {
     pub node: ItemKind,
     pub vis: Visibility,
     pub span: Span,
+
+    /// Original tokens this item was parsed from. This isn't necessarily
+    /// available for all items, although over time more and more items should
+    /// have this be `Some`. Right now this is primarily used for procedural
+    /// macros, notably custom attributes.
+    pub tokens: Option<TokenStream>,
 }
 
 #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)]

src/libsyntax/diagnostics/plugin.rs

@@ -236,6 +236,7 @@ pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt,
             ),
             vis: ast::Visibility::Public,
             span: span,
+            tokens: None,
         })
     ]))
 }

src/libsyntax/ext/build.rs

@@ -979,7 +979,8 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
             id: ast::DUMMY_NODE_ID,
             node: node,
             vis: ast::Visibility::Inherited,
-            span: span
+            span: span,
+            tokens: None,
         })
     }
 
@@ -1147,7 +1148,8 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
             attrs: vec![],
             node: ast::ItemKind::Use(vp),
             vis: vis,
-            span: sp
+            span: sp,
+            tokens: None,
         })
     }
 

src/libsyntax/ext/expand.rs

@@ -214,6 +214,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             ident: keywords::Invalid.ident(),
             id: ast::DUMMY_NODE_ID,
             vis: ast::Visibility::Public,
+            tokens: None,
         })));
 
         match self.expand(krate_item).make_items().pop().map(P::unwrap) {

src/libsyntax/ext/placeholders.rs

@@ -46,6 +46,7 @@ pub fn placeholder(kind: ExpansionKind, id: ast::NodeId) -> Expansion {
         ExpansionKind::Items => Expansion::Items(SmallVector::one(P(ast::Item {
             id: id, span: span, ident: ident, vis: vis, attrs: attrs,
             node: ast::ItemKind::Mac(mac_placeholder()),
+            tokens: None,
         }))),
         ExpansionKind::TraitItems => Expansion::TraitItems(SmallVector::one(ast::TraitItem {
             id: id, span: span, ident: ident, attrs: attrs,

src/libsyntax/fold.rs

@@ -1000,6 +1000,7 @@ pub fn noop_fold_crate<T: Folder>(Crate {module, attrs, span}: Crate,
         vis: ast::Visibility::Public,
         span: span,
         node: ast::ItemKind::Mod(module),
+        tokens: None,
     })).into_iter();
 
     let (module, attrs, span) = match items.next() {
@@ -1032,15 +1033,18 @@ pub fn noop_fold_item<T: Folder>(i: P<Item>, folder: &mut T) -> SmallVector<P<It
 }
 
 // fold one item into exactly one item
-pub fn noop_fold_item_simple<T: Folder>(Item {id, ident, attrs, node, vis, span}: Item,
+pub fn noop_fold_item_simple<T: Folder>(Item {id, ident, attrs, node, vis, span, tokens}: Item,
                                         folder: &mut T) -> Item {
     Item {
         id: folder.new_id(id),
         vis: folder.fold_vis(vis),
         ident: folder.fold_ident(ident),
         attrs: fold_attrs(attrs, folder),
         node: folder.fold_item_kind(node),
-        span: folder.new_span(span)
+        span: folder.new_span(span),
+        tokens: tokens.map(|tokens| {
+            folder.fold_tts(tokens.into()).into()
+        }),
     }
 }
 

src/libsyntax/parse/parser.rs

@@ -4653,6 +4653,7 @@ impl<'a> Parser<'a> {
             node: node,
             vis: vis,
             span: span,
+            tokens: None, // TODO: fill this in
         })
     }
 

src/libsyntax/std_inject.rs

@@ -60,6 +60,7 @@ pub fn maybe_inject_crates_ref(mut krate: ast::Crate, alt_std_name: Option<Strin
         ident: ast::Ident::from_str(name),
         id: ast::DUMMY_NODE_ID,
         span: DUMMY_SP,
+        tokens: None,
     }));
 
     let span = ignored_span(DUMMY_SP);
@@ -82,6 +83,7 @@ pub fn maybe_inject_crates_ref(mut krate: ast::Crate, alt_std_name: Option<Strin
         id: ast::DUMMY_NODE_ID,
         ident: keywords::Invalid.ident(),
         span: span,
+        tokens: None,
     }));
 
     krate

src/libsyntax/test.rs

@@ -192,7 +192,7 @@ impl fold::Folder for EntryPointCleaner {
             EntryPointType::MainNamed |
             EntryPointType::MainAttr |
             EntryPointType::Start =>
-                folded.map(|ast::Item {id, ident, attrs, node, vis, span}| {
+                folded.map(|ast::Item {id, ident, attrs, node, vis, span, tokens}| {
                     let allow_str = Symbol::intern("allow");
                     let dead_code_str = Symbol::intern("dead_code");
                     let word_vec = vec![attr::mk_list_word_item(dead_code_str)];
@@ -212,7 +212,8 @@ impl fold::Folder for EntryPointCleaner {
                             .collect(),
                         node: node,
                         vis: vis,
-                        span: span
+                        span: span,
+                        tokens: tokens,
                     }
                 }),
             EntryPointType::None |
@@ -255,6 +256,7 @@ fn mk_reexport_mod(cx: &mut TestCtxt,
         node: ast::ItemKind::Mod(reexport_mod),
         vis: ast::Visibility::Public,
         span: DUMMY_SP,
+        tokens: None,
     })).pop().unwrap();
 
     (it, sym)
@@ -465,7 +467,8 @@ fn mk_std(cx: &TestCtxt) -> P<ast::Item> {
         node: vi,
         attrs: vec![],
         vis: vis,
-        span: sp
+        span: sp,
+        tokens: None,
     })
 }
 
@@ -506,7 +509,8 @@ fn mk_main(cx: &mut TestCtxt) -> P<ast::Item> {
         id: ast::DUMMY_NODE_ID,
         node: main,
         vis: ast::Visibility::Public,
-        span: sp
+        span: sp,
+        tokens: None,
     })
 }
 
@@ -536,6 +540,7 @@ fn mk_test_module(cx: &mut TestCtxt) -> (P<ast::Item>, Option<P<ast::Item>>) {
         node: item_,
         vis: ast::Visibility::Public,
         span: DUMMY_SP,
+        tokens: None,
     })).pop().unwrap();
     let reexport = cx.reexport_test_harness_main.map(|s| {
         // building `use <ident> = __test::main`
@@ -551,7 +556,8 @@ fn mk_test_module(cx: &mut TestCtxt) -> (P<ast::Item>, Option<P<ast::Item>>) {
             attrs: vec![],
             node: ast::ItemKind::Use(P(use_path)),
             vis: ast::Visibility::Inherited,
-            span: DUMMY_SP
+            span: DUMMY_SP,
+            tokens: None,
         })).pop().unwrap()
     });
 

src/libsyntax_ext/global_asm.rs

@@ -61,5 +61,6 @@ pub fn expand_global_asm<'cx>(cx: &'cx mut ExtCtxt,
         })),
         vis: ast::Visibility::Inherited,
         span: sp,
+        tokens: None,
     })))
 }