diff --git a/Cargo.lock b/Cargo.lock index a592c82225e6d..f3726b1e8684e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -102,6 +102,12 @@ dependencies = [ "nodrop", ] +[[package]] +name = "arrayvec" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" + [[package]] name = "atty" version = "0.2.11" @@ -181,7 +187,7 @@ version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" dependencies = [ - "arrayvec", + "arrayvec 0.4.7", "constant_time_eq", ] @@ -731,7 +737,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fedcd6772e37f3da2a9af9bf12ebe046c0dfe657992377b4df982a2b54cd37a9" dependencies = [ - "arrayvec", + "arrayvec 0.4.7", "cfg-if", "crossbeam-utils 0.6.5", "lazy_static 1.4.0", @@ -948,6 +954,12 @@ dependencies = [ "rustc-std-workspace-core", ] +[[package]] +name = "drop_bomb" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b26e475fd29098530e709294e94e661974c851aed42512793f120fed4e199f" + [[package]] name = "dtoa" version = "0.4.4" @@ -1107,7 +1119,7 @@ dependencies = [ name = "fmt_macros" version = "0.0.0" dependencies = [ - "rustc_lexer", + "rustc_lexer 0.1.0", "rustc_span", ] @@ -2247,9 +2259,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.1.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6a04cb71e910d0034815600180f62a95bf6e67942d7ab52a166a68c7d7e9cd0" +checksum = "b1c601810575c99596d4afc46f78a678c80105117c379eb3650cf99b8a21ce5b" [[package]] name = "open" @@ -2692,6 +2704,37 @@ dependencies = [ "proc-macro2 1.0.3", ] +[[package]] +name = "ra_parser" +version = "0.1.0" +dependencies = [ + "drop_bomb", +] + +[[package]] +name = "ra_syntax" +version = "0.1.0" +dependencies = [ + "arrayvec 0.5.1", + "itertools 0.9.0", + "once_cell", + "ra_parser", + "ra_text_edit", + "rowan", + "rustc-hash", + "rustc_lexer 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde", + "smol_str", + "stdx", +] + +[[package]] +name = "ra_text_edit" +version = "0.1.0" +dependencies = [ + "text_unit", +] + [[package]] name = "racer" version = "2.1.31" @@ -3120,6 +3163,18 @@ dependencies = [ "rls-span", ] +[[package]] +name = "rowan" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ea7cadf87a9d8432e85cb4eb86bd2e765ace60c24ef86e79084dcae5d1c5a19" +dependencies = [ + "rustc-hash", + "smol_str", + "text_unit", + "thin-dst", +] + [[package]] name = "rustbook" version = "0.1.0" @@ -3417,12 +3472,9 @@ dependencies = [ [[package]] name = "rustc-hash" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7540fc8b0c49f096ee9c961cda096467dce8084bec6bdca2fc83895fd9b28cb8" -dependencies = [ - "byteorder", -] +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-main" @@ -3507,7 +3559,7 @@ dependencies = [ "log", "rustc_data_structures", "rustc_index", - "rustc_lexer", + "rustc_lexer 0.1.0", "rustc_macros", "rustc_span", "scoped-tls", @@ -3742,7 +3794,7 @@ dependencies = [ "rustc_data_structures", "rustc_errors", "rustc_feature", - "rustc_lexer", + "rustc_lexer 0.1.0", "rustc_parse", "rustc_session", "rustc_span", @@ -3861,6 +3913,7 @@ dependencies = [ "rustc_mir", "rustc_mir_build", "rustc_parse", + "rustc_parse_ra", "rustc_passes", "rustc_plugin_impl", "rustc_privacy", @@ -3886,6 +3939,15 @@ dependencies = [ "unicode-xid 0.2.0", ] +[[package]] +name = "rustc_lexer" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c86aae0c77166108c01305ee1a36a1e77289d7dc6ca0a3cd91ff4992de2d16a5" +dependencies = [ + "unicode-xid 0.2.0", +] + [[package]] name = "rustc_lint" version = "0.0.0" @@ -3972,7 +4034,7 @@ dependencies = [ "rustc_hir", "rustc_index", "rustc_infer", - "rustc_lexer", + "rustc_lexer 0.1.0", "rustc_macros", "rustc_session", "rustc_span", @@ -4018,7 +4080,25 @@ dependencies = [ "rustc_data_structures", "rustc_errors", "rustc_feature", - "rustc_lexer", + "rustc_lexer 0.1.0", + "rustc_session", + "rustc_span", + "smallvec 1.0.0", + "unicode-normalization", +] + +[[package]] +name = "rustc_parse_ra" +version = "0.0.0" +dependencies = [ + "bitflags", + "log", + "ra_syntax", + "rustc_ast", + "rustc_ast_pretty", + "rustc_data_structures", + "rustc_errors", + "rustc_parse", "rustc_session", "rustc_span", "smallvec 1.0.0", @@ -4466,22 +4546,22 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.99" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec2851eb56d010dc9a21b89ca53ee75e6528bab60c11e89d38390904982da9f" +checksum = "e707fbbf255b8fc8c3b99abb91e7257a622caeb20a9818cbadbeeede4e0932ff" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.81" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "477b13b646f5b5b56fc95bedfc3b550d12141ce84f466f6c44b9a17589923885" +checksum = "ac5d00fc561ba2724df6758a17de23df5914f20e41cb00f94d5b7ae42fffaff8" dependencies = [ - "proc-macro2 0.4.30", - "quote 0.6.12", - "syn 0.15.35", + "proc-macro2 1.0.3", + "quote 1.0.2", + "syn 1.0.11", ] [[package]] @@ -4603,6 +4683,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ecf3b85f68e8abaa7555aa5abdb1153079387e60b718283d732f03897fcfc86" +[[package]] +name = "smol_str" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34836c9a295c62c2ce3514471117c5cb269891e8421b2aafdd910050576c4d8b" +dependencies = [ + "serde", +] + [[package]] name = "socket2" version = "0.3.11" @@ -4643,6 +4732,10 @@ dependencies = [ "wasi", ] +[[package]] +name = "stdx" +version = "0.1.0" + [[package]] name = "string" version = "0.2.1" @@ -4899,6 +4992,12 @@ dependencies = [ "term 0.6.0", ] +[[package]] +name = "text_unit" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20431e104bfecc1a40872578dbc390e10290a0e9c35fffe3ce6f73c15a9dbfc2" + [[package]] name = "textwrap" version = "0.11.0" @@ -4908,6 +5007,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "thin-dst" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c46be180f1af9673ebb27bc1235396f61ef6965b3fe0dbb2e624deb604f0e" + [[package]] name = "thiserror" version = "1.0.5" diff --git a/config.toml.example b/config.toml.example index ce21b63467f53..df57b5d38b0fa 100644 --- a/config.toml.example +++ b/config.toml.example @@ -436,6 +436,10 @@ # override the default allocator for rustc and LLVM. #jemalloc = false +# Enable experimental features that depend on rust-analyzer. Assumes that you +# have a rust-analyzer directory in the same directory as the rustc directory +#rust-analyzer = false + # Run tests in various test suites with the "nll compare mode" in addition to # running the tests in normal mode. Largely only used on CI and during local # development of NLL diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 56164b74f3088..8e371e7984ca7 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -116,6 +116,7 @@ pub struct Config { pub targets: Vec>, pub local_rebuild: bool, pub jemalloc: bool, + pub rust_analyzer: bool, pub control_flow_guard: bool, // dist misc @@ -341,6 +342,7 @@ struct Rust { thin_lto_import_instr_limit: Option, remap_debuginfo: Option, jemalloc: Option, + rust_analyzer: Option, test_compare_mode: Option, llvm_libunwind: Option, control_flow_guard: Option, @@ -569,6 +571,7 @@ impl Config { set(&mut config.codegen_tests, rust.codegen_tests); set(&mut config.rust_rpath, rust.rpath); set(&mut config.jemalloc, rust.jemalloc); + set(&mut config.rust_analyzer, rust.rust_analyzer); set(&mut config.test_compare_mode, rust.test_compare_mode); set(&mut config.llvm_libunwind, rust.llvm_libunwind); set(&mut config.backtrace, rust.backtrace); diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 6436fa756558b..535cba05938b8 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -509,6 +509,9 @@ impl Build { if self.config.jemalloc { features.push_str("jemalloc"); } + if self.config.rust_analyzer { + features.push_str("rust_analyzer"); + } if self.config.llvm_enabled() { features.push_str(" llvm"); } diff --git a/src/librustc_driver/Cargo.toml b/src/librustc_driver/Cargo.toml index 3ca39b24c5276..dc07a6c3ed78c 100644 --- a/src/librustc_driver/Cargo.toml +++ b/src/librustc_driver/Cargo.toml @@ -40,3 +40,4 @@ winapi = { version = "0.3", features = ["consoleapi", "debugapi", "processenv"] [features] llvm = ['rustc_interface/llvm'] +rust_analyzer = ['rustc_interface/rust_analyzer'] diff --git a/src/librustc_interface/Cargo.toml b/src/librustc_interface/Cargo.toml index 2e055ff183f2c..8b60b20a16e12 100644 --- a/src/librustc_interface/Cargo.toml +++ b/src/librustc_interface/Cargo.toml @@ -18,6 +18,7 @@ rustc_attr = { path = "../librustc_attr" } rustc_builtin_macros = { path = "../librustc_builtin_macros" } rustc_expand = { path = "../librustc_expand" } rustc_parse = { path = "../librustc_parse" } +rustc_parse_ra = { path = "../librustc_parse_ra", optional = true } rustc_session = { path = "../librustc_session" } rustc_span = { path = "../librustc_span" } rustc_serialize = { path = "../libserialize", package = "serialize" } @@ -55,3 +56,4 @@ rustc_target = { path = "../librustc_target" } [features] llvm = ['rustc_codegen_llvm'] +rust_analyzer = ['rustc_parse_ra'] diff --git a/src/librustc_interface/passes.rs b/src/librustc_interface/passes.rs index 7cf0b4c44a6aa..db1ee25ebf17e 100644 --- a/src/librustc_interface/passes.rs +++ b/src/librustc_interface/passes.rs @@ -51,13 +51,40 @@ use std::path::PathBuf; use std::rc::Rc; use std::{env, fs, iter, mem}; -pub fn parse<'a>(sess: &'a Session, input: &Input) -> PResult<'a, ast::Crate> { - let krate = sess.time("parse_crate", || match input { - Input::File(file) => parse_crate_from_file(file, &sess.parse_sess), +#[cfg(rust_analyzer)] +fn parse_with_ra<'a>(sess: &'a Session, input: &Input) -> PResult<'a, ast::Crate> +{ + match input { + Input::File(file) => { + rustc_parse_ra::parse_crate_from_file(file, &sess.parse_sess) + }, Input::Str { input, name } => { - parse_crate_from_source_str(name.clone(), input.clone(), &sess.parse_sess) + rustc_parse_ra::parse_crate_from_source_str(name.clone(), input.clone(), &sess.parse_sess) } - })?; + } +} + +#[cfg(not(rust_analyzer))] +fn parse_with_ra<'a>(sess: &'a Session, _input: &Input) -> PResult<'a, ast::Crate> +{ + Err(sess.struct_fatal("parsing with rust-analyzer is not supported because this version of rustc was not compiled with the \"rust_analyzer\" feature flag enabled")) +} + +pub fn parse<'a>(sess: &'a Session, input: &Input) -> PResult<'a, ast::Crate> { + let krate = sess.time("parse_crate", || + if sess.opts.debugging_opts.parse_with_rust_analyzer { + parse_with_ra(sess, input) + } else { + match input { + Input::File(file) => { + parse_crate_from_file(file, &sess.parse_sess) + }, + Input::Str { input, name } => { + parse_crate_from_source_str(name.clone(), input.clone(), &sess.parse_sess) + } + } + } + )?; if sess.opts.debugging_opts.ast_json_noexpand { println!("{}", json::as_json(&krate)); diff --git a/src/librustc_parse/lexer/mod.rs b/src/librustc_parse/lexer/mod.rs index ac58cbb9e8dae..c5156e9b1c1da 100644 --- a/src/librustc_parse/lexer/mod.rs +++ b/src/librustc_parse/lexer/mod.rs @@ -83,6 +83,7 @@ impl<'a> StringReader<'a> { let mut sr = StringReader::new(sess, begin.sf, None); // Seek the lexer to the right byte range. + sr.pos = span.lo(); sr.end_src_index = sr.src_index(span.hi()); sr diff --git a/src/librustc_parse/lexer/tokentrees.rs b/src/librustc_parse/lexer/tokentrees.rs index b65b894172844..0f215396d2ba4 100644 --- a/src/librustc_parse/lexer/tokentrees.rs +++ b/src/librustc_parse/lexer/tokentrees.rs @@ -12,7 +12,7 @@ use rustc_errors::PResult; use rustc_span::Span; impl<'a> StringReader<'a> { - crate fn into_token_trees(self) -> (PResult<'a, TokenStream>, Vec) { + pub fn into_token_trees(self) -> (PResult<'a, TokenStream>, Vec) { let mut tt_reader = TokenTreesReader { string_reader: self, token: Token::dummy(), diff --git a/src/librustc_parse_ra/Cargo.toml b/src/librustc_parse_ra/Cargo.toml new file mode 100644 index 0000000000000..d55dfd7b56d49 --- /dev/null +++ b/src/librustc_parse_ra/Cargo.toml @@ -0,0 +1,24 @@ +[package] +authors = ["The Rust Project Developers"] +name = "rustc_parse_ra" +version = "0.0.0" +edition = "2018" + +[lib] +name = "rustc_parse_ra" +path = "lib.rs" +doctest = false + +[dependencies] +bitflags = "1.0" +log = "0.4" +rustc_errors = { path = "../librustc_errors" } +smallvec = { version = "1.0", features = ["union", "may_dangle"] } +rustc_session = { path = "../librustc_session" } +rustc_span = { path = "../librustc_span" } +rustc_ast = { path = "../librustc_ast" } +rustc_ast_pretty = { path = "../librustc_ast_pretty" } +rustc_data_structures = { path = "../librustc_data_structures" } +rustc_parse = { path = "../librustc_parse" } +unicode-normalization = "0.1.11" +ra_syntax = { path = "../../../rust-analyzer/crates/ra_syntax" } diff --git a/src/librustc_parse_ra/lib.rs b/src/librustc_parse_ra/lib.rs new file mode 100644 index 0000000000000..5fa478cbdc535 --- /dev/null +++ b/src/librustc_parse_ra/lib.rs @@ -0,0 +1,3037 @@ +use ra_syntax::ast as ra; +#[allow(unused_imports)] +use ra_syntax::ast::{ + ArgListOwner, AstElement, AstNode, AstToken, AttrsOwner, LoopBodyOwner, NameOwner, + TypeAscriptionOwner, TypeBoundsOwner, TypeParamsOwner, VisibilityOwner, +}; +use ra_syntax::{ + NodeOrToken, Parse, SmolStr, SourceFile as RaSourceFile, SyntaxElement, SyntaxToken, TextRange, + TextUnit, +}; +use rustc_ast::ast; +use rustc_ast::ptr::P; +use rustc_ast::token; +use rustc_ast::tokenstream::{IsJoint, TokenStream}; +use rustc_data_structures::sync::Lrc; +use rustc_errors::{Diagnostic, DiagnosticBuilder, FatalError, Level, PResult}; +use rustc_parse::lexer; +use rustc_session::parse::ParseSess; +use rustc_span::source_map::Spanned; +use rustc_span::symbol::{kw, Symbol}; +use rustc_span::{BytePos, FileName, Pos, SourceFile, Span, SyntaxContext}; + +use std::convert::{TryFrom, TryInto}; +use std::path::Path; + +pub fn parse_crate_from_file<'tcx>( + input: &Path, + sess: &'tcx ParseSess, +) -> PResult<'tcx, ast::Crate> { + parse_crate_from_source_file(file_to_source_file(sess, input, None), sess) +} + +pub fn parse_crate_from_source_str( + name: FileName, + source: String, + sess: &ParseSess, +) -> PResult<'_, ast::Crate> { + parse_crate_from_source_file(sess.source_map().new_source_file(name, source), sess) +} + +/// Given a session and a path and an optional span (for error reporting), +/// add the path to the session's source_map and return the new source_file or +/// error when a file can't be read. +fn try_file_to_source_file( + sess: &ParseSess, + path: &Path, + spanopt: Option, +) -> Result, Diagnostic> { + sess.source_map().load_file(path).map_err(|e| { + let msg = format!("couldn't read {}: {}", path.display(), e); + let mut diag = Diagnostic::new(Level::Fatal, &msg); + if let Some(sp) = spanopt { + diag.set_span(sp); + } + diag + }) +} + +/// Given a session and a path and an optional span (for error reporting), +/// adds the path to the session's `source_map` and returns the new `source_file`. +fn file_to_source_file(sess: &ParseSess, path: &Path, spanopt: Option) -> Lrc { + match try_file_to_source_file(sess, path, spanopt) { + Ok(source_file) => source_file, + Err(d) => { + sess.span_diagnostic.emit_diagnostic(&d); + FatalError.raise(); + } + } +} + +fn parse_crate_from_source_file<'tcx>( + source_file: Lrc, + sess: &'tcx ParseSess, +) -> PResult<'tcx, ast::Crate> { + let ctxt = SyntaxContext::root(); + let src = match source_file.src.as_ref() { + Some(src) => src, + None => { + sess.span_diagnostic + .bug(&format!("cannot lex `source_file` without source: {}", source_file.name)); + } + }; + let parse = RaSourceFile::parse(src); + let parser = Parser::new(source_file, ctxt, sess); + let krate = parser.to_crate(parse); + + // we recover from all errors after reporting them as diagnostics + Ok(krate) +} + +fn raw_text_range(element: &impl ra::AstElement) -> TextRange { + match element.syntax_element() { + NodeOrToken::Node(node) => node.text_range(), + NodeOrToken::Token(token) => token.text_range(), + } +} + +// TODO: maybe we should move those helpers to rust-analyzer, although it's also possible they are only useful here + +/// whitespace and non-doc comments +fn is_ignorable_token(token: &SyntaxToken) -> bool { + if ra::Whitespace::can_cast(token.kind()) { + true + } else if ra::Comment::can_cast(token.kind()) { + ra::Comment::cast(token.clone()).unwrap().kind().doc.is_none() + } else { + false + } +} + +/// whitespace, non-doc and outer doc comments and outer attributes +fn is_outer_element(element: &SyntaxElement) -> bool { + match element { + NodeOrToken::Node(node) => { + ra::Attr::can_cast(node.kind()) + && ra::Attr::cast(node.clone()).unwrap().excl().is_none() + } + NodeOrToken::Token(token) => is_outer_token(token), + } +} + +fn is_outer_token(token: &SyntaxToken) -> bool { + if ra::Whitespace::can_cast(token.kind()) { + true + } else if ra::Comment::can_cast(token.kind()) { + ra::Comment::cast(token.clone()).unwrap().kind().doc != Some(ra::CommentPlacement::Inner) + } else { + false + } +} + +/// start of first non-ignorable token +fn outer_start(element: &impl ra::AstElement) -> TextUnit { + match element.syntax_element() { + NodeOrToken::Token(token) => token.text_range().start(), + NodeOrToken::Node(node) => { + // TODO: we might not need this, if the rust-analyzer parser never adds whitespace/comments at the start of items + if let Some(mut token) = node.first_token() { + loop { + if !is_ignorable_token(&token) { + break token.text_range().start(); + } else if let Some(token_to_move_to) = token.next_token() { + token = token_to_move_to; + } else { + break token.text_range().end(); + } + } + } else { + node.text_range().start() + } + } + } +} + +/// end of last non-ignorable token +fn outer_end(element: &impl ra::AstElement) -> TextUnit { + match element.syntax_element() { + NodeOrToken::Token(token) => token.text_range().end(), + NodeOrToken::Node(node) => { + // TODO: we might not need this, except for SourceFile, if the rust-analyzer parser never adds whitespace/comments at the end of items + if let Some(mut token) = node.last_token() { + loop { + if !is_ignorable_token(&token) { + break token.text_range().end(); + } else if let Some(token_to_move_to) = token.prev_token() { + token = token_to_move_to; + } else { + break token.text_range().start(); + } + } + } else { + node.text_range().end() + } + } + } +} + +/// trims whitespace and comments, includes all attributes and doc comments +fn outer_range(element: &impl ra::AstElement) -> TextRange { + TextRange::from_to(outer_start(element), outer_end(element)) +} + +/// start of first non-outer token +fn inner_start(element: &impl ra::AstElement) -> TextUnit { + match element.syntax_element() { + NodeOrToken::Token(token) => token.text_range().start(), + NodeOrToken::Node(node) => { + if let Some(element) = + node.children_with_tokens().skip_while(|element| is_outer_element(element)).next() + { + match element { + NodeOrToken::Token(token) => token.text_range().start(), + NodeOrToken::Node(node) => { + // TODO: we might not need this, if the rust-analyzer parser never adds whitespace/comments at the start of items + if let Some(mut token) = node.first_token() { + loop { + if !is_outer_token(&token) { + break token.text_range().start(); + } else if let Some(token_to_move_to) = token.next_token() { + token = token_to_move_to; + } else { + break token.text_range().end(); + } + } + } else { + node.text_range().start() + } + } + } + } else { + node.text_range().end() + } + } + } +} + +/// end of last non-outer token +fn inner_end(element: &impl ra::AstElement) -> TextUnit { + match element.syntax_element() { + NodeOrToken::Token(token) => token.text_range().end(), + NodeOrToken::Node(node) => { + // TODO: we might not need this, except for SourceFile, if the rust-analyzer parser never adds whitespace/comments at the end of items + if let Some(mut token) = node.last_token() { + loop { + if !is_outer_token(&token) { + break token.text_range().end(); + } else if let Some(token_to_move_to) = token.prev_token() { + token = token_to_move_to; + } else { + break token.text_range().start(); + } + } + } else { + node.text_range().end() + } + } + } +} + +/// end of last non-outer token +fn inner_last_start(element: &impl ra::AstElement) -> TextUnit { + match element.syntax_element() { + NodeOrToken::Token(token) => token.text_range().start(), + NodeOrToken::Node(node) => { + // TODO: we might not need this, except for SourceFile, if the rust-analyzer parser never adds whitespace/comments at the end of items + if let Some(mut token) = node.last_token() { + loop { + if !is_outer_token(&token) { + break token.text_range().start(); + } else if let Some(token_to_move_to) = token.prev_token() { + token = token_to_move_to; + } else { + break node.text_range().start(); + } + } + } else { + node.text_range().start() + } + } + } +} + +/// trims whitespace, comments, outer attributes and doc comments, includes inner attributes and inner doc comments +fn inner_range(element: &impl ra::AstElement) -> TextRange { + TextRange::from_to(inner_start(element), inner_end(element)) +} + +/// start of first following non-ignorable token +fn start_of_next(element: &impl ra::AstElement) -> TextUnit { + if let Some(mut token) = match element.syntax_element() { + NodeOrToken::Node(node) => node.last_token().and_then(|token| token.next_token()), + NodeOrToken::Token(token) => token.next_token(), + } { + loop { + if !is_ignorable_token(&token) { + break token.text_range().start(); + } else if let Some(next) = token.next_token() { + token = next; + } else { + break token.text_range().end(); + } + } + } else { + raw_text_range(element).end() + } +} + +fn end_of_next(element: &impl ra::AstElement) -> TextUnit { + if let Some(mut token) = match element.syntax_element() { + NodeOrToken::Node(node) => node.last_token().and_then(|token| token.next_token()), + NodeOrToken::Token(token) => token.next_token(), + } { + loop { + if !is_ignorable_token(&token) { + break token.text_range().end(); + } else if let Some(next) = token.next_token() { + token = next; + } else { + break token.text_range().end(); + } + } + } else { + raw_text_range(element).end() + } +} + +/// end of preceding non-ignorable token +fn end_of_prev(element: &impl ra::AstElement) -> TextUnit { + if let Some(mut token) = match element.syntax_element() { + NodeOrToken::Node(node) => node.first_token().and_then(|token| token.prev_token()), + NodeOrToken::Token(token) => token.prev_token(), + } { + loop { + if !is_ignorable_token(&token) { + break token.text_range().end(); + } else if let Some(prev) = token.prev_token() { + token = prev; + } else { + break token.text_range().start(); + } + } + } else { + raw_text_range(element).start() + } +} + +fn empty_text_range(unit: TextUnit) -> TextRange { + TextRange::from_to(unit, unit) +} + +fn remove_joint(stream: &mut TokenStream) { + for tree in Lrc::make_mut(&mut stream.0).iter_mut() { + tree.1 = IsJoint::NonJoint; + /* + match &mut tree.0 { + TokenTree::Delimited(_, _, sub_stream) => remove_joint(sub_stream), + _ => {} + } + */ + } +} + +struct Parser<'tcx> { + file: Lrc, + ctxt: SyntaxContext, + sess: &'tcx ParseSess, +} + +#[derive(Debug, Clone, Copy)] +struct FnParamMode { + require_pat: bool, + require_type: bool, +} + +impl<'tcx> Parser<'tcx> { + pub fn new(file: Lrc, ctxt: SyntaxContext, sess: &'tcx ParseSess) -> Self { + Self { file, ctxt, sess } + } + + pub fn to_crate(&self, parse: Parse) -> ast::Crate { + for error in parse.errors() { + let msg = error.to_string(); + self.error_at_span(self.to_span(error.range()), &msg); + } + + self.to_crate_from_source_file(parse.tree()) + } + + fn error_at_span(&self, span: Span, msg: &str) { + let mut diag = Diagnostic::new(Level::Error, msg); + diag.set_span(span); + self.sess.span_diagnostic.emit_diagnostic(&diag); + } + + fn error(&self, element: &impl ra::AstElement, msg: &str) { + self.error_at_span(self.get_inner_span(element), msg) + } + + fn missing_in(&self, element: &impl ra::AstElement, object: &str, container: &str) { + self.error(element, &format!("missing {} in {}", object, container)) + } + + fn invalid_token(&self, token: Option, container: &str) { + if let Some(token) = token { + self.error_at_span( + self.get_inner_span(&token), + &format!("{} token is not valid in {}", token.text(), container), + ) + } + } + + fn invalid_element( + &self, + element: Option, + _parent: &impl ra::AstElement, + object: &str, + container: &str, + ) { + if let Some(node) = element { + self.error_at_span( + self.get_inner_span(&node), + &format!("{} is not valid in {}", object, container), + ) + } + } + + fn expect_in( + &self, + opt: Option, + element: &impl ra::AstElement, + object: &str, + container: &str, + ) -> Option { + if opt.is_none() { + self.missing_in(element, object, container); + } + opt + } + + fn expect_in_res( + &self, + res: Result, + object: &str, + container: &str, + ) -> Option { + match res { + Ok(node) => Some(node), + Err(node) => { + self.error(&node, &format!("missing {} in {}", object, container)); + None + } + } + } + + fn to_byte_pos(&self, unit: TextUnit) -> BytePos { + BytePos::from_usize(self.file.start_pos.to_usize() + unit.to_usize()) + } + + fn to_span(&self, range: TextRange) -> Span { + Span::new(self.to_byte_pos(range.start()), self.to_byte_pos(range.end()), self.ctxt) + } + + fn to_span_from_unit(&self, unit: TextUnit) -> Span { + let byte_pos = self.to_byte_pos(unit); + Span::new(byte_pos, byte_pos, self.ctxt) + } + + fn get_inner_span(&self, element: &impl ra::AstElement) -> Span { + self.to_span(inner_range(element)) + } + + fn get_inner_to_next_span(&self, element: &impl ra::AstElement) -> Span { + self.to_span(TextRange::from_to(inner_start(element), end_of_next(element))) + } + + fn get_last_token_span(&self, element: &impl ra::AstElement) -> Span { + self.to_span(TextRange::from_to(inner_last_start(element), inner_end(element))) + } + + fn to_crate_from_source_file(&self, source_file: RaSourceFile) -> ast::Crate { + let span = self.to_span(outer_range(&source_file)); + ast::Crate { + span, + attrs: self.get_attributes(&source_file).collect(), + module: ast::Mod { + inner: span, + items: self.to_items(Some(source_file), "crate"), + inline: true, + }, + proc_macros: Default::default(), // TODO: proc macros + } + } + + fn to_defaultness(&self, default_kw: Option) -> ast::Defaultness { + if let Some(default_token) = default_kw { + ast::Defaultness::Default(self.get_inner_span(&default_token)) + } else { + ast::Defaultness::Final + } + } + + fn to_constness(&self, const_kw: Option) -> ast::Const { + if let Some(const_token) = const_kw { + ast::Const::Yes(self.get_inner_span(&const_token)) + } else { + ast::Const::No + } + } + + fn get_visibility( + &self, + node: &(impl VisibilityOwner + AstElement), + ) -> Spanned { + self.to_visibility(node.visibility(), node) + } + + fn to_visibility( + &self, + visibility: Option, + element: &impl ra::AstElement, + ) -> Spanned { + Spanned { + span: if let Some(visibility) = visibility.as_ref() { + self.get_inner_span(visibility) + } else { + self.get_inner_span(element).shrink_to_lo() + }, + node: if let Some(visibility) = visibility { + match visibility.kind() { + ra::VisibilityKind::Pub => ast::VisibilityKind::Public, + ra::VisibilityKind::PubCrate => { + ast::VisibilityKind::Crate(if visibility.pub_kw().is_some() { + ast::CrateSugar::PubCrate + } else { + ast::CrateSugar::JustCrate + }) + } + ra::VisibilityKind::PubSuper => ast::VisibilityKind::Restricted { + id: ast::DUMMY_NODE_ID, + path: P(self.mk_path_for_ident( + self.to_ident_from_token(visibility.super_kw().unwrap()), + )), + }, + ra::VisibilityKind::PubSelf => ast::VisibilityKind::Restricted { + id: ast::DUMMY_NODE_ID, + path: P(self.mk_path_for_ident( + self.to_ident_from_token(visibility.self_kw().unwrap()), + )), + }, + ra::VisibilityKind::In(path) => ast::VisibilityKind::Restricted { + id: ast::DUMMY_NODE_ID, + path: P(self.to_simple_path(path, "pub(in path)")), + }, + } + } else { + ast::VisibilityKind::Inherited + }, + } + } + + fn to_variant_data(&self, field_def_list: Option) -> ast::VariantData { + match field_def_list { + None => ast::VariantData::Unit(ast::DUMMY_NODE_ID), + Some(ra::FieldDefList::TupleFieldDefList(tuple_field_def_list)) => { + ast::VariantData::Tuple( + tuple_field_def_list + .fields() + .map(|field| ast::StructField { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(&field), + attrs: self.get_attributes(&field).collect(), + vis: self.get_visibility(&field), + ident: None, + ty: self.to_ty_or_err( + field.type_ref(), + &field, + "tuple struct field type", + ), + is_placeholder: false, + }) + .collect(), + ast::DUMMY_NODE_ID, + ) + } + Some(ra::FieldDefList::RecordFieldDefList(record_field_def_list)) => { + ast::VariantData::Struct( + record_field_def_list + .fields() + .map(|field| ast::StructField { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(&field), + attrs: self.get_attributes(&field).collect(), + vis: self.get_visibility(&field), + ident: Some(self.to_ident_from_name_or_err( + field.name(), + &field, + "record field name", + )), + ty: self.to_ty_or_err( + field.ascribed_type(), + &field, + "record field type", + ), + is_placeholder: false, + }) + .collect(), + false, // TODO: this is a "recovered" flag that is apparently set when the parser encounters errors in the struct; why is it there only for this ast node? should we set it? how? + ) + } + } + } + + fn get_fn_sig(&self, fn_def: &ra::FnDef) -> ast::FnSig { + let end = if let Some(body) = fn_def.body() { + inner_start(&body) + } else if let Some(semi) = fn_def.semi() { + inner_start(&semi) + } else { + start_of_next(fn_def) + }; + ast::FnSig { + decl: self.to_fn_decl( + fn_def.param_list(), + fn_def.ret_type(), + end, + FnParamMode { require_pat: true, require_type: true }, + ), + header: ast::FnHeader { + asyncness: self.to_async(fn_def.async_kw()), + constness: self.to_constness(fn_def.const_kw()), + ext: self.to_extern(fn_def.abi()), + unsafety: self.to_unsafe(fn_def.unsafe_kw()), + }, + } + } + + fn to_use_tree_or_err( + &self, + use_tree_opt: Option, + element: &impl ra::AstElement, + container: &str, + ) -> ast::UseTree { + self.expect_in(use_tree_opt, element, "use tree", container) + .map(|use_tree| self.to_use_tree(use_tree)) + .unwrap_or_else(|| { + let span = self.get_inner_span(element); + ast::UseTree { + span, + prefix: self.mk_path_for_ident(ast::Ident::new(kw::Invalid, span)), + kind: ast::UseTreeKind::Simple(None, ast::DUMMY_NODE_ID, ast::DUMMY_NODE_ID), + } + }) + } + + fn to_use_tree(&self, use_tree: ra::UseTree) -> ast::UseTree { + let span = self.get_inner_span(&use_tree); + ast::UseTree { + span, + prefix: if let Some(path) = use_tree.path() { + self.to_simple_path(path, "use tree prefix") + } else { + ast::Path { segments: Default::default(), span: span.shrink_to_lo() } + }, + kind: if use_tree.star().is_some() { + ast::UseTreeKind::Glob + } else if let Some(use_tree_list) = use_tree.use_tree_list() { + ast::UseTreeKind::Nested( + use_tree_list + .use_trees() + .map(|use_tree| (self.to_use_tree(use_tree), ast::DUMMY_NODE_ID)) + .collect(), + ) + } else { + ast::UseTreeKind::Simple( + if let Some(alias) = use_tree.alias() { + Some(self.to_ident_from_name_or_err(alias.name(), &alias, "use as")) + } else { + None + }, + ast::DUMMY_NODE_ID, + ast::DUMMY_NODE_ID, + ) + }, + } + } + + fn to_items + 'static>( + &self, + owner: Option, + container: &str, + ) -> Vec>> { + owner + .into_iter() + .flat_map(|owner| owner.items().flat_map(|item| self.to_item_opt(item, container))) + .collect() + } + + fn to_module_item( + &self, + module_item: ra::ModuleItem, + container: &str, + ) -> P> { + self.to_item_opt(module_item, container).unwrap() /* guaranteed to succeed because it's a try_into from T to T */ + } + + fn to_item_opt + 'static>( + &self, + module_item: ra::ModuleItem, + container: &str, + ) -> Option>> { + let span = self.get_inner_span(&module_item); // must be inner for extern crate + let attrs = self.get_attributes(&module_item).collect(); + let tokens = self.get_collected_tokens_from_range(inner_range(&module_item)); + let vis = self.get_visibility(&module_item); + + // TODO: global asm, trait alias, macro 2.0 def + let (ident, kind) = match module_item { + ra::ModuleItem::ConstDef(const_def) => ( + self.to_ident_from_name_or_err(const_def.name(), &const_def, "const definition"), + ast::ItemKind::Const( + self.to_defaultness(const_def.default_kw()), + self.to_ty_or_err(const_def.ascribed_type(), &const_def, "const definition"), + const_def.body().map(|expr| self.to_expr(expr)), + ), + ), + ra::ModuleItem::EnumDef(enum_def) => ( + self.to_ident_from_name_or_err(enum_def.name(), &enum_def, "enum definition"), + ast::ItemKind::Enum( + ast::EnumDef { + variants: enum_def + .variant_list() + .into_iter() + .flat_map(|variant_list| { + variant_list.variants().map(|variant| ast::Variant { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(&variant), + attrs: self.get_attributes(&variant).collect(), + vis: self.to_visibility(variant.visibility(), &variant), + disr_expr: variant.expr().map(|expr| self.to_anon_const(expr)), + data: self.to_variant_data(variant.field_def_list()), + ident: self.to_ident_from_name_or_err( + variant.name(), + &variant, + "enum variant name", + ), + is_placeholder: false, + }) + }) + .collect(), + }, + self.get_generics_for_named( + &enum_def, + || enum_def.enum_kw().map(|element| inner_end(&element)), + || enum_def.variant_list().map(|element| end_of_prev(&element)), + ), + ), + ), + ra::ModuleItem::ExternBlock(extern_block) => ( + ast::Ident::invalid(), + ast::ItemKind::ForeignMod(ast::ForeignMod { + abi: if let Some(abi) = + self.expect_in(extern_block.abi(), &extern_block, "extern", "extern block") + { + if let Some(str) = abi.string() { + Some(self.to_str_lit(str)) + } else { + None + } + } else { + None + }, + items: self.to_items(extern_block.extern_item_list(), "extern block"), + }), + ), + ra::ModuleItem::ExternCrateItem(extern_crate_item) => { + if let Some(alias) = extern_crate_item.alias() { + ( + self.to_ident_from_name_or_err(alias.name(), &alias, "extern crate alias"), + ast::ItemKind::ExternCrate(Some( + self.to_ident_from_name_ref_or_err( + extern_crate_item.name_ref(), + &extern_crate_item, + "extern crate reference", + ) + .name, + )), + ) + } else { + ( + self.to_ident_from_name_ref_or_err( + extern_crate_item.name_ref(), + &extern_crate_item, + "extern crate", + ), + ast::ItemKind::ExternCrate(None), + ) + } + } + ra::ModuleItem::FnDef(fn_def) => ( + self.to_ident_from_name_or_err(fn_def.name(), &fn_def, "function definition"), + ast::ItemKind::Fn( + self.to_defaultness(fn_def.default_kw()), + self.get_fn_sig(&fn_def), + self.get_generics_for_named( + &fn_def, + || fn_def.fn_kw().map(|element| inner_end(&element)), + || { + fn_def + .body() + .map(|element| end_of_prev(&element)) + .or(fn_def.semi().map(|element| end_of_prev(&element))) + }, + ), + fn_def + .body() + .and_then(|block_expr| block_expr.block()) + .map(|block| self.to_block(block)), + ), + ), + ra::ModuleItem::ImplDef(impl_def) => ( + ast::Ident::invalid(), + ast::ItemKind::Impl { + constness: self.to_constness(impl_def.const_kw()), + defaultness: self.to_defaultness(impl_def.default_kw()), + unsafety: self.to_unsafe(impl_def.unsafe_kw()), + polarity: match impl_def.excl() { + Some(excl) => ast::ImplPolarity::Negative(self.get_inner_span(&excl)), + None => ast::ImplPolarity::Positive, + }, + generics: self.get_generics_for_unnamed( + &impl_def, + || impl_def.impl_kw().map(|element| inner_end(&element)), + || impl_def.item_list().map(|element| end_of_prev(&element)), + ), + self_ty: self.to_ty_or_err( + impl_def.target_type(), + &impl_def, + "impl target type", + ), + of_trait: impl_def.target_trait().map(|trait_type_ref| ast::TraitRef { + ref_id: ast::DUMMY_NODE_ID, + path: self.to_trait_path_from_type(trait_type_ref), + }), + items: self.to_items(impl_def.item_list(), "impl"), + }, + ), + ra::ModuleItem::MacroCall(macro_call) => { + if let Some(item_kind) = self.get_item_kind_opt_from_macro_call(¯o_call) { + ( + self.to_ident_from_name_or_err( + macro_call.name(), + ¯o_call, + "macro_rules! macro definition", + ), + item_kind, + ) + } else { + (ast::Ident::invalid(), ast::ItemKind::MacCall(self.to_mac_call(macro_call))) + } + } + ra::ModuleItem::Module(module) => ( + self.to_ident_from_name_or_err(module.name(), &module, "module definition"), + ast::ItemKind::Mod(if let Some(item_list) = module.item_list() { + ast::Mod { + inner: self.to_span(TextRange::from_to( + item_list + .l_curly() + .map(|l_curly| start_of_next(&l_curly)) + .unwrap_or_else(|| inner_start(&item_list)), + item_list + .r_curly() + .map(|r_curly| inner_end(&r_curly)) + .unwrap_or_else(|| inner_end(&item_list)), + )), + items: self.to_items(Some(item_list), "module"), + inline: true, + } + } else { + ast::Mod { inner: Span::default(), items: Default::default(), inline: false } + }), + ), + ra::ModuleItem::StaticDef(static_def) => ( + self.to_ident_from_name_or_err( + static_def.name(), + &static_def, + "static variable definition", + ), + ast::ItemKind::Static( + self.to_ty_or_err( + static_def.ascribed_type(), + &static_def, + "static variable definition", + ), + self.to_mutability(static_def.mut_kw()), + static_def.body().map(|expr| self.to_expr(expr)), + ), + ), + ra::ModuleItem::StructDef(struct_def) => { + ( + self.to_ident_from_name_or_err(struct_def.name(), &struct_def, "struct definition"), + ast::ItemKind::Struct( + self.to_variant_data(struct_def.field_def_list()), + self.get_generics_for_named(&struct_def, + || struct_def.struct_kw().map(|element| inner_end(&element)), + || None + .or( + struct_def.field_def_list().and_then(|field_def_list| // where clauses are allowed after tuple struct fields + match field_def_list {ra::FieldDefList::RecordFieldDefList(record_field_def_list) => Some(record_field_def_list), _ => None} + ).map(|element| end_of_prev(&element)) + ).or(struct_def.semi().map(|element| end_of_prev(&element))) + ) + ) + ) + } + ra::ModuleItem::TraitDef(trait_def) => ( + self.to_ident_from_name_or_err(trait_def.name(), &trait_def, "trait definition"), + ast::ItemKind::Trait( + if trait_def.auto_kw().is_some() { ast::IsAuto::Yes } else { ast::IsAuto::No }, + self.to_unsafe(trait_def.unsafe_kw()), + self.get_generics_for_named( + &trait_def, + || trait_def.trait_kw().map(|element| inner_end(&element)), + || trait_def.item_list().map(|element| end_of_prev(&element)), + ), + self.to_generic_bounds(trait_def.type_bound_list()), + self.to_items(trait_def.item_list(), "trait definition"), + ), + ), + ra::ModuleItem::TypeAliasDef(type_alias_def) => ( + self.to_ident_from_name_or_err( + type_alias_def.name(), + &type_alias_def, + "type alias definition", + ), + ast::ItemKind::TyAlias( + self.to_defaultness(type_alias_def.default_kw()), + self.get_generics_for_named( + &type_alias_def, + || type_alias_def.type_kw().map(|element| inner_end(&element)), + || { + type_alias_def + .eq() + .map(|element| end_of_prev(&element)) + .or(type_alias_def.semi().map(|element| end_of_prev(&element))) + }, + ), + self.to_generic_bounds(type_alias_def.type_bound_list()), + type_alias_def.type_ref().map(|type_ref| self.to_ty(type_ref)), + ), + ), + ra::ModuleItem::UnionDef(union_def) => ( + self.to_ident_from_name_or_err(union_def.name(), &union_def, "union definition"), + ast::ItemKind::Union( + self.to_variant_data( + if let Some(record_field_def_list) = self.expect_in( + union_def.record_field_def_list(), + &union_def, + "field list", + "union", + ) { + Some(ra::FieldDefList::RecordFieldDefList(record_field_def_list)) + } else { + None + }, + ), + self.get_generics_for_named( + &union_def, + || union_def.union_kw().map(|element| inner_end(&element)), + || union_def.record_field_def_list().map(|element| end_of_prev(&element)), + ), + ), + ), + ra::ModuleItem::UseItem(use_item) => ( + ast::Ident::invalid(), + ast::ItemKind::Use(P(self.to_use_tree_or_err( + use_item.use_tree(), + &use_item, + "use", + ))), + ), + }; + if let Ok(kind) = kind.try_into() { + Some(P(ast::Item { + attrs, + id: ast::DUMMY_NODE_ID, // this is correct + span, + vis, + ident, + kind, + tokens, + })) + } else { + self.error_at_span(span, &format!("unexpected item kind in {}", container)); + None + } + } + + fn get_attributes<'a>( + &'a self, + node: &'a impl ra::AttrsOwner, + ) -> impl Iterator + 'a { + self.to_attributes(node.attr_or_comments()) + } + + fn to_mac_args_from_token_tree(&self, token_tree: ra::TokenTree) -> ast::MacArgs { + let left = self.expect_in( + token_tree.left_delimiter(), + &token_tree, + "left delimiter among '{', '(', '['", + "attribute value", + ); + let right = self.expect_in( + token_tree.right_delimiter(), + &token_tree, + "right delimiter among '}', ')', ']'", + "attribute value", + ); + + let ast_left = left.as_ref().map(|left| match left { + ra::LeftDelimiter::LParen(_) => ast::MacDelimiter::Parenthesis, + ra::LeftDelimiter::LBrack(_) => ast::MacDelimiter::Bracket, + ra::LeftDelimiter::LCurly(_) => ast::MacDelimiter::Brace, + }); + + let ast_right = right.as_ref().map(|right| match right { + ra::RightDelimiter::RParen(_) => ast::MacDelimiter::Parenthesis, + ra::RightDelimiter::RBrack(_) => ast::MacDelimiter::Bracket, + ra::RightDelimiter::RCurly(_) => ast::MacDelimiter::Brace, + }); + + let ast_delim = match (ast_left, ast_right) { + (Some(ast_left), Some(ast_right)) if ast_left == ast_right => Some(ast_left), + (Some(_), Some(_)) => { + self.missing_in( + &token_tree, + "right delimiter matching with left delimiter", + "attribute value", + ); + None + } + _ => None, + } + .unwrap_or(ast::MacDelimiter::Brace); + + let open_range = left + .map(|left| inner_range(&left)) + .unwrap_or_else(|| empty_text_range(inner_start(&token_tree))); + let close_range = right + .map(|right| inner_range(&right)) + .unwrap_or_else(|| empty_text_range(inner_end(&token_tree))); + let delim_span = rustc_ast::tokenstream::DelimSpan { + open: self.to_span(open_range), + close: self.to_span(close_range), + }; + ast::MacArgs::Delimited( + delim_span, + ast_delim, + self.get_tokens_from_range(TextRange::from_to(open_range.end(), close_range.start())) + .unwrap_or_else(|| TokenStream::default()), + ) + } + + fn to_attributes<'a>( + &'a self, + attr_or_comments: ra::AstChildElements, + ) -> impl Iterator + 'a { + attr_or_comments.filter_map(move |attr_or_comment: ra::AttrOrComment| match attr_or_comment + { + ra::AttrOrComment::Attr(attr) => { + let mac_args = match attr.input() { + Some(ra::AttrInput::TokenTree(token_tree)) => { + self.to_mac_args_from_token_tree(token_tree) + } + Some(ra::AttrInput::Literal(literal)) => ast::MacArgs::Eq( + self.expect_in(attr.eq(), &attr, "equals sign", "literal attribute") + .map(|eq| self.get_inner_span(&eq)) + .unwrap_or_else(|| self.get_inner_span(&literal).shrink_to_lo()), + self.to_tokens(literal).unwrap_or_else(|| TokenStream::default()), + ), + None => ast::MacArgs::Empty, + }; + + Some(rustc_ast::attr::mk_attr( + match attr.kind() { + ra::AttrKind::Inner => ast::AttrStyle::Inner, + ra::AttrKind::Outer => ast::AttrStyle::Outer, + }, + self.to_simple_path_or_err(attr.path(), &attr, "attribute path"), + mac_args, + self.get_inner_span(&attr), + )) + } + ra::AttrOrComment::Comment(comment) => comment.kind().doc.map(|doc| { + let span = self.get_inner_span(&comment); + rustc_ast::attr::mk_doc_comment( + match doc { + ra::CommentPlacement::Outer => ast::AttrStyle::Outer, + ra::CommentPlacement::Inner => ast::AttrStyle::Inner, + }, + self.to_symbol_from_token(comment), + span, + ) + }), + }) + } + + fn invalid_qself(&self, span: Span, container: &str) { + self.error_at_span( + span, + &format!("{} must not include a fully qualified path prefix", container), + ); + } + + fn to_simple_path(&self, path: ra::Path, container: &str) -> ast::Path { + let (qself, path) = self.to_qself_and_path(path); + if let Some((_qself, qself_span)) = qself { + self.invalid_qself(qself_span, container); + } + path + } + + fn to_simple_path_or_err( + &self, + path: Option, + element: &impl ra::AstElement, + container: &str, + ) -> ast::Path { + let (qself, path) = self.to_qself_and_path_or_err(path, element, container); + if let Some((_qself, qself_span)) = qself { + self.invalid_qself(qself_span, container); + } + path + } + + fn to_trait_path_from_path_type(&self, node: ra::PathType) -> ast::Path { + self.to_simple_path_or_err(node.path(), &node, "trait reference") + } + + fn to_trait_path_from_type(&self, node: ra::TypeRef) -> ast::Path { + match node { + ra::TypeRef::PathType(path_type) => self.to_trait_path_from_path_type(path_type), + _ => { + self.error(&node, "non-path type used as trait reference"); + self.mk_path_for_ident(ast::Ident::new(kw::Invalid, self.get_inner_span(&node))) + } + } + } + + fn to_qself_and_path_or_err( + &self, + path: Option, + element: &impl ra::AstElement, + container: &str, + ) -> (Option<(ast::QSelf, Span)>, ast::Path) { + self.expect_in(path, element, "path", container) + .map(|path| self.to_qself_and_path(path)) + .unwrap_or_else(|| { + ( + None, + ast::Path { span: self.get_inner_span(element), segments: Default::default() }, + ) + }) + } + + fn to_qself_and_path(&self, node: ra::Path) -> (Option<(ast::QSelf, Span)>, ast::Path) { + let mut qself = None; + let path = ast::Path { + span: self.get_inner_span(&node), + segments: { + let mut segments = Vec::new(); + let mut path_opt = Some(node); + while let Some(path) = path_opt { + let qualifier = path.qualifier(); + let ast_segment = self.expect_in(path.segment(), &path, "path segment", "path").map(|segment| { + let mut ast_segment = self.expect_in(segment.kind(), &segment, "valid path segment", "path segment").map(|kind| + match kind { + ra::PathSegmentKind::Name(name_ref) => Some(ast::PathSegment::from_ident(self.to_ident_from_name_ref(name_ref))), + ra::PathSegmentKind::CrateKw => Some(ast::PathSegment::from_ident(ast::Ident::new(kw::Crate, self.get_inner_span(&segment)))), + ra::PathSegmentKind::SuperKw => Some(ast::PathSegment::from_ident(ast::Ident::new(kw::Super, self.get_inner_span(&segment)))), + ra::PathSegmentKind::SelfKw => Some(ast::PathSegment::from_ident(ast::Ident::new(kw::SelfLower, self.get_inner_span(&segment)))), + ra::PathSegmentKind::Type {type_ref, trait_ref} => { + if qualifier.is_some() { + self.error(&segment, "a qualified path segment can only be the first segment in a path"); + } else { + let mut path_span = if let Some(type_ref) = type_ref.as_ref() { + self.get_inner_span(type_ref).shrink_to_hi() + } else { + // TODO: is this correct? it should be + if let Some(l_angle) = segment.l_angle() { + self.get_inner_span(&l_angle).shrink_to_lo() + } else { + self.get_inner_span(&segment).shrink_to_lo() + } + }; + let mut position = 0; + if let Some(trait_ref) = trait_ref { + let trait_path = self.to_trait_path_from_path_type(trait_ref); + for trait_segment in trait_path.segments.into_iter().rev() { + segments.push(trait_segment); + position += 1; + } + path_span = trait_path.span; + } + qself = Some((ast::QSelf { + path_span, + ty: self.to_ty_or_err(type_ref, &segment, "fully qualified path segment"), + position + }, self.get_inner_span(&segment))); + } + None + } + } + ).unwrap_or_else(|| Some(ast::PathSegment { + id: ast::DUMMY_NODE_ID, + ident: ast::Ident::new(kw::Invalid, self.get_inner_span(&segment)), + args: None + })); + + if let Some(ast_segment) = ast_segment.as_mut() { + if let Some(type_arg_list) = segment.type_arg_list() { + ast_segment.args = Some(P(ast::GenericArgs::AngleBracketed(self.to_angle_bracketed_args(&type_arg_list)))) + } else if let Some(param_list) = segment.param_list() { + ast_segment.args = Some(P(ast::GenericArgs::Parenthesized(ast::ParenthesizedArgs { + span: self.to_span(TextRange::from_to(inner_start(&segment), inner_end(¶m_list))), + inputs: self.to_tys(param_list), + output: self.to_fn_ret_ty(segment.ret_type(), start_of_next(&segment)), + }))); + } + } + ast_segment + }).unwrap_or_else(|| Some(ast::PathSegment { + id: ast::DUMMY_NODE_ID, + ident: ast::Ident::new(kw::Invalid, self.get_inner_span(&path)), + args: None + })); + + if let Some(ast_segment) = ast_segment { + segments.push(ast_segment); + } + path_opt = qualifier; + if path_opt.is_none() { + match path.segment().and_then(|x| x.coloncolon()) { + Some(coloncolon) => segments.push(ast::PathSegment::path_root( + self.get_inner_span(&coloncolon), + )), + _ => {} + } + } + } + segments.reverse(); + segments + }, + }; + (qself, path) + } + + fn to_fn_ret_ty(&self, ret_type: Option, end: TextUnit) -> ast::FnRetTy { + if let Some(ret_type) = ret_type { + ast::FnRetTy::Ty(self.to_ty_or_err(ret_type.type_ref(), &ret_type, "return type")) + } else { + ast::FnRetTy::Default(self.to_span_from_unit(end)) + } + } + + fn to_tys(&self, param_list: ra::ParamList) -> Vec> { + param_list + .params() + .map(|param| self.to_ty_or_err(param.ascribed_type(), ¶m, "parameter")) + .collect() + } + + fn to_ty_or_err( + &self, + type_ref_opt: Option, + element: &impl ra::AstElement, + container: &str, + ) -> P { + self.expect_in(type_ref_opt, element, "type", container) + .map(|type_ref| self.to_ty(type_ref)) + .unwrap_or_else(|| { + P(ast::Ty { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(element), + kind: ast::TyKind::Err, + }) + }) + } + + fn get_token(&self, token: &impl ra::AstToken) -> token::Token { + let mut srdr = lexer::StringReader::retokenize(&self.sess, self.get_inner_span(token)); + srdr.next_token() + } + + fn to_lit_or_err( + &self, + token: Option, + element: &impl ra::AstElement, + container: &str, + ) -> ast::Lit { + self.expect_in(token, element, "literal", container) + .map(|token| self.to_lit(token)) + .unwrap_or_else(|| ast::Lit { + span: self.get_inner_span(element), + kind: ast::LitKind::Err(kw::Invalid), + token: token::Lit { kind: token::LitKind::Err, suffix: None, symbol: kw::Invalid }, + }) + } + + fn to_lit(&self, token: ra::LiteralToken) -> ast::Lit { + let span = self.get_inner_span(&token); + ast::Lit::from_token(&self.get_token(&token)).unwrap_or_else(|_| { + self.error_at_span(span, "invalid literal token"); + ast::Lit { + span, + kind: ast::LitKind::Err(kw::Invalid), + token: token::Lit { kind: token::LitKind::Err, suffix: None, symbol: kw::Invalid }, + } + }) + } + + fn to_str_lit(&self, string: ra::String) -> ast::StrLit { + let lit = self.to_lit(ra::LiteralToken::String(string)); + match lit.kind { + ast::LitKind::Str(symbol_unescaped, style) => ast::StrLit { + span: lit.span, + style, + symbol: lit.token.symbol, + suffix: lit.token.suffix, + symbol_unescaped, + }, + _ => ast::StrLit { + span: lit.span, + style: ast::StrStyle::Cooked, + symbol: lit.token.symbol, + suffix: lit.token.suffix, + symbol_unescaped: lit.token.symbol, + }, + } + } + + fn to_async(&self, async_kw: Option) -> ast::Async { + if let Some(async_kw) = async_kw { + ast::Async::Yes { + span: self.get_inner_span(&async_kw), + closure_id: ast::DUMMY_NODE_ID, + return_impl_trait_id: ast::DUMMY_NODE_ID, + } + } else { + ast::Async::No + } + } + + fn to_extern(&self, abi: Option) -> ast::Extern { + if let Some(abi) = abi { + if let Some(str) = abi.string() { + ast::Extern::Explicit(self.to_str_lit(str)) + } else { + ast::Extern::Implicit + } + } else { + ast::Extern::None + } + } + + fn to_unsafe(&self, unsafe_kw: Option) -> ast::Unsafe { + if let Some(unsafe_kw) = unsafe_kw { + ast::Unsafe::Yes(self.get_inner_span(&unsafe_kw)) + } else { + ast::Unsafe::No + } + } + + fn to_bare_fn_ty( + &self, + fn_pointer_type: ra::FnPointerType, + type_param_list_opt: Option, + ) -> P { + P(ast::BareFnTy { + generic_params: self.to_generic_params(type_param_list_opt), + unsafety: self.to_unsafe(fn_pointer_type.unsafe_kw()), + ext: self.to_extern(fn_pointer_type.abi()), + decl: self.to_fn_decl( + fn_pointer_type.param_list(), + fn_pointer_type.ret_type(), + start_of_next(&fn_pointer_type), + FnParamMode { require_pat: false, require_type: true }, + ), + }) + } + + fn to_fn_decl( + &self, + param_list: Option, + ret_type: Option, + end: TextUnit, + fn_param_mode: FnParamMode, + ) -> P { + P(ast::FnDecl { + inputs: self.to_params(param_list, fn_param_mode), + output: self.to_fn_ret_ty(ret_type, end), + }) + } + + fn to_pat_or_err( + &self, + pat_opt: Option, + element: &impl ra::AstElement, + container: &str, + ) -> P { + self.expect_in(pat_opt, element, "pattern", container) + .map(|pat| self.to_pat(pat)) + .unwrap_or_else(|| { + let span = self.get_inner_span(element); + P(ast::Pat { + id: ast::DUMMY_NODE_ID, + span, + kind: ast::PatKind::Ident( + ast::BindingMode::ByValue(ast::Mutability::Not), + ast::Ident::new(kw::Invalid, span), + None, + ), + }) + }) + } + + fn to_mutability(&self, mut_kw: Option) -> ast::Mutability { + if mut_kw.is_some() { + ast::Mutability::Mut + } else { + ast::Mutability::Not + } + } + + fn to_mutability_from_const(&self, const_kw: Option) -> ast::Mutability { + if const_kw.is_none() { + ast::Mutability::Mut + } else { + ast::Mutability::Not + } + } + + fn to_borrow_kind(&self, raw_kw: Option) -> ast::BorrowKind { + if raw_kw.is_some() { + ast::BorrowKind::Raw + } else { + ast::BorrowKind::Ref + } + } + + fn to_binding_mode( + &self, + ref_kw: Option, + mut_kw: Option, + ) -> ast::BindingMode { + let mutability = self.to_mutability(mut_kw); + if ref_kw.is_some() { + ast::BindingMode::ByRef(mutability) + } else { + ast::BindingMode::ByValue(mutability) + } + } + + fn to_range_end(&self, separator: ra::RangeSeparator) -> ast::RangeEnd { + match separator { + ra::RangeSeparator::Dotdot(_) => ast::RangeEnd::Excluded, + ra::RangeSeparator::Dotdotdot(_) => { + ast::RangeEnd::Included(ast::RangeSyntax::DotDotDot) + } + ra::RangeSeparator::Dotdoteq(_) => ast::RangeEnd::Included(ast::RangeSyntax::DotDotEq), + } + } + + fn to_range_end_or_err( + &self, + separator: Option, + element: &impl ra::AstElement, + container: &str, + ) -> ast::RangeEnd { + if let Some(separator) = self.expect_in(separator, element, "range separator", container) { + self.to_range_end(separator) + } else { + ast::RangeEnd::Excluded + } + } + + // currently only for usage to parse RangeExpr + fn to_expr_from_pat(&self, pat: ra::Pat) -> P { + match pat { + ra::Pat::LiteralPat(literal_pat) => self.to_expr_from_literal_pat(literal_pat), + ra::Pat::BindPat(bind_pat) => { + self.invalid_token(bind_pat.ref_kw(), "expression inside pattern"); + self.invalid_token(bind_pat.mut_kw(), "expression inside pattern"); + self.invalid_element( + bind_pat.pat(), + &bind_pat, + "guarded pattern", + "expression inside pattern", + ); + self.mk_expr_for_ident(self.to_ident_from_name_or_err( + bind_pat.name(), + &bind_pat, + "expression inside pattern", + )) + } + ra::Pat::PathPat(path_pat) => { + let (qself, path) = + self.to_qself_and_path_or_err(path_pat.path(), &path_pat, "path pattern"); + P(ast::Expr { + id: ast::DUMMY_NODE_ID, + attrs: Default::default(), + span: self.get_inner_span(&path_pat), + kind: ast::ExprKind::Path(qself.map(|(qself, _)| qself), path), + }) + } + node => { + self.error(&node, "invalid expression inside pattern"); + P(ast::Expr { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(&node), + attrs: Default::default(), + kind: ast::ExprKind::Err, + }) + } + } + } + + fn to_expr_from_condition_or_err( + &self, + condition: Option, + element: &impl ra::AstElement, + container: &str, + ) -> P { + if let Some(condition) = self.expect_in(condition, element, "condition", container) { + let expr = self.to_expr_or_err(condition.expr(), &condition, "condition expression"); + if let Some(pat) = condition.pat() { + P(ast::Expr { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(&condition), + attrs: Default::default(), + kind: ast::ExprKind::Let(self.to_pat(pat), expr), + }) + } else { + expr + } + } else { + P(ast::Expr { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(element), + attrs: Default::default(), + kind: ast::ExprKind::Err, + }) + } + } + + fn to_expr_from_literal_pat(&self, literal_pat: ra::LiteralPat) -> P { + P(ast::Expr { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(&literal_pat), + attrs: Default::default(), + kind: ast::ExprKind::Lit(self.to_lit_or_err( + literal_pat.literal().and_then(|literal| literal.literal_token()), + &literal_pat, + "literal pattern", + )), + }) + } + + fn to_pat(&self, pat: ra::Pat) -> P { + let span = self.get_inner_span(&pat); + let kind = match pat { + ra::Pat::BindPat(bind_pat) => ast::PatKind::Ident( + self.to_binding_mode(bind_pat.ref_kw(), bind_pat.mut_kw()), + self.to_ident_from_name_or_err(bind_pat.name(), &bind_pat, "bind pattern"), + bind_pat.pat().map(|inner_pat| self.to_pat(inner_pat)), + ), + ra::Pat::LiteralPat(literal_pat) => { + ast::PatKind::Lit(self.to_expr_from_literal_pat(literal_pat)) + } + ra::Pat::OrPat(or_pat) => { + ast::PatKind::Or(or_pat.pats().map(|inner_pat| self.to_pat(inner_pat)).collect()) + } + ra::Pat::PathPat(path_pat) => { + let (qself, path) = + self.to_qself_and_path_or_err(path_pat.path(), &path_pat, "path pattern"); + ast::PatKind::Path(qself.map(|(qself, _)| qself), path) + } + ra::Pat::RefPat(ref_pat) => ast::PatKind::Ref( + self.to_pat_or_err(ref_pat.pat(), &ref_pat, "reference pattern"), + self.to_mutability(ref_pat.mut_kw()), + ), + ra::Pat::TuplePat(tuple_pat) => ast::PatKind::Tuple( + tuple_pat.args().map(|inner_pat| self.to_pat(inner_pat)).collect(), + ), + ra::Pat::TupleStructPat(tuple_struct_pat) => ast::PatKind::TupleStruct( + self.to_simple_path_or_err( + tuple_struct_pat.path(), + &tuple_struct_pat, + "tuple struct pattern", + ), + tuple_struct_pat.args().map(|arg| self.to_pat(arg)).collect(), + ), + ra::Pat::PlaceholderPat(_placeholder_pat) => ast::PatKind::Wild, + ra::Pat::ParenPat(paren_pat) => ast::PatKind::Paren(self.to_pat_or_err( + paren_pat.pat(), + &paren_pat, + "parenthesized pattern", + )), + ra::Pat::BoxPat(box_pat) => { + ast::PatKind::Box(self.to_pat_or_err(box_pat.pat(), &box_pat, "box pattern")) + } + ra::Pat::DotDotPat(_) => ast::PatKind::Rest, + ra::Pat::SlicePat(slice_pat) => { + ast::PatKind::Slice(slice_pat.args().map(|arg| self.to_pat(arg)).collect()) + } + ra::Pat::RangePat(range_pat) => { + let separator = range_pat.range_separator(); + ast::PatKind::Range( + range_pat.start().map(|inner_pat| self.to_expr_from_pat(inner_pat)), + range_pat.end().map(|inner_pat| self.to_expr_from_pat(inner_pat)), + Spanned { + span: if let Some(separator) = separator.as_ref() { + self.get_inner_span(separator) + } else { + self.get_inner_span(&range_pat) + }, + node: self.to_range_end_or_err(separator, &range_pat, "range pattern"), + }, + ) + } + ra::Pat::RecordPat(record_pat) => { + let (fields, dotdot) = + if let Some(record_field_pat_list) = record_pat.record_field_pat_list() { + ( + record_field_pat_list + .pats() + .map(|inner_pat| match inner_pat { + ra::RecordInnerPat::RecordFieldPat(record_field_pat) => { + ast::FieldPat { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(&record_field_pat), + attrs: self + .get_attributes(&record_field_pat) + .collect::>() + .into(), + pat: self.to_pat_or_err( + record_field_pat.pat(), + &record_field_pat, + "record field pattern", + ), + ident: self.to_ident_from_name_or_err( + record_field_pat.name(), + &record_field_pat, + "record field pattern", + ), + is_shorthand: false, + is_placeholder: false, + } + } + ra::RecordInnerPat::BindPat(bind_pat) => ast::FieldPat { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(&bind_pat), + attrs: self + .get_attributes(&bind_pat) + .collect::>() + .into(), + ident: self.to_ident_from_name_or_err( + bind_pat.name(), + &bind_pat, + "bind pattern", + ), + pat: self.to_pat(ra::Pat::BindPat(bind_pat)), + is_shorthand: true, + is_placeholder: false, + }, + }) + .collect(), + record_field_pat_list.dotdot().is_some(), + ) + } else { + (Default::default(), false) + }; + + ast::PatKind::Struct( + self.to_simple_path_or_err(record_pat.path(), &record_pat, "record pattern"), + fields, + dotdot, + ) + } + ra::Pat::MacroCall(macro_call) => ast::PatKind::MacCall(self.to_mac_call(macro_call)), + }; + P(ast::Pat { id: ast::DUMMY_NODE_ID, span, kind }) + } + + fn to_param_from_self_param(&self, self_param: ra::SelfParam) -> ast::Param { + let self_ident = if let Some(self_kw) = + self.expect_in(self_param.self_kw(), &self_param, "self keyword", "self parameter") + { + self.to_ident_from_token(self_kw) + } else { + ast::Ident::new(kw::Invalid, self.get_inner_span(&self_param)) + }; + + let span = self.get_inner_span(&self_param); + ast::Param { + id: ast::DUMMY_NODE_ID, + span, + attrs: self.get_attributes(&self_param).collect::>().into(), + ty: if let Some(self_type) = self_param.ascribed_type() { + self.to_ty(self_type) + } else { + let implicit_self_ty = + P(ast::Ty { id: ast::DUMMY_NODE_ID, span, kind: ast::TyKind::ImplicitSelf }); + if let Some(_) = self_param.amp() { + P(ast::Ty { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(&self_param), + kind: ast::TyKind::Rptr( + self_param.lifetime().map(|t| self.to_lifetime(t)), + ast::MutTy { + mutbl: self.to_mutability(self_param.amp_mut_kw()), + ty: implicit_self_ty, + }, + ), + }) + } else { + implicit_self_ty + } + }, + pat: P(ast::Pat { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(&self_param), + kind: ast::PatKind::Ident( + ast::BindingMode::ByValue(self.to_mutability(self_param.mut_kw())), + self_ident, + None, + ), + }), + is_placeholder: false, + } + } + + fn to_param(&self, param: ra::Param, fn_param_mode: FnParamMode) -> ast::Param { + if let Some(dotdotdot) = param.dotdotdot() { + let span = self.get_inner_span(&dotdotdot); + ast::Param { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_to_next_span(¶m), + attrs: self.get_attributes(¶m).collect::>().into(), + ty: P(ast::Ty { id: ast::DUMMY_NODE_ID, span, kind: ast::TyKind::CVarArgs }), + pat: P(ast::Pat { + id: ast::DUMMY_NODE_ID, + span, + kind: ast::PatKind::Ident( + ast::BindingMode::ByValue(ast::Mutability::Not), + ast::Ident::from_str_and_span("", span), + None, + ), + }), + is_placeholder: false, + } + } else { + ast::Param { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_to_next_span(¶m), + attrs: self.get_attributes(¶m).collect::>().into(), + ty: if fn_param_mode.require_type { + self.to_ty_or_err(param.ascribed_type(), ¶m, "parameter") + } else { + param.ascribed_type().map(|type_ref| self.to_ty(type_ref)).unwrap_or_else( + || { + P(ast::Ty { + id: ast::DUMMY_NODE_ID, + span: self.get_last_token_span(¶m), + kind: ast::TyKind::Infer, + }) + }, + ) + }, + pat: if fn_param_mode.require_pat { + self.to_pat_or_err(param.pat(), ¶m, "parameter") + } else { + let span = self.get_inner_span(¶m); + param.pat().map(|pat| self.to_pat(pat)).unwrap_or_else(|| { + P(ast::Pat { + id: ast::DUMMY_NODE_ID, + span, + kind: ast::PatKind::Ident( + ast::BindingMode::ByValue(ast::Mutability::Not), + ast::Ident::from_str_and_span("", span), + None, + ), + }) + }) + }, + is_placeholder: false, + } + } + } + + fn to_params( + &self, + param_list: Option, + fn_param_mode: FnParamMode, + ) -> Vec { + if let Some(param_list) = param_list { + if let Some(self_param) = param_list.self_param() { + Some(self.to_param_from_self_param(self_param)) + } else { + None + } + .into_iter() + .chain(param_list.params().map(|param| self.to_param(param, fn_param_mode))) + .collect() + } else { + Default::default() + } + } + + fn to_ty(&self, mut type_ref: ra::TypeRef) -> P { + loop { + let span = self.get_inner_span(&type_ref); + let kind = match type_ref { + ra::TypeRef::PathType(path_type) => { + let (qself, path) = + self.to_qself_and_path_or_err(path_type.path(), &path_type, "path type"); + ast::TyKind::Path(qself.map(|(qself, _)| qself), path) + } + ra::TypeRef::NeverType(_) => ast::TyKind::Never, + ra::TypeRef::PlaceholderType(_) => ast::TyKind::Infer, + ra::TypeRef::TupleType(tuple_type) => { + ast::TyKind::Tup(tuple_type.fields().map(|field| self.to_ty(field)).collect()) + } + ra::TypeRef::ArrayType(array_type) => ast::TyKind::Array( + self.to_ty_or_err(array_type.type_ref(), &array_type, "array type"), + self.to_anon_const_or_err(array_type.expr(), &array_type, "array type size"), + ), + ra::TypeRef::SliceType(slice_type) => ast::TyKind::Slice(self.to_ty_or_err( + slice_type.type_ref(), + &slice_type, + "slice type", + )), + ra::TypeRef::PointerType(pointer_type) => ast::TyKind::Ptr(ast::MutTy { + ty: self.to_ty_or_err(pointer_type.type_ref(), &pointer_type, "pointer type"), + mutbl: self.to_mutability_from_const(pointer_type.const_kw()), + }), + ra::TypeRef::ReferenceType(reference_type) => ast::TyKind::Rptr( + reference_type.lifetime().map(|lifetime| self.to_lifetime(lifetime)), + ast::MutTy { + ty: self.to_ty_or_err( + reference_type.type_ref(), + &reference_type, + "reference type", + ), + mutbl: self.to_mutability(reference_type.mut_kw()), + }, + ), + ra::TypeRef::ParenType(paren_type) => ast::TyKind::Paren(self.to_ty_or_err( + paren_type.type_ref(), + &paren_type, + "parenthesized type", + )), + ra::TypeRef::DynTraitType(dyn_trait_type) => ast::TyKind::TraitObject( + self.to_generic_bounds(dyn_trait_type.type_bound_list()), + if dyn_trait_type.dyn_kw().is_some() { + ast::TraitObjectSyntax::Dyn + } else { + ast::TraitObjectSyntax::None + }, + ), + ra::TypeRef::ImplTraitType(impl_trait_type) => ast::TyKind::ImplTrait( + ast::DUMMY_NODE_ID, + self.to_generic_bounds(impl_trait_type.type_bound_list()), + ), + ra::TypeRef::FnPointerType(fn_pointer_type) => { + ast::TyKind::BareFn(self.to_bare_fn_ty(fn_pointer_type, None)) + } + ra::TypeRef::ForType(for_type) => { + if let Some(inner_type_ref) = for_type.type_ref() { + match inner_type_ref { + ra::TypeRef::FnPointerType(fn_pointer_type) => ast::TyKind::BareFn( + self.to_bare_fn_ty(fn_pointer_type, for_type.type_param_list()), + ), + inner_type_ref => { + // other cases, only allowed in where clauses, are handled by special casing the for-type + self.missing_in(&for_type, "fn() type", "for<> type"); + type_ref = inner_type_ref; + continue; + } + } + } else { + self.missing_in(&for_type, "inner type", "for<> type"); + ast::TyKind::Err + } + } + }; + return P(ast::Ty { id: ast::DUMMY_NODE_ID, span, kind }); + } + } + + fn to_lifetime_or_err( + &self, + lifetime_opt: Option, + element: &impl ra::AstElement, + container: &str, + ) -> ast::Lifetime { + self.expect_in(lifetime_opt, element, "lifetime", container) + .map(|lifetime| self.to_lifetime(lifetime)) + .unwrap_or_else(|| ast::Lifetime { + id: ast::DUMMY_NODE_ID, + ident: ast::Ident::new(kw::Invalid, self.get_inner_span(element)), + }) + } + + fn to_lifetime(&self, lifetime: ra::Lifetime) -> ast::Lifetime { + ast::Lifetime { id: ast::DUMMY_NODE_ID, ident: self.to_ident_from_token(lifetime) } + } + + fn to_label(&self, label: ra::Label) -> ast::Label { + ast::Label { ident: self.to_lifetime_or_err(label.lifetime(), &label, "label").ident } + } + + fn to_label_from_lifetime(&self, lifetime: ra::Lifetime) -> ast::Label { + ast::Label { ident: self.to_lifetime(lifetime).ident } + } + + fn to_mac_call(&self, macro_call: ra::MacroCall) -> ast::MacCall { + self.invalid_element( + macro_call.name(), + ¯o_call, + "name", + "macro call other than macro_rules!", + ); + ast::MacCall { + path: self.to_simple_path_or_err(macro_call.path(), ¯o_call, "invoked macro name"), + args: P(if let Some(token_tree) = macro_call.token_tree() { + self.to_mac_args_from_token_tree(token_tree) + } else { + self.error(¯o_call, "macro calls must have arguments"); + ast::MacArgs::Empty + }), + // TODO: this is just used for a better error in a very niche case, maybe we should somehow support it anyway + prior_type_ascription: None, + } + } + + fn mk_path_for_ident(&self, ident: ast::Ident) -> ast::Path { + ast::Path { + span: ident.span, + segments: vec![ast::PathSegment { + id: ast::DUMMY_NODE_ID, + args: Default::default(), + ident, + }], + } + } + + fn mk_expr_for_ident(&self, ident: ast::Ident) -> P { + P(ast::Expr { + id: ast::DUMMY_NODE_ID, + attrs: Default::default(), + span: ident.span, + kind: ast::ExprKind::Path(None, self.mk_path_for_ident(ident)), + }) + } + + fn to_expr(&self, expr: ra::Expr) -> P { + self.to_expr_from_expr_and_expr_stmt_opt(expr, None) + } + + fn to_expr_from_expr_and_expr_stmt_opt( + &self, + expr: ra::Expr, + expr_stmt: Option, + ) -> P { + let span = self.get_inner_span(&expr); + let attrs = expr_stmt + .iter() + .flat_map(|expr_stmt| self.get_attributes(expr_stmt)) + .chain(self.get_attributes(&expr)) + .collect::>() + .into(); + let kind = match expr { + ra::Expr::ArrayExpr(array_expr) => { + if let Some(_semi) = array_expr.semi() { + let mut exprs = array_expr.exprs(); + let expr_kind = ast::ExprKind::Repeat( + self.to_expr_or_err( + exprs.next(), + &array_expr, + "array repeat expression value", + ), + self.to_anon_const_or_err( + exprs.next(), + &array_expr, + "array repeat expression repetition count", + ), + ); + self.invalid_element( + exprs.next(), + &array_expr, + "3rd expression", + "array repeat expression", + ); + expr_kind + } else { + ast::ExprKind::Array( + array_expr.exprs().map(|expr| self.to_expr(expr)).collect(), + ) + } + } + ra::Expr::AwaitExpr(await_expr) => ast::ExprKind::Await(self.to_expr_or_err( + await_expr.expr(), + &await_expr, + "await expression", + )), + ra::Expr::BinExpr(bin_expr) => { + if let Some(bin_op) = self.expect_in( + bin_expr.bin_op(), + &bin_expr, + "recognized binary operator", + "binary expression", + ) { + let span = self.get_inner_span(&bin_op); + let lhs = self.to_expr_or_err( + bin_expr.lhs(), + &bin_expr, + "left-hand side of binary operator", + ); + let rhs = self.to_expr_or_err( + bin_expr.rhs(), + &bin_expr, + "right-hand side of binary operator", + ); + enum Kind { + Assign, + AssignOp, + Binary, + }; + + // TODO: maybe rewrite to match bin_op directly + let (node, kind) = match bin_expr.op_kind().unwrap() { + ra::BinOp::Assignment => (ast::BinOpKind::Eq, Kind::Assign), + + ra::BinOp::AddAssign => (ast::BinOpKind::Add, Kind::AssignOp), + ra::BinOp::BitAndAssign => (ast::BinOpKind::BitAnd, Kind::AssignOp), + ra::BinOp::BitOrAssign => (ast::BinOpKind::BitOr, Kind::AssignOp), + ra::BinOp::BitXorAssign => (ast::BinOpKind::BitXor, Kind::AssignOp), + ra::BinOp::DivAssign => (ast::BinOpKind::Div, Kind::AssignOp), + ra::BinOp::MulAssign => (ast::BinOpKind::Mul, Kind::AssignOp), + ra::BinOp::RemAssign => (ast::BinOpKind::Rem, Kind::AssignOp), + ra::BinOp::ShlAssign => (ast::BinOpKind::Shl, Kind::AssignOp), + ra::BinOp::ShrAssign => (ast::BinOpKind::Shr, Kind::AssignOp), + ra::BinOp::SubAssign => (ast::BinOpKind::Sub, Kind::AssignOp), + + ra::BinOp::Addition => (ast::BinOpKind::Add, Kind::Binary), + ra::BinOp::BitwiseAnd => (ast::BinOpKind::BitAnd, Kind::Binary), + ra::BinOp::BitwiseOr => (ast::BinOpKind::BitOr, Kind::Binary), + ra::BinOp::BitwiseXor => (ast::BinOpKind::BitXor, Kind::Binary), + ra::BinOp::BooleanAnd => (ast::BinOpKind::And, Kind::Binary), + ra::BinOp::BooleanOr => (ast::BinOpKind::Or, Kind::Binary), + ra::BinOp::Division => (ast::BinOpKind::Div, Kind::Binary), + ra::BinOp::EqualityTest => (ast::BinOpKind::Eq, Kind::Binary), + ra::BinOp::GreaterEqualTest => (ast::BinOpKind::Ge, Kind::Binary), + ra::BinOp::GreaterTest => (ast::BinOpKind::Gt, Kind::Binary), + ra::BinOp::LeftShift => (ast::BinOpKind::Shl, Kind::Binary), + ra::BinOp::LesserEqualTest => (ast::BinOpKind::Le, Kind::Binary), + ra::BinOp::LesserTest => (ast::BinOpKind::Lt, Kind::Binary), + ra::BinOp::Multiplication => (ast::BinOpKind::Mul, Kind::Binary), + ra::BinOp::NegatedEqualityTest => (ast::BinOpKind::Ne, Kind::Binary), + ra::BinOp::Remainder => (ast::BinOpKind::Rem, Kind::Binary), + ra::BinOp::RightShift => (ast::BinOpKind::Shr, Kind::Binary), + ra::BinOp::Subtraction => (ast::BinOpKind::Sub, Kind::Binary), + }; + + match kind { + Kind::Assign => ast::ExprKind::Assign(lhs, rhs, span), + Kind::AssignOp => ast::ExprKind::AssignOp(Spanned { span, node }, lhs, rhs), + Kind::Binary => ast::ExprKind::Binary(Spanned { span, node }, lhs, rhs), + } + } else { + ast::ExprKind::Err + } + } + ra::Expr::BlockExpr(block_expr) => { + let label = block_expr.label().map(|label| self.to_label(label)); + ast::ExprKind::Block(self.to_block_from_block_expr(block_expr), label) + } + ra::Expr::BoxExpr(box_expr) => ast::ExprKind::Box(self.to_expr_or_err( + box_expr.expr(), + &box_expr, + "box expression", + )), + ra::Expr::BreakExpr(break_expr) => ast::ExprKind::Break( + break_expr.lifetime().map(|lifetime| self.to_label_from_lifetime(lifetime)), + break_expr.expr().map(|expr| self.to_expr(expr)), + ), + ra::Expr::CallExpr(call_expr) => ast::ExprKind::Call( + self.to_expr_or_err(call_expr.expr(), &call_expr, "function to be called"), + call_expr + .arg_list() + .into_iter() + .flat_map(|arg_list| arg_list.args().map(|expr| self.to_expr(expr))) + .collect(), + ), + ra::Expr::CastExpr(cast_expr) => ast::ExprKind::Cast( + self.to_expr_or_err(cast_expr.expr(), &cast_expr, "cast expression"), + self.to_ty_or_err(cast_expr.type_ref(), &cast_expr, "cast expression"), + ), + ra::Expr::ContinueExpr(continue_expr) => ast::ExprKind::Continue( + continue_expr.lifetime().map(|lifetime| self.to_label_from_lifetime(lifetime)), + ), + ra::Expr::FieldExpr(field_expr) => ast::ExprKind::Field( + self.to_expr_or_err(field_expr.expr(), &field_expr, "field access expression"), + self.to_ident_from_name_ref_or_err( + field_expr.name_ref(), + &field_expr, + "field access expression", + ), + ), + ra::Expr::ForExpr(for_expr) => ast::ExprKind::ForLoop( + self.to_pat_or_err(for_expr.pat(), &for_expr, "for loop"), + self.to_expr_or_err(for_expr.iterable(), &for_expr, "for loop"), + self.to_block_or_err( + for_expr.loop_body().and_then(|block_expr| block_expr.block()), + &for_expr, + "for loop", + ), + for_expr.label().map(|label| self.to_label(label)), + ), + ra::Expr::IfExpr(if_expr) => ast::ExprKind::If( + self.to_expr_from_condition_or_err(if_expr.condition(), &if_expr, "if conditional"), + self.to_block_or_err( + if_expr.then_branch().and_then(|block_expr| block_expr.block()), + &if_expr, + "if conditional", + ), + if_expr.else_branch().map(|else_branch| match else_branch { + ra::ElseBranch::Block(else_expr) => { + self.to_expr(ra::Expr::BlockExpr(else_expr)) + } + ra::ElseBranch::IfExpr(else_if_expr) => { + self.to_expr(ra::Expr::IfExpr(else_if_expr)) + } + }), + ), + ra::Expr::IndexExpr(index_expr) => ast::ExprKind::Index( + self.to_expr_or_err(index_expr.base(), &index_expr, "index expression base"), + self.to_expr_or_err(index_expr.index(), &index_expr, "index expression index"), + ), + ra::Expr::Label(label) => { + self.error(&label, "unexpected label expression in generic context"); + ast::ExprKind::Err + } + ra::Expr::LambdaExpr(lambda_expr) => { + let end = if let Some(body) = lambda_expr.body() { + inner_start(&body) + } else { + start_of_next(&lambda_expr) + }; + ast::ExprKind::Closure( + if lambda_expr.move_kw().is_some() { + ast::CaptureBy::Value + } else { + ast::CaptureBy::Ref + }, + self.to_async(lambda_expr.async_kw()), + if lambda_expr.static_kw().is_some() { + ast::Movability::Static + } else { + ast::Movability::Movable + }, + self.to_fn_decl( + lambda_expr.param_list(), + lambda_expr.ret_type(), + end, + FnParamMode { require_pat: true, require_type: false }, + ), + self.to_expr_or_err(lambda_expr.body(), &lambda_expr, "closure body"), + if let Some(param_list) = lambda_expr.param_list() { + self.get_inner_span(¶m_list) + } else { + self.get_inner_span(&lambda_expr) + }, + ) + } + ra::Expr::Literal(literal) => ast::ExprKind::Lit(self.to_lit_or_err( + literal.literal_token(), + &literal, + "literal expression", + )), + ra::Expr::LoopExpr(loop_expr) => ast::ExprKind::Loop( + self.to_block_or_err( + loop_expr.loop_body().and_then(|block_expr| block_expr.block()), + &loop_expr, + "loop", + ), + loop_expr.label().map(|label| self.to_label(label)), + ), + ra::Expr::MacroCall(macro_call) => ast::ExprKind::MacCall(self.to_mac_call(macro_call)), + ra::Expr::MatchExpr(match_expr) => ast::ExprKind::Match( + self.to_expr_or_err(match_expr.expr(), &match_expr, "match selector"), + if let Some(match_arm_list) = match_expr.match_arm_list() { + match_arm_list + .arms() + .map(|match_arm| ast::Arm { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_to_next_span(&match_arm), + attrs: self.get_attributes(&match_arm).collect(), + pat: self.to_pat_or_err( + match_arm.pat(), + &match_arm, + "match arm condition", + ), + guard: match_arm.guard().map(|guard| { + self.to_expr_or_err(guard.expr(), &guard, "match guard") + }), + body: self.to_expr_or_err( + match_arm.expr(), + &match_arm, + "match arm body", + ), + is_placeholder: false, + }) + .collect() + } else { + Default::default() + }, + ), + ra::Expr::MethodCallExpr(method_call_expr) => ast::ExprKind::MethodCall( + ast::PathSegment { + id: ast::DUMMY_NODE_ID, + ident: self.to_ident_from_name_ref_or_err( + method_call_expr.name_ref(), + &method_call_expr, + "method name", + ), + args: if let Some(type_arg_list) = method_call_expr.type_arg_list() { + Some(P(ast::GenericArgs::AngleBracketed( + self.to_angle_bracketed_args(&type_arg_list), + ))) + } else { + None + }, + }, + Some(self.to_expr_or_err( + method_call_expr.expr(), + &method_call_expr, + "method call receiver", + )) + .into_iter() + .chain( + method_call_expr + .arg_list() + .into_iter() + .flat_map(|arg_list| arg_list.args().map(|expr| self.to_expr(expr))), + ) + .collect(), + ), + ra::Expr::ParenExpr(paren_expr) => ast::ExprKind::Paren(self.to_expr_or_err( + paren_expr.expr(), + &paren_expr, + "parenthesized expression", + )), + ra::Expr::PathExpr(path_expr) => { + let (qself, path) = + self.to_qself_and_path_or_err(path_expr.path(), &path_expr, "path expression"); + ast::ExprKind::Path(qself.map(|(qself, _)| qself), path) + } + ra::Expr::PrefixExpr(prefix_expr) => { + if let Some(op_kind) = self.expect_in( + prefix_expr.op_kind(), + &prefix_expr, + "recognized prefix operator", + "prefix operator expression", + ) { + ast::ExprKind::Unary( + match op_kind { + ra::PrefixOp::Deref => ast::UnOp::Deref, + ra::PrefixOp::Neg => ast::UnOp::Neg, + ra::PrefixOp::Not => ast::UnOp::Not, + }, + self.to_expr_or_err( + prefix_expr.expr(), + &prefix_expr, + "prefix operator expression", + ), + ) + } else { + ast::ExprKind::Err + } + } + ra::Expr::RangeExpr(range_expr) => { + if let Some(op_kind) = self.expect_in( + range_expr.op_kind(), + &range_expr, + "recognized range separator", + "range expression", + ) { + ast::ExprKind::Range( + range_expr.start().map(|start| self.to_expr(start)), + range_expr.end().map(|end| self.to_expr(end)), + match op_kind { + ra::RangeOp::Exclusive => ast::RangeLimits::HalfOpen, + ra::RangeOp::Inclusive => ast::RangeLimits::Closed, + }, + ) + } else { + ast::ExprKind::Err + } + } + ra::Expr::RecordLit(record_lit) => ast::ExprKind::Struct( + self.to_simple_path_or_err(record_lit.path(), &record_lit, "struct initializer"), + if let Some(record_field_list) = record_lit.record_field_list() { + record_field_list + .fields() + .map(|field| { + let ident = self.to_ident_from_name_ref_or_err( + field.name_ref(), + &field, + "record field initialization", + ); + ast::Field { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(&field), + attrs: self.get_attributes(&field).collect::>().into(), + ident, + expr: if let Some(expr) = field.expr() { + self.to_expr(expr) + } else { + self.mk_expr_for_ident(ident) + }, + is_shorthand: field.expr().is_none(), + is_placeholder: false, + } + }) + .collect() + } else { + Default::default() + }, + if let Some(record_field_list) = record_lit.record_field_list() { + record_field_list.spread().map(|expr| self.to_expr(expr)) + } else { + None + }, + ), + ra::Expr::RefExpr(ref_expr) => ast::ExprKind::AddrOf( + self.to_borrow_kind(ref_expr.raw_kw()), + self.to_mutability(ref_expr.mut_kw()), + self.to_expr_or_err(ref_expr.expr(), &ref_expr, "reference expression"), + ), + ra::Expr::ReturnExpr(return_expr) => { + ast::ExprKind::Ret(return_expr.expr().map(|value_expr| self.to_expr(value_expr))) + } + ra::Expr::TryBlockExpr(try_block_expr) => { + ast::ExprKind::TryBlock(self.to_block_or_err( + try_block_expr.body().and_then(|block_expr| block_expr.block()), + &try_block_expr, + "try block", + )) + } + ra::Expr::TryExpr(try_expr) => ast::ExprKind::Try(self.to_expr_or_err( + try_expr.expr(), + &try_expr, + "error propagation expression", + )), + ra::Expr::TupleExpr(tuple_expr) => ast::ExprKind::Tup( + tuple_expr.exprs().map(|inner_expr| self.to_expr(inner_expr)).collect(), + ), + ra::Expr::WhileExpr(while_expr) => ast::ExprKind::While( + self.to_expr_from_condition_or_err( + while_expr.condition(), + &while_expr, + "while loop", + ), + self.to_block_or_err( + while_expr.loop_body().and_then(|block_expr| block_expr.block()), + &while_expr, + "while loop", + ), + while_expr.label().map(|label| self.to_label(label)), + ), + }; + P(ast::Expr { id: ast::DUMMY_NODE_ID, span, attrs, kind }) + } + + fn to_expr_or_err( + &self, + expr_opt: Option, + element: &impl ra::AstElement, + container: &str, + ) -> P { + self.expect_in(expr_opt, element, "expression", container) + .map(|expr| self.to_expr(expr)) + .unwrap_or_else(|| { + P(ast::Expr { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(element), + attrs: Default::default(), + kind: ast::ExprKind::Err, + }) + }) + } + + fn get_item_kind_opt_from_macro_call( + &self, + macro_call: &ra::MacroCall, + ) -> Option { + if let Some(path) = macro_call.path() { + if path.qualifier().is_none() { + if let Some(segment) = path.segment() { + if let Some(name_ref) = segment.name_ref() { + if let Some(ra::NameRefToken::Ident(ident)) = name_ref.name_ref_token() { + if ident.text() == "macro_rules" { + return Some(ast::ItemKind::MacroDef(ast::MacroDef { + body: P(if let Some(token_tree) = macro_call.token_tree() { + self.to_mac_args_from_token_tree(token_tree) + } else { + self.error( + macro_call, + "macro_rules definitions must have arguments", + ); + ast::MacArgs::Empty + }), + macro_rules: true, + })); + } + } + } + } + } + } + None + } + + fn to_stmt_kind_from_macro_call( + &self, + macro_call: ra::MacroCall, + force_mac_stmt_style_opt: Option, + ) -> ast::StmtKind { + if let Some(item_kind) = self.get_item_kind_opt_from_macro_call(¯o_call) { + let span = self.get_inner_span(¯o_call); + ast::StmtKind::Item(P(ast::Item { + id: ast::DUMMY_NODE_ID, + span, + attrs: self.get_attributes(¯o_call).collect(), + vis: Spanned { span: span.shrink_to_lo(), node: ast::VisibilityKind::Inherited }, + ident: self.to_ident_from_name_or_err( + macro_call.name(), + ¯o_call, + "macro_rules! macro name", + ), + kind: item_kind, + tokens: self.get_collected_tokens_from_range(inner_range(¯o_call)), + })) + } else { + let mac_stmt_style = if let Some(force_mac_stmt_style) = force_mac_stmt_style_opt { + force_mac_stmt_style + } else if macro_call.semi().is_some() { + ast::MacStmtStyle::Semicolon + } else if macro_call + .token_tree() + .filter(|x| match x.left_delimiter() { + Some(ra::LeftDelimiter::LCurly(_)) => true, + _ => false, + }) + .is_some() + { + ast::MacStmtStyle::Braces + } else { + ast::MacStmtStyle::NoBraces + }; + let attrs = self.get_attributes(¯o_call).collect::>().into(); + + if mac_stmt_style == ast::MacStmtStyle::NoBraces { + ast::StmtKind::Expr(self.to_expr(ra::Expr::MacroCall(macro_call))) + } else { + ast::StmtKind::MacCall(P((self.to_mac_call(macro_call), mac_stmt_style, attrs))) + } + } + } + + fn to_stmt(&self, stmt: ra::StmtOrSemi) -> ast::Stmt { + ast::Stmt { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(&stmt), + kind: match stmt { + ra::StmtOrSemi::Stmt(ra::Stmt::ExprStmt(expr_stmt)) => { + let semi = expr_stmt.semi(); + match expr_stmt.expr() { + Some(ra::Expr::MacroCall(macro_call)) => self.to_stmt_kind_from_macro_call( + macro_call, + Some(ast::MacStmtStyle::Semicolon), + ), + expr => { + let ast_expr = if let Some(expr) = self.expect_in( + expr, + &expr_stmt, + "expression", + "expression statement", + ) { + self.to_expr_from_expr_and_expr_stmt_opt(expr, Some(expr_stmt)) + } else { + P(ast::Expr { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(&expr_stmt), + attrs: Default::default(), + kind: ast::ExprKind::Err, + }) + }; + if semi.is_some() { + ast::StmtKind::Semi(ast_expr) + } else { + ast::StmtKind::Expr(ast_expr) + } + } + } + } + ra::StmtOrSemi::Stmt(ra::Stmt::LetStmt(let_stmt)) => { + ast::StmtKind::Local(P(ast::Local { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(&let_stmt), + attrs: self.get_attributes(&let_stmt).collect::>().into(), + pat: self.to_pat_or_err(let_stmt.pat(), &let_stmt, "let statement"), + ty: let_stmt.ascribed_type().map(|type_ref| self.to_ty(type_ref)), + init: let_stmt.initializer().map(|expr| self.to_expr(expr)), + })) + } + ra::StmtOrSemi::Stmt(ra::Stmt::ModuleItem(module_item)) => { + ast::StmtKind::Item(self.to_module_item(module_item, "code block")) + } + ra::StmtOrSemi::Semi(_) => ast::StmtKind::Empty, + }, + } + } + + fn to_block(&self, block: ra::Block) -> P { + self.to_block_from_block_and_block_expr_opt(block, None) + } + + fn to_block_from_block_and_block_expr_opt( + &self, + block: ra::Block, + block_expr: Option, + ) -> P { + P(ast::Block { + id: ast::DUMMY_NODE_ID, + span: if let Some(block_expr) = &block_expr { + self.get_inner_span(block_expr) + } else { + self.get_inner_span(&block) + }, + rules: if block_expr.and_then(|block_expr| block_expr.unsafe_kw()).is_some() { + // compiler generated seems to only be used in derive expansions + ast::BlockCheckMode::Unsafe(ast::UnsafeSource::UserProvided) + } else { + ast::BlockCheckMode::Default + }, + stmts: block + .statements_or_semi() + .filter_map(|stmt| match stmt { + ra::StmtOrSemi::Stmt(ra::Stmt::ModuleItem(ra::ModuleItem::MacroCall(_))) => { + // TODO: HACK: this is actually a macro call as the final block expression, which is wrongly included in statements_or_semi + None + } + stmt => Some(self.to_stmt(stmt)), + }) + .chain(block.expr().into_iter().map(|expr| ast::Stmt { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(&expr), + kind: match expr { + ra::Expr::MacroCall(macro_call) => { + self.to_stmt_kind_from_macro_call(macro_call, None) + } + expr => ast::StmtKind::Expr(self.to_expr(expr)), + }, + })) + .collect(), + }) + } + + fn to_block_or_err( + &self, + block_opt: Option, + element: &impl ra::AstElement, + container: &str, + ) -> P { + if let Some(block) = self.expect_in(block_opt, element, "block", container) { + self.to_block_from_block_and_block_expr_opt(block, None) + } else { + P(ast::Block { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(element), + rules: ast::BlockCheckMode::Default, + stmts: Default::default(), + }) + } + } + + fn to_block_from_block_expr(&self, block_expr: ra::BlockExpr) -> P { + if let Some(block) = + self.expect_in(block_expr.block(), &block_expr, "block", "block expression") + { + self.to_block_from_block_and_block_expr_opt(block, Some(block_expr)) + } else { + P(ast::Block { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(&block_expr), + rules: if block_expr.unsafe_kw().is_some() { + // compiler generated seems to only be used in derive expansions + ast::BlockCheckMode::Unsafe(ast::UnsafeSource::UserProvided) + } else { + ast::BlockCheckMode::Default + }, + stmts: Default::default(), + }) + } + } + + fn to_anon_const(&self, expr: ra::Expr) -> ast::AnonConst { + ast::AnonConst { id: ast::DUMMY_NODE_ID, value: self.to_expr(expr) } + } + + fn to_anon_const_or_err( + &self, + expr_opt: Option, + element: &impl ra::AstElement, + container: &str, + ) -> ast::AnonConst { + ast::AnonConst { + id: ast::DUMMY_NODE_ID, + value: self.to_expr_or_err(expr_opt, element, container), + } + } + + fn to_assoc_ty_constraint(&self, assoc_type_arg: ra::AssocTypeArg) -> ast::AssocTyConstraint { + ast::AssocTyConstraint { + id: ast::DUMMY_NODE_ID, + span: self.get_inner_span(&assoc_type_arg), + ident: self.to_ident_from_name_ref_or_err( + assoc_type_arg.name_ref(), + &assoc_type_arg, + "associated type constraint", + ), + kind: if let Some(type_ref) = assoc_type_arg.type_ref() { + ast::AssocTyConstraintKind::Equality { ty: self.to_ty(type_ref) } + } else { + ast::AssocTyConstraintKind::Bound { + bounds: self.to_generic_bounds(assoc_type_arg.type_bound_list()), + } + }, + } + } + + fn to_angle_bracketed_args(&self, type_arg_list: &ra::TypeArgList) -> ast::AngleBracketedArgs { + ast::AngleBracketedArgs { + span: self.to_span(TextRange::from_to( + if let Some(coloncolon) = type_arg_list.coloncolon() { + start_of_next(&coloncolon) + } else { + inner_start(type_arg_list) + }, + inner_end(type_arg_list), + )), + args: type_arg_list + .generic_args() + .map(|arg| match arg { + ra::GenericArg::TypeArg(type_arg) => { + ast::AngleBracketedArg::Arg(ast::GenericArg::Type(self.to_ty_or_err( + type_arg.type_ref(), + &type_arg, + "type argument", + ))) + } + ra::GenericArg::LifetimeArg(lifetime_arg) => ast::AngleBracketedArg::Arg( + ast::GenericArg::Lifetime(self.to_lifetime_or_err( + lifetime_arg.lifetime(), + &lifetime_arg, + "lifetime argument", + )), + ), + ra::GenericArg::ConstArg(const_arg) => { + ast::AngleBracketedArg::Arg(ast::GenericArg::Const( + self.to_anon_const_or_err( + const_arg + .block_expr() + .map(|block_expr| ra::Expr::BlockExpr(block_expr)), + &const_arg, + "constant generic argument", + ), + )) + } + ra::GenericArg::AssocTypeArg(assoc_type_arg) => { + ast::AngleBracketedArg::Constraint( + self.to_assoc_ty_constraint(assoc_type_arg), + ) + } + }) + .collect(), + } + } + + fn get_generics_for_named( + &self, + element: &(impl TypeParamsOwner + NameOwner), + type_param_list_end: impl FnOnce() -> Option, + where_clause_end: impl FnOnce() -> Option, + ) -> ast::Generics { + self.get_generics_for_unnamed( + element, + || { + if let Some(name) = element.name() { + Some(inner_end(&name)) + } else { + type_param_list_end() + } + }, + where_clause_end, + ) + } + + fn get_generics_for_unnamed( + &self, + element: &impl ra::TypeParamsOwner, + type_param_list_end: impl FnOnce() -> Option, + where_clause_end: impl FnOnce() -> Option, + ) -> ast::Generics { + self.to_generics( + element.type_param_list().ok_or_else(|| { + if let Some(type_param_list_end) = type_param_list_end() { + type_param_list_end + } else { + inner_start(element) + } + }), + element.where_clause().ok_or_else(|| { + if let Some(where_clause_end) = where_clause_end() { + where_clause_end + } else { + start_of_next(element) + } + }), + ) + } + + fn to_generics( + &self, + type_param_list: Result, + where_clause: Result, + ) -> ast::Generics { + ast::Generics { + span: match type_param_list.as_ref() { + Ok(type_param_list) => self.get_inner_span(type_param_list), + Err(insert_type_param_list) => self.to_span_from_unit(*insert_type_param_list), + }, + params: self.to_generic_params(type_param_list.ok()), + where_clause: match where_clause { + Ok(where_clause) => self.to_where_clause(where_clause), + Err(insert_where_clause) => ast::WhereClause { + span: self.to_span_from_unit(insert_where_clause), + predicates: Default::default(), + }, + }, + } + } + + fn unwrap_for_type( + &self, + type_ref_opt: Option, + ) -> (Option, Option) { + match type_ref_opt { + Some(ra::TypeRef::ForType(for_type)) => { + (for_type.type_ref(), for_type.type_param_list()) + } + type_ref_opt => (type_ref_opt, None), + } + } + + fn to_where_clause(&self, node: ra::WhereClause) -> ast::WhereClause { + ast::WhereClause { + span: self.get_inner_span(&node), + predicates: node + .predicates() + .map(|pred: ra::WherePred| { + if let Some(lifetime) = pred.lifetime() { + ast::WherePredicate::RegionPredicate(ast::WhereRegionPredicate { + span: self.get_inner_span(&pred), + lifetime: self.to_lifetime(lifetime), + bounds: self.get_generic_bounds(&pred), + }) + } else { + let (type_ref_opt, type_param_list_opt) = + self.unwrap_for_type(pred.type_ref()); + ast::WherePredicate::BoundPredicate(ast::WhereBoundPredicate { + span: self.get_inner_span(&pred), + bound_generic_params: self.to_generic_params(type_param_list_opt), + bounded_ty: self.to_ty_or_err(type_ref_opt, &pred, "where predicate"), + bounds: self.get_generic_bounds(&pred), + }) + } + }) + .collect(), + } + } + + fn get_generic_bounds(&self, node: &impl TypeBoundsOwner) -> ast::GenericBounds { + self.to_generic_bounds(node.type_bound_list()) + } + + fn to_generic_bounds(&self, node: Option) -> ast::GenericBounds { + if let Some(bounds) = node { + bounds + .bounds() + .map(|bound: ra::TypeBound| { + let bound_mod = if bound.const_kw().is_some() { + self.expect_in(bound.const_question(), &bound, "?", "? const bound"); + if bound.question().is_some() { + ast::TraitBoundModifier::MaybeConstMaybe + } else { + ast::TraitBoundModifier::MaybeConst + } + } else { + if bound.question().is_some() { + ast::TraitBoundModifier::Maybe + } else { + ast::TraitBoundModifier::None + } + }; + match bound.kind() { + ra::TypeBoundKind::Lifetime(lifetime) => { + if bound_mod != ast::TraitBoundModifier::None { + self.error(&bound, "? modifier on lifetime bound"); + } + ast::GenericBound::Outlives(self.to_lifetime(lifetime)) + } + ra::TypeBoundKind::PathType(path_type) => ast::GenericBound::Trait( + ast::PolyTraitRef::new( + Default::default(), + self.to_trait_path_from_path_type(path_type), + self.get_inner_span(&bound), + ), + bound_mod, + ), + ra::TypeBoundKind::ForType(for_type) => self + .expect_in(for_type.type_ref(), &for_type, "type", "for type") + .and_then(|type_ref| { + self.expect_in_res( + match type_ref { + ra::TypeRef::PathType(path_type) => Ok(path_type), + x => Err(x), + }, + "path type", + "for type", + ) + .map(|path_type| { + ast::GenericBound::Trait( + ast::PolyTraitRef::new( + self.to_generic_params(for_type.type_param_list()), + self.to_trait_path_from_path_type(path_type), + self.get_inner_span(&bound), + ), + bound_mod, + ) + }) + }) + .unwrap_or_else(|| { + ast::GenericBound::Trait( + ast::PolyTraitRef::new( + self.to_generic_params(for_type.type_param_list()), + ast::Path { + span: self.get_inner_span(&for_type), + segments: Default::default(), + }, + self.get_inner_span(&bound), + ), + bound_mod, + ) + }), + } + }) + .collect() + } else { + Default::default() + } + } + + fn to_generic_params(&self, node: Option) -> Vec { + if let Some(type_params) = node { + type_params + .generic_params() + .map(|param| match param { + ra::GenericParam::TypeParam(type_param) => ast::GenericParam { + id: ast::DUMMY_NODE_ID, + ident: self.to_ident_from_name_or_err( + type_param.name(), + &type_param, + "type parameter", + ), + kind: ast::GenericParamKind::Type { + default: type_param + .default_type() + .map(|default_type| self.to_ty(default_type)), + }, + attrs: self.get_attributes(&type_param).collect::>().into(), + bounds: self.get_generic_bounds(&type_param), + is_placeholder: false, + }, + ra::GenericParam::LifetimeParam(lifetime_param) => ast::GenericParam { + id: ast::DUMMY_NODE_ID, + ident: self + .to_lifetime_or_err( + lifetime_param.lifetime(), + &lifetime_param, + "lifetime parameter", + ) + .ident, + kind: ast::GenericParamKind::Lifetime, + attrs: self.get_attributes(&lifetime_param).collect::>().into(), + bounds: lifetime_param + .lifetime_bounds() + .map(|lifetime| ast::GenericBound::Outlives(self.to_lifetime(lifetime))) + .collect(), + is_placeholder: false, + }, + ra::GenericParam::ConstParam(const_param) => ast::GenericParam { + id: ast::DUMMY_NODE_ID, + ident: self.to_ident_from_name_or_err( + const_param.name(), + &const_param, + "const parameter", + ), + kind: ast::GenericParamKind::Const { + ty: self.to_ty_or_err( + const_param.ascribed_type(), + &const_param, + "const parameter", + ), + }, + attrs: self.get_attributes(&const_param).collect::>().into(), + bounds: Default::default(), + is_placeholder: false, + }, + }) + .collect() + } else { + Default::default() + } + } + + fn to_symbol_from_text(&self, text: &SmolStr) -> Symbol { + // TODO: ideally we should change the rustc interner to use SmolStr instead, to avoid double interning + Symbol::intern(text.as_str()) + } + + fn to_symbol_from_token(&self, token: impl ra::AstToken) -> Symbol { + self.to_symbol_from_text(&token.text()) + } + + fn to_ident_from_text(&self, range: TextRange, text: &SmolStr) -> ast::Ident { + ast::Ident { span: self.to_span(range), name: self.to_symbol_from_text(text) } + } + + fn to_ident_from_name_ref(&self, node: ra::NameRef) -> ast::Ident { + self.to_ident_from_text(inner_range(&node), &node.text()) + } + + fn to_ident_from_name_ref_or_err( + &self, + name_ref_opt: Option, + element: &impl ra::AstElement, + container: &str, + ) -> ast::Ident { + self.expect_in(name_ref_opt, element, "name reference", container) + .map(|name| self.to_ident_from_name_ref(name)) + .unwrap_or_else(|| ast::Ident::new(kw::Invalid, self.get_inner_span(element))) + } + + fn to_ident_from_name(&self, node: ra::Name) -> ast::Ident { + self.to_ident_from_text(inner_range(&node), &node.text()) + } + + fn to_ident_from_name_or_err( + &self, + name_opt: Option, + element: &impl ra::AstElement, + container: &str, + ) -> ast::Ident { + self.expect_in(name_opt, element, "name", container) + .map(|name| self.to_ident_from_name(name)) + .unwrap_or_else(|| ast::Ident::new(kw::Invalid, self.get_inner_span(element))) + } + + fn to_ident_from_token(&self, token: impl ra::AstToken) -> ast::Ident { + self.to_ident_from_text(inner_range(&token), &token.text()) + } + + fn to_tokens(&self, element: impl ra::AstElement) -> Option { + self.get_tokens_from_range(inner_range(&element)) + } + + fn get_tokens_from_range(&self, range: TextRange) -> Option { + let span = self.to_span(range); + let mut srdr = lexer::StringReader::retokenize(&self.sess, span); + srdr.pos = span.lo(); // somehow retokenize() doesn't set this: not sure how the other callers can possibly work properly, but we definitely need this + let (token_trees, unmatched_braces) = srdr.into_token_trees(); + + match token_trees { + Ok(stream) => { + //fix_token_stream(&mut stream); + Some(stream) + } + Err(err) => { + self.sess.span_diagnostic.emit_diagnostic(&err); + for unmatched in unmatched_braces { + if let Some(err) = make_unclosed_delims_error(unmatched, &self.sess) { + self.sess.span_diagnostic.emit_diagnostic(&err); + } + } + None + } + } + } + + fn get_collected_tokens_from_range(&self, range: TextRange) -> Option { + if let Some(mut token_stream) = self.get_tokens_from_range(range) { + remove_joint(&mut token_stream); + Some(token_stream) + } else { + None + } + } +} + +fn make_unclosed_delims_error( + unmatched: lexer::UnmatchedBrace, + sess: &ParseSess, +) -> Option> { + // `None` here means an `Eof` was found. We already emit those errors elsewhere, we add them to + // `unmatched_braces` only for error recovery in the `Parser`. + let found_delim = unmatched.found_delim?; + let mut err = sess.span_diagnostic.struct_span_err( + unmatched.found_span, + &format!( + "mismatched closing delimiter: `{}`", + rustc_ast_pretty::pprust::token_kind_to_string(&rustc_ast::token::CloseDelim( + found_delim + )), + ), + ); + err.span_label(unmatched.found_span, "mismatched closing delimiter"); + if let Some(sp) = unmatched.candidate_span { + err.span_label(sp, "closing delimiter possibly meant for this"); + } + if let Some(sp) = unmatched.unclosed_span { + err.span_label(sp, "unclosed delimiter"); + } + Some(err) +} diff --git a/src/librustc_session/options.rs b/src/librustc_session/options.rs index e43908a79143b..e73bf0968214e 100644 --- a/src/librustc_session/options.rs +++ b/src/librustc_session/options.rs @@ -959,4 +959,6 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options, "use new LLVM pass manager"), link_native_libraries: Option = (None, parse_opt_bool, [UNTRACKED], "Link native libraries in the linker invocation."), + parse_with_rust_analyzer: bool = (false, parse_bool, [UNTRACKED], + "Use rust_analyzer to parse Rust instead of the rustc parser"), } diff --git a/src/rustc/Cargo.toml b/src/rustc/Cargo.toml index 5e0f167bb3801..4ab370d244c5b 100644 --- a/src/rustc/Cargo.toml +++ b/src/rustc/Cargo.toml @@ -23,3 +23,4 @@ features = ['unprefixed_malloc_on_supported_platforms'] [features] jemalloc = ['jemalloc-sys'] llvm = ['rustc_driver/llvm'] +rust_analyzer = ['rustc_driver/rust_analyzer']