Model the pr131656 restriction on e-suffixes

This adds two more reserved forms to reject the things that rustc currently
rejects.

Author: mattheww

src/lex_via_peg/pretokenisation/pest_pretokeniser.rs

@@ -192,9 +192,10 @@ fn interpret_pest_pair(pair: Pair<Rule>) -> Result<PretokenData, &'static str> {
                 suffix: suffix.map(Into::into),
             })
         }
-        Rule::Reserved_float_empty_exponent | Rule::Reserved_float_based => {
-            Ok(PretokenData::Reserved)
-        }
+        Rule::Reserved_float_empty_exponent
+        | Rule::Reserved_float_based
+        | Rule::Reserved_float_e_suffix_restriction
+        | Rule::Reserved_integer_e_suffix_restriction => Ok(PretokenData::Reserved),
         Rule::Integer_literal => {
             let mut base = None;
             let mut digits = None;

src/lex_via_peg/pretokenisation/pretokenise.pest

@@ -10,8 +10,10 @@ PRETOKEN_2015 = {
     Unterminated_literal_2015 |
     Float_literal_1 |
     Reserved_float_empty_exponent |
+    Reserved_float_e_suffix_restriction |
     Float_literal_2 |
     Reserved_float_based |
+    Reserved_integer_e_suffix_restriction |
     Integer_literal |
     Lifetime_or_label |
     Raw_identifier |
@@ -31,8 +33,10 @@ PRETOKEN_2021 = {
     Reserved_literal_2021 |
     Float_literal_1 |
     Reserved_float_empty_exponent |
+    Reserved_float_e_suffix_restriction |
     Float_literal_2 |
     Reserved_float_based |
+    Reserved_integer_e_suffix_restriction |
     Integer_literal |
     Raw_lifetime_or_label_2021 |
     Reserved_lifetime_or_label_prefix_2021 |
@@ -55,8 +59,10 @@ PRETOKEN_2024 = {
     Reserved_guard_2024 |
     Float_literal_1 |
     Reserved_float_empty_exponent |
+    Reserved_float_e_suffix_restriction |
     Float_literal_2 |
     Reserved_float_based |
+    Reserved_integer_e_suffix_restriction |
     Integer_literal |
     Raw_lifetime_or_label_2021 |
     Reserved_lifetime_or_label_prefix_2021 |
@@ -168,6 +174,8 @@ DECIMAL_DIGITS = { ('0'..'9' | "_") * }
 HEXADECIMAL_DIGITS = { ('0'..'9' | 'a' .. 'f' | 'A' .. 'F' | "_") * }
 LOW_BASE_PRETOKEN_DIGITS = { DECIMAL_DIGITS }
 DECIMAL_PART = { '0'..'9' ~ DECIMAL_DIGITS }
+
+RESTRICTED_E_SUFFIX = { ("e"|"E") ~ "_"+ ~ !XID_START ~ XID_CONTINUE }
 // ANCHOR_END: numeric_common
 
 
@@ -200,6 +208,10 @@ Reserved_float_empty_exponent = {
     DECIMAL_PART ~ ("." ~ DECIMAL_PART ) ? ~
     ("e"|"E") ~ ("+"|"-")
 }
+Reserved_float_e_suffix_restriction = {
+    DECIMAL_PART ~ "." ~ DECIMAL_PART ~
+    RESTRICTED_E_SUFFIX
+}
 Reserved_float_based = {
     (
         ("0b" | "0o") ~ LOW_BASE_PRETOKEN_DIGITS |
@@ -211,6 +223,15 @@ Reserved_float_based = {
 }
 // ANCHOR_END: reserved_float
 
+// ANCHOR: reserved_integer
+Reserved_integer_e_suffix_restriction = {
+    ( INTEGER_BINARY_LITERAL |
+      INTEGER_OCTAL_LITERAL |
+      INTEGER_DECIMAL_LITERAL ) ~
+    RESTRICTED_E_SUFFIX
+}
+// ANCHOR_END: reserved_integer
+
 // ANCHOR: integer_literals
 Integer_literal = {
     ( INTEGER_BINARY_LITERAL |

writeup/numeric_literal_pretokens.md

@@ -47,6 +47,25 @@ The following nonterminals are common to the definitions below:
 > Note: The `Reserved_float_empty_exponent` pretoken nonterminal is placed between `Float_literal_1` and `Float_literal_2` in priority order.
 > This ordering makes sure that forms like `123.4e+` are reserved, rather than being accepted by `FLOAT_BODY_WITHOUT_EXPONENT`).
 
+> See [e-suffix-restriction] for discussion of `Reserved_float_e_suffix_restriction`.
+
+
+#### Reserved integer { .rule }
+
+##### Grammar
+```
+{{#include pretokenise_anchored.pest:reserved_integer}}
+```
+
+##### Pretoken kind
+`Reserved`
+
+##### Attributes
+(none)
+
+See [e-suffix-restriction] for discussion.
+
+
 #### Integer literal { .rule }
 
 ##### Grammar
@@ -85,3 +104,4 @@ The <var>base</var> attribute is determined from the following table, depending
 > - `0x·` (which is rejected, not accepted as `0` with suffix `x·`)
 
 [rfc0879]: https://github.com/rust-lang/rfcs/pull/0879
+[e-suffix-restriction]: rustc_oddities.md#e-suffix-restriction

writeup/rustc_oddities.md

@@ -35,11 +35,29 @@ The grammar production in the Reference seems to be written to assume that these
 I haven't seen any discussion of whether this rustc behaviour is considered desirable.
 
 
+### Restriction on e-suffixes { #e-suffix-restriction }
+
+With the implementation of [pr131656] as of 2025-04-27,
+support for numeric literal suffixes beginning with `e` or `E` is incomplete,
+and rejects some (very obscure) cases.
+
+A numeric literal token is rejected if:
+ - it doesn't have an exponent; and
+ - it has a suffix of the following form:
+   - begins with <b>e</b> or <b>E</b>
+   - immediately followed by one or more <b>_</b> characters
+   - immediately followed by a character which has the `XID_Continue` property but not `XID_Start`.
+
+For example, `123e_·` is rejected.
+
+The `Reserved_float_e_suffix_restriction` and `Reserved_integer_e_suffix_restriction` nonterminals describe this restriction in the grammar.
+
+
 [playground-lifetime]: https://play.rust-lang.org/?version=stable&mode=debug&edition=2021&gist=31fc06e4d678e1a38d8d39f521e8a11c
 [playground-ident]: https://play.rust-lang.org/?version=stable&mode=debug&edition=2021&gist=aad27eb75b2774f16fc6b0981b770d56
 
 [rfc2457]: https://rust-lang.github.io/rfcs/2457-non-ascii-idents.html
 
 [#126759]: https://github.com/rust-lang/rust/issues/126759
-
+[pr131656]: https://github.com/rust-lang/rust/pull/131656