Only accept carriage returns before line feeds

Previous commit was much less strict about this, this more closely
matches the desired spec of only allow CR characters in a CRLF pair, but
not otherwise.

Author: moosichu

lib/std/zig/tokenizer.zig

@@ -442,7 +442,7 @@ pub const Tokenizer = struct {
             switch (state) {
                 .start => switch (c) {
                     0 => break,
-                    ' ', '\n', '\t', '\r' => {
+                    ' ', '\n', '\t' => {
                         result.loc.start = self.index + 1;
                     },
                     '"' => {
@@ -565,6 +565,18 @@ pub const Tokenizer = struct {
                         state = .int_literal_dec;
                         result.tag = .integer_literal;
                     },
+                    '\r' => {
+                        // Carriage returns are *only* allowed just before a linefeed as part of a CRLF pair, otherwise
+                        // they constitute an illegal byte!
+                        if (self.index + 1 < self.buffer.len and self.buffer[self.index + 1] == '\n') {
+                            result.loc.start = self.index + 1;
+                        } else {
+                            result.tag = .invalid;
+                            result.loc.end = self.index;
+                            self.index += 1;
+                            return result;
+                        }
+                    },
                     else => {
                         result.tag = .invalid;
                         result.loc.end = self.index;
@@ -903,7 +915,7 @@ pub const Tokenizer = struct {
                         self.index += 1;
                         break;
                     },
-                    '\t', '\r' => {},
+                    '\t' => {},
                     else => self.checkLiteralCharacter(),
                 },
 
@@ -1137,7 +1149,7 @@ pub const Tokenizer = struct {
                         state = .start;
                         result.loc.start = self.index + 1;
                     },
-                    '\t', '\r' => state = .line_comment,
+                    '\t' => state = .line_comment,
                     else => {
                         state = .line_comment;
                         self.checkLiteralCharacter();
@@ -1151,7 +1163,7 @@ pub const Tokenizer = struct {
                         result.tag = .doc_comment;
                         break;
                     },
-                    '\t', '\r' => {
+                    '\t' => {
                         state = .doc_comment;
                         result.tag = .doc_comment;
                     },
@@ -1167,12 +1179,12 @@ pub const Tokenizer = struct {
                         state = .start;
                         result.loc.start = self.index + 1;
                     },
-                    '\t', '\r' => {},
+                    '\t' => {},
                     else => self.checkLiteralCharacter(),
                 },
                 .doc_comment => switch (c) {
                     0, '\n' => break,
-                    '\t', '\r' => {},
+                    '\t' => {},
                     else => self.checkLiteralCharacter(),
                 },
                 .zero => switch (c) {
@@ -1433,7 +1445,15 @@ pub const Tokenizer = struct {
     fn getInvalidCharacterLength(self: *Tokenizer) u3 {
         const c0 = self.buffer[self.index];
         if (std.ascii.isASCII(c0)) {
-            if (std.ascii.isCntrl(c0)) {
+            if (c0 == '\r') {
+                if (self.index + 1 < self.buffer.len and self.buffer[self.index + 1] == '\n') {
+                    // Carriage returns are *only* allowed just before a linefeed as part of a CRLF pair, otherwise
+                    // they constitute an illegal byte!
+                    return 0;
+                } else {
+                    return 1;
+                }
+            } else if (std.ascii.isCntrl(c0)) {
                 // ascii control codes are never allowed
                 // (note that \n was checked before we got here)
                 return 1;

src/AstGen.zig

@@ -9961,35 +9961,19 @@ fn strLitNodeAsString(astgen: *AstGen, node: Ast.Node.Index) !IndexSlice {
     var tok_i = start;
     {
         const slice = tree.tokenSlice(tok_i);
-        const line_bytes = slice[2 .. slice.len - 1];
-        const carriage_return_count = mem.count(u8, line_bytes, "\r");
-        if (carriage_return_count > 0) {
-            try string_bytes.ensureUnusedCapacity(gpa, line_bytes.len - carriage_return_count);
-            for (line_bytes) |line_byte| {
-                if (line_byte == '\r') continue;
-                string_bytes.appendAssumeCapacity(line_byte);
-            }
-        } else {
-            try string_bytes.appendSlice(gpa, line_bytes);
-        }
+        const carriage_return_ending: usize = if (slice[slice.len - 2] == '\r') 2 else 1;
+        const line_bytes = slice[2 .. slice.len - carriage_return_ending];
+        try string_bytes.appendSlice(gpa, line_bytes);
         tok_i += 1;
     }
     // Following lines: each line prepends a newline.
     while (tok_i <= end) : (tok_i += 1) {
         const slice = tree.tokenSlice(tok_i);
-        const line_bytes = slice[2 .. slice.len - 1];
-
-        const carriage_return_count = mem.count(u8, line_bytes, "\r");
-        try string_bytes.ensureUnusedCapacity(gpa, line_bytes.len - carriage_return_count + 1);
+        const carriage_return_ending: usize = if (slice[slice.len - 2] == '\r') 2 else 1;
+        const line_bytes = slice[2 .. slice.len - carriage_return_ending];
+        try string_bytes.ensureUnusedCapacity(gpa, line_bytes.len + 1);
         string_bytes.appendAssumeCapacity('\n');
-        if (carriage_return_count > 0) {
-            for (line_bytes) |line_byte| {
-                if (line_byte == '\r') continue;
-                string_bytes.appendAssumeCapacity(line_byte);
-            }
-        } else {
-            string_bytes.appendSliceAssumeCapacity(line_bytes);
-        }
+        string_bytes.appendSliceAssumeCapacity(line_bytes);
     }
     const len = string_bytes.items.len - str_index;
     try string_bytes.append(gpa, 0);