Skip to content

Commit 01ace4a

Browse files
committed
Skip over CRs at the end of multiline literals
Fixes ziglang#9257. This is needed when tokenizing input containing DOS line endings, i.e. the CRLF sequence.
1 parent acf2e8f commit 01ace4a

File tree

1 file changed

+5
-1
lines changed

1 file changed

+5
-1
lines changed

lib/std/zig/tokenizer.zig

+5-1
Original file line numberDiff line numberDiff line change
@@ -834,7 +834,7 @@ pub const Tokenizer = struct {
834834
},
835835

836836
.multiline_string_literal_line => switch (c) {
837-
'\n' => {
837+
'\r', '\n' => {
838838
self.index += 1;
839839
break;
840840
},
@@ -2049,6 +2049,10 @@ test "tokenizer - invalid builtin identifiers" {
20492049
try testTokenize("@0()", &.{ .invalid, .integer_literal, .l_paren, .r_paren });
20502050
}
20512051

2052+
test "tokenizer - multiline string literal with a DOS-encoded CRLF line endings (issue #9257)" {
2053+
try testTokenize("\\\\foobar\r\n", &.{.multiline_string_literal_line});
2054+
}
2055+
20522056
fn testTokenize(source: []const u8, expected_tokens: []const Token.Tag) !void {
20532057
var tokenizer = Tokenizer.init(source);
20542058
for (expected_tokens) |expected_token_id| {

0 commit comments

Comments
 (0)