perf(es/fast-lexer): Use memchr for skip_line_comments (#10173)

Author: kdy1

Cargo.lock

@@ -19,6 +19,7 @@ swc_common   = { version = "8.0.0", path = "../swc_common" }
 swc_ecma_ast = { version = "8.0.0", path = "../swc_ecma_ast" }
 
 assume     = { workspace = true }
+memchr     = { workspace = true }
 num-bigint = { workspace = true }
 phf        = { workspace = true, features = ["macros"] }
 wide       = { workspace = true }

crates/swc_ecma_fast_parser/Cargo.toml

@@ -4,7 +4,6 @@
 
 use assume::assume;
 use swc_common::BytePos;
-use wide::u8x16;
 
 use crate::util::unlikely;
 
@@ -173,69 +172,4 @@ impl<'a> Cursor<'a> {
     pub fn reset_to(&mut self, pos: BytePos) {
         self.pos = pos.0;
     }
-
-    /// Find the next occurrence of a byte
-    #[inline]
-    pub fn find_byte(&self, byte: u8) -> Option<u32> {
-        // If we're at or near EOF, use the standard implementation
-        if unlikely(self.pos + 16 > self.len) {
-            return self.find_byte_scalar(byte);
-        }
-
-        // SIMD implementation using wide crate
-        self.find_byte_simd(byte)
-    }
-
-    /// SIMD-accelerated implementation of find_byte
-    #[inline]
-    fn find_byte_simd(&self, byte: u8) -> Option<u32> {
-        let input = &self.input[self.pos as usize..];
-        let mut position = 0u32;
-
-        // Process 16 bytes at a time
-        while position + 16 <= input.len() as u32 {
-            // Create a vector with our pattern
-            let needle = u8x16::splat(byte);
-
-            // Create a vector with current chunk of data
-            let mut data = [0u8; 16];
-            data.copy_from_slice(&input[position as usize..(position + 16) as usize]);
-            let chunk = u8x16::new(data);
-
-            // Compare for equality
-            let mask = chunk.cmp_eq(needle);
-
-            // Converting to array to check byte-by-byte (no move_mask available)
-            let mask_array = mask.to_array();
-
-            // Check for any matches
-            #[allow(clippy::needless_range_loop)]
-            for i in 0..16 {
-                if mask_array[i] != 0 {
-                    return Some(self.pos + position + i as u32);
-                }
-            }
-
-            position += 16;
-        }
-
-        // Handle the remainder with the scalar implementation
-        if position < input.len() as u32 {
-            return input[position as usize..]
-                .iter()
-                .position(|&b| b == byte)
-                .map(|pos| self.pos + position + pos as u32);
-        }
-
-        None
-    }
-
-    /// Standard fallback implementation
-    #[inline]
-    fn find_byte_scalar(&self, byte: u8) -> Option<u32> {
-        self.input[self.pos as usize..]
-            .iter()
-            .position(|&b| b == byte)
-            .map(|pos| self.pos + pos as u32)
-    }
 }

crates/swc_ecma_fast_parser/src/lexer/cursor.rs

@@ -20,6 +20,7 @@ mod tests;
 use std::rc::Rc;
 
 use cursor::Cursor;
+use memchr::memchr2;
 use swc_common::{BytePos, Span, DUMMY_SP};
 use wide::u8x16;
 
@@ -606,30 +607,30 @@ impl<'a> Lexer<'a> {
 
     #[inline(always)]
     fn skip_line_comment(&mut self) {
-        // Fast path using find_byte (which uses SIMD internally when available)
-        if let Some(newline_pos) = self.cursor.find_byte(b'\n') {
-            // Skip to the newline
-            let from_cursor = newline_pos - self.cursor.position();
-            self.cursor.advance_n(from_cursor);
-            self.cursor.advance(); // Skip the newline
-            self.had_line_break = LineBreak::Present;
-            return;
-        }
+        // Fast path using memchr2 to find either \n or \r
+        if let Some(pos) = memchr2(b'\n', b'\r', self.cursor.rest()) {
+            let ch = self.cursor.rest()[pos];
+            self.cursor.advance_n(pos as u32);
 
-        // Slower fallback path for when no newline is found
-        while let Some(ch) = self.cursor.peek() {
-            self.cursor.advance();
             if ch == b'\n' {
+                // Simple newline
+                self.cursor.advance(); // Skip the newline
                 self.had_line_break = LineBreak::Present;
-                break;
-            } else if ch == b'\r' {
-                self.had_line_break = LineBreak::Present;
-                // Skip the following \n if it exists (CRLF sequence)
+            } else {
+                // Carriage return - check if followed by newline (CRLF)
+                self.cursor.advance(); // Skip the \r
                 if let Some(b'\n') = self.cursor.peek() {
-                    self.cursor.advance();
+                    self.cursor.advance(); // Skip the \n in CRLF
                 }
-                break;
-            } else if ch == 0xe2 {
+                self.had_line_break = LineBreak::Present;
+            }
+            return;
+        }
+
+        // Slower fallback path for Unicode line breaks or when no line break is found
+        while let Some(ch) = self.cursor.peek() {
+            self.cursor.advance();
+            if ch == 0xe2 {
                 // Check for line separator (U+2028) and paragraph separator (U+2029)
                 let bytes = self.cursor.peek_n(2);
                 if bytes.len() == 2 && bytes[0] == 0x80 && (bytes[1] == 0xa8 || bytes[1] == 0xa9) {