[Syntax] support nul character as garbage trivia

Author: omochi

lib/Parse/Lexer.cpp

@@ -2341,7 +2341,6 @@ void Lexer::lexTrivia(syntax::Trivia &Pieces, bool IsForTrailingTrivia) {
 Restart:
   const char *TriviaStart = CurPtr;
 
-  // TODO: Handle random nul('\0') character in the middle of a buffer.
   // TODO: Handle invalid UTF8 sequence which is skipped in lexImpl().
   switch (*CurPtr++) {
   case '\n':
@@ -2421,6 +2420,19 @@ void Lexer::lexTrivia(syntax::Trivia &Pieces, bool IsForTrailingTrivia) {
       goto Restart;
     }
     break;
+  case 0:
+    switch (getNulCharacterKind(CurPtr - 1)) {
+    case NulCharacterKind::Embedded: {
+      diagnoseEmbeddedNul(Diags, CurPtr - 1);
+      size_t Length = CurPtr - TriviaStart;
+      Pieces.push_back(TriviaPiece::garbageText({TriviaStart, Length}));
+      goto Restart;
+    }
+    case NulCharacterKind::CodeCompletion:
+    case NulCharacterKind::BufferEnd:
+      break;
+    }
+    break;
   default:
     break;
   }

test/Syntax/lexer_invalid_nul.swift

@@ -0,0 +1,5 @@
+// RUN: cat %s | tr '\132' '\0' > %t.tr
+// RUN: cp -f %t.tr %t
+// RUN: %round-trip-syntax-test --swift-syntax-test %swift-syntax-test --file %t
+let a = Z3Z // nul(Z)
+func b() {}

test/Syntax/round_trip_nul.swift

@@ -0,0 +1,28 @@
+// RUN: cat %s | tr '\132' '\0' > %t.tmp
+// RUN: cp -f %t.tmp %t
+// RUN: %swift-syntax-test -input-source-filename %t -dump-full-tokens 2>&1 | %FileCheck %t
+let a = Z3Z // nul(Z)
+func b() {}
+
+// CHECK: 4:9: warning: nul character embedded in middle of file
+// CHECK: 4:11: warning: nul character embedded in middle of file
+// CHECK: 4:20: warning: nul character embedded in middle of file
+
+// CHECK-LABEL: 4:7
+// CHECK-NEXT:(Token equal
+// CHECK-NEXT: (text="=")
+// CHECK-NEXT: (trivia space 1)
+// CHECK-NEXT: (trivia garbage_text \000))
+
+// CHECK-LABEL: 4:10
+// CHECK-NEXT:(Token integer_literal
+// CHECK-NEXT: (text="3")
+// CHECK-NEXT: (trivia garbage_text \000)
+// CHECK-NEXT: (trivia space 1))
+
+// CHECK-LABEL: 5:1
+// CHECK-NEXT:(Token kw_func
+// CHECK-NEXT: (trivia line_comment // nul(\000))
+// CHECK-NEXT: (trivia newline 1)
+// CHECK-NEXT: (text="func")
+// CHECK-NEXT: (trivia space 1))