[Syntax] test diagnostics in Lexer with libSyntax (#14954)

Author: omochi

include/swift/Parse/Parser.h

@@ -1444,7 +1444,8 @@ tokenizeWithTrivia(const LangOptions &LangOpts,
                    const SourceManager &SM,
                    unsigned BufferID,
                    unsigned Offset = 0,
-                   unsigned EndOffset = 0);
+                   unsigned EndOffset = 0,
+                   DiagnosticEngine *Diags = nullptr);
 } // end namespace swift
 
 #endif

include/swift/Subsystems.h

@@ -128,6 +128,7 @@ namespace swift {
   std::vector<Token> tokenize(const LangOptions &LangOpts,
                               const SourceManager &SM, unsigned BufferID,
                               unsigned Offset = 0, unsigned EndOffset = 0,
+                              DiagnosticEngine *Diags = nullptr,
                               bool KeepComments = true,
                               bool TokenizeInterpolatedString = true,
                               ArrayRef<Token> SplitTokens = ArrayRef<Token>());

lib/Parse/Parser.cpp

@@ -45,6 +45,7 @@ namespace swift {
 template <typename DF>
 void tokenize(const LangOptions &LangOpts, const SourceManager &SM,
               unsigned BufferID, unsigned Offset, unsigned EndOffset,
+              DiagnosticEngine * Diags,
               CommentRetentionMode RetainComments,
               TriviaRetentionMode TriviaRetention,
               bool TokenizeInterpolatedString, ArrayRef<Token> SplitTokens,
@@ -56,7 +57,7 @@ void tokenize(const LangOptions &LangOpts, const SourceManager &SM,
   if (Offset == 0 && EndOffset == 0)
     EndOffset = SM.getRangeForBuffer(BufferID).getByteLength();
 
-  Lexer L(LangOpts, SM, BufferID, /*Diags=*/nullptr, /*InSILMode=*/false,
+  Lexer L(LangOpts, SM, BufferID, Diags, /*InSILMode=*/false,
           RetainComments, TriviaRetention, Offset, EndOffset);
 
   auto TokComp = [&](const Token &A, const Token &B) {
@@ -260,6 +261,7 @@ static void getStringPartTokens(const Token &Tok, const LangOptions &LangOpts,
 
       std::vector<Token> NewTokens = swift::tokenize(LangOpts, SM, BufID,
                                                      Offset, EndOffset,
+                                                     /*Diags=*/nullptr,
                                                      /*KeepComments=*/true);
       Toks.insert(Toks.end(), NewTokens.begin(), NewTokens.end());
 
@@ -278,12 +280,14 @@ static void getStringPartTokens(const Token &Tok, const LangOptions &LangOpts,
 std::vector<Token> swift::tokenize(const LangOptions &LangOpts,
                                    const SourceManager &SM, unsigned BufferID,
                                    unsigned Offset, unsigned EndOffset,
+                                   DiagnosticEngine *Diags,
                                    bool KeepComments,
                                    bool TokenizeInterpolatedString,
                                    ArrayRef<Token> SplitTokens) {
   std::vector<Token> Tokens;
 
   tokenize(LangOpts, SM, BufferID, Offset, EndOffset,
+           Diags,
            KeepComments ? CommentRetentionMode::ReturnAsTokens
                         : CommentRetentionMode::AttachToNextToken,
            TriviaRetentionMode::WithoutTrivia, TokenizeInterpolatedString,
@@ -299,13 +303,15 @@ std::vector<Token> swift::tokenize(const LangOptions &LangOpts,
 std::vector<std::pair<RC<syntax::RawSyntax>, syntax::AbsolutePosition>>
 swift::tokenizeWithTrivia(const LangOptions &LangOpts, const SourceManager &SM,
                           unsigned BufferID, unsigned Offset,
-                          unsigned EndOffset) {
+                          unsigned EndOffset,
+                          DiagnosticEngine *Diags) {
   std::vector<std::pair<RC<syntax::RawSyntax>, syntax::AbsolutePosition>>
       Tokens;
   syntax::AbsolutePosition RunningPos;
 
   tokenize(
       LangOpts, SM, BufferID, Offset, EndOffset,
+      Diags,
       CommentRetentionMode::AttachToNextToken, TriviaRetentionMode::WithTrivia,
       /*TokenizeInterpolatedString=*/false,
       /*SplitTokens=*/ArrayRef<Token>(),

test/Syntax/lexer_invalid_nul.swift

@@ -0,0 +1,5 @@
+// RUN: cat %s | tr '\132' '\0' > %t
+// RUN: %swift-syntax-test -input-source-filename %t -dump-full-tokens 2>&1 >/dev/null | %FileCheck %t
+
+// CHECK: 5:18: warning: nul character embedded in middle of file
+let a = 3 // nul(Z)

tools/swift-syntax-test/swift-syntax-test.cpp

@@ -112,11 +112,13 @@ namespace {
 int getTokensFromFile(unsigned BufferID,
                       LangOptions &LangOpts,
                       SourceManager &SourceMgr,
-                      DiagnosticEngine &Diags,
+                      swift::DiagnosticEngine &Diags,
                       std::vector<std::pair<RC<syntax::RawSyntax>,
                       syntax::AbsolutePosition>> &Tokens) {
-  Tokens = tokenizeWithTrivia(LangOpts, SourceMgr, BufferID);
-  return Diags.hadAnyError() ? EXIT_FAILURE : EXIT_SUCCESS;
+  Tokens = tokenizeWithTrivia(LangOpts, SourceMgr, BufferID,
+                              /*Offset=*/0, /*EndOffset=*/0,
+                              &Diags);
+  return EXIT_SUCCESS;
 }
 
 
@@ -191,7 +193,7 @@ int doFullLexRoundTrip(const StringRef InputFilename) {
     TokAndPos.first->print(llvm::outs(), {});
   }
 
-  return Diags.hadAnyError() ? EXIT_FAILURE : EXIT_SUCCESS;
+  return EXIT_SUCCESS;
 }
 
 int doDumpRawTokenSyntax(const StringRef InputFilename) {
@@ -215,7 +217,7 @@ int doDumpRawTokenSyntax(const StringRef InputFilename) {
     llvm::outs() << "\n";
   }
 
-  return Diags.hadAnyError() ? EXIT_FAILURE : EXIT_SUCCESS;
+  return EXIT_SUCCESS;
 }
 
 int doFullParseRoundTrip(const char *MainExecutablePath,

unittests/Parse/LexerTests.cpp

@@ -35,7 +35,7 @@ class LexerTest : public ::testing::Test {
     if (KeepEOF)
       Toks = tokenizeAndKeepEOF(BufID);
     else
-      Toks = tokenize(LangOpts, SourceMgr, BufID, 0, 0, KeepComments);
+      Toks = tokenize(LangOpts, SourceMgr, BufID, 0, 0, /*Diags=*/nullptr, KeepComments);
     EXPECT_EQ(ExpectedTokens.size(), Toks.size());
     for (unsigned i = 0, e = ExpectedTokens.size(); i != e; ++i) {
       EXPECT_EQ(ExpectedTokens[i], Toks[i].getKind()) << "i = " << i;
@@ -685,7 +685,7 @@ TEST_F(LexerTest, TokenizePlaceholder) {
 TEST_F(LexerTest, NoPlaceholder) {
   auto checkTok = [&](StringRef Source) {
     unsigned BufID = SourceMgr.addMemBufferCopy(Source);
-    std::vector<Token> Toks = tokenize(LangOpts, SourceMgr, BufID, 0, 0, false);
+    std::vector<Token> Toks = tokenize(LangOpts, SourceMgr, BufID, 0, 0, /*Diags=*/nullptr, false);
     ASSERT_FALSE(Toks.empty());
     EXPECT_NE(tok::identifier, Toks[0].getKind());
   };

unittests/Parse/TokenizerTests.cpp

@@ -98,6 +98,7 @@ class TokenizerTest : public ::testing::Test {
                            BufID, 
                            /* Offset = */ 0,
                            /* EndOffset = */ 0,
+                           /* Diags = */nullptr,
                            /* KeepComments = */ true,
                            /* TokenizeInterpolatedString = */ true,
                            SplitTokens);