Remove preprocessor for fuzzyc (#1506)

* Remove all scala code related to failed preproc experiment

* Remove CPP code related to failed preproc experiment

* Remove preprocessor from build

Author: fabsx00

.github/workflows/pr.yml

@@ -27,51 +27,3 @@ jobs:
       run: sbt scalafmtCheck test:scalafmtCheck "scalafixAll --check OrganizeImports"
     - run: echo "Previous step failed because code is not formatted. Run 'sbt format'"
       if: ${{ failure() }}
-
-  test-fuzzypp-linux:
-    runs-on: ubuntu-18.04
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          lfs: true
-      - name: Clone submodules
-        run: git submodule update --init --recursive
-      - name: Install g++-8
-        run: sudo apt-get install gcc-8 g++-8
-      - name: Run cmake on fuzzypp
-        run: cmake -DCMAKE_CXX_COMPILER=/usr/bin/g++-8 ./fuzzypp
-      - name: Run cmake build
-        run: cmake --build . -- -j4
-      - name: Run fuzzyppcli-tests
-        run: "./fuzzypp/bin/fuzzyppcli-tests"
-      - name: Zip artifact
-        run: zip -j ./fuzzyppcli.zip ./fuzzypp/bin/fuzzyppcli
-      - name: Upload zip artifact
-        uses: actions/upload-artifact@v2
-        with:
-          name: unix-zip
-          path: ./fuzzyppcli.zip
-
-  test-fuzzypp-windows:
-    runs-on: windows-2019
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          lfs: true
-      - name: Clone submodules
-        run: git submodule update --init --recursive
-      - name: Run cmake on fuzzypp
-        run: cmake -G "Visual Studio 16 2019" -A "x64" ./fuzzypp
-      - name: Run cmake build
-        run: cmake --build . --config Release
-      - name: Run fuzzyppcli-tests
-        run: "./fuzzypp/bin/Release/fuzzyppcli-tests.exe"
-      - name: Zip artifact
-        run: 7z a -r -tzip ./fuzzyppcli-win.zip ./fuzzypp/bin/Release/fuzzyppcli.exe
-      - name: Upload zip artifact
-        uses: actions/upload-artifact@v2
-        with:
-          name: windows-zip
-          path: ./fuzzyppcli-win.zip

.github/workflows/push.yml

@@ -7,7 +7,6 @@ on:
 jobs:
   test-codepropertygraph:
     runs-on: ubuntu-18.04
-
     steps:
     - uses: actions/checkout@v2
       with:
@@ -29,58 +28,9 @@ jobs:
     - run: echo "Previous step failed because code is not formatted. Run 'sbt format'"
       if: ${{ failure() }}
 
-  test-fuzzypp-linux:
-    runs-on: ubuntu-18.04
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          lfs: true
-      - name: Clone submodules
-        run: git submodule update --init --recursive
-      - name: Install g++-8
-        run: sudo apt-get install gcc-8 g++-8
-      - name: Run cmake on fuzzypp
-        run: cmake -DCMAKE_CXX_COMPILER=/usr/bin/g++-8 ./fuzzypp
-      - name: Run cmake build
-        run: cmake --build . -- -j4
-      - name: Run fuzzyppcli-tests
-        run: "./fuzzypp/bin/fuzzyppcli-tests"
-      - name: Zip artifact
-        run: zip -j ./fuzzyppcli.zip ./fuzzypp/bin/fuzzyppcli
-      - name: Upload zip artifact
-        uses: actions/upload-artifact@v2
-        with:
-          name: unix-zip
-          path: ./fuzzyppcli.zip
-
-  test-fuzzypp-windows:
-    runs-on: windows-2019
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          lfs: true
-      - name: Clone submodules
-        run: git submodule update --init --recursive
-      - name: Run cmake on fuzzypp
-        run: cmake -G "Visual Studio 16 2019" -A "x64" ./fuzzypp
-      - name: Run cmake build
-        run: cmake --build . --config Release
-      - name: Run fuzzyppcli-tests
-        run: "./fuzzypp/bin/Release/fuzzyppcli-tests.exe"
-      - name: Zip artifact
-        run: 7z a -r -tzip ./fuzzyppcli-win.zip ./fuzzypp/bin/Release/fuzzyppcli.exe
-      - name: Upload zip artifact
-        uses: actions/upload-artifact@v2
-        with:
-          name: windows-zip
-          path: ./fuzzyppcli-win.zip
-
   release:
-    needs: [test-codepropertygraph, test-fuzzypp-linux, test-fuzzypp-windows]
+    needs: [test-codepropertygraph]
     runs-on: ubuntu-18.04
-
     steps:
       - uses: actions/checkout@v2
         with:
@@ -119,32 +69,6 @@ jobs:
           release_name: ${{ env.LATEST_TAG }}
           draft: false
           prerelease: false
-      - name: Download unix zip
-        uses: actions/download-artifact@v2
-        with:
-          name: unix-zip
-      - name: Upload fuzzypp zip
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./fuzzyppcli.zip
-          asset_name: fuzzyppcli.zip
-          asset_content_type: application/zip
-      - name: Download windows zip
-        uses: actions/download-artifact@v2
-        with:
-          name: windows-zip
-      - name: Upload fuzzypp Windows zip
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./fuzzyppcli-win.zip
-          asset_name: fuzzyppcli-win.zip
-          asset_content_type: application/zip
       - name: Package frontends for release
         run: sbt clean fuzzyc2cpg/Universal/packageBin
       - name: Upload fuzzyc2cpg to release

.gitmodules

@@ -1,9 +0,0 @@
-[submodule "fuzzypp/external/cxxopts"]
-	path = fuzzypp/external/cxxopts
-	url = https://github.com/jarro2783/cxxopts.git
-[submodule "fuzzypp/external/catch2"]
-	path = fuzzypp/external/catch2
-	url = https://github.com/catchorg/Catch2.git
-[submodule "fuzzypp/external/simplecpp"]
-	path = fuzzypp/external/simplecpp
-	url = https://github.com/ShiftLeftSecurity/simplecpp.git

fuzzyc2cpg/src/main/scala/io/shiftleft/fuzzyc2cpg/FuzzyC2Cpg.scala

@@ -10,62 +10,13 @@ import io.shiftleft.x2cpg.{SourceFiles, X2Cpg, X2CpgConfig}
 import org.slf4j.LoggerFactory
 import scopt.OParser
 
-import java.nio.file.Files
 import java.util.concurrent.ConcurrentHashMap
-import scala.collection.mutable.ListBuffer
 import scala.jdk.CollectionConverters._
 import scala.util.control.NonFatal
 
 case class Global(usedTypes: ConcurrentHashMap[String, Boolean] = new ConcurrentHashMap[String, Boolean]())
 
 class FuzzyC2Cpg() {
-  import FuzzyC2Cpg.logger
-
-  def runWithPreprocessorAndOutput(sourcePaths: Set[String],
-                                   sourceFileExtensions: Set[String],
-                                   includeFiles: Set[String],
-                                   includePaths: Set[String],
-                                   defines: Set[String],
-                                   undefines: Set[String],
-                                   preprocessorExecutable: String,
-                                   optionalOutputPath: Option[String] = None): Unit = {
-    // Create temp dir to store preprocessed source.
-    val preprocessedPath = Files.createTempDirectory("fuzzyc2cpg_preprocessed_")
-    logger.info(s"Writing preprocessed files to [$preprocessedPath]")
-
-    val preprocessorLogFile = Files.createTempFile("fuzzyc2cpg_preprocessor_log", ".txt").toFile
-    logger.info(s"Writing preprocessor logs to [$preprocessorLogFile]")
-
-    val sourceFileNames = SourceFiles.determine(sourcePaths, sourceFileExtensions)
-
-    val commandBuffer = new ListBuffer[String]()
-    commandBuffer.appendAll(List(preprocessorExecutable, "--verbose", "-o", preprocessedPath.toString))
-    if (sourceFileNames.nonEmpty) commandBuffer.appendAll(List("-f", sourceFileNames.mkString(",")))
-    if (includeFiles.nonEmpty) commandBuffer.appendAll(List("--include", includeFiles.mkString(",")))
-    if (includePaths.nonEmpty) commandBuffer.appendAll(List("-I", includePaths.mkString(",")))
-    if (defines.nonEmpty) commandBuffer.appendAll(List("-D", defines.mkString(",")))
-    if (undefines.nonEmpty) commandBuffer.appendAll(List("-U", defines.mkString(",")))
-
-    val cmd = commandBuffer.toList
-
-    // Run preprocessor
-    logger.info("Running preprocessor...")
-    val process = new ProcessBuilder()
-      .redirectOutput(preprocessorLogFile)
-      .redirectError(preprocessorLogFile)
-      .command(cmd: _*)
-      .start()
-    val exitCode = process.waitFor()
-
-    if (exitCode == 0) {
-      logger.info(s"Preprocessing complete, files written to [$preprocessedPath], starting CPG generation...")
-      val cpg = runAndOutput(Set(preprocessedPath.toString), sourceFileExtensions, optionalOutputPath)
-      cpg.close()
-    } else {
-      logger.error(
-        s"Error occurred whilst running preprocessor. Log written to [$preprocessorLogFile]. Exit code [$exitCode].")
-    }
-  }
 
   def runAndOutput(sourcePaths: Set[String],
                    sourceFileExtensions: Set[String],
@@ -93,15 +44,8 @@ object FuzzyC2Cpg {
 
   final case class Config(inputPaths: Set[String] = Set.empty,
                           outputPath: String = X2CpgConfig.defaultOutputPath,
-                          sourceFileExtensions: Set[String] = Set(".c", ".cc", ".cpp", ".h", ".hpp"),
-                          includeFiles: Set[String] = Set.empty,
-                          includePaths: Set[String] = Set.empty,
-                          defines: Set[String] = Set.empty,
-                          undefines: Set[String] = Set.empty,
-                          preprocessorExecutable: String = "./fuzzypp/bin/fuzzyppcli")
+                          sourceFileExtensions: Set[String] = Set(".c", ".cc", ".cpp", ".h", ".hpp"))
       extends X2CpgConfig[Config] {
-    lazy val usePreprocessor: Boolean =
-      includeFiles.nonEmpty || includePaths.nonEmpty || defines.nonEmpty || undefines.nonEmpty
 
     override def withAdditionalInputPath(inputPath: String): Config = copy(inputPaths = inputPaths + inputPath)
     override def withOutputPath(x: String): Config = copy(outputPath = x)
@@ -125,47 +69,15 @@ object FuzzyC2Cpg {
           .text(
             "source file extensions to include when gathering source files. Defaults are .c, .cc, .cpp, .h and .hpp")
           .action((pat, cfg) => cfg.copy(sourceFileExtensions = cfg.sourceFileExtensions + pat)),
-        opt[String]("include")
-          .unbounded()
-          .text("header include files")
-          .action((incl, cfg) => cfg.copy(includeFiles = cfg.includeFiles + incl)),
-        opt[String]('I', "")
-          .unbounded()
-          .text("header include paths")
-          .action((incl, cfg) => cfg.copy(includePaths = cfg.includePaths + incl)),
-        opt[String]('D', "define")
-          .unbounded()
-          .text("define a name")
-          .action((d, cfg) => cfg.copy(defines = cfg.defines + d)),
-        opt[String]('U', "undefine")
-          .unbounded()
-          .text("undefine a name")
-          .action((u, cfg) => cfg.copy(undefines = cfg.undefines + u)),
-        opt[String]("preprocessor-executable")
-          .text("path to the preprocessor executable")
-          .action((s, cfg) => cfg.copy(preprocessorExecutable = s)),
       )
     }
 
     X2Cpg.parseCommandLine(args, frontendSpecificOptions, Config()) match {
       case Some(config) =>
         try {
           val fuzzyc = new FuzzyC2Cpg()
-          if (config.usePreprocessor) {
-            fuzzyc.runWithPreprocessorAndOutput(
-              config.inputPaths,
-              config.sourceFileExtensions,
-              config.includeFiles,
-              config.includePaths,
-              config.defines,
-              config.undefines,
-              config.preprocessorExecutable,
-              Some(config.outputPath)
-            )
-          } else {
-            val cpg = fuzzyc.runAndOutput(config.inputPaths, config.sourceFileExtensions, Some(config.outputPath))
-            cpg.close()
-          }
+          val cpg = fuzzyc.runAndOutput(config.inputPaths, config.sourceFileExtensions, Some(config.outputPath))
+          cpg.close()
         } catch {
           case NonFatal(ex) =>
             logger.error("Failed to generate CPG.", ex)