[Serialization] Removing flatbuffer/flexbuffer usage and library from BuildCC (#226)

Author: coder137

.gitmodules

@@ -1,6 +1,6 @@
-[submodule "flatbuffers"]
-	path = third_party/flatbuffers
-	url = https://github.com/google/flatbuffers.git
+[submodule "json"]
+	path = third_party/json
+	url = https://github.com/nlohmann/json.git
 [submodule "spdlog"]
 	path = third_party/spdlog
 	url = https://github.com/gabime/spdlog.git
@@ -22,6 +22,3 @@
 [submodule "optional"]
 	path = third_party/tl_optional
 	url = https://github.com/TartanLlama/optional.git
-[submodule "json"]
-	path = third_party/json
-	url = https://github.com/nlohmann/json.git

CMakeLists.txt

@@ -12,8 +12,6 @@ project(BuildCC
 
 # User options
 option(BUILDCC_INSTALL "Enable BuildCC Installation" ON)
-option(BUILDCC_FLATBUFFERS_FLATC "Build Flatbuffer::Flatc Compiler" ON)
-
 option(BUILDCC_BUILD_AS_SINGLE_LIB "Build all internal libs and modules as part of the buildcc library" ON)
 option(BUILDCC_BUILD_AS_INTERFACE "Build all internal libs and modules seperately and link" OFF)
 
@@ -34,7 +32,6 @@ option(BUILDCC_DOCUMENTATION "Enable Documentation" OFF)
 
 # Compiler options
 # NOTE, This option is required for clang compilers, architecture x86_64-pc-windows-msvc
-# Flatbuffers library uses `std::system` internally which causes a deprecated error
 option(BUILDCC_NO_DEPRECATED "Disable Deprecated" OFF)
 if (${BUILDCC_NO_DEPRECATED})
     add_compile_options("-Wno-deprecated")
@@ -72,9 +69,7 @@ include(cmake/tool/cppcheck.cmake)
 include(cmake/tool/doxygen.cmake)
 
 # Libraries
-include(cmake/target/flatbuffers.cmake)
 include(cmake/target/json.cmake)
-
 include(cmake/target/fmt.cmake)
 include(cmake/target/spdlog.cmake)
 include(cmake/target/cli11.cmake)

CMakePresets.json

@@ -17,7 +17,6 @@
                 "CMAKE_C_COMPILER": "gcc",
                 "CMAKE_CXX_COMPILER": "g++",
                 "BUILDCC_INSTALL": true,
-                "BUILDCC_FLATBUFFERS_FLATC": true,
                 "BUILDCC_BUILD_AS_SINGLE_LIB": true,
                 "BUILDCC_BUILD_AS_INTERFACE": true,
                 "BUILDCC_BUILDEXE": true,
@@ -42,7 +41,6 @@
                 "CMAKE_C_COMPILER": "gcc",
                 "CMAKE_CXX_COMPILER": "g++",
                 "BUILDCC_INSTALL": true,
-                "BUILDCC_FLATBUFFERS_FLATC": true,
                 "BUILDCC_BUILD_AS_SINGLE_LIB": true,
                 "BUILDCC_BUILD_AS_INTERFACE": false,
                 "BUILDCC_BUILDEXE": true,
@@ -67,7 +65,6 @@
                 "CMAKE_C_COMPILER": "gcc",
                 "CMAKE_CXX_COMPILER": "g++",
                 "BUILDCC_INSTALL": true,
-                "BUILDCC_FLATBUFFERS_FLATC": true,
                 "BUILDCC_BUILD_AS_SINGLE_LIB": false,
                 "BUILDCC_BUILD_AS_INTERFACE": true,
                 "BUILDCC_BUILDEXE": false,
@@ -92,7 +89,6 @@
                 "CMAKE_C_COMPILER": "clang",
                 "CMAKE_CXX_COMPILER": "clang++",
                 "BUILDCC_INSTALL": true,
-                "BUILDCC_FLATBUFFERS_FLATC": true,
                 "BUILDCC_BUILD_AS_SINGLE_LIB": true,
                 "BUILDCC_BUILD_AS_INTERFACE": true,
                 "BUILDCC_BUILDEXE": true,
@@ -114,7 +110,6 @@
             "binaryDir": "${sourceDir}/_build_msvc_dev_all",
             "cacheVariables": {
                 "BUILDCC_INSTALL": true,
-                "BUILDCC_FLATBUFFERS_FLATC": true,
                 "BUILDCC_BUILD_AS_SINGLE_LIB": true,
                 "BUILDCC_BUILD_AS_INTERFACE": true,
                 "BUILDCC_BUILDEXE": true,
@@ -136,7 +131,6 @@
             "binaryDir": "${sourceDir}/_build_msvc_analysis",
             "cacheVariables": {
                 "BUILDCC_INSTALL": false,
-                "BUILDCC_FLATBUFFERS_FLATC": true,
                 "BUILDCC_BUILD_AS_SINGLE_LIB": true,
                 "BUILDCC_BUILD_AS_INTERFACE": false,
                 "BUILDCC_BUILDEXE": false,

DEPENDENCIES.md

@@ -2,13 +2,21 @@
 
 These third party libraries are added as submodules since they aren't meant to be modified by this project.
 
+### Adding a submodule
+
 `git submodule add [git_url] third_party/[foldername]`
 
+### Removing a submodule
+
+- `git rm --cached path_to_submodule` (no trailing slash)
+- Delete relevant line from `.gitmodules` file
+- Delete relevant section from `.git/config` file
+
 ## Main
 
 - fmt (Formatting)
 - spdlog (Logging)
-- flatbuffers (Serialization)
+- json (Serialization)
 - CLI11 (Argument Parsing)
 - Taskflow (Parallel Programming)
 

README.md

@@ -27,7 +27,6 @@ Build C, C++ and ASM files in C++
   - `C++17 filesystem` library support
   - `C++11 thread` library support
 - Third Party Libraries (See License below)
-  - Flatbuffers v2.0.0
   - Nlohmann::Json v3.11.2
   - Taskflow v3.1.0
   - CLI11 v2.1.0
@@ -178,7 +177,6 @@ _BuildCC_ is licensed under the Apache License, Version 2.0. See [LICENSE](LICEN
 - [Taskflow](https://github.com/taskflow/taskflow) (Parallel Programming) [MIT License] [Header Only]
   - See also [3rd-Party](https://github.com/taskflow/taskflow/tree/master/3rd-party) used by Taskflow
 - [Nlohmann::Json](https://github.com/nlohmann/json) (JSON Serialization) [MIT License] [Header Only]
-- [Flatbuffers](https://github.com/google/flatbuffers) (Serialization) [Apache-2.0 License]
 - [CLI11](https://github.com/CLIUtils/CLI11) (Argument Parsing) [BSD-3-Clause License] [Header Only]
 - [CppUTest](https://github.com/cpputest/cpputest) (Unit Testing/Mocking) [BSD-3-Clause License]
 - [Tl::Optional](https://github.com/TartanLlama/optional) (Optional support) [CC0-1.0 License]

bootstrap/CMakeLists.txt

@@ -2,7 +2,6 @@ add_executable(buildcc_lib_bootstrap
     main.buildcc.cpp
 )
 target_sources(buildcc_lib_bootstrap PRIVATE
-    src/build_flatbuffers.cpp
     src/build_nlohmann_json.cpp
     src/build_cli11.cpp
     src/build_fmtlib.cpp

bootstrap/include/bootstrap/build_buildcc.h

@@ -20,7 +20,6 @@
 #include "buildcc.h"
 
 #include "build_cli11.h"
-#include "build_flatbuffers.h"
 #include "build_fmtlib.h"
 #include "build_nlohmann_json.h"
 #include "build_spdlog.h"
@@ -30,11 +29,10 @@
 
 namespace buildcc {
 
-void buildcc_cb(BaseTarget &target, const TargetInfo &flatbuffers_ho,
-                const TargetInfo &nlohmann_json_ho, const TargetInfo &fmt_ho,
-                const TargetInfo &spdlog_ho, const TargetInfo &cli11_ho,
-                const TargetInfo &taskflow_ho, const TargetInfo &tl_optional_ho,
-                const BaseTarget &tpl);
+void buildcc_cb(BaseTarget &target, const TargetInfo &nlohmann_json_ho,
+                const TargetInfo &fmt_ho, const TargetInfo &spdlog_ho,
+                const TargetInfo &cli11_ho, const TargetInfo &taskflow_ho,
+                const TargetInfo &tl_optional_ho, const BaseTarget &tpl);
 
 /**
  * @brief
@@ -43,7 +41,6 @@ void buildcc_cb(BaseTarget &target, const TargetInfo &flatbuffers_ho,
 class BuildBuildCC {
 public:
   // TargetInfo / Header Only
-  static constexpr const char *const kFlatbuffersHoName = "flatbuffers_ho";
   static constexpr const char *const kNlohmannJsonHoName = "nlohmann_json_ho";
   static constexpr const char *const kCli11HoName = "cli11_ho";
   static constexpr const char *const kFmtHoName = "fmtlib_ho";
@@ -77,12 +74,6 @@ class BuildBuildCC {
 
 private:
   void Initialize();
-  ExecutableTarget_generic &GetFlatc() {
-    return storage_.Ref<ExecutableTarget_generic>(kFlatcExeName);
-  }
-  TargetInfo &GetFlatbuffersHo() {
-    return storage_.Ref<TargetInfo>(kFlatbuffersHoName);
-  }
   TargetInfo &GetNlohmannJsonHo() {
     return storage_.Ref<TargetInfo>(kNlohmannJsonHoName);
   }

bootstrap/main.buildcc.cpp

@@ -17,7 +17,6 @@
 #include "buildcc.h"
 
 #include "bootstrap/build_cli11.h"
-#include "bootstrap/build_flatbuffers.h"
 #include "bootstrap/build_fmtlib.h"
 #include "bootstrap/build_spdlog.h"
 #include "bootstrap/build_taskflow.h"

bootstrap/src/build_buildcc.cpp

@@ -18,11 +18,10 @@
 
 namespace buildcc {
 
-void buildcc_cb(BaseTarget &target, const TargetInfo &flatbuffers_ho,
-                const TargetInfo &nlohmann_json_ho, const TargetInfo &fmt_ho,
-                const TargetInfo &spdlog_ho, const TargetInfo &cli11_ho,
-                const TargetInfo &taskflow_ho, const TargetInfo &tl_optional_ho,
-                const BaseTarget &tpl) {
+void buildcc_cb(BaseTarget &target, const TargetInfo &nlohmann_json_ho,
+                const TargetInfo &fmt_ho, const TargetInfo &spdlog_ho,
+                const TargetInfo &cli11_ho, const TargetInfo &taskflow_ho,
+                const TargetInfo &tl_optional_ho, const BaseTarget &tpl) {
   // NOTE, Build as single lib
   target.AddIncludeDir("", true);
 
@@ -88,9 +87,6 @@ void buildcc_cb(BaseTarget &target, const TargetInfo &flatbuffers_ho,
       SyncOption::HeaderFiles,
   };
 
-  // FLATBUFFERS HO
-  target.Insert(flatbuffers_ho, kInsertOptions);
-
   // NLOHMANN JSON HO
   target.Insert(nlohmann_json_ho, kInsertOptions);
 
@@ -175,17 +171,6 @@ static void global_flags_cb(TargetInfo &global_info,
 }
 
 void BuildBuildCC::Initialize() {
-  (void)storage_.Add<ExecutableTarget_generic>(
-      kFlatcExeName, kFlatcExeName, toolchain_,
-      TargetEnv(env_.GetTargetRootDir() / "third_party" / "flatbuffers",
-                env_.GetTargetBuildDir()));
-
-  // Flatbuffers HO lib
-  (void)storage_.Add<TargetInfo>(
-      kFlatbuffersHoName, toolchain_,
-      TargetEnv(env_.GetTargetRootDir() / "third_party" / "flatbuffers",
-                env_.GetTargetBuildDir()));
-
   // Nlohmann json HO lib
   (void)storage_.Add<TargetInfo>(
       kNlohmannJsonHoName, toolchain_,
@@ -240,7 +225,6 @@ void BuildBuildCC::Initialize() {
 }
 
 void BuildBuildCC::Setup(const ArgToolchainState &state) {
-  auto &flatbuffers_ho_lib = GetFlatbuffersHo();
   auto &nlohmann_json_ho_lib = GetNlohmannJsonHo();
   auto &cli11_ho_lib = GetCli11Ho();
   auto &fmt_ho_lib = GetFmtHo();
@@ -250,7 +234,6 @@ void BuildBuildCC::Setup(const ArgToolchainState &state) {
   auto &tpl_lib = GetTpl();
   auto &buildcc_lib = GetBuildcc();
   Reg::Toolchain(state)
-      .Func(flatbuffers_ho_cb, flatbuffers_ho_lib)
       .Func(nlohmann_json_ho_cb, nlohmann_json_ho_lib)
       .Func(cli11_ho_cb, cli11_ho_lib)
       .Func(fmt_ho_cb, fmt_ho_lib)
@@ -260,9 +243,9 @@ void BuildBuildCC::Setup(const ArgToolchainState &state) {
       .Func(global_flags_cb, tpl_lib, toolchain_)
       .Build(tpl_cb, tpl_lib)
       .Func(global_flags_cb, buildcc_lib, toolchain_)
-      .Build(buildcc_cb, buildcc_lib, flatbuffers_ho_lib, nlohmann_json_ho_lib,
-             fmt_ho_lib, spdlog_ho_lib, cli11_ho_lib, taskflow_ho_lib,
-             tl_optional_ho_lib, tpl_lib)
+      .Build(buildcc_cb, buildcc_lib, nlohmann_json_ho_lib, fmt_ho_lib,
+             spdlog_ho_lib, cli11_ho_lib, taskflow_ho_lib, tl_optional_ho_lib,
+             tpl_lib)
       .Dep(buildcc_lib, tpl_lib);
 }
 

buildcc/CMakeLists.txt

@@ -13,7 +13,6 @@ if(${BUILDCC_BUILD_AS_SINGLE_LIB})
     target_link_libraries(buildcc PUBLIC
         fmt::fmt
         tl::optional
-        flatbuffers
         nlohmann_json::nlohmann_json
         Taskflow
         CLI11::CLI11
@@ -29,7 +28,7 @@ if(${BUILDCC_BUILD_AS_SINGLE_LIB})
     endif()
 endif()
 
-# Flatbuffers schema
+# Schema
 add_subdirectory(schema)
 
 # Environment

buildcc/lib/target/src/generator/file_generator.cpp

@@ -18,8 +18,6 @@
 
 #include <algorithm>
 
-#include "flatbuffers/flexbuffers.h"
-
 #include "env/assert_fatal.h"
 
 namespace {
@@ -32,29 +30,18 @@ class FileGeneratorBlobHandler : public buildcc::CustomBlobHandler {
   // serialized_data has already been verified
   static std::vector<std::string>
   Deserialize(const std::vector<uint8_t> &serialized_data) {
+    json j = json::from_msgpack(serialized_data, true, false);
     std::vector<std::string> deserialized;
-    auto flex_vect = flexbuffers::GetRoot(serialized_data).AsVector();
-    for (size_t i = 0; i < flex_vect.size(); i++) {
-      deserialized.emplace_back(flex_vect[i].AsString().str());
-    }
+    j.get_to(deserialized);
     return deserialized;
   }
 
 private:
   const std::vector<std::string> &commands_;
 
   bool Verify(const std::vector<uint8_t> &serialized_data) const override {
-    auto flex_ref = flexbuffers::GetRoot(serialized_data);
-    if (!flex_ref.IsVector()) {
-      return false;
-    }
-    auto flex_vect = flex_ref.AsVector();
-    for (size_t i = 0; i < flex_vect.size(); i++) {
-      if (!flex_vect[i].IsString()) {
-        return false;
-      }
-    }
-    return true;
+    json j = json::from_msgpack(serialized_data, true, false);
+    return !j.is_discarded();
   }
 
   bool IsEqual(const std::vector<uint8_t> &previous,
@@ -63,14 +50,8 @@ class FileGeneratorBlobHandler : public buildcc::CustomBlobHandler {
   }
 
   std::vector<uint8_t> Serialize() const override {
-    flexbuffers::Builder builder;
-    builder.Vector([&]() {
-      for (const auto &c : commands_) {
-        builder.Add(c);
-      }
-    });
-    builder.Finish();
-    return builder.GetBuffer();
+    json j = commands_;
+    return json::to_msgpack(j);
   }
 };
 

buildcc/lib/target/test/target/test_custom_generator.cpp

@@ -4,8 +4,6 @@
 #include "expect_custom_generator.h"
 #include "test_target_util.h"
 
-#include "flatbuffers/flexbuffers.h"
-
 #include <memory>
 
 // NOTE, Make sure all these includes are AFTER the system and header includes
@@ -986,7 +984,8 @@ class MyCustomBlobHandler : public buildcc::CustomBlobHandler {
 
 private:
   bool Verify(const std::vector<uint8_t> &serialized_data) const override {
-    return flexbuffers::GetRoot(serialized_data).IsInt();
+    json j = json::from_msgpack(serialized_data);
+    return !j.is_discarded();
   }
 
   bool IsEqual(const std::vector<uint8_t> &previous,
@@ -995,15 +994,16 @@ class MyCustomBlobHandler : public buildcc::CustomBlobHandler {
   }
 
   std::vector<uint8_t> Serialize() const override {
-    flexbuffers::Builder builder;
-    builder.Add(recheck_value);
-    builder.Finish();
-    return builder.GetBuffer();
+    json j = recheck_value;
+    return json::to_msgpack(j);
   }
 
   // serialized_data has already been verified
   int32_t Deserialize(const std::vector<uint8_t> &serialized_data) const {
-    return flexbuffers::GetRoot(serialized_data).AsInt32();
+    json j = json::from_msgpack(serialized_data);
+    int32_t deserialized;
+    j.get_to(deserialized);
+    return deserialized;
   }
 };
 

buildcc/schema/cmake/schema.cmake

@@ -20,7 +20,6 @@ if (${TESTING})
     )
     target_link_libraries(mock_schema PUBLIC
         mock_env
-        flatbuffers
         nlohmann_json::nlohmann_json
 
         CppUTest
@@ -88,7 +87,6 @@ if(${BUILDCC_BUILD_AS_INTERFACE})
     )
     target_link_libraries(schema PUBLIC
         env
-        flatbuffers
         nlohmann_json::nlohmann_json
     )
     target_include_directories(schema PRIVATE