Migrate third_party/ to git submodules.

This replaces third_party/absl/ and third_party/protobuf-lite/ with
submodules checked out at specific versions for abseil-cpp, googletest,
benchmark, and protobuf libraries.

This change allows removing local code copies of absl and googltest
functionality.

Author: ZacharyGarrett

.github/workflows/cifuzz.yml

@@ -23,7 +23,7 @@ jobs:
         dry-run: false
         language: c++
     - name: Upload Crash
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       if: failure() && steps.build.outcome == 'success'
       with:
         name: artifacts

.github/workflows/cmake.yml

@@ -14,19 +14,19 @@ permissions:
 jobs:
   build:
     strategy:
+      fail-fast: false
       matrix:
-        os: [ ubuntu-latest, ubuntu-20.04, windows-latest, macOS-11 ]
+        os: [ ubuntu-latest, ubuntu-22.04, windows-latest ]
         arch: [ x64 ]
-        include:
-          - os: windows-latest
-            arch: x86
     runs-on: ${{ matrix.os }}
 
     permissions:
       contents: write # svenstaro/upload-release-action 
 
     steps:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      with:
+        submodules: 'recursive'
     - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
       with:
         python-version: '3.x'
@@ -35,16 +35,14 @@ jobs:
     - name: Config for Windows
       if: runner.os == 'Windows'
       run: |
-        if ("${{matrix.arch}}" -eq "x64") {
-          $msbuildPlatform = "x64"
-        } else {
-          $msbuildPlatform = "Win32"
-        }
-        cmake -A $msbuildPlatform -B ${{github.workspace}}/build -DSPM_BUILD_TEST=ON -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
+        cmake -A x64 -B ${{github.workspace}}/build -DSPM_BUILD_TEST=ON -DBUILD_SHARED_LIBS=OFF -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
 
     - name: Config for Unix
       if: runner.os != 'Windows'
-      run: cmake -B ${{github.workspace}}/build -DSPM_BUILD_TEST=ON -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y pkgconf
+        cmake -B ${{github.workspace}}/build -DSPM_BUILD_TEST=ON -DBUILD_SHARED_LIBS=OFF -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
       env:
         CMAKE_OSX_ARCHITECTURES: arm64;x86_64
 
@@ -54,6 +52,8 @@ jobs:
     - name: Test
       working-directory: ${{github.workspace}}/build
       run: ctest -C Release --output-on-failure
+      env:
+        TEST_SRCDIR: ${{github.workspace}}/data
 
     - name: Package
       working-directory: ${{github.workspace}}/build
@@ -65,12 +65,16 @@ jobs:
         python -m pip install --require-hashes --no-dependencies -r ../.github/workflows/requirements/base.txt
         python setup.py build
         python setup.py bdist_wheel
+        pip install dist/*.whl
+        pushd test
         python -m pytest
+      env:
+        GITHUB_REF_NAME: "${{github.ref_name}}"
 
     - name: Upload artifcacts
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
-        name: artifcacts
+        name: "build-python-wrapper-artifacts-${{matrix.os}}-${{matrix.arch}}"
         path: ./build/*.7z
 
     - name: Upload Release Assets

.github/workflows/cross_build.yml

@@ -21,20 +21,29 @@ jobs:
 
     steps:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      with:
+        submodules: 'recursive'
 
     - name: Install cross tools
       run: |
         sudo apt-get update
-        sudo apt-get install -y sudo qemu-user gdb zstd dwarfdump {gcc,g++}-10-{i686,aarch64,riscv64,powerpc,powerpc64,powerpc64le,s390x,sparc64,m68k,sh4,alpha}-linux-gnu {gcc,g++}-10-arm-linux-gnueabihf
-        sudo ln -sf /usr/bin/arm-linux-gnueabihf-gcc-10 /usr/bin/arm-linux-gnu-gcc-10
-        sudo ln -sf /usr/bin/arm-linux-gnueabihf-g++-10 /usr/bin/arm-linux-gnu-g++-10
-        sudo ln -sf /usr/arm-linux-gnueabihf /usr/arm-linux-gnu
+        sudo apt-get install -y sudo qemu-user gdb zstd dwarfdump
+        if [[ "${{matrix.arch}}" != "arm" ]]; then
+          sudo apt-get install -y {gcc,g++}-14-${{matrix.arch}}-linux-gnu
+        else
+          sudo apt-get install -y {gcc,g++}-14-arm-linux-gnueabihf
+          sudo ln -sf /usr/bin/arm-linux-gnueabihf-gcc-14 /usr/bin/arm-linux-gnu-gcc-14
+          sudo ln -sf /usr/bin/arm-linux-gnueabihf-g++-14 /usr/bin/arm-linux-gnu-g++-14
+          sudo ln -sf /usr/arm-linux-gnueabihf /usr/arm-linux-gnu
+          sudo ln -sf /usr/arm-linux-gnueabihf/lib/ld-linux-armhf.so.3 /lib/ld-linux-armhf.so.3
+        fi
+
 
     - name: Build
       run: |
         mkdir -p ${{github.workspace}}/build
         cd ${{github.workspace}}/build
-        env CXX=/usr/bin/${{matrix.arch}}-linux-gnu-g++-10 CC=/usr/bin/${{matrix.arch}}-linux-gnu-gcc-10 cmake .. -DSPM_BUILD_TEST=ON -DSPM_ENABLE_SHARED=OFF -DCMAKE_FIND_ROOT_PATH=/usr/${{matrix.arch}}-linux-gnu -DSPM_CROSS_SYSTEM_PROCESSOR=${{matrix.arch}}
+        env CXX=/usr/bin/${{matrix.arch}}-linux-gnu-g++-14 CC=/usr/bin/${{matrix.arch}}-linux-gnu-gcc-14 cmake .. -DSPM_BUILD_TEST=ON -DBUILD_SHARED_LIBS=OFF -DSPM_ENABLE_SHARED=OFF -DCMAKE_FIND_ROOT_PATH=/usr/${{matrix.arch}}-linux-gnu -DSPM_CROSS_SYSTEM_PROCESSOR=${{matrix.arch}}
         make -j$(nproc)
 
     - name: Test on QEMU
@@ -43,3 +52,5 @@ jobs:
         cd ${{github.workspace}}/build
         qemu_arch=`echo ${{matrix.arch}} | sed -e s/powerpc/ppc/ -e s/686/386/`
         qemu-${qemu_arch} -L /usr/${{matrix.arch}}-linux-gnu src/spm_test
+      env:
+        TEST_SRCDIR: ${{github.workspace}}/data

.github/workflows/wheel.yml

@@ -18,8 +18,9 @@ jobs:
       digests-macos: ${{ steps.hash-macos.outputs.digests }}
       digests-windows: ${{ steps.hash-windows.outputs.digests }}
     strategy:
+      fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macOS-11]
+        os: [ubuntu-latest, windows-latest, windows-11-arm]
     runs-on: ${{ matrix.os }}
     name: Build wheels on ${{ matrix.os }}
 
@@ -28,30 +29,36 @@ jobs:
 
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          submodules: 'recursive'
       - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
         with:
-          python-version: "3.x"
+          python-version: ">=3.8"
 
       - name: Set up QEMU
         if: runner.os == 'Linux'
         uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0
         with:
           platforms: arm64
 
-      - name: Build for Windows
-        if: runner.os == 'Windows'
+      - name: Build for Windows x64
+        if: runner.os == 'Windows' && runner.arch == 'X64'
         run: |
-          cmake -A Win32 -B ${{github.workspace}}/build_win32 -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root_win32
+          cmake -A Win32 -B ${{github.workspace}}/build_win32 -DBUILD_SHARED_LIBS=OFF -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root_win32
           cmake --build ${{github.workspace}}/build_win32 --config Release --target install --parallel 8
-          cmake -A x64   -B ${{github.workspace}}/build_amd64 -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root_amd64
+          cmake -A x64   -B ${{github.workspace}}/build_amd64 -DBUILD_SHARED_LIBS=OFF -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root_amd64
           cmake --build ${{github.workspace}}/build_amd64 --config Release --target install --parallel 8
-          cmake -A arm64 -B ${{github.workspace}}/build_arm64 -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root_arm64
+
+      - name: Build for Windows Arm
+        if: runner.os == 'Windows' && runner.arch == 'ARM64'
+        run: |
+          cmake -A arm64 -B ${{github.workspace}}/build_arm64 -DBUILD_SHARED_LIBS=OFF -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root_arm64
           cmake --build ${{github.workspace}}/build_arm64 --config Release --target install --parallel 8
 
       - name: Build for Mac
         if: runner.os == 'macOS'
         run: |
-          cmake -B ${{github.workspace}}/build -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
+          cmake -B ${{github.workspace}}/build -DSPM_ENABLE_SHARED=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
           cmake --build ${{github.workspace}}/build --config Release --target install --parallel 8
         env:
           CMAKE_OSX_ARCHITECTURES: arm64;x86_64
@@ -66,10 +73,11 @@ jobs:
         working-directory: ${{github.workspace}}/python
         run: python -m cibuildwheel --output-dir wheelhouse
         env:
+          CIBW_ENVIRONMENT: "GITHUB_REF_NAME=${{ github.ref_name }}"
           CIBW_ARCHS_LINUX: auto aarch64
           CIBW_ARCHS_MACOS: x86_64 universal2 arm64
           CIBW_ARCHS_WINDOWS: auto ARM64
-          CIBW_SKIP: "pp* *-musllinux_*"
+          CIBW_SKIP: "pp* *-musllinux_* cp3{6,7}-*"
           CIBW_BUILD_VERBOSITY: 1
 
       - name: Build sdist archive
@@ -91,9 +99,9 @@ jobs:
         run: cp -f dist/*.tar.gz wheelhouse/
 
       - name: Upload artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: artifacts
+          name: "build-wheel-artifacts-${{matrix.os}}"
           path: |
             ./python/wheelhouse/*.whl
             ./python/wheelhouse/*.tar.gz

.gitignore

@@ -63,6 +63,7 @@ dist/
 *.swp
 *.swo
 *.pyc
+*.whl
 
 m.model
 m.vocab
@@ -72,6 +73,8 @@ libsentencepiece.so*
 libsentencepiece_train.so*
 python/bundled
 _sentencepiece.*.so
-third_party/abseil-cpp
+
+# Generated proto files.
+src/*.pb.*
 
 python/sentencepiece

.gitmodules

@@ -0,0 +1,12 @@
+[submodule "third_party/googletest"]
+	path = third_party/googletest
+	url = https://github.com/google/googletest.git
+[submodule "third_party/protobuf"]
+	path = third_party/protobuf
+	url = https://github.com/protocolbuffers/protobuf.git
+[submodule "third_party/abseil-cpp"]
+	path = third_party/abseil-cpp
+	url = https://github.com/abseil/abseil-cpp.git
+[submodule "third_party/benchmark"]
+	path = third_party/benchmark
+	url = https://github.com/google/benchmark.git