.github/workflows/ci.yml

@@ -0,0 +1,123 @@
+name: CI
+
+on:
+  push:
+    tags:
+      - '*'
+    branches:
+      - master
+      - develop
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v2
+        with:
+          java-version: 11
+          distribution: adopt
+
+      - name: Install LZO
+        run: sudo apt-get install -y lzop liblzo2-dev
+
+      - name: Run tests
+        run: |
+          sbt "project main" test
+          sbt "project lzo" test
+
+      - name: Check formatting
+        run: sbt scalafmtCheck
+
+  publish_docker:
+    needs: test
+    if: startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        app:
+          - main
+          - lzo
+          - distroless
+        include:
+          - suffix: ""
+          - app: lzo
+            run_snyk: ${{ !contains(github.ref, 'rc') }}
+          - app: distroless
+            run_snyk: ${{ !contains(github.ref, 'rc') }}
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v2
+        with:
+          java-version: 11
+          distribution: adopt
+
+      - name: Install LZO
+        run: sudo apt-get install -y lzop liblzo2-dev
+
+      - name: Login to Docker Hub
+        run: docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Publish to Docker Hub
+        run: sbt "project ${{ matrix.app }}" docker:publish
+
+      - name: Build local image, which is needed to run Snyk
+        if: matrix.run_snyk
+        run: sbt "project ${{ matrix.app }}" docker:publishLocal
+      - name: Run Snyk to check for vulnerabilities
+        uses: snyk/actions/docker@master
+        if: matrix.run_snyk
+        with:
+          image: "snowplow/snowplow-s3-loader:${{ github.ref_name }}-${{ matrix.app }}"
+          args: "--app-vulns --org=data-processing-new"
+          command: monitor
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+
+  create_release:
+    needs: test
+    if: ${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, 'rc') }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v2
+        with:
+          java-version: 11
+          distribution: adopt
+
+      - name: Install LZO
+        run: sudo apt-get install -y lzop liblzo2-dev
+
+      - name: Build artifacts
+        run: |
+          sbt assembly
+      - name: Get current version
+        id: ver
+        run: |
+          export PROJECT_VERSION=$(sbt version -Dsbt.log.noformat=true | perl -ne 'print "$1\n" if /info.*(\d+\.\d+\.\d+[^\r\n]*)/' | tail -n 1 | tr -d '\n')
+          echo "::set-output name=project_version::$PROJECT_VERSION"
+      - name: Create GitHub release and attach artifacts
+        uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          prerelease: true
+          name: Version ${{ steps.ver.outputs.project_version }}
+          tag_name: ${{ steps.ver.outputs.project_version }}
+          files: |
+            modules/main/target/scala-2.13/snowplow-s3-loader-${{ steps.ver.outputs.project_version }}.jar
+            modules/lzo/target/scala-2.13/snowplow-s3-loader-lzo-${{ steps.ver.outputs.project_version }}.jar
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/lacework.yml

@@ -29,9 +29,23 @@ jobs:
       - name: Build docker images
         run: sbt docker:publishLocal
 
-      - name: Scan snowplow-s3-loader
+      - name: Scan snowplow-s3-loader focal
         env:
           LW_ACCESS_TOKEN: ${{ secrets.LW_ACCESS_TOKEN }}
           LW_ACCOUNT_NAME: ${{ secrets.LW_ACCOUNT_NAME }}
           LW_SCANNER_SAVE_RESULTS: ${{ !contains(steps.version.outputs.tag, 'rc') }}
         run: ./lw-scanner image evaluate snowplow/snowplow-s3-loader ${{ steps.ver.outputs.tag }} --build-id ${{ github.run_id }} --no-pull
+
+      - name: Scan snowplow-s3-loader distroless
+        env:
+          LW_ACCESS_TOKEN: ${{ secrets.LW_ACCESS_TOKEN }}
+          LW_ACCOUNT_NAME: ${{ secrets.LW_ACCOUNT_NAME }}
+          LW_SCANNER_SAVE_RESULTS: ${{ !contains(steps.version.outputs.tag, 'rc') }}
+        run: ./lw-scanner image evaluate snowplow/snowplow-s3-loader ${{ steps.ver.outputs.tag }}-distroless --build-id ${{ github.run_id }} --no-pull
+
+      - name: Scan snowplow-s3-loader lzo
+        env:
+          LW_ACCESS_TOKEN: ${{ secrets.LW_ACCESS_TOKEN }}
+          LW_ACCOUNT_NAME: ${{ secrets.LW_ACCOUNT_NAME }}
+          LW_SCANNER_SAVE_RESULTS: ${{ !contains(steps.version.outputs.tag, 'rc') }}
+        run: ./lw-scanner image evaluate snowplow/snowplow-s3-loader ${{ steps.ver.outputs.tag }}-lzo --build-id ${{ github.run_id }} --no-pull

CHANGELOG

@@ -1,3 +1,78 @@
+Version 2.2.9 (2024-07-16)
+--------------------------
+Bump amazon-kinesis-client to 1.15.1
+Add sts module to lzo Docker image
+
+Version 2.2.8 (2023-11-24)
+--------------------------
+Scan Docker images in Snyk Github action (#285)
+Bump pureconfig to 0.15.0 (#286)
+Bump reload4j to 1.2.22 (#286)
+Bump snappy-java to 1.1.10.4 (#286)
+
+Version 2.2.7 (2023-04-14)
+--------------------------
+Bump sbt-snowplow-release to 0.3.1 (#282)
+
+Version 2.2.6 (2023-01-10)
+--------------------------
+Update copyright notice to 2023 (#279)
+Use sbt-snowplow-release to build docker images (#278)
+Bump protobuf-java to 3.21.12 (#277)
+Bump jackson to 2.14.1 (#276)
+
+Version 2.2.5 (2022-11-15)
+--------------------------
+Add STS to runtime dependencies (#275)
+
+Version 2.2.4 (2022-11-02)
+--------------------------
+Ensure docker image has latest libexpat version (#271)
+
+Version 2.2.3 (2022-09-27)
+--------------------------
+Fix: loader cannot start with InitialPosition = AT_TIMESTAMP (#267)
+Bump scala version to 2.13.9 (#270)
+
+Version 2.2.2 (2022-07-21)
+--------------------------
+Ensure docker image has latest libfreetype6 version (#265)
+
+Version 2.2.1 (2022-06-30)
+--------------------------
+Bump hadoop to 3.3.3 (#263)
+
+Version 2.2.0 (2022-05-19)
+--------------------------
+Publish distroless docker image (#258)
+Bump jackson-databind to 2.12.6.1 (#260)
+Bump amazon-kinesis-client to 1.14.8 (#259)
+Split lzo serializers into a separate sbt project (#261)
+
+Version 2.1.4 (2022-02-15)
+--------------------------
+Update copyright notice to 2022 (#255)
+Change docker base image to eclipse-temurin:11-jre-focal (#254)
+Bump protobuf-java to 3.19.4 (#253)
+Bump jackson to 2.12.6 (2.12.6)
+Bump kinesis client to 1.14.7 (#251)
+
+Version 2.1.3 (2021-12-23)
+--------------------------
+Fix partition format in example hocon (#247)
+Clean up terminated shards before expiry (#248)
+
+Version 2.1.2 (2021-12-15)
+--------------------------
+Bump amazon-kinesis-client to 1.14.5 (#245)
+
+Version 2.1.1 (2021-12-15)
+--------------------------
+Exclude transitive dependencies of hadoop (#243)
+Bump commons-collections to 3.2.2 (#242)
+Bump elephant-bird-core to 4.17 (#241)
+Remove log4j (#240)
+
 Version 2.1.0 (2021-11-26)
 --------------------------
 Update readme (#239)
@@ -16,7 +91,7 @@ Use AdoptOpenJDK 11 as docker base image (#224)
 Use snowplow-badrows (#215)
 Use sbt-tpolecat (#222)
 Report metrics to StatsD (#216)
-Integrate Sentry (close #218)
+Integrate Sentry (#218)
 Redesign config file structure (#214)
 Harmonize module structure (#210)
 Drop NSQ support (#211)

LICENSE-2.0.txt

@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2014-2021 Snowplow Analytics Ltd.
+   Copyright 2014-2023 Snowplow Analytics Ltd.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.