Merge branch 'release/0.5.0'

Author: BenFradet

.release.yml

@@ -0,0 +1,42 @@
+# Required: local settings
+local:
+  root_dir : <%= ENV['TRAVIS_BUILD_DIR'] %>
+
+# Required: deployment targets
+targets:
+  - type     : "bintray"
+    user     : <%= ENV['BINTRAY_SNOWPLOW_GENERIC_USER'] %>
+    password : <%= ENV['BINTRAY_SNOWPLOW_GENERIC_API_KEY'] %>
+
+# Required: packages to be deployed
+packages:
+  - repo     : "snowplow-generic"
+    name     : "kinesis-s3"
+    user_org : "snowplow"
+    publish  : true
+
+    # Will attempt to overwrite a published entity if one is found
+    override : false
+
+    # If the artifact already exists will determine whether or not to fail the release
+    continue_on_conflict : false
+
+    # The version of this package
+    version  : <%= ENV['TRAVIS_TAG'] %>
+
+    # Required IF '--check-version' is passed: will assert that both versions are the same
+    build_version : <%= FUNC['sbt_version(.)'] %>
+    
+    build_commands:
+      - sbt assembly
+
+    # Required: Artifact
+    artifacts:
+        # The artifact name is composed like so: {{prefix}}{{version}}{{suffix}}.zip
+      - prefix : "kinesis_s3_"
+        suffix : ""
+        type   : "zip"
+
+        # The binaries to put in the zip
+        binary_paths:
+          - target/scala-2.11/snowplow-kinesis-s3-<%= ENV['TRAVIS_TAG'] %>

.travis.yml

@@ -1,8 +1,21 @@
 language: scala
 scala:
-  - 2.10.1
+  - 2.11.11
 jdk:
-  - oraclejdk7
+  - oraclejdk8
 
 before_script:
-  - echo yes | sudo apt-get install lzop liblzo2-dev
+  - sudo apt-get install -y lzop liblzo2-dev
+
+before_deploy: pip install --user release-manager==0.2.0
+deploy:
+  - provider: script
+    script: release-manager --config .release.yml --make-artifact --upload-artifact --check-version
+    skip_cleanup: true
+    on:
+      tags: true
+
+env:
+  global:
+  - secure: hJTWoOYgFMNaTZZ8gpIVl+SZQS5INschfkUUReuGXAzifaXTvT8J5a/aGryJqEOh/zMEOhKRZn136aHzMMnMDxoOkO4uSXnHnhi61DZHsbaZ51g0YvIjnJyS49ad+IKoIFJlcTKAMjvd+HK3TOHkkcWA/cc7EJWYbYwPEu3xfpkick6Y2+oQd3LYANS4wmTcVYMYKF08teQW3w0XqErHrxb7prIz7lEF8uC9BHHmUYfGEvdVLmphpsP/jJBPB+TB5IRwarEyEBaITEcVL/qeDRM3FxhErY6JI9omT6QRevO9dHx2mfpQFITCjeRf9IjJBiSr7fGVGUusxJSRqg1gMbGyImwUX4a13PaV0ACL1OUHCC3hQa3Rmoh0ObQBJ37rBvkyNKuEjJmt2bX7BMC5Ck7duJFNJhGqWiFpHyZV4MM8oYn+29OcJ7Ju03Qr6MXt04+iD3CjctHI9cbCe4mYuYJcyFYi847PaLicNqDg3Cj2Q2m8KzE9guBSY2PCUvxxFLR0LGFhXXtQYg2LxrAn8pnBE5QNVr6LQ9aoeQLYUscBCjMFyVNpciRLt/IPvzZRGL3vKiKwgJqxaV7CE1LXoTbIUTwCB8jnsqlSOPJbJQCciHA31GlFKz/m7kRRkI8G0Geie7leWjM6G7Pfr3aKXKpTw7HkN9F87AYJ47/YcFU=
+  - secure: LdhTiGwDK5BivJ+/j4CLCMNGSkWN4WiTQRGGrFf2L7wSI2fdOtZTQUcKVCZolvyd6V296BLbPYxgoXs1hdDCQb/jyoQCiWDC54XuJun4gGiKko04ZY7/UCUDQ2oo6995+UcKlvQKhqL9mvGT6stuBIUefB/r+/dQhwR+L9ehFlAiOXDFEzRh3BYgIZDnbJbMvjRptzqz/d88iC7un9e+wf+F8qyObjwJ+7RrL6rddvXg1HUyv0Kt8Yv16uSlFJPuY3fyTqDmiGbm2bYqM7cUVGRokTUOwqwtBhW/uJ071r6SlTxpTNzTJARRV2KvIT+cMvNJflvFZLOL7A55fKfpTDxWCsXcbKu3D842lppqlGH4O+GM5G3+fArSnjHmFhVRwhNMsHlGG/brXHerhCPGb2+hkIaIg/sopWP+OoEWZvlRUihv9nOLTCCZy4F45dz/B38XRM9vXZscw1nX5uu/f4LvRtOIsinOHpOCFWtxloh8qWGarcvj0Q6Ix4f/rb4L/ZIN5PthZJVEswchSK9Ee4b6Pt0EIsM2gEPjdQ5RXvYb0IWRVxbgJH7ljtk+WxHDfr67gvLAzrD2X2bKQCp+mJ3xGdiH+Ddu1r8Pb/Z6g4icFAxBWQYaSuw2etvNNGcH+Y77R7Q+Q5Hbp8omXXBj243uje/zZpNwlEeoaVbOms4=

CHANGELOG

@@ -1,3 +1,32 @@
+Version 0.5.0 (2017-06-30)
+--------------------------
+Add a newline at the end of a gzipped tsv file (#49)
+Resolve environment variables in hocon (#67)
+Add support for the Chinese Kinesis endpoint (#71)
+Do not dismiss failures prior to serialization (#101)
+Reduce INFO logging (#50)
+Bump Scala version to 2.11 (#81)
+Upgrade to Java 8 (#82)
+Bump SBT to 0.13.15 (#83)
+Bump Kinesis Client Library to 1.7.5 (#55)
+Update amazon-kinesis-connectors to 1.3.0 (#17)
+Bump hadoop-common to 2.7.3 (#94)
+Bump elephant-bird to 4.15 (#95)
+Bump hadoop-lzo to 0.4.20 (#93)
+Bump scalaz to 7.0.9 (#86)
+Bump sbt-assembly to 0.14.5 (#84)
+Bump joda-time to 2.9.9 (#96)
+Bump config to 1.3.1 (#97)
+Bump snowplow-scala-tracker to 0.3.0 (#92)
+Bump specs2-core to 3.9.1 (#89)
+Replace argot by scopt (#85)
+Remove scalazon dependency (#88)
+Remove joda-convert dependency (#98)
+Remove scalaz-specs2 dependency (#90)
+Update README markdown in according with CommonMark (#76)
+Add CI/CD to project (#79)
+Add Bintray credentials to .travis.yml (#100)
+
 Version 0.4.1 (2016-06-01)
 --------------------------
 Crash when unable to find stream instead of hanging (#58)

README.md

@@ -1,8 +1,8 @@
 # Kinesis S3
 
-[ ![Build Status] [travis-image] ] [travis]
-[ ![Release] [release-image] ] [releases] 
-[ ![License] [license-image] ] [license]
+[![Build Status][travis-image]][travis]
+[![Release][release-image]][releases] 
+[![License][license-image]][license]
 
 ## Overview
 
@@ -24,7 +24,7 @@ The records are treated as byte arrays containing UTF-8 encoded strings (whether
 
 ## Quickstart
 
-Assuming git, **[Vagrant] [vagrant-install]** and **[VirtualBox] [virtualbox-install]** installed:
+Assuming git, **[Vagrant][vagrant-install]** and **[VirtualBox][virtualbox-install]** installed:
 
 ```bash
  host$ git clone https://github.com/snowplow/kinesis-s3.git
@@ -49,14 +49,11 @@ NOTE: These are already installed in the Vagrant quickstart environment.
 The Kinesis S3 LZO Sink has the following command-line interface:
 
 ```
-snowplow-kinesis-s3: Version 0.4.1. Copyright (c) 2014-2015, Snowplow Analytics
-Ltd.
+snowplow-kinesis-s3: Version 0.5.0
 
-Usage: snowplow-lzo-s3-sink [OPTIONS]
+Usage: snowplow-kinesis-s3 [options]
 
-OPTIONS
---config filename
-                   Configuration file.
+--config <filename>
 ```
 
 ## Running
@@ -78,16 +75,16 @@ guest$ sbt "run --config my.conf"
 
 ## Find out more
 
-| Technical Docs              | Setup Guide           | Roadmap & Contributing               |         
-|-----------------------------|-----------------------|--------------------------------------|
-| ![i1] [techdocs-image]      | ![i2] [setup-image]   | ![i3] [roadmap-image]                |
-| [Technical Docs] [techdocs] | [Setup Guide] [setup] | _coming soon_                        |
+| Technical Docs             | Setup Guide          | Roadmap & Contributing |
+|----------------------------|----------------------|------------------------|
+| ![i1][techdocs-image]      | ![i2][setup-image]   | ![i3][roadmap-image]   |
+| [Technical Docs][techdocs] | [Setup Guide][setup] | _coming soon_          |
 
 ## Copyright and license
 
-Kinesis S3 is copyright 2014-2015 Snowplow Analytics Ltd.
+Kinesis S3 is copyright 2014-2017 Snowplow Analytics Ltd.
 
-Licensed under the [Apache License, Version 2.0] [license] (the "License");
+Licensed under the [Apache License, Version 2.0][license] (the "License");
 you may not use this software except in compliance with the License.
 
 Unless required by applicable law or agreed to in writing, software
@@ -99,7 +96,7 @@ limitations under the License.
 [travis-image]: https://travis-ci.org/snowplow/kinesis-s3.png?branch=master
 [travis]: http://travis-ci.org/snowplow/kinesis-s3
 
-[release-image]: http://img.shields.io/badge/release-0.4.1-blue.svg?style=flat
+[release-image]: http://img.shields.io/badge/release-0.5.0-blue.svg?style=flat
 [releases]: https://github.com/snowplow/kinesis-s3/releases
 
 [license-image]: http://img.shields.io/badge/license-Apache--2-blue.svg?style=flat

build.sbt

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2014-2017 Snowplow Analytics Ltd. All rights reserved.
+ *
+ * This program is licensed to you under the Apache License Version 2.0,
+ * and you may not use this file except in compliance with the Apache License Version 2.0.
+ * You may obtain a copy of the Apache License Version 2.0 at
+ * http://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the Apache License Version 2.0 is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the Apache License Version 2.0 for the specific language governing permissions and
+ * limitations there under.
+ */
+lazy val root = project.in(file("."))
+  .settings(
+    name        := "snowplow-kinesis-s3",
+    version     := "0.5.0",
+    description := "Kinesis sink for S3"
+  )
+  .settings(BuildSettings.buildSettings)
+  .settings(BuildSettings.sbtAssemblySettings)
+  .settings(
+    libraryDependencies ++= Seq(
+      // Java
+      Dependencies.Libraries.kinesisClient,
+      Dependencies.Libraries.kinesisConnector,
+      Dependencies.Libraries.slf4j,
+      Dependencies.Libraries.hadoop,
+      Dependencies.Libraries.elephantbird,
+      Dependencies.Libraries.hadoopLZO,
+      Dependencies.Libraries.jodaTime,
+      // Scala
+      Dependencies.Libraries.scopt,
+      Dependencies.Libraries.config,
+      Dependencies.Libraries.scalaz7,
+      Dependencies.Libraries.json4sJackson,
+      Dependencies.Libraries.snowplowTracker,
+      // Scala (test only)
+      Dependencies.Libraries.specs2,
+      // Thrift (test only)
+      Dependencies.Libraries.collectorPayload
+    )
+  )
+
+shellPrompt := { _ => "kinesis-s3> " }

lib/hadoop-lzo-0.4.20-SNAPSHOT.jar

@@ -18,66 +18,70 @@ import Keys._
 object BuildSettings {
 
   // Basic settings for our app
-  lazy val basicSettings = Seq[Setting[_]](
+  lazy val basicSettings = Seq(
     organization          :=  "com.snowplowanalytics",
-    version               :=  "0.4.1",
-    description           :=  "Kinesis LZO sink for S3",
-    scalaVersion          :=  "2.10.1",
-    scalacOptions         :=  Seq("-deprecation", "-encoding", "utf8",
-                                  "-feature", "-target:jvm-1.7"),
-    scalacOptions in Test :=  Seq("-Yrangepos"),
-    resolvers             ++= Dependencies.resolutionRepos
+    scalaVersion          :=  "2.11.11",
+    scalacOptions         :=  compilerOptions,
+    javacOptions          :=  javaCompilerOptions,
+    resolvers             ++= Dependencies.resolvers
+  )
+
+  lazy val compilerOptions = Seq(
+    "-deprecation",
+    "-encoding", "UTF-8",
+    "-feature",
+    "-language:existentials",
+    "-language:higherKinds",
+    "-language:implicitConversions",
+    "-unchecked",
+    "-Yno-adapted-args",
+    "-Ywarn-dead-code",
+    "-Ywarn-numeric-widen",
+    "-Xfuture",
+    "-Xlint"
+  )
+
+  lazy val javaCompilerOptions = Seq(
+    "-source", "1.8",
+    "-target", "1.8"
   )
 
   // Makes our SBT app settings available from within the app
-  lazy val scalifySettings = Seq(sourceGenerators in Compile <+= (sourceManaged in Compile, version, name, organization) map { (d, v, n, o) =>
-    val file = d / "settings.scala"
-    IO.write(file, """package com.snowplowanalytics.snowplow.storage.kinesis.s3.generated
-      |object Settings {
-      |  val organization = "%s"
-      |  val version = "%s"
-      |  val name = "%s"
-      |}
-      |""".stripMargin.format(o, v, n))
-    Seq(file)
-  })
+  lazy val scalifySettings = Seq(
+    sourceGenerators in Compile += Def.task {
+      val file = (sourceManaged in Compile).value / "settings.scala"
+      IO.write(file, """package com.snowplowanalytics.snowplow.storage.kinesis.s3.generated
+        |object Settings {
+        |  val organization = "%s"
+        |  val version = "%s"
+        |  val name = "%s"
+        |}
+        |""".stripMargin.format(organization.value, version.value, name.value))
+      Seq(file)
+    }.taskValue
+  )
+
+  lazy val buildSettings = basicSettings ++ scalifySettings
 
   // sbt-assembly settings for building a fat jar
-  import sbtassembly.Plugin._
-  import AssemblyKeys._
-  lazy val sbtAssemblySettings = assemblySettings ++ Seq(
+  import sbtassembly.AssemblyPlugin.autoImport._
+  lazy val sbtAssemblySettings = Seq(
     // Executable jarfile
-    assemblyOption in assembly ~= { _.copy(prependShellScript = Some(defaultShellScript)) },
+    assemblyOption in assembly :=
+      (assemblyOption in assembly).value.copy(prependShellScript = Some(
+        Seq("#!/usr/bin/env sh", """exec java -jar "$0" "$@"""" + "\n")
+      )),
     // Name it as an executable
-    jarName in assembly := { s"${name.value}-${version.value}" },
-
-    excludedJars in assembly <<= (fullClasspath in assembly) map { cp =>
-      val excludes = Set(
-        "junit-4.8.2.jar",
-        "jsp-2.1-6.1.14.jar",
-        "jasper-compiler-5.5.12.jar",
-        "jsp-api-2.1-6.1.14.jar",
-        "servlet-api-2.5-6.1.14.jar",
-        "commons-beanutils-1.7.0.jar",
-        "hadoop-lzo-0.4.19.jar",
-        "stax-api-1.0.1.jar",
-        "commons-collections-3.2.1.jar"
-      )
-      cp filter { jar => excludes(jar.data.getName) }
-    },
+    assemblyJarName in assembly := { s"${name.value}-${version.value}" },
 
-    mergeStrategy in assembly := {
+    assemblyMergeStrategy in assembly := {
       case PathList("javax", "servlet", xs @ _*)         => MergeStrategy.first
       case PathList("org", "objectweb", "asm", xs @ _*)  => MergeStrategy.first
       case PathList(ps @ _*) if ps.last endsWith ".html" => MergeStrategy.first
       case "application.conf"                            => MergeStrategy.concat
       case x =>
-        val oldStrategy = (mergeStrategy in assembly).value
+        val oldStrategy = (assemblyMergeStrategy in assembly).value
         oldStrategy(x)
-    },
-
-    assemblyOption in assembly ~= { _.copy(cacheOutput = false) }
+    }
   )
-
-  lazy val buildSettings = basicSettings ++ scalifySettings ++ sbtAssemblySettings
 }