Skip to content

Commit 6cfabba

Browse files
istreeteristreeter
committed
Bump hadoop to 3.3.3 (close #263)
1 parent 81560a1 commit 6cfabba

File tree

3 files changed

+46
-39
lines changed

3 files changed

+46
-39
lines changed

‎build.sbt

+4-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,10 @@ lazy val distroless = project.in(file("modules/distroless"))
3535

3636
lazy val lzo = project.in(file("modules/lzo"))
3737
.settings(BuildSettings.lzoSettings)
38-
.settings(libraryDependencies ++= Dependencies.lzoDependencies)
38+
.settings(
39+
libraryDependencies ++= Dependencies.lzoDependencies,
40+
excludeDependencies ++= Dependencies.hadoopExclusions
41+
)
3942
.dependsOn(main % "compile->compile; test->test")
4043
.enablePlugins(JavaAppPackaging, DockerPlugin)
4144

‎project/BuildSettings.scala

+8-8
Original file line numberDiff line numberDiff line change
@@ -102,14 +102,14 @@ object BuildSettings {
102102
lazy val sbtAssemblySettings = Seq(
103103
assembly / assemblyJarName := { s"${name.value}-${version.value}.jar" },
104104
assembly / assemblyMergeStrategy := {
105-
case PathList("javax", "servlet", xs @ _*) => MergeStrategy.first
106-
case PathList("org", "objectweb", "asm", xs @ _*) => MergeStrategy.first
107-
case PathList("org", "objectweb", "asm", xs @ _*) => MergeStrategy.first
108-
case PathList("org", "apache", "log4j", _*) => MergeStrategy.last
109-
case PathList("org", "apache", "commons", _*) => MergeStrategy.last
110-
case PathList(ps @ _*) if ps.last endsWith ".html" => MergeStrategy.first
111-
case "application.conf" => MergeStrategy.concat
112-
case "module-info.class" => MergeStrategy.discard
105+
case PathList("javax", "servlet", xs @ _*) => MergeStrategy.first
106+
case PathList("org", "objectweb", "asm", xs @ _*) => MergeStrategy.first
107+
case PathList("org", "objectweb", "asm", xs @ _*) => MergeStrategy.first
108+
case PathList("org", "apache", "log4j", _*) => MergeStrategy.last
109+
case PathList("org", "apache", "commons", _*) => MergeStrategy.last
110+
case PathList(ps @ _*) if ps.last endsWith ".html" => MergeStrategy.first
111+
case "application.conf" => MergeStrategy.concat
112+
case PathList(ps @ _*) if ps.last == "module-info.class" => MergeStrategy.first
113113
case PathList("com", "snowplowanalytics", "s3", "loader", "generated", _*) => MergeStrategy.last
114114
case x =>
115115
val oldStrategy = (assembly / assemblyMergeStrategy).value

‎project/Dependencies.scala

+34-30
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,11 @@ object Dependencies {
2525
val log4j = "2.14.0"
2626
val kinesisClient = "1.14.8"
2727
val kinesisConnector = "1.3.0"
28-
val hadoop = "2.10.1"
28+
val hadoop = "3.3.3"
2929
val elephantbird = "4.17"
3030
val hadoopLZO = "0.4.20"
3131
val jackson = "2.12.6.1"
32-
val jacksonCbor = "2.12.6"
32+
val jacksonCbor = "2.12.7"
3333
val sentry = "1.7.30"
3434
val collections = "3.2.2" // Address vulnerability
3535
val jaxbApi = "2.3.1"
@@ -49,37 +49,22 @@ object Dependencies {
4949

5050
object Libraries {
5151
// Java
52-
val slf4j = "org.slf4j" % "slf4j-simple" % V.slf4j
53-
val jclOverSlf4j = "org.slf4j" % "jcl-over-slf4j" % V.slf4j
54-
val kinesisClient = "com.amazonaws" % "amazon-kinesis-client" % V.kinesisClient
55-
val kinesisConnector = "com.amazonaws" % "amazon-kinesis-connectors" % V.kinesisConnector
56-
val jacksonCbor = "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % V.jacksonCbor
57-
val jackson = "com.fasterxml.jackson.core" % "jackson-databind" % V.jackson
58-
val thrift = "org.apache.thrift" % "libthrift" % V.thrift
59-
val hadoop = ("org.apache.hadoop" % "hadoop-common" % V.hadoop)
60-
.exclude("org.slf4j", "slf4j-log4j12")
61-
.exclude("commons-beanutils", "commons-beanutils")
62-
.exclude("commons-beanutils", "commons-beanutils-core")
63-
.exclude("commons-collections", "commons-collections")
64-
.exclude("commons-logging", "commons-logging")
65-
.exclude("org.apache.htrace", "htrace-core")
66-
.exclude("junit", "junit")
67-
.exclude("org.apache.zookeeper", "zookeeper")
68-
.exclude("org.apache.hadoop", "hadoop-auth")
69-
.exclude("org.apache.curator", "curator-client")
70-
.exclude("log4j", "log4j")
71-
.exclude("com.google.code.gson", "gson")
72-
.exclude("org.apache.avro", "avro")
73-
.exclude("org.codehaus.jackson", "jackson-mapper-asl")
74-
.exclude("com.sun.jersey", "jersey-json")
75-
.exclude("org.mortbay.jetty", "jetty-sslengine")
76-
.exclude("org.mortbay.jetty", "jetty-util")
77-
.exclude("org.mortbay.jetty", "jetty")
52+
val slf4j = "org.slf4j" % "slf4j-simple" % V.slf4j
53+
val jclOverSlf4j = "org.slf4j" % "jcl-over-slf4j" % V.slf4j
54+
val kinesisClient = "com.amazonaws" % "amazon-kinesis-client" % V.kinesisClient
55+
val kinesisConnector = "com.amazonaws" % "amazon-kinesis-connectors" % V.kinesisConnector
56+
val jacksonCbor = "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % V.jacksonCbor
57+
val jackson = "com.fasterxml.jackson.core" % "jackson-databind" % V.jackson
58+
val thrift = "org.apache.thrift" % "libthrift" % V.thrift
59+
val hadoopMapReduce = "org.apache.hadoop" % "hadoop-mapreduce-client-core" % V.hadoop
60+
val hadoop = "org.apache.hadoop" % "hadoop-common" % V.hadoop
61+
7862
val collections = "commons-collections" % "commons-collections" % V.collections
7963
val jaxbApi = "javax.xml.bind" % "jaxb-api" % V.jaxbApi % Runtime
8064
val elephantbird = ("com.twitter.elephantbird" % "elephant-bird-core" % V.elephantbird)
8165
.exclude("com.hadoop.gplcompression", "hadoop-lzo")
82-
val hadoopLZO = "com.hadoop.gplcompression" % "hadoop-lzo" % V.hadoopLZO
66+
val hadoopLZO = ("com.hadoop.gplcompression" % "hadoop-lzo" % V.hadoopLZO)
67+
.excludeAll(ExclusionRule(organization = "org.apache.hadoop"))
8368
val sentry = "io.sentry" % "sentry" % V.sentry
8469

8570
val decline = "com.monovore" %% "decline" % V.decline
@@ -97,6 +82,7 @@ object Dependencies {
9782
// Java
9883
Libraries.kinesisClient,
9984
Libraries.kinesisConnector,
85+
Libraries.jacksonCbor,
10086
Libraries.slf4j,
10187
Libraries.jclOverSlf4j,
10288
Libraries.jackson,
@@ -118,6 +104,7 @@ object Dependencies {
118104

119105
val lzoDependencies = Seq(
120106
Libraries.hadoop,
107+
Libraries.hadoopMapReduce,
121108
Libraries.elephantbird,
122109
Libraries.hadoopLZO,
123110
Libraries.thrift,
@@ -126,7 +113,24 @@ object Dependencies {
126113
)
127114

128115
val mainExclusions = Seq(
129-
"commons-logging" % "commons-logging"
116+
ExclusionRule(organization = "commons-logging", name = "commons-logging"),
117+
)
118+
119+
val hadoopExclusions = mainExclusions ++ Seq(
120+
ExclusionRule(organization = "org.apache.avro", name = "avro"),
121+
ExclusionRule(organization = "org.apache.hadoop", name = "hadoop-hdfs-client"),
122+
ExclusionRule(organization = "org.apache.hadoop", name = "hadoop-yarn-client"),
123+
ExclusionRule(organization = "org.apache.hadoop", name = "hadoop-yarn-api"),
124+
ExclusionRule(organization = "org.apache.hadoop", name = "hadoop-auth"),
125+
ExclusionRule(organization = "org.apache.hadoop", name = "hadoop-shaded-protobuf_3_7"),
126+
ExclusionRule(organization = "org.slf4j", name = "slf4j-reload4j"),
127+
ExclusionRule(organization = "com.sun.jersey"),
128+
ExclusionRule(organization = "com.sun.jersey.contribs"),
129+
ExclusionRule(organization = "com.fasterxml.jackson.jaxrs"),
130+
ExclusionRule(organization = "org.eclipse.jetty"),
131+
ExclusionRule(organization = "org.apache.zookeeper"),
132+
ExclusionRule(organization = "jakarta.activation"),
133+
ExclusionRule(organization = "jakarta.xml.bind"),
130134
)
131135

132136
}

0 commit comments

Comments
 (0)