[KYUUBI apache#2115] Update license and enhance collect_licenses script

### _Why are the changes needed?_

This PR updates the LICENSE and NOTICE files for both source release and binary release, and enhances the `collect_licenses.sh` script to support 1)collect `NOTICE*`(previous only `NOTICE`) files, 2) remove redundant or duplicated text from final NOTICE

This PR also fix the LICENSE and NOTICE files for shaded modules

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request

Closes apache#2115 from pan3793/license.

Closes apache#2115

e515eaa [Cheng Pan] nit
53052ee [Cheng Pan] License for static resource
b14837f [Cheng Pan] Remove CCO
2fe01d5 [Cheng Pan] nit
cb617e1 [Cheng Pan] log conf
7aaf2db [Cheng Pan] Update LICENSE
412a04f [Cheng Pan] Fix kyuubi-version-info.properties
0ed01fa [Cheng Pan] recover MANIFEST.MF
f0a5d19 [Cheng Pan] Update LICENSE for shade modules
407b9dd [Cheng Pan] python3
830c7a9 [Cheng Pan] Revert "Setup python3"
239890d [Cheng Pan] Revert "setup-python@v3"
b0b2766 [Cheng Pan] setup-python@v3
3487cd4 [Cheng Pan] Setup python3
82ae8f0 [Cheng Pan] Revert "Update Dockerfile install python3"
4df10c7 [Cheng Pan] Update Dockerfile install python3
bbd87a4 [Cheng Pan] niy
152d2d0 [Cheng Pan] nit
85e89dc [Cheng Pan] update license
df891a6 [Cheng Pan] Revert "Clean LICENSE"
09dcba9 [Cheng Pan] enhance collect_license.sh
acb624c [Cheng Pan] enhance collect-licenses.sh
7e0d565 [Cheng Pan] Clean LICENSE
8eb5e35 [Cheng Pan] [LICENSE] Update LICENSE-binary

Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>

Author: pan3793

LICENSE

@@ -201,15 +201,18 @@
    limitations under the License.
 
 ------------------------------------------------------------------------------------
-
-This product includes code from Apache Spark
-
-* org.apache.kyuubi.Logging copied from classes in org.apache.spark.internal.Logging
-* org.apache.kyuubi.engine.spark.FetchIterator copied from org.apache.spark.sql.hive.thriftserver.FetchIterator
-* org.apache.kyuubi.engine.spark.shim.CatalogShim_v3_0 copied some methods from org.apache.spark.sql.connector.catalog.CatalogV2Implicits
-* org.apache.kyuubi.engine.flink.util.StringUtils copied from classes in org.apache.spark.sql.catalyst.util.StringUtils
-* org.apache.kyuubi.engine.flink.util.StringUtilsSuite copied from classes in org.apache.spark.sql.catalyst.util.StringUtilsSuite
-
-Copyright: 2014 and onwards The Apache Software Foundation
-Home page: https://spark.apache.org/
-License: https://www.apache.org/licenses/LICENSE-2.0
+This product bundles various third-party components under other open source licenses.
+This section summarizes those components and their licenses. See licenses/
+for text of these licenses.
+
+Apache License Version 2.0
+--------------------------
+kyuubi-server/src/main/resources/org/apache/kyuubi/ui/swagger/*
+
+MIT license
+-----------
+kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/assets/fonts/*
+kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/icon.min.css
+kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/semantic.min.css
+kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/semantic.min.js
+kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/jquery-3.6.0.min.js

LICENSE-binary

@@ -201,41 +201,44 @@
    limitations under the License.
 
 ------------------------------------------------------------------------------------
-
-This product includes code from Apache Spark
-
-* org.apache.kyuubi.Logging copied from classes in org.apache.spark.internal.Logging
-* org.apache.kyuubi.engine.spark.FetchIterator copied from org.apache.spark.sql.hive.thriftserver.FetchIterator
-* org.apache.kyuubi.engine.spark.shim.CatalogShim_v3_0 copied some methods from org.apache.spark.sql.connector.catalog.CatalogV2Implicits
-
-Copyright: 2014 and onwards The Apache Software Foundation
-Home page: https://spark.apache.org/
-License: https://www.apache.org/licenses/LICENSE-2.0
-
-------------------------------------------------------------------------------------
-
-This project bundles some components that are licensed under the
+This product bundles various third-party components under other open source licenses.
+This section summarizes those components and their licenses. See licenses/
+for text of these licenses.
 
 Apache License Version 2.0
 --------------------------
-commons-codec:commons-codec
+commons-lang:commons-lang
 org.apache.commons:commons-lang3
 org.apache.curator:curator-client
 org.apache.curator:curator-framework
 org.apache.curator:curator-recipes
-com.google.guava:failureaccess
 com.google.guava:guava
 org.apache.hadoop:hadoop-client-api
 org.apache.hadoop:hadoop-client-runtime
+org.apache.hive:hive-common
+org.apache.hive:hive-metastore
+org.apache.hive:hive-serde
 org.apache.hive:hive-service-rpc
-org.apache.htrace:htrace-core4
+org.apache.hive:hive-shims-0.23
+org.apache.hive:hive-shims-common
 com.fasterxml.jackson.core:jackson-annotations
 com.fasterxml.jackson.core:jackson-core
 com.fasterxml.jackson.core:jackson-databind
 com.fasterxml.jackson.dataformat:jackson-dataformat-yaml
 com.fasterxml.jackson.datatype:jackson-datatype-jsr310
 com.fasterxml.jackson.jaxrs:jackson-jaxrs-base
 com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider
+com.fasterxml.jackson.module:jackson-module-jaxb-annotations
+com.fasterxml.jackson.module:jackson-module-scala_*
+jakarta.validation:jakarta.validation-api
+org.javassist:javassist
+org.glassfish.jersey.core:jersey-client
+org.glassfish.jersey.core:jersey-common
+org.glassfish.jersey.containers:jersey-container-servlet-core
+org.glassfish.jersey.ext:jersey-entity-filtering
+org.glassfish.jersey.inject:jersey-hk2
+org.glassfish.jersey.media:jersey-media-json-jackson
+org.glassfish.jersey.core:jersey-server
 org.eclipse.jetty:jetty-http
 org.eclipse.jetty:jetty-io
 org.eclipse.jetty:jetty-security
@@ -245,13 +248,29 @@ org.eclipse.jetty:jetty-util-ajax
 org.eclipse.jetty:jetty-util
 org.apache.thrift:libfb303
 org.apache.thrift:libthrift
+org.apache.logging.log4j:log4j-1.2-api
+org.apache.logging.log4j:log4j-api
+org.apache.logging.log4j:log4j-core
+org.apache.logging.log4j:log4j-slf4j-impl
 org.webjars:swagger-ui
 org.yaml:snakeyaml
 io.dropwizard.metrics:metrics-core
 io.dropwizard.metrics:metrics-jmx
 io.dropwizard.metrics:metrics-json
 io.dropwizard.metrics:metrics-jvm
 io.netty:netty-all
+io.netty:netty-buffer
+io.netty:netty-codec
+io.netty:netty-common
+io.netty:netty-handler
+io.netty:netty-resolver
+io.netty:netty-tcnative-classes
+io.netty:netty-transport-classes-epoll
+io.netty:netty-transport-classes-kqueue
+io.netty:netty-transport-native-epoll
+io.netty:netty-transport-native-kqueue
+io.netty:netty-transport-native-unix-common
+io.netty:netty-transport
 io.prometheus:simpleclient
 io.prometheus:simpleclient_common
 io.prometheus:simpleclient_dropwizard
@@ -267,30 +286,43 @@ io.swagger.core.v3:swagger-jaxrs2
 io.swagger.core.v3:swagger-models
 org.apache.zookeeper:zookeeper
 
+kyuubi-server/src/main/resources/org/apache/kyuubi/ui/swagger/*
+
 BSD
 ------------
 jline:jline
+com.thoughtworks.paranamer:paranamer
 
 BSD 3-Clause
 ------------
 org.scala-lang:scala-library
 
 Eclipse Distribution License - v 1.0
 ------------------------------------
+jakarta.activation:jakarta.activation-api
 jakarta.xml.bind:jakarta.xml.bind-api
 
 Eclipse Public License (EPL) 2.0
 --------------------------------
-jakarta.servlet:jakarta.servlet-api https://projects.eclipse.org/projects/ee4j.servlet
-
-Common Development and Distribution License (CDDL) 1.1
-------------------------------------------------------
-javax.xml.bind:jaxb-api    https://github.com/javaee/jaxb-v2
+jakarta.annotation:jakarta.annotation-api
+jakarta.servlet:jakarta.servlet-api
+jakarta.ws.rs:jakarta.ws.rs-api
+org.glassfish.hk2:hk2-api
+org.glassfish.hk2:hk2-locator
+org.glassfish.hk2:hk2-utils
+org.glassfish.hk2.external:aopalliance-repackaged
+org.glassfish.hk2.external:jakarta.inject
+org.glassfish.hk2:osgi-resource-locator
 
 MIT License
 -----------
-org.slf4j:slf4j-api
-org.slf4j:slf4j-log4j12
-org.slf4j:jcl-over-slf4j
 com.github.scopt:scopt_*
 io.github.classgraph:classgraph
+org.slf4j:slf4j-api
+org.slf4j:jcl-over-slf4j
+
+kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/assets/fonts/*
+kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/icon.min.css
+kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/semantic.min.css
+kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/semantic.min.js
+kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/jquery-3.6.0.min.js

build/Dockerfile

@@ -46,7 +46,10 @@ ENV CI ${CI}
 ADD . /workspace/kyuubi
 WORKDIR /workspace/kyuubi
 
-RUN ./build/dist ${MVN_ARG} && \
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive \
+    apt-get install -y python3 && \
+    ./build/dist ${MVN_ARG} && \
     mv /workspace/kyuubi/dist /opt/kyuubi && \
     # Removing stuff saves time because docker creates a temporary layer
     rm -rf ~/.m2 && \

build/release/append_notice.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import sys
+
+def usage():
+  print('Usage: %s <NOTICE file> <APPEND file1> <APPEND file2> ...' % sys.argv[0])
+
+redundant_text_list = [
+"""This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).""",
+"""This product includes software developed at
+The Apache Software Foundation (https://www.apache.org/)."""
+]
+
+def append(notice_file, append_file):
+    with open(notice_file, 'r') as f:
+        notice_text = f.read()
+
+    with open(append_file, 'r') as f:
+        append_text = f.read()
+
+    for text in redundant_text_list:
+        append_text = append_text.replace(text, '')
+
+    append_text = append_text.strip()
+
+    if not append_text in notice_text:
+        with open(notice_file, 'a') as f:
+            f.write(append_text)
+            f.write('\n\n')
+
+if __name__ == "__main__":
+    if len(sys.argv) < 3:
+        print(sys.argv)
+        usage()
+        sys.exit(-1)
+
+    notice_file = sys.argv[1]
+    for append_file in sys.argv[2:]:
+      append(notice_file, append_file)

build/release/collect-licenses.sh

@@ -55,7 +55,7 @@ done
 NOTICE="${DST}/NOTICE"
 [ -f "${NOTICE}" ] && rm "${NOTICE}"
 cp "${NOTICE_BINARY_PREAMBLE}" "${NOTICE}"
-(export LC_ALL=C; find "${TMP}" -name "NOTICE" | sort | xargs cat >> "${NOTICE}")
+(export LC_ALL=C; find "${TMP}" -name "NOTICE*" | sort | xargs ${KYUUBI_DIR}/build/release/append_notice.py "${NOTICE}")
 
 LICENSES="${DST}/licenses"
 [ -f "${LICENSES}" ] && rm -r "${LICENSES}"