Skip to content

Commit c74435c

Browse files
nevmmvnevmmv
committedNov 4, 2019
moved to 3.2.1/stable hadoop version
·
3.2.12.0.0-hadoop3.2.1-java8
1 parent bc861a6 commit c74435c

File tree

13 files changed

+28
-26
lines changed

13 files changed

+28
-26
lines changed
 

‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ build:
1313
wordcount:
1414
docker build -t hadoop-wordcount ./submit
1515
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -mkdir -p /input/
16-
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal /opt/hadoop-3.1.3/README.txt /input/
16+
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal -f /opt/hadoop-3.2.1/README.txt /input/
1717
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} hadoop-wordcount
1818
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -cat /output/*
1919
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -rm -r /output

‎base/Dockerfile

+3-3
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ RUN curl -O https://dist.apache.org/repos/dist/release/hadoop/common/KEYS
1717

1818
RUN gpg --import KEYS
1919

20-
ENV HADOOP_VERSION 3.1.3
20+
ENV HADOOP_VERSION 3.2.1
2121
ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz
2222

2323
RUN set -x \
@@ -33,11 +33,11 @@ RUN mkdir /opt/hadoop-$HADOOP_VERSION/logs
3333

3434
RUN mkdir /hadoop-data
3535

36-
ENV HADOOP_PREFIX=/opt/hadoop-$HADOOP_VERSION
36+
ENV HADOOP_HOME=/opt/hadoop-$HADOOP_VERSION
3737
ENV HADOOP_CONF_DIR=/etc/hadoop
3838
ENV MULTIHOMED_NETWORK=1
3939
ENV USER=root
40-
ENV PATH $HADOOP_PREFIX/bin/:$PATH
40+
ENV PATH $HADOOP_HOME/bin/:$PATH
4141

4242
ADD entrypoint.sh /entrypoint.sh
4343

‎datanode/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ MAINTAINER Ivan Ermilov <ivan.s.ermilov@gmail.com>
44

55
HEALTHCHECK CMD curl -f http://localhost:9864/ || exit 1
66

7-
ENV HDFS_CONF_dfs_datanode_data_dir=file:///hadoop/dfs/data
7+
ENV HDFS_CONF_dfs_datanode_data_dir=/hadoop/dfs/data
88
RUN mkdir -p /hadoop/dfs/data
99
VOLUME /hadoop/dfs/data
1010

‎datanode/run.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,4 @@ if [ ! -d $datadir ]; then
66
exit 2
77
fi
88

9-
$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR datanode
9+
$HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR datanode

‎docker-compose-v3.yml

+5-5
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ version: '3'
22

33
services:
44
namenode:
5-
image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8
5+
image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
66
networks:
77
- hbase
88
volumes:
@@ -24,7 +24,7 @@ services:
2424
traefik.port: 50070
2525

2626
datanode:
27-
image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8
27+
image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
2828
networks:
2929
- hbase
3030
volumes:
@@ -42,7 +42,7 @@ services:
4242
traefik.port: 50075
4343

4444
resourcemanager:
45-
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.3-java8
45+
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
4646
networks:
4747
- hbase
4848
environment:
@@ -64,7 +64,7 @@ services:
6464
disable: true
6565

6666
nodemanager:
67-
image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.3-java8
67+
image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
6868
networks:
6969
- hbase
7070
environment:
@@ -80,7 +80,7 @@ services:
8080
traefik.port: 8042
8181

8282
historyserver:
83-
image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.3-java8
83+
image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
8484
networks:
8585
- hbase
8686
volumes:

‎docker-compose.yml

+5-5
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ version: "3"
22

33
services:
44
namenode:
5-
image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8
5+
image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
66
container_name: namenode
77
ports:
88
- 9870:9870
@@ -14,7 +14,7 @@ services:
1414
- ./hadoop.env
1515

1616
datanode:
17-
image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8
17+
image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
1818
container_name: datanode
1919
volumes:
2020
- hadoop_datanode:/hadoop/dfs/data
@@ -24,23 +24,23 @@ services:
2424
- ./hadoop.env
2525

2626
resourcemanager:
27-
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.3-java8
27+
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
2828
container_name: resourcemanager
2929
environment:
3030
SERVICE_PRECONDITION: "namenode:9870 datanode:9864"
3131
env_file:
3232
- ./hadoop.env
3333

3434
nodemanager1:
35-
image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.3-java8
35+
image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
3636
container_name: nodemanager
3737
environment:
3838
SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
3939
env_file:
4040
- ./hadoop.env
4141

4242
historyserver:
43-
image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.3-java8
43+
image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
4444
container_name: historyserver
4545
environment:
4646
SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"

‎hadoop.env

+5-3
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ CORE_CONF_io_compression_codecs=org.apache.hadoop.io.compress.SnappyCodec
77
HDFS_CONF_dfs_webhdfs_enabled=true
88
HDFS_CONF_dfs_permissions_enabled=false
99
HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false
10+
HDFS_CONF_dfs_namenode_name_dir=/hadoop/dfs/name
11+
HDFS_CONF_dfs_datanode_name_dir=/hadoop/dfs/data
1012

1113
YARN_CONF_yarn_log___aggregation___enable=true
1214
YARN_CONF_yarn_log_server_url=http://historyserver:8188/applicationhistory/logs/
@@ -38,6 +40,6 @@ MAPRED_CONF_mapreduce_map_memory_mb=4096
3840
MAPRED_CONF_mapreduce_reduce_memory_mb=8192
3941
MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m
4042
MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m
41-
MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/
42-
MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/
43-
MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/
43+
MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
44+
MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
45+
MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/

‎historyserver/run.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
#!/bin/bash
22

3-
$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR historyserver
3+
$HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR historyserver

‎namenode/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ MAINTAINER Ivan Ermilov <ivan.s.ermilov@gmail.com>
44

55
HEALTHCHECK CMD curl -f http://localhost:9870/ || exit 1
66

7-
ENV HDFS_CONF_dfs_namenode_name_dir=file:///hadoop/dfs/name
7+
ENV HDFS_CONF_dfs_namenode_name_dir=/hadoop/dfs/name
88
RUN mkdir -p /hadoop/dfs/name
99
VOLUME /hadoop/dfs/name
1010

‎namenode/run.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ fi
1313

1414
if [ "`ls -A $namedir`" == "" ]; then
1515
echo "Formatting namenode name directory: $namedir"
16-
$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR namenode -format $CLUSTER_NAME
16+
$HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR namenode -format $CLUSTER_NAME
1717
fi
1818

19-
$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR namenode
19+
$HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR namenode

‎nodemanager/run.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
#!/bin/bash
22

3-
$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR nodemanager
3+
$HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR nodemanager

‎resourcemanager/run.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
#!/bin/bash
22

3-
$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR resourcemanager
3+
$HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR resourcemanager

‎submit/run.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
#!/bin/bash
22

3-
$HADOOP_PREFIX/bin/hadoop jar $JAR_FILEPATH $CLASS_TO_RUN $PARAMS
3+
$HADOOP_HOME/bin/hadoop jar $JAR_FILEPATH $CLASS_TO_RUN $PARAMS

0 commit comments

Comments
 (0)
Please sign in to comment.