notkshitij/docker-hadoop
1
0
Fork 0
Code Issues Pull Requests Activity

moved to 3.2.1/stable hadoop version

Browse Source
This commit is contained in:
Mykola Nevmerzhytskyi 2019-11-04 18:23:06 +02:00
parent bc861a60bb
commit c74435c2f3
13 changed files with 28 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -13,7 +13,7 @@ build:
wordcount: wordcount:
docker build -t hadoop-wordcount ./submit docker build -t hadoop-wordcount ./submit
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -mkdir -p /input/ docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -mkdir -p /input/
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal /opt/hadoop-3.1.3/README.txt /input/ docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal -f /opt/hadoop-3.2.1/README.txt /input/
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} hadoop-wordcount docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} hadoop-wordcount
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -cat /output/* docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -cat /output/*
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -rm -r /output docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -rm -r /output

6
base/Dockerfile
View File

@ -17,7 +17,7 @@ RUN curl -O https://dist.apache.org/repos/dist/release/hadoop/common/KEYS
RUN gpg --import KEYS RUN gpg --import KEYS
ENV HADOOP_VERSION 3.1.3 ENV HADOOP_VERSION 3.2.1
ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz
RUN set -x \ RUN set -x \
@ -33,11 +33,11 @@ RUN mkdir /opt/hadoop-$HADOOP_VERSION/logs
RUN mkdir /hadoop-data RUN mkdir /hadoop-data
ENV HADOOP_PREFIX=/opt/hadoop-$HADOOP_VERSION ENV HADOOP_HOME=/opt/hadoop-$HADOOP_VERSION
ENV HADOOP_CONF_DIR=/etc/hadoop ENV HADOOP_CONF_DIR=/etc/hadoop
ENV MULTIHOMED_NETWORK=1 ENV MULTIHOMED_NETWORK=1
ENV USER=root ENV USER=root
ENV PATH $HADOOP_PREFIX/bin/:$PATH ENV PATH $HADOOP_HOME/bin/:$PATH
ADD entrypoint.sh /entrypoint.sh ADD entrypoint.sh /entrypoint.sh

2
datanode/Dockerfile
View File

@ -4,7 +4,7 @@ MAINTAINER Ivan Ermilov <ivan.s.ermilov@gmail.com>
HEALTHCHECK CMD curl -f http://localhost:9864/ || exit 1 HEALTHCHECK CMD curl -f http://localhost:9864/ || exit 1
ENV HDFS_CONF_dfs_datanode_data_dir=file:///hadoop/dfs/data ENV HDFS_CONF_dfs_datanode_data_dir=/hadoop/dfs/data
RUN mkdir -p /hadoop/dfs/data RUN mkdir -p /hadoop/dfs/data
VOLUME /hadoop/dfs/data VOLUME /hadoop/dfs/data

2
datanode/run.sh
View File

@ -6,4 +6,4 @@ if [ ! -d $datadir ]; then
exit 2 exit 2
fi fi
$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR datanode $HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR datanode

10
docker-compose-v3.yml
View File

@ -2,7 +2,7 @@ version: '3'
services: services:
namenode: namenode:
image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8 image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
networks: networks:
- hbase - hbase
volumes: volumes:
@ -24,7 +24,7 @@ services:
traefik.port: 50070 traefik.port: 50070
datanode: datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8 image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
networks: networks:
- hbase - hbase
volumes: volumes:
@ -42,7 +42,7 @@ services:
traefik.port: 50075 traefik.port: 50075
resourcemanager: resourcemanager:
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.3-java8 image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
networks: networks:
- hbase - hbase
environment: environment:
@ -64,7 +64,7 @@ services:
disable: true disable: true
nodemanager: nodemanager:
image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.3-java8 image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
networks: networks:
- hbase - hbase
environment: environment:
@ -80,7 +80,7 @@ services:
traefik.port: 8042 traefik.port: 8042
historyserver: historyserver:
image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.3-java8 image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
networks: networks:
- hbase - hbase
volumes: volumes:

10
docker-compose.yml
View File

@ -2,7 +2,7 @@ version: "3"
services: services:
namenode: namenode:
image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8 image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
container_name: namenode container_name: namenode
ports: ports:
- 9870:9870 - 9870:9870
@ -14,7 +14,7 @@ services:
- ./hadoop.env - ./hadoop.env
datanode: datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8 image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
container_name: datanode container_name: datanode
volumes: volumes:
- hadoop_datanode:/hadoop/dfs/data - hadoop_datanode:/hadoop/dfs/data
@ -24,7 +24,7 @@ services:
- ./hadoop.env - ./hadoop.env
resourcemanager: resourcemanager:
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.3-java8 image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
container_name: resourcemanager container_name: resourcemanager
environment: environment:
SERVICE_PRECONDITION: "namenode:9870 datanode:9864" SERVICE_PRECONDITION: "namenode:9870 datanode:9864"
@ -32,7 +32,7 @@ services:
- ./hadoop.env - ./hadoop.env
nodemanager1: nodemanager1:
image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.3-java8 image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
container_name: nodemanager container_name: nodemanager
environment: environment:
SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088" SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
@ -40,7 +40,7 @@ services:
- ./hadoop.env - ./hadoop.env
historyserver: historyserver:
image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.3-java8 image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
container_name: historyserver container_name: historyserver
environment: environment:
SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088" SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"

8
hadoop.env
View File

@ -7,6 +7,8 @@ CORE_CONF_io_compression_codecs=org.apache.hadoop.io.compress.SnappyCodec
HDFS_CONF_dfs_webhdfs_enabled=true HDFS_CONF_dfs_webhdfs_enabled=true
HDFS_CONF_dfs_permissions_enabled=false HDFS_CONF_dfs_permissions_enabled=false
HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false
HDFS_CONF_dfs_namenode_name_dir=/hadoop/dfs/name
HDFS_CONF_dfs_datanode_name_dir=/hadoop/dfs/data
YARN_CONF_yarn_log___aggregation___enable=true YARN_CONF_yarn_log___aggregation___enable=true
YARN_CONF_yarn_log_server_url=http://historyserver:8188/applicationhistory/logs/ YARN_CONF_yarn_log_server_url=http://historyserver:8188/applicationhistory/logs/
@ -38,6 +40,6 @@ MAPRED_CONF_mapreduce_map_memory_mb=4096
MAPRED_CONF_mapreduce_reduce_memory_mb=8192 MAPRED_CONF_mapreduce_reduce_memory_mb=8192
MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m
MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m
MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/ MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/ MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/ MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/

2
historyserver/run.sh
View File

@ -1,3 +1,3 @@
#!/bin/bash #!/bin/bash
$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR historyserver $HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR historyserver

2
namenode/Dockerfile
View File

@ -4,7 +4,7 @@ MAINTAINER Ivan Ermilov <ivan.s.ermilov@gmail.com>
HEALTHCHECK CMD curl -f http://localhost:9870/ || exit 1 HEALTHCHECK CMD curl -f http://localhost:9870/ || exit 1
ENV HDFS_CONF_dfs_namenode_name_dir=file:///hadoop/dfs/name ENV HDFS_CONF_dfs_namenode_name_dir=/hadoop/dfs/name
RUN mkdir -p /hadoop/dfs/name RUN mkdir -p /hadoop/dfs/name
VOLUME /hadoop/dfs/name VOLUME /hadoop/dfs/name

4
namenode/run.sh
View File

@ -13,7 +13,7 @@ fi
if [ "`ls -A $namedir`" == "" ]; then if [ "`ls -A $namedir`" == "" ]; then
echo "Formatting namenode name directory: $namedir" echo "Formatting namenode name directory: $namedir"
$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR namenode -format $CLUSTER_NAME $HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR namenode -format $CLUSTER_NAME
fi fi
$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR namenode $HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR namenode

2
nodemanager/run.sh
View File

@ -1,3 +1,3 @@
#!/bin/bash #!/bin/bash
$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR nodemanager $HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR nodemanager

2
resourcemanager/run.sh
View File

@ -1,3 +1,3 @@
#!/bin/bash #!/bin/bash
$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR resourcemanager $HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR resourcemanager

2
submit/run.sh
View File

@ -1,3 +1,3 @@
#!/bin/bash #!/bin/bash
$HADOOP_PREFIX/bin/hadoop jar $JAR_FILEPATH $CLASS_TO_RUN $PARAMS $HADOOP_HOME/bin/hadoop jar $JAR_FILEPATH $CLASS_TO_RUN $PARAMS