notkshitij/docker-hadoop
1
0
Fork 0
Code Issues Pull Requests Activity

Merge pull request #52 from nevmmv/2.0.0-hadoop3.2.1-java8

Browse Source 
moved to 3.2.1/stable hadoop version
This commit is contained in:
gmouchakis 2020-02-04 17:29:09 +02:00 committed by GitHub
commit e1e1e21ec9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 39 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -13,7 +13,7 @@ build:
wordcount:
docker build -t hadoop-wordcount ./submit
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -mkdir -p /input/
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal /opt/hadoop-3.1.3/README.txt /input/
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal -f /opt/hadoop-3.2.1/README.txt /input/
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} hadoop-wordcount
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -cat /output/*
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -rm -r /output

6
base/Dockerfile
View File

@ -17,7 +17,7 @@ RUN curl -O https://dist.apache.org/repos/dist/release/hadoop/common/KEYS
RUN gpg --import KEYS
ENV HADOOP_VERSION 3.1.3
ENV HADOOP_VERSION 3.2.1
ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz
RUN set -x \
@ -33,11 +33,11 @@ RUN mkdir /opt/hadoop-$HADOOP_VERSION/logs
RUN mkdir /hadoop-data
ENV HADOOP_PREFIX=/opt/hadoop-$HADOOP_VERSION
ENV HADOOP_HOME=/opt/hadoop-$HADOOP_VERSION
ENV HADOOP_CONF_DIR=/etc/hadoop
ENV MULTIHOMED_NETWORK=1
ENV USER=root
ENV PATH $HADOOP_PREFIX/bin/:$PATH
ENV PATH $HADOOP_HOME/bin/:$PATH
ADD entrypoint.sh /entrypoint.sh

2
datanode/Dockerfile
View File

@ -1,4 +1,4 @@
FROM bde2020/hadoop-base
FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8
MAINTAINER Ivan Ermilov <ivan.s.ermilov@gmail.com>

2
datanode/run.sh
View File

@ -6,4 +6,4 @@ if [ ! -d $datadir ]; then
exit 2
fi
$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR datanode
$HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR datanode

10
docker-compose-v3.yml
View File

@ -2,7 +2,7 @@ version: '3'
services:
namenode:
image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8
image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
networks:
- hbase
volumes:
@ -24,7 +24,7 @@ services:
traefik.port: 50070
datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8
image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
networks:
- hbase
volumes:
@ -42,7 +42,7 @@ services:
traefik.port: 50075
resourcemanager:
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.3-java8
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
networks:
- hbase
environment:
@ -64,7 +64,7 @@ services:
disable: true
nodemanager:
image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.3-java8
image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
networks:
- hbase
environment:
@ -80,7 +80,7 @@ services:
traefik.port: 8042
historyserver:
image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.3-java8
image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
networks:
- hbase
volumes:

22
docker-compose.yml
View File

@ -2,10 +2,12 @@ version: "3"
services:
namenode:
image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8
image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
container_name: namenode
restart: always
ports:
- 9870:9870
- 9000:9000
volumes:
- hadoop_namenode:/hadoop/dfs/name
environment:
@ -14,8 +16,9 @@ services:
- ./hadoop.env
datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8
image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
container_name: datanode
restart: always
volumes:
- hadoop_datanode:/hadoop/dfs/data
environment:
@ -24,26 +27,29 @@ services:
- ./hadoop.env
resourcemanager:
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.3-java8
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
container_name: resourcemanager
restart: always
environment:
SERVICE_PRECONDITION: "namenode:9870 datanode:9864"
SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864"
env_file:
- ./hadoop.env
nodemanager1:
image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.3-java8
image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
container_name: nodemanager
restart: always
environment:
SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
env_file:
- ./hadoop.env
historyserver:
image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.3-java8
image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
container_name: historyserver
restart: always
environment:
SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
volumes:
- hadoop_historyserver:/hadoop/yarn/timeline
env_file:

6
hadoop.env
View File

@ -38,6 +38,6 @@ MAPRED_CONF_mapreduce_map_memory_mb=4096
MAPRED_CONF_mapreduce_reduce_memory_mb=8192
MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m
MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m
MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/
MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/
MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/
MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/

2
historyserver/Dockerfile
View File

@ -1,4 +1,4 @@
FROM bde2020/hadoop-base
FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8
MAINTAINER Ivan Ermilov <ivan.s.ermilov@gmail.com>

2
historyserver/run.sh
View File

@ -1,3 +1,3 @@
#!/bin/bash
$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR historyserver
$HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR historyserver

2
namenode/Dockerfile
View File

@ -1,4 +1,4 @@
FROM bde2020/hadoop-base
FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8
MAINTAINER Ivan Ermilov <ivan.s.ermilov@gmail.com>

4
namenode/run.sh
View File

@ -13,7 +13,7 @@ fi
if [ "`ls -A $namedir`" == "" ]; then
echo "Formatting namenode name directory: $namedir"
$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR namenode -format $CLUSTER_NAME
$HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR namenode -format $CLUSTER_NAME
fi
$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR namenode
$HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR namenode

2
nodemanager/Dockerfile
View File

@ -1,4 +1,4 @@
FROM bde2020/hadoop-base
FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8
MAINTAINER Ivan Ermilov <ivan.s.ermilov@gmail.com>

2
nodemanager/run.sh
View File

@ -1,3 +1,3 @@
#!/bin/bash
$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR nodemanager
$HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR nodemanager

2
resourcemanager/Dockerfile
View File

@ -1,4 +1,4 @@
FROM bde2020/hadoop-base
FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8
MAINTAINER Ivan Ermilov <ivan.s.ermilov@gmail.com>

2
resourcemanager/run.sh
View File

@ -1,3 +1,3 @@
#!/bin/bash
$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR resourcemanager
$HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR resourcemanager

2
submit/Dockerfile
View File

@ -1,4 +1,4 @@
FROM bde2020/hadoop-base
FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8
MAINTAINER Ivan Ermilov <ivan.s.ermilov@gmail.com>

2
submit/run.sh
View File

@ -1,3 +1,3 @@
#!/bin/bash
$HADOOP_PREFIX/bin/hadoop jar $JAR_FILEPATH $CLASS_TO_RUN $PARAMS
$HADOOP_HOME/bin/hadoop jar $JAR_FILEPATH $CLASS_TO_RUN $PARAMS