notkshitij/docker-hadoop
1
0
Fork 0
Code Issues Pull Requests Activity

Merge pull request #45 from big-data-europe/2.0.0-hadoop3.1.2-java8

Browse Source 
Move to Hadoop 3.1.2.
This commit is contained in:
gmouchakis 2019-09-12 20:34:25 +03:00 committed by GitHub
commit d3e6dd2e91
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 15 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -13,7 +13,7 @@ build:
wordcount:
docker build -t hadoop-wordcount ./submit
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -mkdir -p /input/
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal /opt/hadoop-3.1.1/README.txt /input/
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal /opt/hadoop-3.1.2/README.txt /input/
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} hadoop-wordcount
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -cat /output/*
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -rm -r /output

2
base/Dockerfile
View File

@ -17,7 +17,7 @@ RUN curl -O https://dist.apache.org/repos/dist/release/hadoop/common/KEYS
RUN gpg --import KEYS
ENV HADOOP_VERSION 3.1.1
ENV HADOOP_VERSION 3.1.2
ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz
RUN set -x \

10
docker-compose-v3.yml
View File

@ -2,7 +2,7 @@ version: '3'
services:
namenode:
image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.1-java8
image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.2-java8
networks:
- hbase
volumes:
@ -24,7 +24,7 @@ services:
traefik.port: 50070
datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.1-java8
image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.2-java8
networks:
- hbase
volumes:
@ -42,7 +42,7 @@ services:
traefik.port: 50075
resourcemanager:
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.1-java8
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.2-java8
networks:
- hbase
environment:
@ -64,7 +64,7 @@ services:
disable: true
nodemanager:
image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.1-java8
image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.2-java8
networks:
- hbase
environment:
@ -80,7 +80,7 @@ services:
traefik.port: 8042
historyserver:
image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.1-java8
image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.2-java8
networks:
- hbase
volumes:

10
docker-compose.yml
View File

@ -2,7 +2,7 @@ version: "3"
services:
namenode:
image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.1-java8
image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.2-java8
container_name: namenode
ports:
- 9870:9870
@ -14,7 +14,7 @@ services:
- ./hadoop.env
datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.1-java8
image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.2-java8
container_name: datanode
volumes:
- hadoop_datanode:/hadoop/dfs/data
@ -24,7 +24,7 @@ services:
- ./hadoop.env
resourcemanager:
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.1-java8
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.2-java8
container_name: resourcemanager
environment:
SERVICE_PRECONDITION: "namenode:9870 datanode:9864"
@ -32,7 +32,7 @@ services:
- ./hadoop.env
nodemanager1:
image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.1-java8
image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.2-java8
container_name: nodemanager
environment:
SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
@ -40,7 +40,7 @@ services:
- ./hadoop.env
historyserver:
image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.1-java8
image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.2-java8
container_name: historyserver
environment:
SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"

6
hadoop.env
View File

@ -38,6 +38,6 @@ MAPRED_CONF_mapreduce_map_memory_mb=4096
MAPRED_CONF_mapreduce_reduce_memory_mb=8192
MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m
MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m
MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.1/
MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.1/
MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.1/
MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.2/
MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.2/
MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.2/