notkshitij/docker-hadoop
1
0
Fork 0
Code Issues Pull Requests Activity

Bump hadoop version to 3.1.3

Browse Source
This commit is contained in:
Eugene Agafonov 2019-10-25 22:48:14 +03:00
parent d3e6dd2e91
commit f9afb9103b
5 changed files with 15 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -13,7 +13,7 @@ build:
wordcount: wordcount:
docker build -t hadoop-wordcount ./submit docker build -t hadoop-wordcount ./submit
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -mkdir -p /input/ docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -mkdir -p /input/
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal /opt/hadoop-3.1.2/README.txt /input/ docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal /opt/hadoop-3.1.3/README.txt /input/
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} hadoop-wordcount docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} hadoop-wordcount
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -cat /output/* docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -cat /output/*
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -rm -r /output docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -rm -r /output

2
base/Dockerfile
View File

@ -17,7 +17,7 @@ RUN curl -O https://dist.apache.org/repos/dist/release/hadoop/common/KEYS
RUN gpg --import KEYS RUN gpg --import KEYS
ENV HADOOP_VERSION 3.1.2 ENV HADOOP_VERSION 3.1.3
ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz
RUN set -x \ RUN set -x \

10
docker-compose-v3.yml
View File

@ -2,7 +2,7 @@ version: '3'
services: services:
namenode: namenode:
image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.2-java8 image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8
networks: networks:
- hbase - hbase
volumes: volumes:
@ -24,7 +24,7 @@ services:
traefik.port: 50070 traefik.port: 50070
datanode: datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.2-java8 image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8
networks: networks:
- hbase - hbase
volumes: volumes:
@ -42,7 +42,7 @@ services:
traefik.port: 50075 traefik.port: 50075
resourcemanager: resourcemanager:
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.2-java8 image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.3-java8
networks: networks:
- hbase - hbase
environment: environment:
@ -64,7 +64,7 @@ services:
disable: true disable: true
nodemanager: nodemanager:
image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.2-java8 image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.3-java8
networks: networks:
- hbase - hbase
environment: environment:
@ -80,7 +80,7 @@ services:
traefik.port: 8042 traefik.port: 8042
historyserver: historyserver:
image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.2-java8 image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.3-java8
networks: networks:
- hbase - hbase
volumes: volumes:

10
docker-compose.yml
View File

@ -2,7 +2,7 @@ version: "3"
services: services:
namenode: namenode:
image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.2-java8 image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8
container_name: namenode container_name: namenode
ports: ports:
- 9870:9870 - 9870:9870
@ -14,7 +14,7 @@ services:
- ./hadoop.env - ./hadoop.env
datanode: datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.2-java8 image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8
container_name: datanode container_name: datanode
volumes: volumes:
- hadoop_datanode:/hadoop/dfs/data - hadoop_datanode:/hadoop/dfs/data
@ -24,7 +24,7 @@ services:
- ./hadoop.env - ./hadoop.env
resourcemanager: resourcemanager:
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.2-java8 image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.3-java8
container_name: resourcemanager container_name: resourcemanager
environment: environment:
SERVICE_PRECONDITION: "namenode:9870 datanode:9864" SERVICE_PRECONDITION: "namenode:9870 datanode:9864"
@ -32,7 +32,7 @@ services:
- ./hadoop.env - ./hadoop.env
nodemanager1: nodemanager1:
image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.2-java8 image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.3-java8
container_name: nodemanager container_name: nodemanager
environment: environment:
SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088" SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
@ -40,7 +40,7 @@ services:
- ./hadoop.env - ./hadoop.env
historyserver: historyserver:
image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.2-java8 image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.3-java8
container_name: historyserver container_name: historyserver
environment: environment:
SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088" SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"

6
hadoop.env
View File

@ -38,6 +38,6 @@ MAPRED_CONF_mapreduce_map_memory_mb=4096
MAPRED_CONF_mapreduce_reduce_memory_mb=8192 MAPRED_CONF_mapreduce_reduce_memory_mb=8192
MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m
MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m
MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.2/ MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/
MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.2/ MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/
MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.2/ MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/