From f9afb9103b759b0807b8179d272f2be22c323902 Mon Sep 17 00:00:00 2001 From: Eugene Agafonov Date: Fri, 25 Oct 2019 22:48:14 +0300 Subject: [PATCH] Bump hadoop version to 3.1.3 --- Makefile | 2 +- base/Dockerfile | 2 +- docker-compose-v3.yml | 10 +++++----- docker-compose.yml | 10 +++++----- hadoop.env | 6 +++--- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 4bdeb82..30ddb28 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ build: wordcount: docker build -t hadoop-wordcount ./submit docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -mkdir -p /input/ - docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal /opt/hadoop-3.1.2/README.txt /input/ + docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal /opt/hadoop-3.1.3/README.txt /input/ docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} hadoop-wordcount docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -cat /output/* docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -rm -r /output diff --git a/base/Dockerfile b/base/Dockerfile index df68993..9016b73 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -17,7 +17,7 @@ RUN curl -O https://dist.apache.org/repos/dist/release/hadoop/common/KEYS RUN gpg --import KEYS -ENV HADOOP_VERSION 3.1.2 +ENV HADOOP_VERSION 3.1.3 ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz RUN set -x \ diff --git a/docker-compose-v3.yml b/docker-compose-v3.yml index 6e93fb8..8cb46a9 100644 --- a/docker-compose-v3.yml +++ b/docker-compose-v3.yml @@ -2,7 +2,7 @@ version: '3' services: namenode: - image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.2-java8 + image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8 networks: - hbase volumes: @@ -24,7 +24,7 @@ services: traefik.port: 50070 datanode: - image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.2-java8 + image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8 networks: - hbase volumes: @@ -42,7 +42,7 @@ services: traefik.port: 50075 resourcemanager: - image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.2-java8 + image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.3-java8 networks: - hbase environment: @@ -64,7 +64,7 @@ services: disable: true nodemanager: - image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.2-java8 + image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.3-java8 networks: - hbase environment: @@ -80,7 +80,7 @@ services: traefik.port: 8042 historyserver: - image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.2-java8 + image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.3-java8 networks: - hbase volumes: diff --git a/docker-compose.yml b/docker-compose.yml index 9547641..8fde204 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: "3" services: namenode: - image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.2-java8 + image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8 container_name: namenode ports: - 9870:9870 @@ -14,7 +14,7 @@ services: - ./hadoop.env datanode: - image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.2-java8 + image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8 container_name: datanode volumes: - hadoop_datanode:/hadoop/dfs/data @@ -24,7 +24,7 @@ services: - ./hadoop.env resourcemanager: - image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.2-java8 + image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.3-java8 container_name: resourcemanager environment: SERVICE_PRECONDITION: "namenode:9870 datanode:9864" @@ -32,7 +32,7 @@ services: - ./hadoop.env nodemanager1: - image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.2-java8 + image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.3-java8 container_name: nodemanager environment: SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088" @@ -40,7 +40,7 @@ services: - ./hadoop.env historyserver: - image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.2-java8 + image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.3-java8 container_name: historyserver environment: SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088" diff --git a/hadoop.env b/hadoop.env index 7076e1c..1ffa665 100644 --- a/hadoop.env +++ b/hadoop.env @@ -38,6 +38,6 @@ MAPRED_CONF_mapreduce_map_memory_mb=4096 MAPRED_CONF_mapreduce_reduce_memory_mb=8192 MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m -MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.2/ -MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.2/ -MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.2/ +MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/ +MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/ +MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/