moved to 3.2.1/stable hadoop version

2019-11-04 18:23:06 +02:00 · 2019-11-04 18:23:06 +02:00 · c74435c2f3
commit c74435c2f3
parent bc861a60bb
13 changed files with 28 additions and 26 deletions
--- a/2
+++ b/2
@ -13,7 +13,7 @@ build:
 wordcount:
 	docker build -t hadoop-wordcount ./submit
 	docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -mkdir -p /input/
-	docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal /opt/hadoop-3.1.3/README.txt /input/
+	docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal -f /opt/hadoop-3.2.1/README.txt /input/
 	docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} hadoop-wordcount
 	docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -cat /output/*
 	docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -rm -r /output
--- a/base/Dockerfile
+++ b/base/Dockerfile
@ -17,7 +17,7 @@ RUN curl -O https://dist.apache.org/repos/dist/release/hadoop/common/KEYS

 RUN gpg --import KEYS

-ENV HADOOP_VERSION 3.1.3
+ENV HADOOP_VERSION 3.2.1
 ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz

 RUN set -x \
@ -33,11 +33,11 @@ RUN mkdir /opt/hadoop-$HADOOP_VERSION/logs

 RUN mkdir /hadoop-data

-ENV HADOOP_PREFIX=/opt/hadoop-$HADOOP_VERSION
+ENV HADOOP_HOME=/opt/hadoop-$HADOOP_VERSION
 ENV HADOOP_CONF_DIR=/etc/hadoop
 ENV MULTIHOMED_NETWORK=1
 ENV USER=root
-ENV PATH $HADOOP_PREFIX/bin/:$PATH
+ENV PATH $HADOOP_HOME/bin/:$PATH

 ADD entrypoint.sh /entrypoint.sh

--- a/datanode/Dockerfile
+++ b/datanode/Dockerfile
@ -4,7 +4,7 @@ MAINTAINER Ivan Ermilov <ivan.s.ermilov@gmail.com>

 HEALTHCHECK CMD curl -f http://localhost:9864/ || exit 1

-ENV HDFS_CONF_dfs_datanode_data_dir=file:///hadoop/dfs/data
+ENV HDFS_CONF_dfs_datanode_data_dir=/hadoop/dfs/data
 RUN mkdir -p /hadoop/dfs/data
 VOLUME /hadoop/dfs/data

--- a/datanode/run.sh
+++ b/datanode/run.sh
@ -6,4 +6,4 @@ if [ ! -d $datadir ]; then
  exit 2
 fi

-$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR datanode
+$HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR datanode
--- a/docker-compose-v3.yml
+++ b/docker-compose-v3.yml
@ -2,7 +2,7 @@ version: '3'

 services:
  namenode:
-    image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
    networks:
      - hbase
    volumes:
@ -24,7 +24,7 @@ services:
        traefik.port: 50070

  datanode:
-    image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
    networks:
      - hbase
    volumes:
@ -42,7 +42,7 @@ services:
        traefik.port: 50075

  resourcemanager:
-    image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
    networks:
      - hbase
    environment:
@ -64,7 +64,7 @@ services:
      disable: true

  nodemanager:
-    image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
    networks:
      - hbase
    environment:
@ -80,7 +80,7 @@ services:
        traefik.port: 8042

  historyserver:
-    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
    networks:
      - hbase
    volumes:
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -2,7 +2,7 @@ version: "3"

 services:
  namenode:
-    image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
    container_name: namenode
    ports:
      - 9870:9870
@ -14,7 +14,7 @@ services:
      - ./hadoop.env

  datanode:
-    image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
    container_name: datanode
    volumes:
      - hadoop_datanode:/hadoop/dfs/data
@ -24,7 +24,7 @@ services:
      - ./hadoop.env
  
  resourcemanager:
-    image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
    container_name: resourcemanager
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864"
@ -32,7 +32,7 @@ services:
      - ./hadoop.env

  nodemanager1:
-    image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
    container_name: nodemanager
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
@ -40,7 +40,7 @@ services:
      - ./hadoop.env
  
  historyserver:
-    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
    container_name: historyserver
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
--- a/hadoop.env
+++ b/hadoop.env
@ -7,6 +7,8 @@ CORE_CONF_io_compression_codecs=org.apache.hadoop.io.compress.SnappyCodec
 HDFS_CONF_dfs_webhdfs_enabled=true
 HDFS_CONF_dfs_permissions_enabled=false
 HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false
+HDFS_CONF_dfs_namenode_name_dir=/hadoop/dfs/name
+HDFS_CONF_dfs_datanode_name_dir=/hadoop/dfs/data

 YARN_CONF_yarn_log___aggregation___enable=true
 YARN_CONF_yarn_log_server_url=http://historyserver:8188/applicationhistory/logs/
@ -38,6 +40,6 @@ MAPRED_CONF_mapreduce_map_memory_mb=4096
 MAPRED_CONF_mapreduce_reduce_memory_mb=8192
 MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m
 MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m
-MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/
-MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/
-MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/
+MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
+MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
+MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
--- a/historyserver/run.sh
+++ b/historyserver/run.sh
@ -1,3 +1,3 @@
 #!/bin/bash

-$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR historyserver
+$HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR historyserver
--- a/namenode/Dockerfile
+++ b/namenode/Dockerfile
@ -4,7 +4,7 @@ MAINTAINER Ivan Ermilov <ivan.s.ermilov@gmail.com>

 HEALTHCHECK CMD curl -f http://localhost:9870/ || exit 1

-ENV HDFS_CONF_dfs_namenode_name_dir=file:///hadoop/dfs/name
+ENV HDFS_CONF_dfs_namenode_name_dir=/hadoop/dfs/name
 RUN mkdir -p /hadoop/dfs/name
 VOLUME /hadoop/dfs/name

--- a/namenode/run.sh
+++ b/namenode/run.sh
@ -13,7 +13,7 @@ fi

 if [ "`ls -A $namedir`" == "" ]; then
  echo "Formatting namenode name directory: $namedir"
-  $HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR namenode -format $CLUSTER_NAME 
+  $HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR namenode -format $CLUSTER_NAME
 fi

-$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR namenode
+$HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR namenode
--- a/nodemanager/run.sh
+++ b/nodemanager/run.sh
@ -1,3 +1,3 @@
 #!/bin/bash

-$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR nodemanager
+$HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR nodemanager
--- a/resourcemanager/run.sh
+++ b/resourcemanager/run.sh
@ -1,3 +1,3 @@
 #!/bin/bash

-$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR resourcemanager
+$HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR resourcemanager
--- a/submit/run.sh
+++ b/submit/run.sh
@ -1,3 +1,3 @@
 #!/bin/bash

-$HADOOP_PREFIX/bin/hadoop jar $JAR_FILEPATH $CLASS_TO_RUN $PARAMS 
+$HADOOP_HOME/bin/hadoop jar $JAR_FILEPATH $CLASS_TO_RUN $PARAMS