diff --git a/Makefile b/Makefile index 30ddb28..da7cb7d 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ build: wordcount: docker build -t hadoop-wordcount ./submit docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -mkdir -p /input/ - docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal /opt/hadoop-3.1.3/README.txt /input/ + docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal -f /opt/hadoop-3.2.1/README.txt /input/ docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} hadoop-wordcount docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -cat /output/* docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -rm -r /output diff --git a/base/Dockerfile b/base/Dockerfile index 9016b73..fded474 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -17,7 +17,7 @@ RUN curl -O https://dist.apache.org/repos/dist/release/hadoop/common/KEYS RUN gpg --import KEYS -ENV HADOOP_VERSION 3.1.3 +ENV HADOOP_VERSION 3.2.1 ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz RUN set -x \ @@ -33,11 +33,11 @@ RUN mkdir /opt/hadoop-$HADOOP_VERSION/logs RUN mkdir /hadoop-data -ENV HADOOP_PREFIX=/opt/hadoop-$HADOOP_VERSION +ENV HADOOP_HOME=/opt/hadoop-$HADOOP_VERSION ENV HADOOP_CONF_DIR=/etc/hadoop ENV MULTIHOMED_NETWORK=1 ENV USER=root -ENV PATH $HADOOP_PREFIX/bin/:$PATH +ENV PATH $HADOOP_HOME/bin/:$PATH ADD entrypoint.sh /entrypoint.sh diff --git a/datanode/Dockerfile b/datanode/Dockerfile index acb8a89..ba5a3ea 100644 --- a/datanode/Dockerfile +++ b/datanode/Dockerfile @@ -4,7 +4,7 @@ MAINTAINER Ivan Ermilov HEALTHCHECK CMD curl -f http://localhost:9864/ || exit 1 -ENV HDFS_CONF_dfs_datanode_data_dir=file:///hadoop/dfs/data +ENV HDFS_CONF_dfs_datanode_data_dir=/hadoop/dfs/data RUN mkdir -p /hadoop/dfs/data VOLUME /hadoop/dfs/data diff --git a/datanode/run.sh b/datanode/run.sh index 9f57ee2..bbbb9e3 100644 --- a/datanode/run.sh +++ b/datanode/run.sh @@ -6,4 +6,4 @@ if [ ! -d $datadir ]; then exit 2 fi -$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR datanode +$HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR datanode diff --git a/docker-compose-v3.yml b/docker-compose-v3.yml index 8cb46a9..84587ec 100644 --- a/docker-compose-v3.yml +++ b/docker-compose-v3.yml @@ -2,7 +2,7 @@ version: '3' services: namenode: - image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8 + image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8 networks: - hbase volumes: @@ -24,7 +24,7 @@ services: traefik.port: 50070 datanode: - image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8 + image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8 networks: - hbase volumes: @@ -42,7 +42,7 @@ services: traefik.port: 50075 resourcemanager: - image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.3-java8 + image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8 networks: - hbase environment: @@ -64,7 +64,7 @@ services: disable: true nodemanager: - image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.3-java8 + image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8 networks: - hbase environment: @@ -80,7 +80,7 @@ services: traefik.port: 8042 historyserver: - image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.3-java8 + image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8 networks: - hbase volumes: diff --git a/docker-compose.yml b/docker-compose.yml index 8fde204..5a0b673 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: "3" services: namenode: - image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8 + image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8 container_name: namenode ports: - 9870:9870 @@ -14,7 +14,7 @@ services: - ./hadoop.env datanode: - image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8 + image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8 container_name: datanode volumes: - hadoop_datanode:/hadoop/dfs/data @@ -24,7 +24,7 @@ services: - ./hadoop.env resourcemanager: - image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.3-java8 + image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8 container_name: resourcemanager environment: SERVICE_PRECONDITION: "namenode:9870 datanode:9864" @@ -32,7 +32,7 @@ services: - ./hadoop.env nodemanager1: - image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.3-java8 + image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8 container_name: nodemanager environment: SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088" @@ -40,7 +40,7 @@ services: - ./hadoop.env historyserver: - image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.3-java8 + image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8 container_name: historyserver environment: SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088" diff --git a/hadoop.env b/hadoop.env index 1ffa665..2a01f0c 100644 --- a/hadoop.env +++ b/hadoop.env @@ -7,6 +7,8 @@ CORE_CONF_io_compression_codecs=org.apache.hadoop.io.compress.SnappyCodec HDFS_CONF_dfs_webhdfs_enabled=true HDFS_CONF_dfs_permissions_enabled=false HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false +HDFS_CONF_dfs_namenode_name_dir=/hadoop/dfs/name +HDFS_CONF_dfs_datanode_name_dir=/hadoop/dfs/data YARN_CONF_yarn_log___aggregation___enable=true YARN_CONF_yarn_log_server_url=http://historyserver:8188/applicationhistory/logs/ @@ -38,6 +40,6 @@ MAPRED_CONF_mapreduce_map_memory_mb=4096 MAPRED_CONF_mapreduce_reduce_memory_mb=8192 MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m -MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/ -MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/ -MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/ +MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/ +MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/ +MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/ diff --git a/historyserver/run.sh b/historyserver/run.sh index 1ce6633..56c8468 100644 --- a/historyserver/run.sh +++ b/historyserver/run.sh @@ -1,3 +1,3 @@ #!/bin/bash -$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR historyserver +$HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR historyserver diff --git a/namenode/Dockerfile b/namenode/Dockerfile index 4bd736f..ddd3e24 100644 --- a/namenode/Dockerfile +++ b/namenode/Dockerfile @@ -4,7 +4,7 @@ MAINTAINER Ivan Ermilov HEALTHCHECK CMD curl -f http://localhost:9870/ || exit 1 -ENV HDFS_CONF_dfs_namenode_name_dir=file:///hadoop/dfs/name +ENV HDFS_CONF_dfs_namenode_name_dir=/hadoop/dfs/name RUN mkdir -p /hadoop/dfs/name VOLUME /hadoop/dfs/name diff --git a/namenode/run.sh b/namenode/run.sh index 8ef0797..d9645a8 100644 --- a/namenode/run.sh +++ b/namenode/run.sh @@ -13,7 +13,7 @@ fi if [ "`ls -A $namedir`" == "" ]; then echo "Formatting namenode name directory: $namedir" - $HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR namenode -format $CLUSTER_NAME + $HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR namenode -format $CLUSTER_NAME fi -$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR namenode +$HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR namenode diff --git a/nodemanager/run.sh b/nodemanager/run.sh index 115bcdb..bf8a39d 100644 --- a/nodemanager/run.sh +++ b/nodemanager/run.sh @@ -1,3 +1,3 @@ #!/bin/bash -$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR nodemanager +$HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR nodemanager diff --git a/resourcemanager/run.sh b/resourcemanager/run.sh index c1bdb94..338c28f 100644 --- a/resourcemanager/run.sh +++ b/resourcemanager/run.sh @@ -1,3 +1,3 @@ #!/bin/bash -$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR resourcemanager +$HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR resourcemanager diff --git a/submit/run.sh b/submit/run.sh index 5ebe79e..0991ff8 100644 --- a/submit/run.sh +++ b/submit/run.sh @@ -1,3 +1,3 @@ #!/bin/bash -$HADOOP_PREFIX/bin/hadoop jar $JAR_FILEPATH $CLASS_TO_RUN $PARAMS +$HADOOP_HOME/bin/hadoop jar $JAR_FILEPATH $CLASS_TO_RUN $PARAMS