From 4707efa4121de2aa5b7dbe756097b7396c757b79 Mon Sep 17 00:00:00 2001 From: Ivan Ermilov Date: Mon, 15 Aug 2016 11:51:38 +0200 Subject: [PATCH 1/7] added healthchecks based on http requests to WebGUI --- README.md | 4 +++ datanode/Dockerfile | 2 ++ docker-compose-local.yml | 67 ++++++++++++++++++++++++++++++++++++++ historyserver/Dockerfile | 2 ++ namenode/Dockerfile | 2 ++ nodemanager/Dockerfile | 2 ++ resourcemanager/Dockerfile | 2 ++ 7 files changed, 81 insertions(+) create mode 100644 docker-compose-local.yml diff --git a/README.md b/README.md index c30b936..94195c6 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +# Changes + +Version 1.1.0 introduces healthchecks for the containers. + # Hadoop Docker To deploy an example HDFS cluster, run: diff --git a/datanode/Dockerfile b/datanode/Dockerfile index 44bc712..60e34f3 100644 --- a/datanode/Dockerfile +++ b/datanode/Dockerfile @@ -1,6 +1,8 @@ FROM bde2020/hadoop-base:1.0.0 MAINTAINER Ivan Ermilov +HEALTHCHECK CMD curl -f http://localhost:50075/ || exit 1 + ENV HDFS_CONF_dfs_datanode_data_dir=file:///hadoop/dfs/data RUN mkdir -p /hadoop/dfs/data VOLUME /hadoop/dfs/data diff --git a/docker-compose-local.yml b/docker-compose-local.yml new file mode 100644 index 0000000..80b37bf --- /dev/null +++ b/docker-compose-local.yml @@ -0,0 +1,67 @@ +namenode: + build: ./namenode + hostname: namenode + container_name: namenode + net: hadoop + volumes: + - ./data/namenode:/hadoop/dfs/name + environment: + - CLUSTER_NAME=test + env_file: + - ./hadoop.env + +resourcemanager: + build: ./resourcemanager + hostname: resourcemanager + container_name: resourcemanager + net: hadoop + env_file: + - ./hadoop.env + +historyserver: + build: ./historyserver + hostname: historyserver + container_name: historyserver + net: hadoop + volumes: + - ./data/historyserver:/hadoop/yarn/timeline + env_file: + - ./hadoop.env + +nodemanager1: + build: ./nodemanager + hostname: nodemanager1 + container_name: nodemanager1 + net: hadoop + env_file: + - ./hadoop.env + +datanode1: + build: ./datanode + hostname: datanode1 + container_name: datanode1 + net: hadoop + volumes: + - ./data/datanode1:/hadoop/dfs/data + env_file: + - ./hadoop.env + +datanode2: + build: ./datanode + hostname: datanode2 + container_name: datanode2 + net: hadoop + volumes: + - ./data/datanode2:/hadoop/dfs/data + env_file: + - ./hadoop.env + +datanode3: + build: ./datanode + hostname: datanode3 + container_name: datanode3 + net: hadoop + volumes: + - ./data/datanode3:/hadoop/dfs/data + env_file: + - ./hadoop.env diff --git a/historyserver/Dockerfile b/historyserver/Dockerfile index 358fad8..ae60640 100644 --- a/historyserver/Dockerfile +++ b/historyserver/Dockerfile @@ -1,6 +1,8 @@ FROM bde2020/hadoop-base:1.0.0 MAINTAINER Ivan Ermilov +HEALTHCHECK CMD curl -f http://localhost:8188/ || exit 1 + ENV YARN_CONF_yarn_timeline___service_leveldb___timeline___store_path=/hadoop/yarn/timeline RUN mkdir -p /hadoop/yarn/timeline VOLUME /hadoop/yarn/timeline diff --git a/namenode/Dockerfile b/namenode/Dockerfile index ee947bf..641b864 100644 --- a/namenode/Dockerfile +++ b/namenode/Dockerfile @@ -1,6 +1,8 @@ FROM bde2020/hadoop-base:1.0.0 MAINTAINER Ivan Ermilov +HEALTHCHECK CMD curl -f http://localhost:50070/ || exit 1 + ENV HDFS_CONF_dfs_namenode_name_dir=file:///hadoop/dfs/name RUN mkdir -p /hadoop/dfs/name VOLUME /hadoop/dfs/name diff --git a/nodemanager/Dockerfile b/nodemanager/Dockerfile index a026fd1..3411b63 100644 --- a/nodemanager/Dockerfile +++ b/nodemanager/Dockerfile @@ -1,6 +1,8 @@ FROM bde2020/hadoop-base:1.0.0 MAINTAINER Ivan Ermilov +HEALTHCHECK CMD curl -f http://localhost:8042/ || exit 1 + ADD run.sh /run.sh RUN chmod a+x /run.sh diff --git a/resourcemanager/Dockerfile b/resourcemanager/Dockerfile index a026fd1..13486da 100644 --- a/resourcemanager/Dockerfile +++ b/resourcemanager/Dockerfile @@ -1,6 +1,8 @@ FROM bde2020/hadoop-base:1.0.0 MAINTAINER Ivan Ermilov +HEALTHCHECK CMD curl -f http://localhost:8088/ || exit 1 + ADD run.sh /run.sh RUN chmod a+x /run.sh From afc253b47cec07c89d2110ae4532672e95819e92 Mon Sep 17 00:00:00 2001 From: Ivan Ermilov Date: Mon, 15 Aug 2016 12:25:08 +0200 Subject: [PATCH 2/7] tried depends_on and links, not working with healthchecks --- docker-compose-local.yml | 144 ++++++++++++++++++++++----------------- 1 file changed, 83 insertions(+), 61 deletions(-) diff --git a/docker-compose-local.yml b/docker-compose-local.yml index 80b37bf..8bb61e9 100644 --- a/docker-compose-local.yml +++ b/docker-compose-local.yml @@ -1,67 +1,89 @@ -namenode: - build: ./namenode - hostname: namenode - container_name: namenode - net: hadoop - volumes: - - ./data/namenode:/hadoop/dfs/name - environment: - - CLUSTER_NAME=test - env_file: - - ./hadoop.env +version: "2" -resourcemanager: - build: ./resourcemanager - hostname: resourcemanager - container_name: resourcemanager - net: hadoop - env_file: - - ./hadoop.env +services: + namenode: + build: ./namenode + hostname: namenode + container_name: namenode + volumes: + - ./data/namenode:/hadoop/dfs/name + environment: + - CLUSTER_NAME=test + env_file: + - ./hadoop.env -historyserver: - build: ./historyserver - hostname: historyserver - container_name: historyserver - net: hadoop - volumes: - - ./data/historyserver:/hadoop/yarn/timeline - env_file: - - ./hadoop.env + resourcemanager: + build: ./resourcemanager + hostname: resourcemanager + container_name: resourcemanager + depends_on: + - "namenode" + links: + - "namenode" + env_file: + - ./hadoop.env -nodemanager1: - build: ./nodemanager - hostname: nodemanager1 - container_name: nodemanager1 - net: hadoop - env_file: - - ./hadoop.env + historyserver: + build: ./historyserver + hostname: historyserver + container_name: historyserver + volumes: + - ./data/historyserver:/hadoop/yarn/timeline + depends_on: + - "namenode" + links: + - "namenode" + env_file: + - ./hadoop.env -datanode1: - build: ./datanode - hostname: datanode1 - container_name: datanode1 - net: hadoop - volumes: - - ./data/datanode1:/hadoop/dfs/data - env_file: - - ./hadoop.env + nodemanager1: + build: ./nodemanager + hostname: nodemanager1 + container_name: nodemanager1 + depends_on: + - "namenode" + - "resourcemanager" + links: + - "namenode" + - "resourcemanager" + env_file: + - ./hadoop.env -datanode2: - build: ./datanode - hostname: datanode2 - container_name: datanode2 - net: hadoop - volumes: - - ./data/datanode2:/hadoop/dfs/data - env_file: - - ./hadoop.env + datanode1: + build: ./datanode + hostname: datanode1 + container_name: datanode1 + depends_on: + - "namenode" + links: + - "namenode" + volumes: + - ./data/datanode1:/hadoop/dfs/data + env_file: + - ./hadoop.env -datanode3: - build: ./datanode - hostname: datanode3 - container_name: datanode3 - net: hadoop - volumes: - - ./data/datanode3:/hadoop/dfs/data - env_file: - - ./hadoop.env + datanode2: + build: ./datanode + hostname: datanode2 + container_name: datanode2 + depends_on: + - "namenode" + links: + - "namenode" + volumes: + - ./data/datanode2:/hadoop/dfs/data + env_file: + - ./hadoop.env + + datanode3: + build: ./datanode + hostname: datanode3 + container_name: datanode3 + depends_on: + - "namenode" + links: + - "namenode" + volumes: + - ./data/datanode3:/hadoop/dfs/data + env_file: + - ./hadoop.env From ece591753181d4bbacdd5eebe2b9982c5ecb89ed Mon Sep 17 00:00:00 2001 From: Ivan Ermilov Date: Mon, 15 Aug 2016 12:38:57 +0200 Subject: [PATCH 3/7] added ports exposed -- use local.yml only for testing --- docker-compose-local.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker-compose-local.yml b/docker-compose-local.yml index 8bb61e9..8de012b 100644 --- a/docker-compose-local.yml +++ b/docker-compose-local.yml @@ -20,6 +20,8 @@ services: - "namenode" links: - "namenode" + ports: + - "58088:8088" env_file: - ./hadoop.env @@ -33,6 +35,8 @@ services: - "namenode" links: - "namenode" + ports: + - "58188:8188" env_file: - ./hadoop.env @@ -46,6 +50,8 @@ services: links: - "namenode" - "resourcemanager" + ports: + - "58042:8042" env_file: - ./hadoop.env From 2ded4959632aad0df04cee3ed6aee740a93040a4 Mon Sep 17 00:00:00 2001 From: Ivan Ermilov Date: Fri, 2 Sep 2016 16:27:04 +0200 Subject: [PATCH 4/7] fixed docker-compose for this branch --- docker-compose.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2998aae..bce7e9e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ namenode: - image: bde2020/hadoop-namenode:1.0.0 + image: bde2020/hadoop-namenode:1.1.0-hadoop2.7.1 hostname: namenode container_name: namenode domainname: hadoop @@ -12,7 +12,7 @@ namenode: - ./hadoop.env resourcemanager: - image: bde2020/hadoop-resourcemanager:1.0.0 + image: bde2020/hadoop-resourcemanager:1.1.0-hadoop2.7.1 hostname: resourcemanager container_name: resourcemanager domainname: hadoop @@ -21,7 +21,7 @@ resourcemanager: - ./hadoop.env historyserver: - image: bde2020/hadoop-historyserver:1.0.0 + image: bde2020/hadoop-historyserver:1.1.0-hadoop2.7.1 hostname: historyserver container_name: historyserver domainname: hadoop @@ -32,7 +32,7 @@ historyserver: - ./hadoop.env nodemanager1: - image: bde2020/hadoop-nodemanager:1.0.0 + image: bde2020/hadoop-nodemanager:1.1.0-hadoop2.7.1 hostname: nodemanager1 container_name: nodemanager1 domainname: hadoop @@ -41,7 +41,7 @@ nodemanager1: - ./hadoop.env datanode1: - image: bde2020/hadoop-datanode:1.0.0 + image: bde2020/hadoop-datanode:1.1.0-hadoop2.7.1 hostname: datanode1 container_name: datanode1 domainname: hadoop @@ -52,7 +52,7 @@ datanode1: - ./hadoop.env datanode2: - image: bde2020/hadoop-datanode:1.0.0 + image: bde2020/hadoop-datanode:1.1.0-hadoop2.7.1 hostname: datanode2 container_name: datanode2 domainname: hadoop @@ -63,7 +63,7 @@ datanode2: - ./hadoop.env datanode3: - image: bde2020/hadoop-datanode:1.0.0 + image: bde2020/hadoop-datanode:1.1.0-hadoop2.7.1 hostname: datanode3 container_name: datanode3 domainname: hadoop From e5431251b8f0a2c29d41947eb0494cdf453333e8 Mon Sep 17 00:00:00 2001 From: Ivan Ermilov Date: Wed, 30 Nov 2016 11:53:47 +0100 Subject: [PATCH 5/7] debian 7 --> 8, java 7 --> 8 --- base/Dockerfile | 8 +++++--- datanode/Dockerfile | 2 +- namenode/Dockerfile | 2 +- nodemanager/Dockerfile | 2 +- resourcemanager/Dockerfile | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/base/Dockerfile b/base/Dockerfile index 15bef58..db2e51b 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -1,8 +1,10 @@ -FROM debian:7 +FROM debian:8 MAINTAINER Ivan Ermilov -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openjdk-7-jdk -ENV JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64/ +RUN echo "deb http://ftp.debian.org/debian jessie-backports main" >> /etc/apt/sources.list + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openjdk-8-jdk +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends net-tools curl diff --git a/datanode/Dockerfile b/datanode/Dockerfile index 60e34f3..ed40b21 100644 --- a/datanode/Dockerfile +++ b/datanode/Dockerfile @@ -1,4 +1,4 @@ -FROM bde2020/hadoop-base:1.0.0 +FROM bde2020/hadoop-base:1.1.0-hadoop2.7.1-java8 MAINTAINER Ivan Ermilov HEALTHCHECK CMD curl -f http://localhost:50075/ || exit 1 diff --git a/namenode/Dockerfile b/namenode/Dockerfile index 641b864..cdaa19f 100644 --- a/namenode/Dockerfile +++ b/namenode/Dockerfile @@ -1,4 +1,4 @@ -FROM bde2020/hadoop-base:1.0.0 +FROM bde2020/hadoop-base:1.1.0-hadoop2.7.1-java8 MAINTAINER Ivan Ermilov HEALTHCHECK CMD curl -f http://localhost:50070/ || exit 1 diff --git a/nodemanager/Dockerfile b/nodemanager/Dockerfile index 3411b63..4495939 100644 --- a/nodemanager/Dockerfile +++ b/nodemanager/Dockerfile @@ -1,4 +1,4 @@ -FROM bde2020/hadoop-base:1.0.0 +FROM bde2020/hadoop-base:1.1.0-hadoop2.7.1-java8 MAINTAINER Ivan Ermilov HEALTHCHECK CMD curl -f http://localhost:8042/ || exit 1 diff --git a/resourcemanager/Dockerfile b/resourcemanager/Dockerfile index 13486da..2921e97 100644 --- a/resourcemanager/Dockerfile +++ b/resourcemanager/Dockerfile @@ -1,4 +1,4 @@ -FROM bde2020/hadoop-base:1.0.0 +FROM bde2020/hadoop-base:1.1.0-hadoop2.7.1-java8 MAINTAINER Ivan Ermilov HEALTHCHECK CMD curl -f http://localhost:8088/ || exit 1 From e1cde39d5a70eee655ea4f0be349e3a6607fe10c Mon Sep 17 00:00:00 2001 From: ivan Date: Tue, 24 Jan 2017 13:16:58 +0100 Subject: [PATCH 6/7] Added EXPOSE to Dockerfile for nginx proxy --- datanode/Dockerfile | 2 ++ historyserver/Dockerfile | 2 ++ namenode/Dockerfile | 2 ++ nodemanager/Dockerfile | 2 ++ resourcemanager/Dockerfile | 2 ++ 5 files changed, 10 insertions(+) diff --git a/datanode/Dockerfile b/datanode/Dockerfile index ed40b21..37aad11 100644 --- a/datanode/Dockerfile +++ b/datanode/Dockerfile @@ -10,4 +10,6 @@ VOLUME /hadoop/dfs/data ADD run.sh /run.sh RUN chmod a+x /run.sh +EXPOSE 50075 + CMD ["/run.sh"] diff --git a/historyserver/Dockerfile b/historyserver/Dockerfile index ae60640..55da9e5 100644 --- a/historyserver/Dockerfile +++ b/historyserver/Dockerfile @@ -10,4 +10,6 @@ VOLUME /hadoop/yarn/timeline ADD run.sh /run.sh RUN chmod a+x /run.sh +EXPOSE 8188 + CMD ["/run.sh"] diff --git a/namenode/Dockerfile b/namenode/Dockerfile index cdaa19f..6d3d8f5 100644 --- a/namenode/Dockerfile +++ b/namenode/Dockerfile @@ -10,4 +10,6 @@ VOLUME /hadoop/dfs/name ADD run.sh /run.sh RUN chmod a+x /run.sh +EXPOSE 50070 + CMD ["/run.sh"] diff --git a/nodemanager/Dockerfile b/nodemanager/Dockerfile index 4495939..1e44977 100644 --- a/nodemanager/Dockerfile +++ b/nodemanager/Dockerfile @@ -6,4 +6,6 @@ HEALTHCHECK CMD curl -f http://localhost:8042/ || exit 1 ADD run.sh /run.sh RUN chmod a+x /run.sh +EXPOSE 8042 + CMD ["/run.sh"] diff --git a/resourcemanager/Dockerfile b/resourcemanager/Dockerfile index 2921e97..710915f 100644 --- a/resourcemanager/Dockerfile +++ b/resourcemanager/Dockerfile @@ -6,4 +6,6 @@ HEALTHCHECK CMD curl -f http://localhost:8088/ || exit 1 ADD run.sh /run.sh RUN chmod a+x /run.sh +EXPOSE 8088 + CMD ["/run.sh"] From 67cc3cdd6807eff10427441ed9c4d3e3e0241a65 Mon Sep 17 00:00:00 2001 From: ivan Date: Tue, 24 Jan 2017 14:44:13 +0100 Subject: [PATCH 7/7] volumes and depends_on clauses --- docker-compose.yml | 151 ++++++++++++++++++++++++--------------------- 1 file changed, 79 insertions(+), 72 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index bce7e9e..bef8ae7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,74 +1,81 @@ -namenode: - image: bde2020/hadoop-namenode:1.1.0-hadoop2.7.1 - hostname: namenode - container_name: namenode - domainname: hadoop - net: hadoop - volumes: - - ./data/namenode:/hadoop/dfs/name - environment: - - CLUSTER_NAME=test - env_file: - - ./hadoop.env +version: "2" -resourcemanager: - image: bde2020/hadoop-resourcemanager:1.1.0-hadoop2.7.1 - hostname: resourcemanager - container_name: resourcemanager - domainname: hadoop - net: hadoop - env_file: - - ./hadoop.env +services: + namenode: + image: bde2020/hadoop-namenode:1.1.0-hadoop2.7.1-java8 + container_name: namenode + volumes: + - hadoop_namenode:/hadoop/dfs/name + environment: + - CLUSTER_NAME=test + env_file: + - ./hadoop.env + + resourcemanager: + image: bde2020/hadoop-resourcemanager:1.1.0-hadoop2.7.1-java8 + container_name: resourcemanager + depends_on: + - namenode + - datanode1 + - datanode2 + env_file: + - ./hadoop.env + + historyserver: + image: bde2020/hadoop-historyserver:1.1.0-hadoop2.7.1-java8 + container_name: historyserver + depends_on: + - namenode + - datanode1 + - datanode2 + volumes: + - hadoop_historyserver:/hadoop/yarn/timeline + env_file: + - ./hadoop.env + + nodemanager1: + image: bde2020/hadoop-nodemanager:1.1.0-hadoop2.7.1-java8 + container_name: nodemanager1 + depends_on: + - namenode + - datanode1 + - datanode2 + env_file: + - ./hadoop.env + + datanode1: + image: bde2020/hadoop-datanode:1.1.0-hadoop2.7.1-java8 + container_name: datanode1 + depends_on: + - namenode + volumes: + - hadoop_datanode1:/hadoop/dfs/data + env_file: + - ./hadoop.env + + datanode2: + image: bde2020/hadoop-datanode:1.1.0-hadoop2.7.1-java8 + container_name: datanode2 + depends_on: + - namenode + volumes: + - hadoop_datanode2:/hadoop/dfs/data + env_file: + - ./hadoop.env + + datanode3: + image: bde2020/hadoop-datanode:1.1.0-hadoop2.7.1-java8 + container_name: datanode3 + depends_on: + - namenode + volumes: + - hadoop_datanode3:/hadoop/dfs/data + env_file: + - ./hadoop.env -historyserver: - image: bde2020/hadoop-historyserver:1.1.0-hadoop2.7.1 - hostname: historyserver - container_name: historyserver - domainname: hadoop - net: hadoop - volumes: - - ./data/historyserver:/hadoop/yarn/timeline - env_file: - - ./hadoop.env - -nodemanager1: - image: bde2020/hadoop-nodemanager:1.1.0-hadoop2.7.1 - hostname: nodemanager1 - container_name: nodemanager1 - domainname: hadoop - net: hadoop - env_file: - - ./hadoop.env - -datanode1: - image: bde2020/hadoop-datanode:1.1.0-hadoop2.7.1 - hostname: datanode1 - container_name: datanode1 - domainname: hadoop - net: hadoop - volumes: - - ./data/datanode1:/hadoop/dfs/data - env_file: - - ./hadoop.env - -datanode2: - image: bde2020/hadoop-datanode:1.1.0-hadoop2.7.1 - hostname: datanode2 - container_name: datanode2 - domainname: hadoop - net: hadoop - volumes: - - ./data/datanode2:/hadoop/dfs/data - env_file: - - ./hadoop.env - -datanode3: - image: bde2020/hadoop-datanode:1.1.0-hadoop2.7.1 - hostname: datanode3 - container_name: datanode3 - domainname: hadoop - net: hadoop - volumes: - - ./data/datanode3:/hadoop/dfs/data - env_file: - - ./hadoop.env +volumes: + hadoop_namenode: + hadoop_datanode1: + hadoop_datanode2: + hadoop_datanode3: + hadoop_historyserver: