diff --git a/README.md b/README.md index 7bbd9ec..c19c9db 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +# Changes + +Version 1.1.0 introduces healthchecks for the containers. + # Hadoop Docker ## Supported Hadoop Versions diff --git a/base/Dockerfile b/base/Dockerfile index 15bef58..db2e51b 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -1,8 +1,10 @@ -FROM debian:7 +FROM debian:8 MAINTAINER Ivan Ermilov -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openjdk-7-jdk -ENV JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64/ +RUN echo "deb http://ftp.debian.org/debian jessie-backports main" >> /etc/apt/sources.list + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openjdk-8-jdk +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends net-tools curl diff --git a/datanode/Dockerfile b/datanode/Dockerfile index c8d36e4..37aad11 100644 --- a/datanode/Dockerfile +++ b/datanode/Dockerfile @@ -1,6 +1,8 @@ -FROM bde2020/hadoop-base:1.0.0 +FROM bde2020/hadoop-base:1.1.0-hadoop2.7.1-java8 MAINTAINER Ivan Ermilov +HEALTHCHECK CMD curl -f http://localhost:50075/ || exit 1 + ENV HDFS_CONF_dfs_datanode_data_dir=file:///hadoop/dfs/data RUN mkdir -p /hadoop/dfs/data VOLUME /hadoop/dfs/data diff --git a/docker-compose-local.yml b/docker-compose-local.yml new file mode 100644 index 0000000..8de012b --- /dev/null +++ b/docker-compose-local.yml @@ -0,0 +1,95 @@ +version: "2" + +services: + namenode: + build: ./namenode + hostname: namenode + container_name: namenode + volumes: + - ./data/namenode:/hadoop/dfs/name + environment: + - CLUSTER_NAME=test + env_file: + - ./hadoop.env + + resourcemanager: + build: ./resourcemanager + hostname: resourcemanager + container_name: resourcemanager + depends_on: + - "namenode" + links: + - "namenode" + ports: + - "58088:8088" + env_file: + - ./hadoop.env + + historyserver: + build: ./historyserver + hostname: historyserver + container_name: historyserver + volumes: + - ./data/historyserver:/hadoop/yarn/timeline + depends_on: + - "namenode" + links: + - "namenode" + ports: + - "58188:8188" + env_file: + - ./hadoop.env + + nodemanager1: + build: ./nodemanager + hostname: nodemanager1 + container_name: nodemanager1 + depends_on: + - "namenode" + - "resourcemanager" + links: + - "namenode" + - "resourcemanager" + ports: + - "58042:8042" + env_file: + - ./hadoop.env + + datanode1: + build: ./datanode + hostname: datanode1 + container_name: datanode1 + depends_on: + - "namenode" + links: + - "namenode" + volumes: + - ./data/datanode1:/hadoop/dfs/data + env_file: + - ./hadoop.env + + datanode2: + build: ./datanode + hostname: datanode2 + container_name: datanode2 + depends_on: + - "namenode" + links: + - "namenode" + volumes: + - ./data/datanode2:/hadoop/dfs/data + env_file: + - ./hadoop.env + + datanode3: + build: ./datanode + hostname: datanode3 + container_name: datanode3 + depends_on: + - "namenode" + links: + - "namenode" + volumes: + - ./data/datanode3:/hadoop/dfs/data + env_file: + - ./hadoop.env diff --git a/docker-compose.yml b/docker-compose.yml index 2998aae..bef8ae7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,74 +1,81 @@ -namenode: - image: bde2020/hadoop-namenode:1.0.0 - hostname: namenode - container_name: namenode - domainname: hadoop - net: hadoop - volumes: - - ./data/namenode:/hadoop/dfs/name - environment: - - CLUSTER_NAME=test - env_file: - - ./hadoop.env +version: "2" -resourcemanager: - image: bde2020/hadoop-resourcemanager:1.0.0 - hostname: resourcemanager - container_name: resourcemanager - domainname: hadoop - net: hadoop - env_file: - - ./hadoop.env +services: + namenode: + image: bde2020/hadoop-namenode:1.1.0-hadoop2.7.1-java8 + container_name: namenode + volumes: + - hadoop_namenode:/hadoop/dfs/name + environment: + - CLUSTER_NAME=test + env_file: + - ./hadoop.env + + resourcemanager: + image: bde2020/hadoop-resourcemanager:1.1.0-hadoop2.7.1-java8 + container_name: resourcemanager + depends_on: + - namenode + - datanode1 + - datanode2 + env_file: + - ./hadoop.env + + historyserver: + image: bde2020/hadoop-historyserver:1.1.0-hadoop2.7.1-java8 + container_name: historyserver + depends_on: + - namenode + - datanode1 + - datanode2 + volumes: + - hadoop_historyserver:/hadoop/yarn/timeline + env_file: + - ./hadoop.env + + nodemanager1: + image: bde2020/hadoop-nodemanager:1.1.0-hadoop2.7.1-java8 + container_name: nodemanager1 + depends_on: + - namenode + - datanode1 + - datanode2 + env_file: + - ./hadoop.env + + datanode1: + image: bde2020/hadoop-datanode:1.1.0-hadoop2.7.1-java8 + container_name: datanode1 + depends_on: + - namenode + volumes: + - hadoop_datanode1:/hadoop/dfs/data + env_file: + - ./hadoop.env + + datanode2: + image: bde2020/hadoop-datanode:1.1.0-hadoop2.7.1-java8 + container_name: datanode2 + depends_on: + - namenode + volumes: + - hadoop_datanode2:/hadoop/dfs/data + env_file: + - ./hadoop.env + + datanode3: + image: bde2020/hadoop-datanode:1.1.0-hadoop2.7.1-java8 + container_name: datanode3 + depends_on: + - namenode + volumes: + - hadoop_datanode3:/hadoop/dfs/data + env_file: + - ./hadoop.env -historyserver: - image: bde2020/hadoop-historyserver:1.0.0 - hostname: historyserver - container_name: historyserver - domainname: hadoop - net: hadoop - volumes: - - ./data/historyserver:/hadoop/yarn/timeline - env_file: - - ./hadoop.env - -nodemanager1: - image: bde2020/hadoop-nodemanager:1.0.0 - hostname: nodemanager1 - container_name: nodemanager1 - domainname: hadoop - net: hadoop - env_file: - - ./hadoop.env - -datanode1: - image: bde2020/hadoop-datanode:1.0.0 - hostname: datanode1 - container_name: datanode1 - domainname: hadoop - net: hadoop - volumes: - - ./data/datanode1:/hadoop/dfs/data - env_file: - - ./hadoop.env - -datanode2: - image: bde2020/hadoop-datanode:1.0.0 - hostname: datanode2 - container_name: datanode2 - domainname: hadoop - net: hadoop - volumes: - - ./data/datanode2:/hadoop/dfs/data - env_file: - - ./hadoop.env - -datanode3: - image: bde2020/hadoop-datanode:1.0.0 - hostname: datanode3 - container_name: datanode3 - domainname: hadoop - net: hadoop - volumes: - - ./data/datanode3:/hadoop/dfs/data - env_file: - - ./hadoop.env +volumes: + hadoop_namenode: + hadoop_datanode1: + hadoop_datanode2: + hadoop_datanode3: + hadoop_historyserver: diff --git a/historyserver/Dockerfile b/historyserver/Dockerfile index 07c10f2..55da9e5 100644 --- a/historyserver/Dockerfile +++ b/historyserver/Dockerfile @@ -1,6 +1,8 @@ FROM bde2020/hadoop-base:1.0.0 MAINTAINER Ivan Ermilov +HEALTHCHECK CMD curl -f http://localhost:8188/ || exit 1 + ENV YARN_CONF_yarn_timeline___service_leveldb___timeline___store_path=/hadoop/yarn/timeline RUN mkdir -p /hadoop/yarn/timeline VOLUME /hadoop/yarn/timeline diff --git a/namenode/Dockerfile b/namenode/Dockerfile index e54e816..6d3d8f5 100644 --- a/namenode/Dockerfile +++ b/namenode/Dockerfile @@ -1,6 +1,8 @@ -FROM bde2020/hadoop-base:1.0.0 +FROM bde2020/hadoop-base:1.1.0-hadoop2.7.1-java8 MAINTAINER Ivan Ermilov +HEALTHCHECK CMD curl -f http://localhost:50070/ || exit 1 + ENV HDFS_CONF_dfs_namenode_name_dir=file:///hadoop/dfs/name RUN mkdir -p /hadoop/dfs/name VOLUME /hadoop/dfs/name diff --git a/nodemanager/Dockerfile b/nodemanager/Dockerfile index 4514fbd..1e44977 100644 --- a/nodemanager/Dockerfile +++ b/nodemanager/Dockerfile @@ -1,6 +1,8 @@ -FROM bde2020/hadoop-base:1.0.0 +FROM bde2020/hadoop-base:1.1.0-hadoop2.7.1-java8 MAINTAINER Ivan Ermilov +HEALTHCHECK CMD curl -f http://localhost:8042/ || exit 1 + ADD run.sh /run.sh RUN chmod a+x /run.sh diff --git a/resourcemanager/Dockerfile b/resourcemanager/Dockerfile index 9244ab9..710915f 100644 --- a/resourcemanager/Dockerfile +++ b/resourcemanager/Dockerfile @@ -1,6 +1,8 @@ -FROM bde2020/hadoop-base:1.0.0 +FROM bde2020/hadoop-base:1.1.0-hadoop2.7.1-java8 MAINTAINER Ivan Ermilov +HEALTHCHECK CMD curl -f http://localhost:8088/ || exit 1 + ADD run.sh /run.sh RUN chmod a+x /run.sh