From 8f5eab82c6734311180f19911fd147092f51c0c2 Mon Sep 17 00:00:00 2001 From: Ivan Ermilov Date: Fri, 20 Jan 2017 15:50:38 +0100 Subject: [PATCH 1/9] Update README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index c30b936..7bbd9ec 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ # Hadoop Docker +## Supported Hadoop Versions +* 2.7.1 with OpenJDK 7 +* 2.7.1 with OpenJDK 8 + +## Description + To deploy an example HDFS cluster, run: ``` docker network create hadoop From a7161d6889f9fa15ed9bb18995e49d168035f7b4 Mon Sep 17 00:00:00 2001 From: ivan Date: Tue, 24 Jan 2017 12:57:46 +0100 Subject: [PATCH 2/9] added EXPOSE to Dockerfile for nginx proxy --- datanode/Dockerfile | 2 ++ historyserver/Dockerfile | 2 ++ namenode/Dockerfile | 2 ++ nodemanager/Dockerfile | 2 ++ resourcemanager/Dockerfile | 2 ++ 5 files changed, 10 insertions(+) diff --git a/datanode/Dockerfile b/datanode/Dockerfile index 44bc712..c8d36e4 100644 --- a/datanode/Dockerfile +++ b/datanode/Dockerfile @@ -8,4 +8,6 @@ VOLUME /hadoop/dfs/data ADD run.sh /run.sh RUN chmod a+x /run.sh +EXPOSE 50075 + CMD ["/run.sh"] diff --git a/historyserver/Dockerfile b/historyserver/Dockerfile index 358fad8..07c10f2 100644 --- a/historyserver/Dockerfile +++ b/historyserver/Dockerfile @@ -8,4 +8,6 @@ VOLUME /hadoop/yarn/timeline ADD run.sh /run.sh RUN chmod a+x /run.sh +EXPOSE 8188 + CMD ["/run.sh"] diff --git a/namenode/Dockerfile b/namenode/Dockerfile index ee947bf..e54e816 100644 --- a/namenode/Dockerfile +++ b/namenode/Dockerfile @@ -8,4 +8,6 @@ VOLUME /hadoop/dfs/name ADD run.sh /run.sh RUN chmod a+x /run.sh +EXPOSE 50070 + CMD ["/run.sh"] diff --git a/nodemanager/Dockerfile b/nodemanager/Dockerfile index a026fd1..4514fbd 100644 --- a/nodemanager/Dockerfile +++ b/nodemanager/Dockerfile @@ -4,4 +4,6 @@ MAINTAINER Ivan Ermilov ADD run.sh /run.sh RUN chmod a+x /run.sh +EXPOSE 8042 + CMD ["/run.sh"] diff --git a/resourcemanager/Dockerfile b/resourcemanager/Dockerfile index a026fd1..9244ab9 100644 --- a/resourcemanager/Dockerfile +++ b/resourcemanager/Dockerfile @@ -4,4 +4,6 @@ MAINTAINER Ivan Ermilov ADD run.sh /run.sh RUN chmod a+x /run.sh +EXPOSE 8088 + CMD ["/run.sh"] From d75e146ccc424d621f1ce1caea33203e53f4aedb Mon Sep 17 00:00:00 2001 From: ivan Date: Tue, 24 Jan 2017 14:46:20 +0100 Subject: [PATCH 3/9] removed unnecessary instruction from README file --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index c19c9db..039aa72 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,6 @@ Version 1.1.0 introduces healthchecks for the containers. To deploy an example HDFS cluster, run: ``` - docker network create hadoop docker-compose up ``` From 734b63a31e33aa234e74e13c8628037194d989a5 Mon Sep 17 00:00:00 2001 From: Sangwon Lee Date: Tue, 21 Mar 2017 16:15:16 +0900 Subject: [PATCH 4/9] fixed typo of yarn tracker address --- hadoop.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop.env b/hadoop.env index 8bf3211..304592c 100644 --- a/hadoop.env +++ b/hadoop.env @@ -19,4 +19,4 @@ YARN_CONF_yarn_resourcemanager_hostname=resourcemanager YARN_CONF_yarn_timeline___service_hostname=historyserver YARN_CONF_yarn_resourcemanager_address=resourcemanager:8032 YARN_CONF_yarn_resourcemanager_scheduler_address=resourcemanager:8030 -YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031 +YARN_CONF_yarn_resourcemanager_resource___tracker_address=resourcemanager:8031 From 139df964c0699350d6220f8e55928bc385104141 Mon Sep 17 00:00:00 2001 From: Sangwon Lee Date: Tue, 21 Mar 2017 16:21:05 +0900 Subject: [PATCH 5/9] fixed replacing underscores --- base/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/entrypoint.sh b/base/entrypoint.sh index d7b049b..eb061b0 100644 --- a/base/entrypoint.sh +++ b/base/entrypoint.sh @@ -23,7 +23,7 @@ function configure() { echo "Configuring $module" for c in `printenv | perl -sne 'print "$1 " if m/^${envPrefix}_(.+?)=.*/' -- -envPrefix=$envPrefix`; do - name=`echo ${c} | perl -pe 's/___/-/g; s/__/_/g; s/_/./g'` + name=`echo ${c} | perl -pe 's/___/-/g; s/__/@/g; s/_/./g; s/@/_/g;'` var="${envPrefix}_${c}" value=${!var} echo " - Setting $name=$value" From fd1c6643b8a86f05b8cb78eabd0c6837afe73019 Mon Sep 17 00:00:00 2001 From: Zed Date: Thu, 14 Dec 2017 11:59:32 +0800 Subject: [PATCH 6/9] Cleaned up readme by relabelling sections: - "Quick Start": docker-compose command plus published interfaces and how to access them. Clarified how to get docker internal network ip address - "Configure Environment Variables": moved hadoop.env configuration to its own section for clarity --- README.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 039aa72..d2a7b84 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,25 @@ Version 1.1.0 introduces healthchecks for the containers. * 2.7.1 with OpenJDK 7 * 2.7.1 with OpenJDK 8 -## Description +## Quick Start To deploy an example HDFS cluster, run: ``` docker-compose up ``` +`docker-compose` creates a docker network that can be found by running `docker network list`, e.g. `dockerhadoop_default`. + +Run `docker network inpect` on the network (e.g. `dockerhadoop_default`) to find the IP the hadoop interfaces are published on. Access these interfaces with the following URLs: + +* Namenode: http://:50070/dfshealth.html#tab-overview +* History server: http://:8188/applicationhistory +* Datanode: http://:50075/ +* Nodemanager: http://:8042/node +* Resource manager: http://:8088/ + +## Configure Environment Variables + The configuration parameters can be specified in the hadoop.env file or as environmental variables for specific services (e.g. namenode, datanode etc.): ``` CORE_CONF_fs_defaultFS=hdfs://namenode:8020 @@ -36,11 +48,4 @@ The available configurations are: * /etc/hadoop/httpfs-site.xml HTTPFS_CONF * /etc/hadoop/kms-site.xml KMS_CONF -If you need to extend some other configuration file, refer to base/entrypoint.sh bash script. - -After starting the example Hadoop cluster, you should be able to access interfaces of all the components (substitute domain names by IP addresses from ```network inspect hadoop``` command): -* Namenode: http://namenode:50070/dfshealth.html#tab-overview -* History server: http://historyserver:8188/applicationhistory -* Datanode: http://datanode:50075/ -* Nodemanager: http://nodemanager:8042/node -* Resource manager: http://resourcemanager:8088/ +If you need to extend some other configuration file, refer to base/entrypoint.sh bash script. \ No newline at end of file From fe7cb6d87a74de6caf72e0cb41d92c294299901c Mon Sep 17 00:00:00 2001 From: Ivan Ermilov Date: Fri, 29 Dec 2017 22:49:08 +0100 Subject: [PATCH 7/9] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 039aa72..c29521f 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/big-data-europe/Lobby) + # Changes Version 1.1.0 introduces healthchecks for the containers. From 68a058d73c849c84e845c0661395e50812974172 Mon Sep 17 00:00:00 2001 From: Giannis Mouchakis Date: Tue, 22 May 2018 15:40:28 +0300 Subject: [PATCH 8/9] fixed dash instrunctions --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d0173d7..ab9908a 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ CORE_CONF corresponds to core-site.xml. fs_defaultFS=hdfs://namenode:8020 will b ``` fs.defaultFShdfs://namenode:8020 ``` -To define dash inside a configuration parameter, use double underscore, such as YARN_CONF_yarn_log___aggregation___enable=true (yarn-site.xml): +To define dash inside a configuration parameter, use triple underscore, such as YARN_CONF_yarn_log___aggregation___enable=true (yarn-site.xml): ``` yarn.log-aggregation-enabletrue ``` @@ -50,4 +50,4 @@ The available configurations are: * /etc/hadoop/httpfs-site.xml HTTPFS_CONF * /etc/hadoop/kms-site.xml KMS_CONF -If you need to extend some other configuration file, refer to base/entrypoint.sh bash script. \ No newline at end of file +If you need to extend some other configuration file, refer to base/entrypoint.sh bash script. From fb75d0cb86646e9e8814885ad1f4f7b447a17450 Mon Sep 17 00:00:00 2001 From: detobel36 Date: Fri, 3 Aug 2018 11:32:53 +0200 Subject: [PATCH 9/9] Typing mistake `inspect` and not `inpect` --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ab9908a..a998ae1 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ To deploy an example HDFS cluster, run: `docker-compose` creates a docker network that can be found by running `docker network list`, e.g. `dockerhadoop_default`. -Run `docker network inpect` on the network (e.g. `dockerhadoop_default`) to find the IP the hadoop interfaces are published on. Access these interfaces with the following URLs: +Run `docker network inspect` on the network (e.g. `dockerhadoop_default`) to find the IP the hadoop interfaces are published on. Access these interfaces with the following URLs: * Namenode: http://:50070/dfshealth.html#tab-overview * History server: http://:8188/applicationhistory