sppu-te-comp-content/DataScienceAndBigDataAnalytics
10
0
Fork 0
Code Issues Activity

Added codes, datasets and Jupyter notebooks directory.

Browse Source
This commit is contained in:
K
2025-06-11 13:48:53 +05:30
parent b3a22e9b79
commit 76dc1de8db
32 changed files with 8930 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Codes/Code-B1 (Hadoop Word Count)/Hadoop Setup.md
Executable
+198
View File
@@ -0,0 +1,198 @@
# Setup for Hadoop
This file contains instructions to install and setup Hadoop for assignment B1.
---
## Prerequisites
- OpenJDK 17
- ssh
- curl
- gpg (optional)
---
1. Installing OpenJDK 17:
```shell
sudo apt update # Update packages
sudo apt install -y openjdk-17-jdk gpg curl ssh # Install OpenJDK 17
java -version # Check java (JRE) version
javac -version # Check javac (JDK) version
```
2. Setup Hadoop user:
```shell
sudo adduser --home /home/hadoop hadoop # Set default password as Pass@123
sudo usermod -aG sudo hadoop
su - hadoop
# Hit enter to skip entering all the information such as Full name, Room Number, etc.
```
3. SSH setup:
```shell
ssh-keygen -t ed25519 # Hit enter to save in defauly location. You can skip setting a passphrase for simplicity
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 640 ~/.ssh/authorized_keys
ssh localhost # You will be asked for conformation, type "yes"
su - hadoop
```
4. Download Hadoop (version 3.4.1 in this case):
```shell
cd ~
curl -o hadoop-3.4.1.tar.gz https://dlcdn.apache.org/hadoop/common/hadoop-3.4.1/hadoop-3.4.1.tar.gz # Download Hadoop
curl -o hadoop-3.4.1.tar.gz.sha512 https://downloads.apache.org/hadoop/common/hadoop-3.4.1/hadoop-3.4.1.tar.gz.sha512 # Download the hash
curl -o hadoop-3.4.1.tar.gz.asc https://downloads.apache.org/hadoop/common/hadoop-3.4.1/hadoop-3.4.1.tar.gz.asc # Download the signature
curl -o KEYS https://downloads.apache.org/hadoop/common/KEYS # Download the keys
```
5. Verify the hash and extract the tarball:
```shell
gpg --import KEYS # Import the keys
gpg --verify hadoop-3.4.1.tar.gz.asc # Verify hash signature
sha512sum -c hadoop-3.4.1.tar.gz.sha512 # Verify hash for file
tar -xvf hadoop-3.4.1.tar.gz
mv hadoop-3.4.1/ hadoop/
cd hadoop/
mkdir -p ~/hadoopdata/hdfs/{namenode,datanode}
```
6. Modify `~/.bashrc` & `$HADOOP_HOME/etc/hadoop/hadoop-env.sh`:
```shell
cp /etc/bash.bashrc ~/.bashrc # Reset .bashrc file
chown $USER ~/.bashrc # Change ownership of .bashrc file
echo -e "\n#JAVA+HADOOP CONFIG FROM KSKA GIT\nexport JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64\nexport PATH=\$JAVA_HOME/bin:\$PATH" >> ~/.bashrc # Java env var
echo -e "export HADOOP_HOME=/home/hadoop/hadoop/\nexport HADOOP_INSTALL=\$HADOOP_HOME\nexport HADOOP_MAPRED_HOME=\$HADOOP_HOME\nexport HADOOP_COMMON_HOME=\$HADOOP_HOME\nexport HADOOP_HDFS_HOME=\$HADOOP_HOME\nexport YARN_HOME=\$HADOOP_HOME\nexport HADOOP_COMMON_LIB_NATIVE_DIR=\$HADOOP_HOME/lib/native\nexport PATH=\$PATH:\$HADOOP_HOME/sbin:\$HADOOP_HOME/bin\nexport HADOOP_OPTS=\"-Djava.library.path=\$HADOOP_HOME/lib/native\"" >> ~/.bashrc # Hadoop env var
echo -e "PATH=\$PATH:\$HADOOP_HOME/sbin" >> ~/.bashrc
source ~/.bashrc
sed -i 's|^# export JAVA_HOME=.*|export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64/|' "$HADOOP_HOME/etc/hadoop/hadoop-env.sh"
```
7. Modify hadoop config files:
```shell
sed -i '/<configuration>/,/<\/configuration>/d' $HADOOP_HOME/etc/hadoop/core-site.xml
echo "<configuration>
<property>
<name>hadoop.tmp.dir</name>
<value>/home/hadoop/tmp</value>
</property>
<property>
<name>fs.default.name</name>
<value>hdfs://localhost:9000</value>
</property>
</configuration>" >> $HADOOP_HOME/etc/hadoop/core-site.xml
sed -i '/<configuration>/,/<\/configuration>/d' $HADOOP_HOME/etc/hadoop/hdfs-site.xml
echo "<configuration>
<property>
<name>dfs.namenode.dir</name>
<value>file:///home/hadoop/hadoopdata/hdfs/namenode</value>
</property>
<property>
<name>dfs.data.dir</name>
<value>file:///home/hadoop/hadoopdata/hdfs/datanode</value>
</property>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
</configuration>" >> $HADOOP_HOME/etc/hadoop/hdfs-site.xml
sed -i '/<configuration>/,/<\/configuration>/d' $HADOOP_HOME/etc/hadoop/mapred-site.xml
echo "<configuration>
<property>
<name>yarn.app.mapreduce.am.env</name>
<value>HADOOP_MAPRED_HOME=$HADOOP_HOME/home/hadoop/hadoop/bin/hadoop</value>
</property>
<property>
<name>mapreduce.map.env</name>
<value>HADOOP_MAPRED_HOME=$HADOOP_HOME/home/hadoop/hadoop/bin/hadoop</value>
</property>
<property>
<name>mapreduce.reduce.env</name>
<value>HADOOP_MAPRED_HOME=$HADOOP_HOME/home/hadoop/hadoop/bin/hadoop</value>
</property>
</configuration>" >> $HADOOP_HOME/etc/hadoop/mapred-site.xml
sed -i '/<configuration>/,/<\/configuration>/d' $HADOOP_HOME/etc/hadoop/yarn-site.xml
echo "<configuration>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.resourcemanager.hostname</name>
<value>localhost</value>
</property>
</configuration>" >> $HADOOP_HOME/etc/hadoop/yarn-site.xml
```
8. Format HDFS namenode:
```shell
hdfs namenode -format # Format hdfs namenode
```
9. Start hadoop cluster:
```shell
start-all.sh
jps
```
> [!NOTE]
> Visit `localhost:9870` on your browser!
---
## Manually start/stop services
### Start all services
```shell
# start
hdfs --daemon start namenode
hdfs --daemon start datanode
yarn --daemon start nodemanager
yarn --daemon start resourcemanager
hdfs --daemon start secondarynamenode
hdfs dfsadmin -report
yarn node -list
jps # Check status
```
### Stop all services
```shell
# stop
hdfs --daemon stop namenode
hdfs --daemon stop datanode
yarn --daemon stop resourcemanager
yarn --daemon stop nodemanager
hdfs --daemon stop secondarynamenode
jps # Check status
```
---
## References
1. https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/SingleCluster.html
2. https://medium.com/@abhikdey06/apache-hadoop-3-3-6-installation-on-ubuntu-22-04-14516bceec85
Codes/Code-B1 (Hadoop Word Count)/Steps.md
+130
View File
@@ -0,0 +1,130 @@
# B1 - Hadoop Word Count
> [!NOTE]
> These are generic instructions, need to refine them.
---
1. Copy and paste the following code in `WordCount.java` file:
```java
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class WordCount {
// Mapper Class
public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> {
private final static IntWritable one = new IntWritable(1);
private Text word = new Text();
public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
StringTokenizer itr = new StringTokenizer(value.toString());
while (itr.hasMoreTokens()) {
word.set(itr.nextToken().toLowerCase().replaceAll("[^a-zA-Z0-9]", ""));
if (!word.toString().isEmpty()) {
context.write(word, one);
}
}
}
}
// Reducer Class
public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
private IntWritable result = new IntWritable();
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
result.set(sum);
context.write(key, result);
}
}
public static void main(String[] args) throws Exception {
if (args.length != 2) {
System.err.println("Usage: WordCount <input path> <output path>");
System.exit(-1);
}
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "word count");
job.setJarByClass(WordCount.class);
job.setMapperClass(TokenizerMapper.class);
job.setCombinerClass(IntSumReducer.class); // optional
job.setReducerClass(IntSumReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
```
2. Create an `input.txt` file in the same directory as the above code:
```text
This is a sample code.
All the way from KSKA Git.
Hello world
Meow meow meow meow
```
3. In the same directory, open a `Terminal` window and compile the Java code:
```shell
javac -classpath `hadoop classpath` -d . WordCount.java
jar cvf WordCount.jar *.class
jar tf WordCount.jar
```
> [!NOTE]
> Compiled code will be saved in the current working directory.
4. Create an input and output directory in Hadoop FS:
```shell
hadoop fs -mkdir /user/hadoop/input
hadoop fs -mkdir /user/hadoop/output
```
5. Upload the `input.txt` file to Hadoop FS:
```shell
hadoop fs -put input.txt /user/hadoop/input/
```
6. Run the WordCount job:
```shell
hadoop jar WordCount.jar WordCount /user/hadoop/input/input.txt /user/hadoop/output
```
7. View the output:
```shell
hadoop fs -cat /user/hadoop/output/part-r-00000
```
---
Codes/Code-B1 (Hadoop Word Count)/WordCount.jar
Executable
BIN
View File
Binary file not shown.
Codes/Code-B1 (Hadoop Word Count)/WordCount.java
+72
View File
@@ -0,0 +1,72 @@
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class WordCount {
// Mapper Class
public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> {
private final static IntWritable one = new IntWritable(1);
private Text word = new Text();
public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
StringTokenizer itr = new StringTokenizer(value.toString());
while (itr.hasMoreTokens()) {
word.set(itr.nextToken().toLowerCase().replaceAll("[^a-zA-Z0-9]", ""));
if (!word.toString().isEmpty()) {
context.write(word, one);
}
}
}
}
// Reducer Class
public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
private IntWritable result = new IntWritable();
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
result.set(sum);
context.write(key, result);
}
}
public static void main(String[] args) throws Exception {
if (args.length != 2) {
System.err.println("Usage: WordCount <input path> <output path>");
System.exit(-1);
}
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "word count");
job.setJarByClass(WordCount.class);
job.setMapperClass(TokenizerMapper.class);
job.setCombinerClass(IntSumReducer.class); // optional
job.setReducerClass(IntSumReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
Codes/Code-B1 (Hadoop Word Count)/hadoop java program steps.pdf
BIN
View File
Binary file not shown.