From 3cd77346f73217701f84f44c6126d8c51f7d4aa7 Mon Sep 17 00:00:00 2001 From: vedang29 Date: Mon, 14 Apr 2025 00:22:24 +0530 Subject: [PATCH] Set up spring app with menu driven HDFS --- .gitattributes | 2 + .gitignore | 33 +++ .mvn/wrapper/maven-wrapper.properties | 19 ++ mvnw | 259 ++++++++++++++++++ mvnw.cmd | 149 ++++++++++ pom.xml | 178 ++++++++++++ .../SkycrateBackendApplication.java | 54 ++++ .../skycrateBackend/config/HDFSConfig.java | 52 ++++ .../controller/HDFScontroller.java | 239 ++++++++++++++++ .../backend/skycrateBackend/dto/User.java | 23 ++ .../repository/UserManager.java | 25 ++ .../services/EncryptionUtil.java | 75 +++++ .../services/HDFSOperations.java | 106 +++++++ src/main/resources/application.properties | 10 + .../SkycrateBackendApplicationTests.java | 13 + 15 files changed, 1237 insertions(+) create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 .mvn/wrapper/maven-wrapper.properties create mode 100644 mvnw create mode 100644 mvnw.cmd create mode 100644 pom.xml create mode 100644 src/main/java/com/skycrate/backend/skycrateBackend/SkycrateBackendApplication.java create mode 100644 src/main/java/com/skycrate/backend/skycrateBackend/config/HDFSConfig.java create mode 100644 src/main/java/com/skycrate/backend/skycrateBackend/controller/HDFScontroller.java create mode 100644 src/main/java/com/skycrate/backend/skycrateBackend/dto/User.java create mode 100644 src/main/java/com/skycrate/backend/skycrateBackend/repository/UserManager.java create mode 100644 src/main/java/com/skycrate/backend/skycrateBackend/services/EncryptionUtil.java create mode 100644 src/main/java/com/skycrate/backend/skycrateBackend/services/HDFSOperations.java create mode 100644 src/main/resources/application.properties create mode 100644 src/test/java/com/skycrate/backend/skycrateBackend/SkycrateBackendApplicationTests.java diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3b41682 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +/mvnw text eol=lf +*.cmd text eol=crlf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..549e00a --- /dev/null +++ b/.gitignore @@ -0,0 +1,33 @@ +HELP.md +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 0000000..d58dfb7 --- /dev/null +++ b/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +wrapperVersion=3.3.2 +distributionType=only-script +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip diff --git a/mvnw b/mvnw new file mode 100644 index 0000000..19529dd --- /dev/null +++ b/mvnw @@ -0,0 +1,259 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.3.2 +# +# Optional ENV vars +# ----------------- +# JAVA_HOME - location of a JDK home dir, required when download maven via java source +# MVNW_REPOURL - repo url base for downloading maven distribution +# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output +# ---------------------------------------------------------------------------- + +set -euf +[ "${MVNW_VERBOSE-}" != debug ] || set -x + +# OS specific support. +native_path() { printf %s\\n "$1"; } +case "$(uname)" in +CYGWIN* | MINGW*) + [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")" + native_path() { cygpath --path --windows "$1"; } + ;; +esac + +# set JAVACMD and JAVACCMD +set_java_home() { + # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched + if [ -n "${JAVA_HOME-}" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACCMD="$JAVA_HOME/jre/sh/javac" + else + JAVACMD="$JAVA_HOME/bin/java" + JAVACCMD="$JAVA_HOME/bin/javac" + + if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then + echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2 + echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2 + return 1 + fi + fi + else + JAVACMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v java + )" || : + JAVACCMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v javac + )" || : + + if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then + echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2 + return 1 + fi + fi +} + +# hash string like Java String::hashCode +hash_string() { + str="${1:-}" h=0 + while [ -n "$str" ]; do + char="${str%"${str#?}"}" + h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296)) + str="${str#?}" + done + printf %x\\n $h +} + +verbose() { :; } +[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; } + +die() { + printf %s\\n "$1" >&2 + exit 1 +} + +trim() { + # MWRAPPER-139: + # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds. + # Needed for removing poorly interpreted newline sequences when running in more + # exotic environments such as mingw bash on Windows. + printf "%s" "${1}" | tr -d '[:space:]' +} + +# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties +while IFS="=" read -r key value; do + case "${key-}" in + distributionUrl) distributionUrl=$(trim "${value-}") ;; + distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;; + esac +done <"${0%/*}/.mvn/wrapper/maven-wrapper.properties" +[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in ${0%/*}/.mvn/wrapper/maven-wrapper.properties" + +case "${distributionUrl##*/}" in +maven-mvnd-*bin.*) + MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ + case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in + *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;; + :Darwin*x86_64) distributionPlatform=darwin-amd64 ;; + :Darwin*arm64) distributionPlatform=darwin-aarch64 ;; + :Linux*x86_64*) distributionPlatform=linux-amd64 ;; + *) + echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2 + distributionPlatform=linux-amd64 + ;; + esac + distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip" + ;; +maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;; +*) MVN_CMD="mvn${0##*/mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;; +esac + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}" +distributionUrlName="${distributionUrl##*/}" +distributionUrlNameMain="${distributionUrlName%.*}" +distributionUrlNameMain="${distributionUrlNameMain%-bin}" +MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}" +MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")" + +exec_maven() { + unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || : + exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD" +} + +if [ -d "$MAVEN_HOME" ]; then + verbose "found existing MAVEN_HOME at $MAVEN_HOME" + exec_maven "$@" +fi + +case "${distributionUrl-}" in +*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;; +*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;; +esac + +# prepare tmp dir +if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then + clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; } + trap clean HUP INT TERM EXIT +else + die "cannot create temp dir" +fi + +mkdir -p -- "${MAVEN_HOME%/*}" + +# Download and Install Apache Maven +verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +verbose "Downloading from: $distributionUrl" +verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +# select .zip or .tar.gz +if ! command -v unzip >/dev/null; then + distributionUrl="${distributionUrl%.zip}.tar.gz" + distributionUrlName="${distributionUrl##*/}" +fi + +# verbose opt +__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR='' +[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v + +# normalize http auth +case "${MVNW_PASSWORD:+has-password}" in +'') MVNW_USERNAME='' MVNW_PASSWORD='' ;; +has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;; +esac + +if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then + verbose "Found wget ... using wget" + wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl" +elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then + verbose "Found curl ... using curl" + curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl" +elif set_java_home; then + verbose "Falling back to use Java to download" + javaSource="$TMP_DOWNLOAD_DIR/Downloader.java" + targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName" + cat >"$javaSource" <<-END + public class Downloader extends java.net.Authenticator + { + protected java.net.PasswordAuthentication getPasswordAuthentication() + { + return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() ); + } + public static void main( String[] args ) throws Exception + { + setDefault( new Downloader() ); + java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() ); + } + } + END + # For Cygwin/MinGW, switch paths to Windows format before running javac and java + verbose " - Compiling Downloader.java ..." + "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java" + verbose " - Running Downloader.java ..." + "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")" +fi + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +if [ -n "${distributionSha256Sum-}" ]; then + distributionSha256Result=false + if [ "$MVN_CMD" = mvnd.sh ]; then + echo "Checksum validation is not supported for maven-mvnd." >&2 + echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + elif command -v sha256sum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + elif command -v shasum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2 + echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + fi + if [ $distributionSha256Result = false ]; then + echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2 + echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2 + exit 1 + fi +fi + +# unzip and move +if command -v unzip >/dev/null; then + unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip" +else + tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar" +fi +printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/mvnw.url" +mv -- "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME" + +clean || : +exec_maven "$@" diff --git a/mvnw.cmd b/mvnw.cmd new file mode 100644 index 0000000..249bdf3 --- /dev/null +++ b/mvnw.cmd @@ -0,0 +1,149 @@ +<# : batch portion +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Apache Maven Wrapper startup batch script, version 3.3.2 +@REM +@REM Optional ENV vars +@REM MVNW_REPOURL - repo url base for downloading maven distribution +@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output +@REM ---------------------------------------------------------------------------- + +@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0) +@SET __MVNW_CMD__= +@SET __MVNW_ERROR__= +@SET __MVNW_PSMODULEP_SAVE=%PSModulePath% +@SET PSModulePath= +@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @( + IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B) +) +@SET PSModulePath=%__MVNW_PSMODULEP_SAVE% +@SET __MVNW_PSMODULEP_SAVE= +@SET __MVNW_ARG0_NAME__= +@SET MVNW_USERNAME= +@SET MVNW_PASSWORD= +@IF NOT "%__MVNW_CMD__%"=="" (%__MVNW_CMD__% %*) +@echo Cannot start maven from wrapper >&2 && exit /b 1 +@GOTO :EOF +: end batch / begin powershell #> + +$ErrorActionPreference = "Stop" +if ($env:MVNW_VERBOSE -eq "true") { + $VerbosePreference = "Continue" +} + +# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties +$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl +if (!$distributionUrl) { + Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" +} + +switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) { + "maven-mvnd-*" { + $USE_MVND = $true + $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip" + $MVN_CMD = "mvnd.cmd" + break + } + default { + $USE_MVND = $false + $MVN_CMD = $script -replace '^mvnw','mvn' + break + } +} + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +if ($env:MVNW_REPOURL) { + $MVNW_REPO_PATTERN = if ($USE_MVND) { "/org/apache/maven/" } else { "/maven/mvnd/" } + $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace '^.*'+$MVNW_REPO_PATTERN,'')" +} +$distributionUrlName = $distributionUrl -replace '^.*/','' +$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$','' +$MAVEN_HOME_PARENT = "$HOME/.m2/wrapper/dists/$distributionUrlNameMain" +if ($env:MAVEN_USER_HOME) { + $MAVEN_HOME_PARENT = "$env:MAVEN_USER_HOME/wrapper/dists/$distributionUrlNameMain" +} +$MAVEN_HOME_NAME = ([System.Security.Cryptography.MD5]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join '' +$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME" + +if (Test-Path -Path "$MAVEN_HOME" -PathType Container) { + Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME" + Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" + exit $? +} + +if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) { + Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl" +} + +# prepare tmp dir +$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile +$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir" +$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null +trap { + if ($TMP_DOWNLOAD_DIR.Exists) { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } + } +} + +New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null + +# Download and Install Apache Maven +Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +Write-Verbose "Downloading from: $distributionUrl" +Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +$webclient = New-Object System.Net.WebClient +if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) { + $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD) +} +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum +if ($distributionSha256Sum) { + if ($USE_MVND) { + Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." + } + Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash + if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) { + Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property." + } +} + +# unzip and move +Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null +Rename-Item -Path "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" -NewName $MAVEN_HOME_NAME | Out-Null +try { + Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null +} catch { + if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) { + Write-Error "fail to move MAVEN_HOME" + } +} finally { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } +} + +Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..fc7b716 --- /dev/null +++ b/pom.xml @@ -0,0 +1,178 @@ + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 3.4.4 + + + com.skycrate.backend + skycrateBackend + 0.0.1-SNAPSHOT + skycrateBackend + Cloud Storage App using HDFS + + + + + + + + + + + + + + + 17 + + + + org.springframework.boot + spring-boot-starter + + + + org.projectlombok + lombok + true + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + + org.springframework.boot + spring-boot-starter-web + 3.2.4 + + + + + org.springframework.boot + spring-boot-starter-security + 3.2.4 + + + + + org.springframework.boot + spring-boot-starter-data-jpa + 3.2.4 + + + + + com.mysql + mysql-connector-j + 8.3.0 + + + + + org.springframework.boot + spring-boot-starter-validation + 3.2.4 + + + + + io.jsonwebtoken + jjwt-api + 0.11.5 + + + io.jsonwebtoken + jjwt-impl + 0.11.5 + + + io.jsonwebtoken + jjwt-jackson + 0.11.5 + + + + + org.apache.hadoop + hadoop-common + 3.4.1 + + + org.slf4j + slf4j-log4j12 + + + + + + org.apache.hadoop + hadoop-hdfs + 3.4.1 + + + + org.apache.hadoop + hadoop-client + 3.4.1 + + + + + + + + + + + + + + + + + + commons-io + commons-io + 2.11.0 + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + org.projectlombok + lombok + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + org.projectlombok + lombok + + + + + + + + diff --git a/src/main/java/com/skycrate/backend/skycrateBackend/SkycrateBackendApplication.java b/src/main/java/com/skycrate/backend/skycrateBackend/SkycrateBackendApplication.java new file mode 100644 index 0000000..a697e48 --- /dev/null +++ b/src/main/java/com/skycrate/backend/skycrateBackend/SkycrateBackendApplication.java @@ -0,0 +1,54 @@ +package com.skycrate.backend.skycrateBackend; + +import com.skycrate.backend.skycrateBackend.controller.HDFScontroller; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class SkycrateBackendApplication implements CommandLineRunner { + + @Autowired + private HDFScontroller hdfsController; + + public static void main(String[] args) { + SpringApplication.run(SkycrateBackendApplication.class, args); + } + + @Override + public void run(String... args) { + try { + hdfsController.run(); // ✅ Run the instance method, not static + } catch (Exception e) { + System.err.println("❌ Error running CLI: " + e.getMessage()); + e.printStackTrace(); + } + } +} + + +//package com.skycrate.backend.skycrateBackend; +// +//import com.skycrate.backend.skycrateBackend.controller.HDFScontroller; +//import org.springframework.boot.SpringApplication; +//import org.springframework.boot.autoconfigure.SpringBootApplication; +// +//@SpringBootApplication +//public class SkycrateBackendApplication { +// +// public static void main(String[] args) { +// SpringApplication.run(SkycrateBackendApplication.class, args); +// } +// +// @Override +// public void run(String... args) { +// try { +// HDFScontroller.run(); // ✅ Run the instance method, not static +// } catch (Exception e) { +// System.err.println("❌ Error running CLI: " + e.getMessage()); +// e.printStackTrace(); +// } +// } +// +//} diff --git a/src/main/java/com/skycrate/backend/skycrateBackend/config/HDFSConfig.java b/src/main/java/com/skycrate/backend/skycrateBackend/config/HDFSConfig.java new file mode 100644 index 0000000..6089da5 --- /dev/null +++ b/src/main/java/com/skycrate/backend/skycrateBackend/config/HDFSConfig.java @@ -0,0 +1,52 @@ +//package com.skycrate.backend.skycrateBackend.config; +// +//import org.apache.hadoop.conf.Configuration; +//import org.apache.hadoop.fs.FileSystem; +//import org.apache.hadoop.security.UserGroupInformation; +//import org.springframework.context.annotation.Bean; +// +//import java.io.IOException; +//import java.net.URI; +//import java.security.PrivilegedExceptionAction; +// +//@org.springframework.context.annotation.Configuration +//public class HDFSConfig { +// +// public static FileSystem getHDFS() throws Exception { +// Configuration conf = new Configuration(); +// conf.set("fs.defaultFS", "hdfs://192.168.29.56:9000"); +// conf.set("hadoop.security.authentication", "simple"); +// +// // Disable security manager +// System.setProperty("java.security.manager", "allow"); +// +// // Required for Java 17+: Explicitly configure UGI +// UserGroupInformation.setConfiguration(conf); +// UserGroupInformation ugi = UserGroupInformation.createRemoteUser("hdfs"); +// +// // Login user explicitly and return FileSystem instance +// return ugi.doAs((PrivilegedExceptionAction) () -> +// FileSystem.get(new URI("hdfs://192.168.29.56:9000"), conf) +// ); +// } +//} + + +package com.skycrate.backend.skycrateBackend.config; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.security.UserGroupInformation; +import org.springframework.context.annotation.Bean; + +import java.net.URI; +import java.security.PrivilegedExceptionAction; + + +public class HDFSConfig { + public static FileSystem getHDFS() throws Exception { + Configuration conf = new Configuration(); + conf.set("fs.defaultFS", "hdfs://192.168.29.56:9000"); + return FileSystem.get(new URI("hdfs://192.168.29.56:9000"), conf); + } +} diff --git a/src/main/java/com/skycrate/backend/skycrateBackend/controller/HDFScontroller.java b/src/main/java/com/skycrate/backend/skycrateBackend/controller/HDFScontroller.java new file mode 100644 index 0000000..e46e535 --- /dev/null +++ b/src/main/java/com/skycrate/backend/skycrateBackend/controller/HDFScontroller.java @@ -0,0 +1,239 @@ +package com.skycrate.backend.skycrateBackend.controller; + +import com.skycrate.backend.skycrateBackend.dto.User; +import com.skycrate.backend.skycrateBackend.repository.UserManager; +import com.skycrate.backend.skycrateBackend.services.HDFSOperations; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.Scanner; + +@Component +public class HDFScontroller implements Runnable { + + private final UserManager userManager; + private final HDFSOperations hdfsOperations; + + @Autowired + public HDFScontroller(UserManager userManager, HDFSOperations hdfsOperations) { + this.userManager = userManager; + this.hdfsOperations = hdfsOperations; + } + + @Override + public void run() { + Scanner scanner = new Scanner(System.in); + + while (true) { + System.out.print("Enter your username: "); + String username = scanner.nextLine(); + + System.out.print("Enter your password: "); + String password = scanner.nextLine(); + + if (!userManager.authenticate(username, password)) { + System.out.println("❌ Authentication failed. Exiting..."); + return; + } + + User user; + try { + user = userManager.getUser(username); + } catch (Exception e) { + System.out.println("❌ Error creating user: " + e.getMessage()); + return; + } + + while (true) { + System.out.println("\nChoose an option:"); + System.out.println("1. Create a folder"); + System.out.println("2. Upload a file (encrypted)"); + System.out.println("3. Upload a file to a specific folder (encrypted)"); + System.out.println("4. Download a file"); + System.out.println("5. Delete a file"); + System.out.println("6. Delete a folder"); + System.out.println("7. List files and folders"); + System.out.println("8. Exit"); + + int choice = scanner.nextInt(); + scanner.nextLine(); // Consume newline + + try { + switch (choice) { + case 1: + System.out.print("Enter folder name: "); + hdfsOperations.createFolder(scanner.nextLine()); + break; + + case 2: + System.out.print("Enter file path to upload: "); + String filePath = scanner.nextLine(); + System.out.print("Enter HDFS destination path: "); + String hdfsDestinationFolder = scanner.nextLine(); + System.out.print("Enter the name for the uploaded file: "); + String uploadedFileName = scanner.nextLine(); + hdfsOperations.uploadFile(filePath, hdfsDestinationFolder, uploadedFileName, user); + break; + + case 3: + System.out.print("Enter folder name to upload to: "); + String targetFolder = scanner.nextLine(); + System.out.print("Enter file path to upload: "); + String fileToUpload = scanner.nextLine(); + System.out.print("Enter the name for the uploaded file: "); + String uploadedFileNameSpecific = scanner.nextLine(); + hdfsOperations.uploadFile(fileToUpload, targetFolder, uploadedFileNameSpecific, user); + break; + + case 4: + System.out.print("Enter HDFS file path to download: "); + String hdfsFilePath = scanner.nextLine(); + System.out.print("Enter local path to save the downloaded file: "); + String localDownloadPath = scanner.nextLine(); + hdfsOperations.downloadFile(hdfsFilePath, localDownloadPath, user); + break; + + case 5: + System.out.print("Enter HDFS file path to delete: "); + hdfsOperations.deleteFile(scanner.nextLine()); + break; + + case 6: + System.out.print("Enter HDFS folder path to delete: "); + hdfsOperations.deleteFolder(scanner.nextLine()); + break; + + case 7: + System.out.print("Enter HDFS path to list files and folders: "); + hdfsOperations.listFilesAndFolders(scanner.nextLine()); + break; + + case 8: + System.out.println("✅ Exiting..."); + return; + + default: + System.out.println("⚠️ Invalid option. Please try again."); + } + } catch (Exception e) { + System.out.println("❌ Operation failed: " + e.getMessage()); + e.printStackTrace(); + } + } + } + } +} + + +//package com.skycrate.backend.skycrateBackend.controller; +// +//import com.skycrate.backend.skycrateBackend.dto.User; +//import com.skycrate.backend.skycrateBackend.repository.UserManager; +//import com.skycrate.backend.skycrateBackend.services.HDFSOperations; +//import org.springframework.beans.factory.annotation.Autowired; +//import org.springframework.stereotype.Component; +// +//import java.util.Scanner; +// +//@Component +//public class HDFScontroller implements Runnable { +// +// private final UserManager userManager; +// private final HDFSOperations hdfsOperations; +// +// @Autowired +// public HDFScontroller(UserManager userManager, HDFSOperations hdfsOperations) { +// this.userManager = userManager; +// this.hdfsOperations = hdfsOperations; +// } +// +// @Override +// public void run() { +// Scanner scanner = new Scanner(System.in); +// +// while (true) { +// System.out.print("Enter your username: "); +// String username = scanner.nextLine(); +// +// System.out.print("Enter your password: "); +// String password = scanner.nextLine(); +// +// if (!userManager.authenticate(username, password)) { +// System.out.println("X Authentication failed. Exiting..."); +// return; +// } +// +// User user; +// try { +// user = userManager.getUser(username); +// } catch (Exception e) { +// System.out.println("X Error creating user: " + e.getMessage()); +// return; +// } +// +// while (true) { +// System.out.println("\nChoose an option:"); +// System.out.println("1. Create a folder"); +// System.out.println("2. Upload a file (encrypted)"); +// System.out.println("3. Upload a file to a specific folder (encrypted)"); +// System.out.println("4. Download a file"); +// System.out.println("5. Delete a file"); +// System.out.println("6. Delete a folder"); +// System.out.println("7. List files and folders"); +// System.out.println("8. Exit"); +// +// int choice = scanner.nextInt(); +// scanner.nextLine(); // Consume newline +// +// switch (choice) { +// case 1: +// System.out.print("Enter folder name: "); +// hdfsOperations.createFolder(scanner.nextLine()); +// break; +// case 2: +// System.out.print("Enter file path to upload: "); +// String filePath = scanner.nextLine(); +// System.out.print("Enter HDFS destination path: "); +// String hdfsDestinationFolder = scanner.nextLine(); +// System.out.print("Enter the name for the uploaded file: "); +// String uploadedFileName = scanner.nextLine(); +// hdfsOperations.uploadFile(filePath, hdfsDestinationFolder, uploadedFileName, user); +// break; +// case 3: +// System.out.print("Enter folder name to upload to: "); +// String targetFolder = scanner.nextLine(); +// System.out.print("Enter file path to upload: "); +// String fileToUpload = scanner.nextLine(); +// System.out.print("Enter the name for the uploaded file: "); +// String uploadedFileNameSpecific = scanner.nextLine(); +// hdfsOperations.uploadFile(fileToUpload, targetFolder, uploadedFileNameSpecific, user); +// break; +// case 4: +// System.out.print("Enter HDFS file path to download: "); +// String hdfsFilePath = scanner.nextLine(); +// System.out.print("Enter local path to save the downloaded file: "); +// String localDownloadPath = scanner.nextLine(); +// hdfsOperations.downloadFile(hdfsFilePath, localDownloadPath, user); +// break; +// case 5: +// System.out.print("Enter HDFS file path to delete: "); +// hdfsOperations.deleteFile(scanner.nextLine()); +// break; +// case 6: +// System.out.print("Enter HDFS folder path to delete: "); +// hdfsOperations.deleteFolder(scanner.nextLine()); +// break; +// case 7: +// System.out.print("Enter HDFS path to list files and folders: "); +// hdfsOperations.listFilesAndFolders(scanner.nextLine()); +// break; +// case 8: +// System.out.println("Exiting..."); +// return; +// default: +// System.out.println("Invalid option. Please try again."); +// } +// } +// } +// } +//} diff --git a/src/main/java/com/skycrate/backend/skycrateBackend/dto/User.java b/src/main/java/com/skycrate/backend/skycrateBackend/dto/User.java new file mode 100644 index 0000000..7078c7a --- /dev/null +++ b/src/main/java/com/skycrate/backend/skycrateBackend/dto/User.java @@ -0,0 +1,23 @@ +package com.skycrate.backend.skycrateBackend.dto; + +import com.skycrate.backend.skycrateBackend.services.EncryptionUtil; + +import java.security.KeyPair; + +public class User { + private String username; + private KeyPair keyPair; + + public User(String username) throws Exception { + this.username = username; + this.keyPair = EncryptionUtil.generateKeyPair(); + } + + public String getUsername() { + return username; + } + + public KeyPair getKeyPair() { + return keyPair; + } +} diff --git a/src/main/java/com/skycrate/backend/skycrateBackend/repository/UserManager.java b/src/main/java/com/skycrate/backend/skycrateBackend/repository/UserManager.java new file mode 100644 index 0000000..b37a326 --- /dev/null +++ b/src/main/java/com/skycrate/backend/skycrateBackend/repository/UserManager.java @@ -0,0 +1,25 @@ +package com.skycrate.backend.skycrateBackend.repository; + +import com.skycrate.backend.skycrateBackend.dto.User; +import org.springframework.stereotype.Service; + +import java.util.HashMap; +import java.util.Map; + +@Service +public class UserManager { + private Map users = new HashMap<>(); + + public User getUser(String username) throws Exception { + if (!users.containsKey(username)) { + users.put(username, new User(username)); + } + return users.get(username); + } + + public boolean authenticate(String username, String password) { + // Implement your authentication logic here + return "admin".equals(username) && "password123".equals(password); + } +} + diff --git a/src/main/java/com/skycrate/backend/skycrateBackend/services/EncryptionUtil.java b/src/main/java/com/skycrate/backend/skycrateBackend/services/EncryptionUtil.java new file mode 100644 index 0000000..2fbe89b --- /dev/null +++ b/src/main/java/com/skycrate/backend/skycrateBackend/services/EncryptionUtil.java @@ -0,0 +1,75 @@ +package com.skycrate.backend.skycrateBackend.services; + + +import javax.crypto.Cipher; +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; +import java.security.*; + +public class EncryptionUtil { + private static final String RSA_ALGORITHM = "RSA"; + private static final String AES_ALGORITHM = "AES"; + private static final int RSA_KEY_SIZE = 2048; + private static final int AES_KEY_SIZE = 256; + + public static KeyPair generateKeyPair() throws NoSuchAlgorithmException { + KeyPairGenerator keyGen = KeyPairGenerator.getInstance(RSA_ALGORITHM); + keyGen.initialize(RSA_KEY_SIZE); + return keyGen.generateKeyPair(); + } + + public static byte[] encrypt(byte[] data, PublicKey publicKey) throws Exception { + // Generate a random AES key + KeyGenerator keyGen = KeyGenerator.getInstance(AES_ALGORITHM); + keyGen.init(AES_KEY_SIZE); + SecretKey aesKey = keyGen.generateKey(); + + // Encrypt the data with the AES key + Cipher aesCipher = Cipher.getInstance(AES_ALGORITHM); + aesCipher.init(Cipher.ENCRYPT_MODE, aesKey); + byte[] encryptedData = aesCipher.doFinal(data); + + // Encrypt the AES key with the RSA public key + Cipher rsaCipher = Cipher.getInstance(RSA_ALGORITHM); + rsaCipher.init(Cipher.ENCRYPT_MODE, publicKey); + byte[] encryptedAesKey = rsaCipher.doFinal(aesKey.getEncoded()); + + // Combine the encrypted AES key and the encrypted data + byte[] combined = new byte[encryptedAesKey.length + encryptedData.length + 4]; + System.arraycopy(encryptedAesKey, 0, combined, 0, encryptedAesKey.length); + System.arraycopy(encryptedData, 0, combined, encryptedAesKey.length, encryptedData.length); + // Store the length of the encrypted AES key at the beginning + combined[encryptedAesKey.length + encryptedData.length] = (byte) (encryptedAesKey.length >> 24); + combined[encryptedAesKey.length + encryptedData.length + 1] = (byte) (encryptedAesKey.length >> 16); + combined[encryptedAesKey.length + encryptedData.length + 2] = (byte) (encryptedAesKey.length >> 8); + combined[encryptedAesKey.length + encryptedData.length + 3] = (byte) (encryptedAesKey.length); + + return combined; + } + + public static byte[] decrypt(byte[] encryptedData, PrivateKey privateKey) throws Exception { + // Read the length of the encrypted AES key + int aesKeyLength = ((encryptedData[encryptedData.length - 4] & 0xFF) << 24) | + ((encryptedData[encryptedData.length - 3] & 0xFF) << 16) | + ((encryptedData[encryptedData.length - 2] & 0xFF) << 8) | + (encryptedData[encryptedData.length - 1] & 0xFF); + + // Extract the encrypted AES key and the encrypted data + byte[] encryptedAesKey = new byte[aesKeyLength]; + byte[] encryptedDataBytes = new byte[encryptedData.length - aesKeyLength - 4]; + System.arraycopy(encryptedData, 0, encryptedAesKey, 0, aesKeyLength); + System.arraycopy(encryptedData, aesKeyLength, encryptedDataBytes, 0, encryptedDataBytes.length); + + // Decrypt the AES key with the RSA private key + Cipher rsaCipher = Cipher.getInstance(RSA_ALGORITHM); + rsaCipher.init(Cipher.DECRYPT_MODE, privateKey); + byte[] aesKeyBytes = rsaCipher.doFinal(encryptedAesKey); + SecretKey aesKey = new SecretKeySpec(aesKeyBytes, AES_ALGORITHM); + + // Decrypt the data with the AES key + Cipher aesCipher = Cipher.getInstance(AES_ALGORITHM); + aesCipher.init(Cipher.DECRYPT_MODE, aesKey); + return aesCipher.doFinal(encryptedDataBytes); + } +} diff --git a/src/main/java/com/skycrate/backend/skycrateBackend/services/HDFSOperations.java b/src/main/java/com/skycrate/backend/skycrateBackend/services/HDFSOperations.java new file mode 100644 index 0000000..7fe58f5 --- /dev/null +++ b/src/main/java/com/skycrate/backend/skycrateBackend/services/HDFSOperations.java @@ -0,0 +1,106 @@ +package com.skycrate.backend.skycrateBackend.services; + + +import com.skycrate.backend.skycrateBackend.config.HDFSConfig; +import com.skycrate.backend.skycrateBackend.dto.User; +import org.apache.hadoop.fs.*; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; + +@Service +public class HDFSOperations { + public static void uploadFile(String localPath, String hdfsPath, String uploadedFileName, User user) throws Exception { + FileSystem fs = HDFSConfig.getHDFS(); + byte[] data = Files.readAllBytes(Paths.get(localPath)); // Read file as bytes + byte[] encryptedData = EncryptionUtil.encrypt(data, user.getKeyPair().getPublic()); + + // Save encryptedData to a temporary file and upload it + String tempFilePath = localPath + ".enc"; + Files.write(Paths.get(tempFilePath), encryptedData); // Write bytes to temp file + + // Construct the final HDFS path using the provided uploaded file name + String finalHdfsPath = hdfsPath.endsWith("/") ? hdfsPath + uploadedFileName : hdfsPath + "/" + uploadedFileName; + + fs.copyFromLocalFile(new Path(tempFilePath), new Path(finalHdfsPath)); + System.out.println("✅ File uploaded: " + finalHdfsPath); + + // Clean up temporary file + Files.delete(Paths.get(tempFilePath)); + } + + public static void downloadFile(String hdfsPath, String localPath, User user) throws Exception { + FileSystem fs = HDFSConfig.getHDFS(); + String tempFilePath = localPath + ".enc"; + + fs.copyToLocalFile(new Path(hdfsPath), new Path(tempFilePath)); + + // Read the encrypted file as bytes + byte[] encryptedData = Files.readAllBytes(Paths.get(tempFilePath)); + byte[] decryptedData = EncryptionUtil.decrypt(encryptedData, user.getKeyPair().getPrivate()); + + Files.write(Paths.get(localPath), decryptedData); // Write decrypted bytes to local file + System.out.println("✅ File downloaded: " + localPath); + + // Clean up temporary file + Files.delete(Paths.get(tempFilePath)); + } + + public static void createFolder(String hdfsPath) throws Exception { + FileSystem fs = HDFSConfig.getHDFS(); + Path path = new Path(hdfsPath); + if (!fs.exists(path)) { + fs.mkdirs(path); + System.out.println("✅ Folder created: " + hdfsPath); + } else { + System.out.println("⚠️ Folder already exists: " + hdfsPath); + } + } + + public static void deleteFile(String hdfsFilePath) throws Exception { + FileSystem fs = HDFSConfig.getHDFS(); + Path path = new Path(hdfsFilePath); + if (fs.exists(path)) { + fs.delete(path, false); // false means do not recursively delete + System.out.println("✅ File deleted: " + hdfsFilePath); + } else { + System.out.println("⚠️ File does not exist: " + hdfsFilePath); + } + } + + public static void deleteFolder(String hdfsFolderPath) throws Exception { + FileSystem fs = HDFSConfig.getHDFS(); + Path path = new Path(hdfsFolderPath); + if (fs.exists(path)) { + fs.delete(path, true); // true means recursively delete + System.out.println("✅ Folder deleted: " + hdfsFolderPath); + } else { + System.out.println("⚠️ Folder does not exist: " + hdfsFolderPath); + } + } + + public static void listFilesAndFolders(String hdfsPath) throws Exception { + FileSystem fs = HDFSConfig.getHDFS(); + Path path = new Path(hdfsPath); + + if (!fs.exists(path)) { + System.out.println("⚠️ Path does not exist: " + hdfsPath); + return; + } + + System.out.println("Listing files and folders in: " + hdfsPath); + listFilesAndFoldersRecursively(fs, path, ""); + } + + private static void listFilesAndFoldersRecursively(FileSystem fs, Path path, String indent) throws IOException { + FileStatus[] fileStatuses = fs.listStatus(path); + for (FileStatus fileStatus : fileStatuses) { + System.out.println(indent + (fileStatus.isDirectory() ? "📁 " : "📄 ") + fileStatus.getPath().getName()); + if (fileStatus.isDirectory()) { + listFilesAndFoldersRecursively(fs, fileStatus.getPath(), indent + " "); // Indent for subdirectories + } + } + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties new file mode 100644 index 0000000..70b807c --- /dev/null +++ b/src/main/resources/application.properties @@ -0,0 +1,10 @@ +spring.application.name=skycrateBackend + +spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.MySQLDialect +spring.datasource.username=kshitij +spring.datasource.password=loa_dngLLA8729 +spring.datasource.url=jdbc:mysql://192.168.29.55:3306/skycrate + +spring.jpa.hibernate.ddl-auto=update +spring.jpa.show-sql=true +spring.jpa.properties.hibernate.format_sql=true diff --git a/src/test/java/com/skycrate/backend/skycrateBackend/SkycrateBackendApplicationTests.java b/src/test/java/com/skycrate/backend/skycrateBackend/SkycrateBackendApplicationTests.java new file mode 100644 index 0000000..8195f98 --- /dev/null +++ b/src/test/java/com/skycrate/backend/skycrateBackend/SkycrateBackendApplicationTests.java @@ -0,0 +1,13 @@ +package com.skycrate.backend.skycrateBackend; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest +class SkycrateBackendApplicationTests { + + @Test + void contextLoads() { + } + +}