Deleted tmp folder. This branch is now on hold. tmp folders are no longer being produced. All APIs hopefully work in this one, didn't test tho 🤞

pom.xml

@@ -144,6 +144,16 @@
 			<artifactId>commons-io</artifactId>
 			<version>2.11.0</version>
 		</dependency>
+
+
+
+<!--		NEWLY ADDED DEPENDENCY FOR DOWNLOAD ENDPOINT-->
+		<dependency>
+			<groupId>commons-io</groupId>
+			<artifactId>commons-io</artifactId>
+			<version>2.11.0</version>
+		</dependency>
+
 	</dependencies>
 
 	<build>

src/main/java/com/skycrate/backend/skycrateBackend/controller/HDFScontroller.java

@@ -5,7 +5,9 @@ import com.skycrate.backend.skycrateBackend.services.EncryptionUtil;
 import com.skycrate.backend.skycrateBackend.services.HDFSOperations;
 import com.skycrate.backend.skycrateBackend.utils.KeyUtil;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.InputStreamResource;
 import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
@@ -49,69 +51,103 @@ public class HDFScontroller {
             return new ResponseDTO("Failed to create folder: " + e.getMessage(), false);
         }
     }
-
-    @PostMapping("/uploadFile")
-    public ResponseDTO uploadFile(
-            @RequestParam("file") MultipartFile file,
-            @RequestParam String hdfsPath,
-            @RequestParam String uploadedFileName,
-            @RequestParam String username) {
-        try {
-            // Save file locally first
-            String localPath = saveFileLocally(file);
-            System.out.println("File saved locally at: " + localPath);
-
-            // Upload file to HDFS
-            hdfsOperations.uploadFile(localPath, hdfsPath, uploadedFileName, username);
-            return new ResponseDTO("File uploaded successfully", true);
-        } catch (IOException e) {
-            e.printStackTrace();
-            return new ResponseDTO("Failed to upload file locally: " + e.getMessage(), false);
-        } catch (Exception e) {
-            e.printStackTrace();
-            return new ResponseDTO("Failed to upload file to HDFS: " + e.getMessage(), false);
-        }
+//
+//    @PostMapping("/uploadFile")
+//    public ResponseDTO uploadFile(
+//            @RequestParam("file") MultipartFile file,
+//            @RequestParam String hdfsPath,
+//            @RequestParam String uploadedFileName,
+//            @RequestParam String username) {
+//        try {
+//            // Save file locally first
+//            String localPath = saveFileLocally(file);
+//            System.out.println("File saved locally at: " + localPath);
+//
+//            // Upload file to HDFS
+//            hdfsOperations.uploadFile(localPath, hdfsPath, uploadedFileName, username);
+//            return new ResponseDTO("File uploaded successfully", true);
+//        } catch (IOException e) {
+//            e.printStackTrace();
+//            return new ResponseDTO("Failed to upload file locally: " + e.getMessage(), false);
+//        } catch (Exception e) {
+//            e.printStackTrace();
+//            return new ResponseDTO("Failed to upload file to HDFS: " + e.getMessage(), false);
+//        }
+//    }
+@PostMapping("/uploadFile")
+public ResponseDTO uploadFile(
+        @RequestParam("file") MultipartFile file,
+        @RequestParam String hdfsPath,
+        @RequestParam String uploadedFileName,
+        @RequestParam String username) {
+    try {
+        // Upload file directly to HDFS without saving locally
+        hdfsOperations.uploadFile(file, hdfsPath, uploadedFileName, username);
+        return new ResponseDTO("File uploaded successfully", true);
+    } catch (Exception e) {
+        e.printStackTrace();
+        return new ResponseDTO("Failed to upload file to HDFS: " + e.getMessage(), false);
     }
+}
 
 
-    private String saveFileLocally(MultipartFile file) throws IOException {
-        // Create a temporary directory if it doesn't exist
-        Path tmpDir = Paths.get("tmp");
-        if (!Files.exists(tmpDir)) {
-            Files.createDirectories(tmpDir);  // Create the directory if it doesn't exist
-        }
+//
+//    private String saveFileLocally(MultipartFile file) throws IOException {
+//        // Create a temporary directory if it doesn't exist
+//        Path tmpDir = Paths.get("tmp");
+//        if (!Files.exists(tmpDir)) {
+//            Files.createDirectories(tmpDir);  // Create the directory if it doesn't exist
+//        }
+//
+//        Path path = tmpDir.resolve(file.getOriginalFilename());
+//
+//        // Copy the file to the local directory
+//        Files.copy(file.getInputStream(), path, StandardCopyOption.REPLACE_EXISTING);
+//
+//        return path.toString(); // Return the local path for further processing
+//    }
 
-        Path path = tmpDir.resolve(file.getOriginalFilename());
-
-        // Copy the file to the local directory
-        Files.copy(file.getInputStream(), path, StandardCopyOption.REPLACE_EXISTING);
-
-        return path.toString(); // Return the local path for further processing
-    }
 
+//    @PostMapping("/downloadFile")
+//    public ResponseEntity<?> downloadFile(
+//            @RequestParam String hdfsPath,
+//            @RequestParam String username) {
+//        try {
+//            // Define a temporary local path to download the file
+//            String localPath = "/app/tmp/downloaded/" + new File(hdfsPath).getName(); // Adjust the path as needed
+//
+//            // Download the file from HDFS to the local path
+//            hdfsOperations.downloadFile(hdfsPath, localPath, username);
+//
+//            // Create a File object for the downloaded file
+//            File file = new File(localPath);
+//            if (!file.exists()) {
+//                return ResponseEntity.status(HttpStatus.NOT_FOUND)
+//                        .body(new ResponseDTO("File not found", false));
+//            }
+//
+//            // Create a Resource from the file
+//            Resource resource = new FileSystemResource(file);
+//            return ResponseEntity.ok()
+//                    .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + resource.getFilename() + "\"")
+//                    .body(resource);
+//        } catch (Exception e) {
+//            return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
+//                    .body(new ResponseDTO("Failed to download file: " + e.getMessage(), false));
+//        }
+//    }
 
     @PostMapping("/downloadFile")
     public ResponseEntity<?> downloadFile(
             @RequestParam String hdfsPath,
             @RequestParam String username) {
         try {
-            // Define a temporary local path to download the file
-            String localPath = "/app/tmp/downloaded/" + new File(hdfsPath).getName(); // Adjust the path as needed
+            String fileName = new File(hdfsPath).getName();
+            InputStreamResource resource = hdfsOperations.downloadFile(hdfsPath, username);
 
-            // Download the file from HDFS to the local path
-            hdfsOperations.downloadFile(hdfsPath, localPath, username);
-
-            // Create a File object for the downloaded file
-            File file = new File(localPath);
-            if (!file.exists()) {
-                return ResponseEntity.status(HttpStatus.NOT_FOUND)
-                        .body(new ResponseDTO("File not found", false));
-            }
-
-            // Create a Resource from the file
-            Resource resource = new FileSystemResource(file);
             return ResponseEntity.ok()
-                    .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + resource.getFilename() + "\"")
+                    .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + fileName + "\"")
+                    .contentType(MediaType.APPLICATION_OCTET_STREAM)
                     .body(resource);
         } catch (Exception e) {
             return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
@@ -119,6 +155,7 @@ public class HDFScontroller {
         }
     }
 
+
     @DeleteMapping("/deleteFile")
     public ResponseDTO deleteFile(@RequestParam String hdfsPath) {
         try {

src/main/java/com/skycrate/backend/skycrateBackend/services/HDFSOperations.java

@@ -2,16 +2,18 @@ package com.skycrate.backend.skycrateBackend.services;
 
 import com.skycrate.backend.skycrateBackend.config.HDFSConfig;
 import com.skycrate.backend.skycrateBackend.utils.KeyUtil;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.fs.*;
+import org.springframework.core.io.InputStreamResource;
 import org.springframework.stereotype.Service;
+import org.springframework.web.multipart.MultipartFile;
 
 import javax.crypto.Cipher;
 import javax.crypto.KeyGenerator;
 import javax.crypto.SecretKey;
 import javax.crypto.spec.IvParameterSpec;
 import javax.crypto.spec.SecretKeySpec;
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Paths;
@@ -26,51 +28,97 @@ import java.util.List;
 @Service
 public class HDFSOperations {
 
-    public void uploadFile(String localPath, String hdfsPath, String uploadedFileName, String username) {
+//    public void uploadFile(String localPath, String hdfsPath, String uploadedFileName, String username) {
+//        try {
+//            FileSystem fs = HDFSConfig.getHDFS();
+//            byte[] data = Files.readAllBytes(Paths.get(localPath)); // Read file as bytes
+//
+//            // Encrypt file (consider adding encryption here as needed)
+//            byte[] encryptedData = data;
+//
+//            String tempFilePath = localPath + ".enc";
+//            Files.write(Paths.get(tempFilePath), encryptedData);
+//
+//            String finalHdfsPath = hdfsPath.endsWith("/") ? hdfsPath + uploadedFileName : hdfsPath + "/" + uploadedFileName;
+//            fs.copyFromLocalFile(new Path(tempFilePath), new Path(finalHdfsPath));
+//
+//            Files.delete(Paths.get(tempFilePath));
+//        } catch (IOException e) {
+//            // Handle I/O exception and log the error
+//            throw new RuntimeException("Failed to upload file due to I/O issue: " + e.getMessage(), e);
+//        } catch (Exception e) {
+//            // Catch any other exceptions
+//            throw new RuntimeException("Failed to upload file: " + e.getMessage(), e);
+//        }
+//    }
+
+    public void uploadFile(MultipartFile file, String hdfsPath, String uploadedFileName, String username) {
         try {
             FileSystem fs = HDFSConfig.getHDFS();
-            byte[] data = Files.readAllBytes(Paths.get(localPath)); // Read file as bytes
 
-            // Encrypt file (consider adding encryption here as needed)
-            byte[] encryptedData = data;
-
-            String tempFilePath = localPath + ".enc";
-            Files.write(Paths.get(tempFilePath), encryptedData);
+            // Encrypt file data if needed (currently direct pass-through)
+            byte[] fileBytes = file.getBytes(); // directly get bytes from MultipartFile
+            byte[] encryptedData = fileBytes;
 
+            // Define final HDFS file path
             String finalHdfsPath = hdfsPath.endsWith("/") ? hdfsPath + uploadedFileName : hdfsPath + "/" + uploadedFileName;
-            fs.copyFromLocalFile(new Path(tempFilePath), new Path(finalHdfsPath));
 
-            Files.delete(Paths.get(tempFilePath));
+            // Write encrypted bytes directly to HDFS
+            Path hdfsDestPath = new Path(finalHdfsPath);
+            try (FSDataOutputStream outputStream = fs.create(hdfsDestPath, true)) {
+                outputStream.write(encryptedData);
+            }
+
         } catch (IOException e) {
-            // Handle I/O exception and log the error
             throw new RuntimeException("Failed to upload file due to I/O issue: " + e.getMessage(), e);
         } catch (Exception e) {
-            // Catch any other exceptions
             throw new RuntimeException("Failed to upload file: " + e.getMessage(), e);
         }
     }
 
-    public void downloadFile(String hdfsPath, String localPath, String username) {
+
+//    public void downloadFile(String hdfsPath, String localPath, String username) {
+//        try {
+//            FileSystem fs = HDFSConfig.getHDFS();
+//            String tempFilePath = localPath + ".enc";
+//
+//            fs.copyToLocalFile(new Path(hdfsPath), new Path(tempFilePath));
+//
+//            byte[] encryptedData = Files.readAllBytes(Paths.get(tempFilePath));
+//            byte[] decryptedData = encryptedData; // Decrypt if needed
+//
+//            Files.write(Paths.get(localPath), decryptedData);
+//            Files.delete(Paths.get(tempFilePath));
+//        } catch (IOException e) {
+//            // Handle I/O exception and log the error
+//            throw new RuntimeException("Failed to download file due to I/O issue: " + e.getMessage(), e);
+//        } catch (Exception e) {
+//            // Catch any other exceptions
+//            throw new RuntimeException("Failed to download file: " + e.getMessage(), e);
+//        }
+//    }
+
+    public InputStreamResource downloadFile(String hdfsPath, String username) {
         try {
             FileSystem fs = HDFSConfig.getHDFS();
-            String tempFilePath = localPath + ".enc";
 
-            fs.copyToLocalFile(new Path(hdfsPath), new Path(tempFilePath));
+            // Read encrypted file directly from HDFS
+            FSDataInputStream hdfsInputStream = fs.open(new Path(hdfsPath));
 
-            byte[] encryptedData = Files.readAllBytes(Paths.get(tempFilePath));
-            byte[] decryptedData = encryptedData; // Decrypt if needed
+            // Optional: If decryption is needed, read bytes, decrypt, and wrap in ByteArrayInputStream
+            byte[] encryptedData = IOUtils.toByteArray(hdfsInputStream);
+            byte[] decryptedData = encryptedData; // Apply decryption if needed
+
+            return new InputStreamResource(new ByteArrayInputStream(decryptedData));
 
-            Files.write(Paths.get(localPath), decryptedData);
-            Files.delete(Paths.get(tempFilePath));
         } catch (IOException e) {
-            // Handle I/O exception and log the error
-            throw new RuntimeException("Failed to download file due to I/O issue: " + e.getMessage(), e);
+            throw new RuntimeException("Failed to stream file due to I/O issue: " + e.getMessage(), e);
         } catch (Exception e) {
-            // Catch any other exceptions
-            throw new RuntimeException("Failed to download file: " + e.getMessage(), e);
+            throw new RuntimeException("Failed to stream file: " + e.getMessage(), e);
         }
     }
 
+
     public void createFolder(String hdfsPath) {
         try {
             FileSystem fs = HDFSConfig.getHDFS();

src/main/resources/application.properties

@@ -17,7 +17,7 @@ spring.jpa.properties.hibernate.format_sql=true
 spring.jpa.open-in-view=false
 logging.level.org.springframework.security.config.annotation.authentication.configuration.InitializeUserDetailsBeanManagerConfigurer=ERROR
 
-server.port=8081
+server.port=8080
 
 
 

src/test/java/com/skycrate/backend/skycrateBackend/SkycrateBackendApplicationTests.java

@@ -1,13 +0,0 @@
-package com.skycrate.backend.skycrateBackend;
-
-import org.junit.jupiter.api.Test;
-import org.springframework.boot.test.context.SpringBootTest;
-
-@SpringBootTest
-class SkycrateBackendApplicationTests {
-
-	@Test
-	void contextLoads() {
-	}
-
-}

tmp/Hello.txt

@@ -1 +0,0 @@
-Hello World!!!