notkshitij/SkycrateBackend
1
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Removed tmp folder while uploading

Browse Source
This commit is contained in:
vedang29
2025-04-15 22:35:44 +05:30
parent 36acd75eb3
commit 508405077d
2 changed files with 90 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/skycrate/backend/skycrateBackend/controller/HDFScontroller.java
+52 -36
View File
@@ -49,45 +49,61 @@ public class HDFScontroller {
return new ResponseDTO("Failed to create folder: " + e.getMessage(), false); return new ResponseDTO("Failed to create folder: " + e.getMessage(), false);
} }
} }
//
@PostMapping("/uploadFile") // @PostMapping("/uploadFile")
public ResponseDTO uploadFile( // public ResponseDTO uploadFile(
@RequestParam("file") MultipartFile file, // @RequestParam("file") MultipartFile file,
@RequestParam String hdfsPath, // @RequestParam String hdfsPath,
@RequestParam String uploadedFileName, // @RequestParam String uploadedFileName,
@RequestParam String username) { // @RequestParam String username) {
try { // try {
// Save file locally first // // Save file locally first
String localPath = saveFileLocally(file); // String localPath = saveFileLocally(file);
System.out.println("File saved locally at: " + localPath); // System.out.println("File saved locally at: " + localPath);
//
// Upload file to HDFS // // Upload file to HDFS
hdfsOperations.uploadFile(localPath, hdfsPath, uploadedFileName, username); // hdfsOperations.uploadFile(localPath, hdfsPath, uploadedFileName, username);
return new ResponseDTO("File uploaded successfully", true); // return new ResponseDTO("File uploaded successfully", true);
} catch (IOException e) { // } catch (IOException e) {
e.printStackTrace(); // e.printStackTrace();
return new ResponseDTO("Failed to upload file locally: " + e.getMessage(), false); // return new ResponseDTO("Failed to upload file locally: " + e.getMessage(), false);
} catch (Exception e) { // } catch (Exception e) {
e.printStackTrace(); // e.printStackTrace();
return new ResponseDTO("Failed to upload file to HDFS: " + e.getMessage(), false); // return new ResponseDTO("Failed to upload file to HDFS: " + e.getMessage(), false);
} // }
// }
@PostMapping("/uploadFile")
public ResponseDTO uploadFile(
@RequestParam("file") MultipartFile file,
@RequestParam String hdfsPath,
@RequestParam String uploadedFileName,
@RequestParam String username) {
try {
// Upload file directly to HDFS without saving locally
hdfsOperations.uploadFile(file, hdfsPath, uploadedFileName, username);
return new ResponseDTO("File uploaded successfully", true);
} catch (Exception e) {
e.printStackTrace();
return new ResponseDTO("Failed to upload file to HDFS: " + e.getMessage(), false);
} }
}
private String saveFileLocally(MultipartFile file) throws IOException { //
// Create a temporary directory if it doesn't exist // private String saveFileLocally(MultipartFile file) throws IOException {
Path tmpDir = Paths.get("tmp"); // // Create a temporary directory if it doesn't exist
if (!Files.exists(tmpDir)) { // Path tmpDir = Paths.get("tmp");
Files.createDirectories(tmpDir); // Create the directory if it doesn't exist // if (!Files.exists(tmpDir)) {
} // Files.createDirectories(tmpDir); // Create the directory if it doesn't exist
// }
Path path = tmpDir.resolve(file.getOriginalFilename()); //
// Path path = tmpDir.resolve(file.getOriginalFilename());
// Copy the file to the local directory //
Files.copy(file.getInputStream(), path, StandardCopyOption.REPLACE_EXISTING); // // Copy the file to the local directory
// Files.copy(file.getInputStream(), path, StandardCopyOption.REPLACE_EXISTING);
return path.toString(); // Return the local path for further processing //
} // return path.toString(); // Return the local path for further processing
// }
@PostMapping("/downloadFile") @PostMapping("/downloadFile")
src/main/java/com/skycrate/backend/skycrateBackend/services/HDFSOperations.java
+38 -11
View File
@@ -2,10 +2,12 @@ package com.skycrate.backend.skycrateBackend.services;
import com.skycrate.backend.skycrateBackend.config.HDFSConfig; import com.skycrate.backend.skycrateBackend.config.HDFSConfig;
import com.skycrate.backend.skycrateBackend.utils.KeyUtil; import com.skycrate.backend.skycrateBackend.utils.KeyUtil;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import javax.crypto.Cipher; import javax.crypto.Cipher;
import javax.crypto.KeyGenerator; import javax.crypto.KeyGenerator;
@@ -26,30 +28,55 @@ import java.util.List;
@Service @Service
public class HDFSOperations { public class HDFSOperations {
public void uploadFile(String localPath, String hdfsPath, String uploadedFileName, String username) { // public void uploadFile(String localPath, String hdfsPath, String uploadedFileName, String username) {
// try {
// FileSystem fs = HDFSConfig.getHDFS();
// byte[] data = Files.readAllBytes(Paths.get(localPath)); // Read file as bytes
//
// // Encrypt file (consider adding encryption here as needed)
// byte[] encryptedData = data;
//
// String tempFilePath = localPath + ".enc";
// Files.write(Paths.get(tempFilePath), encryptedData);
//
// String finalHdfsPath = hdfsPath.endsWith("/") ? hdfsPath + uploadedFileName : hdfsPath + "/" + uploadedFileName;
// fs.copyFromLocalFile(new Path(tempFilePath), new Path(finalHdfsPath));
//
// Files.delete(Paths.get(tempFilePath));
// } catch (IOException e) {
// // Handle I/O exception and log the error
// throw new RuntimeException("Failed to upload file due to I/O issue: " + e.getMessage(), e);
// } catch (Exception e) {
// // Catch any other exceptions
// throw new RuntimeException("Failed to upload file: " + e.getMessage(), e);
// }
// }
public void uploadFile(MultipartFile file, String hdfsPath, String uploadedFileName, String username) {
try { try {
FileSystem fs = HDFSConfig.getHDFS(); FileSystem fs = HDFSConfig.getHDFS();
byte[] data = Files.readAllBytes(Paths.get(localPath)); // Read file as bytes
// Encrypt file (consider adding encryption here as needed) // Encrypt file data if needed (currently direct pass-through)
byte[] encryptedData = data; byte[] fileBytes = file.getBytes(); // directly get bytes from MultipartFile
byte[] encryptedData = fileBytes;
String tempFilePath = localPath + ".enc";
Files.write(Paths.get(tempFilePath), encryptedData);
// Define final HDFS file path
String finalHdfsPath = hdfsPath.endsWith("/") ? hdfsPath + uploadedFileName : hdfsPath + "/" + uploadedFileName; String finalHdfsPath = hdfsPath.endsWith("/") ? hdfsPath + uploadedFileName : hdfsPath + "/" + uploadedFileName;
fs.copyFromLocalFile(new Path(tempFilePath), new Path(finalHdfsPath));
Files.delete(Paths.get(tempFilePath)); // Write encrypted bytes directly to HDFS
Path hdfsDestPath = new Path(finalHdfsPath);
try (FSDataOutputStream outputStream = fs.create(hdfsDestPath, true)) {
outputStream.write(encryptedData);
}
} catch (IOException e) { } catch (IOException e) {
// Handle I/O exception and log the error
throw new RuntimeException("Failed to upload file due to I/O issue: " + e.getMessage(), e); throw new RuntimeException("Failed to upload file due to I/O issue: " + e.getMessage(), e);
} catch (Exception e) { } catch (Exception e) {
// Catch any other exceptions
throw new RuntimeException("Failed to upload file: " + e.getMessage(), e); throw new RuntimeException("Failed to upload file: " + e.getMessage(), e);
} }
} }
public void downloadFile(String hdfsPath, String localPath, String username) { public void downloadFile(String hdfsPath, String localPath, String username) {
try { try {
FileSystem fs = HDFSConfig.getHDFS(); FileSystem fs = HDFSConfig.getHDFS();