springboot整合hadoop

Spring Boot 可以通过使用 Hadoop 客户端库来实现与 Hadoop 的集成。下面是实现步骤：

1. 在 pom.xml 文件中添加以下依赖项：

<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-client</artifactId>
    <version>${hadoop.version}</version>
</dependency>

2. 在 application.properties 文件中添加以下配置：

hadoop.fs.defaultFS=hdfs://localhost:9000

3. 创建一个 Hadoop 配置 Bean，用于创建 Hadoop 配置对象：

@Configuration
public class HadoopConfiguration {

    @Value("${hadoop.fs.defaultFS}")
    private String hdfsUri;

    @Bean
    public org.apache.hadoop.conf.Configuration configuration() {
        return new org.apache.hadoop.conf.Configuration();
    }

    @Bean
    public FileSystem fileSystem() throws IOException {
        return FileSystem.get(URI.create(hdfsUri), configuration());
    }
}

4. 创建一个服务类，用于执行 Hadoop 操作：

@Service
public class HadoopService {

    private final FileSystem fileSystem;

    public HadoopService(FileSystem fileSystem) {
        this.fileSystem = fileSystem;
    }

    public void createDirectory(String directoryPath) throws IOException {
        Path path = new Path(directoryPath);
        if (!fileSystem.exists(path)) {
            fileSystem.mkdirs(path);
        }
    }

    public void uploadFile(String localFilePath, String hdfsFilePath) throws IOException {
        Path localPath = new Path(localFilePath);
        Path hdfsPath = new Path(hdfsFilePath);
        fileSystem.copyFromLocalFile(localPath, hdfsPath);
    }

    public void downloadFile(String hdfsFilePath, String localFilePath) throws IOException {
        Path hdfsPath = new Path(hdfsFilePath);
        Path localPath = new Path(localFilePath);
        fileSystem.copyToLocalFile(hdfsPath, localPath);
    }
}

5. 在控制器中注入 HadoopService，并调用需要的操作：

@RestController
public class HadoopController {

    private final HadoopService hadoopService;

    public HadoopController(HadoopService hadoopService) {
        this.hadoopService = hadoopService;
    }

    @PostMapping("/create-directory")
    public void createDirectory(@RequestParam String directoryPath) throws IOException {
        hadoopService.createDirectory(directoryPath);
    }

    @PostMapping("/upload-file")
    public void uploadFile(@RequestParam String localFilePath, @RequestParam String hdfsFilePath) throws IOException {
        hadoopService.uploadFile(localFilePath, hdfsFilePath);
    }

    @PostMapping("/download-file")
    public void downloadFile(@RequestParam String hdfsFilePath, @RequestParam String localFilePath) throws IOException {
        hadoopService.downloadFile(hdfsFilePath, localFilePath);
    }
}

这样就可以在 Spring Boot 应用中使用 Hadoop 了。