spring-boot - Springboot Batch 从 REST API 获取 CSV

问题描述

我有一个 Springboot 批处理应用程序的场景，我需要读取 CSV-11 并写入另一个 CSV-2，当我需要从 REST 端点获取 CSV-1 时出现问题，例如当我启动它的工作时应该从端点获取 CSV-1，然后继续批处理以写入 CSV-2。但似乎处理 CSV-1 我们需要在应用程序启动时提前提供“资源”[我对批处理相当陌生，所以不确定它是否 100% 正确]。谁能指导我正确的方法来解决这个问题？

编辑：添加代码，我能够解决它，但如果它是正确的做事方式需要建议（请忽略硬编码数据）。

@Configuration
@EnableBatchProcessing
public class BatchConfig {

    @Autowired
    private JobBuilderFactory jobbuilder;

    @Autowired
    private StepBuilderFactory stepbuilder;

    @Autowired
    private CSVProcessor processor;

    @Autowired
    private CustomSkipPolicy customSkipPolicy;

    @Autowired
    private CSVResponse response;

    @Bean(name="csv")
    public Job job_csv() throws Exception {

        Step step = stepbuilder
                .get("csv-step")
                .<Person, Person>chunk(5)
                .reader(new CSVReader(response.getResource()))
                .processor(processor)
                .writer(new CSVWriter().write(response.getFilename()))
                .faultTolerant()
                .skipPolicy(customSkipPolicy)
                .listener(new StepListener())
                .build();

        return jobbuilder
                .get("csv-job")
                .incrementer(new RunIdIncrementer())
                .listener(new JobListener())
                .flow(step)
                .end()
                .build();

    }
}


@Slf4j
public class CSVReader extends FlatFileItemReader<Person> {

    public CSVReader(Resource resource) throws Exception {
        super();
        setResource(resource);
        setStrict(false);
        setLinesToSkip(1);

        doOpen();

        DelimitedLineTokenizer dlt = new DelimitedLineTokenizer();
        dlt.setNames(new String[] {"id","first_name","last_name","email","gender","ip_address","dob"});
        dlt.setDelimiter(",");
        dlt.setStrict(false);

        BeanWrapperFieldSetMapper<Person> fsp = new BeanWrapperFieldSetMapper<>();
        fsp.setTargetType(Person.class);

        DefaultLineMapper<Person> dlp = new DefaultLineMapper<>();
        dlp.setLineTokenizer(dlt);
        dlp.setFieldSetMapper(fsp);

        setLineMapper(dlp);
    }
}

public class CSVWriter {
    private static final String DATA_PROCESSED = "C:/data/processed";

    public FlatFileItemWriter<Person> write(String filename) throws Exception {
        FlatFileItemWriter<Person> writer = new FlatFileItemWriter<>();
        writer.setResource(resource(filename));
        writer.setHeaderCallback(new Header());

        BeanWrapperFieldExtractor<Person> fe = new BeanWrapperFieldExtractor<>();
        fe.setNames(new String[] { "id", "first_name", "last_name", "age" });

        DelimitedLineAggregator<Person> dla = new DelimitedLineAggregator<>();
        dla.setDelimiter(",");
        dla.setFieldExtractor(fe);

        writer.setLineAggregator(dla);
        return writer;
    }

    private Resource resource(String filename) throws IOException {
        String processed_file = filename.replace(".csv", "").concat("_PROCESSED").concat(".csv");
        if (Files.notExists(Paths.get(DATA_PROCESSED), new LinkOption[] { LinkOption.NOFOLLOW_LINKS })) {
            Files.createDirectory(Paths.get(DATA_PROCESSED));
        }
        if (Files.notExists(Paths.get(DATA_PROCESSED + "/" + processed_file),
                new LinkOption[] { LinkOption.NOFOLLOW_LINKS })) {
            Files.createFile(Paths.get(DATA_PROCESSED + "/" + processed_file));
        }
        return new FileSystemResource(DATA_PROCESSED + "/" + processed_file);
    }

}


@Component
@Slf4j
public class CSVResponse {

    private static final String DATA_RECEIVE = "C:/data/receive";
    private String filename;

    public Resource getResource() throws IOException {
        ResponseEntity<Resource> resp = new RestTemplate().getForEntity("http://localhost:8081/csv", Resource.class);
        log.info("File Received : "+resp.getBody().getFilename());
        setFilename(resp.getBody().getFilename());
        Path path = Paths.get(DATA_RECEIVE);
        if(!Files.exists(path, new LinkOption[]{ LinkOption.NOFOLLOW_LINKS})) {
            Files.createDirectory(path);
        }
        Files.copy(resp.getBody().getInputStream(), Paths.get(DATA_RECEIVE + "/" +resp.getBody().getFilename()), StandardCopyOption.REPLACE_EXISTING);
        return resp.getBody();
    }

    public String getFilename() {
        return filename;
    }

    public void setFilename(String filename) {
        this.filename = filename;
    }

}

标签： spring-bootspring-batch

spring-boot - Springboot Batch 从 REST API 获取 CSV

问题描述

解决方案

推荐阅读