spring-batch - Spring批量读取对文件夹中的大量文件进行分区

Question

我们的要求是将 XML 加载到 DB。文件夹有超过 50K XML 文件。MultiResourceParitioner 正在创建 50K slaveStep:partitionXXX 并且由于 ThreadPoolTask​​Executor 中的线程数量较少而失败。请告知最好的方法。为 Partitioner、master 和 slave 附加了代码片段。

@Bean("partitioner")
    @JobScope
    public Partitioner partitioner()throws Exception {
        MultiResourcePartitioner partitioner = new MultiResourcePartitioner();
        PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
        partitioner.setResources(resolver.getResources("file:" + resourcesPath + fileType));
        partitioner.partition(10);
        logger.info("--------partitioner()---end -No of files--->"+ resolver.getResources("file:" + resourcesPath + fileType).length);
        return partitioner;
    }
@Bean
    public Step slaveStep() throws Exception{
        return stepBuilderFactory.get("slaveStep")
                .<UnifiedInvoiceDTO, UnifiedInvoiceDTO>chunk(1)
                .reader(xmlItemReader(null))
                .processor(xmlFileItemProcessor())
                .writer(writer())
                .build();
    }

    @Bean
    public ThreadPoolTaskExecutor taskExecutor() {
        ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
        taskExecutor.setMaxPoolSize(100);
        taskExecutor.setCorePoolSize(70);
        taskExecutor.setQueueCapacity(70);
        taskExecutor.setWaitForTasksToCompleteOnShutdown(true);
        taskExecutor.afterPropertiesSet();
        return taskExecutor;
    }

    @Bean
    @Qualifier("masterStep")
    public Step masterStep() throws Exception{
        return stepBuilderFactory.get("masterStep")
                .partitioner("slaveStep", partitioner())
                .step(slaveStep())
                .taskExecutor(taskExecutor()).listener(new InvoiceStepListener())
                .build();
    }

    @Bean
    @StepScope
    @Qualifier("xmlItemReader")
    @DependsOn("partitioner")
    public StaxEventItemReader<Invoice> xmlItemReader(@Value("#{stepExecutionContext['fileName']}") String filename)throws MalformedURLException {
         logger.info("----StaxEventItemReader----fileName--->" + filename);
         StaxEventItemReader<Invoice> xmlFileReader = new StaxEventItemReader<>();
            xmlFileReader.setStrict(false);
            xmlFileReader.setResource(new UrlResource(filename));
            xmlFileReader.setFragmentRootElementName("Invoice");
            Jaxb2Marshaller invoiceMarshaller = new Jaxb2Marshaller();
            invoiceMarshaller.setClassesToBeBound(Invoice.class);
            xmlFileReader.setUnmarshaller(invoiceMarshaller);
            return xmlFileReader;
    }

标签： spring-batchpartitioning