文章目录

  • 前言
  • 第一节 ItemWriter
  • 第二节 简单的ItemWriter入门
  • 第三节 JdbcBatchItemWriter向数据库中批量写
  • 1. 建表
  • 2. 批量插入数据
  • 3. 字段映射转换器
  • 第四节 FlatFileItemWriter向文件中写
  • 1. 依赖说明
  • 2. 向文件中写
  • 3. 写入文件的方式
  • 4. append写入
  • 第五节 StaxEventItemWriter向xml写入
  • 1. 依赖说明
  • 2. 向xml写入内容
  • 第六节 数据输出到多个文件
  • 1. CompositeItemWriter 委派数据写入到多文件
  • 2. ClassifierCompositeItemWriter 分类多文件写入
  • 第七节 遇到的问题


前言

SpringBatch用于数据的批处理，那么它包含数据的输入ItemReader，处理ItemProcessor，输出ItemWriter。

第一节 ItemWriter

ItemReader提供了多种数据输出的方式。

springbatch Step方法 springbatch writer_spring

第二节 简单的ItemWriter入门

  1. 实现ItemWriter 进行数据打印
package com.it2.springbootspringbatch01.itemwriter;

import com.alibaba.fastjson.JSONObject;
import com.it2.springbootspringbatch01.itemreaderdb.User;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.support.ListItemWriter;
import org.springframework.stereotype.Component;

import java.util.List;

@Component("myItemWriter")
public class MyItemWriter implements ItemWriter<User> {
    @Override
    public void write(List<? extends User> list) throws Exception {
        System.out.println("输出list");
        for(User user:list){
            System.out.println(JSONObject.toJSONString(user));
        }
    }
}
  1. 创建自己定义的job,使用自己定义的ItemWrtier进行数据打印
package com.it2.springbootspringbatch01.itemwriter;

import com.alibaba.fastjson.JSONObject;
import com.it2.springbootspringbatch01.itemreaderdb.User;
import com.it2.springbootspringbatch01.restart.RestartReader;
import lombok.extern.slf4j.Slf4j;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import java.util.ArrayList;
import java.util.List;

@Configuration
@EnableBatchProcessing
@Slf4j
public class ItemWriterDemo {

    //注入任务对象工厂
    @Autowired
    private JobBuilderFactory jobBuilderFactory;

    //任务的执行由Step决定,注入step对象的factory
    @Autowired
    private StepBuilderFactory stepBuilderFactory;

    @Autowired
    private MyItemWriter myItemWriter;//注入写

    //创建Job对象
    @Bean
    public Job jobwritr() {
        return jobBuilderFactory.get("jobwritr2")
                .start(step_writer())
                .build();
    }

    //创建Step对象
    @Bean
    public Step step_writer() {
        return stepBuilderFactory.get("step_writer")
                .<User,User>chunk(4)
                .reader(dataReader())
                .writer(myItemWriter)
                .build();
    }

    @Bean
    @StepScope //设定范围,避免重名bean的问题
    public ListItemReader<User> dataReader(){
        List<User> users=new ArrayList<>();
        for(int i=0;i<10;i++){
            User user = new User();
            user.setId(i);
            user.setName("user-"+i);
            users.add(user);
        }
        return new ListItemReader<>(users);
    }
}
  1. 启动服务器，可以看到数据的Writer方法是按照chunk的设置大小进行的批量输出。

第三节 JdbcBatchItemWriter向数据库中批量写

1. 建表

既然要向数据库中插入数据，那么肯定要先有表

CREATE TABLE `user2` (
  `id` int NOT NULL AUTO_INCREMENT,
  `name` varchar(10) CHARACTER SET utf8 DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 ;

2. 批量插入数据

  1. 使用JdbcBatchItemWriter向数据库中写入数据
package com.it2.springbootspringbatch01.itemwriterdb;

import com.alibaba.fastjson.JSONObject;
import com.it2.springbootspringbatch01.itemreaderdb.User;
import com.it2.springbootspringbatch01.itemwriter.MyItemWriter;
import com.it2.springbootspringbatch01.restart.RestartReader;
import lombok.extern.slf4j.Slf4j;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.database.BeanPropertyItemSqlParameterSourceProvider;
import org.springframework.batch.item.database.JdbcBatchItemWriter;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
import org.springframework.jdbc.core.namedparam.SqlParameterSource;

import javax.sql.DataSource;
import java.util.ArrayList;
import java.util.List;

@Configuration
@EnableBatchProcessing
@Slf4j
public class ItemWriterDbDemo {

    //注入任务对象工厂
    @Autowired
    private JobBuilderFactory jobBuilderFactory;

    //任务的执行由Step决定,注入step对象的factory
    @Autowired
    private StepBuilderFactory stepBuilderFactory;


    //创建Job对象
    @Bean
    public Job jobwritr_db() {
        return jobBuilderFactory.get("jobwritr_db")
                .start(step_writerdb())
                .build();
    }

    //创建Step对象
    @Bean
    public Step step_writerdb() {
        return stepBuilderFactory.get("step_writerdb")
                .<User,User>chunk(4)
                .reader(dataReader2())
                .writer(writerdb())
                .build();
    }


    @Bean
    @StepScope
    public ListItemReader<User> dataReader2() {
        List<User> users = new ArrayList<>();
        for (int i = 0; i < 10; i++) {
            User user = new User();
            user.setId(i+1);
            user.setName("user-" + i);
            user.setOrderId(i+10000);
            users.add(user);
        }
        return new ListItemReader<>(users);
    }

    @Autowired
    private DataSource dataSource;

    //创建Step对象
    @Bean
    @StepScope //设定范围,避免重名bean的问题
    public JdbcBatchItemWriter<User> writerdb() {
        JdbcBatchItemWriter<User> writer = new JdbcBatchItemWriter<>();
        writer.setDataSource(dataSource);
        writer.setSql("insert into user2 (id,name) values (:id,:name)");
        writer.setItemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<User>(){});//Bean转换器
        return writer;
    }

}
  1. 启动服务器运行，可以看到user2这个表里包含了刚刚插入的数据

3. 字段映射转换器

前面我们使用了默认的映射BeanPropertyItemSqlParameterSourceProvider，它的要求是对象的字段名和数据库的字段名一致，如果读取的对象和数据库字段名不一致怎么办?
假设现在的User的orderId属性向传给数据库的id，我们可以自己定义转换器

//创建Step对象
    @Bean
    public JdbcBatchItemWriter<User> writerdb() {
        JdbcBatchItemWriter<User> writer = new JdbcBatchItemWriter<>();
        writer.setDataSource(dataSource);
        writer.setSql("insert into user2 (id,name) values (:id,:name)");
//        writer.setItemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<User>(){});//Bean转换器
        writer.setItemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<User>(){ //自定义的字段转换匹配规则
            @Override
            public SqlParameterSource createSqlParameterSource(User item) {
                MapSqlParameterSource parameter=new MapSqlParameterSource();
                parameter.addValue("id",item.getOrderId());//将对象的OrderId传给数据库参数id
                parameter.addValue("name",item.getName());
                return parameter;
            }
        });
        return writer;
    }

修改转换器后测试

springbatch Step方法 springbatch writer_spring_02

第四节 FlatFileItemWriter向文件中写

1. 依赖说明

代码里使用到ObjectMapper进行对象转字符需要用到jackson

<!-- https://mvnrepository.com/artifact/org.codehaus.jackson/jackson-mapper-asl -->
        <dependency>
            <groupId>org.codehaus.jackson</groupId>
            <artifactId>jackson-mapper-asl</artifactId>
            <version>1.9.13</version>
        </dependency>

2. 向文件中写

package com.it2.springbootspringbatch01.itemwriterdb;

import com.alibaba.fastjson.JSONObject;
import com.it2.springbootspringbatch01.itemreaderdb.User;
import com.it2.springbootspringbatch01.itemwriter.MyItemWriter;
import com.it2.springbootspringbatch01.restart.RestartReader;
import lombok.extern.slf4j.Slf4j;
import org.codehaus.jackson.map.ObjectMapper;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.database.BeanPropertyItemSqlParameterSourceProvider;
import org.springframework.batch.item.database.JdbcBatchItemWriter;
import org.springframework.batch.item.file.FlatFileItemWriter;
import org.springframework.batch.item.file.transform.LineAggregator;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import .FileSystemResource;
import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
import org.springframework.jdbc.core.namedparam.SqlParameterSource;

import javax.sql.DataSource;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

@Configuration
@EnableBatchProcessing
@Slf4j
public class ItemWriterDbDemo {

    //注入任务对象工厂
    @Autowired
    private JobBuilderFactory jobBuilderFactory;

    //任务的执行由Step决定,注入step对象的factory
    @Autowired
    private StepBuilderFactory stepBuilderFactory;


    //创建Job对象
    @Bean
    public Job jobwritr_db() {
        return jobBuilderFactory.get("jobwritr_db")
                .start(step_writerdb())
                .build();
    }

    //创建Step对象
    @Bean
    public Step step_writerdb() {
        return stepBuilderFactory.get("step_writerdb")
                .<User,User>chunk(4)
                .reader(dataReader2())
                .writer(writerdb())
                .writer(writerFile())
                .build();
    }


    @Bean
    @StepScope
    public ListItemReader<User> dataReader2() {
        List<User> users = new ArrayList<>();
        for (int i = 0; i < 10; i++) {
            User user = new User();
            user.setId(i+1);
            user.setName("user-" + i);
            user.setOrderId(i+10000);
            users.add(user);
        }
        return new ListItemReader<>(users);
    }

    @Bean
    @StepScope
    public FlatFileItemWriter<User> writerFile() {
        FlatFileItemWriter<User> writer = new FlatFileItemWriter<>();
        writer.setResource(new FileSystemResource("d:\\user.txt"));
        writer.setLineAggregator(new LineAggregator<User>() {
            ObjectMapper mapper = new ObjectMapper();

            @Override
            public String aggregate(User user) { //将对象转字符串
                try {
                    return mapper.writeValueAsString(user);
                } catch (IOException e) {
                    e.printStackTrace();
                }
                return null;
            }
        });
        return writer;
    }
}

运行服务器，可以看到文件中包含的内容。(运行时不需要提前创建user.txt空文件)

springbatch Step方法 springbatch writer_junit_03

3. 写入文件的方式

我们在user.txt文件中先提前准备部分内容

springbatch Step方法 springbatch writer_springbatch Step方法_04


被插入的文件不是空文件，启动服务器向文件中写数据，发现原来的内容还在，则表示它写文件的方式是覆盖的方式，即原文件中有内容，会被覆盖。

springbatch Step方法 springbatch writer_junit_03

4. append写入

设置为append写入，那么写入文件时，会向原来的文件最后的位置添加，不会覆盖原文件的内容。

writer.setAppendAllowed(true);//追加写入

springbatch Step方法 springbatch writer_springbatch Step方法_06


springbatch Step方法 springbatch writer_spring_07

第五节 StaxEventItemWriter向xml写入

1. 依赖说明

xml的解析依赖于

<dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-oxm</artifactId>
        </dependency>
        <dependency>
            <groupId>com.thoughtworks.xstream</groupId>
            <artifactId>xstream</artifactId>
            <version>1.4.11.1</version>
        </dependency>

2. 向xml写入内容

package com.it2.springbootspringbatch01.itemwriterdb;

import com.alibaba.fastjson.JSONObject;
import com.it2.springbootspringbatch01.itemreaderdb.User;
import com.it2.springbootspringbatch01.itemwriter.MyItemWriter;
import com.it2.springbootspringbatch01.restart.RestartReader;
import lombok.extern.slf4j.Slf4j;
import org.codehaus.jackson.map.ObjectMapper;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.database.BeanPropertyItemSqlParameterSourceProvider;
import org.springframework.batch.item.database.JdbcBatchItemWriter;
import org.springframework.batch.item.file.FlatFileItemWriter;
import org.springframework.batch.item.file.transform.LineAggregator;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.batch.item.xml.StaxEventItemReader;
import org.springframework.batch.item.xml.StaxEventItemWriter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import .FileSystemResource;
import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
import org.springframework.jdbc.core.namedparam.SqlParameterSource;
import org.springframework.oxm.xstream.XStreamMarshaller;

import javax.sql.DataSource;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

@Configuration
@EnableBatchProcessing
@Slf4j
public class ItemWriterDbDemo {

    //注入任务对象工厂
    @Autowired
    private JobBuilderFactory jobBuilderFactory;

    //任务的执行由Step决定,注入step对象的factory
    @Autowired
    private StepBuilderFactory stepBuilderFactory;


    //创建Job对象
    @Bean
    public Job jobwritr_db() throws Exception {
        return jobBuilderFactory.get("jobwritr_db")
                .start(step_writerdb())
                .build();
    }

    //创建Step对象
    @Bean
    public Step step_writerdb() throws Exception {
        return stepBuilderFactory.get("step_writerdb")
                .<User, User>chunk(4)
                .reader(dataReader2())
                .writer(writerXmlFile())
                .build();
    }
    @Bean
    public StaxEventItemWriter<User> writerXmlFile() throws Exception {
        StaxEventItemWriter<User> writer = new StaxEventItemWriter<>();
        writer.setResource(new FileSystemResource("d:\\user.xml"));//设置写入路径

        /**
         * 设置转换
         */
        XStreamMarshaller xStreamMarshaller=new XStreamMarshaller();
        Map<String,Class> aliases=new HashMap<>();
        aliases.put("user",User.class);
        xStreamMarshaller.setAliases(aliases);

        writer.setMarshaller(xStreamMarshaller);
        writer.setRootTagName("users");//设置根标签
        writer.afterPropertiesSet();
        return writer;
    }

    @Bean
    @StepScope
    public ListItemReader<User> dataReader2() {
        List<User> users = new ArrayList<>();
        for (int i = 0; i < 10; i++) {
            User user = new User();
            user.setId(i + 1);
            user.setName("user-" + i);
            user.setOrderId(i + 10000);
            users.add(user);
        }
        return new ListItemReader<>(users);
    }
}
  1. 运行服务器，不需要提前创建user.xml文件

第六节 数据输出到多个文件

1. CompositeItemWriter 委派数据写入到多文件

CompositeItemWriter通过委派模式，将数据委派给其它的具体执行者，可以指派多个执行者去执行写操作。多个执行者都会获取reader的全部内容，并由执行者具体处理数据。

@Bean
    public CompositeItemWriter<User> multiFileItemWriter() throws Exception {
        CompositeItemWriter<User> writer = new CompositeItemWriter<>();
        writer.setDelegates(Arrays.asList(writerXmlFile(), writerFile()));//委派模式
        return writer;
    }

springbatch Step方法 springbatch writer_junit_08

启动服务器，可以看到两个文件都写入了reader的内容

springbatch Step方法 springbatch writer_junit_09

2. ClassifierCompositeItemWriter 分类多文件写入

我们读取reader数据之后，想要根据数据进行分类。例如id为奇数，选择A写入器，id为偶数选择B写入器。

//创建Step对象
    @Bean
    public Step step_writerdb() throws Exception {
        return stepBuilderFactory.get("step_writerdb")
                .<User, User>chunk(4)
                .reader(dataReader2())
//                .writer(writerXmlFile())
//                .writer(multiFileItemWriter())
                .writer(classifierMultiFileItemWriter())
                .stream(writerFile())  //使用ClassifierCompositeItemWriter需要由于ClassifierCompositeItemWriter并没有ItemStream,
                // 所以需要将具体的实现的ItemWriter加入到stream
                .stream(writerXmlFile())//使用ClassifierCompositeItemWriter需要由于ClassifierCompositeItemWriter并没有ItemStream,
                // 所以需要将具体的实现的ItemWriter加入到stream
                .build();
    }

 @Bean
    public ClassifierCompositeItemWriter<User> classifierMultiFileItemWriter() {
        ClassifierCompositeItemWriter<User> writer = new ClassifierCompositeItemWriter();
        writer.setClassifier(new Classifier<User, ItemWriter<? super User>>() {
            @Override
            public ItemWriter<? super User> classify(User user) {
                if (user.getId() % 2 == 0) { //根据id是奇数还是偶数，选择不同的ItemWriter
                    return writerFile();
                } else {
                    try{
                        return writerXmlFile();
                    }catch (Exception e){
                    }
                    return null;
                }
            }
        });
        return writer;
    }

springbatch Step方法 springbatch writer_User_10


启动服务器运行，插入写入的文件，可以看到数据按照设定的规则分别写到了不同的文件

springbatch Step方法 springbatch writer_spring_11

第七节 遇到的问题

org.springframework.batch.item.WriterNotOpenException: Writer must be open before it can be written