Springboot 手动分页查询分批批量插入数据的实现流程

        //获取数据总计数
        UserQueryCondition userQueryCondition=new UserQueryCondition();
        Integer totalCount = userService.getAllUserCount(userQueryCondition);
        //每批同步的数据条数
        Integer batchSizeLimit = 500;
        //分批切割处理
        List<PageLimitDTO> pageLimitGroupList = getPageLimitGroupList(totalCount, batchSizeLimit);
        int count=1;
        //物理批次查询
        for (PageLimitDTO pageBatchLimit:pageLimitGroupList){
            List<User> pageBatchList = userService.getPageList(
                    userQueryCondition, pageBatchLimit.getCurrIndex(), pageBatchLimit.getPageSize()
            );
            if (!CollectionUtils.isEmpty(pageBatchList)){
                //批量插入
                Boolean syncAddResult = userSyncService.batchSyncAdd(pageBatchList);
                //做其余业务
                if (syncAddResult){
                    log.info("第{}次,user数据批量插入成功",count);
                }
            }
            log.info("第{}批次,user数据同步批量插入业务结束执行",count);
            count=count+1;
        }

    public List<PageLimitDTO> getPageLimitGroupList(Integer totalCount, Integer batchSizeLimit ) {
        log.info("这一次处理的总数据条数为 ={} 条, 每一批次处理条数为 ={} 条,现在开始做分批切割处理。",totalCount,batchSizeLimit);
        int pageNum = totalCount / batchSizeLimit;
        int surplus = totalCount % batchSizeLimit;
        if (surplus > 0) {
            pageNum = pageNum + 1;
        }
        List<PageLimitDTO> pageLimitGroupList =new LinkedList<>();
        for(int i = 0; i < pageNum; i++){
            Integer currIndex = i * batchSizeLimit;
            PageLimitDTO pageLimitDTO=new PageLimitDTO();
            pageLimitDTO.setPageSize(batchSizeLimit);
            pageLimitDTO.setCurrIndex(currIndex);
            pageLimitDTO.setDealDataCount(currIndex+batchSizeLimit);
            pageLimitGroupList.add(pageLimitDTO);
            log.info("分批切割,第={}次,每次={}条,最终会处理到={}条。",pageLimitGroupList.size(),batchSizeLimit,currIndex+batchSizeLimit);
 
        }
        log.info("这一次处理的总数据条数为 ={} 条, 每一批次处理条数为 ={} 条,总共切割分成了 ={} 次,一切准备就绪,可以开始批量插入。",totalCount,batchSizeLimit,pageLimitGroupList.size());
        return pageLimitGroupList;
    }

    <select id="getPageList" resultMap="BaseResultMap">
        SELECT *
        FROM user
        <where>
            <if test="merchantId != null and merchantId != 0">
                and MERCHANT_ID = #{merchantId}
            </if>
            <if test="nameList != null and !nameList.isEmpty()">
                and NAME in
                <foreach collection="nameList" separator="," open="(" close=")" item="name">
                    #{name}
                </foreach>
            </if>
        </where>
        LIMIT #{currIndex} , #{pageSize}
 
    </select>

    <!--批量插入信息-->
    <insert id="batchSyncAdd" parameterType="java.util.List">
        insert into user(
        id,
        name,
        age
        )
        values
        <foreach collection="list" item="item" index="index" separator=",">
            (
            #{item.id,jdbcType=BIGINT},
            #{item.name,jdbcType=VARCHAR},
            #{item.age,jdbcType=INTEGER}
            )
        </foreach>
    </insert>