MySQL中实现大数据快速插入的全攻略

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
public class FastInsertDemo {
    public static void main(String[] args) {
        String url = "jdbc:mysql://localhost:3306/idea_demo?serverTimezone=Asia/Shanghai&useSSL=false&rewriteBatchedStatements=true";
        String user = "idea_user";
        String password = "IdeaDemo@2026";
        try (Connection conn = DriverManager.getConnection(url, user, password)) {
            // 1. 关闭自动提交
            conn.setAutoCommit(false);
            String sql = "INSERT INTO user_info (username, phone, age) VALUES (?, ?, ?)";
            try (PreparedStatement ps = conn.prepareStatement(sql)) {
                // 2. 批量添加参数
                for (int i = 1; i <= 30000; i++) {
                    ps.setString(1, "用户" + i);
                    ps.setString(2, "13800" + String.format("%06d", i));
                    ps.setInt(3, 20 + (i % 30));
                    ps.addBatch(); // 添加到批量
                    // 每 1000 条提交一次，避免大事务
                    if (i % 1000 == 0) {
                        ps.executeBatch();
                        conn.commit();
                        ps.clearBatch(); // 清空批量
                    }
                }
                // 3. 提交剩余的数据
                ps.executeBatch();
                conn.commit();
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

<!-- UserMapper.xml -->
<insert id="batchInsert" parameterType="java.util.List">
    INSERT INTO user_info (username, phone, age)
    VALUES
    <foreach collection="list" item="item" separator=",">
        (#{item.username}, #{item.phone}, #{item.age})
    </foreach>
</insert>

// UserService.java
@Service
public class UserService {
    @Autowired
    private UserMapper userMapper;
    @Transactional(rollbackFor = Exception.class)
    public void batchInsert(List<User> userList) {
        // 每 1000 条插入一次
        int batchSize = 1000;
        for (int i = 0; i < userList.size(); i += batchSize) {
            int end = Math.min(i + batchSize, userList.size());
            userMapper.batchInsert(userList.subList(i, end));
        }
    }
}

import pymysql
def fast_insert():
    conn = pymysql.connect(
        host='localhost',
        port=3306,
        user='idea_user',
        password='IdeaDemo@2026',
        database='idea_demo',
        charset='utf8mb4'
    )
    try:
        with conn.cursor() as cursor:
            # 1. 关闭自动提交
            conn.autocommit(False)
            sql = "INSERT INTO user_info (username, phone, age) VALUES (%s, %s, %s)"
            data = []
            for i in range(1, 30001):
                data.append((f"用户{i}", f"13800{i:06d}", 20 + (i % 30)))
                # 每 1000 条提交一次
                if i % 1000 == 0:
                    cursor.executemany(sql, data)
                    conn.commit()
                    data = []
            # 提交剩余数据
            if data:
                cursor.executemany(sql, data)
                conn.commit()
    finally:
        conn.close()
if __name__ == '__main__':
    fast_insert()

用户1,13800000001,25
用户2,13800000002,30
用户3,13800000003,28
...
用户30000,13800030000,22

-- 本地文件导入（需要开启 local_infile）
LOAD DATA LOCAL INFILE '/path/to/user_data.csv'
INTO TABLE user_info
FIELDS TERMINATED BY ','  -- 字段分隔符
ENCLOSED BY '"'           -- 字段包裹符（可选）
LINES TERMINATED BY '\n'  -- 行分隔符
IGNORE 1 LINES            -- 忽略第一行（如果有表头）
(username, phone, age);   -- 对应字段

-- 临时开启
SET GLOBAL local_infile = 1;
-- 永久开启（修改 my.cnf/my.ini）
[mysqld]
local_infile = 1

-- 禁用表的非唯一索引
ALTER TABLE user_info DISABLE KEYS;

-- 重建非唯一索引
ALTER TABLE user_info ENABLE KEYS;

[mysqld]
# 比如服务器内存 16G，设置为 10G
innodb_buffer_pool_size = 10G

[mysqld]
# Redo Log 文件大小，建议 1G-4G
innodb_log_file_size = 2G
# Redo Log 缓冲区，建议 16M-64M
innodb_log_buffer_size = 64M

[mysqld]
innodb_flush_log_at_trx_commit = 2

[mysqld]
innodb_flush_method = O_DIRECT

[mysqld]
max_allowed_packet = 64M

[mysqld]
# 关闭查询缓存（MySQL 8.0 已移除，5.7 建议关闭）
query_cache_type = 0
query_cache_size = 0
# 临时表大小
tmp_table_size = 64M
max_heap_table_size = 64M
# 连接数
max_connections = 1000