MySQL 批量插入的原理和实战方法(快速提升大数据导入效率)

CREATE TABLE students (
    id INT PRIMARY KEY AUTO_INCREMENT,
    name VARCHAR(100),
    age INT,
    gender ENUM('M', 'F'),
    grade VARCHAR(10)
);

pip install pymysql faker

import random
from faker import Faker
# 初始化 Faker
fake = Faker()
# 随机生成学生数据
def generate_random_students(num_records=10000):
    students_data = []
    for _ in range(num_records):
        name = fake.name()
        age = random.randint(18, 25)  # 随机年龄在 18 到 25 岁之间
        gender = random.choice(['M', 'F'])  # 随机选择性别
        grade = random.choice(['A', 'B', 'C', 'D', 'F'])  # 随机选择成绩
        students_data.append((name, age, gender, grade))
    return students_data
# 生成 1 万条学生数据
students_data = generate_random_students(10000)
# 输出前 5 条数据查看
for student in students_data[:5]:
    print(student)

import pymysql
from tqdm import tqdm
# 创建数据库连接
connection = pymysql.connect(
    host='localhost',
    user='your_username',
    password='your_password',
    database='your_database',
    charset='utf8mb4',
    cursorclass=pymysql.cursors.DictCursor
)
# 批量插入的批次大小
BATCH_SIZE = 1000
try:
    with connection.cursor() as cursor:
        batch = []
        for student in tqdm(students_data, total=len(students_data)):
            batch.append(student)
            # 当批次达到 BATCH_SIZE 时执行批量插入
            if len(batch) >= BATCH_SIZE:
                sql = """
                INSERT INTO students (name, age, gender, grade)
                VALUES (%s, %s, %s, %s)
                """
                cursor.executemany(sql, batch)
                batch = []  # 清空批次
        # 插入剩余的未满批次的数据
        if batch:
            sql = """
            INSERT INTO students (name, age, gender, grade)
            VALUES (%s, %s, %s, %s)
            """
            cursor.executemany(sql, batch)
        # 提交事务
        connection.commit()
except Exception as e:
    print(f"插入数据时出现错误: {e}")
    connection.rollback()
finally:
    # 关闭数据库连接
    connection.close()