Python之accelerator包语法、参数、实际应用案例和常见错误详解

# 基础安装
pip install accelerate

# 安装时包含所有依赖（推荐）
pip install 'accelerate[torch]'

# 从源码安装最新版本
pip install git+https://github.com/huggingface/accelerate.git

accelerate env

from accelerate import Accelerator
import torch
from torch.utils.data import DataLoader, Dataset

# 简单数据集
class MyDataset(Dataset):
    def __len__(self): return 1000
    def __getitem__(self, idx): return torch.tensor([idx]), torch.tensor([idx%2])

# 初始化加速器
accelerator = Accelerator()

# 模型、数据加载器、优化器
model = torch.nn.Linear(1, 1)
dataloader = DataLoader(MyDataset(), batch_size=32)
optimizer = torch.optim.Adam(model.parameters())

# 准备组件
model, optimizer, dataloader = accelerator.prepare(
    model, optimizer, dataloader
)

# 训练循环
for epoch in range(3):
    for inputs, targets in dataloader:
        outputs = model(inputs.float())
        loss = torch.nn.functional.mse_loss(outputs, targets.float())
        accelerator.backward(loss)
        optimizer.step()
        optimizer.zero_grad()
    print(f"Epoch {epoch} completed")

# 启用FP16混合精度
accelerator = Accelerator(mixed_precision='fp16')

# 其余代码与案例1相同，但会自动使用混合精度

from accelerate import Accelerator
import torch

accelerator = Accelerator()

# 假设我们有模型和测试数据加载器
model, test_dataloader = accelerator.prepare(model, test_dataloader)

model.eval()
total_correct = 0
total = 0

with torch.no_grad():
    for inputs, labels in test_dataloader:
        outputs = model(inputs)
        predictions = outputs.argmax(dim=1)
        # 收集所有进程的结果
        all_predictions = accelerator.gather(predictions)
        all_labels = accelerator.gather(labels)
        total_correct += (all_predictions == all_labels).sum().item()
        total += all_labels.size(0)

# 只在主进程打印结果
if accelerator.is_main_process:
    print(f"Accuracy: {total_correct / total:.2f}")

accelerate launch --num_processes=2 train.py

# 保存训练状态
accelerator.save_state('checkpoint')

# 加载训练状态
accelerator.load_state('checkpoint')

from accelerate import Accelerator
from transformers import AutoModelForSequenceClassification, AutoTokenizer
from datasets import load_dataset

accelerator = Accelerator()

# 加载模型、分词器和数据集
model = AutoModelForSequenceClassification.from_pretrained("bert-base-uncased")
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
dataset = load_dataset("imdb")

# 预处理数据
def preprocess_function(examples):
    return tokenizer(examples["text"], truncation=True, max_length=512)
tokenized_dataset = dataset.map(preprocess_function, batched=True)

# 准备训练器组件
train_loader = DataLoader(tokenized_dataset["train"], batch_size=8)
optimizer = torch.optim.Adam(model.parameters(), lr=5e-5)

# 加速准备
model, optimizer, train_loader = accelerator.prepare(model, optimizer, train_loader)

# 训练循环（简化版）
for batch in train_loader:
    outputs = model(**batch)
    loss = outputs.loss
    accelerator.backward(loss)
    optimizer.step()
    optimizer.zero_grad()