Python搭建NLP模型的详细步骤

# 1. 创建虚拟环境
conda create -n nlp python=3.11 -y
conda activate nlp

# 2. 安装核心库（2026 年推荐组合）
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu   # CPU版（有GPU换cuda版本）
pip install transformers datasets accelerate evaluate pandas scikit-learn

from datasets import load_dataset

# 加载中文情感分析数据集（Seamew/ChnSentiCorp）
dataset = load_dataset("seamew/ChnSentiCorp")

print(dataset)
# 输出：DatasetDict({
#     train: Dataset({...}),
#     validation: Dataset({...}),
#     test: Dataset({...})
# })

# 查看前3条数据
print(dataset["train"][0:3])

from transformers import AutoTokenizer

model_name = "bert-base-chinese"   # 中文预训练模型

tokenizer = AutoTokenizer.from_pretrained(model_name)

def preprocess_function(examples):
    return tokenizer(examples["text"], 
                     truncation=True,      # 超长自动截断
                     padding="max_length", # 统一长度
                     max_length=128)

# 应用到整个数据集
tokenized_datasets = dataset.map(preprocess_function, batched=True)

from transformers import AutoModelForSequenceClassification, TrainingArguments, Trainer
import evaluate
import numpy as np

# 加载模型（2分类：正面/负面）
model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2)

# 评估指标（准确率 + F1）
accuracy = evaluate.load("accuracy")
f1 = evaluate.load("f1")

def compute_metrics(eval_pred):
    logits, labels = eval_pred
    predictions = np.argmax(logits, axis=-1)
    acc = accuracy.compute(predictions=predictions, references=labels)
    f1_score = f1.compute(predictions=predictions, references=labels, average="macro")
    return {"accuracy": acc["accuracy"], "f1": f1_score["f1"]}

# 训练参数（可根据显存调整）
training_args = TrainingArguments(
    output_dir="./results",          # 输出目录
    eval_strategy="epoch",           # 每轮评估
    save_strategy="epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=16,
    num_train_epochs=3,              # 建议 3-5 轮
    weight_decay=0.01,
    logging_dir="./logs",
    report_to="none",                # 不上传 wandb
    load_best_model_at_end=True,
    metric_for_best_model="f1"
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_datasets["train"],
    eval_dataset=tokenized_datasets["validation"],
    compute_metrics=compute_metrics,
    tokenizer=tokenizer,   # 自动保存 tokenizer
)

print("开始训练...")
trainer.train()

# 训练结束后在测试集上评估
test_results = trainer.evaluate(tokenized_datasets["test"])
print("测试集结果：", test_results)

# 保存模型
trainer.save_model("./my_chinese_sentiment_model")
tokenizer.save_pretrained("./my_chinese_sentiment_model")

# 加载推理（最常用方式）
from transformers import pipeline

sentiment_pipeline = pipeline(
    "sentiment-analysis",
    model="./my_chinese_sentiment_model",
    tokenizer="./my_chinese_sentiment_model"
)

# 测试
texts = [
    "这家餐厅的服务态度非常好，菜也超级美味！",
    "完全是浪费钱，东西难吃还贵，差评！",
    "一般般吧，没什么特别的。"
]

results = sentiment_pipeline(texts)
for text, res in zip(texts, results):
    label = "正面" if res["label"] == "LABEL_1" else "负面"
    print(f"文本: {text}\n预测: {label} (置信度: {res['score']:.4f})\n")

nlp_sentiment_project/
├── data/                  # 原始数据
├── models/                # 保存的模型
├── results/               # 训练输出
├── logs/
├── main.py                # 训练主文件
├── inference.py           # 推理脚本
├── requirements.txt
└── README.md