从基础方法到智能算法详解Python查找文本错别字的常见方法

def dictionary_check(text, dictionary):
    words = text.split()  # 简单分词（实际场景需更复杂的分词逻辑）
    errors = []
    for word in words:
        if word.lower() not in dictionary:
            errors.append(word)
    return errors

# 示例词典
standard_dict = {"今天", "天气", "很好", "我", "去", "学校", "学习"}
text = "今天天汽很好 我区学校学习"
errors = dictionary_check(text, standard_dict)
print("检测到的错别字:", errors)  # 输出: ['天汽', '区']

from Levenshtein import distance

def find_closest_word(word, dictionary):
    candidates = []
    for dict_word in dictionary:
        edit_dist = distance(word, dict_word)
        candidates.append((dict_word, edit_dist))
    candidates.sort(key=lambda x: x[1])
    return candidates[0][0] if candidates else None

# 扩展词典检查函数
def enhanced_dictionary_check(text, dictionary):
    words = text.split()
    errors = []
    for word in words:
        if word.lower() not in dictionary:
            correction = find_closest_word(word, dictionary)
            if correction:
                errors.append((word, correction))
    return errors

text = "我喜换编程"
errors = enhanced_dictionary_check(text, {"我", "喜欢", "编程"})
print("错别字及建议:", errors)  # 输出: [('喜换', '喜欢')]

import enchant

def enchant_check(text, lang='en_US'):
    d = enchant.Dict(lang)
    words = text.split()
    errors = [word for word in words if not d.check(word)]
    return errors

text = "I havv a speling eror"
errors = enchant_check(text)
print("检测到的错误:", errors)  # 输出: ['havv', 'speling', 'eror']

import symspellpy

def symspell_check(text):
    sym_spell = symspellpy.SymSpell()
    sym_spell.load_dictionary("frequency_dictionary_en_82_765.txt", 0, 1)
    
    suggestions = sym_spell.lookup_compound(text, max_edit_distance=2)
    errors = []
    for suggestion in suggestions:
        if suggestion.term != text:
            errors.append((text, suggestion.term))
    return errors

text = "I am goig to the park"
errors = symspell_check(text)
print("错误及建议:", errors)  # 输出: [('I am goig to the park', 'I am going to the park')]

from transformers import BertTokenizer, BertForMaskedLM
import torch

def bert_check(text, model_name='bert-base-chinese'):
    tokenizer = BertTokenizer.from_pretrained(model_name)
    model = BertForMaskedLM.from_pretrained(model_name)
    
    # 模拟检测逻辑（实际需更复杂的实现）
    suspicious_words = []
    for i, char in enumerate(text):
        # 简单示例：检测连续相同字符（如"天天"）
        if i > 0 and text[i] == text[i-1]:
            suspicious_words.append((i-1, i+1, text[i-1:i+1]))
    
    # 实际应通过模型预测最可能正确词
    return suspicious_words

text = "今天天气天晴朗"
errors = bert_check(text)
print("可疑片段:", errors)  # 输出: [(5, 7, '天天')]

# 伪代码示例（实际需训练模型）
from transformers import AutoTokenizer, AutoModelForTokenClassification

def sequence_labeling_check(text):
    tokenizer = AutoTokenizer.from_pretrained("bert-base-chinese")
    model = AutoModelForTokenClassification.from_pretrained("your-trained-model")
    
    inputs = tokenizer(text, return_tensors="pt")
    outputs = model(**inputs)
    predictions = torch.argmax(outputs.logits, dim=2)
    
    # 假设标签1表示错误
    errors = []
    for i, pred in enumerate(predictions[0]):
        if pred == 1:
            start = inputs.char_to_token(i, position_id=0)
            # 实际需更复杂的字符位置映射
            errors.append((i, text[i]))
    return errors

def hybrid_check(text):
    # 第一层：词典检查
    base_errors = dictionary_check(text, standard_dict)
    
    # 第二层：规则检查（如重复词、特殊符号）
    rule_errors = []
    for i in range(len(text)-1):
        if text[i] == text[i+1]:
            rule_errors.append((i, text[i:i+2]))
    
    # 第三层：模型检查（需加载预训练模型）
    # model_errors = bert_check(text)
    
    return {
        "dictionary_errors": base_errors,
        "rule_errors": rule_errors,
        # "model_errors": model_errors
    }

def domain_specific_check(text, domain="medical"):
    domain_dict = {
        "medical": {"抗生素", "炎症", "处方"},
        "legal": {"合同", "甲方", "违约"}
    }.get(domain, set())
    
    # 加载领域词典进行检测
    return dictionary_check(text, domain_dict)

import jieba
from collections import defaultdict

class ChineseSpellChecker:
    def __init__(self):
        # 加载基础词典
        self.base_dict = self.load_dictionary("base_dict.txt")
        # 加载常见错别字映射
        self.error_map = self.load_error_map("common_errors.csv")
    
    def load_dictionary(self, filepath):
        with open(filepath, 'r', encoding='utf-8') as f:
            return set(line.strip() for line in f)
    
    def load_error_map(self, filepath):
        error_map = defaultdict(list)
        with open(filepath, 'r', encoding='utf-8') as f:
            for line in f:
                wrong, correct = line.strip().split(',')
                error_map[wrong].append(correct)
        return error_map
    
    def check(self, text):
        words = jieba.lcut(text)
        errors = []
        
        for word in words:
            # 1. 直接词典匹配
            if word not in self.base_dict:
                # 2. 检查常见错别字映射
                if word in self.error_map:
                    suggestions = self.error_map[word]
                    errors.append((word, suggestions[0]))  # 取第一个建议
                else:
                    # 3. 未来可集成模型预测
                    errors.append((word, None))
        
        return errors

# 使用示例
checker = ChineseSpellChecker()
text = "今天天气晴郎，我很高兴。"
errors = checker.check(text)
print("检测结果:", errors)  # 输出: [('晴郎', '晴朗')]