6种Python中提高文本处理效率的技巧分享

import re

text = "Contact us at info@example.com or support@example.com"
email_pattern = r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b'
emails = re.findall(email_pattern, text)
print(emails)

text = "The price is $10.99"
new_text = re.sub(r'\$(\d+\.\d{2})', lambda m: f"￥{float(m.group(1)) * 7.33:.2f}", text)
print(new_text)

import string

text = "Hello, World! How are you?"
translator = str.maketrans("", "", string.punctuation)
cleaned_text = text.translate(translator)
print(cleaned_text)

from difflib import get_close_matches

words = ["python", "programming", "code", "developer"]
similar = get_close_matches("pythonic", words, n=1, cutoff=0.6)
print(similar)

from difflib import SequenceMatcher

def similarity(a, b):
    return SequenceMatcher(None, a, b).ratio()

print(similarity("python", "pyhton"))

import Levenshtein

def spell_check(word, dictionary):
    return min(dictionary, key=lambda x: Levenshtein.distance(word, x))

dictionary = ["python", "programming", "code", "developer"]
print(spell_check("progamming", dictionary))

def find_similar(word, words, max_distance=2):
    return [w for w in words if Levenshtein.distance(word, w) <= max_distance]

words = ["python", "programming", "code", "developer", "coder"]
print(find_similar("code", words))

import ftfy

text = "The Mona Lisa doesn’t have eyebrows."
fixed_text = ftfy.fix_text(text)
print(fixed_text)

weird_text = "Ｔｈｉｓ ｉｓ Ｆｕｌｌｗｉｄｔｈ ｔｅｘｔ"
normal_text = ftfy.fix_text(weird_text)
print(normal_text)

import spacy

nlp = spacy.load("en_core_web_sm")
text = "The quick brown fox jumps over the lazy dog."
doc = nlp(text)
tokens = [token.text for token in doc]
print(tokens)

import nltk
nltk.download('punkt')

from nltk.tokenize import word_tokenize

text = "The quick brown fox jumps over the lazy dog."
tokens = word_tokenize(text)
print(tokens)

import jieba

text = "我爱Python编程，Python是个很棒的语言！"

# 使用jieba进行精确模式分词
tokens = jieba.cut(text, cut_all=False)

print(list(tokens))

import nltk
nltk.download('vader_lexicon')

from nltk.sentiment import SentimentIntensityAnalyzer

def analyze_sentiment(text):
    sia = SentimentIntensityAnalyzer()
    return sia.polarity_scores(text)

text = "I love Python! It's such a versatile and powerful language."
sentiment = analyze_sentiment(text)
print(sentiment)

def process_large_file(filename):
    with open(filename, 'r') as file:
        for line in file:
            yield line.strip()

for line in process_large_file('large_text_file.txt'):
    # 处理每一行
    pass

from multiprocessing import Pool

def process_text(text):
    # 一些CPU密集型的文本处理
    return processed_text

if __name__ == '__main__':
    with Pool() as p:
        results = p.map(process_text, large_text_list)

 stopwords = set(['the', 'a', 'an', 'in', 'of', 'on'])

 def remove_stopwords(text):
     return ' '.join([word for word in text.split() if word.lower() not in stopwords])

import re

email_pattern = re.compile(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b')

def find_emails(text):
    return email_pattern.findall(text)

import pandas as pd

df = pd.read_csv('large_text_data.csv')
processed_df = df['text_column'].apply(process_text)