一文分享5个Python文本处理的高效操作

import re

def clean_text(text):
    # 去除HTML标签
    text = re.sub(r'<[^>]+>', '', text)
    # 去除特殊字符和数字
    text = re.sub(r'[^a-zA-Z\s]', '', text)
    # 转换为小写
    text = text.lower()
    # 去除多余空格
    text = ' '.join(text.split())
    return text

sample_text = "<p>This is a sample text! 123</p>"
print(clean_text(sample_text))  # 输出: this is a sample text

# 英文分词
import nltk
nltk.download('punkt')  # 第一次使用需要下载数据

from nltk.tokenize import word_tokenize
text = "Natural language processing is fascinating."
tokens = word_tokenize(text)
print(tokens)  # 输出: ['Natural', 'language', 'processing', 'is', 'fascinating', '.']

# 中文分词
import jieba
text_chinese = "自然语言处理非常有趣"
tokens_chinese = jieba.lcut(text_chinese)
print(tokens_chinese)  # 输出: ['自然语言', '处理', '非常', '有趣']

from nltk.corpus import stopwords
nltk.download('stopwords')  # 第一次使用需要下载数据

stop_words = set(stopwords.words('english'))
filtered_tokens = [word for word in tokens if word.lower() not in stop_words]
print(filtered_tokens)  # 输出: ['Natural', 'language', 'processing', 'fascinating', '.']

# 中文停用词示例
stopwords_chinese = {"的", "是", "在", "非常"}
filtered_chinese = [word for word in tokens_chinese if word not in stopwords_chinese]
print(filtered_chinese)  # 输出: ['自然语言', '处理', '有趣']

from collections import Counter
from wordcloud import WordCloud
import matplotlib.pyplot as plt

# 词频统计
word_counts = Counter(filtered_tokens)
print(word_counts.most_common(3))  # 输出: [('Natural', 1), ('language', 1), ('processing', 1)]

# 生成词云
text_for_wordcloud = " ".join(filtered_tokens)
wordcloud = WordCloud(width=800, height=400).generate(text_for_wordcloud)

plt.figure(figsize=(10, 5))
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.show()

from textblob import TextBlob
nltk.download('averaged_perceptron_tagger')  # 第一次使用需要下载数据

feedback = "I love this product. It's amazing!"
analysis = TextBlob(feedback)
print(f"情感极性: {analysis.sentiment.polarity}")  # 范围从-1到1
print(f"主观性: {analysis.sentiment.subjectivity}")  # 范围从0到1

# 中文情感分析示例（需要先翻译或使用中文专用库）
chinese_feedback = "这个产品太糟糕了，我非常失望"
# 实际应用中应使用SnowNLP等中文库

from sklearn.feature_extraction.text import TfidfVectorizer

documents = [
    "Python is a popular programming language",
    "Java is another programming language",
    "Python and Java are both object-oriented"
]

vectorizer = TfidfVectorizer()
X = vectorizer.fit_transform(documents)
print(vectorizer.get_feature_names_out())  # 输出特征词
print(X.shape)  # 文档-词矩阵形状