Python中自然语言处理和文本挖掘的常规操作详解

import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
import string

# 下载 NLTK 数据
nltk.download('punkt')
nltk.download('stopwords')

# 示例文本
text = "This is a sample text for natural language processing. It includes punctuation and stopwords."

# 分词
tokens = word_tokenize(text)

# 去除标点符号和停用词
stop_words = set(stopwords.words('english'))
filtered_tokens = [word for word in tokens if word.lower() not in stop_words and word not in string.punctuation]

print(filtered_tokens)

from nltk import pos_tag

# 词性标注
tagged = pos_tag(filtered_tokens)
print(tagged)

from nltk import ne_chunk

# 命名实体识别
entities = ne_chunk(tagged)
print(entities)

from textblob import TextBlob

# 示例文本
text = "I love this product! It is amazing."
blob = TextBlob(text)

# 情感分析
sentiment = blob.sentiment
print(sentiment)

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation

# 示例文本
documents = ["This is a sample document.", "Another document for NLP.", "Text mining is fun."]

# 向量化
vectorizer = CountVectorizer(stop_words='english')
X = vectorizer.fit_transform(documents)

# 主题建模
lda = LatentDirichletAllocation(n_components=2, random_state=42)
lda.fit(X)

# 输出主题
for topic_idx, topic in enumerate(lda.components_):
    print(f"Topic {topic_idx}:")
    print(" ".join([vectorizer.get_feature_names_out()[i] for i in topic.argsort()[:-11:-1]]))

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import make_pipeline

# 示例数据
texts = ["I love this product!", "This is a bad product.", "I am happy with the service."]
labels = [1, 0, 1]  # 1 表示正面，0 表示负面

# 创建分类器
model = make_pipeline(TfidfVectorizer(), MultinomialNB())

# 训练模型
model.fit(texts, labels)

# 预测
predicted_labels = model.predict(["I am very satisfied with the product."])
print(predicted_labels)

from sklearn.cluster import KMeans

# 向量化
vectorizer = TfidfVectorizer(stop_words='english')
X = vectorizer.fit_transform(documents)

# 聚类
kmeans = KMeans(n_clusters=2, random_state=42)
kmeans.fit(X)

# 输出聚类结果
print(kmeans.labels_)

from rake_nltk import Rake

# 示例文本
text = "Natural language processing is a field of study that focuses on the interactions between computers and human language."

# 关键词提取
rake = Rake()
rake.extract_keywords_from_text(text)
keywords = rake.get_ranked_phrases()
print(keywords)

from gensim.summarization import summarize

# 示例文本
text = "Natural language processing is a field of study that focuses on the interactions between computers and human language. It involves various tasks such as text classification, sentiment analysis, and machine translation."

# 文本摘要
summary = summarize(text)
print(summary)