Python中实现文本预处理的方法小结

import re

def remove_special_characters(text):
    pattern = r'[^a-zA-Z0-9\s]'
    return re.sub(pattern, '', text)

text = "Hello, world! This is an example text with @special characters."
cleaned_text = remove_special_characters(text)
print(cleaned_text)

def convert_to_lowercase(text):
    return text.lower()

lowercased_text = convert_to_lowercase(text)
print(lowercased_text)

import nltk
from nltk.tokenize import word_tokenize

nltk.download('punkt')

def tokenize_text(text):
    return word_tokenize(text)

tokenized_text = tokenize_text(text)
print(tokenized_text)

from nltk.corpus import stopwords

nltk.download('stopwords')

def remove_stopwords(tokens):
    stop_words = set(stopwords.words('english'))
    return [word for word in tokens if word.lower() not in stop_words]

filtered_tokens = remove_stopwords(tokenized_text)
print(filtered_tokens)

from nltk.stem import PorterStemmer

def stem_words(tokens):
    stemmer = PorterStemmer()
    return [stemmer.stem(word) for word in tokens]

stemmed_words = stem_words(filtered_tokens)
print(stemmed_words)

from nltk.stem import WordNetLemmatizer

nltk.download('wordnet')

def lemmatize_words(tokens):
    lemmatizer = WordNetLemmatizer()
    return [lemmatizer.lemmatize(word) for word in tokens]

lemmatized_words = lemmatize_words(filtered_tokens)
print(lemmatized_words)

from sklearn.feature_extraction.text import CountVectorizer

corpus = ["This is the first document.",
          "This document is the second document.",
          "And this is the third one."]

vectorizer = CountVectorizer()
X = vectorizer.fit_transform(corpus)

print(vectorizer.get_feature_names())
print(X.toarray())

from sklearn.feature_extraction.text import TfidfVectorizer

tfidf_vectorizer = TfidfVectorizer()
X_tfidf = tfidf_vectorizer.fit_transform(corpus)

print(vectorizer.get_feature_names())
print(X_tfidf.toarray())