Python实现轻松找出文本文件中的高频单词

import re

def read_and_preprocess(file_path):
    with open(file_path, 'r', encoding='utf-8') as file:
        text = file.read()
        text = re.sub(r'[^\w\s]', '', text)  # 去除标点符号
        text = text.lower()  # 转换为小写
    return text

from collections import Counter

def calculate_word_frequency(text):
    words = text.split()
    word_frequency = Counter(words)
    return word_frequency

def find_most_common_word(word_frequency):
    most_common_word = word_frequency.most_common(1)
    return most_common_word[0][0] if most_common_word else None

def main(file_path):
    # 读取并预处理文本
    text = read_and_preprocess(file_path)

    # 计算单词频率
    word_frequency = calculate_word_frequency(text)

    # 找到最高频率的单词
    most_common_word = find_most_common_word(word_frequency)

    print(f"The most common word is: {most_common_word}")

if __name__ == "__main__":
    file_path = "your_text_file.txt"
    main(file_path)

def words_generator(text):
    return (word for word in text.split())

import string

def remove_punctuation(text):
    translator = str.maketrans("", "", string.punctuation)
    return text.translate(translator)

from concurrent.futures import ThreadPoolExecutor

def parallel_word_frequency(text):
    parts = text.split('\n')  # 将文本拆分成多个部分
    with ThreadPoolExecutor() as executor:
        word_frequencies = executor.map(calculate_word_frequency, parts)
    return sum(word_frequencies, Counter())

def read_large_file(file_path, chunk_size=1024):
    with open(file_path, 'r', encoding='utf-8') as file:
        while True:
            chunk = file.read(chunk_size)
            if not chunk:
                break
            yield chunk

def process_large_file_line_by_line(file_path):
    with open(file_path, 'r', encoding='utf-8') as file:
        for line in file:
            process_line(line)

import matplotlib.pyplot as plt

def plot_word_frequency(word_frequency):
    words, frequencies = zip(*word_frequency.items())
    
    plt.bar(words, frequencies)
    plt.xlabel('Words')
    plt.ylabel('Frequency')
    plt.title('Word Frequency Distribution')
    plt.xticks(rotation=45, ha='right')
    plt.show()

from wordcloud import WordCloud

def generate_wordcloud(text):
    wordcloud = WordCloud(width=800, height=400, background_color='white').generate(text)
    
    plt.figure(figsize=(10, 5))
    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis('off')
    plt.show()