使用Python实现文本词频分析

with open('规划.txt', 'r', encoding='utf-8') as f:
    # encoding 参数需要根据文本文件实际的编码来设置
    Text = f.read()  # 读取全部内容
f.close()            # 关闭文件，必要的操作
# 输出正文的前 200 个字符
Text[2024:2224]

import re
# 使用正则表达式去除文本中所有的换行符和空白符
Text = re.sub('\s+', '', Text) 

import jieba
jieba.add_word('特色词')

# Special_words.txt 是特色词语的路径
jieba.load_userdict('Special_words.txt')

Words_list = jieba.lcut(Text)  # 进行分词，结果为词语列表
print(len(Words_list)) # 输出词语总数量，结果为 32789
Words_list[:10]  # 输出前 10 个词语

Stopwords = open('./停用词/baidu_stopwords.txt', 'r', encoding='utf-8').readlines()
Stopwords = [word.strip() for word in Stopwords]

Words_list = [word for word in Words_list if word not in Stopwords]
# 输出去停用词后的总词数
print(len(Words_list)) # 输出：28337, 较去除之前减少了 4452 个词语

# 去除不含中文的词语
Words_list = [word for word in Words_list if bool(re.search('[\u4e00-\u9fa5]', word)) == True]
# 去除字数为 1 的词语
Words_list = [word for word in Words_list if len(word) != 1]
# 输出剩余词语的数量
print(len(Words_list))    # 输出：21913

from collections import Counter # 导入频率统计函数
Word_count = Counter(Words_list) # 统计词频，所得结果为 Counter 对象
Word_count = dict(Word_count) # 将结果转为字典对象
Word_count

import pandas as pd
Table = pd.DataFrame(columns=['词语', '词频']) # 生成空表
Table['词语'] = list(Word_count.keys())   # 词语字段保存词语
Table['词频'] = list(Word_count.values()) # 词频字段保存词语对应的词频
Table = Table.sort_values(by=['词频'], ascending=False) # 按照词频降序排序
Table.to_excel('词频统计表.xlsx', index=False) # 将结果保存为 excel 表

# 读取特色词库为词列表
Specialwords = open('Special_words.txt', 'r', encoding='utf-8').readlines()
Specialwords = [word.strip() for word in Stopwords]
# 获取分词处理结果中所有的特色词，这个结果可以直接用来制作词云图
Special_words = [word for word in Words_list if word in Specialwords]
# 对特色词做词频统计
Special_count = dict(Counter(Special_words))
# 生成词频统计表并保存
Table_special = pd.DataFrame(columns=['特色词语', '词频'])
Table_special['特色词语'] = list(Special_count.keys())
Table_special['词频'] = list(Special_count.values())
Table_special = Table_special.sort_values(by=['词频'], ascending=False)
Table_special.to_excel('词频统计表.xlsx', index=False)

# 导入工具包
import re, jieba
import pandas as pd
from collections import Counter
# 读取文本
with open('规划.txt', 'r', encoding='utf-8') as f:
    # encoding 参数需要根据文本文件实际的编码来设置，也可以
    # 事先对 txt 进行转码
    Text = f.read()  # 读取全部内容
f.close()            # 关闭文件，必要的操作
# 使用正则表达式去除文本中所有的换行符和空白符
Text = re.sub('\s+', '', Text)
# 导入特色词库
jieba.load_userdict('Special_words.txt')
# 进行分词，所得结果为词语列表
Words_list = jieba.lcut(Text)
# 加载停用词、去除停用词
Stopwords = open('./停用词/baidu_stopwords.txt', 'r', encoding='utf-8').readlines()
Stopwords = [word.strip() for word in Stopwords]
Words_list = [word for word in Words_list if word not in Stopwords]
# 去除不含中文的词语
Words_list = [word for word in Words_list if bool(re.search('[\u4e00-\u9fa5]', word)) == True]
# 去除字数为 1 的词语
Words_list = [word for word in Words_list if len(word) != 1]
# 统计词频 
Word_count = dict(Counter(Words_list))
# 生成词频统计表
Table = pd.DataFrame(columns=['词语', '词频']) # 生成空表
Table['词语'] = list(Word_count.keys())   # 词语字段保存词语
Table['词频'] = list(Word_count.values()) # 词频字段保存词语对应的词频
Table = Table.sort_values(by=['词频'], ascending=False) # 按照词频降序排序
Table.to_excel('词频统计表.xlsx', index=False) # 将结果保存为 excel 表