python jieba库的基本使用

pip install jieba

>>> jieba.lcut("中国是一个伟大的国家")
['中国', '是', '一个', '伟大', '的', '国家']

>>> jieba.lcut("中国是一个伟大的国家", cut_all=True)
['中国', '国是', '一个', '伟大', '的', '国家']

>>> jieba.lcut_for_search("中华人民共和国是伟大的")
['中华', '华人', '人民', '共和', '共和国', '中华人民共和国', '是', '伟大', '的']

def getText():
 # 打开 hamlet.txt 这个文件
 txt = open("hamlet.txt", "r").read()
 # 避免大小写对词频统计的干扰，将所有单词转换为小写
 txt = txt.lower()
 # 将文中出现的所有特殊字符替换为空格
 for ch in '|"#$%^&*()_+-=\\`~{}[];:<>?/':
 txt = txt.replace(ch, " ")
 # 返回一个所以后单词都是小写的，单词间以空格间隔的文本
 return txt

hamletTxt = getText()
# split() 默认使用空格作为分隔符
words = hamletTxt.split()
counts = {}
for word in words:
 counts[word] = counts.get(word,0) + 1
items = list(counts.items())
items.sort(key=lambda x:x[1], reverse=True)
for i in range(10):
 word, count = items[i]
 print("{0:<10}{1:>5}".format(word,count))

items.sort(key=lambda x:x[1], reverse=True)

import jieba
txt = open("threekingdoms.txt","r",encoding="utf-8").read()
words = jieba.lcut(txt)
counts = {}
for word in words:
 if len(word) == 1:
 continue
 else:
 counts[word] = counts.get(word, 0) + 1

items = list(counts.items())
items.sort(key=lambda x:x[1], reverse=True)
for i in range(15):
 word, count = items[i]
 print("{0:<10}{1:>5}".format(word,count))

曹操  953
孔明  836
将军  772
却说  656
玄德  585
关公  510
丞相  491
二人  469
不可  440
荆州  425
玄德曰  390
孔明曰  390
不能  384
如此  378
张飞  358

import jieba
txt = open("threekingdoms.txt","r",encoding="utf-8").read()
excludes = {"将军","却说","荆州","二人","不可","不能","如此"}
words = jieba.lcut(txt)
counts = {}
for word in words:
 if len(word) == 1:
 continue
 elif word == "诸葛亮" or word == "孔明曰":
 rword == "孔明"
 elif word == "关公" or word == "云长":
 rword == "关羽"
 elif word == "玄德" or word == "玄德曰":
 rword == "刘备"
 elif word == "孟德" or word == "丞相":
 rword == "曹操"
 else:
 rword = word
 counts[rword] = counts.get(rword, 0) + 1

items = list(counts.items())
items.sort(key=lambda x:x[1], reverse=True)
for i in range(15):
 word, count = items[i]
 print("{0:<10}{1:>5}".format(word,count))

曹操  963
孔明  847
张飞  366
商议  359
如何  352
主公  340
军士  320
吕布  303
左右  298
军马  297
赵云  283
刘备  282
引兵  279
次日  278
大喜  274