Python实现文本相似度分析

import math
class Vector:
    """笛卡尔坐标系向量"""
    def __init__(self, a):
        """构造函数：切片拷贝列表参数a到对象实例变量_coords"""
        self._coords = a[:] # 坐标列表
        self._n = len(a) # 维度
    def __getitem__(self, i):
        """返回第i个元素，即第i维坐标"""
        return self._coords[i]
    def __add__(self, other):
        """返回2个向量之和"""
        result = []
        for i in range(self._n):
            result.append(self._coords[i] + other._coords[i])
        return Vector(result)
    def __sub__(self, other):
        """返回2个向量之差"""
        result = []
        for i in range(self._n):
            result.append(self._coords[i] - other._coords[i])
        return Vector(result)
    def scale(self, n):
        """返回向量与数值的乘积差"""
        result = []
        for i in range(self._n):
            result.append(self._coords[i] * n)
        return Vector(result)
    def dot(self, other):
        """返回2向量的内积"""
        result = 0
        for i in range(self._n):
            result += self._coords[i] * other._coords[i]
        return result
    def __abs__(self):
        """返回向量的模"""
        return math.sqrt(self.dot(self))
    def direction(self):
        """返回向量的单位向量"""
        return self.scale(1.0 / abs(self))
    def __str__(self):
        """返回向量的字符串表示"""
        return str(self._coords)
    def __len__(self):
        """返回向量的维度"""
        return self._n
#测试代码
def main():
    xCoords = [2.0, 2.0, 2.0]
    yCoords = [5.0, 5.0, 0.0]
    x = Vector(xCoords)
    y = Vector(yCoords)
    print('x = {}, y = {}'.format(x, y))
    print('x + y = {}'.format(x + y))
    print('10x = {}'.format(x.scale(10.0)))
    print('|x| = {}'.format(abs(x)))
    print(' = {}'.format(x.dot(y)))
    print('|x-y| = {}'.format(abs(x-y)))
if __name__ == '__main__': main()

import sys
from vector import Vector
class Sketch:
    """计算文本text的k-grams的文档摘要向量（d维）"""
    def __init__(self, text, k, d):
        """初始化函数：计算文本text的文档摘要向量"""
        freq = [0 for i in range(d)] #创建长度为d的列表，初始值0
        for i in range(len(text) - k): #循环抽取k-grams，计算频率
            kgram = text[i:i+k]
            freq[hash(kgram) % d] += 1
        vector = Vector(freq) #创建文档摘要向量
        self._sketch = vector.direction() #归一化并赋值给对象实例变量
    def similarTo(self, other):
        """比较两个文档摘要对象Sketch的余弦相似度"""
        return self._sketch.dot(other._sketch)
    def __str__(self):
        return str(self._sketch)
#测试代码
def main():
    with open("tomsawyer.txt","r") as f:
        text = f.read()
        sketch = Sketch(text, 5, 100)
        print(sketch)
if __name__ == '__main__': main()

import sys
from vector import Vector
from sketch import Sketch
#测试文档列表
filenames = [ 'gene.txt', 'pride.txt', 'tomsawyer.txt']
k = 5    #k-grams
d = 100000 #文档摘要向量维度
sketches = [0 for i in filenames]
for i in range(len(filenames)):
    with open(filenames[i], 'r') as f:
        text = f.read()
        sketches[i] = Sketch(text, k, d)
#输出结果标题
print(' '*15, end='')
for filename in filenames:
    print('{:>22}'.format(filename), end='')
print()
#输出结果比较明细
for i in range(len(filenames)):
    print('{:15}'.format(filenames[i]), end='')
    for j in range(len(filenames)):
        print('{:22}'.format(sketches[i].similarTo(sketches[j])), end='')
    print()