Python检测两个文本文件相似性的三种方法

import Levenshtein

def similarity_levenshtein(text1, text2):
    distance = Levenshtein.distance(text1, text2)
    max_len = max(len(text1), len(text2))
    return 1 - (distance / max_len)

# 读取文件
with open("file1.txt", "r") as f1, open("file2.txt", "r") as f2:
    text1 = f1.read()
    text2 = f2.read()

similarity = similarity_levenshtein(text1, text2)
print(f"Similarity (Levenshtein): {similarity:.2f}")

def similarity_jaccard(text1, text2):
    set1 = set(text1.split())
    set2 = set(text2.split())
    intersection = set1.intersection(set2)
    union = set1.union(set2)
    return len(intersection) / len(union)

# 读取文件
with open("file1.txt", "r") as f1, open("file2.txt", "r") as f2:
    text1 = f1.read()
    text2 = f2.read()

similarity = similarity_jaccard(text1, text2)
print(f"Similarity (Jaccard): {similarity:.2f}")

import Levenshtein

def compare_text_files_edit_distance(file1_path, file2_path):
    try:
        with open(file1_path, 'r', encoding='utf-8') as file1:
            text1 = file1.read()
        with open(file2_path, 'r', encoding='utf-8') as file2:
            text2 = file2.read()

        distance = Levenshtein.distance(text1, text2)
        max_length = max(len(text1), len(text2))
        similarity = 1 - (distance / max_length)
        return similarity
    except FileNotFoundError:
        print("错误: 文件未找到!")
    except Exception as e:
        print(f"错误: 发生了一个未知错误: {e}")
    return None

if __name__ == "__main__":
    file1_path = 'file1.txt'
    file2_path = 'file2.txt'
    similarity = compare_text_files_edit_distance(file1_path, file2_path)
    if similarity is not None:
        print(f"两个文件基于编辑距离的相似度为: {similarity:.2f}")
    

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics.pairwise import cosine_similarity

def similarity_cosine(text1, text2):
    vectorizer = CountVectorizer().fit_transform([text1, text2])
    vectors = vectorizer.toarray()
    return cosine_similarity([vectors[0]], [vectors[1]])[0][0]

# 读取文件
with open("file1.txt", "r") as f1, open("file2.txt", "r") as f2:
    text1 = f1.read()
    text2 = f2.read()

similarity = similarity_cosine(text1, text2)
print(f"Similarity (Cosine): {similarity:.2f}")

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

def similarity_tfidf(text1, text2):
    vectorizer = TfidfVectorizer().fit_transform([text1, text2])
    vectors = vectorizer.toarray()
    return cosine_similarity([vectors[0]], [vectors[1]])[0][0]

# 读取文件
with open("file1.txt", "r") as f1, open("file2.txt", "r") as f2:
    text1 = f1.read()
    text2 = f2.read()

similarity = similarity_tfidf(text1, text2)
print(f"Similarity (TF-IDF): {similarity:.2f}")

from gensim.models import KeyedVectors
import numpy as np

# 加载预训练的词向量模型
word2vec_model = KeyedVectors.load_word2vec_format("path/to/word2vec.bin", binary=True)

def text_to_vector(text):
    words = text.split()
    vectors = [word2vec_model[word] for word in words if word in word2vec_model]
    return np.mean(vectors, axis=0) if vectors else np.zeros(word2vec_model.vector_size)

def similarity_word2vec(text1, text2):
    vec1 = text_to_vector(text1)
    vec2 = text_to_vector(text2)
    return np.dot(vec1, vec2) / (np.linalg.norm(vec1) * np.linalg.norm(vec2))

# 读取文件
with open("file1.txt", "r") as f1, open("file2.txt", "r") as f2:
    text1 = f1.read()
    text2 = f2.read()

similarity = similarity_word2vec(text1, text2)
print(f"Similarity (Word2Vec): {similarity:.2f}")

from transformers import BertTokenizer, BertModel
import torch
import numpy as np

# 加载预训练的 BERT 模型和分词器
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertModel.from_pretrained('bert-base-uncased')

def text_to_bert_vector(text):
    inputs = tokenizer(text, return_tensors='pt', truncation=True, padding=True)
    outputs = model(**inputs)
    return outputs.last_hidden_state.mean(dim=1).detach().numpy()

def similarity_bert(text1, text2):
    vec1 = text_to_bert_vector(text1)
    vec2 = text_to_bert_vector(text2)
    return np.dot(vec1, vec2.T) / (np.linalg.norm(vec1) * np.linalg.norm(vec2))

# 读取文件
with open("file1.txt", "r") as f1, open("file2.txt", "r") as f2:
    text1 = f1.read()
    text2 = f2.read()

similarity = similarity_bert(text1, text2)
print(f"Similarity (BERT): {similarity:.2f}")