从基础到高级全解析Python文本相似度计算

def jaccard_similarity(str1, str2):
    set1 = set(str1.split())
    set2 = set(str2.split())
    intersection = len(set1 & set2)
    union = len(set1 | set2)
    return intersection / union if union != 0 else 0

text1 = "Python is a programming language"
text2 = "Python is used for data science"
print(jaccard_similarity(text1, text2))  # 输出: 0.375

def levenshtein_distance(s1, s2):
    if len(s1) < len(s2):
        return levenshtein_distance(s2, s1)
    if len(s2) == 0:
        return len(s1)
    prev_row = range(len(s2) + 1)
    for i, c1 in enumerate(s1):
        curr_row = [i + 1]
        for j, c2 in enumerate(s2):
            insertions = prev_row[j + 1] + 1
            deletions = curr_row[j] + 1
            substitutions = prev_row[j] + (c1 != c2)
            curr_row.append(min(insertions, deletions, substitutions))
        prev_row = curr_row
    return prev_row[-1]

def normalized_levenshtein(s1, s2):
    distance = levenshtein_distance(s1, s2)
    max_len = max(len(s1), len(s2))
    return 1 - distance / max_len if max_len != 0 else 0

text1 = "kitten"
text2 = "sitting"
print(normalized_levenshtein(text1, text2))  # 输出: 0.571

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

texts = [
    "Python is a programming language",
    "Python is used for data science",
    "Java is another programming language"
]

vectorizer = TfidfVectorizer()
tfidf_matrix = vectorizer.(texts)

# 计算两两之间的相似度
similarity_matrix = cosine_similarity(tfidf_matrix)
print(similarity_matrix)

# 计算text1和text2的相似度
text1 = texts[0]
text2 = texts[1]
vec1 = vectorizer.transform([text1])
vec2 = vectorizer.transform([text2])
print(cosine_similarity(vec1, vec2)[0][0])  # 输出: 0.5

from gensim.models import Word2Vec
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

# 训练Word2Vec模型（示例数据）
sentences = [
    "Python is a programming language".split(),
    "Python is used for data science".split(),
    "Java is another programming language".split()
]
model = Word2Vec(sentences, vector_size=100, window=5, min_count=1, workers=4)

def sentence_vector(sentence, model):
    vectors = [model.wv[word] for word in sentence if word in model.wv]
    return np.mean(vectors, axis=0) if vectors else np.zeros(model.vector_size)

vec1 = sentence_vector("Python is a programming language".split(), model)
vec2 = sentence_vector("Python is used for data science".split(), model)
print(cosine_similarity([vec1], [vec2])[0][0])  # 输出: ~0.7

from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity

model = SentenceTransformer('all-MiniLM-L6-v2')  # 轻量级模型

text1 = "Python is a programming language"
text2 = "Python is used for data science"

embeddings = model.encode([text1, text2])
similarity = cosine_similarity([embeddings[0]], [embeddings[1]])[0][0]
print(similarity)  # 输出: ~0.8

from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity

# 加载预训练模型
model = SentenceTransformer('all-MiniLM-L6-v2')

# 待比较的文本
texts = [
    "Python is a programming language",
    "Python is used for data science",
    "Java is another programming language",
    "I love coding in Python"
]

# 计算所有文本对的相似度
embeddings = model.encode(texts)
similarity_matrix = cosine_similarity(embeddings)

# 打印相似度矩阵
for i in range(len(texts)):
    for j in range(len(texts)):
        print(f"Similarity('{texts[i]}', '{texts[j]}'): {similarity_matrix[i][j]:.2f}")