Python基于文本内容实现隐私信息提取与评估

pip install transformers scikit-learn numpy pandas spacy
python -m spacy download en_core_web_sm

import numpy as np
import pandas as pd
from transformers import BertTokenizer, BertModel
from sklearn.cluster import KMeans
import spacy
from collections import Counter
import math

def get_bert_embeddings(texts, model_name='bert-base-uncased'):
    tokenizer = BertTokenizer.from_pretrained(model_name)
    model = BertModel.from_pretrained(model_name)
    inputs = tokenizer(texts, return_tensors='pt', padding=True, truncation=True, max_length=512)
    outputs = model(**inputs)
    embeddings = outputs.last_hidden_state.mean(dim=1).detach().numpy()
    return embeddings

def cluster_texts(embeddings, n_clusters=5):
    kmeans = KMeans(n_clusters=n_clusters)
    kmeans.fit(embeddings)
    return kmeans.labels_

def extract_private_info(texts):
    nlp = spacy.load("en_core_web_sm")
    private_info = []
    for text in texts:
        doc = nlp(text)
        entities = [ent.text for ent in doc.ents if ent.label_ in ['PERSON', 'GPE', 'ORG', 'DATE']]
        private_info.append(entities)
    return private_info

def calculate_entropy(private_info):
    all_entities = [entity for sublist in private_info for entity in sublist]
    entity_counts = Counter(all_entities)
    total_entities = len(all_entities)
    entropy = 0.0
    for count in entity_counts.values():
        probability = count / total_entities
        entropy -= probability * math.log(probability, 2)
    return entropy

def assess_privacy_risk(texts):
    # Step 1: Get BERT embeddings
    embeddings = get_bert_embeddings(texts)
    
    # Step 2: Cluster texts
    labels = cluster_texts(embeddings)
    
    # Step 3: Extract private information
    private_info = extract_private_info(texts)
    
    # Step 4: Calculate information entropy
    entropy = calculate_entropy(private_info)
    
    # Step 5: Assess privacy risk based on entropy
    if entropy > 2.0:
        return "High Privacy Risk"
    elif entropy > 1.0:
        return "Medium Privacy Risk"
    else:
        return "Low Privacy Risk"

if __name__ == "__main__":
    # Example texts
    texts = [
        "My name is John Doe and I live in New York.",
        "I work at Google and my birthday is on 1990-01-01.",
        "The meeting is scheduled for next Monday at 10 AM.",
        "Alice and Bob are working on the project together."
    ]
    
    # Assess privacy risk
    risk_level = assess_privacy_risk(texts)
    print(f"Privacy Risk Level: {risk_level}")