Python通过计算图片MD5值实现图片去重

base64_str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="

import hashlib

def md5_from_base64(base64_str):
    clean_str = base64_str.strip().replace('\n', '').replace(' ', '')
    return hashlib.md5(clean_str.encode()).hexdigest()

md5 = md5_from_base64(base64_str)
print(md5)  # a1b2c3d4e5f6...

import hashlib
import base64

def md5_from_binary(base64_str):
    # 1. base64 → 二进制
    image_data = base64.b64decode(base64_str)
    # 2. 二进制 → MD5
    return hashlib.md5(image_data).hexdigest()

md5 = md5_from_binary(base64_str)
print(md5)

import hashlib

def md5_from_file(file_path):
    md5 = hashlib.md5()
    with open(file_path, 'rb') as f:
        for chunk in iter(lambda: f.read(8192), b''):
            md5.update(chunk)
    return md5.hexdigest()

md5 = md5_from_file('photo.jpg')
print(md5)

data:image/png;base64,iVBORw0KGgoAAAANSUhEUg...

def extract_base64(s):
    if ',' in s:
        return s.split(',', 1)[1]  # 只分割一次
    return s

clean_b64 = extract_base64(base64_str)
md5 = md5_from_binary(clean_b64)

# 来源A（干净）
"iVBORw0KGgoAAAANSUhEUg..."

# 来源B（带换行）
"iVBORw0KGgoAAAANSUhEUg...
AAAAAAAAAAAAAAAAA..."

# pip install imagehash
import imagehash
from PIL import Image
import io
import base64

def phash_from_base64(base64_str):
    image_data = base64.b64decode(base64_str)
    img = Image.open(io.BytesIO(image_data))
    return str(imagehash.phash(img))  # 返回感知哈希值

# 即使压缩过，pHash 也可能相同！

import hashlib
import base64
import json

class ImageDeduplicator:
    """图片去重管理器"""
    
    def __init__(self):
        self.seen = {}  # {md5: count} 记录见过的图片
    
    def get_md5(self, base64_str):
        """计算图片MD5（推荐方式）"""
        # 去除前缀
        if ',' in base64_str:
            base64_str = base64_str.split(',', 1)[1]
        # base64 → 二进制 → MD5
        image_data = base64.b64decode(base64_str)
        return hashlib.md5(image_data).hexdigest()
    
    def check(self, base64_str, label=""):
        """
        检查图片是否重复
        返回: (是否重复, md5值)
        """
        md5 = self.get_md5(base64_str)
        
        if md5 in self.seen:
            self.seen[md5] += 1
            return True, md5, self.seen[md5]
        else:
            self.seen[md5] = 1
            return False, md5, 1
    
    def save(self, filepath='seen_images.json'):
        """保存去重记录"""
        with open(filepath, 'w') as f:
            json.dump(self.seen, f, indent=2)
    
    def load(self, filepath='seen_images.json'):
        """加载去重记录"""
        try:
            with open(filepath, 'r') as f:
                self.seen = json.load(f)
        except FileNotFoundError:
            self.seen = {}


# ==================== 使用示例 ====================

dedup = ImageDeduplicator()

# 模拟3张图片（第1张和第3张是同一张）
images = [
    ("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", "图片A"),
    ("data:image/png;base64,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "图片B"),
    ("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", "图片A（重复）"),
]

for b64, name in images:
    is_dup, md5, count = dedup.check(b64, name)
    status = "🔁 重复" if is_dup else "✅ 新图片"
    print(f"{name}: {status} | MD5: {md5} | 已出现{count}次")

# 保存记录
dedup.save()