Python中CLIP多模态模型的库的实现

pip install git+https://github.com/openai/CLIP.git

import clip
import torch
from PIL import Image

# 加载模型和预处理方法
device = "cuda" if torch.cuda.is_available() else "cpu"
model, preprocess = clip.load("ViT-B/32", device=device)

# 加载图像并预处理
image = preprocess(Image.open("cat.jpg")).unsqueeze(0).to(device)

# 编写文本描述
text = clip.tokenize(["a photo of a cat", "a photo of a dog"]).to(device)

# 提取特征并计算相似度
with torch.no_grad():
    image_features = model.encode_image(image)
    text_features = model.encode_text(text)
    logits_per_image, logits_per_text = model(image, text)
    probs = logits_per_image.softmax(dim=-1).cpu().numpy()

print("Label probabilities:", probs)

text = ["a photo of a banana", "a dog", "a car"]
tokens = clip.tokenize(text).to(device)

with torch.no_grad():
    text_features = model.encode_text(tokens)

from PIL import Image

image = Image.open("example.jpg")
image_input = preprocess(image).unsqueeze(0).to(device)

with torch.no_grad():
    image_features = model.encode_image(image_input)

import torch.nn.functional as F

# 余弦相似度
similarity = F.cosine_similarity(image_features, text_features)
print(similarity)

labels = ["a dog", "a cat", "a car"]
text_inputs = clip.tokenize([f"a photo of {label}" for label in labels]).to(device)

with torch.no_grad():
    text_features = model.encode_text(text_inputs)
    image_features = model.encode_image(image)

# 归一化
image_features /= image_features.norm(dim=-1, keepdim=True)
text_features /= text_features.norm(dim=-1, keepdim=True)

# 相似度得分
logits = (image_features @ text_features.T)
pred = logits.argmax().item()

print(f"Predicted label: {labels[pred]}")

pip install open_clip_torch

import open_clip

model, preprocess, tokenizer = open_clip.create_model_and_transforms('ViT-B-32', pretrained='laion2b_s34b_b79k')