通過ModelScope開源多模態Embedding模型進行向量產生

from modelscope.utils.constant import Tasks
from modelscope.pipelines import pipeline
from modelscope.preprocessors.image import load_image
from typing import List
from dashvector import Client


pipeline = pipeline(task=Tasks.multi_modal_embedding, model='{model_id}')


def generate_text_embeddings(texts: List[str]):
    inputs = {'text': texts}
    result = pipeline.forward(input=inputs)
    return result['text_embedding'].numpy()


def generate_img_embeddings(img: str):
    input_img = load_image(img)
    inputs = {'img': input_img}
    result = pipeline.forward(input=inputs)
    return result['img_embedding'].numpy()[0]


# 建立DashVector Client
client = Client(
    api_key='{your-dashvector-api-key}',
    endpoint='{your-dashvector-cluster-endpoint}'
)

# 建立DashVector Collection
rsp = client.create('CLIP-embedding', dimension={model_dim})
assert rsp
collection = client.get('CLIP-embedding')
assert collection

# 向量入庫DashVector
collection.insert(
    [
        ('ID1', generate_text_embeddings(['阿里雲向量檢索服務DashVector是效能、性價比具佳的向量資料庫之一'])[0]),
        ('ID2', generate_img_embeddings('https://clip-cn-beijing.oss-cn-beijing.aliyuncs.com/pokemon.jpeg'))
    ]
)

# 向量檢索
docs = collection.query(
    generate_text_embeddings(['The best vector database'])[0]
)
print(docs)