[Model] Adding support for MiniCPM-V by HwwwwwwwH · Pull Request #4087 · vllm-project/vllm (original) (raw)

By the way, how can i use minicpmv2-6's fewshot feature wtih VLLM structure.

here is official fewshot feature usage with transformers:

import torch
from PIL import Image
from transformers import AutoModel, AutoTokenizer

model = AutoModel.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True,
   attn_implementation='sdpa', torch_dtype=torch.bfloat16) # sdpa or flash_attention_2, no eager
model = model.eval().cuda()
tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True)

question = "production date" 
image1 = Image.open('example1.jpg').convert('RGB')
answer1 = "2023.08.04"
image2 = Image.open('example2.jpg').convert('RGB')
answer2 = "2007.04.24"
image_test = Image.open('test.jpg').convert('RGB')

msgs = [
   {'role': 'user', 'content': [image1, question]}, {'role': 'assistant', 'content': [answer1]},
   {'role': 'user', 'content': [image2, question]}, {'role': 'assistant', 'content': [answer2]},
   {'role': 'user', 'content': [image_test, question]}
]

answer = model.chat(
   image=None,
   msgs=msgs,
   tokenizer=tokenizer
)
print(answer)

I think this could work

msgs = [ {'role': 'user', 'content': "(./)" + question}, {'role': 'assistant', 'content': answer1}, {'role': 'user', 'content': "(./)" + question}, {'role': 'assistant', 'content': answer2}, {'role': 'user', 'content': "(./)" + question} ] prompt = tokenizer.apply_chat_template( msgs, tokenize=False, add_generation_prompt=True ) inputs = { "prompt": prompt, "multi_modal_data": { "image": [image1, image2, image_test] }, }