。

主要解决2.6版本默认绑定flash_atten问题#xff0c;pip

install

flash_attn也无法安装#xff0c;因为强制依赖cuda。

主要解决的就是这个问题#xff0c;还…Mac平台M1PRO芯片MiniCPM-V-2.6网页部署跑通

2.6的小钢炮可以输入视频了我必须拉到本地跑跑。

主要解决2.6版本默认绑定flash_atten问题pip

install

flash_attn也无法安装因为强制依赖cuda。

主要解决的就是这个问题还有

BFloat16

pro代码仓库https://github.com/OpenBMB/MiniCPM-V.git分支main代码版本b0125d8a

yiranyyu

[https://github.com/OpenBMB/MiniCPM-V.git](https://github.com/OpenBMB/MiniCPM-V.git)

#modelscope需要手动安装

http://thunlp.oss-cn-qingdao.aliyuncs.com/multi_modal/never_delete/modelscope_studio-0.4.0.9-py3-none-any.whl

#dcord如果安装有问题参考我LAVIS博客#找到根目录web_demo_2.6.py运行

#首先添加环境变量mps参数见下图

PYTORCH_ENABLE_MPS_FALLBACK1#第一次运行web_demo_2.6.py报错如下

ImportError:

transformers.dynamic_module_utils

import

imports:imports.remove(flash_attn)return

imports#79行左右修改为

patch(transformers.dynamic_module_utils.get_imports,

AutoModel.from_pretrained(model_path,

torch_dtypetorch.bfloat16)model

#!/usr/bin/env

transformers.dynamic_module_utils

import

argparse.ArgumentParser(descriptiondemo)

typestr,

parser.add_argument(--multi-gpus,

actionstore_true,

imports:imports.remove(flash_attn)return

imports#

AutoModel.from_pretrained(model_path,

else:if

AutoModel.from_pretrained(model_path,

torch_dtypetorch.bfloat16)device_map

max_memory{0:

10GB},no_split_module_classes[SiglipVisionTransformer,

device_map[llm.model.embed_tokens]device_map[llm.lm_head]

device_id

device_map[llm.model.layers.26]device_map[llm.model.layers.8]

device_id2device_map[llm.model.layers.9]

device_id2device_map[llm.model.layers.10]

device_id2device_map[llm.model.layers.11]

device_id2device_map[llm.model.layers.12]

device_id2device_map[llm.model.layers.13]

device_id2device_map[llm.model.layers.14]

device_id2device_map[llm.model.layers.15]

device_id2device_map[llm.model.layers.16]

device_id2#print(device_map)model

load_checkpoint_and_dispatch(model,

model_path,

patch(transformers.dynamic_module_utils.get_imports,

AutoModel.from_pretrained(model_path,

torch_dtypetorch.bfloat16)model

tokenizer

AutoTokenizer.from_pretrained(model_path,

Error,

get_file_extension(filename):return

os.path.splitext(filename)[1].lower()def

IMAGE_EXTENSIONSdef

gr.Slider(minimumparams[minimum],maximumparams[maximum],valueparams[value],stepparams[step],interactiveparams[interactive],labelparams[label])elif

comp

gr.Radio(choicesparams[choices],valueparams[value],interactiveparams[interactive],labelparams[label])elif

comp

gr.Button(valueparams[value],interactiveTrue)def

create_multimodal_input(upload_image_disabledFalse,

upload_video_disabledFalse):return

mgr.MultimodalInput(upload_image_button_props{label:

Upload

multiple},upload_video_button_props{label:

Upload

single},submit_button_props{label:

Submit})def

vision_hidden_statesNone):try:print(msgs:,

msgs)answer

model.chat(imageNone,msgsmsgs,tokenizertokenizer,**params)res

re.sub(r(box.*/box),

e:print(e)traceback.print_exc()return

-1,

Image.open(image.path).convert(RGB)else:image

Image.open(image.file.path).convert(RGB)#

resize

io.BytesIO()#image.save(buffered,

formatpng)#im_b64

base64.b64encode(buffered.getvalue()).decode()#return

{type:

len(frame_idx)MAX_NUM_FRAMES:frame_idx

MAX_NUM_FRAMES)video

vr.get_batch(frame_idx).asnumpy()video

[Image.fromarray(v.astype(uint8))

for

encode_message(_question):files

r\[mm_media\]\d\[/mm_media\]matches

re.split(pattern,

text:message.append(make_text(text))for

matches[i

text:message.append(make_text(text))return

messagedef

check_has_videos(_question):images_cnt

0videos_cnt

count_video_frames(_context):num_frames

0for

_app_cfg[ctx].copy()_context.append({role:

user,

encode_message(_question)})images_cnt

files_cnts[1]

files_cnts[1]_context.append({role:

assistant,

[make_text(_answer)]})_chat_bot.append((_question,

_answer))if

0:_app_cfg[ctx]_context_app_cfg[sts]sts_app_cfg[images_cnt]

videos_cntupload_image_disabled

videos_cnt

create_multimodal_input(upload_image_disabled,

_chat_bot,

fewshot_add_demonstration(_image,

_user_message,

Image.open(_image).convert(RGB)ctx.append({role:

user,

make_text(_user_message)]})message_item.append({text:

_user_message,

_user_message:ctx.append({role:

user,

[make_text(_user_message)]})message_item.append({text:

_user_message,

[]})else:message_item.append(None)if

_assistant_message:ctx.append({role:

assistant,

[make_text(_assistant_message)]})message_item.append({text:

_assistant_message,

[]})else:message_item.append(None)_chat_bot.append(message_item)return

None,

params_form):user_message_contents

[]_context

Image.open(_image).convert(RGB)user_message_contents

_user_message:user_message_contents

user_message_contents:_context.append({role:

user,

_image:_chat_bot.append([{text:

_user_message,

[]}])else:_chat_bot.append([{text:

_user_message,

0:_app_cfg[ctx]_context_app_cfg[sts]stsreturn

None,

regenerate_button_clicked(_question,

_image,

check_has_videos(_question)images_cnt

files_cnts[1]_app_cfg[images_cnt]

videos_cntupload_image_disabled

videos_cnt

last_message.text:last_user_message

last_message.textif

last_message.files[0].file.path_chat_bot

_user_message,

app_session):txt_message.files.clear()txt_message.text

chat_bot

copy.deepcopy(init_conversation)app_session[sts]

Noneapp_session[ctx]

min_width300):gr.Markdown(valueintroduction)params_form

compRadio)regenerate

valuecopy.deepcopy(init_conversation),

height600,

create_multimodal_input()chat_tab_label

interactiveFalse,

visibleFalse)txt_message.submit(respond,[txt_message,

chat_bot,

gr.Textbox(labelUser)assistant_message

gr.Row():add_demonstration_button

gr.Button(Add

variantprimary)add_demonstration_button.click(fewshot_add_demonstration,[image_input,

user_message,

app_session])generate_button.click(fewshot_respond,[image_input,

user_message,

app_session])chat_tab.select(select_chat_type,[chat_tab_label,

app_session],[app_session])chat_tab.select(

chat_bot,

assistant_message])fewshot_tab.select(select_chat_type,[fewshot_tab_label,

app_session],[app_session])fewshot_tab.select(

chat_bot,

assistant_message])chat_bot.flushed(flushed,outputs[txt_message])regenerate.click(regenerate_button_clicked,[txt_message,

image_input,

app_session])clear_button.click(clear,[txt_message,

chat_bot,

gr.Image(valuehttp://thunlp.oss-cn-qingdao.aliyuncs.com/multi_modal/never_delete/m_bear2.gif,

label1.

gr.Image(valuehttp://thunlp.oss-cn-qingdao.aliyuncs.com/multi_modal/never_delete/video2.gif,

label2.

gr.Image(valuehttp://thunlp.oss-cn-qingdao.aliyuncs.com/multi_modal/never_delete/fshot.gif,

label3.

/Usxxxxxxxckages/torch/nn/modules/module.py,

line

https://download.pytorch.org/whl/nightly/cpu#再次运行就没问题了

#这里下载模型20g可能会等一段时间最后借助魔法下载看这网速在疯狂跑就没问题

#成功运行输出如下

解决flash_attn强制依赖问题解决bfloat16在mps无法使用问题看系统占用是没走mps添加的环境变量也可以看出Sampling瞎回答Beam

Search回答很惊喜Beam

Search处理视频4秒在m1pro下当前代码中需要230s左右ollama部署还在研究中…

写到最后