。

刚好我有H20的GPU所以部署来看看效果#xff0c;因为我的宿主机是cuda-12.1所以为了防止环境污染采用docker部署#xff0c;通过一系列的披荆斩棘…多模态视频大模型Aria在Docker部署

闲逛HuggingFace的时候发现一个25.3B的多模态大模型支持图片和视频。

刚好我有H20的GPU所以部署来看看效果因为我的宿主机是cuda-12.1所以为了防止环境污染采用docker部署通过一系列的披荆斩棘比如Segmentation

fault

dumped)异常最终成功运行在单卡h20服务器上python3.10cuda12.4ubuntu20.04,程序在推理图片的时候占用50g显存推理5s视频20fps的时候占用60g左右显存。

项目简介

https://github.com/rhymes-ai/Aria

线上demo尝试

线上demo响应很快并且描述得很详细并且可以描述什么时间发生了啥介绍里面说的是Cutting

long

timestamps.(通过带有时间戳的场景过渡来剪切长视频。

),这不是自动剪分镜吗我有一个好想法先写完这篇再说

docker环境

宿主机cuda是12.4以上的可以忽略宿主机可以随便升降级cuda的也可以忽略要不然会出现以下异常ImportError:

/usr/local/lib/python3.10/dist-packages/torch/lib/…/…/nvidia/cusparse/lib/libcusparse.so.12:

undefined

https://nvidia.github.io/nvidia-docker/gpgkey

sudo

https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list

sudo

/etc/apt/sources.list.d/nvidia-docker.list#安装docker和nvidia-docker

sudo

:[https://hub.rat.dev,https://docker.1panel.live,https://docker.rainbond.cc,https://mirror.ccs.tencentyun.com,http://registry.docker-cn.com,http://docker.mirrors.ustc.edu.cn,http://hub-mirror.c.163.com],data-root:

/home/docker

docker#运行cuda:12.4.1容器指定使用哪块gpu指定挂载路径

#cuda:12.4.1-devel-ubuntu20.04。

这个镜像包含了

nvcc

nvidia/cuda:12.4.1-devel-ubuntu20.04

tail

#这只是我的磁盘满了需要搞到其他盘我自己记录一下你不用运行sudo

rsync

#下载conda有些云厂商不支持tsinghua所以任意选一个就行

wget

https://mirrors.tuna.tsinghua.edu.cn/anaconda/miniconda/Miniconda3-latest-Linux-x86_64.sh

wget

https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh#安装conda配置环境变量如果选择了自动配置环境可以不修改bashrc

Miniconda3-latest-Linux-x86_64.sh#添加conda

vim

__conda_setup$(/xxx/miniconda3/bin/conda

shell.bash

/xxx/miniconda3/etc/profile.d/conda.sh

***n.

/xxx/miniconda3/etc/profile.d/conda.shelseexport

PATH/xxx/miniconda3/bin:$PATHfi

unset

https://github.com/rhymes-ai/Aria.git#进入Aria工程目录

conda

https://mirrors.aliyun.com/pypi/simple

pip

https://mirrors.aliyun.com/pypi/simple

pip

https://mirrors.aliyun.com/pypi/simple下载模型

import

logging.basicConfig(levellogging.INFO)def

local_name,

1):try:snapshot_download(repo_idmodel_name,ignore_patterns[*.bin],local_dirlocal_name,force_downloadFalse)logging.info(Download

Exception

max_retries:time.sleep(retry_interval)else:logging.critical(Download

failed,

argparse.ArgumentParser(descriptionDownload

model

Hub)parser.add_argument(--model_name,

requiredTrue,

download)parser.add_argument(--local_name,

requiredTrue,

parser.parse_args()download_model(args.model_name,

args.local_name)if

HF_ENDPOINThttps://hf-mirror.com

#命令行直接运行如果缺少依赖手动装下就行

/home/models/huggingface/rhymes-ai/Aria#建议使用nohup

export

HF_ENDPOINThttps://hf-mirror.com

nohup

/home/models/huggingface/rhymes-ai/Ariamodel

AutoModelForCausalLM.from_pretrained(model_id_or_path,

device_mapauto,

trust_remote_codeTrue)processor

AutoProcessor.from_pretrained(model_id_or_path,

trust_remote_codeTrue)#你自己搞一个你图片

image_path

Image.open(requests.get(image_path,

[{role:

processor.apply_chat_template(messages,

inputs

inputs[pixel_values].to(model.dtype)

inputs

torch.cuda.amp.autocast(dtypetorch.bfloat16):output

model.generate(**inputs,max_new_tokens500,stop_strings[|im_end|],tokenizerprocessor.tokenizer,do_sampleTrue,temperature0.9,)output_ids

output[0][inputs[input_ids].shape[1]:]result

skip_special_tokensTrue)print(result)结果

视频测试

os.environ[CUDA_VISIBLE_DEVICES]

import

/home/models/huggingface/rhymes-ai/Aria

model

AutoModelForCausalLM.from_pretrained(model_id_or_path,

device_mapauto,

torch_dtypetorch.bfloat16,trust_remote_codeTrue)

processor

AutoProcessor.from_pretrained(model_id_or_path,

trust_remote_codeTrue)#这个一定放在模型加载下面要不然要报错Segmentation

fault

cache_dir/home/lzy/cached_video_frames,

verbosityDEBUG):#

os.path.basename(video_file)cache_subdir

f{video_basename}_{num_frames})os.makedirs(cache_subdir,

[]missing_frames

os.path.exists(frame_path):cached_frames.append(frame_path)else:missing_frames.append(i)frame_indices.append(i)vr

VideoReader(video_file)duration

len(vr)fps

vr.get_avg_fps()frame_timestamps

[int(duration

speed!.format(len(cached_frames),

num_frames,

[Image.open(frame_path).convert(RGB)

for

frame_timestampsactual_frame_indices

[int(duration

missing_frames]missing_frames_data

vr.get_batch(actual_frame_indices).asnumpy()for

idx,

Image.fromarray(missing_frames_data[idx]).convert(RGB)frame_path

fframe_{frame_index}.jpg)img.save(frame_path)cached_frames.append(frame_path)cached_frames.sort(keylambda

int(os.path.basename(x).split(_)[1].split(.)[0]))return

[Image.open(frame_path).convert(RGB)

for

get_placeholders_for_videos(frames:

List,

enumerate(frames):contents.append({text:

None,

timestamps)):contents.extend([{text:

f[{int(ts)

file.endswith(video_extensions):video_path

os.path.join(root,

get_placeholders_for_videos(frames,

frame_timestamps)###

get_placeholders_for_videos(frames)start

time.time()messages

processor.apply_chat_template(messages,

add_generation_promptTrue)inputs

processor(texttext,

max_image_size980)inputs[pixel_values]

inputs[pixel_values].to(model.dtype)inputs

{k:

torch.cuda.amp.autocast(dtypetorch.bfloat16):output

model.generate(**inputs,max_new_tokens2048,stop_strings[|im_end|],tokenizerprocessor.tokenizer,do_sampleFalse,)output_ids

output[0][inputs[input_ids].shape[1]:]result

skip_special_tokensTrue)print(result)print(time.time()

start)

我是分析/home/下面的所有视频你要分析单个改改就行max_image_size可改成490num_frames你根据自己视频来选我的5秒视频分析20fps相当于一秒4fps

aria显存占用还可以60g左右好像默认使用的是attn_implementation“flash_attention_2”对比qwen和cpm来说可以做到通过带有时间戳的场景过渡来剪切长视频core

dumped调整下import就行