多模态模型部署

多模态模型（Vision-Language, Audio-Language）的部署比纯文本 LLM 复杂：输入预处理多样、GPU 显存占用更大、推理延迟更高。

部署架构

多模态 vs 纯文本部署

多模态服务框架

"""
多模态模型推理服务
"""
from dataclasses import dataclass, field
from enum import Enum
from typing import Any
import base64
import io
class ModalityType(Enum):
TEXT = "text"
IMAGE = "image"
AUDIO = "audio"
VIDEO = "video"
@dataclass
class MultimodalInput:
"""多模态输入"""
modality: ModalityType
data: Any
metadata: dict = field(default_factory=dict)
@dataclass
class ProcessedInput:
"""预处理后的输入"""
embeddings: list[float]
tokens: int
modality: ModalityType
class ImagePreprocessor:
"""图像预处理"""
def __init__(self, target_size: tuple[int, int] = (336, 336)):
self.target_size = target_size
def process(self, image_data: bytes) -> ProcessedInput:
"""处理图像"""
# 解码 → 缩放 → 归一化 → 分 patch
patches = self._split_patches(image_data)
tokens = len(patches) * 256  # 每个 patch ≈ 256 tokens
return ProcessedInput(
embeddings=patches,
tokens=tokens,
modality=ModalityType.IMAGE,
)
def _split_patches(self, image_data: bytes) -> list:
"""将图像分割为 patches（简化示意）"""
# 实际实现使用 PIL/torchvision
return [0.0] * 576  # 简化：576 个 patch embeddings
class MultimodalServer:
"""多模态推理服务"""
# 不同模态的 Token 换算
TOKEN_RATIOS = {
ModalityType.TEXT: 1,
ModalityType.IMAGE: 256,     # 1张图 ≈ 256-1024 tokens
ModalityType.AUDIO: 25,      # 1秒音频 ≈ 25 tokens
}
def __init__(self, model_name: str, max_batch_size: int = 8):
self.model_name = model_name
self.max_batch_size = max_batch_size
self.image_processor = ImagePreprocessor()
self._request_count = 0
def infer(self, inputs: list[MultimodalInput]) -> str:
"""推理"""
processed = []
total_tokens = 0
for inp in inputs:
if inp.modality == ModalityType.IMAGE:
p = self.image_processor.process(inp.data)
elif inp.modality == ModalityType.TEXT:
p = ProcessedInput(
embeddings=[],
tokens=len(str(inp.data)) // 4,
modality=ModalityType.TEXT,
)
else:
continue
processed.append(p)
total_tokens += p.tokens
self._request_count += 1
# 实际推理（调用模型的 forward）
return self._forward(processed, total_tokens)
def _forward(self, inputs: list[ProcessedInput], total_tokens: int) -> str:
"""模型前向推理（简化示意）"""
return f"[多模态推理结果] 处理 {len(inputs)} 个输入，共 {total_tokens} tokens"
def estimate_cost(self, inputs: list[MultimodalInput]) -> dict:
"""估算成本"""
total_tokens = 0
for inp in inputs:
ratio = self.TOKEN_RATIOS.get(inp.modality, 1)
if inp.modality == ModalityType.IMAGE:
total_tokens += ratio * 4  # 高分辨率可达 4x
elif inp.modality == ModalityType.TEXT:
total_tokens += len(str(inp.data)) // 4
elif inp.modality == ModalityType.AUDIO:
duration = inp.metadata.get("duration_sec", 10)
total_tokens += ratio * duration
return {
"estimated_tokens": total_tokens,
"estimated_cost_usd": total_tokens * 0.00001,
}

GPU 显存规划

优化策略

本章小结

下一章：LLM 生产趋势