class MiniMaxM3VLProcessingInfo(BaseProcessingInfo):
IMAGE_TOKEN = "]<]image[>["
VIDEO_TOKEN = "]<]video[>["
VISION_START_TOKEN = "]<]start of image[>["
VISION_END_TOKEN = "]<]end of image[>["
def get_hf_config(self) -> MiniMaxM3Config:
return self.ctx.get_hf_config(MiniMaxM3Config)
def get_hf_processor(self, **kwargs: object) -> MiniMaxVLProcessor:
# The released checkpoint only ships the processor as remote code
# (via ``auto_map``). Construct the vendored processor directly so the
# model loads without ``--trust-remote-code``.
return self.ctx.get_hf_processor(MiniMaxVLProcessor, **kwargs)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None, "video": None}
def get_mm_max_tokens_per_item(
self,
seq_len: int,
mm_counts: Mapping[str, int],
) -> Mapping[str, int]:
return {
"image": self.get_max_image_tokens(),
"video": self.get_max_video_tokens(seq_len, mm_counts),
}
def get_image_processor(self, **kwargs: object) -> MiniMaxM3VLImageProcessor:
return self.get_hf_processor(**kwargs).image_processor
def get_video_processor(self, **kwargs: object) -> MiniMaxM3VLVideoProcessor:
return self.get_hf_processor(**kwargs).video_processor
def _get_vision_info(
self,
*,
image_width: int,
image_height: int,
num_frames: int,
image_processor,
mm_kwargs: Mapping[str, object],
modality: str,
) -> tuple[ImageSize, int]:
"""Compute resized image size and number of vision tokens.
Mirrors the processor's ``compute_aligned_size`` so token counts
match the actual processor output.
"""
patch_size: int = image_processor.patch_size
merge_size: int = image_processor.merge_size
temporal_patch_size: int = image_processor.temporal_patch_size
factor = patch_size * merge_size
# ``image_processor`` is the *video* processor when counting video
# tokens, so modality-scoped request kwargs (flat keys overlaid with
# ``images_kwargs``/``videos_kwargs`` via ``modality=``) and processor
# defaults are read per modality. Per-key fallback mirrors the
# processor, whose own defaults feed ``resolve_max_long_side_pixel``
# the same way.
merged = self.ctx.get_merged_mm_kwargs(mm_kwargs)
# Kwargs scoped to this modality (``images_kwargs`` / ``videos_kwargs``)
# override the flat ones.
scoped = merged.get("images_kwargs" if modality == "image" else "videos_kwargs")
if isinstance(scoped, Mapping):
merged = {**merged, **scoped}
detail = merged.get("detail", getattr(image_processor, "detail", None))
max_long_side_pixel = merged.get(
"max_long_side_pixel", getattr(image_processor, "max_long_side_pixel", None)
)
resolved_long_side = resolve_max_long_side_pixel(
detail, max_long_side_pixel, modality=modality
)
# No volumetric cap here: token counting must not raise; the video
# processor enforces its volume cap on the real inputs.
new_h, new_w = compute_aligned_size(
image_height,
image_width,
factor,
max_long_side_pixel=resolved_long_side,
)
grid_h = new_h // patch_size
grid_w = new_w // patch_size
# Pad frames to be divisible by temporal_patch_size
padded_frames = num_frames + (-num_frames % temporal_patch_size)
grid_t = max(padded_frames // temporal_patch_size, 1)
num_tokens = grid_t * grid_h * grid_w // (merge_size**2)
return ImageSize(width=new_w, height=new_h), num_tokens
def get_num_image_tokens(
self,
*,
image_width: int,
image_height: int,
image_processor,
mm_kwargs: Mapping[str, object],
) -> int:
_, n = self._get_vision_info(
image_width=image_width,
image_height=image_height,
num_frames=1,
image_processor=image_processor,
mm_kwargs=mm_kwargs,
modality="image",
)
return n
def get_num_video_tokens(
self,
*,
image_width: int,
image_height: int,
num_frames: int,
image_processor,
mm_kwargs: Mapping[str, object],
) -> int:
_, n = self._get_vision_info(
image_width=image_width,
image_height=image_height,
num_frames=num_frames,
image_processor=image_processor,
mm_kwargs=mm_kwargs,
modality="video",
)
return n
def get_image_size_with_most_features(self) -> ImageSize:
# Worst case is a detail=high image: a square whose side is the max
# long side (3584 px, a multiple of patch_size*merge_size).
image_processor = self.get_image_processor()
factor = image_processor.patch_size * image_processor.merge_size
side = (_IMAGE_MAX_LONG_SIDE_PIXEL // factor) * factor
return ImageSize(width=side, height=side)
def get_video_size_with_most_features(self) -> ImageSize:
# Worst single frame is a detail=high square capped by the video
# long-side max (2188 px), floored to a patch_size*merge_size
# multiple (2184).
video_processor = self.get_video_processor()
factor = video_processor.patch_size * video_processor.merge_size
side = (_VIDEO_MAX_LONG_SIDE_PIXEL // factor) * factor
return ImageSize(width=side, height=side)
def get_max_image_tokens(self) -> int:
image_processor = self.get_image_processor()
size = self.get_image_size_with_most_features()
return self.get_num_image_tokens(
image_width=size.width,
image_height=size.height,
image_processor=image_processor,
# Budgets must cover the worst case (detail=high); the default
# detail=auto cap would undercount.
mm_kwargs={"max_long_side_pixel": _IMAGE_MAX_LONG_SIDE_PIXEL},
)
def _get_max_video_frames(self, max_tokens: int) -> int:
video_processor = self.get_video_processor()
size = self.get_video_size_with_most_features()
num_frames = 1
while True:
next_n = self.get_num_video_tokens(
image_width=size.width,
image_height=size.height,
num_frames=num_frames + 1,
image_processor=video_processor,
# Worst case (detail=high); see get_max_image_tokens.
mm_kwargs={"max_long_side_pixel": _VIDEO_MAX_LONG_SIDE_PIXEL},
)
if next_n > max_tokens:
break
num_frames += 1
return num_frames
def get_num_frames_with_most_features(
self,
seq_len: int,
mm_counts: Mapping[str, int],
max_frames_per_video: int = _MAX_FRAMES_PER_VIDEO,
) -> int:
max_videos = mm_counts.get("video", 0)
max_total_frames = self._get_max_video_frames(seq_len)
max_frames_per_video = min(
max_total_frames // max(max_videos, 1), max_frames_per_video
)
return max(max_frames_per_video, 1)
def get_max_video_tokens(
self,
seq_len: int,
mm_counts: Mapping[str, int],
) -> int:
video_processor = self.get_video_processor()
size = self.get_video_size_with_most_features()
return self.get_num_video_tokens(
image_width=size.width,
image_height=size.height,
num_frames=self.get_num_frames_with_most_features(seq_len, mm_counts),
image_processor=video_processor,
# Worst case (detail=high); see get_max_image_tokens.
mm_kwargs={"max_long_side_pixel": _VIDEO_MAX_LONG_SIDE_PIXEL},
)