Unverified Commit 27c0ca50 authored by Collin McCarthy's avatar Collin McCarthy Committed by GitHub
Browse files

Update registry for Nemotron-v3 VL Nano/Super (#39747)


Signed-off-by: default avatarCollin McCarthy <cmccarthy@nvidia.com>
parent 7c636432
......@@ -1038,6 +1038,50 @@ _MULTIMODAL_EXAMPLE_MODELS = {
},
trust_remote_code=True,
),
# NemotronH_Nano_Omni_Reasoning_V3 is an alias for NemotronH_Nano_VL_V2
# Use the same registry test as NemotronH_Nano_VL_V2 above
"NemotronH_Nano_Omni_Reasoning_V3": _HfExamplesInfo(
"nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16",
max_model_len=4096,
use_original_num_layers=True,
hf_overrides={
"vision_config": PretrainedConfig(
args={
"min_num_patches": 1,
"max_num_patches": 12,
"model": "vit_huge_patch16_224",
},
video_temporal_patch_size=2,
),
"text_config": {
"num_hidden_layers": 2,
"hybrid_override_pattern": "M*",
},
},
trust_remote_code=True,
),
# NemotronH_Super_Omni_Reasoning_V3 is an alias for NemotronH_Nano_VL_V2 as well
# Use the same registry test as NemotronH_Nano_VL_V2 above
"NemotronH_Super_Omni_Reasoning_V3": _HfExamplesInfo(
"nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16",
max_model_len=4096,
use_original_num_layers=True,
hf_overrides={
"vision_config": PretrainedConfig(
args={
"min_num_patches": 1,
"max_num_patches": 12,
"model": "vit_huge_patch16_224",
},
video_temporal_patch_size=2,
),
"text_config": {
"num_hidden_layers": 2,
"hybrid_override_pattern": "M*",
},
},
trust_remote_code=True,
),
"OpenCUAForConditionalGeneration": _HfExamplesInfo(
"xlangai/OpenCUA-7B", trust_remote_code=True
),
......
......@@ -300,6 +300,10 @@ class SpeculativeConfig:
{"n_predict": n_predict, "architectures": ["ErnieMTPModel"]}
)
if hf_config.architectures[0] == "NemotronH_Super_Omni_Reasoning_V3":
# Promote VLM's text_config so MTP detection below fires correctly
hf_config = hf_config.text_config
if (
hf_config.model_type in {"nemotron_h", "nemotron_h_puzzle"}
and hasattr(hf_config, "num_nextn_predict_layers")
......
......@@ -473,6 +473,8 @@ _MULTIMODAL_MODELS = {
"MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
"Molmo2ForConditionalGeneration": ("molmo2", "Molmo2ForConditionalGeneration"),
"NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
"NemotronH_Nano_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
"NemotronH_Super_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
"NVLM_D": ("nvlm_d", "NVLM_D_Model"),
"OpenCUAForConditionalGeneration": ("opencua", "OpenCUAForConditionalGeneration"),
"OpenPanguVLForConditionalGeneration": (
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment