hunyuan-offline

benchmarks/benchmark_hunyuan.py

+#!/usr/bin/env python3
+"""Minimal classify demo using token IDs as input.
+
+This mirrors the docs example:
+    llm = LLM(model="...", runner="pooling")
+    (output,) = llm.classify("Hello, my name is")
+
+but feeds DEFAULT_PROMPT_TOKEN_IDS via token_inputs instead of text.
+"""
+
+from vllm import LLM
+from vllm.inputs import token_inputs
+
+
+DEFAULT_PROMPT_TOKEN_IDS = [
+    127958, 58, 10172, 24575, 8437, 7489, 51, 60,
+    220, 57668, 102832, 80073, 75761, 102245, 39045, 57668,
+    105982, 103429, 88852, 9743, 34208, 2929, 3922, 101423,
+    83125, 110357, 107759, 82317, 101505, 101009, 1811, 15225,
+    61633, 3922, 101992, 80073, 120702, 17, 15, 17,
+    19, 8107, 16, 17, 9953, 17, 17, 9080,
+    3490, 2929, 5232, 82, 8910, 6704, 25451, 43032,
+    198, 12, 11615, 101241, 5232, 101016, 198, 12,
+    220, 104780, 101526, 5232, 43292, 104780, 198, 12,
+    61696, 225, 101028, 101526, 5232, 16325, 106444, 271,
+    9743, 44915, 29411, 12, 52561, 229, 34972, 5232,
+    11144, 19378, 101814, 106742, 11199, 19378, 100904, 101130,
+    198, 12, 73028, 96, 17161, 5232, 83125, 25580,
+    78244, 105996, 119022, 117130, 103702, 101021, 28542, 104156,
+    101526, 100765, 5486, 101526, 112370, 101526, 107717, 101026,
+    3922, 101093, 101406, 102193, 92780, 105328, 102715, 101697,
+    99480, 71600, 101026, 69636, 105219, 33764, 83800, 100502,
+    78698, 101300, 83800, 100476, 105259, 106329, 69636, 106536,
+    124213, 100893, 9554, 108473, 3922, 78657, 5232, 100588,
+    105219, 33764, 100502, 79656, 106555, 116251, 97150, 83800,
+    101697, 99480, 101026, 16325, 106246, 106408, 20600, 100580,
+    20675, 28469, 121838, 72917, 113081, 113633, 108149, 103463,
+    69636, 103575, 100614, 100860, 70616, 115163, 100886, 83800,
+    102674, 17297, 101006, 105740, 26016, 100588, 83800, 29391,
+    41401, 121110, 105278, 30046, 111058, 105278, 102341, 73548,
+    101987, 101995, 38129, 69636, 110858, 102674, 17297, 44368,
+    72917, 110991, 105529, 50667, 3922, 101374, 114621, 83800,
+    106773, 34547, 101356, 107000, 106212, 87219, 69636, 103575,
+    113429, 102072, 21082, 101593, 106874, 19378, 51611, 91940,
+    113846, 1811, 19378, 101924, 19378, 3844, 18936, 11050,
+    75492, 3922, 100502, 100632, 117130, 8, 105299, 109665,
+    101657, 105148, 46281, 100502, 102172, 101420, 119022, 106334,
+    101393, 3922, 100582, 100486, 123002, 105996, 119022, 117130,
+    9554, 104090, 101909, 100746, 102900, 103394, 106212, 100639,
+    37507, 101745, 34208, 101364, 83800, 106368, 100502, 100943,
+    19378, 115256, 35304, 119022, 109364, 100502, 101792, 101025,
+    3922, 107105, 100632, 20834, 108149, 102715, 51611, 271,
+    9743, 58521, 29411, 482, 220, 58521, 31091, 5232,
+    100655, 104091, 78519, 105689, 198, 482, 220, 58521,
+    101241, 5232, 100827, 62, 107940, 198, 482, 220,
+    58521, 105302, 5232, 100580, 271, 9743, 90261, 5232,
+    106425, 32239, 198, 482, 75677, 111, 55038, 101241,
+    5232, 100992, 111155, 198, 482, 41766, 229, 81742,
+    33005, 5232, 100359, 198, 482, 75677, 111, 55038,
+    105344, 5232, 16, 15, 198, 482, 61696, 225,
+    101028, 105344, 5232, 16, 271, 9743, 21082, 5232,
+    17, 15, 17, 18, 8107, 15, 18, 9953,
+    15, 20, 9080, 320, 101396, 37271, 5232, 21,
+    20, 23, 36827, 696, 2929, 5232, 82, 8910,
+    6704, 25451, 43032, 127962, 127960, 127967,
+]
+
+
+if __name__ == "__main__":
+    llm = LLM(model="/tools/gy_model/hunyuan_model", task="classify",trust_remote_code=True,runner="pooling")
+    (output,) = llm.classify(token_inputs(DEFAULT_PROMPT_TOKEN_IDS))
+    probs = output.outputs.probs
+    print(f"Class Probabilities: {probs!r} (size={len(probs)})")