update multi people for seko (#335)

configs/seko_talk/multi_person/01_base.json

 {
-  "infer_steps": 4,
-  "target_fps": 16,
-  "video_duration": 360,
-  "audio_sr": 16000,
-  "target_video_length": 81,
-  "resize_mode": "adaptive",
-  "self_attn_1_type": "flash_attn3",
-  "cross_attn_1_type": "flash_attn3",
-  "cross_attn_2_type": "flash_attn3",
-  "seed": 42,
-  "sample_guide_scale": 1.0,
-  "sample_shift": 5,
-  "enable_cfg": false,
-  "cpu_offload": false,
-  "use_31_block": false
+    "infer_steps": 4,
+    "target_fps": 16,
+    "video_duration": 360,
+    "audio_sr": 16000,
+    "target_video_length": 81,
+    "resize_mode": "adaptive",
+    "self_attn_1_type": "flash_attn3",
+    "cross_attn_1_type": "flash_attn3",
+    "cross_attn_2_type": "flash_attn3",
+    "seed": 42,
+    "sample_guide_scale": 1.0,
+    "sample_shift": 5,
+    "enable_cfg": false,
+    "cpu_offload": false,
+    "use_31_block": false
 }

configs/seko_talk/multi_person/02_base_fp8.json

 {
-  "infer_steps": 4,
-  "target_fps": 16,
-  "video_duration": 5,
-  "audio_sr": 16000,
-  "target_video_length": 81,
-  "resize_mode": "adaptive",
-  "self_attn_1_type": "sage_attn2",
-  "cross_attn_1_type": "sage_attn2",
-  "cross_attn_2_type": "sage_attn2",
-  "seed": 42,
-  "sample_guide_scale": 1.0,
-  "sample_shift": 5,
-  "enable_cfg": false,
-  "cpu_offload": false,
-  "use_31_block": false,
-  "mm_config": {
-      "mm_type": "W-fp8-channel-sym-A-fp8-channel-sym-dynamic-Sgl"
-  },
-  "adapter_quantized": true,
-  "adapter_quant_scheme": "fp8",
-  "t5_quantized": true,
-  "t5_quant_scheme": "fp8"
-
+    "infer_steps": 4,
+    "target_fps": 16,
+    "video_duration": 360,
+    "audio_sr": 16000,
+    "target_video_length": 81,
+    "resize_mode": "adaptive",
+    "self_attn_1_type": "sage_attn2",
+    "cross_attn_1_type": "sage_attn2",
+    "cross_attn_2_type": "sage_attn2",
+    "seed": 42,
+    "sample_guide_scale": 1.0,
+    "sample_shift": 5,
+    "enable_cfg": false,
+    "cpu_offload": false,
+    "use_31_block": false,
+    "mm_config": {
+        "mm_type": "W-fp8-channel-sym-A-fp8-channel-sym-dynamic-Sgl"
+    },
+    "adapter_quantized": true,
+    "adapter_quant_scheme": "fp8",
+    "t5_quantized": true,
+    "t5_quant_scheme": "fp8"
 }

configs/seko_talk/multi_person/03_dist.json

@@ -5,18 +5,20 @@
     "audio_sr": 16000,
     "target_video_length": 81,
     "resize_mode": "adaptive",
-    "self_attn_1_type": "flash_attn3",
-    "cross_attn_1_type": "flash_attn3",
-    "cross_attn_2_type": "flash_attn3",
+    "self_attn_1_type": "sage_attn2",
+    "cross_attn_1_type": "sage_attn2",
+    "cross_attn_2_type": "sage_attn2",
     "seed": 42,
     "sample_guide_scale": 1.0,
     "sample_shift": 5,
     "enable_cfg": false,
     "cpu_offload": false,
     "use_31_block": false,
-    "parallel": {
-        "seq_p_size": 4,
-        "seq_p_attn_type": "ulysses"
+    "mm_config": {
+        "mm_type": "W-fp8-channel-sym-A-fp8-channel-sym-dynamic-Sgl"
     },
-    "person_mask_path": "assets/inputs/audio/multi_person"
+    "adapter_quantized": true,
+    "adapter_quant_scheme": "fp8",
+    "t5_quantized": true,
+    "t5_quant_scheme": "fp8"
 }

configs/seko_talk/multi_person/04_dist_fp8.json

+{
+    "infer_steps": 4,
+    "target_fps": 16,
+    "video_duration": 360,
+    "audio_sr": 16000,
+    "target_video_length": 81,
+    "resize_mode": "adaptive",
+    "self_attn_1_type": "sage_attn2",
+    "cross_attn_1_type": "sage_attn2",
+    "cross_attn_2_type": "sage_attn2",
+    "seed": 42,
+    "sample_guide_scale": 1.0,
+    "sample_shift": 5,
+    "enable_cfg": false,
+    "cpu_offload": false,
+    "use_31_block": false,
+    "mm_config": {
+        "mm_type": "W-fp8-channel-sym-A-fp8-channel-sym-dynamic-Sgl"
+    },
+    "adapter_quantized": true,
+    "adapter_quant_scheme": "fp8",
+    "t5_quantized": true,
+    "t5_quant_scheme": "fp8",
+    "parallel": {
+        "seq_p_size": 4,
+        "seq_p_attn_type": "ulysses"
+    }
+}

configs/seko_talk/multi_person/15_base_compile.json

@@ -15,7 +15,5 @@
     "cpu_offload": false,
     "use_31_block": false,
     "compile": true,
-    "compile_shapes": [[480, 832], [720, 1280]],
-    "compile_max_audios": 3,
-    "person_mask_path": "assets/inputs/audio/multi_person"
+    "compile_shapes": [[480, 832], [720, 1280]]
 }

scripts/seko_talk/multi_person/00_base.sh

-#!/bin/bash
-
-lightx2v_path=/path/to/LightX2V
-model_path=/path/to/model
-
-export CUDA_VISIBLE_DEVICES=0
-
-# set environment variables
-source ${lightx2v_path}/scripts/base/base.sh
-
-export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
-
-export SENSITIVE_LAYER_DTYPE=None
-
-python  -m lightx2v.infer \
---model_cls seko_talk \
---task i2v \
---model_path $model_path \
---config_json ${lightx2v_path}/configs/seko_talk/multi_person/02_base_fp8.json \
---prompt  "The video features a man and a woman standing by a bench in the park, their expressions tense and voices raised as they argue. The man gestures with both hands, his arms swinging slightly as if to emphasize each heated word, while the woman stands with her hands on her waist, her brows furrowed in frustration. The background is a wide expanse of sunlit grass, the golden light contrasting with the sharp energy of their quarrel. Their voices seem to clash in the air, and the rhythm of their hand movements and body postures interweaves with the rising tension, creating a vivid scene of confrontation." \
---negative_prompt 色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走 \
---image_path ${lightx2v_path}/assets/inputs/audio/seko_input.png \
---audio_path ${lightx2v_path}/assets/inputs/audio/seko_input.mp3 \
---save_video_path ${lightx2v_path}/save_results/seko_talk_multi_person_00.mp4

scripts/seko_talk/multi_person/02_base_single.sh

-#!/bin/bash
-
-lightx2v_path=/path/to/LightX2V
-model_path=/path/to/SekoTalk-Distill
-
-export CUDA_VISIBLE_DEVICES=0
-
-# set environment variables
-source ${lightx2v_path}/scripts/base/base.sh
-
-export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
-
-export SENSITIVE_LAYER_DTYPE=None
-
-python  -m lightx2v.infer \
---model_cls seko_talk \
---task i2v \
---model_path $model_path \
---config_json ${lightx2v_path}/configs/seko_talk/multi_person/02_base_fp8.json \
---prompt  "The video features a man and a woman standing by a bench in the park, their expressions tense and voices raised as they argue. The man gestures with both hands, his arms swinging slightly as if to emphasize each heated word, while the woman stands with her hands on her waist, her brows furrowed in frustration. The background is a wide expanse of sunlit grass, the golden light contrasting with the sharp energy of their quarrel. Their voices seem to clash in the air, and the rhythm of their hand movements and body postures interweaves with the rising tension, creating a vivid scene of confrontation." \
---negative_prompt 色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走 \
---image_path ${lightx2v_path}/assets/inputs/audio/multi_person/seko_input.png \
---audio_path ${lightx2v_path}/assets/inputs/audio/multi_person \
---save_video_path ${lightx2v_path}/save_results/seko_talk_multi_person_single.mp4

scripts/seko_talk/multi_person/03_dist.sh

@@ -19,5 +19,6 @@ torchrun --nproc-per-node 4 -m lightx2v.infer \
 --config_json ${lightx2v_path}/configs/seko_talk/multi_person/03_dist.json \
 --prompt  "The video features a man and a woman standing by a bench in the park, their expressions tense and voices raised as they argue. The man gestures with both hands, his arms swinging slightly as if to emphasize each heated word, while the woman stands with her hands on her waist, her brows furrowed in frustration. The background is a wide expanse of sunlit grass, the golden light contrasting with the sharp energy of their quarrel. Their voices seem to clash in the air, and the rhythm of their hand movements and body postures interweaves with the rising tension, creating a vivid scene of confrontation." \
 --negative_prompt 色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走 \
+--image_path ${lightx2v_path}/assets/inputs/audio/multi_person/seko_input.png \
 --audio_path ${lightx2v_path}/assets/inputs/audio/multi_person \
 --save_video_path ${lightx2v_path}/save_results/seko_talk_multi_person_dist.mp4

scripts/seko_talk/multi_person/15_base_compile.sh

-#!/bin/bash
-
-lightx2v_path=/path/to/Lightx2v
-model_path=/path/to/SekoTalk-Distill
-
-export CUDA_VISIBLE_DEVICES=0
-
-# set environment variables
-source ${lightx2v_path}/scripts/base/base.sh
-
-export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
-
-export SENSITIVE_LAYER_DTYPE=None
-
-python -m lightx2v.infer \
---model_cls seko_talk \
---task i2v \
---model_path $model_path \
---config_json ${lightx2v_path}/configs/seko_talk/multi_person/15_base_compile.json \
---prompt  "The video features a man and a woman standing by a bench in the park, their expressions tense and voices raised as they argue. The man gestures with both hands, his arms swinging slightly as if to emphasize each heated word, while the woman stands with her hands on her waist, her brows furrowed in frustration. The background is a wide expanse of sunlit grass, the golden light contrasting with the sharp energy of their quarrel. Their voices seem to clash in the air, and the rhythm of their hand movements and body postures interweaves with the rising tension, creating a vivid scene of confrontation." \
---negative_prompt 色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走 \
---image_path ${lightx2v_path}/assets/inputs/audio/multi_person/seko_input.png \
---audio_path ${lightx2v_path}/assets/inputs/audio/multi_person \
---save_video_path ${lightx2v_path}/save_results/seko_talk_multi_person.mp4