add run init_run_segment & run end_run_segment latency metrics (#359)

Co-authored-by: yihuiwen <yihuiwen@sensetime.com>

Dockerfile_deploy

@@ -5,7 +5,7 @@ RUN cd /opt/lightx2v/deploy/server/frontend \
     && npm install \
     && npm run build
 
-FROM lightx2v:25092201-cu128 AS base
+FROM lightx2v/lightx2v:25092201-cu128 AS base
 
 RUN mkdir /workspace/LightX2V
 WORKDIR /workspace/LightX2V

lightx2v/models/runners/default_runner.py

@@ -259,7 +259,7 @@ class DefaultRunner(BaseRunner):
             with ProfilingContext4DebugL1(
                 f"segment end2end {segment_idx + 1}/{self.video_segment_num}",
                 recorder_mode=GET_RECORDER_MODE(),
-                metrics_func=monitor_cli.lightx2v_run_pre_step_dit_duration,
+                metrics_func=monitor_cli.lightx2v_run_per_step_dit_duration,
                 metrics_labels=[segment_idx + 1, self.video_segment_num],
             ):
                 self.check_stop()

lightx2v/models/runners/wan/wan_animate_runner.py

@@ -292,6 +292,12 @@ class WanAnimateRunner(WanRunner):
             gc.collect()
         return images
 
+    @ProfilingContext4DebugL1(
+        "Init run segment",
+        recorder_mode=GET_RECORDER_MODE(),
+        metrics_func=monitor_cli.lightx2v_run_init_run_segment_duration,
+        metrics_labels=["WanAnimateRunner"],
+    )
     def init_run_segment(self, segment_idx):
         start = segment_idx * self.move_frames
         end = start + self.config["target_video_length"]

lightx2v/models/runners/wan/wan_audio_runner.py

@@ -587,7 +587,12 @@ class WanAudioRunner(WanRunner):  # type:ignore
             self.gen_video_final = None
             self.cut_audio_final = None
 
-    @ProfilingContext4DebugL1("Init run segment")
+    @ProfilingContext4DebugL1(
+        "Init run segment",
+        recorder_mode=GET_RECORDER_MODE(),
+        metrics_func=monitor_cli.lightx2v_run_init_run_segment_duration,
+        metrics_labels=["WanAudioRunner"],
+    )
     def init_run_segment(self, segment_idx, audio_array=None):
         self.segment_idx = segment_idx
         if audio_array is not None:
@@ -618,7 +623,12 @@ class WanAudioRunner(WanRunner):  # type:ignore
         if segment_idx > 0:
             self.model.scheduler.reset(self.input_info.seed, self.input_info.latent_shape, self.inputs["previmg_encoder_output"])
 
-    @ProfilingContext4DebugL1("End run segment")
+    @ProfilingContext4DebugL1(
+        "End run segment",
+        recorder_mode=GET_RECORDER_MODE(),
+        metrics_func=monitor_cli.lightx2v_run_end_run_segment_duration,
+        metrics_labels=["WanAudioRunner"],
+    )
     def end_run_segment(self, segment_idx):
         self.gen_video = torch.clamp(self.gen_video, -1, 1).to(torch.float)
         useful_length = self.segment.end_frame - self.segment.start_frame

lightx2v/server/metrics/metrics.py

@@ -48,6 +48,7 @@ METRICS_INFO = {
         name="lightx2v_worker_request_duration",
         desc="Duration of the request (s)",
         type_="histogram",
+        labels=["model_cls"],
     ),
     "lightx2v_input_audio_len": MetricsConfig(
         name="lightx2v_input_audio_len",
@@ -86,7 +87,7 @@ METRICS_INFO = {
         type_="histogram",
     ),
     "lightx2v_run_per_step_dit_duration": MetricsConfig(
-        name="lightx2v_run_pre_step_dit_duration",
+        name="lightx2v_run_per_step_dit_duration",
         desc="Duration of run per step Dit (s)",
         type_="histogram",
         labels=["step_no", "total_steps"],
@@ -115,6 +116,18 @@ METRICS_INFO = {
         type_="histogram",
         labels=["model_cls"],
     ),
+    "lightx2v_run_init_run_segment_duration": MetricsConfig(
+        name="lightx2v_run_init_run_segment_duration",
+        desc="Duration of run init_run_segment (s)",
+        type_="histogram",
+        labels=["model_cls"],
+    ),
+    "lightx2v_run_end_run_segment_duration": MetricsConfig(
+        name="lightx2v_run_end_run_segment_duration",
+        desc="Duration of run end_run_segment (s)",
+        type_="histogram",
+        labels=["model_cls"],
+    ),
 }
 
 

lightx2v/utils/profiler.py

@@ -36,7 +36,7 @@ class _ProfilingContext:
         elapsed = time.perf_counter() - self.start_time
         if self.enable_recorder and self.metrics_func:
             if self.metrics_labels:
-                self.metrics_func.labels(self.metrics_labels).observe(elapsed)
+                self.metrics_func.labels(*self.metrics_labels).observe(elapsed)
             else:
                 self.metrics_func.observe(elapsed)
         if self.enable_logger:
@@ -53,7 +53,7 @@ class _ProfilingContext:
         elapsed = time.perf_counter() - self.start_time
         if self.enable_recorder and self.metrics_func:
             if self.metrics_labels:
-                self.metrics_func.labels(self.metrics_labels).observe(elapsed)
+                self.metrics_func.labels(*self.metrics_labels).observe(elapsed)
             else:
                 self.metrics_func.observe(elapsed)
         if self.enable_logger: