"Initial commit"

demo/frontend/src/common/components/toolbar/ToolbarHeaderWrapper.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {ReactNode} from 'react';
+import ToolbarProgressChip from './ToolbarProgressChip';
+
+type Props = {
+  title: string;
+  description?: string;
+  bottomSection?: ReactNode;
+  showProgressChip?: boolean;
+  className?: string;
+};
+
+export default function ToolbarHeaderWrapper({
+  title,
+  description,
+  bottomSection,
+  showProgressChip = true,
+  className,
+}: Props) {
+  return (
+    <div
+      className={`flex flex-col gap-2 p-8 border-b border-b-black ${className}`}>
+      <div className="flex items-center">
+        {showProgressChip && <ToolbarProgressChip />}
+        <h2 className="text-xl">{title}</h2>
+      </div>
+
+      {description != null && (
+        <div className="flex-1 text-gray-400">{description}</div>
+      )}
+      {bottomSection != null && bottomSection}
+    </div>
+  );
+}

demo/frontend/src/common/components/toolbar/ToolbarProgressChip.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {OBJECT_TOOLBAR_INDEX} from '@/common/components/toolbar/ToolbarConfig';
+import useToolbarTabs from '@/common/components/toolbar/useToolbarTabs';
+import {streamingStateAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+import {useMemo} from 'react';
+import {Loading} from 'react-daisyui';
+
+const TOTAL_DEMO_STEPS = 3;
+
+export default function ToolbarProgressChip() {
+  const [toolbarIndex] = useToolbarTabs();
+  const streamingState = useAtomValue(streamingStateAtom);
+
+  const showLoader = useMemo(() => {
+    return streamingState === 'partial' || streamingState === 'requesting';
+  }, [streamingState]);
+
+  function getStepValue() {
+    if (toolbarIndex === OBJECT_TOOLBAR_INDEX) {
+      return streamingState !== 'full' ? 1 : 2;
+    }
+    return 3;
+  }
+
+  return (
+    <span className="inline-flex items-center justify-center rounded-full text-xs md:text-sm font-medium bg-white text-black w-10 md:w-12 h-5 md:h-6 mr-2 shrink-0 ">
+      {showLoader ? (
+        <Loading className="w-2 md:w-4" />
+      ) : (
+        `${getStepValue()}/${TOTAL_DEMO_STEPS}`
+      )}
+    </span>
+  );
+}

demo/frontend/src/common/components/toolbar/ToolbarSection.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {PropsWithChildren} from 'react';
+
+type Props = PropsWithChildren<{
+  title: string;
+  borderBottom?: boolean;
+}>;
+
+export default function ToolbarSection({
+  children,
+  title,
+  borderBottom = false,
+}: Props) {
+  return (
+    <div className={`p-6 ${borderBottom && 'border-b border-black'}`}>
+      <div className="font-bold ml-2">{title}</div>
+      <div className="grid grid-cols-4 gap-2 mt-2 md:mt-6">{children}</div>
+    </div>
+  );
+}

demo/frontend/src/common/components/toolbar/useListenToStreamingState.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {StreamingStateUpdateEvent} from '@/common/components/video/VideoWorkerBridge';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {StreamingState} from '@/common/tracker/Tracker';
+import {isStreamingAtom, streamingStateAtom} from '@/demo/atoms';
+import {useAtom} from 'jotai';
+import {useEffect} from 'react';
+
+export default function useListenToStreamingState(): {
+  isStreaming: boolean;
+  streamingState: StreamingState;
+} {
+  const [streamingState, setStreamingState] = useAtom(streamingStateAtom);
+  const [isStreaming, setIsStreaming] = useAtom(isStreamingAtom);
+  const video = useVideo();
+
+  useEffect(() => {
+    function onStreamingStateUpdate(event: StreamingStateUpdateEvent) {
+      setStreamingState(event.state);
+    }
+    function onStreamingStarted() {
+      setIsStreaming(true);
+    }
+    function onStreamingCompleted() {
+      setIsStreaming(false);
+    }
+    video?.addEventListener('streamingStateUpdate', onStreamingStateUpdate);
+    video?.addEventListener('streamingStarted', onStreamingStarted);
+    video?.addEventListener('streamingCompleted', onStreamingCompleted);
+
+    return () => {
+      video?.removeEventListener(
+        'streamingStateUpdate',
+        onStreamingStateUpdate,
+      );
+      video?.removeEventListener('streamingStarted', onStreamingStarted);
+      video?.removeEventListener('streamingCompleted', onStreamingCompleted);
+    };
+  }, [video, setStreamingState, setIsStreaming]);
+
+  return {isStreaming, streamingState};
+}

demo/frontend/src/common/components/toolbar/useToolbarTabs.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {toolbarTabIndex} from '@/demo/atoms';
+import {useAtom} from 'jotai';
+
+type State = [tabIndex: number, setTabIndex: (tabIndex: number) => void];
+
+export default function useToolbarTabs(): State {
+  return useAtom(toolbarTabIndex);
+}

demo/frontend/src/common/components/useFunctionThrottle.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {useCallback, useState} from 'react';
+
+type ThrottleOptions = {
+  enableThrottling?: boolean;
+};
+type State = {
+  isThrottled: boolean;
+  maxThrottles: boolean;
+  throttle: (callback: () => void, options?: ThrottleOptions) => void;
+};
+
+export default function useFunctionThrottle(
+  initialDelay: number,
+  numThrottles: number,
+): State {
+  const [isThrottled, setIsThrottled] = useState<boolean>(false);
+  const [lastClickTime, setLastClickTime] = useState<number | null>(null);
+  const [numTimesThrottled, setNumTimesThrottled] = useState<number>(1);
+
+  /**
+   * The following function's callback gets throttled when the time between two
+   * executions is less than a threshold.
+   *
+   * The threshold is calculated linearly by multiplying the initial delay
+   * and the number of times the button has been throttled. The button can be
+   * throttled up to numThrottles times.
+   *
+   * The function has an optional flag - enableThrottling - which allows a callsite
+   * to optionally disable throttling. This is useful in cases where throttling may
+   * not be necessary. (e.g. for the Track & Play button, we would only like to
+   * throttle after a stream is aborted.)
+   */
+  const throttle = useCallback(
+    (
+      callback: () => void,
+      options: ThrottleOptions = {
+        enableThrottling: true,
+      },
+    ) => {
+      if (isThrottled) {
+        return;
+      }
+
+      const currentTime = Date.now();
+      if (lastClickTime == null) {
+        callback();
+        setLastClickTime(currentTime);
+        return;
+      }
+
+      const timeBetweenClicks = currentTime - lastClickTime;
+      const delay = initialDelay * numTimesThrottled;
+      const shouldThrottle =
+        options.enableThrottling && delay > timeBetweenClicks;
+
+      if (shouldThrottle) {
+        setIsThrottled(true);
+        setTimeout(() => {
+          setIsThrottled(false);
+        }, delay);
+        setNumTimesThrottled(prev => {
+          return prev === numThrottles ? numThrottles : prev + 1;
+        });
+      }
+
+      callback();
+      setLastClickTime(currentTime);
+    },
+    [initialDelay, numThrottles, isThrottled, lastClickTime, numTimesThrottled],
+  );
+
+  return {
+    isThrottled,
+    maxThrottles: numTimesThrottled === numThrottles,
+    throttle,
+  };
+}

demo/frontend/src/common/components/video/ChangeVideoModal.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import type {VideoGalleryTriggerProps} from '@/common/components/gallery/DemoVideoGalleryModal';
+import DemoVideoGalleryModal from '@/common/components/gallery/DemoVideoGalleryModal';
+import useVideo from '@/common/components/video/editor/useVideo';
+import Logger from '@/common/logger/Logger';
+import {isStreamingAtom, uploadingStateAtom, VideoData} from '@/demo/atoms';
+import {useAtomValue, useSetAtom} from 'jotai';
+import {ComponentType, useCallback} from 'react';
+import {useNavigate} from 'react-router-dom';
+
+type Props = {
+  videoGalleryModalTrigger?: ComponentType<VideoGalleryTriggerProps>;
+  showUploadInGallery?: boolean;
+  onChangeVideo?: () => void;
+};
+
+export default function ChangeVideoModal({
+  videoGalleryModalTrigger: VideoGalleryModalTriggerComponent,
+  showUploadInGallery = true,
+  onChangeVideo,
+}: Props) {
+  const isStreaming = useAtomValue(isStreamingAtom);
+  const setUploadingState = useSetAtom(uploadingStateAtom);
+  const video = useVideo();
+  const navigate = useNavigate();
+
+  const handlePause = useCallback(() => {
+    video?.pause();
+  }, [video]);
+
+  function handlePauseOrAbortVideo() {
+    if (isStreaming) {
+      video?.abortStreamMasks();
+    } else {
+      handlePause();
+    }
+  }
+
+  function handleSwitchVideos(video: VideoData) {
+    // Retain any search parameter
+    navigate(
+      {
+        pathname: location.pathname,
+        search: location.search,
+      },
+      {
+        state: {
+          video,
+        },
+      },
+    );
+    onChangeVideo?.();
+  }
+
+  function handleUploadVideoError(error: Error) {
+    setUploadingState('error');
+    Logger.error(error);
+  }
+
+  return (
+    <DemoVideoGalleryModal
+      trigger={VideoGalleryModalTriggerComponent}
+      showUploadInGallery={showUploadInGallery}
+      onOpen={handlePauseOrAbortVideo}
+      onSelect={handleSwitchVideos}
+      onUploadVideoError={handleUploadVideoError}
+    />
+  );
+}

demo/frontend/src/common/components/video/EventEmitter.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+type EventMap<WorkerEventMap> = {
+  type: keyof WorkerEventMap;
+  listener: (ev: WorkerEventMap[keyof WorkerEventMap]) => unknown;
+};
+
+export class EventEmitter<WorkerEventMap> {
+  listeners: EventMap<WorkerEventMap>[] = [];
+
+  trigger<K extends keyof WorkerEventMap>(type: K, ev: WorkerEventMap[K]) {
+    this.listeners
+      .filter(listener => type === listener.type)
+      .forEach(({listener}) => {
+        setTimeout(() => listener(ev), 0);
+      });
+  }
+
+  addEventListener<K extends keyof WorkerEventMap>(
+    type: K,
+    listener: (ev: WorkerEventMap[K]) => unknown,
+  ): void {
+    // @ts-expect-error Incorrect typing. Not sure how to correctly type it
+    this.listeners.push({type, listener});
+  }
+
+  removeEventListener<K extends keyof WorkerEventMap>(
+    type: K,
+    listener: (ev: WorkerEventMap[K]) => unknown,
+  ): void {
+    this.listeners = this.listeners.filter(
+      existingListener =>
+        !(
+          existingListener.type === type &&
+          existingListener.listener === listener
+        ),
+    );
+  }
+
+  destroy() {
+    this.listeners.length = 0;
+  }
+}

demo/frontend/src/common/components/video/Video.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {BaseTracklet, SegmentationPoint} from '@/common/tracker/Tracker';
+import {TrackerOptions, Trackers} from '@/common/tracker/Trackers';
+import {PauseFilled, PlayFilledAlt} from '@carbon/icons-react';
+import stylex, {StyleXStyles} from '@stylexjs/stylex';
+import {
+  CSSProperties,
+  forwardRef,
+  useEffect,
+  useImperativeHandle,
+  useMemo,
+  useRef,
+} from 'react';
+import {Button} from 'react-daisyui';
+
+import {EffectIndex, Effects} from '@/common/components/video/effects/Effects';
+import useReportError from '@/common/error/useReportError';
+import Logger from '@/common/logger/Logger';
+import {isPlayingAtom, isVideoLoadingAtom} from '@/demo/atoms';
+import {color} from '@/theme/tokens.stylex';
+import {useAtom} from 'jotai';
+import useResizeObserver from 'use-resize-observer';
+import VideoLoadingOverlay from './VideoLoadingOverlay';
+import {
+  StreamingStateUpdateEvent,
+  VideoWorkerEventMap,
+} from './VideoWorkerBridge';
+import {EffectOptions} from './effects/Effect';
+import useVideoWorker from './useVideoWorker';
+
+const styles = stylex.create({
+  container: {
+    position: 'relative',
+    width: '100%',
+    height: '100%',
+  },
+  canvasContainer: {
+    display: 'flex',
+    justifyContent: 'center',
+    alignItems: 'center',
+    backgroundColor: color['gray-800'],
+    width: '100%',
+    height: '100%',
+  },
+  controls: {
+    position: 'absolute',
+    bottom: 0,
+    left: 0,
+    width: '100%',
+    padding: 8,
+    background: 'linear-gradient(#00000000, #000000ff)',
+  },
+  controlButton: {
+    color: 'white',
+  },
+});
+
+type Props = {
+  src: string;
+  width: number;
+  height: number;
+  loading?: boolean;
+  containerStyle?: StyleXStyles<{
+    position: CSSProperties['position'];
+  }>;
+  canvasStyle?: StyleXStyles<{
+    width: CSSProperties['width'];
+  }>;
+  controls?: boolean;
+  createVideoWorker?: () => Worker;
+};
+
+export type VideoRef = {
+  getCanvas(): HTMLCanvasElement | null;
+  get width(): number;
+  get height(): number;
+  get frame(): number;
+  set frame(index: number);
+  get numberOfFrames(): number;
+  play(): void;
+  pause(): void;
+  stop(): void;
+  previousFrame(): void;
+  nextFrame(): void;
+  setEffect(
+    name: keyof Effects,
+    index: EffectIndex,
+    options?: EffectOptions,
+  ): void;
+  encode(): void;
+  streamMasks(): void;
+  abortStreamMasks(): Promise<void>;
+  addEventListener<K extends keyof VideoWorkerEventMap>(
+    type: K,
+    listener: (ev: VideoWorkerEventMap[K]) => unknown,
+  ): void;
+  removeEventListener<K extends keyof VideoWorkerEventMap>(
+    type: K,
+    listener: (ev: VideoWorkerEventMap[K]) => unknown,
+  ): void;
+  createFilmstrip(width: number, height: number): Promise<ImageBitmap>;
+  // Tracker
+  initializeTracker(name: keyof Trackers, options?: TrackerOptions): void;
+  startSession(videoUrl: string): Promise<string | null>;
+  closeSession(): void;
+  logAnnotations(): void;
+  createTracklet(): Promise<BaseTracklet>;
+  deleteTracklet(trackletId: number): Promise<void>;
+  updatePoints(trackletId: number, points: SegmentationPoint[]): void;
+  clearPointsInVideo(): Promise<boolean>;
+  getWorker_ONLY_USE_WITH_CAUTION(): Worker;
+};
+
+export default forwardRef<VideoRef, Props>(function Video(
+  {
+    src,
+    width,
+    height,
+    containerStyle,
+    canvasStyle,
+    createVideoWorker,
+    controls = false,
+    loading = false,
+  },
+  ref,
+) {
+  const reportError = useReportError();
+  const canvasRef = useRef<HTMLCanvasElement>(null);
+  const [isPlaying, setIsPlaying] = useAtom(isPlayingAtom);
+  const [isVideoLoading, setIsVideoLoading] = useAtom(isVideoLoadingAtom);
+
+  const bridge = useVideoWorker(src, canvasRef, {
+    createVideoWorker,
+  });
+
+  const {
+    ref: resizeObserverRef,
+    width: resizeWidth = 1,
+    height: resizeHeight = 1,
+  } = useResizeObserver<HTMLDivElement>();
+
+  const canvasHeight = useMemo(() => {
+    const resizeRatio = resizeWidth / width;
+    return Math.min(height * resizeRatio, resizeHeight);
+  }, [resizeWidth, height, width, resizeHeight]);
+
+  useImperativeHandle(
+    ref,
+    () => ({
+      getCanvas() {
+        return canvasRef.current;
+      },
+      get width() {
+        return bridge.width;
+      },
+      get height() {
+        return bridge.width;
+      },
+      get frame() {
+        return bridge.frame;
+      },
+      set frame(index: number) {
+        bridge.frame = index;
+      },
+      get numberOfFrames() {
+        return bridge.numberOfFrames;
+      },
+      play(): void {
+        bridge.play();
+      },
+      pause(): void {
+        bridge.pause();
+      },
+      stop(): void {
+        bridge.stop();
+      },
+      previousFrame(): void {
+        bridge.previousFrame();
+      },
+      nextFrame(): void {
+        bridge.nextFrame();
+      },
+      setEffect(
+        name: keyof Effects,
+        index: number,
+        options?: EffectOptions,
+      ): void {
+        bridge.setEffect(name, index, options);
+      },
+      encode(): void {
+        bridge.encode();
+      },
+      streamMasks(): void {
+        bridge.streamMasks();
+      },
+      abortStreamMasks(): Promise<void> {
+        return bridge.abortStreamMasks();
+      },
+      addEventListener<K extends keyof VideoWorkerEventMap>(
+        type: K,
+        listener: (ev: VideoWorkerEventMap[K]) => unknown,
+      ): void {
+        bridge.addEventListener(type, listener);
+      },
+      removeEventListener<K extends keyof VideoWorkerEventMap>(
+        type: K,
+        listener: (ev: VideoWorkerEventMap[K]) => unknown,
+      ): void {
+        bridge.removeEventListener(type, listener);
+      },
+      createFilmstrip(width: number, height: number): Promise<ImageBitmap> {
+        return bridge.createFilmstrip(width, height);
+      },
+      // Tracker
+      initializeTracker(name: keyof Trackers, options: TrackerOptions): void {
+        bridge.initializeTracker(name, options);
+      },
+      startSession(videoUrl: string): Promise<string | null> {
+        return bridge.startSession(videoUrl);
+      },
+      closeSession(): void {
+        bridge.closeSession();
+      },
+      logAnnotations(): void {
+        bridge.logAnnotations();
+      },
+      createTracklet(): Promise<BaseTracklet> {
+        return bridge.createTracklet();
+      },
+      deleteTracklet(trackletId: number): Promise<void> {
+        return bridge.deleteTracklet(trackletId);
+      },
+      updatePoints(trackletId: number, points: SegmentationPoint[]): void {
+        bridge.updatePoints(trackletId, points);
+      },
+      clearPointsInVideo(): Promise<boolean> {
+        return bridge.clearPointsInVideo();
+      },
+      getWorker_ONLY_USE_WITH_CAUTION() {
+        return bridge.getWorker_ONLY_USE_WITH_CAUTION();
+      },
+    }),
+    [bridge],
+  );
+
+  // Handle video playback events (get playback state to main thread)
+  useEffect(() => {
+    let isPlaying = false;
+
+    function onFocus() {
+      // Workaround for Safari where the video frame renders black on
+      // unknown events. Trigger re-render frame on focus.
+      if (!isPlaying) {
+        bridge.goToFrame(bridge.frame);
+      }
+    }
+
+    function onVisibilityChange() {
+      // Workaround for Safari where the video frame renders black on
+      // visibility change hidden. Returning to visible shows a black
+      // frame instead of rendering the current frame.
+      if (document.visibilityState === 'visible' && !isPlaying) {
+        bridge.goToFrame(bridge.frame);
+      }
+    }
+
+    function onError(event: ErrorEvent) {
+      const error = event.error;
+      Logger.error(error);
+      reportError(error);
+    }
+
+    function onPlay() {
+      isPlaying = true;
+      setIsPlaying(true);
+    }
+    function onPause() {
+      isPlaying = false;
+      setIsPlaying(false);
+    }
+
+    function onStreamingDone(event: StreamingStateUpdateEvent) {
+      // continue to play after streaming is done (state is "full")
+      if (event.state === 'full') {
+        bridge.play();
+      }
+    }
+
+    function onLoadStart() {
+      setIsVideoLoading(true);
+    }
+
+    function onDecodeStart() {
+      setIsVideoLoading(false);
+    }
+
+    window.addEventListener('focus', onFocus);
+    window.addEventListener('visibilitychange', onVisibilityChange);
+    bridge.addEventListener('error', onError);
+    bridge.addEventListener('play', onPlay);
+    bridge.addEventListener('pause', onPause);
+    bridge.addEventListener('streamingStateUpdate', onStreamingDone);
+    bridge.addEventListener('loadstart', onLoadStart);
+    bridge.addEventListener('decode', onDecodeStart);
+    return () => {
+      window.removeEventListener('focus', onFocus);
+      window.removeEventListener('visibilitychange', onVisibilityChange);
+      bridge.removeEventListener('error', onError);
+      bridge.removeEventListener('play', onPlay);
+      bridge.removeEventListener('pause', onPause);
+      bridge.removeEventListener('streamingStateUpdate', onStreamingDone);
+      bridge.removeEventListener('loadstart', onLoadStart);
+      bridge.removeEventListener('decode', onDecodeStart);
+    };
+  }, [bridge, reportError, setIsPlaying, setIsVideoLoading]);
+
+  return (
+    <div
+      {...stylex.props(containerStyle ?? styles.container)}
+      ref={resizeObserverRef}>
+      <div {...stylex.props(styles.canvasContainer)}>
+        {(isVideoLoading || loading) && <VideoLoadingOverlay />}
+        <canvas
+          ref={canvasRef}
+          {...stylex.props(canvasStyle)}
+          className="lg:rounded-[4px]"
+          width={width}
+          height={height}
+          style={{
+            height: canvasHeight,
+          }}
+        />
+      </div>
+      {controls && (
+        <div {...stylex.props(styles.controls)}>
+          <Button
+            color="ghost"
+            size="xs"
+            startIcon={
+              isPlaying ? (
+                <PauseFilled
+                  {...stylex.props(styles.controlButton)}
+                  size={14}
+                />
+              ) : (
+                <PlayFilledAlt
+                  {...stylex.props(styles.controlButton)}
+                  size={14}
+                />
+              )
+            }
+            onClick={() => {
+              isPlaying ? bridge.pause() : bridge.play();
+            }}
+          />
+        </div>
+      )}
+    </div>
+  );
+});

demo/frontend/src/common/components/video/VideoFilmstripWithPlayback.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import PlaybackButton from '@/common/components/button/PlaybackButton';
+import VideoFilmstrip from '@/common/components/video/filmstrip/VideoFilmstrip';
+import {spacing, w} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    alignItems: 'end',
+    gap: spacing[4],
+    paddingHorizontal: spacing[4],
+    width: '100%',
+  },
+  playbackButtonContainer: {
+    display: 'flex',
+    justifyContent: 'center',
+    alignItems: 'center',
+    width: w[12],
+    height: w[12],
+  },
+  filmstripContainer: {
+    flexGrow: 1,
+  },
+});
+
+export default function VideoFilmstripWithPlayback() {
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div {...stylex.props(styles.playbackButtonContainer)}>
+        <PlaybackButton />
+      </div>
+      <div {...stylex.props(styles.filmstripContainer)}>
+        <VideoFilmstrip />
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/video/VideoLoadingOverlay.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {fontSize, fontWeight, spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+import {Loading} from 'react-daisyui';
+
+const styles = stylex.create({
+  overlay: {
+    position: 'absolute',
+    width: '100%',
+    height: '100%',
+    background: 'rgba(0,0,0,0.5)',
+  },
+  indicatorContainer: {
+    position: 'absolute',
+    top: '50%',
+    left: '50%',
+    transform: 'translate(-50%, -50%)',
+    display: 'flex',
+    alignItems: 'center',
+    gap: spacing[4],
+    color: 'white',
+  },
+  indicatorText: {
+    color: 'white',
+    fontSize: fontSize['sm'],
+    fontWeight: fontWeight['medium'],
+  },
+});
+
+type Props = {
+  label?: string;
+};
+
+export default function VideoLoadingOverlay({label}: Props) {
+  return (
+    <div {...stylex.props(styles.overlay)}>
+      <div {...stylex.props(styles.indicatorContainer)}>
+        <Loading size="sm" />
+        <div {...stylex.props(styles.indicatorText)}>
+          {label ?? 'Loading video...'}
+        </div>
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/video/VideoWorker.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {registerSerializableConstructors} from '@/common/error/ErrorSerializationUtils';
+import {Tracker} from '@/common/tracker/Tracker';
+import {TrackerRequestMessageEvent} from '@/common/tracker/TrackerTypes';
+import {TRACKER_MAPPING} from '@/common/tracker/Trackers';
+import {serializeError} from 'serialize-error';
+import VideoWorkerContext from './VideoWorkerContext';
+import {
+  ErrorResponse,
+  VideoWorkerRequestMessageEvent,
+} from './VideoWorkerTypes';
+
+registerSerializableConstructors();
+
+const context = new VideoWorkerContext();
+let tracker: Tracker | null = null;
+
+let statsEnabled = false;
+
+self.addEventListener(
+  'message',
+  async (
+    event: VideoWorkerRequestMessageEvent | TrackerRequestMessageEvent,
+  ) => {
+    try {
+      switch (event.data.action) {
+        // Initialize context
+        case 'setCanvas':
+          context.setCanvas(event.data.canvas);
+          break;
+        case 'setSource':
+          context.setSource(event.data.source);
+          break;
+
+        // Playback
+        case 'play':
+          context.play();
+          break;
+        case 'pause':
+          context.pause();
+          break;
+        case 'stop':
+          context.stop();
+          break;
+        case 'frameUpdate':
+          context.goToFrame(event.data.index);
+          break;
+
+        // Filmstrip
+        case 'filmstrip': {
+          const {width, height} = event.data;
+          await context.createFilmstrip(width, height);
+          break;
+        }
+
+        // Effects
+        case 'setEffect': {
+          const {name, index, options} = event.data;
+          await context.setEffect(name, index, options);
+          break;
+        }
+
+        // Encode
+        case 'encode': {
+          await context.encode();
+          break;
+        }
+
+        case 'enableStats': {
+          statsEnabled = true;
+          context.enableStats();
+          tracker?.enableStats();
+          break;
+        }
+
+        // Tracker
+        case 'initializeTracker': {
+          const {name, options} = event.data;
+          const Tracker = TRACKER_MAPPING[name];
+          // Update the endpoint for the streaming API
+          tracker = new Tracker(context, options);
+          if (statsEnabled) {
+            tracker.enableStats();
+          }
+          break;
+        }
+        case 'startSession': {
+          const {videoUrl} = event.data;
+          await tracker?.startSession(videoUrl);
+          break;
+        }
+        case 'createTracklet':
+          tracker?.createTracklet();
+          break;
+        case 'deleteTracklet':
+          await tracker?.deleteTracklet(event.data.trackletId);
+          break;
+        case 'closeSession':
+          tracker?.closeSession();
+          break;
+        case 'updatePoints': {
+          const {frameIndex, objectId, points} = event.data;
+          context.allowEffectAnimation(true, objectId, points);
+          await tracker?.updatePoints(frameIndex, objectId, points);
+          break;
+        }
+        case 'clearPointsInFrame': {
+          const {frameIndex, objectId} = event.data;
+          await tracker?.clearPointsInFrame(frameIndex, objectId);
+          break;
+        }
+        case 'clearPointsInVideo':
+          await tracker?.clearPointsInVideo();
+          break;
+        case 'streamMasks': {
+          const {frameIndex} = event.data;
+          context.allowEffectAnimation(false);
+          await tracker?.streamMasks(frameIndex);
+          break;
+        }
+        case 'abortStreamMasks':
+          tracker?.abortStreamMasks();
+          break;
+      }
+    } catch (error) {
+      const serializedError = serializeError(error);
+      const errorResponse: ErrorResponse = {
+        action: 'error',
+        error: serializedError,
+      };
+      self.postMessage(errorResponse);
+    }
+  },
+);

demo/frontend/src/common/components/video/VideoWorkerBridge.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {EffectIndex, Effects} from '@/common/components/video/effects/Effects';
+import {registerSerializableConstructors} from '@/common/error/ErrorSerializationUtils';
+import {
+  BaseTracklet,
+  SegmentationPoint,
+  StreamingState,
+} from '@/common/tracker/Tracker';
+import {
+  AbortStreamMasksRequest,
+  AddPointsResponse,
+  ClearPointsInFrameRequest,
+  ClearPointsInVideoRequest,
+  ClearPointsInVideoResponse,
+  CloseSessionRequest,
+  CreateTrackletRequest,
+  DeleteTrackletRequest,
+  InitializeTrackerRequest,
+  LogAnnotationsRequest,
+  SessionStartFailedResponse,
+  SessionStartedResponse,
+  StartSessionRequest,
+  StreamMasksRequest,
+  StreamingStateUpdateResponse,
+  TrackerRequest,
+  TrackerResponseMessageEvent,
+  TrackletCreatedResponse,
+  TrackletDeletedResponse,
+  UpdatePointsRequest,
+} from '@/common/tracker/TrackerTypes';
+import {TrackerOptions, Trackers} from '@/common/tracker/Trackers';
+import {MP4ArrayBuffer} from 'mp4box';
+import {deserializeError, type ErrorObject} from 'serialize-error';
+import {EventEmitter} from './EventEmitter';
+import {
+  EncodeVideoRequest,
+  FilmstripRequest,
+  FilmstripResponse,
+  FrameUpdateRequest,
+  PauseRequest,
+  PlayRequest,
+  SetCanvasRequest,
+  SetEffectRequest,
+  SetSourceRequest,
+  StopRequest,
+  VideoWorkerRequest,
+  VideoWorkerResponseMessageEvent,
+} from './VideoWorkerTypes';
+import {EffectOptions} from './effects/Effect';
+
+registerSerializableConstructors();
+
+export type DecodeEvent = {
+  totalFrames: number;
+  numFrames: number;
+  fps: number;
+  width: number;
+  height: number;
+  done: boolean;
+};
+
+export type LoadStartEvent = unknown;
+
+export type EffectUpdateEvent = {
+  name: keyof Effects;
+  index: EffectIndex;
+  variant: number;
+  numVariants: number;
+};
+
+export type EncodingStateUpdateEvent = {
+  progress: number;
+};
+
+export type EncodingCompletedEvent = {
+  file: MP4ArrayBuffer;
+};
+
+export interface PlayEvent {}
+
+export interface PauseEvent {}
+
+export interface FilmstripEvent {
+  filmstrip: ImageBitmap;
+}
+
+export interface FrameUpdateEvent {
+  index: number;
+}
+
+export interface SessionStartedEvent {
+  sessionId: string;
+}
+
+export interface SessionStartFailedEvent {}
+
+export interface TrackletCreatedEvent {
+  // Do not send masks between workers and main thread because they are huge,
+  // and sending them would eventually slow down the main thread.
+  tracklet: BaseTracklet;
+}
+
+export interface TrackletsEvent {
+  // Do not send masks between workers and main thread because they are huge,
+  // and sending them would eventually slow down the main thread.
+  tracklets: BaseTracklet[];
+}
+
+export interface TrackletDeletedEvent {
+  isSuccessful: boolean;
+}
+
+export interface AddPointsEvent {
+  isSuccessful: boolean;
+}
+
+export interface ClearPointsInVideoEvent {
+  isSuccessful: boolean;
+}
+
+export interface StreamingStartedEvent {}
+
+export interface StreamingCompletedEvent {}
+
+export interface StreamingStateUpdateEvent {
+  state: StreamingState;
+}
+
+export interface RenderingErrorEvent {
+  error: ErrorObject;
+}
+
+export interface VideoWorkerEventMap {
+  error: ErrorEvent;
+  decode: DecodeEvent;
+  encodingStateUpdate: EncodingStateUpdateEvent;
+  encodingCompleted: EncodingCompletedEvent;
+  play: PlayEvent;
+  pause: PauseEvent;
+  filmstrip: FilmstripEvent;
+  frameUpdate: FrameUpdateEvent;
+  sessionStarted: SessionStartedEvent;
+  sessionStartFailed: SessionStartFailedEvent;
+  trackletCreated: TrackletCreatedEvent;
+  trackletsUpdated: TrackletsEvent;
+  trackletDeleted: TrackletDeletedEvent;
+  addPoints: AddPointsEvent;
+  clearPointsInVideo: ClearPointsInVideoEvent;
+  streamingStarted: StreamingStartedEvent;
+  streamingCompleted: StreamingCompletedEvent;
+  streamingStateUpdate: StreamingStateUpdateEvent;
+  // HTMLVideoElement events https://developer.mozilla.org/en-US/docs/Web/HTML/Element/video#events
+  loadstart: LoadStartEvent;
+  effectUpdate: EffectUpdateEvent;
+  renderingError: RenderingErrorEvent;
+}
+
+type Metadata = {
+  totalFrames: number;
+  fps: number;
+  width: number;
+  height: number;
+};
+
+export default class VideoWorkerBridge extends EventEmitter<VideoWorkerEventMap> {
+  static create(workerFactory: () => Worker) {
+    const worker = workerFactory();
+    return new VideoWorkerBridge(worker);
+  }
+
+  protected worker: Worker;
+  private metadata: Metadata | null = null;
+  private frameIndex: number = 0;
+
+  private _sessionId: string | null = null;
+
+  public get sessionId() {
+    return this._sessionId;
+  }
+
+  public get width() {
+    return this.metadata?.width ?? 0;
+  }
+
+  public get height() {
+    return this.metadata?.height ?? 0;
+  }
+
+  public get numberOfFrames() {
+    return this.metadata?.totalFrames ?? 0;
+  }
+
+  public get fps() {
+    return this.metadata?.fps ?? 0;
+  }
+
+  public get frame() {
+    return this.frameIndex;
+  }
+
+  constructor(worker: Worker) {
+    super();
+    this.worker = worker;
+
+    worker.addEventListener(
+      'message',
+      (
+        event: VideoWorkerResponseMessageEvent | TrackerResponseMessageEvent,
+      ) => {
+        switch (event.data.action) {
+          case 'error':
+            // Deserialize error before triggering the event
+            event.data.error = deserializeError(event.data.error);
+            break;
+          case 'decode':
+            this.metadata = event.data;
+            break;
+          case 'frameUpdate':
+            this.frameIndex = event.data.index;
+            break;
+          case 'sessionStarted':
+            this._sessionId = event.data.sessionId;
+            break;
+        }
+        this.trigger(event.data.action, event.data);
+      },
+    );
+  }
+
+  public setCanvas(canvas: HTMLCanvasElement): void {
+    const offscreenCanvas = canvas.transferControlToOffscreen();
+    this.sendRequest<SetCanvasRequest>(
+      'setCanvas',
+      {
+        canvas: offscreenCanvas,
+      },
+      [offscreenCanvas],
+    );
+  }
+
+  public setSource(source: string): void {
+    this.sendRequest<SetSourceRequest>('setSource', {
+      source,
+    });
+  }
+
+  public terminate(): void {
+    super.destroy();
+    this.worker.terminate();
+  }
+
+  public play(): void {
+    this.sendRequest<PlayRequest>('play');
+  }
+
+  public pause(): void {
+    this.sendRequest<PauseRequest>('pause');
+  }
+
+  public stop(): void {
+    this.sendRequest<StopRequest>('stop');
+  }
+
+  public goToFrame(index: number): void {
+    this.sendRequest<FrameUpdateRequest>('frameUpdate', {
+      index,
+    });
+  }
+
+  public previousFrame(): void {
+    const index = Math.max(0, this.frameIndex - 1);
+    this.goToFrame(index);
+  }
+
+  public nextFrame(): void {
+    const index = Math.min(this.frameIndex + 1, this.numberOfFrames - 1);
+    this.goToFrame(index);
+  }
+
+  public set frame(index: number) {
+    this.sendRequest<FrameUpdateRequest>('frameUpdate', {index});
+  }
+
+  createFilmstrip(width: number, height: number): Promise<ImageBitmap> {
+    return new Promise((resolve, _reject) => {
+      const handleFilmstripResponse = (
+        event: MessageEvent<FilmstripResponse>,
+      ) => {
+        if (event.data.action === 'filmstrip') {
+          this.worker.removeEventListener('message', handleFilmstripResponse);
+          resolve(event.data.filmstrip);
+        }
+      };
+
+      this.worker.addEventListener('message', handleFilmstripResponse);
+
+      this.sendRequest<FilmstripRequest>('filmstrip', {
+        width,
+        height,
+      });
+    });
+  }
+
+  setEffect(name: keyof Effects, index: EffectIndex, options?: EffectOptions) {
+    this.sendRequest<SetEffectRequest>('setEffect', {
+      name,
+      index,
+      options,
+    });
+  }
+
+  encode(): void {
+    this.sendRequest<EncodeVideoRequest>('encode');
+  }
+
+  initializeTracker(name: keyof Trackers, options: TrackerOptions): void {
+    this.sendRequest<InitializeTrackerRequest>('initializeTracker', {
+      name,
+      options,
+    });
+  }
+
+  startSession(videoUrl: string): Promise<string | null> {
+    return new Promise(resolve => {
+      const handleResponse = (
+        event: MessageEvent<
+          SessionStartedResponse | SessionStartFailedResponse
+        >,
+      ) => {
+        if (event.data.action === 'sessionStarted') {
+          this.worker.removeEventListener('message', handleResponse);
+          resolve(event.data.sessionId);
+        }
+        if (event.data.action === 'sessionStartFailed') {
+          this.worker.removeEventListener('message', handleResponse);
+          resolve(null);
+        }
+      };
+
+      this.worker.addEventListener('message', handleResponse);
+      this.sendRequest<StartSessionRequest>('startSession', {
+        videoUrl,
+      });
+    });
+  }
+
+  closeSession(): void {
+    this.sendRequest<CloseSessionRequest>('closeSession');
+  }
+
+  logAnnotations(): void {
+    this.sendRequest<LogAnnotationsRequest>('logAnnotations');
+  }
+
+  createTracklet(): Promise<BaseTracklet> {
+    return new Promise(resolve => {
+      const handleResponse = (event: MessageEvent<TrackletCreatedResponse>) => {
+        if (event.data.action === 'trackletCreated') {
+          this.worker.removeEventListener('message', handleResponse);
+          resolve(event.data.tracklet);
+        }
+      };
+
+      this.worker.addEventListener('message', handleResponse);
+
+      this.sendRequest<CreateTrackletRequest>('createTracklet');
+    });
+  }
+
+  deleteTracklet(trackletId: number): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const handleResponse = (event: MessageEvent<TrackletDeletedResponse>) => {
+        if (event.data.action === 'trackletDeleted') {
+          this.worker.removeEventListener('message', handleResponse);
+          if (event.data.isSuccessful) {
+            resolve();
+          } else {
+            reject(`could not delete tracklet ${trackletId}`);
+          }
+        }
+      };
+      this.worker.addEventListener('message', handleResponse);
+      this.sendRequest<DeleteTrackletRequest>('deleteTracklet', {trackletId});
+    });
+  }
+
+  updatePoints(
+    objectId: number,
+    points: SegmentationPoint[],
+  ): Promise<boolean> {
+    return new Promise(resolve => {
+      const handleResponse = (event: MessageEvent<AddPointsResponse>) => {
+        if (event.data.action === 'addPoints') {
+          this.worker.removeEventListener('message', handleResponse);
+          resolve(event.data.isSuccessful);
+        }
+      };
+
+      this.worker.addEventListener('message', handleResponse);
+
+      this.sendRequest<UpdatePointsRequest>('updatePoints', {
+        frameIndex: this.frame,
+        objectId,
+        points,
+      });
+    });
+  }
+
+  clearPointsInFrame(objectId: number) {
+    this.sendRequest<ClearPointsInFrameRequest>('clearPointsInFrame', {
+      frameIndex: this.frame,
+      objectId,
+    });
+  }
+
+  clearPointsInVideo(): Promise<boolean> {
+    return new Promise(resolve => {
+      const handleResponse = (
+        event: MessageEvent<ClearPointsInVideoResponse>,
+      ) => {
+        if (event.data.action === 'clearPointsInVideo') {
+          this.worker.removeEventListener('message', handleResponse);
+          resolve(event.data.isSuccessful);
+        }
+      };
+      this.worker.addEventListener('message', handleResponse);
+      this.sendRequest<ClearPointsInVideoRequest>('clearPointsInVideo');
+    });
+  }
+
+  streamMasks(): void {
+    this.sendRequest<StreamMasksRequest>('streamMasks', {
+      frameIndex: this.frame,
+    });
+  }
+
+  abortStreamMasks(): Promise<void> {
+    return new Promise(resolve => {
+      const handleAbortResponse = (
+        event: MessageEvent<StreamingStateUpdateResponse>,
+      ) => {
+        if (
+          event.data.action === 'streamingStateUpdate' &&
+          event.data.state === 'aborted'
+        ) {
+          this.worker.removeEventListener('message', handleAbortResponse);
+          resolve();
+        }
+      };
+
+      this.worker.addEventListener('message', handleAbortResponse);
+      this.sendRequest<AbortStreamMasksRequest>('abortStreamMasks');
+    });
+  }
+
+  getWorker_ONLY_USE_WITH_CAUTION(): Worker {
+    return this.worker;
+  }
+
+  /**
+   * Convenient function to have typed postMessage.
+   *
+   * @param action Video worker action
+   * @param message Actual payload
+   * @param transfer Any object that should be transferred instead of cloned
+   */
+  protected sendRequest<T extends VideoWorkerRequest | TrackerRequest>(
+    action: T['action'],
+    payload?: Omit<T, 'action'>,
+    transfer?: Transferable[],
+  ) {
+    this.worker.postMessage(
+      {
+        action,
+        ...payload,
+      },
+      {
+        transfer,
+      },
+    );
+  }
+
+  // // Override EventEmitter
+
+  // addEventListener<K extends keyof WorkerEventMap>(
+  //   type: K,
+  //   listener: (ev: WorkerEventMap[K]) => unknown,
+  // ): void {
+  //   switch (type) {
+  //     case 'frameUpdate':
+  //       {
+  //         const event: FrameUpdateEvent = {
+  //           index: this.frameIndex,
+  //         };
+  //         // @ts-expect-error Incorrect typing. Not sure how to correctly type it
+  //         listener(event);
+  //       }
+  //       break;
+  //     case 'sessionStarted': {
+  //       if (this.sessionId !== null) {
+  //         const event: SessionStartedEvent = {
+  //           sessionId: this.sessionId,
+  //         };
+  //         // @ts-expect-error Incorrect typing. Not sure how to correctly type it
+  //         listener(event);
+  //       }
+  //     }
+  //   }
+  //   super.addEventListener(type, listener);
+  // }
+}

demo/frontend/src/common/components/video/VideoWorkerContext.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  DecodedVideo,
+  ImageFrame,
+  decodeStream,
+} from '@/common/codecs/VideoDecoder';
+import {encode as encodeVideo} from '@/common/codecs/VideoEncoder';
+import {
+  Effect,
+  EffectActionPoint,
+  EffectFrameContext,
+  EffectOptions,
+} from '@/common/components/video/effects/Effect';
+import AllEffects, {
+  EffectIndex,
+  Effects,
+} from '@/common/components/video/effects/Effects';
+import Logger from '@/common/logger/Logger';
+import {Mask, SegmentationPoint, Tracklet} from '@/common/tracker/Tracker';
+import {streamFile} from '@/common/utils/FileUtils';
+import {Stats} from '@/debug/stats/Stats';
+import {VIDEO_WATERMARK_TEXT} from '@/demo/DemoConfig';
+import CreateFilmstripError from '@/graphql/errors/CreateFilmstripError';
+import DrawFrameError from '@/graphql/errors/DrawFrameError';
+import WebGLContextError from '@/graphql/errors/WebGLContextError';
+import {RLEObject} from '@/jscocotools/mask';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+import {serializeError} from 'serialize-error';
+import {
+  DecodeResponse,
+  EffectUpdateResponse,
+  EncodingCompletedResponse,
+  EncodingStateUpdateResponse,
+  FilmstripResponse,
+  FrameUpdateResponse,
+  PauseRequest,
+  PlayRequest,
+  RenderingErrorResponse,
+  VideoWorkerResponse,
+} from './VideoWorkerTypes';
+
+function getEvenlySpacedItems(decodedVideo: DecodedVideo, x: number) {
+  const p = Math.floor(decodedVideo.numFrames / Math.max(1, x - 1));
+  const middleFrames = decodedVideo.frames
+    .slice(p, p * x)
+    .filter(function (_, i) {
+      return 0 == i % p;
+    });
+  return [
+    decodedVideo.frames[0],
+    ...middleFrames,
+    decodedVideo.frames[decodedVideo.numFrames - 1],
+  ];
+}
+
+export type FrameInfo = {
+  tracklet: Tracklet;
+  mask: Mask;
+};
+
+const WATERMARK_BOX_HORIZONTAL_PADDING = 10;
+const WATERMARK_BOX_VERTICAL_PADDING = 10;
+
+export type VideoStats = {
+  fps?: Stats;
+  videoFps?: Stats;
+  total?: Stats;
+  effect0?: Stats;
+  effect1?: Stats;
+  frameBmp?: Stats;
+  maskBmp?: Stats;
+  memory?: Stats;
+};
+
+export default class VideoWorkerContext {
+  private _canvas: OffscreenCanvas | null = null;
+  private _stats: VideoStats = {};
+  private _ctx: OffscreenCanvasRenderingContext2D | null = null;
+  private _form: CanvasForm | null = null;
+  private _decodedVideo: DecodedVideo | null = null;
+  private _frameIndex: number = 0;
+  private _isPlaying: boolean = false;
+  private _playbackRAFHandle: number | null = null;
+  private _playbackTimeoutHandle: NodeJS.Timeout | null = null;
+  private _isDrawing: boolean = false;
+  private _glObjects: WebGL2RenderingContext | null = null;
+  private _glBackground: WebGL2RenderingContext | null = null;
+  private _canvasHighlights: OffscreenCanvas | null = null;
+  private _canvasBackground: OffscreenCanvas | null = null;
+  private _allowAnimation: boolean = false;
+  private _currentSegmetationPoint: EffectActionPoint | null = null;
+
+  private _effects: Effect[];
+  private _tracklets: Tracklet[] = [];
+
+  public get width(): number {
+    return this._decodedVideo?.width ?? 0;
+  }
+
+  public get height(): number {
+    return this._decodedVideo?.height ?? 0;
+  }
+
+  public get frameIndex(): number {
+    return this._frameIndex;
+  }
+
+  public get currentFrame(): VideoFrame | null {
+    return this._decodedVideo?.frames[this._frameIndex].bitmap ?? null;
+  }
+
+  constructor() {
+    this._effects = [
+      AllEffects.Original, // Image as background
+      AllEffects.Overlay, // Masks on top
+    ];
+
+    // Loading watermark fonts. This is going to be async, but by the time of
+    // video encoding, the fonts should be available.
+    this._loadWatermarkFonts();
+  }
+
+  private initializeWebGLContext(width: number, height: number): void {
+    // Given that we use highlight and background effects as layers,
+    // we need to create two WebGL contexts, one for each set.
+    // To avoid memory leaks and too many active contexts,
+    // these contexts must be re-used over the lifecycle of the session.
+
+    if (this._canvasHighlights == null && this._glObjects == null) {
+      this._canvasHighlights = new OffscreenCanvas(width, height);
+      this._glObjects = this._canvasHighlights.getContext('webgl2');
+
+      this._canvasHighlights.addEventListener(
+        'webglcontextlost',
+        event => {
+          event.preventDefault();
+          this._sendRenderingError(
+            new WebGLContextError('WebGL context lost.'),
+          );
+        },
+        false,
+      );
+    } else if (
+      this._canvasHighlights != null &&
+      (this._canvasHighlights.width !== width ||
+        this._canvasHighlights.height !== height)
+    ) {
+      // Resize canvas and webgl viewport
+      this._canvasHighlights.width = width;
+      this._canvasHighlights.height = height;
+      if (this._glObjects != null) {
+        this._glObjects.viewport(0, 0, width, height);
+      }
+    }
+
+    if (this._canvasBackground == null && this._glBackground == null) {
+      this._canvasBackground = new OffscreenCanvas(width, height);
+      this._glBackground = this._canvasBackground.getContext('webgl2');
+
+      this._canvasBackground.addEventListener(
+        'webglcontextlost',
+        event => {
+          event.preventDefault();
+          this._sendRenderingError(
+            new WebGLContextError('WebGL context lost.'),
+          );
+        },
+        false,
+      );
+    } else if (
+      this._canvasBackground != null &&
+      (this._canvasBackground.width != width ||
+        this._canvasBackground.height != height)
+    ) {
+      // Resize canvas and webgl viewport
+      this._canvasBackground.width = width;
+      this._canvasBackground.height = height;
+      if (this._glBackground != null) {
+        this._glBackground.viewport(0, 0, width, height);
+      }
+    }
+  }
+
+  public setCanvas(canvas: OffscreenCanvas) {
+    this._canvas = canvas;
+    this._ctx = canvas.getContext('2d');
+    if (this._ctx == null) {
+      throw new Error('could not initialize drawing context');
+    }
+    this._form = new CanvasForm(this._ctx);
+  }
+
+  public setSource(src: string) {
+    this.close();
+
+    // Clear state of previous source.
+    this.updateFrameIndex(0);
+    this._tracklets = [];
+
+    this._decodeVideo(src);
+  }
+
+  public goToFrame(index: number): void {
+    // Cancel any ongoing render
+    this._cancelRender();
+    this.updateFrameIndex(index);
+    this._playbackRAFHandle = requestAnimationFrame(this._drawFrame.bind(this));
+  }
+
+  public play(): void {
+    // Video already playing
+    if (this._isPlaying) {
+      return;
+    }
+
+    // Cannot playback without frames
+    if (this._decodedVideo === null) {
+      throw new Error('no decoded video');
+    }
+
+    const {numFrames, fps} = this._decodedVideo;
+    const timePerFrame = 1000 / (fps ?? 30);
+    let startTime: number | null = null;
+    // The offset frame index compensate for cases where the video playback
+    // does not start at frame index 0.
+    const offsetFrameIndex = this._frameIndex;
+
+    const updateFrame = (time: number) => {
+      if (startTime === null) {
+        startTime = time;
+      }
+
+      this._stats.fps?.begin();
+
+      const diff = time - startTime;
+      const expectedFrame =
+        (Math.floor(diff / timePerFrame) + offsetFrameIndex) % numFrames;
+
+      if (this._frameIndex !== expectedFrame && !this._isDrawing) {
+        // Update to the next expected frame
+        this.updateFrameIndex(expectedFrame);
+        this._drawFrame();
+      }
+
+      this._playbackRAFHandle = requestAnimationFrame(updateFrame);
+
+      this._stats.fps?.end();
+    };
+
+    this.updatePlayback(true);
+    this._playbackRAFHandle = requestAnimationFrame(updateFrame);
+  }
+
+  public pause(): void {
+    this.updatePlayback(false);
+    this._cancelRender();
+  }
+
+  public stop(): void {
+    this.pause();
+    this.updateFrameIndex(0);
+  }
+
+  public async createFilmstrip(width: number, height: number): Promise<void> {
+    if (width < 1 || height < 1) {
+      Logger.warn(
+        `Cannot create filmstrip because width ${width} or height ${height} is too small.`,
+      );
+      return;
+    }
+
+    try {
+      const canvas = new OffscreenCanvas(width, height);
+      const ctx = canvas.getContext('2d');
+
+      if (this._decodedVideo !== null) {
+        const scale = canvas.height / this._decodedVideo.height;
+        const resizeWidth = this._decodedVideo.width * scale;
+
+        const spacedFrames = getEvenlySpacedItems(
+          this._decodedVideo,
+          Math.ceil(canvas.width / resizeWidth),
+        );
+
+        spacedFrames.forEach((frame, idx) => {
+          if (frame != null) {
+            ctx?.drawImage(
+              frame.bitmap,
+              resizeWidth * idx,
+              0,
+              resizeWidth,
+              canvas.height,
+            );
+          }
+        });
+      }
+
+      const filmstrip = await createImageBitmap(canvas);
+
+      this.sendResponse<FilmstripResponse>(
+        'filmstrip',
+        {
+          filmstrip,
+        },
+        [filmstrip],
+      );
+    } catch {
+      this._sendRenderingError(
+        new CreateFilmstripError('Failed to create filmstrip'),
+      );
+    }
+  }
+
+  public async setEffect(
+    name: keyof Effects,
+    index: EffectIndex,
+    options?: EffectOptions,
+  ): Promise<void> {
+    const effect: Effect = AllEffects[name];
+
+    // The effect has changed.
+    if (this._effects[index] !== effect) {
+      // Effect changed. Cleanup old effect first. Effects are responsible for
+      // cleaning up their memory.
+      await this._effects[index].cleanup();
+
+      const offCanvas =
+        index === EffectIndex.BACKGROUND
+          ? this._canvasBackground
+          : this._canvasHighlights;
+      invariant(offCanvas != null, 'need OffscreenCanvas to render effects');
+      const webglContext =
+        index === EffectIndex.BACKGROUND ? this._glBackground : this._glObjects;
+      invariant(webglContext != null, 'need WebGL context to render effects');
+
+      // Initialize the effect. This can be used by effects to prepare
+      // resources needed for rendering. If the video wasn't decoded yet, the
+      // effect setup will happen in the _decodeVideo function.
+      if (this._decodedVideo != null) {
+        await effect.setup({
+          width: this._decodedVideo.width,
+          height: this._decodedVideo.height,
+          canvas: offCanvas,
+          gl: webglContext,
+        });
+      }
+    }
+
+    // Update effect if already set effect was clicked again. This can happen
+    // when there is a new variant of the effect.
+    if (options != null) {
+      // Update effect if already set effect was clicked again. This can happen
+      // when there is a new variant of the effect.
+      await effect.update(options);
+    }
+
+    // Notify the frontend about the effect state including its variant.
+    this.sendResponse<EffectUpdateResponse>('effectUpdate', {
+      name,
+      index,
+      variant: effect.variant,
+      numVariants: effect.numVariants,
+    });
+
+    this._effects[index] = effect;
+    this._playbackRAFHandle = requestAnimationFrame(this._drawFrame.bind(this));
+  }
+
+  async encode() {
+    const decodedVideo = this._decodedVideo;
+    invariant(
+      decodedVideo !== null,
+      'cannot encode video because there is no decoded video available',
+    );
+
+    const canvas = new OffscreenCanvas(this.width, this.height);
+    const ctx = canvas.getContext('2d', {willReadFrequently: true});
+    invariant(
+      ctx !== null,
+      'cannot encode video because failed to construct offscreen canvas context',
+    );
+
+    const form = new CanvasForm(ctx);
+
+    const file = await encodeVideo(
+      this.width,
+      this.height,
+      decodedVideo.frames.length,
+      this._framesGenerator(decodedVideo, canvas, form),
+      progress => {
+        this.sendResponse<EncodingStateUpdateResponse>('encodingStateUpdate', {
+          progress,
+        });
+      },
+    );
+    this.sendResponse<EncodingCompletedResponse>(
+      'encodingCompleted',
+      {
+        file,
+      },
+      [file],
+    );
+  }
+
+  private async *_framesGenerator(
+    decodedVideo: DecodedVideo,
+    canvas: OffscreenCanvas,
+    form: CanvasForm,
+  ): AsyncGenerator<ImageFrame, undefined> {
+    const frames = decodedVideo.frames;
+
+    for (let frameIndex = 0; frameIndex < frames.length; ++frameIndex) {
+      await this._drawFrameImpl(form, frameIndex, true);
+
+      const frame = frames[frameIndex];
+      const videoFrame = new VideoFrame(canvas, {
+        timestamp: frame.bitmap.timestamp,
+      });
+
+      yield {
+        bitmap: videoFrame,
+        timestamp: frame.timestamp,
+        duration: frame.duration,
+      };
+
+      videoFrame.close();
+    }
+  }
+
+  public enableStats() {
+    this._stats.fps = new Stats('fps');
+    this._stats.videoFps = new Stats('fps', 'V');
+    this._stats.total = new Stats('ms', 'T');
+    this._stats.effect0 = new Stats('ms', 'B');
+    this._stats.effect1 = new Stats('ms', 'H');
+    this._stats.frameBmp = new Stats('ms', 'F');
+    this._stats.maskBmp = new Stats('ms', 'M');
+    this._stats.memory = new Stats('memory');
+  }
+
+  public allowEffectAnimation(
+    allow: boolean = true,
+    objectId?: number,
+    points?: SegmentationPoint[],
+  ) {
+    if (objectId != null && points != null && points.length) {
+      const last_point_position = points[points.length - 1];
+      this._currentSegmetationPoint = {
+        objectId,
+        position: [last_point_position[0], last_point_position[1]],
+      };
+    }
+
+    if (!allow) {
+      this._currentSegmetationPoint = null;
+    }
+
+    this._allowAnimation = allow;
+  }
+
+  public close(): void {
+    // Clear any frame content
+    this._ctx?.reset();
+
+    // Close frames of previously decoded video.
+    this._decodedVideo?.frames.forEach(f => f.bitmap.close());
+    this._decodedVideo = null;
+  }
+
+  // TRACKER
+
+  public updateTracklets(
+    frameIndex: number,
+    tracklets: Tracklet[],
+    shouldGoToFrame: boolean = true,
+  ): void {
+    this._tracklets = tracklets;
+    if (shouldGoToFrame) {
+      this.goToFrame(frameIndex);
+    }
+  }
+
+  public clearTrackletMasks(tracklet: Tracklet): void {
+    this._tracklets = this._tracklets.filter(t => t.id != tracklet.id);
+  }
+
+  public clearMasks(): void {
+    this._tracklets = [];
+  }
+
+  // PRIVATE FUNCTIONS
+
+  private sendResponse<T extends VideoWorkerResponse>(
+    action: T['action'],
+    message?: Omit<T, 'action'>,
+    transfer?: Transferable[],
+  ): void {
+    self.postMessage(
+      {
+        action,
+        ...message,
+      },
+      {
+        transfer,
+      },
+    );
+  }
+
+  private async _decodeVideo(src: string): Promise<void> {
+    const canvas = this._canvas;
+    invariant(canvas != null, 'need canvas to render decoded video');
+
+    this.sendResponse('loadstart');
+
+    const fileStream = streamFile(src, {
+      credentials: 'same-origin',
+      cache: 'no-store',
+    });
+
+    let renderedFirstFrame = false;
+    this._decodedVideo = await decodeStream(fileStream, async progress => {
+      const {fps, height, width, numFrames, frames} = progress;
+      this._decodedVideo = progress;
+      if (!renderedFirstFrame) {
+        renderedFirstFrame = true;
+        canvas.width = width;
+        canvas.height = height;
+        // Set WebGL contexts right after the first frame decoded
+        this.initializeWebGLContext(width, height);
+
+        // Initialize effect once first frame was decoded.
+        for (const [i, effect] of this._effects.entries()) {
+          const offCanvas =
+            i === EffectIndex.BACKGROUND
+              ? this._canvasBackground
+              : this._canvasHighlights;
+          invariant(offCanvas != null, 'need canvas to render effects');
+          const webglContext =
+            i === EffectIndex.BACKGROUND ? this._glBackground : this._glObjects;
+          invariant(
+            webglContext != null,
+            'need WebGL context to render effects',
+          );
+          await effect.setup({
+            width,
+            height,
+            canvas: offCanvas,
+            gl: webglContext,
+          });
+        }
+
+        // Need to render frame immediately. Cannot go through
+        // requestAnimationFrame because then rendering this frame would be
+        // delayed until the full video has finished decoding.
+        this._drawFrame();
+
+        this._stats.videoFps?.updateMaxValue(fps);
+        this._stats.total?.updateMaxValue(1000 / fps);
+        this._stats.effect0?.updateMaxValue(1000 / fps);
+        this._stats.effect1?.updateMaxValue(1000 / fps);
+        this._stats.frameBmp?.updateMaxValue(1000 / fps);
+        this._stats.maskBmp?.updateMaxValue(1000 / fps);
+      }
+      this.sendResponse<DecodeResponse>('decode', {
+        totalFrames: numFrames,
+        numFrames: frames.length,
+        fps: fps,
+        width: width,
+        height: height,
+        done: false,
+      });
+    });
+
+    if (!renderedFirstFrame) {
+      canvas.width = this._decodedVideo.width;
+      canvas.height = this._decodedVideo.height;
+      this._drawFrame();
+    }
+
+    this.sendResponse<DecodeResponse>('decode', {
+      totalFrames: this._decodedVideo.numFrames,
+      numFrames: this._decodedVideo.frames.length,
+      fps: this._decodedVideo.fps,
+      width: this._decodedVideo.width,
+      height: this._decodedVideo.height,
+      done: true,
+    });
+  }
+
+  private _drawFrame(): void {
+    if (this._canvas !== null && this._form !== null) {
+      this._drawFrameImpl(this._form, this._frameIndex);
+    }
+  }
+
+  private async _drawFrameImpl(
+    form: CanvasForm,
+    frameIndex: number,
+    enableWatermark: boolean = false,
+    step: number = 0,
+    maxSteps: number = 40,
+  ): Promise<void> {
+    if (this._decodedVideo === null) {
+      return;
+    }
+
+    {
+      this._stats.videoFps?.begin();
+      this._stats.total?.begin();
+      this._stats.memory?.begin();
+    }
+
+    try {
+      const frame = this._decodedVideo.frames[frameIndex];
+      const {bitmap} = frame;
+
+      this._stats.frameBmp?.begin();
+
+      // Need to convert VideoFrame to ImageBitmap because Safari can only apply
+      // globalCompositeOperation on ImageBitmap and fails on VideoFrame. FWIW,
+      // Chrome treats VideoFrame similarly to ImageBitmap.
+      const frameBitmap = await createImageBitmap(bitmap);
+
+      this._stats.frameBmp?.end();
+
+      const masks: Mask[] = [];
+      const colors: string[] = [];
+      const tracklets: Tracklet[] = [];
+      this._tracklets.forEach(tracklet => {
+        const mask = tracklet.masks[frameIndex];
+        if (mask != null) {
+          masks.push(mask);
+          tracklets.push(tracklet);
+          colors.push(tracklet.color);
+        }
+      });
+      const effectActionPoint = this._currentSegmetationPoint;
+
+      this._stats.maskBmp?.begin();
+
+      const effectMaskPromises = masks.map(async ({data, bounds}) => {
+        return {
+          bounds,
+          bitmap: data as RLEObject,
+        };
+      });
+      const effectMasks = await Promise.all(effectMaskPromises);
+
+      this._stats.maskBmp?.end();
+
+      form.ctx.fillStyle = 'rgba(0, 0, 0, 0)';
+      form.ctx.fillRect(0, 0, this.width, this.height);
+
+      const effectParams: EffectFrameContext = {
+        frame: frameBitmap,
+        masks: effectMasks,
+        maskColors: colors,
+        frameIndex: frameIndex,
+        totalFrames: this._decodedVideo.frames.length,
+        fps: this._decodedVideo.fps,
+        width: frameBitmap.width,
+        height: frameBitmap.height,
+        actionPoint: null,
+      };
+
+      // Allows animation within a single frame.
+      if (this._allowAnimation && step < maxSteps) {
+        const animationDuration = 2; // Total duration of the animation in seconds
+        const progress = step / maxSteps;
+        const timeParameter = progress * animationDuration;
+        // Pass dynamic effect params
+        effectParams.timeParameter = timeParameter;
+        effectParams.actionPoint = effectActionPoint;
+
+        this._processEffects(form, effectParams, tracklets);
+
+        // Use RAF to draw frame, and update the display,
+        // this avoids to wait until the javascript call stack is cleared.
+        requestAnimationFrame(() =>
+          this._drawFrameImpl(form, frameIndex, false, step + 1, maxSteps),
+        );
+      } else {
+        this._processEffects(form, effectParams, tracklets);
+      }
+
+      if (enableWatermark) {
+        this._drawWatermark(form, frameBitmap);
+      }
+
+      // Do not simply drop the JavaScript reference to the ImageBitmap; doing so
+      // will keep its graphics resource alive until the next time the garbage
+      // collector runs.
+      frameBitmap.close();
+
+      {
+        this._stats.videoFps?.end();
+        this._stats.total?.end();
+        this._stats.memory?.end();
+      }
+
+      this._isDrawing = false;
+    } catch {
+      this._sendRenderingError(new DrawFrameError('Failed to draw frame'));
+    }
+  }
+
+  private _drawWatermark(form: CanvasForm, frameBitmap: ImageBitmap): void {
+    const frameWidth = this._canvas?.width || frameBitmap.width;
+    const frameHeight = this._canvas?.height || frameBitmap.height;
+    // Font size is either 12 or smaller based on available width
+    // since the font is not monospaced, we approximate it'll fit 1.5 more characters than monospaced
+    const approximateFontSize = Math.min(
+      Math.floor(frameWidth / (VIDEO_WATERMARK_TEXT.length / 1.5)),
+      12,
+    );
+
+    form.ctx.font = `${approximateFontSize}px "Inter", sans-serif`;
+    const measureGeneratedBy = form.ctx.measureText(VIDEO_WATERMARK_TEXT);
+
+    const textBoxWidth =
+      measureGeneratedBy.width + 2 * WATERMARK_BOX_HORIZONTAL_PADDING;
+    const textBoxHeight =
+      measureGeneratedBy.actualBoundingBoxAscent +
+      2 * WATERMARK_BOX_VERTICAL_PADDING;
+    const textBoxX = frameWidth - textBoxWidth;
+    const textBoxY = frameHeight - textBoxHeight;
+
+    form.ctx.fillStyle = 'rgba(0, 0, 0, 0.4)';
+    form.ctx.beginPath();
+    form.ctx.roundRect(
+      Math.round(textBoxX),
+      Math.round(textBoxY),
+      Math.round(textBoxWidth),
+      Math.round(textBoxHeight),
+      [WATERMARK_BOX_HORIZONTAL_PADDING, 0, 0, 0],
+    );
+    form.ctx.fill();
+
+    // Always reset the text style because some effects may change text styling in the same ctx
+    form.ctx.fillStyle = 'rgba(255, 255, 255, 0.8)';
+    form.ctx.textAlign = 'left';
+
+    form.ctx.fillText(
+      VIDEO_WATERMARK_TEXT,
+      Math.round(textBoxX + WATERMARK_BOX_HORIZONTAL_PADDING),
+      Math.round(
+        textBoxY +
+          WATERMARK_BOX_VERTICAL_PADDING +
+          measureGeneratedBy.actualBoundingBoxAscent,
+      ),
+    );
+  }
+
+  private updateFrameIndex(index: number): void {
+    this._frameIndex = index;
+    this.sendResponse<FrameUpdateResponse>('frameUpdate', {
+      index,
+    });
+  }
+
+  private _loadWatermarkFonts() {
+    const requiredFonts = [
+      {
+        url: '/fonts/Inter-VariableFont.ttf',
+        format: 'truetype-variations',
+      },
+    ];
+    requiredFonts.forEach(requiredFont => {
+      const fontFace = new FontFace(
+        'Inter',
+        `url(${requiredFont.url}) format('${requiredFont.format}')`,
+      );
+      fontFace.load().then(font => {
+        self.fonts.add(font);
+      });
+    });
+  }
+
+  private updatePlayback(playing: boolean): void {
+    if (playing) {
+      this.sendResponse<PlayRequest>('play');
+    } else {
+      this.sendResponse<PauseRequest>('pause');
+    }
+    this._isPlaying = playing;
+  }
+
+  private _cancelRender(): void {
+    if (this._playbackTimeoutHandle !== null) {
+      clearTimeout(this._playbackTimeoutHandle);
+      this._playbackTimeoutHandle = null;
+    }
+    if (this._playbackRAFHandle !== null) {
+      cancelAnimationFrame(this._playbackRAFHandle);
+      this._playbackRAFHandle = null;
+    }
+  }
+
+  private _sendRenderingError(error: Error): void {
+    this.sendResponse<RenderingErrorResponse>('renderingError', {
+      error: serializeError(error),
+    });
+  }
+
+  private _processEffects(
+    form: CanvasForm,
+    effectParams: EffectFrameContext,
+    tracklets: Tracklet[],
+  ) {
+    for (let i = 0; i < this._effects.length; i++) {
+      const effect = this._effects[i];
+
+      if (i === 0) {
+        this._stats.effect0?.begin();
+      } else if (i === 1) {
+        this._stats.effect1?.begin();
+      }
+
+      effect.apply(form, effectParams, tracklets);
+
+      if (i === 0) {
+        this._stats.effect0?.end();
+      } else if (i === 1) {
+        this._stats.effect1?.end();
+      }
+    }
+  }
+}

demo/frontend/src/common/components/video/VideoWorkerTypes.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  DecodeEvent,
+  EffectUpdateEvent,
+  EncodingCompletedEvent,
+  EncodingStateUpdateEvent,
+  FilmstripEvent,
+  FrameUpdateEvent,
+  LoadStartEvent,
+  RenderingErrorEvent,
+} from './VideoWorkerBridge';
+import {EffectOptions} from './effects/Effect';
+import type {Effects} from './effects/Effects';
+
+export type Request<A, P> = {
+  action: A;
+} & P;
+
+// REQUESTS
+
+export type SetCanvasRequest = Request<
+  'setCanvas',
+  {
+    canvas: OffscreenCanvas;
+  }
+>;
+export type SetSourceRequest = Request<
+  'setSource',
+  {
+    source: string;
+  }
+>;
+export type PlayRequest = Request<'play', unknown>;
+export type PauseRequest = Request<'pause', unknown>;
+export type StopRequest = Request<'stop', unknown>;
+export type FrameUpdateRequest = Request<
+  'frameUpdate',
+  {
+    index: number;
+  }
+>;
+export type FilmstripRequest = Request<
+  'filmstrip',
+  {
+    width: number;
+    height: number;
+  }
+>;
+export type SetEffectRequest = Request<
+  'setEffect',
+  {
+    name: keyof Effects;
+    index: number;
+    options?: EffectOptions;
+  }
+>;
+
+export type EncodeVideoRequest = Request<'encode', unknown>;
+
+export type EnableStatsRequest = Request<'enableStats', unknown>;
+
+export type VideoWorkerRequest =
+  | SetCanvasRequest
+  | SetSourceRequest
+  | PlayRequest
+  | PauseRequest
+  | StopRequest
+  | FrameUpdateRequest
+  | FilmstripRequest
+  | SetEffectRequest
+  | EncodeVideoRequest
+  | EnableStatsRequest;
+
+export type VideoWorkerRequestMessageEvent = MessageEvent<VideoWorkerRequest>;
+
+// RESPONSES
+
+export type ErrorResponse = Request<
+  'error',
+  {
+    error: unknown;
+  }
+>;
+
+export type DecodeResponse = Request<'decode', DecodeEvent>;
+
+export type EncodingStateUpdateResponse = Request<
+  'encodingStateUpdate',
+  EncodingStateUpdateEvent
+>;
+
+export type EncodingCompletedResponse = Request<
+  'encodingCompleted',
+  EncodingCompletedEvent
+>;
+
+export type FilmstripResponse = Request<'filmstrip', FilmstripEvent>;
+
+export type PlayResponse = Request<'play', unknown>;
+
+export type PauseResponse = Request<'pause', unknown>;
+
+export type FrameUpdateResponse = Request<'frameUpdate', FrameUpdateEvent>;
+
+export type RenderingErrorResponse = Request<
+  'renderingError',
+  RenderingErrorEvent
+>;
+
+// HTMLVideoElement events https://developer.mozilla.org/en-US/docs/Web/HTML/Element/video#events
+
+export type LoadStartResponse = Request<'loadstart', LoadStartEvent>;
+
+export type EffectUpdateResponse = Request<'effectUpdate', EffectUpdateEvent>;
+
+export type VideoWorkerResponse =
+  | ErrorResponse
+  | FilmstripResponse
+  | DecodeResponse
+  | EncodingStateUpdateResponse
+  | EncodingCompletedResponse
+  | PlayResponse
+  | PauseResponse
+  | FrameUpdateResponse
+  | LoadStartResponse
+  | RenderingErrorResponse
+  | EffectUpdateResponse;
+
+export type VideoWorkerResponseMessageEvent = MessageEvent<VideoWorkerResponse>;

demo/frontend/src/common/components/video/editor/DemoVideoEditor.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import TrackletsAnnotation from '@/common/components/annotations/TrackletsAnnotation';
+import useCloseSessionBeforeUnload from '@/common/components/session/useCloseSessionBeforeUnload';
+import MessagesSnackbar from '@/common/components/snackbar/MessagesSnackbar';
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import {OBJECT_TOOLBAR_INDEX} from '@/common/components/toolbar/ToolbarConfig';
+import useToolbarTabs from '@/common/components/toolbar/useToolbarTabs';
+import VideoFilmstripWithPlayback from '@/common/components/video/VideoFilmstripWithPlayback';
+import {
+  FrameUpdateEvent,
+  RenderingErrorEvent,
+  SessionStartedEvent,
+  TrackletsEvent,
+} from '@/common/components/video/VideoWorkerBridge';
+import VideoEditor from '@/common/components/video/editor/VideoEditor';
+import useResetDemoEditor from '@/common/components/video/editor/useResetEditor';
+import useVideo from '@/common/components/video/editor/useVideo';
+import InteractionLayer from '@/common/components/video/layers/InteractionLayer';
+import {PointsLayer} from '@/common/components/video/layers/PointsLayer';
+import LoadingStateScreen from '@/common/loading/LoadingStateScreen';
+import UploadLoadingScreen from '@/common/loading/UploadLoadingScreen';
+import useScreenSize from '@/common/screen/useScreenSize';
+import {SegmentationPoint} from '@/common/tracker/Tracker';
+import {
+  activeTrackletObjectIdAtom,
+  frameIndexAtom,
+  isAddObjectEnabledAtom,
+  isPlayingAtom,
+  isVideoLoadingAtom,
+  pointsAtom,
+  sessionAtom,
+  streamingStateAtom,
+  trackletObjectsAtom,
+  uploadingStateAtom,
+  VideoData,
+} from '@/demo/atoms';
+import useSettingsContext from '@/settings/useSettingsContext';
+import {color, spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+import {useAtom, useAtomValue, useSetAtom} from 'jotai';
+import {useEffect, useState} from 'react';
+import type {ErrorObject} from 'serialize-error';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    flexDirection: 'column',
+    overflow: 'auto',
+    width: '100%',
+    borderColor: color['gray-800'],
+    backgroundColor: color['gray-800'],
+    borderWidth: 8,
+    borderRadius: 12,
+    '@media screen and (max-width: 768px)': {
+      // on mobile, we want to grow the editor container so that the editor
+      // fills the remaining vertical space between the navbar and bottom
+      // of the page
+      flexGrow: 1,
+      borderWidth: 0,
+      borderRadius: 0,
+      paddingBottom: spacing[4],
+    },
+  },
+  loadingScreenWrapper: {
+    position: 'absolute',
+    top: 0,
+    left: 0,
+    width: '100%',
+    height: '100%',
+    background: 'white',
+    overflow: 'hidden',
+    overflowY: 'auto',
+    zIndex: 999,
+  },
+});
+
+type Props = {
+  video: VideoData;
+};
+
+export default function DemoVideoEditor({video: inputVideo}: Props) {
+  const {settings} = useSettingsContext();
+  const video = useVideo();
+
+  const [isSessionStartFailed, setIsSessionStartFailed] =
+    useState<boolean>(false);
+
+  const [session, setSession] = useAtom(sessionAtom);
+
+  const [activeTrackletId, setActiveTrackletObjectId] = useAtom(
+    activeTrackletObjectIdAtom,
+  );
+  const setTrackletObjects = useSetAtom(trackletObjectsAtom);
+  const setFrameIndex = useSetAtom(frameIndexAtom);
+  const points = useAtomValue(pointsAtom);
+  const isAddObjectEnabled = useAtomValue(isAddObjectEnabledAtom);
+  const streamingState = useAtomValue(streamingStateAtom);
+  const isPlaying = useAtomValue(isPlayingAtom);
+  const isVideoLoading = useAtomValue(isVideoLoadingAtom);
+  const uploadingState = useAtomValue(uploadingStateAtom);
+
+  const [renderingError, setRenderingError] = useState<ErrorObject | null>(
+    null,
+  );
+
+  const {isMobile} = useScreenSize();
+
+  const [tabIndex] = useToolbarTabs();
+  const {enqueueMessage} = useMessagesSnackbar();
+
+  useCloseSessionBeforeUnload();
+
+  const {resetEditor, resetSession} = useResetDemoEditor();
+  useEffect(() => {
+    resetEditor();
+  }, [inputVideo, resetEditor]);
+
+  useEffect(() => {
+    function onFrameUpdate(event: FrameUpdateEvent) {
+      setFrameIndex(event.index);
+    }
+
+    // Listen to frame updates to fetch the frame index in the main thread,
+    // which is then used downstream to render points per frame.
+    video?.addEventListener('frameUpdate', onFrameUpdate);
+
+    function onSessionStarted(event: SessionStartedEvent) {
+      setSession({id: event.sessionId, ranPropagation: false});
+    }
+
+    video?.addEventListener('sessionStarted', onSessionStarted);
+
+    function onSessionStartFailed() {
+      setIsSessionStartFailed(true);
+    }
+
+    video?.addEventListener('sessionStartFailed', onSessionStartFailed);
+
+    function onTrackletsUpdated(event: TrackletsEvent) {
+      const tracklets = event.tracklets;
+      if (tracklets.length === 0) {
+        resetSession();
+      }
+      setTrackletObjects(tracklets);
+    }
+
+    video?.addEventListener('trackletsUpdated', onTrackletsUpdated);
+
+    function onRenderingError(event: RenderingErrorEvent) {
+      setRenderingError(event.error);
+    }
+
+    video?.addEventListener('renderingError', onRenderingError);
+
+    video?.initializeTracker('SAM 2', {
+      inferenceEndpoint: settings.inferenceAPIEndpoint,
+    });
+
+    video?.startSession(inputVideo.path);
+
+    return () => {
+      video?.closeSession();
+      video?.removeEventListener('frameUpdate', onFrameUpdate);
+      video?.removeEventListener('sessionStarted', onSessionStarted);
+      video?.removeEventListener('sessionStartFailed', onSessionStartFailed);
+      video?.removeEventListener('trackletsUpdated', onTrackletsUpdated);
+      video?.removeEventListener('renderingError', onRenderingError);
+    };
+  }, [
+    setFrameIndex,
+    setSession,
+    setTrackletObjects,
+    resetSession,
+    inputVideo,
+    video,
+    settings.inferenceAPIEndpoint,
+    settings.videoAPIEndpoint,
+  ]);
+
+  async function handleOptimisticPointUpdate(newPoints: SegmentationPoint[]) {
+    if (session == null) {
+      return;
+    }
+
+    async function createActiveTracklet() {
+      if (!isAddObjectEnabled || newPoints.length === 0) {
+        return;
+      }
+      const tracklet = await video?.createTracklet();
+      if (tracklet != null && newPoints.length > 0) {
+        setActiveTrackletObjectId(tracklet.id);
+        video?.updatePoints(tracklet.id, [newPoints[newPoints.length - 1]]);
+      }
+    }
+
+    if (activeTrackletId != null) {
+      video?.updatePoints(activeTrackletId, newPoints);
+    } else {
+      await createActiveTracklet();
+    }
+    enqueueMessage('pointClick');
+  }
+
+  async function handleAddPoint(point: SegmentationPoint) {
+    if (streamingState === 'partial' || streamingState === 'requesting') {
+      return;
+    }
+    if (isPlaying) {
+      return video?.pause();
+    }
+    handleOptimisticPointUpdate([...points, point]);
+  }
+
+  function handleRemovePoint(point: SegmentationPoint) {
+    if (
+      isPlaying ||
+      streamingState === 'partial' ||
+      streamingState === 'requesting'
+    ) {
+      return;
+    }
+    handleOptimisticPointUpdate(points.filter(p => p !== point));
+  }
+
+  // The interaction layer handles clicks onto the video canvas. It is used
+  // to get absolute point clicks within the video's coordinate system.
+  // The PointsLayer handles rendering of input points and allows removing
+  // individual points by clicking on them.
+  const layers = (
+    <>
+      {tabIndex === OBJECT_TOOLBAR_INDEX && (
+        <>
+          <InteractionLayer
+            key="interaction-layer"
+            onPoint={point => handleAddPoint(point)}
+          />
+          <PointsLayer
+            key="points-layer"
+            points={points}
+            onRemovePoint={handleRemovePoint}
+          />
+        </>
+      )}
+      {!isMobile && <MessagesSnackbar key="snackbar-layer" />}
+    </>
+  );
+
+  return (
+    <>
+      {(isVideoLoading || session === null) && !isSessionStartFailed && (
+        <div {...stylex.props(styles.loadingScreenWrapper)}>
+          <LoadingStateScreen
+            title="Loading demo..."
+            description="This may take a few moments, you're almost there!"
+          />
+        </div>
+      )}
+      {isSessionStartFailed && (
+        <div {...stylex.props(styles.loadingScreenWrapper)}>
+          <LoadingStateScreen
+            title="Did we just break the internet?"
+            description={
+              <>Uh oh, it looks like there was an issue starting a session.</>
+            }
+            linkProps={{to: '..', label: 'Back to homepage'}}
+          />
+        </div>
+      )}
+      {isMobile && renderingError != null && (
+        <div {...stylex.props(styles.loadingScreenWrapper)}>
+          <LoadingStateScreen
+            title="Well, this is embarrassing..."
+            description="This demo is not optimized for your device. Please try again on a different device with a larger screen."
+            linkProps={{to: '..', label: 'Back to homepage'}}
+          />
+        </div>
+      )}
+      {uploadingState !== 'default' && (
+        <div {...stylex.props(styles.loadingScreenWrapper)}>
+          <UploadLoadingScreen />
+        </div>
+      )}
+      <div {...stylex.props(styles.container)}>
+        <VideoEditor
+          video={inputVideo}
+          layers={layers}
+          loading={session == null}>
+          <div className="bg-graydark-800 w-full">
+            <VideoFilmstripWithPlayback />
+            <TrackletsAnnotation />
+          </div>
+        </VideoEditor>
+      </div>
+    </>
+  );
+}

demo/frontend/src/common/components/video/editor/ImageUtils.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export function convertVideoFrameToImageData(
+  videoFrame: VideoFrame,
+): ImageData | undefined {
+  const canvas = new OffscreenCanvas(
+    videoFrame.displayWidth,
+    videoFrame.displayHeight,
+  );
+  const ctx = canvas.getContext('2d');
+  ctx?.drawImage(videoFrame, 0, 0);
+  return ctx?.getImageData(0, 0, canvas.width, canvas.height);
+}
+
+/**
+ * This utility provides two functions:
+ * `process`: to find the bounding box of non-empty pixels from an ImageData, when looping through all its pixels
+ * `crop` to cut out the subsection found in `process`
+ * @returns
+ */
+export function findBoundingBox() {
+  let xMin = Number.MAX_VALUE;
+  let yMin = Number.MAX_VALUE;
+  let xMax = Number.MIN_VALUE;
+  let yMax = Number.MIN_VALUE;
+  return {
+    process: function (x: number, y: number, hasData: boolean) {
+      if (hasData) {
+        xMin = Math.min(x, xMin);
+        xMax = Math.max(x, xMax);
+        yMin = Math.min(y, yMin);
+        yMax = Math.max(y, yMax);
+      }
+      return [xMin, xMax, yMin, yMax];
+    },
+    crop(imageData: ImageData): ImageData | null {
+      const canvas = new OffscreenCanvas(imageData.width, imageData.height);
+      const ctx = canvas.getContext('2d');
+
+      const boundingBoxWidth = xMax - xMin;
+      const boundingBoxHeight = yMax - yMin;
+      if (ctx && boundingBoxWidth > 0 && boundingBoxHeight > 0) {
+        ctx.clearRect(0, 0, canvas.width, canvas.height);
+        ctx.putImageData(imageData, 0, 0);
+        return ctx.getImageData(
+          xMin,
+          yMin,
+          boundingBoxWidth,
+          boundingBoxHeight,
+        );
+      } else {
+        return null;
+      }
+    },
+    getBox(): [[number, number], [number, number]] {
+      return [
+        [xMin, yMin],
+        [xMax, yMax],
+      ];
+    },
+  };
+}
+
+export function magnifyImageRegion(
+  canvas: HTMLCanvasElement | null,
+  x: number,
+  y: number,
+  radius: number = 25,
+  scale: number = 2,
+): string {
+  if (canvas == null) {
+    return '';
+  }
+  const ctx = canvas.getContext('2d');
+  if (ctx) {
+    const minX = x - radius < 0 ? radius - x : 0;
+    const minY = y - radius < 0 ? radius - y : 0;
+    const region = ctx.getImageData(
+      Math.max(x - radius, 0),
+      Math.max(y - radius, 0),
+      radius * 2,
+      radius * 2,
+    );
+
+    // ImageData doesn't scale-transform correctly on canvas
+    // So we first draw the original size on an offscreen canvas, and then scale it
+    const regionCanvas = new OffscreenCanvas(region.width, region.height);
+    const regionCtx = regionCanvas.getContext('2d');
+    regionCtx?.putImageData(region, minX > 0 ? minX : 0, minY > 0 ? minY : 0);
+
+    const scaleCanvas = document.createElement('canvas');
+    scaleCanvas.width = Math.round(region.width * scale);
+    scaleCanvas.height = Math.round(region.height * scale);
+    const scaleCtx = scaleCanvas.getContext('2d');
+    scaleCtx?.scale(scale, scale);
+    scaleCtx?.drawImage(regionCanvas, 0, 0);
+
+    return scaleCanvas.toDataURL();
+  }
+  return '';
+}

demo/frontend/src/common/components/video/editor/VideoEditor.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {VideoData} from '@/demo/atoms';
+import stylex, {StyleXStyles} from '@stylexjs/stylex';
+import {useSetAtom} from 'jotai';
+import {PropsWithChildren, RefObject, useEffect, useRef} from 'react';
+import Video, {VideoRef} from '../Video';
+import {videoAtom} from './atoms';
+
+const MAX_VIDEO_WIDTH = 1280;
+
+const styles = stylex.create({
+  editorContainer: {
+    position: 'relative',
+    display: 'flex',
+    flexDirection: 'column',
+    alignItems: 'center',
+    width: '100%',
+    height: '100%',
+    borderRadius: '0.375rem',
+    overflow: {
+      default: 'clip',
+      '@media screen and (max-width: 768px)': 'visible',
+    },
+  },
+  videoContainer: {
+    position: 'relative',
+    flexGrow: 1,
+    overflow: 'hidden',
+    width: '100%',
+    maxWidth: MAX_VIDEO_WIDTH,
+  },
+  layers: {
+    position: 'absolute',
+    left: 0,
+    top: 0,
+    bottom: 0,
+    right: 0,
+  },
+  loadingMessage: {
+    position: 'absolute',
+    top: '8px',
+    right: '8px',
+    padding: '6px 10px',
+    backgroundColor: '#6441D2CC',
+    color: '#FFF',
+    display: 'flex',
+    alignItems: 'center',
+    gap: '8px',
+    borderRadius: '8px',
+    fontSize: '0.8rem',
+  },
+});
+
+export type InteractionLayerProps = {
+  style: StyleXStyles;
+  videoRef: RefObject<VideoRef>;
+};
+
+export type ControlsProps = {
+  isPlaying: boolean;
+  onPlay: () => void;
+  onPause: () => void;
+  onPreviousFrame?: () => void;
+  onNextFrame?: () => void;
+};
+
+type Props = PropsWithChildren<{
+  video: VideoData;
+  layers?: React.ReactNode;
+  loading?: boolean;
+}>;
+
+export default function VideoEditor({
+  video: inputVideo,
+  layers,
+  loading,
+  children,
+}: Props) {
+  const videoRef = useRef<VideoRef>(null);
+  const setVideo = useSetAtom(videoAtom);
+
+  // Initialize video atom
+  useEffect(() => {
+    setVideo(videoRef.current);
+    return () => {
+      setVideo(null);
+    };
+  }, [setVideo]);
+
+  return (
+    <div {...stylex.props(styles.editorContainer)}>
+      <div {...stylex.props(styles.videoContainer)}>
+        <Video
+          ref={videoRef}
+          src={inputVideo.url}
+          width={inputVideo.width}
+          height={inputVideo.height}
+          loading={loading}
+        />
+        <div {...stylex.props(styles.layers)}>{layers}</div>
+      </div>
+      {children}
+    </div>
+  );
+}

demo/frontend/src/common/components/video/editor/VideoEditorUtils.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Mask, Tracklet} from '@/common/tracker/Tracker';
+import {
+  convertVideoFrameToImageData,
+  findBoundingBox,
+} from '@/common/utils/ImageUtils';
+import {DataArray} from '@/jscocotools/mask';
+import invariant from 'invariant';
+
+function getCanvas(
+  width: number,
+  height: number,
+  isOffscreen: boolean = false,
+): HTMLCanvasElement | OffscreenCanvas {
+  if (isOffscreen || typeof document === 'undefined') {
+    return new OffscreenCanvas(width, height);
+  }
+  const canvas = document.createElement('canvas');
+  canvas.width = width;
+  canvas.height = height;
+  return canvas;
+}
+
+export function drawFrame(
+  ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
+  frame: VideoFrame | HTMLImageElement,
+  width: number,
+  height: number,
+) {
+  ctx?.drawImage(frame, 0, 0, width, height);
+}
+
+/**
+ * Given a mask and the image frame, get the masked image cropped to its bounding box.
+ */
+export function getThumbnailImageDataOld(
+  mask: DataArray,
+  videoFrame: VideoFrame,
+): ImageData | null {
+  const data = mask.data;
+  if (!ArrayBuffer.isView(data) || !(data instanceof Uint8Array)) {
+    return new ImageData(0, 0);
+  }
+
+  const frame = convertVideoFrameToImageData(videoFrame);
+  if (!frame) {
+    return new ImageData(0, 0);
+  }
+
+  const frameData = frame.data;
+  const scaleX = frame.width / mask.shape[1];
+  const scaleY = frame.height / mask.shape[0];
+  const boundingBox = findBoundingBox();
+  const transformedData = new Uint8ClampedArray(data.length * 4);
+
+  for (let i = 0; i < data.length; i++) {
+    // Since the mask is rotated, new width is the mask's height = mask.shape[1];
+    // Transform matrix: doing a rotate 90deg and then flip horizontal is the same as flipping x and y
+    // [ 0 1 ]   [ -1 0 ]   =   [ 0 1 ]
+    // [-1 0 ] x [  0 1 ]   =   [ 1 0 ]
+    // So, we can find the new index as: newY * newWidth + newX
+    const newX = Math.floor(i / mask.shape[0]); // ie, new x is the current y
+    const newY = i % mask.shape[0];
+    const transformedIndex = (newY * mask.shape[1] + newX) * 4;
+    const frameDataIndex = (newY * mask.shape[1] * scaleY + newX * scaleX) * 4;
+
+    transformedData[transformedIndex] = frameData[frameDataIndex];
+    transformedData[transformedIndex + 1] = frameData[frameDataIndex + 1];
+    transformedData[transformedIndex + 2] = frameData[frameDataIndex + 2];
+    transformedData[transformedIndex + 3] = (data[i] && 255) || 0; // A value
+
+    boundingBox.process(newX, newY, data[i] > 0);
+  }
+
+  const rotatedData = new ImageData(
+    transformedData,
+    mask.shape[1],
+    mask.shape[0],
+  ); // flip w and h of the mask
+
+  return boundingBox.crop(rotatedData);
+}
+
+/**
+ * Given a mask, the mask rendering context, and the video frame, get the
+ * masked image cropped to its bounding box.
+ */
+function getThumbnailImageData(
+  mask: Mask,
+  maskCtx: OffscreenCanvasRenderingContext2D,
+  frameBitmap: ImageBitmap,
+): ImageData | null {
+  const x = mask.bounds[0][0];
+  const y = mask.bounds[0][1];
+  const w = mask.bounds[1][0] - mask.bounds[0][0];
+  const h = mask.bounds[1][1] - mask.bounds[0][1];
+
+  if (w <= 0 || h <= 0) {
+    return null;
+  }
+
+  const thumbnailMaskData = maskCtx.getImageData(x, y, w, h);
+
+  const canvas = new OffscreenCanvas(w, h);
+  const ctx = canvas.getContext('2d');
+  invariant(ctx !== null, '2d context cannot be null');
+
+  ctx.putImageData(thumbnailMaskData, 0, 0);
+  ctx.globalCompositeOperation = 'source-in';
+  ctx.drawImage(frameBitmap, x, y, w, h, 0, 0, w, h);
+
+  return ctx.getImageData(0, 0, w, h);
+}
+
+export async function generateThumbnail(
+  track: Tracklet,
+  frameIndex: number,
+  mask: Mask,
+  frame: VideoFrame,
+  ctx: OffscreenCanvasRenderingContext2D,
+): Promise<void> {
+  // If a frame doesn't have points, the points will be undefined.
+  const hasPoints = (track.points[frameIndex]?.length ?? 0) > 0;
+  if (!hasPoints) {
+    return;
+  }
+  invariant(frame !== null, 'frame must be ready');
+  const bitmap = await createImageBitmap(frame);
+  const thumbnailImageData = getThumbnailImageData(
+    mask,
+    ctx as OffscreenCanvasRenderingContext2D,
+    bitmap,
+  );
+
+  bitmap.close();
+  if (thumbnailImageData != null) {
+    const thumbnailDataURL = await getDataURLFromImageData(thumbnailImageData);
+    track.thumbnail = thumbnailDataURL;
+  }
+}
+
+export async function getDataURLFromImageData(
+  imageData: ImageData | null,
+): Promise<string> {
+  if (!imageData) {
+    return '';
+  }
+
+  const canvas = getCanvas(imageData.width, imageData.height);
+  const ctx = canvas.getContext('2d');
+
+  if (ctx === null) {
+    return '';
+  }
+
+  ctx?.putImageData(imageData, 0, 0);
+
+  if (canvas instanceof OffscreenCanvas) {
+    const blob = await canvas.convertToBlob();
+    return new Promise(resolve => {
+      const reader = new FileReader();
+      reader.addEventListener(
+        'load',
+        () => {
+          const result = reader.result;
+          if (typeof result === 'string') {
+            resolve(result);
+          } else {
+            resolve('');
+          }
+        },
+        false,
+      );
+      reader.readAsDataURL(blob);
+    });
+  }
+  return canvas.toDataURL();
+}
+
+export function hexToRgb(hex: string): {
+  r: number;
+  g: number;
+  b: number;
+  a: number;
+} {
+  const result = /^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})?$/i.exec(
+    hex,
+  );
+  return result
+    ? {
+        r: parseInt(result[1], 16),
+        g: parseInt(result[2], 16),
+        b: parseInt(result[3], 16),
+        a: result[4] != null ? parseInt(result[4], 16) : 128,
+      }
+    : {r: 255, g: 0, b: 0, a: 128};
+}
+
+export function getPointInImage(
+  event: React.MouseEvent<HTMLElement>,
+  canvas: HTMLCanvasElement,
+  normalized: boolean = false,
+): [x: number, y: number] {
+  const rect = canvas.getBoundingClientRect();
+
+  const matrix = new DOMMatrix();
+
+  // First, center the image
+  const elementCenter = new DOMPoint(
+    canvas.clientWidth / 2,
+    canvas.clientHeight / 2,
+  );
+  const imageCenter = new DOMPoint(canvas.width / 2, canvas.height / 2);
+  matrix.translateSelf(
+    elementCenter.x - imageCenter.x,
+    elementCenter.y - imageCenter.y,
+  );
+
+  // Containing the object take the minimal scale
+  const scale = Math.min(
+    canvas.clientWidth / canvas.width,
+    canvas.clientHeight / canvas.height,
+  );
+  matrix.scaleSelf(scale, scale, 1, imageCenter.x, imageCenter.y);
+
+  const point = new DOMPoint(
+    event.clientX - rect.left,
+    event.clientY - rect.top,
+  );
+  const imagePoint = matrix.inverse().transformPoint(point);
+
+  const x = Math.max(Math.min(imagePoint.x, canvas.width), 0);
+  const y = Math.max(Math.min(imagePoint.y, canvas.height), 0);
+
+  if (normalized) {
+    return [x / canvas.width, y / canvas.height];
+  }
+  return [x, y];
+}

demo/frontend/src/common/components/video/editor/atoms.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {atom} from 'jotai';
+import {VideoRef} from '../Video';
+
+export const videoAtom = atom<VideoRef | null>(null);