Initial commit

demo/frontend/src/common/components/video/VideoWorker.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {registerSerializableConstructors} from '@/common/error/ErrorSerializationUtils';
+import {Tracker} from '@/common/tracker/Tracker';
+import {TrackerRequestMessageEvent} from '@/common/tracker/TrackerTypes';
+import {TRACKER_MAPPING} from '@/common/tracker/Trackers';
+import {serializeError} from 'serialize-error';
+import VideoWorkerContext from './VideoWorkerContext';
+import {
+  ErrorResponse,
+  VideoWorkerRequestMessageEvent,
+} from './VideoWorkerTypes';
+
+registerSerializableConstructors();
+
+const context = new VideoWorkerContext();
+let tracker: Tracker | null = null;
+
+let statsEnabled = false;
+
+self.addEventListener(
+  'message',
+  async (
+    event: VideoWorkerRequestMessageEvent | TrackerRequestMessageEvent,
+  ) => {
+    try {
+      switch (event.data.action) {
+        // Initialize context
+        case 'setCanvas':
+          context.setCanvas(event.data.canvas);
+          break;
+        case 'setSource':
+          context.setSource(event.data.source);
+          break;
+
+        // Playback
+        case 'play':
+          context.play();
+          break;
+        case 'pause':
+          context.pause();
+          break;
+        case 'stop':
+          context.stop();
+          break;
+        case 'frameUpdate':
+          context.goToFrame(event.data.index);
+          break;
+
+        // Filmstrip
+        case 'filmstrip': {
+          const {width, height} = event.data;
+          await context.createFilmstrip(width, height);
+          break;
+        }
+
+        // Effects
+        case 'setEffect': {
+          const {name, index, options} = event.data;
+          await context.setEffect(name, index, options);
+          break;
+        }
+
+        // Encode
+        case 'encode': {
+          await context.encode();
+          break;
+        }
+
+        case 'enableStats': {
+          statsEnabled = true;
+          context.enableStats();
+          tracker?.enableStats();
+          break;
+        }
+
+        // Tracker
+        case 'initializeTracker': {
+          const {name, options} = event.data;
+          const Tracker = TRACKER_MAPPING[name];
+          // Update the endpoint for the streaming API
+          tracker = new Tracker(context, options);
+          if (statsEnabled) {
+            tracker.enableStats();
+          }
+          break;
+        }
+        case 'startSession': {
+          const {videoUrl} = event.data;
+          await tracker?.startSession(videoUrl);
+          break;
+        }
+        case 'createTracklet':
+          tracker?.createTracklet();
+          break;
+        case 'deleteTracklet':
+          await tracker?.deleteTracklet(event.data.trackletId);
+          break;
+        case 'closeSession':
+          tracker?.closeSession();
+          break;
+        case 'updatePoints': {
+          const {frameIndex, objectId, points} = event.data;
+          context.allowEffectAnimation(true, objectId, points);
+          await tracker?.updatePoints(frameIndex, objectId, points);
+          break;
+        }
+        case 'clearPointsInFrame': {
+          const {frameIndex, objectId} = event.data;
+          await tracker?.clearPointsInFrame(frameIndex, objectId);
+          break;
+        }
+        case 'clearPointsInVideo':
+          await tracker?.clearPointsInVideo();
+          break;
+        case 'streamMasks': {
+          const {frameIndex} = event.data;
+          context.allowEffectAnimation(false);
+          await tracker?.streamMasks(frameIndex);
+          break;
+        }
+        case 'abortStreamMasks':
+          tracker?.abortStreamMasks();
+          break;
+      }
+    } catch (error) {
+      const serializedError = serializeError(error);
+      const errorResponse: ErrorResponse = {
+        action: 'error',
+        error: serializedError,
+      };
+      self.postMessage(errorResponse);
+    }
+  },
+);

demo/frontend/src/common/components/video/VideoWorkerBridge.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {EffectIndex, Effects} from '@/common/components/video/effects/Effects';
+import {registerSerializableConstructors} from '@/common/error/ErrorSerializationUtils';
+import {
+  BaseTracklet,
+  SegmentationPoint,
+  StreamingState,
+} from '@/common/tracker/Tracker';
+import {
+  AbortStreamMasksRequest,
+  AddPointsResponse,
+  ClearPointsInFrameRequest,
+  ClearPointsInVideoRequest,
+  ClearPointsInVideoResponse,
+  CloseSessionRequest,
+  CreateTrackletRequest,
+  DeleteTrackletRequest,
+  InitializeTrackerRequest,
+  LogAnnotationsRequest,
+  SessionStartFailedResponse,
+  SessionStartedResponse,
+  StartSessionRequest,
+  StreamMasksRequest,
+  StreamingStateUpdateResponse,
+  TrackerRequest,
+  TrackerResponseMessageEvent,
+  TrackletCreatedResponse,
+  TrackletDeletedResponse,
+  UpdatePointsRequest,
+} from '@/common/tracker/TrackerTypes';
+import {TrackerOptions, Trackers} from '@/common/tracker/Trackers';
+import {MP4ArrayBuffer} from 'mp4box';
+import {deserializeError, type ErrorObject} from 'serialize-error';
+import {EventEmitter} from './EventEmitter';
+import {
+  EncodeVideoRequest,
+  FilmstripRequest,
+  FilmstripResponse,
+  FrameUpdateRequest,
+  PauseRequest,
+  PlayRequest,
+  SetCanvasRequest,
+  SetEffectRequest,
+  SetSourceRequest,
+  StopRequest,
+  VideoWorkerRequest,
+  VideoWorkerResponseMessageEvent,
+} from './VideoWorkerTypes';
+import {EffectOptions} from './effects/Effect';
+
+registerSerializableConstructors();
+
+export type DecodeEvent = {
+  totalFrames: number;
+  numFrames: number;
+  fps: number;
+  width: number;
+  height: number;
+  done: boolean;
+};
+
+export type LoadStartEvent = unknown;
+
+export type EffectUpdateEvent = {
+  name: keyof Effects;
+  index: EffectIndex;
+  variant: number;
+  numVariants: number;
+};
+
+export type EncodingStateUpdateEvent = {
+  progress: number;
+};
+
+export type EncodingCompletedEvent = {
+  file: MP4ArrayBuffer;
+};
+
+export interface PlayEvent {}
+
+export interface PauseEvent {}
+
+export interface FilmstripEvent {
+  filmstrip: ImageBitmap;
+}
+
+export interface FrameUpdateEvent {
+  index: number;
+}
+
+export interface SessionStartedEvent {
+  sessionId: string;
+}
+
+export interface SessionStartFailedEvent {}
+
+export interface TrackletCreatedEvent {
+  // Do not send masks between workers and main thread because they are huge,
+  // and sending them would eventually slow down the main thread.
+  tracklet: BaseTracklet;
+}
+
+export interface TrackletsEvent {
+  // Do not send masks between workers and main thread because they are huge,
+  // and sending them would eventually slow down the main thread.
+  tracklets: BaseTracklet[];
+}
+
+export interface TrackletDeletedEvent {
+  isSuccessful: boolean;
+}
+
+export interface AddPointsEvent {
+  isSuccessful: boolean;
+}
+
+export interface ClearPointsInVideoEvent {
+  isSuccessful: boolean;
+}
+
+export interface StreamingStartedEvent {}
+
+export interface StreamingCompletedEvent {}
+
+export interface StreamingStateUpdateEvent {
+  state: StreamingState;
+}
+
+export interface RenderingErrorEvent {
+  error: ErrorObject;
+}
+
+export interface VideoWorkerEventMap {
+  error: ErrorEvent;
+  decode: DecodeEvent;
+  encodingStateUpdate: EncodingStateUpdateEvent;
+  encodingCompleted: EncodingCompletedEvent;
+  play: PlayEvent;
+  pause: PauseEvent;
+  filmstrip: FilmstripEvent;
+  frameUpdate: FrameUpdateEvent;
+  sessionStarted: SessionStartedEvent;
+  sessionStartFailed: SessionStartFailedEvent;
+  trackletCreated: TrackletCreatedEvent;
+  trackletsUpdated: TrackletsEvent;
+  trackletDeleted: TrackletDeletedEvent;
+  addPoints: AddPointsEvent;
+  clearPointsInVideo: ClearPointsInVideoEvent;
+  streamingStarted: StreamingStartedEvent;
+  streamingCompleted: StreamingCompletedEvent;
+  streamingStateUpdate: StreamingStateUpdateEvent;
+  // HTMLVideoElement events https://developer.mozilla.org/en-US/docs/Web/HTML/Element/video#events
+  loadstart: LoadStartEvent;
+  effectUpdate: EffectUpdateEvent;
+  renderingError: RenderingErrorEvent;
+}
+
+type Metadata = {
+  totalFrames: number;
+  fps: number;
+  width: number;
+  height: number;
+};
+
+export default class VideoWorkerBridge extends EventEmitter<VideoWorkerEventMap> {
+  static create(workerFactory: () => Worker) {
+    const worker = workerFactory();
+    return new VideoWorkerBridge(worker);
+  }
+
+  protected worker: Worker;
+  private metadata: Metadata | null = null;
+  private frameIndex: number = 0;
+
+  private _sessionId: string | null = null;
+
+  public get sessionId() {
+    return this._sessionId;
+  }
+
+  public get width() {
+    return this.metadata?.width ?? 0;
+  }
+
+  public get height() {
+    return this.metadata?.height ?? 0;
+  }
+
+  public get numberOfFrames() {
+    return this.metadata?.totalFrames ?? 0;
+  }
+
+  public get fps() {
+    return this.metadata?.fps ?? 0;
+  }
+
+  public get frame() {
+    return this.frameIndex;
+  }
+
+  constructor(worker: Worker) {
+    super();
+    this.worker = worker;
+
+    worker.addEventListener(
+      'message',
+      (
+        event: VideoWorkerResponseMessageEvent | TrackerResponseMessageEvent,
+      ) => {
+        switch (event.data.action) {
+          case 'error':
+            // Deserialize error before triggering the event
+            event.data.error = deserializeError(event.data.error);
+            break;
+          case 'decode':
+            this.metadata = event.data;
+            break;
+          case 'frameUpdate':
+            this.frameIndex = event.data.index;
+            break;
+          case 'sessionStarted':
+            this._sessionId = event.data.sessionId;
+            break;
+        }
+        this.trigger(event.data.action, event.data);
+      },
+    );
+  }
+
+  public setCanvas(canvas: HTMLCanvasElement): void {
+    const offscreenCanvas = canvas.transferControlToOffscreen();
+    this.sendRequest<SetCanvasRequest>(
+      'setCanvas',
+      {
+        canvas: offscreenCanvas,
+      },
+      [offscreenCanvas],
+    );
+  }
+
+  public setSource(source: string): void {
+    this.sendRequest<SetSourceRequest>('setSource', {
+      source,
+    });
+  }
+
+  public terminate(): void {
+    super.destroy();
+    this.worker.terminate();
+  }
+
+  public play(): void {
+    this.sendRequest<PlayRequest>('play');
+  }
+
+  public pause(): void {
+    this.sendRequest<PauseRequest>('pause');
+  }
+
+  public stop(): void {
+    this.sendRequest<StopRequest>('stop');
+  }
+
+  public goToFrame(index: number): void {
+    this.sendRequest<FrameUpdateRequest>('frameUpdate', {
+      index,
+    });
+  }
+
+  public previousFrame(): void {
+    const index = Math.max(0, this.frameIndex - 1);
+    this.goToFrame(index);
+  }
+
+  public nextFrame(): void {
+    const index = Math.min(this.frameIndex + 1, this.numberOfFrames - 1);
+    this.goToFrame(index);
+  }
+
+  public set frame(index: number) {
+    this.sendRequest<FrameUpdateRequest>('frameUpdate', {index});
+  }
+
+  createFilmstrip(width: number, height: number): Promise<ImageBitmap> {
+    return new Promise((resolve, _reject) => {
+      const handleFilmstripResponse = (
+        event: MessageEvent<FilmstripResponse>,
+      ) => {
+        if (event.data.action === 'filmstrip') {
+          this.worker.removeEventListener('message', handleFilmstripResponse);
+          resolve(event.data.filmstrip);
+        }
+      };
+
+      this.worker.addEventListener('message', handleFilmstripResponse);
+
+      this.sendRequest<FilmstripRequest>('filmstrip', {
+        width,
+        height,
+      });
+    });
+  }
+
+  setEffect(name: keyof Effects, index: EffectIndex, options?: EffectOptions) {
+    this.sendRequest<SetEffectRequest>('setEffect', {
+      name,
+      index,
+      options,
+    });
+  }
+
+  encode(): void {
+    this.sendRequest<EncodeVideoRequest>('encode');
+  }
+
+  initializeTracker(name: keyof Trackers, options: TrackerOptions): void {
+    this.sendRequest<InitializeTrackerRequest>('initializeTracker', {
+      name,
+      options,
+    });
+  }
+
+  startSession(videoUrl: string): Promise<string | null> {
+    return new Promise(resolve => {
+      const handleResponse = (
+        event: MessageEvent<
+          SessionStartedResponse | SessionStartFailedResponse
+        >,
+      ) => {
+        if (event.data.action === 'sessionStarted') {
+          this.worker.removeEventListener('message', handleResponse);
+          resolve(event.data.sessionId);
+        }
+        if (event.data.action === 'sessionStartFailed') {
+          this.worker.removeEventListener('message', handleResponse);
+          resolve(null);
+        }
+      };
+
+      this.worker.addEventListener('message', handleResponse);
+      this.sendRequest<StartSessionRequest>('startSession', {
+        videoUrl,
+      });
+    });
+  }
+
+  closeSession(): void {
+    this.sendRequest<CloseSessionRequest>('closeSession');
+  }
+
+  logAnnotations(): void {
+    this.sendRequest<LogAnnotationsRequest>('logAnnotations');
+  }
+
+  createTracklet(): Promise<BaseTracklet> {
+    return new Promise(resolve => {
+      const handleResponse = (event: MessageEvent<TrackletCreatedResponse>) => {
+        if (event.data.action === 'trackletCreated') {
+          this.worker.removeEventListener('message', handleResponse);
+          resolve(event.data.tracklet);
+        }
+      };
+
+      this.worker.addEventListener('message', handleResponse);
+
+      this.sendRequest<CreateTrackletRequest>('createTracklet');
+    });
+  }
+
+  deleteTracklet(trackletId: number): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const handleResponse = (event: MessageEvent<TrackletDeletedResponse>) => {
+        if (event.data.action === 'trackletDeleted') {
+          this.worker.removeEventListener('message', handleResponse);
+          if (event.data.isSuccessful) {
+            resolve();
+          } else {
+            reject(`could not delete tracklet ${trackletId}`);
+          }
+        }
+      };
+      this.worker.addEventListener('message', handleResponse);
+      this.sendRequest<DeleteTrackletRequest>('deleteTracklet', {trackletId});
+    });
+  }
+
+  updatePoints(
+    objectId: number,
+    points: SegmentationPoint[],
+  ): Promise<boolean> {
+    return new Promise(resolve => {
+      const handleResponse = (event: MessageEvent<AddPointsResponse>) => {
+        if (event.data.action === 'addPoints') {
+          this.worker.removeEventListener('message', handleResponse);
+          resolve(event.data.isSuccessful);
+        }
+      };
+
+      this.worker.addEventListener('message', handleResponse);
+
+      this.sendRequest<UpdatePointsRequest>('updatePoints', {
+        frameIndex: this.frame,
+        objectId,
+        points,
+      });
+    });
+  }
+
+  clearPointsInFrame(objectId: number) {
+    this.sendRequest<ClearPointsInFrameRequest>('clearPointsInFrame', {
+      frameIndex: this.frame,
+      objectId,
+    });
+  }
+
+  clearPointsInVideo(): Promise<boolean> {
+    return new Promise(resolve => {
+      const handleResponse = (
+        event: MessageEvent<ClearPointsInVideoResponse>,
+      ) => {
+        if (event.data.action === 'clearPointsInVideo') {
+          this.worker.removeEventListener('message', handleResponse);
+          resolve(event.data.isSuccessful);
+        }
+      };
+      this.worker.addEventListener('message', handleResponse);
+      this.sendRequest<ClearPointsInVideoRequest>('clearPointsInVideo');
+    });
+  }
+
+  streamMasks(): void {
+    this.sendRequest<StreamMasksRequest>('streamMasks', {
+      frameIndex: this.frame,
+    });
+  }
+
+  abortStreamMasks(): Promise<void> {
+    return new Promise(resolve => {
+      const handleAbortResponse = (
+        event: MessageEvent<StreamingStateUpdateResponse>,
+      ) => {
+        if (
+          event.data.action === 'streamingStateUpdate' &&
+          event.data.state === 'aborted'
+        ) {
+          this.worker.removeEventListener('message', handleAbortResponse);
+          resolve();
+        }
+      };
+
+      this.worker.addEventListener('message', handleAbortResponse);
+      this.sendRequest<AbortStreamMasksRequest>('abortStreamMasks');
+    });
+  }
+
+  getWorker_ONLY_USE_WITH_CAUTION(): Worker {
+    return this.worker;
+  }
+
+  /**
+   * Convenient function to have typed postMessage.
+   *
+   * @param action Video worker action
+   * @param message Actual payload
+   * @param transfer Any object that should be transferred instead of cloned
+   */
+  protected sendRequest<T extends VideoWorkerRequest | TrackerRequest>(
+    action: T['action'],
+    payload?: Omit<T, 'action'>,
+    transfer?: Transferable[],
+  ) {
+    this.worker.postMessage(
+      {
+        action,
+        ...payload,
+      },
+      {
+        transfer,
+      },
+    );
+  }
+
+  // // Override EventEmitter
+
+  // addEventListener<K extends keyof WorkerEventMap>(
+  //   type: K,
+  //   listener: (ev: WorkerEventMap[K]) => unknown,
+  // ): void {
+  //   switch (type) {
+  //     case 'frameUpdate':
+  //       {
+  //         const event: FrameUpdateEvent = {
+  //           index: this.frameIndex,
+  //         };
+  //         // @ts-expect-error Incorrect typing. Not sure how to correctly type it
+  //         listener(event);
+  //       }
+  //       break;
+  //     case 'sessionStarted': {
+  //       if (this.sessionId !== null) {
+  //         const event: SessionStartedEvent = {
+  //           sessionId: this.sessionId,
+  //         };
+  //         // @ts-expect-error Incorrect typing. Not sure how to correctly type it
+  //         listener(event);
+  //       }
+  //     }
+  //   }
+  //   super.addEventListener(type, listener);
+  // }
+}

demo/frontend/src/common/components/video/VideoWorkerContext.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  DecodedVideo,
+  ImageFrame,
+  decodeStream,
+} from '@/common/codecs/VideoDecoder';
+import {encode as encodeVideo} from '@/common/codecs/VideoEncoder';
+import {
+  Effect,
+  EffectActionPoint,
+  EffectFrameContext,
+  EffectOptions,
+} from '@/common/components/video/effects/Effect';
+import AllEffects, {
+  EffectIndex,
+  Effects,
+} from '@/common/components/video/effects/Effects';
+import Logger from '@/common/logger/Logger';
+import {Mask, SegmentationPoint, Tracklet} from '@/common/tracker/Tracker';
+import {streamFile} from '@/common/utils/FileUtils';
+import {Stats} from '@/debug/stats/Stats';
+import {VIDEO_WATERMARK_TEXT} from '@/demo/DemoConfig';
+import CreateFilmstripError from '@/graphql/errors/CreateFilmstripError';
+import DrawFrameError from '@/graphql/errors/DrawFrameError';
+import WebGLContextError from '@/graphql/errors/WebGLContextError';
+import {RLEObject} from '@/jscocotools/mask';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+import {serializeError} from 'serialize-error';
+import {
+  DecodeResponse,
+  EffectUpdateResponse,
+  EncodingCompletedResponse,
+  EncodingStateUpdateResponse,
+  FilmstripResponse,
+  FrameUpdateResponse,
+  PauseRequest,
+  PlayRequest,
+  RenderingErrorResponse,
+  VideoWorkerResponse,
+} from './VideoWorkerTypes';
+
+function getEvenlySpacedItems(decodedVideo: DecodedVideo, x: number) {
+  const p = Math.floor(decodedVideo.numFrames / Math.max(1, x - 1));
+  const middleFrames = decodedVideo.frames
+    .slice(p, p * x)
+    .filter(function (_, i) {
+      return 0 == i % p;
+    });
+  return [
+    decodedVideo.frames[0],
+    ...middleFrames,
+    decodedVideo.frames[decodedVideo.numFrames - 1],
+  ];
+}
+
+export type FrameInfo = {
+  tracklet: Tracklet;
+  mask: Mask;
+};
+
+const WATERMARK_BOX_HORIZONTAL_PADDING = 10;
+const WATERMARK_BOX_VERTICAL_PADDING = 10;
+
+export type VideoStats = {
+  fps?: Stats;
+  videoFps?: Stats;
+  total?: Stats;
+  effect0?: Stats;
+  effect1?: Stats;
+  frameBmp?: Stats;
+  maskBmp?: Stats;
+  memory?: Stats;
+};
+
+export default class VideoWorkerContext {
+  private _canvas: OffscreenCanvas | null = null;
+  private _stats: VideoStats = {};
+  private _ctx: OffscreenCanvasRenderingContext2D | null = null;
+  private _form: CanvasForm | null = null;
+  private _decodedVideo: DecodedVideo | null = null;
+  private _frameIndex: number = 0;
+  private _isPlaying: boolean = false;
+  private _playbackRAFHandle: number | null = null;
+  private _playbackTimeoutHandle: NodeJS.Timeout | null = null;
+  private _isDrawing: boolean = false;
+  private _glObjects: WebGL2RenderingContext | null = null;
+  private _glBackground: WebGL2RenderingContext | null = null;
+  private _canvasHighlights: OffscreenCanvas | null = null;
+  private _canvasBackground: OffscreenCanvas | null = null;
+  private _allowAnimation: boolean = false;
+  private _currentSegmetationPoint: EffectActionPoint | null = null;
+
+  private _effects: Effect[];
+  private _tracklets: Tracklet[] = [];
+
+  public get width(): number {
+    return this._decodedVideo?.width ?? 0;
+  }
+
+  public get height(): number {
+    return this._decodedVideo?.height ?? 0;
+  }
+
+  public get frameIndex(): number {
+    return this._frameIndex;
+  }
+
+  public get currentFrame(): VideoFrame | null {
+    return this._decodedVideo?.frames[this._frameIndex].bitmap ?? null;
+  }
+
+  constructor() {
+    this._effects = [
+      AllEffects.Original, // Image as background
+      AllEffects.Overlay, // Masks on top
+    ];
+
+    // Loading watermark fonts. This is going to be async, but by the time of
+    // video encoding, the fonts should be available.
+    this._loadWatermarkFonts();
+  }
+
+  private initializeWebGLContext(width: number, height: number): void {
+    // Given that we use highlight and background effects as layers,
+    // we need to create two WebGL contexts, one for each set.
+    // To avoid memory leaks and too many active contexts,
+    // these contexts must be re-used over the lifecycle of the session.
+
+    if (this._canvasHighlights == null && this._glObjects == null) {
+      this._canvasHighlights = new OffscreenCanvas(width, height);
+      this._glObjects = this._canvasHighlights.getContext('webgl2');
+
+      this._canvasHighlights.addEventListener(
+        'webglcontextlost',
+        event => {
+          event.preventDefault();
+          this._sendRenderingError(
+            new WebGLContextError('WebGL context lost.'),
+          );
+        },
+        false,
+      );
+    } else if (
+      this._canvasHighlights != null &&
+      (this._canvasHighlights.width !== width ||
+        this._canvasHighlights.height !== height)
+    ) {
+      // Resize canvas and webgl viewport
+      this._canvasHighlights.width = width;
+      this._canvasHighlights.height = height;
+      if (this._glObjects != null) {
+        this._glObjects.viewport(0, 0, width, height);
+      }
+    }
+
+    if (this._canvasBackground == null && this._glBackground == null) {
+      this._canvasBackground = new OffscreenCanvas(width, height);
+      this._glBackground = this._canvasBackground.getContext('webgl2');
+
+      this._canvasBackground.addEventListener(
+        'webglcontextlost',
+        event => {
+          event.preventDefault();
+          this._sendRenderingError(
+            new WebGLContextError('WebGL context lost.'),
+          );
+        },
+        false,
+      );
+    } else if (
+      this._canvasBackground != null &&
+      (this._canvasBackground.width != width ||
+        this._canvasBackground.height != height)
+    ) {
+      // Resize canvas and webgl viewport
+      this._canvasBackground.width = width;
+      this._canvasBackground.height = height;
+      if (this._glBackground != null) {
+        this._glBackground.viewport(0, 0, width, height);
+      }
+    }
+  }
+
+  public setCanvas(canvas: OffscreenCanvas) {
+    this._canvas = canvas;
+    this._ctx = canvas.getContext('2d');
+    if (this._ctx == null) {
+      throw new Error('could not initialize drawing context');
+    }
+    this._form = new CanvasForm(this._ctx);
+  }
+
+  public setSource(src: string) {
+    this.close();
+
+    // Clear state of previous source.
+    this.updateFrameIndex(0);
+    this._tracklets = [];
+
+    this._decodeVideo(src);
+  }
+
+  public goToFrame(index: number): void {
+    // Cancel any ongoing render
+    this._cancelRender();
+    this.updateFrameIndex(index);
+    this._playbackRAFHandle = requestAnimationFrame(this._drawFrame.bind(this));
+  }
+
+  public play(): void {
+    // Video already playing
+    if (this._isPlaying) {
+      return;
+    }
+
+    // Cannot playback without frames
+    if (this._decodedVideo === null) {
+      throw new Error('no decoded video');
+    }
+
+    const {numFrames, fps} = this._decodedVideo;
+    const timePerFrame = 1000 / (fps ?? 30);
+    let startTime: number | null = null;
+    // The offset frame index compensate for cases where the video playback
+    // does not start at frame index 0.
+    const offsetFrameIndex = this._frameIndex;
+
+    const updateFrame = (time: number) => {
+      if (startTime === null) {
+        startTime = time;
+      }
+
+      this._stats.fps?.begin();
+
+      const diff = time - startTime;
+      const expectedFrame =
+        (Math.floor(diff / timePerFrame) + offsetFrameIndex) % numFrames;
+
+      if (this._frameIndex !== expectedFrame && !this._isDrawing) {
+        // Update to the next expected frame
+        this.updateFrameIndex(expectedFrame);
+        this._drawFrame();
+      }
+
+      this._playbackRAFHandle = requestAnimationFrame(updateFrame);
+
+      this._stats.fps?.end();
+    };
+
+    this.updatePlayback(true);
+    this._playbackRAFHandle = requestAnimationFrame(updateFrame);
+  }
+
+  public pause(): void {
+    this.updatePlayback(false);
+    this._cancelRender();
+  }
+
+  public stop(): void {
+    this.pause();
+    this.updateFrameIndex(0);
+  }
+
+  public async createFilmstrip(width: number, height: number): Promise<void> {
+    if (width < 1 || height < 1) {
+      Logger.warn(
+        `Cannot create filmstrip because width ${width} or height ${height} is too small.`,
+      );
+      return;
+    }
+
+    try {
+      const canvas = new OffscreenCanvas(width, height);
+      const ctx = canvas.getContext('2d');
+
+      if (this._decodedVideo !== null) {
+        const scale = canvas.height / this._decodedVideo.height;
+        const resizeWidth = this._decodedVideo.width * scale;
+
+        const spacedFrames = getEvenlySpacedItems(
+          this._decodedVideo,
+          Math.ceil(canvas.width / resizeWidth),
+        );
+
+        spacedFrames.forEach((frame, idx) => {
+          if (frame != null) {
+            ctx?.drawImage(
+              frame.bitmap,
+              resizeWidth * idx,
+              0,
+              resizeWidth,
+              canvas.height,
+            );
+          }
+        });
+      }
+
+      const filmstrip = await createImageBitmap(canvas);
+
+      this.sendResponse<FilmstripResponse>(
+        'filmstrip',
+        {
+          filmstrip,
+        },
+        [filmstrip],
+      );
+    } catch {
+      this._sendRenderingError(
+        new CreateFilmstripError('Failed to create filmstrip'),
+      );
+    }
+  }
+
+  public async setEffect(
+    name: keyof Effects,
+    index: EffectIndex,
+    options?: EffectOptions,
+  ): Promise<void> {
+    const effect: Effect = AllEffects[name];
+
+    // The effect has changed.
+    if (this._effects[index] !== effect) {
+      // Effect changed. Cleanup old effect first. Effects are responsible for
+      // cleaning up their memory.
+      await this._effects[index].cleanup();
+
+      const offCanvas =
+        index === EffectIndex.BACKGROUND
+          ? this._canvasBackground
+          : this._canvasHighlights;
+      invariant(offCanvas != null, 'need OffscreenCanvas to render effects');
+      const webglContext =
+        index === EffectIndex.BACKGROUND ? this._glBackground : this._glObjects;
+      invariant(webglContext != null, 'need WebGL context to render effects');
+
+      // Initialize the effect. This can be used by effects to prepare
+      // resources needed for rendering. If the video wasn't decoded yet, the
+      // effect setup will happen in the _decodeVideo function.
+      if (this._decodedVideo != null) {
+        await effect.setup({
+          width: this._decodedVideo.width,
+          height: this._decodedVideo.height,
+          canvas: offCanvas,
+          gl: webglContext,
+        });
+      }
+    }
+
+    // Update effect if already set effect was clicked again. This can happen
+    // when there is a new variant of the effect.
+    if (options != null) {
+      // Update effect if already set effect was clicked again. This can happen
+      // when there is a new variant of the effect.
+      await effect.update(options);
+    }
+
+    // Notify the frontend about the effect state including its variant.
+    this.sendResponse<EffectUpdateResponse>('effectUpdate', {
+      name,
+      index,
+      variant: effect.variant,
+      numVariants: effect.numVariants,
+    });
+
+    this._effects[index] = effect;
+    this._playbackRAFHandle = requestAnimationFrame(this._drawFrame.bind(this));
+  }
+
+  async encode() {
+    const decodedVideo = this._decodedVideo;
+    invariant(
+      decodedVideo !== null,
+      'cannot encode video because there is no decoded video available',
+    );
+
+    const canvas = new OffscreenCanvas(this.width, this.height);
+    const ctx = canvas.getContext('2d', {willReadFrequently: true});
+    invariant(
+      ctx !== null,
+      'cannot encode video because failed to construct offscreen canvas context',
+    );
+
+    const form = new CanvasForm(ctx);
+
+    const file = await encodeVideo(
+      this.width,
+      this.height,
+      decodedVideo.frames.length,
+      this._framesGenerator(decodedVideo, canvas, form),
+      progress => {
+        this.sendResponse<EncodingStateUpdateResponse>('encodingStateUpdate', {
+          progress,
+        });
+      },
+    );
+    this.sendResponse<EncodingCompletedResponse>(
+      'encodingCompleted',
+      {
+        file,
+      },
+      [file],
+    );
+  }
+
+  private async *_framesGenerator(
+    decodedVideo: DecodedVideo,
+    canvas: OffscreenCanvas,
+    form: CanvasForm,
+  ): AsyncGenerator<ImageFrame, undefined> {
+    const frames = decodedVideo.frames;
+
+    for (let frameIndex = 0; frameIndex < frames.length; ++frameIndex) {
+      await this._drawFrameImpl(form, frameIndex, true);
+
+      const frame = frames[frameIndex];
+      const videoFrame = new VideoFrame(canvas, {
+        timestamp: frame.bitmap.timestamp,
+      });
+
+      yield {
+        bitmap: videoFrame,
+        timestamp: frame.timestamp,
+        duration: frame.duration,
+      };
+
+      videoFrame.close();
+    }
+  }
+
+  public enableStats() {
+    this._stats.fps = new Stats('fps');
+    this._stats.videoFps = new Stats('fps', 'V');
+    this._stats.total = new Stats('ms', 'T');
+    this._stats.effect0 = new Stats('ms', 'B');
+    this._stats.effect1 = new Stats('ms', 'H');
+    this._stats.frameBmp = new Stats('ms', 'F');
+    this._stats.maskBmp = new Stats('ms', 'M');
+    this._stats.memory = new Stats('memory');
+  }
+
+  public allowEffectAnimation(
+    allow: boolean = true,
+    objectId?: number,
+    points?: SegmentationPoint[],
+  ) {
+    if (objectId != null && points != null && points.length) {
+      const last_point_position = points[points.length - 1];
+      this._currentSegmetationPoint = {
+        objectId,
+        position: [last_point_position[0], last_point_position[1]],
+      };
+    }
+
+    if (!allow) {
+      this._currentSegmetationPoint = null;
+    }
+
+    this._allowAnimation = allow;
+  }
+
+  public close(): void {
+    // Clear any frame content
+    this._ctx?.reset();
+
+    // Close frames of previously decoded video.
+    this._decodedVideo?.frames.forEach(f => f.bitmap.close());
+    this._decodedVideo = null;
+  }
+
+  // TRACKER
+
+  public updateTracklets(
+    frameIndex: number,
+    tracklets: Tracklet[],
+    shouldGoToFrame: boolean = true,
+  ): void {
+    this._tracklets = tracklets;
+    if (shouldGoToFrame) {
+      this.goToFrame(frameIndex);
+    }
+  }
+
+  public clearTrackletMasks(tracklet: Tracklet): void {
+    this._tracklets = this._tracklets.filter(t => t.id != tracklet.id);
+  }
+
+  public clearMasks(): void {
+    this._tracklets = [];
+  }
+
+  // PRIVATE FUNCTIONS
+
+  private sendResponse<T extends VideoWorkerResponse>(
+    action: T['action'],
+    message?: Omit<T, 'action'>,
+    transfer?: Transferable[],
+  ): void {
+    self.postMessage(
+      {
+        action,
+        ...message,
+      },
+      {
+        transfer,
+      },
+    );
+  }
+
+  private async _decodeVideo(src: string): Promise<void> {
+    const canvas = this._canvas;
+    invariant(canvas != null, 'need canvas to render decoded video');
+
+    this.sendResponse('loadstart');
+
+    const fileStream = streamFile(src, {
+      credentials: 'same-origin',
+      cache: 'no-store',
+    });
+
+    let renderedFirstFrame = false;
+    this._decodedVideo = await decodeStream(fileStream, async progress => {
+      const {fps, height, width, numFrames, frames} = progress;
+      this._decodedVideo = progress;
+      if (!renderedFirstFrame) {
+        renderedFirstFrame = true;
+        canvas.width = width;
+        canvas.height = height;
+        // Set WebGL contexts right after the first frame decoded
+        this.initializeWebGLContext(width, height);
+
+        // Initialize effect once first frame was decoded.
+        for (const [i, effect] of this._effects.entries()) {
+          const offCanvas =
+            i === EffectIndex.BACKGROUND
+              ? this._canvasBackground
+              : this._canvasHighlights;
+          invariant(offCanvas != null, 'need canvas to render effects');
+          const webglContext =
+            i === EffectIndex.BACKGROUND ? this._glBackground : this._glObjects;
+          invariant(
+            webglContext != null,
+            'need WebGL context to render effects',
+          );
+          await effect.setup({
+            width,
+            height,
+            canvas: offCanvas,
+            gl: webglContext,
+          });
+        }
+
+        // Need to render frame immediately. Cannot go through
+        // requestAnimationFrame because then rendering this frame would be
+        // delayed until the full video has finished decoding.
+        this._drawFrame();
+
+        this._stats.videoFps?.updateMaxValue(fps);
+        this._stats.total?.updateMaxValue(1000 / fps);
+        this._stats.effect0?.updateMaxValue(1000 / fps);
+        this._stats.effect1?.updateMaxValue(1000 / fps);
+        this._stats.frameBmp?.updateMaxValue(1000 / fps);
+        this._stats.maskBmp?.updateMaxValue(1000 / fps);
+      }
+      this.sendResponse<DecodeResponse>('decode', {
+        totalFrames: numFrames,
+        numFrames: frames.length,
+        fps: fps,
+        width: width,
+        height: height,
+        done: false,
+      });
+    });
+
+    if (!renderedFirstFrame) {
+      canvas.width = this._decodedVideo.width;
+      canvas.height = this._decodedVideo.height;
+      this._drawFrame();
+    }
+
+    this.sendResponse<DecodeResponse>('decode', {
+      totalFrames: this._decodedVideo.numFrames,
+      numFrames: this._decodedVideo.frames.length,
+      fps: this._decodedVideo.fps,
+      width: this._decodedVideo.width,
+      height: this._decodedVideo.height,
+      done: true,
+    });
+  }
+
+  private _drawFrame(): void {
+    if (this._canvas !== null && this._form !== null) {
+      this._drawFrameImpl(this._form, this._frameIndex);
+    }
+  }
+
+  private async _drawFrameImpl(
+    form: CanvasForm,
+    frameIndex: number,
+    enableWatermark: boolean = false,
+    step: number = 0,
+    maxSteps: number = 40,
+  ): Promise<void> {
+    if (this._decodedVideo === null) {
+      return;
+    }
+
+    {
+      this._stats.videoFps?.begin();
+      this._stats.total?.begin();
+      this._stats.memory?.begin();
+    }
+
+    try {
+      const frame = this._decodedVideo.frames[frameIndex];
+      const {bitmap} = frame;
+
+      this._stats.frameBmp?.begin();
+
+      // Need to convert VideoFrame to ImageBitmap because Safari can only apply
+      // globalCompositeOperation on ImageBitmap and fails on VideoFrame. FWIW,
+      // Chrome treats VideoFrame similarly to ImageBitmap.
+      const frameBitmap = await createImageBitmap(bitmap);
+
+      this._stats.frameBmp?.end();
+
+      const masks: Mask[] = [];
+      const colors: string[] = [];
+      const tracklets: Tracklet[] = [];
+      this._tracklets.forEach(tracklet => {
+        const mask = tracklet.masks[frameIndex];
+        if (mask != null) {
+          masks.push(mask);
+          tracklets.push(tracklet);
+          colors.push(tracklet.color);
+        }
+      });
+      const effectActionPoint = this._currentSegmetationPoint;
+
+      this._stats.maskBmp?.begin();
+
+      const effectMaskPromises = masks.map(async ({data, bounds}) => {
+        return {
+          bounds,
+          bitmap: data as RLEObject,
+        };
+      });
+      const effectMasks = await Promise.all(effectMaskPromises);
+
+      this._stats.maskBmp?.end();
+
+      form.ctx.fillStyle = 'rgba(0, 0, 0, 0)';
+      form.ctx.fillRect(0, 0, this.width, this.height);
+
+      const effectParams: EffectFrameContext = {
+        frame: frameBitmap,
+        masks: effectMasks,
+        maskColors: colors,
+        frameIndex: frameIndex,
+        totalFrames: this._decodedVideo.frames.length,
+        fps: this._decodedVideo.fps,
+        width: frameBitmap.width,
+        height: frameBitmap.height,
+        actionPoint: null,
+      };
+
+      // Allows animation within a single frame.
+      if (this._allowAnimation && step < maxSteps) {
+        const animationDuration = 2; // Total duration of the animation in seconds
+        const progress = step / maxSteps;
+        const timeParameter = progress * animationDuration;
+        // Pass dynamic effect params
+        effectParams.timeParameter = timeParameter;
+        effectParams.actionPoint = effectActionPoint;
+
+        this._processEffects(form, effectParams, tracklets);
+
+        // Use RAF to draw frame, and update the display,
+        // this avoids to wait until the javascript call stack is cleared.
+        requestAnimationFrame(() =>
+          this._drawFrameImpl(form, frameIndex, false, step + 1, maxSteps),
+        );
+      } else {
+        this._processEffects(form, effectParams, tracklets);
+      }
+
+      if (enableWatermark) {
+        this._drawWatermark(form, frameBitmap);
+      }
+
+      // Do not simply drop the JavaScript reference to the ImageBitmap; doing so
+      // will keep its graphics resource alive until the next time the garbage
+      // collector runs.
+      frameBitmap.close();
+
+      {
+        this._stats.videoFps?.end();
+        this._stats.total?.end();
+        this._stats.memory?.end();
+      }
+
+      this._isDrawing = false;
+    } catch {
+      this._sendRenderingError(new DrawFrameError('Failed to draw frame'));
+    }
+  }
+
+  private _drawWatermark(form: CanvasForm, frameBitmap: ImageBitmap): void {
+    const frameWidth = this._canvas?.width || frameBitmap.width;
+    const frameHeight = this._canvas?.height || frameBitmap.height;
+    // Font size is either 12 or smaller based on available width
+    // since the font is not monospaced, we approximate it'll fit 1.5 more characters than monospaced
+    const approximateFontSize = Math.min(
+      Math.floor(frameWidth / (VIDEO_WATERMARK_TEXT.length / 1.5)),
+      12,
+    );
+
+    form.ctx.font = `${approximateFontSize}px "Inter", sans-serif`;
+    const measureGeneratedBy = form.ctx.measureText(VIDEO_WATERMARK_TEXT);
+
+    const textBoxWidth =
+      measureGeneratedBy.width + 2 * WATERMARK_BOX_HORIZONTAL_PADDING;
+    const textBoxHeight =
+      measureGeneratedBy.actualBoundingBoxAscent +
+      2 * WATERMARK_BOX_VERTICAL_PADDING;
+    const textBoxX = frameWidth - textBoxWidth;
+    const textBoxY = frameHeight - textBoxHeight;
+
+    form.ctx.fillStyle = 'rgba(0, 0, 0, 0.4)';
+    form.ctx.beginPath();
+    form.ctx.roundRect(
+      Math.round(textBoxX),
+      Math.round(textBoxY),
+      Math.round(textBoxWidth),
+      Math.round(textBoxHeight),
+      [WATERMARK_BOX_HORIZONTAL_PADDING, 0, 0, 0],
+    );
+    form.ctx.fill();
+
+    // Always reset the text style because some effects may change text styling in the same ctx
+    form.ctx.fillStyle = 'rgba(255, 255, 255, 0.8)';
+    form.ctx.textAlign = 'left';
+
+    form.ctx.fillText(
+      VIDEO_WATERMARK_TEXT,
+      Math.round(textBoxX + WATERMARK_BOX_HORIZONTAL_PADDING),
+      Math.round(
+        textBoxY +
+          WATERMARK_BOX_VERTICAL_PADDING +
+          measureGeneratedBy.actualBoundingBoxAscent,
+      ),
+    );
+  }
+
+  private updateFrameIndex(index: number): void {
+    this._frameIndex = index;
+    this.sendResponse<FrameUpdateResponse>('frameUpdate', {
+      index,
+    });
+  }
+
+  private _loadWatermarkFonts() {
+    const requiredFonts = [
+      {
+        url: '/fonts/Inter-VariableFont.ttf',
+        format: 'truetype-variations',
+      },
+    ];
+    requiredFonts.forEach(requiredFont => {
+      const fontFace = new FontFace(
+        'Inter',
+        `url(${requiredFont.url}) format('${requiredFont.format}')`,
+      );
+      fontFace.load().then(font => {
+        self.fonts.add(font);
+      });
+    });
+  }
+
+  private updatePlayback(playing: boolean): void {
+    if (playing) {
+      this.sendResponse<PlayRequest>('play');
+    } else {
+      this.sendResponse<PauseRequest>('pause');
+    }
+    this._isPlaying = playing;
+  }
+
+  private _cancelRender(): void {
+    if (this._playbackTimeoutHandle !== null) {
+      clearTimeout(this._playbackTimeoutHandle);
+      this._playbackTimeoutHandle = null;
+    }
+    if (this._playbackRAFHandle !== null) {
+      cancelAnimationFrame(this._playbackRAFHandle);
+      this._playbackRAFHandle = null;
+    }
+  }
+
+  private _sendRenderingError(error: Error): void {
+    this.sendResponse<RenderingErrorResponse>('renderingError', {
+      error: serializeError(error),
+    });
+  }
+
+  private _processEffects(
+    form: CanvasForm,
+    effectParams: EffectFrameContext,
+    tracklets: Tracklet[],
+  ) {
+    for (let i = 0; i < this._effects.length; i++) {
+      const effect = this._effects[i];
+
+      if (i === 0) {
+        this._stats.effect0?.begin();
+      } else if (i === 1) {
+        this._stats.effect1?.begin();
+      }
+
+      effect.apply(form, effectParams, tracklets);
+
+      if (i === 0) {
+        this._stats.effect0?.end();
+      } else if (i === 1) {
+        this._stats.effect1?.end();
+      }
+    }
+  }
+}

demo/frontend/src/common/components/video/VideoWorkerTypes.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  DecodeEvent,
+  EffectUpdateEvent,
+  EncodingCompletedEvent,
+  EncodingStateUpdateEvent,
+  FilmstripEvent,
+  FrameUpdateEvent,
+  LoadStartEvent,
+  RenderingErrorEvent,
+} from './VideoWorkerBridge';
+import {EffectOptions} from './effects/Effect';
+import type {Effects} from './effects/Effects';
+
+export type Request<A, P> = {
+  action: A;
+} & P;
+
+// REQUESTS
+
+export type SetCanvasRequest = Request<
+  'setCanvas',
+  {
+    canvas: OffscreenCanvas;
+  }
+>;
+export type SetSourceRequest = Request<
+  'setSource',
+  {
+    source: string;
+  }
+>;
+export type PlayRequest = Request<'play', unknown>;
+export type PauseRequest = Request<'pause', unknown>;
+export type StopRequest = Request<'stop', unknown>;
+export type FrameUpdateRequest = Request<
+  'frameUpdate',
+  {
+    index: number;
+  }
+>;
+export type FilmstripRequest = Request<
+  'filmstrip',
+  {
+    width: number;
+    height: number;
+  }
+>;
+export type SetEffectRequest = Request<
+  'setEffect',
+  {
+    name: keyof Effects;
+    index: number;
+    options?: EffectOptions;
+  }
+>;
+
+export type EncodeVideoRequest = Request<'encode', unknown>;
+
+export type EnableStatsRequest = Request<'enableStats', unknown>;
+
+export type VideoWorkerRequest =
+  | SetCanvasRequest
+  | SetSourceRequest
+  | PlayRequest
+  | PauseRequest
+  | StopRequest
+  | FrameUpdateRequest
+  | FilmstripRequest
+  | SetEffectRequest
+  | EncodeVideoRequest
+  | EnableStatsRequest;
+
+export type VideoWorkerRequestMessageEvent = MessageEvent<VideoWorkerRequest>;
+
+// RESPONSES
+
+export type ErrorResponse = Request<
+  'error',
+  {
+    error: unknown;
+  }
+>;
+
+export type DecodeResponse = Request<'decode', DecodeEvent>;
+
+export type EncodingStateUpdateResponse = Request<
+  'encodingStateUpdate',
+  EncodingStateUpdateEvent
+>;
+
+export type EncodingCompletedResponse = Request<
+  'encodingCompleted',
+  EncodingCompletedEvent
+>;
+
+export type FilmstripResponse = Request<'filmstrip', FilmstripEvent>;
+
+export type PlayResponse = Request<'play', unknown>;
+
+export type PauseResponse = Request<'pause', unknown>;
+
+export type FrameUpdateResponse = Request<'frameUpdate', FrameUpdateEvent>;
+
+export type RenderingErrorResponse = Request<
+  'renderingError',
+  RenderingErrorEvent
+>;
+
+// HTMLVideoElement events https://developer.mozilla.org/en-US/docs/Web/HTML/Element/video#events
+
+export type LoadStartResponse = Request<'loadstart', LoadStartEvent>;
+
+export type EffectUpdateResponse = Request<'effectUpdate', EffectUpdateEvent>;
+
+export type VideoWorkerResponse =
+  | ErrorResponse
+  | FilmstripResponse
+  | DecodeResponse
+  | EncodingStateUpdateResponse
+  | EncodingCompletedResponse
+  | PlayResponse
+  | PauseResponse
+  | FrameUpdateResponse
+  | LoadStartResponse
+  | RenderingErrorResponse
+  | EffectUpdateResponse;
+
+export type VideoWorkerResponseMessageEvent = MessageEvent<VideoWorkerResponse>;

demo/frontend/src/common/components/video/editor/DemoVideoEditor.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import TrackletsAnnotation from '@/common/components/annotations/TrackletsAnnotation';
+import useCloseSessionBeforeUnload from '@/common/components/session/useCloseSessionBeforeUnload';
+import MessagesSnackbar from '@/common/components/snackbar/MessagesSnackbar';
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import {OBJECT_TOOLBAR_INDEX} from '@/common/components/toolbar/ToolbarConfig';
+import useToolbarTabs from '@/common/components/toolbar/useToolbarTabs';
+import VideoFilmstripWithPlayback from '@/common/components/video/VideoFilmstripWithPlayback';
+import {
+  FrameUpdateEvent,
+  RenderingErrorEvent,
+  SessionStartedEvent,
+  TrackletsEvent,
+} from '@/common/components/video/VideoWorkerBridge';
+import VideoEditor from '@/common/components/video/editor/VideoEditor';
+import useResetDemoEditor from '@/common/components/video/editor/useResetEditor';
+import useVideo from '@/common/components/video/editor/useVideo';
+import InteractionLayer from '@/common/components/video/layers/InteractionLayer';
+import {PointsLayer} from '@/common/components/video/layers/PointsLayer';
+import LoadingStateScreen from '@/common/loading/LoadingStateScreen';
+import UploadLoadingScreen from '@/common/loading/UploadLoadingScreen';
+import useScreenSize from '@/common/screen/useScreenSize';
+import {SegmentationPoint} from '@/common/tracker/Tracker';
+import {
+  activeTrackletObjectIdAtom,
+  frameIndexAtom,
+  isAddObjectEnabledAtom,
+  isPlayingAtom,
+  isVideoLoadingAtom,
+  pointsAtom,
+  sessionAtom,
+  streamingStateAtom,
+  trackletObjectsAtom,
+  uploadingStateAtom,
+  VideoData,
+} from '@/demo/atoms';
+import useSettingsContext from '@/settings/useSettingsContext';
+import {color, spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+import {useAtom, useAtomValue, useSetAtom} from 'jotai';
+import {useEffect, useState} from 'react';
+import type {ErrorObject} from 'serialize-error';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    flexDirection: 'column',
+    overflow: 'auto',
+    width: '100%',
+    borderColor: color['gray-800'],
+    backgroundColor: color['gray-800'],
+    borderWidth: 8,
+    borderRadius: 12,
+    '@media screen and (max-width: 768px)': {
+      // on mobile, we want to grow the editor container so that the editor
+      // fills the remaining vertical space between the navbar and bottom
+      // of the page
+      flexGrow: 1,
+      borderWidth: 0,
+      borderRadius: 0,
+      paddingBottom: spacing[4],
+    },
+  },
+  loadingScreenWrapper: {
+    position: 'absolute',
+    top: 0,
+    left: 0,
+    width: '100%',
+    height: '100%',
+    background: 'white',
+    overflow: 'hidden',
+    overflowY: 'auto',
+    zIndex: 999,
+  },
+});
+
+type Props = {
+  video: VideoData;
+};
+
+export default function DemoVideoEditor({video: inputVideo}: Props) {
+  const {settings} = useSettingsContext();
+  const video = useVideo();
+
+  const [isSessionStartFailed, setIsSessionStartFailed] =
+    useState<boolean>(false);
+
+  const [session, setSession] = useAtom(sessionAtom);
+
+  const [activeTrackletId, setActiveTrackletObjectId] = useAtom(
+    activeTrackletObjectIdAtom,
+  );
+  const setTrackletObjects = useSetAtom(trackletObjectsAtom);
+  const setFrameIndex = useSetAtom(frameIndexAtom);
+  const points = useAtomValue(pointsAtom);
+  const isAddObjectEnabled = useAtomValue(isAddObjectEnabledAtom);
+  const streamingState = useAtomValue(streamingStateAtom);
+  const isPlaying = useAtomValue(isPlayingAtom);
+  const isVideoLoading = useAtomValue(isVideoLoadingAtom);
+  const uploadingState = useAtomValue(uploadingStateAtom);
+
+  const [renderingError, setRenderingError] = useState<ErrorObject | null>(
+    null,
+  );
+
+  const {isMobile} = useScreenSize();
+
+  const [tabIndex] = useToolbarTabs();
+  const {enqueueMessage} = useMessagesSnackbar();
+
+  useCloseSessionBeforeUnload();
+
+  const {resetEditor, resetSession} = useResetDemoEditor();
+  useEffect(() => {
+    resetEditor();
+  }, [inputVideo, resetEditor]);
+
+  useEffect(() => {
+    function onFrameUpdate(event: FrameUpdateEvent) {
+      setFrameIndex(event.index);
+    }
+
+    // Listen to frame updates to fetch the frame index in the main thread,
+    // which is then used downstream to render points per frame.
+    video?.addEventListener('frameUpdate', onFrameUpdate);
+
+    function onSessionStarted(event: SessionStartedEvent) {
+      setSession({id: event.sessionId, ranPropagation: false});
+    }
+
+    video?.addEventListener('sessionStarted', onSessionStarted);
+
+    function onSessionStartFailed() {
+      setIsSessionStartFailed(true);
+    }
+
+    video?.addEventListener('sessionStartFailed', onSessionStartFailed);
+
+    function onTrackletsUpdated(event: TrackletsEvent) {
+      const tracklets = event.tracklets;
+      if (tracklets.length === 0) {
+        resetSession();
+      }
+      setTrackletObjects(tracklets);
+    }
+
+    video?.addEventListener('trackletsUpdated', onTrackletsUpdated);
+
+    function onRenderingError(event: RenderingErrorEvent) {
+      setRenderingError(event.error);
+    }
+
+    video?.addEventListener('renderingError', onRenderingError);
+
+    video?.initializeTracker('SAM 2', {
+      inferenceEndpoint: settings.inferenceAPIEndpoint,
+    });
+
+    video?.startSession(inputVideo.path);
+
+    return () => {
+      video?.closeSession();
+      video?.removeEventListener('frameUpdate', onFrameUpdate);
+      video?.removeEventListener('sessionStarted', onSessionStarted);
+      video?.removeEventListener('sessionStartFailed', onSessionStartFailed);
+      video?.removeEventListener('trackletsUpdated', onTrackletsUpdated);
+      video?.removeEventListener('renderingError', onRenderingError);
+    };
+  }, [
+    setFrameIndex,
+    setSession,
+    setTrackletObjects,
+    resetSession,
+    inputVideo,
+    video,
+    settings.inferenceAPIEndpoint,
+    settings.videoAPIEndpoint,
+  ]);
+
+  async function handleOptimisticPointUpdate(newPoints: SegmentationPoint[]) {
+    if (session == null) {
+      return;
+    }
+
+    async function createActiveTracklet() {
+      if (!isAddObjectEnabled || newPoints.length === 0) {
+        return;
+      }
+      const tracklet = await video?.createTracklet();
+      if (tracklet != null && newPoints.length > 0) {
+        setActiveTrackletObjectId(tracklet.id);
+        video?.updatePoints(tracklet.id, [newPoints[newPoints.length - 1]]);
+      }
+    }
+
+    if (activeTrackletId != null) {
+      video?.updatePoints(activeTrackletId, newPoints);
+    } else {
+      await createActiveTracklet();
+    }
+    enqueueMessage('pointClick');
+  }
+
+  async function handleAddPoint(point: SegmentationPoint) {
+    if (streamingState === 'partial' || streamingState === 'requesting') {
+      return;
+    }
+    if (isPlaying) {
+      return video?.pause();
+    }
+    handleOptimisticPointUpdate([...points, point]);
+  }
+
+  function handleRemovePoint(point: SegmentationPoint) {
+    if (
+      isPlaying ||
+      streamingState === 'partial' ||
+      streamingState === 'requesting'
+    ) {
+      return;
+    }
+    handleOptimisticPointUpdate(points.filter(p => p !== point));
+  }
+
+  // The interaction layer handles clicks onto the video canvas. It is used
+  // to get absolute point clicks within the video's coordinate system.
+  // The PointsLayer handles rendering of input points and allows removing
+  // individual points by clicking on them.
+  const layers = (
+    <>
+      {tabIndex === OBJECT_TOOLBAR_INDEX && (
+        <>
+          <InteractionLayer
+            key="interaction-layer"
+            onPoint={point => handleAddPoint(point)}
+          />
+          <PointsLayer
+            key="points-layer"
+            points={points}
+            onRemovePoint={handleRemovePoint}
+          />
+        </>
+      )}
+      {!isMobile && <MessagesSnackbar key="snackbar-layer" />}
+    </>
+  );
+
+  return (
+    <>
+      {(isVideoLoading || session === null) && !isSessionStartFailed && (
+        <div {...stylex.props(styles.loadingScreenWrapper)}>
+          <LoadingStateScreen
+            title="Loading demo..."
+            description="This may take a few moments, you're almost there!"
+          />
+        </div>
+      )}
+      {isSessionStartFailed && (
+        <div {...stylex.props(styles.loadingScreenWrapper)}>
+          <LoadingStateScreen
+            title="Did we just break the internet?"
+            description={
+              <>Uh oh, it looks like there was an issue starting a session.</>
+            }
+            linkProps={{to: '..', label: 'Back to homepage'}}
+          />
+        </div>
+      )}
+      {isMobile && renderingError != null && (
+        <div {...stylex.props(styles.loadingScreenWrapper)}>
+          <LoadingStateScreen
+            title="Well, this is embarrassing..."
+            description="This demo is not optimized for your device. Please try again on a different device with a larger screen."
+            linkProps={{to: '..', label: 'Back to homepage'}}
+          />
+        </div>
+      )}
+      {uploadingState !== 'default' && (
+        <div {...stylex.props(styles.loadingScreenWrapper)}>
+          <UploadLoadingScreen />
+        </div>
+      )}
+      <div {...stylex.props(styles.container)}>
+        <VideoEditor
+          video={inputVideo}
+          layers={layers}
+          loading={session == null}>
+          <div className="bg-graydark-800 w-full">
+            <VideoFilmstripWithPlayback />
+            <TrackletsAnnotation />
+          </div>
+        </VideoEditor>
+      </div>
+    </>
+  );
+}

demo/frontend/src/common/components/video/editor/ImageUtils.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export function convertVideoFrameToImageData(
+  videoFrame: VideoFrame,
+): ImageData | undefined {
+  const canvas = new OffscreenCanvas(
+    videoFrame.displayWidth,
+    videoFrame.displayHeight,
+  );
+  const ctx = canvas.getContext('2d');
+  ctx?.drawImage(videoFrame, 0, 0);
+  return ctx?.getImageData(0, 0, canvas.width, canvas.height);
+}
+
+/**
+ * This utility provides two functions:
+ * `process`: to find the bounding box of non-empty pixels from an ImageData, when looping through all its pixels
+ * `crop` to cut out the subsection found in `process`
+ * @returns
+ */
+export function findBoundingBox() {
+  let xMin = Number.MAX_VALUE;
+  let yMin = Number.MAX_VALUE;
+  let xMax = Number.MIN_VALUE;
+  let yMax = Number.MIN_VALUE;
+  return {
+    process: function (x: number, y: number, hasData: boolean) {
+      if (hasData) {
+        xMin = Math.min(x, xMin);
+        xMax = Math.max(x, xMax);
+        yMin = Math.min(y, yMin);
+        yMax = Math.max(y, yMax);
+      }
+      return [xMin, xMax, yMin, yMax];
+    },
+    crop(imageData: ImageData): ImageData | null {
+      const canvas = new OffscreenCanvas(imageData.width, imageData.height);
+      const ctx = canvas.getContext('2d');
+
+      const boundingBoxWidth = xMax - xMin;
+      const boundingBoxHeight = yMax - yMin;
+      if (ctx && boundingBoxWidth > 0 && boundingBoxHeight > 0) {
+        ctx.clearRect(0, 0, canvas.width, canvas.height);
+        ctx.putImageData(imageData, 0, 0);
+        return ctx.getImageData(
+          xMin,
+          yMin,
+          boundingBoxWidth,
+          boundingBoxHeight,
+        );
+      } else {
+        return null;
+      }
+    },
+    getBox(): [[number, number], [number, number]] {
+      return [
+        [xMin, yMin],
+        [xMax, yMax],
+      ];
+    },
+  };
+}
+
+export function magnifyImageRegion(
+  canvas: HTMLCanvasElement | null,
+  x: number,
+  y: number,
+  radius: number = 25,
+  scale: number = 2,
+): string {
+  if (canvas == null) {
+    return '';
+  }
+  const ctx = canvas.getContext('2d');
+  if (ctx) {
+    const minX = x - radius < 0 ? radius - x : 0;
+    const minY = y - radius < 0 ? radius - y : 0;
+    const region = ctx.getImageData(
+      Math.max(x - radius, 0),
+      Math.max(y - radius, 0),
+      radius * 2,
+      radius * 2,
+    );
+
+    // ImageData doesn't scale-transform correctly on canvas
+    // So we first draw the original size on an offscreen canvas, and then scale it
+    const regionCanvas = new OffscreenCanvas(region.width, region.height);
+    const regionCtx = regionCanvas.getContext('2d');
+    regionCtx?.putImageData(region, minX > 0 ? minX : 0, minY > 0 ? minY : 0);
+
+    const scaleCanvas = document.createElement('canvas');
+    scaleCanvas.width = Math.round(region.width * scale);
+    scaleCanvas.height = Math.round(region.height * scale);
+    const scaleCtx = scaleCanvas.getContext('2d');
+    scaleCtx?.scale(scale, scale);
+    scaleCtx?.drawImage(regionCanvas, 0, 0);
+
+    return scaleCanvas.toDataURL();
+  }
+  return '';
+}

demo/frontend/src/common/components/video/editor/VideoEditor.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {VideoData} from '@/demo/atoms';
+import stylex, {StyleXStyles} from '@stylexjs/stylex';
+import {useSetAtom} from 'jotai';
+import {PropsWithChildren, RefObject, useEffect, useRef} from 'react';
+import Video, {VideoRef} from '../Video';
+import {videoAtom} from './atoms';
+
+const MAX_VIDEO_WIDTH = 1280;
+
+const styles = stylex.create({
+  editorContainer: {
+    position: 'relative',
+    display: 'flex',
+    flexDirection: 'column',
+    alignItems: 'center',
+    width: '100%',
+    height: '100%',
+    borderRadius: '0.375rem',
+    overflow: {
+      default: 'clip',
+      '@media screen and (max-width: 768px)': 'visible',
+    },
+  },
+  videoContainer: {
+    position: 'relative',
+    flexGrow: 1,
+    overflow: 'hidden',
+    width: '100%',
+    maxWidth: MAX_VIDEO_WIDTH,
+  },
+  layers: {
+    position: 'absolute',
+    left: 0,
+    top: 0,
+    bottom: 0,
+    right: 0,
+  },
+  loadingMessage: {
+    position: 'absolute',
+    top: '8px',
+    right: '8px',
+    padding: '6px 10px',
+    backgroundColor: '#6441D2CC',
+    color: '#FFF',
+    display: 'flex',
+    alignItems: 'center',
+    gap: '8px',
+    borderRadius: '8px',
+    fontSize: '0.8rem',
+  },
+});
+
+export type InteractionLayerProps = {
+  style: StyleXStyles;
+  videoRef: RefObject<VideoRef>;
+};
+
+export type ControlsProps = {
+  isPlaying: boolean;
+  onPlay: () => void;
+  onPause: () => void;
+  onPreviousFrame?: () => void;
+  onNextFrame?: () => void;
+};
+
+type Props = PropsWithChildren<{
+  video: VideoData;
+  layers?: React.ReactNode;
+  loading?: boolean;
+}>;
+
+export default function VideoEditor({
+  video: inputVideo,
+  layers,
+  loading,
+  children,
+}: Props) {
+  const videoRef = useRef<VideoRef>(null);
+  const setVideo = useSetAtom(videoAtom);
+
+  // Initialize video atom
+  useEffect(() => {
+    setVideo(videoRef.current);
+    return () => {
+      setVideo(null);
+    };
+  }, [setVideo]);
+
+  return (
+    <div {...stylex.props(styles.editorContainer)}>
+      <div {...stylex.props(styles.videoContainer)}>
+        <Video
+          ref={videoRef}
+          src={inputVideo.url}
+          width={inputVideo.width}
+          height={inputVideo.height}
+          loading={loading}
+        />
+        <div {...stylex.props(styles.layers)}>{layers}</div>
+      </div>
+      {children}
+    </div>
+  );
+}

demo/frontend/src/common/components/video/editor/VideoEditorUtils.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Mask, Tracklet} from '@/common/tracker/Tracker';
+import {
+  convertVideoFrameToImageData,
+  findBoundingBox,
+} from '@/common/utils/ImageUtils';
+import {DataArray} from '@/jscocotools/mask';
+import invariant from 'invariant';
+
+function getCanvas(
+  width: number,
+  height: number,
+  isOffscreen: boolean = false,
+): HTMLCanvasElement | OffscreenCanvas {
+  if (isOffscreen || typeof document === 'undefined') {
+    return new OffscreenCanvas(width, height);
+  }
+  const canvas = document.createElement('canvas');
+  canvas.width = width;
+  canvas.height = height;
+  return canvas;
+}
+
+export function drawFrame(
+  ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
+  frame: VideoFrame | HTMLImageElement,
+  width: number,
+  height: number,
+) {
+  ctx?.drawImage(frame, 0, 0, width, height);
+}
+
+/**
+ * Given a mask and the image frame, get the masked image cropped to its bounding box.
+ */
+export function getThumbnailImageDataOld(
+  mask: DataArray,
+  videoFrame: VideoFrame,
+): ImageData | null {
+  const data = mask.data;
+  if (!ArrayBuffer.isView(data) || !(data instanceof Uint8Array)) {
+    return new ImageData(0, 0);
+  }
+
+  const frame = convertVideoFrameToImageData(videoFrame);
+  if (!frame) {
+    return new ImageData(0, 0);
+  }
+
+  const frameData = frame.data;
+  const scaleX = frame.width / mask.shape[1];
+  const scaleY = frame.height / mask.shape[0];
+  const boundingBox = findBoundingBox();
+  const transformedData = new Uint8ClampedArray(data.length * 4);
+
+  for (let i = 0; i < data.length; i++) {
+    // Since the mask is rotated, new width is the mask's height = mask.shape[1];
+    // Transform matrix: doing a rotate 90deg and then flip horizontal is the same as flipping x and y
+    // [ 0 1 ]   [ -1 0 ]   =   [ 0 1 ]
+    // [-1 0 ] x [  0 1 ]   =   [ 1 0 ]
+    // So, we can find the new index as: newY * newWidth + newX
+    const newX = Math.floor(i / mask.shape[0]); // ie, new x is the current y
+    const newY = i % mask.shape[0];
+    const transformedIndex = (newY * mask.shape[1] + newX) * 4;
+    const frameDataIndex = (newY * mask.shape[1] * scaleY + newX * scaleX) * 4;
+
+    transformedData[transformedIndex] = frameData[frameDataIndex];
+    transformedData[transformedIndex + 1] = frameData[frameDataIndex + 1];
+    transformedData[transformedIndex + 2] = frameData[frameDataIndex + 2];
+    transformedData[transformedIndex + 3] = (data[i] && 255) || 0; // A value
+
+    boundingBox.process(newX, newY, data[i] > 0);
+  }
+
+  const rotatedData = new ImageData(
+    transformedData,
+    mask.shape[1],
+    mask.shape[0],
+  ); // flip w and h of the mask
+
+  return boundingBox.crop(rotatedData);
+}
+
+/**
+ * Given a mask, the mask rendering context, and the video frame, get the
+ * masked image cropped to its bounding box.
+ */
+function getThumbnailImageData(
+  mask: Mask,
+  maskCtx: OffscreenCanvasRenderingContext2D,
+  frameBitmap: ImageBitmap,
+): ImageData | null {
+  const x = mask.bounds[0][0];
+  const y = mask.bounds[0][1];
+  const w = mask.bounds[1][0] - mask.bounds[0][0];
+  const h = mask.bounds[1][1] - mask.bounds[0][1];
+
+  if (w <= 0 || h <= 0) {
+    return null;
+  }
+
+  const thumbnailMaskData = maskCtx.getImageData(x, y, w, h);
+
+  const canvas = new OffscreenCanvas(w, h);
+  const ctx = canvas.getContext('2d');
+  invariant(ctx !== null, '2d context cannot be null');
+
+  ctx.putImageData(thumbnailMaskData, 0, 0);
+  ctx.globalCompositeOperation = 'source-in';
+  ctx.drawImage(frameBitmap, x, y, w, h, 0, 0, w, h);
+
+  return ctx.getImageData(0, 0, w, h);
+}
+
+export async function generateThumbnail(
+  track: Tracklet,
+  frameIndex: number,
+  mask: Mask,
+  frame: VideoFrame,
+  ctx: OffscreenCanvasRenderingContext2D,
+): Promise<void> {
+  // If a frame doesn't have points, the points will be undefined.
+  const hasPoints = (track.points[frameIndex]?.length ?? 0) > 0;
+  if (!hasPoints) {
+    return;
+  }
+  invariant(frame !== null, 'frame must be ready');
+  const bitmap = await createImageBitmap(frame);
+  const thumbnailImageData = getThumbnailImageData(
+    mask,
+    ctx as OffscreenCanvasRenderingContext2D,
+    bitmap,
+  );
+
+  bitmap.close();
+  if (thumbnailImageData != null) {
+    const thumbnailDataURL = await getDataURLFromImageData(thumbnailImageData);
+    track.thumbnail = thumbnailDataURL;
+  }
+}
+
+export async function getDataURLFromImageData(
+  imageData: ImageData | null,
+): Promise<string> {
+  if (!imageData) {
+    return '';
+  }
+
+  const canvas = getCanvas(imageData.width, imageData.height);
+  const ctx = canvas.getContext('2d');
+
+  if (ctx === null) {
+    return '';
+  }
+
+  ctx?.putImageData(imageData, 0, 0);
+
+  if (canvas instanceof OffscreenCanvas) {
+    const blob = await canvas.convertToBlob();
+    return new Promise(resolve => {
+      const reader = new FileReader();
+      reader.addEventListener(
+        'load',
+        () => {
+          const result = reader.result;
+          if (typeof result === 'string') {
+            resolve(result);
+          } else {
+            resolve('');
+          }
+        },
+        false,
+      );
+      reader.readAsDataURL(blob);
+    });
+  }
+  return canvas.toDataURL();
+}
+
+export function hexToRgb(hex: string): {
+  r: number;
+  g: number;
+  b: number;
+  a: number;
+} {
+  const result = /^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})?$/i.exec(
+    hex,
+  );
+  return result
+    ? {
+        r: parseInt(result[1], 16),
+        g: parseInt(result[2], 16),
+        b: parseInt(result[3], 16),
+        a: result[4] != null ? parseInt(result[4], 16) : 128,
+      }
+    : {r: 255, g: 0, b: 0, a: 128};
+}
+
+export function getPointInImage(
+  event: React.MouseEvent<HTMLElement>,
+  canvas: HTMLCanvasElement,
+  normalized: boolean = false,
+): [x: number, y: number] {
+  const rect = canvas.getBoundingClientRect();
+
+  const matrix = new DOMMatrix();
+
+  // First, center the image
+  const elementCenter = new DOMPoint(
+    canvas.clientWidth / 2,
+    canvas.clientHeight / 2,
+  );
+  const imageCenter = new DOMPoint(canvas.width / 2, canvas.height / 2);
+  matrix.translateSelf(
+    elementCenter.x - imageCenter.x,
+    elementCenter.y - imageCenter.y,
+  );
+
+  // Containing the object take the minimal scale
+  const scale = Math.min(
+    canvas.clientWidth / canvas.width,
+    canvas.clientHeight / canvas.height,
+  );
+  matrix.scaleSelf(scale, scale, 1, imageCenter.x, imageCenter.y);
+
+  const point = new DOMPoint(
+    event.clientX - rect.left,
+    event.clientY - rect.top,
+  );
+  const imagePoint = matrix.inverse().transformPoint(point);
+
+  const x = Math.max(Math.min(imagePoint.x, canvas.width), 0);
+  const y = Math.max(Math.min(imagePoint.y, canvas.height), 0);
+
+  if (normalized) {
+    return [x / canvas.width, y / canvas.height];
+  }
+  return [x, y];
+}

demo/frontend/src/common/components/video/editor/atoms.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {atom} from 'jotai';
+import {VideoRef} from '../Video';
+
+export const videoAtom = atom<VideoRef | null>(null);

demo/frontend/src/common/components/video/editor/useResetEditor.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {OBJECT_TOOLBAR_INDEX} from '@/common/components/toolbar/ToolbarConfig';
+import useToolbarTabs from '@/common/components/toolbar/useToolbarTabs';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {
+  activeTrackletObjectIdAtom,
+  frameIndexAtom,
+  isPlayingAtom,
+  isStreamingAtom,
+  sessionAtom,
+  streamingStateAtom,
+  trackletObjectsAtom,
+} from '@/demo/atoms';
+import {DEFAULT_EFFECT_LAYERS} from '@/demo/DemoConfig';
+import {useSetAtom} from 'jotai';
+import {useCallback} from 'react';
+
+type State = {
+  resetEditor: () => void;
+  resetEffects: () => void;
+  resetSession: () => void;
+};
+
+export default function useResetEditor(): State {
+  const video = useVideo();
+
+  const setSession = useSetAtom(sessionAtom);
+  const setActiveTrackletObjectId = useSetAtom(activeTrackletObjectIdAtom);
+  const setTrackletObjects = useSetAtom(trackletObjectsAtom);
+  const setFrameIndex = useSetAtom(frameIndexAtom);
+  const setStreamingState = useSetAtom(streamingStateAtom);
+  const setIsPlaying = useSetAtom(isPlayingAtom);
+  const setIsStreaming = useSetAtom(isStreamingAtom);
+  const [, setDemoTabIndex] = useToolbarTabs();
+
+  const resetEffects = useCallback(() => {
+    video?.setEffect(DEFAULT_EFFECT_LAYERS.background, 0, {variant: 0});
+    video?.setEffect(DEFAULT_EFFECT_LAYERS.highlight, 1, {variant: 0});
+  }, [video]);
+
+  const resetEditor = useCallback(() => {
+    setFrameIndex(0);
+    setSession(null);
+    setActiveTrackletObjectId(0);
+    setTrackletObjects([]);
+    setStreamingState('none');
+    setIsPlaying(false);
+    setIsStreaming(false);
+    resetEffects();
+    setDemoTabIndex(OBJECT_TOOLBAR_INDEX);
+  }, [
+    setFrameIndex,
+    setSession,
+    setActiveTrackletObjectId,
+    setTrackletObjects,
+    setStreamingState,
+    setIsPlaying,
+    setIsStreaming,
+    resetEffects,
+    setDemoTabIndex,
+  ]);
+
+  const resetSession = useCallback(() => {
+    setSession(prev => {
+      if (prev === null) {
+        return prev;
+      }
+      return {...prev, ranPropagation: false};
+    });
+    setActiveTrackletObjectId(null);
+    resetEffects();
+  }, [setSession, setActiveTrackletObjectId, resetEffects]);
+
+  return {resetEditor, resetEffects, resetSession};
+}

demo/frontend/src/common/components/video/editor/useVideo.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {useAtomValue} from 'jotai';
+import {videoAtom} from './atoms';
+
+export default function useVideo() {
+  return useAtomValue(videoAtom);
+}

demo/frontend/src/common/components/video/editor/useVideoEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useVideo from '@/common/components/video/editor/useVideo';
+import {
+  activeBackgroundEffectAtom,
+  activeHighlightEffectAtom,
+} from '@/demo/atoms';
+import {useSetAtom} from 'jotai';
+import {useCallback, useEffect} from 'react';
+import {EffectUpdateEvent} from '../VideoWorkerBridge';
+import {EffectOptions} from '../effects/Effect';
+import Effects, {EffectIndex, Effects as EffectsType} from '../effects/Effects';
+
+export default function useVideoEffect() {
+  const video = useVideo();
+  const setBackgroundEffect = useSetAtom(activeBackgroundEffectAtom);
+  const setHighlightEffect = useSetAtom(activeHighlightEffectAtom);
+
+  // The useEffect will listen to any effect updates from the worker. The
+  // worker is the source of truth, which effect and effect variant is
+  // currently applied. The main thread will be notified whenever an effect
+  // or effect variant changes.
+  useEffect(() => {
+    function onEffectUpdate(event: EffectUpdateEvent) {
+      if (event.index === EffectIndex.BACKGROUND) {
+        setBackgroundEffect(event);
+      } else {
+        setHighlightEffect(event);
+      }
+    }
+    video?.addEventListener('effectUpdate', onEffectUpdate);
+    return () => {
+      video?.removeEventListener('effectUpdate', onEffectUpdate);
+    };
+  }, [video, setBackgroundEffect, setHighlightEffect]);
+
+  return useCallback(
+    (name: keyof EffectsType, index: EffectIndex, options?: EffectOptions) => {
+      video?.setEffect(name, index, options);
+      const effect = Effects[name];
+      const effectVariant = options?.variant ?? 0;
+
+      if (index === EffectIndex.BACKGROUND) {
+        setBackgroundEffect({
+          name,
+          variant: effectVariant,
+          numVariants: effect.numVariants,
+        });
+      } else {
+        setHighlightEffect({
+          name,
+          variant: options?.variant ?? 0,
+          numVariants: effect.numVariants,
+        });
+      }
+    },
+    [video, setBackgroundEffect, setHighlightEffect],
+  );
+}

demo/frontend/src/common/components/video/effects/ArrowGLEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/Arrow.frag?raw';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import {normalizeBounds} from '@/common/utils/ShaderUtils';
+import {RLEObject, decode} from '@/jscocotools/mask';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class ArrowGLEffect extends BaseGLEffect {
+  private _numMasks: number = 0;
+  private _numMasksUniformLocation: WebGLUniformLocation | null = null;
+
+  // Must from start 1, main texture takes.
+  private _masksTextureUnitStart: number = 1;
+
+  constructor() {
+    super(4);
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+
+    this._numMasksUniformLocation = gl.getUniformLocation(program, 'uNumMasks');
+    gl.uniform1i(this._numMasksUniformLocation, this._numMasks);
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    if (!program) {
+      return;
+    }
+    invariant(gl !== null, 'WebGL2 context is required');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    // dynamic uniforms per frame
+    const styleIndex = Math.floor(this.variant / 2) % 2;
+    gl.uniform1i(this._numMasksUniformLocation, context.masks.length);
+    gl.uniform1f(
+      gl.getUniformLocation(program, 'uCurrentFrame'),
+      context.frameIndex,
+    );
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uLineColor'),
+      this.variant % 2 === 0 ? 0 : 1,
+    );
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uArrow'),
+      styleIndex === 0 ? 1 : 0,
+    );
+
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, this._frameTexture);
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      context.width,
+      context.height,
+      0,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      context.frame,
+    );
+
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+
+    // Create and bind 2D textures for each mask
+    context.masks.forEach((mask, index) => {
+      const maskTexture = gl.createTexture();
+      const decodedMask = decode([mask.bitmap as RLEObject]);
+      const maskData = decodedMask.data as Uint8Array;
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, maskTexture);
+
+      const boundaries = normalizeBounds(
+        mask.bounds[0],
+        mask.bounds[1],
+        context.width,
+        context.height,
+      );
+
+      gl.uniform1i(
+        gl.getUniformLocation(program, `uMaskTexture${index}`),
+        index + this._masksTextureUnitStart,
+      );
+      gl.uniform4fv(gl.getUniformLocation(program, `bbox${index}`), boundaries);
+
+      // dynamic uniforms per mask
+      gl.uniform1i(
+        gl.getUniformLocation(program, `uMaskTexture${index}`),
+        this._masksTextureUnitStart + index,
+      );
+
+      gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
+      gl.texImage2D(
+        gl.TEXTURE_2D,
+        0,
+        gl.LUMINANCE,
+        context.height,
+        context.width,
+        0,
+        gl.LUMINANCE,
+        gl.UNSIGNED_BYTE,
+        maskData,
+      );
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+
+      gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+    });
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+    ctx.drawImage(this._canvas, 0, 0);
+  }
+}

demo/frontend/src/common/components/video/effects/BackgroundBlurEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/BackgroundBlur.frag?raw';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class BackgroundBlurEffect extends BaseGLEffect {
+  private _blurRadius: number = 3;
+
+  constructor() {
+    super(3);
+
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uBlurRadius'),
+      this._blurRadius,
+    );
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    if (!program) {
+      return;
+    }
+    invariant(gl !== null, 'WebGL2 context is required');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    const blurRadius = [3, 6, 12][this.variant % 3];
+    gl.uniform1i(gl.getUniformLocation(program, 'uBlurRadius'), blurRadius);
+
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, this._frameTexture);
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      context.width,
+      context.height,
+      0,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      context.frame,
+    );
+
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+    ctx.drawImage(this._canvas, 0, 0);
+  }
+}

demo/frontend/src/common/components/video/effects/BackgroundTextEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Tracklet} from '@/common/tracker/Tracker';
+import {DEMO_SHORT_NAME} from '@/demo/DemoConfig';
+import {Bound, CanvasForm, Num, Pt, Shaping} from 'pts';
+import {AbstractEffect, EffectFrameContext} from './Effect';
+
+export default class BackgroundTextEffect extends AbstractEffect {
+  constructor() {
+    super(2);
+  }
+
+  apply(
+    form: CanvasForm,
+    context: EffectFrameContext,
+    _tracklets: Tracklet[],
+  ): void {
+    form.image([0, 0], context.frame);
+
+    const words = ['SEGMENT', 'ANYTHING', 'WOW'];
+    const paragraph = `${DEMO_SHORT_NAME} is designed for efficient video processing with streaming inference to enable real-time, interactive applications.`;
+    const progress = context.frameIndex / context.totalFrames;
+
+    // Zooming heading
+    if (this.variant % 2 === 0) {
+      const step = context.totalFrames / words.length;
+      const wordIndex = Math.floor(progress * words.length);
+      const fontSize = context.width / Math.max(4, words[wordIndex].length - 1);
+      const sizeMax = fontSize * 1.2;
+
+      const t = Shaping.quadraticInOut(
+        Num.cycle((context.frameIndex - wordIndex * step) / step),
+      );
+      const currentSize = fontSize + Shaping.sineInOut(t, sizeMax - fontSize);
+      form.fillOnly('#fff').font(currentSize, 'bold');
+      const area = new Pt(
+        context.width,
+        context.height - (context.height / 4) * (1 - t),
+      )
+        .toBound()
+        .scale(1.5, [context.width / 2, 0]);
+
+      form
+        .alignText('center', 'middle')
+        .textBox(area, words[wordIndex], 'middle');
+
+      // Scrolling paragraph
+    } else {
+      const t = Shaping.quadraticInOut(Num.cycle(progress));
+      const offset = t * context.height;
+      const area = Bound.fromArray([
+        [0, -context.height + offset],
+        [context.width, context.height],
+      ]);
+      form.fillOnly('#00000066').rect(area);
+      form.fillOnly('#fff').font(context.width / 8, 'bold');
+      form
+        .fillOnly('#fff')
+        .alignText('start')
+        .paragraphBox(area, paragraph, 0.8, 'top', false);
+    }
+  }
+}

demo/frontend/src/common/components/video/effects/BaseGLEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import Logger from '@/common/logger/Logger';
+import {Tracklet} from '@/common/tracker/Tracker';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+import {AbstractEffect, EffectFrameContext, EffectInit} from './Effect';
+
+export default abstract class BaseGLEffect extends AbstractEffect {
+  protected _canvas: OffscreenCanvas | null = null;
+  protected _gl: WebGL2RenderingContext | null = null;
+  protected _program: WebGLProgram | null = null;
+
+  protected _frameTextureUnit: number = 0;
+  protected _frameTexture: WebGLTexture | null = null;
+
+  protected vertexShaderSource: string = '';
+  protected fragmentShaderSource: string = '';
+
+  protected _vertexShader: WebGLShader | null = null;
+  protected _fragmentShader: WebGLShader | null = null;
+
+  async setup(init: EffectInit): Promise<void> {
+    const {canvas, gl} = init;
+
+    if (canvas != null && gl != null) {
+      this._canvas = canvas;
+      this._gl = gl;
+    }
+
+    invariant(this._gl !== null, 'WebGL2 context is required');
+
+    const program = this._gl.createProgram();
+    this._program = program;
+
+    {
+      const vertexShader = this._gl.createShader(this._gl.VERTEX_SHADER);
+      this._vertexShader = vertexShader;
+      invariant(vertexShader !== null, 'vertexShader required');
+      this._gl.shaderSource(vertexShader, this.vertexShaderSource);
+      this._gl.compileShader(vertexShader);
+      invariant(program !== null, 'program required');
+      this._gl.attachShader(program, vertexShader);
+
+      const fragmentShader = this._gl.createShader(this._gl.FRAGMENT_SHADER);
+      this._fragmentShader = fragmentShader;
+      invariant(fragmentShader !== null, 'fragmentShader required');
+      this._gl.shaderSource(fragmentShader, this.fragmentShaderSource);
+      this._gl.compileShader(fragmentShader);
+      this._gl.attachShader(program, fragmentShader);
+
+      this._gl.linkProgram(program);
+
+      if (!this._gl.getProgramParameter(program, this._gl.LINK_STATUS)) {
+        Logger.error(this._gl.getShaderInfoLog(vertexShader));
+        Logger.error(this._gl.getShaderInfoLog(fragmentShader));
+      }
+    }
+    this._gl.useProgram(program);
+
+    this.setupBuffers(this._gl);
+    this.setupUniforms(this._gl, program, init);
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    invariant(gl !== null, 'WebGL2 context is required');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    gl.activeTexture(gl.TEXTURE0 + this._frameTextureUnit);
+    gl.bindTexture(gl.TEXTURE_2D, this._frameTexture);
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      context.frame.width,
+      context.frame.height,
+      0,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      context.frame,
+    );
+
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+    gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
+
+    // Apply shader
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+    ctx.drawImage(this._canvas, 0, 0);
+  }
+
+  async cleanup(): Promise<void> {
+    if (this._gl != null) {
+      // Dispose of WebGL resources, e.g., textures, buffers, etc.
+      if (this._frameTexture != null) {
+        this._gl.deleteTexture(this._frameTexture);
+        this._frameTexture = null;
+      }
+
+      if (
+        this._program != null &&
+        this._vertexShader != null &&
+        this._fragmentShader != null
+      ) {
+        this._gl.detachShader(this._program, this._vertexShader);
+        this._gl.deleteShader(this._vertexShader);
+        this._gl.detachShader(this._program, this._fragmentShader);
+        this._gl.deleteShader(this._fragmentShader);
+      }
+    }
+  }
+
+  protected setupBuffers(gl: WebGL2RenderingContext) {
+    const vertexBufferData = new Float32Array([
+      1.0, 1.0, -1.0, 1.0, 1.0, -1.0, -1.0, -1.0,
+    ]);
+
+    const texCoordBufferData = new Float32Array([
+      1.0, 1.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0,
+    ]);
+
+    const vertexBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, vertexBufferData, gl.STATIC_DRAW);
+    gl.vertexAttribPointer(0, 2, gl.FLOAT, false, 0, 0);
+    gl.enableVertexAttribArray(0);
+
+    const texCoordBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, texCoordBufferData, gl.STATIC_DRAW);
+    gl.vertexAttribPointer(1, 2, gl.FLOAT, false, 0, 0);
+    gl.enableVertexAttribArray(1);
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ) {
+    this._frameTexture = gl.createTexture();
+
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uSampler'),
+      this._frameTextureUnit,
+    );
+
+    gl.uniform2f(
+      gl.getUniformLocation(program, 'uSize'),
+      init.width,
+      init.height,
+    );
+  }
+}

demo/frontend/src/common/components/video/effects/BurstGLEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {hexToRgb} from '@/common/components/video/editor/VideoEditorUtils';
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/Burst.frag?raw';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import {normalizeBounds, preAllocateTextures} from '@/common/utils/ShaderUtils';
+import {RLEObject, decode} from '@/jscocotools/mask';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class BurstGLEffect extends BaseGLEffect {
+  private _numMasks: number = 0;
+  private _numMasksUniformLocation: WebGLUniformLocation | null = null;
+
+  // Must from start 1, main texture takes.
+  private _masksTextureUnitStart: number = 1;
+  private _maskTextures: WebGLTexture[] = [];
+
+  constructor() {
+    super(4);
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+
+    this._numMasksUniformLocation = gl.getUniformLocation(program, 'uNumMasks');
+    gl.uniform1i(this._numMasksUniformLocation, this._numMasks);
+
+    // We know the max number of textures, pre-allocate 3.
+    this._maskTextures = preAllocateTextures(gl, 3);
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    if (!program) {
+      return;
+    }
+    invariant(gl !== null, 'WebGL2 context is required');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    const styleIndex = Math.floor(this.variant / 2) % 2;
+
+    // dynamic uniforms per frame
+    gl.uniform1i(this._numMasksUniformLocation, context.masks.length);
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uLineColor'),
+      this.variant % 2 === 0 ? 1 : 0,
+    );
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uInterleave'),
+      styleIndex === 0 ? 0 : 1,
+    );
+
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, this._frameTexture);
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      context.width,
+      context.height,
+      0,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      context.frame,
+    );
+
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+
+    // Create and bind 2D textures for each mask
+    context.masks.forEach((mask, index) => {
+      const decodedMask = decode([mask.bitmap as RLEObject]);
+      const maskData = decodedMask.data as Uint8Array;
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, this._maskTextures[index]);
+
+      const boundaries = normalizeBounds(
+        mask.bounds[0],
+        mask.bounds[1],
+        context.width,
+        context.height,
+      );
+
+      // dynamic uniforms per mask
+      gl.uniform1i(
+        gl.getUniformLocation(program, `uMaskTexture${index}`),
+        this._masksTextureUnitStart + index,
+      );
+      const color = hexToRgb(context.maskColors[index]);
+      gl.uniform4f(
+        gl.getUniformLocation(program, `uMaskColor${index}`),
+        color.r,
+        color.g,
+        color.b,
+        color.a,
+      );
+      gl.uniform4fv(gl.getUniformLocation(program, `bbox${index}`), boundaries);
+
+      gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
+      gl.texImage2D(
+        gl.TEXTURE_2D,
+        0,
+        gl.LUMINANCE,
+        context.height,
+        context.width,
+        0,
+        gl.LUMINANCE,
+        gl.UNSIGNED_BYTE,
+        maskData,
+      );
+    });
+
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+    // Unbind textures
+    gl.bindTexture(gl.TEXTURE_2D, null);
+    context.masks.forEach((_, index) => {
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, null);
+    });
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+    ctx.drawImage(this._canvas, 0, 0);
+  }
+
+  async cleanup(): Promise<void> {
+    super.cleanup();
+
+    if (this._gl != null) {
+      // Delete mask textures to prevent memory leaks
+      this._maskTextures.forEach(texture => {
+        if (texture != null && this._gl != null) {
+          this._gl.deleteTexture(texture);
+        }
+      });
+      this._maskTextures = [];
+    }
+  }
+}

demo/frontend/src/common/components/video/effects/CutoutGLEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/Cutout.frag?raw';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import {preAllocateTextures} from '@/common/utils/ShaderUtils';
+import {RLEObject, decode} from '@/jscocotools/mask';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class CutoutGLEffect extends BaseGLEffect {
+  private _numMasks: number = 0;
+  private _numMasksUniformLocation: WebGLUniformLocation | null = null;
+
+  // Must from start 1, main texture takes.
+  private _masksTextureUnitStart: number = 1;
+  private _maskTextures: WebGLTexture[] = [];
+
+  constructor() {
+    super(4);
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+
+    this._numMasksUniformLocation = gl.getUniformLocation(program, 'uNumMasks');
+    gl.uniform1i(this._numMasksUniformLocation, this._numMasks);
+
+    // We know the max number of textures, pre-allocate 3.
+    this._maskTextures = preAllocateTextures(gl, 3);
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    if (!program) {
+      return;
+    }
+    invariant(gl !== null, 'WebGL2 context is required');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    // dynamic uniforms per frame
+    const contrastValue = [1.0, 1.6, 0.75, 0.0][this.variant % 4];
+    gl.uniform1f(gl.getUniformLocation(program, 'uContrast'), contrastValue);
+    gl.uniform1i(this._numMasksUniformLocation, context.masks.length);
+
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, this._frameTexture);
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      context.width,
+      context.height,
+      0,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      context.frame,
+    );
+
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+
+    // Create and bind 2D textures for each mask
+    context.masks.forEach((mask, index) => {
+      const decodedMask = decode([mask.bitmap as RLEObject]);
+      const maskData = decodedMask.data as Uint8Array;
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, this._maskTextures[index]);
+
+      // dynamic uniforms per mask
+      gl.uniform1i(
+        gl.getUniformLocation(program, `uMaskTexture${index}`),
+        this._masksTextureUnitStart + index,
+      );
+
+      gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
+      gl.texImage2D(
+        gl.TEXTURE_2D,
+        0,
+        gl.LUMINANCE,
+        context.height,
+        context.width,
+        0,
+        gl.LUMINANCE,
+        gl.UNSIGNED_BYTE,
+        maskData,
+      );
+    });
+
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+    // Unbind textures
+    gl.bindTexture(gl.TEXTURE_2D, null);
+    context.masks.forEach((_, index) => {
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, null);
+    });
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+    ctx.drawImage(this._canvas, 0, 0);
+  }
+
+  async cleanup(): Promise<void> {
+    super.cleanup();
+
+    if (this._gl != null) {
+      // Delete mask textures to prevent memory leaks
+      this._maskTextures.forEach(texture => {
+        if (texture != null && this._gl != null) {
+          this._gl.deleteTexture(texture);
+        }
+      });
+      this._maskTextures = [];
+    }
+  }
+}

demo/frontend/src/common/components/video/effects/DesaturateEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Tracklet} from '@/common/tracker/Tracker';
+import {CanvasForm} from 'pts';
+import {AbstractEffect, EffectFrameContext} from './Effect';
+
+export default class DesaturateEffect extends AbstractEffect {
+  constructor() {
+    super(3);
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    form.ctx.save();
+    form.ctx.filter = ['contrast(100%)', 'contrast(150%)', 'contrast(50%)'][
+      this.variant % 3
+    ];
+    form.image([0, 0], context.frame);
+    form.ctx.globalCompositeOperation = 'hue';
+    form.fillOnly('#fff').rect([
+      [0, 0],
+      [context.width, context.height],
+    ]);
+    form.ctx.restore();
+  }
+}

demo/frontend/src/common/components/video/effects/Effect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Effects} from '@/common/components/video/effects/Effects';
+import {Tracklet} from '@/common/tracker/Tracker';
+import {RLEObject} from '@/jscocotools/mask';
+import {CanvasForm} from 'pts';
+
+export type EffectLayers = {
+  background: keyof Effects;
+  highlight: keyof Effects;
+};
+
+export type EffectOptions = {
+  variant: number;
+};
+
+export type EffectInit = {
+  width: number;
+  height: number;
+  gl?: WebGL2RenderingContext;
+  canvas?: OffscreenCanvas;
+};
+
+export type EffectMask = {
+  bitmap: ImageBitmap | RLEObject;
+  bounds: [[number, number], [number, number]];
+};
+
+export type EffectActionPoint = {
+  objectId: number;
+  position: [number, number];
+};
+
+export type EffectFrameContext = {
+  frameIndex: number;
+  totalFrames: number;
+  fps: number;
+  width: number;
+  height: number;
+  masks: EffectMask[];
+  maskColors: string[];
+  frame: ImageBitmap;
+  timeParameter?: number;
+  actionPoint: EffectActionPoint | null;
+};
+
+export interface Effect {
+  variant: number;
+  numVariants: number;
+  nextVariant(): void;
+  setup(init: EffectInit): Promise<void>;
+  update(options: EffectOptions): Promise<void>;
+  cleanup(): Promise<void>;
+  apply(
+    form: CanvasForm,
+    context: EffectFrameContext,
+    tracklets: Tracklet[],
+  ): void;
+}
+
+export abstract class AbstractEffect implements Effect {
+  public numVariants: number;
+  public variant: number;
+
+  constructor(numVariants: number) {
+    this.numVariants = numVariants;
+    this.variant = 0;
+  }
+
+  nextVariant() {
+    // Cycle through variants
+    this.variant = (this.variant + 1) % this.numVariants;
+  }
+
+  async setup(_init: EffectInit): Promise<void> {
+    // noop
+  }
+
+  async update(options: EffectOptions): Promise<void> {
+    this.variant = options.variant;
+  }
+
+  async cleanup(): Promise<void> {
+    // noop
+  }
+
+  abstract apply(
+    form: CanvasForm,
+    context: EffectFrameContext,
+    tracklets: Tracklet[],
+  ): void;
+}