"Initial commit"

demo/frontend/src/common/components/video/editor/useResetEditor.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {OBJECT_TOOLBAR_INDEX} from '@/common/components/toolbar/ToolbarConfig';
+import useToolbarTabs from '@/common/components/toolbar/useToolbarTabs';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {
+  activeTrackletObjectIdAtom,
+  frameIndexAtom,
+  isPlayingAtom,
+  isStreamingAtom,
+  sessionAtom,
+  streamingStateAtom,
+  trackletObjectsAtom,
+} from '@/demo/atoms';
+import {DEFAULT_EFFECT_LAYERS} from '@/demo/DemoConfig';
+import {useSetAtom} from 'jotai';
+import {useCallback} from 'react';
+
+type State = {
+  resetEditor: () => void;
+  resetEffects: () => void;
+  resetSession: () => void;
+};
+
+export default function useResetEditor(): State {
+  const video = useVideo();
+
+  const setSession = useSetAtom(sessionAtom);
+  const setActiveTrackletObjectId = useSetAtom(activeTrackletObjectIdAtom);
+  const setTrackletObjects = useSetAtom(trackletObjectsAtom);
+  const setFrameIndex = useSetAtom(frameIndexAtom);
+  const setStreamingState = useSetAtom(streamingStateAtom);
+  const setIsPlaying = useSetAtom(isPlayingAtom);
+  const setIsStreaming = useSetAtom(isStreamingAtom);
+  const [, setDemoTabIndex] = useToolbarTabs();
+
+  const resetEffects = useCallback(() => {
+    video?.setEffect(DEFAULT_EFFECT_LAYERS.background, 0, {variant: 0});
+    video?.setEffect(DEFAULT_EFFECT_LAYERS.highlight, 1, {variant: 0});
+  }, [video]);
+
+  const resetEditor = useCallback(() => {
+    setFrameIndex(0);
+    setSession(null);
+    setActiveTrackletObjectId(0);
+    setTrackletObjects([]);
+    setStreamingState('none');
+    setIsPlaying(false);
+    setIsStreaming(false);
+    resetEffects();
+    setDemoTabIndex(OBJECT_TOOLBAR_INDEX);
+  }, [
+    setFrameIndex,
+    setSession,
+    setActiveTrackletObjectId,
+    setTrackletObjects,
+    setStreamingState,
+    setIsPlaying,
+    setIsStreaming,
+    resetEffects,
+    setDemoTabIndex,
+  ]);
+
+  const resetSession = useCallback(() => {
+    setSession(prev => {
+      if (prev === null) {
+        return prev;
+      }
+      return {...prev, ranPropagation: false};
+    });
+    setActiveTrackletObjectId(null);
+    resetEffects();
+  }, [setSession, setActiveTrackletObjectId, resetEffects]);
+
+  return {resetEditor, resetEffects, resetSession};
+}

demo/frontend/src/common/components/video/editor/useVideo.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {useAtomValue} from 'jotai';
+import {videoAtom} from './atoms';
+
+export default function useVideo() {
+  return useAtomValue(videoAtom);
+}

demo/frontend/src/common/components/video/editor/useVideoEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useVideo from '@/common/components/video/editor/useVideo';
+import {
+  activeBackgroundEffectAtom,
+  activeHighlightEffectAtom,
+} from '@/demo/atoms';
+import {useSetAtom} from 'jotai';
+import {useCallback, useEffect} from 'react';
+import {EffectUpdateEvent} from '../VideoWorkerBridge';
+import {EffectOptions} from '../effects/Effect';
+import Effects, {EffectIndex, Effects as EffectsType} from '../effects/Effects';
+
+export default function useVideoEffect() {
+  const video = useVideo();
+  const setBackgroundEffect = useSetAtom(activeBackgroundEffectAtom);
+  const setHighlightEffect = useSetAtom(activeHighlightEffectAtom);
+
+  // The useEffect will listen to any effect updates from the worker. The
+  // worker is the source of truth, which effect and effect variant is
+  // currently applied. The main thread will be notified whenever an effect
+  // or effect variant changes.
+  useEffect(() => {
+    function onEffectUpdate(event: EffectUpdateEvent) {
+      if (event.index === EffectIndex.BACKGROUND) {
+        setBackgroundEffect(event);
+      } else {
+        setHighlightEffect(event);
+      }
+    }
+    video?.addEventListener('effectUpdate', onEffectUpdate);
+    return () => {
+      video?.removeEventListener('effectUpdate', onEffectUpdate);
+    };
+  }, [video, setBackgroundEffect, setHighlightEffect]);
+
+  return useCallback(
+    (name: keyof EffectsType, index: EffectIndex, options?: EffectOptions) => {
+      video?.setEffect(name, index, options);
+      const effect = Effects[name];
+      const effectVariant = options?.variant ?? 0;
+
+      if (index === EffectIndex.BACKGROUND) {
+        setBackgroundEffect({
+          name,
+          variant: effectVariant,
+          numVariants: effect.numVariants,
+        });
+      } else {
+        setHighlightEffect({
+          name,
+          variant: options?.variant ?? 0,
+          numVariants: effect.numVariants,
+        });
+      }
+    },
+    [video, setBackgroundEffect, setHighlightEffect],
+  );
+}

demo/frontend/src/common/components/video/effects/ArrowGLEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/Arrow.frag?raw';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import {normalizeBounds} from '@/common/utils/ShaderUtils';
+import {RLEObject, decode} from '@/jscocotools/mask';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class ArrowGLEffect extends BaseGLEffect {
+  private _numMasks: number = 0;
+  private _numMasksUniformLocation: WebGLUniformLocation | null = null;
+
+  // Must from start 1, main texture takes.
+  private _masksTextureUnitStart: number = 1;
+
+  constructor() {
+    super(4);
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+
+    this._numMasksUniformLocation = gl.getUniformLocation(program, 'uNumMasks');
+    gl.uniform1i(this._numMasksUniformLocation, this._numMasks);
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    if (!program) {
+      return;
+    }
+    invariant(gl !== null, 'WebGL2 context is required');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    // dynamic uniforms per frame
+    const styleIndex = Math.floor(this.variant / 2) % 2;
+    gl.uniform1i(this._numMasksUniformLocation, context.masks.length);
+    gl.uniform1f(
+      gl.getUniformLocation(program, 'uCurrentFrame'),
+      context.frameIndex,
+    );
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uLineColor'),
+      this.variant % 2 === 0 ? 0 : 1,
+    );
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uArrow'),
+      styleIndex === 0 ? 1 : 0,
+    );
+
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, this._frameTexture);
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      context.width,
+      context.height,
+      0,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      context.frame,
+    );
+
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+
+    // Create and bind 2D textures for each mask
+    context.masks.forEach((mask, index) => {
+      const maskTexture = gl.createTexture();
+      const decodedMask = decode([mask.bitmap as RLEObject]);
+      const maskData = decodedMask.data as Uint8Array;
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, maskTexture);
+
+      const boundaries = normalizeBounds(
+        mask.bounds[0],
+        mask.bounds[1],
+        context.width,
+        context.height,
+      );
+
+      gl.uniform1i(
+        gl.getUniformLocation(program, `uMaskTexture${index}`),
+        index + this._masksTextureUnitStart,
+      );
+      gl.uniform4fv(gl.getUniformLocation(program, `bbox${index}`), boundaries);
+
+      // dynamic uniforms per mask
+      gl.uniform1i(
+        gl.getUniformLocation(program, `uMaskTexture${index}`),
+        this._masksTextureUnitStart + index,
+      );
+
+      gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
+      gl.texImage2D(
+        gl.TEXTURE_2D,
+        0,
+        gl.LUMINANCE,
+        context.height,
+        context.width,
+        0,
+        gl.LUMINANCE,
+        gl.UNSIGNED_BYTE,
+        maskData,
+      );
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+
+      gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+    });
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+    ctx.drawImage(this._canvas, 0, 0);
+  }
+}

demo/frontend/src/common/components/video/effects/BackgroundBlurEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/BackgroundBlur.frag?raw';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class BackgroundBlurEffect extends BaseGLEffect {
+  private _blurRadius: number = 3;
+
+  constructor() {
+    super(3);
+
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uBlurRadius'),
+      this._blurRadius,
+    );
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    if (!program) {
+      return;
+    }
+    invariant(gl !== null, 'WebGL2 context is required');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    const blurRadius = [3, 6, 12][this.variant % 3];
+    gl.uniform1i(gl.getUniformLocation(program, 'uBlurRadius'), blurRadius);
+
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, this._frameTexture);
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      context.width,
+      context.height,
+      0,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      context.frame,
+    );
+
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+    ctx.drawImage(this._canvas, 0, 0);
+  }
+}

demo/frontend/src/common/components/video/effects/BackgroundTextEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Tracklet} from '@/common/tracker/Tracker';
+import {DEMO_SHORT_NAME} from '@/demo/DemoConfig';
+import {Bound, CanvasForm, Num, Pt, Shaping} from 'pts';
+import {AbstractEffect, EffectFrameContext} from './Effect';
+
+export default class BackgroundTextEffect extends AbstractEffect {
+  constructor() {
+    super(2);
+  }
+
+  apply(
+    form: CanvasForm,
+    context: EffectFrameContext,
+    _tracklets: Tracklet[],
+  ): void {
+    form.image([0, 0], context.frame);
+
+    const words = ['SEGMENT', 'ANYTHING', 'WOW'];
+    const paragraph = `${DEMO_SHORT_NAME} is designed for efficient video processing with streaming inference to enable real-time, interactive applications.`;
+    const progress = context.frameIndex / context.totalFrames;
+
+    // Zooming heading
+    if (this.variant % 2 === 0) {
+      const step = context.totalFrames / words.length;
+      const wordIndex = Math.floor(progress * words.length);
+      const fontSize = context.width / Math.max(4, words[wordIndex].length - 1);
+      const sizeMax = fontSize * 1.2;
+
+      const t = Shaping.quadraticInOut(
+        Num.cycle((context.frameIndex - wordIndex * step) / step),
+      );
+      const currentSize = fontSize + Shaping.sineInOut(t, sizeMax - fontSize);
+      form.fillOnly('#fff').font(currentSize, 'bold');
+      const area = new Pt(
+        context.width,
+        context.height - (context.height / 4) * (1 - t),
+      )
+        .toBound()
+        .scale(1.5, [context.width / 2, 0]);
+
+      form
+        .alignText('center', 'middle')
+        .textBox(area, words[wordIndex], 'middle');
+
+      // Scrolling paragraph
+    } else {
+      const t = Shaping.quadraticInOut(Num.cycle(progress));
+      const offset = t * context.height;
+      const area = Bound.fromArray([
+        [0, -context.height + offset],
+        [context.width, context.height],
+      ]);
+      form.fillOnly('#00000066').rect(area);
+      form.fillOnly('#fff').font(context.width / 8, 'bold');
+      form
+        .fillOnly('#fff')
+        .alignText('start')
+        .paragraphBox(area, paragraph, 0.8, 'top', false);
+    }
+  }
+}

demo/frontend/src/common/components/video/effects/BaseGLEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import Logger from '@/common/logger/Logger';
+import {Tracklet} from '@/common/tracker/Tracker';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+import {AbstractEffect, EffectFrameContext, EffectInit} from './Effect';
+
+export default abstract class BaseGLEffect extends AbstractEffect {
+  protected _canvas: OffscreenCanvas | null = null;
+  protected _gl: WebGL2RenderingContext | null = null;
+  protected _program: WebGLProgram | null = null;
+
+  protected _frameTextureUnit: number = 0;
+  protected _frameTexture: WebGLTexture | null = null;
+
+  protected vertexShaderSource: string = '';
+  protected fragmentShaderSource: string = '';
+
+  protected _vertexShader: WebGLShader | null = null;
+  protected _fragmentShader: WebGLShader | null = null;
+
+  async setup(init: EffectInit): Promise<void> {
+    const {canvas, gl} = init;
+
+    if (canvas != null && gl != null) {
+      this._canvas = canvas;
+      this._gl = gl;
+    }
+
+    invariant(this._gl !== null, 'WebGL2 context is required');
+
+    const program = this._gl.createProgram();
+    this._program = program;
+
+    {
+      const vertexShader = this._gl.createShader(this._gl.VERTEX_SHADER);
+      this._vertexShader = vertexShader;
+      invariant(vertexShader !== null, 'vertexShader required');
+      this._gl.shaderSource(vertexShader, this.vertexShaderSource);
+      this._gl.compileShader(vertexShader);
+      invariant(program !== null, 'program required');
+      this._gl.attachShader(program, vertexShader);
+
+      const fragmentShader = this._gl.createShader(this._gl.FRAGMENT_SHADER);
+      this._fragmentShader = fragmentShader;
+      invariant(fragmentShader !== null, 'fragmentShader required');
+      this._gl.shaderSource(fragmentShader, this.fragmentShaderSource);
+      this._gl.compileShader(fragmentShader);
+      this._gl.attachShader(program, fragmentShader);
+
+      this._gl.linkProgram(program);
+
+      if (!this._gl.getProgramParameter(program, this._gl.LINK_STATUS)) {
+        Logger.error(this._gl.getShaderInfoLog(vertexShader));
+        Logger.error(this._gl.getShaderInfoLog(fragmentShader));
+      }
+    }
+    this._gl.useProgram(program);
+
+    this.setupBuffers(this._gl);
+    this.setupUniforms(this._gl, program, init);
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    invariant(gl !== null, 'WebGL2 context is required');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    gl.activeTexture(gl.TEXTURE0 + this._frameTextureUnit);
+    gl.bindTexture(gl.TEXTURE_2D, this._frameTexture);
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      context.frame.width,
+      context.frame.height,
+      0,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      context.frame,
+    );
+
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+    gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
+
+    // Apply shader
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+    ctx.drawImage(this._canvas, 0, 0);
+  }
+
+  async cleanup(): Promise<void> {
+    if (this._gl != null) {
+      // Dispose of WebGL resources, e.g., textures, buffers, etc.
+      if (this._frameTexture != null) {
+        this._gl.deleteTexture(this._frameTexture);
+        this._frameTexture = null;
+      }
+
+      if (
+        this._program != null &&
+        this._vertexShader != null &&
+        this._fragmentShader != null
+      ) {
+        this._gl.detachShader(this._program, this._vertexShader);
+        this._gl.deleteShader(this._vertexShader);
+        this._gl.detachShader(this._program, this._fragmentShader);
+        this._gl.deleteShader(this._fragmentShader);
+      }
+    }
+  }
+
+  protected setupBuffers(gl: WebGL2RenderingContext) {
+    const vertexBufferData = new Float32Array([
+      1.0, 1.0, -1.0, 1.0, 1.0, -1.0, -1.0, -1.0,
+    ]);
+
+    const texCoordBufferData = new Float32Array([
+      1.0, 1.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0,
+    ]);
+
+    const vertexBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, vertexBufferData, gl.STATIC_DRAW);
+    gl.vertexAttribPointer(0, 2, gl.FLOAT, false, 0, 0);
+    gl.enableVertexAttribArray(0);
+
+    const texCoordBuffer = gl.createBuffer();
+    gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
+    gl.bufferData(gl.ARRAY_BUFFER, texCoordBufferData, gl.STATIC_DRAW);
+    gl.vertexAttribPointer(1, 2, gl.FLOAT, false, 0, 0);
+    gl.enableVertexAttribArray(1);
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ) {
+    this._frameTexture = gl.createTexture();
+
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uSampler'),
+      this._frameTextureUnit,
+    );
+
+    gl.uniform2f(
+      gl.getUniformLocation(program, 'uSize'),
+      init.width,
+      init.height,
+    );
+  }
+}

demo/frontend/src/common/components/video/effects/BurstGLEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {hexToRgb} from '@/common/components/video/editor/VideoEditorUtils';
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/Burst.frag?raw';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import {normalizeBounds, preAllocateTextures} from '@/common/utils/ShaderUtils';
+import {RLEObject, decode} from '@/jscocotools/mask';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class BurstGLEffect extends BaseGLEffect {
+  private _numMasks: number = 0;
+  private _numMasksUniformLocation: WebGLUniformLocation | null = null;
+
+  // Must from start 1, main texture takes.
+  private _masksTextureUnitStart: number = 1;
+  private _maskTextures: WebGLTexture[] = [];
+
+  constructor() {
+    super(4);
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+
+    this._numMasksUniformLocation = gl.getUniformLocation(program, 'uNumMasks');
+    gl.uniform1i(this._numMasksUniformLocation, this._numMasks);
+
+    // We know the max number of textures, pre-allocate 3.
+    this._maskTextures = preAllocateTextures(gl, 3);
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    if (!program) {
+      return;
+    }
+    invariant(gl !== null, 'WebGL2 context is required');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    const styleIndex = Math.floor(this.variant / 2) % 2;
+
+    // dynamic uniforms per frame
+    gl.uniform1i(this._numMasksUniformLocation, context.masks.length);
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uLineColor'),
+      this.variant % 2 === 0 ? 1 : 0,
+    );
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uInterleave'),
+      styleIndex === 0 ? 0 : 1,
+    );
+
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, this._frameTexture);
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      context.width,
+      context.height,
+      0,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      context.frame,
+    );
+
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+
+    // Create and bind 2D textures for each mask
+    context.masks.forEach((mask, index) => {
+      const decodedMask = decode([mask.bitmap as RLEObject]);
+      const maskData = decodedMask.data as Uint8Array;
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, this._maskTextures[index]);
+
+      const boundaries = normalizeBounds(
+        mask.bounds[0],
+        mask.bounds[1],
+        context.width,
+        context.height,
+      );
+
+      // dynamic uniforms per mask
+      gl.uniform1i(
+        gl.getUniformLocation(program, `uMaskTexture${index}`),
+        this._masksTextureUnitStart + index,
+      );
+      const color = hexToRgb(context.maskColors[index]);
+      gl.uniform4f(
+        gl.getUniformLocation(program, `uMaskColor${index}`),
+        color.r,
+        color.g,
+        color.b,
+        color.a,
+      );
+      gl.uniform4fv(gl.getUniformLocation(program, `bbox${index}`), boundaries);
+
+      gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
+      gl.texImage2D(
+        gl.TEXTURE_2D,
+        0,
+        gl.LUMINANCE,
+        context.height,
+        context.width,
+        0,
+        gl.LUMINANCE,
+        gl.UNSIGNED_BYTE,
+        maskData,
+      );
+    });
+
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+    // Unbind textures
+    gl.bindTexture(gl.TEXTURE_2D, null);
+    context.masks.forEach((_, index) => {
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, null);
+    });
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+    ctx.drawImage(this._canvas, 0, 0);
+  }
+
+  async cleanup(): Promise<void> {
+    super.cleanup();
+
+    if (this._gl != null) {
+      // Delete mask textures to prevent memory leaks
+      this._maskTextures.forEach(texture => {
+        if (texture != null && this._gl != null) {
+          this._gl.deleteTexture(texture);
+        }
+      });
+      this._maskTextures = [];
+    }
+  }
+}

demo/frontend/src/common/components/video/effects/CutoutGLEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/Cutout.frag?raw';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import {preAllocateTextures} from '@/common/utils/ShaderUtils';
+import {RLEObject, decode} from '@/jscocotools/mask';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class CutoutGLEffect extends BaseGLEffect {
+  private _numMasks: number = 0;
+  private _numMasksUniformLocation: WebGLUniformLocation | null = null;
+
+  // Must from start 1, main texture takes.
+  private _masksTextureUnitStart: number = 1;
+  private _maskTextures: WebGLTexture[] = [];
+
+  constructor() {
+    super(4);
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+
+    this._numMasksUniformLocation = gl.getUniformLocation(program, 'uNumMasks');
+    gl.uniform1i(this._numMasksUniformLocation, this._numMasks);
+
+    // We know the max number of textures, pre-allocate 3.
+    this._maskTextures = preAllocateTextures(gl, 3);
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    if (!program) {
+      return;
+    }
+    invariant(gl !== null, 'WebGL2 context is required');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    // dynamic uniforms per frame
+    const contrastValue = [1.0, 1.6, 0.75, 0.0][this.variant % 4];
+    gl.uniform1f(gl.getUniformLocation(program, 'uContrast'), contrastValue);
+    gl.uniform1i(this._numMasksUniformLocation, context.masks.length);
+
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, this._frameTexture);
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      context.width,
+      context.height,
+      0,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      context.frame,
+    );
+
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+
+    // Create and bind 2D textures for each mask
+    context.masks.forEach((mask, index) => {
+      const decodedMask = decode([mask.bitmap as RLEObject]);
+      const maskData = decodedMask.data as Uint8Array;
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, this._maskTextures[index]);
+
+      // dynamic uniforms per mask
+      gl.uniform1i(
+        gl.getUniformLocation(program, `uMaskTexture${index}`),
+        this._masksTextureUnitStart + index,
+      );
+
+      gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
+      gl.texImage2D(
+        gl.TEXTURE_2D,
+        0,
+        gl.LUMINANCE,
+        context.height,
+        context.width,
+        0,
+        gl.LUMINANCE,
+        gl.UNSIGNED_BYTE,
+        maskData,
+      );
+    });
+
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+    // Unbind textures
+    gl.bindTexture(gl.TEXTURE_2D, null);
+    context.masks.forEach((_, index) => {
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, null);
+    });
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+    ctx.drawImage(this._canvas, 0, 0);
+  }
+
+  async cleanup(): Promise<void> {
+    super.cleanup();
+
+    if (this._gl != null) {
+      // Delete mask textures to prevent memory leaks
+      this._maskTextures.forEach(texture => {
+        if (texture != null && this._gl != null) {
+          this._gl.deleteTexture(texture);
+        }
+      });
+      this._maskTextures = [];
+    }
+  }
+}

demo/frontend/src/common/components/video/effects/DesaturateEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Tracklet} from '@/common/tracker/Tracker';
+import {CanvasForm} from 'pts';
+import {AbstractEffect, EffectFrameContext} from './Effect';
+
+export default class DesaturateEffect extends AbstractEffect {
+  constructor() {
+    super(3);
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    form.ctx.save();
+    form.ctx.filter = ['contrast(100%)', 'contrast(150%)', 'contrast(50%)'][
+      this.variant % 3
+    ];
+    form.image([0, 0], context.frame);
+    form.ctx.globalCompositeOperation = 'hue';
+    form.fillOnly('#fff').rect([
+      [0, 0],
+      [context.width, context.height],
+    ]);
+    form.ctx.restore();
+  }
+}

demo/frontend/src/common/components/video/effects/Effect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Effects} from '@/common/components/video/effects/Effects';
+import {Tracklet} from '@/common/tracker/Tracker';
+import {RLEObject} from '@/jscocotools/mask';
+import {CanvasForm} from 'pts';
+
+export type EffectLayers = {
+  background: keyof Effects;
+  highlight: keyof Effects;
+};
+
+export type EffectOptions = {
+  variant: number;
+};
+
+export type EffectInit = {
+  width: number;
+  height: number;
+  gl?: WebGL2RenderingContext;
+  canvas?: OffscreenCanvas;
+};
+
+export type EffectMask = {
+  bitmap: ImageBitmap | RLEObject;
+  bounds: [[number, number], [number, number]];
+};
+
+export type EffectActionPoint = {
+  objectId: number;
+  position: [number, number];
+};
+
+export type EffectFrameContext = {
+  frameIndex: number;
+  totalFrames: number;
+  fps: number;
+  width: number;
+  height: number;
+  masks: EffectMask[];
+  maskColors: string[];
+  frame: ImageBitmap;
+  timeParameter?: number;
+  actionPoint: EffectActionPoint | null;
+};
+
+export interface Effect {
+  variant: number;
+  numVariants: number;
+  nextVariant(): void;
+  setup(init: EffectInit): Promise<void>;
+  update(options: EffectOptions): Promise<void>;
+  cleanup(): Promise<void>;
+  apply(
+    form: CanvasForm,
+    context: EffectFrameContext,
+    tracklets: Tracklet[],
+  ): void;
+}
+
+export abstract class AbstractEffect implements Effect {
+  public numVariants: number;
+  public variant: number;
+
+  constructor(numVariants: number) {
+    this.numVariants = numVariants;
+    this.variant = 0;
+  }
+
+  nextVariant() {
+    // Cycle through variants
+    this.variant = (this.variant + 1) % this.numVariants;
+  }
+
+  async setup(_init: EffectInit): Promise<void> {
+    // noop
+  }
+
+  async update(options: EffectOptions): Promise<void> {
+    this.variant = options.variant;
+  }
+
+  async cleanup(): Promise<void> {
+    // noop
+  }
+
+  abstract apply(
+    form: CanvasForm,
+    context: EffectFrameContext,
+    tracklets: Tracklet[],
+  ): void;
+}

demo/frontend/src/common/components/video/effects/EffectUtils.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import invariant from 'invariant';
+import {Group} from 'pts';
+import {EffectFrameContext} from './Effect';
+
+export type MaskCanvas = {
+  maskCanvas: OffscreenCanvas;
+  bounds: number[][];
+  scaleX: number;
+  scaleY: number;
+};
+
+import {Effects} from '@/common/components/video/effects/Effects';
+import type {CarbonIconType} from '@carbon/icons-react';
+import {
+  AppleDash,
+  Asterisk,
+  Barcode,
+  CenterCircle,
+  ColorPalette,
+  ColorSwitch,
+  Development,
+  Erase,
+  FaceWink,
+  Humidity,
+  Image,
+  Overlay,
+  TextFont,
+} from '@carbon/icons-react';
+
+export type DemoEffect = {
+  title: string;
+  Icon: CarbonIconType;
+  effectName: keyof Effects;
+};
+
+export const backgroundEffects: DemoEffect[] = [
+  {title: 'Original', Icon: Image, effectName: 'Original'},
+  {title: 'Erase', Icon: Erase, effectName: 'EraseBackground'},
+  {
+    title: 'Gradient',
+    Icon: ColorPalette,
+    effectName: 'Gradient',
+  },
+  {
+    title: 'Pixelate',
+    Icon: Development,
+    effectName: 'Pixelate',
+  },
+  {title: 'Desaturate', Icon: ColorSwitch, effectName: 'Desaturate'},
+  {title: 'Text', Icon: TextFont, effectName: 'BackgroundText'},
+  {title: 'Blur', Icon: Humidity, effectName: 'BackgroundBlur'},
+  {title: 'Outline', Icon: AppleDash, effectName: 'Sobel'},
+];
+
+export const highlightEffects: DemoEffect[] = [
+  {title: 'Original', Icon: Image, effectName: 'Cutout'},
+  {title: 'Erase', Icon: Erase, effectName: 'EraseForeground'},
+  {title: 'Gradient', Icon: ColorPalette, effectName: 'VibrantMask'},
+  {title: 'Pixelate', Icon: Development, effectName: 'PixelateMask'},
+  {
+    title: 'Overlay',
+    Icon: Overlay,
+    effectName: 'Overlay',
+  },
+  {title: 'Emoji', Icon: FaceWink, effectName: 'Replace'},
+  {title: 'Burst', Icon: Asterisk, effectName: 'Burst'},
+  {title: 'Spotlight', Icon: CenterCircle, effectName: 'Scope'},
+];
+
+export const moreEffects: DemoEffect[] = [
+  {title: 'Noisy', Icon: Barcode, effectName: 'NoisyMask'},
+];
+
+// Store existing content in a temporary canvas
+// This can be used in HighlightEffect composite blending, so that the existing background effect can be put back via "destination-over"
+export function copyCanvasContent(
+  ctx: CanvasRenderingContext2D,
+  effectContext: EffectFrameContext,
+): OffscreenCanvas {
+  const {width, height} = effectContext;
+  const previousContent = ctx.getImageData(0, 0, width, height);
+  const tempCanvas = new OffscreenCanvas(width, height);
+  const tempCtx = tempCanvas.getContext('2d');
+  tempCtx?.putImageData(previousContent, 0, 0);
+  return tempCanvas;
+}
+
+export function isInvalidMask(bound: number[][] | Group) {
+  return (
+    bound[0].length < 2 ||
+    bound[1].length < 2 ||
+    bound[1][0] - bound[0][0] < 1 ||
+    bound[1][1] - bound[0][1] < 1
+  );
+}
+
+export type MaskRenderingData = {
+  canvas: OffscreenCanvas;
+  scale: number[];
+  bounds: number[][];
+};
+
+export class EffectLayer {
+  canvas: OffscreenCanvas;
+  ctx: OffscreenCanvasRenderingContext2D;
+  width: number;
+  height: number;
+
+  constructor(context: EffectFrameContext) {
+    this.canvas = new OffscreenCanvas(context.width, context.height);
+    const ctx = this.canvas.getContext('2d');
+    invariant(ctx !== null, 'context cannot be  null');
+    this.ctx = ctx;
+    this.width = context.width;
+    this.height = context.height;
+  }
+
+  image(source: CanvasImageSourceWebCodecs) {
+    this.ctx.drawImage(source, 0, 0);
+  }
+
+  filter(filterString: string) {
+    this.ctx.filter = filterString;
+  }
+
+  composite(blend: GlobalCompositeOperation) {
+    this.ctx.globalCompositeOperation = blend;
+  }
+
+  fill(color: string) {
+    this.ctx.fillStyle = color;
+    this.ctx.fillRect(0, 0, this.width, this.height);
+  }
+
+  clear() {
+    this.ctx.clearRect(0, 0, this.width, this.height);
+  }
+}

demo/frontend/src/common/components/video/effects/Effects.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BackgroundTextEffect from './BackgroundTextEffect';
+import DesaturateEffect from './DesaturateEffect';
+import {Effect} from './Effect';
+import EraseBackgroundEffect from './EraseBackgroundEffect';
+import OriginalEffect from './OriginalEffect';
+import OverlayEffect from './OverlayEffect';
+import ArrowGLEffect from './ArrowGLEffect';
+import BackgroundBlurEffect from './BackgroundBlurEffect';
+import BurstGLEffect from './BurstGLEffect';
+import CutoutGLEffect from './CutoutGLEffect';
+import EraseForegroundGLEffect from './EraseForegroundGLEffect';
+import GradientEffect from './GradientEffect';
+import NoisyMaskEffect from './NoisyMaskEffect';
+import PixelateEffect from './PixelateEffect';
+import PixelateMaskGLEffect from './PixelateMaskGLEffect';
+import ReplaceGLEffect from './ReplaceGLEffect';
+import ScopeGLEffect from './ScopeGLEffect';
+import SobelEffect from './SobelEffect';
+import VibrantMaskEffect from './VibrantMaskEffect';
+
+export type Effects = {
+  /* Backgrounds */
+  Original: Effect;
+  EraseBackground: Effect;
+  Desaturate: Effect;
+  Pixelate: Effect;
+  Sobel: Effect;
+  BackgroundText: Effect;
+  BackgroundBlur: Effect;
+  Gradient: Effect;
+
+  /* Highlights */
+  Overlay: Effect;
+  EraseForeground: Effect;
+  Cutout: Effect;
+  Scope: Effect;
+  VibrantMask: Effect;
+  Replace: Effect;
+  Burst: Effect;
+  PixelateMask: Effect;
+  Arrow: Effect;
+
+  /* More Effects */
+  NoisyMask: Effect;
+};
+
+export default {
+  /* Backgrounds */
+  Original: new OriginalEffect(),
+  EraseBackground: new EraseBackgroundEffect(),
+  Desaturate: new DesaturateEffect(),
+  Pixelate: new PixelateEffect(),
+  Sobel: new SobelEffect(),
+  BackgroundText: new BackgroundTextEffect(),
+  BackgroundBlur: new BackgroundBlurEffect(),
+  Gradient: new GradientEffect(),
+
+  /* Highlights */
+  Overlay: new OverlayEffect(),
+  EraseForeground: new EraseForegroundGLEffect(),
+  Cutout: new CutoutGLEffect(),
+  Scope: new ScopeGLEffect(),
+  VibrantMask: new VibrantMaskEffect(),
+  Replace: new ReplaceGLEffect(),
+  Burst: new BurstGLEffect(),
+  PixelateMask: new PixelateMaskGLEffect(),
+  Arrow: new ArrowGLEffect(),
+
+  /* More Effects */
+  NoisyMask: new NoisyMaskEffect(),
+} as Effects;
+
+export enum EffectIndex {
+  BACKGROUND = 0,
+  HIGHLIGHT = 1,
+}
+
+type EffectComboItem = {name: keyof Effects; variant: number};
+
+export type EffectsCombo = [EffectComboItem, EffectComboItem];
+
+export const effectPresets: EffectsCombo[] = [
+  [
+    {name: 'Original', variant: 0},
+    {name: 'Overlay', variant: 0},
+  ],
+  [
+    {name: 'Desaturate', variant: 0},
+    {name: 'Burst', variant: 2},
+  ],
+  [
+    {name: 'Desaturate', variant: 1},
+    {name: 'VibrantMask', variant: 0},
+  ],
+  [
+    {name: 'BackgroundText', variant: 1},
+    {name: 'Cutout', variant: 0},
+  ],
+  [
+    {name: 'Original', variant: 0},
+    {name: 'PixelateMask', variant: 1},
+  ],
+  [
+    {name: 'Desaturate', variant: 2},
+    {name: 'Cutout', variant: 0},
+  ],
+  [
+    {name: 'Sobel', variant: 3},
+    {name: 'Cutout', variant: 1},
+  ],
+  [
+    {name: 'Sobel', variant: 2},
+    {name: 'EraseForeground', variant: 2},
+  ],
+  [
+    {name: 'EraseBackground', variant: 0},
+    {name: 'EraseForeground', variant: 0},
+  ],
+];

demo/frontend/src/common/components/video/effects/EraseBackgroundEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Tracklet} from '@/common/tracker/Tracker';
+import {CanvasForm} from 'pts';
+import {AbstractEffect, EffectFrameContext} from './Effect';
+
+export default class EraseBackgroundEffect extends AbstractEffect {
+  constructor() {
+    super(3);
+  }
+
+  apply(
+    form: CanvasForm,
+    context: EffectFrameContext,
+    _tracklets: Tracklet[],
+  ): void {
+    const fillColor = ['#000', '#fff', '#0f0'][this.variant % 3];
+    form.fillOnly(fillColor).rect([
+      [0, 0],
+      [context.width, context.height],
+    ]);
+  }
+}

demo/frontend/src/common/components/video/effects/EraseForegroundEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Tracklet} from '@/common/tracker/Tracker';
+import {CanvasForm} from 'pts';
+import {AbstractEffect, EffectFrameContext} from './Effect';
+import {EffectLayer} from './EffectUtils';
+
+export default class EraseForegroundEffect extends AbstractEffect {
+  constructor() {
+    super(3);
+  }
+
+  apply(
+    form: CanvasForm,
+    context: EffectFrameContext,
+    _tracklets: Tracklet[],
+  ): void {
+    const effect = new EffectLayer(context);
+    const fillColor = ['#fff', '#000', '#0f0'][this.variant % 3];
+    for (const mask of context.masks) {
+      effect.image(mask.bitmap as ImageBitmap);
+      effect.composite('source-in');
+      effect.fill(fillColor);
+    }
+    form.image([0, 0], effect.canvas);
+  }
+}

demo/frontend/src/common/components/video/effects/EraseForegroundGLEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/EraseForeground.frag?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import {preAllocateTextures} from '@/common/utils/ShaderUtils';
+import {RLEObject, decode} from '@/jscocotools/mask';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class EraseForegroundGLEffect extends BaseGLEffect {
+  private _numMasks: number = 0;
+  private _numMasksUniformLocation: WebGLUniformLocation | null = null;
+
+  private _maskTextures: WebGLTexture[] = [];
+
+  constructor() {
+    super(3);
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+
+    this._numMasksUniformLocation = gl.getUniformLocation(program, 'uNumMasks');
+    gl.uniform1i(this._numMasksUniformLocation, this._numMasks);
+
+    // We know the max number of textures, pre-allocate 3.
+    this._maskTextures = preAllocateTextures(gl, 3);
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    invariant(gl !== null, 'WebGL2 context is required');
+    invariant(program !== null, 'Not WebGL program found');
+
+    const fillColor = [
+      [1, 1, 1],
+      [0, 0, 0],
+      [0, 1, 0],
+    ][this.variant % 3];
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    gl.uniform1i(this._numMasksUniformLocation, context.masks.length);
+    gl.uniform3fv(gl.getUniformLocation(program, 'uBgColor'), fillColor);
+
+    context.masks.forEach((mask, index) => {
+      const decodedMask = decode([mask.bitmap as RLEObject]);
+      const maskData = decodedMask.data as Uint8Array;
+      gl.activeTexture(gl.TEXTURE0 + index);
+      gl.bindTexture(gl.TEXTURE_2D, this._maskTextures[index]);
+
+      gl.uniform1i(
+        gl.getUniformLocation(program, `uMaskTexture${index}`),
+        index,
+      );
+
+      gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
+      gl.texImage2D(
+        gl.TEXTURE_2D,
+        0,
+        gl.LUMINANCE,
+        context.height,
+        context.width,
+        0,
+        gl.LUMINANCE,
+        gl.UNSIGNED_BYTE,
+        maskData,
+      );
+    });
+
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+    // Unbind textures
+    gl.bindTexture(gl.TEXTURE_2D, null);
+    context.masks.forEach((_, index) => {
+      gl.activeTexture(gl.TEXTURE0 + index);
+      gl.bindTexture(gl.TEXTURE_2D, null);
+    });
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+
+    if (context.masks.length) {
+      ctx.drawImage(this._canvas, 0, 0);
+    }
+  }
+
+  async cleanup(): Promise<void> {
+    super.cleanup();
+
+    if (this._gl != null) {
+      // Delete mask textures to prevent memory leaks
+      this._maskTextures.forEach(texture => {
+        if (texture != null && this._gl != null) {
+          this._gl.deleteTexture(texture);
+        }
+      });
+      this._maskTextures = [];
+    }
+  }
+}

demo/frontend/src/common/components/video/effects/GradientEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/Gradient.frag?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import {generateLUTDATA, load3DLUT} from '@/common/utils/ShaderUtils';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class GradientEffect extends BaseGLEffect {
+  private lutSize: number = 2;
+  private _lutTextures: WebGLTexture[] = [];
+
+  // Must be 1, main background texture takes 0.
+  private _extraTextureUnit: number = 1;
+
+  constructor() {
+    super(3);
+
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uColorGradeLUT'),
+      this._extraTextureUnit,
+    );
+
+    this._lutTextures = []; // clear any previous pool of textures
+
+    for (let i = 0; i < this.numVariants; i++) {
+      const _lutData = generateLUTDATA(this.lutSize);
+      const _extraTexture = load3DLUT(gl, this.lutSize, _lutData);
+      this._lutTextures.push(_extraTexture as WebGLTexture);
+    }
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    if (!program) {
+      return;
+    }
+    invariant(gl !== null, 'WebGL2 context is required');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    // Bind the LUT texture to texture unit 1
+    const lutTexture = this._lutTextures[this.variant];
+    gl.activeTexture(gl.TEXTURE1);
+    gl.bindTexture(gl.TEXTURE_3D, lutTexture);
+
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, this._frameTexture);
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      context.width,
+      context.height,
+      0,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      context.frame,
+    );
+
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+    ctx.drawImage(this._canvas, 0, 0);
+  }
+}

demo/frontend/src/common/components/video/effects/NoisyMaskEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/NoisyMask.frag?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import {RLEObject, decode} from '@/jscocotools/mask';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class NoisyMaskEffect extends BaseGLEffect {
+  private _numMasks: number = 0;
+  private _numMasksUniformLocation: WebGLUniformLocation | null = null;
+  private _currentFrameLocation: WebGLUniformLocation | null = null;
+
+  constructor() {
+    super(1);
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+
+    this._numMasksUniformLocation = gl.getUniformLocation(program, 'uNumMasks');
+    gl.uniform1i(this._numMasksUniformLocation, this._numMasks);
+
+    this._currentFrameLocation = gl.getUniformLocation(
+      program,
+      'uCurrentFrame',
+    );
+    gl.uniform1f(this._currentFrameLocation, 0);
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    if (!program) {
+      return;
+    }
+    invariant(gl !== null, 'WebGL2 context is required');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    // dynamic uniforms per frame
+    gl.uniform1f(this._currentFrameLocation, context.frameIndex);
+    gl.uniform1i(this._numMasksUniformLocation, context.masks.length);
+
+    // Create and bind 2D textures for each mask
+    context.masks.forEach((mask, index) => {
+      const maskTexture = gl.createTexture();
+      const decodedMask = decode([mask.bitmap as RLEObject]);
+      const maskData = decodedMask.data as Uint8Array;
+      gl.activeTexture(gl.TEXTURE0 + index);
+      gl.bindTexture(gl.TEXTURE_2D, maskTexture);
+
+      // dynamic uniforms per mask
+      gl.uniform1i(
+        gl.getUniformLocation(program, `uMaskTexture${index}`),
+        index,
+      );
+
+      gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
+      gl.texImage2D(
+        gl.TEXTURE_2D,
+        0,
+        gl.LUMINANCE,
+        context.height,
+        context.width,
+        0,
+        gl.LUMINANCE,
+        gl.UNSIGNED_BYTE,
+        maskData,
+      );
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+
+      gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+    });
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+    ctx.drawImage(this._canvas, 0, 0);
+  }
+}

demo/frontend/src/common/components/video/effects/OriginalEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Tracklet} from '@/common/tracker/Tracker';
+import {CanvasForm} from 'pts';
+import {AbstractEffect, EffectFrameContext} from './Effect';
+
+export default class OriginalEffect extends AbstractEffect {
+  constructor() {
+    super(3);
+  }
+
+  apply(
+    form: CanvasForm,
+    context: EffectFrameContext,
+    _tracklets: Tracklet[],
+  ): void {
+    form.ctx.save();
+    if (this.variant % 3 === 1) {
+      form.ctx.filter = 'saturate(120%) contrast(120%)';
+    } else if (this.variant % 3 === 2) {
+      form.ctx.filter = 'brightness(70%) contrast(115%)';
+    }
+
+    form.image([0, 0], context.frame);
+    form.ctx.restore();
+
+    if (this.variant % 3 === 2) {
+      form.fillOnly('#00000066').rect([
+        [0, 0],
+        [context.width, context.height],
+      ]);
+    }
+  }
+}

demo/frontend/src/common/components/video/effects/OverlayEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {hexToRgb} from '@/common/components/video/editor/VideoEditorUtils';
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/Overlay.frag?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import {
+  findIndexByTrackletId,
+  preAllocateTextures,
+} from '@/common/utils/ShaderUtils';
+import {RLEObject, decode} from '@/jscocotools/mask';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class OverlayEffect extends BaseGLEffect {
+  private _numMasks: number = 0;
+  private _numMasksUniformLocation: WebGLUniformLocation | null = null;
+
+  // Must start from 1, main texture takes 0.
+  private _masksTextureUnitStart: number = 1;
+  private _maskTextures: WebGLTexture[] = [];
+  private _clickPosition: number[] | null = null;
+  private _activeMask: number = 0;
+
+  constructor() {
+    super(8);
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+
+    this._numMasksUniformLocation = gl.getUniformLocation(program, 'uNumMasks');
+    gl.uniform1i(this._numMasksUniformLocation, this._numMasks);
+
+    // We know the max number of textures, pre-allocate 3.
+    this._maskTextures = preAllocateTextures(gl, 3);
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    invariant(gl !== null, 'WebGL2 context is required');
+    invariant(program !== null, 'Not WebGL program found');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    const opacity = [0.5, 0.75, 0.35, 0.95][this.variant % 4];
+    gl.uniform1f(
+      gl.getUniformLocation(program, 'uTime'),
+      context.timeParameter ?? 1.5, // Pass a constant value when no time parameter
+    );
+    gl.uniform1f(gl.getUniformLocation(program, 'uOpacity'), opacity);
+    gl.uniform1i(this._numMasksUniformLocation, context.masks.length);
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uBorder'),
+      this.variant % this.numVariants < 4 ? 1 : 0,
+    );
+
+    if (context.actionPoint) {
+      const clickPos = [
+        context.actionPoint.position[0] / context.width,
+        context.actionPoint.position[1] / context.height,
+      ];
+      this._clickPosition = clickPos;
+      this._activeMask = findIndexByTrackletId(
+        context.actionPoint.objectId,
+        _tracklets,
+      );
+    }
+
+    gl.uniform2fv(
+      gl.getUniformLocation(program, 'uClickPos'),
+      this._clickPosition ?? [0, 0],
+    );
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uActiveMask'),
+      this._activeMask,
+    );
+
+    // Activate original frame texture
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, this._frameTexture);
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      context.width,
+      context.height,
+      0,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      context.frame,
+    );
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+
+    context.masks.forEach((mask, index) => {
+      const decodedMask = decode([mask.bitmap as RLEObject]);
+      const maskData = decodedMask.data as Uint8Array;
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, this._maskTextures[index]);
+
+      gl.uniform1i(
+        gl.getUniformLocation(program, `uMaskTexture${index}`),
+        this._masksTextureUnitStart + index,
+      );
+
+      const color = hexToRgb(context.maskColors[index]);
+      gl.uniform4f(
+        gl.getUniformLocation(program, `uMaskColor${index}`),
+        color.r,
+        color.g,
+        color.b,
+        color.a,
+      );
+
+      // 1 byte aligment
+      gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
+      gl.texImage2D(
+        gl.TEXTURE_2D,
+        0,
+        gl.LUMINANCE,
+        context.height,
+        context.width,
+        0,
+        gl.LUMINANCE,
+        gl.UNSIGNED_BYTE,
+        maskData,
+      );
+    });
+
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+    // Unbind textures
+    gl.bindTexture(gl.TEXTURE_2D, null);
+    context.masks.forEach((_, index) => {
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, null);
+    });
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+
+    ctx.drawImage(this._canvas, 0, 0);
+    this._clickPosition = null;
+  }
+
+  async cleanup(): Promise<void> {
+    super.cleanup();
+
+    if (this._gl != null) {
+      // Delete mask textures to prevent memory leaks
+      this._maskTextures.forEach(texture => {
+        if (texture != null && this._gl != null) {
+          this._gl.deleteTexture(texture);
+        }
+      });
+      this._maskTextures = [];
+    }
+  }
+}