Initial commit

demo/frontend/src/common/components/video/effects/EffectUtils.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import invariant from 'invariant';
+import {Group} from 'pts';
+import {EffectFrameContext} from './Effect';
+
+export type MaskCanvas = {
+  maskCanvas: OffscreenCanvas;
+  bounds: number[][];
+  scaleX: number;
+  scaleY: number;
+};
+
+import {Effects} from '@/common/components/video/effects/Effects';
+import type {CarbonIconType} from '@carbon/icons-react';
+import {
+  AppleDash,
+  Asterisk,
+  Barcode,
+  CenterCircle,
+  ColorPalette,
+  ColorSwitch,
+  Development,
+  Erase,
+  FaceWink,
+  Humidity,
+  Image,
+  Overlay,
+  TextFont,
+} from '@carbon/icons-react';
+
+export type DemoEffect = {
+  title: string;
+  Icon: CarbonIconType;
+  effectName: keyof Effects;
+};
+
+export const backgroundEffects: DemoEffect[] = [
+  {title: 'Original', Icon: Image, effectName: 'Original'},
+  {title: 'Erase', Icon: Erase, effectName: 'EraseBackground'},
+  {
+    title: 'Gradient',
+    Icon: ColorPalette,
+    effectName: 'Gradient',
+  },
+  {
+    title: 'Pixelate',
+    Icon: Development,
+    effectName: 'Pixelate',
+  },
+  {title: 'Desaturate', Icon: ColorSwitch, effectName: 'Desaturate'},
+  {title: 'Text', Icon: TextFont, effectName: 'BackgroundText'},
+  {title: 'Blur', Icon: Humidity, effectName: 'BackgroundBlur'},
+  {title: 'Outline', Icon: AppleDash, effectName: 'Sobel'},
+];
+
+export const highlightEffects: DemoEffect[] = [
+  {title: 'Original', Icon: Image, effectName: 'Cutout'},
+  {title: 'Erase', Icon: Erase, effectName: 'EraseForeground'},
+  {title: 'Gradient', Icon: ColorPalette, effectName: 'VibrantMask'},
+  {title: 'Pixelate', Icon: Development, effectName: 'PixelateMask'},
+  {
+    title: 'Overlay',
+    Icon: Overlay,
+    effectName: 'Overlay',
+  },
+  {title: 'Emoji', Icon: FaceWink, effectName: 'Replace'},
+  {title: 'Burst', Icon: Asterisk, effectName: 'Burst'},
+  {title: 'Spotlight', Icon: CenterCircle, effectName: 'Scope'},
+];
+
+export const moreEffects: DemoEffect[] = [
+  {title: 'Noisy', Icon: Barcode, effectName: 'NoisyMask'},
+];
+
+// Store existing content in a temporary canvas
+// This can be used in HighlightEffect composite blending, so that the existing background effect can be put back via "destination-over"
+export function copyCanvasContent(
+  ctx: CanvasRenderingContext2D,
+  effectContext: EffectFrameContext,
+): OffscreenCanvas {
+  const {width, height} = effectContext;
+  const previousContent = ctx.getImageData(0, 0, width, height);
+  const tempCanvas = new OffscreenCanvas(width, height);
+  const tempCtx = tempCanvas.getContext('2d');
+  tempCtx?.putImageData(previousContent, 0, 0);
+  return tempCanvas;
+}
+
+export function isInvalidMask(bound: number[][] | Group) {
+  return (
+    bound[0].length < 2 ||
+    bound[1].length < 2 ||
+    bound[1][0] - bound[0][0] < 1 ||
+    bound[1][1] - bound[0][1] < 1
+  );
+}
+
+export type MaskRenderingData = {
+  canvas: OffscreenCanvas;
+  scale: number[];
+  bounds: number[][];
+};
+
+export class EffectLayer {
+  canvas: OffscreenCanvas;
+  ctx: OffscreenCanvasRenderingContext2D;
+  width: number;
+  height: number;
+
+  constructor(context: EffectFrameContext) {
+    this.canvas = new OffscreenCanvas(context.width, context.height);
+    const ctx = this.canvas.getContext('2d');
+    invariant(ctx !== null, 'context cannot be  null');
+    this.ctx = ctx;
+    this.width = context.width;
+    this.height = context.height;
+  }
+
+  image(source: CanvasImageSourceWebCodecs) {
+    this.ctx.drawImage(source, 0, 0);
+  }
+
+  filter(filterString: string) {
+    this.ctx.filter = filterString;
+  }
+
+  composite(blend: GlobalCompositeOperation) {
+    this.ctx.globalCompositeOperation = blend;
+  }
+
+  fill(color: string) {
+    this.ctx.fillStyle = color;
+    this.ctx.fillRect(0, 0, this.width, this.height);
+  }
+
+  clear() {
+    this.ctx.clearRect(0, 0, this.width, this.height);
+  }
+}

demo/frontend/src/common/components/video/effects/Effects.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BackgroundTextEffect from './BackgroundTextEffect';
+import DesaturateEffect from './DesaturateEffect';
+import {Effect} from './Effect';
+import EraseBackgroundEffect from './EraseBackgroundEffect';
+import OriginalEffect from './OriginalEffect';
+import OverlayEffect from './OverlayEffect';
+import ArrowGLEffect from './ArrowGLEffect';
+import BackgroundBlurEffect from './BackgroundBlurEffect';
+import BurstGLEffect from './BurstGLEffect';
+import CutoutGLEffect from './CutoutGLEffect';
+import EraseForegroundGLEffect from './EraseForegroundGLEffect';
+import GradientEffect from './GradientEffect';
+import NoisyMaskEffect from './NoisyMaskEffect';
+import PixelateEffect from './PixelateEffect';
+import PixelateMaskGLEffect from './PixelateMaskGLEffect';
+import ReplaceGLEffect from './ReplaceGLEffect';
+import ScopeGLEffect from './ScopeGLEffect';
+import SobelEffect from './SobelEffect';
+import VibrantMaskEffect from './VibrantMaskEffect';
+
+export type Effects = {
+  /* Backgrounds */
+  Original: Effect;
+  EraseBackground: Effect;
+  Desaturate: Effect;
+  Pixelate: Effect;
+  Sobel: Effect;
+  BackgroundText: Effect;
+  BackgroundBlur: Effect;
+  Gradient: Effect;
+
+  /* Highlights */
+  Overlay: Effect;
+  EraseForeground: Effect;
+  Cutout: Effect;
+  Scope: Effect;
+  VibrantMask: Effect;
+  Replace: Effect;
+  Burst: Effect;
+  PixelateMask: Effect;
+  Arrow: Effect;
+
+  /* More Effects */
+  NoisyMask: Effect;
+};
+
+export default {
+  /* Backgrounds */
+  Original: new OriginalEffect(),
+  EraseBackground: new EraseBackgroundEffect(),
+  Desaturate: new DesaturateEffect(),
+  Pixelate: new PixelateEffect(),
+  Sobel: new SobelEffect(),
+  BackgroundText: new BackgroundTextEffect(),
+  BackgroundBlur: new BackgroundBlurEffect(),
+  Gradient: new GradientEffect(),
+
+  /* Highlights */
+  Overlay: new OverlayEffect(),
+  EraseForeground: new EraseForegroundGLEffect(),
+  Cutout: new CutoutGLEffect(),
+  Scope: new ScopeGLEffect(),
+  VibrantMask: new VibrantMaskEffect(),
+  Replace: new ReplaceGLEffect(),
+  Burst: new BurstGLEffect(),
+  PixelateMask: new PixelateMaskGLEffect(),
+  Arrow: new ArrowGLEffect(),
+
+  /* More Effects */
+  NoisyMask: new NoisyMaskEffect(),
+} as Effects;
+
+export enum EffectIndex {
+  BACKGROUND = 0,
+  HIGHLIGHT = 1,
+}
+
+type EffectComboItem = {name: keyof Effects; variant: number};
+
+export type EffectsCombo = [EffectComboItem, EffectComboItem];
+
+export const effectPresets: EffectsCombo[] = [
+  [
+    {name: 'Original', variant: 0},
+    {name: 'Overlay', variant: 0},
+  ],
+  [
+    {name: 'Desaturate', variant: 0},
+    {name: 'Burst', variant: 2},
+  ],
+  [
+    {name: 'Desaturate', variant: 1},
+    {name: 'VibrantMask', variant: 0},
+  ],
+  [
+    {name: 'BackgroundText', variant: 1},
+    {name: 'Cutout', variant: 0},
+  ],
+  [
+    {name: 'Original', variant: 0},
+    {name: 'PixelateMask', variant: 1},
+  ],
+  [
+    {name: 'Desaturate', variant: 2},
+    {name: 'Cutout', variant: 0},
+  ],
+  [
+    {name: 'Sobel', variant: 3},
+    {name: 'Cutout', variant: 1},
+  ],
+  [
+    {name: 'Sobel', variant: 2},
+    {name: 'EraseForeground', variant: 2},
+  ],
+  [
+    {name: 'EraseBackground', variant: 0},
+    {name: 'EraseForeground', variant: 0},
+  ],
+];

demo/frontend/src/common/components/video/effects/EraseBackgroundEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Tracklet} from '@/common/tracker/Tracker';
+import {CanvasForm} from 'pts';
+import {AbstractEffect, EffectFrameContext} from './Effect';
+
+export default class EraseBackgroundEffect extends AbstractEffect {
+  constructor() {
+    super(3);
+  }
+
+  apply(
+    form: CanvasForm,
+    context: EffectFrameContext,
+    _tracklets: Tracklet[],
+  ): void {
+    const fillColor = ['#000', '#fff', '#0f0'][this.variant % 3];
+    form.fillOnly(fillColor).rect([
+      [0, 0],
+      [context.width, context.height],
+    ]);
+  }
+}

demo/frontend/src/common/components/video/effects/EraseForegroundEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Tracklet} from '@/common/tracker/Tracker';
+import {CanvasForm} from 'pts';
+import {AbstractEffect, EffectFrameContext} from './Effect';
+import {EffectLayer} from './EffectUtils';
+
+export default class EraseForegroundEffect extends AbstractEffect {
+  constructor() {
+    super(3);
+  }
+
+  apply(
+    form: CanvasForm,
+    context: EffectFrameContext,
+    _tracklets: Tracklet[],
+  ): void {
+    const effect = new EffectLayer(context);
+    const fillColor = ['#fff', '#000', '#0f0'][this.variant % 3];
+    for (const mask of context.masks) {
+      effect.image(mask.bitmap as ImageBitmap);
+      effect.composite('source-in');
+      effect.fill(fillColor);
+    }
+    form.image([0, 0], effect.canvas);
+  }
+}

demo/frontend/src/common/components/video/effects/EraseForegroundGLEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/EraseForeground.frag?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import {preAllocateTextures} from '@/common/utils/ShaderUtils';
+import {RLEObject, decode} from '@/jscocotools/mask';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class EraseForegroundGLEffect extends BaseGLEffect {
+  private _numMasks: number = 0;
+  private _numMasksUniformLocation: WebGLUniformLocation | null = null;
+
+  private _maskTextures: WebGLTexture[] = [];
+
+  constructor() {
+    super(3);
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+
+    this._numMasksUniformLocation = gl.getUniformLocation(program, 'uNumMasks');
+    gl.uniform1i(this._numMasksUniformLocation, this._numMasks);
+
+    // We know the max number of textures, pre-allocate 3.
+    this._maskTextures = preAllocateTextures(gl, 3);
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    invariant(gl !== null, 'WebGL2 context is required');
+    invariant(program !== null, 'Not WebGL program found');
+
+    const fillColor = [
+      [1, 1, 1],
+      [0, 0, 0],
+      [0, 1, 0],
+    ][this.variant % 3];
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    gl.uniform1i(this._numMasksUniformLocation, context.masks.length);
+    gl.uniform3fv(gl.getUniformLocation(program, 'uBgColor'), fillColor);
+
+    context.masks.forEach((mask, index) => {
+      const decodedMask = decode([mask.bitmap as RLEObject]);
+      const maskData = decodedMask.data as Uint8Array;
+      gl.activeTexture(gl.TEXTURE0 + index);
+      gl.bindTexture(gl.TEXTURE_2D, this._maskTextures[index]);
+
+      gl.uniform1i(
+        gl.getUniformLocation(program, `uMaskTexture${index}`),
+        index,
+      );
+
+      gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
+      gl.texImage2D(
+        gl.TEXTURE_2D,
+        0,
+        gl.LUMINANCE,
+        context.height,
+        context.width,
+        0,
+        gl.LUMINANCE,
+        gl.UNSIGNED_BYTE,
+        maskData,
+      );
+    });
+
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+    // Unbind textures
+    gl.bindTexture(gl.TEXTURE_2D, null);
+    context.masks.forEach((_, index) => {
+      gl.activeTexture(gl.TEXTURE0 + index);
+      gl.bindTexture(gl.TEXTURE_2D, null);
+    });
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+
+    if (context.masks.length) {
+      ctx.drawImage(this._canvas, 0, 0);
+    }
+  }
+
+  async cleanup(): Promise<void> {
+    super.cleanup();
+
+    if (this._gl != null) {
+      // Delete mask textures to prevent memory leaks
+      this._maskTextures.forEach(texture => {
+        if (texture != null && this._gl != null) {
+          this._gl.deleteTexture(texture);
+        }
+      });
+      this._maskTextures = [];
+    }
+  }
+}

demo/frontend/src/common/components/video/effects/GradientEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/Gradient.frag?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import {generateLUTDATA, load3DLUT} from '@/common/utils/ShaderUtils';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class GradientEffect extends BaseGLEffect {
+  private lutSize: number = 2;
+  private _lutTextures: WebGLTexture[] = [];
+
+  // Must be 1, main background texture takes 0.
+  private _extraTextureUnit: number = 1;
+
+  constructor() {
+    super(3);
+
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uColorGradeLUT'),
+      this._extraTextureUnit,
+    );
+
+    this._lutTextures = []; // clear any previous pool of textures
+
+    for (let i = 0; i < this.numVariants; i++) {
+      const _lutData = generateLUTDATA(this.lutSize);
+      const _extraTexture = load3DLUT(gl, this.lutSize, _lutData);
+      this._lutTextures.push(_extraTexture as WebGLTexture);
+    }
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    if (!program) {
+      return;
+    }
+    invariant(gl !== null, 'WebGL2 context is required');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    // Bind the LUT texture to texture unit 1
+    const lutTexture = this._lutTextures[this.variant];
+    gl.activeTexture(gl.TEXTURE1);
+    gl.bindTexture(gl.TEXTURE_3D, lutTexture);
+
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, this._frameTexture);
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      context.width,
+      context.height,
+      0,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      context.frame,
+    );
+
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+    ctx.drawImage(this._canvas, 0, 0);
+  }
+}

demo/frontend/src/common/components/video/effects/NoisyMaskEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/NoisyMask.frag?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import {RLEObject, decode} from '@/jscocotools/mask';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class NoisyMaskEffect extends BaseGLEffect {
+  private _numMasks: number = 0;
+  private _numMasksUniformLocation: WebGLUniformLocation | null = null;
+  private _currentFrameLocation: WebGLUniformLocation | null = null;
+
+  constructor() {
+    super(1);
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+
+    this._numMasksUniformLocation = gl.getUniformLocation(program, 'uNumMasks');
+    gl.uniform1i(this._numMasksUniformLocation, this._numMasks);
+
+    this._currentFrameLocation = gl.getUniformLocation(
+      program,
+      'uCurrentFrame',
+    );
+    gl.uniform1f(this._currentFrameLocation, 0);
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    if (!program) {
+      return;
+    }
+    invariant(gl !== null, 'WebGL2 context is required');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    // dynamic uniforms per frame
+    gl.uniform1f(this._currentFrameLocation, context.frameIndex);
+    gl.uniform1i(this._numMasksUniformLocation, context.masks.length);
+
+    // Create and bind 2D textures for each mask
+    context.masks.forEach((mask, index) => {
+      const maskTexture = gl.createTexture();
+      const decodedMask = decode([mask.bitmap as RLEObject]);
+      const maskData = decodedMask.data as Uint8Array;
+      gl.activeTexture(gl.TEXTURE0 + index);
+      gl.bindTexture(gl.TEXTURE_2D, maskTexture);
+
+      // dynamic uniforms per mask
+      gl.uniform1i(
+        gl.getUniformLocation(program, `uMaskTexture${index}`),
+        index,
+      );
+
+      gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
+      gl.texImage2D(
+        gl.TEXTURE_2D,
+        0,
+        gl.LUMINANCE,
+        context.height,
+        context.width,
+        0,
+        gl.LUMINANCE,
+        gl.UNSIGNED_BYTE,
+        maskData,
+      );
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+
+      gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+    });
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+    ctx.drawImage(this._canvas, 0, 0);
+  }
+}

demo/frontend/src/common/components/video/effects/OriginalEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Tracklet} from '@/common/tracker/Tracker';
+import {CanvasForm} from 'pts';
+import {AbstractEffect, EffectFrameContext} from './Effect';
+
+export default class OriginalEffect extends AbstractEffect {
+  constructor() {
+    super(3);
+  }
+
+  apply(
+    form: CanvasForm,
+    context: EffectFrameContext,
+    _tracklets: Tracklet[],
+  ): void {
+    form.ctx.save();
+    if (this.variant % 3 === 1) {
+      form.ctx.filter = 'saturate(120%) contrast(120%)';
+    } else if (this.variant % 3 === 2) {
+      form.ctx.filter = 'brightness(70%) contrast(115%)';
+    }
+
+    form.image([0, 0], context.frame);
+    form.ctx.restore();
+
+    if (this.variant % 3 === 2) {
+      form.fillOnly('#00000066').rect([
+        [0, 0],
+        [context.width, context.height],
+      ]);
+    }
+  }
+}

demo/frontend/src/common/components/video/effects/OverlayEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {hexToRgb} from '@/common/components/video/editor/VideoEditorUtils';
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/Overlay.frag?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import {
+  findIndexByTrackletId,
+  preAllocateTextures,
+} from '@/common/utils/ShaderUtils';
+import {RLEObject, decode} from '@/jscocotools/mask';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class OverlayEffect extends BaseGLEffect {
+  private _numMasks: number = 0;
+  private _numMasksUniformLocation: WebGLUniformLocation | null = null;
+
+  // Must start from 1, main texture takes 0.
+  private _masksTextureUnitStart: number = 1;
+  private _maskTextures: WebGLTexture[] = [];
+  private _clickPosition: number[] | null = null;
+  private _activeMask: number = 0;
+
+  constructor() {
+    super(8);
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+
+    this._numMasksUniformLocation = gl.getUniformLocation(program, 'uNumMasks');
+    gl.uniform1i(this._numMasksUniformLocation, this._numMasks);
+
+    // We know the max number of textures, pre-allocate 3.
+    this._maskTextures = preAllocateTextures(gl, 3);
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    invariant(gl !== null, 'WebGL2 context is required');
+    invariant(program !== null, 'Not WebGL program found');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    const opacity = [0.5, 0.75, 0.35, 0.95][this.variant % 4];
+    gl.uniform1f(
+      gl.getUniformLocation(program, 'uTime'),
+      context.timeParameter ?? 1.5, // Pass a constant value when no time parameter
+    );
+    gl.uniform1f(gl.getUniformLocation(program, 'uOpacity'), opacity);
+    gl.uniform1i(this._numMasksUniformLocation, context.masks.length);
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uBorder'),
+      this.variant % this.numVariants < 4 ? 1 : 0,
+    );
+
+    if (context.actionPoint) {
+      const clickPos = [
+        context.actionPoint.position[0] / context.width,
+        context.actionPoint.position[1] / context.height,
+      ];
+      this._clickPosition = clickPos;
+      this._activeMask = findIndexByTrackletId(
+        context.actionPoint.objectId,
+        _tracklets,
+      );
+    }
+
+    gl.uniform2fv(
+      gl.getUniformLocation(program, 'uClickPos'),
+      this._clickPosition ?? [0, 0],
+    );
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uActiveMask'),
+      this._activeMask,
+    );
+
+    // Activate original frame texture
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, this._frameTexture);
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      context.width,
+      context.height,
+      0,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      context.frame,
+    );
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+
+    context.masks.forEach((mask, index) => {
+      const decodedMask = decode([mask.bitmap as RLEObject]);
+      const maskData = decodedMask.data as Uint8Array;
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, this._maskTextures[index]);
+
+      gl.uniform1i(
+        gl.getUniformLocation(program, `uMaskTexture${index}`),
+        this._masksTextureUnitStart + index,
+      );
+
+      const color = hexToRgb(context.maskColors[index]);
+      gl.uniform4f(
+        gl.getUniformLocation(program, `uMaskColor${index}`),
+        color.r,
+        color.g,
+        color.b,
+        color.a,
+      );
+
+      // 1 byte aligment
+      gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
+      gl.texImage2D(
+        gl.TEXTURE_2D,
+        0,
+        gl.LUMINANCE,
+        context.height,
+        context.width,
+        0,
+        gl.LUMINANCE,
+        gl.UNSIGNED_BYTE,
+        maskData,
+      );
+    });
+
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+    // Unbind textures
+    gl.bindTexture(gl.TEXTURE_2D, null);
+    context.masks.forEach((_, index) => {
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, null);
+    });
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+
+    ctx.drawImage(this._canvas, 0, 0);
+    this._clickPosition = null;
+  }
+
+  async cleanup(): Promise<void> {
+    super.cleanup();
+
+    if (this._gl != null) {
+      // Delete mask textures to prevent memory leaks
+      this._maskTextures.forEach(texture => {
+        if (texture != null && this._gl != null) {
+          this._gl.deleteTexture(texture);
+        }
+      });
+      this._maskTextures = [];
+    }
+  }
+}

demo/frontend/src/common/components/video/effects/PixelateEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/Pixelate.frag?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class PixelateEffect extends BaseGLEffect {
+  private _blockSize: number = 10.0;
+
+  constructor() {
+    super(3);
+
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+
+    gl.uniform1f(gl.getUniformLocation(program, 'uBlockSize'), this._blockSize);
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    if (!program) {
+      return;
+    }
+    invariant(gl !== null, 'WebGL2 context is required');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    const blockSize = [10, 20, 30][this.variant];
+
+    // dynamic uniforms per frame
+    gl.uniform1f(gl.getUniformLocation(program, 'uBlockSize'), blockSize);
+
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, this._frameTexture);
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      context.width,
+      context.height,
+      0,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      context.frame,
+    );
+
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+    gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
+
+    // Apply shader
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+    ctx.drawImage(this._canvas, 0, 0);
+  }
+}

demo/frontend/src/common/components/video/effects/PixelateMaskGLEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/PixelateMask.frag?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import {preAllocateTextures} from '@/common/utils/ShaderUtils';
+import {RLEObject, decode} from '@/jscocotools/mask';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class PixelateMaskGLEffect extends BaseGLEffect {
+  private _numMasks: number = 0;
+  private _numMasksUniformLocation: WebGLUniformLocation | null = null;
+
+  // Must from start 1, main texture takes.
+  private _masksTextureUnitStart: number = 1;
+  private _maskTextures: WebGLTexture[] = [];
+
+  constructor() {
+    super(3);
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+
+    this._numMasksUniformLocation = gl.getUniformLocation(program, 'uNumMasks');
+    gl.uniform1i(this._numMasksUniformLocation, this._numMasks);
+
+    // We know the max number of textures, pre-allocate 3.
+    this._maskTextures = preAllocateTextures(gl, 3);
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    if (!program) {
+      return;
+    }
+    invariant(gl !== null, 'WebGL2 context is required');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    const blockSize = [10, 20, 30][this.variant];
+
+    // dynamic uniforms per frame
+    gl.uniform1i(this._numMasksUniformLocation, context.masks.length);
+    gl.uniform1f(gl.getUniformLocation(program, 'uBlockSize'), blockSize);
+
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, this._frameTexture);
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      context.width,
+      context.height,
+      0,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      context.frame,
+    );
+
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+
+    // Create and bind 2D textures for each mask
+    context.masks.forEach((mask, index) => {
+      const decodedMask = decode([mask.bitmap as RLEObject]);
+      const maskData = decodedMask.data as Uint8Array;
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, this._maskTextures[index]);
+      gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
+      gl.texImage2D(
+        gl.TEXTURE_2D,
+        0,
+        gl.LUMINANCE,
+        context.height,
+        context.width,
+        0,
+        gl.LUMINANCE,
+        gl.UNSIGNED_BYTE,
+        maskData,
+      );
+
+      // dynamic uniforms per mask
+      gl.uniform1i(
+        gl.getUniformLocation(program, `uMaskTexture${index}`),
+        this._masksTextureUnitStart + index,
+      );
+    });
+
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+    // Unbind textures
+    gl.bindTexture(gl.TEXTURE_2D, null);
+    context.masks.forEach((_, index) => {
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, null);
+    });
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+    ctx.drawImage(this._canvas, 0, 0);
+  }
+
+  async cleanup(): Promise<void> {
+    super.cleanup();
+
+    if (this._gl != null) {
+      // Delete mask textures to prevent memory leaks
+      this._maskTextures.forEach(texture => {
+        if (texture != null && this._gl != null) {
+          this._gl.deleteTexture(texture);
+        }
+      });
+      this._maskTextures = [];
+    }
+  }
+}

demo/frontend/src/common/components/video/effects/ReplaceGLEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import angeryIcon from '@/assets/icons/angery.png';
+import heartIcon from '@/assets/icons/heart.png';
+import whistleIcon from '@/assets/icons/whistle.png';
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/Replace.frag?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import {normalizeBounds, preAllocateTextures} from '@/common/utils/ShaderUtils';
+import {RLEObject, decode} from '@/jscocotools/mask';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class ReplaceGLEffect extends BaseGLEffect {
+  private _numMasks: number = 0;
+  private _numMasksUniformLocation: WebGLUniformLocation | null = null;
+  private _bitmap: ImageBitmap[] = [];
+  private _extraTextureUnit: number = 1;
+  private _extraTexture: WebGLTexture | null = null;
+  private _fillBg: number = 0;
+  private _fillBgLocation: WebGLUniformLocation | null = null;
+  private _masksTextureUnitStart: number = 2;
+  private _maskTextures: WebGLTexture[] = [];
+
+  constructor() {
+    super(6);
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected async setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ) {
+    super.setupUniforms(gl, program, init);
+    this._extraTexture = gl.createTexture();
+
+    this._numMasksUniformLocation = gl.getUniformLocation(program, 'uNumMasks');
+    gl.uniform1i(this._numMasksUniformLocation, this._numMasks);
+
+    this._fillBgLocation = gl.getUniformLocation(program, 'uFill');
+    gl.uniform1i(this._fillBgLocation, this._fillBg);
+
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uEmojiTexture'),
+      this._extraTextureUnit,
+    );
+
+    // We know the max number of textures, pre-allocate 3.
+    this._maskTextures = preAllocateTextures(gl, 3);
+
+    this._bitmap = []; // clear any previous pool of texture
+
+    let response = await fetch(angeryIcon);
+    let blob = await response.blob();
+    const angery = await createImageBitmap(blob);
+
+    response = await fetch(heartIcon);
+    blob = await response.blob();
+    const heart = await createImageBitmap(blob);
+
+    response = await fetch(whistleIcon);
+    blob = await response.blob();
+    const whistle = await createImageBitmap(blob);
+
+    this._bitmap = [angery, heart, whistle];
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    invariant(gl !== null, 'WebGL2 context is required');
+    invariant(program !== null, 'Not WebGL program found');
+
+    const iconIndex = Math.floor(this.variant / 2) % this._bitmap.length;
+
+    if (this._bitmap === null) {
+      return;
+    }
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    // dynamic uniforms per frame
+    gl.uniform1i(this._numMasksUniformLocation, context.masks.length);
+    gl.uniform1i(this._fillBgLocation, this.variant % 2 === 0 ? 0 : 1);
+
+    // Bind the extra texture/emoji to texture unit 1
+    if (this._bitmap.length) {
+      gl.activeTexture(gl.TEXTURE0 + this._extraTextureUnit);
+      gl.bindTexture(gl.TEXTURE_2D, this._extraTexture);
+
+      gl.texImage2D(
+        gl.TEXTURE_2D,
+        0,
+        gl.RGBA,
+        this._bitmap[iconIndex].width,
+        this._bitmap[iconIndex].height,
+        0,
+        gl.RGBA,
+        gl.UNSIGNED_BYTE,
+        this._bitmap[iconIndex],
+      );
+
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+    }
+
+    context.masks.forEach((mask, index) => {
+      const decodedMask = decode([mask.bitmap as RLEObject]);
+      const maskData = decodedMask.data as Uint8Array;
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, this._maskTextures[index]);
+
+      const boundaries = normalizeBounds(
+        mask.bounds[0],
+        mask.bounds[1],
+        context.width,
+        context.height,
+      );
+
+      gl.uniform1i(
+        gl.getUniformLocation(program, `uMaskTexture${index}`),
+        index + this._masksTextureUnitStart,
+      );
+      gl.uniform4fv(gl.getUniformLocation(program, `bbox${index}`), boundaries);
+
+      gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
+      gl.texImage2D(
+        gl.TEXTURE_2D,
+        0,
+        gl.LUMINANCE,
+        context.height,
+        context.width,
+        0,
+        gl.LUMINANCE,
+        gl.UNSIGNED_BYTE,
+        maskData,
+      );
+    });
+
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+    // Unbind textures
+    gl.bindTexture(gl.TEXTURE_2D, null);
+    context.masks.forEach((_, index) => {
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, null);
+    });
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+    ctx.drawImage(this._canvas, 0, 0);
+  }
+
+  async cleanup(): Promise<void> {
+    super.cleanup();
+
+    if (this._gl != null) {
+      // Delete mask textures to prevent memory leaks
+      this._maskTextures.forEach(texture => {
+        if (texture != null && this._gl != null) {
+          this._gl.deleteTexture(texture);
+        }
+      });
+      this._maskTextures = [];
+    }
+  }
+}

demo/frontend/src/common/components/video/effects/ScopeGLEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {hexToRgb} from '@/common/components/video/editor/VideoEditorUtils';
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/Scope.frag?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import {normalizeBounds, preAllocateTextures} from '@/common/utils/ShaderUtils';
+import {RLEObject, decode} from '@/jscocotools/mask';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class ScopeGLEffect extends BaseGLEffect {
+  private _numMasks: number = 0;
+  private _numMasksUniformLocation: WebGLUniformLocation | null = null;
+
+  // Must from start 2, main texture takes 0 and 1.
+  private _masksTextureUnitStart: number = 2;
+  private _maskTextures: WebGLTexture[] = [];
+
+  constructor() {
+    super(6);
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+
+    this._numMasksUniformLocation = gl.getUniformLocation(program, 'uNumMasks');
+    gl.uniform1i(this._numMasksUniformLocation, this._numMasks);
+
+    // We know the max number of textures, pre-allocate 3.
+    this._maskTextures = preAllocateTextures(gl, 3);
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    if (!program) {
+      return;
+    }
+    invariant(gl !== null, 'WebGL2 context is required');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    // dynamic uniforms per frame
+    gl.uniform1i(this._numMasksUniformLocation, context.masks.length);
+
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, this._frameTexture);
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      context.width,
+      context.height,
+      0,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      context.frame,
+    );
+
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+
+    // Create and bind 2D textures for each mask
+    context.masks.forEach((mask, index) => {
+      const decodedMask = decode([mask.bitmap as RLEObject]);
+      const maskData = decodedMask.data as Uint8Array;
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, this._maskTextures[index]);
+
+      const boundaries = normalizeBounds(
+        mask.bounds[0],
+        mask.bounds[1],
+        context.width,
+        context.height,
+      );
+
+      const styleIndex = Math.floor(this.variant / 2) % 2;
+
+      // dynamic uniforms per mask
+      gl.uniform1i(
+        gl.getUniformLocation(program, `uMaskTexture${index}`),
+        this._masksTextureUnitStart + index,
+      );
+      const color = hexToRgb(context.maskColors[index]);
+      gl.uniform4f(
+        gl.getUniformLocation(program, `uMaskColor${index}`),
+        color.r,
+        color.g,
+        color.b,
+        color.a,
+      );
+      gl.uniform4fv(gl.getUniformLocation(program, `bbox${index}`), boundaries);
+      gl.uniform1i(
+        gl.getUniformLocation(program, 'uFillColor'),
+        this.variant % 2 === 0 ? 0 : 1,
+      );
+      gl.uniform1i(
+        gl.getUniformLocation(program, 'uLight'),
+        styleIndex === 0 ? 0 : 1,
+      );
+      gl.uniform1i(
+        gl.getUniformLocation(program, 'uTransparency'),
+        Math.floor(this.variant / 2) % 3 === 2 ? 1 : 0,
+      );
+
+      gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
+      gl.texImage2D(
+        gl.TEXTURE_2D,
+        0,
+        gl.LUMINANCE,
+        context.height,
+        context.width,
+        0,
+        gl.LUMINANCE,
+        gl.UNSIGNED_BYTE,
+        maskData,
+      );
+    });
+
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+    // Unbind textures
+    gl.bindTexture(gl.TEXTURE_2D, null);
+    context.masks.forEach((_, index) => {
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, null);
+    });
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+    ctx.drawImage(this._canvas, 0, 0);
+  }
+
+  async cleanup(): Promise<void> {
+    super.cleanup();
+
+    if (this._gl != null) {
+      // Delete mask textures to prevent memory leaks
+      this._maskTextures.forEach(texture => {
+        if (texture != null && this._gl != null) {
+          this._gl.deleteTexture(texture);
+        }
+      });
+      this._maskTextures = [];
+    }
+  }
+}

demo/frontend/src/common/components/video/effects/SobelEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/Sobel.frag?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class SobelEffect extends BaseGLEffect {
+  constructor() {
+    super(4);
+
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    if (!program) {
+      return;
+    }
+    invariant(gl !== null, 'WebGL2 context is required');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    const pairIndex = Math.floor(this.variant / 2) % 2;
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uSwapColor'),
+      this.variant % 2 === 0 ? 1 : 0,
+    );
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uMonocolor'),
+      pairIndex === 0 ? 0 : 1,
+    );
+
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, this._frameTexture);
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      context.width,
+      context.height,
+      0,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      context.frame,
+    );
+
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+    ctx.drawImage(this._canvas, 0, 0);
+  }
+}

demo/frontend/src/common/components/video/effects/VibrantMaskEffect.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BaseGLEffect from '@/common/components/video/effects/BaseGLEffect';
+import {
+  EffectFrameContext,
+  EffectInit,
+} from '@/common/components/video/effects/Effect';
+import vertexShaderSource from '@/common/components/video/effects/shaders/DefaultVert.vert?raw';
+import fragmentShaderSource from '@/common/components/video/effects/shaders/VibrantMask.frag?raw';
+import {Tracklet} from '@/common/tracker/Tracker';
+import {
+  generateLUTDATA,
+  load3DLUT,
+  preAllocateTextures,
+} from '@/common/utils/ShaderUtils';
+import {RLEObject, decode} from '@/jscocotools/mask';
+import invariant from 'invariant';
+import {CanvasForm} from 'pts';
+
+export default class VibrantMaskEffect extends BaseGLEffect {
+  private lutSize: number = 4;
+
+  private _numMasks: number = 0;
+  private _numMasksUniformLocation: WebGLUniformLocation | null = null;
+  private _currentFrameLocation: WebGLUniformLocation | null = null;
+  private _lutTextures: WebGLTexture[] = [];
+  private _maskTextures: WebGLTexture[] = [];
+
+  // Must be 1, main background texture takes 0.
+  private _extraTextureUnit: number = 1;
+
+  // Must from start 2, main texture takes 0 and 1.
+  private _masksTextureUnitStart: number = 2;
+
+  constructor() {
+    super(3);
+    this.vertexShaderSource = vertexShaderSource;
+    this.fragmentShaderSource = fragmentShaderSource;
+  }
+
+  protected setupUniforms(
+    gl: WebGL2RenderingContext,
+    program: WebGLProgram,
+    init: EffectInit,
+  ): void {
+    super.setupUniforms(gl, program, init);
+
+    gl.uniform1i(
+      gl.getUniformLocation(program, 'uColorGradeLUT'),
+      this._extraTextureUnit,
+    );
+
+    this._numMasksUniformLocation = gl.getUniformLocation(program, 'uNumMasks');
+    gl.uniform1i(this._numMasksUniformLocation, this._numMasks);
+
+    this._currentFrameLocation = gl.getUniformLocation(
+      program,
+      'uCurrentFrame',
+    );
+    gl.uniform1f(this._currentFrameLocation, 0);
+
+    // We know the max number of textures, pre-allocate 3.
+    this._maskTextures = preAllocateTextures(gl, 3);
+
+    this._lutTextures = []; // clear any previous pool of textures
+
+    for (let i = 0; i < this.numVariants; i++) {
+      const _lutData = generateLUTDATA(this.lutSize);
+      const _extraTexture = load3DLUT(gl, this.lutSize, _lutData);
+      this._lutTextures.push(_extraTexture as WebGLTexture);
+    }
+  }
+
+  apply(form: CanvasForm, context: EffectFrameContext, _tracklets: Tracklet[]) {
+    const gl = this._gl;
+    const program = this._program;
+
+    if (!program) {
+      return;
+    }
+    invariant(gl !== null, 'WebGL2 context is required');
+
+    gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    // dynamic uniforms per frame
+    gl.uniform1f(this._currentFrameLocation, context.frameIndex);
+    gl.uniform1i(this._numMasksUniformLocation, context.masks.length);
+
+    // Bind the LUT texture to texture unit 1
+    const lutTexture = this._lutTextures[this.variant];
+    gl.activeTexture(gl.TEXTURE1);
+    gl.bindTexture(gl.TEXTURE_3D, lutTexture);
+
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, this._frameTexture);
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      context.width,
+      context.height,
+      0,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      context.frame,
+    );
+
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+
+    // Create and bind 2D textures for each mask
+    context.masks.forEach((mask, index) => {
+      const decodedMask = decode([mask.bitmap as RLEObject]);
+      const maskData = decodedMask.data as Uint8Array;
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, this._maskTextures[index]);
+
+      // dynamic uniforms per mask
+      gl.uniform1i(
+        gl.getUniformLocation(program, `uMaskTexture${index}`),
+        this._masksTextureUnitStart + index,
+      );
+
+      gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1);
+      gl.texImage2D(
+        gl.TEXTURE_2D,
+        0,
+        gl.LUMINANCE,
+        context.height,
+        context.width,
+        0,
+        gl.LUMINANCE,
+        gl.UNSIGNED_BYTE,
+        maskData,
+      );
+    });
+
+    gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+
+    // Unbind textures
+    gl.bindTexture(gl.TEXTURE_2D, null);
+    context.masks.forEach((_, index) => {
+      gl.activeTexture(gl.TEXTURE0 + index + this._masksTextureUnitStart);
+      gl.bindTexture(gl.TEXTURE_2D, null);
+    });
+
+    const ctx = form.ctx;
+    invariant(this._canvas !== null, 'canvas is required');
+    ctx.drawImage(this._canvas, 0, 0);
+  }
+
+  async cleanup(): Promise<void> {
+    super.cleanup();
+
+    if (this._gl != null) {
+      // Delete mask textures to prevent memory leaks
+      this._maskTextures.forEach(texture => {
+        if (texture != null && this._gl != null) {
+          this._gl.deleteTexture(texture);
+        }
+      });
+      this._maskTextures = [];
+    }
+  }
+}

demo/frontend/src/common/components/video/effects/shaders/Arrow.frag

+#version 300 es
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+precision mediump float;
+
+in vec2 vTexCoord;
+
+uniform sampler2D uSampler;
+uniform vec2 uSize;
+uniform int uNumMasks;
+uniform float uCurrentFrame;
+uniform bool uLineColor;
+uniform bool uArrow;
+uniform sampler2D uMaskTexture0;
+uniform sampler2D uMaskTexture1;
+uniform sampler2D uMaskTexture2;
+
+uniform vec4 bbox0;
+uniform vec4 bbox1;
+uniform vec4 bbox2;
+
+out vec4 fragColor;
+
+float addv(vec2 a) {
+  return a.x + a.y;
+}
+
+#define dd(a) dot(a,a)
+
+vec2 solveCubic2(vec3 a) {
+  float p = a.y - a.x * a.x / 3.0f;
+  float p3 = p * p * p;
+  float q = a.x * (2.0f * a.x * a.x - 9.0f * a.y) / 27.0f + a.z;
+  float d = q * q + 4.0f * p3 / 27.0f;
+  if(d > 0.0f) {
+    vec2 x = (vec2(1.0f, -1.0f) * sqrt(d) - q) * 0.5f;
+    return vec2(addv(sign(x) * pow(abs(x), vec2(1.0f / 3.0f))) - a.x / 3.0f);
+  }
+  float v = acos(-sqrt(-27.0f / p3) * q * 0.5f) / 3.0f;
+  float m = cos(v);
+  float n = sin(v) * 1.732050808f;
+  return vec2(m + m, -n - m) * sqrt(-p / 3.0f) - a.x / 3.0f;
+}
+
+float calculateDistanceToQuadraticBezier(vec2 p, vec2 a, vec2 b, vec2 c) {
+  b += mix(vec2(1e-4f), vec2(0.0f), abs(sign(b * 2.0f - a - c)));
+  vec2 A = b - a;
+  vec2 B = c - b - A;
+  vec2 C = p - a;
+  vec2 D = A * 2.0f;
+  vec2 T = clamp((solveCubic2(vec3(-3.0f * dot(A, B), dot(C, B) - 2.0f * dd(A), dot(C, A)) / -dd(B))), 0.0f, 1.0f);
+  return sqrt(min(dd(C - (D + B * T.x) * T.x), dd(C - (D + B * T.y) * T.y)));
+}
+
+float crossProduct(vec2 a, vec2 b) {
+  return a.x * b.y - a.y * b.x;
+}
+
+bool pointInTriangle(vec2 pt, vec2 v0, vec2 v1, vec2 v2) {
+  vec2 v0v1 = v1 - v0;
+  vec2 v1v2 = v2 - v1;
+  vec2 v2v0 = v0 - v2;
+  float d0 = sign(crossProduct(v0v1, pt - v0));
+  float d1 = sign(crossProduct(v1v2, pt - v1));
+  float d2 = sign(crossProduct(v2v0, pt - v2));
+  bool has_neg = (d0 < 0.0f) || (d1 < 0.0f) || (d2 < 0.0f);
+  bool has_pos = (d0 > 0.0f) || (d1 > 0.0f) || (d2 > 0.0f);
+  return !(has_neg && has_pos);
+}
+
+void main() {
+  vec4 color = texture(uSampler, vTexCoord);
+  vec2 fragCoord = vTexCoord * uSize;
+  float aspectRatio = uSize.y / uSize.x;
+  float time = uCurrentFrame * 0.05f;
+  vec3 multicolor = vec3(0.5f + 0.5f * sin(time), 0.5f + 0.5f * cos(time), 0.5f - 0.5f * sin(time));
+
+  vec4 mask1 = vec4(0.0f);
+  vec4 mask2 = vec4(0.0f);
+  vec4 mask3 = vec4(0.0f);
+
+  bool scoped = false;
+  bool intersected = false;
+  float threshold = 0.75f;
+  float circleRadius = 0.015f;
+
+  if(uNumMasks > 0) {
+    mask1 = texture(uMaskTexture0, vec2(vTexCoord.y, vTexCoord.x));
+    bool visible = bbox0 != vec4(0.0f);
+
+    vec2 p0 = vec2((bbox0.x + bbox0.z) * 0.5f, bbox0.y); // Top center
+    vec2 p1 = vec2(bbox0.x + 0.5f * (bbox0.z - bbox0.x) * (0.5f + 0.5f * sin(time)), bbox0.y - 0.25f);
+    //vec2 p1 = vec2(0.5f, 0.5f);
+    vec2 p2 = vec2(bbox0.x + 0.5f * (bbox0.z - bbox0.x) * (0.5f + 0.5f * cos(time)), (bbox0.w + bbox0.y) * 0.5f);
+    float d = calculateDistanceToQuadraticBezier(vTexCoord, p0, p1, p2);
+    d *= length(uSize.xy) * 0.25f;
+
+    vec2 v0 = p0 + vec2(-0.020f, -0.020f); // Left vertex
+    vec2 v1 = p0 + vec2(0.020f, -0.020f);  // Right vertex
+    vec2 v2 = p0 + vec2(0.0f, 0.020f);  // Bottom vertex
+    // Check if the point is inside the triangle
+    bool inside = pointInTriangle(vTexCoord, v0, v1, v2);
+
+    // Circle drawing
+    vec2 adjustedCoord = vTexCoord - p0;
+    adjustedCoord.x /= aspectRatio;
+    float circleDistance = length(adjustedCoord);
+
+    if(d < threshold && visible) {
+      scoped = true;
+    }
+
+    if(uArrow && inside && visible) {
+      intersected = true;
+    } else if(!uArrow && circleDistance < circleRadius && visible) {
+      intersected = true;
+    }
+  }
+
+  if(uNumMasks > 1) {
+    mask2 = texture(uMaskTexture1, vec2(vTexCoord.y, vTexCoord.x));
+    bool visible = bbox1 != vec4(0.0f);
+
+    vec2 p0 = vec2((bbox1.x + bbox1.z) * 0.5f, bbox1.y);
+    vec2 p1 = vec2(bbox1.x + 0.5f * (bbox1.z - bbox1.x) * (0.5f + 0.5f * sin(time)), bbox1.y - 0.25f);
+    vec2 p2 = vec2(bbox1.x + 0.5f * (bbox1.z - bbox1.x) * (0.5f + 0.5f * cos(time)), (bbox1.w + bbox1.y) * 0.5f);
+    float d = calculateDistanceToQuadraticBezier(vTexCoord, p0, p1, p2);
+    d *= length(uSize.xy) * 0.25f;
+
+    vec2 v0 = p0 + vec2(-0.020f, -0.020f);
+    vec2 v1 = p0 + vec2(0.020f, -0.020f);
+    vec2 v2 = p0 + vec2(0.0f, 0.020f);
+    bool inside = pointInTriangle(vTexCoord, v0, v1, v2);
+
+    // Circle drawing
+    vec2 adjustedCoord = vTexCoord - p0;
+    adjustedCoord.x /= aspectRatio;
+    float circleDistance = length(adjustedCoord);
+
+    if(d < threshold && visible) {
+      scoped = true;
+    }
+
+    if(uArrow && inside && visible) {
+      intersected = true;
+    } else if(!uArrow && circleDistance < circleRadius && visible) {
+      intersected = true;
+    }
+  }
+
+  if(uNumMasks > 2) {
+    mask3 = texture(uMaskTexture2, vec2(vTexCoord.y, vTexCoord.x));
+    bool visible = bbox2 != vec4(0.0f);
+
+    vec2 p0 = vec2((bbox2.x + bbox2.z) * 0.5f, bbox2.y);
+    vec2 p1 = vec2(bbox2.x + 0.5f * (bbox2.z - bbox2.x) * (0.5f + 0.5f * sin(time)), bbox2.y - 0.25f);
+    vec2 p2 = vec2(bbox2.x + 0.5f * (bbox2.z - bbox2.x) * (0.5f + 0.5f * cos(time)), (bbox2.w + bbox2.y) * 0.5f);
+    float d = calculateDistanceToQuadraticBezier(vTexCoord, p0, p1, p2);
+    d *= length(uSize.xy) * 0.25f;
+
+    vec2 v0 = p0 + vec2(-0.020f, -0.020f);
+    vec2 v1 = p0 + vec2(0.020f, -0.020f);
+    vec2 v2 = p0 + vec2(0.0f, 0.020f);
+    bool inside = pointInTriangle(vTexCoord, v0, v1, v2);
+
+    vec2 adjustedCoord = vTexCoord - p0;
+    adjustedCoord.x /= aspectRatio;
+    float circleDistance = length(adjustedCoord);
+
+    if(d < threshold && visible) {
+      scoped = true;
+    }
+
+    if(uArrow && inside && visible) {
+      intersected = true;
+    } else if(!uArrow && circleDistance < circleRadius && visible) {
+      intersected = true;
+    }
+  }
+
+  bool overlap = (mask1.r > 0.0f || mask2.r > 0.0f || mask3.r > 0.0f);
+  if(overlap) {
+    fragColor = color;
+  }
+
+  if(scoped || intersected) {
+    fragColor = uLineColor ? vec4(multicolor, 1.0f) : vec4(1.0f);
+    if(intersected) {
+      fragColor = vec4(multicolor, 1.0f);
+    }
+  } else {
+    fragColor = overlap ? color : vec4(0.0f, 0.0f, 0.0f, 0.0f);
+  }
+}
\ No newline at end of file

demo/frontend/src/common/components/video/effects/shaders/BackgroundBlur.frag

+#version 300 es
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+precision mediump float;
+
+in vec2 vTexCoord;
+
+uniform sampler2D uSampler;
+uniform vec2 uSize;
+uniform int uBlurRadius;
+
+out vec4 fragColor;
+
+void main() {
+  vec2 texOffset = 1.0f / uSize;
+  // texel color
+  vec3 color = texture(uSampler, vTexCoord).rgb;
+  float sampleCount = 0.0f;
+
+  // sample the surrounding pixels based on the blur radius
+  for(int x = -uBlurRadius; x <= uBlurRadius; x++) {
+    for(int y = -uBlurRadius; y <= uBlurRadius; y++) {
+      vec2 offset = vec2(float(x), float(y)) * texOffset;
+      color += texture(uSampler, vTexCoord + offset).rgb;
+      sampleCount += 1.0f;
+    }
+  }
+
+  // average the colors of the sampled pixels
+  color /= sampleCount;
+  fragColor = vec4(color, 1.0f);
+}
\ No newline at end of file

demo/frontend/src/common/components/video/effects/shaders/Burst.frag

+#version 300 es
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+precision highp float;
+
+in vec2 vTexCoord;
+
+uniform sampler2D uSampler;
+uniform vec2 uSize; // resolution
+uniform int uNumMasks;
+uniform bool uLineColor;
+uniform bool uInterleave;
+uniform sampler2D uMaskTexture0;
+uniform sampler2D uMaskTexture1;
+uniform sampler2D uMaskTexture2;
+
+uniform vec4 uMaskColor0;
+uniform vec4 uMaskColor1;
+uniform vec4 uMaskColor2;
+
+uniform vec4 bbox0;
+uniform vec4 bbox1;
+uniform vec4 bbox2;
+
+out vec4 fragColor;
+
+void main() {
+  float PI = radians(180.0f);
+  float lines = uInterleave ? 12.0f : 80.0f;
+  vec4 color = texture(uSampler, vTexCoord);
+  vec4 color1 = uMaskColor0 / 255.0;
+  vec4 color2 = uMaskColor1 / 255.0;
+  vec4 color3 = uMaskColor2 / 255.0;
+
+  vec4 mask1 = vec4(0.0f);
+  vec4 mask2 = vec4(0.0f);
+  vec4 mask3 = vec4(0.0f);
+  vec4 scopedColor = vec4(0.0f);
+
+  vec2 fragCoord = vTexCoord * uSize; // transform to pixel space
+  bool scoped = false;
+  vec4 transparent = vec4(0.0);
+  float p = PI / lines;
+
+  if(uNumMasks > 0) {
+    mask1 = texture(uMaskTexture0, vec2(vTexCoord.y, vTexCoord.x));
+
+    vec2 center1 = (bbox0.xy + bbox0.zw) * 0.5f * uSize;
+    vec2 fragCoordT = (fragCoord - center1) / uSize.y;
+    float a = mod(atan(fragCoordT.y, fragCoordT.x) + p, p + p) - p; // angle of fragment
+
+    float pattern = sin(a * lines);
+    // smoothstep for antialiasing
+    float line = smoothstep(2.8 / uSize.y, 0.0, length(fragCoordT) * abs(sin(a)));
+    
+    vec4 colorToBlend = uLineColor ? vec4(color1.rgb, 0.80f) : vec4(1.0f);
+    bool visible = bbox0 != vec4(0.0f);
+
+    if (uInterleave && visible) {
+      vec4 tempColor = mix(transparent, colorToBlend, step(0.0, pattern));
+      scopedColor += tempColor;
+      scoped = true;
+    } else if (!uInterleave && visible) {
+      vec4 tempColor = uLineColor ? vec4(color1.rgb * line, line) : vec4(line);
+      scopedColor += tempColor;
+      scoped = true;
+    }
+  }
+
+  if(uNumMasks > 1) {
+    mask2 = texture(uMaskTexture1, vec2(vTexCoord.y, vTexCoord.x));
+
+    vec2 center2 = (bbox1.xy + bbox1.zw) * 0.5f * uSize;
+    vec2 fragCoordT = (fragCoord - center2) / uSize.y;
+    float a = mod(atan(fragCoordT.y, fragCoordT.x) + p, p + p) - p; // angle of fragment
+
+    float pattern = sin(a * lines);
+    float line = smoothstep(2.8 / uSize.y, 0.0, length(fragCoordT) * abs(sin(a)));
+    
+    vec4 colorToBlend = uLineColor ? vec4(color2.rgb, 0.8f) : vec4(1.0f);
+    bool visible = bbox1 != vec4(0.0f);
+
+    if (uInterleave && visible) {
+      vec4 tempColor = mix(transparent, colorToBlend, step(0.0, pattern));
+      if (scopedColor == vec4(0.0)) {
+        scopedColor += tempColor;
+      }
+      scoped = true;
+    } else if (!uInterleave && visible) {
+      vec4 tempColor = uLineColor ? vec4(color2.rgb * line, line) : vec4(line);
+      scopedColor += tempColor;    
+      scoped = true;
+    }
+  }
+
+  if (uNumMasks > 2) {
+    mask3 = texture(uMaskTexture2, vec2(vTexCoord.y, vTexCoord.x));
+
+    vec2 center3 = (bbox2.xy + bbox2.zw) * 0.5f * uSize;
+    vec2 fragCoordT = (fragCoord - center3) / uSize.y;
+
+    float a = mod(atan(fragCoordT.y, fragCoordT.x) + p, p + p) - p; // angle of fragment
+
+    float pattern = sin(a * lines);
+    float line = smoothstep(2.8 / uSize.y, 0.0, length(fragCoordT) * abs(sin(a)));
+
+    vec4 colorToBlend = uLineColor ? vec4(color3.rgb, 0.8f) : vec4(1.0f);
+    bool visible = bbox2 != vec4(0.0f);
+
+    if (uInterleave && visible) {
+      vec4 tempColor = mix(transparent, colorToBlend, step(0.0, pattern));
+      if (scopedColor == vec4(0.0)) {
+        scopedColor += tempColor;
+      }
+      scoped = true;
+    } else if (!uInterleave && visible) {
+      vec4 tempColor = uLineColor ? vec4(color3.rgb * line, line) : vec4(line);
+      scopedColor += tempColor;
+      scoped = true;
+    }
+  }
+
+  bool overlap = (mask1.r > 0.0f || mask2.r > 0.0f || mask3.r > 0.0f);
+  if(scoped) {
+    fragColor = overlap ? color : scopedColor;
+  } else {
+    fragColor = overlap ? color : vec4(0.0f, 0.0f, 0.0f, 0.0f);
+  }
+}

demo/frontend/src/common/components/video/effects/shaders/Cutout.frag

+#version 300 es
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+precision mediump float;
+
+in vec2 vTexCoord;
+
+uniform sampler2D uSampler;
+uniform float uContrast;
+uniform int uNumMasks;
+uniform sampler2D uMaskTexture0;
+uniform sampler2D uMaskTexture1;
+uniform sampler2D uMaskTexture2;
+
+out vec4 fragColor;
+
+vec3 applySepia(vec4 color) {
+  float gray = dot(color.rgb, vec3(0.3, 0.59, 0.11));
+  vec3 sepia = vec3(gray) * vec3(1.2, 1.0, 0.8);
+  sepia.r = min(sepia.r, 1.0);
+  sepia.g = min(sepia.g, 1.0);
+  sepia.b = min(sepia.b, 1.0);
+
+  return sepia;
+}
+
+void main() {
+  vec4 color = texture(uSampler, vTexCoord);
+
+  vec4 color1 = vec4(0.0f);
+  vec4 color2 = vec4(0.0f);
+  vec4 color3 = vec4(0.0f);
+
+  if(uNumMasks > 0) {
+    color1 = texture(uMaskTexture0, vec2(vTexCoord.y, vTexCoord.x));
+  }
+  if(uNumMasks > 1) {
+    color2 = texture(uMaskTexture1, vec2(vTexCoord.y, vTexCoord.x));
+  }
+  if(uNumMasks > 2) {
+    color3 = texture(uMaskTexture2, vec2(vTexCoord.y, vTexCoord.x));
+  }
+
+  bool overlap = (color1.r > 0.0f || color2.r > 0.0f || color3.r > 0.0f);
+  if(overlap) {    
+    if (uContrast == 0.0) {
+      color = vec4(applySepia(color), color.a);
+    } else {
+      color.rgb = ((color.rgb - 0.5) * max(uContrast, 0.0)) + 0.5;   
+    }
+    fragColor = color;
+  } else {
+    fragColor = vec4(0.0f);
+  }
+}
\ No newline at end of file

demo/frontend/src/common/components/video/effects/shaders/DefaultVert.vert

+#version 300 es
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+layout(location = 0) in vec4 aPosition;
+layout(location = 1) in vec2 aTexCoord;
+
+out vec2 vTexCoord;
+
+void main() {
+  vTexCoord = vec2(aTexCoord.s, 1.0f - aTexCoord.t);
+  gl_Position = aPosition;
+}
\ No newline at end of file