Initial commit

demo/frontend/src/common/components/video/effects/shaders/EraseForeground.frag

+#version 300 es
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+precision lowp float;
+
+in vec2 vTexCoord;
+uniform int uNumMasks;
+uniform vec3 uBgColor;
+uniform sampler2D uMaskTexture0;
+uniform sampler2D uMaskTexture1;
+uniform sampler2D uMaskTexture2;
+
+out vec4 fragColor;
+
+void main() {
+  vec4 finalColor = vec4(0.0f, 0.0f, 0.0f, 0.0f);
+  float totalMaskValue = 0.0f;
+
+  if(uNumMasks > 0) {
+    float maskValue0 = texture(uMaskTexture0, vec2(vTexCoord.y, vTexCoord.x)).r;
+    totalMaskValue += maskValue0;
+  }
+  if(uNumMasks > 1) {
+    float maskValue1 = texture(uMaskTexture1, vec2(vTexCoord.y, vTexCoord.x)).r;
+    totalMaskValue += maskValue1;
+  }
+  if(uNumMasks > 2) {
+    float maskValue2 = texture(uMaskTexture2, vec2(vTexCoord.y, vTexCoord.x)).r;
+    totalMaskValue += maskValue2;
+  }
+
+  if(totalMaskValue > 0.0f) {
+    finalColor = vec4(uBgColor, 1.0f);
+  } else {
+    finalColor.a = 0.0f;
+  }
+  fragColor = finalColor;
+}
\ No newline at end of file

demo/frontend/src/common/components/video/effects/shaders/Gradient.frag

+#version 300 es
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+precision mediump float;
+precision mediump sampler3D;
+
+in vec2 vTexCoord;
+
+uniform sampler2D uSampler;
+uniform sampler3D uColorGradeLUT;
+uniform mediump vec2 uSize;
+
+out vec4 fragColor;
+
+void main() {
+
+  // texel color
+  vec3 color = texture(uSampler, vTexCoord).rgb;
+  vec3 gradedColor = texture(uColorGradeLUT, color).rgb;
+  fragColor = vec4(gradedColor, 1);
+}
\ No newline at end of file

demo/frontend/src/common/components/video/effects/shaders/NoisyMask.frag

+#version 300 es
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+precision mediump float;
+
+in vec2 vTexCoord;
+
+uniform float uCurrentFrame;
+uniform int uNumMasks;
+uniform sampler2D uMaskTexture0;
+uniform sampler2D uMaskTexture1;
+uniform sampler2D uMaskTexture2;
+
+out vec4 fragColor;
+
+vec3 startColor = vec3(0.0f, 0.67f, 1.0f);
+vec3 endColor = vec3(0.05f, 0.06f, 0.05f);
+
+float random(vec2 st) {
+  return fract(sin(dot(st.xy, vec2(12.9898f, 78.233f))) *
+    43758.5453123f);
+}
+
+void main() {
+  vec4 finalColor = vec4(0.0f, 0.0f, 0.0f, 0.0f);
+  float totalMaskValue = 0.0f;
+
+  if(uNumMasks > 0) {
+    float maskValue0 = texture(uMaskTexture0, vec2(vTexCoord.y, vTexCoord.x)).r;
+    totalMaskValue += maskValue0;
+  }
+  if(uNumMasks > 1) {
+    float maskValue1 = texture(uMaskTexture1, vec2(vTexCoord.y, vTexCoord.x)).r;
+    totalMaskValue += maskValue1;
+  }
+  if(uNumMasks > 2) {
+    float maskValue2 = texture(uMaskTexture2, vec2(vTexCoord.y, vTexCoord.x)).r;
+    totalMaskValue += maskValue2;
+  }
+
+  // Dynamic color alteration using sin(time)
+  float time = uCurrentFrame * 0.1f;
+  vec3 dynamicColor = mix(startColor, endColor, sin(time));
+  vec3 colorVariation = mix(vec3(0.0f, 0.0f, 0.0f), vec3(1.0f, 1.0f, 1.0f), vTexCoord.y);
+
+  // apply randomness to the final color
+  float rnd = random(vTexCoord.xy);
+ 
+  if(totalMaskValue > 0.0f) {
+    finalColor = vec4(mix(dynamicColor, colorVariation, rnd), 1.0f);
+  } else {
+    finalColor.a = 0.0f;
+  }
+  fragColor = finalColor;
+}
\ No newline at end of file

demo/frontend/src/common/components/video/effects/shaders/Overlay.frag

+#version 300 es
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+precision highp float;
+
+in vec2 vTexCoord;
+
+uniform sampler2D uSampler;
+uniform vec2 uSize;
+uniform int uNumMasks;
+uniform float uOpacity;
+uniform bool uBorder;
+uniform sampler2D uMaskTexture0;
+uniform sampler2D uMaskTexture1;
+uniform sampler2D uMaskTexture2;
+
+uniform vec4 uMaskColor0;
+uniform vec4 uMaskColor1;
+uniform vec4 uMaskColor2;
+
+uniform float uTime;
+uniform vec2 uClickPos;
+uniform int uActiveMask;
+
+out vec4 fragColor;
+
+vec4 lowerSaturation(vec4 color, float saturationFactor) {
+  float luminance = 0.299f * color.r + 0.587f * color.g + 0.114f * color.b; // Calculate luminance
+  vec3 gray = vec3(luminance);
+  vec3 saturated = mix(gray, color.rgb, saturationFactor); // Mix gray with original color based on saturation factor
+  return vec4(saturated, color.a);
+}
+
+vec4 detectEdges(sampler2D textureSampler, float coverage, vec4 edgeColor) {
+  vec2 tvTexCoord = vec2(vTexCoord.y, vTexCoord.x);
+  vec2 texOffset = 1.0f / uSize;
+  vec3 result = vec3(0.0f);
+  // neighboring pixels
+  vec3 tLeft = texture(textureSampler, tvTexCoord + texOffset * vec2(-coverage, coverage)).rgb;
+  vec3 tRight = texture(textureSampler, tvTexCoord + texOffset * vec2(coverage, -coverage)).rgb;
+  vec3 bLeft = texture(textureSampler, tvTexCoord + texOffset * vec2(-coverage, -coverage)).rgb;
+  vec3 bRight = texture(textureSampler, tvTexCoord + texOffset * vec2(coverage, coverage)).rgb;
+
+  // calculate the gradient edge of the current pixel using [3x3] sobel operator.
+  vec3 xEdge = tLeft + 2.0f * texture(textureSampler, tvTexCoord + texOffset * vec2(-coverage, 0)).rgb + bLeft - tRight - 2.0f * texture(textureSampler, tvTexCoord + texOffset * vec2(coverage, 0)).rgb - bRight;
+  vec3 yEdge = tLeft + 2.0f * texture(textureSampler, tvTexCoord + texOffset * vec2(0, coverage)).rgb + tRight - bLeft - 2.0f * texture(textureSampler, tvTexCoord + texOffset * vec2(0, -coverage)).rgb - bRight;
+
+  // magnitude of the gradient at the current pixel.
+  result = sqrt(xEdge * xEdge + yEdge * yEdge);
+  return result.r > 1e-6f ? edgeColor : vec4(0.0f, 0.0f, 0.0f, 0.0f);
+}
+
+vec2 calculateAdjustedTexCoord(vec2 vTexCoord, vec4 bbox, float aspectRatio) {
+  vec2 center = vec2((bbox.x + bbox.z) * 0.5f, bbox.w);
+  float radiusX = abs(bbox.z - bbox.x);
+  float radiusY = radiusX / aspectRatio;
+  float scale = 1.0f;
+  radiusX *= scale;
+  radiusY *= scale;
+  vec2 adjustedTexCoord = (vTexCoord - center) / vec2(radiusX, radiusY) + vec2(0.5f);
+  return adjustedTexCoord;
+}
+
+void main() {
+  vec4 color = texture(uSampler, vTexCoord);
+  vec4 color1 = uMaskColor0 / 255.0;
+  vec4 color2 = uMaskColor1 / 255.0;
+  vec4 color3 = uMaskColor2 / 255.0;
+  float saturationFactor = 0.7;
+  float aspectRatio = uSize.y / uSize.x;
+  vec2 tvTexCoord = vec2(vTexCoord.y, vTexCoord.x);
+
+  vec4 finalColor = vec4(0.0f, 0.0f, 0.0f, 0.0f);
+  float totalMaskValue = 0.0f;
+  vec4 edgeColor = vec4(0.0f, 0.0f, 0.0f, 0.0f);
+  float numRipples = 1.75;
+  float timeThreshold = 1.1; // can take any value from [0.0, 1.5]
+  vec2 adjustedClickCoord =  calculateAdjustedTexCoord(vTexCoord, vec4(uClickPos, uClickPos + 0.1), aspectRatio);
+
+  if(uNumMasks > 0) {
+    float maskValue0 = texture(uMaskTexture0, tvTexCoord).r;
+    vec4 saturatedColor = lowerSaturation(color1, saturationFactor);
+    vec4 plainColor= vec4(vec3(saturatedColor).rgb, 1.0);
+    vec4 rippleColor = vec4(color1.rgb, 0.2);
+    
+    if (uActiveMask == 0 && uTime < timeThreshold) {
+      float dist = length(adjustedClickCoord);
+      float colorFactor = abs(sin((dist - uTime) * numRipples));
+      plainColor = vec4(mix(rippleColor, plainColor, colorFactor));
+    };
+    
+    if (uTime >= timeThreshold) {
+      plainColor= vec4(vec3(saturatedColor).rgb, 1.0);
+    }
+    finalColor += maskValue0 * plainColor;
+    totalMaskValue += maskValue0;
+
+    edgeColor = detectEdges(uMaskTexture0, 1.25, color1);
+  }
+  if(uNumMasks > 1) {
+    float maskValue1 = texture(uMaskTexture1, tvTexCoord).r;
+    vec4 saturatedColor = lowerSaturation(color2, saturationFactor);
+    vec4 plainColor= vec4(vec3(saturatedColor).rgb, 1.0);
+    vec4 rippleColor = vec4(color2.rgb, 0.2);
+
+    if (uActiveMask == 1 && uTime < timeThreshold) {
+      float dist = length(adjustedClickCoord);
+      float colorFactor = abs(sin((dist - uTime) * numRipples));
+      plainColor = vec4(mix(rippleColor, plainColor, colorFactor));
+    }
+
+    if (uTime >= timeThreshold) {
+      plainColor= vec4(vec3(saturatedColor).rgb, 1.0);
+    }
+    finalColor += maskValue1 * plainColor;
+    totalMaskValue += maskValue1;
+
+    if(edgeColor.a <= 0.0f) {
+      edgeColor = detectEdges(uMaskTexture1, 1.25, color2);
+    }
+  }
+  if(uNumMasks > 2) {
+    float maskValue2 = texture(uMaskTexture2, tvTexCoord).r;
+    vec4 saturatedColor = lowerSaturation(color3, saturationFactor);
+    vec4 plainColor= vec4(vec3(saturatedColor).rgb, 1.0);
+    vec4 rippleColor = vec4(color3.rgb, 0.2);
+
+    if (uActiveMask == 2 && uTime < timeThreshold) {
+      float dist = length(adjustedClickCoord);
+      float colorFactor = abs(sin((dist - uTime) * numRipples));
+      plainColor = vec4(mix(rippleColor, plainColor, colorFactor));
+    }
+
+    if (uTime >= timeThreshold) {
+      plainColor= vec4(vec3(saturatedColor).rgb, 1.0);
+    }
+
+    finalColor += maskValue2 * plainColor;
+    totalMaskValue += maskValue2;
+
+    if(edgeColor.a <= 0.0f) {
+      edgeColor = detectEdges(uMaskTexture2, 1.25, color3);
+    }
+  }
+
+  if(totalMaskValue > 0.0f) {
+    finalColor /= totalMaskValue;
+    finalColor = mix(color, finalColor, uOpacity);
+  } else {
+    finalColor.a = 0.0f;
+  }
+
+  if(edgeColor.a > 0.0f && uBorder) {
+    finalColor = vec4(vec3(edgeColor), 1.0f);
+  }
+  fragColor = finalColor;
+}
\ No newline at end of file

demo/frontend/src/common/components/video/effects/shaders/Overlay.vert

+#version 300 es
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+layout(location = 0) in vec4 aPosition;
+layout(location = 1) in vec2 aTexCoord;
+
+out vec2 vTexCoord;
+
+void main() {
+  // Rotate texture 90 degrees clockwise
+  vTexCoord = vec2(1.0f - aTexCoord.t, aTexCoord.s);
+  gl_Position = aPosition;
+}
\ No newline at end of file

demo/frontend/src/common/components/video/effects/shaders/Pixelate.frag

+#version 300 es
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+precision mediump float;
+
+in vec2 vTexCoord;
+
+uniform sampler2D uSampler;
+uniform mediump vec2 uSize;
+uniform lowp float uBlockSize;
+
+out vec4 fragColor;
+
+void main() {
+  vec2 uv = vTexCoord.xy;
+  float dx = uBlockSize / uSize.x;
+  float dy = uBlockSize / uSize.y;
+
+  // Sample from 2 places to get a better average texel color
+  vec2 sampleCoord = (vec2(dx * floor((uv.x / dx)), dy * floor((uv.y / dy))) +
+    vec2(dx * ceil((uv.x / dx)), dy * ceil((uv.y / dy)))) / 2.0f;
+
+  vec4 frameColor = texture(uSampler, sampleCoord);
+  fragColor = frameColor;
+}
\ No newline at end of file

demo/frontend/src/common/components/video/effects/shaders/PixelateMask.frag

+#version 300 es
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+precision mediump float;
+
+in vec2 vTexCoord;
+
+uniform sampler2D uSampler;
+uniform mediump vec2 uSize;
+uniform lowp float uBlockSize;
+uniform int uNumMasks;
+uniform sampler2D uMaskTexture0;
+uniform sampler2D uMaskTexture1;
+uniform sampler2D uMaskTexture2;
+
+out vec4 fragColor;
+
+void main() {
+  vec4 color = texture(uSampler, vTexCoord);
+  vec2 uv = vTexCoord.xy;
+  float dx = uBlockSize / uSize.x;
+  float dy = uBlockSize / uSize.y;
+
+  vec4 color1 = vec4(0.0f);
+  vec4 color2 = vec4(0.0f);
+  vec4 color3 = vec4(0.0f);
+
+  vec2 sampleCoord = (vec2(dx * floor((uv.x / dx)), dy * floor((uv.y / dy))) +
+  vec2(dx * ceil((uv.x / dx)), dy * ceil((uv.y / dy)))) / 2.0f;
+  vec4 frameColor = texture(uSampler, sampleCoord);
+  color = frameColor;
+
+  if(uNumMasks > 0) {
+    color1 = texture(uMaskTexture0, vec2(vTexCoord.y, vTexCoord.x));
+  }
+  if(uNumMasks > 1) {
+    color2 = texture(uMaskTexture1, vec2(vTexCoord.y, vTexCoord.x));    
+  }
+  if(uNumMasks > 2) {
+    color3 = texture(uMaskTexture2, vec2(vTexCoord.y, vTexCoord.x));
+  }
+
+  bool overlap = (color1.r > 0.0f || color2.r > 0.0f || color3.r > 0.0f);
+  if(overlap) {
+    fragColor = color;
+  } else {
+    fragColor = vec4(0.0f);
+  }
+}
\ No newline at end of file

demo/frontend/src/common/components/video/effects/shaders/Replace.frag

+#version 300 es
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+precision lowp float;
+
+in vec2 vTexCoord;
+uniform vec2 uSize;
+uniform int uNumMasks;
+uniform sampler2D uEmojiTexture;
+uniform bool uFill; // use all emoji texture
+uniform sampler2D uMaskTexture0;
+uniform sampler2D uMaskTexture1;
+uniform sampler2D uMaskTexture2;
+
+uniform vec4 bbox0;
+uniform vec4 bbox1;
+uniform vec4 bbox2;
+
+out vec4 fragColor;
+
+vec2 calculateAdjustedTexCoord(vec2 vTexCoord, vec4 bbox, float aspectRatio, out float distanceFromCenter) {
+  vec2 center = (bbox.xy + bbox.zw) * 0.5f;
+  float radiusX = abs(bbox.z - bbox.x);
+  float radiusY = radiusX / aspectRatio;
+  float scale = 1.25f;
+  radiusX *= scale;
+  radiusY *= scale;
+  vec2 adjustedTexCoord = (vTexCoord - center) / vec2(radiusX, radiusY) + vec2(0.5f);
+  distanceFromCenter = length((vTexCoord - center) / vec2(radiusX * 0.5f, radiusY * 0.5f));
+  return adjustedTexCoord;
+}
+
+void main() {
+  vec4 finalColor = vec4(0.0f);
+
+  float aspectRatio = uSize.y / uSize.x;
+  float totalMaskValue = 0.0f;
+  vec4 bgFill = vec4(1.0f, 0.0f, 0.0f, 1.0f);
+
+  vec4 emojiColor;
+
+  if(uNumMasks > 0) {
+    float maskValue0 = texture(uMaskTexture0, vec2(vTexCoord.y, vTexCoord.x)).r;
+    float distanceFromCenter;
+    vec2 adjustedTexCoord = calculateAdjustedTexCoord(vTexCoord, bbox0, aspectRatio, distanceFromCenter);
+
+    if(maskValue0 > 0.0f) {
+      emojiColor = texture(uEmojiTexture, adjustedTexCoord);
+      if(distanceFromCenter > 0.85f && !uFill) {
+        emojiColor = bgFill;
+      }
+    }
+    if(uFill) {
+      emojiColor = texture(uEmojiTexture, adjustedTexCoord);
+    }
+    
+    totalMaskValue += maskValue0;
+  }
+  if(uNumMasks > 1) {
+    float maskValue1 = texture(uMaskTexture1, vec2(vTexCoord.y, vTexCoord.x)).r;
+    float distanceFromCenter;
+    vec2 adjustedTexCoord = calculateAdjustedTexCoord(vTexCoord, bbox1, aspectRatio, distanceFromCenter);
+
+    if(maskValue1 > 0.0f) {
+      emojiColor = texture(uEmojiTexture, adjustedTexCoord);
+      if(distanceFromCenter > 0.85f && !uFill) {
+        emojiColor = bgFill;
+      }
+    }
+    if(uFill && emojiColor.a == 0.0f) {
+      emojiColor = texture(uEmojiTexture, adjustedTexCoord);
+    }
+      
+    totalMaskValue += maskValue1;
+  }
+  if(uNumMasks > 2) {
+    float maskValue2 = texture(uMaskTexture2, vec2(vTexCoord.y, vTexCoord.x)).r;
+    float distanceFromCenter;
+    vec2 adjustedTexCoord = calculateAdjustedTexCoord(vTexCoord, bbox2, aspectRatio, distanceFromCenter);
+    if(maskValue2 > 0.0f) {
+      emojiColor = texture(uEmojiTexture, adjustedTexCoord);
+      if(distanceFromCenter > 0.85f && !uFill) {
+        emojiColor = bgFill;
+      }
+    }
+    if(uFill && emojiColor.a == 0.0f) {
+      emojiColor = texture(uEmojiTexture, adjustedTexCoord);
+    }
+
+    totalMaskValue += maskValue2;
+  }
+
+  if(totalMaskValue > 0.0f) {
+    finalColor = emojiColor;
+  } else {
+    finalColor = uFill ? emojiColor : vec4(0.0f);
+  }
+  fragColor = finalColor;
+}

demo/frontend/src/common/components/video/effects/shaders/Scope.frag

+#version 300 es
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+precision mediump float;
+
+in vec2 vTexCoord;
+
+uniform sampler2D uSampler;
+uniform vec2 uSize;
+uniform int uNumMasks;
+uniform bool uFillColor;
+uniform bool uLight;
+uniform bool uTransparency;
+uniform sampler2D uMaskTexture0;
+uniform sampler2D uMaskTexture1;
+uniform sampler2D uMaskTexture2;
+
+uniform vec4 uMaskColor0;
+uniform vec4 uMaskColor1;
+uniform vec4 uMaskColor2;
+
+uniform vec4 bbox0;
+uniform vec4 bbox1;
+uniform vec4 bbox2;
+
+out vec4 fragColor;
+
+void main() {
+  vec4 color = texture(uSampler, vTexCoord);
+  float aspectRatio = uSize.y / uSize.x;
+  float radiusThreshold = 0.8f;
+  float tickness = 0.085f;
+
+  vec4 mask1 = vec4(0.0f);
+  vec4 mask2 = vec4(0.0f);
+  vec4 mask3 = vec4(0.0f);
+  vec4 color1 = uMaskColor0 / 255.0;
+  vec4 color2 = uMaskColor1 / 255.0;
+  vec4 color3 = uMaskColor2 / 255.0;
+  vec4 scopedColor = vec4(0.0f);
+
+  bool scoped = false;
+  vec4 whiteVariation = uTransparency ? vec4(0.0,0.0,0.0,1.0) : vec4(1.0);
+
+  if(uNumMasks > 0) {
+    mask1 = texture(uMaskTexture0, vec2(vTexCoord.y, vTexCoord.x));
+
+    vec2 center1 = (bbox0.xy + bbox0.zw) * 0.5f;
+    float radiusX1 = abs(bbox0.y - bbox0.w) * 0.5f;
+    float radiusY1 = radiusX1 / aspectRatio;
+
+    float distX1 = (vTexCoord.x - center1.x) / radiusX1;
+    float distY1 = (vTexCoord.y - center1.y) / radiusY1;
+    float dist1 = sqrt(pow(distX1, 2.0f) + pow(distY1, 2.0f));
+   
+    if(uFillColor) {
+      if(dist1 >= radiusThreshold - tickness && dist1 <= radiusThreshold) {
+        scoped = true;
+        scopedColor = uLight ? whiteVariation : color1;
+      }
+    } else if(dist1 <= radiusThreshold) {
+      scoped = true;
+      scopedColor = uLight ? whiteVariation : color1;
+    }
+  }
+  if(uNumMasks > 1) {
+    mask2 = texture(uMaskTexture1, vec2(vTexCoord.y, vTexCoord.x));
+
+    vec2 center2 = (bbox1.xy + bbox1.zw) * 0.5f;
+    float radiusX2 = abs(bbox1.y - bbox1.w) * 0.5f;
+    float radiusY2 = radiusX2 / aspectRatio;
+
+    float distX2 = (vTexCoord.x - center2.x) / radiusX2;
+    float distY2 = (vTexCoord.y - center2.y) / radiusY2;
+    float dist2 = sqrt(pow(distX2, 2.0f) + pow(distY2, 2.0f));
+
+    if(uFillColor) {
+      if(dist2 >= radiusThreshold - tickness && dist2 <= radiusThreshold) {
+        scoped = true;
+        scopedColor = uLight ? whiteVariation : color2;
+      }
+    } else if(dist2 <= radiusThreshold) {
+      scoped = true;
+      scopedColor = uLight ? whiteVariation : color2;
+    }
+  }
+  if(uNumMasks > 2) {
+    mask3 = texture(uMaskTexture2, vec2(vTexCoord.y, vTexCoord.x));
+
+    vec2 center3 = (bbox2.xy + bbox2.zw) * 0.5f;
+    float radiusX3 = abs(bbox2.y - bbox2.w) * 0.5f;
+    float radiusY3 = radiusX3 / aspectRatio;
+
+    float distX3 = (vTexCoord.x - center3.x) / radiusX3;
+    float distY3 = (vTexCoord.y - center3.y) / radiusY3;
+    float dist3 = sqrt(pow(distX3, 2.0f) + pow(distY3, 2.0f));
+
+    if(uFillColor) {
+      if(dist3 >= radiusThreshold - tickness && dist3 <= radiusThreshold) {
+        scoped = true;
+        scopedColor = uLight ? whiteVariation : color3;
+      }
+    } else if(dist3 <= radiusThreshold) {
+      scoped = true;
+      scopedColor = uLight ? whiteVariation : color3;
+    }
+  }
+
+  bool overlap = (mask1.r > 0.0f || mask2.r > 0.0f || mask3.r > 0.0f);
+
+  if(scoped) {
+    fragColor = overlap ? color : scopedColor;
+    fragColor.a = uTransparency ? fragColor.a : 1.0;
+  } else {
+    fragColor = overlap ? color : vec4(0.0f, 0.0f, 0.0f, 0.0f);
+  }
+}

demo/frontend/src/common/components/video/effects/shaders/Sobel.frag

+#version 300 es
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+precision mediump float;
+
+in vec2 vTexCoord;
+
+uniform sampler2D uSampler;
+uniform vec2 uSize;
+uniform bool uSwapColor;
+uniform bool uMonocolor;
+
+out vec4 fragColor;
+
+void main() {
+  // calculate the offset for one pixel in texture coordinates
+  vec2 texOffset = 1.0f / uSize;
+  vec3 result = vec3(0.0f);
+  // neighboring pixels
+  vec3 tLeft = texture(uSampler, vTexCoord + texOffset * vec2(-1, 1)).rgb;
+  vec3 tRight = texture(uSampler, vTexCoord + texOffset * vec2(1, -1)).rgb;
+  vec3 bLeft = texture(uSampler, vTexCoord + texOffset * vec2(-1, -1)).rgb;
+  vec3 bRight = texture(uSampler, vTexCoord + texOffset * vec2(1, 1)).rgb;
+  
+  // calculate the gradient edge of the current pixel using [3x3] sobel operator.
+  vec3 xEdge = tLeft + 2.0f * texture(uSampler, vTexCoord + texOffset * vec2(-1, 0)).rgb + bLeft - tRight - 2.0f * texture(uSampler, vTexCoord + texOffset * vec2(1, 0)).rgb - bRight;
+  vec3 yEdge = tLeft + 2.0f * texture(uSampler, vTexCoord + texOffset * vec2(0, 1)).rgb + tRight - bLeft - 2.0f * texture(uSampler, vTexCoord + texOffset * vec2(0, -1)).rgb - bRight;
+
+  // magnitude of the gradient at the current pixel.
+  result = sqrt(xEdge * xEdge + yEdge * yEdge);
+  
+  if (uMonocolor) {
+    // Convert result to a grayscale intensity
+    float intensity = length(result) / sqrt(3.0);
+    // Threshold to determine if the color should be white or black
+    float threshold = 0.2;
+    if (intensity > threshold) {
+      fragColor = uSwapColor ? vec4(1.0) : vec4(0.0, 0.0, 0.0, 1.0);
+    } else {
+      fragColor = uSwapColor ? vec4(0.0, 0.0, 0.0, 1.0) : vec4(1.0);
+    }
+  } else {
+    result = uSwapColor ? result : vec3(0.0, 1.0, 0.0) * result;
+    vec4 finalColor = vec4(result, 1.0f);
+    fragColor = finalColor;
+  }
+}
\ No newline at end of file

demo/frontend/src/common/components/video/effects/shaders/VibrantMask.frag

+#version 300 es
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+precision mediump float;
+precision mediump sampler3D;
+
+in vec2 vTexCoord;
+
+uniform sampler2D uSampler;
+uniform float uCurrentFrame;
+uniform sampler3D uColorGradeLUT;
+uniform int uNumMasks;
+uniform sampler2D uMaskTexture0;
+uniform sampler2D uMaskTexture1;
+uniform sampler2D uMaskTexture2;
+
+out vec4 fragColor;
+
+void main() {
+  vec4 color = texture(uSampler, vTexCoord);
+  vec3 gradedColor = texture(uColorGradeLUT, color.rgb).rgb;
+
+  vec4 color1 = vec4(0.0f);
+  vec4 color2 = vec4(0.0f);
+  vec4 color3 = vec4(0.0f);
+
+  // Apply edge detection for each mask
+  // We can't use dynamic indexing with samplers in GLSL ES 3.0.
+  // https://registry.khronos.org/OpenGL/specs/es/3.0/GLSL_ES_Specification_3.00.pdf Ch 12.30
+  if(uNumMasks > 0) {
+    color1 = texture(uMaskTexture0, vec2(vTexCoord.y, vTexCoord.x));
+  }
+  if(uNumMasks > 1) {
+    color2 = texture(uMaskTexture1, vec2(vTexCoord.y, vTexCoord.x));
+  }
+  if(uNumMasks > 2) {
+    color3 = texture(uMaskTexture2, vec2(vTexCoord.y, vTexCoord.x));
+  }
+
+  bool overlap = (color1.r > 0.0f || color2.r > 0.0f || color3.r > 0.0f);
+  if(overlap) {
+    fragColor = vec4(gradedColor, 1);
+  } else {
+    fragColor = vec4(0.0f);
+  }
+}
\ No newline at end of file

demo/frontend/src/common/components/video/filmstrip/FilmstripUtil.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {CanvasForm, CanvasSpace, Font, Group, Pt, Triangle} from 'pts';
+import SelectedFrameHelper from './SelectedFrameHelper';
+import {PADDING_BOTTOM, PADDING_TOP} from './VideoFilmstrip';
+
+export function getPointerPosition(
+  event: React.PointerEvent<HTMLCanvasElement>,
+) {
+  const rect = event.currentTarget.getBoundingClientRect();
+  return new Pt(event.clientX - rect.left, event.clientY - rect.top);
+}
+
+export function drawFilmstrip(
+  filmstrip: ImageBitmap | null,
+  space: CanvasSpace | undefined,
+  form: CanvasForm | undefined,
+) {
+  if (filmstrip == null || space == undefined || form?.ctx == undefined) {
+    return;
+  }
+
+  const ratio =
+    filmstrip.width / (filmstrip.height + PADDING_TOP + PADDING_BOTTOM);
+
+  form.image(
+    [
+      [0, PADDING_TOP],
+      [space.size.x, space.size.x / ratio],
+    ],
+    filmstrip,
+  );
+}
+
+export function getTimeFromFrame(frame: number, fps: number): string {
+  const seconds = Math.floor(frame / fps);
+  const frameRemaining = frame - fps * seconds;
+  return `${seconds}:${frameRemaining.toFixed().toString().padStart(2, '0')}`;
+}
+
+export function drawMarker(
+  space: CanvasSpace | undefined,
+  form: CanvasForm | undefined,
+  selectedFrameHelper: SelectedFrameHelper,
+  pointerPosition: Pt | null,
+  scanLabel: string | false,
+  fps: number,
+) {
+  if (space == undefined || form?.ctx == undefined) {
+    return;
+  }
+
+  const marker = Group.fromArray([
+    [0, PADDING_TOP],
+    [0, space.height - PADDING_BOTTOM],
+  ]);
+
+  const currentMarker = marker
+    .clone()
+    .add(Math.max(5, selectedFrameHelper.position), 0);
+
+  const getTextPosition = (label: string, marker: Group) => {
+    const textWidth = form.ctx.measureText(label).width;
+    return marker[0]
+      .$subtract(textWidth / 2, 0)
+      .$min(space.width - textWidth, PADDING_TOP - 10)
+      .$max(textWidth / 2 - 2, 0);
+  };
+
+  // draw current marker
+  form
+    .strokeOnly('#00000066', 5)
+    .line(currentMarker)
+    .strokeOnly('#fff', 1)
+    .line(currentMarker)
+    .fill('#000')
+    .polygon(
+      Triangle.fromCenter(currentMarker[0].$add(0, 10), 5).rotate2D(Math.PI),
+    );
+
+  // draw text
+  const frameLabel = getTimeFromFrame(selectedFrameHelper.index, fps);
+  form
+    .font(new Font(12, 'monospace'))
+    .fillOnly('#fff')
+    .text(getTextPosition(frameLabel, currentMarker), frameLabel);
+
+  // draw scanning ghost marker
+  if (
+    selectedFrameHelper.isScanning &&
+    pointerPosition != null &&
+    scanLabel != false
+  ) {
+    const scanMarker = marker.clone().add(pointerPosition.x, 0);
+    form.strokeOnly('#ffffff66', 5).line(scanMarker);
+
+    form
+      .font(new Font(12, 'monospace'))
+      .fillOnly('#8595A4')
+      .text(getTextPosition(scanLabel, scanMarker), scanLabel);
+  }
+}

demo/frontend/src/common/components/video/filmstrip/SelectedFrameHelper.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export default class SelectedFrameHelper {
+  private frames = 0;
+  private frameToWidthRatio = 1;
+  private selectedIndex = 0;
+  private scanning = false;
+
+  constructor(totalFrames: number, totalWidth: number, index?: number) {
+    this.reset(totalFrames, totalWidth, index);
+  }
+
+  reset(totalFrames: number, totalWidth: number, index?: number) {
+    this.frames = totalFrames;
+    this.frameToWidthRatio = totalWidth / this.frames;
+    if (index != null) {
+      this.select(index);
+    }
+  }
+
+  select(index: number) {
+    this.selectedIndex = index >= this.frames ? this.frames - index : index;
+  }
+
+  toPosition(index: number) {
+    return index * this.frameToWidthRatio;
+  }
+
+  toIndex(position: number) {
+    return Math.floor(position / this.frameToWidthRatio);
+  }
+
+  get index(): number {
+    return this.selectedIndex;
+  }
+
+  get position(): number {
+    return this.selectedIndex * this.frameToWidthRatio;
+  }
+
+  scan(state: boolean) {
+    this.scanning = state;
+  }
+
+  get isScanning(): boolean {
+    return this.scanning;
+  }
+}

demo/frontend/src/common/components/video/filmstrip/VideoFilmstrip.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import SelectedFrameHelper from '@/common/components/video/filmstrip/SelectedFrameHelper';
+import {isPlayingAtom} from '@/demo/atoms';
+import stylex from '@stylexjs/stylex';
+import {useAtomValue, useSetAtom} from 'jotai';
+import {CanvasSpace, Pt} from 'pts';
+import {useCallback, useEffect, useMemo, useRef} from 'react';
+import {PtsCanvas, PtsCanvasImperative} from 'react-pts-canvas';
+import {VideoRef} from '../Video';
+import {DecodeEvent, FrameUpdateEvent} from '../VideoWorkerBridge';
+import useVideo from '../editor/useVideo';
+import {
+  drawFilmstrip,
+  drawMarker,
+  getPointerPosition,
+  getTimeFromFrame,
+} from './FilmstripUtil';
+import {selectedFrameHelperAtom} from './atoms';
+import useDisableScrolling from './useDisableScrolling';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    flexDirection: 'column',
+  },
+  filmstripWrapper: {
+    position: 'relative',
+    width: '100%',
+    height: '5rem' /* 80px */,
+  },
+  filmstrip: {
+    position: 'absolute',
+    top: 0,
+    left: 0,
+    bottom: 0,
+    right: 0,
+    cursor: 'col-resize',
+    overflow: 'hidden',
+  },
+  canvas: {
+    width: '100%',
+    height: '100%',
+  },
+});
+
+export const PADDING_TOP = 30;
+export const PADDING_BOTTOM = 0;
+
+export default function VideoFilmstrip() {
+  const video = useVideo();
+  const ptsCanvasRef = useRef<PtsCanvasImperative | null>(null);
+  const filmstripRef = useRef<ImageBitmap | null>(null);
+  const isPlayingOnPointerDownRef = useRef<boolean>(false);
+  const isPlaying = useAtomValue(isPlayingAtom);
+
+  const {enable: enableScrolling, disable: disableScrolling} =
+    useDisableScrolling();
+
+  const pointerPositionRef = useRef<Pt | null>(null);
+  const animateRAFHandle = useRef<number | null>(null);
+
+  const selectedFrameHelper = useMemo(() => new SelectedFrameHelper(1, 1), []);
+  const setSelectedFrameHelper = useSetAtom(selectedFrameHelperAtom);
+
+  const fpsRef = useRef<number>(30);
+
+  useEffect(() => {
+    function onDecode(event: DecodeEvent) {
+      video?.removeEventListener('decode', onDecode);
+      fpsRef.current = event.fps;
+    }
+    video?.addEventListener('decode', onDecode);
+    return () => {
+      video?.removeEventListener('decode', onDecode);
+    };
+  }, [video]);
+
+  useEffect(() => {
+    setSelectedFrameHelper(selectedFrameHelper);
+  }, [setSelectedFrameHelper, selectedFrameHelper]);
+
+  const computeFrame = useCallback(
+    (normalizedPosition: number): {index: number} | null => {
+      if (video == null) {
+        return null;
+      }
+
+      const numFrames = video.numberOfFrames;
+      const index = Math.min(
+        Math.max(0, Math.floor(normalizedPosition * numFrames)),
+        numFrames - 1,
+      );
+      // The frame is needed for the CAE model. Do we still want to support it?
+      // return {image: decodedVideo.frames[index], index: index};
+      return {index};
+    },
+    [video],
+  );
+
+  const createFilmstrip = useCallback(
+    async (
+      video: VideoRef | null,
+      space: CanvasSpace | undefined,
+      frameIndex?: number,
+    ) => {
+      if (video === null || space == undefined) {
+        return;
+      }
+
+      const bitmap: ImageBitmap = await video?.createFilmstrip(
+        space.width,
+        space.height - (PADDING_TOP - PADDING_BOTTOM),
+      );
+
+      filmstripRef.current = bitmap;
+
+      selectedFrameHelper.reset(video.numberOfFrames, space.width, frameIndex); // also reset index to first frame
+
+      return bitmap;
+    },
+    [selectedFrameHelper],
+  );
+
+  // Custom animation handler
+  const handleRAF = useCallback(() => {
+    animateRAFHandle.current = null;
+    const space = ptsCanvasRef.current?.getSpace();
+    const form = ptsCanvasRef.current?.getForm();
+    if (space == undefined || form == undefined) {
+      return;
+    }
+
+    // Clear space, in particular clearing the frame index number of
+    // previous renders.
+    space.clear();
+
+    drawFilmstrip(filmstripRef.current, space, form);
+
+    const scanLabel =
+      selectedFrameHelper.isScanning &&
+      pointerPositionRef.current !== null &&
+      fpsRef.current !== null &&
+      getTimeFromFrame(
+        computeFrame(pointerPositionRef.current.x / space.width)?.index ?? 0,
+        fpsRef.current,
+      );
+
+    drawMarker(
+      space,
+      form,
+      selectedFrameHelper,
+      pointerPositionRef.current,
+      scanLabel,
+      fpsRef.current,
+    );
+  }, [computeFrame, selectedFrameHelper]);
+
+  const handleAnimate = useCallback(() => {
+    if (animateRAFHandle.current === null) {
+      animateRAFHandle.current = requestAnimationFrame(handleRAF);
+    }
+  }, [handleRAF]);
+
+  const handleFrameUpdate = useCallback(
+    (event: FrameUpdateEvent) => {
+      if (!selectedFrameHelper.isScanning) {
+        selectedFrameHelper.select(event.index);
+      }
+      handleAnimate();
+    },
+    [handleAnimate, selectedFrameHelper],
+  );
+
+  // Register a frame update listener on the video to update the filmstrip
+  // indicator when the video changes frames.
+  useEffect(() => {
+    video?.addEventListener('frameUpdate', handleFrameUpdate);
+    return () => {
+      video?.removeEventListener('frameUpdate', handleFrameUpdate);
+    };
+  }, [video, handleFrameUpdate]);
+
+  // Initiate filmstrip image
+  useEffect(() => {
+    const space = ptsCanvasRef.current?.getSpace();
+
+    async function onLoadStart() {
+      await createFilmstrip(video, space, 0);
+      handleAnimate();
+    }
+
+    async function progress() {
+      await createFilmstrip(video, space, 0);
+      handleAnimate();
+    }
+
+    void progress();
+
+    video?.addEventListener('loadstart', onLoadStart);
+    video?.addEventListener('decode', progress);
+
+    return () => {
+      video?.removeEventListener('loadstart', onLoadStart);
+      video?.removeEventListener('decode', progress);
+    };
+  }, [createFilmstrip, selectedFrameHelper, handleAnimate, video]);
+
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div {...stylex.props(styles.filmstripWrapper)}>
+        <div {...stylex.props(styles.filmstrip)}>
+          <PtsCanvas
+            {...stylex.props(styles.canvas)}
+            ref={ptsCanvasRef}
+            background="transparent"
+            resize={true}
+            refresh={false}
+            play={false}
+            onPtsResize={async space => {
+              if (video != null && space != undefined) {
+                selectedFrameHelper.reset(video.numberOfFrames, space.width);
+              }
+              if (video !== null) {
+                await createFilmstrip(video, space);
+              }
+              handleAnimate();
+            }}
+            onPointerDown={event => {
+              const canvas = ptsCanvasRef.current?.getCanvas();
+              canvas?.setPointerCapture(event.pointerId);
+
+              // Disable page scrolling while interacting with the filmstrip
+              disableScrolling();
+
+              pointerPositionRef.current = getPointerPosition(event);
+              selectedFrameHelper.scan(true);
+
+              // Pause the video when a user initially has their pointer down.
+              // Playback will resume once the onPointerUp event is triggered.
+              isPlayingOnPointerDownRef.current = isPlaying;
+              if (isPlaying) {
+                video?.pause();
+              }
+            }}
+            onPointerUp={event => {
+              // Enable page scrolling after interaction with filmstrip is done
+              enableScrolling();
+
+              const space = ptsCanvasRef.current?.getSpace();
+              if (space != undefined) {
+                pointerPositionRef.current = getPointerPosition(event);
+
+                selectedFrameHelper.scan(false);
+                const frame = computeFrame(
+                  pointerPositionRef.current.x / space.size.x,
+                );
+                if (
+                  frame != null &&
+                  selectedFrameHelper.index !== frame.index
+                ) {
+                  selectedFrameHelper.select(frame.index);
+                  if (video !== null) {
+                    video.frame = frame.index;
+                    if (isPlayingOnPointerDownRef.current) {
+                      video.play();
+                    }
+                  }
+                }
+                handleAnimate();
+              }
+
+              pointerPositionRef.current = null;
+            }}
+            onPointerMove={event => {
+              if (
+                !selectedFrameHelper.isScanning ||
+                pointerPositionRef.current === null
+              ) {
+                return;
+              }
+
+              const space = ptsCanvasRef.current?.getSpace();
+              const form = ptsCanvasRef.current?.getForm();
+              if (
+                selectedFrameHelper.isScanning &&
+                space != null &&
+                form != null
+              ) {
+                pointerPositionRef.current = getPointerPosition(event);
+
+                const frame = computeFrame(
+                  pointerPositionRef.current.x / space.size.x,
+                );
+                if (frame != null) {
+                  handleAnimate();
+                  if (video !== null) {
+                    video.frame = frame.index;
+                  }
+                }
+              }
+            }}
+          />
+        </div>
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/video/filmstrip/atoms.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {atom} from 'jotai';
+import SelectedFrameHelper from './SelectedFrameHelper';
+
+export const selectedFrameHelperAtom = atom<SelectedFrameHelper | null>(null);

demo/frontend/src/common/components/video/filmstrip/useDisableScrolling.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {useCallback, useEffect, useRef} from 'react';
+
+function preventDefault(event: TouchEvent) {
+  event.preventDefault();
+}
+
+export default function useDisableScrolling() {
+  const isDisabledRef = useRef<boolean>(false);
+
+  const disable = useCallback(() => {
+    // Scrolling is already disabled
+    if (isDisabledRef.current) {
+      return;
+    }
+    isDisabledRef.current = true;
+    document.body.addEventListener('touchmove', preventDefault, {
+      passive: false,
+    });
+  }, []);
+
+  const enable = useCallback(() => {
+    // Scrolling is not disabled
+    if (!isDisabledRef.current) {
+      return;
+    }
+    isDisabledRef.current = false;
+    document.body.removeEventListener('touchmove', preventDefault);
+  }, []);
+
+  useEffect(() => {
+    // Enable scrolling again on unmount
+    return () => {
+      enable();
+    };
+  }, [enable]);
+
+  return {
+    disable,
+    enable,
+  };
+}

demo/frontend/src/common/components/video/filmstrip/useSelectedFrameHelper.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {useAtomValue} from 'jotai';
+import {selectedFrameHelperAtom} from './atoms';
+
+export default function useSelectedFrameHelper() {
+  return useAtomValue(selectedFrameHelperAtom);
+}

demo/frontend/src/common/components/video/layers/InteractionLayer.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useVideo from '@/common/components/video/editor/useVideo';
+import {getPointInImage} from '@/common/components/video/editor/VideoEditorUtils';
+import {SegmentationPoint} from '@/common/tracker/Tracker';
+import {labelTypeAtom} from '@/demo/atoms';
+import stylex from '@stylexjs/stylex';
+import {useAtomValue} from 'jotai';
+import {MouseEvent} from 'react';
+
+const styles = stylex.create({
+  container: {
+    position: 'absolute',
+    left: 0,
+    top: 0,
+    right: 0,
+    bottom: 0,
+  },
+});
+
+type Props = {
+  onPoint: (point: SegmentationPoint) => void;
+};
+
+export default function InteractionLayer({onPoint}: Props) {
+  const video = useVideo();
+  // Use labelType to swap positive and negative points. The most important use
+  // case is the switch between positive and negative label for left mouse
+  // clicks.
+  const labelType = useAtomValue(labelTypeAtom);
+
+  return (
+    <div
+      {...stylex.props(styles.container)}
+      onClick={(event: MouseEvent<HTMLDivElement>) => {
+        const canvas = video?.getCanvas();
+        if (canvas != null) {
+          const point = getPointInImage(event, canvas);
+          onPoint([...point, labelType === 'positive' ? 1 : 0]);
+        }
+      }}
+      onContextMenu={event => {
+        event.preventDefault();
+        const canvas = video?.getCanvas();
+        if (canvas != null) {
+          const point = getPointInImage(event, canvas);
+          onPoint([...point, labelType === 'positive' ? 0 : 1]);
+        }
+      }}
+    />
+  );
+}

demo/frontend/src/common/components/video/layers/PointsLayer.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {SegmentationPoint} from '@/common/tracker/Tracker';
+import stylex from '@stylexjs/stylex';
+import {useMemo} from 'react';
+import useResizeObserver from 'use-resize-observer';
+import useVideo from '../editor/useVideo';
+
+const styles = stylex.create({
+  container: {
+    position: 'absolute',
+    width: '100%',
+    height: '100%',
+    pointerEvents: 'none',
+  },
+});
+
+type Props = {
+  points: SegmentationPoint[];
+  onRemovePoint: (point: SegmentationPoint) => void;
+};
+
+export function PointsLayer({points, onRemovePoint}: Props) {
+  const video = useVideo();
+  const videoCanvas = useMemo(() => video?.getCanvas(), [video]);
+
+  const {
+    ref,
+    width: containerWidth = 1,
+    height: containerHeight = 1,
+  } = useResizeObserver<SVGElement>();
+
+  const canvasWidth = videoCanvas?.width ?? 1;
+  const canvasHeight = videoCanvas?.height ?? 1;
+
+  const sizeMultiplier = useMemo(() => {
+    const widthMultiplier = canvasWidth / containerWidth;
+    const heightMultiplier = canvasHeight / containerHeight;
+
+    return Math.max(widthMultiplier, heightMultiplier);
+  }, [canvasWidth, canvasHeight, containerWidth, containerHeight]);
+
+  const pointRadius = useMemo(() => 8 * sizeMultiplier, [sizeMultiplier]);
+  const pointStroke = useMemo(() => 2 * sizeMultiplier, [sizeMultiplier]);
+
+  return (
+    <svg
+      ref={ref}
+      {...stylex.props(styles.container)}
+      xmlns="http://www.w3.org/2000/svg"
+      viewBox={`0 0 ${canvasWidth} ${canvasHeight}`}>
+      {/*
+       * This is a debug element to verify the SVG element overlays
+       * perfectly with the canvas element.
+       */}
+      {/*
+      <rect
+        fill="rgba(255, 255, 0, 0.5)"
+        width={decodedVideo?.width}
+        height={decodedVideo?.height}
+      />
+      */}
+      {/* Render points */}
+      {points.map((point, idx) => {
+        const isAdd = point[2] === 1;
+        return (
+          <g key={idx} className="cursor-pointer">
+            <circle
+              className="stroke-white hover:stroke-gray-400"
+              pointerEvents="visiblePainted"
+              cx={point[0]}
+              cy={point[1]}
+              r={pointRadius}
+              fill={isAdd ? '#000000' : '#E6193B'}
+              strokeWidth={pointStroke}
+              onClick={event => {
+                event.stopPropagation();
+                onRemovePoint(point);
+              }}
+            />
+            <line
+              x1={point[0] - pointRadius / 2}
+              y1={point[1]}
+              x2={point[0] + pointRadius / 2}
+              y2={point[1]}
+              strokeWidth={pointStroke}
+              stroke="white"
+            />
+            {isAdd && (
+              <line
+                x1={point[0]}
+                y1={point[1] - pointRadius / 2}
+                x2={point[0]}
+                y2={point[1] + pointRadius / 2}
+                strokeWidth={pointStroke}
+                stroke="white"
+              />
+            )}
+          </g>
+        );
+      })}
+    </svg>
+  );
+}

demo/frontend/src/common/components/video/useInputVideo.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {inputVideoAtom} from '@/demo/atoms';
+import {useAtom} from 'jotai';
+
+export default function useInputVideo() {
+  const [inputVideo, setInputVideo] = useAtom(inputVideoAtom);
+  return {inputVideo, setInputVideo};
+}