"Initial commit"

demo/frontend/src/common/components/video/effects/shaders/Sobel.frag

+#version 300 es
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+precision mediump float;
+
+in vec2 vTexCoord;
+
+uniform sampler2D uSampler;
+uniform vec2 uSize;
+uniform bool uSwapColor;
+uniform bool uMonocolor;
+
+out vec4 fragColor;
+
+void main() {
+  // calculate the offset for one pixel in texture coordinates
+  vec2 texOffset = 1.0f / uSize;
+  vec3 result = vec3(0.0f);
+  // neighboring pixels
+  vec3 tLeft = texture(uSampler, vTexCoord + texOffset * vec2(-1, 1)).rgb;
+  vec3 tRight = texture(uSampler, vTexCoord + texOffset * vec2(1, -1)).rgb;
+  vec3 bLeft = texture(uSampler, vTexCoord + texOffset * vec2(-1, -1)).rgb;
+  vec3 bRight = texture(uSampler, vTexCoord + texOffset * vec2(1, 1)).rgb;
+  
+  // calculate the gradient edge of the current pixel using [3x3] sobel operator.
+  vec3 xEdge = tLeft + 2.0f * texture(uSampler, vTexCoord + texOffset * vec2(-1, 0)).rgb + bLeft - tRight - 2.0f * texture(uSampler, vTexCoord + texOffset * vec2(1, 0)).rgb - bRight;
+  vec3 yEdge = tLeft + 2.0f * texture(uSampler, vTexCoord + texOffset * vec2(0, 1)).rgb + tRight - bLeft - 2.0f * texture(uSampler, vTexCoord + texOffset * vec2(0, -1)).rgb - bRight;
+
+  // magnitude of the gradient at the current pixel.
+  result = sqrt(xEdge * xEdge + yEdge * yEdge);
+  
+  if (uMonocolor) {
+    // Convert result to a grayscale intensity
+    float intensity = length(result) / sqrt(3.0);
+    // Threshold to determine if the color should be white or black
+    float threshold = 0.2;
+    if (intensity > threshold) {
+      fragColor = uSwapColor ? vec4(1.0) : vec4(0.0, 0.0, 0.0, 1.0);
+    } else {
+      fragColor = uSwapColor ? vec4(0.0, 0.0, 0.0, 1.0) : vec4(1.0);
+    }
+  } else {
+    result = uSwapColor ? result : vec3(0.0, 1.0, 0.0) * result;
+    vec4 finalColor = vec4(result, 1.0f);
+    fragColor = finalColor;
+  }
+}
\ No newline at end of file

demo/frontend/src/common/components/video/effects/shaders/VibrantMask.frag

+#version 300 es
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+precision mediump float;
+precision mediump sampler3D;
+
+in vec2 vTexCoord;
+
+uniform sampler2D uSampler;
+uniform float uCurrentFrame;
+uniform sampler3D uColorGradeLUT;
+uniform int uNumMasks;
+uniform sampler2D uMaskTexture0;
+uniform sampler2D uMaskTexture1;
+uniform sampler2D uMaskTexture2;
+
+out vec4 fragColor;
+
+void main() {
+  vec4 color = texture(uSampler, vTexCoord);
+  vec3 gradedColor = texture(uColorGradeLUT, color.rgb).rgb;
+
+  vec4 color1 = vec4(0.0f);
+  vec4 color2 = vec4(0.0f);
+  vec4 color3 = vec4(0.0f);
+
+  // Apply edge detection for each mask
+  // We can't use dynamic indexing with samplers in GLSL ES 3.0.
+  // https://registry.khronos.org/OpenGL/specs/es/3.0/GLSL_ES_Specification_3.00.pdf Ch 12.30
+  if(uNumMasks > 0) {
+    color1 = texture(uMaskTexture0, vec2(vTexCoord.y, vTexCoord.x));
+  }
+  if(uNumMasks > 1) {
+    color2 = texture(uMaskTexture1, vec2(vTexCoord.y, vTexCoord.x));
+  }
+  if(uNumMasks > 2) {
+    color3 = texture(uMaskTexture2, vec2(vTexCoord.y, vTexCoord.x));
+  }
+
+  bool overlap = (color1.r > 0.0f || color2.r > 0.0f || color3.r > 0.0f);
+  if(overlap) {
+    fragColor = vec4(gradedColor, 1);
+  } else {
+    fragColor = vec4(0.0f);
+  }
+}
\ No newline at end of file

demo/frontend/src/common/components/video/filmstrip/FilmstripUtil.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {CanvasForm, CanvasSpace, Font, Group, Pt, Triangle} from 'pts';
+import SelectedFrameHelper from './SelectedFrameHelper';
+import {PADDING_BOTTOM, PADDING_TOP} from './VideoFilmstrip';
+
+export function getPointerPosition(
+  event: React.PointerEvent<HTMLCanvasElement>,
+) {
+  const rect = event.currentTarget.getBoundingClientRect();
+  return new Pt(event.clientX - rect.left, event.clientY - rect.top);
+}
+
+export function drawFilmstrip(
+  filmstrip: ImageBitmap | null,
+  space: CanvasSpace | undefined,
+  form: CanvasForm | undefined,
+) {
+  if (filmstrip == null || space == undefined || form?.ctx == undefined) {
+    return;
+  }
+
+  const ratio =
+    filmstrip.width / (filmstrip.height + PADDING_TOP + PADDING_BOTTOM);
+
+  form.image(
+    [
+      [0, PADDING_TOP],
+      [space.size.x, space.size.x / ratio],
+    ],
+    filmstrip,
+  );
+}
+
+export function getTimeFromFrame(frame: number, fps: number): string {
+  const seconds = Math.floor(frame / fps);
+  const frameRemaining = frame - fps * seconds;
+  return `${seconds}:${frameRemaining.toFixed().toString().padStart(2, '0')}`;
+}
+
+export function drawMarker(
+  space: CanvasSpace | undefined,
+  form: CanvasForm | undefined,
+  selectedFrameHelper: SelectedFrameHelper,
+  pointerPosition: Pt | null,
+  scanLabel: string | false,
+  fps: number,
+) {
+  if (space == undefined || form?.ctx == undefined) {
+    return;
+  }
+
+  const marker = Group.fromArray([
+    [0, PADDING_TOP],
+    [0, space.height - PADDING_BOTTOM],
+  ]);
+
+  const currentMarker = marker
+    .clone()
+    .add(Math.max(5, selectedFrameHelper.position), 0);
+
+  const getTextPosition = (label: string, marker: Group) => {
+    const textWidth = form.ctx.measureText(label).width;
+    return marker[0]
+      .$subtract(textWidth / 2, 0)
+      .$min(space.width - textWidth, PADDING_TOP - 10)
+      .$max(textWidth / 2 - 2, 0);
+  };
+
+  // draw current marker
+  form
+    .strokeOnly('#00000066', 5)
+    .line(currentMarker)
+    .strokeOnly('#fff', 1)
+    .line(currentMarker)
+    .fill('#000')
+    .polygon(
+      Triangle.fromCenter(currentMarker[0].$add(0, 10), 5).rotate2D(Math.PI),
+    );
+
+  // draw text
+  const frameLabel = getTimeFromFrame(selectedFrameHelper.index, fps);
+  form
+    .font(new Font(12, 'monospace'))
+    .fillOnly('#fff')
+    .text(getTextPosition(frameLabel, currentMarker), frameLabel);
+
+  // draw scanning ghost marker
+  if (
+    selectedFrameHelper.isScanning &&
+    pointerPosition != null &&
+    scanLabel != false
+  ) {
+    const scanMarker = marker.clone().add(pointerPosition.x, 0);
+    form.strokeOnly('#ffffff66', 5).line(scanMarker);
+
+    form
+      .font(new Font(12, 'monospace'))
+      .fillOnly('#8595A4')
+      .text(getTextPosition(scanLabel, scanMarker), scanLabel);
+  }
+}

demo/frontend/src/common/components/video/filmstrip/SelectedFrameHelper.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export default class SelectedFrameHelper {
+  private frames = 0;
+  private frameToWidthRatio = 1;
+  private selectedIndex = 0;
+  private scanning = false;
+
+  constructor(totalFrames: number, totalWidth: number, index?: number) {
+    this.reset(totalFrames, totalWidth, index);
+  }
+
+  reset(totalFrames: number, totalWidth: number, index?: number) {
+    this.frames = totalFrames;
+    this.frameToWidthRatio = totalWidth / this.frames;
+    if (index != null) {
+      this.select(index);
+    }
+  }
+
+  select(index: number) {
+    this.selectedIndex = index >= this.frames ? this.frames - index : index;
+  }
+
+  toPosition(index: number) {
+    return index * this.frameToWidthRatio;
+  }
+
+  toIndex(position: number) {
+    return Math.floor(position / this.frameToWidthRatio);
+  }
+
+  get index(): number {
+    return this.selectedIndex;
+  }
+
+  get position(): number {
+    return this.selectedIndex * this.frameToWidthRatio;
+  }
+
+  scan(state: boolean) {
+    this.scanning = state;
+  }
+
+  get isScanning(): boolean {
+    return this.scanning;
+  }
+}

demo/frontend/src/common/components/video/filmstrip/VideoFilmstrip.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import SelectedFrameHelper from '@/common/components/video/filmstrip/SelectedFrameHelper';
+import {isPlayingAtom} from '@/demo/atoms';
+import stylex from '@stylexjs/stylex';
+import {useAtomValue, useSetAtom} from 'jotai';
+import {CanvasSpace, Pt} from 'pts';
+import {useCallback, useEffect, useMemo, useRef} from 'react';
+import {PtsCanvas, PtsCanvasImperative} from 'react-pts-canvas';
+import {VideoRef} from '../Video';
+import {DecodeEvent, FrameUpdateEvent} from '../VideoWorkerBridge';
+import useVideo from '../editor/useVideo';
+import {
+  drawFilmstrip,
+  drawMarker,
+  getPointerPosition,
+  getTimeFromFrame,
+} from './FilmstripUtil';
+import {selectedFrameHelperAtom} from './atoms';
+import useDisableScrolling from './useDisableScrolling';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    flexDirection: 'column',
+  },
+  filmstripWrapper: {
+    position: 'relative',
+    width: '100%',
+    height: '5rem' /* 80px */,
+  },
+  filmstrip: {
+    position: 'absolute',
+    top: 0,
+    left: 0,
+    bottom: 0,
+    right: 0,
+    cursor: 'col-resize',
+    overflow: 'hidden',
+  },
+  canvas: {
+    width: '100%',
+    height: '100%',
+  },
+});
+
+export const PADDING_TOP = 30;
+export const PADDING_BOTTOM = 0;
+
+export default function VideoFilmstrip() {
+  const video = useVideo();
+  const ptsCanvasRef = useRef<PtsCanvasImperative | null>(null);
+  const filmstripRef = useRef<ImageBitmap | null>(null);
+  const isPlayingOnPointerDownRef = useRef<boolean>(false);
+  const isPlaying = useAtomValue(isPlayingAtom);
+
+  const {enable: enableScrolling, disable: disableScrolling} =
+    useDisableScrolling();
+
+  const pointerPositionRef = useRef<Pt | null>(null);
+  const animateRAFHandle = useRef<number | null>(null);
+
+  const selectedFrameHelper = useMemo(() => new SelectedFrameHelper(1, 1), []);
+  const setSelectedFrameHelper = useSetAtom(selectedFrameHelperAtom);
+
+  const fpsRef = useRef<number>(30);
+
+  useEffect(() => {
+    function onDecode(event: DecodeEvent) {
+      video?.removeEventListener('decode', onDecode);
+      fpsRef.current = event.fps;
+    }
+    video?.addEventListener('decode', onDecode);
+    return () => {
+      video?.removeEventListener('decode', onDecode);
+    };
+  }, [video]);
+
+  useEffect(() => {
+    setSelectedFrameHelper(selectedFrameHelper);
+  }, [setSelectedFrameHelper, selectedFrameHelper]);
+
+  const computeFrame = useCallback(
+    (normalizedPosition: number): {index: number} | null => {
+      if (video == null) {
+        return null;
+      }
+
+      const numFrames = video.numberOfFrames;
+      const index = Math.min(
+        Math.max(0, Math.floor(normalizedPosition * numFrames)),
+        numFrames - 1,
+      );
+      // The frame is needed for the CAE model. Do we still want to support it?
+      // return {image: decodedVideo.frames[index], index: index};
+      return {index};
+    },
+    [video],
+  );
+
+  const createFilmstrip = useCallback(
+    async (
+      video: VideoRef | null,
+      space: CanvasSpace | undefined,
+      frameIndex?: number,
+    ) => {
+      if (video === null || space == undefined) {
+        return;
+      }
+
+      const bitmap: ImageBitmap = await video?.createFilmstrip(
+        space.width,
+        space.height - (PADDING_TOP - PADDING_BOTTOM),
+      );
+
+      filmstripRef.current = bitmap;
+
+      selectedFrameHelper.reset(video.numberOfFrames, space.width, frameIndex); // also reset index to first frame
+
+      return bitmap;
+    },
+    [selectedFrameHelper],
+  );
+
+  // Custom animation handler
+  const handleRAF = useCallback(() => {
+    animateRAFHandle.current = null;
+    const space = ptsCanvasRef.current?.getSpace();
+    const form = ptsCanvasRef.current?.getForm();
+    if (space == undefined || form == undefined) {
+      return;
+    }
+
+    // Clear space, in particular clearing the frame index number of
+    // previous renders.
+    space.clear();
+
+    drawFilmstrip(filmstripRef.current, space, form);
+
+    const scanLabel =
+      selectedFrameHelper.isScanning &&
+      pointerPositionRef.current !== null &&
+      fpsRef.current !== null &&
+      getTimeFromFrame(
+        computeFrame(pointerPositionRef.current.x / space.width)?.index ?? 0,
+        fpsRef.current,
+      );
+
+    drawMarker(
+      space,
+      form,
+      selectedFrameHelper,
+      pointerPositionRef.current,
+      scanLabel,
+      fpsRef.current,
+    );
+  }, [computeFrame, selectedFrameHelper]);
+
+  const handleAnimate = useCallback(() => {
+    if (animateRAFHandle.current === null) {
+      animateRAFHandle.current = requestAnimationFrame(handleRAF);
+    }
+  }, [handleRAF]);
+
+  const handleFrameUpdate = useCallback(
+    (event: FrameUpdateEvent) => {
+      if (!selectedFrameHelper.isScanning) {
+        selectedFrameHelper.select(event.index);
+      }
+      handleAnimate();
+    },
+    [handleAnimate, selectedFrameHelper],
+  );
+
+  // Register a frame update listener on the video to update the filmstrip
+  // indicator when the video changes frames.
+  useEffect(() => {
+    video?.addEventListener('frameUpdate', handleFrameUpdate);
+    return () => {
+      video?.removeEventListener('frameUpdate', handleFrameUpdate);
+    };
+  }, [video, handleFrameUpdate]);
+
+  // Initiate filmstrip image
+  useEffect(() => {
+    const space = ptsCanvasRef.current?.getSpace();
+
+    async function onLoadStart() {
+      await createFilmstrip(video, space, 0);
+      handleAnimate();
+    }
+
+    async function progress() {
+      await createFilmstrip(video, space, 0);
+      handleAnimate();
+    }
+
+    void progress();
+
+    video?.addEventListener('loadstart', onLoadStart);
+    video?.addEventListener('decode', progress);
+
+    return () => {
+      video?.removeEventListener('loadstart', onLoadStart);
+      video?.removeEventListener('decode', progress);
+    };
+  }, [createFilmstrip, selectedFrameHelper, handleAnimate, video]);
+
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div {...stylex.props(styles.filmstripWrapper)}>
+        <div {...stylex.props(styles.filmstrip)}>
+          <PtsCanvas
+            {...stylex.props(styles.canvas)}
+            ref={ptsCanvasRef}
+            background="transparent"
+            resize={true}
+            refresh={false}
+            play={false}
+            onPtsResize={async space => {
+              if (video != null && space != undefined) {
+                selectedFrameHelper.reset(video.numberOfFrames, space.width);
+              }
+              if (video !== null) {
+                await createFilmstrip(video, space);
+              }
+              handleAnimate();
+            }}
+            onPointerDown={event => {
+              const canvas = ptsCanvasRef.current?.getCanvas();
+              canvas?.setPointerCapture(event.pointerId);
+
+              // Disable page scrolling while interacting with the filmstrip
+              disableScrolling();
+
+              pointerPositionRef.current = getPointerPosition(event);
+              selectedFrameHelper.scan(true);
+
+              // Pause the video when a user initially has their pointer down.
+              // Playback will resume once the onPointerUp event is triggered.
+              isPlayingOnPointerDownRef.current = isPlaying;
+              if (isPlaying) {
+                video?.pause();
+              }
+            }}
+            onPointerUp={event => {
+              // Enable page scrolling after interaction with filmstrip is done
+              enableScrolling();
+
+              const space = ptsCanvasRef.current?.getSpace();
+              if (space != undefined) {
+                pointerPositionRef.current = getPointerPosition(event);
+
+                selectedFrameHelper.scan(false);
+                const frame = computeFrame(
+                  pointerPositionRef.current.x / space.size.x,
+                );
+                if (
+                  frame != null &&
+                  selectedFrameHelper.index !== frame.index
+                ) {
+                  selectedFrameHelper.select(frame.index);
+                  if (video !== null) {
+                    video.frame = frame.index;
+                    if (isPlayingOnPointerDownRef.current) {
+                      video.play();
+                    }
+                  }
+                }
+                handleAnimate();
+              }
+
+              pointerPositionRef.current = null;
+            }}
+            onPointerMove={event => {
+              if (
+                !selectedFrameHelper.isScanning ||
+                pointerPositionRef.current === null
+              ) {
+                return;
+              }
+
+              const space = ptsCanvasRef.current?.getSpace();
+              const form = ptsCanvasRef.current?.getForm();
+              if (
+                selectedFrameHelper.isScanning &&
+                space != null &&
+                form != null
+              ) {
+                pointerPositionRef.current = getPointerPosition(event);
+
+                const frame = computeFrame(
+                  pointerPositionRef.current.x / space.size.x,
+                );
+                if (frame != null) {
+                  handleAnimate();
+                  if (video !== null) {
+                    video.frame = frame.index;
+                  }
+                }
+              }
+            }}
+          />
+        </div>
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/video/filmstrip/atoms.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {atom} from 'jotai';
+import SelectedFrameHelper from './SelectedFrameHelper';
+
+export const selectedFrameHelperAtom = atom<SelectedFrameHelper | null>(null);

demo/frontend/src/common/components/video/filmstrip/useDisableScrolling.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {useCallback, useEffect, useRef} from 'react';
+
+function preventDefault(event: TouchEvent) {
+  event.preventDefault();
+}
+
+export default function useDisableScrolling() {
+  const isDisabledRef = useRef<boolean>(false);
+
+  const disable = useCallback(() => {
+    // Scrolling is already disabled
+    if (isDisabledRef.current) {
+      return;
+    }
+    isDisabledRef.current = true;
+    document.body.addEventListener('touchmove', preventDefault, {
+      passive: false,
+    });
+  }, []);
+
+  const enable = useCallback(() => {
+    // Scrolling is not disabled
+    if (!isDisabledRef.current) {
+      return;
+    }
+    isDisabledRef.current = false;
+    document.body.removeEventListener('touchmove', preventDefault);
+  }, []);
+
+  useEffect(() => {
+    // Enable scrolling again on unmount
+    return () => {
+      enable();
+    };
+  }, [enable]);
+
+  return {
+    disable,
+    enable,
+  };
+}

demo/frontend/src/common/components/video/filmstrip/useSelectedFrameHelper.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {useAtomValue} from 'jotai';
+import {selectedFrameHelperAtom} from './atoms';
+
+export default function useSelectedFrameHelper() {
+  return useAtomValue(selectedFrameHelperAtom);
+}

demo/frontend/src/common/components/video/layers/InteractionLayer.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useVideo from '@/common/components/video/editor/useVideo';
+import {getPointInImage} from '@/common/components/video/editor/VideoEditorUtils';
+import {SegmentationPoint} from '@/common/tracker/Tracker';
+import {labelTypeAtom} from '@/demo/atoms';
+import stylex from '@stylexjs/stylex';
+import {useAtomValue} from 'jotai';
+import {MouseEvent} from 'react';
+
+const styles = stylex.create({
+  container: {
+    position: 'absolute',
+    left: 0,
+    top: 0,
+    right: 0,
+    bottom: 0,
+  },
+});
+
+type Props = {
+  onPoint: (point: SegmentationPoint) => void;
+};
+
+export default function InteractionLayer({onPoint}: Props) {
+  const video = useVideo();
+  // Use labelType to swap positive and negative points. The most important use
+  // case is the switch between positive and negative label for left mouse
+  // clicks.
+  const labelType = useAtomValue(labelTypeAtom);
+
+  return (
+    <div
+      {...stylex.props(styles.container)}
+      onClick={(event: MouseEvent<HTMLDivElement>) => {
+        const canvas = video?.getCanvas();
+        if (canvas != null) {
+          const point = getPointInImage(event, canvas);
+          onPoint([...point, labelType === 'positive' ? 1 : 0]);
+        }
+      }}
+      onContextMenu={event => {
+        event.preventDefault();
+        const canvas = video?.getCanvas();
+        if (canvas != null) {
+          const point = getPointInImage(event, canvas);
+          onPoint([...point, labelType === 'positive' ? 0 : 1]);
+        }
+      }}
+    />
+  );
+}

demo/frontend/src/common/components/video/layers/PointsLayer.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {SegmentationPoint} from '@/common/tracker/Tracker';
+import stylex from '@stylexjs/stylex';
+import {useMemo} from 'react';
+import useResizeObserver from 'use-resize-observer';
+import useVideo from '../editor/useVideo';
+
+const styles = stylex.create({
+  container: {
+    position: 'absolute',
+    width: '100%',
+    height: '100%',
+    pointerEvents: 'none',
+  },
+});
+
+type Props = {
+  points: SegmentationPoint[];
+  onRemovePoint: (point: SegmentationPoint) => void;
+};
+
+export function PointsLayer({points, onRemovePoint}: Props) {
+  const video = useVideo();
+  const videoCanvas = useMemo(() => video?.getCanvas(), [video]);
+
+  const {
+    ref,
+    width: containerWidth = 1,
+    height: containerHeight = 1,
+  } = useResizeObserver<SVGElement>();
+
+  const canvasWidth = videoCanvas?.width ?? 1;
+  const canvasHeight = videoCanvas?.height ?? 1;
+
+  const sizeMultiplier = useMemo(() => {
+    const widthMultiplier = canvasWidth / containerWidth;
+    const heightMultiplier = canvasHeight / containerHeight;
+
+    return Math.max(widthMultiplier, heightMultiplier);
+  }, [canvasWidth, canvasHeight, containerWidth, containerHeight]);
+
+  const pointRadius = useMemo(() => 8 * sizeMultiplier, [sizeMultiplier]);
+  const pointStroke = useMemo(() => 2 * sizeMultiplier, [sizeMultiplier]);
+
+  return (
+    <svg
+      ref={ref}
+      {...stylex.props(styles.container)}
+      xmlns="http://www.w3.org/2000/svg"
+      viewBox={`0 0 ${canvasWidth} ${canvasHeight}`}>
+      {/*
+       * This is a debug element to verify the SVG element overlays
+       * perfectly with the canvas element.
+       */}
+      {/*
+      <rect
+        fill="rgba(255, 255, 0, 0.5)"
+        width={decodedVideo?.width}
+        height={decodedVideo?.height}
+      />
+      */}
+      {/* Render points */}
+      {points.map((point, idx) => {
+        const isAdd = point[2] === 1;
+        return (
+          <g key={idx} className="cursor-pointer">
+            <circle
+              className="stroke-white hover:stroke-gray-400"
+              pointerEvents="visiblePainted"
+              cx={point[0]}
+              cy={point[1]}
+              r={pointRadius}
+              fill={isAdd ? '#000000' : '#E6193B'}
+              strokeWidth={pointStroke}
+              onClick={event => {
+                event.stopPropagation();
+                onRemovePoint(point);
+              }}
+            />
+            <line
+              x1={point[0] - pointRadius / 2}
+              y1={point[1]}
+              x2={point[0] + pointRadius / 2}
+              y2={point[1]}
+              strokeWidth={pointStroke}
+              stroke="white"
+            />
+            {isAdd && (
+              <line
+                x1={point[0]}
+                y1={point[1] - pointRadius / 2}
+                x2={point[0]}
+                y2={point[1] + pointRadius / 2}
+                strokeWidth={pointStroke}
+                stroke="white"
+              />
+            )}
+          </g>
+        );
+      })}
+    </svg>
+  );
+}

demo/frontend/src/common/components/video/useInputVideo.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {inputVideoAtom} from '@/demo/atoms';
+import {useAtom} from 'jotai';
+
+export default function useInputVideo() {
+  const [inputVideo, setInputVideo] = useAtom(inputVideoAtom);
+  return {inputVideo, setInputVideo};
+}

demo/frontend/src/common/components/video/useVideoWorker.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {RefObject, useEffect, useMemo, useRef} from 'react';
+import VideoWorkerBridge from './VideoWorkerBridge';
+
+type Options = {
+  createVideoWorker?: () => Worker;
+  createWorkerBridge?: CreateWorkerBridgeFunction;
+};
+
+const DEFAULT_OPTIONS: Options = {
+  createVideoWorker: () =>
+    new Worker(new URL('./VideoWorker', import.meta.url), {
+      type: 'module',
+    }),
+};
+
+type WorkerFactory = () => Worker;
+
+type CreateWorkerBridgeFunction = (
+  workerFactory: WorkerFactory,
+) => VideoWorkerBridge;
+
+export default function useVideoWorker(
+  src: string,
+  canvasRef: RefObject<HTMLCanvasElement>,
+  options: Options = {},
+) {
+  const isControlTransferredToOffscreenRef = useRef(false);
+
+  const mergedOptions = useMemo(() => {
+    const definedProps = (o: Options) =>
+      Object.fromEntries(
+        Object.entries(o).filter(([_k, v]) => v !== undefined),
+      );
+    return Object.assign(
+      DEFAULT_OPTIONS,
+      definedProps(options),
+    ) as Required<Options>;
+  }, [options]);
+
+  const worker = useMemo(() => {
+    if (mergedOptions.createWorkerBridge) {
+      return mergedOptions.createWorkerBridge(mergedOptions.createVideoWorker);
+    }
+    return VideoWorkerBridge.create(mergedOptions.createVideoWorker);
+  }, [mergedOptions]);
+
+  useEffect(() => {
+    const canvas = canvasRef.current;
+
+    if (canvas == null) {
+      return;
+    }
+
+    if (isControlTransferredToOffscreenRef.current) {
+      return;
+    }
+
+    isControlTransferredToOffscreenRef.current = true;
+
+    worker.setCanvas(canvas);
+
+    return () => {
+      // Cannot terminate worker in DEV mode
+      // workerRef.current?.terminate();
+    };
+  }, [canvasRef, mergedOptions, worker]);
+
+  useEffect(() => {
+    worker.setSource(src);
+  }, [src, worker]);
+
+  return worker;
+}

demo/frontend/src/common/error/ErrorFallback.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useReportError from '@/common/error/useReportError';
+import {Button} from 'react-daisyui';
+import {FallbackProps} from 'react-error-boundary';
+
+export default function ErrorFallback({
+  error,
+  resetErrorBoundary,
+}: FallbackProps) {
+  const reportError = useReportError();
+
+  function handleReportError() {
+    reportError(error);
+  }
+
+  return (
+    <div className="h-full flex flex-col gap-2 items-center justify-center">
+      <p>Please check your connection and retry or report error.</p>
+      <div className="flex flex-row gap-2">
+        <Button color="ghost" onClick={resetErrorBoundary}>
+          Retry
+        </Button>
+        <Button
+          className="text-error"
+          color="ghost"
+          onClick={handleReportError}>
+          Report Error
+        </Button>
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/error/ErrorReport.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {getErrorTitle} from '@/common/error/ErrorUtils';
+import errorReportAtom from '@/common/error/errorReportAtom';
+import emptyFunction from '@/common/utils/emptyFunction';
+import {BugAntIcon} from '@heroicons/react/24/outline';
+import {Editor} from '@monaco-editor/react';
+import {useAtom} from 'jotai';
+import {useEffect, useRef} from 'react';
+import {Button, Modal} from 'react-daisyui';
+
+type Props = {
+  onReport?: (error: Error) => void;
+};
+
+export default function ErrorReport({onReport = emptyFunction}: Props) {
+  const [error, setError] = useAtom(errorReportAtom);
+  const errorModalRef = useRef<HTMLDialogElement>(null);
+
+  // Clean error state on ESC
+  useEffect(() => {
+    function onCloseDialog() {
+      setError(null);
+    }
+    const errorModal = errorModalRef.current;
+    errorModal?.addEventListener('close', onCloseDialog);
+    return () => {
+      errorModal?.removeEventListener('close', onCloseDialog);
+    };
+  }, [setError]);
+
+  useEffect(() => {
+    if (error != null) {
+      errorModalRef.current?.showModal();
+    } else {
+      errorModalRef.current?.close();
+    }
+  }, [error, setError]);
+
+  function handleCloseModal() {
+    errorModalRef.current?.close();
+  }
+
+  function handleReport() {
+    if (error != null) {
+      onReport(error);
+    }
+  }
+
+  return (
+    <Modal ref={errorModalRef} className="max-w-[800px]">
+      <Modal.Header>
+        {error != null ? getErrorTitle(error) : 'Unknown error'}
+      </Modal.Header>
+      <Modal.Body>
+        <Editor
+          className="h-[400px]"
+          language="javascript"
+          value={error?.stack ?? ''}
+          options={{
+            wordWrap: 'wordWrapColumn',
+            scrollBeyondLastLine: false,
+            readOnly: true,
+            minimap: {
+              enabled: false,
+            },
+          }}
+        />
+      </Modal.Body>
+      <Modal.Actions>
+        <Button
+          color="error"
+          startIcon={<BugAntIcon className="w-4 h-4" />}
+          onClick={handleReport}>
+          Report
+        </Button>
+        <Button onClick={handleCloseModal}>Close</Button>
+      </Modal.Actions>
+    </Modal>
+  );
+}

demo/frontend/src/common/error/ErrorSerializationUtils.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import CreateFilmstripError from '@/graphql/errors/CreateFilmstripError';
+import DrawFrameError from '@/graphql/errors/DrawFrameError';
+import WebGLContextError from '@/graphql/errors/WebGLContextError';
+import {errorConstructors} from 'serialize-error';
+
+export function registerSerializableConstructors() {
+  // @ts-expect-error Wrong `errorConstructors` types
+  errorConstructors.set('DrawFrameError', DrawFrameError);
+  // @ts-expect-error Wrong `errorConstructors` types
+  errorConstructors.set('CreateFilmstripError', CreateFilmstripError);
+  // @ts-expect-error Wrong `errorConstructors` types
+  errorConstructors.set('WebGLContextError', WebGLContextError);
+}

demo/frontend/src/common/error/ErrorUtils.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import CreateFilmstripError from '@/graphql/errors/CreateFilmstripError';
+import DrawFrameError from '@/graphql/errors/DrawFrameError';
+import WebGLContextError from '@/graphql/errors/WebGLContextError';
+import {deserializeError, type ErrorObject} from 'serialize-error';
+
+export type RenderingErrorType =
+  | 'webgl_context'
+  | 'draw_frame'
+  | 'create_filmstrip'
+  | 'error';
+
+export function getRenderErrorType(error?: ErrorObject): RenderingErrorType {
+  const deserializedError = deserializeError(error);
+
+  if (deserializedError instanceof WebGLContextError) {
+    return 'webgl_context';
+  }
+  if (deserializedError instanceof DrawFrameError) {
+    return 'draw_frame';
+  }
+  if (deserializedError instanceof CreateFilmstripError) {
+    return 'create_filmstrip';
+  }
+  return 'error';
+}
+
+/**
+ * This function extracts the title from an error message.
+ * The title is defined as the text before the first newline character.
+ *
+ * @param error The error object from which the title is to be extracted.
+ * @returns The title of the error message.
+ * @example
+ * ```ts
+ * const error = new Error('This is the title\nThis is the body');
+ * const title = getErrorTitle(error);
+ * console.log(title); // 'This is the title'
+ * ```
+ */
+export function getErrorTitle({message}: Error): string {
+  const idx = message.indexOf('\n');
+  return idx < 0 ? message : message.substring(0, idx);
+}

demo/frontend/src/common/error/errorReportAtom.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {atom} from 'jotai';
+
+export default atom<Error | null>(null);

demo/frontend/src/common/error/useReportError.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import errorReportAtom from '@/common/error/errorReportAtom';
+import {useSetAtom} from 'jotai';
+import {useCallback} from 'react';
+
+export default function useReportError() {
+  const setError = useSetAtom(errorReportAtom);
+  return useCallback(
+    (error: unknown) => {
+      if (typeof error === 'string') {
+        setError(new Error(error));
+      } else if (error instanceof Error) {
+        setError(error);
+      } else {
+        setError(new Error('unknown error occurred'));
+      }
+    },
+    [setError],
+  );
+}

demo/frontend/src/common/loading/LoadingMessage.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Loading} from 'react-daisyui';
+
+export default function LoadingMessage() {
+  return (
+    <div className="flex flex-col w-full h-full justify-center items-center bg-black text-white">
+      <div className="flex justify-center">
+        <Loading className="mr-2" /> Fetching data
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/loading/LoadingStateScreen.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import introVideo from '@/assets/videos/sam2_720px_dark.mp4';
+import introVideoPoster from '@/assets/videos/sam2_video_poster.png';
+import StaticVideoPlayer from '@/common/loading/StaticVideoPlayer';
+import {borderRadius, fontSize, spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+import {PropsWithChildren, ReactNode} from 'react';
+import {Link} from 'react-router-dom';
+
+const styles = stylex.create({
+  container: {
+    backgroundColor: '#000',
+    minHeight: '100%',
+  },
+  content: {
+    display: 'flex',
+    flexDirection: 'column',
+    gap: spacing[8],
+    maxWidth: '36rem', //* 576px */
+    marginHorizontal: 'auto',
+    paddingVertical: {
+      default: '6rem',
+      '@media screen and (max-width: 768px)': '3rem',
+    },
+    paddingHorizontal: spacing[8],
+    color: '#fff',
+  },
+  animationContainer: {
+    display: 'flex',
+    justifyContent: 'center',
+  },
+  animation: {
+    border: '2px solid white',
+    borderRadius: borderRadius['xl'],
+    maxWidth: 450,
+    maxHeight: 450,
+    height: '100%',
+    overflow: 'hidden',
+    '@media screen and (max-width: 768px)': {
+      height: 300,
+      width: 300,
+    },
+  },
+  title: {
+    textAlign: 'center',
+    lineHeight: '2rem',
+    fontSize: fontSize['2xl'],
+    fontWeight: 400,
+  },
+  description: {
+    textAlign: 'center',
+    color: '#A7B3BF',
+  },
+  link: {
+    textAlign: 'center',
+    textDecorationLine: 'underline',
+    color: '#A7B3BF',
+  },
+});
+
+type Props = PropsWithChildren<{
+  title: string;
+  description?: string | ReactNode;
+  linkProps?: {
+    to: string;
+    label: string;
+  };
+}>;
+
+export default function LoadingStateScreen({
+  title,
+  description,
+  children,
+  linkProps,
+}: Props) {
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div {...stylex.props(styles.content)}>
+        <div {...stylex.props(styles.animationContainer)}>
+          <div {...stylex.props(styles.animation)}>
+            <StaticVideoPlayer
+              src={introVideo}
+              aspectRatio="square"
+              poster={introVideoPoster}
+              muted={true}
+              loop={true}
+              autoPlay={true}
+              playsInline={true}
+              controls={false}
+            />
+          </div>
+        </div>
+        <h2 {...stylex.props(styles.title)}>{title}</h2>
+        {description != null && (
+          <div {...stylex.props(styles.description)}>{description}</div>
+        )}
+        {children}
+        {linkProps != null && (
+          <Link to={linkProps.to} {...stylex.props(styles.link)}>
+            {linkProps.label}
+          </Link>
+        )}
+      </div>
+    </div>
+  );
+}