Initial commit

demo/frontend/src/common/components/video/useVideoWorker.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {RefObject, useEffect, useMemo, useRef} from 'react';
+import VideoWorkerBridge from './VideoWorkerBridge';
+
+type Options = {
+  createVideoWorker?: () => Worker;
+  createWorkerBridge?: CreateWorkerBridgeFunction;
+};
+
+const DEFAULT_OPTIONS: Options = {
+  createVideoWorker: () =>
+    new Worker(new URL('./VideoWorker', import.meta.url), {
+      type: 'module',
+    }),
+};
+
+type WorkerFactory = () => Worker;
+
+type CreateWorkerBridgeFunction = (
+  workerFactory: WorkerFactory,
+) => VideoWorkerBridge;
+
+export default function useVideoWorker(
+  src: string,
+  canvasRef: RefObject<HTMLCanvasElement>,
+  options: Options = {},
+) {
+  const isControlTransferredToOffscreenRef = useRef(false);
+
+  const mergedOptions = useMemo(() => {
+    const definedProps = (o: Options) =>
+      Object.fromEntries(
+        Object.entries(o).filter(([_k, v]) => v !== undefined),
+      );
+    return Object.assign(
+      DEFAULT_OPTIONS,
+      definedProps(options),
+    ) as Required<Options>;
+  }, [options]);
+
+  const worker = useMemo(() => {
+    if (mergedOptions.createWorkerBridge) {
+      return mergedOptions.createWorkerBridge(mergedOptions.createVideoWorker);
+    }
+    return VideoWorkerBridge.create(mergedOptions.createVideoWorker);
+  }, [mergedOptions]);
+
+  useEffect(() => {
+    const canvas = canvasRef.current;
+
+    if (canvas == null) {
+      return;
+    }
+
+    if (isControlTransferredToOffscreenRef.current) {
+      return;
+    }
+
+    isControlTransferredToOffscreenRef.current = true;
+
+    worker.setCanvas(canvas);
+
+    return () => {
+      // Cannot terminate worker in DEV mode
+      // workerRef.current?.terminate();
+    };
+  }, [canvasRef, mergedOptions, worker]);
+
+  useEffect(() => {
+    worker.setSource(src);
+  }, [src, worker]);
+
+  return worker;
+}

demo/frontend/src/common/error/ErrorFallback.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useReportError from '@/common/error/useReportError';
+import {Button} from 'react-daisyui';
+import {FallbackProps} from 'react-error-boundary';
+
+export default function ErrorFallback({
+  error,
+  resetErrorBoundary,
+}: FallbackProps) {
+  const reportError = useReportError();
+
+  function handleReportError() {
+    reportError(error);
+  }
+
+  return (
+    <div className="h-full flex flex-col gap-2 items-center justify-center">
+      <p>Please check your connection and retry or report error.</p>
+      <div className="flex flex-row gap-2">
+        <Button color="ghost" onClick={resetErrorBoundary}>
+          Retry
+        </Button>
+        <Button
+          className="text-error"
+          color="ghost"
+          onClick={handleReportError}>
+          Report Error
+        </Button>
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/error/ErrorReport.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {getErrorTitle} from '@/common/error/ErrorUtils';
+import errorReportAtom from '@/common/error/errorReportAtom';
+import emptyFunction from '@/common/utils/emptyFunction';
+import {BugAntIcon} from '@heroicons/react/24/outline';
+import {Editor} from '@monaco-editor/react';
+import {useAtom} from 'jotai';
+import {useEffect, useRef} from 'react';
+import {Button, Modal} from 'react-daisyui';
+
+type Props = {
+  onReport?: (error: Error) => void;
+};
+
+export default function ErrorReport({onReport = emptyFunction}: Props) {
+  const [error, setError] = useAtom(errorReportAtom);
+  const errorModalRef = useRef<HTMLDialogElement>(null);
+
+  // Clean error state on ESC
+  useEffect(() => {
+    function onCloseDialog() {
+      setError(null);
+    }
+    const errorModal = errorModalRef.current;
+    errorModal?.addEventListener('close', onCloseDialog);
+    return () => {
+      errorModal?.removeEventListener('close', onCloseDialog);
+    };
+  }, [setError]);
+
+  useEffect(() => {
+    if (error != null) {
+      errorModalRef.current?.showModal();
+    } else {
+      errorModalRef.current?.close();
+    }
+  }, [error, setError]);
+
+  function handleCloseModal() {
+    errorModalRef.current?.close();
+  }
+
+  function handleReport() {
+    if (error != null) {
+      onReport(error);
+    }
+  }
+
+  return (
+    <Modal ref={errorModalRef} className="max-w-[800px]">
+      <Modal.Header>
+        {error != null ? getErrorTitle(error) : 'Unknown error'}
+      </Modal.Header>
+      <Modal.Body>
+        <Editor
+          className="h-[400px]"
+          language="javascript"
+          value={error?.stack ?? ''}
+          options={{
+            wordWrap: 'wordWrapColumn',
+            scrollBeyondLastLine: false,
+            readOnly: true,
+            minimap: {
+              enabled: false,
+            },
+          }}
+        />
+      </Modal.Body>
+      <Modal.Actions>
+        <Button
+          color="error"
+          startIcon={<BugAntIcon className="w-4 h-4" />}
+          onClick={handleReport}>
+          Report
+        </Button>
+        <Button onClick={handleCloseModal}>Close</Button>
+      </Modal.Actions>
+    </Modal>
+  );
+}

demo/frontend/src/common/error/ErrorSerializationUtils.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import CreateFilmstripError from '@/graphql/errors/CreateFilmstripError';
+import DrawFrameError from '@/graphql/errors/DrawFrameError';
+import WebGLContextError from '@/graphql/errors/WebGLContextError';
+import {errorConstructors} from 'serialize-error';
+
+export function registerSerializableConstructors() {
+  // @ts-expect-error Wrong `errorConstructors` types
+  errorConstructors.set('DrawFrameError', DrawFrameError);
+  // @ts-expect-error Wrong `errorConstructors` types
+  errorConstructors.set('CreateFilmstripError', CreateFilmstripError);
+  // @ts-expect-error Wrong `errorConstructors` types
+  errorConstructors.set('WebGLContextError', WebGLContextError);
+}

demo/frontend/src/common/error/ErrorUtils.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import CreateFilmstripError from '@/graphql/errors/CreateFilmstripError';
+import DrawFrameError from '@/graphql/errors/DrawFrameError';
+import WebGLContextError from '@/graphql/errors/WebGLContextError';
+import {deserializeError, type ErrorObject} from 'serialize-error';
+
+export type RenderingErrorType =
+  | 'webgl_context'
+  | 'draw_frame'
+  | 'create_filmstrip'
+  | 'error';
+
+export function getRenderErrorType(error?: ErrorObject): RenderingErrorType {
+  const deserializedError = deserializeError(error);
+
+  if (deserializedError instanceof WebGLContextError) {
+    return 'webgl_context';
+  }
+  if (deserializedError instanceof DrawFrameError) {
+    return 'draw_frame';
+  }
+  if (deserializedError instanceof CreateFilmstripError) {
+    return 'create_filmstrip';
+  }
+  return 'error';
+}
+
+/**
+ * This function extracts the title from an error message.
+ * The title is defined as the text before the first newline character.
+ *
+ * @param error The error object from which the title is to be extracted.
+ * @returns The title of the error message.
+ * @example
+ * ```ts
+ * const error = new Error('This is the title\nThis is the body');
+ * const title = getErrorTitle(error);
+ * console.log(title); // 'This is the title'
+ * ```
+ */
+export function getErrorTitle({message}: Error): string {
+  const idx = message.indexOf('\n');
+  return idx < 0 ? message : message.substring(0, idx);
+}

demo/frontend/src/common/error/errorReportAtom.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {atom} from 'jotai';
+
+export default atom<Error | null>(null);

demo/frontend/src/common/error/useReportError.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import errorReportAtom from '@/common/error/errorReportAtom';
+import {useSetAtom} from 'jotai';
+import {useCallback} from 'react';
+
+export default function useReportError() {
+  const setError = useSetAtom(errorReportAtom);
+  return useCallback(
+    (error: unknown) => {
+      if (typeof error === 'string') {
+        setError(new Error(error));
+      } else if (error instanceof Error) {
+        setError(error);
+      } else {
+        setError(new Error('unknown error occurred'));
+      }
+    },
+    [setError],
+  );
+}

demo/frontend/src/common/loading/LoadingMessage.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Loading} from 'react-daisyui';
+
+export default function LoadingMessage() {
+  return (
+    <div className="flex flex-col w-full h-full justify-center items-center bg-black text-white">
+      <div className="flex justify-center">
+        <Loading className="mr-2" /> Fetching data
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/loading/LoadingStateScreen.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import introVideo from '@/assets/videos/sam2_720px_dark.mp4';
+import introVideoPoster from '@/assets/videos/sam2_video_poster.png';
+import StaticVideoPlayer from '@/common/loading/StaticVideoPlayer';
+import {borderRadius, fontSize, spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+import {PropsWithChildren, ReactNode} from 'react';
+import {Link} from 'react-router-dom';
+
+const styles = stylex.create({
+  container: {
+    backgroundColor: '#000',
+    minHeight: '100%',
+  },
+  content: {
+    display: 'flex',
+    flexDirection: 'column',
+    gap: spacing[8],
+    maxWidth: '36rem', //* 576px */
+    marginHorizontal: 'auto',
+    paddingVertical: {
+      default: '6rem',
+      '@media screen and (max-width: 768px)': '3rem',
+    },
+    paddingHorizontal: spacing[8],
+    color: '#fff',
+  },
+  animationContainer: {
+    display: 'flex',
+    justifyContent: 'center',
+  },
+  animation: {
+    border: '2px solid white',
+    borderRadius: borderRadius['xl'],
+    maxWidth: 450,
+    maxHeight: 450,
+    height: '100%',
+    overflow: 'hidden',
+    '@media screen and (max-width: 768px)': {
+      height: 300,
+      width: 300,
+    },
+  },
+  title: {
+    textAlign: 'center',
+    lineHeight: '2rem',
+    fontSize: fontSize['2xl'],
+    fontWeight: 400,
+  },
+  description: {
+    textAlign: 'center',
+    color: '#A7B3BF',
+  },
+  link: {
+    textAlign: 'center',
+    textDecorationLine: 'underline',
+    color: '#A7B3BF',
+  },
+});
+
+type Props = PropsWithChildren<{
+  title: string;
+  description?: string | ReactNode;
+  linkProps?: {
+    to: string;
+    label: string;
+  };
+}>;
+
+export default function LoadingStateScreen({
+  title,
+  description,
+  children,
+  linkProps,
+}: Props) {
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div {...stylex.props(styles.content)}>
+        <div {...stylex.props(styles.animationContainer)}>
+          <div {...stylex.props(styles.animation)}>
+            <StaticVideoPlayer
+              src={introVideo}
+              aspectRatio="square"
+              poster={introVideoPoster}
+              muted={true}
+              loop={true}
+              autoPlay={true}
+              playsInline={true}
+              controls={false}
+            />
+          </div>
+        </div>
+        <h2 {...stylex.props(styles.title)}>{title}</h2>
+        {description != null && (
+          <div {...stylex.props(styles.description)}>{description}</div>
+        )}
+        {children}
+        {linkProps != null && (
+          <Link to={linkProps.to} {...stylex.props(styles.link)}>
+            {linkProps.label}
+          </Link>
+        )}
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/loading/StaticVideoPlayer.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import React from 'react';
+
+export type VideoAspectRatio = 'wide' | 'square' | 'normal' | 'fill';
+
+export type VideoProps = {
+  src: string;
+  aspectRatio?: VideoAspectRatio;
+  className?: string;
+  containerClassName?: string;
+} & React.VideoHTMLAttributes<HTMLVideoElement>;
+
+export default function StaticVideoPlayer({
+  src,
+  aspectRatio,
+  className = '',
+  containerClassName = '',
+  ...props
+}: VideoProps) {
+  let aspect =
+    aspectRatio === 'wide'
+      ? `aspect-video`
+      : aspectRatio === 'square'
+        ? 'aspect-square'
+        : 'aspect-auto';
+
+  let videoSize = '';
+
+  if (aspectRatio === 'fill') {
+    aspect =
+      'absolute object-cover right-0 bottom-0 min-w-full min-h-full h-full';
+    videoSize = 'w-full h-full object-cover object-center';
+  }
+
+  return (
+    <div
+      className={`w-full relative flex flex-col ${aspect} ${containerClassName}`}>
+      <video className={`m-0 ${videoSize} ${className}`} {...props}>
+        <source src={src} type="video/mp4" />
+        Sorry, your browser does not support embedded videos.
+      </video>
+    </div>
+  );
+}

demo/frontend/src/common/loading/UploadLoadingScreen.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ChangeVideoModal from '@/common/components/gallery/ChangeVideoModal';
+import type {VideoGalleryTriggerProps} from '@/common/components/gallery/DemoVideoGalleryModal';
+import LoadingStateScreen from '@/common/loading/LoadingStateScreen';
+import {uploadingStateAtom} from '@/demo/atoms';
+import {ImageCopy} from '@carbon/icons-react';
+import {useAtomValue} from 'jotai';
+import OptionButton from '../components/options/OptionButton';
+
+export default function UploadLoadingScreen() {
+  const uploadingState = useAtomValue(uploadingStateAtom);
+
+  if (uploadingState === 'error') {
+    return (
+      <LoadingStateScreen
+        title="Uh oh, we cannot process this video"
+        description="Please upload another video, and make sure that the video’s file size is less than 70Mb. ">
+        <div className="max-w-[250px] w-full mx-auto">
+          <ChangeVideoModal
+            videoGalleryModalTrigger={UploadLoadingScreenChangeVideoTrigger}
+          />
+        </div>
+      </LoadingStateScreen>
+    );
+  }
+
+  return (
+    <LoadingStateScreen
+      title="Uploading video..."
+      description="Sit tight while we upload your video."
+    />
+  );
+}
+
+function UploadLoadingScreenChangeVideoTrigger({
+  onClick,
+}: VideoGalleryTriggerProps) {
+  return (
+    <OptionButton
+      variant="gradient"
+      title="Change video"
+      Icon={ImageCopy}
+      onClick={onClick}
+    />
+  );
+}

demo/frontend/src/common/logger/DemoLogger.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {RenderingErrorType} from '@/common/error/ErrorUtils';
+import Logger from './Logger';
+
+type UploadSourceType = 'gallery' | 'option';
+
+// Maps event names to an optional payload for each event
+type DemoEventMap = {
+  // User events
+  user_click_canvas: {
+    click_type: 'add_point' | 'remove_point';
+    click_action: 'add_object' | 'refine_object';
+    click_variant?: 'positive' | 'negative';
+  };
+  user_click_object: {
+    tracklet_id: number;
+  };
+  user_click_track_and_play: {
+    track_and_play_click_type: 'stream' | 'abort';
+  };
+  user_click_apply_effect: {
+    effect_type: 'background' | 'object';
+    effect_name: string;
+    effect_variant: number;
+  };
+  user_change_video: {
+    gallery_video_url: string;
+  };
+  user_upload_video: {
+    upload_source: UploadSourceType;
+  };
+  user_click_share: {
+    gallery_video_url: string;
+  };
+  user_click_download: {
+    gallery_video_url: string;
+  };
+  user_click_web_share: undefined;
+  // Error events
+  client_error_rendering: {
+    rendering_error_type: RenderingErrorType;
+  };
+  client_error_start_session: undefined;
+  client_error_upload_video: {
+    upload_source: UploadSourceType;
+    upload_error_message: string;
+  };
+  client_error_unsupported_browser: undefined;
+  client_error_page_not_found: {
+    path: string;
+  };
+  client_error_general: {
+    message: string;
+  };
+  client_error_fallback: {
+    fallback_error_message: string;
+  };
+
+  // Dataset events
+  client_error_fallback_dataset: {
+    dataset_fallback_error_message: string;
+  };
+  dataset_client_impression_event: {
+    impression_type: 'grid_view' | 'detailed_view';
+    video_id?: string;
+  };
+  dataset_client_click_events: {
+    click_type: 'search' | 'next_page' | 'prev_page';
+    video_id?: string;
+  };
+};
+
+export interface LoggerInterface<TEventMap> {
+  event: <K extends keyof TEventMap>(
+    eventName: K,
+    options?: TEventMap[K],
+  ) => void;
+}
+
+export function initialize(): void {
+  // noop
+}
+
+export class DemoLogger implements LoggerInterface<DemoEventMap> {
+  event<K extends keyof DemoEventMap>(eventName: K, options?: DemoEventMap[K]) {
+    Logger.info(eventName, options ?? {});
+  }
+}

demo/frontend/src/common/logger/LogEnvironment.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {LogLevel} from '@/common/logger/Logger';
+
+// Only enable debug logging in modes that are set in MODES_WITH_LOGGER. The
+// default is always error only.
+export const LOG_LEVEL: LogLevel =
+  import.meta.env.MODE === 'production' ? 'debug' : 'error';

demo/frontend/src/common/logger/Logger.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {LOG_LEVEL} from './LogEnvironment';
+
+/** Signature of a logging function */
+export type LogFn = {
+  (message?: unknown, ...optionalParams: unknown[]): void;
+};
+
+/** Basic logger interface */
+export interface Logger {
+  info: LogFn;
+  warn: LogFn;
+  error: LogFn;
+  debug: LogFn;
+}
+
+/** Log levels */
+export type LogLevel = 'info' | 'warn' | 'error' | 'debug';
+
+const NO_OP: LogFn = (_message?: unknown, ..._optionalParams: unknown[]) => {};
+
+/** Logger which outputs to the browser console */
+export class ConsoleLogger implements Logger {
+  readonly info: LogFn;
+  readonly warn: LogFn;
+  readonly error: LogFn;
+  readonly debug: LogFn;
+
+  constructor(options?: {level?: LogLevel}) {
+    const {level} = options || {};
+
+    // eslint-disable-next-line no-console
+    this.error = console.error.bind(console);
+
+    if (level === 'error') {
+      this.debug = NO_OP;
+      this.warn = NO_OP;
+      this.info = NO_OP;
+
+      return;
+    }
+
+    // eslint-disable-next-line no-console
+    this.warn = console.warn.bind(console);
+
+    if (level === 'warn') {
+      this.debug = NO_OP;
+      this.info = NO_OP;
+
+      return;
+    }
+
+    // eslint-disable-next-line no-console
+    this.info = console.log.bind(console);
+
+    if (level === 'info') {
+      this.debug = NO_OP;
+
+      return;
+    }
+
+    // eslint-disable-next-line no-console
+    this.debug = console.debug.bind(console);
+  }
+}
+
+export default new ConsoleLogger({level: LOG_LEVEL});

demo/frontend/src/common/screen/useScreenSize.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {screenSizes} from '@/theme/tokens.stylex';
+import {useLayoutEffect, useState} from 'react';
+
+export default function useScreenSize(): {
+  screenSize: number;
+  isMobile: boolean;
+} {
+  const [screenSize, setScreenSize] = useState<number>(0);
+
+  useLayoutEffect(() => {
+    const updateSize = (): void => {
+      setScreenSize(window.innerWidth);
+    };
+    window.addEventListener('resize', updateSize);
+    updateSize();
+    return (): void => window.removeEventListener('resize', updateSize);
+  }, []);
+
+  return {isMobile: screenSize < screenSizes['md'], screenSize};
+}

demo/frontend/src/common/tracker/SAM2Model.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {generateThumbnail} from '@/common/components/video/editor/VideoEditorUtils';
+import VideoWorkerContext from '@/common/components/video/VideoWorkerContext';
+import Logger from '@/common/logger/Logger';
+import {
+  SAM2ModelAddNewPointsMutation,
+  SAM2ModelAddNewPointsMutation$data,
+} from '@/common/tracker/__generated__/SAM2ModelAddNewPointsMutation.graphql';
+import {SAM2ModelCancelPropagateInVideoMutation} from '@/common/tracker/__generated__/SAM2ModelCancelPropagateInVideoMutation.graphql';
+import {SAM2ModelClearPointsInFrameMutation} from '@/common/tracker/__generated__/SAM2ModelClearPointsInFrameMutation.graphql';
+import {SAM2ModelClearPointsInVideoMutation} from '@/common/tracker/__generated__/SAM2ModelClearPointsInVideoMutation.graphql';
+import {SAM2ModelCloseSessionMutation} from '@/common/tracker/__generated__/SAM2ModelCloseSessionMutation.graphql';
+import {SAM2ModelRemoveObjectMutation} from '@/common/tracker/__generated__/SAM2ModelRemoveObjectMutation.graphql';
+import {SAM2ModelStartSessionMutation} from '@/common/tracker/__generated__/SAM2ModelStartSessionMutation.graphql';
+import {
+  BaseTracklet,
+  Mask,
+  SegmentationPoint,
+  StreamingState,
+  Tracker,
+  Tracklet,
+} from '@/common/tracker/Tracker';
+import {TrackerOptions} from '@/common/tracker/Trackers';
+import {
+  ClearPointsInVideoResponse,
+  SessionStartFailedResponse,
+  SessionStartedResponse,
+  StreamingCompletedResponse,
+  StreamingStartedResponse,
+  StreamingStateUpdateResponse,
+  TrackletCreatedResponse,
+  TrackletDeletedResponse,
+  TrackletsUpdatedResponse,
+} from '@/common/tracker/TrackerTypes';
+import {convertMaskToRGBA} from '@/common/utils/MaskUtils';
+import multipartStream from '@/common/utils/MultipartStream';
+import {Stats} from '@/debug/stats/Stats';
+import {INFERENCE_API_ENDPOINT} from '@/demo/DemoConfig';
+import {createEnvironment} from '@/graphql/RelayEnvironment';
+import {
+  DataArray,
+  Masks,
+  RLEObject,
+  decode,
+  encode,
+  toBbox,
+} from '@/jscocotools/mask';
+import {THEME_COLORS} from '@/theme/colors';
+import invariant from 'invariant';
+import {IEnvironment, commitMutation, graphql} from 'relay-runtime';
+
+type Options = Pick<TrackerOptions, 'inferenceEndpoint'>;
+
+type Session = {
+  id: string | null;
+  tracklets: {[id: number]: Tracklet};
+};
+
+type StreamMasksResult = {
+  frameIndex: number;
+  rleMaskList: Array<{
+    objectId: number;
+    rleMask: RLEObject;
+  }>;
+};
+
+type StreamMasksAbortResult = {
+  aborted: boolean;
+};
+
+export class SAM2Model extends Tracker {
+  private _endpoint: string;
+  private _environment: IEnvironment;
+
+  private abortController: AbortController | null = null;
+  private _session: Session = {
+    id: null,
+    tracklets: {},
+  };
+  private _streamingState: StreamingState = 'none';
+
+  private _emptyMask: RLEObject | null = null;
+
+  private _maskCanvas: OffscreenCanvas;
+  private _maskCtx: OffscreenCanvasRenderingContext2D;
+
+  private _stats?: Stats;
+
+  constructor(
+    context: VideoWorkerContext,
+    options: Options = {
+      inferenceEndpoint: INFERENCE_API_ENDPOINT,
+    },
+  ) {
+    super(context);
+    this._endpoint = options.inferenceEndpoint;
+    this._environment = createEnvironment(options.inferenceEndpoint);
+
+    this._maskCanvas = new OffscreenCanvas(0, 0);
+    const maskCtx = this._maskCanvas.getContext('2d');
+    invariant(maskCtx != null, 'context cannot be null');
+    this._maskCtx = maskCtx;
+  }
+
+  public startSession(videoPath: string): Promise<void> {
+    // Reset streaming state. Force update with the true flag to make sure the
+    // UI updates its state.
+    this._updateStreamingState('none', true);
+
+    return new Promise(resolve => {
+      try {
+        commitMutation<SAM2ModelStartSessionMutation>(this._environment, {
+          mutation: graphql`
+            mutation SAM2ModelStartSessionMutation($input: StartSessionInput!) {
+              startSession(input: $input) {
+                sessionId
+              }
+            }
+          `,
+          variables: {
+            input: {
+              path: videoPath,
+            },
+          },
+          onCompleted: response => {
+            const {sessionId} = response.startSession;
+            this._session.id = sessionId;
+
+            this._sendResponse<SessionStartedResponse>('sessionStarted', {
+              sessionId,
+            });
+
+            // Clear any tracklets from the previous session when
+            // a new session is started
+            this._clearTracklets();
+
+            // Make an empty tracklet
+            this.createTracklet();
+            resolve();
+          },
+          onError: error => {
+            Logger.error(error);
+            this._sendResponse<SessionStartFailedResponse>(
+              'sessionStartFailed',
+            );
+            resolve();
+          },
+        });
+      } catch (error) {
+        Logger.error(error);
+        this._sendResponse<SessionStartFailedResponse>('sessionStartFailed');
+        resolve();
+      }
+    });
+  }
+
+  public closeSession(): Promise<void> {
+    const sessionId = this._session.id;
+
+    // Do not call cleanup before retrieving the session id because cleanup
+    // will reset the session id. If the order would be changed, it would
+    // never execute the closeSession mutation.
+    this._cleanup();
+
+    if (sessionId === null) {
+      return Promise.resolve();
+    }
+    return new Promise((resolve, reject) => {
+      commitMutation<SAM2ModelCloseSessionMutation>(this._environment, {
+        mutation: graphql`
+          mutation SAM2ModelCloseSessionMutation($input: CloseSessionInput!) {
+            closeSession(input: $input) {
+              success
+            }
+          }
+        `,
+        variables: {
+          input: {
+            sessionId,
+          },
+        },
+        onCompleted: response => {
+          const {success} = response.closeSession;
+          if (success === false) {
+            reject(new Error('Failed to close session'));
+            return;
+          }
+          resolve();
+        },
+        onError: error => {
+          Logger.error(error);
+          reject(error);
+        },
+      });
+    });
+  }
+
+  public createTracklet(): void {
+    // This will return 0 for for empty tracklets and otherwise the next
+    // largest number.
+    const nextId =
+      Object.values(this._session.tracklets).reduce(
+        (prev, curr) => Math.max(prev, curr.id),
+        -1,
+      ) + 1;
+
+    const newTracklet = {
+      id: nextId,
+      color: THEME_COLORS[nextId % THEME_COLORS.length],
+      thumbnail: null,
+      points: [],
+      masks: [],
+      isInitialized: false,
+    };
+
+    this._session.tracklets[nextId] = newTracklet;
+
+    // Notify the main thread
+    this._updateTracklets();
+
+    this._sendResponse<TrackletCreatedResponse>('trackletCreated', {
+      tracklet: newTracklet,
+    });
+  }
+
+  public deleteTracklet(trackletId: number): Promise<void> {
+    const sessionId = this._session.id;
+    if (sessionId === null) {
+      return Promise.reject('No active session');
+    }
+
+    const tracklet = this._session.tracklets[trackletId];
+    invariant(
+      tracklet != null,
+      'tracklet for tracklet id %s not initialized',
+      trackletId,
+    );
+
+    return new Promise((resolve, reject) => {
+      commitMutation<SAM2ModelRemoveObjectMutation>(this._environment, {
+        mutation: graphql`
+          mutation SAM2ModelRemoveObjectMutation($input: RemoveObjectInput!) {
+            removeObject(input: $input) {
+              frameIndex
+              rleMaskList {
+                objectId
+                rleMask {
+                  counts
+                  size
+                }
+              }
+            }
+          }
+        `,
+        variables: {
+          input: {objectId: trackletId, sessionId},
+        },
+        onCompleted: response => {
+          const trackletUpdates = response.removeObject;
+          this._sendResponse<TrackletDeletedResponse>('trackletDeleted', {
+            isSuccessful: true,
+          });
+          for (const trackletUpdate of trackletUpdates) {
+            this._updateTrackletMasks(
+              trackletUpdate,
+              trackletUpdate.frameIndex === this._context.frameIndex,
+              false, // shouldGoToFrame
+            );
+          }
+          this._removeTrackletMasks(tracklet);
+          resolve();
+        },
+        onError: error => {
+          this._sendResponse<TrackletDeletedResponse>('trackletDeleted', {
+            isSuccessful: false,
+          });
+          Logger.error(error);
+          reject(error);
+        },
+      });
+    });
+  }
+
+  public updatePoints(
+    frameIndex: number,
+    objectId: number,
+    points: SegmentationPoint[],
+  ): Promise<void> {
+    const sessionId = this._session.id;
+    if (sessionId === null) {
+      return Promise.reject('No active session');
+    }
+
+    // TODO: This is not the right place to initialize the empty mask.
+    // Move this into the constructor and listen to events on the context.
+    // Note, the initial context.width and context.height is 0, so it needs
+    // to happen based on an event, so when the video is initialized, it needs
+    // to notify the tracker to update the empty mask.
+    if (this._emptyMask === null) {
+      // We need to round the height/width to the nearest integer since
+      // Masks.toTensor() expects an integer value for the height/width.
+      const tensor = new Masks(
+        Math.trunc(this._context.height),
+        Math.trunc(this._context.width),
+        1,
+      ).toDataArray();
+      this._emptyMask = encode(tensor)[0];
+    }
+
+    const tracklet = this._session.tracklets[objectId];
+    invariant(
+      tracklet != null,
+      'tracklet for object id %s not initialized',
+      objectId,
+    );
+
+    // Mark session needing propagation when point is set
+    this._updateStreamingState('required');
+
+    // Clear all points in frame if no points are provided.
+    if (points.length === 0) {
+      return this.clearPointsInFrame(frameIndex, objectId);
+    }
+    return new Promise((resolve, reject) => {
+      const normalizedPoints = points.map(p => [
+        p[0] / this._context.width,
+        p[1] / this._context.height,
+      ]);
+      const labels = points.map(p => p[2]);
+      commitMutation<SAM2ModelAddNewPointsMutation>(this._environment, {
+        mutation: graphql`
+          mutation SAM2ModelAddNewPointsMutation($input: AddPointsInput!) {
+            addPoints(input: $input) {
+              frameIndex
+              rleMaskList {
+                objectId
+                rleMask {
+                  counts
+                  size
+                }
+              }
+            }
+          }
+        `,
+        variables: {
+          input: {
+            sessionId,
+            frameIndex,
+            objectId,
+            labels: labels,
+            points: normalizedPoints,
+            clearOldPoints: true,
+          },
+        },
+        onCompleted: response => {
+          tracklet.points[frameIndex] = points;
+          tracklet.isInitialized = true;
+          this._updateTrackletMasks(response.addPoints, true);
+          resolve();
+        },
+        onError: error => {
+          Logger.error(error);
+          reject(error);
+        },
+      });
+    });
+  }
+
+  public clearPointsInFrame(
+    frameIndex: number,
+    objectId: number,
+  ): Promise<void> {
+    const sessionId = this._session.id;
+    if (sessionId === null) {
+      return Promise.reject('No active session');
+    }
+
+    const tracklet = this._session.tracklets[objectId];
+    invariant(
+      tracklet != null,
+      'tracklet for object id %s not initialized',
+      objectId,
+    );
+
+    // Mark session needing propagation when point is set
+    this._updateStreamingState('required');
+
+    return new Promise((resolve, reject) => {
+      commitMutation<SAM2ModelClearPointsInFrameMutation>(this._environment, {
+        mutation: graphql`
+          mutation SAM2ModelClearPointsInFrameMutation(
+            $input: ClearPointsInFrameInput!
+          ) {
+            clearPointsInFrame(input: $input) {
+              frameIndex
+              rleMaskList {
+                objectId
+                rleMask {
+                  counts
+                  size
+                }
+              }
+            }
+          }
+        `,
+        variables: {
+          input: {
+            sessionId,
+            frameIndex,
+            objectId,
+          },
+        },
+        onCompleted: response => {
+          tracklet.points[frameIndex] = [];
+          tracklet.isInitialized = true;
+          this._updateTrackletMasks(response.clearPointsInFrame, true);
+          resolve();
+        },
+        onError: error => {
+          Logger.error(error);
+          reject(error);
+        },
+      });
+    });
+  }
+
+  public clearPointsInVideo(): Promise<void> {
+    const sessionId = this._session.id;
+    if (sessionId === null) {
+      return Promise.reject('No active session');
+    }
+
+    // Mark session needing propagation when point is set
+    this._updateStreamingState('none');
+
+    return new Promise(resolve => {
+      commitMutation<SAM2ModelClearPointsInVideoMutation>(this._environment, {
+        mutation: graphql`
+          mutation SAM2ModelClearPointsInVideoMutation(
+            $input: ClearPointsInVideoInput!
+          ) {
+            clearPointsInVideo(input: $input) {
+              success
+            }
+          }
+        `,
+        variables: {
+          input: {
+            sessionId,
+          },
+        },
+        onCompleted: response => {
+          const {success} = response.clearPointsInVideo;
+          if (!success) {
+            this._sendResponse<ClearPointsInVideoResponse>(
+              'clearPointsInVideo',
+              {isSuccessful: false},
+            );
+            return;
+          }
+
+          // Reset points and masks for each tracklet
+          this._clearTracklets();
+
+          // Notify the main thread
+          this._context.goToFrame(this._context.frameIndex);
+          this._updateTracklets();
+          this._sendResponse<ClearPointsInVideoResponse>('clearPointsInVideo', {
+            isSuccessful: true,
+          });
+          resolve();
+        },
+        onError: error => {
+          this._sendResponse<ClearPointsInVideoResponse>('clearPointsInVideo', {
+            isSuccessful: false,
+          });
+          Logger.error(error);
+        },
+      });
+    });
+  }
+
+  public async streamMasks(frameIndex: number): Promise<void> {
+    const sessionId = this._session.id;
+    if (sessionId === null) {
+      return Promise.reject('No active session');
+    }
+    try {
+      this._sendResponse<StreamingStartedResponse>('streamingStarted');
+
+      // 1. Clear previous masks
+      this._context.clearMasks();
+      this._clearTrackletMasks();
+
+      // 2. Create abort controller and async generator
+      const controller = new AbortController();
+      this.abortController = controller;
+
+      this._updateStreamingState('requesting');
+      const generator = this._streamMasksForSession(
+        controller,
+        sessionId,
+        frameIndex,
+      );
+
+      // 3. parse stream response and update masks in session objects
+      let isAborted = false;
+      for await (const result of generator) {
+        if ('aborted' in result) {
+          this._updateStreamingState('aborting');
+          await this._abortRequest();
+          this._updateStreamingState('aborted');
+          isAborted = true;
+        } else {
+          await this._updateTrackletMasks(result, false);
+          this._updateStreamingState('partial');
+        }
+      }
+
+      if (!isAborted) {
+        // Mark session needing propagation when point is set
+        this._updateStreamingState('full');
+      }
+    } catch (error) {
+      Logger.error(error);
+      throw error;
+    }
+
+    this._sendResponse<StreamingCompletedResponse>('streamingCompleted');
+  }
+
+  public abortStreamMasks() {
+    this.abortController?.abort();
+    this._sendResponse<StreamingCompletedResponse>('streamingCompleted');
+  }
+
+  public enableStats(): void {
+    this._stats = new Stats('ms', 'D', 1000 / 25);
+  }
+
+  // PRIVATE
+
+  private _cleanup() {
+    this._session.id = null;
+    // Clear existing tracklets
+    this._session.tracklets = [];
+  }
+
+  private _clearTracklets() {
+    this._session.tracklets = [];
+    this._context.clearMasks();
+  }
+
+  private _updateStreamingState(
+    state: StreamingState,
+    forceUpdate: boolean = false,
+  ) {
+    if (!forceUpdate && this._streamingState === state) {
+      return;
+    }
+    this._streamingState = state;
+    this._sendResponse<StreamingStateUpdateResponse>('streamingStateUpdate', {
+      state,
+    });
+  }
+
+  private async _removeTrackletMasks(tracklet: Tracklet) {
+    this._context.clearTrackletMasks(tracklet);
+    delete this._session.tracklets[tracklet.id];
+
+    // Notify the main thread
+    this._context.goToFrame(this._context.frameIndex);
+    this._updateTracklets();
+  }
+
+  private async _updateTrackletMasks(
+    data: SAM2ModelAddNewPointsMutation$data['addPoints'],
+    updateThumbnails: boolean,
+    shouldGoToFrame: boolean = true,
+  ) {
+    const {frameIndex, rleMaskList} = data;
+
+    // 1. parse and decode masks for all objects
+    for (const {objectId, rleMask} of rleMaskList) {
+      const track = this._session.tracklets[objectId];
+      const {size, counts} = rleMask;
+      const rleObject: RLEObject = {
+        size: [size[0], size[1]],
+        counts: counts,
+      };
+      const isEmpty = counts === this._emptyMask?.counts;
+
+      this._stats?.begin();
+
+      const decodedMask = decode([rleObject]);
+      const bbox = toBbox([rleObject]);
+
+      const mask: Mask = {
+        data: rleObject as RLEObject,
+        shape: [...decodedMask.shape],
+        bounds: [
+          [bbox[0], bbox[1]],
+          [bbox[0] + bbox[2], bbox[1] + bbox[3]],
+        ],
+        isEmpty,
+      } as const;
+      track.masks[frameIndex] = mask;
+
+      if (updateThumbnails && !isEmpty) {
+        const {ctx} = await this._compressMaskForCanvas(decodedMask);
+        const frame = this._context.currentFrame as VideoFrame;
+        await generateThumbnail(track, frameIndex, mask, frame, ctx);
+      }
+    }
+
+    this._context.updateTracklets(
+      frameIndex,
+      Object.values(this._session.tracklets),
+      shouldGoToFrame,
+    );
+
+    // Notify the main thread
+    this._updateTracklets();
+  }
+
+  private _updateTracklets() {
+    const tracklets: BaseTracklet[] = Object.values(
+      this._session.tracklets,
+    ).map(tracklet => {
+      // Notify the main thread
+      const {
+        id,
+        color,
+        isInitialized,
+        points: trackletPoints,
+        thumbnail,
+        masks,
+      } = tracklet;
+      return {
+        id,
+        color,
+        isInitialized,
+        points: trackletPoints,
+        thumbnail,
+        masks: masks.map(mask => ({
+          shape: mask.shape,
+          bounds: mask.bounds,
+          isEmpty: mask.isEmpty,
+        })),
+      };
+    });
+
+    this._sendResponse<TrackletsUpdatedResponse>('trackletsUpdated', {
+      tracklets,
+    });
+  }
+
+  private _clearTrackletMasks() {
+    const keys = Object.keys(this._session.tracklets);
+    for (const key of keys) {
+      const trackletId = Number(key);
+      const tracklet = {...this._session.tracklets[trackletId], masks: []};
+      this._session.tracklets[trackletId] = tracklet;
+    }
+    this._updateTracklets();
+  }
+
+  private async _compressMaskForCanvas(
+    decodedMask: DataArray,
+  ): Promise<{compressedData: Blob; ctx: OffscreenCanvasRenderingContext2D}> {
+    const data = convertMaskToRGBA(decodedMask.data as Uint8Array);
+
+    this._maskCanvas.width = decodedMask.shape[0];
+    this._maskCanvas.height = decodedMask.shape[1];
+
+    const imageData = new ImageData(
+      data,
+      decodedMask.shape[0],
+      decodedMask.shape[1],
+    );
+    this._maskCtx.putImageData(imageData, 0, 0);
+
+    const canvas = new OffscreenCanvas(
+      decodedMask.shape[1],
+      decodedMask.shape[0],
+    );
+
+    const ctx = canvas.getContext('2d');
+    invariant(ctx != null, 'context cannot be null');
+    ctx.save();
+    ctx.rotate(Math.PI / 2);
+    // Since the image was previously rotated 90° clockwise, after the image is rotated,
+    // we scale the canvas's width using scaleY and height using scaleX.
+    ctx.scale(1, -1);
+    ctx.drawImage(this._maskCanvas, 0, 0);
+    ctx.restore();
+
+    const compressedData = await canvas.convertToBlob({type: 'image/png'});
+
+    return {compressedData, ctx};
+  }
+
+  private async *_streamMasksForSession(
+    abortController: AbortController,
+    sessionId: string,
+    startFrameIndex: undefined | number = 0,
+  ): AsyncGenerator<StreamMasksResult | StreamMasksAbortResult, undefined> {
+    const url = `${this._endpoint}/propagate_in_video`;
+
+    const requestBody = {
+      session_id: sessionId,
+      start_frame_index: startFrameIndex,
+    };
+
+    const headers: {[name: string]: string} = Object.assign({
+      'Content-Type': 'application/json',
+    });
+
+    const response = await fetch(url, {
+      method: 'POST',
+      body: JSON.stringify(requestBody),
+      headers,
+    });
+
+    const contentType = response.headers.get('Content-Type');
+    if (
+      contentType == null ||
+      !contentType.startsWith('multipart/x-savi-stream;')
+    ) {
+      throw new Error(
+        'endpoint needs to support Content-Type "multipart/x-savi-stream"',
+      );
+    }
+
+    const responseBody = response.body;
+    if (responseBody == null) {
+      throw new Error('response body is null');
+    }
+
+    const reader = multipartStream(contentType, responseBody).getReader();
+
+    const textDecoder = new TextDecoder();
+
+    while (true) {
+      if (abortController.signal.aborted) {
+        reader.releaseLock();
+        yield {aborted: true};
+        return;
+      }
+
+      const {done, value} = await reader.read();
+      if (done) {
+        return;
+      }
+
+      const {headers, body} = value;
+
+      const contentType = headers.get('Content-Type') as string;
+
+      if (contentType.startsWith('application/json')) {
+        const jsonResponse = JSON.parse(textDecoder.decode(body));
+        const maskResults = jsonResponse.results;
+        const rleMaskList = maskResults.map(
+          (mask: {object_id: number; mask: RLEObject}) => {
+            return {
+              objectId: mask.object_id,
+              rleMask: mask.mask,
+            };
+          },
+        );
+        yield {
+          frameIndex: jsonResponse.frame_index,
+          rleMaskList,
+        };
+      }
+    }
+  }
+
+  private async _abortRequest(): Promise<void> {
+    const sessionId = this._session.id;
+    invariant(sessionId != null, 'session id cannot be empty');
+    return new Promise((resolve, reject) => {
+      try {
+        commitMutation<SAM2ModelCancelPropagateInVideoMutation>(
+          this._environment,
+          {
+            mutation: graphql`
+              mutation SAM2ModelCancelPropagateInVideoMutation(
+                $input: CancelPropagateInVideoInput!
+              ) {
+                cancelPropagateInVideo(input: $input) {
+                  success
+                }
+              }
+            `,
+            variables: {
+              input: {
+                sessionId,
+              },
+            },
+            onCompleted: response => {
+              const {success} = response.cancelPropagateInVideo;
+              if (!success) {
+                reject(`could not abort session ${sessionId}`);
+                return;
+              }
+              resolve();
+            },
+            onError: error => {
+              Logger.error(error);
+              reject(error);
+            },
+          },
+        );
+      } catch (error) {
+        Logger.error(error);
+        reject(error);
+      }
+    });
+  }
+}

demo/frontend/src/common/tracker/Tracker.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import VideoWorkerContext from '@/common/components/video/VideoWorkerContext';
+
+import {TrackerOptions} from '@/common/tracker/Trackers';
+import {TrackerResponse} from '@/common/tracker/TrackerTypes';
+import {RLEObject} from '@/jscocotools/mask';
+
+export type Point = [x: number, y: number];
+
+export type SegmentationPoint = [...point: Point, label: 0 | 1];
+
+export type FramePoints = Array<SegmentationPoint> | undefined;
+
+export type Mask = DatalessMask & {
+  data: Blob | RLEObject;
+};
+
+export type DatalessMask = {
+  shape: number[];
+  bounds: [[number, number], [number, number]];
+  isEmpty: boolean;
+};
+
+export type Tracklet = {
+  id: number;
+  color: string;
+  thumbnail: string | null;
+  points: FramePoints[];
+  masks: Mask[];
+  isInitialized: boolean;
+};
+
+export type BaseTracklet = Omit<Tracklet, 'masks'> & {
+  masks: DatalessMask[];
+};
+
+export type StreamingState =
+  | 'none'
+  | 'required'
+  | 'requesting'
+  | 'aborting'
+  | 'aborted'
+  | 'partial'
+  | 'full';
+
+export interface ITracker {
+  startSession(videoUrl: string): Promise<void>;
+  closeSession(): Promise<void>;
+  createTracklet(): void;
+  deleteTracklet(trackletId: number): Promise<void>;
+  updatePoints(
+    frameIndex: number,
+    objectId: number,
+    points: SegmentationPoint[],
+  ): Promise<void>;
+  clearPointsInFrame(frameIndex: number, objectId: number): Promise<void>;
+  clearPointsInVideo(): Promise<void>;
+  streamMasks(frameIndex: number): Promise<void>;
+  abortStreamMasks(): void;
+  enableStats(): void;
+}
+
+export abstract class Tracker implements ITracker {
+  protected _context: VideoWorkerContext;
+  constructor(context: VideoWorkerContext, _options?: TrackerOptions) {
+    this._context = context;
+  }
+  abstract startSession(videoUrl: string): Promise<void>;
+  abstract closeSession(): Promise<void>;
+  abstract createTracklet(): void;
+  abstract deleteTracklet(trackletId: number): Promise<void>;
+  abstract updatePoints(
+    frameIndex: number,
+    objectId: number,
+    points: SegmentationPoint[],
+  ): Promise<void>;
+  abstract clearPointsInFrame(
+    frameIndex: number,
+    objectId: number,
+  ): Promise<void>;
+  abstract clearPointsInVideo(): Promise<void>;
+  abstract streamMasks(frameIndex: number): Promise<void>;
+  abstract abortStreamMasks(): void;
+  abstract enableStats(): void;
+
+  // PRIVATE FUNCTIONS
+
+  protected _sendResponse<T extends TrackerResponse>(
+    action: T['action'],
+    message?: Omit<T, 'action'>,
+    transfer?: Transferable[],
+  ): void {
+    self.postMessage(
+      {
+        action,
+        ...message,
+      },
+      {
+        transfer,
+      },
+    );
+  }
+}

demo/frontend/src/common/tracker/TrackerTypes.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {SegmentationPoint} from '@/common/tracker/Tracker';
+import {TrackerOptions, Trackers} from '@/common/tracker/Trackers';
+import {
+  AddPointsEvent,
+  ClearPointsInVideoEvent,
+  SessionStartFailedEvent,
+  SessionStartedEvent,
+  StreamingCompletedEvent,
+  StreamingStartedEvent,
+  StreamingStateUpdateEvent,
+  TrackletCreatedEvent,
+  TrackletDeletedEvent,
+  TrackletsEvent,
+} from '../components/video/VideoWorkerBridge';
+
+export type Flags = {
+  masks: boolean;
+  effect: boolean;
+};
+
+export type Request<A, P> = {
+  action: A;
+} & P;
+
+// REQUESTS
+
+export type InitializeTrackerRequest = Request<
+  'initializeTracker',
+  {
+    name: keyof Trackers;
+    options: TrackerOptions;
+  }
+>;
+export type StartSessionRequest = Request<
+  'startSession',
+  {
+    videoUrl: string;
+  }
+>;
+export type CloseSessionRequest = Request<'closeSession', unknown>;
+export type CreateTrackletRequest = Request<'createTracklet', unknown>;
+export type DeleteTrackletRequest = Request<
+  'deleteTracklet',
+  {
+    trackletId: number;
+  }
+>;
+export type UpdatePointsRequest = Request<
+  'updatePoints',
+  {
+    frameIndex: number;
+    objectId: number;
+    points: SegmentationPoint[];
+  }
+>;
+export type ClearPointsInFrameRequest = Request<
+  'clearPointsInFrame',
+  {
+    frameIndex: number;
+    objectId: number;
+  }
+>;
+export type ClearPointsInVideoRequest = Request<'clearPointsInVideo', unknown>;
+export type StreamMasksRequest = Request<
+  'streamMasks',
+  {
+    frameIndex: number;
+  }
+>;
+export type AbortStreamMasksRequest = Request<'abortStreamMasks', unknown>;
+
+export type LogAnnotationsRequest = Request<'logAnnotations', unknown>;
+
+export type TrackerRequest =
+  | InitializeTrackerRequest
+  | StartSessionRequest
+  | CloseSessionRequest
+  | CreateTrackletRequest
+  | DeleteTrackletRequest
+  | UpdatePointsRequest
+  | ClearPointsInFrameRequest
+  | ClearPointsInVideoRequest
+  | StreamMasksRequest
+  | AbortStreamMasksRequest
+  | LogAnnotationsRequest;
+
+export type TrackerRequestMessageEvent = MessageEvent<TrackerRequest>;
+
+// RESPONSES
+
+export type SessionStartedResponse = Request<
+  'sessionStarted',
+  SessionStartedEvent
+>;
+
+export type SessionStartFailedResponse = Request<
+  'sessionStartFailed',
+  SessionStartFailedEvent
+>;
+
+export type TrackletCreatedResponse = Request<
+  'trackletCreated',
+  TrackletCreatedEvent
+>;
+
+export type TrackletsUpdatedResponse = Request<
+  'trackletsUpdated',
+  TrackletsEvent
+>;
+
+export type TrackletDeletedResponse = Request<
+  'trackletDeleted',
+  TrackletDeletedEvent
+>;
+
+export type AddPointsResponse = Request<'addPoints', AddPointsEvent>;
+
+export type ClearPointsInVideoResponse = Request<
+  'clearPointsInVideo',
+  ClearPointsInVideoEvent
+>;
+
+export type StreamingStartedResponse = Request<
+  'streamingStarted',
+  StreamingStartedEvent
+>;
+
+export type StreamingCompletedResponse = Request<
+  'streamingCompleted',
+  StreamingCompletedEvent
+>;
+
+export type StreamingStateUpdateResponse = Request<
+  'streamingStateUpdate',
+  StreamingStateUpdateEvent
+>;
+
+export type TrackerResponse =
+  | SessionStartedResponse
+  | SessionStartFailedResponse
+  | TrackletCreatedResponse
+  | TrackletsUpdatedResponse
+  | TrackletDeletedResponse
+  | AddPointsResponse
+  | ClearPointsInVideoResponse
+  | StreamingStartedResponse
+  | StreamingCompletedResponse
+  | StreamingStateUpdateResponse;
+
+export type TrackerResponseMessageEvent = MessageEvent<TrackerResponse>;

demo/frontend/src/common/tracker/Trackers.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {SAM2Model} from './SAM2Model';
+
+export type Headers = {[name: string]: string};
+
+export type TrackerOptions = {
+  inferenceEndpoint: string;
+};
+
+export type Trackers = {
+  'SAM 2': typeof SAM2Model;
+};
+
+export const TRACKER_MAPPING: Trackers = {
+  'SAM 2': SAM2Model,
+};

demo/frontend/src/common/tracker/__generated__/SAM2ModelAddNewPointsMutation.graphql.ts

+/**
+ * @generated SignedSource<<db1ee50f3027130f61feafb624026897>>
+ * @lightSyntaxTransform
+ * @nogrep
+ */
+
+/* tslint:disable */
+/* eslint-disable */
+// @ts-nocheck
+
+import { ConcreteRequest, Mutation } from 'relay-runtime';
+export type AddPointsInput = {
+  clearOldPoints: boolean;
+  frameIndex: number;
+  labels: ReadonlyArray<number>;
+  objectId: number;
+  points: ReadonlyArray<ReadonlyArray<number>>;
+  sessionId: string;
+};
+export type SAM2ModelAddNewPointsMutation$variables = {
+  input: AddPointsInput;
+};
+export type SAM2ModelAddNewPointsMutation$data = {
+  readonly addPoints: {
+    readonly frameIndex: number;
+    readonly rleMaskList: ReadonlyArray<{
+      readonly objectId: number;
+      readonly rleMask: {
+        readonly counts: string;
+        readonly size: ReadonlyArray<number>;
+      };
+    }>;
+  };
+};
+export type SAM2ModelAddNewPointsMutation = {
+  response: SAM2ModelAddNewPointsMutation$data;
+  variables: SAM2ModelAddNewPointsMutation$variables;
+};
+
+const node: ConcreteRequest = (function(){
+var v0 = [
+  {
+    "defaultValue": null,
+    "kind": "LocalArgument",
+    "name": "input"
+  }
+],
+v1 = [
+  {
+    "alias": null,
+    "args": [
+      {
+        "kind": "Variable",
+        "name": "input",
+        "variableName": "input"
+      }
+    ],
+    "concreteType": "RLEMaskListOnFrame",
+    "kind": "LinkedField",
+    "name": "addPoints",
+    "plural": false,
+    "selections": [
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "frameIndex",
+        "storageKey": null
+      },
+      {
+        "alias": null,
+        "args": null,
+        "concreteType": "RLEMaskForObject",
+        "kind": "LinkedField",
+        "name": "rleMaskList",
+        "plural": true,
+        "selections": [
+          {
+            "alias": null,
+            "args": null,
+            "kind": "ScalarField",
+            "name": "objectId",
+            "storageKey": null
+          },
+          {
+            "alias": null,
+            "args": null,
+            "concreteType": "RLEMask",
+            "kind": "LinkedField",
+            "name": "rleMask",
+            "plural": false,
+            "selections": [
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "counts",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "size",
+                "storageKey": null
+              }
+            ],
+            "storageKey": null
+          }
+        ],
+        "storageKey": null
+      }
+    ],
+    "storageKey": null
+  }
+];
+return {
+  "fragment": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Fragment",
+    "metadata": null,
+    "name": "SAM2ModelAddNewPointsMutation",
+    "selections": (v1/*: any*/),
+    "type": "Mutation",
+    "abstractKey": null
+  },
+  "kind": "Request",
+  "operation": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Operation",
+    "name": "SAM2ModelAddNewPointsMutation",
+    "selections": (v1/*: any*/)
+  },
+  "params": {
+    "cacheID": "dc86527e91907e696683458ed0943d2f",
+    "id": null,
+    "metadata": {},
+    "name": "SAM2ModelAddNewPointsMutation",
+    "operationKind": "mutation",
+    "text": "mutation SAM2ModelAddNewPointsMutation(\n  $input: AddPointsInput!\n) {\n  addPoints(input: $input) {\n    frameIndex\n    rleMaskList {\n      objectId\n      rleMask {\n        counts\n        size\n      }\n    }\n  }\n}\n"
+  }
+};
+})();
+
+(node as any).hash = "3c96f05877dd91668c1f9e8a3f1203a5";
+
+export default node;