"Initial commit"

demo/frontend/src/common/loading/StaticVideoPlayer.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import React from 'react';
+
+export type VideoAspectRatio = 'wide' | 'square' | 'normal' | 'fill';
+
+export type VideoProps = {
+  src: string;
+  aspectRatio?: VideoAspectRatio;
+  className?: string;
+  containerClassName?: string;
+} & React.VideoHTMLAttributes<HTMLVideoElement>;
+
+export default function StaticVideoPlayer({
+  src,
+  aspectRatio,
+  className = '',
+  containerClassName = '',
+  ...props
+}: VideoProps) {
+  let aspect =
+    aspectRatio === 'wide'
+      ? `aspect-video`
+      : aspectRatio === 'square'
+        ? 'aspect-square'
+        : 'aspect-auto';
+
+  let videoSize = '';
+
+  if (aspectRatio === 'fill') {
+    aspect =
+      'absolute object-cover right-0 bottom-0 min-w-full min-h-full h-full';
+    videoSize = 'w-full h-full object-cover object-center';
+  }
+
+  return (
+    <div
+      className={`w-full relative flex flex-col ${aspect} ${containerClassName}`}>
+      <video className={`m-0 ${videoSize} ${className}`} {...props}>
+        <source src={src} type="video/mp4" />
+        Sorry, your browser does not support embedded videos.
+      </video>
+    </div>
+  );
+}

demo/frontend/src/common/loading/UploadLoadingScreen.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ChangeVideoModal from '@/common/components/gallery/ChangeVideoModal';
+import type {VideoGalleryTriggerProps} from '@/common/components/gallery/DemoVideoGalleryModal';
+import LoadingStateScreen from '@/common/loading/LoadingStateScreen';
+import {uploadingStateAtom} from '@/demo/atoms';
+import {ImageCopy} from '@carbon/icons-react';
+import {useAtomValue} from 'jotai';
+import OptionButton from '../components/options/OptionButton';
+
+export default function UploadLoadingScreen() {
+  const uploadingState = useAtomValue(uploadingStateAtom);
+
+  if (uploadingState === 'error') {
+    return (
+      <LoadingStateScreen
+        title="Uh oh, we cannot process this video"
+        description="Please upload another video, and make sure that the video’s file size is less than 70Mb. ">
+        <div className="max-w-[250px] w-full mx-auto">
+          <ChangeVideoModal
+            videoGalleryModalTrigger={UploadLoadingScreenChangeVideoTrigger}
+          />
+        </div>
+      </LoadingStateScreen>
+    );
+  }
+
+  return (
+    <LoadingStateScreen
+      title="Uploading video..."
+      description="Sit tight while we upload your video."
+    />
+  );
+}
+
+function UploadLoadingScreenChangeVideoTrigger({
+  onClick,
+}: VideoGalleryTriggerProps) {
+  return (
+    <OptionButton
+      variant="gradient"
+      title="Change video"
+      Icon={ImageCopy}
+      onClick={onClick}
+    />
+  );
+}

demo/frontend/src/common/logger/DemoLogger.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {RenderingErrorType} from '@/common/error/ErrorUtils';
+import Logger from './Logger';
+
+type UploadSourceType = 'gallery' | 'option';
+
+// Maps event names to an optional payload for each event
+type DemoEventMap = {
+  // User events
+  user_click_canvas: {
+    click_type: 'add_point' | 'remove_point';
+    click_action: 'add_object' | 'refine_object';
+    click_variant?: 'positive' | 'negative';
+  };
+  user_click_object: {
+    tracklet_id: number;
+  };
+  user_click_track_and_play: {
+    track_and_play_click_type: 'stream' | 'abort';
+  };
+  user_click_apply_effect: {
+    effect_type: 'background' | 'object';
+    effect_name: string;
+    effect_variant: number;
+  };
+  user_change_video: {
+    gallery_video_url: string;
+  };
+  user_upload_video: {
+    upload_source: UploadSourceType;
+  };
+  user_click_share: {
+    gallery_video_url: string;
+  };
+  user_click_download: {
+    gallery_video_url: string;
+  };
+  user_click_web_share: undefined;
+  // Error events
+  client_error_rendering: {
+    rendering_error_type: RenderingErrorType;
+  };
+  client_error_start_session: undefined;
+  client_error_upload_video: {
+    upload_source: UploadSourceType;
+    upload_error_message: string;
+  };
+  client_error_unsupported_browser: undefined;
+  client_error_page_not_found: {
+    path: string;
+  };
+  client_error_general: {
+    message: string;
+  };
+  client_error_fallback: {
+    fallback_error_message: string;
+  };
+
+  // Dataset events
+  client_error_fallback_dataset: {
+    dataset_fallback_error_message: string;
+  };
+  dataset_client_impression_event: {
+    impression_type: 'grid_view' | 'detailed_view';
+    video_id?: string;
+  };
+  dataset_client_click_events: {
+    click_type: 'search' | 'next_page' | 'prev_page';
+    video_id?: string;
+  };
+};
+
+export interface LoggerInterface<TEventMap> {
+  event: <K extends keyof TEventMap>(
+    eventName: K,
+    options?: TEventMap[K],
+  ) => void;
+}
+
+export function initialize(): void {
+  // noop
+}
+
+export class DemoLogger implements LoggerInterface<DemoEventMap> {
+  event<K extends keyof DemoEventMap>(eventName: K, options?: DemoEventMap[K]) {
+    Logger.info(eventName, options ?? {});
+  }
+}

demo/frontend/src/common/logger/LogEnvironment.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {LogLevel} from '@/common/logger/Logger';
+
+// Only enable debug logging in modes that are set in MODES_WITH_LOGGER. The
+// default is always error only.
+export const LOG_LEVEL: LogLevel =
+  import.meta.env.MODE === 'production' ? 'debug' : 'error';

demo/frontend/src/common/logger/Logger.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {LOG_LEVEL} from './LogEnvironment';
+
+/** Signature of a logging function */
+export type LogFn = {
+  (message?: unknown, ...optionalParams: unknown[]): void;
+};
+
+/** Basic logger interface */
+export interface Logger {
+  info: LogFn;
+  warn: LogFn;
+  error: LogFn;
+  debug: LogFn;
+}
+
+/** Log levels */
+export type LogLevel = 'info' | 'warn' | 'error' | 'debug';
+
+const NO_OP: LogFn = (_message?: unknown, ..._optionalParams: unknown[]) => {};
+
+/** Logger which outputs to the browser console */
+export class ConsoleLogger implements Logger {
+  readonly info: LogFn;
+  readonly warn: LogFn;
+  readonly error: LogFn;
+  readonly debug: LogFn;
+
+  constructor(options?: {level?: LogLevel}) {
+    const {level} = options || {};
+
+    // eslint-disable-next-line no-console
+    this.error = console.error.bind(console);
+
+    if (level === 'error') {
+      this.debug = NO_OP;
+      this.warn = NO_OP;
+      this.info = NO_OP;
+
+      return;
+    }
+
+    // eslint-disable-next-line no-console
+    this.warn = console.warn.bind(console);
+
+    if (level === 'warn') {
+      this.debug = NO_OP;
+      this.info = NO_OP;
+
+      return;
+    }
+
+    // eslint-disable-next-line no-console
+    this.info = console.log.bind(console);
+
+    if (level === 'info') {
+      this.debug = NO_OP;
+
+      return;
+    }
+
+    // eslint-disable-next-line no-console
+    this.debug = console.debug.bind(console);
+  }
+}
+
+export default new ConsoleLogger({level: LOG_LEVEL});

demo/frontend/src/common/screen/useScreenSize.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {screenSizes} from '@/theme/tokens.stylex';
+import {useLayoutEffect, useState} from 'react';
+
+export default function useScreenSize(): {
+  screenSize: number;
+  isMobile: boolean;
+} {
+  const [screenSize, setScreenSize] = useState<number>(0);
+
+  useLayoutEffect(() => {
+    const updateSize = (): void => {
+      setScreenSize(window.innerWidth);
+    };
+    window.addEventListener('resize', updateSize);
+    updateSize();
+    return (): void => window.removeEventListener('resize', updateSize);
+  }, []);
+
+  return {isMobile: screenSize < screenSizes['md'], screenSize};
+}

demo/frontend/src/common/tracker/SAM2Model.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {generateThumbnail} from '@/common/components/video/editor/VideoEditorUtils';
+import VideoWorkerContext from '@/common/components/video/VideoWorkerContext';
+import Logger from '@/common/logger/Logger';
+import {
+  SAM2ModelAddNewPointsMutation,
+  SAM2ModelAddNewPointsMutation$data,
+} from '@/common/tracker/__generated__/SAM2ModelAddNewPointsMutation.graphql';
+import {SAM2ModelCancelPropagateInVideoMutation} from '@/common/tracker/__generated__/SAM2ModelCancelPropagateInVideoMutation.graphql';
+import {SAM2ModelClearPointsInFrameMutation} from '@/common/tracker/__generated__/SAM2ModelClearPointsInFrameMutation.graphql';
+import {SAM2ModelClearPointsInVideoMutation} from '@/common/tracker/__generated__/SAM2ModelClearPointsInVideoMutation.graphql';
+import {SAM2ModelCloseSessionMutation} from '@/common/tracker/__generated__/SAM2ModelCloseSessionMutation.graphql';
+import {SAM2ModelRemoveObjectMutation} from '@/common/tracker/__generated__/SAM2ModelRemoveObjectMutation.graphql';
+import {SAM2ModelStartSessionMutation} from '@/common/tracker/__generated__/SAM2ModelStartSessionMutation.graphql';
+import {
+  BaseTracklet,
+  Mask,
+  SegmentationPoint,
+  StreamingState,
+  Tracker,
+  Tracklet,
+} from '@/common/tracker/Tracker';
+import {TrackerOptions} from '@/common/tracker/Trackers';
+import {
+  ClearPointsInVideoResponse,
+  SessionStartFailedResponse,
+  SessionStartedResponse,
+  StreamingCompletedResponse,
+  StreamingStartedResponse,
+  StreamingStateUpdateResponse,
+  TrackletCreatedResponse,
+  TrackletDeletedResponse,
+  TrackletsUpdatedResponse,
+} from '@/common/tracker/TrackerTypes';
+import {convertMaskToRGBA} from '@/common/utils/MaskUtils';
+import multipartStream from '@/common/utils/MultipartStream';
+import {Stats} from '@/debug/stats/Stats';
+import {INFERENCE_API_ENDPOINT} from '@/demo/DemoConfig';
+import {createEnvironment} from '@/graphql/RelayEnvironment';
+import {
+  DataArray,
+  Masks,
+  RLEObject,
+  decode,
+  encode,
+  toBbox,
+} from '@/jscocotools/mask';
+import {THEME_COLORS} from '@/theme/colors';
+import invariant from 'invariant';
+import {IEnvironment, commitMutation, graphql} from 'relay-runtime';
+
+type Options = Pick<TrackerOptions, 'inferenceEndpoint'>;
+
+type Session = {
+  id: string | null;
+  tracklets: {[id: number]: Tracklet};
+};
+
+type StreamMasksResult = {
+  frameIndex: number;
+  rleMaskList: Array<{
+    objectId: number;
+    rleMask: RLEObject;
+  }>;
+};
+
+type StreamMasksAbortResult = {
+  aborted: boolean;
+};
+
+export class SAM2Model extends Tracker {
+  private _endpoint: string;
+  private _environment: IEnvironment;
+
+  private abortController: AbortController | null = null;
+  private _session: Session = {
+    id: null,
+    tracklets: {},
+  };
+  private _streamingState: StreamingState = 'none';
+
+  private _emptyMask: RLEObject | null = null;
+
+  private _maskCanvas: OffscreenCanvas;
+  private _maskCtx: OffscreenCanvasRenderingContext2D;
+
+  private _stats?: Stats;
+
+  constructor(
+    context: VideoWorkerContext,
+    options: Options = {
+      inferenceEndpoint: INFERENCE_API_ENDPOINT,
+    },
+  ) {
+    super(context);
+    this._endpoint = options.inferenceEndpoint;
+    this._environment = createEnvironment(options.inferenceEndpoint);
+
+    this._maskCanvas = new OffscreenCanvas(0, 0);
+    const maskCtx = this._maskCanvas.getContext('2d');
+    invariant(maskCtx != null, 'context cannot be null');
+    this._maskCtx = maskCtx;
+  }
+
+  public startSession(videoPath: string): Promise<void> {
+    // Reset streaming state. Force update with the true flag to make sure the
+    // UI updates its state.
+    this._updateStreamingState('none', true);
+
+    return new Promise(resolve => {
+      try {
+        commitMutation<SAM2ModelStartSessionMutation>(this._environment, {
+          mutation: graphql`
+            mutation SAM2ModelStartSessionMutation($input: StartSessionInput!) {
+              startSession(input: $input) {
+                sessionId
+              }
+            }
+          `,
+          variables: {
+            input: {
+              path: videoPath,
+            },
+          },
+          onCompleted: response => {
+            const {sessionId} = response.startSession;
+            this._session.id = sessionId;
+
+            this._sendResponse<SessionStartedResponse>('sessionStarted', {
+              sessionId,
+            });
+
+            // Clear any tracklets from the previous session when
+            // a new session is started
+            this._clearTracklets();
+
+            // Make an empty tracklet
+            this.createTracklet();
+            resolve();
+          },
+          onError: error => {
+            Logger.error(error);
+            this._sendResponse<SessionStartFailedResponse>(
+              'sessionStartFailed',
+            );
+            resolve();
+          },
+        });
+      } catch (error) {
+        Logger.error(error);
+        this._sendResponse<SessionStartFailedResponse>('sessionStartFailed');
+        resolve();
+      }
+    });
+  }
+
+  public closeSession(): Promise<void> {
+    const sessionId = this._session.id;
+
+    // Do not call cleanup before retrieving the session id because cleanup
+    // will reset the session id. If the order would be changed, it would
+    // never execute the closeSession mutation.
+    this._cleanup();
+
+    if (sessionId === null) {
+      return Promise.resolve();
+    }
+    return new Promise((resolve, reject) => {
+      commitMutation<SAM2ModelCloseSessionMutation>(this._environment, {
+        mutation: graphql`
+          mutation SAM2ModelCloseSessionMutation($input: CloseSessionInput!) {
+            closeSession(input: $input) {
+              success
+            }
+          }
+        `,
+        variables: {
+          input: {
+            sessionId,
+          },
+        },
+        onCompleted: response => {
+          const {success} = response.closeSession;
+          if (success === false) {
+            reject(new Error('Failed to close session'));
+            return;
+          }
+          resolve();
+        },
+        onError: error => {
+          Logger.error(error);
+          reject(error);
+        },
+      });
+    });
+  }
+
+  public createTracklet(): void {
+    // This will return 0 for for empty tracklets and otherwise the next
+    // largest number.
+    const nextId =
+      Object.values(this._session.tracklets).reduce(
+        (prev, curr) => Math.max(prev, curr.id),
+        -1,
+      ) + 1;
+
+    const newTracklet = {
+      id: nextId,
+      color: THEME_COLORS[nextId % THEME_COLORS.length],
+      thumbnail: null,
+      points: [],
+      masks: [],
+      isInitialized: false,
+    };
+
+    this._session.tracklets[nextId] = newTracklet;
+
+    // Notify the main thread
+    this._updateTracklets();
+
+    this._sendResponse<TrackletCreatedResponse>('trackletCreated', {
+      tracklet: newTracklet,
+    });
+  }
+
+  public deleteTracklet(trackletId: number): Promise<void> {
+    const sessionId = this._session.id;
+    if (sessionId === null) {
+      return Promise.reject('No active session');
+    }
+
+    const tracklet = this._session.tracklets[trackletId];
+    invariant(
+      tracklet != null,
+      'tracklet for tracklet id %s not initialized',
+      trackletId,
+    );
+
+    return new Promise((resolve, reject) => {
+      commitMutation<SAM2ModelRemoveObjectMutation>(this._environment, {
+        mutation: graphql`
+          mutation SAM2ModelRemoveObjectMutation($input: RemoveObjectInput!) {
+            removeObject(input: $input) {
+              frameIndex
+              rleMaskList {
+                objectId
+                rleMask {
+                  counts
+                  size
+                }
+              }
+            }
+          }
+        `,
+        variables: {
+          input: {objectId: trackletId, sessionId},
+        },
+        onCompleted: response => {
+          const trackletUpdates = response.removeObject;
+          this._sendResponse<TrackletDeletedResponse>('trackletDeleted', {
+            isSuccessful: true,
+          });
+          for (const trackletUpdate of trackletUpdates) {
+            this._updateTrackletMasks(
+              trackletUpdate,
+              trackletUpdate.frameIndex === this._context.frameIndex,
+              false, // shouldGoToFrame
+            );
+          }
+          this._removeTrackletMasks(tracklet);
+          resolve();
+        },
+        onError: error => {
+          this._sendResponse<TrackletDeletedResponse>('trackletDeleted', {
+            isSuccessful: false,
+          });
+          Logger.error(error);
+          reject(error);
+        },
+      });
+    });
+  }
+
+  public updatePoints(
+    frameIndex: number,
+    objectId: number,
+    points: SegmentationPoint[],
+  ): Promise<void> {
+    const sessionId = this._session.id;
+    if (sessionId === null) {
+      return Promise.reject('No active session');
+    }
+
+    // TODO: This is not the right place to initialize the empty mask.
+    // Move this into the constructor and listen to events on the context.
+    // Note, the initial context.width and context.height is 0, so it needs
+    // to happen based on an event, so when the video is initialized, it needs
+    // to notify the tracker to update the empty mask.
+    if (this._emptyMask === null) {
+      // We need to round the height/width to the nearest integer since
+      // Masks.toTensor() expects an integer value for the height/width.
+      const tensor = new Masks(
+        Math.trunc(this._context.height),
+        Math.trunc(this._context.width),
+        1,
+      ).toDataArray();
+      this._emptyMask = encode(tensor)[0];
+    }
+
+    const tracklet = this._session.tracklets[objectId];
+    invariant(
+      tracklet != null,
+      'tracklet for object id %s not initialized',
+      objectId,
+    );
+
+    // Mark session needing propagation when point is set
+    this._updateStreamingState('required');
+
+    // Clear all points in frame if no points are provided.
+    if (points.length === 0) {
+      return this.clearPointsInFrame(frameIndex, objectId);
+    }
+    return new Promise((resolve, reject) => {
+      const normalizedPoints = points.map(p => [
+        p[0] / this._context.width,
+        p[1] / this._context.height,
+      ]);
+      const labels = points.map(p => p[2]);
+      commitMutation<SAM2ModelAddNewPointsMutation>(this._environment, {
+        mutation: graphql`
+          mutation SAM2ModelAddNewPointsMutation($input: AddPointsInput!) {
+            addPoints(input: $input) {
+              frameIndex
+              rleMaskList {
+                objectId
+                rleMask {
+                  counts
+                  size
+                }
+              }
+            }
+          }
+        `,
+        variables: {
+          input: {
+            sessionId,
+            frameIndex,
+            objectId,
+            labels: labels,
+            points: normalizedPoints,
+            clearOldPoints: true,
+          },
+        },
+        onCompleted: response => {
+          tracklet.points[frameIndex] = points;
+          tracklet.isInitialized = true;
+          this._updateTrackletMasks(response.addPoints, true);
+          resolve();
+        },
+        onError: error => {
+          Logger.error(error);
+          reject(error);
+        },
+      });
+    });
+  }
+
+  public clearPointsInFrame(
+    frameIndex: number,
+    objectId: number,
+  ): Promise<void> {
+    const sessionId = this._session.id;
+    if (sessionId === null) {
+      return Promise.reject('No active session');
+    }
+
+    const tracklet = this._session.tracklets[objectId];
+    invariant(
+      tracklet != null,
+      'tracklet for object id %s not initialized',
+      objectId,
+    );
+
+    // Mark session needing propagation when point is set
+    this._updateStreamingState('required');
+
+    return new Promise((resolve, reject) => {
+      commitMutation<SAM2ModelClearPointsInFrameMutation>(this._environment, {
+        mutation: graphql`
+          mutation SAM2ModelClearPointsInFrameMutation(
+            $input: ClearPointsInFrameInput!
+          ) {
+            clearPointsInFrame(input: $input) {
+              frameIndex
+              rleMaskList {
+                objectId
+                rleMask {
+                  counts
+                  size
+                }
+              }
+            }
+          }
+        `,
+        variables: {
+          input: {
+            sessionId,
+            frameIndex,
+            objectId,
+          },
+        },
+        onCompleted: response => {
+          tracklet.points[frameIndex] = [];
+          tracklet.isInitialized = true;
+          this._updateTrackletMasks(response.clearPointsInFrame, true);
+          resolve();
+        },
+        onError: error => {
+          Logger.error(error);
+          reject(error);
+        },
+      });
+    });
+  }
+
+  public clearPointsInVideo(): Promise<void> {
+    const sessionId = this._session.id;
+    if (sessionId === null) {
+      return Promise.reject('No active session');
+    }
+
+    // Mark session needing propagation when point is set
+    this._updateStreamingState('none');
+
+    return new Promise(resolve => {
+      commitMutation<SAM2ModelClearPointsInVideoMutation>(this._environment, {
+        mutation: graphql`
+          mutation SAM2ModelClearPointsInVideoMutation(
+            $input: ClearPointsInVideoInput!
+          ) {
+            clearPointsInVideo(input: $input) {
+              success
+            }
+          }
+        `,
+        variables: {
+          input: {
+            sessionId,
+          },
+        },
+        onCompleted: response => {
+          const {success} = response.clearPointsInVideo;
+          if (!success) {
+            this._sendResponse<ClearPointsInVideoResponse>(
+              'clearPointsInVideo',
+              {isSuccessful: false},
+            );
+            return;
+          }
+
+          // Reset points and masks for each tracklet
+          this._clearTracklets();
+
+          // Notify the main thread
+          this._context.goToFrame(this._context.frameIndex);
+          this._updateTracklets();
+          this._sendResponse<ClearPointsInVideoResponse>('clearPointsInVideo', {
+            isSuccessful: true,
+          });
+          resolve();
+        },
+        onError: error => {
+          this._sendResponse<ClearPointsInVideoResponse>('clearPointsInVideo', {
+            isSuccessful: false,
+          });
+          Logger.error(error);
+        },
+      });
+    });
+  }
+
+  public async streamMasks(frameIndex: number): Promise<void> {
+    const sessionId = this._session.id;
+    if (sessionId === null) {
+      return Promise.reject('No active session');
+    }
+    try {
+      this._sendResponse<StreamingStartedResponse>('streamingStarted');
+
+      // 1. Clear previous masks
+      this._context.clearMasks();
+      this._clearTrackletMasks();
+
+      // 2. Create abort controller and async generator
+      const controller = new AbortController();
+      this.abortController = controller;
+
+      this._updateStreamingState('requesting');
+      const generator = this._streamMasksForSession(
+        controller,
+        sessionId,
+        frameIndex,
+      );
+
+      // 3. parse stream response and update masks in session objects
+      let isAborted = false;
+      for await (const result of generator) {
+        if ('aborted' in result) {
+          this._updateStreamingState('aborting');
+          await this._abortRequest();
+          this._updateStreamingState('aborted');
+          isAborted = true;
+        } else {
+          await this._updateTrackletMasks(result, false);
+          this._updateStreamingState('partial');
+        }
+      }
+
+      if (!isAborted) {
+        // Mark session needing propagation when point is set
+        this._updateStreamingState('full');
+      }
+    } catch (error) {
+      Logger.error(error);
+      throw error;
+    }
+
+    this._sendResponse<StreamingCompletedResponse>('streamingCompleted');
+  }
+
+  public abortStreamMasks() {
+    this.abortController?.abort();
+    this._sendResponse<StreamingCompletedResponse>('streamingCompleted');
+  }
+
+  public enableStats(): void {
+    this._stats = new Stats('ms', 'D', 1000 / 25);
+  }
+
+  // PRIVATE
+
+  private _cleanup() {
+    this._session.id = null;
+    // Clear existing tracklets
+    this._session.tracklets = [];
+  }
+
+  private _clearTracklets() {
+    this._session.tracklets = [];
+    this._context.clearMasks();
+  }
+
+  private _updateStreamingState(
+    state: StreamingState,
+    forceUpdate: boolean = false,
+  ) {
+    if (!forceUpdate && this._streamingState === state) {
+      return;
+    }
+    this._streamingState = state;
+    this._sendResponse<StreamingStateUpdateResponse>('streamingStateUpdate', {
+      state,
+    });
+  }
+
+  private async _removeTrackletMasks(tracklet: Tracklet) {
+    this._context.clearTrackletMasks(tracklet);
+    delete this._session.tracklets[tracklet.id];
+
+    // Notify the main thread
+    this._context.goToFrame(this._context.frameIndex);
+    this._updateTracklets();
+  }
+
+  private async _updateTrackletMasks(
+    data: SAM2ModelAddNewPointsMutation$data['addPoints'],
+    updateThumbnails: boolean,
+    shouldGoToFrame: boolean = true,
+  ) {
+    const {frameIndex, rleMaskList} = data;
+
+    // 1. parse and decode masks for all objects
+    for (const {objectId, rleMask} of rleMaskList) {
+      const track = this._session.tracklets[objectId];
+      const {size, counts} = rleMask;
+      const rleObject: RLEObject = {
+        size: [size[0], size[1]],
+        counts: counts,
+      };
+      const isEmpty = counts === this._emptyMask?.counts;
+
+      this._stats?.begin();
+
+      const decodedMask = decode([rleObject]);
+      const bbox = toBbox([rleObject]);
+
+      const mask: Mask = {
+        data: rleObject as RLEObject,
+        shape: [...decodedMask.shape],
+        bounds: [
+          [bbox[0], bbox[1]],
+          [bbox[0] + bbox[2], bbox[1] + bbox[3]],
+        ],
+        isEmpty,
+      } as const;
+      track.masks[frameIndex] = mask;
+
+      if (updateThumbnails && !isEmpty) {
+        const {ctx} = await this._compressMaskForCanvas(decodedMask);
+        const frame = this._context.currentFrame as VideoFrame;
+        await generateThumbnail(track, frameIndex, mask, frame, ctx);
+      }
+    }
+
+    this._context.updateTracklets(
+      frameIndex,
+      Object.values(this._session.tracklets),
+      shouldGoToFrame,
+    );
+
+    // Notify the main thread
+    this._updateTracklets();
+  }
+
+  private _updateTracklets() {
+    const tracklets: BaseTracklet[] = Object.values(
+      this._session.tracklets,
+    ).map(tracklet => {
+      // Notify the main thread
+      const {
+        id,
+        color,
+        isInitialized,
+        points: trackletPoints,
+        thumbnail,
+        masks,
+      } = tracklet;
+      return {
+        id,
+        color,
+        isInitialized,
+        points: trackletPoints,
+        thumbnail,
+        masks: masks.map(mask => ({
+          shape: mask.shape,
+          bounds: mask.bounds,
+          isEmpty: mask.isEmpty,
+        })),
+      };
+    });
+
+    this._sendResponse<TrackletsUpdatedResponse>('trackletsUpdated', {
+      tracklets,
+    });
+  }
+
+  private _clearTrackletMasks() {
+    const keys = Object.keys(this._session.tracklets);
+    for (const key of keys) {
+      const trackletId = Number(key);
+      const tracklet = {...this._session.tracklets[trackletId], masks: []};
+      this._session.tracklets[trackletId] = tracklet;
+    }
+    this._updateTracklets();
+  }
+
+  private async _compressMaskForCanvas(
+    decodedMask: DataArray,
+  ): Promise<{compressedData: Blob; ctx: OffscreenCanvasRenderingContext2D}> {
+    const data = convertMaskToRGBA(decodedMask.data as Uint8Array);
+
+    this._maskCanvas.width = decodedMask.shape[0];
+    this._maskCanvas.height = decodedMask.shape[1];
+
+    const imageData = new ImageData(
+      data,
+      decodedMask.shape[0],
+      decodedMask.shape[1],
+    );
+    this._maskCtx.putImageData(imageData, 0, 0);
+
+    const canvas = new OffscreenCanvas(
+      decodedMask.shape[1],
+      decodedMask.shape[0],
+    );
+
+    const ctx = canvas.getContext('2d');
+    invariant(ctx != null, 'context cannot be null');
+    ctx.save();
+    ctx.rotate(Math.PI / 2);
+    // Since the image was previously rotated 90° clockwise, after the image is rotated,
+    // we scale the canvas's width using scaleY and height using scaleX.
+    ctx.scale(1, -1);
+    ctx.drawImage(this._maskCanvas, 0, 0);
+    ctx.restore();
+
+    const compressedData = await canvas.convertToBlob({type: 'image/png'});
+
+    return {compressedData, ctx};
+  }
+
+  private async *_streamMasksForSession(
+    abortController: AbortController,
+    sessionId: string,
+    startFrameIndex: undefined | number = 0,
+  ): AsyncGenerator<StreamMasksResult | StreamMasksAbortResult, undefined> {
+    const url = `${this._endpoint}/propagate_in_video`;
+
+    const requestBody = {
+      session_id: sessionId,
+      start_frame_index: startFrameIndex,
+    };
+
+    const headers: {[name: string]: string} = Object.assign({
+      'Content-Type': 'application/json',
+    });
+
+    const response = await fetch(url, {
+      method: 'POST',
+      body: JSON.stringify(requestBody),
+      headers,
+    });
+
+    const contentType = response.headers.get('Content-Type');
+    if (
+      contentType == null ||
+      !contentType.startsWith('multipart/x-savi-stream;')
+    ) {
+      throw new Error(
+        'endpoint needs to support Content-Type "multipart/x-savi-stream"',
+      );
+    }
+
+    const responseBody = response.body;
+    if (responseBody == null) {
+      throw new Error('response body is null');
+    }
+
+    const reader = multipartStream(contentType, responseBody).getReader();
+
+    const textDecoder = new TextDecoder();
+
+    while (true) {
+      if (abortController.signal.aborted) {
+        reader.releaseLock();
+        yield {aborted: true};
+        return;
+      }
+
+      const {done, value} = await reader.read();
+      if (done) {
+        return;
+      }
+
+      const {headers, body} = value;
+
+      const contentType = headers.get('Content-Type') as string;
+
+      if (contentType.startsWith('application/json')) {
+        const jsonResponse = JSON.parse(textDecoder.decode(body));
+        const maskResults = jsonResponse.results;
+        const rleMaskList = maskResults.map(
+          (mask: {object_id: number; mask: RLEObject}) => {
+            return {
+              objectId: mask.object_id,
+              rleMask: mask.mask,
+            };
+          },
+        );
+        yield {
+          frameIndex: jsonResponse.frame_index,
+          rleMaskList,
+        };
+      }
+    }
+  }
+
+  private async _abortRequest(): Promise<void> {
+    const sessionId = this._session.id;
+    invariant(sessionId != null, 'session id cannot be empty');
+    return new Promise((resolve, reject) => {
+      try {
+        commitMutation<SAM2ModelCancelPropagateInVideoMutation>(
+          this._environment,
+          {
+            mutation: graphql`
+              mutation SAM2ModelCancelPropagateInVideoMutation(
+                $input: CancelPropagateInVideoInput!
+              ) {
+                cancelPropagateInVideo(input: $input) {
+                  success
+                }
+              }
+            `,
+            variables: {
+              input: {
+                sessionId,
+              },
+            },
+            onCompleted: response => {
+              const {success} = response.cancelPropagateInVideo;
+              if (!success) {
+                reject(`could not abort session ${sessionId}`);
+                return;
+              }
+              resolve();
+            },
+            onError: error => {
+              Logger.error(error);
+              reject(error);
+            },
+          },
+        );
+      } catch (error) {
+        Logger.error(error);
+        reject(error);
+      }
+    });
+  }
+}

demo/frontend/src/common/tracker/Tracker.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import VideoWorkerContext from '@/common/components/video/VideoWorkerContext';
+
+import {TrackerOptions} from '@/common/tracker/Trackers';
+import {TrackerResponse} from '@/common/tracker/TrackerTypes';
+import {RLEObject} from '@/jscocotools/mask';
+
+export type Point = [x: number, y: number];
+
+export type SegmentationPoint = [...point: Point, label: 0 | 1];
+
+export type FramePoints = Array<SegmentationPoint> | undefined;
+
+export type Mask = DatalessMask & {
+  data: Blob | RLEObject;
+};
+
+export type DatalessMask = {
+  shape: number[];
+  bounds: [[number, number], [number, number]];
+  isEmpty: boolean;
+};
+
+export type Tracklet = {
+  id: number;
+  color: string;
+  thumbnail: string | null;
+  points: FramePoints[];
+  masks: Mask[];
+  isInitialized: boolean;
+};
+
+export type BaseTracklet = Omit<Tracklet, 'masks'> & {
+  masks: DatalessMask[];
+};
+
+export type StreamingState =
+  | 'none'
+  | 'required'
+  | 'requesting'
+  | 'aborting'
+  | 'aborted'
+  | 'partial'
+  | 'full';
+
+export interface ITracker {
+  startSession(videoUrl: string): Promise<void>;
+  closeSession(): Promise<void>;
+  createTracklet(): void;
+  deleteTracklet(trackletId: number): Promise<void>;
+  updatePoints(
+    frameIndex: number,
+    objectId: number,
+    points: SegmentationPoint[],
+  ): Promise<void>;
+  clearPointsInFrame(frameIndex: number, objectId: number): Promise<void>;
+  clearPointsInVideo(): Promise<void>;
+  streamMasks(frameIndex: number): Promise<void>;
+  abortStreamMasks(): void;
+  enableStats(): void;
+}
+
+export abstract class Tracker implements ITracker {
+  protected _context: VideoWorkerContext;
+  constructor(context: VideoWorkerContext, _options?: TrackerOptions) {
+    this._context = context;
+  }
+  abstract startSession(videoUrl: string): Promise<void>;
+  abstract closeSession(): Promise<void>;
+  abstract createTracklet(): void;
+  abstract deleteTracklet(trackletId: number): Promise<void>;
+  abstract updatePoints(
+    frameIndex: number,
+    objectId: number,
+    points: SegmentationPoint[],
+  ): Promise<void>;
+  abstract clearPointsInFrame(
+    frameIndex: number,
+    objectId: number,
+  ): Promise<void>;
+  abstract clearPointsInVideo(): Promise<void>;
+  abstract streamMasks(frameIndex: number): Promise<void>;
+  abstract abortStreamMasks(): void;
+  abstract enableStats(): void;
+
+  // PRIVATE FUNCTIONS
+
+  protected _sendResponse<T extends TrackerResponse>(
+    action: T['action'],
+    message?: Omit<T, 'action'>,
+    transfer?: Transferable[],
+  ): void {
+    self.postMessage(
+      {
+        action,
+        ...message,
+      },
+      {
+        transfer,
+      },
+    );
+  }
+}

demo/frontend/src/common/tracker/TrackerTypes.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {SegmentationPoint} from '@/common/tracker/Tracker';
+import {TrackerOptions, Trackers} from '@/common/tracker/Trackers';
+import {
+  AddPointsEvent,
+  ClearPointsInVideoEvent,
+  SessionStartFailedEvent,
+  SessionStartedEvent,
+  StreamingCompletedEvent,
+  StreamingStartedEvent,
+  StreamingStateUpdateEvent,
+  TrackletCreatedEvent,
+  TrackletDeletedEvent,
+  TrackletsEvent,
+} from '../components/video/VideoWorkerBridge';
+
+export type Flags = {
+  masks: boolean;
+  effect: boolean;
+};
+
+export type Request<A, P> = {
+  action: A;
+} & P;
+
+// REQUESTS
+
+export type InitializeTrackerRequest = Request<
+  'initializeTracker',
+  {
+    name: keyof Trackers;
+    options: TrackerOptions;
+  }
+>;
+export type StartSessionRequest = Request<
+  'startSession',
+  {
+    videoUrl: string;
+  }
+>;
+export type CloseSessionRequest = Request<'closeSession', unknown>;
+export type CreateTrackletRequest = Request<'createTracklet', unknown>;
+export type DeleteTrackletRequest = Request<
+  'deleteTracklet',
+  {
+    trackletId: number;
+  }
+>;
+export type UpdatePointsRequest = Request<
+  'updatePoints',
+  {
+    frameIndex: number;
+    objectId: number;
+    points: SegmentationPoint[];
+  }
+>;
+export type ClearPointsInFrameRequest = Request<
+  'clearPointsInFrame',
+  {
+    frameIndex: number;
+    objectId: number;
+  }
+>;
+export type ClearPointsInVideoRequest = Request<'clearPointsInVideo', unknown>;
+export type StreamMasksRequest = Request<
+  'streamMasks',
+  {
+    frameIndex: number;
+  }
+>;
+export type AbortStreamMasksRequest = Request<'abortStreamMasks', unknown>;
+
+export type LogAnnotationsRequest = Request<'logAnnotations', unknown>;
+
+export type TrackerRequest =
+  | InitializeTrackerRequest
+  | StartSessionRequest
+  | CloseSessionRequest
+  | CreateTrackletRequest
+  | DeleteTrackletRequest
+  | UpdatePointsRequest
+  | ClearPointsInFrameRequest
+  | ClearPointsInVideoRequest
+  | StreamMasksRequest
+  | AbortStreamMasksRequest
+  | LogAnnotationsRequest;
+
+export type TrackerRequestMessageEvent = MessageEvent<TrackerRequest>;
+
+// RESPONSES
+
+export type SessionStartedResponse = Request<
+  'sessionStarted',
+  SessionStartedEvent
+>;
+
+export type SessionStartFailedResponse = Request<
+  'sessionStartFailed',
+  SessionStartFailedEvent
+>;
+
+export type TrackletCreatedResponse = Request<
+  'trackletCreated',
+  TrackletCreatedEvent
+>;
+
+export type TrackletsUpdatedResponse = Request<
+  'trackletsUpdated',
+  TrackletsEvent
+>;
+
+export type TrackletDeletedResponse = Request<
+  'trackletDeleted',
+  TrackletDeletedEvent
+>;
+
+export type AddPointsResponse = Request<'addPoints', AddPointsEvent>;
+
+export type ClearPointsInVideoResponse = Request<
+  'clearPointsInVideo',
+  ClearPointsInVideoEvent
+>;
+
+export type StreamingStartedResponse = Request<
+  'streamingStarted',
+  StreamingStartedEvent
+>;
+
+export type StreamingCompletedResponse = Request<
+  'streamingCompleted',
+  StreamingCompletedEvent
+>;
+
+export type StreamingStateUpdateResponse = Request<
+  'streamingStateUpdate',
+  StreamingStateUpdateEvent
+>;
+
+export type TrackerResponse =
+  | SessionStartedResponse
+  | SessionStartFailedResponse
+  | TrackletCreatedResponse
+  | TrackletsUpdatedResponse
+  | TrackletDeletedResponse
+  | AddPointsResponse
+  | ClearPointsInVideoResponse
+  | StreamingStartedResponse
+  | StreamingCompletedResponse
+  | StreamingStateUpdateResponse;
+
+export type TrackerResponseMessageEvent = MessageEvent<TrackerResponse>;

demo/frontend/src/common/tracker/Trackers.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {SAM2Model} from './SAM2Model';
+
+export type Headers = {[name: string]: string};
+
+export type TrackerOptions = {
+  inferenceEndpoint: string;
+};
+
+export type Trackers = {
+  'SAM 2': typeof SAM2Model;
+};
+
+export const TRACKER_MAPPING: Trackers = {
+  'SAM 2': SAM2Model,
+};

demo/frontend/src/common/tracker/__generated__/SAM2ModelAddNewPointsMutation.graphql.ts

+/**
+ * @generated SignedSource<<db1ee50f3027130f61feafb624026897>>
+ * @lightSyntaxTransform
+ * @nogrep
+ */
+
+/* tslint:disable */
+/* eslint-disable */
+// @ts-nocheck
+
+import { ConcreteRequest, Mutation } from 'relay-runtime';
+export type AddPointsInput = {
+  clearOldPoints: boolean;
+  frameIndex: number;
+  labels: ReadonlyArray<number>;
+  objectId: number;
+  points: ReadonlyArray<ReadonlyArray<number>>;
+  sessionId: string;
+};
+export type SAM2ModelAddNewPointsMutation$variables = {
+  input: AddPointsInput;
+};
+export type SAM2ModelAddNewPointsMutation$data = {
+  readonly addPoints: {
+    readonly frameIndex: number;
+    readonly rleMaskList: ReadonlyArray<{
+      readonly objectId: number;
+      readonly rleMask: {
+        readonly counts: string;
+        readonly size: ReadonlyArray<number>;
+      };
+    }>;
+  };
+};
+export type SAM2ModelAddNewPointsMutation = {
+  response: SAM2ModelAddNewPointsMutation$data;
+  variables: SAM2ModelAddNewPointsMutation$variables;
+};
+
+const node: ConcreteRequest = (function(){
+var v0 = [
+  {
+    "defaultValue": null,
+    "kind": "LocalArgument",
+    "name": "input"
+  }
+],
+v1 = [
+  {
+    "alias": null,
+    "args": [
+      {
+        "kind": "Variable",
+        "name": "input",
+        "variableName": "input"
+      }
+    ],
+    "concreteType": "RLEMaskListOnFrame",
+    "kind": "LinkedField",
+    "name": "addPoints",
+    "plural": false,
+    "selections": [
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "frameIndex",
+        "storageKey": null
+      },
+      {
+        "alias": null,
+        "args": null,
+        "concreteType": "RLEMaskForObject",
+        "kind": "LinkedField",
+        "name": "rleMaskList",
+        "plural": true,
+        "selections": [
+          {
+            "alias": null,
+            "args": null,
+            "kind": "ScalarField",
+            "name": "objectId",
+            "storageKey": null
+          },
+          {
+            "alias": null,
+            "args": null,
+            "concreteType": "RLEMask",
+            "kind": "LinkedField",
+            "name": "rleMask",
+            "plural": false,
+            "selections": [
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "counts",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "size",
+                "storageKey": null
+              }
+            ],
+            "storageKey": null
+          }
+        ],
+        "storageKey": null
+      }
+    ],
+    "storageKey": null
+  }
+];
+return {
+  "fragment": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Fragment",
+    "metadata": null,
+    "name": "SAM2ModelAddNewPointsMutation",
+    "selections": (v1/*: any*/),
+    "type": "Mutation",
+    "abstractKey": null
+  },
+  "kind": "Request",
+  "operation": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Operation",
+    "name": "SAM2ModelAddNewPointsMutation",
+    "selections": (v1/*: any*/)
+  },
+  "params": {
+    "cacheID": "dc86527e91907e696683458ed0943d2f",
+    "id": null,
+    "metadata": {},
+    "name": "SAM2ModelAddNewPointsMutation",
+    "operationKind": "mutation",
+    "text": "mutation SAM2ModelAddNewPointsMutation(\n  $input: AddPointsInput!\n) {\n  addPoints(input: $input) {\n    frameIndex\n    rleMaskList {\n      objectId\n      rleMask {\n        counts\n        size\n      }\n    }\n  }\n}\n"
+  }
+};
+})();
+
+(node as any).hash = "3c96f05877dd91668c1f9e8a3f1203a5";
+
+export default node;

demo/frontend/src/common/tracker/__generated__/SAM2ModelCancelPropagateInVideoMutation.graphql.ts

+/**
+ * @generated SignedSource<<87827cb79ef9276cd5a66026151e937c>>
+ * @lightSyntaxTransform
+ * @nogrep
+ */
+
+/* tslint:disable */
+/* eslint-disable */
+// @ts-nocheck
+
+import { ConcreteRequest, Mutation } from 'relay-runtime';
+export type CancelPropagateInVideoInput = {
+  sessionId: string;
+};
+export type SAM2ModelCancelPropagateInVideoMutation$variables = {
+  input: CancelPropagateInVideoInput;
+};
+export type SAM2ModelCancelPropagateInVideoMutation$data = {
+  readonly cancelPropagateInVideo: {
+    readonly success: boolean;
+  };
+};
+export type SAM2ModelCancelPropagateInVideoMutation = {
+  response: SAM2ModelCancelPropagateInVideoMutation$data;
+  variables: SAM2ModelCancelPropagateInVideoMutation$variables;
+};
+
+const node: ConcreteRequest = (function(){
+var v0 = [
+  {
+    "defaultValue": null,
+    "kind": "LocalArgument",
+    "name": "input"
+  }
+],
+v1 = [
+  {
+    "alias": null,
+    "args": [
+      {
+        "kind": "Variable",
+        "name": "input",
+        "variableName": "input"
+      }
+    ],
+    "concreteType": "CancelPropagateInVideo",
+    "kind": "LinkedField",
+    "name": "cancelPropagateInVideo",
+    "plural": false,
+    "selections": [
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "success",
+        "storageKey": null
+      }
+    ],
+    "storageKey": null
+  }
+];
+return {
+  "fragment": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Fragment",
+    "metadata": null,
+    "name": "SAM2ModelCancelPropagateInVideoMutation",
+    "selections": (v1/*: any*/),
+    "type": "Mutation",
+    "abstractKey": null
+  },
+  "kind": "Request",
+  "operation": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Operation",
+    "name": "SAM2ModelCancelPropagateInVideoMutation",
+    "selections": (v1/*: any*/)
+  },
+  "params": {
+    "cacheID": "f00f78f24741d27828f0bd95b0f373c2",
+    "id": null,
+    "metadata": {},
+    "name": "SAM2ModelCancelPropagateInVideoMutation",
+    "operationKind": "mutation",
+    "text": "mutation SAM2ModelCancelPropagateInVideoMutation(\n  $input: CancelPropagateInVideoInput!\n) {\n  cancelPropagateInVideo(input: $input) {\n    success\n  }\n}\n"
+  }
+};
+})();
+
+(node as any).hash = "1abafecade479ab3c45f9cecf0360285";
+
+export default node;

demo/frontend/src/common/tracker/__generated__/SAM2ModelClearPointsInFrameMutation.graphql.ts

+/**
+ * @generated SignedSource<<7330d05db0fe66bbd89190cc665dd8d9>>
+ * @lightSyntaxTransform
+ * @nogrep
+ */
+
+/* tslint:disable */
+/* eslint-disable */
+// @ts-nocheck
+
+import { ConcreteRequest, Mutation } from 'relay-runtime';
+export type ClearPointsInFrameInput = {
+  frameIndex: number;
+  objectId: number;
+  sessionId: string;
+};
+export type SAM2ModelClearPointsInFrameMutation$variables = {
+  input: ClearPointsInFrameInput;
+};
+export type SAM2ModelClearPointsInFrameMutation$data = {
+  readonly clearPointsInFrame: {
+    readonly frameIndex: number;
+    readonly rleMaskList: ReadonlyArray<{
+      readonly objectId: number;
+      readonly rleMask: {
+        readonly counts: string;
+        readonly size: ReadonlyArray<number>;
+      };
+    }>;
+  };
+};
+export type SAM2ModelClearPointsInFrameMutation = {
+  response: SAM2ModelClearPointsInFrameMutation$data;
+  variables: SAM2ModelClearPointsInFrameMutation$variables;
+};
+
+const node: ConcreteRequest = (function(){
+var v0 = [
+  {
+    "defaultValue": null,
+    "kind": "LocalArgument",
+    "name": "input"
+  }
+],
+v1 = [
+  {
+    "alias": null,
+    "args": [
+      {
+        "kind": "Variable",
+        "name": "input",
+        "variableName": "input"
+      }
+    ],
+    "concreteType": "RLEMaskListOnFrame",
+    "kind": "LinkedField",
+    "name": "clearPointsInFrame",
+    "plural": false,
+    "selections": [
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "frameIndex",
+        "storageKey": null
+      },
+      {
+        "alias": null,
+        "args": null,
+        "concreteType": "RLEMaskForObject",
+        "kind": "LinkedField",
+        "name": "rleMaskList",
+        "plural": true,
+        "selections": [
+          {
+            "alias": null,
+            "args": null,
+            "kind": "ScalarField",
+            "name": "objectId",
+            "storageKey": null
+          },
+          {
+            "alias": null,
+            "args": null,
+            "concreteType": "RLEMask",
+            "kind": "LinkedField",
+            "name": "rleMask",
+            "plural": false,
+            "selections": [
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "counts",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "size",
+                "storageKey": null
+              }
+            ],
+            "storageKey": null
+          }
+        ],
+        "storageKey": null
+      }
+    ],
+    "storageKey": null
+  }
+];
+return {
+  "fragment": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Fragment",
+    "metadata": null,
+    "name": "SAM2ModelClearPointsInFrameMutation",
+    "selections": (v1/*: any*/),
+    "type": "Mutation",
+    "abstractKey": null
+  },
+  "kind": "Request",
+  "operation": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Operation",
+    "name": "SAM2ModelClearPointsInFrameMutation",
+    "selections": (v1/*: any*/)
+  },
+  "params": {
+    "cacheID": "b4f20e0205c26d5dc3614935ac73fa3f",
+    "id": null,
+    "metadata": {},
+    "name": "SAM2ModelClearPointsInFrameMutation",
+    "operationKind": "mutation",
+    "text": "mutation SAM2ModelClearPointsInFrameMutation(\n  $input: ClearPointsInFrameInput!\n) {\n  clearPointsInFrame(input: $input) {\n    frameIndex\n    rleMaskList {\n      objectId\n      rleMask {\n        counts\n        size\n      }\n    }\n  }\n}\n"
+  }
+};
+})();
+
+(node as any).hash = "880295870f14839040acf8f191fa1409";
+
+export default node;

demo/frontend/src/common/tracker/__generated__/SAM2ModelClearPointsInVideoMutation.graphql.ts

+/**
+ * @generated SignedSource<<092c43655450b8af706e546837e0a01c>>
+ * @lightSyntaxTransform
+ * @nogrep
+ */
+
+/* tslint:disable */
+/* eslint-disable */
+// @ts-nocheck
+
+import { ConcreteRequest, Mutation } from 'relay-runtime';
+export type ClearPointsInVideoInput = {
+  sessionId: string;
+};
+export type SAM2ModelClearPointsInVideoMutation$variables = {
+  input: ClearPointsInVideoInput;
+};
+export type SAM2ModelClearPointsInVideoMutation$data = {
+  readonly clearPointsInVideo: {
+    readonly success: boolean;
+  };
+};
+export type SAM2ModelClearPointsInVideoMutation = {
+  response: SAM2ModelClearPointsInVideoMutation$data;
+  variables: SAM2ModelClearPointsInVideoMutation$variables;
+};
+
+const node: ConcreteRequest = (function(){
+var v0 = [
+  {
+    "defaultValue": null,
+    "kind": "LocalArgument",
+    "name": "input"
+  }
+],
+v1 = [
+  {
+    "alias": null,
+    "args": [
+      {
+        "kind": "Variable",
+        "name": "input",
+        "variableName": "input"
+      }
+    ],
+    "concreteType": "ClearPointsInVideo",
+    "kind": "LinkedField",
+    "name": "clearPointsInVideo",
+    "plural": false,
+    "selections": [
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "success",
+        "storageKey": null
+      }
+    ],
+    "storageKey": null
+  }
+];
+return {
+  "fragment": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Fragment",
+    "metadata": null,
+    "name": "SAM2ModelClearPointsInVideoMutation",
+    "selections": (v1/*: any*/),
+    "type": "Mutation",
+    "abstractKey": null
+  },
+  "kind": "Request",
+  "operation": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Operation",
+    "name": "SAM2ModelClearPointsInVideoMutation",
+    "selections": (v1/*: any*/)
+  },
+  "params": {
+    "cacheID": "c23b3d5afca5b235328a562369056527",
+    "id": null,
+    "metadata": {},
+    "name": "SAM2ModelClearPointsInVideoMutation",
+    "operationKind": "mutation",
+    "text": "mutation SAM2ModelClearPointsInVideoMutation(\n  $input: ClearPointsInVideoInput!\n) {\n  clearPointsInVideo(input: $input) {\n    success\n  }\n}\n"
+  }
+};
+})();
+
+(node as any).hash = "020267989385cb8b8f0e5cdde784d17e";
+
+export default node;

demo/frontend/src/common/tracker/__generated__/SAM2ModelCloseSessionMutation.graphql.ts

+/**
+ * @generated SignedSource<<48ee5db240b8093e9e53bf0329c8bab7>>
+ * @lightSyntaxTransform
+ * @nogrep
+ */
+
+/* tslint:disable */
+/* eslint-disable */
+// @ts-nocheck
+
+import { ConcreteRequest, Mutation } from 'relay-runtime';
+export type CloseSessionInput = {
+  sessionId: string;
+};
+export type SAM2ModelCloseSessionMutation$variables = {
+  input: CloseSessionInput;
+};
+export type SAM2ModelCloseSessionMutation$data = {
+  readonly closeSession: {
+    readonly success: boolean;
+  };
+};
+export type SAM2ModelCloseSessionMutation = {
+  response: SAM2ModelCloseSessionMutation$data;
+  variables: SAM2ModelCloseSessionMutation$variables;
+};
+
+const node: ConcreteRequest = (function(){
+var v0 = [
+  {
+    "defaultValue": null,
+    "kind": "LocalArgument",
+    "name": "input"
+  }
+],
+v1 = [
+  {
+    "alias": null,
+    "args": [
+      {
+        "kind": "Variable",
+        "name": "input",
+        "variableName": "input"
+      }
+    ],
+    "concreteType": "CloseSession",
+    "kind": "LinkedField",
+    "name": "closeSession",
+    "plural": false,
+    "selections": [
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "success",
+        "storageKey": null
+      }
+    ],
+    "storageKey": null
+  }
+];
+return {
+  "fragment": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Fragment",
+    "metadata": null,
+    "name": "SAM2ModelCloseSessionMutation",
+    "selections": (v1/*: any*/),
+    "type": "Mutation",
+    "abstractKey": null
+  },
+  "kind": "Request",
+  "operation": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Operation",
+    "name": "SAM2ModelCloseSessionMutation",
+    "selections": (v1/*: any*/)
+  },
+  "params": {
+    "cacheID": "aa7177838c16536b397bfee2d15a94ee",
+    "id": null,
+    "metadata": {},
+    "name": "SAM2ModelCloseSessionMutation",
+    "operationKind": "mutation",
+    "text": "mutation SAM2ModelCloseSessionMutation(\n  $input: CloseSessionInput!\n) {\n  closeSession(input: $input) {\n    success\n  }\n}\n"
+  }
+};
+})();
+
+(node as any).hash = "6e1008de944562dc1922cd3f9cc40f10";
+
+export default node;

demo/frontend/src/common/tracker/__generated__/SAM2ModelRemoveObjectMutation.graphql.ts

+/**
+ * @generated SignedSource<<3d0d7bdc0d4304f08ea91b7df9efeb1f>>
+ * @lightSyntaxTransform
+ * @nogrep
+ */
+
+/* tslint:disable */
+/* eslint-disable */
+// @ts-nocheck
+
+import { ConcreteRequest, Mutation } from 'relay-runtime';
+export type RemoveObjectInput = {
+  objectId: number;
+  sessionId: string;
+};
+export type SAM2ModelRemoveObjectMutation$variables = {
+  input: RemoveObjectInput;
+};
+export type SAM2ModelRemoveObjectMutation$data = {
+  readonly removeObject: ReadonlyArray<{
+    readonly frameIndex: number;
+    readonly rleMaskList: ReadonlyArray<{
+      readonly objectId: number;
+      readonly rleMask: {
+        readonly counts: string;
+        readonly size: ReadonlyArray<number>;
+      };
+    }>;
+  }>;
+};
+export type SAM2ModelRemoveObjectMutation = {
+  response: SAM2ModelRemoveObjectMutation$data;
+  variables: SAM2ModelRemoveObjectMutation$variables;
+};
+
+const node: ConcreteRequest = (function(){
+var v0 = [
+  {
+    "defaultValue": null,
+    "kind": "LocalArgument",
+    "name": "input"
+  }
+],
+v1 = [
+  {
+    "alias": null,
+    "args": [
+      {
+        "kind": "Variable",
+        "name": "input",
+        "variableName": "input"
+      }
+    ],
+    "concreteType": "RLEMaskListOnFrame",
+    "kind": "LinkedField",
+    "name": "removeObject",
+    "plural": true,
+    "selections": [
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "frameIndex",
+        "storageKey": null
+      },
+      {
+        "alias": null,
+        "args": null,
+        "concreteType": "RLEMaskForObject",
+        "kind": "LinkedField",
+        "name": "rleMaskList",
+        "plural": true,
+        "selections": [
+          {
+            "alias": null,
+            "args": null,
+            "kind": "ScalarField",
+            "name": "objectId",
+            "storageKey": null
+          },
+          {
+            "alias": null,
+            "args": null,
+            "concreteType": "RLEMask",
+            "kind": "LinkedField",
+            "name": "rleMask",
+            "plural": false,
+            "selections": [
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "counts",
+                "storageKey": null
+              },
+              {
+                "alias": null,
+                "args": null,
+                "kind": "ScalarField",
+                "name": "size",
+                "storageKey": null
+              }
+            ],
+            "storageKey": null
+          }
+        ],
+        "storageKey": null
+      }
+    ],
+    "storageKey": null
+  }
+];
+return {
+  "fragment": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Fragment",
+    "metadata": null,
+    "name": "SAM2ModelRemoveObjectMutation",
+    "selections": (v1/*: any*/),
+    "type": "Mutation",
+    "abstractKey": null
+  },
+  "kind": "Request",
+  "operation": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Operation",
+    "name": "SAM2ModelRemoveObjectMutation",
+    "selections": (v1/*: any*/)
+  },
+  "params": {
+    "cacheID": "0accbe68b8deea021539365678e58172",
+    "id": null,
+    "metadata": {},
+    "name": "SAM2ModelRemoveObjectMutation",
+    "operationKind": "mutation",
+    "text": "mutation SAM2ModelRemoveObjectMutation(\n  $input: RemoveObjectInput!\n) {\n  removeObject(input: $input) {\n    frameIndex\n    rleMaskList {\n      objectId\n      rleMask {\n        counts\n        size\n      }\n    }\n  }\n}\n"
+  }
+};
+})();
+
+(node as any).hash = "2dddf010d202332e6e012443cc1d8e55";
+
+export default node;

demo/frontend/src/common/tracker/__generated__/SAM2ModelStartSessionMutation.graphql.ts

+/**
+ * @generated SignedSource<<90910bae5bb646118174e736434aac56>>
+ * @lightSyntaxTransform
+ * @nogrep
+ */
+
+/* tslint:disable */
+/* eslint-disable */
+// @ts-nocheck
+
+import { ConcreteRequest, Mutation } from 'relay-runtime';
+export type StartSessionInput = {
+  path: string;
+};
+export type SAM2ModelStartSessionMutation$variables = {
+  input: StartSessionInput;
+};
+export type SAM2ModelStartSessionMutation$data = {
+  readonly startSession: {
+    readonly sessionId: string;
+  };
+};
+export type SAM2ModelStartSessionMutation = {
+  response: SAM2ModelStartSessionMutation$data;
+  variables: SAM2ModelStartSessionMutation$variables;
+};
+
+const node: ConcreteRequest = (function(){
+var v0 = [
+  {
+    "defaultValue": null,
+    "kind": "LocalArgument",
+    "name": "input"
+  }
+],
+v1 = [
+  {
+    "alias": null,
+    "args": [
+      {
+        "kind": "Variable",
+        "name": "input",
+        "variableName": "input"
+      }
+    ],
+    "concreteType": "StartSession",
+    "kind": "LinkedField",
+    "name": "startSession",
+    "plural": false,
+    "selections": [
+      {
+        "alias": null,
+        "args": null,
+        "kind": "ScalarField",
+        "name": "sessionId",
+        "storageKey": null
+      }
+    ],
+    "storageKey": null
+  }
+];
+return {
+  "fragment": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Fragment",
+    "metadata": null,
+    "name": "SAM2ModelStartSessionMutation",
+    "selections": (v1/*: any*/),
+    "type": "Mutation",
+    "abstractKey": null
+  },
+  "kind": "Request",
+  "operation": {
+    "argumentDefinitions": (v0/*: any*/),
+    "kind": "Operation",
+    "name": "SAM2ModelStartSessionMutation",
+    "selections": (v1/*: any*/)
+  },
+  "params": {
+    "cacheID": "2403f5005f5bb3805109874569f2050e",
+    "id": null,
+    "metadata": {},
+    "name": "SAM2ModelStartSessionMutation",
+    "operationKind": "mutation",
+    "text": "mutation SAM2ModelStartSessionMutation(\n  $input: StartSessionInput!\n) {\n  startSession(input: $input) {\n    sessionId\n  }\n}\n"
+  }
+};
+})();
+
+(node as any).hash = "5cf0005c7a54fc87c539dd4cbd5fef5d";
+
+export default node;

demo/frontend/src/common/utils/FileUtils.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import Logger from '@/common/logger/Logger';
+
+type Range = {
+  start: number;
+  end: number;
+};
+
+type FileStreamPart = {
+  data: Uint8Array;
+  range: Range;
+  contentLength: number;
+};
+
+export type FileStream = AsyncGenerator<FileStreamPart, File | null, null>;
+
+/**
+ * Asynchronously generates a SHA-256 hash for a Blob object.
+ *
+ * DO NOT USE this function casually. Computing the SHA-256 is expensive and can
+ * take several 100 milliseconds to complete.
+ *
+ * @param blob - The Blob object to be hashed.
+ * @returns A Promise that resolves to a string representing the SHA-256 hash of
+ * the Blob.
+ */
+export async function hashBlob(blob: Blob): Promise<string> {
+  const buffer = await blob.arrayBuffer();
+  // Crypto subtle is only availabe in secure contexts. For example, this will
+  // be the case when running the project locally with http protocol.
+  // https://developer.mozilla.org/en-US/docs/Web/API/Crypto/subtle
+  if (crypto.subtle != null) {
+    const hashBuffer = await crypto.subtle.digest('SHA-256', buffer);
+    const hashArray = Array.from(new Uint8Array(hashBuffer));
+    return hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
+  }
+  // If not secure context, return random string
+  return (Math.random() + 1).toString(36).substring(7);
+}
+
+export async function* streamFile(url: string, init?: RequestInit): FileStream {
+  try {
+    const response = await fetch(url, init);
+
+    let blob: Blob;
+
+    // Try to download the file with a stream reader. This has the benefit
+    // of providing progress during the download. It requires the body and
+    // Content-Length. As a fallback, it uses the blob function on the
+    // response object.
+    const contentLength = response.headers.get('Content-Length');
+    if (response.body != null && contentLength != null) {
+      const totalLength = parseInt(contentLength);
+      const chunks: Uint8Array[] = [];
+      let start = 0;
+      let end = 0;
+
+      const reader = response.body.getReader();
+      try {
+        while (true) {
+          const {done, value} = await reader.read();
+          if (done) {
+            break;
+          }
+
+          start = end;
+          end += value.length;
+
+          yield {
+            data: value,
+            range: {start, end},
+            contentLength: totalLength,
+          };
+        }
+      } finally {
+        reader.releaseLock();
+      }
+      blob = new Blob(chunks);
+    } else {
+      blob = await response.blob();
+    }
+
+    const filename = await hashBlob(blob);
+    return new File([blob], `${filename}.mp4`);
+  } catch (error) {
+    Logger.error('aborting download due to component unmount', error);
+  }
+  return null;
+}

demo/frontend/src/common/utils/ImageUtils.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export function convertVideoFrameToImageData(
+  videoFrame: VideoFrame,
+): ImageData | undefined {
+  const canvas = new OffscreenCanvas(
+    videoFrame.displayWidth,
+    videoFrame.displayHeight,
+  );
+  const ctx = canvas.getContext('2d');
+  ctx?.drawImage(videoFrame, 0, 0);
+  return ctx?.getImageData(0, 0, canvas.width, canvas.height);
+}
+
+/**
+ * This utility provides two functions:
+ * `process`: to find the bounding box of non-empty pixels from an ImageData, when looping through all its pixels
+ * `crop` to cut out the subsection found in `process`
+ * @returns
+ */
+export function findBoundingBox() {
+  let xMin = Number.MAX_VALUE;
+  let yMin = Number.MAX_VALUE;
+  let xMax = Number.MIN_VALUE;
+  let yMax = Number.MIN_VALUE;
+  return {
+    process: function (x: number, y: number, hasData: boolean) {
+      if (hasData) {
+        xMin = Math.min(x, xMin);
+        xMax = Math.max(x, xMax);
+        yMin = Math.min(y, yMin);
+        yMax = Math.max(y, yMax);
+      }
+      return [xMin, xMax, yMin, yMax];
+    },
+    crop(imageData: ImageData): ImageData | null {
+      const canvas = new OffscreenCanvas(imageData.width, imageData.height);
+      const ctx = canvas.getContext('2d');
+
+      const boundingBoxWidth = xMax - xMin;
+      const boundingBoxHeight = yMax - yMin;
+      if (ctx && boundingBoxWidth > 0 && boundingBoxHeight > 0) {
+        ctx.clearRect(0, 0, canvas.width, canvas.height);
+        ctx.putImageData(imageData, 0, 0);
+        return ctx.getImageData(
+          xMin,
+          yMin,
+          boundingBoxWidth,
+          boundingBoxHeight,
+        );
+      } else {
+        return null;
+      }
+    },
+    getBox(): [[number, number], [number, number]] {
+      return [
+        [xMin, yMin],
+        [xMax, yMax],
+      ];
+    },
+  };
+}
+
+export function magnifyImageRegion(
+  canvas: HTMLCanvasElement | null,
+  x: number,
+  y: number,
+  radius: number = 25,
+  scale: number = 2,
+): string {
+  if (canvas == null) {
+    return '';
+  }
+  const ctx = canvas.getContext('2d');
+  if (ctx) {
+    const minX = x - radius < 0 ? radius - x : 0;
+    const minY = y - radius < 0 ? radius - y : 0;
+    const region = ctx.getImageData(
+      Math.max(x - radius, 0),
+      Math.max(y - radius, 0),
+      radius * 2,
+      radius * 2,
+    );
+
+    // ImageData doesn't scale-transform correctly on canvas
+    // So we first draw the original size on an offscreen canvas, and then scale it
+    const regionCanvas = new OffscreenCanvas(region.width, region.height);
+    const regionCtx = regionCanvas.getContext('2d');
+    regionCtx?.putImageData(region, minX > 0 ? minX : 0, minY > 0 ? minY : 0);
+
+    const scaleCanvas = document.createElement('canvas');
+    scaleCanvas.width = Math.round(region.width * scale);
+    scaleCanvas.height = Math.round(region.height * scale);
+    const scaleCtx = scaleCanvas.getContext('2d');
+    scaleCtx?.scale(scale, scale);
+    scaleCtx?.drawImage(regionCanvas, 0, 0);
+
+    return scaleCanvas.toDataURL();
+  }
+  return '';
+}

demo/frontend/src/common/utils/MaskUtils.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Converts an image mask represented as a binary image (foreground pixels are
+ * `>1` and background pixels are `0`) stored in a Uint8Array to an RGBA
+ * representation where background pixels have an alpha value of 0 and
+ * foreground pixels have an alpha value of 255. This is useful for compositing
+ * the mask onto another image.
+ *
+ * ```typescript
+ * const rgba = convertMaskDataToRGBA(mask.data);
+ * ```
+ *
+ * @param data - The image mask represented as a Uint8Array
+ * @returns A new Uint8ClampedArray representing the mask in RGBA format
+ */
+export function convertMaskToRGBA(data: Uint8Array): Uint8ClampedArray {
+  // Shifting pixels instead of assigning them individually per pixel is
+  // much faster. See JSPerf benchamrk: https://jsperf.app/morifo
+  const len = data.length;
+  const tempData = new Uint32Array(len);
+  const RGA = 0x00ffffff;
+  const FOREGROUND = 0xff000000;
+  const BACKGROUND = 0x00000000;
+  for (let i = 0; i < len; i++) {
+    const alpha = data[i] > 0 ? FOREGROUND : BACKGROUND; // alpha is the high byte. Bits 24-31
+    tempData[i] = alpha + RGA;
+  }
+  return new Uint8ClampedArray(tempData.buffer);
+}