"Initial commit"

demo/frontend/src/common/components/annotations/ToolbarObject.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ObjectActions from '@/common/components/annotations/ObjectActions';
+import ObjectPlaceholder from '@/common/components/annotations/ObjectPlaceholder';
+import ObjectThumbnail from '@/common/components/annotations/ObjectThumbnail';
+import ToolbarObjectContainer from '@/common/components/annotations/ToolbarObjectContainer';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {BaseTracklet} from '@/common/tracker/Tracker';
+import emptyFunction from '@/common/utils/emptyFunction';
+import {activeTrackletObjectIdAtom} from '@/demo/atoms';
+import {useSetAtom} from 'jotai';
+
+type Props = {
+  label: string;
+  tracklet: BaseTracklet;
+  isActive: boolean;
+  isMobile?: boolean;
+  onClick?: () => void;
+  onThumbnailClick?: () => void;
+};
+
+export default function ToolbarObject({
+  label,
+  tracklet,
+  isActive,
+  isMobile = false,
+  onClick,
+  onThumbnailClick = emptyFunction,
+}: Props) {
+  const video = useVideo();
+  const setActiveTrackletId = useSetAtom(activeTrackletObjectIdAtom);
+
+  async function handleCancelNewObject() {
+    try {
+      await video?.deleteTracklet(tracklet.id);
+    } catch (error) {
+      reportError(error);
+    } finally {
+      setActiveTrackletId(null);
+    }
+  }
+
+  if (!tracklet.isInitialized) {
+    return (
+      <ToolbarObjectContainer
+        alignItems="center"
+        isActive={isActive}
+        title="New object"
+        subtitle="No object is currently selected. Click an object in the video."
+        thumbnail={<ObjectPlaceholder showPlus={false} />}
+        isMobile={isMobile}
+        onClick={onClick}
+        onCancel={handleCancelNewObject}
+      />
+    );
+  }
+
+  return (
+    <ToolbarObjectContainer
+      isActive={isActive}
+      onClick={onClick}
+      title={label}
+      subtitle=""
+      thumbnail={
+        <ObjectThumbnail
+          thumbnail={tracklet.thumbnail}
+          color={tracklet.color}
+          onClick={onThumbnailClick}
+        />
+      }
+      isMobile={isMobile}>
+      <ObjectActions objectId={tracklet.id} active={isActive} />
+    </ToolbarObjectContainer>
+  );
+}

demo/frontend/src/common/components/annotations/ToolbarObjectContainer.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {spacing} from '@/theme/tokens.stylex';
+import {Close} from '@carbon/icons-react';
+import stylex from '@stylexjs/stylex';
+import {PropsWithChildren, ReactNode} from 'react';
+
+const sharedStyles = stylex.create({
+  container: {
+    display: 'flex',
+    overflow: 'hidden',
+    cursor: 'pointer',
+    flexShrink: 0,
+    borderTop: 'none',
+    backgroundColor: {
+      '@media screen and (max-width: 768px)': '#000',
+    },
+    paddingHorizontal: {
+      default: spacing[8],
+      '@media screen and (max-width: 768px)': spacing[5],
+    },
+    paddingBottom: {
+      default: spacing[8],
+      '@media screen and (max-width: 768px)': 10,
+    },
+  },
+  activeContainer: {
+    background: '#000',
+    borderRadius: 16,
+    marginHorizontal: 16,
+    padding: {
+      default: spacing[4],
+      '@media screen and (max-width: 768px)': spacing[5],
+    },
+    marginBottom: {
+      default: spacing[8],
+      '@media screen and (max-width: 768px)': 0,
+    },
+  },
+  itemsCenter: {
+    alignItems: 'center',
+  },
+  rightColumn: {
+    marginStart: {
+      default: spacing[4],
+      '@media screen and (max-width: 768px)': 0,
+    },
+    flexGrow: 1,
+    alignItems: 'center',
+  },
+});
+
+type ToolbarObjectContainerProps = PropsWithChildren<{
+  alignItems?: 'top' | 'center';
+  isActive: boolean;
+  title: string;
+  subtitle: string;
+  thumbnail: ReactNode;
+  isMobile: boolean;
+  onCancel?: () => void;
+  onClick?: () => void;
+}>;
+
+export default function ToolbarObjectContainer({
+  alignItems = 'top',
+  children,
+  isActive,
+  title,
+  subtitle,
+  thumbnail,
+  isMobile,
+  onClick,
+  onCancel,
+}: ToolbarObjectContainerProps) {
+  if (isMobile) {
+    return (
+      <div
+        onClick={onClick}
+        {...stylex.props(sharedStyles.container, sharedStyles.itemsCenter)}>
+        <div {...stylex.props(sharedStyles.rightColumn)}>{children}</div>
+      </div>
+    );
+  }
+
+  return (
+    <div
+      onClick={onClick}
+      {...stylex.props(
+        sharedStyles.container,
+        isActive && sharedStyles.activeContainer,
+        alignItems === 'center' && sharedStyles.itemsCenter,
+      )}>
+      {thumbnail}
+      <div {...stylex.props(sharedStyles.rightColumn)}>
+        <div className="text-md font-semibold ml-2">{title}</div>
+        {subtitle.length > 0 && (
+          <div className="text-sm text-gray-400 leading-5 mt-2 ml-2">
+            {subtitle}
+          </div>
+        )}
+        {children}
+      </div>
+      {onCancel != null && (
+        <div className="items-start self-stretch" onClick={onCancel}>
+          <Close size={32} />
+        </div>
+      )}
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/TrackletSwimlane.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useSelectedFrameHelper from '@/common/components/video/filmstrip/useSelectedFrameHelper';
+import {BaseTracklet, DatalessMask} from '@/common/tracker/Tracker';
+import {spacing, w} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+import {useMemo} from 'react';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    alignItems: 'center',
+    gap: spacing[4],
+    width: '100%',
+  },
+  trackletNameContainer: {
+    width: w[12],
+    textAlign: 'center',
+    fontSize: '10px',
+    color: 'white',
+  },
+  swimlaneContainer: {
+    flexGrow: 1,
+    position: 'relative',
+    display: 'flex',
+    height: 12,
+    marginVertical: '0.25rem' /* 4px */,
+    '@media screen and (max-width: 768px)': {
+      marginVertical: 0,
+    },
+  },
+  swimlane: {
+    position: 'absolute',
+    left: 0,
+    top: '50%',
+    width: '100%',
+    height: 1,
+    transform: 'translate3d(0, -50%, 0)',
+    opacity: 0.4,
+  },
+  segment: {
+    position: 'absolute',
+    top: '50%',
+    height: 1,
+    transform: 'translate3d(0, -50%, 0)',
+  },
+  segmentationPoint: {
+    position: 'absolute',
+    top: '50%',
+    transform: 'translate3d(0, -50%, 0)',
+    borderRadius: '50%',
+    cursor: 'pointer',
+    width: 12,
+    height: 12,
+    '@media screen and (max-width: 768px)': {
+      width: 8,
+      height: 8,
+    },
+  },
+});
+
+type SwimlineSegment = {
+  left: number;
+  width: number;
+};
+
+type Props = {
+  tracklet: BaseTracklet;
+  onSelectFrame: (tracklet: BaseTracklet, index: number) => void;
+};
+
+function getSwimlaneSegments(masks: DatalessMask[]): SwimlineSegment[] {
+  if (masks.length === 0) {
+    return [];
+  }
+
+  const swimlineSegments: SwimlineSegment[] = [];
+  let left = -1;
+
+  for (let frameIndex = 0; frameIndex < masks.length; ++frameIndex) {
+    const isEmpty = masks?.[frameIndex]?.isEmpty ?? true;
+    if (left === -1 && !isEmpty) {
+      left = frameIndex;
+    } else if (left !== -1 && (isEmpty || frameIndex == masks.length - 1)) {
+      swimlineSegments.push({
+        left,
+        width: frameIndex - left + 1,
+      });
+      left = -1;
+    }
+  }
+
+  return swimlineSegments;
+}
+
+export default function TrackletSwimlane({tracklet, onSelectFrame}: Props) {
+  const selection = useSelectedFrameHelper();
+
+  const segments = useMemo(() => {
+    return getSwimlaneSegments(tracklet.masks);
+  }, [tracklet.masks]);
+
+  const framesWithPoints = tracklet.points.reduce<number[]>(
+    (frames, pts, frameIndex) => {
+      if (pts != null && pts.length > 0) {
+        frames.push(frameIndex);
+      }
+      return frames;
+    },
+    [],
+  );
+
+  if (selection === null) {
+    return;
+  }
+
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div {...stylex.props(styles.trackletNameContainer)}>
+        Object {tracklet.id + 1}
+      </div>
+      <div {...stylex.props(styles.swimlaneContainer)}>
+        <div
+          {...stylex.props(styles.swimlane)}
+          style={{
+            backgroundColor: tracklet.color,
+          }}
+        />
+        {segments.map(segment => {
+          return (
+            <div
+              key={segment.left}
+              {...stylex.props(styles.segment)}
+              style={{
+                backgroundColor: tracklet.color,
+                left: selection.toPosition(segment.left),
+                width: selection.toPosition(segment.width),
+              }}
+            />
+          );
+        })}
+        {framesWithPoints.map(index => {
+          return (
+            <div
+              key={`frame${index}`}
+              onClick={() => {
+                onSelectFrame?.(tracklet, index);
+              }}
+              {...stylex.props(styles.segmentationPoint)}
+              style={{
+                left: Math.floor(selection.toPosition(index) - 4),
+                backgroundColor: tracklet.color,
+              }}
+            />
+          );
+        })}
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/TrackletsAnnotation.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import TrackletSwimlane from '@/common/components/annotations/TrackletSwimlane';
+import useTracklets from '@/common/components/annotations/useTracklets';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {BaseTracklet} from '@/common/tracker/Tracker';
+import {m, spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+
+const styles = stylex.create({
+  container: {
+    marginTop: m[3],
+    height: 75,
+    paddingHorizontal: spacing[4],
+    '@media screen and (max-width: 768px)': {
+      height: 25,
+    },
+  },
+});
+
+export default function TrackletsAnnotation() {
+  const video = useVideo();
+  const tracklets = useTracklets();
+
+  function handleSelectFrame(_tracklet: BaseTracklet, index: number) {
+    if (video !== null) {
+      video.frame = index;
+    }
+  }
+
+  return (
+    <div {...stylex.props(styles.container)}>
+      {tracklets.map(tracklet => (
+        <TrackletSwimlane
+          key={tracklet.id}
+          tracklet={tracklet}
+          onSelectFrame={handleSelectFrame}
+        />
+      ))}
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/useTracklets.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {trackletObjectsAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+export default function useTracklets() {
+  return useAtomValue(trackletObjectsAtom);
+}

demo/frontend/src/common/components/button/GradientBorder.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import stylex from '@stylexjs/stylex';
+
+import {gradients} from '@/theme/tokens.stylex';
+
+enum GradientTypes {
+  fullGradient = 'fullGradient',
+  bluePinkGradient = 'bluePinkGradient',
+}
+
+type Props = {
+  gradientType?: GradientTypes;
+  disabled?: boolean;
+  rounded?: boolean;
+  className?: string;
+} & React.DOMAttributes<HTMLDivElement>;
+
+const styles = stylex.create({
+  animationHover: {
+    ':hover': {
+      backgroundPosition: '300% 100%',
+    },
+  },
+
+  fullGradient: {
+    border: '2px solid transparent',
+    background: gradients['rainbow'],
+    backgroundSize: '100% 400%',
+    transition: 'background 0.35s ease-in-out',
+  },
+
+  bluePinkGradient: {
+    border: '2px solid transparent',
+    background: gradients['rainbow'],
+  },
+});
+
+export default function GradientBorder({
+  gradientType = GradientTypes.fullGradient,
+  disabled,
+  rounded = true,
+  className = '',
+  children,
+}: Props) {
+  const gradient = (name: GradientTypes) => {
+    if (name === GradientTypes.fullGradient) {
+      return styles.fullGradient;
+    } else if (name === GradientTypes.bluePinkGradient) {
+      return styles.bluePinkGradient;
+    }
+  };
+
+  return (
+    <div
+      className={`${stylex(gradient(gradientType), !disabled && styles.animationHover)} ${disabled && 'opacity-30'} ${rounded && 'rounded-full'} ${className}`}>
+      {children}
+    </div>
+  );
+}

demo/frontend/src/common/components/button/PlaybackButton.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {OBJECT_TOOLBAR_INDEX} from '@/common/components/toolbar/ToolbarConfig';
+import Tooltip from '@/common/components/Tooltip';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {isPlayingAtom, streamingStateAtom, toolbarTabIndex} from '@/demo/atoms';
+import {PauseFilled, PlayFilledAlt} from '@carbon/icons-react';
+import {useAtomValue} from 'jotai';
+import {useCallback, useEffect} from 'react';
+
+export default function PlaybackButton() {
+  const tabIndex = useAtomValue(toolbarTabIndex);
+  const streamingState = useAtomValue(streamingStateAtom);
+  const isPlaying = useAtomValue(isPlayingAtom);
+  const video = useVideo();
+
+  const isDisabled =
+    tabIndex === OBJECT_TOOLBAR_INDEX &&
+    streamingState !== 'none' &&
+    streamingState !== 'full';
+
+  const handlePlay = useCallback(() => {
+    video?.play();
+  }, [video]);
+
+  const handlePause = useCallback(() => {
+    video?.pause();
+  }, [video]);
+
+  const handleClick = useCallback(() => {
+    if (isDisabled) {
+      return;
+    }
+    if (isPlaying) {
+      handlePause();
+    } else {
+      handlePlay();
+    }
+  }, [isDisabled, isPlaying, handlePlay, handlePause]);
+
+  useEffect(() => {
+    const handleKey = (event: KeyboardEvent) => {
+      const callback = {
+        KeyK: handleClick,
+      }[event.code];
+      if (callback != null) {
+        event.preventDefault();
+        callback();
+      }
+    };
+    document.addEventListener('keydown', handleKey);
+    return () => {
+      document.removeEventListener('keydown', handleKey);
+    };
+  }, [handleClick]);
+
+  return (
+    <Tooltip message={`${isPlaying ? 'Pause' : 'Play'} (k)`}>
+      <button
+        disabled={isDisabled}
+        className={`group !rounded-full !w-10 !h-10 flex items-center justify-center ${getButtonStyles(isDisabled)}`}
+        onClick={handleClick}>
+        {isPlaying ? (
+          <PauseFilled size={18} />
+        ) : (
+          <PlayFilledAlt
+            size={18}
+            className={!isDisabled ? 'group-hover:text-green-500' : ''}
+          />
+        )}
+      </button>
+    </Tooltip>
+  );
+}
+
+function getButtonStyles(isDisabled: boolean): string {
+  if (isDisabled) {
+    return '!bg-gray-600 !text-graydark-700';
+  }
+  return `!text-black bg-white`;
+}

demo/frontend/src/common/components/button/PrimaryCTAButton.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import GradientBorder from '@/common/components/button/GradientBorder';
+import type {ReactNode} from 'react';
+
+type Props = {
+  disabled?: boolean;
+  endIcon?: ReactNode;
+} & React.DOMAttributes<HTMLButtonElement>;
+
+export default function PrimaryCTAButton({
+  children,
+  disabled,
+  endIcon,
+  ...props
+}: Props) {
+  return (
+    <GradientBorder disabled={disabled}>
+      <button
+        className={`btn ${disabled && 'btn-disabled'} !rounded-full !bg-black !text-white !border-none`}
+        {...props}>
+        {children}
+        {endIcon != null && endIcon}
+      </button>
+    </GradientBorder>
+  );
+}

demo/frontend/src/common/components/button/ResponsiveButton.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useScreenSize from '@/common/screen/useScreenSize';
+import type {ReactNode} from 'react';
+import type {ButtonProps} from 'react-daisyui';
+import {Button} from 'react-daisyui';
+
+type Props = ButtonProps & {startIcon: ReactNode};
+
+export default function ResponsiveButton(props: Props) {
+  const {isMobile} = useScreenSize();
+
+  return <Button {...props}>{!isMobile && props.children}</Button>;
+}

demo/frontend/src/common/components/button/TrackAndPlayButton.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import PrimaryCTAButton from '@/common/components/button/PrimaryCTAButton';
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import useFunctionThrottle from '@/common/components/useFunctionThrottle';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {
+  areTrackletObjectsInitializedAtom,
+  isStreamingAtom,
+  sessionAtom,
+  streamingStateAtom,
+} from '@/demo/atoms';
+import {ChevronRight} from '@carbon/icons-react';
+import {useAtom, useAtomValue, useSetAtom} from 'jotai';
+import {useCallback, useEffect} from 'react';
+
+export default function TrackAndPlayButton() {
+  const video = useVideo();
+  const [isStreaming, setIsStreaming] = useAtom(isStreamingAtom);
+  const streamingState = useAtomValue(streamingStateAtom);
+  const areObjectsInitialized = useAtomValue(areTrackletObjectsInitializedAtom);
+  const setSession = useSetAtom(sessionAtom);
+  const {enqueueMessage} = useMessagesSnackbar();
+  const {isThrottled, maxThrottles, throttle} = useFunctionThrottle(250, 4);
+
+  const isTrackAndPlayDisabled =
+    streamingState === 'aborting' || streamingState === 'requesting';
+
+  useEffect(() => {
+    function onStreamingStarted() {
+      setIsStreaming(true);
+    }
+    video?.addEventListener('streamingStarted', onStreamingStarted);
+
+    function onStreamingCompleted() {
+      enqueueMessage('trackAndPlayComplete');
+      setIsStreaming(false);
+    }
+    video?.addEventListener('streamingCompleted', onStreamingCompleted);
+
+    return () => {
+      video?.removeEventListener('streamingStarted', onStreamingStarted);
+      video?.removeEventListener('streamingCompleted', onStreamingCompleted);
+    };
+  }, [video, setIsStreaming, enqueueMessage]);
+
+  const handleTrackAndPlay = useCallback(() => {
+    if (isTrackAndPlayDisabled) {
+      return;
+    }
+    if (maxThrottles && isThrottled) {
+      enqueueMessage('trackAndPlayThrottlingWarning');
+    }
+
+    // Throttling is only applied while streaming because we should
+    // only throttle after a user has aborted inference. This way,
+    // a user can still quickly abort a stream if they notice the
+    // inferred mask is misaligned.
+    throttle(
+      () => {
+        if (!isStreaming) {
+          enqueueMessage('trackAndPlayClick');
+          video?.streamMasks();
+          setSession(previousSession =>
+            previousSession == null
+              ? previousSession
+              : {...previousSession, ranPropagation: true},
+          );
+        } else {
+          video?.abortStreamMasks();
+        }
+      },
+      {enableThrottling: isStreaming},
+    );
+  }, [
+    isTrackAndPlayDisabled,
+    isThrottled,
+    isStreaming,
+    maxThrottles,
+    video,
+    setSession,
+    enqueueMessage,
+    throttle,
+  ]);
+
+  useEffect(() => {
+    const handleKey = (event: KeyboardEvent) => {
+      const callback = {
+        KeyK: handleTrackAndPlay,
+      }[event.code];
+      if (callback != null) {
+        event.preventDefault();
+        callback();
+      }
+    };
+    document.addEventListener('keydown', handleKey);
+    return () => {
+      document.removeEventListener('keydown', handleKey);
+    };
+  }, [handleTrackAndPlay]);
+
+  return (
+    <PrimaryCTAButton
+      disabled={isThrottled || !areObjectsInitialized}
+      onClick={handleTrackAndPlay}
+      endIcon={isStreaming ? undefined : <ChevronRight size={20} />}>
+      {isStreaming ? 'Cancel Tracking' : 'Track objects'}
+    </PrimaryCTAButton>
+  );
+}

demo/frontend/src/common/components/code/InitializeLocalMonaco.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {loader} from '@monaco-editor/react';
+
+import Logger from '@/common/logger/Logger';
+import * as monaco from 'monaco-editor';
+import editorWorker from 'monaco-editor/esm/vs/editor/editor.worker?worker';
+import tsWorker from 'monaco-editor/esm/vs/language/typescript/ts.worker?worker';
+
+self.MonacoEnvironment = {
+  getWorker(_, label) {
+    if (label === 'typescript' || label === 'javascript') {
+      return new tsWorker();
+    }
+    return new editorWorker();
+  },
+};
+
+loader.config({monaco});
+
+loader.init().then(monaco => {
+  Logger.debug('initialized monaco', monaco);
+});

demo/frontend/src/common/components/effects/BackgroundEffects.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {backgroundEffects} from '@/common/components/effects/EffectsUtils';
+import EffectVariantBadge from '@/common/components/effects/EffectVariantBadge';
+import ToolbarActionIcon from '@/common/components/toolbar/ToolbarActionIcon';
+import ToolbarSection from '@/common/components/toolbar/ToolbarSection';
+import useVideoEffect from '@/common/components/video/editor/useVideoEffect';
+import {EffectIndex} from '@/common/components/video/effects/Effects';
+import {activeBackgroundEffectAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+export default function BackgroundEffects() {
+  const setEffect = useVideoEffect();
+  const activeEffect = useAtomValue(activeBackgroundEffectAtom);
+
+  return (
+    <ToolbarSection title="Background" borderBottom={false}>
+      {backgroundEffects.map(backgroundEffect => {
+        return (
+          <ToolbarActionIcon
+            variant="toggle"
+            key={backgroundEffect.title}
+            icon={backgroundEffect.Icon}
+            title={backgroundEffect.title}
+            isActive={activeEffect.name === backgroundEffect.effectName}
+            badge={
+              activeEffect.name === backgroundEffect.effectName && (
+                <EffectVariantBadge
+                  label={`${activeEffect.variant + 1}/${activeEffect.numVariants}`}
+                />
+              )
+            }
+            onClick={() => {
+              if (activeEffect.name === backgroundEffect.effectName) {
+                setEffect(backgroundEffect.effectName, EffectIndex.BACKGROUND, {
+                  variant:
+                    (activeEffect.variant + 1) % activeEffect.numVariants,
+                });
+              } else {
+                setEffect(backgroundEffect.effectName, EffectIndex.BACKGROUND);
+              }
+            }}
+          />
+        );
+      })}
+    </ToolbarSection>
+  );
+}

demo/frontend/src/common/components/effects/EffectVariantBadge.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {right, top} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+
+const styles = stylex.create({
+  variantBadge: {
+    position: 'absolute',
+    top: top[1],
+    right: right[1],
+    backgroundColor: '#280578',
+    color: '#D2D2FF',
+    fontVariantNumeric: 'tabular-nums',
+    paddingHorizontal: 4,
+    paddingVertical: 1,
+    fontSize: 9,
+    borderRadius: 6,
+    fontWeight: 'bold',
+  },
+});
+
+type Props = {
+  label: string;
+};
+
+export default function VariantBadge({label}: Props) {
+  return <div {...stylex.props(styles.variantBadge)}>{label}</div>;
+}

demo/frontend/src/common/components/effects/EffectsCarousel.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {CarouselContainerShadow} from '@/common/components/effects/EffectsCarouselShadow';
+import {DemoEffect} from '@/common/components/effects/EffectsUtils';
+import useVideoEffect from '@/common/components/video/editor/useVideoEffect';
+import type {EffectIndex} from '@/common/components/video/effects/Effects';
+import {Effects} from '@/common/components/video/effects/Effects';
+import {color, fontSize, spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+
+type Props = {
+  label: string;
+  effects: DemoEffect[];
+  activeEffect: keyof Effects;
+  index: EffectIndex;
+};
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    flexDirection: 'column',
+    gap: spacing[2],
+    width: '100%',
+  },
+  label: {
+    fontSize: fontSize['xs'],
+    color: '#A6ACB2',
+    textAlign: 'center',
+  },
+  carouselContainer: {
+    position: 'relative',
+    borderRadius: '8px',
+    overflow: 'hidden',
+    width: '100%',
+    height: '120px',
+    backgroundColor: color['gray-700'],
+  },
+});
+
+export default function EffectsCarousel({
+  label,
+  effects,
+  activeEffect,
+  index: effectIndex,
+}: Props) {
+  const setEffect = useVideoEffect();
+
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div {...stylex.props(styles.label)}>{label}</div>
+      <div {...stylex.props(styles.carouselContainer)}>
+        <CarouselContainerShadow isTop={true} />
+        <div className="carousel carousel-vertical w-full h-full text-white">
+          <div className={`carousel-item h-6`} />
+          {effects.map(({effectName, Icon, title}, index) => {
+            const isActive = activeEffect === effectName;
+            return (
+              <div
+                key={index}
+                className={`carousel-item flex items-center h-6 gap-2 px-4`}
+                onClick={() => setEffect(effectName, effectIndex)}>
+                <Icon
+                  color={isActive ? '#FB73A5' : undefined}
+                  size={18}
+                  fontWeight={10}
+                />
+                <div
+                  className={`text-sm ${isActive ? 'text-[#FB73A5] font-bold' : 'font-medium'}`}>
+                  {title}
+                </div>
+              </div>
+            );
+          })}
+          <div className={`carousel-item h-6`} />
+        </div>
+        <CarouselContainerShadow isTop={false} />
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/effects/EffectsCarouselShadow.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+
+const styles = stylex.create({
+  container: {
+    position: 'absolute',
+    width: '100%',
+    height: spacing[8],
+    pointerEvents: 'none',
+  },
+});
+
+type CarouselContainerShadowProps = {
+  isTop: boolean;
+};
+
+const edgeColor = 'rgba(55, 62, 65, 1)';
+const transitionColor = 'rgba(55, 62, 65, 0.2)';
+
+export function CarouselContainerShadow({isTop}: CarouselContainerShadowProps) {
+  return (
+    <div
+      {...stylex.props(styles.container)}
+      style={{
+        background: `linear-gradient(${isTop ? `${edgeColor}, ${transitionColor}` : `${transitionColor}, ${edgeColor}`})`,
+        top: isTop ? 0 : undefined,
+        bottom: isTop ? undefined : 0,
+      }}
+    />
+  );
+}

demo/frontend/src/common/components/effects/EffectsToolbar.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BackgroundEffects from '@/common/components/effects/BackgroundEffects';
+import EffectsToolbarBottomActions from '@/common/components/effects/EffectsToolbarBottomActions';
+import EffectsToolbarHeader from '@/common/components/effects/EffectsToolbarHeader';
+import HighlightEffects from '@/common/components/effects/HighlightEffects';
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import {useEffect, useRef} from 'react';
+
+type Props = {
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function EffectsToolbar({onTabChange}: Props) {
+  const isEffectsMessageShown = useRef(false);
+  const {enqueueMessage} = useMessagesSnackbar();
+
+  useEffect(() => {
+    if (!isEffectsMessageShown.current) {
+      isEffectsMessageShown.current = true;
+      enqueueMessage('effectsMessage');
+    }
+  }, [enqueueMessage]);
+
+  return (
+    <div className="flex flex-col h-full">
+      <EffectsToolbarHeader />
+      <div className="grow overflow-y-auto">
+        <HighlightEffects />
+        <BackgroundEffects />
+      </div>
+      <EffectsToolbarBottomActions onTabChange={onTabChange} />
+    </div>
+  );
+}

demo/frontend/src/common/components/effects/EffectsToolbarBottomActions.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import PrimaryCTAButton from '@/common/components/button/PrimaryCTAButton';
+import RestartSessionButton from '@/common/components/session/RestartSessionButton';
+import ToolbarBottomActionsWrapper from '@/common/components/toolbar/ToolbarBottomActionsWrapper';
+import {
+  MORE_OPTIONS_TOOLBAR_INDEX,
+  OBJECT_TOOLBAR_INDEX,
+} from '@/common/components/toolbar/ToolbarConfig';
+import {ChevronRight} from '@carbon/icons-react';
+
+type Props = {
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function EffectsToolbarBottomActions({onTabChange}: Props) {
+  function handleSwitchToMoreOptionsTab() {
+    onTabChange(MORE_OPTIONS_TOOLBAR_INDEX);
+  }
+
+  return (
+    <ToolbarBottomActionsWrapper>
+      <RestartSessionButton
+        onRestartSession={() => onTabChange(OBJECT_TOOLBAR_INDEX)}
+      />
+      <PrimaryCTAButton
+        onClick={handleSwitchToMoreOptionsTab}
+        endIcon={<ChevronRight />}>
+        Next
+      </PrimaryCTAButton>
+    </ToolbarBottomActionsWrapper>
+  );
+}

demo/frontend/src/common/components/effects/EffectsToolbarHeader.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ToolbarHeaderWrapper from '@/common/components/toolbar/ToolbarHeaderWrapper';
+import useVideoEffect from '@/common/components/video/editor/useVideoEffect';
+import {
+  EffectIndex,
+  effectPresets,
+} from '@/common/components/video/effects/Effects';
+import {BLUE_PINK_FILL} from '@/theme/gradientStyle';
+import {MagicWandFilled} from '@carbon/icons-react';
+import {useCallback, useRef} from 'react';
+import {Button} from 'react-daisyui';
+
+export default function EffectsToolbarHeader() {
+  const preset = useRef(0);
+  const setEffect = useVideoEffect();
+
+  const handleTogglePreset = useCallback(() => {
+    preset.current++;
+    const [background, highlight] =
+      effectPresets[preset.current % effectPresets.length];
+    setEffect(background.name, EffectIndex.BACKGROUND, {
+      variant: background.variant,
+    });
+    setEffect(highlight.name, EffectIndex.HIGHLIGHT, {
+      variant: highlight.variant,
+    });
+  }, [setEffect]);
+
+  return (
+    <ToolbarHeaderWrapper
+      title="Add effects"
+      description="Apply visual effects to your selected objects and the background. Keeping clicking the same effect for different variations."
+      bottomSection={
+        <div className="flex mt-1">
+          <Button
+            color="ghost"
+            size="md"
+            className={`font-medium bg-black !rounded-full hover:!bg-gradient-to-br ${BLUE_PINK_FILL} border-none`}
+            endIcon={<MagicWandFilled size={20} className="text-white " />}
+            onClick={handleTogglePreset}>
+            Surprise Me
+          </Button>
+        </div>
+      }
+      className="pb-4"
+    />
+  );
+}

demo/frontend/src/common/components/effects/EffectsUtils.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Effects} from '@/common/components/video/effects/Effects';
+import type {CarbonIconType} from '@carbon/icons-react';
+import {
+  AppleDash,
+  Asterisk,
+  Barcode,
+  CenterCircle,
+  ColorPalette,
+  ColorSwitch,
+  Development,
+  Erase,
+  FaceWink,
+  Humidity,
+  Image,
+  Overlay,
+  TextFont,
+} from '@carbon/icons-react';
+
+export type DemoEffect = {
+  title: string;
+  Icon: CarbonIconType;
+  effectName: keyof Effects;
+};
+
+export const backgroundEffects: DemoEffect[] = [
+  {title: 'Original', Icon: Image, effectName: 'Original'},
+  {title: 'Erase', Icon: Erase, effectName: 'EraseBackground'},
+  {
+    title: 'Gradient',
+    Icon: ColorPalette,
+    effectName: 'Gradient',
+  },
+  {
+    title: 'Pixelate',
+    Icon: Development,
+    effectName: 'Pixelate',
+  },
+  {title: 'Desaturate', Icon: ColorSwitch, effectName: 'Desaturate'},
+  {title: 'Text', Icon: TextFont, effectName: 'BackgroundText'},
+  {title: 'Blur', Icon: Humidity, effectName: 'BackgroundBlur'},
+  {title: 'Outline', Icon: AppleDash, effectName: 'Sobel'},
+];
+
+export const highlightEffects: DemoEffect[] = [
+  {title: 'Original', Icon: Image, effectName: 'Cutout'},
+  {title: 'Erase', Icon: Erase, effectName: 'EraseForeground'},
+  {title: 'Gradient', Icon: ColorPalette, effectName: 'VibrantMask'},
+  {title: 'Pixelate', Icon: Development, effectName: 'PixelateMask'},
+  {
+    title: 'Overlay',
+    Icon: Overlay,
+    effectName: 'Overlay',
+  },
+  {title: 'Emoji', Icon: FaceWink, effectName: 'Replace'},
+  {title: 'Burst', Icon: Asterisk, effectName: 'Burst'},
+  {title: 'Spotlight', Icon: CenterCircle, effectName: 'Scope'},
+];
+
+export const moreEffects: DemoEffect[] = [
+  {title: 'Noisy', Icon: Barcode, effectName: 'NoisyMask'},
+];

demo/frontend/src/common/components/effects/HighlightEffects.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import EffectVariantBadge from '@/common/components/effects/EffectVariantBadge';
+import ToolbarActionIcon from '@/common/components/toolbar/ToolbarActionIcon';
+import ToolbarSection from '@/common/components/toolbar/ToolbarSection';
+import useVideoEffect from '@/common/components/video/editor/useVideoEffect';
+import {EffectIndex} from '@/common/components/video/effects/Effects';
+import {
+  activeHighlightEffectAtom,
+  activeHighlightEffectGroupAtom,
+} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+export default function HighlightEffects() {
+  const setEffect = useVideoEffect();
+  const activeEffect = useAtomValue(activeHighlightEffectAtom);
+  const activeEffectsGroup = useAtomValue(activeHighlightEffectGroupAtom);
+
+  return (
+    <ToolbarSection title="Selected Objects" borderBottom={true}>
+      {activeEffectsGroup.map(highlightEffect => {
+        return (
+          <ToolbarActionIcon
+            variant="toggle"
+            key={highlightEffect.title}
+            icon={highlightEffect.Icon}
+            title={highlightEffect.title}
+            isActive={activeEffect.name === highlightEffect.effectName}
+            badge={
+              activeEffect.name === highlightEffect.effectName && (
+                <EffectVariantBadge
+                  label={`${activeEffect.variant + 1}/${activeEffect.numVariants}`}
+                />
+              )
+            }
+            onClick={() => {
+              if (activeEffect.name === highlightEffect.effectName) {
+                setEffect(highlightEffect.effectName, EffectIndex.HIGHLIGHT, {
+                  variant:
+                    (activeEffect.variant + 1) % activeEffect.numVariants,
+                });
+              } else {
+                setEffect(highlightEffect.effectName, EffectIndex.HIGHLIGHT);
+              }
+            }}
+          />
+        );
+      })}
+    </ToolbarSection>
+  );
+}