Initial commit

demo/frontend/src/common/components/annotations/ObjectActions.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import PointsToggle from '@/common/components/annotations/PointsToggle';
+import useVideo from '@/common/components/video/editor/useVideo';
+import useReportError from '@/common/error/useReportError';
+import {
+  activeTrackletObjectIdAtom,
+  isPlayingAtom,
+  isStreamingAtom,
+} from '@/demo/atoms';
+import {
+  AddFilled,
+  Select_02,
+  SubtractFilled,
+  TrashCan,
+} from '@carbon/icons-react';
+import {useAtom, useAtomValue} from 'jotai';
+import {useState} from 'react';
+import type {ButtonProps} from 'react-daisyui';
+import {Button} from 'react-daisyui';
+
+type Props = {
+  objectId: number;
+  active: boolean;
+};
+
+function CustomButton({className, ...props}: ButtonProps) {
+  return (
+    <Button
+      size="sm"
+      color="ghost"
+      className={`font-medium border-none hover:bg-black  px-2 h-10 ${className}`}
+      {...props}>
+      {props.children}
+    </Button>
+  );
+}
+
+export default function ObjectActions({objectId, active}: Props) {
+  const [isRemovingObject, setIsRemovingObject] = useState<boolean>(false);
+  const [activeTrackId, setActiveTrackletId] = useAtom(
+    activeTrackletObjectIdAtom,
+  );
+  const isStreaming = useAtomValue(isStreamingAtom);
+  const isPlaying = useAtom(isPlayingAtom);
+
+  const video = useVideo();
+  const reportError = useReportError();
+
+  async function handleRemoveObject(
+    event: React.MouseEvent<HTMLButtonElement>,
+  ) {
+    try {
+      event.stopPropagation();
+      setIsRemovingObject(true);
+      if (isStreaming) {
+        await video?.abortStreamMasks();
+      }
+      if (isPlaying) {
+        video?.pause();
+      }
+      await video?.deleteTracklet(objectId);
+    } catch (error) {
+      reportError(error);
+    } finally {
+      setIsRemovingObject(false);
+      if (activeTrackId === objectId) {
+        setActiveTrackletId(null);
+      }
+    }
+  }
+
+  return (
+    <div>
+      {active && (
+        <div className="text-sm mt-1 leading-snug text-gray-400 hidden md:block ml-2 md:mb-4">
+          Select <AddFilled size={14} className="inline" /> to add areas to the
+          object and <SubtractFilled size={14} className="inline" /> to remove
+          areas from the object in the video. Click on an existing point to
+          delete it.
+        </div>
+      )}
+
+      <div className="flex justify-between items-center md:mt-2 mt-0">
+        {active ? (
+          <PointsToggle />
+        ) : (
+          <>
+            <CustomButton startIcon={<Select_02 size={24} />}>
+              Edit selection
+            </CustomButton>
+            <CustomButton
+              loading={isRemovingObject}
+              onClick={handleRemoveObject}
+              startIcon={!isRemovingObject && <TrashCan size={24} />}>
+              <span className="hidden md:inline">Clear</span>
+            </CustomButton>
+          </>
+        )}
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/ObjectPlaceholder.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {BLUE_PINK_FILL_BR} from '@/theme/gradientStyle';
+
+type Props = {
+  showPlus?: boolean;
+  onClick?: () => void;
+};
+
+export default function ObjectPlaceholder({showPlus = true, onClick}: Props) {
+  return (
+    <div
+      className={`relative ${BLUE_PINK_FILL_BR} h-12 w-12 md:h-20 md:w-20 shrink-0 rounded-lg`}
+      onClick={onClick}>
+      {showPlus && (
+        <div className="absolute left-1/2 top-1/2 -translate-x-1/2 -translate-y-1/2">
+          <svg
+            xmlns="http://www.w3.org/2000/svg"
+            width="16"
+            height="16"
+            viewBox="0 0 16 16"
+            fill="none">
+            <path
+              d="M16 7H9V0H7V7H0V9H7V16H9V9H16V7Z"
+              fill="#667788"
+              fillOpacity={1}
+            />
+          </svg>
+        </div>
+      )}
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/ObjectThumbnail.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+type Props = {
+  thumbnail: string | null;
+  color: string;
+  onClick?: () => void;
+};
+
+export default function ObjectThumbnail({thumbnail, color, onClick}: Props) {
+  return (
+    <div
+      className="relative h-12 w-12 md:w-20 md:h-20 shrink-0 p-2 rounded-lg bg-contain bg-no-repeat bg-center"
+      style={{
+        backgroundColor: color,
+      }}
+      onClick={onClick}>
+      <div
+        className="w-full h-full bg-contain bg-no-repeat bg-center"
+        style={{
+          backgroundImage: thumbnail == null ? 'none' : `url(${thumbnail})`,
+        }}></div>
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/ObjectUtils.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {BaseTracklet} from '@/common/tracker/Tracker';
+
+export function getObjectLabel(tracklet: BaseTracklet) {
+  return `Object ${tracklet.id + 1}`;
+}

demo/frontend/src/common/components/annotations/ObjectsToolbar.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import AddObjectButton from '@/common/components/annotations/AddObjectButton';
+import FirstClickView from '@/common/components/annotations/FirstClickView';
+import LimitNotice from '@/common/components/annotations/LimitNotice';
+import ObjectsToolbarBottomActions from '@/common/components/annotations/ObjectsToolbarBottomActions';
+import ObjectsToolbarHeader from '@/common/components/annotations/ObjectsToolbarHeader';
+import {getObjectLabel} from '@/common/components/annotations/ObjectUtils';
+import ToolbarObject from '@/common/components/annotations/ToolbarObject';
+import {
+  activeTrackletObjectAtom,
+  activeTrackletObjectIdAtom,
+  isAddObjectEnabledAtom,
+  isFirstClickMadeAtom,
+  isTrackletObjectLimitReachedAtom,
+  trackletObjectsAtom,
+} from '@/demo/atoms';
+import {useAtomValue, useSetAtom} from 'jotai';
+
+type Props = {
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function ObjectsToolbar({onTabChange}: Props) {
+  const tracklets = useAtomValue(trackletObjectsAtom);
+  const activeTracklet = useAtomValue(activeTrackletObjectAtom);
+  const setActiveTrackletId = useSetAtom(activeTrackletObjectIdAtom);
+  const isFirstClickMade = useAtomValue(isFirstClickMadeAtom);
+  const isObjectLimitReached = useAtomValue(isTrackletObjectLimitReachedAtom);
+  const isAddObjectEnabled = useAtomValue(isAddObjectEnabledAtom);
+
+  if (!isFirstClickMade) {
+    return <FirstClickView />;
+  }
+
+  return (
+    <div className="flex flex-col h-full">
+      <ObjectsToolbarHeader />
+      <div className="grow w-full overflow-y-auto">
+        {tracklets.map(tracklet => {
+          return (
+            <ToolbarObject
+              key={tracklet.id}
+              label={getObjectLabel(tracklet)}
+              tracklet={tracklet}
+              isActive={activeTracklet?.id === tracklet.id}
+              onClick={() => {
+                setActiveTrackletId(tracklet.id);
+              }}
+            />
+          );
+        })}
+        {isAddObjectEnabled && <AddObjectButton />}
+        {isObjectLimitReached && <LimitNotice />}
+      </div>
+      <ObjectsToolbarBottomActions onTabChange={onTabChange} />
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/ObjectsToolbarBottomActions.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ClearAllPointsInVideoButton from '@/common/components/annotations/ClearAllPointsInVideoButton';
+import CloseSessionButton from '@/common/components/annotations/CloseSessionButton';
+import TrackAndPlayButton from '@/common/components/button/TrackAndPlayButton';
+import ToolbarBottomActionsWrapper from '@/common/components/toolbar/ToolbarBottomActionsWrapper';
+import {
+  EFFECT_TOOLBAR_INDEX,
+  OBJECT_TOOLBAR_INDEX,
+} from '@/common/components/toolbar/ToolbarConfig';
+import {streamingStateAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+type Props = {
+  onTabChange: (newIndex: number) => void;
+};
+
+export default function ObjectsToolbarBottomActions({onTabChange}: Props) {
+  const streamingState = useAtomValue(streamingStateAtom);
+
+  const isTrackingEnabled =
+    streamingState !== 'none' && streamingState !== 'full';
+
+  function handleSwitchToEffectsTab() {
+    onTabChange(EFFECT_TOOLBAR_INDEX);
+  }
+
+  return (
+    <ToolbarBottomActionsWrapper>
+      <ClearAllPointsInVideoButton
+        onRestart={() => onTabChange(OBJECT_TOOLBAR_INDEX)}
+      />
+      {isTrackingEnabled && <TrackAndPlayButton />}
+      {streamingState === 'full' && (
+        <CloseSessionButton onSessionClose={handleSwitchToEffectsTab} />
+      )}
+    </ToolbarBottomActionsWrapper>
+  );
+}

demo/frontend/src/common/components/annotations/ObjectsToolbarHeader.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ToolbarHeaderWrapper from '@/common/components/toolbar/ToolbarHeaderWrapper';
+import {isStreamingAtom, streamingStateAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+export default function ObjectsToolbarHeader() {
+  const isStreaming = useAtomValue(isStreamingAtom);
+  const streamingState = useAtomValue(streamingStateAtom);
+
+  return (
+    <ToolbarHeaderWrapper
+      title={
+        streamingState === 'full'
+          ? 'Review tracked objects'
+          : isStreaming
+            ? 'Tracking objects'
+            : 'Select objects'
+      }
+      description={
+        streamingState === 'full'
+          ? 'Review your selected objects across the video, and continue to edit if needed. Once everything looks good, press “Next” to continue.'
+          : isStreaming
+            ? 'Watch the video closely for any places where your objects aren’t tracked correctly. You can also stop tracking to make additional edits.'
+            : 'Adjust the selection of your object, or add additional objects. Press “Track objects” to track your objects throughout the video.'
+      }
+      className="mb-8"
+    />
+  );
+}

demo/frontend/src/common/components/annotations/PointsToggle.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {labelTypeAtom} from '@/demo/atoms';
+import {AddFilled, SubtractFilled} from '@carbon/icons-react';
+import {useAtom} from 'jotai';
+
+export default function PointsToggle() {
+  const [labelType, setLabelType] = useAtom(labelTypeAtom);
+  const isPositive = labelType === 'positive';
+
+  const buttonStyle = (selected: boolean) =>
+    `btn-md bg-graydark-800 !text-white md:px-2 lg:px-4 py-0.5 ${selected ? `border border-white hover:bg-graydark-800` : `border-graydark-700 hover:bg-graydark-700`}`;
+
+  return (
+    <div className="flex items-center w-full md:ml-2">
+      <div className="join group grow gap-[1px]">
+        <button
+          className={`w-1/2  btn join-item text-white ${buttonStyle(isPositive)}`}
+          onClick={() => setLabelType('positive')}>
+          <AddFilled size={24} className="text-blue-500" /> Add
+        </button>
+        <button
+          className={`w-1/2 btn join-item text-red-700 ${buttonStyle(!isPositive)}`}
+          onClick={() => setLabelType('negative')}>
+          <SubtractFilled size={24} className="text-red-400" />
+          Remove
+        </button>
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/PrimaryCTAButton.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import GradientBorder from '@/common/components/button/GradientBorder';
+import type {ReactNode} from 'react';
+
+type Props = {
+  disabled?: boolean;
+  endIcon?: ReactNode;
+} & React.DOMAttributes<HTMLButtonElement>;
+
+export default function PrimaryCTAButton({
+  children,
+  disabled,
+  endIcon,
+  ...props
+}: Props) {
+  return (
+    <GradientBorder disabled={disabled}>
+      <button
+        className={`btn ${disabled && 'btn-disabled'} !rounded-full !bg-black !text-white !border-none`}
+        {...props}>
+        {children}
+        {endIcon != null && endIcon}
+      </button>
+    </GradientBorder>
+  );
+}

demo/frontend/src/common/components/annotations/ToolbarObject.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ObjectActions from '@/common/components/annotations/ObjectActions';
+import ObjectPlaceholder from '@/common/components/annotations/ObjectPlaceholder';
+import ObjectThumbnail from '@/common/components/annotations/ObjectThumbnail';
+import ToolbarObjectContainer from '@/common/components/annotations/ToolbarObjectContainer';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {BaseTracklet} from '@/common/tracker/Tracker';
+import emptyFunction from '@/common/utils/emptyFunction';
+import {activeTrackletObjectIdAtom} from '@/demo/atoms';
+import {useSetAtom} from 'jotai';
+
+type Props = {
+  label: string;
+  tracklet: BaseTracklet;
+  isActive: boolean;
+  isMobile?: boolean;
+  onClick?: () => void;
+  onThumbnailClick?: () => void;
+};
+
+export default function ToolbarObject({
+  label,
+  tracklet,
+  isActive,
+  isMobile = false,
+  onClick,
+  onThumbnailClick = emptyFunction,
+}: Props) {
+  const video = useVideo();
+  const setActiveTrackletId = useSetAtom(activeTrackletObjectIdAtom);
+
+  async function handleCancelNewObject() {
+    try {
+      await video?.deleteTracklet(tracklet.id);
+    } catch (error) {
+      reportError(error);
+    } finally {
+      setActiveTrackletId(null);
+    }
+  }
+
+  if (!tracklet.isInitialized) {
+    return (
+      <ToolbarObjectContainer
+        alignItems="center"
+        isActive={isActive}
+        title="New object"
+        subtitle="No object is currently selected. Click an object in the video."
+        thumbnail={<ObjectPlaceholder showPlus={false} />}
+        isMobile={isMobile}
+        onClick={onClick}
+        onCancel={handleCancelNewObject}
+      />
+    );
+  }
+
+  return (
+    <ToolbarObjectContainer
+      isActive={isActive}
+      onClick={onClick}
+      title={label}
+      subtitle=""
+      thumbnail={
+        <ObjectThumbnail
+          thumbnail={tracklet.thumbnail}
+          color={tracklet.color}
+          onClick={onThumbnailClick}
+        />
+      }
+      isMobile={isMobile}>
+      <ObjectActions objectId={tracklet.id} active={isActive} />
+    </ToolbarObjectContainer>
+  );
+}

demo/frontend/src/common/components/annotations/ToolbarObjectContainer.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {spacing} from '@/theme/tokens.stylex';
+import {Close} from '@carbon/icons-react';
+import stylex from '@stylexjs/stylex';
+import {PropsWithChildren, ReactNode} from 'react';
+
+const sharedStyles = stylex.create({
+  container: {
+    display: 'flex',
+    overflow: 'hidden',
+    cursor: 'pointer',
+    flexShrink: 0,
+    borderTop: 'none',
+    backgroundColor: {
+      '@media screen and (max-width: 768px)': '#000',
+    },
+    paddingHorizontal: {
+      default: spacing[8],
+      '@media screen and (max-width: 768px)': spacing[5],
+    },
+    paddingBottom: {
+      default: spacing[8],
+      '@media screen and (max-width: 768px)': 10,
+    },
+  },
+  activeContainer: {
+    background: '#000',
+    borderRadius: 16,
+    marginHorizontal: 16,
+    padding: {
+      default: spacing[4],
+      '@media screen and (max-width: 768px)': spacing[5],
+    },
+    marginBottom: {
+      default: spacing[8],
+      '@media screen and (max-width: 768px)': 0,
+    },
+  },
+  itemsCenter: {
+    alignItems: 'center',
+  },
+  rightColumn: {
+    marginStart: {
+      default: spacing[4],
+      '@media screen and (max-width: 768px)': 0,
+    },
+    flexGrow: 1,
+    alignItems: 'center',
+  },
+});
+
+type ToolbarObjectContainerProps = PropsWithChildren<{
+  alignItems?: 'top' | 'center';
+  isActive: boolean;
+  title: string;
+  subtitle: string;
+  thumbnail: ReactNode;
+  isMobile: boolean;
+  onCancel?: () => void;
+  onClick?: () => void;
+}>;
+
+export default function ToolbarObjectContainer({
+  alignItems = 'top',
+  children,
+  isActive,
+  title,
+  subtitle,
+  thumbnail,
+  isMobile,
+  onClick,
+  onCancel,
+}: ToolbarObjectContainerProps) {
+  if (isMobile) {
+    return (
+      <div
+        onClick={onClick}
+        {...stylex.props(sharedStyles.container, sharedStyles.itemsCenter)}>
+        <div {...stylex.props(sharedStyles.rightColumn)}>{children}</div>
+      </div>
+    );
+  }
+
+  return (
+    <div
+      onClick={onClick}
+      {...stylex.props(
+        sharedStyles.container,
+        isActive && sharedStyles.activeContainer,
+        alignItems === 'center' && sharedStyles.itemsCenter,
+      )}>
+      {thumbnail}
+      <div {...stylex.props(sharedStyles.rightColumn)}>
+        <div className="text-md font-semibold ml-2">{title}</div>
+        {subtitle.length > 0 && (
+          <div className="text-sm text-gray-400 leading-5 mt-2 ml-2">
+            {subtitle}
+          </div>
+        )}
+        {children}
+      </div>
+      {onCancel != null && (
+        <div className="items-start self-stretch" onClick={onCancel}>
+          <Close size={32} />
+        </div>
+      )}
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/TrackletSwimlane.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useSelectedFrameHelper from '@/common/components/video/filmstrip/useSelectedFrameHelper';
+import {BaseTracklet, DatalessMask} from '@/common/tracker/Tracker';
+import {spacing, w} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+import {useMemo} from 'react';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    alignItems: 'center',
+    gap: spacing[4],
+    width: '100%',
+  },
+  trackletNameContainer: {
+    width: w[12],
+    textAlign: 'center',
+    fontSize: '10px',
+    color: 'white',
+  },
+  swimlaneContainer: {
+    flexGrow: 1,
+    position: 'relative',
+    display: 'flex',
+    height: 12,
+    marginVertical: '0.25rem' /* 4px */,
+    '@media screen and (max-width: 768px)': {
+      marginVertical: 0,
+    },
+  },
+  swimlane: {
+    position: 'absolute',
+    left: 0,
+    top: '50%',
+    width: '100%',
+    height: 1,
+    transform: 'translate3d(0, -50%, 0)',
+    opacity: 0.4,
+  },
+  segment: {
+    position: 'absolute',
+    top: '50%',
+    height: 1,
+    transform: 'translate3d(0, -50%, 0)',
+  },
+  segmentationPoint: {
+    position: 'absolute',
+    top: '50%',
+    transform: 'translate3d(0, -50%, 0)',
+    borderRadius: '50%',
+    cursor: 'pointer',
+    width: 12,
+    height: 12,
+    '@media screen and (max-width: 768px)': {
+      width: 8,
+      height: 8,
+    },
+  },
+});
+
+type SwimlineSegment = {
+  left: number;
+  width: number;
+};
+
+type Props = {
+  tracklet: BaseTracklet;
+  onSelectFrame: (tracklet: BaseTracklet, index: number) => void;
+};
+
+function getSwimlaneSegments(masks: DatalessMask[]): SwimlineSegment[] {
+  if (masks.length === 0) {
+    return [];
+  }
+
+  const swimlineSegments: SwimlineSegment[] = [];
+  let left = -1;
+
+  for (let frameIndex = 0; frameIndex < masks.length; ++frameIndex) {
+    const isEmpty = masks?.[frameIndex]?.isEmpty ?? true;
+    if (left === -1 && !isEmpty) {
+      left = frameIndex;
+    } else if (left !== -1 && (isEmpty || frameIndex == masks.length - 1)) {
+      swimlineSegments.push({
+        left,
+        width: frameIndex - left + 1,
+      });
+      left = -1;
+    }
+  }
+
+  return swimlineSegments;
+}
+
+export default function TrackletSwimlane({tracklet, onSelectFrame}: Props) {
+  const selection = useSelectedFrameHelper();
+
+  const segments = useMemo(() => {
+    return getSwimlaneSegments(tracklet.masks);
+  }, [tracklet.masks]);
+
+  const framesWithPoints = tracklet.points.reduce<number[]>(
+    (frames, pts, frameIndex) => {
+      if (pts != null && pts.length > 0) {
+        frames.push(frameIndex);
+      }
+      return frames;
+    },
+    [],
+  );
+
+  if (selection === null) {
+    return;
+  }
+
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div {...stylex.props(styles.trackletNameContainer)}>
+        Object {tracklet.id + 1}
+      </div>
+      <div {...stylex.props(styles.swimlaneContainer)}>
+        <div
+          {...stylex.props(styles.swimlane)}
+          style={{
+            backgroundColor: tracklet.color,
+          }}
+        />
+        {segments.map(segment => {
+          return (
+            <div
+              key={segment.left}
+              {...stylex.props(styles.segment)}
+              style={{
+                backgroundColor: tracklet.color,
+                left: selection.toPosition(segment.left),
+                width: selection.toPosition(segment.width),
+              }}
+            />
+          );
+        })}
+        {framesWithPoints.map(index => {
+          return (
+            <div
+              key={`frame${index}`}
+              onClick={() => {
+                onSelectFrame?.(tracklet, index);
+              }}
+              {...stylex.props(styles.segmentationPoint)}
+              style={{
+                left: Math.floor(selection.toPosition(index) - 4),
+                backgroundColor: tracklet.color,
+              }}
+            />
+          );
+        })}
+      </div>
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/TrackletsAnnotation.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import TrackletSwimlane from '@/common/components/annotations/TrackletSwimlane';
+import useTracklets from '@/common/components/annotations/useTracklets';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {BaseTracklet} from '@/common/tracker/Tracker';
+import {m, spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+
+const styles = stylex.create({
+  container: {
+    marginTop: m[3],
+    height: 75,
+    paddingHorizontal: spacing[4],
+    '@media screen and (max-width: 768px)': {
+      height: 25,
+    },
+  },
+});
+
+export default function TrackletsAnnotation() {
+  const video = useVideo();
+  const tracklets = useTracklets();
+
+  function handleSelectFrame(_tracklet: BaseTracklet, index: number) {
+    if (video !== null) {
+      video.frame = index;
+    }
+  }
+
+  return (
+    <div {...stylex.props(styles.container)}>
+      {tracklets.map(tracklet => (
+        <TrackletSwimlane
+          key={tracklet.id}
+          tracklet={tracklet}
+          onSelectFrame={handleSelectFrame}
+        />
+      ))}
+    </div>
+  );
+}

demo/frontend/src/common/components/annotations/useTracklets.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {trackletObjectsAtom} from '@/demo/atoms';
+import {useAtomValue} from 'jotai';
+
+export default function useTracklets() {
+  return useAtomValue(trackletObjectsAtom);
+}

demo/frontend/src/common/components/button/GradientBorder.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import stylex from '@stylexjs/stylex';
+
+import {gradients} from '@/theme/tokens.stylex';
+
+enum GradientTypes {
+  fullGradient = 'fullGradient',
+  bluePinkGradient = 'bluePinkGradient',
+}
+
+type Props = {
+  gradientType?: GradientTypes;
+  disabled?: boolean;
+  rounded?: boolean;
+  className?: string;
+} & React.DOMAttributes<HTMLDivElement>;
+
+const styles = stylex.create({
+  animationHover: {
+    ':hover': {
+      backgroundPosition: '300% 100%',
+    },
+  },
+
+  fullGradient: {
+    border: '2px solid transparent',
+    background: gradients['rainbow'],
+    backgroundSize: '100% 400%',
+    transition: 'background 0.35s ease-in-out',
+  },
+
+  bluePinkGradient: {
+    border: '2px solid transparent',
+    background: gradients['rainbow'],
+  },
+});
+
+export default function GradientBorder({
+  gradientType = GradientTypes.fullGradient,
+  disabled,
+  rounded = true,
+  className = '',
+  children,
+}: Props) {
+  const gradient = (name: GradientTypes) => {
+    if (name === GradientTypes.fullGradient) {
+      return styles.fullGradient;
+    } else if (name === GradientTypes.bluePinkGradient) {
+      return styles.bluePinkGradient;
+    }
+  };
+
+  return (
+    <div
+      className={`${stylex(gradient(gradientType), !disabled && styles.animationHover)} ${disabled && 'opacity-30'} ${rounded && 'rounded-full'} ${className}`}>
+      {children}
+    </div>
+  );
+}

demo/frontend/src/common/components/button/PlaybackButton.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {OBJECT_TOOLBAR_INDEX} from '@/common/components/toolbar/ToolbarConfig';
+import Tooltip from '@/common/components/Tooltip';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {isPlayingAtom, streamingStateAtom, toolbarTabIndex} from '@/demo/atoms';
+import {PauseFilled, PlayFilledAlt} from '@carbon/icons-react';
+import {useAtomValue} from 'jotai';
+import {useCallback, useEffect} from 'react';
+
+export default function PlaybackButton() {
+  const tabIndex = useAtomValue(toolbarTabIndex);
+  const streamingState = useAtomValue(streamingStateAtom);
+  const isPlaying = useAtomValue(isPlayingAtom);
+  const video = useVideo();
+
+  const isDisabled =
+    tabIndex === OBJECT_TOOLBAR_INDEX &&
+    streamingState !== 'none' &&
+    streamingState !== 'full';
+
+  const handlePlay = useCallback(() => {
+    video?.play();
+  }, [video]);
+
+  const handlePause = useCallback(() => {
+    video?.pause();
+  }, [video]);
+
+  const handleClick = useCallback(() => {
+    if (isDisabled) {
+      return;
+    }
+    if (isPlaying) {
+      handlePause();
+    } else {
+      handlePlay();
+    }
+  }, [isDisabled, isPlaying, handlePlay, handlePause]);
+
+  useEffect(() => {
+    const handleKey = (event: KeyboardEvent) => {
+      const callback = {
+        KeyK: handleClick,
+      }[event.code];
+      if (callback != null) {
+        event.preventDefault();
+        callback();
+      }
+    };
+    document.addEventListener('keydown', handleKey);
+    return () => {
+      document.removeEventListener('keydown', handleKey);
+    };
+  }, [handleClick]);
+
+  return (
+    <Tooltip message={`${isPlaying ? 'Pause' : 'Play'} (k)`}>
+      <button
+        disabled={isDisabled}
+        className={`group !rounded-full !w-10 !h-10 flex items-center justify-center ${getButtonStyles(isDisabled)}`}
+        onClick={handleClick}>
+        {isPlaying ? (
+          <PauseFilled size={18} />
+        ) : (
+          <PlayFilledAlt
+            size={18}
+            className={!isDisabled ? 'group-hover:text-green-500' : ''}
+          />
+        )}
+      </button>
+    </Tooltip>
+  );
+}
+
+function getButtonStyles(isDisabled: boolean): string {
+  if (isDisabled) {
+    return '!bg-gray-600 !text-graydark-700';
+  }
+  return `!text-black bg-white`;
+}

demo/frontend/src/common/components/button/PrimaryCTAButton.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import GradientBorder from '@/common/components/button/GradientBorder';
+import type {ReactNode} from 'react';
+
+type Props = {
+  disabled?: boolean;
+  endIcon?: ReactNode;
+} & React.DOMAttributes<HTMLButtonElement>;
+
+export default function PrimaryCTAButton({
+  children,
+  disabled,
+  endIcon,
+  ...props
+}: Props) {
+  return (
+    <GradientBorder disabled={disabled}>
+      <button
+        className={`btn ${disabled && 'btn-disabled'} !rounded-full !bg-black !text-white !border-none`}
+        {...props}>
+        {children}
+        {endIcon != null && endIcon}
+      </button>
+    </GradientBorder>
+  );
+}

demo/frontend/src/common/components/button/ResponsiveButton.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import useScreenSize from '@/common/screen/useScreenSize';
+import type {ReactNode} from 'react';
+import type {ButtonProps} from 'react-daisyui';
+import {Button} from 'react-daisyui';
+
+type Props = ButtonProps & {startIcon: ReactNode};
+
+export default function ResponsiveButton(props: Props) {
+  const {isMobile} = useScreenSize();
+
+  return <Button {...props}>{!isMobile && props.children}</Button>;
+}

demo/frontend/src/common/components/button/TrackAndPlayButton.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import PrimaryCTAButton from '@/common/components/button/PrimaryCTAButton';
+import useMessagesSnackbar from '@/common/components/snackbar/useDemoMessagesSnackbar';
+import useFunctionThrottle from '@/common/components/useFunctionThrottle';
+import useVideo from '@/common/components/video/editor/useVideo';
+import {
+  areTrackletObjectsInitializedAtom,
+  isStreamingAtom,
+  sessionAtom,
+  streamingStateAtom,
+} from '@/demo/atoms';
+import {ChevronRight} from '@carbon/icons-react';
+import {useAtom, useAtomValue, useSetAtom} from 'jotai';
+import {useCallback, useEffect} from 'react';
+
+export default function TrackAndPlayButton() {
+  const video = useVideo();
+  const [isStreaming, setIsStreaming] = useAtom(isStreamingAtom);
+  const streamingState = useAtomValue(streamingStateAtom);
+  const areObjectsInitialized = useAtomValue(areTrackletObjectsInitializedAtom);
+  const setSession = useSetAtom(sessionAtom);
+  const {enqueueMessage} = useMessagesSnackbar();
+  const {isThrottled, maxThrottles, throttle} = useFunctionThrottle(250, 4);
+
+  const isTrackAndPlayDisabled =
+    streamingState === 'aborting' || streamingState === 'requesting';
+
+  useEffect(() => {
+    function onStreamingStarted() {
+      setIsStreaming(true);
+    }
+    video?.addEventListener('streamingStarted', onStreamingStarted);
+
+    function onStreamingCompleted() {
+      enqueueMessage('trackAndPlayComplete');
+      setIsStreaming(false);
+    }
+    video?.addEventListener('streamingCompleted', onStreamingCompleted);
+
+    return () => {
+      video?.removeEventListener('streamingStarted', onStreamingStarted);
+      video?.removeEventListener('streamingCompleted', onStreamingCompleted);
+    };
+  }, [video, setIsStreaming, enqueueMessage]);
+
+  const handleTrackAndPlay = useCallback(() => {
+    if (isTrackAndPlayDisabled) {
+      return;
+    }
+    if (maxThrottles && isThrottled) {
+      enqueueMessage('trackAndPlayThrottlingWarning');
+    }
+
+    // Throttling is only applied while streaming because we should
+    // only throttle after a user has aborted inference. This way,
+    // a user can still quickly abort a stream if they notice the
+    // inferred mask is misaligned.
+    throttle(
+      () => {
+        if (!isStreaming) {
+          enqueueMessage('trackAndPlayClick');
+          video?.streamMasks();
+          setSession(previousSession =>
+            previousSession == null
+              ? previousSession
+              : {...previousSession, ranPropagation: true},
+          );
+        } else {
+          video?.abortStreamMasks();
+        }
+      },
+      {enableThrottling: isStreaming},
+    );
+  }, [
+    isTrackAndPlayDisabled,
+    isThrottled,
+    isStreaming,
+    maxThrottles,
+    video,
+    setSession,
+    enqueueMessage,
+    throttle,
+  ]);
+
+  useEffect(() => {
+    const handleKey = (event: KeyboardEvent) => {
+      const callback = {
+        KeyK: handleTrackAndPlay,
+      }[event.code];
+      if (callback != null) {
+        event.preventDefault();
+        callback();
+      }
+    };
+    document.addEventListener('keydown', handleKey);
+    return () => {
+      document.removeEventListener('keydown', handleKey);
+    };
+  }, [handleTrackAndPlay]);
+
+  return (
+    <PrimaryCTAButton
+      disabled={isThrottled || !areObjectsInitialized}
+      onClick={handleTrackAndPlay}
+      endIcon={isStreaming ? undefined : <ChevronRight size={20} />}>
+      {isStreaming ? 'Cancel Tracking' : 'Track objects'}
+    </PrimaryCTAButton>
+  );
+}

demo/frontend/src/common/components/code/InitializeLocalMonaco.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {loader} from '@monaco-editor/react';
+
+import Logger from '@/common/logger/Logger';
+import * as monaco from 'monaco-editor';
+import editorWorker from 'monaco-editor/esm/vs/editor/editor.worker?worker';
+import tsWorker from 'monaco-editor/esm/vs/language/typescript/ts.worker?worker';
+
+self.MonacoEnvironment = {
+  getWorker(_, label) {
+    if (label === 'typescript' || label === 'javascript') {
+      return new tsWorker();
+    }
+    return new editorWorker();
+  },
+};
+
+loader.config({monaco});
+
+loader.init().then(monaco => {
+  Logger.debug('initialized monaco', monaco);
+});