Skip to content

GitLab

Commit 17d316f3 authored by suily's avatar suily
Browse files

Initial commit

parents
Pipeline #3368 failed with stages
in 0 seconds
Hide whitespace changes
Inline Side-by-side
Showing with 2104 additions and 0 deletions
demo/frontend/src/common/tracker/__generated__/SAM2ModelCancelPropagateInVideoMutation.graphql.ts 0 → 100644
View file @ 17d316f3
/**
* @generated SignedSource<<87827cb79ef9276cd5a66026151e937c>>
* @lightSyntaxTransform
* @nogrep
*/
/* tslint:disable */
/* eslint-disable */
// @ts-nocheck
import { ConcreteRequest, Mutation } from 'relay-runtime';
export type CancelPropagateInVideoInput = {
sessionId: string;
};
export type SAM2ModelCancelPropagateInVideoMutation$variables = {
input: CancelPropagateInVideoInput;
};
export type SAM2ModelCancelPropagateInVideoMutation$data = {
readonly cancelPropagateInVideo: {
readonly success: boolean;
};
};
export type SAM2ModelCancelPropagateInVideoMutation = {
response: SAM2ModelCancelPropagateInVideoMutation$data;
variables: SAM2ModelCancelPropagateInVideoMutation$variables;
};
const node: ConcreteRequest = (function(){
var v0 = [
{
"defaultValue": null,
"kind": "LocalArgument",
"name": "input"
}
],
v1 = [
{
"alias": null,
"args": [
{
"kind": "Variable",
"name": "input",
"variableName": "input"
}
],
"concreteType": "CancelPropagateInVideo",
"kind": "LinkedField",
"name": "cancelPropagateInVideo",
"plural": false,
"selections": [
{
"alias": null,
"args": null,
"kind": "ScalarField",
"name": "success",
"storageKey": null
}
],
"storageKey": null
}
];
return {
"fragment": {
"argumentDefinitions": (v0/*: any*/),
"kind": "Fragment",
"metadata": null,
"name": "SAM2ModelCancelPropagateInVideoMutation",
"selections": (v1/*: any*/),
"type": "Mutation",
"abstractKey": null
},
"kind": "Request",
"operation": {
"argumentDefinitions": (v0/*: any*/),
"kind": "Operation",
"name": "SAM2ModelCancelPropagateInVideoMutation",
"selections": (v1/*: any*/)
},
"params": {
"cacheID": "f00f78f24741d27828f0bd95b0f373c2",
"id": null,
"metadata": {},
"name": "SAM2ModelCancelPropagateInVideoMutation",
"operationKind": "mutation",
"text": "mutation SAM2ModelCancelPropagateInVideoMutation(\n $input: CancelPropagateInVideoInput!\n) {\n cancelPropagateInVideo(input: $input) {\n success\n }\n}\n"
}
};
})();
(node as any).hash = "1abafecade479ab3c45f9cecf0360285";
export default node;
demo/frontend/src/common/tracker/__generated__/SAM2ModelClearPointsInFrameMutation.graphql.ts 0 → 100644
View file @ 17d316f3
/**
* @generated SignedSource<<7330d05db0fe66bbd89190cc665dd8d9>>
* @lightSyntaxTransform
* @nogrep
*/
/* tslint:disable */
/* eslint-disable */
// @ts-nocheck
import { ConcreteRequest, Mutation } from 'relay-runtime';
export type ClearPointsInFrameInput = {
frameIndex: number;
objectId: number;
sessionId: string;
};
export type SAM2ModelClearPointsInFrameMutation$variables = {
input: ClearPointsInFrameInput;
};
export type SAM2ModelClearPointsInFrameMutation$data = {
readonly clearPointsInFrame: {
readonly frameIndex: number;
readonly rleMaskList: ReadonlyArray<{
readonly objectId: number;
readonly rleMask: {
readonly counts: string;
readonly size: ReadonlyArray<number>;
};
}>;
};
};
export type SAM2ModelClearPointsInFrameMutation = {
response: SAM2ModelClearPointsInFrameMutation$data;
variables: SAM2ModelClearPointsInFrameMutation$variables;
};
const node: ConcreteRequest = (function(){
var v0 = [
{
"defaultValue": null,
"kind": "LocalArgument",
"name": "input"
}
],
v1 = [
{
"alias": null,
"args": [
{
"kind": "Variable",
"name": "input",
"variableName": "input"
}
],
"concreteType": "RLEMaskListOnFrame",
"kind": "LinkedField",
"name": "clearPointsInFrame",
"plural": false,
"selections": [
{
"alias": null,
"args": null,
"kind": "ScalarField",
"name": "frameIndex",
"storageKey": null
},
{
"alias": null,
"args": null,
"concreteType": "RLEMaskForObject",
"kind": "LinkedField",
"name": "rleMaskList",
"plural": true,
"selections": [
{
"alias": null,
"args": null,
"kind": "ScalarField",
"name": "objectId",
"storageKey": null
},
{
"alias": null,
"args": null,
"concreteType": "RLEMask",
"kind": "LinkedField",
"name": "rleMask",
"plural": false,
"selections": [
{
"alias": null,
"args": null,
"kind": "ScalarField",
"name": "counts",
"storageKey": null
},
{
"alias": null,
"args": null,
"kind": "ScalarField",
"name": "size",
"storageKey": null
}
],
"storageKey": null
}
],
"storageKey": null
}
],
"storageKey": null
}
];
return {
"fragment": {
"argumentDefinitions": (v0/*: any*/),
"kind": "Fragment",
"metadata": null,
"name": "SAM2ModelClearPointsInFrameMutation",
"selections": (v1/*: any*/),
"type": "Mutation",
"abstractKey": null
},
"kind": "Request",
"operation": {
"argumentDefinitions": (v0/*: any*/),
"kind": "Operation",
"name": "SAM2ModelClearPointsInFrameMutation",
"selections": (v1/*: any*/)
},
"params": {
"cacheID": "b4f20e0205c26d5dc3614935ac73fa3f",
"id": null,
"metadata": {},
"name": "SAM2ModelClearPointsInFrameMutation",
"operationKind": "mutation",
"text": "mutation SAM2ModelClearPointsInFrameMutation(\n $input: ClearPointsInFrameInput!\n) {\n clearPointsInFrame(input: $input) {\n frameIndex\n rleMaskList {\n objectId\n rleMask {\n counts\n size\n }\n }\n }\n}\n"
}
};
})();
(node as any).hash = "880295870f14839040acf8f191fa1409";
export default node;
demo/frontend/src/common/tracker/__generated__/SAM2ModelClearPointsInVideoMutation.graphql.ts 0 → 100644
View file @ 17d316f3
/**
* @generated SignedSource<<092c43655450b8af706e546837e0a01c>>
* @lightSyntaxTransform
* @nogrep
*/
/* tslint:disable */
/* eslint-disable */
// @ts-nocheck
import { ConcreteRequest, Mutation } from 'relay-runtime';
export type ClearPointsInVideoInput = {
sessionId: string;
};
export type SAM2ModelClearPointsInVideoMutation$variables = {
input: ClearPointsInVideoInput;
};
export type SAM2ModelClearPointsInVideoMutation$data = {
readonly clearPointsInVideo: {
readonly success: boolean;
};
};
export type SAM2ModelClearPointsInVideoMutation = {
response: SAM2ModelClearPointsInVideoMutation$data;
variables: SAM2ModelClearPointsInVideoMutation$variables;
};
const node: ConcreteRequest = (function(){
var v0 = [
{
"defaultValue": null,
"kind": "LocalArgument",
"name": "input"
}
],
v1 = [
{
"alias": null,
"args": [
{
"kind": "Variable",
"name": "input",
"variableName": "input"
}
],
"concreteType": "ClearPointsInVideo",
"kind": "LinkedField",
"name": "clearPointsInVideo",
"plural": false,
"selections": [
{
"alias": null,
"args": null,
"kind": "ScalarField",
"name": "success",
"storageKey": null
}
],
"storageKey": null
}
];
return {
"fragment": {
"argumentDefinitions": (v0/*: any*/),
"kind": "Fragment",
"metadata": null,
"name": "SAM2ModelClearPointsInVideoMutation",
"selections": (v1/*: any*/),
"type": "Mutation",
"abstractKey": null
},
"kind": "Request",
"operation": {
"argumentDefinitions": (v0/*: any*/),
"kind": "Operation",
"name": "SAM2ModelClearPointsInVideoMutation",
"selections": (v1/*: any*/)
},
"params": {
"cacheID": "c23b3d5afca5b235328a562369056527",
"id": null,
"metadata": {},
"name": "SAM2ModelClearPointsInVideoMutation",
"operationKind": "mutation",
"text": "mutation SAM2ModelClearPointsInVideoMutation(\n $input: ClearPointsInVideoInput!\n) {\n clearPointsInVideo(input: $input) {\n success\n }\n}\n"
}
};
})();
(node as any).hash = "020267989385cb8b8f0e5cdde784d17e";
export default node;
demo/frontend/src/common/tracker/__generated__/SAM2ModelCloseSessionMutation.graphql.ts 0 → 100644
View file @ 17d316f3
/**
* @generated SignedSource<<48ee5db240b8093e9e53bf0329c8bab7>>
* @lightSyntaxTransform
* @nogrep
*/
/* tslint:disable */
/* eslint-disable */
// @ts-nocheck
import { ConcreteRequest, Mutation } from 'relay-runtime';
export type CloseSessionInput = {
sessionId: string;
};
export type SAM2ModelCloseSessionMutation$variables = {
input: CloseSessionInput;
};
export type SAM2ModelCloseSessionMutation$data = {
readonly closeSession: {
readonly success: boolean;
};
};
export type SAM2ModelCloseSessionMutation = {
response: SAM2ModelCloseSessionMutation$data;
variables: SAM2ModelCloseSessionMutation$variables;
};
const node: ConcreteRequest = (function(){
var v0 = [
{
"defaultValue": null,
"kind": "LocalArgument",
"name": "input"
}
],
v1 = [
{
"alias": null,
"args": [
{
"kind": "Variable",
"name": "input",
"variableName": "input"
}
],
"concreteType": "CloseSession",
"kind": "LinkedField",
"name": "closeSession",
"plural": false,
"selections": [
{
"alias": null,
"args": null,
"kind": "ScalarField",
"name": "success",
"storageKey": null
}
],
"storageKey": null
}
];
return {
"fragment": {
"argumentDefinitions": (v0/*: any*/),
"kind": "Fragment",
"metadata": null,
"name": "SAM2ModelCloseSessionMutation",
"selections": (v1/*: any*/),
"type": "Mutation",
"abstractKey": null
},
"kind": "Request",
"operation": {
"argumentDefinitions": (v0/*: any*/),
"kind": "Operation",
"name": "SAM2ModelCloseSessionMutation",
"selections": (v1/*: any*/)
},
"params": {
"cacheID": "aa7177838c16536b397bfee2d15a94ee",
"id": null,
"metadata": {},
"name": "SAM2ModelCloseSessionMutation",
"operationKind": "mutation",
"text": "mutation SAM2ModelCloseSessionMutation(\n $input: CloseSessionInput!\n) {\n closeSession(input: $input) {\n success\n }\n}\n"
}
};
})();
(node as any).hash = "6e1008de944562dc1922cd3f9cc40f10";
export default node;
demo/frontend/src/common/tracker/__generated__/SAM2ModelRemoveObjectMutation.graphql.ts 0 → 100644
View file @ 17d316f3
/**
* @generated SignedSource<<3d0d7bdc0d4304f08ea91b7df9efeb1f>>
* @lightSyntaxTransform
* @nogrep
*/
/* tslint:disable */
/* eslint-disable */
// @ts-nocheck
import { ConcreteRequest, Mutation } from 'relay-runtime';
export type RemoveObjectInput = {
objectId: number;
sessionId: string;
};
export type SAM2ModelRemoveObjectMutation$variables = {
input: RemoveObjectInput;
};
export type SAM2ModelRemoveObjectMutation$data = {
readonly removeObject: ReadonlyArray<{
readonly frameIndex: number;
readonly rleMaskList: ReadonlyArray<{
readonly objectId: number;
readonly rleMask: {
readonly counts: string;
readonly size: ReadonlyArray<number>;
};
}>;
}>;
};
export type SAM2ModelRemoveObjectMutation = {
response: SAM2ModelRemoveObjectMutation$data;
variables: SAM2ModelRemoveObjectMutation$variables;
};
const node: ConcreteRequest = (function(){
var v0 = [
{
"defaultValue": null,
"kind": "LocalArgument",
"name": "input"
}
],
v1 = [
{
"alias": null,
"args": [
{
"kind": "Variable",
"name": "input",
"variableName": "input"
}
],
"concreteType": "RLEMaskListOnFrame",
"kind": "LinkedField",
"name": "removeObject",
"plural": true,
"selections": [
{
"alias": null,
"args": null,
"kind": "ScalarField",
"name": "frameIndex",
"storageKey": null
},
{
"alias": null,
"args": null,
"concreteType": "RLEMaskForObject",
"kind": "LinkedField",
"name": "rleMaskList",
"plural": true,
"selections": [
{
"alias": null,
"args": null,
"kind": "ScalarField",
"name": "objectId",
"storageKey": null
},
{
"alias": null,
"args": null,
"concreteType": "RLEMask",
"kind": "LinkedField",
"name": "rleMask",
"plural": false,
"selections": [
{
"alias": null,
"args": null,
"kind": "ScalarField",
"name": "counts",
"storageKey": null
},
{
"alias": null,
"args": null,
"kind": "ScalarField",
"name": "size",
"storageKey": null
}
],
"storageKey": null
}
],
"storageKey": null
}
],
"storageKey": null
}
];
return {
"fragment": {
"argumentDefinitions": (v0/*: any*/),
"kind": "Fragment",
"metadata": null,
"name": "SAM2ModelRemoveObjectMutation",
"selections": (v1/*: any*/),
"type": "Mutation",
"abstractKey": null
},
"kind": "Request",
"operation": {
"argumentDefinitions": (v0/*: any*/),
"kind": "Operation",
"name": "SAM2ModelRemoveObjectMutation",
"selections": (v1/*: any*/)
},
"params": {
"cacheID": "0accbe68b8deea021539365678e58172",
"id": null,
"metadata": {},
"name": "SAM2ModelRemoveObjectMutation",
"operationKind": "mutation",
"text": "mutation SAM2ModelRemoveObjectMutation(\n $input: RemoveObjectInput!\n) {\n removeObject(input: $input) {\n frameIndex\n rleMaskList {\n objectId\n rleMask {\n counts\n size\n }\n }\n }\n}\n"
}
};
})();
(node as any).hash = "2dddf010d202332e6e012443cc1d8e55";
export default node;
demo/frontend/src/common/tracker/__generated__/SAM2ModelStartSessionMutation.graphql.ts 0 → 100644
View file @ 17d316f3
/**
* @generated SignedSource<<90910bae5bb646118174e736434aac56>>
* @lightSyntaxTransform
* @nogrep
*/
/* tslint:disable */
/* eslint-disable */
// @ts-nocheck
import { ConcreteRequest, Mutation } from 'relay-runtime';
export type StartSessionInput = {
path: string;
};
export type SAM2ModelStartSessionMutation$variables = {
input: StartSessionInput;
};
export type SAM2ModelStartSessionMutation$data = {
readonly startSession: {
readonly sessionId: string;
};
};
export type SAM2ModelStartSessionMutation = {
response: SAM2ModelStartSessionMutation$data;
variables: SAM2ModelStartSessionMutation$variables;
};
const node: ConcreteRequest = (function(){
var v0 = [
{
"defaultValue": null,
"kind": "LocalArgument",
"name": "input"
}
],
v1 = [
{
"alias": null,
"args": [
{
"kind": "Variable",
"name": "input",
"variableName": "input"
}
],
"concreteType": "StartSession",
"kind": "LinkedField",
"name": "startSession",
"plural": false,
"selections": [
{
"alias": null,
"args": null,
"kind": "ScalarField",
"name": "sessionId",
"storageKey": null
}
],
"storageKey": null
}
];
return {
"fragment": {
"argumentDefinitions": (v0/*: any*/),
"kind": "Fragment",
"metadata": null,
"name": "SAM2ModelStartSessionMutation",
"selections": (v1/*: any*/),
"type": "Mutation",
"abstractKey": null
},
"kind": "Request",
"operation": {
"argumentDefinitions": (v0/*: any*/),
"kind": "Operation",
"name": "SAM2ModelStartSessionMutation",
"selections": (v1/*: any*/)
},
"params": {
"cacheID": "2403f5005f5bb3805109874569f2050e",
"id": null,
"metadata": {},
"name": "SAM2ModelStartSessionMutation",
"operationKind": "mutation",
"text": "mutation SAM2ModelStartSessionMutation(\n $input: StartSessionInput!\n) {\n startSession(input: $input) {\n sessionId\n }\n}\n"
}
};
})();
(node as any).hash = "5cf0005c7a54fc87c539dd4cbd5fef5d";
export default node;
demo/frontend/src/common/utils/FileUtils.ts 0 → 100644
View file @ 17d316f3
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import Logger from '@/common/logger/Logger';
type Range = {
start: number;
end: number;
};
type FileStreamPart = {
data: Uint8Array;
range: Range;
contentLength: number;
};
export type FileStream = AsyncGenerator<FileStreamPart, File | null, null>;
/**
* Asynchronously generates a SHA-256 hash for a Blob object.
*
* DO NOT USE this function casually. Computing the SHA-256 is expensive and can
* take several 100 milliseconds to complete.
*
* @param blob - The Blob object to be hashed.
* @returns A Promise that resolves to a string representing the SHA-256 hash of
* the Blob.
*/
export async function hashBlob(blob: Blob): Promise<string> {
const buffer = await blob.arrayBuffer();
// Crypto subtle is only availabe in secure contexts. For example, this will
// be the case when running the project locally with http protocol.
// https://developer.mozilla.org/en-US/docs/Web/API/Crypto/subtle
if (crypto.subtle != null) {
const hashBuffer = await crypto.subtle.digest('SHA-256', buffer);
const hashArray = Array.from(new Uint8Array(hashBuffer));
return hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
}
// If not secure context, return random string
return (Math.random() + 1).toString(36).substring(7);
}
export async function* streamFile(url: string, init?: RequestInit): FileStream {
try {
const response = await fetch(url, init);
let blob: Blob;
// Try to download the file with a stream reader. This has the benefit
// of providing progress during the download. It requires the body and
// Content-Length. As a fallback, it uses the blob function on the
// response object.
const contentLength = response.headers.get('Content-Length');
if (response.body != null && contentLength != null) {
const totalLength = parseInt(contentLength);
const chunks: Uint8Array[] = [];
let start = 0;
let end = 0;
const reader = response.body.getReader();
try {
while (true) {
const {done, value} = await reader.read();
if (done) {
break;
}
start = end;
end += value.length;
yield {
data: value,
range: {start, end},
contentLength: totalLength,
};
}
} finally {
reader.releaseLock();
}
blob = new Blob(chunks);
} else {
blob = await response.blob();
}
const filename = await hashBlob(blob);
return new File([blob], `${filename}.mp4`);
} catch (error) {
Logger.error('aborting download due to component unmount', error);
}
return null;
}
demo/frontend/src/common/utils/ImageUtils.ts 0 → 100644
View file @ 17d316f3
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export function convertVideoFrameToImageData(
videoFrame: VideoFrame,
): ImageData | undefined {
const canvas = new OffscreenCanvas(
videoFrame.displayWidth,
videoFrame.displayHeight,
);
const ctx = canvas.getContext('2d');
ctx?.drawImage(videoFrame, 0, 0);
return ctx?.getImageData(0, 0, canvas.width, canvas.height);
}
/**
* This utility provides two functions:
* `process`: to find the bounding box of non-empty pixels from an ImageData, when looping through all its pixels
* `crop` to cut out the subsection found in `process`
* @returns
*/
export function findBoundingBox() {
let xMin = Number.MAX_VALUE;
let yMin = Number.MAX_VALUE;
let xMax = Number.MIN_VALUE;
let yMax = Number.MIN_VALUE;
return {
process: function (x: number, y: number, hasData: boolean) {
if (hasData) {
xMin = Math.min(x, xMin);
xMax = Math.max(x, xMax);
yMin = Math.min(y, yMin);
yMax = Math.max(y, yMax);
}
return [xMin, xMax, yMin, yMax];
},
crop(imageData: ImageData): ImageData | null {
const canvas = new OffscreenCanvas(imageData.width, imageData.height);
const ctx = canvas.getContext('2d');
const boundingBoxWidth = xMax - xMin;
const boundingBoxHeight = yMax - yMin;
if (ctx && boundingBoxWidth > 0 && boundingBoxHeight > 0) {
ctx.clearRect(0, 0, canvas.width, canvas.height);
ctx.putImageData(imageData, 0, 0);
return ctx.getImageData(
xMin,
yMin,
boundingBoxWidth,
boundingBoxHeight,
);
} else {
return null;
}
},
getBox(): [[number, number], [number, number]] {
return [
[xMin, yMin],
[xMax, yMax],
];
},
};
}
export function magnifyImageRegion(
canvas: HTMLCanvasElement | null,
x: number,
y: number,
radius: number = 25,
scale: number = 2,
): string {
if (canvas == null) {
return '';
}
const ctx = canvas.getContext('2d');
if (ctx) {
const minX = x - radius < 0 ? radius - x : 0;
const minY = y - radius < 0 ? radius - y : 0;
const region = ctx.getImageData(
Math.max(x - radius, 0),
Math.max(y - radius, 0),
radius * 2,
radius * 2,
);
// ImageData doesn't scale-transform correctly on canvas
// So we first draw the original size on an offscreen canvas, and then scale it
const regionCanvas = new OffscreenCanvas(region.width, region.height);
const regionCtx = regionCanvas.getContext('2d');
regionCtx?.putImageData(region, minX > 0 ? minX : 0, minY > 0 ? minY : 0);
const scaleCanvas = document.createElement('canvas');
scaleCanvas.width = Math.round(region.width * scale);
scaleCanvas.height = Math.round(region.height * scale);
const scaleCtx = scaleCanvas.getContext('2d');
scaleCtx?.scale(scale, scale);
scaleCtx?.drawImage(regionCanvas, 0, 0);
return scaleCanvas.toDataURL();
}
return '';
}
demo/frontend/src/common/utils/MaskUtils.ts 0 → 100644
View file @ 17d316f3
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Converts an image mask represented as a binary image (foreground pixels are
* `>1` and background pixels are `0`) stored in a Uint8Array to an RGBA
* representation where background pixels have an alpha value of 0 and
* foreground pixels have an alpha value of 255. This is useful for compositing
* the mask onto another image.
*
* ```typescript
* const rgba = convertMaskDataToRGBA(mask.data);
* ```
*
* @param data - The image mask represented as a Uint8Array
* @returns A new Uint8ClampedArray representing the mask in RGBA format
*/
export function convertMaskToRGBA(data: Uint8Array): Uint8ClampedArray {
// Shifting pixels instead of assigning them individually per pixel is
// much faster. See JSPerf benchamrk: https://jsperf.app/morifo
const len = data.length;
const tempData = new Uint32Array(len);
const RGA = 0x00ffffff;
const FOREGROUND = 0xff000000;
const BACKGROUND = 0x00000000;
for (let i = 0; i < len; i++) {
const alpha = data[i] > 0 ? FOREGROUND : BACKGROUND; // alpha is the high byte. Bits 24-31
tempData[i] = alpha + RGA;
}
return new Uint8ClampedArray(tempData.buffer);
}
demo/frontend/src/common/utils/MultipartStream.ts 0 → 100644
View file @ 17d316f3
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const decoder = new TextDecoder();
const encoder = new TextEncoder();
const blankLine = encoder.encode('\r\n');
const STATE_BOUNDARY = 0;
const STATE_HEADERS = 1;
const STATE_BODY = 2;
/**
* Compares two Uint8Array objects for equality.
* @param {Uint8Array} a
* @param {Uint8Array} b
* @return {bool}
*/
function compareArrays(a: Uint8Array, b: Uint8Array): boolean {
if (a.length != b.length) {
return false;
}
for (let i = 0; i < a.length; i++) {
if (a[i] != b[i]) {
return false;
}
}
return true;
}
/**
* Parses a Content-Type into a multipart boundary.
* @param {string} contentType
* @return {Uint8Array} boundary line, including preceding -- and trailing \r\n
*/
function getBoundary(contentType: string): Uint8Array | null {
// Expects the form "multipart/...; boundary=...".
// This is not a full MIME media type parser but should be good enough.
const MULTIPART_TYPE = 'multipart/';
const BOUNDARY_PARAM = '; boundary=';
if (!contentType.startsWith(MULTIPART_TYPE)) {
return null;
}
const i = contentType.indexOf(BOUNDARY_PARAM, MULTIPART_TYPE.length);
if (i == -1) {
return null;
}
const suffix = contentType.substring(i + BOUNDARY_PARAM.length);
return encoder.encode('--' + suffix + '\r\n');
}
/**
* Creates a multipart stream.
* @param {string} contentType A Content-Type header.
* @param {ReadableStream} body The body of a HTTP response.
* @return {ReadableStream} a stream of {headers: Headers, body: Uint8Array}
* objects.
*/
export default function multipartStream(
contentType: string,
body: ReadableStream,
): ReadableStream {
const reader = body.getReader();
return new ReadableStream({
async start(controller) {
// Define the boundary.
const boundary = getBoundary(contentType);
if (boundary === null) {
controller.error(
new Error(
'Invalid content type for multipart stream: ' + contentType,
),
);
return;
}
let pos = 0;
let buf = new Uint8Array(); // buf.slice(pos) has unprocessed data.
let state = STATE_BOUNDARY;
let headers: Headers | null = null; // non-null in STATE_HEADERS and STATE_BODY.
let contentLength: number | null = null; // non-null in STATE_BODY.
/**
* Consumes all complete data in buf or raises an Error.
* May leave incomplete data at buf.slice(pos).
*/
function processBuf() {
// The while(true) condition is reqired
// eslint-disable-next-line no-constant-condition
while (true) {
if (boundary === null) {
controller.error(
new Error(
'Invalid content type for multipart stream: ' + contentType,
),
);
return;
}
switch (state) {
case STATE_BOUNDARY:
// Read blank lines (if any) then boundary.
while (
buf.length >= pos + blankLine.length &&
compareArrays(buf.slice(pos, pos + blankLine.length), blankLine)
) {
pos += blankLine.length;
}
// Check that it starts with a boundary.
if (buf.length < pos + boundary.length) {
return;
}
if (
!compareArrays(buf.slice(pos, pos + boundary.length), boundary)
) {
throw new Error('bad part boundary');
}
pos += boundary.length;
state = STATE_HEADERS;
headers = new Headers();
break;
case STATE_HEADERS: {
const cr = buf.indexOf('\r'.charCodeAt(0), pos);
if (cr == -1 || buf.length == cr + 1) {
return;
}
if (buf[cr + 1] != '\n'.charCodeAt(0)) {
throw new Error('bad part header line (CR without NL)');
}
const line = decoder.decode(buf.slice(pos, cr));
pos = cr + 2;
if (line == '') {
const rawContentLength = headers?.get('Content-Length');
if (rawContentLength == null) {
throw new Error('missing/invalid part Content-Length');
}
contentLength = parseInt(rawContentLength, 10);
if (isNaN(contentLength)) {
throw new Error('missing/invalid part Content-Length');
}
state = STATE_BODY;
break;
}
const colon = line.indexOf(':');
const name = line.substring(0, colon);
if (colon == line.length || line[colon + 1] != ' ') {
throw new Error('bad part header line (no ": ")');
}
const value = line.substring(colon + 2);
headers?.append(name, value);
break;
}
case STATE_BODY: {
if (contentLength === null) {
throw new Error('content length not set');
}
if (buf.length < pos + contentLength) {
return;
}
const body = buf.slice(pos, pos + contentLength);
pos += contentLength;
controller.enqueue({
headers: headers,
body: body,
});
headers = null;
contentLength = null;
state = STATE_BOUNDARY;
break;
}
}
}
}
// The while(true) condition is required
// eslint-disable-next-line no-constant-condition
while (true) {
const {done, value} = await reader.read();
const buffered = buf.length - pos;
if (done) {
if (state != STATE_BOUNDARY || buffered > 0) {
throw Error('multipart stream ended mid-part');
}
controller.close();
return;
}
// Update buf.slice(pos) to include the new data from value.
if (buffered == 0) {
buf = value;
} else {
const newLen = buffered + value.length;
const newBuf = new Uint8Array(newLen);
newBuf.set(buf.slice(pos), 0);
newBuf.set(value, buffered);
buf = newBuf;
}
pos = 0;
processBuf();
}
},
cancel(reason) {
return body.cancel(reason);
},
});
}
demo/frontend/src/common/utils/ShaderUtils.ts 0 → 100644
View file @ 17d316f3
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {Tracklet} from '@/common/tracker/Tracker';
/**
* util funtion to generate a WebGL texture using a look up table.
* @param {WebGL2RenderingContext} gl - The WebGL2 rendering context.
* @param {number} lutSize - The size of the LUT in each dimension.
* @param {Uint8Array} lutData - The LUT data as an array of unsigned 8-bit integers.
* @returns {WebGLTexture} - The WebGL texture object representing the loaded LUT.
*/
export function load3DLUT(
gl: WebGL2RenderingContext,
lutSize: number,
lutData: Uint8Array,
) {
const texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_3D, texture);
gl.texParameteri(gl.TEXTURE_3D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_3D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_3D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_3D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_3D, gl.TEXTURE_WRAP_R, gl.CLAMP_TO_EDGE);
// Pixel storage modes must be set to default for 3D textures
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, false);
gl.texImage3D(
gl.TEXTURE_3D,
0,
gl.RGBA,
lutSize,
lutSize,
lutSize,
0,
gl.RGBA,
gl.UNSIGNED_BYTE,
lutData,
);
gl.bindTexture(gl.TEXTURE_3D, null);
return texture;
}
/**
* Generates a 3D lookup table (LUT) data with random RGBA values.
* @param {number} lutSize - The size of the LUT in each dimension.
* @returns {Uint8Array} - The LUT data as an array of unsigned 8-bit integers.
*/
export function generateLUTDATA(lutSize: number) {
const totalEntries = lutSize * lutSize * lutSize; // 3D LUT nodes
const lutData = new Uint8Array(totalEntries * 4); // Each entry has an RGBA value
for (let i = 0; i < totalEntries; i++) {
lutData[i * 4 + 0] = Math.floor(Math.random() * 256); // Random red value
lutData[i * 4 + 1] = Math.floor(Math.random() * 256); // Random green value
lutData[i * 4 + 2] = Math.floor(Math.random() * 256); // Random blue value
lutData[i * 4 + 3] = 1; // alpha value
}
return lutData;
}
/**
* Normalizes the bounds of a rectangle defined by two points (A and B) within a given width and height.
* @param {number[]} pointA - The coordinates of the first point defining the rectangle.
* @param {number[]} pointB - The coordinates of the second point defining the rectangle.
* @param {number} width - The width of the canvas or container where the rectangle is drawn.
* @param {number} height - The height of the canvas or container where the rectangle is drawn.
* @returns {number[]} - An array containing the normalized x and y coordinates of the rectangle's corners.
*/
export function normalizeBounds(
pointA: number[],
pointB: number[],
width: number,
height: number,
) {
return [
pointA[0] / width,
pointA[1] / height,
pointB[0] / width,
pointB[1] / height,
];
}
/**
* Pre-allocates a specified number of 2D textures for use in WebGL2 rendering.
* @param {WebGL2RenderingContext} gl - The WebGL2 rendering context.
* @param {number} numTextures - The number of textures to be pre-allocated.
* @returns {WebGLTexture[]} - An array of WebGL textures, each pre-allocated and ready for use.
*/
export function preAllocateTextures(
gl: WebGL2RenderingContext,
numTextures: number,
) {
const maskTextures = [];
for (let i = 0; i < numTextures; i++) {
const maskTexture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, maskTexture);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
maskTextures.push(maskTexture);
}
return maskTextures as WebGLTexture[];
}
/**
* Finds the index of a Tracklet object within an array based on its unique identifier.
* @param objects - The array of Tracklet objects to search within.
* @param id - The unique identifier of the Tracklet object to find.
* @returns The index of the `Tracklet` object with the specified `id` in the `objects` array.
*/
export function findIndexByTrackletId(id: number, objects: Tracklet[]): number {
return objects.findIndex(obj => obj.id === id);
}
demo/frontend/src/common/utils/emptyFunction.ts 0 → 100644
View file @ 17d316f3
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* This function accepts and discards inputs; it has no side effects. This is
* primarily useful idiomatically for overridable function endpoints which
* always need to be callable, since JS lacks a null-call idiom ala Cocoa.
*/
export default function () {}
demo/frontend/src/common/utils/uuid.ts 0 → 100644
View file @ 17d316f3
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Generates a random UUID (Universally Unique Identifier) following the version
* 4 standard.
*
* The function replaces each 'x' and 'y' in the template
* 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx' with random hexadecimal digits. For
* 'y', the function ensures the first hexadecimal digit is '8', '9', 'A', or
* 'B' as per the UUID v4 standard.
*
* @returns A string representing a version 4 UUID.
*
* @example
*
* const id = uuidv4();
* console.log(id); // Outputs: '3f0d2c77-4f69-4c1e-8a6e-35e866e8a5d1'
*/
export function uuidv4() {
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
const r = (Math.random() * 16) | 0,
v = c == 'x' ? r : (r & 0x3) | 0x8;
return v.toString(16);
});
}
demo/frontend/src/debug/stats/Stats.ts 0 → 100644
View file @ 17d316f3
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Derived from mrdoob / http://mrdoob.com/
*/
import Logger from '@/common/logger/Logger';
import {uuidv4} from '@/common/utils/uuid';
import invariant from 'invariant';
export type Request<A, P> = {
action: A;
} & P;
export type Response<A, P> = Request<A, P>;
export type GetStatsCanvasRequest = Request<
'getStatsCanvas',
{
id: string;
width: number;
height: number;
}
>;
export type GetMemoryStatsRequest = Request<
'getMemoryStats',
{
id: string;
jsHeapSizeLimit: number;
totalJSHeapSize: number;
usedJSHeapSize: number;
}
>;
export type SetStatsCanvasResponse = Response<
'setStatsCanvas',
{
id: string;
canvas: OffscreenCanvas;
devicePixelRatio: number;
}
>;
export type MemoryStatsResponse = Response<
'memoryStats',
{
id: string;
jsHeapSizeLimit: number;
totalJSHeapSize: number;
usedJSHeapSize: number;
}
>;
export type StatsType = 'fps' | 'ms' | 'memory';
export class Stats {
private maxValue: number;
private beginTime: number;
private prevTime: number;
private frames: number;
private fpsPanel: Panel | null = null;
private msPanel: Panel | null = null;
private memPanel: Panel | null = null;
constructor(type: StatsType, label: string = '', maxValue: number = 100) {
const id = uuidv4();
this.maxValue = maxValue;
this.beginTime = (performance || Date).now();
this.prevTime = this.beginTime;
this.frames = 0;
const onMessage = (event: MessageEvent<SetStatsCanvasResponse>) => {
if (event.data.action === 'setStatsCanvas' && event.data.id === id) {
const {canvas, devicePixelRatio} = event.data;
if (type === 'fps') {
this.fpsPanel = new Panel(
canvas,
devicePixelRatio,
`FPS ${label}`.trim(),
'#0ff',
'#002',
);
} else if (type === 'ms') {
this.msPanel = new Panel(
canvas,
devicePixelRatio,
`MS ${label}`.trim(),
'#0f0',
'#020',
);
} else if (type === 'memory') {
this.memPanel = new Panel(
canvas,
devicePixelRatio,
`MB ${label}`.trim(),
'#f08',
'#201',
);
}
self.removeEventListener('message', onMessage);
}
};
self.addEventListener('message', onMessage);
self.postMessage({
action: 'getStatsCanvas',
id,
width: 80,
height: 48,
} as GetStatsCanvasRequest);
}
updateMaxValue(maxValue: number) {
this.maxValue = maxValue;
}
begin() {
this.beginTime = (performance || Date).now();
}
end() {
this.frames++;
const time = (performance || Date).now();
this.msPanel?.update(time - this.beginTime, this.maxValue);
if (time >= this.prevTime + 1000) {
this.fpsPanel?.update(
(this.frames * 1000) / (time - this.prevTime),
this.maxValue,
);
this.prevTime = time;
this.frames = 0;
const id = uuidv4();
const onMessage = (event: MessageEvent<MemoryStatsResponse>) => {
if (event.data.action === 'memoryStats' && event.data.id === id) {
const {usedJSHeapSize, jsHeapSizeLimit} = event.data;
this.memPanel?.update(
usedJSHeapSize / 1048576,
jsHeapSizeLimit / 1048576,
);
}
};
self.addEventListener('message', onMessage);
self.postMessage({
action: 'getMemoryStats',
id,
} as GetMemoryStatsRequest);
}
return time;
}
update() {
this.beginTime = this.end();
}
}
export class Panel {
private min = Infinity;
private max = 0;
private round = Math.round;
private PR: number;
private WIDTH: number;
private HEIGHT: number;
private TEXT_X: number;
private TEXT_Y: number;
private GRAPH_X: number;
private GRAPH_Y: number;
private GRAPH_WIDTH: number;
private GRAPH_HEIGHT: number;
public canvas: HTMLCanvasElement | OffscreenCanvas;
private context:
| CanvasRenderingContext2D
| OffscreenCanvasRenderingContext2D
| null = null;
private name: string;
private fg: string;
private bg: string;
constructor(
canvas: HTMLCanvasElement | OffscreenCanvas,
devicePixelRatio: number,
name: string,
fg: string,
bg: string,
) {
this.canvas = canvas;
this.name = name;
this.fg = fg;
this.bg = bg;
this.PR = this.round(devicePixelRatio || 1);
this.WIDTH = 80 * this.PR;
this.HEIGHT = 48 * this.PR;
this.TEXT_X = 3 * this.PR;
this.TEXT_Y = 2 * this.PR;
this.GRAPH_X = 3 * this.PR;
this.GRAPH_Y = 15 * this.PR;
this.GRAPH_WIDTH = 74 * this.PR;
this.GRAPH_HEIGHT = 30 * this.PR;
const context: OffscreenCanvasRenderingContext2D | RenderingContext | null =
canvas.getContext('2d');
invariant(context !== null, 'context 2d is required');
if (
!(context instanceof CanvasRenderingContext2D) &&
!(context instanceof OffscreenCanvasRenderingContext2D)
) {
Logger.warn(
'rendering stats requires CanvasRenderingContext2D or OffscreenCanvasRenderingContext2D',
);
return;
}
context.font = 'bold ' + 9 * this.PR + 'px Helvetica,Arial,sans-serif';
context.textBaseline = 'top';
context.fillStyle = bg;
context.fillRect(0, 0, this.WIDTH, this.HEIGHT);
context.fillStyle = fg;
context.fillText(name, this.TEXT_X, this.TEXT_Y);
context.fillRect(
this.GRAPH_X,
this.GRAPH_Y,
this.GRAPH_WIDTH,
this.GRAPH_HEIGHT,
);
context.fillStyle = bg;
context.globalAlpha = 0.9;
context.fillRect(
this.GRAPH_X,
this.GRAPH_Y,
this.GRAPH_WIDTH,
this.GRAPH_HEIGHT,
);
this.context = context;
}
update(value: number, maxValue: number) {
invariant(this.context !== null, 'context 2d is required');
this.min = Math.min(this.min, value);
this.max = Math.max(this.max, value);
this.context.fillStyle = this.bg;
this.context.globalAlpha = 1;
this.context.fillRect(0, 0, this.WIDTH, this.GRAPH_Y);
this.context.fillStyle = this.fg;
this.context.fillText(
this.round(value) +
' ' +
this.name +
' (' +
this.round(this.min) +
'-' +
this.round(this.max) +
')',
this.TEXT_X,
this.TEXT_Y,
);
this.context.drawImage(
this.canvas,
this.GRAPH_X + this.PR,
this.GRAPH_Y,
this.GRAPH_WIDTH - this.PR,
this.GRAPH_HEIGHT,
this.GRAPH_X,
this.GRAPH_Y,
this.GRAPH_WIDTH - this.PR,
this.GRAPH_HEIGHT,
);
this.context.fillRect(
this.GRAPH_X + this.GRAPH_WIDTH - this.PR,
this.GRAPH_Y,
this.PR,
this.GRAPH_HEIGHT,
);
this.context.fillStyle = this.bg;
this.context.globalAlpha = 0.9;
this.context.fillRect(
this.GRAPH_X + this.GRAPH_WIDTH - this.PR,
this.GRAPH_Y,
this.PR,
this.round((1 - value / maxValue) * this.GRAPH_HEIGHT),
);
}
}
demo/frontend/src/debug/stats/StatsView.tsx 0 → 100644
View file @ 17d316f3
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {EnableStatsRequest} from '@/common/components/video/VideoWorkerTypes';
import stylex from '@stylexjs/stylex';
import {useEffect, useMemo, useRef, useState} from 'react';
import {useLocation} from 'react-router-dom';
import useVideo from '../../common/components/video/editor/useVideo';
import {
GetMemoryStatsRequest,
GetStatsCanvasRequest,
MemoryStatsResponse,
SetStatsCanvasResponse,
} from './Stats';
const styles = stylex.create({
container: {
position: 'fixed',
top: 0,
left: 0,
width: '100%',
overflowX: 'auto',
display: 'flex',
flexDirection: 'row',
cursor: 'pointer',
opacity: 0.9,
zIndex: 10000,
},
});
const URL_PARAM = 'monitors';
export default function StatsView() {
const {search} = useLocation();
const video = useVideo();
const containerRef = useRef<HTMLDivElement | null>(null);
const [isWrapped, setIsWrapped] = useState<boolean>(false);
const isEnabled = useMemo(() => {
const urlSearchParams = new URLSearchParams(search);
return (
urlSearchParams.has(URL_PARAM) &&
['true', ''].includes(urlSearchParams.get('monitors') ?? '')
);
}, [search]);
useEffect(() => {
if (!isEnabled) {
return;
}
const worker = video?.getWorker_ONLY_USE_WITH_CAUTION();
// Enable stats for video worker
worker?.postMessage({
action: 'enableStats',
} as EnableStatsRequest);
function onMessage(
event: MessageEvent<GetStatsCanvasRequest | GetMemoryStatsRequest>,
) {
if (event.data.action === 'getStatsCanvas') {
// Add stats canvas and hand control over to worker
const canvas = document.createElement('canvas');
canvas.width = event.data.width * window.devicePixelRatio;
canvas.height = event.data.height * window.devicePixelRatio;
canvas.style.width = `${event.data.width}px`;
canvas.style.height = `${event.data.height}px`;
containerRef.current?.appendChild(canvas);
const offscreenCanvas = canvas.transferControlToOffscreen();
worker?.postMessage(
{
action: 'setStatsCanvas',
id: event.data.id,
canvas: offscreenCanvas,
devicePixelRatio: window.devicePixelRatio,
} as SetStatsCanvasResponse,
{
transfer: [offscreenCanvas],
},
);
} else if (event.data.action === 'getMemoryStats') {
// @ts-expect-error performance.memory might not exist
const memory = performance.memory ?? {
jsHeapSizeLimit: 0,
totalJSHeapSize: 0,
usedJSHeapSize: 0,
};
worker?.postMessage({
action: 'memoryStats',
id: event.data.id,
jsHeapSizeLimit: memory.jsHeapSizeLimit,
totalJSHeapSize: memory.totalJSHeapSize,
usedJSHeapSize: memory.usedJSHeapSize,
} as MemoryStatsResponse);
}
}
worker?.addEventListener('message', onMessage);
return () => {
worker?.removeEventListener('message', onMessage);
};
}, [video, isEnabled]);
function handleClick() {
setIsWrapped(w => !w);
}
if (!isEnabled) {
return null;
}
return (
<div
ref={containerRef}
{...stylex.props(styles.container)}
style={{flexWrap: isWrapped ? 'wrap' : 'unset'}}
onDoubleClick={handleClick}
/>
);
}
demo/frontend/src/demo/DemoConfig.tsx 0 → 100644
View file @ 17d316f3
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {Effects} from '@/common/components/video/effects/Effects';
type EffectLayers = {
background: keyof Effects;
highlight: keyof Effects;
};
export const DEMO_SHORT_NAME = 'SAM 2 Demo';
export const RESEARCH_BY_META_AI = 'By Meta FAIR';
export const DEMO_FRIENDLY_NAME = 'Segment Anything 2 Demo';
export const VIDEO_WATERMARK_TEXT = `Modified with ${DEMO_FRIENDLY_NAME}`;
export const PROJECT_GITHUB_URL =
'https://github.com/facebookresearch/sam2';
export const AIDEMOS_URL = 'https://aidemos.meta.com';
export const ABOUT_URL = 'https://ai.meta.com/sam2';
export const EMAIL_ADDRESS = 'segment-anything@meta.com';
export const BLOG_URL = 'http://ai.meta.com/blog/sam2';
export const VIDEO_API_ENDPOINT = 'http://localhost:7263';
export const INFERENCE_API_ENDPOINT = 'http://localhost:7263';
export const demoObjectLimit = 3;
export const DEFAULT_EFFECT_LAYERS: EffectLayers = {
background: 'Original',
highlight: 'Overlay',
};
export const MAX_UPLOAD_FILE_SIZE = '70MB';
demo/frontend/src/demo/DemoErrorFallback.tsx 0 → 100644
View file @ 17d316f3
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import LoadingStateScreen from '@/common/loading/LoadingStateScreen';
import {FallbackProps} from 'react-error-boundary';
export default function DemoErrorFallback(_props: FallbackProps) {
return (
<LoadingStateScreen
title="Well, this is embarrassing..."
description="This demo is not optimized for your device. Please try again on a different device with a larger screen."
linkProps={{to: '..', label: 'Back to homepage'}}
/>
);
}
demo/frontend/src/demo/DemoSuspenseFallback.tsx 0 → 100644
View file @ 17d316f3
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import LoadingStateScreen from '@/common/loading/LoadingStateScreen';
export default function DemoSuspenseFallback() {
return <LoadingStateScreen title="Fetching data" />;
}
demo/frontend/src/demo/SAM2DemoApp.tsx 0 → 100644
View file @ 17d316f3
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import '@/assets/scss/App.scss';
import ErrorReport from '@/common/error/ErrorReport';
import DemoErrorFallback from '@/demo/DemoErrorFallback';
import DemoSuspenseFallback from '@/demo/DemoSuspenseFallback';
import RelayEnvironmentProvider from '@/graphql/RelayEnvironmentProvider';
import RootLayout from '@/layouts/RootLayout';
import SAM2DemoPage from '@/routes/DemoPageWrapper';
import PageNotFoundPage from '@/routes/PageNotFoundPage';
import useSettingsContext from '@/settings/useSettingsContext';
import {Route, Routes} from 'react-router-dom';
export default function DemoAppWrapper() {
const {settings} = useSettingsContext();
return (
<RelayEnvironmentProvider
endpoint={settings.videoAPIEndpoint}
suspenseFallback={<DemoSuspenseFallback />}
errorFallback={DemoErrorFallback}>
<DemoApp />
</RelayEnvironmentProvider>
);
}
function DemoApp() {
return (
<>
<Routes>
<Route element={<RootLayout />}>
<Route index={true} element={<SAM2DemoPage />} />
<Route path="*" element={<PageNotFoundPage />} />
</Route>
</Routes>
<ErrorReport />
</>
);
}
demo/frontend/src/demo/atoms.ts 0 → 100644
View file @ 17d316f3
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
defaultMessageMap,
MessagesEventMap,
} from '@/common/components/snackbar/DemoMessagesSnackbarUtils';
import {Effects} from '@/common/components/video/effects/Effects';
import {
DemoEffect,
highlightEffects,
} from '@/common/components/video/effects/EffectUtils';
import {
BaseTracklet,
SegmentationPoint,
StreamingState,
} from '@/common/tracker/Tracker';
import type {DataArray} from '@/jscocotools/mask';
import {atom} from 'jotai';
export type VideoData = {
path: string;
posterPath: string | null | undefined;
url: string;
posterUrl: string;
width: number;
height: number;
};
export const frameIndexAtom = atom<number>(0);
export const inputVideoAtom = atom<VideoData | null>(null);
// #####################
// SESSION
// #####################
export type Session = {
id: string;
ranPropagation: boolean;
};
export const sessionAtom = atom<Session | null>(null);
// #####################
// STREAMING/PLAYBACK
// #####################
export const isVideoLoadingAtom = atom<boolean>(false);
export const streamingStateAtom = atom<StreamingState>('none');
export const isPlayingAtom = atom<boolean>(false);
export const isStreamingAtom = atom<boolean>(false);
// #####################
// OBJECTS
// #####################
export type TrackletMask = {
mask: DataArray;
isEmpty: boolean;
};
export type TrackletObject = {
id: number;
color: string;
thumbnail: string | null;
points: SegmentationPoint[][];
masks: TrackletMask[];
isInitialized: boolean;
};
const MAX_NUMBER_TRACKLET_OBJECTS = 3;
export const activeTrackletObjectIdAtom = atom<number | null>(0);
export const activeTrackletObjectAtom = atom<BaseTracklet | null>(get => {
const objectId = get(activeTrackletObjectIdAtom);
const tracklets = get(trackletObjectsAtom);
return tracklets.find(obj => obj.id === objectId) ?? null;
});
export const trackletObjectsAtom = atom<BaseTracklet[]>([]);
export const maxTrackletObjectIdAtom = atom<number>(get => {
const tracklets = get(trackletObjectsAtom);
return tracklets.reduce((prev, curr) => Math.max(prev, curr.id), 0);
});
export const isTrackletObjectLimitReachedAtom = atom<boolean>(
get => get(trackletObjectsAtom).length >= MAX_NUMBER_TRACKLET_OBJECTS,
);
export const areTrackletObjectsInitializedAtom = atom<boolean>(get =>
get(trackletObjectsAtom).every(obj => obj.isInitialized),
);
export const isFirstClickMadeAtom = atom(get => {
const tracklets = get(trackletObjectsAtom);
return tracklets.some(tracklet => tracklet.points.length > 0);
});
export const pointsAtom = atom<SegmentationPoint[]>(get => {
const frameIndex = get(frameIndexAtom);
const activeTracklet = get(activeTrackletObjectAtom);
return activeTracklet?.points[frameIndex] ?? [];
});
export const labelTypeAtom = atom<'positive' | 'negative'>('positive');
export const isAddObjectEnabledAtom = atom<boolean>(get => {
const session = get(sessionAtom);
const trackletsInitialized = get(areTrackletObjectsInitializedAtom);
const isObjectLimitReached = get(isTrackletObjectLimitReachedAtom);
return (
session?.ranPropagation === false &&
trackletsInitialized &&
!isObjectLimitReached
);
});
export const codeEditorOpenedAtom = atom<boolean>(false);
export const tutorialVideoEnabledAtom = atom<boolean>(true);
// #####################
// Effects
// #####################
type EffectConfig = {
name: keyof Effects;
variant: number;
numVariants: number;
};
export const activeBackgroundEffectAtom = atom<EffectConfig>({
name: 'Original',
variant: 0,
numVariants: 0,
});
export const activeHighlightEffectAtom = atom<EffectConfig>({
name: 'Overlay',
variant: 0,
numVariants: 0,
});
export const activeHighlightEffectGroupAtom =
atom<DemoEffect[]>(highlightEffects);
// #####################
// Toolbar
// #####################
export const toolbarTabIndex = atom<number>(0);
// #####################
// Messages snackbar
// #####################
export const messageMapAtom = atom<MessagesEventMap>(defaultMessageMap);
// #####################
// Upload state
// #####################
export const uploadingStateAtom = atom<'default' | 'uploading' | 'error'>(
'default',
);
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment