"Initial commit"

demo/frontend/src/common/utils/MultipartStream.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const decoder = new TextDecoder();
+const encoder = new TextEncoder();
+const blankLine = encoder.encode('\r\n');
+
+const STATE_BOUNDARY = 0;
+const STATE_HEADERS = 1;
+const STATE_BODY = 2;
+
+/**
+ * Compares two Uint8Array objects for equality.
+ * @param {Uint8Array} a
+ * @param {Uint8Array} b
+ * @return {bool}
+ */
+function compareArrays(a: Uint8Array, b: Uint8Array): boolean {
+  if (a.length != b.length) {
+    return false;
+  }
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] != b[i]) {
+      return false;
+    }
+  }
+  return true;
+}
+
+/**
+ * Parses a Content-Type into a multipart boundary.
+ * @param {string} contentType
+ * @return {Uint8Array} boundary line, including preceding -- and trailing \r\n
+ */
+function getBoundary(contentType: string): Uint8Array | null {
+  // Expects the form "multipart/...; boundary=...".
+  // This is not a full MIME media type parser but should be good enough.
+  const MULTIPART_TYPE = 'multipart/';
+  const BOUNDARY_PARAM = '; boundary=';
+  if (!contentType.startsWith(MULTIPART_TYPE)) {
+    return null;
+  }
+  const i = contentType.indexOf(BOUNDARY_PARAM, MULTIPART_TYPE.length);
+  if (i == -1) {
+    return null;
+  }
+  const suffix = contentType.substring(i + BOUNDARY_PARAM.length);
+  return encoder.encode('--' + suffix + '\r\n');
+}
+
+/**
+ * Creates a multipart stream.
+ * @param {string} contentType A Content-Type header.
+ * @param {ReadableStream} body The body of a HTTP response.
+ * @return {ReadableStream} a stream of {headers: Headers, body: Uint8Array}
+ *     objects.
+ */
+export default function multipartStream(
+  contentType: string,
+  body: ReadableStream,
+): ReadableStream {
+  const reader = body.getReader();
+  return new ReadableStream({
+    async start(controller) {
+      // Define the boundary.
+      const boundary = getBoundary(contentType);
+      if (boundary === null) {
+        controller.error(
+          new Error(
+            'Invalid content type for multipart stream: ' + contentType,
+          ),
+        );
+        return;
+      }
+      let pos = 0;
+      let buf = new Uint8Array(); // buf.slice(pos) has unprocessed data.
+      let state = STATE_BOUNDARY;
+      let headers: Headers | null = null; // non-null in STATE_HEADERS and STATE_BODY.
+      let contentLength: number | null = null; // non-null in STATE_BODY.
+
+      /**
+       * Consumes all complete data in buf or raises an Error.
+       * May leave incomplete data at buf.slice(pos).
+       */
+      function processBuf() {
+        // The while(true) condition is reqired
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+          if (boundary === null) {
+            controller.error(
+              new Error(
+                'Invalid content type for multipart stream: ' + contentType,
+              ),
+            );
+            return;
+          }
+          switch (state) {
+            case STATE_BOUNDARY:
+              // Read blank lines (if any) then boundary.
+              while (
+                buf.length >= pos + blankLine.length &&
+                compareArrays(buf.slice(pos, pos + blankLine.length), blankLine)
+              ) {
+                pos += blankLine.length;
+              }
+
+              // Check that it starts with a boundary.
+              if (buf.length < pos + boundary.length) {
+                return;
+              }
+
+              if (
+                !compareArrays(buf.slice(pos, pos + boundary.length), boundary)
+              ) {
+                throw new Error('bad part boundary');
+              }
+              pos += boundary.length;
+              state = STATE_HEADERS;
+              headers = new Headers();
+              break;
+
+            case STATE_HEADERS: {
+              const cr = buf.indexOf('\r'.charCodeAt(0), pos);
+              if (cr == -1 || buf.length == cr + 1) {
+                return;
+              }
+              if (buf[cr + 1] != '\n'.charCodeAt(0)) {
+                throw new Error('bad part header line (CR without NL)');
+              }
+              const line = decoder.decode(buf.slice(pos, cr));
+              pos = cr + 2;
+              if (line == '') {
+                const rawContentLength = headers?.get('Content-Length');
+                if (rawContentLength == null) {
+                  throw new Error('missing/invalid part Content-Length');
+                }
+                contentLength = parseInt(rawContentLength, 10);
+                if (isNaN(contentLength)) {
+                  throw new Error('missing/invalid part Content-Length');
+                }
+                state = STATE_BODY;
+                break;
+              }
+              const colon = line.indexOf(':');
+              const name = line.substring(0, colon);
+              if (colon == line.length || line[colon + 1] != ' ') {
+                throw new Error('bad part header line (no ": ")');
+              }
+              const value = line.substring(colon + 2);
+              headers?.append(name, value);
+              break;
+            }
+            case STATE_BODY: {
+              if (contentLength === null) {
+                throw new Error('content length not set');
+              }
+              if (buf.length < pos + contentLength) {
+                return;
+              }
+              const body = buf.slice(pos, pos + contentLength);
+              pos += contentLength;
+              controller.enqueue({
+                headers: headers,
+                body: body,
+              });
+              headers = null;
+              contentLength = null;
+              state = STATE_BOUNDARY;
+              break;
+            }
+          }
+        }
+      }
+
+      // The while(true) condition is required
+      // eslint-disable-next-line no-constant-condition
+      while (true) {
+        const {done, value} = await reader.read();
+        const buffered = buf.length - pos;
+        if (done) {
+          if (state != STATE_BOUNDARY || buffered > 0) {
+            throw Error('multipart stream ended mid-part');
+          }
+          controller.close();
+          return;
+        }
+
+        // Update buf.slice(pos) to include the new data from value.
+        if (buffered == 0) {
+          buf = value;
+        } else {
+          const newLen = buffered + value.length;
+          const newBuf = new Uint8Array(newLen);
+          newBuf.set(buf.slice(pos), 0);
+          newBuf.set(value, buffered);
+          buf = newBuf;
+        }
+        pos = 0;
+
+        processBuf();
+      }
+    },
+    cancel(reason) {
+      return body.cancel(reason);
+    },
+  });
+}

demo/frontend/src/common/utils/ShaderUtils.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Tracklet} from '@/common/tracker/Tracker';
+
+/**
+ * util funtion to generate a WebGL texture using a look up table.
+ * @param {WebGL2RenderingContext} gl - The WebGL2 rendering context.
+ * @param {number} lutSize - The size of the LUT in each dimension.
+ * @param {Uint8Array} lutData - The LUT data as an array of unsigned 8-bit integers.
+ * @returns {WebGLTexture} - The WebGL texture object representing the loaded LUT.
+ */
+export function load3DLUT(
+  gl: WebGL2RenderingContext,
+  lutSize: number,
+  lutData: Uint8Array,
+) {
+  const texture = gl.createTexture();
+  gl.bindTexture(gl.TEXTURE_3D, texture);
+  gl.texParameteri(gl.TEXTURE_3D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+  gl.texParameteri(gl.TEXTURE_3D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
+  gl.texParameteri(gl.TEXTURE_3D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+  gl.texParameteri(gl.TEXTURE_3D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+  gl.texParameteri(gl.TEXTURE_3D, gl.TEXTURE_WRAP_R, gl.CLAMP_TO_EDGE);
+  // Pixel storage modes must be set to default for 3D textures
+  gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
+  gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, false);
+
+  gl.texImage3D(
+    gl.TEXTURE_3D,
+    0,
+    gl.RGBA,
+    lutSize,
+    lutSize,
+    lutSize,
+    0,
+    gl.RGBA,
+    gl.UNSIGNED_BYTE,
+    lutData,
+  );
+  gl.bindTexture(gl.TEXTURE_3D, null);
+  return texture;
+}
+
+/**
+ * Generates a 3D lookup table (LUT) data with random RGBA values.
+ * @param {number} lutSize - The size of the LUT in each dimension.
+ * @returns {Uint8Array} - The LUT data as an array of unsigned 8-bit integers.
+ */
+export function generateLUTDATA(lutSize: number) {
+  const totalEntries = lutSize * lutSize * lutSize; // 3D LUT nodes
+  const lutData = new Uint8Array(totalEntries * 4); // Each entry has an RGBA value
+
+  for (let i = 0; i < totalEntries; i++) {
+    lutData[i * 4 + 0] = Math.floor(Math.random() * 256); // Random red value
+    lutData[i * 4 + 1] = Math.floor(Math.random() * 256); // Random green value
+    lutData[i * 4 + 2] = Math.floor(Math.random() * 256); // Random blue value
+    lutData[i * 4 + 3] = 1; // alpha value
+  }
+
+  return lutData;
+}
+
+/**
+ * Normalizes the bounds of a rectangle defined by two points (A and B) within a given width and height.
+ * @param {number[]} pointA - The coordinates of the first point defining the rectangle.
+ * @param {number[]} pointB - The coordinates of the second point defining the rectangle.
+ * @param {number} width - The width of the canvas or container where the rectangle is drawn.
+ * @param {number} height - The height of the canvas or container where the rectangle is drawn.
+ * @returns {number[]} - An array containing the normalized x and y coordinates of the rectangle's corners.
+ */
+export function normalizeBounds(
+  pointA: number[],
+  pointB: number[],
+  width: number,
+  height: number,
+) {
+  return [
+    pointA[0] / width,
+    pointA[1] / height,
+    pointB[0] / width,
+    pointB[1] / height,
+  ];
+}
+
+/**
+ * Pre-allocates a specified number of 2D textures for use in WebGL2 rendering.
+ * @param {WebGL2RenderingContext} gl - The WebGL2 rendering context.
+ * @param {number} numTextures - The number of textures to be pre-allocated.
+ * @returns {WebGLTexture[]} - An array of WebGL textures, each pre-allocated and ready for use.
+ */
+export function preAllocateTextures(
+  gl: WebGL2RenderingContext,
+  numTextures: number,
+) {
+  const maskTextures = [];
+
+  for (let i = 0; i < numTextures; i++) {
+    const maskTexture = gl.createTexture();
+
+    gl.bindTexture(gl.TEXTURE_2D, maskTexture);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+    maskTextures.push(maskTexture);
+  }
+
+  return maskTextures as WebGLTexture[];
+}
+
+/**
+ * Finds the index of a Tracklet object within an array based on its unique identifier.
+ * @param objects - The array of Tracklet objects to search within.
+ * @param id - The unique identifier of the Tracklet object to find.
+ * @returns The index of the `Tracklet` object with the specified `id` in the `objects` array.
+ */
+export function findIndexByTrackletId(id: number, objects: Tracklet[]): number {
+  return objects.findIndex(obj => obj.id === id);
+}

demo/frontend/src/common/utils/emptyFunction.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This function accepts and discards inputs; it has no side effects. This is
+ * primarily useful idiomatically for overridable function endpoints which
+ * always need to be callable, since JS lacks a null-call idiom ala Cocoa.
+ */
+export default function () {}

demo/frontend/src/common/utils/uuid.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Generates a random UUID (Universally Unique Identifier) following the version
+ * 4 standard.
+ *
+ * The function replaces each 'x' and 'y' in the template
+ * 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx' with random hexadecimal digits. For
+ * 'y', the function ensures the first hexadecimal digit is '8', '9', 'A', or
+ * 'B' as per the UUID v4 standard.
+ *
+ * @returns A string representing a version 4 UUID.
+ *
+ * @example
+ *
+ * const id = uuidv4();
+ * console.log(id); // Outputs: '3f0d2c77-4f69-4c1e-8a6e-35e866e8a5d1'
+ */
+export function uuidv4() {
+  return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
+    const r = (Math.random() * 16) | 0,
+      v = c == 'x' ? r : (r & 0x3) | 0x8;
+    return v.toString(16);
+  });
+}

demo/frontend/src/debug/stats/Stats.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Derived from mrdoob / http://mrdoob.com/
+ */
+
+import Logger from '@/common/logger/Logger';
+import {uuidv4} from '@/common/utils/uuid';
+import invariant from 'invariant';
+
+export type Request<A, P> = {
+  action: A;
+} & P;
+
+export type Response<A, P> = Request<A, P>;
+
+export type GetStatsCanvasRequest = Request<
+  'getStatsCanvas',
+  {
+    id: string;
+    width: number;
+    height: number;
+  }
+>;
+
+export type GetMemoryStatsRequest = Request<
+  'getMemoryStats',
+  {
+    id: string;
+    jsHeapSizeLimit: number;
+    totalJSHeapSize: number;
+    usedJSHeapSize: number;
+  }
+>;
+
+export type SetStatsCanvasResponse = Response<
+  'setStatsCanvas',
+  {
+    id: string;
+    canvas: OffscreenCanvas;
+    devicePixelRatio: number;
+  }
+>;
+
+export type MemoryStatsResponse = Response<
+  'memoryStats',
+  {
+    id: string;
+    jsHeapSizeLimit: number;
+    totalJSHeapSize: number;
+    usedJSHeapSize: number;
+  }
+>;
+
+export type StatsType = 'fps' | 'ms' | 'memory';
+
+export class Stats {
+  private maxValue: number;
+  private beginTime: number;
+  private prevTime: number;
+  private frames: number;
+
+  private fpsPanel: Panel | null = null;
+  private msPanel: Panel | null = null;
+  private memPanel: Panel | null = null;
+
+  constructor(type: StatsType, label: string = '', maxValue: number = 100) {
+    const id = uuidv4();
+
+    this.maxValue = maxValue;
+    this.beginTime = (performance || Date).now();
+    this.prevTime = this.beginTime;
+    this.frames = 0;
+
+    const onMessage = (event: MessageEvent<SetStatsCanvasResponse>) => {
+      if (event.data.action === 'setStatsCanvas' && event.data.id === id) {
+        const {canvas, devicePixelRatio} = event.data;
+        if (type === 'fps') {
+          this.fpsPanel = new Panel(
+            canvas,
+            devicePixelRatio,
+            `FPS ${label}`.trim(),
+            '#0ff',
+            '#002',
+          );
+        } else if (type === 'ms') {
+          this.msPanel = new Panel(
+            canvas,
+            devicePixelRatio,
+            `MS ${label}`.trim(),
+            '#0f0',
+            '#020',
+          );
+        } else if (type === 'memory') {
+          this.memPanel = new Panel(
+            canvas,
+            devicePixelRatio,
+            `MB ${label}`.trim(),
+            '#f08',
+            '#201',
+          );
+        }
+        self.removeEventListener('message', onMessage);
+      }
+    };
+
+    self.addEventListener('message', onMessage);
+
+    self.postMessage({
+      action: 'getStatsCanvas',
+      id,
+      width: 80,
+      height: 48,
+    } as GetStatsCanvasRequest);
+  }
+
+  updateMaxValue(maxValue: number) {
+    this.maxValue = maxValue;
+  }
+
+  begin() {
+    this.beginTime = (performance || Date).now();
+  }
+
+  end() {
+    this.frames++;
+
+    const time = (performance || Date).now();
+
+    this.msPanel?.update(time - this.beginTime, this.maxValue);
+
+    if (time >= this.prevTime + 1000) {
+      this.fpsPanel?.update(
+        (this.frames * 1000) / (time - this.prevTime),
+        this.maxValue,
+      );
+
+      this.prevTime = time;
+      this.frames = 0;
+
+      const id = uuidv4();
+      const onMessage = (event: MessageEvent<MemoryStatsResponse>) => {
+        if (event.data.action === 'memoryStats' && event.data.id === id) {
+          const {usedJSHeapSize, jsHeapSizeLimit} = event.data;
+          this.memPanel?.update(
+            usedJSHeapSize / 1048576,
+            jsHeapSizeLimit / 1048576,
+          );
+        }
+      };
+
+      self.addEventListener('message', onMessage);
+
+      self.postMessage({
+        action: 'getMemoryStats',
+        id,
+      } as GetMemoryStatsRequest);
+    }
+
+    return time;
+  }
+
+  update() {
+    this.beginTime = this.end();
+  }
+}
+
+export class Panel {
+  private min = Infinity;
+  private max = 0;
+  private round = Math.round;
+
+  private PR: number;
+  private WIDTH: number;
+  private HEIGHT: number;
+  private TEXT_X: number;
+  private TEXT_Y: number;
+  private GRAPH_X: number;
+  private GRAPH_Y: number;
+  private GRAPH_WIDTH: number;
+  private GRAPH_HEIGHT: number;
+
+  public canvas: HTMLCanvasElement | OffscreenCanvas;
+  private context:
+    | CanvasRenderingContext2D
+    | OffscreenCanvasRenderingContext2D
+    | null = null;
+
+  private name: string;
+  private fg: string;
+  private bg: string;
+
+  constructor(
+    canvas: HTMLCanvasElement | OffscreenCanvas,
+    devicePixelRatio: number,
+    name: string,
+    fg: string,
+    bg: string,
+  ) {
+    this.canvas = canvas;
+    this.name = name;
+    this.fg = fg;
+    this.bg = bg;
+
+    this.PR = this.round(devicePixelRatio || 1);
+    this.WIDTH = 80 * this.PR;
+    this.HEIGHT = 48 * this.PR;
+    this.TEXT_X = 3 * this.PR;
+    this.TEXT_Y = 2 * this.PR;
+    this.GRAPH_X = 3 * this.PR;
+    this.GRAPH_Y = 15 * this.PR;
+    this.GRAPH_WIDTH = 74 * this.PR;
+    this.GRAPH_HEIGHT = 30 * this.PR;
+
+    const context: OffscreenCanvasRenderingContext2D | RenderingContext | null =
+      canvas.getContext('2d');
+    invariant(context !== null, 'context 2d is required');
+
+    if (
+      !(context instanceof CanvasRenderingContext2D) &&
+      !(context instanceof OffscreenCanvasRenderingContext2D)
+    ) {
+      Logger.warn(
+        'rendering stats requires CanvasRenderingContext2D or OffscreenCanvasRenderingContext2D',
+      );
+      return;
+    }
+
+    context.font = 'bold ' + 9 * this.PR + 'px Helvetica,Arial,sans-serif';
+    context.textBaseline = 'top';
+
+    context.fillStyle = bg;
+    context.fillRect(0, 0, this.WIDTH, this.HEIGHT);
+
+    context.fillStyle = fg;
+    context.fillText(name, this.TEXT_X, this.TEXT_Y);
+    context.fillRect(
+      this.GRAPH_X,
+      this.GRAPH_Y,
+      this.GRAPH_WIDTH,
+      this.GRAPH_HEIGHT,
+    );
+
+    context.fillStyle = bg;
+    context.globalAlpha = 0.9;
+    context.fillRect(
+      this.GRAPH_X,
+      this.GRAPH_Y,
+      this.GRAPH_WIDTH,
+      this.GRAPH_HEIGHT,
+    );
+
+    this.context = context;
+  }
+
+  update(value: number, maxValue: number) {
+    invariant(this.context !== null, 'context 2d is required');
+
+    this.min = Math.min(this.min, value);
+    this.max = Math.max(this.max, value);
+
+    this.context.fillStyle = this.bg;
+    this.context.globalAlpha = 1;
+    this.context.fillRect(0, 0, this.WIDTH, this.GRAPH_Y);
+    this.context.fillStyle = this.fg;
+    this.context.fillText(
+      this.round(value) +
+        ' ' +
+        this.name +
+        ' (' +
+        this.round(this.min) +
+        '-' +
+        this.round(this.max) +
+        ')',
+      this.TEXT_X,
+      this.TEXT_Y,
+    );
+
+    this.context.drawImage(
+      this.canvas,
+      this.GRAPH_X + this.PR,
+      this.GRAPH_Y,
+      this.GRAPH_WIDTH - this.PR,
+      this.GRAPH_HEIGHT,
+      this.GRAPH_X,
+      this.GRAPH_Y,
+      this.GRAPH_WIDTH - this.PR,
+      this.GRAPH_HEIGHT,
+    );
+
+    this.context.fillRect(
+      this.GRAPH_X + this.GRAPH_WIDTH - this.PR,
+      this.GRAPH_Y,
+      this.PR,
+      this.GRAPH_HEIGHT,
+    );
+
+    this.context.fillStyle = this.bg;
+    this.context.globalAlpha = 0.9;
+    this.context.fillRect(
+      this.GRAPH_X + this.GRAPH_WIDTH - this.PR,
+      this.GRAPH_Y,
+      this.PR,
+      this.round((1 - value / maxValue) * this.GRAPH_HEIGHT),
+    );
+  }
+}

demo/frontend/src/debug/stats/StatsView.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {EnableStatsRequest} from '@/common/components/video/VideoWorkerTypes';
+import stylex from '@stylexjs/stylex';
+import {useEffect, useMemo, useRef, useState} from 'react';
+import {useLocation} from 'react-router-dom';
+import useVideo from '../../common/components/video/editor/useVideo';
+import {
+  GetMemoryStatsRequest,
+  GetStatsCanvasRequest,
+  MemoryStatsResponse,
+  SetStatsCanvasResponse,
+} from './Stats';
+
+const styles = stylex.create({
+  container: {
+    position: 'fixed',
+    top: 0,
+    left: 0,
+    width: '100%',
+    overflowX: 'auto',
+    display: 'flex',
+    flexDirection: 'row',
+    cursor: 'pointer',
+    opacity: 0.9,
+    zIndex: 10000,
+  },
+});
+
+const URL_PARAM = 'monitors';
+
+export default function StatsView() {
+  const {search} = useLocation();
+  const video = useVideo();
+  const containerRef = useRef<HTMLDivElement | null>(null);
+  const [isWrapped, setIsWrapped] = useState<boolean>(false);
+
+  const isEnabled = useMemo(() => {
+    const urlSearchParams = new URLSearchParams(search);
+    return (
+      urlSearchParams.has(URL_PARAM) &&
+      ['true', ''].includes(urlSearchParams.get('monitors') ?? '')
+    );
+  }, [search]);
+
+  useEffect(() => {
+    if (!isEnabled) {
+      return;
+    }
+
+    const worker = video?.getWorker_ONLY_USE_WITH_CAUTION();
+
+    // Enable stats for video worker
+    worker?.postMessage({
+      action: 'enableStats',
+    } as EnableStatsRequest);
+
+    function onMessage(
+      event: MessageEvent<GetStatsCanvasRequest | GetMemoryStatsRequest>,
+    ) {
+      if (event.data.action === 'getStatsCanvas') {
+        // Add stats canvas and hand control over to worker
+        const canvas = document.createElement('canvas');
+        canvas.width = event.data.width * window.devicePixelRatio;
+        canvas.height = event.data.height * window.devicePixelRatio;
+        canvas.style.width = `${event.data.width}px`;
+        canvas.style.height = `${event.data.height}px`;
+        containerRef.current?.appendChild(canvas);
+
+        const offscreenCanvas = canvas.transferControlToOffscreen();
+        worker?.postMessage(
+          {
+            action: 'setStatsCanvas',
+            id: event.data.id,
+            canvas: offscreenCanvas,
+            devicePixelRatio: window.devicePixelRatio,
+          } as SetStatsCanvasResponse,
+          {
+            transfer: [offscreenCanvas],
+          },
+        );
+      } else if (event.data.action === 'getMemoryStats') {
+        // @ts-expect-error performance.memory might not exist
+        const memory = performance.memory ?? {
+          jsHeapSizeLimit: 0,
+          totalJSHeapSize: 0,
+          usedJSHeapSize: 0,
+        };
+        worker?.postMessage({
+          action: 'memoryStats',
+          id: event.data.id,
+          jsHeapSizeLimit: memory.jsHeapSizeLimit,
+          totalJSHeapSize: memory.totalJSHeapSize,
+          usedJSHeapSize: memory.usedJSHeapSize,
+        } as MemoryStatsResponse);
+      }
+    }
+
+    worker?.addEventListener('message', onMessage);
+    return () => {
+      worker?.removeEventListener('message', onMessage);
+    };
+  }, [video, isEnabled]);
+
+  function handleClick() {
+    setIsWrapped(w => !w);
+  }
+
+  if (!isEnabled) {
+    return null;
+  }
+
+  return (
+    <div
+      ref={containerRef}
+      {...stylex.props(styles.container)}
+      style={{flexWrap: isWrapped ? 'wrap' : 'unset'}}
+      onDoubleClick={handleClick}
+    />
+  );
+}

demo/frontend/src/demo/DemoConfig.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {Effects} from '@/common/components/video/effects/Effects';
+
+type EffectLayers = {
+  background: keyof Effects;
+  highlight: keyof Effects;
+};
+
+export const DEMO_SHORT_NAME = 'SAM 2 Demo';
+export const RESEARCH_BY_META_AI = 'By Meta FAIR';
+export const DEMO_FRIENDLY_NAME = 'Segment Anything 2 Demo';
+export const VIDEO_WATERMARK_TEXT = `Modified with ${DEMO_FRIENDLY_NAME}`;
+export const PROJECT_GITHUB_URL =
+  'https://github.com/facebookresearch/sam2';
+export const AIDEMOS_URL = 'https://aidemos.meta.com';
+export const ABOUT_URL = 'https://ai.meta.com/sam2';
+export const EMAIL_ADDRESS = 'segment-anything@meta.com';
+export const BLOG_URL = 'http://ai.meta.com/blog/sam2';
+
+export const VIDEO_API_ENDPOINT = 'http://localhost:7263';
+export const INFERENCE_API_ENDPOINT = 'http://localhost:7263';
+
+export const demoObjectLimit = 3;
+
+export const DEFAULT_EFFECT_LAYERS: EffectLayers = {
+  background: 'Original',
+  highlight: 'Overlay',
+};
+
+export const MAX_UPLOAD_FILE_SIZE = '70MB';

demo/frontend/src/demo/DemoErrorFallback.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import LoadingStateScreen from '@/common/loading/LoadingStateScreen';
+import {FallbackProps} from 'react-error-boundary';
+
+export default function DemoErrorFallback(_props: FallbackProps) {
+  return (
+    <LoadingStateScreen
+      title="Well, this is embarrassing..."
+      description="This demo is not optimized for your device. Please try again on a different device with a larger screen."
+      linkProps={{to: '..', label: 'Back to homepage'}}
+    />
+  );
+}

demo/frontend/src/demo/DemoSuspenseFallback.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import LoadingStateScreen from '@/common/loading/LoadingStateScreen';
+
+export default function DemoSuspenseFallback() {
+  return <LoadingStateScreen title="Fetching data" />;
+}

demo/frontend/src/demo/SAM2DemoApp.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import '@/assets/scss/App.scss';
+import ErrorReport from '@/common/error/ErrorReport';
+import DemoErrorFallback from '@/demo/DemoErrorFallback';
+import DemoSuspenseFallback from '@/demo/DemoSuspenseFallback';
+import RelayEnvironmentProvider from '@/graphql/RelayEnvironmentProvider';
+import RootLayout from '@/layouts/RootLayout';
+import SAM2DemoPage from '@/routes/DemoPageWrapper';
+import PageNotFoundPage from '@/routes/PageNotFoundPage';
+import useSettingsContext from '@/settings/useSettingsContext';
+import {Route, Routes} from 'react-router-dom';
+
+export default function DemoAppWrapper() {
+  const {settings} = useSettingsContext();
+  return (
+    <RelayEnvironmentProvider
+      endpoint={settings.videoAPIEndpoint}
+      suspenseFallback={<DemoSuspenseFallback />}
+      errorFallback={DemoErrorFallback}>
+      <DemoApp />
+    </RelayEnvironmentProvider>
+  );
+}
+
+function DemoApp() {
+  return (
+    <>
+      <Routes>
+        <Route element={<RootLayout />}>
+          <Route index={true} element={<SAM2DemoPage />} />
+          <Route path="*" element={<PageNotFoundPage />} />
+        </Route>
+      </Routes>
+      <ErrorReport />
+    </>
+  );
+}

demo/frontend/src/demo/atoms.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  defaultMessageMap,
+  MessagesEventMap,
+} from '@/common/components/snackbar/DemoMessagesSnackbarUtils';
+import {Effects} from '@/common/components/video/effects/Effects';
+import {
+  DemoEffect,
+  highlightEffects,
+} from '@/common/components/video/effects/EffectUtils';
+import {
+  BaseTracklet,
+  SegmentationPoint,
+  StreamingState,
+} from '@/common/tracker/Tracker';
+import type {DataArray} from '@/jscocotools/mask';
+import {atom} from 'jotai';
+
+export type VideoData = {
+  path: string;
+  posterPath: string | null | undefined;
+  url: string;
+  posterUrl: string;
+  width: number;
+  height: number;
+};
+
+export const frameIndexAtom = atom<number>(0);
+
+export const inputVideoAtom = atom<VideoData | null>(null);
+
+// #####################
+// SESSION
+// #####################
+
+export type Session = {
+  id: string;
+  ranPropagation: boolean;
+};
+
+export const sessionAtom = atom<Session | null>(null);
+
+// #####################
+// STREAMING/PLAYBACK
+// #####################
+
+export const isVideoLoadingAtom = atom<boolean>(false);
+
+export const streamingStateAtom = atom<StreamingState>('none');
+
+export const isPlayingAtom = atom<boolean>(false);
+
+export const isStreamingAtom = atom<boolean>(false);
+
+// #####################
+// OBJECTS
+// #####################
+
+export type TrackletMask = {
+  mask: DataArray;
+  isEmpty: boolean;
+};
+
+export type TrackletObject = {
+  id: number;
+  color: string;
+  thumbnail: string | null;
+  points: SegmentationPoint[][];
+  masks: TrackletMask[];
+  isInitialized: boolean;
+};
+
+const MAX_NUMBER_TRACKLET_OBJECTS = 3;
+
+export const activeTrackletObjectIdAtom = atom<number | null>(0);
+
+export const activeTrackletObjectAtom = atom<BaseTracklet | null>(get => {
+  const objectId = get(activeTrackletObjectIdAtom);
+  const tracklets = get(trackletObjectsAtom);
+  return tracklets.find(obj => obj.id === objectId) ?? null;
+});
+
+export const trackletObjectsAtom = atom<BaseTracklet[]>([]);
+
+export const maxTrackletObjectIdAtom = atom<number>(get => {
+  const tracklets = get(trackletObjectsAtom);
+  return tracklets.reduce((prev, curr) => Math.max(prev, curr.id), 0);
+});
+
+export const isTrackletObjectLimitReachedAtom = atom<boolean>(
+  get => get(trackletObjectsAtom).length >= MAX_NUMBER_TRACKLET_OBJECTS,
+);
+
+export const areTrackletObjectsInitializedAtom = atom<boolean>(get =>
+  get(trackletObjectsAtom).every(obj => obj.isInitialized),
+);
+
+export const isFirstClickMadeAtom = atom(get => {
+  const tracklets = get(trackletObjectsAtom);
+  return tracklets.some(tracklet => tracklet.points.length > 0);
+});
+
+export const pointsAtom = atom<SegmentationPoint[]>(get => {
+  const frameIndex = get(frameIndexAtom);
+  const activeTracklet = get(activeTrackletObjectAtom);
+  return activeTracklet?.points[frameIndex] ?? [];
+});
+
+export const labelTypeAtom = atom<'positive' | 'negative'>('positive');
+
+export const isAddObjectEnabledAtom = atom<boolean>(get => {
+  const session = get(sessionAtom);
+  const trackletsInitialized = get(areTrackletObjectsInitializedAtom);
+  const isObjectLimitReached = get(isTrackletObjectLimitReachedAtom);
+  return (
+    session?.ranPropagation === false &&
+    trackletsInitialized &&
+    !isObjectLimitReached
+  );
+});
+
+export const codeEditorOpenedAtom = atom<boolean>(false);
+
+export const tutorialVideoEnabledAtom = atom<boolean>(true);
+
+// #####################
+// Effects
+// #####################
+
+type EffectConfig = {
+  name: keyof Effects;
+  variant: number;
+  numVariants: number;
+};
+
+export const activeBackgroundEffectAtom = atom<EffectConfig>({
+  name: 'Original',
+  variant: 0,
+  numVariants: 0,
+});
+
+export const activeHighlightEffectAtom = atom<EffectConfig>({
+  name: 'Overlay',
+  variant: 0,
+  numVariants: 0,
+});
+
+export const activeHighlightEffectGroupAtom =
+  atom<DemoEffect[]>(highlightEffects);
+
+// #####################
+// Toolbar
+// #####################
+
+export const toolbarTabIndex = atom<number>(0);
+
+// #####################
+// Messages snackbar
+// #####################
+
+export const messageMapAtom = atom<MessagesEventMap>(defaultMessageMap);
+
+// #####################
+// Upload state
+// #####################
+
+export const uploadingStateAtom = atom<'default' | 'uploading' | 'error'>(
+  'default',
+);

demo/frontend/src/graphql/RelayEnvironment.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import Logger from '@/common/logger/Logger';
+import {
+  CacheConfig,
+  Environment,
+  FetchFunction,
+  GraphQLResponse,
+  LogEvent,
+  Network,
+  ObservableFromValue,
+  RecordSource,
+  RequestParameters,
+  Store,
+  UploadableMap,
+  Variables,
+} from 'relay-runtime';
+import fetchGraphQL from './fetchGraphQL';
+
+function createFetchRelay(endpoint: string): FetchFunction {
+  return (
+    request: RequestParameters,
+    variables: Variables,
+    cacheConfig: CacheConfig,
+    uploadables?: UploadableMap | null,
+  ): ObservableFromValue<GraphQLResponse> => {
+    Logger.debug(
+      `fetching query ${request.name} with ${JSON.stringify(variables)}`,
+    );
+    return fetchGraphQL(endpoint, request, variables, cacheConfig, uploadables);
+  };
+}
+
+export function createEnvironment(endpoint: string): Environment {
+  return new Environment({
+    log: (logEvent: LogEvent) => Logger.debug(logEvent.name, logEvent),
+    network: Network.create(createFetchRelay(endpoint)),
+    store: new Store(new RecordSource()),
+  });
+}

demo/frontend/src/graphql/RelayEnvironmentProvider.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ErrorFallback from '@/common/error/ErrorFallback';
+import LoadingMessage from '@/common/loading/LoadingMessage';
+import {createEnvironment} from '@/graphql/RelayEnvironment';
+import {
+  ComponentType,
+  PropsWithChildren,
+  ReactNode,
+  Suspense,
+  useMemo,
+  useState,
+} from 'react';
+import {ErrorBoundary, FallbackProps} from 'react-error-boundary';
+import {RelayEnvironmentProvider} from 'react-relay';
+
+type Props = PropsWithChildren<{
+  suspenseFallback?: ReactNode;
+  errorFallback?: ComponentType<FallbackProps>;
+  endpoint: string;
+}>;
+
+export default function OnevisionRelayEnvironmentProvider({
+  suspenseFallback,
+  errorFallback = ErrorFallback,
+  endpoint,
+  children,
+}: Props) {
+  const [retryKey, setRetryKey] = useState<number>(0);
+
+  const environment = useMemo(() => {
+    return createEnvironment(endpoint);
+    // The retryKey is needed to force a new Relay Environment
+    // instance when the user retries after an error occurred.
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [endpoint, retryKey]);
+
+  // Force re-creating Relay Environment
+  function handleReset() {
+    setRetryKey(k => k + 1);
+  }
+
+  return (
+    <ErrorBoundary onReset={handleReset} FallbackComponent={errorFallback}>
+      <RelayEnvironmentProvider environment={environment}>
+        <Suspense fallback={suspenseFallback ?? <LoadingMessage />}>
+          {children}
+        </Suspense>
+      </RelayEnvironmentProvider>
+    </ErrorBoundary>
+  );
+}

demo/frontend/src/graphql/errors/CreateFilmstripError.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export default class CreateFilmstripError extends Error {
+  override name = 'CreateFilmstripError';
+  constructor(message?: string) {
+    super(message);
+  }
+}

demo/frontend/src/graphql/errors/DrawFrameError.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export default class DrawFrameError extends Error {
+  override name = 'DrawFrameError';
+  constructor(message?: string) {
+    super(message);
+  }
+}

demo/frontend/src/graphql/errors/WebGLContextError.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export default class WebGLContextError extends Error {
+  override name = 'WebGLContextError';
+  constructor(message?: string) {
+    super(message);
+  }
+}

demo/frontend/src/graphql/fetchGraphQL.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import Logger from '@/common/logger/Logger';
+import {
+  CacheConfig,
+  GraphQLResponse,
+  RequestParameters,
+  UploadableMap,
+  Variables,
+} from 'relay-runtime';
+
+/**
+ * Inspired by https://github.com/facebook/relay/issues/1844
+ */
+export default async function fetchGraphQL(
+  endpoint: string,
+  request: RequestParameters,
+  variables: Variables,
+  cacheConfig: CacheConfig,
+  uploadables?: UploadableMap | null,
+): Promise<GraphQLResponse> {
+  const url = `${endpoint}/graphql`;
+
+  const headers: {[name: string]: string} = {};
+  const requestInit: RequestInit = {
+    method: 'POST',
+    headers,
+    credentials: 'include',
+  };
+
+  const customHeaders = (cacheConfig?.metadata?.headers ?? {}) as {
+    [key: string]: string;
+  };
+
+  requestInit.headers = Object.assign(customHeaders, requestInit.headers);
+
+  if (uploadables != null) {
+    const formData = new FormData();
+    formData.append(
+      'operations',
+      JSON.stringify({
+        query: request.text,
+        variables,
+      }),
+    );
+
+    const uploadableMap: {
+      [key: string]: string[];
+    } = {};
+
+    Object.keys(uploadables).forEach(key => {
+      uploadableMap[key] = [`variables.${key}`];
+    });
+
+    formData.append('map', JSON.stringify(uploadableMap));
+
+    Object.keys(uploadables).forEach(key => {
+      formData.append(key, uploadables[key]);
+    });
+
+    requestInit.body = formData;
+  } else {
+    requestInit.headers = Object.assign(
+      {'Content-Type': 'application/json'},
+      requestInit.headers,
+    );
+
+    requestInit.body = JSON.stringify({
+      query: request.text,
+      variables,
+    });
+  }
+
+  try {
+    const response = await fetch(url, requestInit);
+    const result = await response.json();
+
+    // Handle any intentional GraphQL errors, which are passed through the
+    // errors property in the JSON payload.
+    if ('errors' in result) {
+      for (const error of result.errors) {
+        Logger.error(error);
+      }
+    }
+
+    return result;
+  } catch (error) {
+    Logger.error(`Could not connect to GraphQL endpoint ${url}`, error);
+    throw error;
+  }
+}

demo/frontend/src/jscocotools/mask.ts

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export class DataArray {
+  data: Uint8Array;
+  readonly shape: number[];
+
+  constructor(data: Uint8Array, shape: Array<number>) {
+    this.data = data;
+    this.shape = shape;
+  }
+}
+
+export type RLEObject = {
+  size: [h: number, w: number];
+  counts: string;
+};
+
+type RLE = {
+  h: number;
+  w: number;
+  m: number;
+  cnts: number[];
+};
+
+type BB = number[];
+
+function rleInit(R: RLE, h: number, w: number, m: number, cnts: number[]) {
+  R.h = h;
+  R.w = w;
+  R.m = m;
+  R.cnts = m === 0 ? [0] : cnts;
+}
+
+function rlesInit(R: RLE[], n: number) {
+  let i;
+  for (i = 0; i < n; i++) {
+    R[i] = {h: 0, w: 0, m: 0, cnts: [0]};
+    rleInit(R[i], 0, 0, 0, [0]);
+  }
+}
+
+class RLEs {
+  _R: RLE[];
+  _n: number;
+
+  constructor(n: number) {
+    this._R = [];
+    rlesInit(this._R, n);
+    this._n = n;
+  }
+}
+
+export class Masks {
+  _mask: Uint8Array;
+  _h: number;
+  _w: number;
+  _n: number;
+
+  constructor(h: number, w: number, n: number) {
+    this._mask = new Uint8Array(h * w * n);
+    this._h = h;
+    this._w = w;
+    this._n = n;
+  }
+
+  toDataArray(): DataArray {
+    return new DataArray(this._mask, [this._h, this._w, this._n]);
+  }
+}
+
+// encode mask to RLEs objects
+// list of RLE string can be generated by RLEs member function
+export function encode(mask: DataArray): RLEObject[] {
+  const h = mask.shape[0];
+  const w = mask.shape[1];
+  const n = mask.shape[2];
+  const Rs = new RLEs(n);
+  rleEncode(Rs._R, mask.data, h, w, n);
+  const objs = _toString(Rs);
+  return objs;
+}
+
+// decode mask from compressed list of RLE string or RLEs object
+export function decode(rleObjs: RLEObject[]): DataArray {
+  const Rs = _frString(rleObjs);
+  const h = Rs._R[0].h;
+  const w = Rs._R[0].w;
+  const n = Rs._n;
+  const masks = new Masks(h, w, n);
+  rleDecode(Rs._R, masks._mask, n);
+  return masks.toDataArray();
+}
+
+export function toBbox(rleObjs: RLEObject[]): BB {
+  const Rs = _frString(rleObjs);
+  const n = Rs._n;
+  const bb: BB = [];
+  rleToBbox(Rs._R, bb, n);
+  return bb;
+}
+
+function rleEncode(R: RLE[], M: Uint8Array, h: number, w: number, n: number) {
+  let i;
+  let j;
+  let k;
+  const a = w * h;
+  let c;
+  const cnts: number[] = [];
+  let p;
+  for (i = 0; i < n; i++) {
+    const from = a * i;
+    const to = a * (i + 1);
+    // Slice data for current RLE object
+    const T = M.slice(from, to);
+    k = 0;
+    p = 0;
+    c = 0;
+    for (j = 0; j < a; j++) {
+      if (T[j] !== p) {
+        cnts[k++] = c;
+        c = 0;
+        p = T[j];
+      }
+      c++;
+    }
+    cnts[k++] = c;
+    rleInit(R[i], h, w, k, [...cnts]);
+  }
+}
+
+function rleDecode(R: RLE[], M: Uint8Array, n: number): void {
+  let i;
+  let j;
+  let k;
+  let p = 0;
+  for (i = 0; i < n; i++) {
+    let v = false;
+    for (j = 0; j < R[i].m; j++) {
+      for (k = 0; k < R[i].cnts[j]; k++) {
+        M[p++] = v === false ? 0 : 1;
+      }
+      v = !v;
+    }
+  }
+}
+
+function rleToString(R: RLE): string {
+  /* Similar to LEB128 but using 6 bits/char and ascii chars 48-111. */
+  let i;
+  const m = R.m;
+  let p = 0;
+  let x: number;
+  let more;
+  const s: string[] = [];
+  for (i = 0; i < m; i++) {
+    x = R.cnts[i];
+    if (i > 2) {
+      x -= R.cnts[i - 2];
+    }
+    more = true; // 1;
+    while (more) {
+      let c = x & 0x1f;
+      x >>= 5;
+      more = c & 0x10 ? x != -1 : x != 0;
+      if (more) {
+        c |= 0x20;
+      }
+      c += 48;
+      s[p++] = String.fromCharCode(c);
+    }
+  }
+  return s.join('');
+}
+
+// internal conversion from Python RLEs object to compressed RLE format
+function _toString(Rs: RLEs): RLEObject[] {
+  const n = Rs._n;
+  let py_string;
+  let c_string;
+  const objs: RLEObject[] = [];
+  for (let i = 0; i < n; i++) {
+    c_string = rleToString(Rs._R[i]);
+    py_string = c_string;
+    objs.push({
+      size: [Rs._R[i].h, Rs._R[i].w],
+      counts: py_string,
+    });
+  }
+  return objs;
+}
+
+// internal conversion from compressed RLE format to Python RLEs object
+function _frString(rleObjs: RLEObject[]): RLEs {
+  const n = rleObjs.length;
+  const Rs = new RLEs(n);
+  let py_string;
+  let c_string;
+  for (let i = 0; i < rleObjs.length; i++) {
+    const obj = rleObjs[i];
+    py_string = obj.counts;
+    c_string = py_string;
+    rleFrString(Rs._R[i], c_string, obj.size[0], obj.size[1]);
+  }
+  return Rs;
+}
+
+function rleToBbox(R: RLE[], bb: BB, n: number) {
+  for (let i = 0; i < n; i++) {
+    const h = R[i].h;
+    const w = R[i].w;
+    let m = R[i].m;
+    // The RLE structure likely contains run-length encoded data where each
+    // element represents a count of consecutive pixels with the same value in
+    // a binary image (black or white). Since the counts represent both black
+    // and white pixels, this operation ((siz)(m/2)) * 2 is used to ensure that
+    // m is always an even number. By doing so, the code can later check
+    // whether the current pixel is black or white based on whether the index j
+    // is even or odd.
+    m = Math.floor(m / 2) * 2;
+    let xs = w;
+    let ys = h;
+    let xe = 0;
+    let ye = 0;
+    let cc = 0;
+    let t;
+    let y;
+    let x;
+    let xp = 0;
+    if (m === 0) {
+      bb[4 * i] = bb[4 * i + 1] = bb[4 * i + 2] = bb[4 * i + 3] = 0;
+      continue;
+    }
+    for (let j = 0; j < m; j++) {
+      cc += R[i].cnts[j];
+      t = cc - (j % 2);
+      y = t % h;
+      x = Math.floor((t - y) / h);
+      if (j % 2 === 0) {
+        xp = x;
+      } else if (xp < x) {
+        ys = 0;
+        ye = h - 1;
+      }
+      xs = Math.min(xs, x);
+      xe = Math.max(xe, x);
+      ys = Math.min(ys, y);
+      ye = Math.max(ye, y);
+    }
+    bb[4 * i] = xs;
+    bb[4 * i + 2] = xe - xs + 1;
+    bb[4 * i + 1] = ys;
+    bb[4 * i + 3] = ye - ys + 1;
+  }
+}
+
+function rleFrString(R: RLE, s: string, h: number, w: number): void {
+  let m = 0;
+  let p = 0;
+  let k;
+  let x;
+  let more;
+  let cnts = [];
+  while (s[m]) {
+    m++;
+  }
+  cnts = [];
+  m = 0;
+  while (s[p]) {
+    x = 0;
+    k = 0;
+    more = 1;
+    while (more) {
+      const c = s.charCodeAt(p) - 48;
+      x |= (c & 0x1f) << (5 * k);
+      more = c & 0x20;
+      p++;
+      k++;
+      if (!more && c & 0x10) {
+        x |= -1 << (5 * k);
+      }
+    }
+    if (m > 2) {
+      x += cnts[m - 2];
+    }
+    cnts[m++] = x;
+  }
+  rleInit(R, h, w, m, cnts);
+}

demo/frontend/src/layouts/DemoPageLayout.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {spacing} from '@/theme/tokens.stylex';
+import stylex from '@stylexjs/stylex';
+import {PropsWithChildren} from 'react';
+
+type Props = PropsWithChildren;
+
+const styles = stylex.create({
+  container: {
+    width: '100%',
+    height: '100%',
+    display: 'flex',
+    justifyContent: 'stretch',
+    alignItems: 'stretch',
+    gap: spacing[12],
+    paddingHorizontal: spacing[12],
+    paddingVertical: spacing[4],
+    '@media screen and (max-width: 768px)': {
+      display: 'flex',
+      flexDirection: 'column-reverse',
+      gap: 0,
+      marginTop: spacing[0],
+      marginBottom: spacing[0],
+      paddingHorizontal: spacing[0],
+      paddingBottom: spacing[0],
+    },
+  },
+});
+
+export default function DemoPageLayout({children}: Props) {
+  return <div {...stylex.props(styles.container)}>{children}</div>;
+}

demo/frontend/src/layouts/RootLayout.tsx

+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import LoadingStateScreen from '@/common/loading/LoadingStateScreen';
+import useSettingsContext from '@/settings/useSettingsContext';
+import {Cog6ToothIcon} from '@heroicons/react/24/outline';
+import stylex from '@stylexjs/stylex';
+import {Suspense} from 'react';
+import {Button, Indicator} from 'react-daisyui';
+import {Outlet} from 'react-router-dom';
+
+const styles = stylex.create({
+  container: {
+    display: 'flex',
+    flexDirection: 'column',
+    height: '100%',
+    maxHeight: '100vh',
+    backgroundColor: '#000',
+  },
+  content: {
+    position: 'relative',
+    flex: '1 1 0%',
+    display: 'flex',
+    flexDirection: 'column',
+    overflowX: 'auto',
+    overflowY: {
+      default: 'auto',
+      '@media screen and (max-width: 768px)': 'auto',
+    },
+  },
+  debugActions: {
+    display: 'flex',
+    flexDirection: 'column',
+    position: 'fixed',
+    top: 100,
+    right: 0,
+    backgroundColor: 'white',
+    borderRadius: 3,
+  },
+});
+
+export default function RootLayout() {
+  const {openModal, hasChanged} = useSettingsContext();
+
+  return (
+    <div {...stylex.props(styles.container)}>
+      <div {...stylex.props(styles.content)}>
+        <Suspense
+          fallback={
+            <LoadingStateScreen
+              title="Loading demo..."
+              description="This may take a few moments, you're almost there!"
+            />
+          }>
+          <Outlet />
+        </Suspense>
+      </div>
+      <div {...stylex.props(styles.debugActions)}>
+        <Indicator>
+          {hasChanged && (
+            <Indicator.Item
+              className="badge badge-primary scale-50"
+              horizontal="start"
+              vertical="top"
+            />
+          )}
+          <Button
+            color="ghost"
+            onClick={openModal}
+            shape="circle"
+            size="xs"
+            startIcon={<Cog6ToothIcon className="w-4 h-4" />}
+            title="Bugnub"
+          />
+        </Indicator>
+      </div>
+    </div>
+  );
+}