Skip to content

GitLab

Commit 3af09475 authored by luopl's avatar luopl
Browse files

"Initial commit"

parents
Pipeline #3140 canceled with stages
Hide whitespace changes
Inline Side-by-side
Showing with 1497 additions and 0 deletions
demo/frontend/.prettierrc.json 0 → 100644
View file @ 3af09475
{
"arrowParens": "avoid",
"bracketSameLine": true,
"bracketSpacing": false,
"singleQuote": true,
"tabWidth": 2,
"trailingComma": "all",
"endOfLine": "auto"
}
\ No newline at end of file
demo/frontend/frontend.Dockerfile 0 → 100644
View file @ 3af09475
# Stage 1: Build Stage
FROM node:22.9.0 AS build
WORKDIR /app
# Copy package.json and yarn.lock
COPY package.json ./
COPY yarn.lock ./
# Install dependencies
RUN yarn install --frozen-lockfile
# Copy source code
COPY . .
# Build the application
RUN yarn build
# Stage 2: Production Stage
FROM nginx:latest
# Copy built files from the build stage to the production image
COPY --from=build /app/dist /usr/share/nginx/html
# Container startup command for the web server (nginx in this case)
CMD ["nginx", "-g", "daemon off;"]
demo/frontend/index.html 0 → 100644
View file @ 3af09475
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta
name="viewport"
content="width=device-width, initial-scale=1.0, shrink-to-fit=no" />
<title>SAM 2 Demo | By Meta FAIR</title>
</head>
<body>
<div id="root"></div>
<script type="module" src="/src/main.tsx"></script>
</body>
</html>
demo/frontend/package.json 0 → 100644
View file @ 3af09475
{
"name": "frontend-vite",
"private": true,
"version": "0.0.0",
"type": "module",
"scripts": {
"dev": "vite",
"merge-schemas": "tsx schemas/merge-schemas",
"relay": "yarn merge-schemas && relay-compiler",
"build": "tsc && vite build",
"lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
"preview": "vite preview --open"
},
"dependencies": {
"@carbon/icons-react": "^11.34.1",
"@heroicons/react": "^2.0.18",
"@monaco-editor/react": "^4.6.0",
"@stylexjs/stylex": "^0.6.1",
"graphql": "^16.8.1",
"immer": "^10.0.3",
"immutability-helper": "^3.1.1",
"jotai": "^2.6.1",
"jotai-immer": "^0.3.0",
"localforage": "^1.10.0",
"monaco-editor": "^0.48.0",
"mp4box": "^0.5.2",
"pts": "^0.12.8",
"react": "^18.2.0",
"react-daisyui": "^4.1.0",
"react-device-detect": "^2.2.3",
"react-dom": "^18.2.0",
"react-dropzone": "^14.2.3",
"react-error-boundary": "^4.0.11",
"react-photo-album": "^2.3.0",
"react-pts-canvas": "^0.5.2",
"react-relay": "^16.2.0",
"react-router-dom": "^6.15.0",
"relay-runtime": "^16.2.0",
"serialize-error": "^11.0.3",
"use-immer": "^0.9.0",
"use-resize-observer": "^9.1.0"
},
"devDependencies": {
"@graphql-tools/load-files": "^7.0.0",
"@graphql-tools/merge": "^9.0.4",
"@tailwindcss/typography": "^0.5.9",
"@types/dom-webcodecs": "^0.1.11",
"@types/invariant": "^2.2.37",
"@types/node": "^20.14.10",
"@types/react": "^18.2.47",
"@types/react-dom": "^18.2.7",
"@types/react-relay": "^16.0.6",
"@types/relay-runtime": "^14.1.13",
"@typescript-eslint/eslint-plugin": "^6.18.1",
"@typescript-eslint/parser": "^6.18.1",
"@vitejs/plugin-react": "^4.2.1",
"autoprefixer": "^10.4.15",
"babel-plugin-relay": "^16.2.0",
"babel-plugin-strip-invariant": "^1.0.0",
"daisyui": "^3.6.3",
"eslint": "^8.48.0",
"eslint-config-prettier": "^9.0.0",
"eslint-import-resolver-alias": "^1.1.2",
"eslint-import-resolver-typescript": "^3.6.3",
"eslint-plugin-import": "^2.28.1",
"eslint-plugin-prettier": "^5.1.3",
"eslint-plugin-react": "^7.33.2",
"eslint-plugin-react-hooks": "^4.6.0",
"eslint-plugin-react-refresh": "^0.4.3",
"invariant": "^2.2.4",
"postcss": "^8.4.28",
"postinstall-postinstall": "^2.1.0",
"prettier": "^3.0.3",
"relay-compiler": "^16.2.0",
"sass": "^1.66.1",
"strip-ansi": "^7.1.0",
"tailwindcss": "^3.3.3",
"tsx": "^4.16.2",
"typescript": ">=4.3.5 <5.4.0",
"vite": "^5.0.11",
"vite-plugin-babel": "^1.2.0",
"vite-plugin-relay": "^2.0.0",
"vite-plugin-stylex-dev": "^0.5.2"
},
"resolutions": {
"wrap-ansi": "7.0.0"
},
"relay": {
"src": "./src/",
"schema": "./schema.graphql",
"language": "typescript",
"eagerEsModules": true,
"exclude": [
"**/node_modules/**",
"**/__mocks__/**",
"**/__generated__/**"
]
}
}
\ No newline at end of file
demo/frontend/postcss.config.js 0 → 100644
View file @ 3af09475
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export default {
plugins: {
'postcss-import': {},
'tailwindcss/nesting': {},
tailwindcss: {},
autoprefixer: {},
},
};
demo/frontend/schema.graphql 0 → 100644
View file @ 3af09475
input AddPointsInput {
sessionId: String!
frameIndex: Int!
clearOldPoints: Boolean!
objectId: Int!
labels: [Int!]!
points: [[Float!]!]!
}
type CancelPropagateInVideo {
success: Boolean!
}
input CancelPropagateInVideoInput {
sessionId: String!
}
input ClearPointsInFrameInput {
sessionId: String!
frameIndex: Int!
objectId: Int!
}
type ClearPointsInVideo {
success: Boolean!
}
input ClearPointsInVideoInput {
sessionId: String!
}
type CloseSession {
success: Boolean!
}
input CloseSessionInput {
sessionId: String!
}
type Mutation {
startSession(input: StartSessionInput!): StartSession!
closeSession(input: CloseSessionInput!): CloseSession!
addPoints(input: AddPointsInput!): RLEMaskListOnFrame!
clearPointsInFrame(input: ClearPointsInFrameInput!): RLEMaskListOnFrame!
clearPointsInVideo(input: ClearPointsInVideoInput!): ClearPointsInVideo!
removeObject(input: RemoveObjectInput!): [RLEMaskListOnFrame!]!
cancelPropagateInVideo(
input: CancelPropagateInVideoInput!
): CancelPropagateInVideo!
createDeletionId: String!
acceptTos: Boolean!
acceptTermsOfService: String!
uploadVideo(
file: Upload!
startTimeSec: Float = null
durationTimeSec: Float = null
): Video!
uploadSharedVideo(file: Upload!): SharedVideo!
uploadAnnotations(file: Upload!): Boolean!
}
input PingInput {
sessionId: String!
}
type Pong {
success: Boolean!
}
type Query {
ping(input: PingInput!): Pong!
defaultVideo: Video!
videos(
"""
Returns the items in the list that come before the specified cursor.
"""
before: String = null
"""
Returns the items in the list that come after the specified cursor.
"""
after: String = null
"""
Returns the first n items from the list.
"""
first: Int = null
"""
Returns the items in the list that come after the specified cursor.
"""
last: Int = null
): VideoConnection!
sharedVideo(path: String!): SharedVideo!
}
type RLEMask {
size: [Int!]!
counts: String!
order: String!
}
type RLEMaskForObject {
objectId: Int!
rleMask: RLEMask!
}
type RLEMaskListOnFrame {
frameIndex: Int!
rleMaskList: [RLEMaskForObject!]!
}
input RemoveObjectInput {
sessionId: String!
objectId: Int!
}
type StartSession {
sessionId: String!
}
input StartSessionInput {
path: String!
}
"""
The `ID` scalar type represents a unique identifier, often used to refetch an object or as key for a cache. The ID type appears in a JSON response as a String; however, it is not intended to be human-readable. When expected as an input type, any string (such as `"4"`) or integer (such as `4`) input value will be accepted as an ID.
"""
scalar GlobalID
@specifiedBy(url: "https://relay.dev/graphql/objectidentification.htm")
"""
An object with a Globally Unique ID
"""
interface Node {
"""
The Globally Unique ID of this object
"""
id: GlobalID!
}
"""
Information to aid in pagination.
"""
type PageInfo {
"""
When paginating forwards, are there more items?
"""
hasNextPage: Boolean!
"""
When paginating backwards, are there more items?
"""
hasPreviousPage: Boolean!
"""
When paginating backwards, the cursor to continue.
"""
startCursor: String
"""
When paginating forwards, the cursor to continue.
"""
endCursor: String
}
type SharedVideo {
path: String!
url: String!
}
scalar Upload
type Video implements Node {
"""
The Globally Unique ID of this object
"""
id: GlobalID!
path: String!
posterPath: String
width: Int!
height: Int!
url: String!
posterUrl: String!
}
"""
A connection to a list of items.
"""
type VideoConnection {
"""
Pagination data for this connection
"""
pageInfo: PageInfo!
"""
Contains the nodes in this connection
"""
edges: [VideoEdge!]!
}
"""
An edge in a connection.
"""
type VideoEdge {
"""
A cursor for use in pagination
"""
cursor: String!
"""
The item at the end of the edge
"""
node: Video!
}
schema {
query: Query
mutation: Mutation
}
demo/frontend/schemas/inference-api-schema.graphql 0 → 100644
View file @ 3af09475
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
input AddPointsInput {
sessionId: String!
frameIndex: Int!
clearOldPoints: Boolean!
objectId: Int!
labels: [Int!]!
points: [[Float!]!]!
}
type CancelPropagateInVideo {
success: Boolean!
}
input CancelPropagateInVideoInput {
sessionId: String!
}
input ClearPointsInFrameInput {
sessionId: String!
frameIndex: Int!
objectId: Int!
}
type ClearPointsInVideo {
success: Boolean!
}
input ClearPointsInVideoInput {
sessionId: String!
}
type CloseSession {
success: Boolean!
}
input CloseSessionInput {
sessionId: String!
}
type Mutation {
startSession(input: StartSessionInput!): StartSession!
closeSession(input: CloseSessionInput!): CloseSession!
addPoints(input: AddPointsInput!): RLEMaskListOnFrame!
clearPointsInFrame(input: ClearPointsInFrameInput!): RLEMaskListOnFrame!
clearPointsInVideo(input: ClearPointsInVideoInput!): ClearPointsInVideo!
removeObject(input: RemoveObjectInput!): [RLEMaskListOnFrame!]!
cancelPropagateInVideo(
input: CancelPropagateInVideoInput!
): CancelPropagateInVideo!
}
input PingInput {
sessionId: String!
}
type Pong {
success: Boolean!
}
type Query {
ping(input: PingInput!): Pong!
}
type RLEMask {
size: [Int!]!
counts: String!
order: String!
}
type RLEMaskForObject {
objectId: Int!
rleMask: RLEMask!
}
type RLEMaskListOnFrame {
frameIndex: Int!
rleMaskList: [RLEMaskForObject!]!
}
input RemoveObjectInput {
sessionId: String!
objectId: Int!
}
type StartSession {
sessionId: String!
}
input StartSessionInput {
path: String!
}
demo/frontend/schemas/merge-schemas.ts 0 → 100644
View file @ 3af09475
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {loadFilesSync} from '@graphql-tools/load-files';
import {mergeTypeDefs} from '@graphql-tools/merge';
import fs from 'fs';
import {print} from 'graphql';
import path from 'path';
import * as prettier from 'prettier';
import {fileURLToPath} from 'url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const loadedFiles = loadFilesSync(`${__dirname}/*.graphql`);
const typeDefs = mergeTypeDefs(loadedFiles);
const printedTypeDefs = print(typeDefs);
const prettyTypeDefs = await prettier.format(printedTypeDefs, {
parser: 'graphql',
});
fs.writeFileSync('schema.graphql', prettyTypeDefs);
demo/frontend/schemas/video-api-schema.graphql 0 → 100644
View file @ 3af09475
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
The `ID` scalar type represents a unique identifier, often used to refetch an object or as key for a cache. The ID type appears in a JSON response as a String; however, it is not intended to be human-readable. When expected as an input type, any string (such as `"4"`) or integer (such as `4`) input value will be accepted as an ID.
"""
scalar GlobalID
@specifiedBy(url: "https://relay.dev/graphql/objectidentification.htm")
type Mutation {
createDeletionId: String!
acceptTos: Boolean!
acceptTermsOfService: String!
uploadVideo(
file: Upload!
startTimeSec: Float = null
durationTimeSec: Float = null
): Video!
uploadSharedVideo(file: Upload!): SharedVideo!
uploadAnnotations(file: Upload!): Boolean!
}
"""
An object with a Globally Unique ID
"""
interface Node {
"""
The Globally Unique ID of this object
"""
id: GlobalID!
}
"""
Information to aid in pagination.
"""
type PageInfo {
"""
When paginating forwards, are there more items?
"""
hasNextPage: Boolean!
"""
When paginating backwards, are there more items?
"""
hasPreviousPage: Boolean!
"""
When paginating backwards, the cursor to continue.
"""
startCursor: String
"""
When paginating forwards, the cursor to continue.
"""
endCursor: String
}
type Query {
defaultVideo: Video!
videos(
"""
Returns the items in the list that come before the specified cursor.
"""
before: String = null
"""
Returns the items in the list that come after the specified cursor.
"""
after: String = null
"""
Returns the first n items from the list.
"""
first: Int = null
"""
Returns the items in the list that come after the specified cursor.
"""
last: Int = null
): VideoConnection!
sharedVideo(path: String!): SharedVideo!
}
type SharedVideo {
path: String!
url: String!
}
scalar Upload
type Video implements Node {
"""
The Globally Unique ID of this object
"""
id: GlobalID!
path: String!
posterPath: String
width: Int!
height: Int!
url: String!
posterUrl: String!
}
"""
A connection to a list of items.
"""
type VideoConnection {
"""
Pagination data for this connection
"""
pageInfo: PageInfo!
"""
Contains the nodes in this connection
"""
edges: [VideoEdge!]!
}
"""
An edge in a connection.
"""
type VideoEdge {
"""
A cursor for use in pagination
"""
cursor: String!
"""
The item at the end of the edge
"""
node: Video!
}
demo/frontend/src/App.tsx 0 → 100644
View file @ 3af09475
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import SAM2DemoApp from '@/demo/SAM2DemoApp';
import SettingsContextProvider from '@/settings/SettingsContextProvider';
import {RouterProvider, createBrowserRouter} from 'react-router-dom';
export default function App() {
const router = createBrowserRouter([
{
path: '*',
element: (
<SettingsContextProvider>
<SAM2DemoApp />
</SettingsContextProvider>
),
},
]);
return <RouterProvider router={router} />;
}
demo/frontend/src/assets/scss/App.scss 0 → 100644
View file @ 3af09475
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@tailwind base;
@tailwind components;
@tailwind utilities;
.tab {
display: flex;
padding: 0px 0px;
margin-right: 6px;
align-items: center;
height: 100%;
}
@layer base {
@font-face {
font-family: 'Inter';
src: url(/fonts/Inter-VariableFont.ttf) format('truetype-variations');
}
}
body {
font-family: 'Inter', sans-serif;
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
}
body,
html,
#root {
height: 100%;
@media screen and (max-width: '768px') {
overflow: hidden;
}
}
:root {
--segEv-font: 'Inter', system-ui, -apple-system, BlinkMacSystemFont,
'Segoe UI', Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue',
sans-serif;
--perspective: 4000px;
color-scheme: dark;
}
h1,
h2,
h3,
h4,
h5,
h6 {
font-family: 'Inter', sans-serif;
}
.prose .display h1 {
@apply text-4xl text-gray-800 font-medium leading-tight;
}
.prose .display h2 {
@apply text-gray-800 font-medium leading-tight;
font-size: 2.5rem;
}
.prose h1 {
@apply text-3xl text-gray-800 font-medium leading-tight mt-2 mb-4;
letter-spacing: 0.016rem;
}
.prose h2 {
@apply text-2xl text-gray-800 font-medium leading-tight my-2;
letter-spacing: 0.01rem;
}
.prose h3 {
@apply text-xl text-gray-800 font-medium leading-tight my-2;
letter-spacing: 0.005rem;
}
.prose h4 {
@apply text-lg text-gray-800 font-medium leading-tight my-2;
}
.prose h5 {
@apply text-xl text-gray-700 font-normal leading-normal my-2;
letter-spacing: 0.005rem;
}
.prose h6 {
@apply text-base text-gray-700 font-normal leading-normal my-2;
}
.prose p {
@apply text-sm text-gray-700 font-normal leading-normal;
@apply leading-snug;
}
.prose ol,
.prose ul {
@apply text-sm text-gray-700 font-normal leading-normal;
padding-right: 2rem;
}
.dark-mode h1,
.dark-mode h2,
.dark-mode h3,
.dark-mode h4,
.dark-mode h5,
.dark-mode p,
.dark-mode ol,
.dark-mode ul,
.dark-mode p *,
.dark-mode ol *,
.dark-mode ul *,
≈ {
@apply text-white;
}
.dark-mode h4,
.dark-mode h6,
.dark-mode li::marker,
.dark-mode a {
@apply text-gray-200;
}
.flex-grow-2 {
flex-grow: 2;
}
.flex-grow-3 {
flex-grow: 3;
}
.flex-grow-4 {
flex-grow: 4;
}
.flex-grow-5 {
flex-grow: 5;
}
.nav-title {
font-family: var(--segEv-font);
}
.segment-active {
animation: segment-highlight 2s linear infinite;
stroke-dasharray: 5, 10;
stroke-width: 4px;
}
@keyframes segment-highlight {
to {
stroke-dashoffset: 60;
}
}
.segment-select {
animation: segment-dotted 2s linear infinite;
stroke-dasharray: 3, 5;
stroke-width: 3px;
}
@keyframes segment-dotted {
to {
stroke-dashoffset: 24;
}
}
/**
* Daisy UI customizations
*/
.btn {
@apply normal-case rounded-md;
}
.comp_summary h1,
.comp_summary h2,
.comp_summary h3 {
@apply mb-4;
}
.disabled {
opacity: 0.4;
pointer-events: none;
}
.absolute-center {
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
}
@screen lg {
.drawer .grid {
grid-template-columns: max-content 1fr;
}
}
.fade-in {
transition: opacity 0.5s;
opacity: 1 !important;
}
.react-photo-gallery--gallery > div {
gap: 0.25rem;
}
.sticker {
filter: drop-shadow(0.25rem 0.25rem 5px #fff)
drop-shadow(-0.25rem 0.25rem 5px #fff)
drop-shadow(0.25rem -0.25rem 5px #fff)
drop-shadow(-0.25rem -0.25rem 5px #fff);
transition: filter 0.3s ease-out;
}
.sticker:hover,
.sticker-select {
filter: drop-shadow(0.25rem 0.25rem 1px #2962d9)
drop-shadow(-0.25rem 0.25rem 1px #2962d9)
drop-shadow(0.25rem -0.25rem 1px #2962d9)
drop-shadow(-0.25rem -0.25rem 1px #2962d9);
}
/* keyframe animations */
.mask-path,
.reveal {
opacity: 0;
animation: reveal 0.4s ease-in forwards;
}
.slow-reveal {
animation: reveal 1s ease-in;
}
.reveal-then-conceal {
opacity: 0;
animation: reveal-then-conceal 1.5s ease-in-out forwards;
}
@keyframes reveal {
from {
opacity: 0;
}
to {
opacity: 1;
}
}
@keyframes reveal-then-conceal {
from {
opacity: 0;
}
50% {
opacity: 1;
}
to {
opacity: 0;
}
}
.background-animate {
background-size: 400%;
animation: pulse 3s ease infinite;
}
@keyframes pulse {
0%,
100% {
background-position: 0% 50%;
}
50% {
background-position: 100% 50%;
}
}
/* Fix for Safari and Mobile Safari:
Extracted Tailwind progress-bar styles and applied
them to a <div> instead of a <progress> element */
.loading-bar {
position: relative;
width: 100%;
-webkit-appearance: none;
-moz-appearance: none;
appearance: none;
overflow: hidden;
height: 0.5rem;
border-radius: 1rem;
border-radius: var(--rounded-box, 1rem);
vertical-align: baseline;
background-color: hsl(var(--n) / var(--tw-bg-opacity));
--tw-bg-opacity: 0.2;
&::after {
--tw-bg-opacity: 1;
background-color: hsl(var(--n) / var(--tw-bg-opacity));
content: '';
position: absolute;
top: 0px;
bottom: 0px;
left: -40%;
width: 33.333333%;
border-radius: 1rem;
border-radius: var(--rounded-box, 1rem);
animation: loading 5s infinite ease-in-out;
}
}
@keyframes loading {
50% {
left: 107%;
}
}
@keyframes inAnimation {
0% {
opacity: 0;
max-height: 0px;
}
50% {
opacity: 1;
}
100% {
opacity: 1;
max-height: 600px;
}
}
@keyframes outAnimation {
0% {
opacity: 1;
max-height: 600px;
}
50% {
opacity: 0;
}
100% {
opacity: 0;
max-height: 0px;
}
}
@keyframes ellipsisAnimation {
0% {
content: '';
}
25% {
content: '.';
}
50% {
content: '..';
}
75% {
content: '...';
}
}
.ellipsis::after {
content: '';
animation: ellipsisAnimation 1.5s infinite;
}
demo/frontend/src/common/codecs/VideoDecoder.ts 0 → 100644
View file @ 3af09475
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {cloneFrame} from '@/common/codecs/WebCodecUtils';
import {FileStream} from '@/common/utils/FileUtils';
import {
createFile,
DataStream,
MP4ArrayBuffer,
MP4File,
MP4Sample,
MP4VideoTrack,
} from 'mp4box';
import {isAndroid, isChrome, isEdge, isWindows} from 'react-device-detect';
export type ImageFrame = {
bitmap: VideoFrame;
timestamp: number;
duration: number;
};
export type DecodedVideo = {
width: number;
height: number;
frames: ImageFrame[];
numFrames: number;
fps: number;
};
function decodeInternal(
identifier: string,
onReady: (mp4File: MP4File) => Promise<void>,
onProgress: (decodedVideo: DecodedVideo) => void,
): Promise<DecodedVideo> {
return new Promise((resolve, reject) => {
const imageFrames: ImageFrame[] = [];
const globalSamples: MP4Sample[] = [];
let decoder: VideoDecoder;
let track: MP4VideoTrack | null = null;
const mp4File = createFile();
mp4File.onError = reject;
mp4File.onReady = async info => {
if (info.videoTracks.length > 0) {
track = info.videoTracks[0];
} else {
// The video does not have a video track, so looking if there is an
// "otherTracks" available. Note, I couldn't find any documentation
// about "otherTracks" in WebCodecs [1], but it was available in the
// info for MP4V-ES, which isn't supported by Chrome [2].
// However, we'll still try to get the track and then throw an error
// further down in the VideoDecoder.isConfigSupported if the codec is
// not supported by the browser.
//
// [1] https://www.w3.org/TR/webcodecs/
// [2] https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Video_codecs#mp4v-es
track = info.otherTracks[0];
}
if (track == null) {
reject(new Error(`${identifier} does not contain a video track`));
return;
}
const timescale = track.timescale;
const edits = track.edits;
let frame_n = 0;
decoder = new VideoDecoder({
// Be careful with any await in this function. The VideoDecoder will
// not await output and continue calling it with decoded frames.
async output(inputFrame) {
if (track == null) {
reject(new Error(`${identifier} does not contain a video track`));
return;
}
const saveTrack = track;
// If the track has edits, we'll need to check that only frames are
// returned that are within the edit list. This can happen for
// trimmed videos that have not been transcoded and therefore the
// video track contains more frames than those visually rendered when
// playing back the video.
if (edits != null && edits.length > 0) {
const cts = Math.round(
(inputFrame.timestamp * timescale) / 1_000_000,
);
if (cts < edits[0].media_time) {
inputFrame.close();
return;
}
}
// Workaround for Chrome where the decoding stops at ~17 frames unless
// the VideoFrame is closed. So, the workaround here is to create a
// new VideoFrame and close the decoded VideoFrame.
// The frame has to be cloned, or otherwise some frames at the end of the
// video will be black. Note, the default VideoFrame.clone doesn't work
// and it is using a frame cloning found here:
// https://webcodecs-blogpost-demo.glitch.me/
if (
(isAndroid && isChrome) ||
(isWindows && isChrome) ||
(isWindows && isEdge)
) {
const clonedFrame = await cloneFrame(inputFrame);
inputFrame.close();
inputFrame = clonedFrame;
}
const sample = globalSamples[frame_n];
if (sample != null) {
const duration = (sample.duration * 1_000_000) / sample.timescale;
imageFrames.push({
bitmap: inputFrame,
timestamp: inputFrame.timestamp,
duration,
});
// Sort frames in order of timestamp. This is needed because Safari
// can return decoded frames out of order.
imageFrames.sort((a, b) => (a.timestamp < b.timestamp ? 1 : -1));
// Update progress on first frame and then every 40th frame
if (onProgress != null && frame_n % 100 === 0) {
onProgress({
width: saveTrack.track_width,
height: saveTrack.track_height,
frames: imageFrames,
numFrames: saveTrack.nb_samples,
fps:
(saveTrack.nb_samples / saveTrack.duration) *
saveTrack.timescale,
});
}
}
frame_n++;
if (saveTrack.nb_samples === frame_n) {
// Sort frames in order of timestamp. This is needed because Safari
// can return decoded frames out of order.
imageFrames.sort((a, b) => (a.timestamp > b.timestamp ? 1 : -1));
resolve({
width: saveTrack.track_width,
height: saveTrack.track_height,
frames: imageFrames,
numFrames: saveTrack.nb_samples,
fps:
(saveTrack.nb_samples / saveTrack.duration) *
saveTrack.timescale,
});
}
},
error(error) {
reject(error);
},
});
let description;
const trak = mp4File.getTrackById(track.id);
const entries = trak?.mdia?.minf?.stbl?.stsd?.entries;
if (entries == null) {
return;
}
for (const entry of entries) {
if (entry.avcC || entry.hvcC) {
const stream = new DataStream(undefined, 0, DataStream.BIG_ENDIAN);
if (entry.avcC) {
entry.avcC.write(stream);
} else if (entry.hvcC) {
entry.hvcC.write(stream);
}
description = new Uint8Array(stream.buffer, 8); // Remove the box header.
break;
}
}
const configuration: VideoDecoderConfig = {
codec: track.codec,
codedWidth: track.track_width,
codedHeight: track.track_height,
description,
};
const supportedConfig =
await VideoDecoder.isConfigSupported(configuration);
if (supportedConfig.supported == true) {
decoder.configure(configuration);
mp4File.setExtractionOptions(track.id, null, {
nbSamples: Infinity,
});
mp4File.start();
} else {
reject(
new Error(
`Decoder config faile: config ${JSON.stringify(
supportedConfig.config,
)} is not supported`,
),
);
return;
}
};
mp4File.onSamples = async (
_id: number,
_user: unknown,
samples: MP4Sample[],
) => {
for (const sample of samples) {
globalSamples.push(sample);
decoder.decode(
new EncodedVideoChunk({
type: sample.is_sync ? 'key' : 'delta',
timestamp: (sample.cts * 1_000_000) / sample.timescale,
duration: (sample.duration * 1_000_000) / sample.timescale,
data: sample.data,
}),
);
}
await decoder.flush();
decoder.close();
};
onReady(mp4File);
});
}
export function decode(
file: File,
onProgress: (decodedVideo: DecodedVideo) => void,
): Promise<DecodedVideo> {
return decodeInternal(
file.name,
async (mp4File: MP4File) => {
const reader = new FileReader();
reader.onload = function () {
const result = this.result as MP4ArrayBuffer;
if (result != null) {
result.fileStart = 0;
mp4File.appendBuffer(result);
}
mp4File.flush();
};
reader.readAsArrayBuffer(file);
},
onProgress,
);
}
export function decodeStream(
fileStream: FileStream,
onProgress: (decodedVideo: DecodedVideo) => void,
): Promise<DecodedVideo> {
return decodeInternal(
'stream',
async (mp4File: MP4File) => {
let part = await fileStream.next();
while (part.done === false) {
const result = part.value.data.buffer as MP4ArrayBuffer;
if (result != null) {
result.fileStart = part.value.range.start;
mp4File.appendBuffer(result);
}
mp4File.flush();
part = await fileStream.next();
}
},
onProgress,
);
}
demo/frontend/src/common/codecs/VideoEncoder.ts 0 → 100644
View file @ 3af09475
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {ImageFrame} from '@/common/codecs/VideoDecoder';
import {MP4ArrayBuffer, createFile} from 'mp4box';
// The selection of timescale and seconds/key-frame value are
// explained in the following docs: https://github.com/vjeux/mp4-h264-re-encode
const TIMESCALE = 90000;
const SECONDS_PER_KEY_FRAME = 2;
export function encode(
width: number,
height: number,
numFrames: number,
framesGenerator: AsyncGenerator<ImageFrame, unknown>,
progressCallback?: (progress: number) => void,
): Promise<MP4ArrayBuffer> {
return new Promise((resolve, reject) => {
let encodedFrameIndex = 0;
let nextKeyFrameTimestamp = 0;
let trackID: number | null = null;
const durations: number[] = [];
const outputFile = createFile();
const encoder = new VideoEncoder({
output(chunk, metaData) {
const uint8 = new Uint8Array(chunk.byteLength);
chunk.copyTo(uint8);
const description = metaData?.decoderConfig?.description;
if (trackID === null) {
trackID = outputFile.addTrack({
width: width,
height: height,
timescale: TIMESCALE,
avcDecoderConfigRecord: description,
});
}
const shiftedDuration = durations.shift();
if (shiftedDuration != null) {
outputFile.addSample(trackID, uint8, {
duration: getScaledDuration(shiftedDuration),
is_sync: chunk.type === 'key',
});
encodedFrameIndex++;
progressCallback?.(encodedFrameIndex / numFrames);
}
if (encodedFrameIndex === numFrames) {
resolve(outputFile.getBuffer());
}
},
error(error) {
reject(error);
return;
},
});
const setConfigurationAndEncodeFrames = async () => {
// The codec value was taken from the following implementation and seems
// reasonable for our use case for now:
// https://github.com/vjeux/mp4-h264-re-encode/blob/main/mp4box.html#L103
// Additional details about codecs can be found here:
// - https://developer.mozilla.org/en-US/docs/Web/Media/Formats/codecs_parameter
// - https://www.w3.org/TR/webcodecs-codec-registry/#video-codec-registry
//
// The following setting is a good compromise between output video file
// size and quality. The latencyMode "realtime" is needed for Safari,
// which otherwise will produce 20x larger files when in quality
// latencyMode. Chrome does a really good job with file size even when
// latencyMode is set to quality.
const configuration: VideoEncoderConfig = {
codec: 'avc1.4d0034',
width: roundToNearestEven(width),
height: roundToNearestEven(height),
bitrate: 14_000_000,
alpha: 'discard',
bitrateMode: 'variable',
latencyMode: 'realtime',
};
const supportedConfig =
await VideoEncoder.isConfigSupported(configuration);
if (supportedConfig.supported === true) {
encoder.configure(configuration);
} else {
throw new Error(
`Unsupported video encoder config ${JSON.stringify(supportedConfig)}`,
);
}
for await (const frame of framesGenerator) {
const {bitmap, duration, timestamp} = frame;
durations.push(duration);
let keyFrame = false;
if (timestamp >= nextKeyFrameTimestamp) {
await encoder.flush();
keyFrame = true;
nextKeyFrameTimestamp = timestamp + SECONDS_PER_KEY_FRAME * 1e6;
}
encoder.encode(bitmap, {keyFrame});
bitmap.close();
}
await encoder.flush();
encoder.close();
};
setConfigurationAndEncodeFrames();
});
}
function getScaledDuration(rawDuration: number) {
return rawDuration / (1_000_000 / TIMESCALE);
}
function roundToNearestEven(dim: number) {
const rounded = Math.round(dim);
if (rounded % 2 === 0) {
return rounded;
} else {
return rounded + (rounded > dim ? -1 : 1);
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment