Skip to content

GitLab

"tests/albert/test_modeling_albert.py" did not exist on "c40c7e213bdd0479bdca69df0c500004a7294d39"
Commit 08a21d59 authored by chenpangpang's avatar chenpangpang
Browse files

feat: 初始提交

parent 1a6b26f1
Pipeline #2165 failed with stages
in 0 seconds
Expand all Show whitespace changes
Inline Side-by-side
Showing with 2582 additions and 1 deletion
.gitignore 0 → 100644
View file @ 08a21d59
.idea
chenyh
Dockerfile 0 → 100644
View file @ 08a21d59
FROM image.sourcefind.cn:5000/gpu/admin/base/jupyterlab-pytorch:2.3.1-py3.10-cuda11.8-ubuntu22.04-devel as base
ARG IMAGE=ruyi-mini-7b
ARG IMAGE_UPPER=Ruyi-Mini-7B
ARG BRANCH=gpu
RUN cd /root && git clone -b $BRANCH http://developer.hpccube.com/codes/chenpangpang/$IMAGE.git
WORKDIR /root/$IMAGE/$IMAGE_UPPER
RUN pip install -r requirements.txt
#########
# Prod #
#########
FROM image.sourcefind.cn:5000/gpu/admin/base/jupyterlab-pytorch:2.3.1-py3.10-cuda11.8-ubuntu22.04-devel
ARG IMAGE=ruyi-mini-7b
ARG IMAGE_UPPER=Ruyi-Mini-7B
COPY chenyh/$IMAGE/frpc_linux_amd64_* /opt/conda/lib/python3.10/site-packages/gradio/
RUN chmod +x /opt/conda/lib/python3.10/site-packages/gradio/frpc_linux_amd64_*
COPY chenyh/$IMAGE/ /root/Ruyi-Models/models
COPY --from=base /opt/conda/lib/python3.10/site-packages /opt/conda/lib/python3.10/site-packages
COPY --from=base /root/$IMAGE/$IMAGE_UPPER /root/Ruyi-Models
COPY --from=base /root/$IMAGE/启动器.ipynb /root/$IMAGE/start.sh /root/
COPY --from=base /root/$IMAGE/assets/ /root/assets/
\ No newline at end of file
README.md
View file @ 08a21d59
# Ruyi-Mini-7B
# NVComposer
Ruyi-Models/.gitignore 0 → 100644
View file @ 08a21d59
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
.pdm.toml
.pdm-python
.pdm-build/
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
Ruyi-Models/LICENSE 0 → 100644
View file @ 08a21d59
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Ruyi-Models/README.md 0 → 100644
View file @ 08a21d59
# Ruyi-Models
English | [简体中文](./README_CN.md)
Welcome to Ruyi-Models!
Ruyi is an image-to-video model capable of generating cinematic-quality videos at a **resolution of 768**, with a frame rate of **24 frames per second**, totaling **5 seconds and 120 frames**. It supports **lens control** and **motion amplitude control**. Using a **RTX 3090 or RTX 4090**, you can generate 512 resolution, 120 frames (or 768 resolution, ~72 frames) videos **without any loss of quality**.
## Table of Contents
- [Installation Instructions](#installation-instructions)
- [Download Model (Optional)](#download-model-optional)
- [How to Use](#how-to-use)
- [Showcase](#showcase)
- [GPU Memory Optimization](#gpu-memory-optimization)
- [License](#license)
## Installation Instructions
The installation instructions are simple. Just clone the repo and install the requirements.
```shell
git clone https://github.com/IamCreateAI/Ruyi-Models
cd Ruyi-Models
pip install -r requirements.txt
```
### For ComfyUI Users
#### Method (1): Installation via ComfyUI Manager
Download and install [ComfyUI-Manager](https://github.com/ltdrdata/ComfyUI-Manager).
```shell
cd ComfyUI/custom_nodes/
git clone https://github.com/ltdrdata/ComfyUI-Manager.git
# install requirements
pip install -r ComfyUI-Manager/requirements.txt
```
Next, start ComfyUI and open the Manager. Select Custom Nodes Manager, then search for "Ruyi". You should see ComfyUI-Ruyi as shown in the screenshot below. Click "Install" to proceed.
<div align=center>
<img src="https://github.com/user-attachments/assets/10dda65f-13d5-4da8-9437-9c98b114536c"></img>
</div>
Finally, search for "ComfyUI-VideoHelperSuite" and install it as well.
#### Method (2): Manual Installation
Download and save this repository to the path *ComfyUI/custom_nodes/Ruyi-Models*.
```shell
# download the repo
cd ComfyUI/custom_nodes/
git clone https://github.com/IamCreateAI/Ruyi-Models.git
# install requirements
pip install -r Ruyi-Models/requirements.txt
```
Install the dependency [ComfyUI-VideoHelperSuite](https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite) to display video output (skip this step if already installed).
```shell
# download ComfyUI-VideoHelperSuite
cd ComfyUI/custom_nodes/
git clone https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite.git
# install requirements
pip install -r ComfyUI-VideoHelperSuite/requirements.txt
```
##### For Windows Users
When using the Windows operating system, a common distribution is [ComfyUI_windows_portable_nvidia](https://github.com/comfyanonymous/ComfyUI/releases). When launched with `run_nvidia_gpu.bat`, it utilizes the embedded Python interpreter included with the package. Therefore, the environment needs to be set up within this built-in Python.
For example, if the extracted directory of the distribution is ComfyUI_windows_portable, you can typically use the following command to download the repository and install the runtime environment:
```shell
# download the repo
cd ComfyUI_windows_portable\ComfyUI\custom_nodes
git clone https://github.com/IamCreateAI/Ruyi-Models.git
# install requirements using embedded Python interpreter
..\..\python_embeded\python.exe -m pip install -r Ruyi-Models\requirements.txt
```
## Download Model (Optional)
Download the model and save it to certain path. To directly run our model, it is recommand to save the models into _Ruyi-Models/models_ folder. For ComfyUI users, the path should be _ComfyUI/models/Ruyi_.
| Model Name | Type | Resolution | Max Frames | Frames per Second | Storage Space | Download |
| :---: | :---: | :---: | :---: | :---: | :---: | :---: |
| Ruyi-Mini-7B | Image to Video | 512 & 768 | 120 | 24 | 17 GB | [🤗](https://huggingface.co/IamCreateAI/Ruyi-Mini-7B) |
For example, after downloading Ruyi-Mini-7B, the file path structure should be:
```
📦 Ruyi-Models/models/ or ComfyUI/models/Ruyi/
├── 📂 Ruyi-Mini-7B/
│ ├── 📂 transformers/
│ ├── 📂 vae/
│ └── 📂 ...
```
> This repository **supports automatic model downloading**, but manual downloading provides more control. For instance, you can download the model to another location and then link it to the *ComfyUI/models/Ruyi* path using symbolic links or similar methods.
## How to Use
We provide two ways to run our model. The first is directly using python code.
```
python3 predict_i2v.py
```
Specifically, the script downloads the model to the _Ruyi-Models/models_ folder and uses images from the [_assets_](./assets/) folder as the start and end frames for video inference. You can modify the variables in the script to replace the input images and set parameters such as video length and resolution.
For users with more than 24GB of GPU memory, you can use predict_i2v_80g.py to enhance generation speed. For those with less GPU memory, we offer parameters to optimize memory usage, enabling the generation of higher resolution and longer videos by extending the inference time. The effects of these parameters can be found in the [GPU memory optimization section](#gpu-memory-optimization) section below.
Or use ComfyUI wrapper in our github repo, the detail of ComfyUI nodes is described in [_comfyui/README.md_](./comfyui/README.md).
## Showcase
### Image to Video Effects
<table>
<tr>
<td><video src="https://github.com/user-attachments/assets/4dedf40b-82f2-454c-9a67-5f4ed243f5ea"></video></td>
<td><video src="https://github.com/user-attachments/assets/905fef17-8c5d-49b0-a49a-6ae7e212fa07"></video></td>
<td><video src="https://github.com/user-attachments/assets/20daab12-b510-448a-9491-389d7bdbbf2e"></video></td>
<td><video src="https://github.com/user-attachments/assets/f1bb0a91-d52a-4611-bac2-8fcf9658cac0"></video></td>
</tr>
</table>
### Camera Control
<table>
<tr>
<td align=center><img src="https://github.com/user-attachments/assets/8aedcea6-3b8e-4c8b-9fed-9ceca4d41954" height=200></img>input</td>
<td align=center><video src="https://github.com/user-attachments/assets/d9d027d4-0d4f-45f5-9d46-49860b562c69"></video>left</td>
<td align=center><video src="https://github.com/user-attachments/assets/7716a67b-1bb8-4d44-b128-346cbc35e4ee"></video>right</td>
</tr>
<tr>
<td align=center><video src="https://github.com/user-attachments/assets/cc1f1928-cab7-4c4b-90af-928936102e66"></video>static</td>
<td align=center><video src="https://github.com/user-attachments/assets/c742ea2c-503a-454f-a61a-10b539100cd9"></video>up</td>
<td align=center><video src="https://github.com/user-attachments/assets/442839fa-cc53-4b75-b015-909e44c065e0"></video>down</td>
</tr>
</table>
### Motion Amplitude Control
<table>
<tr>
<td align=center><video src="https://github.com/user-attachments/assets/0020bd54-0ff6-46ad-91ee-d9f0df013772"></video>motion 1</td>
<td align=center><video src="https://github.com/user-attachments/assets/d1c26419-54e3-4b86-8ae3-98e12de3022e"></video>motion 2</td>
<td align=center><video src="https://github.com/user-attachments/assets/535147a2-049a-4afc-8d2a-017bc778977e"></video>motion 3</td>
<td align=center><video src="https://github.com/user-attachments/assets/bf893d53-2e11-406f-bb9a-2aacffcecd44"></video>motion 4</td>
</tr>
</table>
## GPU Memory Optimization
We provide the options **`GPU_memory_mode` and `GPU_offload_steps` to reduce GPU memory usage**, catering to different user needs.
Generally speaking, using **less GPU memory requires more RAM and results in longer generation times**. Below is a reference table of expected GPU memory usage and generation times. Note that, the GPU memory reported below is the `max_memory_allocated()` value. The values read from nvidia-smi may be higher than the reported values because CUDA occupies some GPU memory (usually between 500 - 800 MiB), and PyTorch's caching mechanism also requests additional GPU memory.
Additionally, the community and we have created a detailed table featuring various resolutions and option combinations, which can be found in the [gpu_memory_appendix.md](assets/gpu_memory_appendix.md). We encourage community members to help us complete the table.
### A100 Results
- Resolution of 512
| Num frames | normal_mode + 0 steps | normal_mode + 10 steps | normal_mode + 7 steps | normal_mode + 5 steps | normal_mode + 1 steps | low_gpu_mode + 0 steps |
| :---: | :---: | :---: | :---: | :---: | :---: | :---: |
| 24 frames | 16119MiB <br> _01:01s_ | 15535MiB <br> _01:07s_ | 15340MiB <br> _01:13s_ | 15210MiB <br> _01:20s_ | 14950MiB <br> _01:32s_ | 4216MiB <br> _05:14s_ |
| 48 frames | 18398MiB <br> _01:53s_ | 17230MiB <br> _02:15s_ | 16840MiB <br> _02:29s_ | 16580MiB <br> _02:32s_ | 16060MiB <br> _02:54s_ | 4590MiB <br> _09:59s_ |
| 72 frames | 20678MiB <br> _03:00s_ | 18925MiB <br> _03:31s_ | 18340MiB <br> _03:53s_ | 17951MiB <br> _03:57s_ | 17171MiB <br> _04:25s_ | 6870MiB <br> _14:42s_ |
| 96 frames | 22958MiB <br> _04:11s_ | 20620MiB <br> _04:54s_ | 19841MiB <br> _05:10s_ | 19321MiB <br> _05:14s_ | 18281MiB <br> _05:47s_ | 9150MiB <br> _19:17s_ |
| 120 frames | 25238MiB <br> _05:42s_ | 22315MiB <br> _06:34s_ | 21341MiB <br> _06:59s_ | 20691MiB <br> _07:07s_ | 19392MiB <br> _07:41s_ | 11430MiB <br> _24:08s_ |
- Resolution of 768
| Num frames | normal_mode + 0 steps | normal_mode + 10 steps | normal_mode + 7 steps | normal_mode + 5 steps | normal_mode + 1 steps | low_gpu_mode + 0 steps |
| :---: | :---: | :---: | :---: | :---: | :---: | :---: |
| 24 frames | 18971MiB <br> _02:06s_ | 17655MiB <br> _02:40s_ | 17217MiB <br> _02:39s_ | 16925MiB <br> _02:41s_ | 16339MiB <br> _03:13s_ | 5162MiB <br> _13:42s_ |
| 48 frames | 24101MiB <br> _04:52s_ | 21469MiB <br> _05:44s_ | 20592MiB <br> _05:51s_ | 20008MiB <br> _06:00s_ | 18837MiB <br> _06:49s_ | 10292MiB <br> _20:58s_ |
| 72 frames | 29230MiB <br> _08:24s_ | 25283MiB <br> _09:45s_ | 25283MiB <br> _09:45s_ | 23091MiB <br> _10:10s_ | 21335MiB <br> _11:10s_ | 15421MiB <br> _39:12s_ |
| 96 frames | 34360MiB <br> _12:49s_ | 29097MiB <br> _14:41s_ | 27343MiB <br> _15:33s_ | 26174MiB <br> _15:44s_ | 23834MiB <br> _16:33s_ | 20550MiB <br> _43:47s_ |
| 120 frames | 39489MiB <br> _18:21s_ | 32911MiB <br> _20:39s_ | 30719MiB <br> _21:34s_ | 29257MiB <br> _21:48s_ | 26332MiB <br> _23:02s_ | 25679MiB <br> _63:01s_ |
### RTX 4090 Results
The values marked with `---` in the table indicate that an out-of-memory (OOM) error occurred, preventing generation.
- Resolution of 512
| Num frames | normal_mode + 0 steps | normal_mode + 10 steps | normal_mode + 7 steps | normal_mode + 5 steps | normal_mode + 1 steps | low_gpu_mode + 0 steps |
| :---: | :---: | :---: | :---: | :---: | :---: | :---: |
| 24 frames | 16366MiB <br> _01:18s_ | 15805MiB <br> _01:26s_ | 15607MiB <br> _01:37s_ | 15475MiB <br> _01:36s_ | 15211MiB <br> _01:39s_ | 4211MiB <br> _03:57s_ |
| 48 frames | 18720MiB <br> _02:21s_ | 17532MiB <br> _02:49s_ | 17136MiB <br> _02:55s_ | 16872MiB <br> _02:58s_ | 16344MiB <br> _03:01s_ | 4666MiB <br> _05:01s_ |
| 72 frames | 21036MiB <br> _03:41s_ | 19254MiB <br> _04:25s_ | 18660MiB <br> _04:34s_ | 18264MiB <br> _04:36s_ | 17472MiB <br> _04:51s_ | 6981MiB <br> _06:36s_ |
| 96 frames | -----MiB <br> _--:--s_ | 20972MiB <br> _06:18s_ | 20180MiB <br> _06:24s_ | 19652MiB <br> _06:36s_ | 18596MiB <br> _06:56s_ | 9298MiB <br> _10:03s_ |
| 120 frames | -----MiB <br> _--:--s_ | -----MiB <br> _--:--s_ | 21704MiB <br> _08:50s_ | 21044MiB <br> _08:53s_ | 19724MiB <br> _09:08s_ | 11613MiB <br> _13:57s_ |
- Resolution of 768
| Num frames | normal_mode + 0 steps | normal_mode + 10 steps | normal_mode + 7 steps | normal_mode + 5 steps | normal_mode + 1 steps | low_gpu_mode + 0 steps |
| :---: | :---: | :---: | :---: | :---: | :---: | :---: |
| 24 frames | 19223MiB <br> _02:38s_ | 17900MiB <br> _03:06s_ | 17448MiB <br> _03:18s_ | 17153MiB <br> _03:23s_ | 16624MiB <br> _03:34s_ | 5251MiB <br> _05:54s_ |
| 48 frames | -----MiB <br> _--:--s_ | -----MiB <br> _--:--s_ | 20946MiB <br> _07:28s_ | 20352MiB <br> _07:35s_ | 19164MiB <br> _08:04s_ | 10457MiB <br> _10:55s_ |
| 72 frames | -----MiB <br> _--:--s_ | -----MiB <br> _--:--s_ | -----MiB <br> _--:--s_ | -----MiB <br> _--:--s_ | -----MiB <br> _--:--s_ | 15671MiB <br> _18:52s_ |
## License
We’re releasing the model under a permissive **Apache 2.0 license**.
## BibTeX
```
@misc{createai2024ruyi,
title={Ruyi-Mini-7B},
author={CreateAI Team},
year={2024},
publisher = {GitHub},
journal = {GitHub repository},
howpublished={\url{https://github.com/IamCreateAI/Ruyi-Models}}
}
```
## Welcome Feedback and Collaborative Optimization
We sincerely welcome everyone to actively provide valuable feedback and suggestions, and we hope to work together to optimize our services and products. Your words will help us better understand user needs, allowing us to continuously enhance the user experience. Thank you for your support and attention to our work!
You are welcomed to join our [Discord](https://discord.com/invite/nueQFQwwGw) or Wechat Group (Scan QR code to add Ruyi Assistant and join the official group) for further discussion!
<img src="https://github.com/user-attachments/assets/cc5e25c6-34ab-4be1-a59b-7d5789264a9c" style="width:300px"></img>
Ruyi-Models/README_CN.md 0 → 100644
View file @ 08a21d59
# Ruyi-Models
欢迎使用 Ruyi-Models！
Ruyi 是一款图生视频模型，能够生成 **768 分辨率、每秒 24 帧总计 5 秒 120 帧的影视级视频**，支持**镜头控制**与**运动幅度控制**，使用 **RTX 3090** 或 **RTX 4090** 可**无精度损失**地生成 512 分辨率、120 帧（768分辨率、~72帧）的视频。
## 安装方法
克隆本仓库并安装所需的依赖。
```shell
git clone https://github.com/IamCreateAI/Ruyi-Models
cd Ruyi-Models
pip install -r requirements.txt
```
### ComfyUI 的安装方法
#### 方法（1）：通过 ComfyUI Manager 安装
下载并安装 [ComfyUI-Manager](https://github.com/ltdrdata/ComfyUI-Manager)。
```shell
cd ComfyUI/custom_nodes/
git clone https://github.com/ltdrdata/ComfyUI-Manager.git
# install requirements
pip install -r ComfyUI-Manager/requirements.txt
```
启动 ComfyUI 并打开 Manager。选择 Custom Nodes Manager，然后搜索 “Ruyi”。选择搜索结果中的 ComfyUI-Ruyi（如下方截图所示），点击 “Install” 按钮安装。
<div align=center>
<img src="https://github.com/user-attachments/assets/10dda65f-13d5-4da8-9437-9c98b114536c"></img>
</div>
最后，搜索 “ComfyUI-VideoHelperSuite” 并安装。
#### 方法（2）：手动安装
- 下载并保存本仓库到 *ComfyUI/custom_nodes/Ruyi-Models* 路径。
```shell
# download the repo
cd ComfyUI/custom_nodes/
git clone https://github.com/IamCreateAI/Ruyi-Models.git
# install requirements
pip install -r Ruyi-Models/requirements.txt
```
- 安装依赖项 [ComfyUI-VideoHelperSuite](https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite) 以显示视频输出（如已安装，请跳过本步骤）。
```shell
# download ComfyUI-VideoHelperSuite
cd ComfyUI/custom_nodes/
git clone https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite.git
# install requirements
pip install -r ComfyUI-VideoHelperSuite/requirements.txt
```
##### Windows 操作系统下的安装方法
在使用Windows操作系统时，[ComfyUI_windows_portable_nvidia](https://github.com/comfyanonymous/ComfyUI/releases) 是一种常见的发行版。当通过 `run_nvidia_gpu.bat` 启动时，会使用其中嵌入的 Python 解释器。因此，需要在这个内置的 Python 环境中安装运行环境。
例如，如果提取后的发行版目录是 _ComfyUI_windows_portable_，通常可使用以下命令下载仓库并安装运行时环境：
```shell
# download the repo
cd ComfyUI_windows_portable\ComfyUI\custom_nodes
git clone https://github.com/IamCreateAI/Ruyi-Models.git
# install requirements using embedded Python interpreter
..\..\python_embeded\python.exe -m pip install -r Ruyi-Models\requirements.txt
```
## 下载模型（可选）
下载模型并将其保存到指定路径。为了直接运行 Ruyi 模型，建议将模型保存到 _Ruyi-Models/models_ 文件夹中。对于 ComfyUI 用户，路径应为 _ComfyUI/models/Ruyi_。
| 名称 | 类型 | 分辨率 | 最大帧数 | 每秒帧数 | 存储空间 | 下载地址 |
| :---: | :---: | :---: | :---: | :---: | :---: | :---: |
| Ruyi-Mini-7B | 图生视频 | 512 & 768 | 120 | 24 | 17 GB | [🤗](https://huggingface.co/IamCreateAI/Ruyi-Mini-7B) |
例如，下载 Ruyi-Mini-7B 后，文件的路径结构应该为：
```
📦 Ruyi-Models/models/ or ComfyUI/models/Ruyi/
├── 📂 Ruyi-Mini-7B/
│ ├── 📂 transformers/
│ ├── 📂 vae/
│ └── 📂 ...
```
> 本仓库**支持自动下载模型功能**，但手动下载提供了更多的可控性。例如，可以下载模型到其他位置，再通过软链接等方式链接到 *ComfyUI/models/Ruyi* 路径。
## 使用方法
我们提供两种运行模型的方法。第一种是直接使用Python代码。
```shell
python3 predict_i2v.py
```
具体来说，该脚本将模型下载到 _Ruyi-Models/models_ 文件夹，并使用 _[assets](./assets)_ 文件夹中的图像作为视频推理的起始帧和结束帧。您可以修改脚本中的变量来替换输入图像，并设置视频长度和分辨率等参数。
对于显存超过 24GB 的用户，可以使用 predict_i2v_80g.py 来提高生成速度。对于显存较少的用户，提供了优化显存使用的参数，这些参数可以通过延长推理时间来生成更高分辨率和更长时长的视频。这些参数的影响可以在下面的显存优化选项小节找到。
或者，您可以使用我们 GitHub 仓库中的 ComfyUI 封装，ComfyUI 节点的详细信息在 _[comfyui/README_CN.md](comfyui/README_CN.md)_ 中描述。
## 效果展示
### 图生视频效果
<table>
<tr>
<td><video src="https://github.com/user-attachments/assets/4dedf40b-82f2-454c-9a67-5f4ed243f5ea"></video></td>
<td><video src="https://github.com/user-attachments/assets/905fef17-8c5d-49b0-a49a-6ae7e212fa07"></video></td>
<td><video src="https://github.com/user-attachments/assets/20daab12-b510-448a-9491-389d7bdbbf2e"></video></td>
<td><video src="https://github.com/user-attachments/assets/f1bb0a91-d52a-4611-bac2-8fcf9658cac0"></video></td>
</tr>
</table>
### 镜头控制
<table>
<tr>
<td align=center><img src="https://github.com/user-attachments/assets/8aedcea6-3b8e-4c8b-9fed-9ceca4d41954" height=200></img>input</td>
<td align=center><video src="https://github.com/user-attachments/assets/d9d027d4-0d4f-45f5-9d46-49860b562c69"></video>left</td>
<td align=center><video src="https://github.com/user-attachments/assets/7716a67b-1bb8-4d44-b128-346cbc35e4ee"></video>right</td>
</tr>
<tr>
<td align=center><video src="https://github.com/user-attachments/assets/cc1f1928-cab7-4c4b-90af-928936102e66"></video>static</td>
<td align=center><video src="https://github.com/user-attachments/assets/c742ea2c-503a-454f-a61a-10b539100cd9"></video>up</td>
<td align=center><video src="https://github.com/user-attachments/assets/442839fa-cc53-4b75-b015-909e44c065e0"></video>down</td>
</tr>
</table>
### 运动幅度控制
<table>
<tr>
<td align=center><video src="https://github.com/user-attachments/assets/0020bd54-0ff6-46ad-91ee-d9f0df013772"></video>motion 1</td>
<td align=center><video src="https://github.com/user-attachments/assets/d1c26419-54e3-4b86-8ae3-98e12de3022e"></video>motion 2</td>
<td align=center><video src="https://github.com/user-attachments/assets/535147a2-049a-4afc-8d2a-017bc778977e"></video>motion 3</td>
<td align=center><video src="https://github.com/user-attachments/assets/bf893d53-2e11-406f-bb9a-2aacffcecd44"></video>motion 4</td>
</tr>
</table>
## 显存优化选项
提供了 **`GPU_memory_mode` 和 `GPU_offload_steps`** 选项以**降低显存占用**，满足不同用户的需求。
通常来说，**使用更少的显存，需要更多的内存和更长的生成时间**。以下列出预期显存使用和生成时间的参考表格。请注意，下面报告的显存是 `max_memory_allocated()` 的返回值，而 nvidia-smi 的显存数值通常会高于报告的数值。因为 CUDA 会占用一些显存（通常在500 - 800 MiB之间），而 PyTorch 的缓存机制也会请求额外的显存。
此外，在社区的帮助下创建了一张详细的表格 [gpu_memory_appendix.md](assets/gpu_memory_appendix.md)，包含了更多分辨率和选项组合。欢迎社区成员帮助完善表格。
### A100 的显存占用与运行时间
- Resolution of 512
| 帧数 | normal_mode + 0 steps | normal_mode + 10 steps | normal_mode + 7 steps | normal_mode + 5 steps | normal_mode + 1 steps | low_gpu_mode + 0 steps |
| :---: | :---: | :---: | :---: | :---: | :---: | :---: |
| 24 frames | 16119MiB <br> _01:01s_ | 15535MiB <br> _01:07s_ | 15340MiB <br> _01:13s_ | 15210MiB <br> _01:20s_ | 14950MiB <br> _01:32s_ | 4216MiB <br> _05:14s_ |
| 48 frames | 18398MiB <br> _01:53s_ | 17230MiB <br> _02:15s_ | 16840MiB <br> _02:29s_ | 16580MiB <br> _02:32s_ | 16060MiB <br> _02:54s_ | 4590MiB <br> _09:59s_ |
| 72 frames | 20678MiB <br> _03:00s_ | 18925MiB <br> _03:31s_ | 18340MiB <br> _03:53s_ | 17951MiB <br> _03:57s_ | 17171MiB <br> _04:25s_ | 6870MiB <br> _14:42s_ |
| 96 frames | 22958MiB <br> _04:11s_ | 20620MiB <br> _04:54s_ | 19841MiB <br> _05:10s_ | 19321MiB <br> _05:14s_ | 18281MiB <br> _05:47s_ | 9150MiB <br> _19:17s_ |
| 120 frames | 25238MiB <br> _05:42s_ | 22315MiB <br> _06:34s_ | 21341MiB <br> _06:59s_ | 20691MiB <br> _07:07s_ | 19392MiB <br> _07:41s_ | 11430MiB <br> _24:08s_ |
- Resolution of 768
| 帧数 | normal_mode + 0 steps | normal_mode + 10 steps | normal_mode + 7 steps | normal_mode + 5 steps | normal_mode + 1 steps | low_gpu_mode + 0 steps |
| :---: | :---: | :---: | :---: | :---: | :---: | :---: |
| 24 frames | 18971MiB <br> _02:06s_ | 17655MiB <br> _02:40s_ | 17217MiB <br> _02:39s_ | 16925MiB <br> _02:41s_ | 16339MiB <br> _03:13s_ | 5162MiB <br> _13:42s_ |
| 48 frames | 24101MiB <br> _04:52s_ | 21469MiB <br> _05:44s_ | 20592MiB <br> _05:51s_ | 20008MiB <br> _06:00s_ | 18837MiB <br> _06:49s_ | 10292MiB <br> _20:58s_ |
| 72 frames | 29230MiB <br> _08:24s_ | 25283MiB <br> _09:45s_ | 25283MiB <br> _09:45s_ | 23091MiB <br> _10:10s_ | 21335MiB <br> _11:10s_ | 15421MiB <br> _39:12s_ |
| 96 frames | 34360MiB <br> _12:49s_ | 29097MiB <br> _14:41s_ | 27343MiB <br> _15:33s_ | 26174MiB <br> _15:44s_ | 23834MiB <br> _16:33s_ | 20550MiB <br> _43:47s_ |
| 120 frames | 39489MiB <br> _18:21s_ | 32911MiB <br> _20:39s_ | 30719MiB <br> _21:34s_ | 29257MiB <br> _21:48s_ | 26332MiB <br> _23:02s_ | 25679MiB <br> _63:01s_ |
### RTX 4090 的显存占用与运行时间
表格中以 `---` 显示的值表示触发了显存溢出（OOM），无法生成视频。
- Resolution of 512
| 帧数 | normal_mode + 0 steps | normal_mode + 10 steps | normal_mode + 7 steps | normal_mode + 5 steps | normal_mode + 1 steps | low_gpu_mode + 0 steps |
| :---: | :---: | :---: | :---: | :---: | :---: | :---: |
| 24 frames | 16366MiB <br> _01:18s_ | 15805MiB <br> _01:26s_ | 15607MiB <br> _01:37s_ | 15475MiB <br> _01:36s_ | 15211MiB <br> _01:39s_ | 4211MiB <br> _03:57s_ |
| 48 frames | 18720MiB <br> _02:21s_ | 17532MiB <br> _02:49s_ | 17136MiB <br> _02:55s_ | 16872MiB <br> _02:58s_ | 16344MiB <br> _03:01s_ | 4666MiB <br> _05:01s_ |
| 72 frames | 21036MiB <br> _03:41s_ | 19254MiB <br> _04:25s_ | 18660MiB <br> _04:34s_ | 18264MiB <br> _04:36s_ | 17472MiB <br> _04:51s_ | 6981MiB <br> _06:36s_ |
| 96 frames | -----MiB <br> _--:--s_ | 20972MiB <br> _06:18s_ | 20180MiB <br> _06:24s_ | 19652MiB <br> _06:36s_ | 18596MiB <br> _06:56s_ | 9298MiB <br> _10:03s_ |
| 120 frames | -----MiB <br> _--:--s_ | -----MiB <br> _--:--s_ | 21704MiB <br> _08:50s_ | 21044MiB <br> _08:53s_ | 19724MiB <br> _09:08s_ | 11613MiB <br> _13:57s_ |
- Resolution of 768
| 帧数 | normal_mode + 0 steps | normal_mode + 10 steps | normal_mode + 7 steps | normal_mode + 5 steps | normal_mode + 1 steps | low_gpu_mode + 0 steps |
| :---: | :---: | :---: | :---: | :---: | :---: | :---: |
| 24 frames | 19223MiB <br> _02:38s_ | 17900MiB <br> _03:06s_ | 17448MiB <br> _03:18s_ | 17153MiB <br> _03:23s_ | 16624MiB <br> _03:34s_ | 5251MiB <br> _05:54s_ |
| 48 frames | -----MiB <br> _--:--s_ | -----MiB <br> _--:--s_ | 20946MiB <br> _07:28s_ | 20352MiB <br> _07:35s_ | 19164MiB <br> _08:04s_ | 10457MiB <br> _10:55s_ |
| 72 frames | -----MiB <br> _--:--s_ | -----MiB <br> _--:--s_ | -----MiB <br> _--:--s_ | -----MiB <br> _--:--s_ | -----MiB <br> _--:--s_ | 15671MiB <br> _18:52s_ |
## 许可证
我们将以宽松的 **Apache 2.0** 许可证发布该模型。
## 引用
```
@misc{createai2024ruyi,
title={Ruyi-Mini-7B},
author={CreateAI Team},
year={2024},
publisher = {GitHub},
journal = {GitHub repository},
howpublished={\url{https://github.com/IamCreateAI/Ruyi-Models}}
}
```
## 欢迎建议反馈与协同优化
我们真诚欢迎大家积极提供宝贵的反馈和建议，希望能够共同努力优化我们的服务和产品。您的意见将帮助我们更好地理解用户需求，从而不断提升用户体验。感谢您对我们工作的支持和关注！欢迎加入我们的[Discord](https://discord.com/invite/nueQFQwwGw) 或者微信群（扫描下方二维码，添加 Ruyi 小助手，加入官方交流群）！
<img src="https://github.com/user-attachments/assets/cc5e25c6-34ab-4be1-a59b-7d5789264a9c" style="width:300px"></img>
Ruyi-Models/__init__.py 0 → 100644
View file @ 08a21d59
from .comfyui.comfyui_nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
Ruyi-Models/assets/gpu_memory_appendix.md 0 → 100644
View file @ 08a21d59
This diff is collapsed.
Ruyi-Models/comfyui/README.md 0 → 100644
View file @ 08a21d59
## ComfyUI Nodes
After installation, you can find the three nodes provided by this repository in the *Add Node - Ruyi* menu, as shown in the image below:
<div align=center>
<img src="https://github.com/user-attachments/assets/0760399c-c57f-465d-9685-ef910b60421c"></img>
</div>
The following sections will introduce the functions and parameters of each node.
> Note: The new version of ComfyUI nodes is displayed in the NODE LIBRARY on the left side of the interface.
### Load Model
The Load Model node is used to load the model from disk. It also provides the functionality for **automatic model downloading** (auto_download parameter).
<div align=center>
<img src="https://github.com/user-attachments/assets/2e9fcbb5-6f7c-451c-b742-5b844e949a01"></img>
</div>
- **model**: Select which model to use. Currently, Ruyi-Mini-7B is the only option.
- **auto_download**: Whether to automatically download. Defaults to yes. If the model is detected as missing or incomplete, it will automatically download the model to the *ComfyUI/models/Ruyi* path.
- **auto_update**: Whether to automatically check for and update the current model. Defaults to yes. When auto_download is enabled, the system will automatically check for updates to the model and download any updates to the *ComfyUI/models/Ruyi* directory. Please note that this feature relies on the caching mechanism of huggingface_hub, so do not delete the *.cache* folder in the model directory to ensure a smooth update process.
### Load LoRA
The Load LoRA node is used to load LoRA models, which need to be placed in the *ComfyUI/models/loras* path.
<div align=center>
<img src="https://github.com/user-attachments/assets/33f860ca-94c9-4686-afe4-7e00de15b0ce"></img>
</div>
- **lora_name**: The LoRA to be loaded; it will automatically search and display all model files in the *ComfyUI/models/loras* path.
- **strength_model**: The degree of influence of the LoRA, typically set between 1.0 and 1.4 for optimal results based on experience.
### Sampler for Image to Video
The Sampler for Image to Video node is used to generate videos based on input images. The starting frame image (start_img) is a required input, while the ending frame image (end_img) is optional. This node also supports **camera control** (camera_direction parameter) and **motion amplitude control** of the video subject (motion parameter).
<div align=center>
<img src="https://github.com/user-attachments/assets/e5078e95-7100-4e9b-9baa-0d534b085e7a"></img>
</div>
- **start_img**: The starting frame image.
- **end_img**: The ending frame image, optional input.
- **video_length**: The length of the video, which must be divisible by 8, with a maximum of 120 frames.
- **base_resolution**: The video resolution, such as 512, indicates that the generated video will have pixel dimensions close to 512 x 512. The model will automatically select the closest output video aspect ratio based on the input image.
- **seed**: A random number; different random numbers usually generate different videos. If the generated video does not meet requirements, this value can be adjusted to try other generation possibilities.
- **control_after_generate**: The method of changing the random number after each generation.
- **Fixed** indicates the seed is fixed.
- **Increment** indicates the seed is increased by one each time.
- **Decrement** indicates the seed is decreased by one each time.
- **Randomize** indicates the seed is randomly set each time.
- **steps**: The number of iterations for video generation. **More iterations require more time**. Typically, 25 iterations yield good results.
- **cfg**: The guidance of instructions (such as input images). A higher value indicates better guidance, with values between 7 and 10 usually achieving better generation results.
- **motion**: Controls the motion amplitude of the video subject.
- **1** indicates minimal motion, which means the video subject is nearly static.
- **2** indicates normal motion, which could be used in most case.
- **3** indicates significant motion. The video subject is trying to move as much as possible.
- **4** indicates a very large motion. Sometimes the video subject may move out of the camera frame.
- **Auto** indicates the motion is automatically determined by the model.
- **camera_direction**: Camera movement.
- **Static** indicates a stationary camera.
- **Left** indicates the camera moves left.
- **Right** indicates the camera moves right.
- **Up** indicates the camera moves up.
- **Down** indicates the camera moves down.
- **Auto** indicates automatic determination.
- **GPU_memory_mode**: Determines how GPU memory is utilized.
- **normal_mode** is the default mode, using more GPU memory and generating faster.
- **low_memory_mode** is the low memory mode, which significantly reduces GPU memory usage but severely impacts generation speed.
- **GPU_offload_steps**: Used to **optimize GPU memory usage** by moving some temporary variables from GPU memory to RAM, which **increases memory usage and decreases generation speed**.
- **0** indicates no optimization.
- **1 - 10**, where 1 has the least GPU memory usage and the slowest generation speed; 10 has the most GPU memory usage (less than the non-optimized case) and the fastest generation speed.
- Generally, with 24GB of GPU memory, you can use 7 to generate a 512 resolution, 120 frame video. For more detailed data, please refer to the following.
## Workflow Example
This section presents an example workflow for generating videos from images. You can import the workflow using the *Load* button in the (bottom right) menu. Note that the new version of ComfyUI move the menu to the top left, and allows you to load workflows through the *Workflow - Open* option.
The workflows are located in the *[comfyui/workflows/](workflows/)* directory, while the assets can be found in the *[assets/](../assets/)* directory.
After importing the workflow, you **need to manually re-specify the input image for the LoadImage input node**. Since the workflow file can only record the names of input files, it does require manual configuration.
### Image to Video (Starting Frame)
The workflow corresponds to the *[workflow-ruyi-i2v-start-frame.json](workflows/workflow-ruyi-i2v-start-frame.json)* file. For users with larger GPU memory, you can also use *[workflow-ruyi-i2v-start-frame-80g.json](workflows/workflow-ruyi-i2v-start-frame-80g.json)* to enhance the generation speed.
<div align=center>
<img style="width:80%" src="https://github.com/user-attachments/assets/4c0a58b8-ea04-4656-bf1f-b8665a3802a3"></img>
</div>
### Image to Video (Starting and Ending Frames)
The workflow corresponds to the *[workflow-ruyi-i2v-start-end-frames.json](workflows/workflow-ruyi-i2v-start-end-frames.json)* file. For users with larger GPU memory, you can also use *[workflow-ruyi-i2v-start-end-frames-80g.json](workflows/workflow-ruyi-i2v-start-end-frames-80g.json)* to enhance the generation speed.
<div align=center>
<img style="width:80%" src="https://github.com/user-attachments/assets/42b685a4-35ad-4dd8-afa2-79ded3e936cd"></img>
</div>
## Frequently Asked Questions
### Model loading error: LoadModel: ConnectionError (MaxRetryError)
This is usually caused by network issues leading to a failure in downloading from huggingface_hub. If the network is functioning properly, simply rerunning the LoadModel node should resolve the issue.
### Video generation speed is slow and far below expectations
- First, check if the low_memory_mode in the Load Model node is enabled. This mode significantly reduces video generation speed.
- Second, verify the version of PyTorch. PyTorch version 2.2 supports FlashAttention-2 ([link](https://pytorch.org/blog/pytorch2-2/)), which can greatly enhance computational efficiency. Installing the latest version of PyTorch can effectively improve generation speed.
Ruyi-Models/comfyui/README_CN.md 0 → 100644
View file @ 08a21d59
## 节点功能
安装完成后，可在 ComfyUI 的 *Add Node - Ruyi* 菜单找到本仓库提供的 3 个节点，如下图所示：
<div align=center>
<img src="https://github.com/user-attachments/assets/0760399c-c57f-465d-9685-ef910b60421c"></img>
</div>
下文依次介绍各个节点的功能与参数。
> 注：新版 ComfyUI 节点在界面左侧 NODE LIBRARY 中显示。
### Load Model
用于加载模型，并提供**自动下载模型**（通过 auto_download 选项设置）的功能。
<div align=center>
<img src="https://github.com/user-attachments/assets/2e9fcbb5-6f7c-451c-b742-5b844e949a01"></img>
</div>
- **model**: 选择使用哪个模型，目前只有 Ruyi-Mini-7B 一个选项。
- **auto_download**: 是否自动下载，默认为 yes，检测到模型不存在（或不完整）时，将自动下载模型到 *ComfyUI/models/Ruyi* 路径。
- **auto_update**: 是否自动检查并更新当前模型，默认为 yes。当启用 auto_download 时，系统将自动检查模型是否有更新，并将更新内容下载到 *ComfyUI/models/Ruyi* 路径。请注意，此功能依赖于 huggingface_hub 的缓存机制，因此请勿删除模型路径中的 *.cache* 文件夹，以确保更新过程顺利进行。
### Load LoRA
用于加载 LoRA 模型，LoRA 模型需要放在 *ComfyUI/models/loras* 路径下。
<div align=center>
<img src="https://github.com/user-attachments/assets/33f860ca-94c9-4686-afe4-7e00de15b0ce"></img>
</div>
- **lora_name**: 需要加载的 LoRA，将自动搜索并显示 *ComfyUI/models/loras* 路径下所有模型文件。
- **strength_model**: LoRA 的影响程度，根据经验通常设置在 1.0 ~ 1.4 效果较好。
### Sampler for Image to Video
用于根据输入图片生成视频，首帧图片（start_img）为必须输入，尾帧图片（end_img）为可选输入。同时，该节点支持**镜头控制**（camera_direction）与**视频主体的运动幅度控制**（motion）。
<div align=center>
<img src="https://github.com/user-attachments/assets/e5078e95-7100-4e9b-9baa-0d534b085e7a"></img>
</div>
- **start_img**: 首帧图片。
- **end_img**: 尾帧图片，可选输入。
- **video_length**: 视频长度，必须能**被 8 整除**，**最大支持 120 帧**。
- **base_resolution**: 视频分辨率，**模型将根据输入图片的长宽比自动选择最接近的输出视频长宽**。
- **512** 表示生成的视频像素数接近 512 * 512。
- **768** 表示生成的视频像素数接近 768 * 768。
- **seed**: 随机种子，用于控制随机数生成器产生随机数的序列。**不同的随机种子通常能生成不同的视频**，当生成的视频不符合需求时候，可调整此值以尝试其他的生成可能。
- **control_after_generate**: 每次生成后随机种子的变化方式。
- **Fixed** 表示随机种子固定不变。
- **Increment** 表示随机种子每次增加一。
- **Decrement** 表示随机种子每次减少一。
- **Randomize** 表示随机种子运行后随机设置。
- **steps**: 视频生成的迭代次数，**迭代次数越多，需要的时间越久**，通常 25 次能够得到较好的结果。
- **cfg**: 指令（如输入图片）的遵循程度，数值越大遵循程度越好，取值 7 ～ 10 通常能取得较好的生成效果。
- **motion**: 控制视频主体的运动幅度。
- **1** 基本不运动，适用于静态场景。
- **2** 正常运动幅度，适用于谈话、转头等常见场合。
- **3** 运动幅度较大，可能出现转身、走动等情况。
- **4** 运动幅度非常大，可能出现视频主体离开画面的情况。
- **Auto** 表示模型自动判断运动幅度大小。
- **camera_direction**: 镜头运动。
- **Static** 表示静止镜头。
- **Left** 表示镜头向左移动。
- **Right** 表示镜头向右移动。
- **Up** 表示镜头向上移动。
- **Down** 表示镜头向下移动。
- **Auto** 表示模型自动判断镜头运动方向。
- **GPU_memory_mode**:
- **normal_mode** 是默认模式，**使用显存较多，生成速度较快**。
- **low_memory_mode** 是低显存模式，**能大幅降低显存用量，但严重影响生成速度**。
- **GPU_offload_steps**: 用于**优化显存占用**，通过将部分临时变量从显存移动到内存而实现，会**增加内存的占用并降低生成速度**。
- **0** 表示不优化。
- **1 - 10**，1 显存占用最小，生成速度最慢；10 显存占用最多（少于不优化情况），生成速度最快。
- 通常情况下，24G 显存可以使用 7 生成 512 分辨率，120 帧视频。更详细数据请参考下文。
## 工作流样例
本节展示了图生视频的工作流样例，可通过菜单中的 *Load* 按钮导入工作流。新版 ComfyUI 可通过左上方菜单的 *Workflow - Open* 加载工作流。
工作流位于 *[comfyui/workflows/](workflows/)* 目录中，素材位于 *[assets/](../assets/)* 目录中。
导入工作流后，**需要手动重新指定输入节点 LoadImage 的输入图片**，由于工作流文件只能记录输入文件的名字，所以目前只能手动设置。
### 图生视频（首帧）
工作流对应 *[workflow-ruyi-i2v-start-frame.json](workflows/workflow-ruyi-i2v-start-frame.json)* 文件。对于显存较大的用户，也可以使用 *[workflow-ruyi-i2v-start-frame-80g.json](workflows/workflow-ruyi-i2v-start-frame-80g.json)* 以提高生成速度。
<div align=center>
<img style="width:80%" src="https://github.com/user-attachments/assets/4c0a58b8-ea04-4656-bf1f-b8665a3802a3"></img>
</div>
### 图生视频（首尾帧）
工作流对应 *[workflow-ruyi-i2v-start-end-frames.json](workflows/workflow-ruyi-i2v-start-end-frames.json)* 文件。对于显存较大的用户，也可以使用 *[workflow-ruyi-i2v-start-end-frames-80g.json](workflows/workflow-ruyi-i2v-start-end-frames-80g.json)* 以提高生成速度。
<div align=center>
<img style="width:80%" src="https://github.com/user-attachments/assets/42b685a4-35ad-4dd8-afa2-79ded3e936cd"></img>
</div>
## 常见问题
### 模型加载错误，LoadModel: ConnectionError (MaxRetryError)
通常是网络问题导致 huggingface_hub 下载失败。网络正常的情况下，再次运行 LoadModel 节点即可解决。
### 视频生成速度慢、远低于预期
- 首先，请检查是否开启 Load Model 节点中 GPU_memory_mode 的 low_memory_mode，此模式会大幅降低视频生成速度。
- 其次，请检查 PyTorch 版本。PyTorch 在 2.2 版本支持了 FlashAttention-2（[链接](https://pytorch.org/blog/pytorch2-2/)），能大幅提升计算效率。安装新版本的 PyTorch 能够有效提升生成速度。
Ruyi-Models/comfyui/comfyui_nodes.py 0 → 100644
View file @ 08a21d59
"""Modified from https://github.com/kijai/ComfyUI-EasyAnimateWrapper/blob/main/nodes.py
"""
import gc
import os
import comfy.model_management as mm
import cv2
import folder_paths
import numpy as np
import torch
from comfy.utils import ProgressBar
from diffusers import (DDIMScheduler, DPMSolverMultistepScheduler, PNDMScheduler,
EulerAncestralDiscreteScheduler, EulerDiscreteScheduler)
from einops import rearrange
from omegaconf import OmegaConf
from PIL import Image
from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection
from safetensors.torch import load_file as load_safetensors
from huggingface_hub import snapshot_download
from ..ruyi.data.bucket_sampler import ASPECT_RATIO_512, get_closest_ratio
from ..ruyi.models.autoencoder_magvit import AutoencoderKLMagvit
from ..ruyi.models.transformer3d import HunyuanTransformer3DModel
from ..ruyi.pipeline.pipeline_ruyi_inpaint import RuyiInpaintPipeline
from ..ruyi.utils.lora_utils import merge_lora, unmerge_lora
from ..ruyi.utils.utils import get_image_to_video_latent
# The directory of scripts
script_directory = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
class Ruyi_LoadModel:
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"model": (
[
"Ruyi-Mini-7B",
],
{ "default": "Ruyi-Mini-7B", }
),
"auto_download": (
["yes", "no"],
{
"default": "yes"
}
),
"auto_update": (
["yes", "no"],
{
"default": "yes"
}
)
},
}
RETURN_TYPES = ("RUYI_MODEL",)
RETURN_NAMES = ("ruyi_model",)
FUNCTION = "load_model"
CATEGORY = "Ruyi"
RUYI_MODEL_TYPE_DICT = {
"Ruyi-Mini-7B": "Inpaint"
}
def get_model_type(self, model_name):
return self.RUYI_MODEL_TYPE_DICT.get(model_name, "Inpaint")
def try_setup_pipeline(self, model_path, weight_dtype, config):
try:
# Init processbar
pbar = ProgressBar(5)
# Get Vae
vae = AutoencoderKLMagvit.from_pretrained(
model_path,
subfolder="vae"
).to(weight_dtype)
# Update pbar
pbar.update(1)
# Get Transformer
transformer_additional_kwargs = OmegaConf.to_container(config['transformer_additional_kwargs'])
transformer = HunyuanTransformer3DModel.from_pretrained_2d(
model_path,
subfolder="transformer",
transformer_additional_kwargs=transformer_additional_kwargs
).to(weight_dtype)
# Update pbar
pbar.update(1)
# Load Clip
clip_image_encoder = CLIPVisionModelWithProjection.from_pretrained(
model_path, subfolder="image_encoder"
).to(weight_dtype)
clip_image_processor = CLIPImageProcessor.from_pretrained(
model_path, subfolder="image_encoder"
)
# Update pbar
pbar.update(1)
# Load sampler and create pipeline
Choosen_Scheduler = DDIMScheduler
scheduler = Choosen_Scheduler.from_pretrained(
model_path,
subfolder="scheduler"
)
pipeline = RuyiInpaintPipeline.from_pretrained(
model_path,
vae=vae,
transformer=transformer,
scheduler=scheduler,
torch_dtype=weight_dtype,
clip_image_encoder=clip_image_encoder,
clip_image_processor=clip_image_processor,
)
# Load embeddings
embeddings = load_safetensors(os.path.join(model_path, "embeddings.safetensors"))
pipeline.embeddings = embeddings
# Update pbar
pbar.update(1)
return pipeline
except Exception as e:
print("[Ruyi] Setup pipeline failed:", e)
return None
def load_model(self, model, auto_download, auto_update):
# Init weight_dtype and device
device = mm.get_torch_device()
offload_device = mm.unet_offload_device()
# Init model name and type
model_name = model
model_type = self.get_model_type(model_name)
weight_dtype = torch.bfloat16
# Load config
config_path = os.path.join(script_directory, "config", "default.yaml")
config = OmegaConf.load(config_path)
# Check for update
repo_id = f"IamCreateAI/{model_name}"
model_path = os.path.join(folder_paths.models_dir, "Ruyi", model_name)
if auto_download == "yes" and auto_update == "yes":
print(f"Checking for {model_name} updates ...")
# Download the model
snapshot_download(repo_id=repo_id, local_dir=model_path)
# Init model
pipeline = self.try_setup_pipeline(model_path, weight_dtype, config)
if pipeline is None and auto_download == "yes":
mm.soft_empty_cache()
gc.collect()
# Download the model
snapshot_download(repo_id=repo_id, local_dir=model_path)
pipeline = self.try_setup_pipeline(model_path, weight_dtype, config)
if pipeline is None:
message = (f"[ Load Model {model_name} Failed ]\n"
f"Please download Ruyi model from huggingface repo '{repo_id}',\n"
f"And put it into '{model_path}'.")
if auto_download == "no":
message += "\n\nOr just set auto_download to 'yes'."
raise FileNotFoundError(message)
pipeline.enable_model_cpu_offload()
ruyi_model = {
'pipeline': pipeline,
'dtype': weight_dtype,
'model_path': model_path,
'model_type': model_type,
'loras': [],
'strength_model': [],
}
return (ruyi_model,)
class Ruyi_LoadLora:
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"ruyi_model": ("RUYI_MODEL",),
"lora_name": (folder_paths.get_filename_list("loras"), {"default": None,}),
"strength_model": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01}),
}
}
RETURN_TYPES = ("RUYI_MODEL",)
RETURN_NAMES = ("ruyi_model",)
FUNCTION = "load_lora"
CATEGORY = "Ruyi"
def load_lora(self, ruyi_model, lora_name, strength_model):
if lora_name is not None:
return (
{
'pipeline': ruyi_model["pipeline"],
'dtype': ruyi_model["dtype"],
'model_path': ruyi_model["model_path"],
'model_type': ruyi_model["model_type"],
'loras': ruyi_model.get("loras", []) + [folder_paths.get_full_path("loras", lora_name)],
'strength_model': ruyi_model.get("strength_model", []) + [strength_model],
},
)
else:
return (ruyi_model,)
class Ruyi_I2VSampler:
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"ruyi_model": (
"RUYI_MODEL",
),
"video_length": (
"INT", {"default": 72, "min": 8, "max": 120, "step": 8}
),
"base_resolution": (
"INT", {"default": 512, "min": 384, "max": 1024, "step": 16}
),
"seed": (
"INT", {"default": 42, "min": 0, "max": 0xffffffffffffffff}
),
"steps": (
"INT", {"default": 25, "min": 1, "max": 200, "step": 1}
),
"cfg": (
"FLOAT", {"default": 7.0, "min": 1.0, "max": 20.0, "step": 0.01}
),
"scheduler": (
[
"Euler",
"Euler A",
"DPM++",
"PNDM",
"DDIM",
],
{
"default": 'DDIM'
}
),
"motion": (
[ "1", "2", "3", "4", "auto" ],
{ "default": "2" }
),
"camera_direction": (
[ "static", "left", "right", "up", "down", "auto" ],
{ "default": "static" }
),
"GPU_memory_mode": (
["normal_mode", "low_memory_mode"],
{
"default": "normal_mode",
}
),
"GPU_offload_steps": (
[ "0", "1", "5", "7", "10" ],
{ "default": "0" }
),
"start_img": (
"IMAGE",
),
},
"optional":{
"end_img": ("IMAGE",),
},
}
RETURN_TYPES = ("IMAGE",)
RETURN_NAMES =("images",)
FUNCTION = "process"
CATEGORY = "Ruyi"
def tensor2pil(self, image):
return Image.fromarray(np.clip(255. * image.cpu().numpy(), 0, 255).astype(np.uint8))
def numpy2pil(self, image):
return Image.fromarray(np.clip(255. * image, 0, 255).astype(np.uint8))
def to_pil(self, image):
if isinstance(image, Image.Image):
return image
if isinstance(image, torch.Tensor):
return self.tensor2pil(image)
if isinstance(image, np.ndarray):
return self.numpy2pil(image)
raise ValueError(f"Cannot convert {type(image)} to PIL.Image")
def get_control_embeddings(self, pipeline, aspect_ratio, motion, camera_direction):
# Default keys
p_default_key = "p.default"
n_default_key = "n.default"
# Load embeddings
if motion == "auto":
motion = "0"
p_key = f"p.{aspect_ratio.replace(':', 'x')}movie{motion}{camera_direction}"
embeddings = pipeline.embeddings
# Get embeddings
positive_embeds = embeddings.get(f"{p_key}.emb1", embeddings[f"{p_default_key}.emb1"])
positive_attention_mask = embeddings.get(f"{p_key}.mask1", embeddings[f"{p_default_key}.mask1"])
positive_embeds_2 = embeddings.get(f"{p_key}.emb2", embeddings[f"{p_default_key}.emb2"])
positive_attention_mask_2 = embeddings.get(f"{p_key}.mask2", embeddings[f"{p_default_key}.mask2"])
negative_embeds = embeddings[f"{n_default_key}.emb1"]
negative_attention_mask = embeddings[f"{n_default_key}.mask1"]
negative_embeds_2 = embeddings[f"{n_default_key}.emb2"]
negative_attention_mask_2 = embeddings[f"{n_default_key}.mask2"]
return {
"positive_embeds": positive_embeds,
"positive_attention_mask": positive_attention_mask,
"positive_embeds_2": positive_embeds_2,
"positive_attention_mask_2": positive_attention_mask_2,
"negative_embeds": negative_embeds,
"negative_attention_mask": negative_attention_mask,
"negative_embeds_2": negative_embeds_2,
"negative_attention_mask_2": negative_attention_mask_2,
}
def get_scheduler(self, model_path, scheduler_name):
if scheduler_name == "DPM++":
noise_scheduler = DPMSolverMultistepScheduler.from_pretrained(model_path, subfolder='scheduler')
elif scheduler_name == "Euler":
noise_scheduler = EulerDiscreteScheduler.from_pretrained(model_path, subfolder='scheduler')
elif scheduler_name == "Euler A":
noise_scheduler = EulerAncestralDiscreteScheduler.from_pretrained(model_path, subfolder='scheduler')
elif scheduler_name == "PNDM":
noise_scheduler = PNDMScheduler.from_pretrained(model_path, subfolder='scheduler')
elif scheduler_name == "DDIM":
noise_scheduler = DDIMScheduler.from_pretrained(model_path, subfolder='scheduler')
return noise_scheduler
def process(
self, ruyi_model, video_length, base_resolution, seed, steps, cfg, scheduler,
motion, camera_direction, GPU_memory_mode, GPU_offload_steps,
start_img, end_img=None
):
device = mm.get_torch_device()
offload_device = mm.unet_offload_device()
mm.soft_empty_cache()
gc.collect()
start_img = [self.to_pil(_start_img) for _start_img in start_img] if start_img is not None else None
end_img = [self.to_pil(_end_img) for _end_img in end_img] if end_img is not None else None
# Count most suitable height and width
aspect_ratio_sample_size = {key : [x / 512 * base_resolution for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()}
original_width, original_height = start_img[0].size if type(start_img) is list else Image.open(start_img).size
closest_size, closest_ratio = get_closest_ratio(original_height, original_width, ratios=aspect_ratio_sample_size)
height, width = [int(x / 16) * 16 for x in closest_size]
aspect_ratio = "16:9" if width > height else "9:16"
# Get Pipeline
pipeline = ruyi_model['pipeline']
model_path = ruyi_model['model_path']
# Set GPU memory mode
if GPU_memory_mode == "low_memory_mode" and pipeline.model_cpu_offload_flag:
# Switch to low_memory_mode
pipeline.enable_sequential_cpu_offload()
elif GPU_memory_mode == "normal_mode" and not pipeline.model_cpu_offload_flag:
# Switch to normal_mode
pipeline.enable_model_cpu_offload()
# Set GPU offload steps
pipeline.transformer.hidden_cache_size = int(GPU_offload_steps)
# Load Sampler
pipeline.scheduler = self.get_scheduler(model_path, scheduler)
# Set random seed
generator= torch.Generator(device).manual_seed(seed)
# Load control embeddings
embeddings = self.get_control_embeddings(pipeline, aspect_ratio, motion, camera_direction)
# Inference
with torch.no_grad():
video_length = int(video_length // pipeline.vae.mini_batch_encoder * pipeline.vae.mini_batch_encoder) if video_length != 1 else 1
input_video, input_video_mask, clip_image = get_image_to_video_latent(start_img, end_img, video_length=video_length, sample_size=(height, width))
for _lora_path, _lora_weight in zip(ruyi_model.get("loras", []), ruyi_model.get("strength_model", [])):
pipeline = merge_lora(pipeline, _lora_path, _lora_weight)
sample = pipeline(
prompt_embeds = embeddings["positive_embeds"],
prompt_attention_mask = embeddings["positive_attention_mask"],
prompt_embeds_2 = embeddings["positive_embeds_2"],
prompt_attention_mask_2 = embeddings["positive_attention_mask_2"],
negative_prompt_embeds = embeddings["negative_embeds"],
negative_prompt_attention_mask = embeddings["negative_attention_mask"],
negative_prompt_embeds_2 = embeddings["negative_embeds_2"],
negative_prompt_attention_mask_2 = embeddings["negative_attention_mask_2"],
video_length = video_length,
height = height,
width = width,
generator = generator,
guidance_scale = cfg,
num_inference_steps = steps,
video = input_video,
mask_video = input_video_mask,
clip_image = clip_image,
comfyui_progressbar = True,
).videos
videos = rearrange(sample, "b c t h w -> (b t) h w c")
for _lora_path, _lora_weight in zip(ruyi_model.get("loras", []), ruyi_model.get("strength_model", [])):
pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight)
return (videos,)
NODE_CLASS_MAPPINGS = {
"Ruyi_LoadModel": Ruyi_LoadModel,
"Ruyi_LoadLora": Ruyi_LoadLora,
"Ruyi_I2VSampler": Ruyi_I2VSampler,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"Ruyi_LoadModel": "Load Model",
"Ruyi_LoadLora": "Load LoRA",
"Ruyi_I2VSampler": "Sampler for Image to Video",
}
Ruyi-Models/comfyui/workflows/workflow-ruyi-i2v-start-end-frames-80g.json 0 → 100644
View file @ 08a21d59
{
"last_node_id": 5,
"last_link_id": 4,
"nodes": [
{
"id": 1,
"type": "Ruyi_LoadModel",
"pos": {
"0": 295,
"1": 238
},
"size": {
"0": 315,
"1": 82
},
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "ruyi_model",
"type": "RUYI_MODEL",
"links": [
2
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "Ruyi_LoadModel"
},
"widgets_values": [
"Ruyi-Mini-7B",
"yes",
"yes"
]
},
{
"id": 2,
"type": "Ruyi_I2VSampler",
"pos": {
"0": 753,
"1": 326
},
"size": {
"0": 327.5999755859375,
"1": 338
},
"flags": {},
"order": 3,
"mode": 0,
"inputs": [
{
"name": "ruyi_model",
"type": "RUYI_MODEL",
"link": 2
},
{
"name": "start_img",
"type": "IMAGE",
"link": 3
},
{
"name": "end_img",
"type": "IMAGE",
"link": 4,
"shape": 7
}
],
"outputs": [
{
"name": "images",
"type": "IMAGE",
"links": [
1
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "Ruyi_I2VSampler"
},
"widgets_values": [
120,
512,
112876945129710,
"randomize",
25,
7,
"DDIM",
"auto",
"auto",
"normal_mode",
"0"
]
},
{
"id": 5,
"type": "VHS_VideoCombine",
"pos": {
"0": 1183,
"1": 89
},
"size": [
317,
758.0769230769231
],
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 1
},
{
"name": "audio",
"type": "AUDIO",
"link": null,
"shape": 7
},
{
"name": "meta_batch",
"type": "VHS_BatchManager",
"link": null,
"shape": 7
},
{
"name": "vae",
"type": "VAE",
"link": null,
"shape": 7
}
],
"outputs": [
{
"name": "Filenames",
"type": "VHS_FILENAMES",
"links": null
}
],
"properties": {
"Node name for S&R": "VHS_VideoCombine"
},
"widgets_values": {
"frame_rate": 24,
"loop_count": 0,
"filename_prefix": "Ruyi-I2V-StartEndFrames",
"format": "video/h264-mp4",
"pix_fmt": "yuv420p",
"crf": 19,
"save_metadata": true,
"pingpong": false,
"save_output": true,
"videopreview": {
"hidden": false,
"paused": false,
"params": {
"filename": "Ruyi-I2V-StartEndFrames_00001.mp4",
"subfolder": "",
"type": "output",
"format": "video/h264-mp4",
"frame_rate": 24
},
"muted": false
}
}
},
{
"id": 3,
"type": "LoadImage",
"pos": {
"0": 180,
"1": 398
},
"size": {
"0": 240.88999938964844,
"1": 419.6499938964844
},
"flags": {},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
3
],
"slot_index": 0
},
{
"name": "MASK",
"type": "MASK",
"links": null
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"girl_01.jpg",
"image"
]
},
{
"id": 4,
"type": "LoadImage",
"pos": {
"0": 453,
"1": 398
},
"size": [
242.0999999999999,
421.6500000000001
],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
4
],
"slot_index": 0
},
{
"name": "MASK",
"type": "MASK",
"links": null
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"girl_02.jpg",
"image"
]
}
],
"links": [
[
1,
2,
0,
5,
0,
"IMAGE"
],
[
2,
1,
0,
2,
0,
"RUYI_MODEL"
],
[
3,
3,
0,
2,
1,
"IMAGE"
],
[
4,
4,
0,
2,
2,
"IMAGE"
]
],
"groups": [],
"config": {},
"extra": {
"ds": {
"scale": 1,
"offset": [
-43.00999999999996,
33.75000000000006
]
}
},
"version": 0.4
}
\ No newline at end of file
Ruyi-Models/comfyui/workflows/workflow-ruyi-i2v-start-end-frames.json 0 → 100644
View file @ 08a21d59
{
"last_node_id": 5,
"last_link_id": 4,
"nodes": [
{
"id": 1,
"type": "Ruyi_LoadModel",
"pos": {
"0": 295,
"1": 238
},
"size": {
"0": 315,
"1": 82
},
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "ruyi_model",
"type": "RUYI_MODEL",
"links": [
2
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "Ruyi_LoadModel"
},
"widgets_values": [
"Ruyi-Mini-7B",
"yes",
"yes"
]
},
{
"id": 5,
"type": "VHS_VideoCombine",
"pos": {
"0": 1183,
"1": 89
},
"size": [
317,
758.0769230769231
],
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 1
},
{
"name": "audio",
"type": "AUDIO",
"link": null,
"shape": 7
},
{
"name": "meta_batch",
"type": "VHS_BatchManager",
"link": null,
"shape": 7
},
{
"name": "vae",
"type": "VAE",
"link": null,
"shape": 7
}
],
"outputs": [
{
"name": "Filenames",
"type": "VHS_FILENAMES",
"links": null
}
],
"properties": {
"Node name for S&R": "VHS_VideoCombine"
},
"widgets_values": {
"frame_rate": 24,
"loop_count": 0,
"filename_prefix": "Ruyi-I2V-StartEndFrames",
"format": "video/h264-mp4",
"pix_fmt": "yuv420p",
"crf": 19,
"save_metadata": true,
"pingpong": false,
"save_output": true,
"videopreview": {
"hidden": false,
"paused": false,
"params": {
"filename": "Ruyi-I2V-StartEndFrames_00001.mp4",
"subfolder": "",
"type": "output",
"format": "video/h264-mp4",
"frame_rate": 24
},
"muted": false
}
}
},
{
"id": 3,
"type": "LoadImage",
"pos": {
"0": 180,
"1": 398
},
"size": {
"0": 240.88999938964844,
"1": 419.6499938964844
},
"flags": {},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
3
],
"slot_index": 0
},
{
"name": "MASK",
"type": "MASK",
"links": null
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"gir_01.jpg",
"image"
]
},
{
"id": 4,
"type": "LoadImage",
"pos": {
"0": 453,
"1": 398
},
"size": [
242.0999999999999,
421.6500000000001
],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
4
],
"slot_index": 0
},
{
"name": "MASK",
"type": "MASK",
"links": null
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"girl_02.jpg",
"image"
]
},
{
"id": 2,
"type": "Ruyi_I2VSampler",
"pos": {
"0": 753,
"1": 326
},
"size": {
"0": 327.5999755859375,
"1": 338
},
"flags": {},
"order": 3,
"mode": 0,
"inputs": [
{
"name": "ruyi_model",
"type": "RUYI_MODEL",
"link": 2
},
{
"name": "start_img",
"type": "IMAGE",
"link": 3
},
{
"name": "end_img",
"type": "IMAGE",
"link": 4,
"shape": 7
}
],
"outputs": [
{
"name": "images",
"type": "IMAGE",
"links": [
1
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "Ruyi_I2VSampler"
},
"widgets_values": [
120,
512,
112876945129710,
"randomize",
25,
7,
"DDIM",
"auto",
"auto",
"normal_mode",
"5"
]
}
],
"links": [
[
1,
2,
0,
5,
0,
"IMAGE"
],
[
2,
1,
0,
2,
0,
"RUYI_MODEL"
],
[
3,
3,
0,
2,
1,
"IMAGE"
],
[
4,
4,
0,
2,
2,
"IMAGE"
]
],
"groups": [],
"config": {},
"extra": {
"ds": {
"scale": 1,
"offset": [
-43.00999999999996,
33.75000000000006
]
}
},
"version": 0.4
}
\ No newline at end of file
Ruyi-Models/comfyui/workflows/workflow-ruyi-i2v-start-frame-80g.json 0 → 100644
View file @ 08a21d59
{
"last_node_id": 6,
"last_link_id": 3,
"nodes": [
{
"id": 2,
"type": "Ruyi_I2VSampler",
"pos": {
"0": 628,
"1": 284
},
"size": {
"0": 327.5999755859375,
"1": 338
},
"flags": {},
"order": 2,
"mode": 0,
"inputs": [
{
"name": "ruyi_model",
"type": "RUYI_MODEL",
"link": 1
},
{
"name": "start_img",
"type": "IMAGE",
"link": 2
},
{
"name": "end_img",
"type": "IMAGE",
"link": null,
"shape": 7
}
],
"outputs": [
{
"name": "images",
"type": "IMAGE",
"links": [
3
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "Ruyi_I2VSampler"
},
"widgets_values": [
120,
512,
925247271358454,
"randomize",
25,
7,
"DDIM",
"2",
"static",
"normal_mode",
"0"
]
},
{
"id": 1,
"type": "Ruyi_LoadModel",
"pos": {
"0": 210,
"1": 162
},
"size": {
"0": 315,
"1": 82
},
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "ruyi_model",
"type": "RUYI_MODEL",
"links": [
1
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "Ruyi_LoadModel"
},
"widgets_values": [
"Ruyi-Mini-7B",
"yes",
"yes"
]
},
{
"id": 4,
"type": "VHS_VideoCombine",
"pos": {
"0": 1045,
"1": 133
},
"size": [
404.73553466796875,
601.8645528157551
],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [
{
"name": "images",
"type": "IMAGE",
"link": 3
},
{
"name": "audio",
"type": "AUDIO",
"link": null,
"shape": 7
},
{
"name": "meta_batch",
"type": "VHS_BatchManager",
"link": null,
"shape": 7
},
{
"name": "vae",
"type": "VAE",
"link": null,
"shape": 7
}
],
"outputs": [
{
"name": "Filenames",
"type": "VHS_FILENAMES",
"links": null
}
],
"properties": {
"Node name for S&R": "VHS_VideoCombine"
},
"widgets_values": {
"frame_rate": 24,
"loop_count": 0,
"filename_prefix": "Ruyi-I2V-StartFrame",
"format": "video/h264-mp4",
"pix_fmt": "yuv420p",
"crf": 19,
"save_metadata": true,
"pingpong": false,
"save_output": true,
"videopreview": {
"hidden": false,
"paused": false,
"params": {
"filename": "Ruyi-I2V-StartFrame_00001.mp4",
"subfolder": "",
"type": "output",
"format": "video/h264-mp4",
"frame_rate": 24
},
"muted": false
}
}
},
{
"id": 3,
"type": "LoadImage",
"pos": {
"0": 200,
"1": 439
},
"size": {
"0": 315,
"1": 314
},
"flags": {},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "IMAGE",
"type": "IMAGE",
"links": [
2
],
"slot_index": 0
},
{
"name": "MASK",
"type": "MASK",
"links": null
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"example_03.jpg",
"image"
]
}
],
"links": [
[
1,
1,
0,
2,
0,
"RUYI_MODEL"
],
[
2,
3,
0,
2,
1,
"IMAGE"
],
[
3,
2,
0,
4,
0,
"IMAGE"
]
],
"groups": [],
"config": {},
"extra": {
"ds": {
"scale": 1,
"offset": [
0,
0
]
}
},
"version": 0.4
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment