v1.0

facechain_demo.ipynb

+{
+ "nbformat": 4,
+ "nbformat_minor": 0,
+ "metadata": {
+  "colab": {
+   "private_outputs": true,
+   "provenance": [],
+   "machine_shape": "hm",
+   "name": "facechain-demo.ipynb"
+  },
+  "kernelspec": {
+   "name": "python3",
+   "display_name": "Python 3"
+  }
+ },
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "source": [
+    "Requirements:\n",
+    "- GPU Mem Usage: 19G\n",
+    "- Disk Usage: About 50G"
+   ],
+   "metadata": {
+    "id": "AzOycrSUkK-0"
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "## Get facechain source code from GitHub"
+   ],
+   "metadata": {
+    "id": "8yQEWTkekIx9"
+   }
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "!GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/modelscope/facechain.git --depth 1"
+   ],
+   "metadata": {
+    "id": "WK_uDw0NkHTP"
+   },
+   "execution_count": null,
+   "outputs": []
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "## Installation requirements\n",
+    "- Note that you can ignore warning on dependencies conflicts at the end\n",
+    "- You may use conda virtual env to avoid warning info."
+   ],
+   "metadata": {
+    "id": "jUWWSGDG1ZGK"
+   }
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "!pip3 install -r facechain/requirements.txt"
+   ],
+   "metadata": {
+    "id": "QrwRDBipxKZw"
+   },
+   "execution_count": null,
+   "outputs": []
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "## Check environment informaiton"
+   ],
+   "metadata": {
+    "id": "0miaB1nC2VrA"
+   }
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "!nvidia-smi\n",
+    "!pip3 show torch\n",
+    "\n",
+    "# Note that this setup is verified on (cuda 12.0, torch2.0.1+cu118, Nvidia-A100 40G)"
+   ],
+   "metadata": {
+    "id": "uvNE0kRxyI4q"
+   },
+   "execution_count": null,
+   "outputs": []
+  },
+  {
+   "cell_type": "markdown",
+   "source": [],
+   "metadata": {
+    "id": "klba298h2bbu"
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "## Installing mmcv-full\n",
+    "- Several of the underlying models depend on mmcv-full, which could be tricky to install since it is environment-dependent, you may refer to [mmcv's official documentation](https://mmcv.readthedocs.io/zh_CN/latest/get_started/installation.html) for more details..\n",
+    "- A prebuilt package is provided here for convenience, and was verified on (cuda 12.0, torch2.0.1+cu118, Nvidia-A100 40G)\n",
+    "- If the prebuilt package does not work on your env setup, please use the alternative installation as suggested by [mmcv's official documentation](https://mmcv.readthedocs.io/zh_CN/latest/get_started/)"
+   ],
+   "metadata": {
+    "id": "ciSxxSzI4l_u"
+   }
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "# use prebuilt mmcv-full package provided by ModelScope (verified on cuda 12.0, torch2.0.1+cu118, Nvidia-A100 40G)\n",
+    "!pip3 install https://modelscope.oss-cn-beijing.aliyuncs.com/packages/mmcv/mmcv_full-1.7.0-cp310-cp310-linux_x86_64.whl\n",
+    "\n",
+    "# alternative manual installtation. note that it may take 30+ mins for building dependencies.\n",
+    "# !pip3 install -U openmim\n",
+    "# !mim install mmcv-full==1.7.0\n"
+   ],
+   "metadata": {
+    "id": "t2lciBXyyuzA"
+   },
+   "execution_count": null,
+   "outputs": []
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "## Pre-download models to mitigate connection timeout issue"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "os.chdir('/content/facechain') # Note: replace with your facechain root\n",
+    "print(os.getcwd())\n",
+    "\n",
+    "# Not required\n",
+    "from facechain.utils import pre_download_models\n",
+    "pre_download_models()"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "## Start facechain WebUI service"
+   ],
+   "metadata": {
+    "id": "OxDgRY8H4vta"
+   }
+  },
+  {
+   "cell_type": "code",
+   "source": [
+    "# Install service dependencies\n",
+    "!pip3 install gradio==3.50.2\n",
+    "!pip3 install controlnet_aux==0.0.6\n",
+    "!pip3 install python-slugify\n",
+    "!pip3 install onnxruntime==1.15.1\n",
+    "!pip3 install edge-tts\n",
+    "\n",
+    "# Click on the gradio URL to start building your FaceChain digital-twin!\n",
+    "!python3 app.py"
+   ],
+   "metadata": {
+    "id": "NW5o0iiezTCg"
+   },
+   "execution_count": null,
+   "outputs": []
+  }
+ ]
+}

install.py

+import launch
+import os
+
+# TODO: add pip dependency if need extra module only on extension
+
+if not launch.is_installed("slugify"):
+    print("--installing slugify...")
+    launch.run_pip("install slugify", "requirements for slugify")
+    launch.run_pip("install python-slugify==8.0.1", "requirements for python-slugify")
+
+if not launch.is_installed("diffusers"):
+    print("--installing diffusers...")
+    launch.run_pip("install diffusers", "requirements for diffusers")
+
+if not launch.is_installed("onnxruntime") and not launch.is_installed("onnxruntime-gpu"):
+    import torch.cuda as cuda
+    print("Installing onnxruntime")
+    launch.run_pip("install onnxruntime-gpu" if cuda.is_available() else "install onnxruntime")
+
+if not launch.is_installed("modelscope"):
+    print("--installing modelscope...")
+    launch.run_pip("install modelscope", "requirements for modelscope")
+
+if not launch.is_installed("controlnet_aux"):
+    print("--installing controlnet_aux...")
+    launch.run_pip("install controlnet_aux==0.0.6", "requirements for controlnet_aux")
+
+if not launch.is_installed("mmcv"):
+    print("--installing mmcv...")
+    # Todo 这里有坑
+    try:
+        launch.run_pip("install mmcv-full==1.7.0", "requirements for mmcv")
+    except Exception as e:
+        print(e)
+        if os.name == 'nt':  # Windows
+            print('ERROR facechain: failed to install mmcv, make sure to have "CUDA Toolkit" and "Build Tools for Visual Studio" installed')
+        else:
+            print('ERROR facechain: failed to install mmcv')
+
+if not launch.is_installed("mmdet"):
+    print("--installing mmdet...")
+    launch.run_pip("install mmdet==2.26.0", "requirements for mmdet")
+
+if not launch.is_installed("mediapipe"):
+    print("--installing mmdet...")
+    launch.run_pip("install mediapipe==0.10.3", "requirements for mediapipe")
+
+if not launch.is_installed("edge_tts"):
+    print("--installing edge_tts...")
+    launch.run_pip("install edge_tts", "requirements for mediapipe")
+
+if not launch.is_installed("cv2"):
+    launch.run_pip("install opencv-python", "requirements for opencv")
+
+if not launch.is_installed("diffusers"):
+    launch.run_pip("install diffusers", "requirements for diffusers")
+
+if not launch.is_installed("protobuf==3.20.1"):
+    launch.run_pip("install protobuf==3.20.1", "requirements for diffusers")
+
+# there seems to be a bug in fsspec 2023.10.0 that triggers an Error during training
+# NotImplementedError: Loading a dataset cached in a LocalFileSystem is not supported.
+# currently webui by default will install 2023.10.0
+# Todo remove fsspec version change after issue is resolved, please monitor situation, it's possible in the future that webui might specify a specific version of fsspec
+import pkg_resources
+required_fsspec_version = '2023.9.2'
+try:
+    fsspec_version = pkg_resources.get_distribution('fsspec').version
+    if fsspec_version != required_fsspec_version:
+        print("--installing fsspec...")
+        launch.run_pip(f"install -U fsspec=={required_fsspec_version}", f"facechain changing fsspec version from {fsspec_version} to {required_fsspec_version}")
+except Exception:
+    # pkg_resources.get_distribution will throw if fsspec installed, since webui install by default fsspec this section shouldn't be necessary
+    print("--installing fsspec...")
+    launch.run_pip(f"install -U fsspec=={required_fsspec_version}", f"requirements for facechain")

model.properties

+# 模型编码
+modelCode=510
+# 模型名称
+modelName=facechain_pytorch
+# 模型描述
+modelDescription=足不出户拍摄个人写真大片!一种数字形象生成算法，包括个人写真、证件照、虚拟试装、视频口播等多种效果。
+# 应用场景
+appScenario=推理,训练,零售,广媒,金融,医疗,家居,医疗,环保
+# 框架类型
+frameType=pytorch

requirements.txt

+accelerate
+transformers
+onnxruntime==1.15.1
+diffusers>=0.23.0
+invisible-watermark>=0.2.0
+modelscope==1.10.0
+Pillow
+opencv-contrib-python
+opencv-python==4.6.0.66
+# torchvision
+mmdet==2.26.0
+mmengine
+numpy==1.24.0
+protobuf==3.20.1
+timm
+scikit-image==0.19.3
+gradio==3.50
+controlnet_aux==0.0.6
+mediapipe
+python-slugify
+edge-tts
+sentencepiece
+
+# mmcv-full (need mim install)

run_inference.py

+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import json
+from facechain.inference import GenPortrait
+import cv2
+from facechain.utils import snapshot_download
+from facechain.constants import neg_prompt, pos_prompt_with_cloth, pos_prompt_with_style, base_models
+
+
+def generate_pos_prompt(style_model, prompt_cloth):
+    if style_model is not None:
+        matched = list(filter(lambda style: style_model == style['name'], styles))
+        if len(matched) == 0:
+            raise ValueError(f'styles not found: {style_model}')
+        matched = matched[0]
+        if matched['model_id'] is None:
+            pos_prompt = pos_prompt_with_cloth.format(prompt_cloth)
+        else:
+            pos_prompt = pos_prompt_with_style.format(matched['add_prompt_style'])
+    else:
+        pos_prompt = pos_prompt_with_cloth.format(prompt_cloth)
+    return pos_prompt
+
+styles = []
+for base_model in base_models:
+    style_in_base = []
+    folder_path = f"/mnt/workspace/new_facechain/facechain/styles/{base_model['name']}"
+    files = os.listdir(folder_path)
+    files.sort()
+    for file in files:
+        file_path = os.path.join(folder_path, file)
+        with open(file_path, "r") as f:
+            data = json.load(f)
+            style_in_base.append(data['name'])
+            styles.append(data)
+    base_model['style_list'] = style_in_base
+
+use_main_model = True
+use_face_swap = True
+use_post_process = True
+use_stylization = False
+use_depth_control = False
+use_pose_model = False
+pose_image = 'poses/man/pose1.png'
+processed_dir = './processed'
+num_generate = 5
+multiplier_style = 0.25
+multiplier_human = 0.85
+train_output_dir = './output'
+output_dir = './generated'
+base_model = base_models[0]
+style = styles[0]
+model_id = style['model_id']
+
+if model_id == None:
+    style_model_path = None
+    pos_prompt = generate_pos_prompt(style['name'], style['add_prompt_style'])
+else:
+    if os.path.exists(model_id):
+        model_dir = model_id
+    else:
+        model_dir = snapshot_download(model_id, revision=style['revision'])
+    style_model_path = os.path.join(model_dir, style['bin_file'])
+    pos_prompt = generate_pos_prompt(style['name'], style['add_prompt_style'])  # style has its own prompt
+
+if not use_pose_model:
+    pose_model_path = None
+    use_depth_control = False
+    pose_image = None
+else:
+    model_dir = snapshot_download('damo/face_chain_control_model', revision='v1.0.1')
+    pose_model_path = os.path.join(model_dir, 'model_controlnet/control_v11p_sd15_openpose')
+
+gen_portrait = GenPortrait(pose_model_path, pose_image, use_depth_control, pos_prompt, neg_prompt, style_model_path,
+                           multiplier_style, multiplier_human, use_main_model,
+                           use_face_swap, use_post_process,
+                           use_stylization)
+
+outputs = gen_portrait(processed_dir, num_generate, base_model['model_id'],
+                       train_output_dir, base_model['sub_path'], base_model['revision'])
+
+os.makedirs(output_dir, exist_ok=True)
+
+for i, out_tmp in enumerate(outputs):
+    cv2.imwrite(os.path.join(output_dir, f'{i}.png'), out_tmp)
+

scripts/facechain_sdwebui.py

+import modules.scripts as scripts
+import gradio as gr
+import os
+
+from modules import script_callbacks
+from modules import shared
+import sys
+
+def on_ui_tabs():
+    # TODO initialize facechain UI here
+    with gr.Blocks(analytics_enabled=False) as ui_component:
+        parent_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        if parent_path not in sys.path:
+            sys.path.append(parent_path)
+        import app
+        return [(ui_component, "FaceChain", "FaceChain_tab")]
+
+def on_ui_settings():
+    # TODO initialize facechain setting here
+    section = ('FaceChain', "FaceChain")
+    shared.opts.add_option(
+        "option1",
+        shared.OptionInfo(
+            False,
+            "option1 description",
+            gr.Checkbox,
+            {"interactive": True},
+            section=section)
+    )
+
+
+
+# register setting
+# script_callbacks.on_ui_tabs(on_ui_settings)
+
+# register ui
+script_callbacks.on_ui_tabs(on_ui_tabs)
+
+# register setting
+# script_callbacks.on_ui_settings(on_ui_settings)