Merge remote-tracking branch 'origin/master' into a1111-meta-v2

.ci/nightly/update_windows/update.py

+import pygit2
+from datetime import datetime
+import sys
+
+def pull(repo, remote_name='origin', branch='master'):
+    for remote in repo.remotes:
+        if remote.name == remote_name:
+            remote.fetch()
+            remote_master_id = repo.lookup_reference('refs/remotes/origin/%s' % (branch)).target
+            merge_result, _ = repo.merge_analysis(remote_master_id)
+            # Up to date, do nothing
+            if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE:
+                return
+            # We can just fastforward
+            elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD:
+                repo.checkout_tree(repo.get(remote_master_id))
+                try:
+                    master_ref = repo.lookup_reference('refs/heads/%s' % (branch))
+                    master_ref.set_target(remote_master_id)
+                except KeyError:
+                    repo.create_branch(branch, repo.get(remote_master_id))
+                repo.head.set_target(remote_master_id)
+            elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL:
+                repo.merge(remote_master_id)
+
+                if repo.index.conflicts is not None:
+                    for conflict in repo.index.conflicts:
+                        print('Conflicts found in:', conflict[0].path)
+                    raise AssertionError('Conflicts, ahhhhh!!')
+
+                user = repo.default_signature
+                tree = repo.index.write_tree()
+                commit = repo.create_commit('HEAD',
+                                            user,
+                                            user,
+                                            'Merge!',
+                                            tree,
+                                            [repo.head.target, remote_master_id])
+                # We need to do this or git CLI will think we are still merging.
+                repo.state_cleanup()
+            else:
+                raise AssertionError('Unknown merge analysis result')
+
+
+repo = pygit2.Repository(str(sys.argv[1]))
+ident = pygit2.Signature('comfyui', 'comfy@ui')
+try:
+    print("stashing current changes")
+    repo.stash(ident)
+except KeyError:
+    print("nothing to stash")
+backup_branch_name = 'backup_branch_{}'.format(datetime.today().strftime('%Y-%m-%d_%H_%M_%S'))
+print("creating backup branch: {}".format(backup_branch_name))
+repo.branches.local.create(backup_branch_name, repo.head.peel())
+
+print("checking out master branch")
+branch = repo.lookup_branch('master')
+ref = repo.lookup_reference(branch.name)
+repo.checkout(ref)
+
+print("pulling latest changes")
+pull(repo)
+
+print("Done!")
+

.ci/nightly/update_windows/update_comfyui.bat

+..\python_embeded\python.exe .\update.py ..\ComfyUI\
+pause

.ci/nightly/update_windows/update_comfyui_and_python_dependencies.bat

+..\python_embeded\python.exe .\update.py ..\ComfyUI\
+..\python_embeded\python.exe -s -m pip install --upgrade --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu118 -r ../ComfyUI/requirements.txt pygit2
+pause

.ci/nightly/windows_base_files/README_VERY_IMPORTANT.txt

+HOW TO RUN:
+
+if you have a NVIDIA gpu:
+
+run_nvidia_gpu.bat
+
+
+
+To run it in slow CPU mode:
+
+run_cpu.bat
+
+
+
+IF YOU GET A RED ERROR IN THE UI MAKE SURE YOU HAVE A MODEL/CHECKPOINT IN: ComfyUI\models\checkpoints
+
+You can download the stable diffusion 1.5 one from: https://huggingface.co/runwayml/stable-diffusion-v1-5/blob/main/v1-5-pruned-emaonly.ckpt
+
+
+
+RECOMMENDED WAY TO UPDATE:
+To update the ComfyUI code: update\update_comfyui.bat
+
+
+
+To update ComfyUI with the python dependencies:
+update\update_comfyui_and_python_dependencies.bat

.ci/nightly/windows_base_files/run_cpu.bat

+.\python_embeded\python.exe -s ComfyUI\main.py --cpu --windows-standalone-build
+pause

.ci/nightly/windows_base_files/run_nvidia_gpu.bat

+.\python_embeded\python.exe -s ComfyUI\main.py --windows-standalone-build --use-pytorch-cross-attention
+pause

.ci/update_windows/update.py

+import pygit2
+from datetime import datetime
+import sys
+
+def pull(repo, remote_name='origin', branch='master'):
+    for remote in repo.remotes:
+        if remote.name == remote_name:
+            remote.fetch()
+            remote_master_id = repo.lookup_reference('refs/remotes/origin/%s' % (branch)).target
+            merge_result, _ = repo.merge_analysis(remote_master_id)
+            # Up to date, do nothing
+            if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE:
+                return
+            # We can just fastforward
+            elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD:
+                repo.checkout_tree(repo.get(remote_master_id))
+                try:
+                    master_ref = repo.lookup_reference('refs/heads/%s' % (branch))
+                    master_ref.set_target(remote_master_id)
+                except KeyError:
+                    repo.create_branch(branch, repo.get(remote_master_id))
+                repo.head.set_target(remote_master_id)
+            elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL:
+                repo.merge(remote_master_id)
+
+                if repo.index.conflicts is not None:
+                    for conflict in repo.index.conflicts:
+                        print('Conflicts found in:', conflict[0].path)
+                    raise AssertionError('Conflicts, ahhhhh!!')
+
+                user = repo.default_signature
+                tree = repo.index.write_tree()
+                commit = repo.create_commit('HEAD',
+                                            user,
+                                            user,
+                                            'Merge!',
+                                            tree,
+                                            [repo.head.target, remote_master_id])
+                # We need to do this or git CLI will think we are still merging.
+                repo.state_cleanup()
+            else:
+                raise AssertionError('Unknown merge analysis result')
+
+
+repo = pygit2.Repository(str(sys.argv[1]))
+ident = pygit2.Signature('comfyui', 'comfy@ui')
+try:
+    print("stashing current changes")
+    repo.stash(ident)
+except KeyError:
+    print("nothing to stash")
+backup_branch_name = 'backup_branch_{}'.format(datetime.today().strftime('%Y-%m-%d_%H_%M_%S'))
+print("creating backup branch: {}".format(backup_branch_name))
+repo.branches.local.create(backup_branch_name, repo.head.peel())
+
+print("checking out master branch")
+branch = repo.lookup_branch('master')
+ref = repo.lookup_reference(branch.name)
+repo.checkout(ref)
+
+print("pulling latest changes")
+pull(repo)
+
+print("Done!")
+

.ci/update_windows/update_comfyui.bat

+..\python_embeded\python.exe .\update.py ..\ComfyUI\
+pause

.ci/update_windows/update_comfyui_and_python_dependencies.bat

+..\python_embeded\python.exe .\update.py ..\ComfyUI\
+..\python_embeded\python.exe -s -m pip install --upgrade torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu117 xformers -r ../ComfyUI/requirements.txt pygit2
+pause

.ci/update_windows_cu118/update_comfyui_and_python_dependencies.bat

+..\python_embeded\python.exe .\update.py ..\ComfyUI\
+..\python_embeded\python.exe -s -m pip install --upgrade torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu118 xformers -r ../ComfyUI/requirements.txt pygit2
+echo NOTE If you get an error with pip you can ignore it, it's pip being pip as usual, your ComfyUI should have updated anyways.
+pause

.ci/windows_base_files/README_VERY_IMPORTANT.txt

+HOW TO RUN:
+
+if you have a NVIDIA gpu:
+
+run_nvidia_gpu.bat
+
+
+
+To run it in slow CPU mode:
+
+run_cpu.bat
+
+
+
+IF YOU GET A RED ERROR IN THE UI MAKE SURE YOU HAVE A MODEL/CHECKPOINT IN: ComfyUI\models\checkpoints
+
+You can download the stable diffusion 1.5 one from: https://huggingface.co/runwayml/stable-diffusion-v1-5/blob/main/v1-5-pruned-emaonly.ckpt
+
+
+RECOMMENDED WAY TO UPDATE:
+To update the ComfyUI code: update\update_comfyui.bat
+
+
+
+To update ComfyUI with the python dependencies, note that you should ONLY run this if you have issues with python dependencies.
+update\update_comfyui_and_python_dependencies.bat
+

.ci/windows_base_files/run_cpu.bat

+.\python_embeded\python.exe -s ComfyUI\main.py --cpu --windows-standalone-build
+pause

.ci/windows_base_files/run_nvidia_gpu.bat

+.\python_embeded\python.exe -s ComfyUI\main.py --windows-standalone-build
+pause

.github/workflows/windows_release.yml

+name: "Windows Release"
+
+on:
+  workflow_dispatch:
+#  push:
+#    branches:
+#      - master
+
+jobs:
+  build:
+    permissions:
+        contents: "write"
+        packages: "write"
+        pull-requests: "read"
+    runs-on: windows-latest
+    steps:
+        - uses: actions/checkout@v3
+          with:
+            fetch-depth: 0
+
+        - shell: bash
+          run: |
+            cd ..
+            cp -r ComfyUI ComfyUI_copy
+            curl https://www.python.org/ftp/python/3.10.9/python-3.10.9-embed-amd64.zip -o python_embeded.zip
+            unzip python_embeded.zip -d python_embeded
+            cd python_embeded
+            echo 'import site' >> ./python310._pth
+            curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+            ./python.exe get-pip.py
+            ./python.exe -s -m pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu117 xformers -r ../ComfyUI/requirements.txt pygit2
+            sed -i '1i../ComfyUI' ./python310._pth
+            cd ..
+
+
+            mkdir ComfyUI_windows_portable
+            mv python_embeded ComfyUI_windows_portable
+            mv ComfyUI_copy ComfyUI_windows_portable/ComfyUI
+
+            cd ComfyUI_windows_portable
+
+            mkdir update
+            cp -r ComfyUI/.ci/update_windows/* ./update/
+            cp -r ComfyUI/.ci/windows_base_files/* ./
+
+            cd ..
+
+            "C:\Program Files\7-Zip\7z.exe" a -t7z -m0=lzma -mx=8 -mfb=64 -md=32m -ms=on ComfyUI_windows_portable.7z ComfyUI_windows_portable
+            mv ComfyUI_windows_portable.7z ComfyUI/ComfyUI_windows_portable_nvidia_or_cpu.7z
+
+            cd ComfyUI_windows_portable
+            python_embeded/python.exe -s ComfyUI/main.py --quick-test-for-ci --cpu
+
+            ls
+
+
+        - name: Upload binaries to release
+          uses: svenstaro/upload-release-action@v2
+          with:
+                repo_token: ${{ secrets.GITHUB_TOKEN }}
+                file: ComfyUI_windows_portable_nvidia_or_cpu.7z
+                tag: "latest"
+                overwrite: true
+

.github/workflows/windows_release_cu118.yml

+name: "Windows Release cu118"
+
+on:
+  workflow_dispatch:
+#  push:
+#    branches:
+#      - master
+
+jobs:
+  build_dependencies:
+    env:
+        # you need at least cuda 5.0 for some of the stuff compiled here.
+        TORCH_CUDA_ARCH_LIST: "5.0+PTX 6.0 6.1 7.0 7.5 8.0 8.6 8.9"
+        FORCE_CUDA: 1
+        MAX_JOBS: 1 # will crash otherwise
+        DISTUTILS_USE_SDK: 1 # otherwise distutils will complain on windows about multiple versions of msvc
+        XFORMERS_BUILD_TYPE: "Release"
+    runs-on: windows-latest
+    steps:
+        - name: Cache Built Dependencies
+          uses: actions/cache@v3
+          id: cache-cu118_python_stuff
+          with:
+            path: cu118_python_deps.tar
+            key: ${{ runner.os }}-build-cu118
+
+        - if: ${{ steps.cache-cu118_python_stuff.cache-hit != 'true' }}
+          uses: actions/checkout@v3
+
+        - if: ${{ steps.cache-cu118_python_stuff.cache-hit != 'true' }}
+          uses: actions/setup-python@v4
+          with:
+            python-version: '3.10.9'
+
+        - if: ${{ steps.cache-cu118_python_stuff.cache-hit != 'true' }}
+          uses: comfyanonymous/cuda-toolkit@test
+          id: cuda-toolkit
+          with:
+            cuda: '11.8.0'
+        # copied from xformers github
+        - name: Setup MSVC
+          uses: ilammy/msvc-dev-cmd@v1
+        - name: Configure Pagefile
+          # windows runners will OOM with many CUDA architectures
+          # we cheat here with a page file
+          uses: al-cheb/configure-pagefile-action@v1.3
+          with:
+            minimum-size: 2GB
+        # really unfortunate: https://github.com/ilammy/msvc-dev-cmd#name-conflicts-with-shell-bash
+        - name: Remove link.exe
+          shell: bash
+          run: rm /usr/bin/link
+
+        - if: ${{ steps.cache-cu118_python_stuff.cache-hit != 'true' }}
+          shell: bash
+          run: |
+            python -m pip wheel --no-cache-dir torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu118 -r requirements.txt pygit2 -w ./temp_wheel_dir
+            python -m pip install --no-cache-dir ./temp_wheel_dir/*
+            echo installed basic
+            git clone --recurse-submodules https://github.com/facebookresearch/xformers.git
+            cd xformers
+            python -m pip install --no-cache-dir wheel setuptools twine
+            echo building xformers
+            python setup.py bdist_wheel -d ../temp_wheel_dir/
+            cd ..
+            rm -rf xformers
+            ls -lah temp_wheel_dir
+            mv temp_wheel_dir cu118_python_deps
+            tar cf cu118_python_deps.tar cu118_python_deps
+        - uses: actions/upload-artifact@v3
+          with:
+            name: cu118_python_deps
+            path: cu118_python_deps.tar
+
+
+  package_comfyui:
+    needs: build_dependencies
+    permissions:
+        contents: "write"
+        packages: "write"
+        pull-requests: "read"
+    runs-on: windows-latest
+    steps:
+        - uses: actions/download-artifact@v3
+          with:
+            name: cu118_python_deps
+        - shell: bash
+          run: |
+            mv cu118_python_deps.tar ../
+            cd ..
+            tar xf cu118_python_deps.tar
+            pwd
+            ls
+
+        - uses: actions/checkout@v3
+          with:
+            fetch-depth: 0
+        - shell: bash
+          run: |
+            cd ..
+            cp -r ComfyUI ComfyUI_copy
+            curl https://www.python.org/ftp/python/3.10.9/python-3.10.9-embed-amd64.zip -o python_embeded.zip
+            unzip python_embeded.zip -d python_embeded
+            cd python_embeded
+            echo 'import site' >> ./python310._pth
+            curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+            ./python.exe get-pip.py
+            ./python.exe -s -m pip install ../cu118_python_deps/*
+            sed -i '1i../ComfyUI' ./python310._pth
+            cd ..
+
+
+            mkdir ComfyUI_windows_portable
+            mv python_embeded ComfyUI_windows_portable
+            mv ComfyUI_copy ComfyUI_windows_portable/ComfyUI
+
+            cd ComfyUI_windows_portable
+
+            mkdir update
+            cp -r ComfyUI/.ci/update_windows/* ./update/
+            cp -r ComfyUI/.ci/update_windows_cu118/* ./update/
+            cp -r ComfyUI/.ci/windows_base_files/* ./
+
+            cd ..
+
+            "C:\Program Files\7-Zip\7z.exe" a -t7z -m0=lzma -mx=8 -mfb=64 -md=32m -ms=on ComfyUI_windows_portable.7z ComfyUI_windows_portable
+            mv ComfyUI_windows_portable.7z ComfyUI/ComfyUI_windows_portable_nvidia_cu118_or_cpu.7z
+
+            cd ComfyUI_windows_portable
+            python_embeded/python.exe -s ComfyUI/main.py --quick-test-for-ci --cpu
+
+            ls
+
+        - name: Upload binaries to release
+          uses: svenstaro/upload-release-action@v2
+          with:
+                repo_token: ${{ secrets.GITHUB_TOKEN }}
+                file: ComfyUI_windows_portable_nvidia_cu118_or_cpu.7z
+                tag: "latest"
+                overwrite: true
+

.github/workflows/windows_release_nightly_pytorch.yml

+name: "Windows Release Nightly pytorch"
+
+on:
+  workflow_dispatch:
+#  push:
+#    branches:
+#      - master
+
+jobs:
+  build:
+    permissions:
+        contents: "write"
+        packages: "write"
+        pull-requests: "read"
+    runs-on: windows-latest
+    steps:
+        - uses: actions/checkout@v3
+          with:
+            fetch-depth: 0
+        - uses: actions/setup-python@v4
+          with:
+            python-version: '3.10.9'
+        - shell: bash
+          run: |
+            cd ..
+            cp -r ComfyUI ComfyUI_copy
+            curl https://www.python.org/ftp/python/3.10.9/python-3.10.9-embed-amd64.zip -o python_embeded.zip
+            unzip python_embeded.zip -d python_embeded
+            cd python_embeded
+            echo 'import site' >> ./python310._pth
+            curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+            ./python.exe get-pip.py
+            python -m pip wheel torch torchvision torchaudio --pre --extra-index-url https://download.pytorch.org/whl/nightly/cu118 -r ../ComfyUI/requirements.txt pygit2 -w ../temp_wheel_dir
+            ls ../temp_wheel_dir
+            ./python.exe -s -m pip install --pre ../temp_wheel_dir/*
+            sed -i '1i../ComfyUI' ./python310._pth
+            cd ..
+
+
+            mkdir ComfyUI_windows_portable_nightly_pytorch
+            mv python_embeded ComfyUI_windows_portable_nightly_pytorch
+            mv ComfyUI_copy ComfyUI_windows_portable_nightly_pytorch/ComfyUI
+
+            cd ComfyUI_windows_portable_nightly_pytorch
+
+            mkdir update
+            cp -r ComfyUI/.ci/update_windows/* ./update/
+            cp -r ComfyUI/.ci/windows_base_files/* ./
+
+            cd ..
+
+            "C:\Program Files\7-Zip\7z.exe" a -t7z -m0=lzma -mx=8 -mfb=64 -md=32m -ms=on ComfyUI_windows_portable_nightly_pytorch.7z ComfyUI_windows_portable_nightly_pytorch
+            mv ComfyUI_windows_portable_nightly_pytorch.7z ComfyUI/ComfyUI_windows_portable_nvidia_or_cpu_nightly_pytorch.7z
+
+            cd ComfyUI_windows_portable_nightly_pytorch
+            python_embeded/python.exe -s ComfyUI/main.py --quick-test-for-ci --cpu
+
+            ls
+
+        - name: Upload binaries to release
+          uses: svenstaro/upload-release-action@v2
+          with:
+                repo_token: ${{ secrets.GITHUB_TOKEN }}
+                file: ComfyUI_windows_portable_nvidia_or_cpu_nightly_pytorch.7z
+                tag: "latest"
+                overwrite: true

README.md

@@ -31,6 +31,20 @@ Workflow examples can be found on the [Examples page](https://comfyanonymous.git
 
 # Installing
 
+## Windows
+
+There is a portable standalone build for Windows that should work for running on Nvidia GPUs or for running on your CPU only on the [releases page](https://github.com/comfyanonymous/ComfyUI/releases).
+
+### [Direct link to download](https://github.com/comfyanonymous/ComfyUI/releases/download/latest/ComfyUI_windows_portable_nvidia_or_cpu.7z)
+
+Just download, extract and run. Make sure you put your Stable Diffusion checkpoints/models (the huge ckpt/safetensors files) in: ComfyUI\models\checkpoints
+
+## Colab Notebook
+
+To run it on colab or paperspace you can use my [Colab Notebook](notebooks/comfyui_colab.ipynb) here: [Link to open with google colab](https://colab.research.google.com/github/comfyanonymous/ComfyUI/blob/master/notebooks/comfyui_colab.ipynb)
+
+## Manual Install (Windows, Linux)
+
 Git clone this repo.
 
 Put your SD checkpoints (the huge ckpt/safetensors files) in: models/checkpoints
@@ -39,20 +53,17 @@ Put your VAE in: models/vae
 
 At the time of writing this pytorch has issues with python versions higher than 3.10 so make sure your python/pip versions are 3.10.
 
-### AMD
-AMD users can install rocm and pytorch with pip if you don't have it already installed:
-
-```pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/rocm5.2```
+### AMD (Linux only)
+AMD users can install rocm and pytorch with pip if you don't have it already installed, this is the command to install the stable version:
 
-### NVIDIA
+```pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/rocm5.4.2```
 
-Nvidia users should install torch using this command:
 
-```pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu117```
+### NVIDIA
 
-Nvidia users should also install Xformers for a speed boost but can still run the software without it.
+Nvidia users should install torch and xformers using this command:
 
-```pip install xformers```
+```pip install torch==1.13.1 torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu117 xformers```
 
 #### Troubleshooting
 
@@ -112,10 +123,6 @@ To use a textual inversion concepts/embeddings in a text prompt put them in the
 
 ```embedding:embedding_filename.pt```
 
-### Colab Notebook
-
-To run it on colab or paperspace you can use my [Colab Notebook](notebooks/comfyui_colab.ipynb) here: [Link to open with google colab](https://colab.research.google.com/github/comfyanonymous/ComfyUI/blob/master/notebooks/comfyui_colab.ipynb)
-
 ### Fedora
 
 To get python 3.10 on fedora:

comfy/cldm/cldm.py

@@ -59,9 +59,9 @@ class ControlNet(nn.Module):
 
         if context_dim is not None:
             assert use_spatial_transformer, 'Fool!! You forgot to use the spatial transformer for your cross-attention conditioning...'
-            from omegaconf.listconfig import ListConfig
-            if type(context_dim) == ListConfig:
-                context_dim = list(context_dim)
+            # from omegaconf.listconfig import ListConfig
+            # if type(context_dim) == ListConfig:
+            #     context_dim = list(context_dim)
 
         if num_heads_upsample == -1:
             num_heads_upsample = num_heads

comfy/ldm/models/diffusion/ddpm.py

@@ -18,7 +18,6 @@ import itertools
 from tqdm import tqdm
 from torchvision.utils import make_grid
 # from pytorch_lightning.utilities.distributed import rank_zero_only
-from omegaconf import ListConfig
 
 from ldm.util import log_txt_as_img, exists, default, ismap, isimage, mean_flat, count_params, instantiate_from_config
 from ldm.modules.ema import LitEma
@@ -1124,8 +1123,8 @@ class LatentDiffusion(DDPM):
     def get_unconditional_conditioning(self, batch_size, null_label=None):
         if null_label is not None:
             xc = null_label
-            if isinstance(xc, ListConfig):
-                xc = list(xc)
+            # if isinstance(xc, ListConfig):
+            #     xc = list(xc)
             if isinstance(xc, dict) or isinstance(xc, list):
                 c = self.get_learned_conditioning(xc)
             else:

comfy/ldm/modules/attention.py

@@ -11,12 +11,10 @@ from .sub_quadratic_attention import efficient_dot_product_attention
 
 import model_management
 
-try:
+
+if model_management.xformers_enabled():
     import xformers
     import xformers.ops
-    XFORMERS_IS_AVAILBLE = True
-except:
-    XFORMERS_IS_AVAILBLE = False
 
 # CrossAttn precision handling
 import os
@@ -481,23 +479,19 @@ class CrossAttentionPytorch(nn.Module):
         return self.to_out(out)
 
 import sys
-if XFORMERS_IS_AVAILBLE == False or "--disable-xformers" in sys.argv:
+if model_management.xformers_enabled():
+    print("Using xformers cross attention")
+    CrossAttention = MemoryEfficientCrossAttention
+elif model_management.pytorch_attention_enabled():
+    print("Using pytorch cross attention")
+    CrossAttention = CrossAttentionPytorch
+else:
     if "--use-split-cross-attention" in sys.argv:
         print("Using split optimization for cross attention")
         CrossAttention = CrossAttentionDoggettx
-    else:
-        if "--use-pytorch-cross-attention" in sys.argv:
-            print("Using pytorch cross attention")
-            torch.backends.cuda.enable_math_sdp(False)
-            torch.backends.cuda.enable_flash_sdp(True)
-            torch.backends.cuda.enable_mem_efficient_sdp(True)
-            CrossAttention = CrossAttentionPytorch
     else:
         print("Using sub quadratic optimization for cross attention, if you have memory or speed issues try using: --use-split-cross-attention")
         CrossAttention = CrossAttentionBirchSan
-else:
-    print("Using xformers cross attention")
-    CrossAttention = MemoryEfficientCrossAttention
 
 
 class BasicTransformerBlock(nn.Module):