"git@developer.sourcefind.cn:chenpangpang/diffusers.git" did not exist on "6d6a08f1f1a62c90e501f1b2771af94fe2ac0678"
Commit 14764aa2 authored by comfyanonymous's avatar comfyanonymous
Browse files

Rename LLAMATokenizer to SPieceTokenizer.

parent b2c995f6
from comfy import sd1_clip from comfy import sd1_clip
from .llama_tokenizer import LLAMATokenizer from .spiece_tokenizer import SPieceTokenizer
import comfy.text_encoders.t5 import comfy.text_encoders.t5
import os import os
...@@ -11,7 +11,7 @@ class PT5XlModel(sd1_clip.SDClipModel): ...@@ -11,7 +11,7 @@ class PT5XlModel(sd1_clip.SDClipModel):
class PT5XlTokenizer(sd1_clip.SDTokenizer): class PT5XlTokenizer(sd1_clip.SDTokenizer):
def __init__(self, embedding_directory=None): def __init__(self, embedding_directory=None):
tokenizer_path = os.path.join(os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_pile_tokenizer"), "tokenizer.model") tokenizer_path = os.path.join(os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_pile_tokenizer"), "tokenizer.model")
super().__init__(tokenizer_path, pad_with_end=False, embedding_size=2048, embedding_key='pile_t5xl', tokenizer_class=LLAMATokenizer, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=256, pad_token=1) super().__init__(tokenizer_path, pad_with_end=False, embedding_size=2048, embedding_key='pile_t5xl', tokenizer_class=SPieceTokenizer, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=256, pad_token=1)
class AuraT5Tokenizer(sd1_clip.SD1Tokenizer): class AuraT5Tokenizer(sd1_clip.SD1Tokenizer):
def __init__(self, embedding_directory=None): def __init__(self, embedding_directory=None):
......
import os import os
class LLAMATokenizer: class SPieceTokenizer:
@staticmethod @staticmethod
def from_pretrained(path): def from_pretrained(path):
return LLAMATokenizer(path) return SPieceTokenizer(path)
def __init__(self, tokenizer_path): def __init__(self, tokenizer_path):
import sentencepiece import sentencepiece
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment