chore: update tokenizer.go (#4571)

PreTokenziers -> PreTokenizers

chore: update tokenizer.go (#4571)
PreTokenziers -> PreTokenizers
955c317c · Ikko Eltociear Ashimine · GitHub · 9f18b88a · 955c317c
Unverified Commit 955c317c authored May 22, 2024 by Ikko Eltociear Ashimine Committed by GitHub May 22, 2024
Hide whitespace changes
Inline Side-by-side

Showing with 2 additions and 2 deletions

convert/tokenizer.go convert/tokenizer.go +2 -2

No files found.
--- a/convert/tokenizer.go
+++ b/convert/tokenizer.go
@@ -18,7 +18,7 @@ type Tokenizer struct {
 	Model       TokenizerModel `json:"model"`

 	PreTokenizer struct {
-		PreTokenziers []struct {
+		PreTokenizers []struct {
 			Type    string `json:"type"`
 			Pattern struct {
 				Regex string `json:"Regex"`
@@ -84,7 +84,7 @@ func parseTokens(dirpath string) (pre string, tokens []Token, merges []string, e
 	}

 	sha256sum := sha256.New()
-	for _, pt := range t.PreTokenizer.PreTokenziers {
+	for _, pt := range t.PreTokenizer.PreTokenizers {
 		switch pt.Type {
 		case "Split":
 			if pt.Pattern.Regex != "" {