Skip to content

GitLab

Unverified Commit 955c317c authored by Ikko Eltociear Ashimine's avatar Ikko Eltociear Ashimine Committed by GitHub
Browse files

chore: update tokenizer.go (#4571) 

PreTokenziers -> PreTokenizers
parent 9f18b88a
Show whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
convert/tokenizer.go
View file @ 955c317c
...@@ -18,7 +18,7 @@ type Tokenizer struct { ...@@ -18,7 +18,7 @@ type Tokenizer struct {
Model TokenizerModel `json:"model"` Model TokenizerModel `json:"model"`
PreTokenizer struct { PreTokenizer struct {
PreTokenziers []struct { PreTokenizers []struct {
Type string `json:"type"` Type string `json:"type"`
Pattern struct { Pattern struct {
Regex string `json:"Regex"` Regex string `json:"Regex"`
...@@ -84,7 +84,7 @@ func parseTokens(dirpath string) (pre string, tokens []Token, merges []string, e ...@@ -84,7 +84,7 @@ func parseTokens(dirpath string) (pre string, tokens []Token, merges []string, e
} }
sha256sum := sha256.New() sha256sum := sha256.New()
for _, pt := range t.PreTokenizer.PreTokenziers { for _, pt := range t.PreTokenizer.PreTokenizers {
switch pt.Type { switch pt.Type {
case "Split": case "Split":
if pt.Pattern.Regex != "" { if pt.Pattern.Regex != "" {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment