Skip to content

GitLab

You need to sign in or sign up before continuing.
Commit 631fecc6 authored by Patrick Devine's avatar Patrick Devine Committed by Michael Yang
Browse files

temporary work around for converting spm

parent 4346c240
Hide whitespace changes
Inline Side-by-side
Showing with 6 additions and 0 deletions
convert/tokenizer_spm.go
View file @ 631fecc6
......@@ -47,6 +47,12 @@ func parseSentencePiece(fsys fs.FS) (*Vocabulary, error) {
v.Types = append(v.Types, int32(t))
default:
tt := int32(sentencepiece.ModelProto_SentencePiece_NORMAL)
// temporary fix to handle gemma3 broken configs
if slices.Contains([]string{"<end_of_turn>", "<start_of_turn>"}, piece.GetPiece()) {
tt = int32(sentencepiece.ModelProto_SentencePiece_CONTROL)
}
for _, t := range ast {
if t.Content == piece.GetPiece() {
tt = int32(sentencepiece.ModelProto_SentencePiece_CONTROL)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment