skip tokenizer.model if possible (#11050)

if tokenizer.json is already copied, skip tokenizer.model

parser/parser.go

@@ -292,6 +292,10 @@ func filesForModel(path string) ([]string, error) {
 	}
 	files = append(files, js...)
 
+	// only include tokenizer.model is tokenizer.json is not present
+	if !slices.ContainsFunc(files, func(s string) bool {
+		return slices.Contains(strings.Split(s, string(os.PathSeparator)), "tokenizer.json")
+	}) {
 		if tks, _ := glob(filepath.Join(path, "tokenizer.model"), "application/octet-stream"); len(tks) > 0 {
 			// add tokenizer.model if it exists, tokenizer.json is automatically picked up by the previous glob
 			// tokenizer.model might be a unresolved git lfs reference; error if it is
@@ -300,6 +304,7 @@ func filesForModel(path string) ([]string, error) {
 			// some times tokenizer.model is in a subdirectory (e.g. meta-llama/Meta-Llama-3-8B)
 			files = append(files, tks...)
 		}
+	}
 
 	return files, nil
 }