llama: update vendored code to commit 46e3556 (#8308)

llama/unicode.h

 /**
- * llama.cpp - commit ba1cb19cdd0d92e012e0f6e009e0620f854b6afd - do not edit this file
+ * llama.cpp - commit 46e3556e01b824e52395fb050b29804b6cff2a7c - do not edit this file
  *
  * MIT License
  *
@@ -30,9 +30,7 @@
 #include <string>
 #include <vector>
 
-// TODO: prefix all symbols with "llama_"
-
-struct codepoint_flags {
+struct unicode_cpt_flags {
     enum {
         UNDEFINED       = 0x0001,
         NUMBER          = 0x0002,  // regex: \p{N}
@@ -61,7 +59,7 @@ struct codepoint_flags {
     uint16_t is_nfd         : 1;
 
     // decode from uint16
-    inline codepoint_flags(const uint16_t flags=0) {
+    inline unicode_cpt_flags(const uint16_t flags = 0) {
         *reinterpret_cast<uint16_t*>(this) = flags;
     }
 
@@ -76,18 +74,19 @@ struct codepoint_flags {
 
 size_t unicode_len_utf8(char src);
 
-std::string unicode_cpt_to_utf8(uint32_t cp);
-uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset);
+std::string unicode_cpt_to_utf8  (uint32_t cpt);
+uint32_t    unicode_cpt_from_utf8(const std::string & utf8, size_t & offset);
+
 std::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8);
 
 std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & cpts);
 
-codepoint_flags unicode_cpt_flags(const uint32_t cp);
-codepoint_flags unicode_cpt_flags(const std::string & utf8);
+unicode_cpt_flags unicode_cpt_flags_from_cpt (uint32_t cpt);
+unicode_cpt_flags unicode_cpt_flags_from_utf8(const std::string & utf8);
 
 std::string unicode_byte_to_utf8(uint8_t byte);
-uint8_t unicode_utf8_to_byte(const std::string & utf8);
+uint8_t     unicode_utf8_to_byte(const std::string & utf8);
 
-uint32_t unicode_tolower(uint32_t cp);
+uint32_t unicode_tolower(uint32_t cpt);
 
 std::vector<std::string> unicode_regex_split(const std::string & text, const std::vector<std::string> & regex_exprs);

llama/vendoring

-LLAMACPP_BASE_COMMIT=ba1cb19cdd0d92e012e0f6e009e0620f854b6afd
+LLAMACPP_BASE_COMMIT=46e3556e01b824e52395fb050b29804b6cff2a7c

llm/server.go

@@ -692,7 +692,6 @@ func (s *llmServer) Completion(ctx context.Context, req CompletionRequest, fn fu
 		"mirostat":          req.Options.Mirostat,
 		"mirostat_tau":      req.Options.MirostatTau,
 		"mirostat_eta":      req.Options.MirostatEta,
-		"penalize_nl":       req.Options.PenalizeNewline,
 		"seed":              req.Options.Seed,
 		"stop":              req.Options.Stop,
 		"image_data":        req.Images,

make/Makefile.sync

@@ -78,14 +78,40 @@ LLAMACPP_FILES=\
 	src/unicode-data.cpp \
 	src/unicode-data.h \
 	src/llama.cpp \
-	src/llama-impl.h \
-	src/llama-vocab.cpp \
-	src/llama-vocab.h \
+	src/llama-adapter.cpp \
+	src/llama-adapter.h \
+	src/llama-arch.cpp \
+	src/llama-arch.h \
+	src/llama-batch.cpp \
+	src/llama-batch.h \
+	src/llama-chat.cpp \
+	src/llama-chat.h \
+	src/llama-context.cpp \
+	src/llama-context.h \
+	src/llama-cparams.cpp \
+	src/llama-cparams.h \
 	src/llama-grammar.cpp \
 	src/llama-grammar.h \
+	src/llama-hparams.cpp \
+	src/llama-hparams.h \
+	src/llama-impl.cpp \
+	src/llama-impl.h \
+	src/llama-kv-cache.cpp \
+	src/llama-kv-cache.h \
+	src/llama-mmap.cpp \
+	src/llama-mmap.h \
+	src/llama-model-loader.cpp \
+	src/llama-model-loader.h \
+	src/llama-model.cpp \
+	src/llama-model.h \
+	src/llama-quant.cpp \
+	src/llama-quant.h \
 	src/llama-sampling.cpp \
 	src/llama-sampling.h \
+	src/llama-vocab.cpp \
+	src/llama-vocab.h \
 	include/llama.h \
+	include/llama-cpp.h \
 	ggml/include/ggml-cpu.h \
 	ggml/src/ggml-cpu/llamafile/sgemm.cpp \
 	ggml/src/ggml-cpu/llamafile/sgemm.h

parser/parser.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"os/user"
 	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"
 
@@ -35,6 +36,8 @@ func (f Modelfile) String() string {
 	return sb.String()
 }
 
+var deprecatedParameters = []string{"penalize_newline"}
+
 // CreateRequest creates a new *api.CreateRequest from an existing Modelfile
 func (f Modelfile) CreateRequest() (*api.CreateRequest, error) {
 	req := &api.CreateRequest{}
@@ -82,6 +85,11 @@ func (f Modelfile) CreateRequest() (*api.CreateRequest, error) {
 			role, msg, _ := strings.Cut(c.Args, ": ")
 			messages = append(messages, api.Message{Role: role, Content: msg})
 		default:
+			if slices.Contains(deprecatedParameters, c.Name) {
+				fmt.Printf("warning: parameter %s is deprecated\n", c.Name)
+				break
+			}
+
 			ps, err := api.FormatParams(map[string][]string{c.Name: {c.Args}})
 			if err != nil {
 				return nil, err