Update llama.cpp submodule to `5921b8f0` (#4731)

* update llama.cpp submodule to `5921b8f089d3b7bda86aac5a66825df6a6c10603` * add patch

Update llama.cpp submodule to `5921b8f0` (#4731)
* update llama.cpp submodule to `5921b8f089d3b7bda86aac5a66825df6a6c10603` * add patch
22f5c12c · Jeffrey Morgan · GitHub · 0fc0cfc6 · 5921b8f0 · 74f33adf
Unverified Commit 22f5c12c authored May 30, 2024 by Jeffrey Morgan Committed by GitHub May 30, 2024
Hide whitespace changes
Inline Side-by-side

Showing with 22 additions and 25 deletions

llm/llama.cpp llm/llama.cpp +1 -1

llm/patches/05-default-pretokenizer.diff llm/patches/05-default-pretokenizer.diff +21 -24

No files found.
--- a/llama.cpp @ 5921b8f0
+++ b/llama.cpp @ 5921b8f0
-Subproject commit 74f33adf5f8b20b08fc5a6aa17ce081abe86ef2f
+Subproject commit 5921b8f089d3b7bda86aac5a66825df6a6c10603
--- a/llm/patches/05-default-pretokenizer.diff
+++ b/llm/patches/05-default-pretokenizer.diff
-From d02a06f3f45a09255ace8684a66590e06ce44605 Mon Sep 17 00:00:00 2001
-From: Michael Yang <mxyng@pm.me>
-Date: Thu, 23 May 2024 11:33:20 -0700
-Subject: [PATCH] default pretokenizer on unrecognized type
---
- llama.cpp | 5 +----
- 1 file changed, 1 insertion(+), 4 deletions(-)
 diff --git a/llama.cpp b/llama.cpp
-index 15c66077..af1aede3 100644
+index 40d2ec2c..74f3ee9c 100644
 --- a/llama.cpp
 +++ b/llama.cpp
-@@ -4504,9 +4504,6 @@ static void llm_load_vocab(
+@@ -4642,16 +4642,7 @@ static void llm_load_vocab(
-                 LLAMA_LOG_WARN("%s: ************************************        \n", __func__);
-                 LLAMA_LOG_WARN("%s:                                             \n", __func__);
+         // for now, only BPE models have pre-tokenizers
-                 vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+         if (vocab.type == LLAMA_VOCAB_TYPE_BPE) {
-            } else if (
+-            if (tokenizer_pre.empty()) {
-                    tokenizer_pre == "default") {
+-                LLAMA_LOG_WARN("%s: missing pre-tokenizer type, using: 'default'\n", __func__);
+-                LLAMA_LOG_WARN("%s:                                             \n", __func__);
+-                LLAMA_LOG_WARN("%s: ************************************        \n", __func__);
+-                LLAMA_LOG_WARN("%s: GENERATION QUALITY WILL BE DEGRADED!        \n", __func__);
+-                LLAMA_LOG_WARN("%s: CONSIDER REGENERATING THE MODEL             \n", __func__);
+-                LLAMA_LOG_WARN("%s: ************************************        \n", __func__);
+-                LLAMA_LOG_WARN("%s:                                             \n", __func__);
 -                vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+-            } else if (
+            if (
+                     tokenizer_pre == "default") {
+                 vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
             } else if (
-                     tokenizer_pre == "llama3"   ||
+@@ -4703,7 +4694,8 @@ static void llm_load_vocab(
-                     tokenizer_pre == "llama-v3" ||
+                 tokenizer_pre == "smaug-bpe") {
-@@ -4553,7 +4550,7 @@ static void llm_load_vocab(
+                 vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_SMAUG;
-                 tokenizer_pre == "dbrx") {
-                 vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DBRX;
             } else {
 -                throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));
+                LLAMA_LOG_WARN("%s: missing or unrecognized pre-tokenizer type, using: 'default'\n", __func__);
 +                vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
             }
         } else {
             vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
-- 
-2.45.1