Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
OpenDAS
ollama
Commits
571dc619
Unverified
Commit
571dc619
authored
Jul 07, 2024
by
Jeffrey Morgan
Committed by
GitHub
Jul 07, 2024
Browse files
Update llama.cpp submodule to `a8db2a9c` (#5530)
parent
0e09c380
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
6 additions
and
6 deletions
+6
-6
llm/llama.cpp
llm/llama.cpp
+1
-1
llm/patches/05-default-pretokenizer.diff
llm/patches/05-default-pretokenizer.diff
+5
-5
No files found.
llama.cpp
@
a8db2a9c
Compare
d7fd29ff
...
a8db2a9c
Subproject commit
d7fd29fff16456ce9c3a23fd2d09a66256b05aff
Subproject commit
a8db2a9ce64cd4417f6a312ab61858f17f0f8584
llm/patches/05-default-pretokenizer.diff
View file @
571dc619
diff --git a/src/llama.cpp b/src/llama.cpp
diff --git a/src/llama.cpp b/src/llama.cpp
index
73f52435..2b81b4bd
100644
index
2b9ace28..172640e2
100644
--- a/src/llama.cpp
--- a/src/llama.cpp
+++ b/src/llama.cpp
+++ b/src/llama.cpp
@@ -5092,16 +5092,7 @@
static void llm_load_vocab(
@@ -5357,16 +5357,7 @@
static void llm_load_vocab(
// for now, only BPE models have pre-tokenizers
if (vocab.type == LLAMA_VOCAB_TYPE_BPE) {
if (vocab.type == LLAMA_VOCAB_TYPE_BPE) {
vocab.tokenizer_add_space_prefix = false;
vocab.tokenizer_clean_spaces = true;
- if (tokenizer_pre.empty()) {
- if (tokenizer_pre.empty()) {
- LLAMA_LOG_WARN("%s: missing pre-tokenizer type, using: 'default'\n", __func__);
- LLAMA_LOG_WARN("%s: missing pre-tokenizer type, using: 'default'\n", __func__);
- LLAMA_LOG_WARN("%s: \n", __func__);
- LLAMA_LOG_WARN("%s: \n", __func__);
...
@@ -20,7 +20,7 @@ index 73f52435..2b81b4bd 100644
...
@@ -20,7 +20,7 @@ index 73f52435..2b81b4bd 100644
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
} else if (
} else if (
tokenizer_pre == "llama3" ||
tokenizer_pre == "llama3" ||
@@ -5
164
,7 +5
155
,8 @@
static void llm_load_vocab(
@@ -5
439
,7 +5
430
,8 @@
static void llm_load_vocab(
tokenizer_pre == "jais") {
tokenizer_pre == "jais") {
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_JAIS;
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_JAIS;
} else {
} else {
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment