patch: use default locale in wpm tokenizer (#3034)

908005d9 · Jeffrey Morgan · GitHub · cdf65e79 · 908005d9
Unverified Commit 908005d9 authored Mar 09, 2024 by Jeffrey Morgan Committed by GitHub Mar 09, 2024
Hide whitespace changes
Inline Side-by-side

Showing with 13 additions and 0 deletions

llm/patches/03-locale.diff llm/patches/03-locale.diff +13 -0

No files found.
--- a/llm/patches/03-locale.diff
+++ b/llm/patches/03-locale.diff
+diff --git a/llama.cpp b/llama.cpp
+index b19616e8..519b9602 100644
+--- a/llama.cpp
+++ b/llama.cpp
+@@ -9938,7 +9938,7 @@ struct llm_tokenizer_wpm {
+     }
+
+     uint32_t to_lower(uint32_t code) {
+-        static const std::locale locale("en_US.UTF-8");
+        static const std::locale locale("");
+ #if defined(_WIN32)
+         if (code > 0xFFFF) {
+             return code;