Update Src/Utility/tokenization.cpp

a2b41d78 · yangql · d8f53cee · a2b41d78
Commit a2b41d78 authored Jul 12, 2023 by yangql
Hide whitespace changes
Inline Side-by-side

Showing with 0 additions and 1 deletion

Src/Utility/tokenization.cpp Src/Utility/tokenization.cpp +0 -1

No files found.
--- a/Src/Utility/tokenization.cpp
+++ b/Src/Utility/tokenization.cpp
@@ -6,7 +6,6 @@
 #include "./tokenization.h"
 namespace cuBERT {
    void FullTokenizer::convert_tokens_to_ids(const std::vector<std::string> &tokens, uint64_t *ids) {