CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-10-27 08:21:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

convert : refactor vocab selection logic (#6355)

Browse Source
This commit is contained in:
Jared Van Bortel
2024-03-28 11:44:36 -04:00
committed by GitHub
parent 66ba560256
commit be55134a53
4 changed files with 204 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
llama.h
View File

@@ -60,9 +60,9 @@ extern "C" {
enum llama_vocab_type {
LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab
LLAMA_VOCAB_TYPE_SPM = 1, // SentencePiece
LLAMA_VOCAB_TYPE_BPE = 2, // Byte Pair Encoding
LLAMA_VOCAB_TYPE_WPM = 3, // WordPiece
LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback
LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE
LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece
};
// note: these values should be synchronized with ggml_rope