CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-10-31 08:51:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

gguf : make token scores and types optional (#3347)

Browse Source
This commit is contained in:
Cebtenzzre
2023-09-28 14:30:15 -04:00
committed by GitHub
parent 2619109ad5
commit ecf90b1a51
3 changed files with 8 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
llama.cpp
View File

@@ -1931,20 +1931,18 @@ static void llm_load_vocab(
throw std::runtime_error("cannot find tokenizer vocab in model file\n");
}
const float * scores = nullptr;
const int score_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_SCORES).c_str());
if (score_idx == -1) {
throw std::runtime_error("cannot find tokenizer scores in model file\n");
if (score_idx != -1) {
scores = (const float * ) gguf_get_arr_data(ctx, score_idx);
}
const float * scores = (const float * ) gguf_get_arr_data(ctx, score_idx);
const int * toktypes = nullptr;
const int toktype_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_TOKEN_TYPE).c_str());
if (toktype_idx == -1) {
throw std::runtime_error("cannot find token type list in GGUF file\n");
if (toktype_idx != -1) {
toktypes = (const int * ) gguf_get_arr_data(ctx, toktype_idx);
}
const int * toktypes = (const int * ) gguf_get_arr_data(ctx, toktype_idx);
// determine vocab type
{
std::string tokenizer_name;
@@ -2012,8 +2010,8 @@ static void llm_load_vocab(
auto & token_data = vocab.id_to_token[i];
token_data.text = std::move(word);
token_data.score = scores[i];
token_data.type = (llama_token_type) toktypes[i];
token_data.score = scores ? scores[i] : 0.0f;
token_data.type = toktypes ? (llama_token_type) toktypes[i] : LLAMA_TOKEN_TYPE_NORMAL;
}
// determine the newline token: LLaMA "<0x0A>" == 10 == '\n', Falcon 193 == '\n'