CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-11-04 09:32:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

convert-llama-7b-pth-to-gguf.py : rope scale and added tokens

Browse Source
This commit is contained in:
klosax
2023-08-21 13:27:53 +02:00
committed by GitHub
parent 5f6ff387ca
commit dc1f051013
1 changed files with 17 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
convert-llama-7b-pth-to-gguf.py
View File

@@ -118,6 +118,11 @@ gguf_writer.add_head_count(head_count)
gguf_writer.add_head_count_kv(head_count_kv) gguf_writer.add_head_count_kv(head_count_kv)
gguf_writer.add_layer_norm_rms_eps(hparams["rms_norm_eps"]) gguf_writer.add_layer_norm_rms_eps(hparams["rms_norm_eps"])
if "rope_scaling" in hparams and "factor" in hparams["rope_scaling"]:
if "type" in hparams["rope_scaling"]:
if hparams["rope_scaling"]["type"] == "linear":
gguf_writer.add_rope_scale_linear(hparams["rope_scaling"]["factor"])
# TOKENIZATION # TOKENIZATION
@@ -147,9 +152,7 @@ if Path(dir_model + "/tokenizer.model").is_file():
if tokenizer.is_control(i): if tokenizer.is_control(i):
toktype = 3 toktype = 3
# TODO: How to determinate if a token is user defined? # toktype = 4 is user-defined = tokens from added_tokens.json
# ref: https://github.com/google/sentencepiece/blob/master/src/sentencepiece_model.proto
# if tokenizer.is_user_defined(i): toktype = 4
if tokenizer.is_unused(i): if tokenizer.is_unused(i):
toktype = 5 toktype = 5
@@ -160,6 +163,17 @@ if Path(dir_model + "/tokenizer.model").is_file():
scores.append(score) scores.append(score)
toktypes.append(toktype) toktypes.append(toktype)
if Path(dir_model + "/added_tokens.json").is_file():
with open(dir_model + "/added_tokens.json", "r", encoding="utf-8") as f:
addtokens_json = json.load(f)
print("gguf: get added tokens")
for key in addtokens_json:
tokens.append( key.encode("utf-8") )
scores.append(-1000.0)
toktypes.append(4) # user-defined token type
gguf_writer.add_tokenizer_model("llama") gguf_writer.add_tokenizer_model("llama")
gguf_writer.add_token_list(tokens) gguf_writer.add_token_list(tokens)
gguf_writer.add_token_scores(scores) gguf_writer.add_token_scores(scores)