CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-11-04 09:32:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

gguf : write tokenizer data

Browse Source
This commit is contained in:
M. Yusuf Sarıgöz
2023-07-29 13:30:22 +03:00
parent 8a76dd8a85
commit cc3dd7f042
1 changed files with 9 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
gguf.py
View File

@@ -267,6 +267,15 @@ class GGUFWriter:
def write_rope_scale(self, llm: str, value: float): def write_rope_scale(self, llm: str, value: float):
self.write_float32(constants.KEY_ROPE_SCALE.format(llm=llm), value) self.write_float32(constants.KEY_ROPE_SCALE.format(llm=llm), value)
def write_tokenizer_model(self, model: str):
self.write_string(constants.KEY_TOKENIZER_MODEL, model)
def write_token_list(self, tokens: List[str]):
self.write_array(constants.KEY_TOKENIZER_LIST, tokens)
def write_token_scores(self, scores: List[float]):
self.write_array(constants.KEY_TOKENIZER_SCORES, scores)
# Example usage: # Example usage:
if __name__ == "__main__": if __name__ == "__main__":