CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-10-31 08:51:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

llama-vocab : add SuperBPE pre-tokenizer

Browse Source
This commit is contained in:
Francis Couture-Harpin
2025-03-23 16:19:03 -04:00
parent 77f9c6bbe5
commit 35c2f8b9ff
4 changed files with 15 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
convert_hf_to_gguf.py
View File

@@ -705,6 +705,9 @@ class Model:
if chkhsh == "ccc2ef013c104be7bae2965776d611e1d7a8a2a9c547dd93a682c9a9fc80352e":
# ref: https://huggingface.co/Xenova/gpt-4o
res = "gpt-4o"
if chkhsh == "7dec86086fcc38b66b7bc1575a160ae21cf705be7718b9d5598190d7c12db76f":
# ref: https://huggingface.co/UW/OLMo2-8B-SuperBPE-t180k
res = "superbpe"
if res is None:
logger.warning("\n")