CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-11-03 09:22:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

gguf.py : bytesarray for gpt2bpe tokenizer

Browse Source
This commit is contained in:
klosax
2023-08-04 04:07:57 +02:00
committed by GitHub
parent fb0b243705
commit 278ada9572
1 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
gguf.py
View File

@@ -10,7 +10,7 @@ from enum import IntEnum
from typing import Any, IO, List from typing import Any, IO, List
import numpy as np import numpy as np
import sys
class GGMLQuantizationType(IntEnum): class GGMLQuantizationType(IntEnum):
F32 = 0 F32 = 0
@@ -45,7 +45,7 @@ class GGUFValueType(IntEnum):
@staticmethod @staticmethod
def get_type(val): def get_type(val):
if isinstance(val, str) or isinstance(val, bytes): if isinstance(val, str) or isinstance(val, bytes) or isinstance(val, bytearray):
return GGUFValueType.STRING return GGUFValueType.STRING
elif isinstance(val, list): elif isinstance(val, list):
return GGUFValueType.ARRAY return GGUFValueType.ARRAY
@@ -53,8 +53,11 @@ class GGUFValueType(IntEnum):
return GGUFValueType.FLOAT32 return GGUFValueType.FLOAT32
elif isinstance(val, bool): elif isinstance(val, bool):
return GGUFValueType.BOOL return GGUFValueType.BOOL
else: elif isinstance(val, int):
return GGUFValueType.INT32 return GGUFValueType.INT32
else:
print("Unknown type: "+str(type(val)))
sys.exit()
class GGUFWriter: class GGUFWriter: