CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-10-31 08:51:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

gguf.py : some code style changes

Browse Source
This commit is contained in:
Georgi Gerganov
2023-07-27 15:37:06 +03:00
parent d2b6ca13ad
commit 158be8f7f4
2 changed files with 89 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
constants.py
View File

@@ -1,32 +1,32 @@
GGUF_MAGIC = 0x47475546 GGUF_MAGIC = 0x47475546
GGUF_VERSION = 1 GGUF_VERSION = 1
# general # general
KEY_GENERAL_ARCHITECTURE = "general.architecture" KEY_GENERAL_ARCHITECTURE = "general.architecture"
KEY_GENERAL_QUANTIZATION_VERSION = "general.quantization_version" KEY_GENERAL_QUANTIZATION_VERSION = "general.quantization_version"
KEY_GENERAL_NAME = "general.name" KEY_GENERAL_NAME = "general.name"
KEY_GENERAL_AUTHOR = "general.author" KEY_GENERAL_AUTHOR = "general.author"
KEY_GENERAL_URL = "general.url" KEY_GENERAL_URL = "general.url"
KEY_GENERAL_DESCRIPTION = "general.description" KEY_GENERAL_DESCRIPTION = "general.description"
KEY_GENERAL_FILE_TYPE = "general.file_type" KEY_GENERAL_FILE_TYPE = "general.file_type"
KEY_GENERAL_LICENSE = "general.license" KEY_GENERAL_LICENSE = "general.license"
KEY_GENERAL_SOURCE_URL = "general.source.url" KEY_GENERAL_SOURCE_URL = "general.source.url"
KEY_GENERAL_SOURCE_HF_REPO = "general.source.hugginface.repository" KEY_GENERAL_SOURCE_HF_REPO = "general.source.hugginface.repository"
# LLM # LLM
KEY_LLM_CONTEXT_LENGTH = "{llm}.context_length" KEY_LLM_CONTEXT_LENGTH = "{llm}.context_length"
KEY_LLM_EMBEDDING_LENGTH = "{llm}.embedding_length" KEY_LLM_EMBEDDING_LENGTH = "{llm}.embedding_length"
KEY_LLM_LAYER_COUNT = "{llm}.layer_count" KEY_LLM_LAYER_COUNT = "{llm}.layer_count"
KEY_LLM_FEED_FORWARD_LENGTH = "{llm}.feed_forward_length" KEY_LLM_FEED_FORWARD_LENGTH = "{llm}.feed_forward_length"
KEY_LLM_USE_PARALLEL_RESIDUAL = "{llm}.use_parallel_residual" KEY_LLM_USE_PARALLEL_RESIDUAL = "{llm}.use_parallel_residual"
KEY_LLM_TENSOR_DATA_LAYOUT = "{llm}.tensor_data_layout" KEY_LLM_TENSOR_DATA_LAYOUT = "{llm}.tensor_data_layout"
# attention # attention
KEY_ATTENTION_HEAD_COUNT = "{llm}.attention.head_count" KEY_ATTENTION_HEAD_COUNT = "{llm}.attention.head_count"
KEY_ATTENTION_HEAD_COUNT_KV = "{llm}.attention.head_count_kv" KEY_ATTENTION_HEAD_COUNT_KV = "{llm}.attention.head_count_kv"
KEY_ATTENTION_MAX_ALIBI_BIAS = "{llm}.attention.max_alibi_bias" KEY_ATTENTION_MAX_ALIBI_BIAS = "{llm}.attention.max_alibi_bias"
KEY_ATTENTION_CLAMP_KQV = "{llm}.attention.clamp_kqv" KEY_ATTENTION_CLAMP_KQV = "{llm}.attention.clamp_kqv"
# RoPE # RoPE
KEY_ROPE_DIMENSION_COUNT = "{llm}.rope.dimension_count" KEY_ROPE_DIMENSION_COUNT = "{llm}.rope.dimension_count"
KEY_ROPE_SCALE = "{llm}.rope.scale" KEY_ROPE_SCALE = "{llm}.rope.scale"

136
gguf.py
View File

@@ -6,14 +6,13 @@
""" """
import struct import struct
import constants
from enum import IntEnum from enum import IntEnum
from typing import List, Any from typing import List, Any
import constants
class GGMLQuantizationType(IntEnum): class GGMLQuantizationType(IntEnum):
F32 = 0 F32 = 0
F16 = 1 F16 = 1
QR_0 = 2 QR_0 = 2
Q4_1 = 3 Q4_1 = 3
# Q4_2 = 4 # support has been removed # Q4_2 = 4 # support has been removed
@@ -31,31 +30,30 @@ class GGMLQuantizationType(IntEnum):
class GGUFValueType(IntEnum): class GGUFValueType(IntEnum):
UINT8 = 0 UINT8 = 0
INT8 = 1 INT8 = 1
UINT16 = 2 UINT16 = 2
INT16 = 3 INT16 = 3
UINT32 = 4 UINT32 = 4
INT32 = 5 INT32 = 5
FLOAT32 = 6 FLOAT32 = 6
BOOL = 7 BOOL = 7
STRING = 8 STRING = 8
ARRAY = 9 ARRAY = 9
@staticmethod @staticmethod
def get_type(value): def get_type(val):
if isinstance(value, str): if isinstance(val, str):
return GGUFValueType.STRING return GGUFValueType.STRING
elif isinstance(value, list): elif isinstance(val, list):
return GGUFValueType.ARRAY return GGUFValueType.ARRAY
elif isinstance(value, float): elif isinstance(val, float):
return GGUFValueType.FLOAT32 return GGUFValueType.FLOAT32
elif isinstance(value, bool): elif isinstance(val, bool):
return GGUFValueType.BOOL return GGUFValueType.BOOL
else: else:
return GGUFValueType.INT32 return GGUFValueType.INT32
class GGUFWriter: class GGUFWriter:
def __init__(self, buffered_writer): def __init__(self, buffered_writer):
self.buffered_writer = buffered_writer self.buffered_writer = buffered_writer
@@ -72,81 +70,81 @@ class GGUFWriter:
return cls(f) return cls(f)
def write_key(self, key: str): def write_key(self, key: str):
self.write_value(key, GGUFValueType.STRING) self.write_val(key, GGUFValueType.STRING)
def write_uint8(self, key: str, value: int): def write_uint8(self, key: str, val: int):
self.write_key(key) self.write_key(key)
self.write_value(value, GGUFValueType.UINT8) self.write_val(val, GGUFValueType.UINT8)
def write_int8(self, key: str, value: int): def write_int8(self, key: str, val: int):
self.write_key(key) self.write_key(key)
self.write_value(value, GGUFValueType.INT8) self.write_val(val, GGUFValueType.INT8)
def write_uint16(self, key: str, value: int): def write_uint16(self, key: str, val: int):
self.write_key(key) self.write_key(key)
self.write_value(value, GGUFValueType.UINT16) self.write_val(val, GGUFValueType.UINT16)
def write_int16(self, key: str, value: int): def write_int16(self, key: str, val: int):
self.write_key(key) self.write_key(key)
self.write_value(value, GGUFValueType.INT16) self.write_val(val, GGUFValueType.INT16)
def write_uint32(self, key: str, value: int): def write_uint32(self, key: str, val: int):
self.write_key(key) self.write_key(key)
self.write(value, GGUFValueType.UINT32) self.write_val(val, GGUFValueType.UINT32)
def write_int32(self, key: str, value: int): def write_int32(self, key: str, val: int):
self.write_key(key) self.write_key(key)
self.write_value(value, GGUFValueType.INT32) self.write_val(val, GGUFValueType.INT32)
def write_float32(self, key: str, value: float): def write_float32(self, key: str, val: float):
self.write_key(key) self.write_key(key)
self.write_value(value, GGUFValueType.FLOAT32) self.write_val(val, GGUFValueType.FLOAT32)
def write_bool(self, key: str, value: bool): def write_bool(self, key: str, val: bool):
self.write_key(key) self.write_key(key)
self.write_value(value, GGUFValueType.BOOL) self.write_val(val, GGUFValueType.BOOL)
def write_string(self, key: str, value: str): def write_string(self, key: str, val: str):
self.write_key(key) self.write_key(key)
self.write_value(value, GGUFValueType.STRING) self.write_val(val, GGUFValueType.STRING)
def write_array(self, key: str, value: list): def write_array(self, key: str, val: list):
if not isinstance(value, list): if not isinstance(val, list):
raise ValueError("Value must be a list for array type") raise ValueError("Value must be a list for array type")
self.write_key(key) self.write_key(key)
self.write_value(value, GGUFValueType.ARRAY) self.write_val(val, GGUFValueType.ARRAY)
def write_value(self: str, value: Any, value_type: GGUFValueType = None): def write_val(self: str, val: Any, vtype: GGUFValueType = None):
if value_type is None: if vtype is None:
value_type = GGUFValueType.get_type(value) vtype = GGUFValueType.get_type(val)
self.buffered_writer.write(struct.pack("<I", value_type)) self.buffered_writer.write(struct.pack("<I", vtype))
if value_type == GGUFValueType.UINT8: if vtype == GGUFValueType.UINT8:
self.buffered_writer.write(struct.pack("<B", value)) self.buffered_writer.write(struct.pack("<B", val))
elif value_type == GGUFValueType.INT8: elif vtype == GGUFValueType.INT8:
self.buffered_writer.write(struct.pack("<b", value)) self.buffered_writer.write(struct.pack("<b", val))
elif value_type == GGUFValueType.UINT16: elif vtype == GGUFValueType.UINT16:
self.buffered_writer.write(struct.pack("<H", value)) self.buffered_writer.write(struct.pack("<H", val))
elif value_type == GGUFValueType.INT16: elif vtype == GGUFValueType.INT16:
self.buffered_writer.write(struct.pack("<h", value)) self.buffered_writer.write(struct.pack("<h", val))
elif value_type == GGUFValueType.UINT32: elif vtype == GGUFValueType.UINT32:
self.buffered_writer.write(struct.pack("<I", value)) self.buffered_writer.write(struct.pack("<I", val))
elif value_type == GGUFValueType.INT32: elif vtype == GGUFValueType.INT32:
self.buffered_writer.write(struct.pack("<i", value)) self.buffered_writer.write(struct.pack("<i", val))
elif value_type == GGUFValueType.FLOAT32: elif vtype == GGUFValueType.FLOAT32:
self.buffered_writer.write(struct.pack("<f", value)) self.buffered_writer.write(struct.pack("<f", val))
elif value_type == GGUFValueType.BOOL: elif vtype == GGUFValueType.BOOL:
self.buffered_writer.write(struct.pack("?", value)) self.buffered_writer.write(struct.pack("?", val))
elif value_type == GGUFValueType.STRING: elif vtype == GGUFValueType.STRING:
encoded_value = value.encode("utf8") encoded_val = val.encode("utf8")
self.buffered_writer.write(struct.pack("<I", len(encoded_value))) self.buffered_writer.write(struct.pack("<I", len(encoded_val)))
self.buffered_writer.write(encoded_value) self.buffered_writer.write(encoded_val)
elif value_type == GGUFValueType.ARRAY: elif vtype == GGUFValueType.ARRAY:
self.buffered_writer.write(struct.pack("<I", len(value))) self.buffered_writer.write(struct.pack("<I", len(val)))
for item in value: for item in val:
self.write_value(item) self.write_val(item)
else: else:
raise ValueError("Invalid GGUF metadata value type") raise ValueError("Invalid GGUF metadata value type")