CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-11-16 11:27:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

convert-hf : allow converting the weird BitNet 1.3B

Browse Source 
Its FFN size is 5460 which is not convenient.
The offending tensors are kept in F16,
which makes the final model 5.01 bpw.
This commit is contained in:
Francis Couture-Harpin
2024-06-26 22:10:12 -04:00
parent 961e293833
commit 0996149911
2 changed files with 14 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
gguf-py/gguf/quants.py
View File

@@ -126,6 +126,10 @@ def quantize_q8_0(data: np.ndarray):
__q1_3_block_size, __q1_3_type_size = GGML_QUANT_SIZES[GGMLQuantizationType.Q1_3]
def can_quantize_to_q1_3(n: np.ndarray) -> bool:
return n.shape[-1] % __q1_3_block_size == 0
def __quantize_q1_3_shape_change(s: tuple[int, ...]) -> tuple[int, ...]:
return (*s[:-1], s[-1] // __q1_3_block_size * __q1_3_type_size)