CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-11-01 09:01:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

convert : for FP8, use scale type to decide auto type

Browse Source
This commit is contained in:
Francis Couture-Harpin
2025-09-09 14:28:10 -04:00
parent 34680f07d2
commit 833d03c25d
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
convert_hf_to_gguf.py
View File

@@ -376,11 +376,13 @@ class ModelBase:
weight_name = name.removesuffix("_scale_inv") weight_name = name.removesuffix("_scale_inv")
w = self.model_tensors[weight_name] w = self.model_tensors[weight_name]
s = self.model_tensors[name] s = self.model_tensors[name]
# TODO: change to FP8 once natively supported
auto_qtype = s.auto_qtype if s.auto_qtype is not gguf.GGMLQuantizationType.F32 else gguf.GGMLQuantizationType.BF16
self.model_tensors[weight_name] = ModelTensorInfo( self.model_tensors[weight_name] = ModelTensorInfo(
load=lambda w=w, s=s: dequant_simple(w.load(), s.load()), load=lambda w=w, s=s: dequant_simple(w.load(), s.load()),
size=w.size, size=w.size,
src_type=w.src_type, src_type=w.src_type,
auto_qtype=gguf.GGMLQuantizationType.BF16, # TODO: change to FP8 once natively supported auto_qtype=auto_qtype,
) )
tensors_to_remove.append(name) tensors_to_remove.append(name)
elif quant_method == "gptq": elif quant_method == "gptq":