CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-11-10 10:27:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

quantize : assume the neutral prior is equal imatrix weights

Browse Source
This commit is contained in:
Francis Couture-Harpin
2025-08-05 13:34:01 -04:00
parent 92383bfab3
commit 46a8601140
1 changed files with 8 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
tools/quantize/quantize.cpp
View File

@@ -291,8 +291,15 @@ static int load_imatrix(const std::string & imatrix_file, std::vector<std::strin
for (int64_t j = 0; j < ne1; ++j) { for (int64_t j = 0; j < ne1; ++j) {
const float count = ((const float *) counts->data)[j]; const float count = ((const float *) counts->data)[j];
if (count > 0.0f) { if (count > 0.0f) {
float sumw = 0.0f;
for (int64_t i = 0; i < ne0; ++i) { for (int64_t i = 0; i < ne0; ++i) {
e[j*ne0 + i] = (((const float *) sums->data)[j*ne0 + i] + prior_weight) / (count + prior_weight); sumw += ((const float *) sums->data)[j*ne0 + i];
}
// the neutral prior is equal weights, and it should reduce the variance by weighted-averaging with the mean
const float prior_value = sumw / ne0;
for (int64_t i = 0; i < ne0; ++i) {
e[j*ne0 + i] = (((const float *) sums->data)[j*ne0 + i] + prior_value * prior_weight) / (count + prior_weight);
} }
} else { } else {
// Partial imatrix data, this tensor never got any input during calibration // Partial imatrix data, this tensor never got any input during calibration