CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-11-04 09:32:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

llama : remember and restore kv cache data pointers (#1104)

Browse Source 
because their value is stored in buf and overwritten by memcpy
This commit is contained in:
xaedes
2023-04-21 17:25:21 +02:00
committed by GitHub
parent 1bfc153e2f
commit 8687c1f258
1 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
llama.cpp
View File

@@ -2092,7 +2092,11 @@ void llama_set_kv_cache(
int n_token_count) { int n_token_count) {
// Make sure we have the same kv cache setup // Make sure we have the same kv cache setup
LLAMA_ASSERT(ctx->model.kv_self.buf.size == n_size); LLAMA_ASSERT(ctx->model.kv_self.buf.size == n_size);
void * k_data = ctx->model.kv_self.k->data; // remember data pointers
void * v_data = ctx->model.kv_self.v->data; // because their value is stored in buf and overwritten by memcpy
memcpy(ctx->model.kv_self.buf.addr, kv_cache, n_size); memcpy(ctx->model.kv_self.buf.addr, kv_cache, n_size);
ctx->model.kv_self.k->data = k_data; // restore correct data pointers
ctx->model.kv_self.v->data = v_data;
ctx->model.kv_self.n = n_token_count; ctx->model.kv_self.n = n_token_count;
} }