CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-10-30 08:42:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

speculative : add infill mode

Browse Source 
ggml-ci
This commit is contained in:
Georgi Gerganov
2024-11-26 11:14:17 +02:00
parent 0eb4e12bee
commit b83cae088c
3 changed files with 26 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
examples/server/server.cpp
View File

@@ -2315,6 +2315,7 @@ struct server_context {
params_spec.n_draft = slot.params.speculative.n_max;
params_spec.n_reuse = llama_n_ctx(slot.ctx_dft) - slot.params.speculative.n_max;
params_spec.p_min = slot.params.speculative.p_min;
params_spec.infill = slot.inf_type == SERVER_TASK_INF_TYPE_INFILL;
llama_tokens draft = common_speculative_gen_draft(slot.spec, params_spec, slot.cache_tokens, id);