CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-10-30 08:42:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

server : fix extra BOS in infill endpoint (#11106)

Browse Source 
* server : fix extra BOS in infill endpoing

ggml-ci

* server : update infill tests
This commit is contained in:
Georgi Gerganov
2025-01-06 15:36:08 +02:00
committed by GitHub
parent 09186fabbe
commit e6e7c75d94
2 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
examples/server/server.cpp
View File

@@ -3797,7 +3797,7 @@ int main(int argc, char ** argv) {
data["input_extra"] = input_extra; // default to empty array if it's not exist data["input_extra"] = input_extra; // default to empty array if it's not exist
std::string prompt = json_value(data, "prompt", std::string()); std::string prompt = json_value(data, "prompt", std::string());
std::vector<llama_tokens> tokenized_prompts = tokenize_input_prompts(ctx_server.ctx, prompt, true, true); std::vector<llama_tokens> tokenized_prompts = tokenize_input_prompts(ctx_server.ctx, prompt, false, true);
SRV_DBG("creating infill tasks, n_prompts = %d\n", (int) tokenized_prompts.size()); SRV_DBG("creating infill tasks, n_prompts = %d\n", (int) tokenized_prompts.size());
data["prompt"] = format_infill( data["prompt"] = format_infill(
ctx_server.ctx, ctx_server.ctx,

2
examples/server/tests/unit/test_infill.py
View File

@@ -18,7 +18,7 @@ def test_infill_without_input_extra():
"input_suffix": "}\n", "input_suffix": "}\n",
}) })
assert res.status_code == 200 assert res.status_code == 200
assert match_regex("(Ann|small|shiny)+", res.body["content"]) assert match_regex("(Ann|small|shiny|Daddy)+", res.body["content"])
def test_infill_with_input_extra(): def test_infill_with_input_extra():