CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-11-16 11:27:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

server : fix "can batch with" bug (#17263)

Browse Source
This commit is contained in:
Georgi Gerganov
2025-11-14 14:03:45 +02:00
committed by GitHub
parent 45c6ef7307
commit d396b43748
1 changed files with 10 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
tools/server/server.cpp
View File

@@ -3591,13 +3591,13 @@ struct server_context {
// next, batch any pending prompts without exceeding n_batch // next, batch any pending prompts without exceeding n_batch
if (params_base.cont_batching || batch.n_tokens == 0) { if (params_base.cont_batching || batch.n_tokens == 0) {
for (auto & slot : slots) { for (auto & slot : slots) {
if (!slot.is_processing()) {
continue;
}
// check if we can batch this slot with the previous one // check if we can batch this slot with the previous one
if (slot.is_processing()) { if (slot_batched && !slot_batched->can_batch_with(slot)) {
if (!slot_batched) { continue;
slot_batched = &slot;
} else if (!slot_batched->can_batch_with(slot)) {
continue;
}
} }
// this slot still has a prompt to be processed // this slot still has a prompt to be processed
@@ -4028,6 +4028,10 @@ struct server_context {
} }
} }
if (!slot_batched) {
slot_batched = &slot;
}
if (batch.n_tokens >= n_batch) { if (batch.n_tokens >= n_batch) {
break; break;
} }