CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-11-03 09:22:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

cont : update todos [no ci]

Browse Source
This commit is contained in:
Georgi Gerganov
2025-10-30 20:39:26 +02:00
parent 9d342994f5
commit 93373cc54e
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tools/server/server.cpp
View File

@@ -2699,7 +2699,7 @@ struct server_context {
// return true if at least one slot has been purged
// TODO: improve logic
// - smarter decision which slot to purge
// - smarter decision which slot to purge (LRU or longest prompt?)
// - move slot to level 2 cache instead of removing?
// - instead of purging, try to store and resume later?
bool try_purge_idle_slots() {
@@ -4159,7 +4159,7 @@ struct server_context {
std::string err;
if (n_batch == 1 && ret == 1) {
// TODO: try to terminate only the largest active slot and continue
// TODO: try to terminate only the largest active slot/sequence and continue with the rest
// need to remove the tokens from the current batch too
err = "Context size has been exceeded.";
}