CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-10-31 08:51:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

server: fix SSE and OpenAI compatibility for error messages when streaming (#16109)

Browse Source 
* server: fix SSE and OpenAI compatibility for error messages when streaming

* server: remove obsolete event parameter and use required data fieldname instead
This commit is contained in:
Benni
2025-09-20 07:56:30 +02:00
committed by GitHub
parent be79d9fdd9
commit 459c0c2c1a
2 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
tools/server/server.cpp
View File

@@ -4679,17 +4679,17 @@ int main(int argc, char ** argv) {
json res_json = result->to_json();
if (res_json.is_array()) {
for (const auto & res : res_json) {
if (!server_sent_event(sink, "data", res)) {
if (!server_sent_event(sink, res)) {
// sending failed (HTTP connection closed), cancel the generation
return false;
}
}
return true;
} else {
return server_sent_event(sink, "data", res_json);
return server_sent_event(sink, res_json);
}
}, [&](const json & error_data) {
server_sent_event(sink, "error", error_data);
server_sent_event(sink, json{{"error", error_data}});
}, [&sink]() {
// note: do not use req.is_connection_closed here because req is already destroyed
return !sink.is_writable();

4
tools/server/utils.hpp
View File

@@ -459,9 +459,9 @@ static std::string tokens_to_output_formatted_string(const llama_context * ctx,
return out;
}
static bool server_sent_event(httplib::DataSink & sink, const char * event, const json & data) {
static bool server_sent_event(httplib::DataSink & sink, const json & data) {
const std::string str =
std::string(event) + ": " +
"data: " +
data.dump(-1, ' ', false, json::error_handler_t::replace) +
"\n\n"; // required by RFC 8895 - A message is terminated by a blank line (two line terminators in a row).