CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-10-27 08:21:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

server: fix SSE and OpenAI compatibility for error messages when streaming (#16109)

Browse Source 
* server: fix SSE and OpenAI compatibility for error messages when streaming

* server: remove obsolete event parameter and use required data fieldname instead
This commit is contained in:
Benni
2025-09-20 07:56:30 +02:00
committed by GitHub
parent be79d9fdd9
commit 459c0c2c1a
2 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
tools/server/server.cpp
View File

@@ -4679,17 +4679,17 @@ int main(int argc, char ** argv) {
json res_json = result->to_json(); json res_json = result->to_json();
if (res_json.is_array()) { if (res_json.is_array()) {
for (const auto & res : res_json) { for (const auto & res : res_json) {
if (!server_sent_event(sink, "data", res)) { if (!server_sent_event(sink, res)) {
// sending failed (HTTP connection closed), cancel the generation // sending failed (HTTP connection closed), cancel the generation
return false; return false;
} }
} }
return true; return true;
} else { } else {
return server_sent_event(sink, "data", res_json); return server_sent_event(sink, res_json);
} }
}, [&](const json & error_data) { }, [&](const json & error_data) {
server_sent_event(sink, "error", error_data); server_sent_event(sink, json{{"error", error_data}});
}, [&sink]() { }, [&sink]() {
// note: do not use req.is_connection_closed here because req is already destroyed // note: do not use req.is_connection_closed here because req is already destroyed
return !sink.is_writable(); return !sink.is_writable();

4
tools/server/utils.hpp
View File

@@ -459,9 +459,9 @@ static std::string tokens_to_output_formatted_string(const llama_context * ctx,
return out; return out;
} }
static bool server_sent_event(httplib::DataSink & sink, const char * event, const json & data) { static bool server_sent_event(httplib::DataSink & sink, const json & data) {
const std::string str = const std::string str =
std::string(event) + ": " + "data: " +
data.dump(-1, ' ', false, json::error_handler_t::replace) + data.dump(-1, ' ', false, json::error_handler_t::replace) +
"\n\n"; // required by RFC 8895 - A message is terminated by a blank line (two line terminators in a row). "\n\n"; // required by RFC 8895 - A message is terminated by a blank line (two line terminators in a row).