CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-11-01 09:01:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

parallel : fix bug (extra BOS) + smaller token_prev array

Browse Source
This commit is contained in:
Georgi Gerganov
2023-09-20 17:32:21 +03:00
parent 1be2b8c19b
commit ee1d670cc6
1 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
examples/parallel/parallel.cpp
View File

@@ -114,7 +114,7 @@ int main(int argc, char ** argv) {
for (size_t i = 0; i < clients.size(); ++i) {
auto & client = clients[i];
client.id = i;
client.tokens_prev.resize(n_ctx);
client.tokens_prev.resize(params.n_predict);
std::fill(client.tokens_prev.begin(), client.tokens_prev.end(), 0);
}
@@ -191,6 +191,8 @@ int main(int argc, char ** argv) {
for (int i = 0; i < n_clients; ++i) {
llama_kv_cache_seq_rm(ctx, i, n_tokens_system, -1);
}
LOG_TEE("%s: clearing the KV cache\n", __func__);
}
// insert new sequences for decoding
@@ -208,8 +210,9 @@ int main(int argc, char ** argv) {
std::fill(client.tokens_prev.begin(), client.tokens_prev.end(), 0);
// do not prepend BOS because we have a system prompt!
std::vector<llama_token> tokens_prompt;
tokens_prompt = ::llama_tokenize(ctx, client.prompt, true);
tokens_prompt = ::llama_tokenize(ctx, client.prompt, false);
for (size_t i = 0; i < tokens_prompt.size(); ++i) {
batch.token [batch.n_tokens] = tokens_prompt[i];