CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-10-30 08:42:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

llama-graph : fix text position for mrope (#13159)

Browse Source 
* llama-graph : fix text position for mrope

* fix typo

* explicitly set 4th dim in the loop
This commit is contained in:
Xuan-Son Nguyen
2025-04-29 08:45:49 +02:00
committed by GitHub
parent 5f5e39e1ba
commit b6ce7430b7
1 changed files with 7 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/llama-graph.cpp
View File

@@ -55,13 +55,16 @@ void llm_graph_input_pos::set_input(const llama_ubatch * ubatch) {
if (ubatch->pos && pos) { if (ubatch->pos && pos) {
const int64_t n_tokens = ubatch->n_tokens; const int64_t n_tokens = ubatch->n_tokens;
if (ubatch->token && n_pos_per_embd > 1) { if (ubatch->token && n_pos_per_embd == 4) {
// in case we're using M-RoPE with text tokens, convert the 1D positions to 4D // in case we're using M-RoPE with text tokens, convert the 1D positions to 4D
// the other dimensions are all 0, they are unused for text tokens // the 3 first dims are the same, and 4th dim is all 0
std::vector<llama_pos> pos_data(n_tokens*n_pos_per_embd, 0); std::vector<llama_pos> pos_data(n_tokens*n_pos_per_embd);
// copy the first dimension // copy the first dimension
for (int i = 0; i < n_tokens; ++i) { for (int i = 0; i < n_tokens; ++i) {
pos_data[i] = ubatch->pos[i]; pos_data[ i] = ubatch->pos[i];
pos_data[ n_tokens + i] = ubatch->pos[i];
pos_data[2 * n_tokens + i] = ubatch->pos[i];
pos_data[3 * n_tokens + i] = 0; // 4th dim is 0
} }
ggml_backend_tensor_set(pos, pos_data.data(), 0, pos_data.size()*ggml_element_size(pos)); ggml_backend_tensor_set(pos, pos_data.data(), 0, pos_data.size()*ggml_element_size(pos));
} else { } else {