CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-11-06 09:46:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

pre-norm -> norm

Browse Source
This commit is contained in:
younesbelkada
2025-07-04 14:58:33 +04:00
parent 243e4d1a50
commit cce35498d5
3 changed files with 3 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/llama-arch.cpp
View File

@@ -1068,7 +1068,7 @@ static const std::map<llm_arch, std::map<llm_tensor, const char *>> LLM_TENSOR_N
{ LLM_TENSOR_SSM_D, "blk.%d.ssm_d" }, { LLM_TENSOR_SSM_D, "blk.%d.ssm_d" },
{ LLM_TENSOR_SSM_NORM, "blk.%d.ssm_norm" }, { LLM_TENSOR_SSM_NORM, "blk.%d.ssm_norm" },
{ LLM_TENSOR_SSM_OUT, "blk.%d.ssm_out" }, { LLM_TENSOR_SSM_OUT, "blk.%d.ssm_out" },
{ LLM_TENSOR_FFN_PRE_NORM, "blk.%d.ffn_pre_norm" }, { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_pre_norm" },
{ LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" }, { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },

4
src/llama-model.cpp
View File

@@ -4595,7 +4595,7 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
// feed forward (w/ optional biases) // feed forward (w/ optional biases)
layer.ffn_pre_norm = create_tensor(tn(LLM_TENSOR_FFN_PRE_NORM, i), {hidden_size}, 0); layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_PRE_NORM, i), {hidden_size}, 0);
layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_rot/2}, llama_model_loader::TENSOR_NOT_REQUIRED | (i != 0 ? llama_model_loader::TENSOR_DUPLICATED : 0)); layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_rot/2}, llama_model_loader::TENSOR_NOT_REQUIRED | (i != 0 ? llama_model_loader::TENSOR_DUPLICATED : 0));
layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {hidden_size, ffn_intermediate_size}, 0); layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {hidden_size, ffn_intermediate_size}, 0);
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { ffn_intermediate_size, hidden_size}, 0); layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { ffn_intermediate_size, hidden_size}, 0);
@@ -14751,7 +14751,7 @@ struct llm_build_falcon_h1 : public llm_graph_context {
// feed-forward network // feed-forward network
cur = build_norm(ffn_inp, cur = build_norm(ffn_inp,
model.layers[il].ffn_pre_norm, NULL, model.layers[il].ffn_norm, NULL,
LLM_NORM_RMS, il); LLM_NORM_RMS, il);
cb(cur, "ffn_norm", il); cb(cur, "ffn_norm", il);

1
src/llama-model.h
View File

@@ -216,7 +216,6 @@ struct llama_layer {
struct ggml_tensor * layer_out_norm_b = nullptr; struct ggml_tensor * layer_out_norm_b = nullptr;
struct ggml_tensor * ffn_norm_exps = nullptr; struct ggml_tensor * ffn_norm_exps = nullptr;
struct ggml_tensor * ffn_norm_enc = nullptr; struct ggml_tensor * ffn_norm_enc = nullptr;
struct ggml_tensor * ffn_pre_norm = nullptr;
// ff // ff
struct ggml_tensor * ffn_gate = nullptr; // w1 struct ggml_tensor * ffn_gate = nullptr; // w1