diff --git a/.clang-format b/.clang-format
index 45232b80ed..47d96b6b40 100644
--- a/.clang-format
+++ b/.clang-format
@@ -22,8 +22,8 @@ AllowShortIfStatementsOnASingleLine: Never
AllowShortLambdasOnASingleLine: Inline
AllowShortLoopsOnASingleLine: false
AlwaysBreakBeforeMultilineStrings: true
-BinPackArguments: true
-BinPackParameters: true # OnePerLine
+BinPackArguments: false
+BinPackParameters: false # OnePerLine
BitFieldColonSpacing: Both
BreakBeforeBraces: Custom # Attach
BraceWrapping:
@@ -70,15 +70,18 @@ ExperimentalAutoDetectBinPacking: false
FixNamespaceComments: true
IncludeBlocks: Regroup
IncludeCategories:
- - Regex: '^<.*\.h>'
+ - Regex: '".*"'
Priority: 1
SortPriority: 0
- - Regex: '^<.*'
+ - Regex: '^<.*\.h>'
Priority: 2
SortPriority: 0
- - Regex: '.*'
+ - Regex: '^<.*'
Priority: 3
SortPriority: 0
+ - Regex: '.*'
+ Priority: 4
+ SortPriority: 0
IncludeIsMainRegex: '([-_](test|unittest))?$'
IncludeIsMainSourceRegex: ''
IndentAccessModifiers: false
diff --git a/.devops/musa.Dockerfile b/.devops/musa.Dockerfile
index 87ce2393f6..b0c86dccd5 100644
--- a/.devops/musa.Dockerfile
+++ b/.devops/musa.Dockerfile
@@ -1,10 +1,10 @@
ARG UBUNTU_VERSION=22.04
# This needs to generally match the container host's environment.
-ARG MUSA_VERSION=rc4.0.1
+ARG MUSA_VERSION=rc4.2.0
# Target the MUSA build image
-ARG BASE_MUSA_DEV_CONTAINER=mthreads/musa:${MUSA_VERSION}-mudnn-devel-ubuntu${UBUNTU_VERSION}
+ARG BASE_MUSA_DEV_CONTAINER=mthreads/musa:${MUSA_VERSION}-devel-ubuntu${UBUNTU_VERSION}-amd64
-ARG BASE_MUSA_RUN_CONTAINER=mthreads/musa:${MUSA_VERSION}-mudnn-runtime-ubuntu${UBUNTU_VERSION}
+ARG BASE_MUSA_RUN_CONTAINER=mthreads/musa:${MUSA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}-amd64
FROM ${BASE_MUSA_DEV_CONTAINER} AS build
diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix
index 6e8050a499..651a54db4c 100644
--- a/.devops/nix/package.nix
+++ b/.devops/nix/package.nix
@@ -47,6 +47,7 @@ let
inherit (lib)
cmakeBool
cmakeFeature
+ optionalAttrs
optionals
strings
;
@@ -197,7 +198,7 @@ effectiveStdenv.mkDerivation (finalAttrs: {
];
# Environment variables needed for ROCm
- env = optionals useRocm {
+ env = optionalAttrs useRocm {
ROCM_PATH = "${rocmPackages.clr}";
HIP_DEVICE_LIB_PATH = "${rocmPackages.rocm-device-libs}/amdgcn/bitcode";
};
diff --git a/.devops/rocm.Dockerfile b/.devops/rocm.Dockerfile
index 1c00f1b9c2..cf19e6e028 100644
--- a/.devops/rocm.Dockerfile
+++ b/.devops/rocm.Dockerfile
@@ -1,8 +1,8 @@
ARG UBUNTU_VERSION=24.04
# This needs to generally match the container host's environment.
-ARG ROCM_VERSION=6.3
-ARG AMDGPU_VERSION=6.3
+ARG ROCM_VERSION=6.4
+ARG AMDGPU_VERSION=6.4
# Target the CUDA build image
ARG BASE_ROCM_DEV_CONTAINER=rocm/dev-ubuntu-${UBUNTU_VERSION}:${ROCM_VERSION}-complete
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 5bd988b7f7..c6d51fb0c2 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -515,7 +515,7 @@ jobs:
ubuntu-22-cmake-musa:
runs-on: ubuntu-22.04
- container: mthreads/musa:rc4.0.1-mudnn-devel-ubuntu22.04
+ container: mthreads/musa:rc4.2.0-devel-ubuntu22.04-amd64
steps:
- name: Clone
diff --git a/.github/workflows/close-issue.yml b/.github/workflows/close-issue.yml
index 276a217d45..19e7854745 100644
--- a/.github/workflows/close-issue.yml
+++ b/.github/workflows/close-issue.yml
@@ -17,7 +17,7 @@ jobs:
steps:
- uses: actions/stale@v5
with:
- exempt-issue-labels: "refactor,help wanted,good first issue,research,bug,roadmap"
+ exempt-issue-labels: "refactoring,help wanted,good first issue,research,bug,roadmap"
days-before-issue-stale: 30
days-before-issue-close: 14
stale-issue-label: "stale"
diff --git a/.gitignore b/.gitignore
index f8ceb1560a..f48ce4cacd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -82,6 +82,7 @@ models/*
models-mnt
!models/.editorconfig
!models/ggml-vocab-*.gguf*
+!models/templates
# Zig
zig-out/
diff --git a/CODEOWNERS b/CODEOWNERS
index f87035343c..b698b1f4cb 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -9,4 +9,5 @@
/ggml/src/ggml-cuda/mmvq.* @JohannesGaessler
/ggml/src/ggml-opt.cpp @JohannesGaessler
/ggml/src/gguf.cpp @JohannesGaessler
+/ggml/src/ggml-vulkan/ @0cc4m
/ggml/src/ggml-zdnn/ @taronaeo
diff --git a/README.md b/README.md
index edde61238c..9b2e0f851c 100644
--- a/README.md
+++ b/README.md
@@ -270,7 +270,6 @@ Instructions for adding support for new models: [HOWTO-add-model.md](docs/develo
| [CANN](docs/build.md#cann) | Ascend NPU |
| [OpenCL](docs/backend/OPENCL.md) | Adreno GPU |
| [WebGPU [In Progress]](docs/build.md#webgpu) | All |
-
| [RPC](https://github.com/ggml-org/llama.cpp/tree/master/tools/rpc) | All |
## Obtaining and quantizing models
@@ -436,7 +435,7 @@ To learn more about model quantization, [read this documentation](tools/quantize
## [`llama-perplexity`](tools/perplexity)
-#### A tool for measuring the perplexity [^1][^2] (and other quality metrics) of a model over a given text.
+#### A tool for measuring the [perplexity](tools/perplexity/README.md) [^1] (and other quality metrics) of a model over a given text.
-
Measure the perplexity over a text file
@@ -459,8 +458,7 @@ To learn more about model quantization, [read this documentation](tools/quantize
-[^1]: [tools/perplexity/README.md](./tools/perplexity/README.md)
-[^2]: [https://huggingface.co/docs/transformers/perplexity](https://huggingface.co/docs/transformers/perplexity)
+[^1]: [https://huggingface.co/docs/transformers/perplexity](https://huggingface.co/docs/transformers/perplexity)
## [`llama-bench`](tools/llama-bench)
diff --git a/ci/README.md b/ci/README.md
index 6e297f1a82..8eebe988d5 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -54,7 +54,7 @@ docker run --privileged -it \
-v $HOME/llama.cpp/ci-cache:/ci-cache \
-v $HOME/llama.cpp/ci-results:/ci-results \
-v $PWD:/ws -w /ws \
- mthreads/musa:rc4.0.1-mudnn-devel-ubuntu22.04
+ mthreads/musa:rc4.2.0-devel-ubuntu22.04-amd64
```
Inside the container, execute the following commands:
diff --git a/common/arg.cpp b/common/arg.cpp
index c1151f51da..060053595d 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -1612,7 +1612,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
[](common_params & params, const std::string & value) {
params.antiprompt.emplace_back(value);
}
- ).set_examples({LLAMA_EXAMPLE_MAIN}));
+ ).set_examples({LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
{"-sp", "--special"},
string_format("special tokens output enabled (default: %s)", params.special ? "true" : "false"),
@@ -2655,6 +2655,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.i_chunk = value;
}
).set_examples({LLAMA_EXAMPLE_IMATRIX}));
+ add_opt(common_arg(
+ {"--show-statistics"},
+ string_format("show imatrix statistics and then exit (default: %s)", params.show_statistics ? "true" : "false"),
+ [](common_params & params) {
+ params.show_statistics = true;
+ }
+ ).set_examples({LLAMA_EXAMPLE_IMATRIX}));
add_opt(common_arg(
{"--parse-special"},
string_format("prase special tokens (chat, tool, etc) (default: %s)", params.parse_special ? "true" : "false"),
diff --git a/common/chat.cpp b/common/chat.cpp
index 114dbfccdb..0c777d7a78 100644
--- a/common/chat.cpp
+++ b/common/chat.cpp
@@ -1944,6 +1944,8 @@ common_chat_msg common_chat_parse(const std::string & input, bool is_partial, co
}
}
auto msg = builder.result();
- LOG_DBG("Parsed message: %s\n", common_chat_msgs_to_json_oaicompat({msg}).at(0).dump().c_str());
+ if (!is_partial) {
+ LOG_DBG("Parsed message: %s\n", common_chat_msgs_to_json_oaicompat({msg}).at(0).dump().c_str());
+ }
return msg;
}
diff --git a/common/common.cpp b/common/common.cpp
index 466271be61..d8c4d988b6 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -448,6 +448,15 @@ void string_replace_all(std::string & s, const std::string & search, const std::
bool string_ends_with(const std::string_view & str, const std::string_view & suffix) {
return str.size() >= suffix.size() && str.compare(str.size()-suffix.size(), suffix.size(), suffix) == 0;
}
+
+bool string_remove_suffix(std::string & str, const std::string_view & suffix) {
+ bool has_suffix = string_ends_with(str, suffix);
+ if (has_suffix) {
+ str = str.substr(0, str.size() - suffix.size());
+ }
+ return has_suffix;
+}
+
size_t string_find_partial_stop(const std::string_view & str, const std::string_view & stop) {
if (!str.empty() && !stop.empty()) {
const char text_last_char = str.back();
diff --git a/common/common.h b/common/common.h
index 27adf55246..00f42694ea 100644
--- a/common/common.h
+++ b/common/common.h
@@ -432,9 +432,10 @@ struct common_params {
int32_t n_save_freq = 0; // save the imatrix every n_save_freq iterations
int32_t i_chunk = 0; // start processing from this chunk
- bool process_output = false; // collect data for the output tensor
- bool compute_ppl = true; // whether to compute perplexity
- bool parse_special = false; // whether to parse special tokens during imatrix tokenization
+ bool process_output = false; // collect data for the output tensor
+ bool compute_ppl = true; // whether to compute perplexity
+ bool show_statistics = false; // show imatrix statistics per tensor
+ bool parse_special = false; // whether to parse special tokens during imatrix tokenization
// cvector-generator params
int n_pca_batch = 100;
@@ -534,6 +535,7 @@ static bool string_starts_with(const std::string & str,
// While we wait for C++20's std::string::ends_with...
bool string_ends_with(const std::string_view & str, const std::string_view & suffix);
+bool string_remove_suffix(std::string & str, const std::string_view & suffix);
size_t string_find_partial_stop(const std::string_view & str, const std::string_view & stop);
bool string_parse_kv_override(const char * data, std::vector & overrides);
diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py
index d802524bba..3f5cefe007 100755
--- a/convert_hf_to_gguf.py
+++ b/convert_hf_to_gguf.py
@@ -843,6 +843,9 @@ class TextModel(ModelBase):
if chkhsh == "169bf0296a13c4d9b7672313f749eb36501d931022de052aad6e36f2bf34dd51":
# ref: https://huggingface.co/LiquidAI/LFM2-Tokenizer
res = "lfm2"
+ if chkhsh == "2085e1638f6c377a0aa4ead21b27bb4cb941bf800df86ed391011769c1758dfb":
+ # ref: https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B
+ res = "exaone4"
if res is None:
logger.warning("\n")
@@ -1897,6 +1900,7 @@ class StableLMModel(TextModel):
"MixtralForCausalLM",
"VLlama3ForCausalLM",
"LlavaForConditionalGeneration",
+ "VoxtralForConditionalGeneration",
"LlamaModel")
class LlamaModel(TextModel):
model_arch = gguf.MODEL_ARCH.LLAMA
@@ -1909,6 +1913,11 @@ class LlamaModel(TextModel):
self.hparams["num_attention_heads"] = self.hparams.get("num_attention_heads", 32)
def set_vocab(self):
+ path_tekken_json = self.dir_model / "tekken.json"
+ path_tokenizer_json = self.dir_model / "tokenizer.json"
+ if path_tekken_json.is_file() and not path_tokenizer_json.is_file():
+ return self.set_vocab_tekken()
+
try:
self._set_vocab_sentencepiece()
except FileNotFoundError:
@@ -1941,6 +1950,52 @@ class LlamaModel(TextModel):
if self.hparams.get("vocab_size", 32000) == 49152:
self.gguf_writer.add_add_bos_token(False)
+ def set_vocab_tekken(self):
+ vocab = gguf.vocab.MistralVocab(self.dir_model)
+ self.gguf_writer.add_tokenizer_model(vocab.gguf_tokenizer_model)
+
+ tokens = []
+ scores = []
+ toktypes = []
+
+ for text, score, toktype in vocab.all_tokens():
+ tokens.append(text)
+ scores.append(score)
+ toktypes.append(toktype)
+
+ assert len(tokens) == vocab.vocab_size, (
+ f"token count ({len(tokens)}) != vocab size ({vocab.vocab_size})"
+ )
+
+ if vocab.tokenizer_type == gguf.vocab.MistralTokenizerType.tekken:
+ self.gguf_writer.add_tokenizer_pre("tekken")
+ self.gguf_writer.add_token_merges(
+ vocab.extract_vocab_merges_from_model()
+ )
+
+ logger.info(
+ f"Setting bos, eos, unk and pad token IDs to {vocab.bos_id}, {vocab.eos_id}, {vocab.unk_id}, {vocab.pad_id}."
+ )
+
+ self.gguf_writer.add_bos_token_id(vocab.bos_id)
+ self.gguf_writer.add_eos_token_id(vocab.eos_id)
+ self.gguf_writer.add_unk_token_id(vocab.unk_id)
+ self.gguf_writer.add_pad_token_id(vocab.pad_id)
+
+ self.gguf_writer.add_token_list(tokens)
+ self.gguf_writer.add_token_scores(scores)
+ self.gguf_writer.add_token_types(toktypes)
+ self.gguf_writer.add_vocab_size(vocab.vocab_size)
+
+ self.gguf_writer.add_add_bos_token(True)
+ self.gguf_writer.add_add_eos_token(False)
+
+ script_dir = Path(__file__).parent
+ template_path = script_dir / "models/templates/unsloth-mistral-Devstral-Small-2507.jinja"
+ with open(template_path, "r", encoding="utf-8") as f:
+ template = f.read()
+ self.gguf_writer.add_chat_template(template)
+
def set_gguf_parameters(self):
super().set_gguf_parameters()
hparams = self.hparams
@@ -1968,12 +2023,13 @@ class LlamaModel(TextModel):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
n_head = self.hparams["num_attention_heads"]
n_kv_head = self.hparams.get("num_key_value_heads")
- is_vision_tensor = "vision_tower" in name \
+ is_multimodal_tensor = "vision_tower" in name \
or "vision_model" in name \
+ or "audio_tower" in name \
or "model.connector" in name \
or "multi_modal_projector" in name
- if is_vision_tensor:
+ if is_multimodal_tensor:
return [] # skip vision tensors
elif self.hf_arch == "LlamaModel":
name = "model." + name
@@ -2861,7 +2917,8 @@ class Ernie4_5Model(TextModel):
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
num_heads = self.hparams["num_attention_heads"]
num_kv_heads = self.hparams["num_key_value_heads"]
- head_dim = self.hparams["head_dim"]
+ if (head_dim := self.hparams.get("head_dim")) is None:
+ head_dim = self.hparams["hidden_size"] // num_heads
if "ernie." in name:
name = name.replace("ernie.", "model.")
@@ -2894,6 +2951,93 @@ class Ernie4_5Model(TextModel):
return [(self.map_tensor_name(name), data_torch)]
+@ModelBase.register("Ernie4_5_MoeForCausalLM")
+class Ernie4_5MoeModel(Ernie4_5Model):
+ model_arch = gguf.MODEL_ARCH.ERNIE4_5_MOE
+ _experts: list[dict[str, Tensor]] | None = None
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._experts = [{} for _ in range(self.block_count)]
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_expert_count(self.hparams["moe_num_experts"])
+ self.gguf_writer.add_expert_used_count(self.hparams["moe_k"])
+ self.gguf_writer.add_interleave_moe_layer_step(self.hparams["moe_layer_interval"])
+ self.gguf_writer.add_leading_dense_block_count(self.hparams["moe_layer_start_index"])
+ if (moe_intermediate_size := self.hparams.get("moe_intermediate_size")) is not None:
+ self.gguf_writer.add_expert_feed_forward_length(moe_intermediate_size)
+ if (shared_expert_count := self.hparams.get('moe_num_shared_experts')) is not None:
+ self.gguf_writer.add_expert_shared_count(shared_expert_count)
+ if shared_expert_count > 0 and (shared_expert_intermediate_size := self.hparams.get('intermediate_size')) is not None and (num_key_value_heads := self.hparams.get('num_key_value_heads')) is not None:
+ self.gguf_writer.add_expert_shared_feed_forward_length(shared_expert_intermediate_size // num_key_value_heads)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # Modify correction bias name as in DeepseekV2
+ if name.endswith("e_score_correction_bias"):
+ name = name.replace("e_score_correction_bias", "e_score_correction.bias")
+
+ # skip Multi-Token Prediction (MTP) layers (again, same as DeepseekV2)
+ match = re.match(r"model.mtp_block.(\d+)", name)
+ if match:
+ return []
+
+ # skip all other MTP tensors for now
+ match = re.match(r"model.mtp_emb_norm.(\d+)", name)
+ if match:
+ return []
+
+ match = re.match(r"model.mtp_hidden_norm.(\d+)", name)
+ if match:
+ return []
+
+ match = re.match(r"model.mtp_linear_proj.(\d+)", name)
+ if match:
+ return []
+
+ # process the experts separately
+ if name.find("mlp.experts") != -1:
+ n_experts = self.hparams["moe_num_experts"]
+ assert bid is not None
+
+ if self._experts is None:
+ self._experts = [{} for _ in range(self.block_count)]
+
+ self._experts[bid][name] = data_torch
+
+ if len(self._experts[bid]) >= n_experts * 3:
+ tensors: list[tuple[str, Tensor]] = []
+
+ # merge the experts into a single 3d tensor
+ for w_name in ["gate_proj", "up_proj", "down_proj"]:
+ datas: list[Tensor] = []
+
+ for xid in range(n_experts):
+ ename_to_retrieve = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight"
+ datas.append(self._experts[bid][ename_to_retrieve])
+ del self._experts[bid][ename_to_retrieve]
+
+ data_torch = torch.stack(datas, dim=0)
+ merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
+ new_name = self.map_tensor_name(merged_name)
+ tensors.append((new_name, data_torch))
+
+ return tensors
+ else:
+ return []
+ return [(self.map_tensor_name(name), data_torch)]
+
+ def prepare_tensors(self):
+ super().prepare_tensors()
+
+ if self._experts is not None:
+ # flatten `list[dict[str, Tensor]]` into `list[str]`
+ experts = [k for d in self._experts for k in d.keys()]
+ if len(experts) > 0:
+ raise ValueError(f"Unprocessed experts: {experts}")
+
+
@ModelBase.register(
"Qwen2VLModel",
"Qwen2VLForConditionalGeneration",
@@ -3700,7 +3844,7 @@ class Plamo2Model(TextModel):
self.gguf_writer.add_block_count(block_count)
self.gguf_writer.add_head_count(hparams.get("num_attention_heads", 32))
self.gguf_writer.add_layer_norm_rms_eps(hparams.get("rms_norm_eps", 1e-06))
- self.gguf_writer.add_rope_freq_base(hparams.get("rope_theta", 1000000.0))
+ self.gguf_writer.add_rope_freq_base(hparams.get("rope_theta", 10000))
# Mamba parameters
self.gguf_writer.add_ssm_state_size(hparams.get("mamba_d_state", 64))
@@ -3711,7 +3855,7 @@ class Plamo2Model(TextModel):
self.gguf_writer.add_ssm_group_count(0)
# MLP feed forward parameters (for attention layers)
- self.gguf_writer.add_feed_forward_length(hparams.get("intermediate_size", 16384))
+ self.gguf_writer.add_feed_forward_length(hparams.get("intermediate_size", 13312))
self.gguf_writer.add_file_type(self.ftype)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
@@ -6395,7 +6539,7 @@ class JaisModel(TextModel):
self.gguf_writer.add_max_alibi_bias(self.max_alibi_bias)
-@ModelBase.register("Glm4ForCausalLM")
+@ModelBase.register("Glm4ForCausalLM", "Glm4vForConditionalGeneration")
class Glm4Model(TextModel):
model_arch = gguf.MODEL_ARCH.GLM4
@@ -6417,7 +6561,8 @@ class Glm4Model(TextModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
- rope_dim = self.hparams["head_dim"]
+ if (rope_dim := self.hparams.get("head_dim")) is None:
+ rope_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.hparams.get("partial_rotary_factor", 0.5)))
rope_scaling = self.hparams.get("rope_scaling") or {}
if rope_scaling.get("rope_type", rope_scaling.get("type")) == "yarn" and "factor" in rope_scaling:
@@ -6425,6 +6570,13 @@ class Glm4Model(TextModel):
self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
self.gguf_writer.add_rope_scaling_orig_ctx_len(rope_scaling["original_max_position_embeddings"])
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name.startswith("model.visual."): # ignore visual part of Glm4v
+ return []
+ elif name.startswith("model.language_model."):
+ name = name.replace("language_model.", "") # for Glm4v
+ return super().modify_tensors(data_torch, name, bid)
+
@ModelBase.register("GlmForCausalLM", "ChatGLMModel", "ChatGLMForConditionalGeneration")
class ChatGLMModel(TextModel):
@@ -6692,6 +6844,75 @@ class ExaoneModel(TextModel):
yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), torch.tensor(rope_factors, dtype=torch.float32))
+@ModelBase.register("Exaone4ForCausalLM")
+class Exaone4Model(TextModel):
+ model_arch = gguf.MODEL_ARCH.EXAONE4
+
+ def set_vocab(self):
+ tokens, toktypes, tokpre = self.get_vocab_base()
+ self.gguf_writer.add_tokenizer_model("gpt2")
+ self.gguf_writer.add_tokenizer_pre(tokpre)
+ self.gguf_writer.add_token_list(tokens)
+ self.gguf_writer.add_token_types(toktypes)
+
+ special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
+ special_vocab.add_to_gguf(self.gguf_writer)
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ hparams = self.hparams
+ self.gguf_writer.add_vocab_size(hparams["vocab_size"])
+
+ if hparams.get("sliding_window") is not None:
+ self.gguf_writer.add_sliding_window(hparams["sliding_window"])
+ if "layer_types" in hparams:
+ self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in hparams["layer_types"]])
+ elif "sliding_window_pattern" in hparams:
+ sliding_window_pattern = []
+ if isinstance(hparams["sliding_window_pattern"], str): # e.g. LLLG
+ for i in range(hparams["num_hidden_layers"]):
+ sliding_window_pattern.append(hparams["sliding_window_pattern"][i % len(hparams["sliding_window_pattern"])] == "L")
+ if isinstance(hparams["sliding_window_pattern"], int): # e.g. 4
+ for i in range(hparams["num_hidden_layers"]):
+ sliding_window_pattern.append((i + 1) % hparams["sliding_window_pattern"] != 0)
+ if len(sliding_window_pattern) == hparams["num_hidden_layers"]:
+ self.gguf_writer.add_sliding_window_pattern(sliding_window_pattern)
+
+ rope_scaling = self.hparams.get("rope_scaling") or {}
+ if rope_scaling.get("rope_type", rope_scaling.get("type")) == "linear" and "factor" in rope_scaling:
+ self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR)
+ self.gguf_writer.add_rope_scaling_factor(rope_scaling["factor"])
+
+ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+ if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):
+ if rope_scaling.get("rope_type", '').lower() == "llama3":
+ base = self.hparams.get("rope_theta", 10_000.0)
+ if (dim := self.hparams.get("head_dim")) is None:
+ dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
+ freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
+
+ factor = rope_scaling.get("factor", 16.0)
+ low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)
+ high_freq_factor = rope_scaling.get("high_freq_factor", 4.0)
+ old_context_len = self.hparams.get("original_max_position_embeddings", 8192)
+
+ low_freq_wavelen = old_context_len / low_freq_factor
+ high_freq_wavelen = old_context_len / high_freq_factor
+
+ rope_factors = []
+ for freq in freqs:
+ wavelen = 2 * math.pi / freq
+ if wavelen <