diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 108f12be22..4aabfd01d0 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -607,7 +607,14 @@ class TextModel(ModelBase): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model) +<<<<<<< HEAD vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab)) +======= + vocab_size = min( + self.hparams.get("vocab_size", len(tokenizer.vocab)), + len(tokenizer.vocab) + ) +>>>>>>> 250b4f1074da288399e71577a925aa83acf316d5 assert max(tokenizer.vocab.values()) < vocab_size tokpre = self.get_vocab_base_pre(tokenizer)