diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 4aabfd01d0..108f12be22 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -607,14 +607,7 @@ class TextModel(ModelBase): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model) -<<<<<<< HEAD vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab)) -======= - vocab_size = min( - self.hparams.get("vocab_size", len(tokenizer.vocab)), - len(tokenizer.vocab) - ) ->>>>>>> 250b4f1074da288399e71577a925aa83acf316d5 assert max(tokenizer.vocab.values()) < vocab_size tokpre = self.get_vocab_base_pre(tokenizer)