mirror of
				https://github.com/ggml-org/llama.cpp.git
				synced 2025-10-31 08:51:55 +00:00 
			
		
		
		
	py : add Phi-1.5/Phi-2 tokenizer (#9361)
* add phi2 tokenizer * add phi name to convert_hf_to_gguf_update.py * make tokenizer_pre consistent; llama.cpp work
This commit is contained in:
		| @@ -626,6 +626,9 @@ class Model: | ||||
|         if chkhsh == "4e2b24cc4770243d65a2c9ec19770a72f08cffc161adbb73fcbb6b7dd45a0aae": | ||||
|             # ref: https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct | ||||
|             res = "exaone" | ||||
|         if chkhsh == "fcace8b9cac38ce847670c970cd5892031a753a1ef381abd1d9af00f713da085": | ||||
|             # ref: https://huggingface.co/microsoft/phi-2 | ||||
|             res = "phi-2" | ||||
|  | ||||
|         if res is None: | ||||
|             logger.warning("\n") | ||||
|   | ||||
		Reference in New Issue
	
	Block a user
	 daminho
					daminho