CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-10-29 08:41:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

gguf-py : make sentencepiece optional (#14200)

Browse Source 
* Make sentencepiece optional

* Bump to 0.18.0

* Bump patch instead of minor

Co-authored-by: compilade <git@compilade.net>

---------

Co-authored-by: compilade <git@compilade.net>
This commit is contained in:
Alex Trotta
2025-06-19 09:56:12 -04:00
committed by GitHub
parent d67341dc18
commit 381174bbda
2 changed files with 9 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
gguf-py/gguf/vocab.py
View File

@@ -7,7 +7,10 @@ import os
from pathlib import Path from pathlib import Path
from typing import Any, Callable, Sequence, Mapping, Iterable, Protocol, ClassVar, runtime_checkable from typing import Any, Callable, Sequence, Mapping, Iterable, Protocol, ClassVar, runtime_checkable
from sentencepiece import SentencePieceProcessor try:
from sentencepiece import SentencePieceProcessor
except ImportError:
SentencePieceProcessor = None
import gguf import gguf
@@ -302,6 +305,9 @@ class SentencePieceVocab(Vocab):
name = "spm" name = "spm"
def __init__(self, base_path: Path): def __init__(self, base_path: Path):
if SentencePieceProcessor is None:
raise RuntimeError("sentencepiece is not installed")
added_tokens: dict[str, int] = {} added_tokens: dict[str, int] = {}
if (fname_tokenizer := base_path / 'tokenizer.model').exists(): if (fname_tokenizer := base_path / 'tokenizer.model').exists():
# normal location # normal location

4
gguf-py/pyproject.toml
View File

@@ -1,6 +1,6 @@
[tool.poetry] [tool.poetry]
name = "gguf" name = "gguf"
version = "0.17.0" version = "0.17.1"
description = "Read and write ML models in GGUF for GGML" description = "Read and write ML models in GGUF for GGML"
authors = ["GGML <ggml@ggml.ai>"] authors = ["GGML <ggml@ggml.ai>"]
packages = [ packages = [
@@ -22,7 +22,7 @@ python = ">=3.8"
numpy = ">=1.17" numpy = ">=1.17"
tqdm = ">=4.27" tqdm = ">=4.27"
pyyaml = ">=5.1" pyyaml = ">=5.1"
sentencepiece = ">=0.1.98,<=0.2.0" sentencepiece = { version = ">=0.1.98,<=0.2.0", optional = true }
PySide6 = { version = "^6.9", python = ">=3.9,<3.14", optional = true } PySide6 = { version = "^6.9", python = ">=3.9,<3.14", optional = true }
[tool.poetry.dev-dependencies] [tool.poetry.dev-dependencies]