CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-11-10 10:27:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

convert : use writeable buffer for remote lazy tensors

Browse Source
This commit is contained in:
Francis Couture-Harpin
2025-04-08 11:40:12 -04:00
parent b584e3979d
commit 78094fc4a7
2 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
convert_hf_to_gguf.py
View File

@@ -5416,7 +5416,8 @@ class LazyTorchTensor(gguf.LazyBase):
dtype = cls._dtype_str_map[remote_tensor.dtype] dtype = cls._dtype_str_map[remote_tensor.dtype]
shape = remote_tensor.shape shape = remote_tensor.shape
meta = cls.meta_with_dtype_and_shape(dtype, shape) meta = cls.meta_with_dtype_and_shape(dtype, shape)
lazy = cls(meta=meta, args=(remote_tensor,), func=lambda r: torch.frombuffer(r.data(), dtype=dtype).reshape(shape)) func = lambda r: torch.frombuffer(r.data(), dtype=dtype).reshape(shape)
lazy = cls(meta=meta, args=(remote_tensor,), func=func)
return cast(torch.Tensor, lazy) return cast(torch.Tensor, lazy)
@classmethod @classmethod

5
gguf-py/gguf/utility.py
View File

@@ -81,9 +81,10 @@ class RemoteTensor:
size: int size: int
url: str url: str
def data(self) -> bytes: def data(self) -> bytearray:
# TODO: handle request errors (maybe with limited retries?) # TODO: handle request errors (maybe with limited retries?)
data = SafetensorRemote.get_data_by_range(url=self.url, start=self.offset_start, size=self.size) # NOTE: using a bytearray, otherwise PyTorch complains the buffer is not writeable
data = bytearray(SafetensorRemote.get_data_by_range(url=self.url, start=self.offset_start, size=self.size))
return data return data