diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py
index 72d2f7a815..b47a3d85ec 100755
--- a/convert_hf_to_gguf.py
+++ b/convert_hf_to_gguf.py
@@ -5416,7 +5416,8 @@ class LazyTorchTensor(gguf.LazyBase):
         dtype = cls._dtype_str_map[remote_tensor.dtype]
         shape = remote_tensor.shape
         meta = cls.meta_with_dtype_and_shape(dtype, shape)
-        lazy = cls(meta=meta, args=(remote_tensor,), func=lambda r: torch.frombuffer(r.data(), dtype=dtype).reshape(shape))
+        func = lambda r: torch.frombuffer(r.data(), dtype=dtype).reshape(shape)
+        lazy = cls(meta=meta, args=(remote_tensor,), func=func)
         return cast(torch.Tensor, lazy)
 
     @classmethod
diff --git a/gguf-py/gguf/utility.py b/gguf-py/gguf/utility.py
index 62ccc4f930..89a9d494db 100644
--- a/gguf-py/gguf/utility.py
+++ b/gguf-py/gguf/utility.py
@@ -81,9 +81,10 @@ class RemoteTensor:
     size: int
     url: str
 
-    def data(self) -> bytes:
+    def data(self) -> bytearray:
         # TODO: handle request errors (maybe with limited retries?)
-        data = SafetensorRemote.get_data_by_range(url=self.url, start=self.offset_start, size=self.size)
+        # NOTE: using a bytearray, otherwise PyTorch complains the buffer is not writeable
+        data = bytearray(SafetensorRemote.get_data_by_range(url=self.url, start=self.offset_start, size=self.size))
         return data