CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-10-31 08:51:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

llava/mtmd : fixes to fully support dl backends (#13303)

Browse Source
This commit is contained in:
Diego Devesa
2025-05-04 17:05:20 +02:00
committed by GitHub
parent 9f2da5871f
commit 86bd60d3fe
3 changed files with 15 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tools/CMakeLists.txt
View File

@@ -27,11 +27,11 @@ else()
add_subdirectory(run) add_subdirectory(run)
add_subdirectory(tokenize) add_subdirectory(tokenize)
add_subdirectory(tts) add_subdirectory(tts)
add_subdirectory(llava)
if (NOT GGML_BACKEND_DL) if (NOT GGML_BACKEND_DL)
# these examples use the backends directly and cannot be built with dynamic loading # these examples use the backends directly and cannot be built with dynamic loading
add_subdirectory(cvector-generator) add_subdirectory(cvector-generator)
add_subdirectory(export-lora) add_subdirectory(export-lora)
add_subdirectory(llava)
if (GGML_RPC) if (GGML_RPC)
add_subdirectory(rpc) add_subdirectory(rpc)
endif() endif()

10
tools/llava/clip.cpp
View File

@@ -3382,7 +3382,15 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
GGML_ABORT("Unknown projector type"); GGML_ABORT("Unknown projector type");
} }
ggml_backend_cpu_set_n_threads(ctx->backend_cpu, n_threads); // ggml_backend_cpu_set_n_threads(ctx->backend_cpu, n_threads);
ggml_backend_dev_t dev = ggml_backend_get_device(ctx->backend_cpu);
ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
if (reg) {
auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");
if (ggml_backend_set_n_threads_fn) {
ggml_backend_set_n_threads_fn(ctx->backend_cpu, n_threads);
}
}
auto status = ggml_backend_sched_graph_compute(ctx->sched.get(), gf); auto status = ggml_backend_sched_graph_compute(ctx->sched.get(), gf);
if (status != GGML_STATUS_SUCCESS) { if (status != GGML_STATUS_SUCCESS) {

6
tools/llava/llava.cpp
View File

@@ -2,6 +2,7 @@
#include "llava.h" #include "llava.h"
#include "llama.h" #include "llama.h"
#include "ggml-cpp.h"
#include <algorithm> #include <algorithm>
#include <cerrno> #include <cerrno>
@@ -209,7 +210,10 @@ static bool clip_llava_handle_patches(clip_ctx * ctx_clip, std::vector<float *>
struct ggml_tensor *flatten = ggml_view_2d(model.ctx, permuted_cont, clip_n_mmproj_embd(ctx_clip), num_patches_height * num_patches_width * num_patches_per_side * num_patches_per_side, size_ele * clip_n_mmproj_embd(ctx_clip), 0); struct ggml_tensor *flatten = ggml_view_2d(model.ctx, permuted_cont, clip_n_mmproj_embd(ctx_clip), num_patches_height * num_patches_width * num_patches_per_side * num_patches_per_side, size_ele * clip_n_mmproj_embd(ctx_clip), 0);
// ggml_tensor_printf(flatten,"flatten",__LINE__,false,false); // ggml_tensor_printf(flatten,"flatten",__LINE__,false,false);
ggml_build_forward_expand(gf, flatten); ggml_build_forward_expand(gf, flatten);
ggml_graph_compute_with_ctx(model.ctx, gf, 1);
ggml_backend_ptr backend { ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr) };
ggml_backend_graph_compute(backend.get(), gf);
struct ggml_tensor* result = ggml_graph_node(gf, -1); struct ggml_tensor* result = ggml_graph_node(gf, -1);
memcpy(image_embd_out, image_embd_v[0], clip_embd_nbytes(ctx_clip)); // main image as global context memcpy(image_embd_out, image_embd_v[0], clip_embd_nbytes(ctx_clip)); // main image as global context