mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2025-11-03 09:22:01 +00:00
aa23412989
* Create llava-survery-v2.py * Update convert-image-encoder-to-gguf.py * Update convert-image-encoder-to-gguf.py * Rename llava-survery-v2.py to llava-surgery-v2.py * Update convert-image-encoder-to-gguf.py will now search for projector * Update convert-image-encoder-to-gguf.py whoops * Update llava-surgery-v2.py * Clip: Bugfix for normalization (it did not loat the 3 std and mean values) Clip: bicubic resize function Clip: added save-to-bmp/pil for debugging and conversion from/to 32/8 images Clip: added normalization with FP16 precision simulation (image tensors match HF implementation, can be switched off, only used for llava-1.6) Clip: added newline tensor, mergetype kv, image-grid kv, new resize-pad function with resolution from gridpoints Clip: clip_image_preprocess now returns a float * vector instead of float, this way llava 1.5 and 1.6 is supported llava: added ggml cpu graph for embedding patching, added spatial_unpad preliminary support, added a lot of comments that need to be cleaned when all is final convert-image-encoder: fixed image-grid flattening * whitespace corrections * ws * Tensors are now properly permuted. Before the embeddings were inserted 1:1, now they are split into the 24x24 patches as in reference. * ws * added verbose_prompt support into cli added stopwords for llava-1.6 into cli * moved llava functions to llava.cpp, made clip.h C compatible API, replaced vector style functions with pointers, added a debug define to remove functions from compilation while not needed * ws * convert : skip unknown tensors (need for LLaVA) * llava : update readme * llava : fix compile warnings * llava : style * convert : add --skip-unknown CLI arg * server : remove clip structs * bugfix for non llava-1.6 It should now work with llava-1.5 as well * clip : minor code rearrange * llava : update readme a bit --------- Co-authored-by: John <cmt-nct@users.noreply.github.com> Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
84 lines
2.7 KiB
C
84 lines
2.7 KiB
C
#ifndef CLIP_H
|
|
#define CLIP_H
|
|
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
|
|
#ifdef LLAMA_SHARED
|
|
# if defined(_WIN32) && !defined(__MINGW32__)
|
|
# ifdef LLAMA_BUILD
|
|
# define CLIP_API __declspec(dllexport)
|
|
# else
|
|
# define CLIP_API __declspec(dllimport)
|
|
# endif
|
|
# else
|
|
# define CLIP_API __attribute__ ((visibility ("default")))
|
|
# endif
|
|
#else
|
|
# define CLIP_API
|
|
#endif
|
|
|
|
struct clip_ctx;
|
|
|
|
#ifdef __cplusplus
|
|
extern "C" {
|
|
#endif
|
|
|
|
struct clip_ctx;
|
|
|
|
struct clip_image_u8_batch {
|
|
struct clip_image_u8 * data;
|
|
size_t size;
|
|
};
|
|
|
|
struct clip_image_f32_batch {
|
|
struct clip_image_f32 * data;
|
|
size_t size;
|
|
};
|
|
|
|
CLIP_API struct clip_ctx * clip_model_load (const char * fname, int verbosity);
|
|
CLIP_API struct clip_ctx * clip_model_load_cpu(const char * fname, int verbosity);
|
|
|
|
CLIP_API void clip_free(struct clip_ctx * ctx);
|
|
|
|
CLIP_API size_t clip_embd_nbytes(const struct clip_ctx * ctx);
|
|
|
|
CLIP_API int32_t clip_image_size (const struct clip_ctx * ctx);
|
|
CLIP_API int32_t clip_patch_size (const struct clip_ctx * ctx);
|
|
CLIP_API int32_t clip_hidden_size(const struct clip_ctx * ctx);
|
|
|
|
// TODO: should be enum, not string
|
|
CLIP_API const char * clip_patch_merge_type(const struct clip_ctx * ctx);
|
|
|
|
CLIP_API const int32_t * clip_image_grid(const struct clip_ctx * ctx);
|
|
|
|
CLIP_API int clip_n_patches (const struct clip_ctx * ctx);
|
|
CLIP_API int clip_n_mmproj_embd(const struct clip_ctx * ctx);
|
|
|
|
CLIP_API struct clip_image_u8 * clip_image_u8_init ();
|
|
CLIP_API struct clip_image_f32 * clip_image_f32_init();
|
|
|
|
CLIP_API void clip_image_u8_free (struct clip_image_u8 * img);
|
|
CLIP_API void clip_image_f32_free(struct clip_image_f32 * img);
|
|
|
|
CLIP_API bool clip_image_load_from_file(const char * fname, struct clip_image_u8 * img);
|
|
|
|
/** interpret bytes as an image file with length bytes_length, and use the result to populate img */
|
|
CLIP_API bool clip_image_load_from_bytes(const unsigned char * bytes, size_t bytes_length, struct clip_image_u8 * img);
|
|
|
|
/** preprocess img and store the result in res_imgs, pad_to_square may be overriden to false depending on model configuration */
|
|
CLIP_API bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32_batch & res_imgs );
|
|
|
|
CLIP_API struct ggml_tensor * clip_get_newline_tensor(const struct clip_ctx * ctx);
|
|
|
|
CLIP_API bool clip_image_encode (struct clip_ctx * ctx, int n_threads, struct clip_image_f32 * img, float * vec);
|
|
CLIP_API bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, float * vec);
|
|
|
|
CLIP_API bool clip_model_quantize(const char * fname_inp, const char * fname_out, int itype);
|
|
|
|
#ifdef __cplusplus
|
|
}
|
|
#endif
|
|
|
|
#endif // CLIP_H
|