mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2025-10-28 08:31:25 +00:00
56b4795842
* model-conversion : add support for SentenceTransformers This commit adds support for models that use SentenceTransformer layers. The motivation for this is that if converted model includes any of the numbered layers specified in the original models repository then these changes enable these models to be used and verified. Currently the model-conversion only support the base model output without any of the additional transformation layers. Usage: Convert the model that also includes the SentenceTransformer layers: ```console (venv) $ export EMBEDDING_MODEL_PATH="~/google/embeddinggemma-300M" (venv) make embedding-convert-model ``` Verify the produced embeddings from the converted model against the original model embeddings: ```console (venv) make embedding-verify-logits-st ``` The original model can be run using SentenceTransformer: ```console (venv) make embedding-run-original-model-st ``` Run the converted model using "SentenceTransformer" layers whic enables pooling and normalization: ```console (venv) make embedding-run-converted-model-st ``` * add model-conversion example requirements * add support for -st flag in embedding model conversion This commit add support for the -st flag in the embedding model conversion script. This will enable models to be converted using sentence transformers dense layers.
60 lines
1.4 KiB
Bash
Executable File
60 lines
1.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -e
|
|
|
|
# Parse command line arguments
|
|
CONVERTED_MODEL=""
|
|
PROMPTS_FILE=""
|
|
USE_POOLING=""
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
-p|--prompts-file)
|
|
PROMPTS_FILE="$2"
|
|
shift 2
|
|
;;
|
|
--pooling)
|
|
USE_POOLING="1"
|
|
shift
|
|
;;
|
|
*)
|
|
if [ -z "$CONVERTED_MODEL" ]; then
|
|
CONVERTED_MODEL="$1"
|
|
fi
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# First try command line argument, then environment variable
|
|
CONVERTED_MODEL="${CONVERTED_MODEL:-"$CONVERTED_EMBEDDING_MODEL"}"
|
|
|
|
# Final check if we have a model path
|
|
if [ -z "$CONVERTED_MODEL" ]; then
|
|
echo "Error: Model path must be provided either as:" >&2
|
|
echo " 1. Command line argument" >&2
|
|
echo " 2. CONVERTED_EMBEDDING_MODEL environment variable" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Read prompt from file or use default
|
|
if [ -n "$PROMPTS_FILE" ]; then
|
|
if [ ! -f "$PROMPTS_FILE" ]; then
|
|
echo "Error: Prompts file '$PROMPTS_FILE' not found" >&2
|
|
exit 1
|
|
fi
|
|
PROMPT=$(cat "$PROMPTS_FILE")
|
|
else
|
|
PROMPT="Hello world today"
|
|
fi
|
|
|
|
echo $CONVERTED_MODEL
|
|
|
|
cmake --build ../../build --target llama-logits -j8
|
|
# TODO: update logits.cpp to accept a --file/-f option for the prompt
|
|
if [ -n "$USE_POOLING" ]; then
|
|
../../build/bin/llama-logits -m "$CONVERTED_MODEL" -embd-mode -pooling "$PROMPT"
|
|
else
|
|
../../build/bin/llama-logits -m "$CONVERTED_MODEL" -embd-mode "$PROMPT"
|
|
fi
|