mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2025-10-31 08:51:55 +00:00
examples : add compare-mlx
This commit is contained in:
Georgi Gerganov
2
examples/compare-mlx/.gitignore
vendored
Normal file
2
examples/compare-mlx/.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
*.txt
|
||||
*/
|
||||
706
examples/compare-mlx/compare-mlx.sh
Executable file
706
examples/compare-mlx/compare-mlx.sh
Executable file
@@ -0,0 +1,706 @@
|
||||
#!/bin/bash
|
||||
|
||||
# a script to compare MLX and GGUF models
|
||||
#
|
||||
# usage:
|
||||
# ./compare-mlx.sh --raw-path wiki.test.raw --no-keep
|
||||
#
|
||||
# TODOs
|
||||
# - add QAT evals
|
||||
|
||||
# check if LLAMA_HOME_DIR is set
|
||||
if [[ -z "$LLAMA_HOME_DIR" ]]; then
|
||||
lcpp_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")"/../../ && pwd)
|
||||
else
|
||||
lcpp_dir="${LLAMA_HOME_DIR}"
|
||||
fi
|
||||
|
||||
echo "Using llama.cpp directory: ${lcpp_dir}"
|
||||
|
||||
# check for convert_hf_to_gguf.py
|
||||
if [[ ! -f "${lcpp_dir}/convert_hf_to_gguf.py" ]]; then
|
||||
echo "convert_hf_to_gguf.py not found in ${lcpp_dir}"
|
||||
echo "Set a LLAMA_HOME_DIR environment variable to point to your llama.cpp directory"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -x
|
||||
|
||||
# sanity checks that all Python dependencies are installed
|
||||
if ! python -c "import mlx.core"; then
|
||||
echo "MLX not found. Please install MLX"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! python ${lcpp_dir}/convert_hf_to_gguf.py --help; then
|
||||
echo "convert_hf_to_gguf.py not working. Please install llama.cpp python requirements"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# by default use the system binaries (for example from brew)
|
||||
llama_perplexity="llama-perplexity"
|
||||
|
||||
if [[ ! -z "$LLAMA_PERPLEXITY" ]]; then
|
||||
llama_perplexity="$LLAMA_PERPLEXITY"
|
||||
fi
|
||||
|
||||
echo "Using llama-perplexity: ${llama_perplexity}"
|
||||
|
||||
if ! command -v "$llama_perplexity" &> /dev/null; then
|
||||
echo "llama-perplexity not found. Please install it."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
llama_quantize="llama-quantize"
|
||||
|
||||
if [[ ! -z "$LLAMA_QUANTIZE" ]]; then
|
||||
llama_quantize="$LLAMA_QUANTIZE"
|
||||
fi
|
||||
|
||||
echo "Using llama-quantize: ${llama_quantize}"
|
||||
|
||||
if ! command -v "$llama_quantize" &> /dev/null; then
|
||||
echo "llama-quantize not found. Please install it."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
llama_batched_bench="llama-batched-bench"
|
||||
|
||||
if [[ ! -z "$LLAMA_BATCHED_BENCH" ]]; then
|
||||
llama_batched_bench="$LLAMA_BATCHED_BENCH"
|
||||
fi
|
||||
|
||||
echo "Using llama-batched-bench: ${llama_batched_bench}"
|
||||
|
||||
if ! command -v "$llama_batched_bench" &> /dev/null; then
|
||||
echo "llama-batched-bench not found. Please install it."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# get the size in GiB
|
||||
get_size() {
|
||||
local path="$1"
|
||||
local bytes=$(du -s "$path" | awk '{print $1}')
|
||||
local res=$(echo "scale=3; ($bytes*512)/1024/1024/1024" | bc)
|
||||
echo "$res"
|
||||
}
|
||||
|
||||
# parameters:
|
||||
# --no-compute : do not compute anything, just summarize the existing results
|
||||
# --no-ppl : do not compute ppl
|
||||
# --no-perf : do not compute performance (speed) metrics
|
||||
# --no-keep : delete intermediate model files
|
||||
# --num-samples : number of text samples to evaluate (default: 512)
|
||||
# --sequence-length : sequence length of the samples in tokens (default: 512)
|
||||
# --raw-path : file with raw text (such as wikitext)
|
||||
|
||||
# extra agruments
|
||||
args_lcpp="-t 1"
|
||||
|
||||
num_samples=512
|
||||
sequence_length=512
|
||||
raw_path=""
|
||||
no_compute=false
|
||||
no_ppl=false
|
||||
no_perf=false
|
||||
no_keep=false
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--no-compute)
|
||||
no_compute=true
|
||||
shift
|
||||
;;
|
||||
--no-ppl)
|
||||
no_ppl=true
|
||||
shift
|
||||
;;
|
||||
--no-perf)
|
||||
no_perf=true
|
||||
shift
|
||||
;;
|
||||
--no-keep)
|
||||
no_keep=true
|
||||
shift
|
||||
;;
|
||||
--num-samples)
|
||||
num_samples="$2"
|
||||
shift 2
|
||||
;;
|
||||
--sequence-length)
|
||||
sequence_length="$2"
|
||||
shift 2
|
||||
;;
|
||||
--raw-path)
|
||||
raw_path="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "Unknown parameter: $1"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "$raw_path" ]]; then
|
||||
echo "No raw path provided"
|
||||
echo "Recommended to use the test set of WikiText from here: https://github.com/ggml-org/llama.cpp/blob/master/scripts/get-wikitext-2.sh"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
eval_model() {
|
||||
org="$1"
|
||||
mid="$2"
|
||||
|
||||
echo "Evaluating ${org}/${mid}"
|
||||
|
||||
huggingface-cli download ${org}/${mid} --local-dir ${org}/${mid}
|
||||
|
||||
# generate and process MLX models
|
||||
|
||||
if [[ "$no_compute" == true ]]; then
|
||||
echo "Skipping computation"
|
||||
else
|
||||
rm -rfv ./${mid}-f32-mlx
|
||||
mlx_lm.convert --hf ./${org}/${mid} --mlx-path ./${mid}-f32-mlx --dtype float32
|
||||
get_size ./${mid}-f32-mlx > ./${mid}-f32-mlx-size.txt
|
||||
|
||||
if [[ "$no_ppl" == false ]]; then
|
||||
python ./mlx-ppl.py --model ./${mid}-f32-mlx --raw-path "$raw_path" --num-samples "$num_samples" --sequence-length "$sequence_length" 2>&1 | tee ./${mid}-f32-mlx-ppl.txt
|
||||
fi
|
||||
|
||||
# no need for F32 perf benchmarks
|
||||
#if [[ "$no_perf" == false ]]; then
|
||||
# mlx_lm.benchmark --model ./${mid}-f32-mlx -p 2048 -g 128 --num-trials 1 2>&1 | tee ./${mid}-f32-mlx-perf-2048.txt
|
||||
# mlx_lm.benchmark --model ./${mid}-f32-mlx -p 4096 -g 128 --num-trials 1 2>&1 | tee ./${mid}-f32-mlx-perf-4096.txt
|
||||
# mlx_lm.benchmark --model ./${mid}-f32-mlx -p 8192 -g 128 --num-trials 1 2>&1 | tee ./${mid}-f32-mlx-perf-8192.txt
|
||||
# mlx_lm.benchmark --model ./${mid}-f32-mlx -p 16384 -g 128 --num-trials 1 2>&1 | tee ./${mid}-f32-mlx-perf-16384.txt
|
||||
# mlx_lm.benchmark --model ./${mid}-f32-mlx -p 32768 -g 128 --num-trials 1 2>&1 | tee ./${mid}-f32-mlx-perf-32768.txt
|
||||
#fi
|
||||
|
||||
if [[ "$no_keep" == true ]]; then
|
||||
echo "Deleting intermediate model files"
|
||||
rm -rfv ./${mid}-f32-mlx
|
||||
fi
|
||||
|
||||
rm -rfv ./${mid}-bf16-mlx
|
||||
mlx_lm.convert --hf ./${org}/${mid} --mlx-path ./${mid}-bf16-mlx --dtype bfloat16
|
||||
get_size ./${mid}-bf16-mlx > ./${mid}-bf16-mlx-size.txt
|
||||
|
||||
if [[ "$no_ppl" == false ]]; then
|
||||
python ./mlx-ppl.py --model ./${mid}-bf16-mlx --raw-path "$raw_path" --num-samples "$num_samples" --sequence-length "$sequence_length" 2>&1 | tee ./${mid}-bf16-mlx-ppl.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_perf" == false ]]; then
|
||||
mlx_lm.benchmark --model ./${mid}-bf16-mlx -p 2048 -g 128 --num-trials 1 2>&1 | tee ./${mid}-bf16-mlx-perf-2048.txt
|
||||
mlx_lm.benchmark --model ./${mid}-bf16-mlx -p 4096 -g 128 --num-trials 1 2>&1 | tee ./${mid}-bf16-mlx-perf-4096.txt
|
||||
mlx_lm.benchmark --model ./${mid}-bf16-mlx -p 8192 -g 128 --num-trials 1 2>&1 | tee ./${mid}-bf16-mlx-perf-8192.txt
|
||||
mlx_lm.benchmark --model ./${mid}-bf16-mlx -p 16384 -g 128 --num-trials 1 2>&1 | tee ./${mid}-bf16-mlx-perf-16384.txt
|
||||
mlx_lm.benchmark --model ./${mid}-bf16-mlx -p 32768 -g 128 --num-trials 1 2>&1 | tee ./${mid}-bf16-mlx-perf-32768.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_keep" == true ]]; then
|
||||
echo "Deleting intermediate model files"
|
||||
rm -rfv ./${mid}-bf16-mlx
|
||||
fi
|
||||
|
||||
rm -rfv ./${mid}-f16-mlx
|
||||
mlx_lm.convert --hf ./${org}/${mid} --mlx-path ./${mid}-f16-mlx --dtype float16
|
||||
get_size ./${mid}-f16-mlx > ./${mid}-f16-mlx-size.txt
|
||||
|
||||
if [[ "$no_ppl" == false ]]; then
|
||||
python ./mlx-ppl.py --model ./${mid}-f16-mlx --raw-path "$raw_path" --num-samples "$num_samples" --sequence-length "$sequence_length" 2>&1 | tee ./${mid}-f16-mlx-ppl.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_perf" == false ]]; then
|
||||
mlx_lm.benchmark --model ./${mid}-f16-mlx -p 2048 -g 128 --num-trials 1 2>&1 | tee ./${mid}-f16-mlx-perf-2048.txt
|
||||
mlx_lm.benchmark --model ./${mid}-f16-mlx -p 4096 -g 128 --num-trials 1 2>&1 | tee ./${mid}-f16-mlx-perf-4096.txt
|
||||
mlx_lm.benchmark --model ./${mid}-f16-mlx -p 8192 -g 128 --num-trials 1 2>&1 | tee ./${mid}-f16-mlx-perf-8192.txt
|
||||
mlx_lm.benchmark --model ./${mid}-f16-mlx -p 16384 -g 128 --num-trials 1 2>&1 | tee ./${mid}-f16-mlx-perf-16384.txt
|
||||
mlx_lm.benchmark --model ./${mid}-f16-mlx -p 32768 -g 128 --num-trials 1 2>&1 | tee ./${mid}-f16-mlx-perf-32768.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_keep" == true ]]; then
|
||||
echo "Deleting intermediate model files"
|
||||
rm -rfv ./${mid}-f16-mlx
|
||||
fi
|
||||
|
||||
rm -rfv ./${mid}-q8-mlx
|
||||
mlx_lm.convert --hf ./${org}/${mid} --mlx-path ./${mid}-q8-mlx --quantize --q-bits 8
|
||||
get_size ./${mid}-q8-mlx > ./${mid}-q8-mlx-size.txt
|
||||
|
||||
if [[ "$no_ppl" == false ]]; then
|
||||
python ./mlx-ppl.py --model ./${mid}-q8-mlx --raw-path "$raw_path" --num-samples "$num_samples" --sequence-length "$sequence_length" 2>&1 | tee ./${mid}-q8-mlx-ppl.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_perf" == false ]]; then
|
||||
mlx_lm.benchmark --model ./${mid}-q8-mlx -p 2048 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q8-mlx-perf-2048.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q8-mlx -p 4096 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q8-mlx-perf-4096.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q8-mlx -p 8192 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q8-mlx-perf-8192.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q8-mlx -p 16384 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q8-mlx-perf-16384.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q8-mlx -p 32768 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q8-mlx-perf-32768.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_keep" == true ]]; then
|
||||
echo "Deleting intermediate model files"
|
||||
rm -rfv ./${mid}-q8-mlx
|
||||
fi
|
||||
|
||||
rm -rfv ./${mid}-q6-mlx
|
||||
mlx_lm.convert --hf ./${org}/${mid} --mlx-path ./${mid}-q6-mlx --quantize --q-bits 6
|
||||
get_size ./${mid}-q6-mlx > ./${mid}-q6-mlx-size.txt
|
||||
|
||||
if [[ "$no_ppl" == false ]]; then
|
||||
python ./mlx-ppl.py --model ./${mid}-q6-mlx --raw-path "$raw_path" --num-samples "$num_samples" --sequence-length "$sequence_length" 2>&1 | tee ./${mid}-q6-mlx-ppl.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_perf" == false ]]; then
|
||||
mlx_lm.benchmark --model ./${mid}-q6-mlx -p 2048 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q6-mlx-perf-2048.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q6-mlx -p 4096 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q6-mlx-perf-4096.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q6-mlx -p 8192 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q6-mlx-perf-8192.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q6-mlx -p 16384 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q6-mlx-perf-16384.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q6-mlx -p 32768 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q6-mlx-perf-32768.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_keep" == true ]]; then
|
||||
echo "Deleting intermediate model files"
|
||||
rm -rfv ./${mid}-q6-mlx
|
||||
fi
|
||||
|
||||
rm -rfv ./${mid}-q5-mlx
|
||||
mlx_lm.convert --hf ./${org}/${mid} --mlx-path ./${mid}-q5-mlx --quantize --q-bits 5
|
||||
get_size ./${mid}-q5-mlx > ./${mid}-q5-mlx-size.txt
|
||||
|
||||
if [[ "$no_ppl" == false ]]; then
|
||||
python ./mlx-ppl.py --model ./${mid}-q5-mlx --raw-path "$raw_path" --num-samples "$num_samples" --sequence-length "$sequence_length" 2>&1 | tee ./${mid}-q5-mlx-ppl.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_perf" == false ]]; then
|
||||
mlx_lm.benchmark --model ./${mid}-q5-mlx -p 2048 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q5-mlx-perf-2048.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q5-mlx -p 4096 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q5-mlx-perf-4096.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q5-mlx -p 8192 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q5-mlx-perf-8192.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q5-mlx -p 16384 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q5-mlx-perf-16384.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q5-mlx -p 32768 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q5-mlx-perf-32768.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_keep" == true ]]; then
|
||||
echo "Deleting intermediate model files"
|
||||
rm -rfv ./${mid}-q5-mlx
|
||||
fi
|
||||
|
||||
# I think this is something similar to q4_k
|
||||
rm -rfv ./${mid}-q4p-mlx
|
||||
mlx_lm.convert --hf ./${org}/${mid} --mlx-path ./${mid}-q4p-mlx --quantize --quant-predicate mixed_4_6
|
||||
get_size ./${mid}-q4p-mlx > ./${mid}-q4p-mlx-size.txt
|
||||
|
||||
if [[ "$no_ppl" == false ]]; then
|
||||
python ./mlx-ppl.py --model ./${mid}-q4p-mlx --raw-path "$raw_path" --num-samples "$num_samples" --sequence-length "$sequence_length" 2>&1 | tee ./${mid}-q4p-mlx-ppl.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_perf" == false ]]; then
|
||||
mlx_lm.benchmark --model ./${mid}-q4p-mlx -p 2048 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q4p-mlx-perf-2048.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q4p-mlx -p 4096 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q4p-mlx-perf-4096.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q4p-mlx -p 8192 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q4p-mlx-perf-8192.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q4p-mlx -p 16384 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q4p-mlx-perf-16384.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q4p-mlx -p 32768 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q4p-mlx-perf-32768.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_keep" == true ]]; then
|
||||
echo "Deleting intermediate model files"
|
||||
rm -rfv ./${mid}-q4p-mlx
|
||||
fi
|
||||
|
||||
rm -rfv ./${mid}-q4-mlx
|
||||
mlx_lm.convert --hf ./${org}/${mid} --mlx-path ./${mid}-q4-mlx --quantize --q-bits 4
|
||||
get_size ./${mid}-q4-mlx > ./${mid}-q4-mlx-size.txt
|
||||
|
||||
if [[ "$no_ppl" == false ]]; then
|
||||
python ./mlx-ppl.py --model ./${mid}-q4-mlx --raw-path "$raw_path" --num-samples "$num_samples" --sequence-length "$sequence_length" 2>&1 | tee ./${mid}-q4-mlx-ppl.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_perf" == false ]]; then
|
||||
mlx_lm.benchmark --model ./${mid}-q4-mlx -p 2048 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q4-mlx-perf-2048.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q4-mlx -p 4096 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q4-mlx-perf-4096.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q4-mlx -p 8192 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q4-mlx-perf-8192.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q4-mlx -p 16384 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q4-mlx-perf-16384.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q4-mlx -p 32768 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q4-mlx-perf-32768.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_keep" == true ]]; then
|
||||
echo "Deleting intermediate model files"
|
||||
rm -rfv ./${mid}-q4-mlx
|
||||
fi
|
||||
|
||||
rm -rfv ./${mid}-q3-mlx
|
||||
mlx_lm.convert --hf ./${org}/${mid} --mlx-path ./${mid}-q3-mlx --quantize --q-bits 3
|
||||
get_size ./${mid}-q3-mlx > ./${mid}-q3-mlx-size.txt
|
||||
|
||||
if [[ "$no_ppl" == false ]]; then
|
||||
python ./mlx-ppl.py --model ./${mid}-q3-mlx --raw-path "$raw_path" --num-samples "$num_samples" --sequence-length "$sequence_length" 2>&1 | tee ./${mid}-q3-mlx-ppl.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_perf" == false ]]; then
|
||||
mlx_lm.benchmark --model ./${mid}-q3-mlx -p 2048 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q3-mlx-perf-2048.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q3-mlx -p 4096 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q3-mlx-perf-4096.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q3-mlx -p 8192 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q3-mlx-perf-8192.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q3-mlx -p 16384 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q3-mlx-perf-16384.txt
|
||||
mlx_lm.benchmark --model ./${mid}-q3-mlx -p 32768 -g 128 --num-trials 1 2>&1 | tee ./${mid}-q3-mlx-perf-32768.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_keep" == true ]]; then
|
||||
echo "Deleting intermediate model files"
|
||||
rm -rfv ./${mid}-q3-mlx
|
||||
fi
|
||||
fi
|
||||
|
||||
# generate and process llama.cpp GGUF models
|
||||
|
||||
if [[ "$no_compute" == true ]]; then
|
||||
echo "Skipping computation"
|
||||
else
|
||||
# the F32 model is the reference - we generate all other models from it
|
||||
mkdir -p ./${mid}-f32-gguf
|
||||
python ${lcpp_dir}/convert_hf_to_gguf.py ./${org}/${mid} --outtype f32 --outfile ./${mid}-f32-gguf/model.gguf
|
||||
get_size ./${mid}-f32-gguf > ./${mid}-f32-gguf-size.txt
|
||||
|
||||
if [[ "$no_ppl" == false ]]; then
|
||||
${llama_perplexity} $args_lcpp -m ./${mid}-f32-gguf/model.gguf -f "$raw_path" --chunks "${num_samples}" -c "${sequence_length}" 2>&1 | tee ./${mid}-f32-gguf-ppl.txt
|
||||
fi
|
||||
|
||||
# no need for F32 perf benchmarks
|
||||
#if [[ "$no_perf" == false ]]; then
|
||||
# ${llama_batched_bench} $args_lcpp -m ./${mid}-f32-gguf/model.gguf -c 33768 -b 2048 -ub 2048 -npp 2048,4096,8192,16384,32768 -ntg 128 -npl 1 2>&1 | tee ./${mid}-f32-gguf-perf.txt
|
||||
#fi
|
||||
|
||||
# this requires to explicitly build llama.cpp with BF16 support
|
||||
rm -rfv ./${mid}-bf16-gguf && mkdir -p ./${mid}-bf16-gguf
|
||||
${llama_quantize} ./${mid}-f32-gguf/model.gguf ./${mid}-bf16-gguf/model.gguf bf16
|
||||
get_size ./${mid}-bf16-gguf > ./${mid}-bf16-gguf-size.txt
|
||||
|
||||
if [[ "$no_ppl" == false ]]; then
|
||||
${llama_perplexity} $args_lcpp -m ./${mid}-bf16-gguf/model.gguf -f "$raw_path" --chunks "${num_samples}" -c "${sequence_length}" 2>&1 | tee ./${mid}-bf16-gguf-ppl.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_perf" == false ]]; then
|
||||
${llama_batched_bench} $args_lcpp -m ./${mid}-bf16-gguf/model.gguf -c 33768 -b 2048 -ub 2048 -npp 2048,4096,8192,16384,32768 -ntg 128 -npl 1 2>&1 | tee ./${mid}-bf16-gguf-perf.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_keep" == true ]]; then
|
||||
echo "Deleting intermediate model files"
|
||||
rm -rfv ./${mid}-bf16-gguf
|
||||
fi
|
||||
|
||||
rm -rfv ./${mid}-f16-gguf && mkdir -p ./${mid}-f16-gguf
|
||||
${llama_quantize} ./${mid}-f32-gguf/model.gguf ./${mid}-f16-gguf/model.gguf f16
|
||||
get_size ./${mid}-f16-gguf > ./${mid}-f16-gguf-size.txt
|
||||
|
||||
if [[ "$no_ppl" == false ]]; then
|
||||
${llama_perplexity} $args_lcpp -m ./${mid}-f16-gguf/model.gguf -f "$raw_path" --chunks "${num_samples}" -c "${sequence_length}" 2>&1 | tee ./${mid}-f16-gguf-ppl.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_perf" == false ]]; then
|
||||
${llama_batched_bench} $args_lcpp -m ./${mid}-f16-gguf/model.gguf -c 33768 -b 2048 -ub 2048 -npp 2048,4096,8192,16384,32768 -ntg 128 -npl 1 2>&1 | tee ./${mid}-f16-gguf-perf.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_keep" == true ]]; then
|
||||
echo "Deleting intermediate model files"
|
||||
rm -rfv ./${mid}-f16-gguf
|
||||
fi
|
||||
|
||||
rm -rfv ./${mid}-q8-gguf && mkdir -p ./${mid}-q8-gguf
|
||||
${llama_quantize} ./${mid}-f32-gguf/model.gguf ./${mid}-q8-gguf/model.gguf q8_0
|
||||
get_size ./${mid}-q8-gguf > ./${mid}-q8-gguf-size.txt
|
||||
|
||||
if [[ "$no_ppl" == false ]]; then
|
||||
${llama_perplexity} $args_lcpp -m ./${mid}-q8-gguf/model.gguf -f "$raw_path" --chunks "${num_samples}" -c "${sequence_length}" 2>&1 | tee ./${mid}-q8-gguf-ppl.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_perf" == false ]]; then
|
||||
${llama_batched_bench} $args_lcpp -m ./${mid}-q8-gguf/model.gguf -c 33768 -b 2048 -ub 2048 -npp 2048,4096,8192,16384,32768 -ntg 128 -npl 1 2>&1 | tee ./${mid}-q8-gguf-perf.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_keep" == true ]]; then
|
||||
echo "Deleting intermediate model files"
|
||||
rm -rfv ./${mid}-q8-gguf
|
||||
fi
|
||||
|
||||
rm -rfv ./${mid}-q6-gguf && mkdir -p ./${mid}-q6-gguf
|
||||
${llama_quantize} ./${mid}-f32-gguf/model.gguf ./${mid}-q6-gguf/model.gguf q6_k
|
||||
get_size ./${mid}-q6-gguf > ./${mid}-q6-gguf-size.txt
|
||||
|
||||
if [[ "$no_ppl" == false ]]; then
|
||||
${llama_perplexity} $args_lcpp -m ./${mid}-q6-gguf/model.gguf -f "$raw_path" --chunks "${num_samples}" -c "${sequence_length}" 2>&1 | tee ./${mid}-q6-gguf-ppl.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_perf" == false ]]; then
|
||||
${llama_batched_bench} $args_lcpp -m ./${mid}-q6-gguf/model.gguf -c 33768 -b 2048 -ub 2048 -npp 2048,4096,8192,16384,32768 -ntg 128 -npl 1 2>&1 | tee ./${mid}-q6-gguf-perf.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_keep" == true ]]; then
|
||||
echo "Deleting intermediate model files"
|
||||
rm -rfv ./${mid}-q6-gguf
|
||||
fi
|
||||
|
||||
rm -rfv ./${mid}-q5-gguf && mkdir -p ./${mid}-q5-gguf
|
||||
${llama_quantize} ./${mid}-f32-gguf/model.gguf ./${mid}-q5-gguf/model.gguf q5_k_s
|
||||
get_size ./${mid}-q5-gguf > ./${mid}-q5-gguf-size.txt
|
||||
|
||||
if [[ "$no_ppl" == false ]]; then
|
||||
${llama_perplexity} $args_lcpp -m ./${mid}-q5-gguf/model.gguf -f "$raw_path" --chunks "${num_samples}" -c "${sequence_length}" 2>&1 | tee ./${mid}-q5-gguf-ppl.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_perf" == false ]]; then
|
||||
${llama_batched_bench} $args_lcpp -m ./${mid}-q5-gguf/model.gguf -c 33768 -b 2048 -ub 2048 -npp 2048,4096,8192,16384,32768 -ntg 128 -npl 1 2>&1 | tee ./${mid}-q5-gguf-perf.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_keep" == true ]]; then
|
||||
echo "Deleting intermediate model files"
|
||||
rm -rfv ./${mid}-q5-gguf
|
||||
fi
|
||||
|
||||
rm -rfv ./${mid}-q4p-gguf && mkdir -p ./${mid}-q4p-gguf
|
||||
${llama_quantize} ./${mid}-f32-gguf/model.gguf ./${mid}-q4p-gguf/model.gguf q4_k
|
||||
get_size ./${mid}-q4p-gguf > ./${mid}-q4p-gguf-size.txt
|
||||
|
||||
if [[ "$no_ppl" == false ]]; then
|
||||
${llama_perplexity} $args_lcpp -m ./${mid}-q4p-gguf/model.gguf -f "$raw_path" --chunks "${num_samples}" -c "${sequence_length}" 2>&1 | tee ./${mid}-q4p-gguf-ppl.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_perf" == false ]]; then
|
||||
${llama_batched_bench} $args_lcpp -m ./${mid}-q4p-gguf/model.gguf -c 33768 -b 2048 -ub 2048 -npp 2048,4096,8192,16384,32768 -ntg 128 -npl 1 2>&1 | tee ./${mid}-q4p-gguf-perf.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_keep" == true ]]; then
|
||||
echo "Deleting intermediate model files"
|
||||
rm -rfv ./${mid}-q4p-gguf
|
||||
fi
|
||||
|
||||
# note: we use --pure here to match the MLX quantization of the embeddings
|
||||
rm -rfv ./${mid}-q4-gguf && mkdir -p ./${mid}-q4-gguf
|
||||
${llama_quantize} --pure ./${mid}-f32-gguf/model.gguf ./${mid}-q4-gguf/model.gguf q4_0
|
||||
get_size ./${mid}-q4-gguf > ./${mid}-q4-gguf-size.txt
|
||||
|
||||
if [[ "$no_ppl" == false ]]; then
|
||||
${llama_perplexity} $args_lcpp -m ./${mid}-q4-gguf/model.gguf -f "$raw_path" --chunks "${num_samples}" -c "${sequence_length}" 2>&1 | tee ./${mid}-q4-gguf-ppl.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_perf" == false ]]; then
|
||||
${llama_batched_bench} $args_lcpp -m ./${mid}-q4-gguf/model.gguf -c 33768 -b 2048 -ub 2048 -npp 2048,4096,8192,16384,32768 -ntg 128 -npl 1 2>&1 | tee ./${mid}-q4-gguf-perf.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_keep" == true ]]; then
|
||||
echo "Deleting intermediate model files"
|
||||
rm -rfv ./${mid}-q4-gguf
|
||||
fi
|
||||
|
||||
rm -rfv ./${mid}-q3-gguf && mkdir -p ./${mid}-q3-gguf
|
||||
${llama_quantize} ./${mid}-f32-gguf/model.gguf ./${mid}-q3-gguf/model.gguf q3_k_s
|
||||
get_size ./${mid}-q3-gguf > ./${mid}-q3-gguf-size.txt
|
||||
|
||||
if [[ "$no_ppl" == false ]]; then
|
||||
${llama_perplexity} $args_lcpp -m ./${mid}-q3-gguf/model.gguf -f "$raw_path" --chunks "${num_samples}" -c "${sequence_length}" 2>&1 | tee ./${mid}-q3-gguf-ppl.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_perf" == false ]]; then
|
||||
${llama_batched_bench} $args_lcpp -m ./${mid}-q3-gguf/model.gguf -c 33768 -b 2048 -ub 2048 -npp 2048,4096,8192,16384,32768 -ntg 128 -npl 1 2>&1 | tee ./${mid}-q3-gguf-perf.txt
|
||||
fi
|
||||
|
||||
if [[ "$no_keep" == true ]]; then
|
||||
echo "Deleting intermediate model files"
|
||||
rm -rfv ./${mid}-q3-gguf
|
||||
fi
|
||||
|
||||
# remove the f32 model at the end
|
||||
if [[ "$no_keep" == true ]]; then
|
||||
rm -rfv ./${mid}-f32-gguf
|
||||
fi
|
||||
fi
|
||||
|
||||
set +x
|
||||
|
||||
# analyze results
|
||||
|
||||
types=("f32" "bf16" "f16" "q8" "q6" "q5" "q4p" "q4" "q3")
|
||||
|
||||
mlx_ppls=()
|
||||
mlx_ppl_deltas=()
|
||||
mlx_sizes=()
|
||||
mlx_pps2k=()
|
||||
mlx_tgs2k=()
|
||||
mlx_pps4k=()
|
||||
mlx_tgs4k=()
|
||||
mlx_pps8k=()
|
||||
mlx_tgs8k=()
|
||||
mlx_pps16k=()
|
||||
mlx_tgs16k=()
|
||||
mlx_pps32k=()
|
||||
mlx_tgs32k=()
|
||||
|
||||
# mlx:
|
||||
for t in ${types[*]}; do
|
||||
cur_ppl="N/A"
|
||||
cur_ppl_delta="N/A"
|
||||
cur_size="N/A"
|
||||
cur_pp2k="N/A"
|
||||
cur_tg2k="N/A"
|
||||
cur_pp4k="N/A"
|
||||
cur_tg4k="N/A"
|
||||
cur_pp8k="N/A"
|
||||
cur_tg8k="N/A"
|
||||
cur_pp16k="N/A"
|
||||
cur_tg16k="N/A"
|
||||
cur_pp32k="N/A"
|
||||
cur_tg32k="N/A"
|
||||
|
||||
if [[ -f ./${mid}-${t}-mlx-ppl.txt ]]; then
|
||||
cur_ppl=$(grep -o 'Perplexity: [0-9.]*' ./${mid}-${t}-mlx-ppl.txt | cut -d' ' -f2)
|
||||
cur_ppl_delta=$(grep -o 'Perplexity: [0-9.]* ± [0-9.]*' ./${mid}-${t}-mlx-ppl.txt | cut -d' ' -f4)
|
||||
cur_size=$(cat ./${mid}-${t}-mlx-size.txt)
|
||||
cur_pp2k=$(grep -o 'Averages.*prompt_tps=[0-9.]*' ./${mid}-${t}-mlx-perf-2048.txt | cut -d'=' -f2)
|
||||
cur_tg2k=$(grep -o 'Averages.*generation_tps=[0-9.]*' ./${mid}-${t}-mlx-perf-2048.txt | cut -d'=' -f3)
|
||||
cur_pp4k=$(grep -o 'Averages.*prompt_tps=[0-9.]*' ./${mid}-${t}-mlx-perf-4096.txt | cut -d'=' -f2)
|
||||
cur_tg4k=$(grep -o 'Averages.*generation_tps=[0-9.]*' ./${mid}-${t}-mlx-perf-4096.txt | cut -d'=' -f3)
|
||||
cur_pp8k=$(grep -o 'Averages.*prompt_tps=[0-9.]*' ./${mid}-${t}-mlx-perf-8192.txt | cut -d'=' -f2)
|
||||
cur_tg8k=$(grep -o 'Averages.*generation_tps=[0-9.]*' ./${mid}-${t}-mlx-perf-8192.txt | cut -d'=' -f3)
|
||||
cur_pp16k=$(grep -o 'Averages.*prompt_tps=[0-9.]*' ./${mid}-${t}-mlx-perf-16384.txt | cut -d'=' -f2)
|
||||
cur_tg16k=$(grep -o 'Averages.*generation_tps=[0-9.]*' ./${mid}-${t}-mlx-perf-16384.txt | cut -d'=' -f3)
|
||||
cur_pp32k=$(grep -o 'Averages.*prompt_tps=[0-9.]*' ./${mid}-${t}-mlx-perf-32768.txt | cut -d'=' -f2)
|
||||
cur_tg32k=$(grep -o 'Averages.*generation_tps=[0-9.]*' ./${mid}-${t}-mlx-perf-32768.txt | cut -d'=' -f3)
|
||||
fi
|
||||
|
||||
mlx_ppls+=("${cur_ppl}")
|
||||
mlx_ppl_deltas+=("${cur_ppl_delta}")
|
||||
mlx_sizes+=("${cur_size}")
|
||||
mlx_pps2k+=("${cur_pp2k}")
|
||||
mlx_tgs2k+=("${cur_tg2k}")
|
||||
mlx_pps4k+=("${cur_pp4k}")
|
||||
mlx_tgs4k+=("${cur_tg4k}")
|
||||
mlx_pps8k+=("${cur_pp8k}")
|
||||
mlx_tgs8k+=("${cur_tg8k}")
|
||||
mlx_pps16k+=("${cur_pp16k}")
|
||||
mlx_tgs16k+=("${cur_tg16k}")
|
||||
mlx_pps32k+=("${cur_pp32k}")
|
||||
mlx_tgs32k+=("${cur_tg32k}")
|
||||
done
|
||||
|
||||
gguf_ppls=()
|
||||
gguf_ppl_deltas=()
|
||||
gguf_sizes=()
|
||||
gguf_pps2k=()
|
||||
gguf_tgs2k=()
|
||||
gguf_pps4k=()
|
||||
gguf_tgs4k=()
|
||||
gguf_pps8k=()
|
||||
gguf_tgs8k=()
|
||||
gguf_pps16k=()
|
||||
gguf_tgs16k=()
|
||||
gguf_pps32k=()
|
||||
gguf_tgs32k=()
|
||||
|
||||
# gguf:
|
||||
for t in ${types[*]}; do
|
||||
cur_ppl="N/A"
|
||||
cur_ppl_delta="N/A"
|
||||
cur_size="N/A"
|
||||
cur_pp2k="N/A"
|
||||
cur_tg2k="N/A"
|
||||
cur_pp4k="N/A"
|
||||
cur_tg4k="N/A"
|
||||
cur_pp8k="N/A"
|
||||
cur_tg8k="N/A"
|
||||
cur_pp16k="N/A"
|
||||
cur_tg16k="N/A"
|
||||
cur_pp32k="N/A"
|
||||
cur_tg32k="N/A"
|
||||
|
||||
if [[ -f ./${mid}-${t}-gguf-ppl.txt ]]; then
|
||||
cur_ppl=$(grep -o 'Final estimate: PPL = [0-9.]*' ./${mid}-${t}-gguf-ppl.txt | sed -e "s/.*Final//" | cut -d' ' -f5)
|
||||
cur_ppl_delta=$(grep -o 'Final estimate: PPL = [0-9.]* +/- [0-9.]*' ./${mid}-${t}-gguf-ppl.txt | sed -e "s/.*Final//" | cut -d' ' -f7)
|
||||
cur_size=$(cat ./${mid}-${t}-gguf-size.txt)
|
||||
cur_pp2k=$(grep -o '| 2048 |.*' ./${mid}-${t}-gguf-perf.txt | awk '{print $12}')
|
||||
cur_tg2k=$(grep -o '| 2048 |.*' ./${mid}-${t}-gguf-perf.txt | awk '{print $16}')
|
||||
cur_pp4k=$(grep -o '| 4096 |.*' ./${mid}-${t}-gguf-perf.txt | awk '{print $12}')
|
||||
cur_tg4k=$(grep -o '| 4096 |.*' ./${mid}-${t}-gguf-perf.txt | awk '{print $16}')
|
||||
cur_pp8k=$(grep -o '| 8192 |.*' ./${mid}-${t}-gguf-perf.txt | awk '{print $12}')
|
||||
cur_tg8k=$(grep -o '| 8192 |.*' ./${mid}-${t}-gguf-perf.txt | awk '{print $16}')
|
||||
cur_pp16k=$(grep -o '| 16384 |.*' ./${mid}-${t}-gguf-perf.txt | awk '{print $12}')
|
||||
cur_tg16k=$(grep -o '| 16384 |.*' ./${mid}-${t}-gguf-perf.txt | awk '{print $16}')
|
||||
cur_pp32k=$(grep -o '| 32768 |.*' ./${mid}-${t}-gguf-perf.txt | awk '{print $12}')
|
||||
cur_tg32k=$(grep -o '| 32768 |.*' ./${mid}-${t}-gguf-perf.txt | awk '{print $16}')
|
||||
fi
|
||||
|
||||
gguf_ppls+=("${cur_ppl}")
|
||||
gguf_ppl_deltas+=("${cur_ppl_delta}")
|
||||
gguf_sizes+=("${cur_size}")
|
||||
gguf_pps2k+=("${cur_pp2k}")
|
||||
gguf_tgs2k+=("${cur_tg2k}")
|
||||
gguf_pps4k+=("${cur_pp4k}")
|
||||
gguf_tgs4k+=("${cur_tg4k}")
|
||||
gguf_pps8k+=("${cur_pp8k}")
|
||||
gguf_tgs8k+=("${cur_tg8k}")
|
||||
gguf_pps16k+=("${cur_pp16k}")
|
||||
gguf_tgs16k+=("${cur_tg16k}")
|
||||
gguf_pps32k+=("${cur_pp32k}")
|
||||
gguf_tgs32k+=("${cur_tg32k}")
|
||||
done
|
||||
|
||||
res="${mid}-results.txt"
|
||||
echo "Results for ${org}/${mid} saved to ${res}"
|
||||
|
||||
printf "\n" | tee ${res}
|
||||
printf "Model ID: ${org}/${mid}\n" | tee -a ${res}
|
||||
#printf "Samples: ${num_samples}\n" | tee -a ${res}
|
||||
#printf "Sequence Length: ${sequence_length}\n" | tee -a ${res}
|
||||
printf "\n" | tee -a ${res}
|
||||
printf "| Type | MLX PPL | GGUF PPL | MLX Size | GGUF Size | MLX PP 2K | GGUF PP 2K | MLX TG 2K | GGUF TG 2K | MLX PP 4K | GGUF PP 4K | MLX TG 4K | GGUF TG 4K | MLX PP 8K | GGUF PP 8K | MLX TG 8K | GGUF TG 8K | MLX PP 16K | GGUF PP 16K | MLX TG 16K | GGUF TG 16K | MLX PP 32K | GGUF PP 32K | MLX TG 32K | GGUF TG 32K |\n" | tee -a ${res}
|
||||
printf "|-------|---------------------|------------------------|----------|-----------| ---------- | ----------- | ---------- | ----------- | ---------- | ----------- | ---------- | ----------- | ---------- | ----------- | ---------- | ----------- | ---------- | ----------- | ---------- | ----------- | ---------- | ----------- | ---------- | ----------- |\n" | tee -a ${res}
|
||||
|
||||
for i in "${!types[@]}"; do
|
||||
printf "| %-5s | %10s ± %6s | %10s ± %9s | %8s | %9s | %10s | %11s | %10s | %11s | %10s | %11s | %10s | %11s | %10s | %11s | %10s | %11s | %10s | %11s | %10s | %11s | %10s | %11s | %10s | %11s |\n" \
|
||||
"${types[i]}" \
|
||||
"${mlx_ppls[i]}" \
|
||||
"${mlx_ppl_deltas[i]}" \
|
||||
"${gguf_ppls[i]}" \
|
||||
"${gguf_ppl_deltas[i]}" \
|
||||
"${mlx_sizes[i]}" \
|
||||
"${gguf_sizes[i]}" \
|
||||
"${mlx_pps2k[i]}" \
|
||||
"${gguf_pps2k[i]}" \
|
||||
"${mlx_tgs2k[i]}" \
|
||||
"${gguf_tgs2k[i]}" \
|
||||
"${mlx_pps4k[i]}" \
|
||||
"${gguf_pps4k[i]}" \
|
||||
"${mlx_tgs4k[i]}" \
|
||||
"${gguf_tgs4k[i]}" \
|
||||
"${mlx_pps8k[i]}" \
|
||||
"${gguf_pps8k[i]}" \
|
||||
"${mlx_tgs8k[i]}" \
|
||||
"${gguf_tgs8k[i]}" \
|
||||
"${mlx_pps16k[i]}" \
|
||||
"${gguf_pps16k[i]}" \
|
||||
"${mlx_tgs16k[i]}" \
|
||||
"${gguf_tgs16k[i]}" \
|
||||
"${mlx_pps32k[i]}" \
|
||||
"${gguf_pps32k[i]}" \
|
||||
"${mlx_tgs32k[i]}" \
|
||||
"${gguf_tgs32k[i]}" | tee -a ${res}
|
||||
done
|
||||
}
|
||||
|
||||
eval_model "meta-llama" "Llama-3.2-1B"
|
||||
eval_model "meta-llama" "Llama-3.2-3B"
|
||||
eval_model "meta-llama" "Llama-3.1-8B"
|
||||
|
||||
eval_model "google" "gemma-3-270m"
|
||||
eval_model "google" "gemma-3-1b-pt"
|
||||
eval_model "google" "gemma-3-4b-pt"
|
||||
|
||||
# the mlx-ppl.y script does not work with these models - not sure why
|
||||
#eval_model "google" "gemma-3n-E2B"
|
||||
#eval_model "google" "gemma-3n-E4B"
|
||||
|
||||
eval_model "Qwen" "Qwen3-0.6B-Base"
|
||||
eval_model "Qwen" "Qwen3-1.7B-Base"
|
||||
eval_model "Qwen" "Qwen3-4B-Base"
|
||||
eval_model "Qwen" "Qwen3-8B-Base"
|
||||
eval_model "Qwen" "Qwen3-30B-A3B-Base"
|
||||
120
examples/compare-mlx/inspect_model.py
Normal file
120
examples/compare-mlx/inspect_model.py
Normal file
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env python3
|
||||
# generated by Claude
|
||||
"""
|
||||
Script to inspect SafeTensors model files and print tensor information.
|
||||
"""
|
||||
|
||||
import json
|
||||
from safetensors import safe_open
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
def inspect_safetensors_model(model_dir="."):
|
||||
"""Inspect all SafeTensors files in the model directory."""
|
||||
|
||||
# First, let's read the index file to see the file structure
|
||||
index_file = Path(model_dir) / "model.safetensors.index.json"
|
||||
|
||||
if index_file.exists():
|
||||
with open(index_file, 'r') as f:
|
||||
index_data = json.load(f)
|
||||
|
||||
print("=== Model Structure ===")
|
||||
print(f"Total parameters: {index_data.get('metadata', {}).get('total_size', 'Unknown')}")
|
||||
print()
|
||||
|
||||
# Get all safetensor files
|
||||
safetensor_files = set(index_data.get('weight_map', {}).values())
|
||||
else:
|
||||
# If no index file, look for safetensor files directly
|
||||
safetensor_files = [f for f in os.listdir(model_dir) if f.endswith('.safetensors')]
|
||||
|
||||
# Sort files for consistent output
|
||||
safetensor_files = sorted(safetensor_files)
|
||||
|
||||
print("=== Tensor Information ===")
|
||||
print(f"{'Tensor Name':<50} {'Shape':<25} {'Data Type':<15} {'File'}")
|
||||
print("-" * 110)
|
||||
|
||||
total_tensors = 0
|
||||
|
||||
for filename in safetensor_files:
|
||||
filepath = Path(model_dir) / filename
|
||||
if not filepath.exists():
|
||||
continue
|
||||
|
||||
print(f"\n--- {filename} ---")
|
||||
|
||||
# Open and inspect the safetensor file
|
||||
with safe_open(filepath, framework="pt") as f: # Use PyTorch framework for better dtype support
|
||||
tensor_names = f.keys()
|
||||
|
||||
for tensor_name in sorted(tensor_names):
|
||||
# Get tensor metadata without loading the full tensor
|
||||
tensor_slice = f.get_slice(tensor_name)
|
||||
shape = tensor_slice.get_shape()
|
||||
dtype = tensor_slice.get_dtype()
|
||||
|
||||
shape_str = str(tuple(shape))
|
||||
dtype_str = str(dtype)
|
||||
|
||||
print(f"{tensor_name:<50} {shape_str:<25} {dtype_str:<15} {filename}")
|
||||
total_tensors += 1
|
||||
|
||||
print(f"\nTotal tensors found: {total_tensors}")
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Inspect SafeTensors model files")
|
||||
parser.add_argument("--model-dir", "-d", default=".",
|
||||
help="Directory containing the model files (default: current directory)")
|
||||
parser.add_argument("--summary", "-s", action="store_true",
|
||||
help="Show only summary statistics")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.summary:
|
||||
print_summary_only(args.model_dir)
|
||||
else:
|
||||
inspect_safetensors_model(args.model_dir)
|
||||
|
||||
def print_summary_only(model_dir="."):
|
||||
"""Print only summary statistics."""
|
||||
safetensor_files = [f for f in os.listdir(model_dir) if f.endswith('.safetensors')]
|
||||
|
||||
total_tensors = 0
|
||||
dtype_counts = {}
|
||||
total_params = 0
|
||||
|
||||
for filename in sorted(safetensor_files):
|
||||
filepath = Path(model_dir) / filename
|
||||
if not filepath.exists():
|
||||
continue
|
||||
|
||||
with safe_open(filepath, framework="pt") as f: # Use PyTorch framework
|
||||
for tensor_name in f.keys():
|
||||
tensor_slice = f.get_slice(tensor_name)
|
||||
shape = tensor_slice.get_shape()
|
||||
dtype = tensor_slice.get_dtype()
|
||||
|
||||
total_tensors += 1
|
||||
|
||||
dtype_str = str(dtype)
|
||||
dtype_counts[dtype_str] = dtype_counts.get(dtype_str, 0) + 1
|
||||
|
||||
# Calculate parameter count
|
||||
param_count = 1
|
||||
for dim in shape:
|
||||
param_count *= dim
|
||||
total_params += param_count
|
||||
|
||||
print("=== Model Summary ===")
|
||||
print(f"Total tensors: {total_tensors}")
|
||||
print(f"Total parameters: {total_params:,}")
|
||||
print(f"Data type distribution:")
|
||||
for dtype, count in sorted(dtype_counts.items()):
|
||||
print(f" {dtype}: {count} tensors")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
305
examples/compare-mlx/mlx-ppl.py
Normal file
305
examples/compare-mlx/mlx-ppl.py
Normal file
@@ -0,0 +1,305 @@
|
||||
# Copyright © 2025 Apple Inc.
|
||||
# modified: https://github.com/ml-explore/mlx-lm/blob/60320dc2347d45dc3ca08be90e5255fb9424bb09/mlx_lm/perplexity.py
|
||||
"""
|
||||
Evaluate perplexity (PPL) of pre-trained MLX models in the same way as llama.cpp's llama-perplexity.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
import types
|
||||
|
||||
import mlx.core as mx
|
||||
import mlx.nn as nn
|
||||
import numpy as np
|
||||
|
||||
from mlx_lm.tuner.datasets import load_dataset
|
||||
from mlx_lm.tuner.utils import get_total_parameters
|
||||
from mlx_lm.utils import load
|
||||
|
||||
|
||||
def load_data(
|
||||
tokenizer,
|
||||
data_path: str,
|
||||
num_samples: int,
|
||||
sequence_length: int,
|
||||
):
|
||||
"""
|
||||
Load a Hugging‑Face dataset (via mlx‑lm’s dataset utilities) and convert it
|
||||
into a token tensor of shape (N, sequence_length).
|
||||
"""
|
||||
args = types.SimpleNamespace(
|
||||
hf_dataset={
|
||||
"path": data_path,
|
||||
"train_split": "train",
|
||||
"valid_split": "train[:1]",
|
||||
},
|
||||
train=True,
|
||||
test=False,
|
||||
)
|
||||
dataset = load_dataset(args, tokenizer)[0]
|
||||
|
||||
perm = np.random.permutation(len(dataset)).tolist()
|
||||
|
||||
num_tokens = sequence_length * num_samples if num_samples > 0 else float("inf")
|
||||
data = []
|
||||
i = 0
|
||||
while len(data) < num_tokens:
|
||||
tokens, _ = dataset.process(dataset[perm[i]])
|
||||
i += 1
|
||||
data.extend(tokens)
|
||||
|
||||
# Convert to MX array, truncate to a multiple of `sequence_length`
|
||||
data = mx.array(data[: (len(data) // sequence_length) * sequence_length])
|
||||
data = data.reshape(-1, sequence_length)
|
||||
if num_samples > 0:
|
||||
data = data[:num_samples]
|
||||
return data
|
||||
|
||||
|
||||
def _tokenize_text(tokenizer, text: str):
|
||||
"""
|
||||
Helper that tokenises a string using the MLX‑LM tokenizer.
|
||||
Supports the common `encode` method or a callable tokenizer.
|
||||
"""
|
||||
# Most mlx‑lm tokenizers expose an `encode` method.
|
||||
if hasattr(tokenizer, "encode"):
|
||||
tokens = tokenizer.encode(text)
|
||||
elif callable(tokenizer):
|
||||
tokens = tokenizer(text)
|
||||
else:
|
||||
raise AttributeError(
|
||||
"Tokenizer does not have an `encode` method nor is it callable."
|
||||
)
|
||||
# Normalise the output to a Python list of ints.
|
||||
if isinstance(tokens, mx.array):
|
||||
tokens = tokens.tolist()
|
||||
return tokens
|
||||
|
||||
|
||||
# load a raw text file and tokenize it
|
||||
# generated with gpt-oss-120b
|
||||
def load_raw_data(
|
||||
tokenizer,
|
||||
raw_path: str,
|
||||
num_samples: int,
|
||||
sequence_length: int,
|
||||
):
|
||||
"""
|
||||
Load a raw text file, tokenize it, and reshape into a (N, sequence_length)
|
||||
tensor suitable for perplexity evaluation.
|
||||
"""
|
||||
if not os.path.isfile(raw_path):
|
||||
raise FileNotFoundError(f"Raw text file not found: {raw_path}")
|
||||
|
||||
# Read the whole file (UTF‑8). Users can supply any plain‑text corpus.
|
||||
with open(raw_path, "r", encoding="utf-8") as fp:
|
||||
raw_text = fp.read()
|
||||
|
||||
# Tokenise the complete text.
|
||||
token_list = _tokenize_text(tokenizer, raw_text)
|
||||
|
||||
if len(token_list) == 0:
|
||||
raise ValueError("Tokenisation of the raw file produced no tokens.")
|
||||
|
||||
# Convert to MX array (int32 is sufficient for token IDs).
|
||||
token_array = mx.array(token_list, dtype=mx.int32)
|
||||
|
||||
# Trim to a length that is an exact multiple of `sequence_length`.
|
||||
total_len = (token_array.shape[0] // sequence_length) * sequence_length
|
||||
token_array = token_array[:total_len]
|
||||
|
||||
# Reshape into (num_sequences, sequence_length)
|
||||
data = token_array.reshape(-1, sequence_length)
|
||||
|
||||
if num_samples > 0:
|
||||
data = data[:num_samples]
|
||||
|
||||
#print(f"First 4 samples of the data:")
|
||||
#for j in range(min(4, len(data))):
|
||||
# print(f" Sample {j}: {tokenizer.decode(data[j].tolist())}\n\n-------------------\n\n")
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def eval_ppl(model, tokenizer, data, batch_size=8):
|
||||
"""
|
||||
Evaluate perplexity on a dataset with standard error calculation.
|
||||
|
||||
Args:
|
||||
model: The model to evaluate.
|
||||
data: Tokenized data tensor (shape: N x L).
|
||||
batch_size: Batch size for evaluation.
|
||||
|
||||
Returns:
|
||||
tuple: (perplexity, standard_error_of_perplexity)
|
||||
"""
|
||||
all_losses = []
|
||||
|
||||
num_batches = (len(data) + batch_size - 1) // batch_size
|
||||
for i, s in enumerate(range(0, len(data), batch_size)):
|
||||
batch = data[s : s + batch_size]
|
||||
|
||||
# Set the first token of all samples to the BOS token
|
||||
if tokenizer.bos_token_id:
|
||||
batch[:, 0] = tokenizer.bos_token_id
|
||||
|
||||
# compute cross entropy only with the second half of the sequence to match llama.cpp behavior
|
||||
# ref: https://github.com/ggml-org/llama.cpp/blob/696fccf354e9dbdfbce135bc40b44c9dcc64dda9/tools/perplexity/perplexity.cpp#L527-L541
|
||||
#
|
||||
#start = 0
|
||||
start = batch.shape[1] // 2
|
||||
|
||||
# Forward pass: get logits for all tokens except last
|
||||
logits = model(batch[:, :-1]).astype(mx.float32)
|
||||
|
||||
# Calculate cross‑entropy loss with next tokens
|
||||
#losses = nn.losses.cross_entropy(logits, batch[:, 1:], reduction="none")
|
||||
losses = nn.losses.cross_entropy(logits[:, start:, :], batch[:, start+1:], reduction="none")
|
||||
|
||||
mx.eval(losses)
|
||||
# Store individual token losses
|
||||
all_losses.append(losses.flatten())
|
||||
|
||||
# Progress indicator
|
||||
if (i + 1) % 1 == 0 or (i + 1) == num_batches:
|
||||
print(f" Processed {i + 1}/{num_batches} batches...", end="\r")
|
||||
|
||||
print() # New line after progress
|
||||
|
||||
# Concatenate all losses into a single array
|
||||
all_losses = mx.concatenate(all_losses)
|
||||
|
||||
# Calculate mean loss and perplexity
|
||||
mean_loss = all_losses.mean().item()
|
||||
ppl = math.exp(mean_loss)
|
||||
|
||||
# Calculate standard error
|
||||
std_dev = mx.sqrt(mx.var(all_losses, ddof=1)).item()
|
||||
num_tokens = all_losses.size
|
||||
standard_error = std_dev / math.sqrt(num_tokens)
|
||||
|
||||
# Delta approximation for standard error of perplexity
|
||||
standard_error_ppl = ppl * standard_error
|
||||
|
||||
return ppl, standard_error_ppl
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Evaluate perplexity of MLX models")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
required=True,
|
||||
help="Path to model or Hugging Face model ID",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch-size", type=int, default=8, help="Batch size for evaluation"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--sequence-length",
|
||||
type=int,
|
||||
default=512,
|
||||
help="Sequence length for evaluation",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-samples",
|
||||
type=int,
|
||||
default=256,
|
||||
help="Number of samples to use (-1 for all available)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--data-path",
|
||||
type=str,
|
||||
default="allenai/tulu-3-sft-mixture",
|
||||
help=(
|
||||
"A Hugging Face dataset compatible with mlx‑lm. "
|
||||
"Ignored if --raw-path is provided."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--raw-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"Path to a local raw‑text file to use for evaluation. "
|
||||
"If specified, the script skips loading a HF dataset."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seed", type=int, default=123, help="Random seed for data sampling"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Set random seed (used for HF dataset shuffling)
|
||||
mx.random.seed(args.seed)
|
||||
|
||||
# Load model
|
||||
print(f"Loading model from {args.model}...")
|
||||
model, tokenizer = load(args.model)
|
||||
|
||||
# Count parameters
|
||||
total_params = get_total_parameters(model)
|
||||
print(f"Model loaded: {total_params/1e6:.1f}M parameters")
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Load evaluation data (raw file vs. HF dataset)
|
||||
# ----------------------------------------------------------------------
|
||||
print("\nLoading dataset...")
|
||||
print(f" Sequence length: {args.sequence_length}")
|
||||
|
||||
if args.raw_path:
|
||||
print(f" Using raw text file: {args.raw_path}")
|
||||
data = load_raw_data(
|
||||
tokenizer,
|
||||
raw_path=args.raw_path,
|
||||
num_samples=args.num_samples,
|
||||
sequence_length=args.sequence_length,
|
||||
)
|
||||
else:
|
||||
print(f" Using HF dataset: {args.data_path}")
|
||||
data = load_data(
|
||||
tokenizer,
|
||||
data_path=args.data_path,
|
||||
num_samples=args.num_samples,
|
||||
sequence_length=args.sequence_length,
|
||||
)
|
||||
|
||||
print(f" Loaded {len(data)} samples")
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Evaluate perplexity
|
||||
# ----------------------------------------------------------------------
|
||||
print(f"\nEvaluating perplexity with batch size {args.batch_size}...")
|
||||
start_time = time.time()
|
||||
|
||||
ppl, se = eval_ppl(model, tokenizer, data, batch_size=args.batch_size)
|
||||
|
||||
eval_time = time.time() - start_time
|
||||
tokens_evaluated = data.shape[0] * (data.shape[1] - 1) # B * (L - 1)
|
||||
|
||||
# Print results
|
||||
print("\n" + "=" * 60)
|
||||
print("EVALUATION RESULTS")
|
||||
print("=" * 60)
|
||||
print(f"Model: {args.model}")
|
||||
print(f"Perplexity: {ppl:.3f} ± {se:.3f}")
|
||||
print(f"Evaluation time: {eval_time:.2f} seconds")
|
||||
print(f"Peak memory: {mx.get_peak_memory() / 1e9:.2f} GB")
|
||||
print(f"Tokens per second: {tokens_evaluated / eval_time:.0f}")
|
||||
|
||||
# Additional statistics
|
||||
print(f"\nDataset statistics:")
|
||||
print(f" Total samples: {len(data)}")
|
||||
print(f" Total tokens: {data.size}")
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Done
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user