CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-10-31 08:51:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added a single test function script and fix debug-test.sh to be more robust (#7279)

Browse Source 
* run-single-test.sh: added a single test function script and fix debug-test.sh to be more robust

* debug-test.sh: combined execute and gdb test mode via -g flag

* debug-test.sh: refactor

* debug-test: refactor for clarity

* debug-test.sh: comment style changes

* debug-test.sh: fix gdb
This commit is contained in:
Brian
2024-05-17 22:40:14 +10:00
committed by GitHub
parent d273c1402b
commit 51e9d02599
2 changed files with 196 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
docs/debugging-tests.md
View File

@@ -1,6 +1,6 @@
# Debugging Tests Tips # Debugging Tests Tips
## How to run & debug a specific test without anything else to keep the feedback loop short? ## How to run & execute or debug a specific test without anything else to keep the feedback loop short?
There is a script called debug-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number. There is a script called debug-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number.
@@ -10,13 +10,27 @@ For example, running the following command will output an interactive list from
It will then build & run in the debugger for you. It will then build & run in the debugger for you.
To just execute a test and get back a PASS or FAIL message run:
```bash ```bash
./scripts/debug-test.sh test-tokenizer ./scripts/debug-test.sh test-tokenizer
```
To test in GDB use the `-g` flag to enable gdb test mode.
```bash
./scripts/debug-test.sh -g test-tokenizer
# Once in the debugger, i.e. at the chevrons prompt, setting a breakpoint could be as follows: # Once in the debugger, i.e. at the chevrons prompt, setting a breakpoint could be as follows:
>>> b main >>> b main
``` ```
To speed up the testing loop, if you know your test number you can just run it similar to below:
```bash
./scripts/debug-test.sh test 23
```
For further reference use `debug-test.sh -h` to print help. For further reference use `debug-test.sh -h` to print help.
   
@@ -41,7 +55,7 @@ cmake -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON ..
make -j make -j
``` ```
#### Step 3.1: Identify Test Command for Debugging #### Step 3: Find all tests available that matches REGEX
The output of this command will give you the command & arguments needed to run GDB. The output of this command will give you the command & arguments needed to run GDB.
@@ -69,11 +83,13 @@ Labels: main
... ...
``` ```
So for test #1 we can tell these two pieces of relevant information: #### Step 4: Identify Test Command for Debugging
So for test #1 above we can tell these two pieces of relevant information:
* Test Binary: `~/llama.cpp/build-ci-debug/bin/test-tokenizer-0` * Test Binary: `~/llama.cpp/build-ci-debug/bin/test-tokenizer-0`
* Test GGUF Model: `~/llama.cpp/tests/../models/ggml-vocab-llama-spm.gguf` * Test GGUF Model: `~/llama.cpp/tests/../models/ggml-vocab-llama-spm.gguf`
#### Step 3.2: Run GDB on test command #### Step 5: Run GDB on test command
Based on the ctest 'test command' report above we can then run a gdb session via this command below: Based on the ctest 'test command' report above we can then run a gdb session via this command below:

266
scripts/debug-test.sh
View File

@@ -1,117 +1,203 @@
#!/bin/bash #!/bin/bash
test_suite=${1:-}
test_number=${2:-}
PROG=${0##*/} PROG=${0##*/}
build_dir="build-ci-debug" build_dir="build-ci-debug"
if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then # Print Color Commands
echo "Usage: $PROG [OPTION]... <test_regex> (test_number)" red=$(tput setaf 1)
echo "Debug specific ctest program." green=$(tput setaf 2)
echo yellow=$(tput setaf 3)
echo "Options:" blue=$(tput setaf 4)
echo " -h, --help Display this help and exit" magenta=$(tput setaf 5)
echo cyan=$(tput setaf 6)
echo "Arguments:" normal=$(tput sgr0)
echo " <test_regex> (Mandatory) Supply one regex to the script to filter tests"
echo " (test_number) (Optional) Test number to run a specific test"
echo
echo "Example:"
echo " $PROG test-tokenizer"
echo " $PROG test-tokenizer 3"
echo
exit 0
fi
# Function to select and debug a test
function select_test() {
test_suite=${1:-test}
test_number=${2:-}
# Sanity Check If Tests Is Detected # Print Help Message
printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n" ####################
tests=($(ctest -R ${test_suite} -V -N | grep -E " +Test +#[0-9]+*" | cut -d':' -f2 | awk '{$1=$1};1'))
if [ ${#tests[@]} -eq 0 ]
then
echo "No tests avaliable... check your compliation process..."
echo "Exiting."
exit 1
fi
if [ -z $test_number ] print_full_help() {
then cat << EOF
# List out avaliable tests Usage: $PROG [OPTION]... <test_regex> (test_number)
printf "Which test would you like to debug?\n" Debug specific ctest program.
id=0
for s in "${tests[@]}"
do
echo "Test# ${id}"
echo " $s"
((id++))
done
# Prompt user which test they wanted to run Options:
printf "\nRun test#? " -h, --help display this help and exit
read test_number -g run in gdb mode
else
printf "\nUser Already Requested #${test_number}"
fi
# Start GDB with the requested test binary and arguments Arguments:
printf "Debugging(GDB) test: ${tests[test_number]}\n" <test_regex> (Mandatory) Supply one regex to the script to filter tests
# Change IFS (Internal Field Separator) (test_number) (Optional) Test number to run a specific test
sIFS=$IFS
IFS=$'\n'
# Get test args Example:
gdb_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' )) $PROG test-tokenizer
IFS=$sIFS $PROG test-tokenizer 3
printf "Debug arguments: ${gdb_args[test_number]}\n\n" EOF
# Expand paths if needed
args=()
for x in $(echo ${gdb_args[test_number]} | sed -e 's/"\/\<//' -e 's/\>"//')
do
args+=($(echo $x | sed -e 's/.*\/..\//..\//'))
done
# Execute debugger
echo "gdb args: ${args[@]}"
gdb --args ${args[@]}
} }
abort() {
echo "Error: $1" >&2
cat << EOF >&2
Usage: $PROG [OPTION]... <test_regex> (test_number)
Debug specific ctest program.
Refer to --help for full instructions.
EOF
exit 1
}
# Dependency Sanity Check
#########################
check_dependency() {
command -v "$1" >/dev/null 2>&1 || {
abort "$1 is required but not found. Please install it and try again."
}
}
check_dependency ctest
check_dependency cmake
# Step 0: Check the args # Step 0: Check the args
if [ -z "$test_suite" ] ########################
then
echo "Usage: $PROG [OPTION]... <test_regex> (test_number)" if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then
echo "Supply one regex to the script to filter tests," print_full_help >&2
echo "and optionally a test number to run a specific test." exit 0
echo "Use --help flag for full instructions"
exit 1
fi fi
# Parse command-line options
gdb_mode=false
while getopts "g" opt; do
case $opt in
g)
gdb_mode=true
echo "gdb_mode Mode Enabled"
;;
esac
done
# Shift the option parameters
shift $((OPTIND - 1))
# Positionial Argument Processing : <test_regex>
if [ -z "${1}" ]; then
abort "Test regex is required"
else
test_suite=${1:-}
fi
# Positionial Argument Processing : (test_number)
test_number=${2:-}
# Step 1: Reset and Setup folder context # Step 1: Reset and Setup folder context
########################################
## Sanity check that we are actually in a git repo ## Sanity check that we are actually in a git repo
repo_root=$(git rev-parse --show-toplevel) repo_root=$(git rev-parse --show-toplevel)
if [ ! -d "$repo_root" ]; then if [ ! -d "$repo_root" ]; then
echo "Error: Not in a Git repository." abort "Not in a Git repository."
exit 1
fi fi
## Reset folder to root context of git repo ## Reset folder to root context of git repo and Create and enter build directory
pushd "$repo_root" || exit 1 pushd "$repo_root"
rm -rf "$build_dir" && mkdir "$build_dir" || abort "Failed to make $build_dir"
## Create and enter build directory
rm -rf "$build_dir" && mkdir "$build_dir" || exit 1
# Step 2: Setup Build Environment and Compile Test Binaries # Step 2: Setup Build Environment and Compile Test Binaries
cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON || exit 1 ###########################################################
pushd "$build_dir" && make -j || exit 1
# Step 3: Debug the Test # Note: test-eval-callback requires -DLLAMA_CURL
select_test "$test_suite" "$test_number" cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_CURL=1 || abort "Failed to build enviroment"
pushd "$build_dir"
make -j || abort "Failed to compile"
popd > /dev/null || exit 1
# Step 4: Return to the directory from which the user ran the command.
popd || exit 1 # Step 3: Find all tests available that matches REGEX
popd || exit 1 ####################################################
popd || exit 1
# Ctest Gather Tests
# `-R test-tokenizer` : looks for all the test files named `test-tokenizer*` (R=Regex)
# `-N` : "show-only" disables test execution & shows test commands that you can feed to GDB.
# `-V` : Verbose Mode
printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n"
pushd "$build_dir"
tests=($(ctest -R ${test_suite} -V -N | grep -E " +Test +#[0-9]+*" | cut -d':' -f2 | awk '{$1=$1};1'))
if [ ${#tests[@]} -eq 0 ]; then
abort "No tests avaliable... check your compliation process..."
fi
popd > /dev/null || exit 1
# Step 4: Identify Test Command for Debugging
#############################################
# Select test number
if [ -z $test_number ]; then
# List out avaliable tests
printf "Which test would you like to debug?\n"
id=0
for s in "${tests[@]}"
do
echo "Test# ${id}"
echo " $s"
((id++))
done
# Prompt user which test they wanted to run
printf "\nRun test#? "
read test_number
else
printf "\nUser Already Requested #${test_number}\n"
fi
# Grab all tests commands
pushd "$build_dir"
sIFS=$IFS # Save Initial IFS (Internal Field Separator)
IFS=$'\n' # Change IFS (Internal Field Separator) (So we split ctest output by newline rather than by spaces)
test_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' )) # Get test args
IFS=$sIFS # Reset IFS (Internal Field Separator)
popd > /dev/null || exit 1
# Grab specific test command
single_test_name="${tests[test_number]}"
single_test_command="${test_args[test_number]}"
# Step 5: Execute or GDB Debug
##############################
printf "${magenta}Running Test #${test_number}: ${single_test_name}${normal}\n"
printf "${cyan}single_test_command: ${single_test_command}${normal}\n"
if [ "$gdb_mode" = "true" ]; then
# Execute debugger
pushd "$repo_root" || exit 1
eval "gdb --args ${single_test_command}"
popd > /dev/null || exit 1
else
# Execute Test
pushd "$repo_root" || exit 1
eval "${single_test_command}"
exit_code=$?
popd > /dev/null || exit 1
# Print Result
printf "${blue}Ran Test #${test_number}: ${single_test_name}${normal}\n"
printf "${yellow}Command: ${single_test_command}${normal}\n"
if [ $exit_code -eq 0 ]; then
printf "${green}TEST PASS${normal}\n"
else
printf "${red}TEST FAIL${normal}\n"
fi
fi
# Return to the directory from which the user ran the command.
popd > /dev/null || exit 1