CS348Project/llama.cpp
5
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2025-10-28 08:31:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

use stride=128 if built for tensor cores

Browse Source
This commit is contained in:
Jared Van Bortel
2023-11-27 13:09:14 -05:00
parent dd71a35cc8
commit 6272b6764a
1 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
ggml-cuda.cu
View File

@@ -468,7 +468,12 @@ static_assert(K_QUANTS_PER_ITERATION == 1 || K_QUANTS_PER_ITERATION == 2, "K_QUA
#endif // GGML_CUDA_PEER_MAX_BATCH_SIZE #endif // GGML_CUDA_PEER_MAX_BATCH_SIZE
#define MUL_MAT_SRC1_COL_STRIDE_MMQ 128 #define MUL_MAT_SRC1_COL_STRIDE_MMQ 128
#define MUL_MAT_SRC1_COL_STRIDE 4096
#ifdef CUDA_USE_TENSOR_CORES
#define MUL_MAT_SRC1_COL_STRIDE 128
#else
#define MUL_MAT_SRC1_COL_STRIDE 4096
#endif
#define MAX_STREAMS 8 #define MAX_STREAMS 8
static cudaStream_t g_cudaStreams[GGML_CUDA_MAX_DEVICES][MAX_STREAMS] = { { nullptr } }; static cudaStream_t g_cudaStreams[GGML_CUDA_MAX_DEVICES][MAX_STREAMS] = { { nullptr } };