21 lines
560 B
Bash
Executable File
21 lines
560 B
Bash
Executable File
#!/bin/bash
|
|
#grab container image name
|
|
MODEL_NAME="Qwen3-Coder-Next"
|
|
MODEL="Qwen3-Coder-Next-IQ4_KSS.gguf"
|
|
MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3-Coder-Next"
|
|
|
|
source ./env.sh
|
|
echo $IK_IMAGE
|
|
docker run --name ik_$MODEL_NAME --network host -v $MODEL_FOLDER:/model --gpus=all $IK_IMAGE /ik_llama.cpp/build/bin/llama-server \
|
|
--model /model/$MODEL \
|
|
-ctk q8_0 -ctv q8_0 \
|
|
--parallel 1 --threads 32 \
|
|
--port 8654 -ngl 99 \
|
|
--alias $MODEL_NAME \
|
|
--no-mmap --jinja \
|
|
-b 4096 -ub 4096 \
|
|
-ns 1 -np 1 -cram 32000 \
|
|
-cram-n-min 128 -crs .9 \
|
|
--ctx-checkpoints 128
|
|
|