jdelony/ik_llama_docker
Template
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Modified cache to hopefully reduce poor hit quality

Browse Source
This commit is contained in:
Jared Delony 2026-05-17 14:36:46 -05:00
parent d2cf1ab5e2
commit 3591a33d37
Signed by: jdelony
SSH Key Fingerprint: SHA256:W5F7KMb6FLjlHNhQhtrDl4NvR5fCfinaDDXSQnA2sF0
1 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
qwen3.5-397B-A17B-IQ4_KSS.sh
View File

@ -1,7 +1,7 @@
#!/bin/bash #!/bin/bash
#grab container image name #grab container image name
MODEL_NAME="Qwen3.5-397B-A17B" MODEL_NAME="Qwen3.5-397B-A17B"
CONT_NAME="ik-$MODEL_NAME" CONT_NAME="LLM-$MODEL_NAME"
MODEL="huggingface/hub/models--ubergarm--Qwen3.5-397B-A17B-GGUF/snapshots/bb9c5e48cdcad8716ef0dfff1f48f43b34b89cb2/IQ4_KSS/Qwen3.5-397B-A17B-IQ4_KSS-00001-of-00006.gguf" MODEL="huggingface/hub/models--ubergarm--Qwen3.5-397B-A17B-GGUF/snapshots/bb9c5e48cdcad8716ef0dfff1f48f43b34b89cb2/IQ4_KSS/Qwen3.5-397B-A17B-IQ4_KSS-00001-of-00006.gguf"
MODEL_FOLDER="/home/jared/.cache" MODEL_FOLDER="/home/jared/.cache"
@ -20,7 +20,7 @@ docker container create --name $CONT_NAME --network llms -p $PORT:$PORT -v $MODE
--health-interval 5s \ --health-interval 5s \
--health-timeout 5s \ --health-timeout 5s \
--health-start-period 20s \ --health-start-period 20s \
$IK_IMAGE \ $IMAGE \
/ik_llama.cpp/build/bin/llama-server \ /ik_llama.cpp/build/bin/llama-server \
--model /model/$MODEL \ --model /model/$MODEL \
--alias $MODEL_NAME \ --alias $MODEL_NAME \
@ -28,8 +28,11 @@ docker container create --name $CONT_NAME --network llms -p $PORT:$PORT -v $MODE
-ngl 999 \ -ngl 999 \
--n-cpu-moe 38 \ --n-cpu-moe 38 \
--parallel 2 --threads 32 \ --parallel 2 --threads 32 \
-cram 32768 -cram-n-min 128 -crs .9 --ctx-checkpoints 512 \
--host $CONT_NAME \ --host $CONT_NAME \
--port $PORT --jinja --no-mmap --port $PORT --jinja --no-mmap
echo "Done!" echo "Done!"
echo $CONT_NAME echo $CONT_NAME
docker container start -a $CONT_NAME