jdelony/ik_llama_docker
Template
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added Qwen3.5 397B, again

Browse Source
This commit is contained in:
Jared Delony 2026-04-02 09:10:30 -05:00
parent 7f79b9b9e8
commit b3babd43b0
Signed by: jdelony
SSH Key Fingerprint: SHA256:W5F7KMb6FLjlHNhQhtrDl4NvR5fCfinaDDXSQnA2sF0
3 changed files with 70 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
qwen3.5-122b-IQ4_KSS.sh Executable file
View File

@ -0,0 +1,23 @@
#!/bin/bash
#grab container image name
MODEL_NAME="Qwen3.5-122B-A10B"
MODEL="Qwen3.5-122B-A10B-IQ4_KSS.gguf"
MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5"
source ./env.sh
echo $IK_IMAGE
docker container create --name ik-IQ4-KSS-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \
--health-interval 5s \
--health-timeout 2s \
--health-start-period 15s \
$IK_IMAGE \
/ik_llama.cpp/build/bin/llama-server \
--model /model/$MODEL \
--alias Qwen3.5-122B-A10B \
-ctk q8_0 -ctv q8_0 \
-ger -ngl 99 \
-ub 4096 -b 4096 \
--parallel 1 --threads 1 \
--host ik-$MODEL_NAME \
--port 10987 --jinja --no-mmap

23
qwen3.5-122b-IQ5_KS.sh Executable file
View File

@ -0,0 +1,23 @@
#!/bin/bash
#grab container image name
MODEL_NAME="Qwen3.5-122B-A10B"
MODEL="Qwen3.5-122B-A10B-smol-IQ5_KS.gguf"
MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5"
source ./env.sh
echo $IK_IMAGE
docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \
--health-interval 5s \
--health-timeout 2s \
--health-start-period 15s \
$IK_IMAGE \
/ik_llama.cpp/build/bin/llama-server \
--model /model/$MODEL \
--alias Qwen3.5-122B-A10B \
-ctk q8_0 -ctv q8_0 \
-ger -ngl 99 \
-ub 4096 -b 4096 \
--parallel 1 --threads 1 \
--host ik-$MODEL_NAME \
--port 10987 --jinja --no-mmap

24
qwen3.5-397B-A17B-IQ4_KSS.sh Executable file
View File

@ -0,0 +1,24 @@
#!/bin/bash
#grab container image name
MODEL_NAME="Qwen3.5-397B-A17B"
MODEL="Qwen3.5-397B-A17B-IQ4_KSS-00001-of-00006.gguf"
MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5"
source ./env.sh
echo $IK_IMAGE
docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \
--health-interval 5s \
--health-timeout 2s \
--health-start-period 15s \
$IK_IMAGE \
/ik_llama.cpp/build/bin/llama-server \
--model /model/$MODEL \
--alias $MODEL_NAME \
-ctk q8_0 -ctv q8_0 \
-ngl 999 \
-ub 4096 -b 4096 \
--n-cpu-moe 38 \
--parallel 1 --threads 32 \
--host ik-$MODEL_NAME \
--port 10987 --jinja --no-mmap