diff --git a/qwen3.5-122b-IQ4_KSS.sh b/qwen3.5-122b-IQ4_KSS.sh new file mode 100755 index 0000000..8d3f3fe --- /dev/null +++ b/qwen3.5-122b-IQ4_KSS.sh @@ -0,0 +1,23 @@ +#!/bin/bash +#grab container image name +MODEL_NAME="Qwen3.5-122B-A10B" +MODEL="Qwen3.5-122B-A10B-IQ4_KSS.gguf" +MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5" + +source ./env.sh +echo $IK_IMAGE +docker container create --name ik-IQ4-KSS-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --gpus=all --restart on-failure:3 \ + --health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \ + --health-interval 5s \ + --health-timeout 2s \ + --health-start-period 15s \ + $IK_IMAGE \ + /ik_llama.cpp/build/bin/llama-server \ + --model /model/$MODEL \ + --alias Qwen3.5-122B-A10B \ + -ctk q8_0 -ctv q8_0 \ + -ger -ngl 99 \ + -ub 4096 -b 4096 \ + --parallel 1 --threads 1 \ + --host ik-$MODEL_NAME \ + --port 10987 --jinja --no-mmap diff --git a/qwen3.5-122b-IQ5_KS.sh b/qwen3.5-122b-IQ5_KS.sh new file mode 100755 index 0000000..c16dd20 --- /dev/null +++ b/qwen3.5-122b-IQ5_KS.sh @@ -0,0 +1,23 @@ +#!/bin/bash +#grab container image name +MODEL_NAME="Qwen3.5-122B-A10B" +MODEL="Qwen3.5-122B-A10B-smol-IQ5_KS.gguf" +MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5" + +source ./env.sh +echo $IK_IMAGE +docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --gpus=all --restart on-failure:3 \ + --health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \ + --health-interval 5s \ + --health-timeout 2s \ + --health-start-period 15s \ + $IK_IMAGE \ + /ik_llama.cpp/build/bin/llama-server \ + --model /model/$MODEL \ + --alias Qwen3.5-122B-A10B \ + -ctk q8_0 -ctv q8_0 \ + -ger -ngl 99 \ + -ub 4096 -b 4096 \ + --parallel 1 --threads 1 \ + --host ik-$MODEL_NAME \ + --port 10987 --jinja --no-mmap diff --git a/qwen3.5-397B-A17B-IQ4_KSS.sh b/qwen3.5-397B-A17B-IQ4_KSS.sh new file mode 100755 index 0000000..56f7df2 --- /dev/null +++ b/qwen3.5-397B-A17B-IQ4_KSS.sh @@ -0,0 +1,24 @@ +#!/bin/bash +#grab container image name +MODEL_NAME="Qwen3.5-397B-A17B" +MODEL="Qwen3.5-397B-A17B-IQ4_KSS-00001-of-00006.gguf" +MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5" + +source ./env.sh +echo $IK_IMAGE +docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --gpus=all --restart on-failure:3 \ + --health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \ + --health-interval 5s \ + --health-timeout 2s \ + --health-start-period 15s \ + $IK_IMAGE \ + /ik_llama.cpp/build/bin/llama-server \ + --model /model/$MODEL \ + --alias $MODEL_NAME \ + -ctk q8_0 -ctv q8_0 \ + -ngl 999 \ + -ub 4096 -b 4096 \ + --n-cpu-moe 38 \ + --parallel 1 --threads 32 \ + --host ik-$MODEL_NAME \ + --port 10987 --jinja --no-mmap