24 lines
724 B
Bash
Executable File
24 lines
724 B
Bash
Executable File
#!/bin/bash
|
|
#grab container image name
|
|
MODEL_NAME="Qwen3.5-122B-A10B"
|
|
MODEL="Qwen3.5-122B-A10B-IQ4_KSS.gguf"
|
|
MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5"
|
|
|
|
source ./env.sh
|
|
echo $IK_IMAGE
|
|
docker container create --name ik-IQ4-KSS-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --gpus=all --restart on-failure:3 \
|
|
--health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \
|
|
--health-interval 5s \
|
|
--health-timeout 2s \
|
|
--health-start-period 15s \
|
|
$IK_IMAGE \
|
|
/ik_llama.cpp/build/bin/llama-server \
|
|
--model /model/$MODEL \
|
|
--alias Qwen3.5-122B-A10B \
|
|
-ctk q8_0 -ctv q8_0 \
|
|
-ger -ngl 99 \
|
|
-ub 4096 -b 4096 \
|
|
--parallel 1 --threads 1 \
|
|
--host ik-$MODEL_NAME \
|
|
--port 10987 --jinja --no-mmap
|