30 lines
782 B
Bash
Executable File
30 lines
782 B
Bash
Executable File
#!/bin/bash
|
|
#grab container image name
|
|
MODEL_NAME="GLM-5.1-IQ3_KW"
|
|
CONT_NAME="ik-$MODEL_NAME"
|
|
|
|
|
|
MODEL="GLM-5.1-IQ3_KS-00001-of-00008.gguf"
|
|
MODEL_FOLDER="/sam4t/ENC/LLM/GLM-5.1"
|
|
|
|
source ./env/ik_env.sh
|
|
echo $IMAGE
|
|
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
|
|
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \
|
|
--health-interval 5s \
|
|
--health-timeout 2s \
|
|
--health-start-period 60s \
|
|
$IMAGE \
|
|
/ik_llama.cpp/build/bin/llama-server \
|
|
--model /model/$MODEL \
|
|
--alias $MODEL_NAME \
|
|
--ctx-size 64000 \
|
|
-amb 512 \
|
|
-muge \
|
|
--merge-qkv \
|
|
-ngl 999 \
|
|
--n-cpu-moe 66 \
|
|
--parallel 1 --threads 32 \
|
|
--host $CONT_NAME \
|
|
--port 10987 --jinja --no-mmap
|