25 lines
1.0 KiB
Bash
Executable File
25 lines
1.0 KiB
Bash
Executable File
#!/bin/bash
|
|
#grab container image name
|
|
MODEL_FOLDER="/home/jared/.cache"
|
|
MODEL_NAME="Qwen3.6-27B-Uncensored"
|
|
MODEL="huggingface/hub/models--HauhauCS--Qwen3.6-27B-Uncensored-HauhauCS-Aggressive/snapshots/f2db94d03eccf3b133aeea3dea62388acf98e864/Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf"
|
|
MMPROJ="huggingface/hub/models--HauhauCS--Qwen3.6-27B-Uncensored-HauhauCS-Aggressive/snapshots/f2db94d03eccf3b133aeea3dea62388acf98e864/mmproj-Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-f16.gguf"
|
|
NAME="LLM-$MODEL_NAME"
|
|
|
|
source ./env.sh
|
|
echo $IK_IMAGE
|
|
docker container create --name $NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
|
|
--health-cmd "curl -f http://$NAME:10987/v1/models || exit 1" \
|
|
--health-interval 5s \
|
|
--health-timeout 5s \
|
|
--health-start-period 20s \
|
|
$IK_IMAGE \
|
|
/llama.cpp/build/bin/llama-server \
|
|
--model /model/$MODEL \
|
|
--alias $MODEL_NAME \
|
|
-ctk q8_0 -ctv q8_0 \
|
|
--image-min-tokens 2048 \
|
|
-mm /model/$MMPROJ \
|
|
--host $NAME \
|
|
--port 10987 --jinja --no-mmap -dio
|