34 lines
1.3 KiB
Bash
Executable File
34 lines
1.3 KiB
Bash
Executable File
#!/bin/bash
|
|
#grab container image name
|
|
MODEL_NAME="GLM-4.7-Flash-Uncen-Hrt"
|
|
CONT_NAME="LLM-$MODEL_NAME"
|
|
|
|
MODEL_FOLDER="/home/jared/.cache"
|
|
MODEL="huggingface/hub/models--DavidAU--GLM-4.7-Flash-Uncensored-Heretic-NEO-CODE-Imatrix-MAX-GGUF/snapshots/af1cb75ecde13d21c19ceaa5fbd4352f8233e0e5/GLM-4.7-Flash-Uncen-Hrt-NEO-CODE-MAX-imat-D_AU-Q8_0.gguf"
|
|
#MODEL_MM="huggingface/hub/models--HauhauCS--Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive/snapshots/f12a584fecbeb5f20001130d8ecd66c9327ae685/mmproj-Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-f16.gguf"
|
|
|
|
PORT=8356
|
|
|
|
# Check if llama_env.sh exists, if not run create_new_image.sh
|
|
if [ ! -f "./env/llama_env.sh" ]; then
|
|
echo "llama_env.sh not found, running create_new_image.sh..."
|
|
./create_new_image.sh
|
|
fi
|
|
source ./env/llama_env.sh
|
|
echo $IMAGE
|
|
docker container create --name $CONT_NAME --network llms -p $PORT:$PORT -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
|
|
--health-cmd "curl -f http://$CONT_NAME:$PORT/v1/models || exit 1" \
|
|
--health-interval 5s \
|
|
--health-timeout 5s \
|
|
--health-start-period 20s \
|
|
$IMAGE \
|
|
/llama.cpp/build/bin/llama-server \
|
|
--model /model/$MODEL \
|
|
--alias $MODEL_NAME \
|
|
-ctk q8_0 -ctv q8_0 \
|
|
--host $CONT_NAME \
|
|
--port $PORT --no-mmap -dio
|
|
|
|
|
|
echo "Done!"
|
|
echo $CONT_NAME |