60 lines
2.0 KiB
Bash
60 lines
2.0 KiB
Bash
#!/bin/bash
|
||
#grab container image name
|
||
MODEL_NAME="sulphur-prompt"
|
||
CONT_NAME="LLM-$MODEL_NAME"
|
||
|
||
MODEL_FOLDER="/home/jared/.cache"
|
||
MODEL="huggingface/hub/models--SulphurAI--Sulphur-2-base/snapshots/b0ba1217a7c1eedca26afff4f5d53f77b951e4db/prompt_enhancer/sulphur_prompt_enhancer_model-q8_0.gguf"
|
||
MODEL_MM="huggingface/hub/models--SulphurAI--Sulphur-2-base/snapshots/b0ba1217a7c1eedca26afff4f5d53f77b951e4db/prompt_enhancer/mmproj-BF16.gguf"
|
||
|
||
PORT=8555
|
||
|
||
set -e
|
||
|
||
# Check if llama_env.sh exists, if not run create_new_image.sh
|
||
if [ ! -f "./env/llama_env.sh" ]; then
|
||
echo "llama_env.sh not found, running create_new_image.sh..."
|
||
./create_new_image.sh
|
||
fi
|
||
source ./env/llama_env.sh
|
||
|
||
# Check and remove container
|
||
if docker inspect "$CONT_NAME" > /dev/null 2>&1; then
|
||
echo "✓ Container '$CONT_NAME' found"
|
||
|
||
# Check if container is running
|
||
if docker inspect -f '{{.State.Running}}' "$CONT_NAME" | grep -q "true"; then
|
||
echo "⚠ Container is running, forcing removal..."
|
||
fi
|
||
|
||
# Remove container
|
||
if docker rm -f "$CONT_NAME"; then
|
||
echo "✓ Container removed successfully. Creating new container."
|
||
exit 0
|
||
else
|
||
echo "✗ Failed to remove container"
|
||
exit 1
|
||
fi
|
||
else
|
||
echo "ℹ Container '$CONT_NAME' does not exist. Creating new container."
|
||
fi
|
||
|
||
#create container
|
||
docker container create --name $CONT_NAME --network llms -p $PORT:$PORT -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
|
||
--health-cmd "curl -f http://$CONT_NAME:$PORT/v1/models || exit 1" \
|
||
--health-interval 5s \
|
||
--health-timeout 5s \
|
||
--health-start-period 20s \
|
||
$IMAGE \
|
||
/llama.cpp/build/bin/llama-server \
|
||
--model /model/$MODEL \
|
||
-mm /model/$MODEL_MM \
|
||
--alias $MODEL_NAME \
|
||
-ctk q8_0 -ctv q8_0 \
|
||
--host $CONT_NAME \
|
||
--port $PORT --no-mmap -dio \
|
||
--reasoning off -c 16384 -np 2
|
||
|
||
echo "Done!"
|
||
docker container start -a $CONT_NAME
|