jdelony/ik_llama_docker
Template
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Script changes

Browse Source
This commit is contained in:
Jared Delony 2026-05-16 15:38:05 -05:00
parent 62e241fd85
commit d7db6c5e02
Signed by: jdelony
SSH Key Fingerprint: SHA256:W5F7KMb6FLjlHNhQhtrDl4NvR5fCfinaDDXSQnA2sF0
10 changed files with 54 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Qwen3.6-27B-Uncensored.sh
View File

@ -9,7 +9,7 @@ MODEL="huggingface/hub/models--HauhauCS--Qwen3.6-27B-Uncensored-HauhauCS-Aggress
MMPROJ="huggingface/hub/models--HauhauCS--Qwen3.6-27B-Uncensored-HauhauCS-Aggressive/snapshots/f2db94d03eccf3b133aeea3dea62388acf98e864/mmproj-Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-f16.gguf"
source ./env.sh
echo $IK_IMAGE
echo $IMAGE
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \
--health-interval 5s \
@ -24,3 +24,6 @@ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODE
-mm /model/$MMPROJ \
--host $CONT_NAME \
--port 10987 --jinja --no-mmap -dio
echo "Done!"
echo $CONT_NAME

18
Qwen3.6-35B-A3B-Uncensored-Aggres-noThink.sh
View File

@ -1,15 +1,15 @@
#!/bin/bash
#grab container image name
MODEL_NAME="Gemma-4-26B-A4B-noThink"
MODEL_NAME="Qwen3.6-35B-A3B-Uncensored-Aggressive-noThink"
CONT_NAME="llama-$MODEL_NAME"
MODEL_FOLDER="/home/jared/.cache/huggingface/hub/models--unsloth--gemma-4-26B-A4B-it-GGUF"
MODEL="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/gemma-4-26B-A4B-it-MXFP4_MOE.gguf"
MODEL_MM="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/mmproj-F16.gguf"
MODEL_FOLDER="/home/jared/.cache/huggingface/hub/models--HauhauCS--Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive"
MODEL="snapshots/f12a584fecbeb5f20001130d8ecd66c9327ae685/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-IQ4_XS.gguf"
MODEL_MM="snapshots/f12a584fecbeb5f20001130d8ecd66c9327ae685/mmproj-Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-f16.gguf"
source ../env/llama_env.sh
echo $IMAGkE
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
echo $IMAGE
docker container create --name $CONT_NAME --network llms -p 8355:8355 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \
--health-interval 5s \
--health-timeout 5s \
@ -22,7 +22,7 @@ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODE
-ctk q8_0 -ctv q8_0 \
--reasoning-budget 0 \
--host $CONT_NAME \
--port 10987 --no-mmap -dio
--port 8355 --no-mmap -dio
./build/bin/llama-server --model /home/jared/.cache/huggingface/hub/models--HauhauCS--Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive/snapshots/f12a584fecbeb5f20001130d8ecd66c9327ae685/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-IQ4_XS.gguf -mm /home/jared/.cache/huggingface/hub/models--HauhauCS--Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive/snapshots/f12a584fecbeb5f20001130d8ecd66c9327ae685/mmproj-Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-f16.gguf -ctk q8_0 -ctv q8_0 --reasoning-budget 0 --port 8355 --host 192.168.222.191 --no-mmap -dio --alias Qwen3.6-35B-A3B-SparkD 2>&1 | awk '{print strftime("[%H:%M:%S]"), $0}'
echo "Done!"
echo $CONT_NAME

18
Qwen3.6-35B-A3B-Uncensored-Aggres.sh
View File

@ -1,15 +1,15 @@
#!/bin/bash
#grab container image name
MODEL_NAME="Gemma-4-26B-A4B-noThink"
MODEL_NAME="Qwen3.6-35B-A3B-Uncensored-Aggressive"
CONT_NAME="llama-$MODEL_NAME"
MODEL_FOLDER="/home/jared/.cache/huggingface/hub/models--unsloth--gemma-4-26B-A4B-it-GGUF"
MODEL="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/gemma-4-26B-A4B-it-MXFP4_MOE.gguf"
MODEL_MM="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/mmproj-F16.gguf"
MODEL_FOLDER="/home/jared/.cache/huggingface/hub/models--HauhauCS--Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive"
MODEL="snapshots/f12a584fecbeb5f20001130d8ecd66c9327ae685/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-IQ4_XS.gguf"
MODEL_MM="snapshots/f12a584fecbeb5f20001130d8ecd66c9327ae685/mmproj-Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-f16.gguf"
source ../env/llama_env.sh
echo $IMAGkE
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
echo $IMAGE
docker container create --name $CONT_NAME --network llms -p 8356:8356 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \
--health-interval 5s \
--health-timeout 5s \
@ -20,9 +20,9 @@ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODE
--mm /model/$MODEL_MM \
--alias $MODEL_NAME \
-ctk q8_0 -ctv q8_0 \
--reasoning-budget 0 \
--host $CONT_NAME \
--port 10987 --no-mmap -dio
--port 8356 --no-mmap -dio
./build/bin/llama-server --model /home/jared/.cache/huggingface/hub/models--HauhauCS--Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive/snapshots/f12a584fecbeb5f20001130d8ecd66c9327ae685/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-IQ4_XS.gguf -mm /home/jared/.cache/huggingface/hub/models--HauhauCS--Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive/snapshots/f12a584fecbeb5f20001130d8ecd66c9327ae685/mmproj-Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-f16.gguf -ctk q8_0 -ctv q8_0 --reasoning-budget 0 --port 8355 --host 192.168.222.191 --no-mmap -dio --alias Qwen3.6-35B-A3B-SparkD 2>&1 | awk '{print strftime("[%H:%M:%S]"), $0}'
echo "Done!"
echo $CONT_NAME

6
gemma-4-26B-A4B-mxfp4-noThink.sh
View File

@ -8,7 +8,8 @@ MODEL="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/gemma-4-26B-A4B-it-MXF
MODEL_MM="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/mmproj-F16.gguf"
source ../env/llama_env.sh
echo $IMAGkE
echo $IMAGE
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \
--health-interval 5s \
@ -23,3 +24,6 @@ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODE
--reasoning-budget 0 \
--host $CONT_NAME \
--port 10987 --no-mmap -dio
echo "Done!"
echo $CONT_NAME

6
gemma-4-26B-A4B-mxfp4.sh
View File

@ -8,7 +8,8 @@ MODEL="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/gemma-4-26B-A4B-it-MXF
MODEL_MM="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/mmproj-F16.gguf"
source ../env/llama_env.sh
echo $IMAGkE
echo $IMAGE
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \
--health-interval 5s \
@ -22,3 +23,6 @@ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODE
-ctk q8_0 -ctv q8_0 \
--host $CONT_NAME \
--port 10987 --no-mmap -dio
echo "Done!"
echo $CONT_NAME

4
glm-5.1-IQ3_KS.sh
View File

@ -8,6 +8,7 @@ MODEL="GLM-5.1-IQ3_KS-00001-of-00008.gguf"
MODEL_FOLDER="/sam4t/ENC/LLM/GLM-5.1"
source ./env/ik_env.sh
echo $IMAGE
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \
@ -27,3 +28,6 @@ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODE
--parallel 1 --threads 32 \
--host $CONT_NAME \
--port 10987 --jinja --no-mmap
echo "Done!"
echo $CONT_NAME

6
qwen3.5-122b-IQ4_KSS.sh
View File

@ -7,7 +7,8 @@ MODEL="Qwen3.5-122B-A10B-IQ4_KSS.gguf"
MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5"
source ./ik_env.sh
echo $IK_IMAGE
echo $IMAGE
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \
--health-interval 5s \
@ -23,3 +24,6 @@ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODE
--parallel 1 --threads 1 \
--host $CONT_NAME \
--port 10987 --jinja --no-mmap
echo "Done!"
echo $CONT_NAME

5
qwen3.5-122b-IQ5_KS.sh
View File

@ -7,7 +7,7 @@ MODEL="Qwen3.5-122B-A10B-smol-IQ5_KS.gguf"
MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5"
source ./ik_env.sh
echo $IK_IMAGE
echo $IMAGE
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \
--health-interval 5s \
@ -23,3 +23,6 @@ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODE
--parallel 1 --threads 1 \
--host $CONT_NAME \
--port 10987 --jinja --no-mmap
echo "Done!"
echo $CONT_NAME

5
qwen3.5-397B-A17B-IQ4_KSS.sh
View File

@ -7,7 +7,7 @@ MODEL="huggingface/hub/models--ubergarm--Qwen3.5-397B-A17B-GGUF/snapshots/bb9c5e
MODEL_FOLDER="/home/jared/.cache"
source ./ik_env.sh
echo $IK_IMAGE
echo $IMAGE
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \
--health-interval 5s \
@ -23,3 +23,6 @@ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODE
--parallel 2 --threads 32 \
--host $CONT_NAME \
--port 10987 --jinja --no-mmap
echo "Done!"
echo $CONT_NAME

7
qwen3.5-coder-next.sh
View File

@ -7,7 +7,7 @@ MODEL="Qwen3-Coder-Next-IQ4_KSS.gguf"
MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3-Coder-Next"
source ./ik_env.sh
echo $IK_IMAGE
echo $IMAGE
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \
--health-interval 5s \
@ -25,4 +25,7 @@ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODE
-b 4096 -ub 4096 \
-ns 1 -np 1 -cram 32000 \
-cram-n-min 128 -crs .9 \
--ctx-checkpoints 128
--ctx-checkpoints 128
echo "Done!"
echo $CONT_NAME