jdelony/ik_llama_docker
Template
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added Port Option

Browse Source
This commit is contained in:
Jared Delony 2026-05-16 15:46:52 -05:00
parent d7db6c5e02
commit dee5dacdbb
Signed by: jdelony
SSH Key Fingerprint: SHA256:W5F7KMb6FLjlHNhQhtrDl4NvR5fCfinaDDXSQnA2sF0
10 changed files with 50 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Qwen3.6-27B-Uncensored.sh
View File

@ -8,10 +8,12 @@ MODEL_FOLDER="/home/jared/.cache"
MODEL="huggingface/hub/models--HauhauCS--Qwen3.6-27B-Uncensored-HauhauCS-Aggressive/snapshots/f2db94d03eccf3b133aeea3dea62388acf98e864/Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf" MODEL="huggingface/hub/models--HauhauCS--Qwen3.6-27B-Uncensored-HauhauCS-Aggressive/snapshots/f2db94d03eccf3b133aeea3dea62388acf98e864/Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf"
MMPROJ="huggingface/hub/models--HauhauCS--Qwen3.6-27B-Uncensored-HauhauCS-Aggressive/snapshots/f2db94d03eccf3b133aeea3dea62388acf98e864/mmproj-Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-f16.gguf" MMPROJ="huggingface/hub/models--HauhauCS--Qwen3.6-27B-Uncensored-HauhauCS-Aggressive/snapshots/f2db94d03eccf3b133aeea3dea62388acf98e864/mmproj-Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-f16.gguf"
PORT=10987
source ./env.sh source ./env.sh
echo $IMAGE echo $IMAGE
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ docker container create --name $CONT_NAME --network llms -p $PORT:$PORT -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \ --health-cmd "curl -f http://$CONT_NAME:$PORT/v1/models || exit 1" \
--health-interval 5s \ --health-interval 5s \
--health-timeout 5s \ --health-timeout 5s \
--health-start-period 20s \ --health-start-period 20s \
@ -23,7 +25,7 @@ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODE
--image-min-tokens 2048 \ --image-min-tokens 2048 \
-mm /model/$MMPROJ \ -mm /model/$MMPROJ \
--host $CONT_NAME \ --host $CONT_NAME \
--port 10987 --jinja --no-mmap -dio --port $PORT --jinja --no-mmap -dio
echo "Done!" echo "Done!"
echo $CONT_NAME echo $CONT_NAME

8
Qwen3.6-35B-A3B-Uncensored-Aggres-noThink.sh
View File

@ -7,10 +7,12 @@ MODEL_FOLDER="/home/jared/.cache/huggingface/hub/models--HauhauCS--Qwen3.6-35B-A
MODEL="snapshots/f12a584fecbeb5f20001130d8ecd66c9327ae685/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-IQ4_XS.gguf" MODEL="snapshots/f12a584fecbeb5f20001130d8ecd66c9327ae685/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-IQ4_XS.gguf"
MODEL_MM="snapshots/f12a584fecbeb5f20001130d8ecd66c9327ae685/mmproj-Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-f16.gguf" MODEL_MM="snapshots/f12a584fecbeb5f20001130d8ecd66c9327ae685/mmproj-Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-f16.gguf"
PORT=8355
source ../env/llama_env.sh source ../env/llama_env.sh
echo $IMAGE echo $IMAGE
docker container create --name $CONT_NAME --network llms -p 8355:8355 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ docker container create --name $CONT_NAME --network llms -p $PORT:$PORT -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \ --health-cmd "curl -f http://$CONT_NAME:$PORT/v1/models || exit 1" \
--health-interval 5s \ --health-interval 5s \
--health-timeout 5s \ --health-timeout 5s \
--health-start-period 20s \ --health-start-period 20s \
@ -22,7 +24,7 @@ docker container create --name $CONT_NAME --network llms -p 8355:8355 -v $MODEL_
-ctk q8_0 -ctv q8_0 \ -ctk q8_0 -ctv q8_0 \
--reasoning-budget 0 \ --reasoning-budget 0 \
--host $CONT_NAME \ --host $CONT_NAME \
--port 8355 --no-mmap -dio --port $PORT --no-mmap -dio
echo "Done!" echo "Done!"
echo $CONT_NAME echo $CONT_NAME

8
Qwen3.6-35B-A3B-Uncensored-Aggres.sh
View File

@ -7,10 +7,12 @@ MODEL_FOLDER="/home/jared/.cache/huggingface/hub/models--HauhauCS--Qwen3.6-35B-A
MODEL="snapshots/f12a584fecbeb5f20001130d8ecd66c9327ae685/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-IQ4_XS.gguf" MODEL="snapshots/f12a584fecbeb5f20001130d8ecd66c9327ae685/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-IQ4_XS.gguf"
MODEL_MM="snapshots/f12a584fecbeb5f20001130d8ecd66c9327ae685/mmproj-Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-f16.gguf" MODEL_MM="snapshots/f12a584fecbeb5f20001130d8ecd66c9327ae685/mmproj-Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-f16.gguf"
PORT=8356
source ../env/llama_env.sh source ../env/llama_env.sh
echo $IMAGE echo $IMAGE
docker container create --name $CONT_NAME --network llms -p 8356:8356 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ docker container create --name $CONT_NAME --network llms -p $PORT:$PORT -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \ --health-cmd "curl -f http://$CONT_NAME:$PORT/v1/models || exit 1" \
--health-interval 5s \ --health-interval 5s \
--health-timeout 5s \ --health-timeout 5s \
--health-start-period 20s \ --health-start-period 20s \
@ -21,7 +23,7 @@ docker container create --name $CONT_NAME --network llms -p 8356:8356 -v $MODEL_
--alias $MODEL_NAME \ --alias $MODEL_NAME \
-ctk q8_0 -ctv q8_0 \ -ctk q8_0 -ctv q8_0 \
--host $CONT_NAME \ --host $CONT_NAME \
--port 8356 --no-mmap -dio --port $PORT --no-mmap -dio
echo "Done!" echo "Done!"

8
gemma-4-26B-A4B-mxfp4-noThink.sh
View File

@ -7,11 +7,13 @@ MODEL_FOLDER="/home/jared/.cache/huggingface/hub/models--unsloth--gemma-4-26B-A4
MODEL="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/gemma-4-26B-A4B-it-MXFP4_MOE.gguf" MODEL="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/gemma-4-26B-A4B-it-MXFP4_MOE.gguf"
MODEL_MM="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/mmproj-F16.gguf" MODEL_MM="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/mmproj-F16.gguf"
PORT=10987
source ../env/llama_env.sh source ../env/llama_env.sh
echo $IMAGE echo $IMAGE
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ docker container create --name $CONT_NAME --network llms -p $PORT:$PORT -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \ --health-cmd "curl -f http://$CONT_NAME:$PORT/v1/models || exit 1" \
--health-interval 5s \ --health-interval 5s \
--health-timeout 5s \ --health-timeout 5s \
--health-start-period 20s \ --health-start-period 20s \
@ -23,7 +25,7 @@ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODE
-ctk q8_0 -ctv q8_0 \ -ctk q8_0 -ctv q8_0 \
--reasoning-budget 0 \ --reasoning-budget 0 \
--host $CONT_NAME \ --host $CONT_NAME \
--port 10987 --no-mmap -dio --port $PORT --no-mmap -dio
echo "Done!" echo "Done!"
echo $CONT_NAME echo $CONT_NAME

8
gemma-4-26B-A4B-mxfp4.sh
View File

@ -7,11 +7,13 @@ MODEL_FOLDER="/home/jared/.cache/huggingface/hub/models--unsloth--gemma-4-26B-A4
MODEL="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/gemma-4-26B-A4B-it-MXFP4_MOE.gguf" MODEL="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/gemma-4-26B-A4B-it-MXFP4_MOE.gguf"
MODEL_MM="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/mmproj-F16.gguf" MODEL_MM="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/mmproj-F16.gguf"
PORT=10987
source ../env/llama_env.sh source ../env/llama_env.sh
echo $IMAGE echo $IMAGE
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ docker container create --name $CONT_NAME --network llms -p $PORT:$PORT -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \ --health-cmd "curl -f http://$CONT_NAME:$PORT/v1/models || exit 1" \
--health-interval 5s \ --health-interval 5s \
--health-timeout 5s \ --health-timeout 5s \
--health-start-period 20s \ --health-start-period 20s \
@ -22,7 +24,7 @@ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODE
--alias $MODEL_NAME \ --alias $MODEL_NAME \
-ctk q8_0 -ctv q8_0 \ -ctk q8_0 -ctv q8_0 \
--host $CONT_NAME \ --host $CONT_NAME \
--port 10987 --no-mmap -dio --port $PORT --no-mmap -dio
echo "Done!" echo "Done!"
echo $CONT_NAME echo $CONT_NAME

8
glm-5.1-IQ3_KS.sh
View File

@ -7,11 +7,13 @@ CONT_NAME="ik-$MODEL_NAME"
MODEL="GLM-5.1-IQ3_KS-00001-of-00008.gguf" MODEL="GLM-5.1-IQ3_KS-00001-of-00008.gguf"
MODEL_FOLDER="/sam4t/ENC/LLM/GLM-5.1" MODEL_FOLDER="/sam4t/ENC/LLM/GLM-5.1"
PORT=10987
source ./env/ik_env.sh source ./env/ik_env.sh
echo $IMAGE echo $IMAGE
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ docker container create --name $CONT_NAME --network llms -p $PORT:$PORT -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \ --health-cmd "curl -f http://$CONT_NAME:$PORT/v1/models || exit 1" \
--health-interval 5s \ --health-interval 5s \
--health-timeout 2s \ --health-timeout 2s \
--health-start-period 60s \ --health-start-period 60s \
@ -27,7 +29,7 @@ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODE
--n-cpu-moe 66 \ --n-cpu-moe 66 \
--parallel 1 --threads 32 \ --parallel 1 --threads 32 \
--host $CONT_NAME \ --host $CONT_NAME \
--port 10987 --jinja --no-mmap --port $PORT --jinja --no-mmap
echo "Done!" echo "Done!"
echo $CONT_NAME echo $CONT_NAME

8
qwen3.5-122b-IQ4_KSS.sh
View File

@ -6,11 +6,13 @@ CONT_NAME="ik-$MODEL_NAME"
MODEL="Qwen3.5-122B-A10B-IQ4_KSS.gguf" MODEL="Qwen3.5-122B-A10B-IQ4_KSS.gguf"
MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5" MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5"
PORT=10987
source ./ik_env.sh source ./ik_env.sh
echo $IMAGE echo $IMAGE
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ docker container create --name $CONT_NAME --network llms -p $PORT:$PORT -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \ --health-cmd "curl -f http://$CONT_NAME:$PORT/v1/models || exit 1" \
--health-interval 5s \ --health-interval 5s \
--health-timeout 2s \ --health-timeout 2s \
--health-start-period 15s \ --health-start-period 15s \
@ -23,7 +25,7 @@ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODE
-ub 4096 -b 4096 \ -ub 4096 -b 4096 \
--parallel 1 --threads 1 \ --parallel 1 --threads 1 \
--host $CONT_NAME \ --host $CONT_NAME \
--port 10987 --jinja --no-mmap --port $PORT --jinja --no-mmap
echo "Done!" echo "Done!"
echo $CONT_NAME echo $CONT_NAME

8
qwen3.5-122b-IQ5_KS.sh
View File

@ -6,10 +6,12 @@ CONT_NAME="ik-$MODEL_NAME"
MODEL="Qwen3.5-122B-A10B-smol-IQ5_KS.gguf" MODEL="Qwen3.5-122B-A10B-smol-IQ5_KS.gguf"
MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5" MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5"
PORT=10987
source ./ik_env.sh source ./ik_env.sh
echo $IMAGE echo $IMAGE
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ docker container create --name $CONT_NAME --network llms -p $PORT:$PORT -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \ --health-cmd "curl -f http://$CONT_NAME:$PORT/v1/models || exit 1" \
--health-interval 5s \ --health-interval 5s \
--health-timeout 2s \ --health-timeout 2s \
--health-start-period 15s \ --health-start-period 15s \
@ -22,7 +24,7 @@ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODE
-ub 4096 -b 4096 \ -ub 4096 -b 4096 \
--parallel 1 --threads 1 \ --parallel 1 --threads 1 \
--host $CONT_NAME \ --host $CONT_NAME \
--port 10987 --jinja --no-mmap --port $PORT --jinja --no-mmap
echo "Done!" echo "Done!"
echo $CONT_NAME echo $CONT_NAME

8
qwen3.5-397B-A17B-IQ4_KSS.sh
View File

@ -6,10 +6,12 @@ CONT_NAME="ik-$MODEL_NAME"
MODEL="huggingface/hub/models--ubergarm--Qwen3.5-397B-A17B-GGUF/snapshots/bb9c5e48cdcad8716ef0dfff1f48f43b34b89cb2/IQ4_KSS/Qwen3.5-397B-A17B-IQ4_KSS-00001-of-00006.gguf" MODEL="huggingface/hub/models--ubergarm--Qwen3.5-397B-A17B-GGUF/snapshots/bb9c5e48cdcad8716ef0dfff1f48f43b34b89cb2/IQ4_KSS/Qwen3.5-397B-A17B-IQ4_KSS-00001-of-00006.gguf"
MODEL_FOLDER="/home/jared/.cache" MODEL_FOLDER="/home/jared/.cache"
PORT=10987
source ./ik_env.sh source ./ik_env.sh
echo $IMAGE echo $IMAGE
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ docker container create --name $CONT_NAME --network llms -p $PORT:$PORT -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \ --health-cmd "curl -f http://$CONT_NAME:$PORT/v1/models || exit 1" \
--health-interval 5s \ --health-interval 5s \
--health-timeout 5s \ --health-timeout 5s \
--health-start-period 20s \ --health-start-period 20s \
@ -22,7 +24,7 @@ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODE
--n-cpu-moe 38 \ --n-cpu-moe 38 \
--parallel 2 --threads 32 \ --parallel 2 --threads 32 \
--host $CONT_NAME \ --host $CONT_NAME \
--port 10987 --jinja --no-mmap --port $PORT --jinja --no-mmap
echo "Done!" echo "Done!"
echo $CONT_NAME echo $CONT_NAME

8
qwen3.5-coder-next.sh
View File

@ -6,10 +6,12 @@ CONT_NAME="ik-$MODEL_NAME"
MODEL="Qwen3-Coder-Next-IQ4_KSS.gguf" MODEL="Qwen3-Coder-Next-IQ4_KSS.gguf"
MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3-Coder-Next" MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3-Coder-Next"
PORT=10987
source ./ik_env.sh source ./ik_env.sh
echo $IMAGE echo $IMAGE
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ docker container create --name $CONT_NAME --network llms -p $PORT:$PORT -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \ --health-cmd "curl -f http://$CONT_NAME:$PORT/v1/models || exit 1" \
--health-interval 5s \ --health-interval 5s \
--health-timeout 2s \ --health-timeout 2s \
--health-start-period 15s \ --health-start-period 15s \
@ -19,7 +21,7 @@ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODE
-ctk q8_0 -ctv q8_0 \ -ctk q8_0 -ctv q8_0 \
--parallel 1 --threads 32 \ --parallel 1 --threads 32 \
--host $CONT_NAME \ --host $CONT_NAME \
--port 10987 -ngl 99 \ --port $PORT -ngl 99 \
--alias $MODEL_NAME \ --alias $MODEL_NAME \
--no-mmap --jinja \ --no-mmap --jinja \
-b 4096 -ub 4096 \ -b 4096 -ub 4096 \