jdelony/ik_llama_docker
Template
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remixed Scripts

Browse Source
This commit is contained in:
Jared Delony 2026-05-16 15:02:45 -05:00
parent 49feca7e2b
commit 074bc3d61f
Signed by: jdelony
SSH Key Fingerprint: SHA256:W5F7KMb6FLjlHNhQhtrDl4NvR5fCfinaDDXSQnA2sF0
14 changed files with 75 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Qwen3.6-27B-Uncensored.sh
View File

@ -1,15 +1,17 @@
#!/bin/bash #!/bin/bash
#grab container image name #grab container image name
MODEL_FOLDER="/home/jared/.cache"
MODEL_NAME="Qwen3.6-27B-Uncensored" MODEL_NAME="Qwen3.6-27B-Uncensored"
CONT_NAME="llama-$MODEL_NAME"
MODEL_FOLDER="/home/jared/.cache"
MODEL="huggingface/hub/models--HauhauCS--Qwen3.6-27B-Uncensored-HauhauCS-Aggressive/snapshots/f2db94d03eccf3b133aeea3dea62388acf98e864/Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf" MODEL="huggingface/hub/models--HauhauCS--Qwen3.6-27B-Uncensored-HauhauCS-Aggressive/snapshots/f2db94d03eccf3b133aeea3dea62388acf98e864/Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf"
MMPROJ="huggingface/hub/models--HauhauCS--Qwen3.6-27B-Uncensored-HauhauCS-Aggressive/snapshots/f2db94d03eccf3b133aeea3dea62388acf98e864/mmproj-Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-f16.gguf" MMPROJ="huggingface/hub/models--HauhauCS--Qwen3.6-27B-Uncensored-HauhauCS-Aggressive/snapshots/f2db94d03eccf3b133aeea3dea62388acf98e864/mmproj-Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-f16.gguf"
NAME="LLM-$MODEL_NAME"
source ./env.sh source ./env.sh
echo $IK_IMAGE echo $IK_IMAGE
docker container create --name $NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$NAME:10987/v1/models || exit 1" \ --health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \
--health-interval 5s \ --health-interval 5s \
--health-timeout 5s \ --health-timeout 5s \
--health-start-period 20s \ --health-start-period 20s \
@ -20,5 +22,5 @@ docker container create --name $NAME --network llms -p 10987:10987 -v $MODEL_FOL
-ctk q8_0 -ctv q8_0 \ -ctk q8_0 -ctv q8_0 \
--image-min-tokens 2048 \ --image-min-tokens 2048 \
-mm /model/$MMPROJ \ -mm /model/$MMPROJ \
--host $NAME \ --host $CONT_NAME \
--port 10987 --jinja --no-mmap -dio --port 10987 --jinja --no-mmap -dio

4
create_new_IK_image.sh
View File

@ -15,8 +15,8 @@ docker rm ik-llama-dw-$(date +%Y-%m-%d)
#Updated env for other scripts #Updated env for other scripts
IMAGE=ik-llama-dw:$(date +%Y-%m-%d) IMAGE=ik-llama-dw:$(date +%Y-%m-%d)
cat > ik_env.sh << EOF cat > ./env/ik_env.sh << EOF
#!/bin/bash #!/bin/bash
#updated from create_new_image.sh #updated from create_new_image.sh
export IK_IMAGE=$IMAGE export IMAGE=$IMAGE
EOF EOF

4
create_new_image.sh
View File

@ -15,8 +15,8 @@ docker rm llama-dw-$(date +%Y-%m-%d)
#Updated env for other scripts #Updated env for other scripts
IMAGE=llama-dw:$(date +%Y-%m-%d) IMAGE=llama-dw:$(date +%Y-%m-%d)
cat > env.sh << EOF cat > ./env/llama_env.sh << EOF
#!/bin/bash #!/bin/bash
#updated from create_new_image.sh #updated from create_new_image.sh
export IK_IMAGE=$IMAGE export IMAGE=$IMAGE
EOF EOF

4
create_new_vllm_image.sh
View File

@ -14,10 +14,10 @@ EOF
docker build -t $IMG_NAME . docker build -t $IMG_NAME .
#Updated env for other scripts #Updated env for other scripts
cat > vllm_env.sh << EOF cat > ./env/vllm_env.sh << EOF
#!/bin/bash #!/bin/bash
#updated from create_new_image_vllm.sh #updated from create_new_image_vllm.sh
export VLLM_IMAGE=$IMG_NAME export IMAGE=$IMG_NAME
EOF EOF
rm Dockerfile rm Dockerfile

3
env.sh
View File

@ -1,3 +0,0 @@
#!/bin/bash
#updated from create_new_image.sh
export IK_IMAGE=llama-dw:2026-04-29

1
env/FolderForAutogenENVARS vendored Normal file
View File

@ -0,0 +1 @@
This is the folder that will hold the auto generated ENVs for the docker image generation.

25
gemma-4-26B-A4B-mxfp4-noThink.sh Executable file
View File

@ -0,0 +1,25 @@
#!/bin/bash
#grab container image name
MODEL_NAME="Gemma-4-26B-A4B-noThink"
CONT_NAME="llama-$MODEL_NAME"
MODEL_FOLDER="/home/jared/.cache/huggingface/hub/models--unsloth--gemma-4-26B-A4B-it-GGUF"
MODEL="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/gemma-4-26B-A4B-it-MXFP4_MOE.gguf"
MODEL_MM="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/mmproj-F16.gguf"
source ../env/llama_env.sh
echo $IMAGkE
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \
--health-interval 5s \
--health-timeout 5s \
--health-start-period 20s \
$IMAGE \
/llama.cpp/build/bin/llama-server \
--model /model/$MODEL \
--mm /model/$MODEL_MM \
--alias $MODEL_NAME \
-ctk q8_0 -ctv q8_0 \
--reasoning-budget 0 \
--host $CONT_NAME \
--port 10987 --no-mmap -dio

12
spark/gemma-4-26B-A4B-mxfp4.sh → gemma-4-26B-A4B-mxfp4.sh
View File

@ -1,24 +1,24 @@
#!/bin/bash #!/bin/bash
#grab container image name #grab container image name
MODEL_NAME="Gemma-4-26B-A4B-it-mxfp4" MODEL_NAME="Gemma-4-26B-A4B"
CONT_NAME="llama-$MODEL_NAME" CONT_NAME="llama-$MODEL_NAME"
MODEL_FOLDER="/home/jared/.cache/huggingface/hub/models--unsloth--gemma-4-26B-A4B-it-GGUF" MODEL_FOLDER="/home/jared/.cache/huggingface/hub/models--unsloth--gemma-4-26B-A4B-it-GGUF"
MODEL="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/gemma-4-26B-A4B-it-MXFP4_MOE.gguf" MODEL="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/gemma-4-26B-A4B-it-MXFP4_MOE.gguf"
MODEL_MM="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/mmproj-F16.gguf" MODEL_MM="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/mmproj-F16.gguf"
source ../env.sh source ../env/llama_env.sh
echo $IK_IMAGkE echo $IMAGkE
docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \ --health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \
--health-interval 5s \ --health-interval 5s \
--health-timeout 5s \ --health-timeout 5s \
--health-start-period 20s \ --health-start-period 20s \
$IK_IMAGE \ $IMAGE \
/llama.cpp/build/bin/llama-server \ /llama.cpp/build/bin/llama-server \
--model /model/$MODEL \ --model /model/$MODEL \
--mm /model/$MODEL_MM \ --mm /model/$MODEL_MM \
--alias $MODEL_NAME \ --alias $MODEL_NAME \
-ctk q8_0 -ctv q8_0 \ -ctk q8_0 -ctv q8_0 \
--reasoning-budget 0 \ --host $CONT_NAME \
--host ik-$MODEL_NAME \
--port 10987 --no-mmap -dio --port 10987 --no-mmap -dio

15
glm-5.1-IQ3_KS.sh
View File

@ -1,17 +1,20 @@
#!/bin/bash #!/bin/bash
#grab container image name #grab container image name
MODEL_NAME="GLM-5.1-IQ3_KW" MODEL_NAME="GLM-5.1-IQ3_KW"
CONT_NAME="ik-$MODEL_NAME"
MODEL="GLM-5.1-IQ3_KS-00001-of-00008.gguf" MODEL="GLM-5.1-IQ3_KS-00001-of-00008.gguf"
MODEL_FOLDER="/sam4t/ENC/LLM/GLM-5.1" MODEL_FOLDER="/sam4t/ENC/LLM/GLM-5.1"
source ./ik_env.sh source ./env/ik_env.sh
echo $IK_IMAGE echo $IMAGE
docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --gpus=all --restart on-failure:3 \ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \ --health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \
--health-interval 5s \ --health-interval 5s \
--health-timeout 2s \ --health-timeout 2s \
--health-start-period 60s \ --health-start-period 60s \
$IK_IMAGE \ $IMAGE \
/ik_llama.cpp/build/bin/llama-server \ /ik_llama.cpp/build/bin/llama-server \
--model /model/$MODEL \ --model /model/$MODEL \
--alias $MODEL_NAME \ --alias $MODEL_NAME \
@ -22,5 +25,5 @@ docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $
-ngl 999 \ -ngl 999 \
--n-cpu-moe 66 \ --n-cpu-moe 66 \
--parallel 1 --threads 32 \ --parallel 1 --threads 32 \
--host ik-$MODEL_NAME \ --host $CONT_NAME \
--port 10987 --jinja --no-mmap --port 10987 --jinja --no-mmap

8
qwen3.5-122b-IQ4_KSS.sh
View File

@ -1,13 +1,15 @@
#!/bin/bash #!/bin/bash
#grab container image name #grab container image name
MODEL_NAME="Qwen3.5-122B-A10B" MODEL_NAME="Qwen3.5-122B-A10B"
CONT_NAME="ik-$MODEL_NAME"
MODEL="Qwen3.5-122B-A10B-IQ4_KSS.gguf" MODEL="Qwen3.5-122B-A10B-IQ4_KSS.gguf"
MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5" MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5"
source ./ik_env.sh source ./ik_env.sh
echo $IK_IMAGE echo $IK_IMAGE
docker container create --name ik-IQ4-KSS-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --gpus=all --restart on-failure:3 \ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \ --health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \
--health-interval 5s \ --health-interval 5s \
--health-timeout 2s \ --health-timeout 2s \
--health-start-period 15s \ --health-start-period 15s \
@ -19,5 +21,5 @@ docker container create --name ik-IQ4-KSS-$MODEL_NAME --network llms -p 10987:10
-ger -ngl 99 \ -ger -ngl 99 \
-ub 4096 -b 4096 \ -ub 4096 -b 4096 \
--parallel 1 --threads 1 \ --parallel 1 --threads 1 \
--host ik-$MODEL_NAME \ --host $CONT_NAME \
--port 10987 --jinja --no-mmap --port 10987 --jinja --no-mmap

8
qwen3.5-122b-IQ5_KS.sh
View File

@ -1,13 +1,15 @@
#!/bin/bash #!/bin/bash
#grab container image name #grab container image name
MODEL_NAME="Qwen3.5-122B-A10B" MODEL_NAME="Qwen3.5-122B-A10B"
CONT_NAME="ik-$MODEL_NAME"
MODEL="Qwen3.5-122B-A10B-smol-IQ5_KS.gguf" MODEL="Qwen3.5-122B-A10B-smol-IQ5_KS.gguf"
MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5" MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5"
source ./ik_env.sh source ./ik_env.sh
echo $IK_IMAGE echo $IK_IMAGE
docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --gpus=all --restart on-failure:3 \ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \ --health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \
--health-interval 5s \ --health-interval 5s \
--health-timeout 2s \ --health-timeout 2s \
--health-start-period 15s \ --health-start-period 15s \
@ -19,5 +21,5 @@ docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $
-ger -ngl 99 \ -ger -ngl 99 \
-ub 4096 -b 4096 \ -ub 4096 -b 4096 \
--parallel 1 --threads 1 \ --parallel 1 --threads 1 \
--host ik-$MODEL_NAME \ --host $CONT_NAME \
--port 10987 --jinja --no-mmap --port 10987 --jinja --no-mmap

8
qwen3.5-397B-A17B-IQ4_KSS.sh
View File

@ -1,13 +1,15 @@
#!/bin/bash #!/bin/bash
#grab container image name #grab container image name
MODEL_NAME="Qwen3.5-397B-A17B" MODEL_NAME="Qwen3.5-397B-A17B"
CONT_NAME="ik-$MODEL_NAME"
MODEL="huggingface/hub/models--ubergarm--Qwen3.5-397B-A17B-GGUF/snapshots/bb9c5e48cdcad8716ef0dfff1f48f43b34b89cb2/IQ4_KSS/Qwen3.5-397B-A17B-IQ4_KSS-00001-of-00006.gguf" MODEL="huggingface/hub/models--ubergarm--Qwen3.5-397B-A17B-GGUF/snapshots/bb9c5e48cdcad8716ef0dfff1f48f43b34b89cb2/IQ4_KSS/Qwen3.5-397B-A17B-IQ4_KSS-00001-of-00006.gguf"
MODEL_FOLDER="/home/jared/.cache" MODEL_FOLDER="/home/jared/.cache"
source ./ik_env.sh source ./ik_env.sh
echo $IK_IMAGE echo $IK_IMAGE
docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \ --health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \
--health-interval 5s \ --health-interval 5s \
--health-timeout 5s \ --health-timeout 5s \
--health-start-period 20s \ --health-start-period 20s \
@ -19,5 +21,5 @@ docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $
-ngl 999 \ -ngl 999 \
--n-cpu-moe 38 \ --n-cpu-moe 38 \
--parallel 2 --threads 32 \ --parallel 2 --threads 32 \
--host ik-$MODEL_NAME \ --host $CONT_NAME \
--port 10987 --jinja --no-mmap --port 10987 --jinja --no-mmap

8
qwen3.5-coder-next.sh
View File

@ -1,13 +1,15 @@
#!/bin/bash #!/bin/bash
#grab container image name #grab container image name
MODEL_NAME="Qwen3-Coder-Next" MODEL_NAME="Qwen3-Coder-Next"
CONT_NAME="ik-$MODEL_NAME"
MODEL="Qwen3-Coder-Next-IQ4_KSS.gguf" MODEL="Qwen3-Coder-Next-IQ4_KSS.gguf"
MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3-Coder-Next" MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3-Coder-Next"
source ./ik_env.sh source ./ik_env.sh
echo $IK_IMAGE echo $IK_IMAGE
docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --gpus=all --restart on-failure:3 \ docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \ --health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \
--health-interval 5s \ --health-interval 5s \
--health-timeout 2s \ --health-timeout 2s \
--health-start-period 15s \ --health-start-period 15s \
@ -16,7 +18,7 @@ docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $
--model /model/$MODEL \ --model /model/$MODEL \
-ctk q8_0 -ctv q8_0 \ -ctk q8_0 -ctv q8_0 \
--parallel 1 --threads 32 \ --parallel 1 --threads 32 \
--host ik-$MODEL_NAME \ --host $CONT_NAME \
--port 10987 -ngl 99 \ --port 10987 -ngl 99 \
--alias $MODEL_NAME \ --alias $MODEL_NAME \
--no-mmap --jinja \ --no-mmap --jinja \

1
spark/IfSparkSpecific Normal file
View File

@ -0,0 +1 @@
This is where spark specific settings will be