From 074bc3d61f92a18bb393e1c686d4a74685d092ba Mon Sep 17 00:00:00 2001 From: Jared Delony Date: Sat, 16 May 2026 15:02:45 -0500 Subject: [PATCH] Remixed Scripts --- Qwen3.6-27B-Uncensored.sh | 12 +++++---- create_new_IK_image.sh | 4 +-- create_new_image.sh | 4 +-- create_new_vllm_image.sh | 4 +-- env.sh | 3 --- env/FolderForAutogenENVARS | 1 + gemma-4-26B-A4B-mxfp4-noThink.sh | 25 +++++++++++++++++++ ...B-A4B-mxfp4.sh => gemma-4-26B-A4B-mxfp4.sh | 12 ++++----- glm-5.1-IQ3_KS.sh | 15 ++++++----- qwen3.5-122b-IQ4_KSS.sh | 8 +++--- qwen3.5-122b-IQ5_KS.sh | 8 +++--- qwen3.5-397B-A17B-IQ4_KSS.sh | 8 +++--- qwen3.5-coder-next.sh | 8 +++--- spark/IfSparkSpecific | 1 + 14 files changed, 75 insertions(+), 38 deletions(-) delete mode 100755 env.sh create mode 100644 env/FolderForAutogenENVARS create mode 100755 gemma-4-26B-A4B-mxfp4-noThink.sh rename spark/gemma-4-26B-A4B-mxfp4.sh => gemma-4-26B-A4B-mxfp4.sh (85%) create mode 100644 spark/IfSparkSpecific diff --git a/Qwen3.6-27B-Uncensored.sh b/Qwen3.6-27B-Uncensored.sh index 946d059..7adc36c 100755 --- a/Qwen3.6-27B-Uncensored.sh +++ b/Qwen3.6-27B-Uncensored.sh @@ -1,15 +1,17 @@ #!/bin/bash #grab container image name -MODEL_FOLDER="/home/jared/.cache" + MODEL_NAME="Qwen3.6-27B-Uncensored" +CONT_NAME="llama-$MODEL_NAME" + +MODEL_FOLDER="/home/jared/.cache" MODEL="huggingface/hub/models--HauhauCS--Qwen3.6-27B-Uncensored-HauhauCS-Aggressive/snapshots/f2db94d03eccf3b133aeea3dea62388acf98e864/Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf" MMPROJ="huggingface/hub/models--HauhauCS--Qwen3.6-27B-Uncensored-HauhauCS-Aggressive/snapshots/f2db94d03eccf3b133aeea3dea62388acf98e864/mmproj-Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-f16.gguf" -NAME="LLM-$MODEL_NAME" source ./env.sh echo $IK_IMAGE -docker container create --name $NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ - --health-cmd "curl -f http://$NAME:10987/v1/models || exit 1" \ +docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ + --health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \ --health-interval 5s \ --health-timeout 5s \ --health-start-period 20s \ @@ -20,5 +22,5 @@ docker container create --name $NAME --network llms -p 10987:10987 -v $MODEL_FOL -ctk q8_0 -ctv q8_0 \ --image-min-tokens 2048 \ -mm /model/$MMPROJ \ - --host $NAME \ + --host $CONT_NAME \ --port 10987 --jinja --no-mmap -dio diff --git a/create_new_IK_image.sh b/create_new_IK_image.sh index f042393..a242100 100755 --- a/create_new_IK_image.sh +++ b/create_new_IK_image.sh @@ -15,8 +15,8 @@ docker rm ik-llama-dw-$(date +%Y-%m-%d) #Updated env for other scripts IMAGE=ik-llama-dw:$(date +%Y-%m-%d) -cat > ik_env.sh << EOF +cat > ./env/ik_env.sh << EOF #!/bin/bash #updated from create_new_image.sh -export IK_IMAGE=$IMAGE +export IMAGE=$IMAGE EOF diff --git a/create_new_image.sh b/create_new_image.sh index 9eb7238..2d88c74 100755 --- a/create_new_image.sh +++ b/create_new_image.sh @@ -15,8 +15,8 @@ docker rm llama-dw-$(date +%Y-%m-%d) #Updated env for other scripts IMAGE=llama-dw:$(date +%Y-%m-%d) -cat > env.sh << EOF +cat > ./env/llama_env.sh << EOF #!/bin/bash #updated from create_new_image.sh -export IK_IMAGE=$IMAGE +export IMAGE=$IMAGE EOF diff --git a/create_new_vllm_image.sh b/create_new_vllm_image.sh index 6c51700..4551ed8 100755 --- a/create_new_vllm_image.sh +++ b/create_new_vllm_image.sh @@ -14,10 +14,10 @@ EOF docker build -t $IMG_NAME . #Updated env for other scripts -cat > vllm_env.sh << EOF +cat > ./env/vllm_env.sh << EOF #!/bin/bash #updated from create_new_image_vllm.sh -export VLLM_IMAGE=$IMG_NAME +export IMAGE=$IMG_NAME EOF rm Dockerfile diff --git a/env.sh b/env.sh deleted file mode 100755 index d92cf0e..0000000 --- a/env.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -#updated from create_new_image.sh -export IK_IMAGE=llama-dw:2026-04-29 diff --git a/env/FolderForAutogenENVARS b/env/FolderForAutogenENVARS new file mode 100644 index 0000000..c1b0b50 --- /dev/null +++ b/env/FolderForAutogenENVARS @@ -0,0 +1 @@ +This is the folder that will hold the auto generated ENVs for the docker image generation. \ No newline at end of file diff --git a/gemma-4-26B-A4B-mxfp4-noThink.sh b/gemma-4-26B-A4B-mxfp4-noThink.sh new file mode 100755 index 0000000..0d762d3 --- /dev/null +++ b/gemma-4-26B-A4B-mxfp4-noThink.sh @@ -0,0 +1,25 @@ +#!/bin/bash +#grab container image name +MODEL_NAME="Gemma-4-26B-A4B-noThink" +CONT_NAME="llama-$MODEL_NAME" + +MODEL_FOLDER="/home/jared/.cache/huggingface/hub/models--unsloth--gemma-4-26B-A4B-it-GGUF" +MODEL="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/gemma-4-26B-A4B-it-MXFP4_MOE.gguf" +MODEL_MM="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/mmproj-F16.gguf" + +source ../env/llama_env.sh +echo $IMAGkE +docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ + --health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \ + --health-interval 5s \ + --health-timeout 5s \ + --health-start-period 20s \ + $IMAGE \ + /llama.cpp/build/bin/llama-server \ + --model /model/$MODEL \ + --mm /model/$MODEL_MM \ + --alias $MODEL_NAME \ + -ctk q8_0 -ctv q8_0 \ + --reasoning-budget 0 \ + --host $CONT_NAME \ + --port 10987 --no-mmap -dio diff --git a/spark/gemma-4-26B-A4B-mxfp4.sh b/gemma-4-26B-A4B-mxfp4.sh similarity index 85% rename from spark/gemma-4-26B-A4B-mxfp4.sh rename to gemma-4-26B-A4B-mxfp4.sh index a9b1695..6e76439 100755 --- a/spark/gemma-4-26B-A4B-mxfp4.sh +++ b/gemma-4-26B-A4B-mxfp4.sh @@ -1,24 +1,24 @@ #!/bin/bash #grab container image name -MODEL_NAME="Gemma-4-26B-A4B-it-mxfp4" +MODEL_NAME="Gemma-4-26B-A4B" CONT_NAME="llama-$MODEL_NAME" + MODEL_FOLDER="/home/jared/.cache/huggingface/hub/models--unsloth--gemma-4-26B-A4B-it-GGUF" MODEL="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/gemma-4-26B-A4B-it-MXFP4_MOE.gguf" MODEL_MM="snapshots/3365c68df1a83799b846d05324ebfadbb8cc70b3/mmproj-F16.gguf" -source ../env.sh -echo $IK_IMAGkE +source ../env/llama_env.sh +echo $IMAGkE docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ --health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \ --health-interval 5s \ --health-timeout 5s \ --health-start-period 20s \ - $IK_IMAGE \ + $IMAGE \ /llama.cpp/build/bin/llama-server \ --model /model/$MODEL \ --mm /model/$MODEL_MM \ --alias $MODEL_NAME \ -ctk q8_0 -ctv q8_0 \ - --reasoning-budget 0 \ - --host ik-$MODEL_NAME \ + --host $CONT_NAME \ --port 10987 --no-mmap -dio diff --git a/glm-5.1-IQ3_KS.sh b/glm-5.1-IQ3_KS.sh index fcc7f64..7b468d8 100755 --- a/glm-5.1-IQ3_KS.sh +++ b/glm-5.1-IQ3_KS.sh @@ -1,17 +1,20 @@ #!/bin/bash #grab container image name MODEL_NAME="GLM-5.1-IQ3_KW" +CONT_NAME="ik-$MODEL_NAME" + + MODEL="GLM-5.1-IQ3_KS-00001-of-00008.gguf" MODEL_FOLDER="/sam4t/ENC/LLM/GLM-5.1" -source ./ik_env.sh -echo $IK_IMAGE -docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --gpus=all --restart on-failure:3 \ - --health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \ +source ./env/ik_env.sh +echo $IMAGE +docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ + --health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \ --health-interval 5s \ --health-timeout 2s \ --health-start-period 60s \ - $IK_IMAGE \ + $IMAGE \ /ik_llama.cpp/build/bin/llama-server \ --model /model/$MODEL \ --alias $MODEL_NAME \ @@ -22,5 +25,5 @@ docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $ -ngl 999 \ --n-cpu-moe 66 \ --parallel 1 --threads 32 \ - --host ik-$MODEL_NAME \ + --host $CONT_NAME \ --port 10987 --jinja --no-mmap diff --git a/qwen3.5-122b-IQ4_KSS.sh b/qwen3.5-122b-IQ4_KSS.sh index 8f7092c..fb023c7 100755 --- a/qwen3.5-122b-IQ4_KSS.sh +++ b/qwen3.5-122b-IQ4_KSS.sh @@ -1,13 +1,15 @@ #!/bin/bash #grab container image name MODEL_NAME="Qwen3.5-122B-A10B" +CONT_NAME="ik-$MODEL_NAME" + MODEL="Qwen3.5-122B-A10B-IQ4_KSS.gguf" MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5" source ./ik_env.sh echo $IK_IMAGE -docker container create --name ik-IQ4-KSS-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --gpus=all --restart on-failure:3 \ - --health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \ +docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ + --health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \ --health-interval 5s \ --health-timeout 2s \ --health-start-period 15s \ @@ -19,5 +21,5 @@ docker container create --name ik-IQ4-KSS-$MODEL_NAME --network llms -p 10987:10 -ger -ngl 99 \ -ub 4096 -b 4096 \ --parallel 1 --threads 1 \ - --host ik-$MODEL_NAME \ + --host $CONT_NAME \ --port 10987 --jinja --no-mmap diff --git a/qwen3.5-122b-IQ5_KS.sh b/qwen3.5-122b-IQ5_KS.sh index eb79e3c..7e6ca15 100755 --- a/qwen3.5-122b-IQ5_KS.sh +++ b/qwen3.5-122b-IQ5_KS.sh @@ -1,13 +1,15 @@ #!/bin/bash #grab container image name MODEL_NAME="Qwen3.5-122B-A10B" +CONT_NAME="ik-$MODEL_NAME" + MODEL="Qwen3.5-122B-A10B-smol-IQ5_KS.gguf" MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5" source ./ik_env.sh echo $IK_IMAGE -docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --gpus=all --restart on-failure:3 \ - --health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \ +docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ + --health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \ --health-interval 5s \ --health-timeout 2s \ --health-start-period 15s \ @@ -19,5 +21,5 @@ docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $ -ger -ngl 99 \ -ub 4096 -b 4096 \ --parallel 1 --threads 1 \ - --host ik-$MODEL_NAME \ + --host $CONT_NAME \ --port 10987 --jinja --no-mmap diff --git a/qwen3.5-397B-A17B-IQ4_KSS.sh b/qwen3.5-397B-A17B-IQ4_KSS.sh index a4a682d..798fdcd 100755 --- a/qwen3.5-397B-A17B-IQ4_KSS.sh +++ b/qwen3.5-397B-A17B-IQ4_KSS.sh @@ -1,13 +1,15 @@ #!/bin/bash #grab container image name MODEL_NAME="Qwen3.5-397B-A17B" +CONT_NAME="ik-$MODEL_NAME" + MODEL="huggingface/hub/models--ubergarm--Qwen3.5-397B-A17B-GGUF/snapshots/bb9c5e48cdcad8716ef0dfff1f48f43b34b89cb2/IQ4_KSS/Qwen3.5-397B-A17B-IQ4_KSS-00001-of-00006.gguf" MODEL_FOLDER="/home/jared/.cache" source ./ik_env.sh echo $IK_IMAGE -docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ - --health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \ +docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ + --health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \ --health-interval 5s \ --health-timeout 5s \ --health-start-period 20s \ @@ -19,5 +21,5 @@ docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $ -ngl 999 \ --n-cpu-moe 38 \ --parallel 2 --threads 32 \ - --host ik-$MODEL_NAME \ + --host $CONT_NAME \ --port 10987 --jinja --no-mmap diff --git a/qwen3.5-coder-next.sh b/qwen3.5-coder-next.sh index d2454c4..bb27629 100755 --- a/qwen3.5-coder-next.sh +++ b/qwen3.5-coder-next.sh @@ -1,13 +1,15 @@ #!/bin/bash #grab container image name MODEL_NAME="Qwen3-Coder-Next" +CONT_NAME="ik-$MODEL_NAME" + MODEL="Qwen3-Coder-Next-IQ4_KSS.gguf" MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3-Coder-Next" source ./ik_env.sh echo $IK_IMAGE -docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --gpus=all --restart on-failure:3 \ - --health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \ +docker container create --name $CONT_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \ + --health-cmd "curl -f http://$CONT_NAME:10987/v1/models || exit 1" \ --health-interval 5s \ --health-timeout 2s \ --health-start-period 15s \ @@ -16,7 +18,7 @@ docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $ --model /model/$MODEL \ -ctk q8_0 -ctv q8_0 \ --parallel 1 --threads 32 \ - --host ik-$MODEL_NAME \ + --host $CONT_NAME \ --port 10987 -ngl 99 \ --alias $MODEL_NAME \ --no-mmap --jinja \ diff --git a/spark/IfSparkSpecific b/spark/IfSparkSpecific new file mode 100644 index 0000000..b9673e1 --- /dev/null +++ b/spark/IfSparkSpecific @@ -0,0 +1 @@ +This is where spark specific settings will be \ No newline at end of file