jdelony/ik_llama_docker
Template
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added a pure llama.cpp image

Browse Source
This commit is contained in:
Jared Delony 2026-05-07 12:35:42 -05:00
parent b3babd43b0
commit cb2346e432
Signed by: jdelony
SSH Key Fingerprint: SHA256:W5F7KMb6FLjlHNhQhtrDl4NvR5fCfinaDDXSQnA2sF0
9 changed files with 90 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
Qwen3.6=27B-Uncensored.sh Executable file
View File

@ -0,0 +1,24 @@
#!/bin/bash
#grab container image name
MODEL_FOLDER="/home/jared/.cache"
MODEL_NAME="Qwen3.6-27B-Uncensored"
MODEL="huggingface/hub/models--HauhauCS--Qwen3.6-27B-Uncensored-HauhauCS-Aggressive/snapshots/f2db94d03eccf3b133aeea3dea62388acf98e864/Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf"
MMPROJ="huggingface/hub/models--HauhauCS--Qwen3.6-27B-Uncensored-HauhauCS-Aggressive/snapshots/f2db94d03eccf3b133aeea3dea62388acf98e864/mmproj-Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-f16.gguf"
NAME="LLM-$MODEL_NAME"
source ./env.sh
echo $IK_IMAGE
docker container create --name $NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://$NAME:10987/v1/models || exit 1" \
--health-interval 5s \
--health-timeout 5s \
--health-start-period 20s \
$IK_IMAGE \
/llama.cpp/build/bin/llama-server \
--model /model/$MODEL \
--alias $MODEL_NAME \
-ctk q8_0 -ctv q8_0 \
--image-min-tokens 2048 \
-mm /model/$MMPROJ \
--host $NAME \
--port 10987 --jinja --no-mmap -dio

22
create_new_IK_image.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
export BASE_IMAGE="nvcr.io/nvidia/cuda:13.2.0-cudnn-devel-ubuntu24.04"
docker run --name ik-llama-dw-$(date +%Y-%m-%d) --gpus=all $BASE_IMAGE /bin/bash -c \
'apt-get update
apt-get --yes upgrade
apt-get --yes install build-essential git libcurl4-openssl-dev curl libgomp1 cmake
git clone https://github.com/ikawrakow/ik_llama.cpp
cd ik_llama.cpp
cmake -B build -DGGML_NATIVE=ON -DGGML_CUDA=ON
cmake --build build --config Release -j$(nproc)'
docker commit ik-llama-dw-$(date +%Y-%m-%d) ik-llama-dw:$(date +%Y-%m-%d)
docker rm ik-llama-dw-$(date +%Y-%m-%d)
#Updated env for other scripts
IMAGE=ik-llama-dw:$(date +%Y-%m-%d)
cat > ik_env.sh << EOF
#!/bin/bash
#updated from create_new_image.sh
export IK_IMAGE=$IMAGE
EOF

12
create_new_image.sh
View File

@ -1,19 +1,19 @@
#!/bin/bash
export BASE_IMAGE="nvcr.io/nvidia/cuda:13.2.0-cudnn-devel-ubuntu24.04"
docker run --name ik-llama-dw-$(date +%Y-%m-%d) --gpus=all $BASE_IMAGE /bin/bash -c \
docker run --name llama-dw-$(date +%Y-%m-%d) --gpus=all $BASE_IMAGE /bin/bash -c \
'apt-get update
apt-get --yes upgrade
apt-get --yes install build-essential git libcurl4-openssl-dev curl libgomp1 cmake
git clone https://github.com/ikawrakow/ik_llama.cpp
cd ik_llama.cpp
git clone https://github.com/ggml-org/llama.cpp
cd llama.cpp
cmake -B build -DGGML_NATIVE=ON -DGGML_CUDA=ON
cmake --build build --config Release -j$(nproc)'
docker commit ik-llama-dw-$(date +%Y-%m-%d) ik-llama-dw:$(date +%Y-%m-%d)
docker rm ik-llama-dw-$(date +%Y-%m-%d)
docker commit llama-dw-$(date +%Y-%m-%d) llama-dw:$(date +%Y-%m-%d)
docker rm llama-dw-$(date +%Y-%m-%d)
#Updated env for other scripts
IMAGE=ik-llama-dw:$(date +%Y-%m-%d)
IMAGE=llama-dw:$(date +%Y-%m-%d)
cat > env.sh << EOF
#!/bin/bash

2
env.sh
View File

@ -1,3 +1,3 @@
#!/bin/bash
#updated from create_new_image.sh
export IK_IMAGE=ik-llama-dw:2026-03-25
export IK_IMAGE=llama-dw:2026-04-29

26
glm-5.1-IQ3_KS.sh Executable file
View File

@ -0,0 +1,26 @@
#!/bin/bash
#grab container image name
MODEL_NAME="GLM-5.1-IQ3_KW"
MODEL="GLM-5.1-IQ3_KS-00001-of-00008.gguf"
MODEL_FOLDER="/sam4t/ENC/LLM/GLM-5.1"
source ./ik_env.sh
echo $IK_IMAGE
docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \
--health-interval 5s \
--health-timeout 2s \
--health-start-period 60s \
$IK_IMAGE \
/ik_llama.cpp/build/bin/llama-server \
--model /model/$MODEL \
--alias $MODEL_NAME \
--ctx-size 64000 \
-amb 512 \
-muge \
--merge-qkv \
-ngl 999 \
--n-cpu-moe 66 \
--parallel 1 --threads 32 \
--host ik-$MODEL_NAME \
--port 10987 --jinja --no-mmap

2
qwen3.5-122b-IQ4_KSS.sh
View File

@ -4,7 +4,7 @@ MODEL_NAME="Qwen3.5-122B-A10B"
MODEL="Qwen3.5-122B-A10B-IQ4_KSS.gguf"
MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5"
source ./env.sh
source ./ik_env.sh
echo $IK_IMAGE
docker container create --name ik-IQ4-KSS-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \

2
qwen3.5-122b-IQ5_KS.sh
View File

@ -4,7 +4,7 @@ MODEL_NAME="Qwen3.5-122B-A10B"
MODEL="Qwen3.5-122B-A10B-smol-IQ5_KS.gguf"
MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5"
source ./env.sh
source ./ik_env.sh
echo $IK_IMAGE
docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \

15
qwen3.5-397B-A17B-IQ4_KSS.sh
View File

@ -1,24 +1,23 @@
#!/bin/bash
#grab container image name
MODEL_NAME="Qwen3.5-397B-A17B"
MODEL="Qwen3.5-397B-A17B-IQ4_KSS-00001-of-00006.gguf"
MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3.5"
MODEL="huggingface/hub/models--ubergarm--Qwen3.5-397B-A17B-GGUF/snapshots/bb9c5e48cdcad8716ef0dfff1f48f43b34b89cb2/IQ4_KSS/Qwen3.5-397B-A17B-IQ4_KSS-00001-of-00006.gguf"
MODEL_FOLDER="/home/jared/.cache"
source ./env.sh
source ./ik_env.sh
echo $IK_IMAGE
docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --gpus=all --restart on-failure:3 \
docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --user $(id -u):$(id -g) --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \
--health-interval 5s \
--health-timeout 2s \
--health-start-period 15s \
--health-timeout 5s \
--health-start-period 20s \
$IK_IMAGE \
/ik_llama.cpp/build/bin/llama-server \
--model /model/$MODEL \
--alias $MODEL_NAME \
-ctk q8_0 -ctv q8_0 \
-ngl 999 \
-ub 4096 -b 4096 \
--n-cpu-moe 38 \
--parallel 1 --threads 32 \
--parallel 2 --threads 32 \
--host ik-$MODEL_NAME \
--port 10987 --jinja --no-mmap

5
qwen3.5-coder-next.sh
View File

@ -4,7 +4,7 @@ MODEL_NAME="Qwen3-Coder-Next"
MODEL="Qwen3-Coder-Next-IQ4_KSS.gguf"
MODEL_FOLDER="/sam4t/ENC/LLM/Qwen3-Coder-Next"
source ./env.sh
source ./ik_env.sh
echo $IK_IMAGE
docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $MODEL_FOLDER:/model --gpus=all --restart on-failure:3 \
--health-cmd "curl -f http://ik-$MODEL_NAME:10987/v1/models || exit 1" \
@ -23,5 +23,4 @@ docker container create --name ik-$MODEL_NAME --network llms -p 10987:10987 -v $
-b 4096 -ub 4096 \
-ns 1 -np 1 -cram 32000 \
-cram-n-min 128 -crs .9 \
--ctx-checkpoints 128
--ctx-checkpoints 128