koboldai/koboldcpp:

services:
  koboldcpp:
    restart: unless-stopped

    tty: true
    image: koboldai/koboldcpp
    container_name: koboldcpp
    working_dir: /home/koboldcpp
    ports:
      - "${PORT}:5001"
    volumes:
      - "${MODEL_PATH}:/models"
      - "${CONFIG_PATH}:/config"
    environment:
      KCPP_ARGS: "${KCPP_ARGS}"
      KCPP_DONT_REMOVE_MODELS: "${KCPP_DONT_REMOVE_MODELS}"
      KCPP_DONT_TUNNEL: "${KCPP_DONT_TUNNEL}"

    env_file:
      - stack.env

stack.env

1
2
3
4
5
6
7
8
MODEL_NAME=
THREADS=7
PORT=5001
KCPP_DONT_TUNNEL=true
KCPP_DONT_REMOVE_MODELS=true
MODEL_PATH=/opt/4tb/files/llm-models
CONFIG_PATH=/opt/4tb/files/llm-models/koboldcpp/config
KCPP_ARGS="--threads=${THREADS} --model /models/${MODEL_NAME}"
Edit Report
Pub: 28 Sep 2024 09:11 UTC
Views: 45