.env

MODEL_PATH = ""
# if MODEL_PATH is "", default llama.cpp/gptq models 
# will be downloaded to: ./models

# Example ggml path:
#MODEL_PATH = "models/llama2_7b_chat.bin"

# options: llama.cpp, gptq, transformers
BACKEND_TYPE = "llama.cpp"
#BACKEND_TYPE = "llama2.cu"

# only for transformers bitsandbytes 8 bit
LOAD_IN_8BIT = False

MAX_MAX_NEW_TOKENS = 2048
DEFAULT_MAX_NEW_TOKENS = 1024
MAX_INPUT_TOKEN_LENGTH = 4000

DEFAULT_SYSTEM_PROMPT = ""