ci: remove the submodule and during the current docker build we download instead the files from the repositories OR copy from the current directory

.dockerignore

@@ -21,5 +21,4 @@ mongo/
 .history/
 .continue/
 .vscode/
-scripts/
 tests/

.gitignore

@@ -667,4 +667,6 @@ $RECYCLE.BIN/
 # custom
 # onnx models
 *.onnx
-tmp/*
+tmp/*
+lite.koboldai.net
+lite.koboldai.net/*

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "lite.koboldai.net"]
-	path = lite.koboldai.net
-	url = [email protected]:aletrn/lite.koboldai.net.git

Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.gitlab.com/aletrn/my_ghost_writer_base:0.5.3
+FROM registry.gitlab.com/aletrn/my_ghost_writer_base:0.6.0
 
 LABEL authors="trincadev"
 
@@ -8,10 +8,28 @@ ENV VIRTUAL_ENV=${HOME}/.venv PATH="${HOME}/.venv/bin:$PATH"
 ENV WRITE_TMP_ON_DISK=""
 ENV MOUNT_GRADIO_APP=""
 ENV SPACY_MODEL="en_core_web_sm"
+ENV BASEURL1=https://raw.githubusercontent.com/trincadev/lite.koboldai.net/refs/heads/mgw_smart_thesaurus
+ENV BASEURL2=https://gitlab.com/aletrn/my_ghost_writer/-/raw/main
 
 # Set working directory to function root directory
 WORKDIR ${HOME}
 
+RUN mkdir ${HOME}/lite.koboldai.net && chown python:python ${HOME}/lite.koboldai.net
+RUN mkdir ${HOME}/my_ghost_writer && chown python:python ${HOME}/my_ghost_writer
+#COPY --chown=python:python ./lite.koboldai.net* ${HOME}/lite.koboldai.net
+#COPY --chown=python:python ./my_ghost_writer* ${HOME}/my_ghost_writer
+
+# for lite.koboldai.net.txt we don't need the folder name: we are splitting the paths to keep only the filenames
+# ${x##*/} will keep only the filename, in case of lite.koboldai.net
+RUN ls -l ${HOME}/lite.koboldai.net/index.html || for x in $(cat ${HOME}/lite.koboldai.net.txt); do ${HOME}/download.sh "${BASEURL1}/${x##*/}" ${HOME}/lite.koboldai.net/; done
+RUN ls -l ${HOME}/lite.koboldai.net/index.html || echo "did you checked if the files list from lite.koboldai.net.txt is ok? Maybe not keeping only the filenames?"
+RUN ls -l ${HOME}/lite.koboldai.net/index.html
+
+RUN ls -l ${HOME}/my_ghost_writer/app.py || for x in $(cat ${HOME}/my_ghost_writer.txt); do ${HOME}/download.sh "${BASEURL2}/$x" ${HOME}/my_ghost_writer/; done
+RUN ls -l ${HOME}/my_ghost_writer/app.py
+RUN ls -l ${HOME}/my_ghost_writer/custom_synonym_handler.py
+RUN rm ./download.sh
+
 # RUN . ${HOME}/.venv && which python && echo "# install samgis #" && pip install .
 RUN if [ "${WRITE_TMP_ON_DISK}" != "" ]; then mkdir {WRITE_TMP_ON_DISK}; fi
 RUN if [ "${WRITE_TMP_ON_DISK}" != "" ]; then ls -l {WRITE_TMP_ON_DISK}; fi

dockerfiles/dockerfile_my_ghost_writer_base

@@ -18,6 +18,7 @@ ARG POETRY_NO_INTERACTION
 ARG POETRY_VIRTUALENVS_IN_PROJECT
 ARG POETRY_VIRTUALENVS_CREATE
 ARG POETRY_CACHE_DIR
+ARG HOME
 
 RUN echo "ARCH: $ARCH ..."
 RUN echo "HOME: ${HOME} ..."
@@ -64,15 +65,14 @@ WORKDIR ${HOME}
 COPY --from=builder_global ${HOME}/.venv ${HOME}/.venv
 RUN . ${HOME}/.venv && which python && pip list
 
-COPY --chown=python:python ./lite.koboldai.net ${HOME}/lite.koboldai.net
-COPY --chown=python:python ./my_ghost_writer ${HOME}/my_ghost_writer
 RUN mkdir ${HOME}/static && chown python:python ${HOME}/static
 COPY --chown=python:python static/index-dark.css static/index.html static/index_dark.html static/index.css static/index.js ${HOME}/static/
+COPY --chown=python:python lite.koboldai.net.txt my_ghost_writer.txt ${HOME}/
+COPY --chown=python:python scripts/download.sh scripts/download_file.py ${HOME}/
+RUN chmod +x ${HOME}/download.sh
 
 RUN echo "new HOME after hidden venv copy => ${HOME}"
 RUN ls -ld ${HOME}/
 RUN ls -lA ${HOME}/
-RUN ls -l ${HOME}/lite.koboldai.net/index.html
 RUN ls -l ${HOME}/static/index.html
-RUN ls -l ${HOME}/my_ghost_writer/app.py
-RUN ls -l ${HOME}/my_ghost_writer/custom_synonym_handler.py
+

lite.koboldai.net

@@ -1 +0,0 @@
-Subproject commit eee9cdcd9978d4285d78c061b16ceafc5acb2192

lite.koboldai.net.txt

@@ -0,0 +1,13 @@
+lite.koboldai.net/niko.png
+lite.koboldai.net/package.json
+lite.koboldai.net/koboldcpp_api.json
+lite.koboldai.net/kudorank.html
+lite.koboldai.net/sw.js
+lite.koboldai.net/README.md
+lite.koboldai.net/CNAME
+lite.koboldai.net/LICENSE.md
+lite.koboldai.net/preview2.png
+lite.koboldai.net/manifest.json
+lite.koboldai.net/koboldcpp_api.html
+lite.koboldai.net/preview1.png
+lite.koboldai.net/index.html

my_ghost_writer.txt

@@ -0,0 +1,15 @@
+my_ghost_writer/__init__.py
+my_ghost_writer/custom_synonym_handler.py
+my_ghost_writer/thesaurus.py
+my_ghost_writer/constants.py
+my_ghost_writer/middlewares.py
+my_ghost_writer/exception_handlers.py
+my_ghost_writer/text_parsers.py
+my_ghost_writer/api_clients/a.txt
+my_ghost_writer/app.py
+my_ghost_writer/pymongo_operations_rw.py
+my_ghost_writer/pymongo_utils.py
+my_ghost_writer/session_logger.py
+my_ghost_writer/text_parsers2.py
+my_ghost_writer/__version__.py
+my_ghost_writer/type_hints.py

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "my-ghost-writer"
-version = "0.5.3"
+version = "0.6.0"
 description = "A simple webapp supporting writers"
 authors = [
     {name = "alessandro trinca tornidor",email = "[email protected]"}

scripts/download.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# Check if a URL is provided as an argument
+OUTPUT_FOLDER=$2
+if [ -z "$1" ] || [ -z "$2" ]; then
+    echo "Usage: $0 <URL> <OUTPUT_FOLDER>"
+    exit 1
+fi
+
+URL="$1"
+FILENAME=$(basename "$URL")
+echo "Downloading '${FILENAME}' from '${URL}'"
+
+# Run the Python script
+python3 ./download_file.py "${URL}" "${OUTPUT_FOLDER}/${FILENAME}"
+
+exit 0

scripts/download_file.py

@@ -0,0 +1,20 @@
+import requests
+import sys
+
+
+if __name__ == '__main__':
+    url = sys.argv[1]
+    filename = sys.argv[2]
+
+    try:
+        response = requests.get(url, stream=True)
+        response.raise_for_status()  # Raise an exception for HTTP errors
+
+        print(f"saving file to '{filename}'")
+        with open(filename, 'wb') as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                f.write(chunk)
+        print(f"Successfully downloaded '{filename}' from '{url}'")
+    except requests.exceptions.RequestException as e:
+        print(f"Error downloading the file: '{e}'")
+        sys.exit(1)