dockerfile.py version

Dockerfile

@@ -13,7 +13,7 @@
 # Base image #
 ##############
 # Adds rstudio server, tiyverse, devtools to rocker/cuda
-FROM rocker/ml:4.2
+FROM dvats-rstudio-base:latest
 
 RUN echo "--> Prior installations" \ 
     apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A4B469963BF863CC \
@@ -34,8 +34,8 @@ SHELL [ "/bin/bash", "--login", "-c" ]
 # Ensure vars # (Only for local test - DON'T USE IT IN HUGGINGFACE. ENSURE .env IS NOT ACCESIBLE IN THE WEB, USE THEIR SECRET MANAGEMENT)
 ###############
 RUN echo "--> ARGS"
-ARG USER_NAME \
-    USER_ID \
+ARG USER_NAME=user \
+    USER_ID=1000 \
     GROUP_ID \
     WANDB_API_KEY \ 
     WANDB_ENTITY \
@@ -63,36 +63,18 @@ ENV RETICULATE_PYTHON_ENV=/home/${USER_NAME}/env \
 # Volumes #
 ###########
 
-RUN echo "---> Setup volumes" \
-    echo ${HOME} \
-    echo ${USER_NAME}"|"${USER_ID}"|"${GROUP_ID} 
 
-COPY --chown=${USER_ID}:${GROUP_ID} ../r_shiny_app $HOME/app
-COPY --chown=${USER_ID}:${GROUP_ID} ${LOCAL_DATA_PATH} $HOME/data/
-COPY --chown=${USER_ID}:${GROUP_ID} ../dvats ${HOME}/dvats
 
 
 ## Install R packages
 
-#COPY docker/DESCRIPTION /tmp/
+# COPY docker/DESCRIPTION /tmp/
 COPY --chown=${USER_ID}:${GROUP_ID} docker/DESCRIPTION /tmp/
 
+RUN echo "--> Install R packages" \ 
+    R -e "devtools::install_deps('/tmp/', dependencies = TRUE)"
 
-RUN echo "--> Install R packages"
-
-#RUN R -e "install.packages(deps_path = '/tmp/DESCRIPTION', remotes::dev_package_deps(dependencies = TRUE), repos = NULL)"
-
-RUN R -e "devtools::install_deps('/tmp/', dependencies = TRUE)"
-
-#RUN R -e "install.packages(c('shiny', 'pals', 'shinyWidgets', 'dygraphs', 'shinycssloaders', 'shinyjs', 'Rcpp', 'reactlog', 'pals',  'feather', 'fasttime', 'zoo', 'shinyWidgets'))"
-#RUN R -e "remotes::install_github('r-lib/later')"
-#RUN R -e "remotes::install_github('apache/arrow/r')"
-
-## Install reticulate and create virtual environment using default Ubuntu installed Python
-#RUN R -e "install.packages('reticulate')"
-
-
-
+## Reticulate configuration
 ENV ENV_PREFIX $HOME/env
 
 RUN echo "---> Make sudo & setup sudoers"
@@ -128,12 +110,13 @@ RUN echo "HOME: ${HOME} | CONDA_DIR = ${CONDA_DIR}" \
     && rm ~/miniconda.sh 
 
     # Make conda activate command available from /bin/bash --login shells
-RUN echo ". $CONDA_DIR/etc/profile.d/conda.sh" >> ~/.profile 
+RUN echo ". $CONDA_DIR/etc/profile.d/conda.sh" >> $HOME/.profile 
     # make conda activate command available from /bin/bash --interative shells
 RUN  conda init bash \
     # create a project directory inside user home
     && mkdir -p $PROJECT_DIR
 
+
 WORKDIR $PROJECT_DIR
 ##########################
 # Install & update MAMBA #
@@ -183,6 +166,15 @@ RUN echo "--> Permissions"
 
 RUN echo "$(id )"
 
+### Copy deepvats
+RUN echo "---> Setup volumes" \
+    echo ${HOME} \
+    echo ${USER_NAME}"|"${USER_ID}"|"${GROUP_ID} 
+
+COPY --chown=${USER_ID}:${GROUP_ID} ../r_shiny_app $HOME/app
+COPY --chown=${USER_ID}:${GROUP_ID} ${LOCAL_DATA_PATH} $HOME/data/
+COPY --chown=${USER_ID}:${GROUP_ID} ../dvats ${HOME}/dvats
+
 #RUN echo "--> Setup miniconda & Mamba"
 
 #RUN echo ". ${HOME}miniconda3/etc/profile.d/conda.sh" >> ${HOME}/rstudio/.bashrc
@@ -204,6 +196,7 @@ RUN echo "---> Setup variables" && \
     echo "$ENV_VAR_NAME=$ENV_VAR_VALUE"; \
     done > ${HOME}/.Renviron
 
+# Install deepvats
 
 RUN pip install -e  /home/${USER_NAME}/work
 RUN echo "--> Go!"

Dockerfile.base

@@ -0,0 +1,72 @@
+#R-Shiny template https://huggingface.co/spaces/posit/shiny-for-r-template/tree/main
+#Py-shiny template https://huggingface.co/spaces/posit/gptneox-chat
+
+#############################
+# RSTUDIO-SERVER            #
+# VOLUME DOCKERFILE         #
+#***************************#
+# Set up a jupyter-lab for  #
+# developing using deepvats #
+#############################
+
+##############
+# Base image #
+##############
+# Adds rstudio server, tiyverse, devtools to rocker/cuda
+FROM rocker/ml:4.2
+
+
+
+###############
+# Ensure vars # (Only for local test - DON'T USE IT IN HUGGINGFACE. ENSURE .env IS NOT ACCESIBLE IN THE WEB, USE THEIR SECRET MANAGEMENT)
+###############
+RUN echo "--> ARGS"
+ARG USER_NAME \
+    USER_ID \
+    GROUP_ID \
+    WANDB_API_KEY \ 
+    WANDB_ENTITY \
+    WANDB_PROJECT \
+    GH_TOKEN 
+
+RUN echo "--> ARGS -> ENV"
+ENV USER_NAME=$USER_NAME \
+    USER_ID=$USER_ID \
+    GROUP_ID=$GROUP_ID \
+    WANDB_ENTITY=$WANDB_ENTITY \
+    WANDB_API_KEY=$WANDB_API_KEY \
+    GH_TOKEN=$ghtoken \
+    WANDB_PROJECT=$WANDB_PROJECT
+
+RUN echo "--> Setup ENV variables"
+ENV RETICULATE_PYTHON_ENV=/home/${USER_NAME}/env \
+    ROOT=TRUE \
+    RETICULATE_MINICONDA_PATH=/home/${USER_NAME}/miniconda \
+    CUDA_VISIBLE_DEVICES=0,1,2 \
+    ENV_VARS=WANDB_ENTITY,WANDB_PROJECT,USER_NAME,USER_ID,GROUP_ID,PASSWORD,ROOT,CUDA_VISIBLE_DEVICES\
+    HOME="/home/${USER_NAME}"
+
+###########
+# Volumes #
+###########
+
+RUN echo "---> Setup volumes" \
+    echo ${HOME} \
+    echo ${USER_NAME}"|"${USER_ID}"|"${GROUP_ID} 
+
+COPY --chown=${USER_ID}:${GROUP_ID} ../r_shiny_app $HOME/app
+COPY --chown=${USER_ID}:${GROUP_ID} ${LOCAL_DATA_PATH} $HOME/data/
+COPY --chown=${USER_ID}:${GROUP_ID} ../dvats ${HOME}/dvats
+
+RUN echo "--> Prior installations" \ 
+    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A4B469963BF863CC \
+    set -e; \
+    apt-get update --fix-missing \
+    && apt-get install -y \
+    python3-pip wget bzip2 curl \
+    #dbus python3-pip wget bzip2 curl \
+    git sudo libarchive-dev libtiff5-dev zsh\
+    #python3 -m pip install --upgrade pip \
+    python3-venv libxt-dev \
+    && apt-get clean  \
+    && rm -rf /var/lib/apt/lists/* 

Dockerfile.py

@@ -0,0 +1,61 @@
+############################################
+# CONDA & MINICONDA WITH PYBASE DOCKERFILE #
+############################################
+##############
+# Base image #
+##############
+##--- Setup Ubuntu
+ARG CUDA_VERSION
+FROM nvidia/cuda:${CUDA_VERSION}
+
+##--- Tags
+LABEL maintainer="vrodriguezf <[email protected]>"
+LABEL cuda_version=${CUDA_VERSION}
+LABEL cuda_version=${CUDA_VERSION}
+
+##--- Setup bash
+SHELL [ "/bin/bash", "--login", "-c" ]
+
+##################
+# Packages setup #
+################## 
+#TODO: Automatizar que lo coja de /etc/timezone
+ARG TZ=Etc/UTC 
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone \
+    && apt-get update --fix-missing \
+    && apt-get install -y wget bzip2 curl git sudo libarchive-dev libtiff5-dev zsh\
+    && apt-get clean  \
+    && rm -rf /var/lib/apt/lists/*
+
+#-- Environmental variables for wandb
+ENV LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 
+
+#####################
+# Install miniconda #
+#####################
+
+ENV MINICONDA_VERSION=4.10.3 \ 
+#ENV MINICONDA_VERSION=23.9.0 \ 
+    CONDA_DIR=$HOME/miniconda3 \
+    # Make non-activate conda commands available
+    PATH=$CONDA_DIR/bin:$PATH \
+    HOME=~\
+    PROJECT_DIR=$HOME
+
+RUN echo "HOME: ${HOME} | CONDA_DIR = ${CONDA_DIR}" \
+    #-- Install MINICONDA
+    && wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py38_$MINICONDA_VERSION-Linux-x86_64.sh -O ~/miniconda.sh  \
+    && chmod +x ~/miniconda.sh  \
+    && ~/miniconda.sh -b -p $CONDA_DIR  \
+    && rm ~/miniconda.sh 
+
+#-- Bring & Install deepvats
+COPY ./ ${HOME}/work
+RUN conda activate $ENV_PREFIX
+RUN pip install -e /home/$USER/work
+
+###### -> Cuando se cree un usuario en el otro docker
+###### habrá que hacer chown y mover al nuevo home todo
+###### o ver cómo hacerlo
+

Dockerfile.rstudio

@@ -0,0 +1,14 @@
+#############################
+# RSTUDIO-SERVER            #
+# VOLUME DOCKERFILE         #
+#***************************#
+# Set up a jupyter-lab for  #
+# developing using deepvats #
+#############################
+
+##############
+# Base image #
+##############
+# Adds rstudio server, tiyverse, devtools to rocker/cuda
+FROM rocker-cuda-base:latest
+

Dockerfile_bkup

@@ -0,0 +1,211 @@
+#R-Shiny template https://huggingface.co/spaces/posit/shiny-for-r-template/tree/main
+#Py-shiny template https://huggingface.co/spaces/posit/gptneox-chat
+
+#############################
+# RSTUDIO-SERVER            #
+# VOLUME DOCKERFILE         #
+#***************************#
+# Set up a jupyter-lab for  #
+# developing using deepvats #
+#############################
+
+##############
+# Base image #
+##############
+# Adds rstudio server, tiyverse, devtools to rocker/cuda
+FROM dvats-rstudio-base:latest
+
+SHELL [ "/bin/bash", "--login", "-c" ]
+
+###############
+# Ensure vars # (Only for local test - DON'T USE IT IN HUGGINGFACE. ENSURE .env IS NOT ACCESIBLE IN THE WEB, USE THEIR SECRET MANAGEMENT)
+###############
+RUN echo "--> ARGS"
+ARG USER_NAME \
+    USER_ID \
+    GROUP_ID \
+    WANDB_API_KEY \ 
+    WANDB_ENTITY \
+    WANDB_PROJECT \
+    GH_TOKEN 
+
+RUN echo "--> ARGS -> ENV"
+ENV USER_NAME=$USER_NAME \
+    USER_ID=$USER_ID \
+    GROUP_ID=$GROUP_ID \
+    WANDB_ENTITY=$WANDB_ENTITY \
+    WANDB_API_KEY=$WANDB_API_KEY \
+    GH_TOKEN=$ghtoken \
+    WANDB_PROJECT=$WANDB_PROJECT
+
+RUN echo "--> Setup ENV variables"
+ENV RETICULATE_PYTHON_ENV=/home/${USER_NAME}/env \
+    ROOT=TRUE \
+    RETICULATE_MINICONDA_PATH=/home/${USER_NAME}/miniconda \
+    CUDA_VISIBLE_DEVICES=0,1,2 \
+    ENV_VARS=WANDB_ENTITY,WANDB_PROJECT,USER_NAME,USER_ID,GROUP_ID,PASSWORD,ROOT,CUDA_VISIBLE_DEVICES\
+    HOME="/home/${USER_NAME}"
+
+###########
+# Volumes #
+###########
+
+RUN echo "---> Setup volumes" \
+    echo ${HOME} \
+    echo ${USER_NAME}"|"${USER_ID}"|"${GROUP_ID} 
+
+COPY --chown=${USER_ID}:${GROUP_ID} ../r_shiny_app $HOME/app
+COPY --chown=${USER_ID}:${GROUP_ID} ${LOCAL_DATA_PATH} $HOME/data/
+COPY --chown=${USER_ID}:${GROUP_ID} ../dvats ${HOME}/dvats
+
+RUN echo "--> Prior installations" \ 
+    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A4B469963BF863CC \
+    set -e; \
+    apt-get update --fix-missing \
+    && apt-get install -y \
+    python3-pip wget bzip2 curl \
+    git sudo libarchive-dev libtiff5-dev zsh\
+    #python3 -m pip install --upgrade pip \
+    python3-venv libxt-dev \
+    && apt-get clean  \
+    && rm -rf /var/lib/apt/lists/* 
+
+## Install R packages
+
+#COPY docker/DESCRIPTION /tmp/
+COPY --chown=${USER_ID}:${GROUP_ID} docker/DESCRIPTION /tmp/
+
+RUN ls -la /tmp/ && sleep 5
+
+RUN echo "--> Install R packages"
+
+#RUN R -e "install.packages(deps_path = '/tmp/DESCRIPTION', remotes::dev_package_deps(dependencies = TRUE), repos = NULL)"
+
+RUN R -e "devtools::install_deps('/tmp/', dependencies = TRUE)"
+
+#RUN R -e "install.packages(c('shiny', 'pals', 'shinyWidgets', 'dygraphs', 'shinycssloaders', 'shinyjs', 'Rcpp', 'reactlog', 'pals',  'feather', 'fasttime', 'zoo', 'shinyWidgets'))"
+#RUN R -e "remotes::install_github('r-lib/later')"
+#RUN R -e "remotes::install_github('apache/arrow/r')"
+
+## Install reticulate and create virtual environment using default Ubuntu installed Python
+#RUN R -e "install.packages('reticulate')"
+
+
+
+ENV ENV_PREFIX $HOME/env
+
+RUN echo "---> Make sudo & setup sudoers"
+RUN echo "$(id)"
+
+ARG RETICULATE_MINICONDA_PATH=/usr/local/share/r-miniconda
+ARG RETICULATE_PYTHON_ENV=/usr/virtualenvs/venv_shiny_app
+ENV RETICULATE_PYTHON=${RETICULATE_PYTHON_ENV}/bin/python/
+
+RUN echo "---> RETICULATE PYTHONS"
+RUN echo "RETICULATE_PYTHON_ENV=${RETICULATE_PYTHON_ENV}" >> ${HOME}/.Renviron
+RUN echo "RETICULATE_PYTHON=${RETICULATE_PYTHON}" >> ${HOME}/.Renviron
+
+RUN echo "---> Export W&B"
+## Export W&B environment variable to Rstudio
+
+ARG WANDB_API_KEY
+RUN echo "WANDB_API_KEY=${WANDB_API_KEY}" >> ${HOME}/.Renviron
+
+
+ENV MINICONDA_VERSION=4.10.3 \ 
+#ENV MINICONDA_VERSION=23.9.0 \ 
+    CONDA_DIR=$HOME/miniconda3 \
+    # Make non-activate conda commands available
+    PATH=$CONDA_DIR/bin:$PATH \
+    PROJECT_DIR=$HOME 
+
+RUN echo "HOME: ${HOME} | CONDA_DIR = ${CONDA_DIR}" \
+    #-- Install MINICONDA
+    && wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py38_$MINICONDA_VERSION-Linux-x86_64.sh -O ~/miniconda.sh  \
+    && chmod +x ~/miniconda.sh  \
+    && ~/miniconda.sh -b -p $CONDA_DIR  \
+    && rm ~/miniconda.sh 
+
+    # Make conda activate command available from /bin/bash --login shells
+RUN echo ". $CONDA_DIR/etc/profile.d/conda.sh" >> ~/.profile 
+    # make conda activate command available from /bin/bash --interative shells
+RUN  conda init bash \
+    # create a project directory inside user home
+    && mkdir -p $PROJECT_DIR
+
+WORKDIR $PROJECT_DIR
+##########################
+# Install & update MAMBA #
+########################## 
+ENV ENV_PREFIX $PROJECT_DIR/env
+RUN conda install --name base --channel conda-forge mamba \
+    && mamba update --name base --channel defaults conda 
+#-- Build the mamba environment
+RUN mamba install conda-lock -c conda-forge
+COPY --chown=$UID:$GID docker/environment.yml docker/requirements.txt /tmp/
+#RUN mamba lock -f /tmp/environment.yml --lockfile /tmp/environment.lock
+#RUN mamba create --prefix ${ENV_PREFIX} --file /tmp/environment.lock
+RUN mamba env create -vv --prefix ${ENV_PREFIX} --file /tmp/environment.yml
+RUN conda clean --all --yes
+
+# run the postBuild script to install the JupyterLab extensions
+COPY --chown=$UID:$GID docker/postBuild /usr/local/bin
+RUN chmod u+x /usr/local/bin/postBuild \
+    && conda activate $ENV_PREFIX  \
+    && /usr/local/bin/postBuild  \
+    && conda deactivate \
+    # Make bash automatically activate the conda environment
+    && echo "conda activate $ENV_PREFIX" >> ~/.bashrc
+
+
+RUN chmod u+x /usr/local/bin/postBuild \
+    && conda activate $ENV_PREFIX  \
+    && /usr/local/bin/postBuild  \
+    && conda deactivate \
+    # Make bash automatically activate the conda environment
+    && echo "conda activate $ENV_PREFIX" >> ~/.bashrc
+
+RUN conda list --prefix ${ENV_PREFIX}
+
+
+
+
+
+RUN echo "--> rewrrite miniconda path"
+# Rewrite the miniconda path environment in case it has been redefined in the compose file
+RUN echo "RETICULATE_MINICONDA_PATH=${RETICULATE_MINICONDA_PATH}" >> ${HOME}/.Renviron
+
+RUN echo "---> Make non-activate conda commands available"
+RUN # make non-activate conda commands available
+ENV PATH=${RETICULATE_MINICONDA_PATH}/bin:${PATH}
+RUN echo "--> Permissions"
+
+RUN echo "$(id )"
+
+#RUN echo "--> Setup miniconda & Mamba"
+
+#RUN echo ". ${HOME}miniconda3/etc/profile.d/conda.sh" >> ${HOME}/rstudio/.bashrc
+
+
+#RUN echo "--> Conda activate"
+# Make bash automatically activate the conda environment
+#RUN echo "conda activate ${ENV_PREFIX}" >> ${HOME}/rstudio/.bashrc
+#echo "export WANDB_ENTITY=${WANDB_ENTITY:-default}" >> ${HOME}/.bashrc
+# echo "WANDB_ENTITY=${WANDB_ENTITY:-default}" >> ${HOME}/.Renviron
+
+# Define an array of environment variable names from the ENV_VARS Compose variable
+ARG ENV_VARS=ENV_VARS
+RUN echo "---> Setup variables" && \
+    IFS=',' read -ra NAMES <<< "ENV_VARS" && \
+    for ENV_VAR_NAsME in "${NAMES[@]}"; do \ 
+    #echo "$ENV_VARS" | tr ',' '\n' | while read ENV_VAR_NAME; do \
+    ENV_VAR_VALUE="${!ENV_VAR_NAME:-default}"; \
+    echo "$ENV_VAR_NAME=$ENV_VAR_VALUE"; \
+    done > ${HOME}/.Renviron
+
+
+RUN echo "--> Go!"
+
+CMD ["R", "--quiet", "-e", "shiny::runApp(host='0.0.0.0', port=${RSTUDIO_PORT:-3838}, launch.browser = FALSE, appDir = '/home/${USER_NAME}/app')"]
+
+# CMD ["/init"]

local_build_base_image.sh

@@ -0,0 +1,13 @@
+# Inicializa un array vacío
+args=()
+
+# Lee el archivo .env línea por línea
+while IFS='=' read -r key value; do
+    if [[ $key != \#* && $key != '' ]]; then  # Excluye comentarios y líneas vacías
+        args+=(--build-arg "$key=$value")  # Agrega --build-arg y la variable como un elemento
+    fi
+done < .env
+
+# Ejecuta docker build con los argumentos
+IMAGE_NAME='dvats-rstudio-base:latest'
+docker build "${args[@]}" . -f Dockerfile.base -t ${IMAGE_NAME}

settings.ini

@@ -0,0 +1,26 @@
+[DEFAULT]
+lib_name = dvats
+user = vrodriguezf
+description = Deep VATS
+keywords = time-series machine-learning visual-analytics
+author = Victor Rodriguez-Fernandez
+author_email = [email protected]
+copyright = Universidad Politécnica de Madrid
+branch = master
+version = 0.0.1
+min_python = 3.6
+audience = Developers
+language = English
+custom_sidebar = False
+license = apache2
+status = 2
+#requirements = tsai
+nbs_path = nbs
+doc_path = docs
+doc_host = https://vrodriguezf.github.io
+doc_baseurl = /dvats/
+git_url = https://github.com/vrodriguezf/deepvats
+lib_path = dvats
+title = deepvats
+host = github
+

setup.py

@@ -0,0 +1,46 @@
+from pkg_resources import parse_version
+from configparser import ConfigParser
+import setuptools
+assert parse_version(setuptools.__version__)>=parse_version('36.2')
+
+# note: all settings are in settings.ini; edit there, not here
+config = ConfigParser(delimiters=['='])
+config.read('settings.ini')
+cfg = config['DEFAULT']
+
+cfg_keys = 'version description keywords author author_email'.split()
+expected = cfg_keys + "lib_name user branch license status min_python audience language".split()
+for o in expected: assert o in cfg, "missing expected setting: {}".format(o)
+setup_cfg = {o:cfg[o] for o in cfg_keys}
+
+licenses = {
+    'apache2': ('Apache Software License 2.0','OSI Approved :: Apache Software License'),
+}
+statuses = [ '1 - Planning', '2 - Pre-Alpha', '3 - Alpha',
+    '4 - Beta', '5 - Production/Stable', '6 - Mature', '7 - Inactive' ]
+py_versions = '2.0 2.1 2.2 2.3 2.4 2.5 2.6 2.7 3.0 3.1 3.2 3.3 3.4 3.5 3.6 3.7 3.8'.split()
+
+requirements = cfg.get('requirements','').split()
+lic = licenses[cfg['license']]
+min_python = cfg['min_python']
+
+setuptools.setup(
+    name = cfg['lib_name'],
+    license = lic[0],
+    classifiers = [
+        'Development Status :: ' + statuses[int(cfg['status'])],
+        'Intended Audience :: ' + cfg['audience'].title(),
+        'License :: ' + lic[1],
+        'Natural Language :: ' + cfg['language'].title(),
+    ] + ['Programming Language :: Python :: '+o for o in py_versions[py_versions.index(min_python):]],
+    url = 'https://github.com/{}/{}'.format(cfg['user'],cfg['lib_name']),
+    packages = setuptools.find_packages(),
+    include_package_data = True,
+    install_requires = requirements,
+    python_requires  = '>=' + cfg['min_python'],
+    long_description = open('README.md').read(),
+    long_description_content_type = 'text/markdown',
+    zip_safe = False,
+    entry_points = { 'console_scripts': cfg.get('console_scripts','').split() },
+    **setup_cfg)
+