import os
import pathlib
import shutil
import sys
import tarfile
import tempfile
import timeit
from datetime import datetime
from logging import Logger
from typing import Dict, Optional, List, Set, Tuple

import docker
from docker.errors import NotFound
from docker.models.containers import Container

from cincan.command_log import FileLog, read_with_hash
from cincan.file_tool import FileMatcher

IGNORE_FILENAME = ".cincanignore"
COMMENT_CHAR = "#"


class TarTool:
    def __init__(self, logger: Logger, container: Container, upload_stats: Dict[str, List],
                 explicit_file: Optional[str] = None):
        self.logger = logger
        self.container = container
        self.upload_stats = upload_stats
        self.explicit_file = explicit_file
        self.time_format_seconds = "%Y-%m-%dT%H:%M:%S"

        self.work_dir: str = container.image.attrs['Config'].get('WorkingDir', '.') or '/'
        if not self.work_dir.endswith('/'):
            self.work_dir += '/'

    def upload(self, upload_files: Dict[pathlib.Path, str], in_files: List[FileLog]):
        if not self.explicit_file and not upload_files:
            return  # nothing to upload

        if self.explicit_file == '-':
            # tar from stdin
            tar_file = tempfile.TemporaryFile()
            try:
                shutil.copyfileobj(sys.stdin.buffer, tar_file)
                tar_file.seek(0)
                self.__list_members(tar_file, in_files)
                tar_file.seek(0)
                self.__put_archive(tar_file)
            finally:
                tar_file.close()  # should be deleted by close
        elif self.explicit_file:
            # tar file provided, use it as-it-is
            explicit_path = pathlib.Path(self.explicit_file)
            with explicit_path.open("rb") as tar_file:
                self.__list_members(tar_file, in_files)
            with explicit_path.open("rb") as tar_file:
                self.__put_archive(tar_file)
        else:
            # collect a tar file, and upload it
            tar_file = self.__create_tar(upload_files, in_files)
            try:
                self.__put_archive(tar_file)
            finally:
                tar_file.close()

    def __put_archive(self, tar_content):
        put_arc_start = timeit.default_timer()
        self.container.put_archive(path=self.work_dir, data=tar_content)
        self.logger.debug("put_archive time %.4f s", timeit.default_timer() - put_arc_start)

    def __list_members(self, tar_content, in_files: List[FileLog]):
        with tarfile.open(fileobj=tar_content, mode="r") as tar:
            for m in tar.getmembers():
                self.logger.debug(f"checking in file {m.name}")
                # file size, modification time, upload time
                self.upload_stats[m.name] = [m.size, m.mtime, datetime.now().timestamp()]
                if not m.isfile():
                    continue
                m_fileobj = tar.extractfile(m)
                m_file = pathlib.Path(m.name)
                m_md = read_with_hash(m_fileobj.read)
                in_files.append(
                    FileLog(m_file.resolve(), m_md, datetime.fromtimestamp(m.mtime)))

    def __create_tar(self, upload_files: Dict[pathlib.Path, str], in_files: List[FileLog]):
        file_out = tempfile.TemporaryFile()
        tar = tarfile.open(mode="w", fileobj=file_out)

        # need to have all directories explicitly, otherwise seen them to be created with root
        # permissions without write possibility for the user
        dirs = set()

        for host_file, arc_name in upload_files.items():
            self.logger.info("<= %s", host_file.as_posix())

            # create all directories leading to the file, unless already added
            a_parent = pathlib.Path(arc_name).parent
            while a_parent and a_parent.as_posix() != '.':
                if a_parent not in dirs:
                    dirs.add(a_parent)
                    tar.addfile(self.__new_directory(a_parent.as_posix()))
                a_parent = a_parent.parent

            if not host_file.exists():
                # no host file, must be explicitly added directory for output
                tar.addfile(self.__new_directory(arc_name))
            else:
                tar_file = tar.gettarinfo(host_file, arcname=arc_name)
                tar_file.mode = 511  # 777 - allow all to access (uid may be different in container)
                # file size, modification time, upload time
                self.upload_stats[arc_name] = [tar_file.size, tar_file.mtime, datetime.now().timestamp()]
                if host_file.is_file():
                    # put file to tar
                    with host_file.open("rb") as f:
                        tar.addfile(tar_file, fileobj=f)
                    # create log entry
                    with host_file.open("rb") as f:
                        file_md = read_with_hash(f.read)
                    in_files.append(
                        FileLog(host_file.resolve(), file_md, datetime.fromtimestamp(host_file.stat().st_mtime)))
                elif host_file.is_dir():
                    # add directory to tar
                    tar.addfile(tar_file)
                else:
                    raise Exception(f"Cannot upload file of unknown type {arc_name}")
        file_out.seek(0)
        return file_out

    @classmethod
    def __new_directory(cls, name: str) -> tarfile.TarInfo:
        p_file = tarfile.TarInfo(name)
        p_file.type = tarfile.DIRTYPE
        p_file.uid = os.getuid()
        p_file.gid = os.getgid()
        p_file.mode = 511  # 777 - allow all to access (uid may be different in container)
        return p_file

    def __read_config_file(self, filepath: pathlib.Path, skip_comment: bool = False) -> List[str]:
        """Read configuration file.
        Returns a list of strings, where one index represents single line of the file.
        """

        file_lines = []
        try:
            chunks, stat = self.container.get_archive(str(filepath))
            tmp_tar = tempfile.TemporaryFile()
            # Write all chunks to construct tar
            for chunk in chunks:
                tmp_tar.write(chunk)
            tmp_tar.seek(0)
            open_tmp_tar = tarfile.open(fileobj=tmp_tar)
            # Extract ignorefile to fileobject
            f = open_tmp_tar.extractfile(filepath.name)
            if skip_comment:
                file_lines = list(filter(None, [line.decode("utf-8") for line in f.read().splitlines() if
                                                not line.decode("utf-8").lstrip().startswith(COMMENT_CHAR)]))
            else:
                file_lines = list(filter(None, [line.decode("utf-8") for line in f.read().splitlines()]))
            tmp_tar.close()
            f.close()
        except NotFound as e:
            self.logger.debug(
                f"Excepted {filepath.name} file not found from path '{filepath}'.")
            self.logger.debug(e)
        return file_lines

    def download_files(self, filters: List[FileMatcher] = None, no_defaults: bool = False,
                       file_paths: List[str] = None, implicit_output=True) -> List[FileLog]:
        """Download modified files, filtered as required"""
        # check all modified (includes the ones we uploaded)
        candidates = sorted(
            [d['Path'] for d in filter(lambda f: 'Path' in f, self.container.diff() or [])], reverse=True)
        # note: candidates start with / as path container absolute
        candidates = self.__filter_files(candidates, filters, no_defaults)

        # write to a tar?
        explicit_file = None
        try:
            if self.explicit_file == '-':
                # write tar to stdout
                explicit_file = tarfile.open(mode="w|", fileobj=sys.stdout.buffer)
            elif self.explicit_file:
                # write tar file
                explicit_file = tarfile.open(self.explicit_file, "w")

            files_to_do = set(candidates)
            out_files = []

            if implicit_output and self.work_dir != '/':
                # container has non-root working directory, get it at once!
                self.logger.debug("%d files to download, looking from %s...", len(files_to_do), self.work_dir)
                log = self.__download_file_set(self.work_dir, files_to_do, write_to=explicit_file)
                out_files.extend(log)

            # explicit result directories
            for fp in file_paths or []:
                if not files_to_do:
                    continue
                fp_in_cont = (pathlib.Path(self.work_dir) / fp).as_posix()
                self.logger.debug("%d files to download, looking from %s...", len(files_to_do), fp_in_cont)
                log = self.__download_file_set(fp_in_cont, files_to_do, write_to=explicit_file)
                out_files.extend(log)

            if implicit_output and files_to_do:
                # go for each missing file individually
                self.logger.debug("%d files to download, fetching each individually", len(files_to_do))
                files_to_load = sorted(files_to_do)
                for f in files_to_load:
                    # separately download and possibly copy remaining result files
                    log = self.__download_file_set(f, files_to_do, write_to=explicit_file)
                    out_files.extend(log)

            if files_to_do:
                self.logger.debug("%d files in diff not downloaded:", len(files_to_do))
                for f in sorted(files_to_do):
                    self.logger.debug("  %s", f)
            return out_files
        finally:
            explicit_file and explicit_file.close()

    def __filter_files(self, candidates: List[str], filters: List[FileMatcher] = None,
                       no_defaults: bool = False) -> List[str]:
        """Filter list of candidate files to download"""
        # Sort by excluding and including filters
        # Including filter has more power than excluding one!
        output_filters_to_exclude = filters.copy() if filters else []
        output_filters_to_include = [output_filters_to_exclude.pop(i) for i, f in
                                     enumerate(output_filters_to_exclude)
                                     if f.include] if filters else []

        # Check if container has .cincanignore file - these are not downloaded by default
        ignore_file = pathlib.Path(self.work_dir) / IGNORE_FILENAME
        ignore_paths = self.__read_config_file(ignore_file, skip_comment=True)
        # Ignore the ignorefile itself..
        ignore_paths.append(IGNORE_FILENAME)
        # remove files which are paths to files
        candidates = candidates.copy()
        skip_set = set()
        for i, c in enumerate(candidates):
            if c in skip_set:
                candidates[i] = None
                continue
            c_parent = pathlib.Path(c).parent
            while c_parent and c_parent.name:
                skip_set.add(c_parent.as_posix())
                c_parent = c_parent.parent
        candidates = list(filter(lambda s: s, candidates))
        # remove candidates which are not in working directory
        candidates = list(filter(lambda s: s.startswith(self.work_dir), candidates))
        # nicely sorted
        candidates.sort()
        # filters?
        ignore_filters = []
        if ignore_paths:
            for file in ignore_paths:
                if file.endswith("/"):
                    file = file + "*"
                    ignore_filters.append(FileMatcher(file, include=False))
                    continue
                if not file.endswith("*"):
                    ignore_filters.append(FileMatcher(file, include=False))
                    ignore_filters.append(FileMatcher(file + "/*", include=False))
                    continue
                ignore_filters.append(FileMatcher(file, include=False))

        # If user has not defined output_filters, use .cincanignore from container if not set to be ignored
        if not filters and ignore_paths and not no_defaults:
            self.logger.debug("No user provided output filters - using .cincanignore")
            for filth in ignore_filters or []:
                candidates = filth.filter_download_files(candidates, self.work_dir)

        elif filters and ignore_paths:

            # Check if user has defined to not use container specific output filters
            if no_defaults:
                for filth in filters or []:
                    candidates = filth.filter_download_files(candidates, self.work_dir)
            elif output_filters_to_include:
                # If we have some including filters, only those are applied
                for filth in output_filters_to_include:
                    candidates = filth.filter_download_files(candidates, self.work_dir)
            else:
                # Merge excluding/ignoring filters, no duplicates
                combined_excluding_filters = set(output_filters_to_exclude)
                if not output_filters_to_exclude:
                    combined_excluding_filters = ignore_filters
                else:
                    for i_f in ignore_filters:
                        for o_f in output_filters_to_exclude:
                            if i_f.match_string == o_f.match_string:
                                break
                            else:
                                combined_excluding_filters.add(i_f)
                for filth in combined_excluding_filters or []:
                    candidates = filth.filter_download_files(candidates, self.work_dir)

        else:
            for filth in filters or []:
                # remove non-matching files
                candidates = filth.filter_download_files(candidates, self.work_dir)
        return candidates

    def __download_file_set(self, file_path: str, files: Set[str],
                            write_to: Optional[tarfile.TarFile] = None) -> List[FileLog]:
        """Download files by a path, copy matching files into host"""
        base_path = pathlib.Path(file_path)

        # fetch the path from container in its own tar ball
        get_arc_start = timeit.default_timer()
        try:
            chunks, _ = self.container.get_archive(file_path)
        except docker.errors.NotFound:
            self.logger.debug("Not found in container: %s", file_path)
            return []

        # read the tarball into temp file
        tmp_tar = tempfile.TemporaryFile()
        for c in chunks:
            tmp_tar.write(c)
        self.logger.debug("get_archive %s time %.4f s", file_path, timeit.default_timer() - get_arc_start)

        tmp_tar.seek(0)
        down_tar = tarfile.open(fileobj=tmp_tar, mode="r|")
        out_files = []
        for tar_file in down_tar:
            file_in_cont = (base_path.parent or base_path) / tar_file.name
            cont_full_name = file_in_cont.as_posix()
            if cont_full_name not in files:
                continue  # not interested in this
            files.remove(cont_full_name)

            if cont_full_name.startswith(self.work_dir):
                file_in_host = pathlib.Path(cont_full_name[len(self.work_dir):])
            elif cont_full_name.startswith('/'):
                file_in_host = pathlib.Path(cont_full_name[1:])
            else:
                self.logger.warning("skipping file %s", cont_full_name)
                continue
            if file_in_host.is_absolute():
                self.logger.warning("skipping suspicious file %s", cont_full_name)
                continue
            modified, unmodified = self.__check_if_modified(file_in_host, tar_file)
            if unmodified:
                continue  # no need to download

            md = ''
            timestamp = datetime.now()
            if write_to:
                # write file to tar, calculate hash
                with tempfile.TemporaryFile() as temp_file:
                    tf_data = down_tar.extractfile(tar_file)
                    md = read_with_hash(tf_data.read, temp_file.write)

                    write_tf = tarfile.TarInfo(file_in_host.as_posix())
                    write_tf.mtime = tar_file.mtime
                    write_tf.mode = tar_file.mode
                    write_tf.size = temp_file.tell()
                    temp_file.seek(0)
                    write_to.addfile(write_tf, fileobj=temp_file)
            elif tar_file.isfile():
                # this is a file we were looking for
                if not file_in_host.exists():
                    # no local file or explicit output asked, this is too easy
                    self.logger.info(f"=> {file_in_host.as_posix()}")
                    tf_data = down_tar.extractfile(tar_file)
                    if file_in_host.parent:
                        file_in_host.parent.mkdir(parents=True, exist_ok=True)
                    with file_in_host.open("wb") as f:
                        md = read_with_hash(tf_data.read, f.write)
                else:
                    # compare by hash, if should override local file
                    tf_data = down_tar.extractfile(tar_file)
                    temp_file = pathlib.Path(file_in_host.as_posix() + '_TEMP')
                    self.logger.debug(f"creating temp file {temp_file.as_posix()}")

                    # calculate hash for file from tar, copy it to temp file
                    if file_in_host.parent:
                        file_in_host.parent.mkdir(parents=True, exist_ok=True)
                    with temp_file.open("wb") as f:
                        md = read_with_hash(tf_data.read, f.write)

                    self.logger.info(f"=> {file_in_host.as_posix()}")
                    if modified:
                        # modified by timestamp or size, overwrite
                        file_in_host.unlink()
                        temp_file.rename(file_in_host)
                    else:
                        # not sure if modified, calculate hash for existing file
                        with file_in_host.open("rb") as f:
                            host_digest = read_with_hash(f.read)

                        if md == host_digest:
                            self.logger.debug(f"identical file {file_in_host.as_posix()} digest {md}, no action")
                            # The file is identical as uploaded
                            # Assuming it was created identical, adding the log entry
                            temp_file.unlink()
                        else:
                            # digest changed, modified despite identical size and timestamps
                            self.logger.debug(
                                f"file {file_in_host.as_posix()} digest in container {md}, in host {host_digest}")
                            file_in_host.unlink()
                            temp_file.rename(file_in_host)
            else:
                if not file_in_host.exists():
                    # must be a directory we need
                    self.logger.info(f"=> {file_in_host.as_posix()}/")
                    file_in_host.mkdir(parents=True, exist_ok=True)
                    timestamp = datetime.fromtimestamp(file_in_host.stat().st_mtime)
                else:
                    pass  # no action required
            out_files.append(FileLog(file_in_host.resolve(), md, timestamp))
        tmp_tar.close()
        return out_files

    def __check_if_modified(self, host_file: pathlib.Path, file_info: tarfile.TarInfo) -> Tuple[bool, bool]:
        """Check if file has been modified, or not modified in the container"""
        up_stat = self.upload_stats.get(host_file.as_posix())
        if up_stat is None:
            return True, False  # not uploaded, must be downloaded

        # original size, upload time, original creation time
        orig_size, orig_time, up_time = up_stat

        # NOTE: for directories (?) size from container is 4096 -> mismatch for directories!!!
        down_size = file_info.size
        if orig_size != down_size:
            self.logger.debug(f"size {host_file.as_posix()} change {orig_size} -> {down_size}")
            return True, False

        # remove fractions for comparison
        down_time_s = int(file_info.mtime)
        orig_time_s = int(orig_time)
        up_time_s = int(up_time)
        if orig_time_s != down_time_s:
            self.logger.debug(f"{host_file.as_posix()} mtime updated at container")
            return True, False
        if orig_time_s != up_time_s:
            self.logger.debug(f"{host_file.as_posix()} old and not modified")
            return False, True
        self.logger.debug(f"{host_file.as_posix()} timestamp nor size tells if modified")
        return False, False

import os
import pathlib
import shutil
import sys
import tarfile
import tempfile
import timeit
from datetime import datetime
from logging import Logger
from typing import Dict, Optional, List, Set, Tuple

import docker
from docker.errors import NotFound
from docker.models.containers import Container

from cincan.command_log import FileLog, read_with_hash
from cincan.file_tool import FileMatcher

IGNORE_FILENAME = ".cincanignore"
COMMENT_CHAR = "#"


class TarTool:
    def __init__(self, logger: Logger, container: Container, upload_stats: Dict[str, List],
                 explicit_file: Optional[str] = None):
        self.logger = logger
        self.container = container
        self.upload_stats = upload_stats
        self.explicit_file = explicit_file
        self.time_format_seconds = "%Y-%m-%dT%H:%M:%S"

        self.work_dir: str = container.image.attrs['Config'].get('WorkingDir', '.') or '/'
        if not self.work_dir.endswith('/'):
            self.work_dir += '/'

    def upload(self, upload_files: Dict[pathlib.Path, str], in_files: List[FileLog]):
        if not self.explicit_file and not upload_files:
            return  # nothing to upload

        if self.explicit_file == '-':
            # tar from stdin
            tar_file = tempfile.TemporaryFile()
            try:
                shutil.copyfileobj(sys.stdin.buffer, tar_file)
                tar_file.seek(0)
                self.__list_members(tar_file, in_files)
                tar_file.seek(0)
                self.__put_archive(tar_file)
            finally:
                tar_file.close()  # should be deleted by close
        elif self.explicit_file:
            # tar file provided, use it as-it-is
            explicit_path = pathlib.Path(self.explicit_file)
            with explicit_path.open("rb") as tar_file:
                self.__list_members(tar_file, in_files)
            with explicit_path.open("rb") as tar_file:
                self.__put_archive(tar_file)
        else:
            # collect a tar file, and upload it
            tar_file = self.__create_tar(upload_files, in_files)
            try:
                self.__put_archive(tar_file)
            finally:
                tar_file.close()

    def __put_archive(self, tar_content):
        put_arc_start = timeit.default_timer()
        self.container.put_archive(path=self.work_dir, data=tar_content)
        self.logger.debug("put_archive time %.4f s", timeit.default_timer() - put_arc_start)

    def __list_members(self, tar_content, in_files: List[FileLog]):
        with tarfile.open(fileobj=tar_content, mode="r") as tar:
            for m in tar.getmembers():
                self.logger.debug(f"checking in file {m.name}")
                # file size, modification time, upload time
                self.upload_stats[m.name] = [m.size, m.mtime, datetime.now().timestamp()]
                if not m.isfile():
                    continue
                m_fileobj = tar.extractfile(m)
                m_file = pathlib.Path(m.name)
                m_md = read_with_hash(m_fileobj.read)
                in_files.append(
                    FileLog(m_file.resolve(), m_md, datetime.fromtimestamp(m.mtime)))

    def __create_tar(self, upload_files: Dict[pathlib.Path, str], in_files: List[FileLog]):
        file_out = tempfile.TemporaryFile()
        tar = tarfile.open(mode="w", fileobj=file_out)

        # need to have all directories explicitly, otherwise seen them to be created with root
        # permissions without write possibility for the user
        dirs = set()

        for host_file, arc_name in upload_files.items():
            self.logger.info("<= %s", host_file.as_posix())

            # create all directories leading to the file, unless already added
            a_parent = pathlib.Path(arc_name).parent
            while a_parent and a_parent.as_posix() != '.':
                if a_parent not in dirs:
                    dirs.add(a_parent)
                    tar.addfile(self.__new_directory(a_parent.as_posix()))
                a_parent = a_parent.parent

            if not host_file.exists():
                # no host file, must be explicitly added directory for output
                tar.addfile(self.__new_directory(arc_name))
            else:
                tar_file = tar.gettarinfo(host_file, arcname=arc_name)
                tar_file.mode = 511  # 777 - allow all to access (uid may be different in container)
                # file size, modification time, upload time
                self.upload_stats[arc_name] = [tar_file.size, tar_file.mtime, datetime.now().timestamp()]
                if host_file.is_file():
                    # put file to tar
                    with host_file.open("rb") as f:
                        tar.addfile(tar_file, fileobj=f)
                    # create log entry
                    with host_file.open("rb") as f:
                        file_md = read_with_hash(f.read)
                    in_files.append(
                        FileLog(host_file.resolve(), file_md, datetime.fromtimestamp(host_file.stat().st_mtime)))
                elif host_file.is_dir():
                    # add directory to tar
                    tar.addfile(tar_file)
                else:
                    raise Exception(f"Cannot upload file of unknown type {arc_name}")
        file_out.seek(0)
        return file_out

    @classmethod
    def __new_directory(cls, name: str) -> tarfile.TarInfo:
        p_file = tarfile.TarInfo(name)
        p_file.type = tarfile.DIRTYPE
        p_file.uid = os.getuid()
        p_file.gid = os.getgid()
        p_file.mode = 511  # 777 - allow all to access (uid may be different in container)
        return p_file

    def __read_config_file(self, filepath: pathlib.Path, skip_comment: bool = False) -> List[str]:
        """Read configuration file.
        Returns a list of strings, where one index represents single line of the file.
        """

        file_lines = []
        try:
            chunks, stat = self.container.get_archive(str(filepath))
            tmp_tar = tempfile.TemporaryFile()
            # Write all chunks to construct tar
            for chunk in chunks:
                tmp_tar.write(chunk)
            tmp_tar.seek(0)
            open_tmp_tar = tarfile.open(fileobj=tmp_tar)
            # Extract ignorefile to fileobject
            f = open_tmp_tar.extractfile(filepath.name)
            if skip_comment:
                file_lines = list(filter(None, [line.decode("utf-8") for line in f.read().splitlines() if
                                                not line.decode("utf-8").lstrip().startswith(COMMENT_CHAR)]))
            else:
                file_lines = list(filter(None, [line.decode("utf-8") for line in f.read().splitlines()]))
            tmp_tar.close()
            f.close()
        except NotFound as e:
            self.logger.debug(
                f"Excepted {filepath.name} file not found from path '{filepath}'.")
            self.logger.debug(e)
        return file_lines

    def download_files(self, filters: List[FileMatcher] = None, no_defaults: bool = False,
                       file_paths: List[str] = None, implicit_output=True) -> List[FileLog]:
        """Download modified files, filtered as required"""
        # check all modified (includes the ones we uploaded)
        candidates = sorted(
            [d['Path'] for d in filter(lambda f: 'Path' in f, self.container.diff() or [])], reverse=True)
        # note: candidates start with / as path container absolute
        candidates = self.__filter_files(candidates, filters, no_defaults)

        # write to a tar?
        explicit_file = None
        try:
            if self.explicit_file == '-':
                # write tar to stdout
                explicit_file = tarfile.open(mode="w|", fileobj=sys.stdout.buffer)
            elif self.explicit_file:
                # write tar file
                explicit_file = tarfile.open(self.explicit_file, "w")

            files_to_do = set(candidates)
            out_files = []

            if implicit_output and self.work_dir != '/':
                # container has non-root working directory, get it at once!
                self.logger.debug("%d files to download, looking from %s...", len(files_to_do), self.work_dir)
                log = self.__download_file_set(self.work_dir, files_to_do, write_to=explicit_file)
                out_files.extend(log)

            # explicit result directories
            for fp in file_paths or []:
                if not files_to_do:
                    continue
                fp_in_cont = (pathlib.Path(self.work_dir) / fp).as_posix()
                self.logger.debug("%d files to download, looking from %s...", len(files_to_do), fp_in_cont)
                log = self.__download_file_set(fp_in_cont, files_to_do, write_to=explicit_file)
                out_files.extend(log)

            if implicit_output and files_to_do:
                # go for each missing file individually
                self.logger.debug("%d files to download, fetching each individually", len(files_to_do))
                files_to_load = sorted(files_to_do)
                for f in files_to_load:
                    # separately download and possibly copy remaining result files
                    log = self.__download_file_set(f, files_to_do, write_to=explicit_file)
                    out_files.extend(log)

            if files_to_do:
                self.logger.debug("%d files in diff not downloaded:", len(files_to_do))
                for f in sorted(files_to_do):
                    self.logger.debug("  %s", f)
            return out_files
        finally:
            explicit_file and explicit_file.close()

    def __filter_files(self, candidates: List[str], filters: List[FileMatcher] = None,
                       no_defaults: bool = False) -> List[str]:
        """Filter list of candidate files to download"""
        # Sort by excluding and including filters
        # Including filter has more power than excluding one!
        output_filters_to_exclude = filters.copy() if filters else []
        output_filters_to_include = [output_filters_to_exclude.pop(i) for i, f in
                                     enumerate(output_filters_to_exclude)
                                     if f.include] if filters else []

        # Check if container has .cincanignore file - these are not downloaded by default
        ignore_file = pathlib.Path(self.work_dir) / IGNORE_FILENAME
        ignore_paths = self.__read_config_file(ignore_file, skip_comment=True)
        # Ignore the ignorefile itself..
        ignore_paths.append(IGNORE_FILENAME)
        # remove files which are paths to files
        candidates = candidates.copy()
        skip_set = set()
        for i, c in enumerate(candidates):
            if c in skip_set:
                candidates[i] = None
                continue
            c_parent = pathlib.Path(c).parent
            while c_parent and c_parent.name:
                skip_set.add(c_parent.as_posix())
                c_parent = c_parent.parent
        candidates = list(filter(lambda s: s, candidates))
        # remove candidates which are not in working directory
        candidates = list(filter(lambda s: s.startswith(self.work_dir), candidates))
        # nicely sorted
        candidates.sort()
        # filters?
        ignore_filters = []
        if ignore_paths:
            for file in ignore_paths:
                if file.endswith("/"):
                    file = file + "*"
                    ignore_filters.append(FileMatcher(file, include=False))
                    continue
                if not file.endswith("*"):
                    ignore_filters.append(FileMatcher(file, include=False))
                    ignore_filters.append(FileMatcher(file + "/*", include=False))
                    continue
                ignore_filters.append(FileMatcher(file, include=False))

        # If user has not defined output_filters, use .cincanignore from container if not set to be ignored
        if not filters and ignore_paths and not no_defaults:
            self.logger.debug("No user provided output filters - using .cincanignore")
            for filth in ignore_filters or []:
                candidates = filth.filter_download_files(candidates, self.work_dir)

        elif filters and ignore_paths:

            # Check if user has defined to not use container specific output filters
            if no_defaults:
                for filth in filters or []:
                    candidates = filth.filter_download_files(candidates, self.work_dir)
            elif output_filters_to_include:
                # If we have some including filters, only those are applied
                for filth in output_filters_to_include:
                    candidates = filth.filter_download_files(candidates, self.work_dir)
            else:
                # Merge excluding/ignoring filters, no duplicates
                combined_excluding_filters = set(output_filters_to_exclude)
                if not output_filters_to_exclude:
                    combined_excluding_filters = ignore_filters
                else:
                    for i_f in ignore_filters:
                        for o_f in output_filters_to_exclude:
                            if i_f.match_string == o_f.match_string:
                                break
                            else:
                                combined_excluding_filters.add(i_f)
                for filth in combined_excluding_filters or []:
                    candidates = filth.filter_download_files(candidates, self.work_dir)

        else:
            for filth in filters or []:
                # remove non-matching files
                candidates = filth.filter_download_files(candidates, self.work_dir)
        return candidates

    def __download_file_set(self, file_path: str, files: Set[str],
                            write_to: Optional[tarfile.TarFile] = None) -> List[FileLog]:
        """Download files by a path, copy matching files into host"""
        base_path = pathlib.Path(file_path)

        # fetch the path from container in its own tar ball
        get_arc_start = timeit.default_timer()
        try:
            chunks, _ = self.container.get_archive(file_path)
        except docker.errors.NotFound:
            self.logger.debug("Not found in container: %s", file_path)
            return []

        # read the tarball into temp file
        tmp_tar = tempfile.TemporaryFile()
        for c in chunks:
            tmp_tar.write(c)
        self.logger.debug("get_archive %s time %.4f s", file_path, timeit.default_timer() - get_arc_start)

        tmp_tar.seek(0)
        down_tar = tarfile.open(fileobj=tmp_tar, mode="r|")
        out_files = []
        for tar_file in down_tar:
            file_in_cont = (base_path.parent or base_path) / tar_file.name
            cont_full_name = file_in_cont.as_posix()
            if cont_full_name not in files:
                continue  # not interested in this
            files.remove(cont_full_name)

            if cont_full_name.startswith(self.work_dir):
                file_in_host = pathlib.Path(cont_full_name[len(self.work_dir):])
            elif cont_full_name.startswith('/'):
                file_in_host = pathlib.Path(cont_full_name[1:])
            else:
                self.logger.warning("skipping file %s", cont_full_name)
                continue
            if file_in_host.is_absolute():
                self.logger.warning("skipping suspicious file %s", cont_full_name)
                continue
            modified, unmodified = self.__check_if_modified(file_in_host, tar_file)
            if unmodified:
                continue  # no need to download

            md = ''
            timestamp = datetime.now()
            if write_to:
                # write file to tar, calculate hash
                with tempfile.TemporaryFile() as temp_file:
                    tf_data = down_tar.extractfile(tar_file)
                    md = read_with_hash(tf_data.read, temp_file.write)

                    write_tf = tarfile.TarInfo(file_in_host.as_posix())
                    write_tf.mtime = tar_file.mtime
                    write_tf.mode = tar_file.mode
                    write_tf.size = temp_file.tell()
                    temp_file.seek(0)
                    write_to.addfile(write_tf, fileobj=temp_file)
            elif tar_file.isfile():
                # this is a file we were looking for
                if not file_in_host.exists():
                    # no local file or explicit output asked, this is too easy
                    self.logger.info(f"=> {file_in_host.as_posix()}")
                    tf_data = down_tar.extractfile(tar_file)
                    if file_in_host.parent:
                        file_in_host.parent.mkdir(parents=True, exist_ok=True)
                    with file_in_host.open("wb") as f:
                        md = read_with_hash(tf_data.read, f.write)
                else:
                    # compare by hash, if should override local file
                    tf_data = down_tar.extractfile(tar_file)
                    temp_file = pathlib.Path(file_in_host.as_posix() + '_TEMP')
                    self.logger.debug(f"creating temp file {temp_file.as_posix()}")

                    # calculate hash for file from tar, copy it to temp file
                    if file_in_host.parent:
                        file_in_host.parent.mkdir(parents=True, exist_ok=True)
                    with temp_file.open("wb") as f:
                        md = read_with_hash(tf_data.read, f.write)

                    self.logger.info(f"=> {file_in_host.as_posix()}")
                    if modified:
                        # modified by timestamp or size, overwrite
                        file_in_host.unlink()
                        temp_file.rename(file_in_host)
                    else:
                        # not sure if modified, calculate hash for existing file
                        with file_in_host.open("rb") as f:
                            host_digest = read_with_hash(f.read)

                        if md == host_digest:
                            self.logger.debug(f"identical file {file_in_host.as_posix()} digest {md}, no action")
                            # The file is identical as uploaded
                            # Assuming it was created identical, adding the log entry
                            temp_file.unlink()
                        else:
                            # digest changed, modified despite identical size and timestamps
                            self.logger.debug(
                                f"file {file_in_host.as_posix()} digest in container {md}, in host {host_digest}")
                            file_in_host.unlink()
                            temp_file.rename(file_in_host)
            else:
                if not file_in_host.exists():
                    # must be a directory we need
                    self.logger.info(f"=> {file_in_host.as_posix()}/")
                    file_in_host.mkdir(parents=True, exist_ok=True)
                    timestamp = datetime.fromtimestamp(file_in_host.stat().st_mtime)
                else:
                    pass  # no action required
            out_files.append(FileLog(file_in_host.resolve(), md, timestamp))
        tmp_tar.close()
        return out_files

    def __check_if_modified(self, host_file: pathlib.Path, file_info: tarfile.TarInfo) -> Tuple[bool, bool]:
        """Check if file has been modified, or not modified in the container"""
        up_stat = self.upload_stats.get(host_file.as_posix())
        if up_stat is None:
            return True, False  # not uploaded, must be downloaded

        # original size, upload time, original creation time
        orig_size, orig_time, up_time = up_stat

        # NOTE: for directories (?) size from container is 4096 -> mismatch for directories!!!
        down_size = file_info.size
        if orig_size != down_size:
            self.logger.debug(f"size {host_file.as_posix()} change {orig_size} -> {down_size}")
            return True, False

        # remove fractions for comparison
        down_time_s = int(file_info.mtime)
        orig_time_s = int(orig_time)
        up_time_s = int(up_time)
        if orig_time_s != down_time_s:
            self.logger.debug(f"{host_file.as_posix()} mtime updated at container")
            return True, False
        if orig_time_s != up_time_s:
            self.logger.debug(f"{host_file.as_posix()} old and not modified")
            return False, True
        self.logger.debug(f"{host_file.as_posix()} timestamp nor size tells if modified")
        return False, False

# nothing to upload # tar from stdin # should be deleted by close # tar file provided, use it as-it-is # collect a tar file, and upload it # file size, modification time, upload time # need to have all directories explicitly, otherwise seen them to be created with root # permissions without write possibility for the user # create all directories leading to the file, unless already added # no host file, must be explicitly added directory for output # 777 - allow all to access (uid may be different in container) # file size, modification time, upload time # put file to tar # create log entry # add directory to tar # 777 - allow all to access (uid may be different in container) Read configuration file.         Returns a list of strings, where one index represents single line of the file. # Write all chunks to construct tar # Extract ignorefile to fileobject Download modified files, filtered as required # check all modified (includes the ones we uploaded) # note: candidates start with / as path container absolute # write to a tar? # write tar to stdout # write tar file # container has non-root working directory, get it at once! # explicit result directories # go for each missing file individually # separately download and possibly copy remaining result files Filter list of candidate files to download # Sort by excluding and including filters # Including filter has more power than excluding one! # Check if container has .cincanignore file - these are not downloaded by default # Ignore the ignorefile itself.. # remove files which are paths to files # remove candidates which are not in working directory # nicely sorted # filters? # If user has not defined output_filters, use .cincanignore from container if not set to be ignored # Check if user has defined to not use container specific output filters # If we have some including filters, only those are applied # Merge excluding/ignoring filters, no duplicates # remove non-matching files Download files by a path, copy matching files into host # fetch the path from container in its own tar ball # read the tarball into temp file # not interested in this # no need to download # write file to tar, calculate hash # this is a file we were looking for # no local file or explicit output asked, this is too easy # compare by hash, if should override local file # calculate hash for file from tar, copy it to temp file # modified by timestamp or size, overwrite # not sure if modified, calculate hash for existing file # The file is identical as uploaded # Assuming it was created identical, adding the log entry # digest changed, modified despite identical size and timestamps # must be a directory we need # no action required Check if file has been modified, or not modified in the container # not uploaded, must be downloaded # original size, upload time, original creation time # NOTE: for directories (?) size from container is 4096 -> mismatch for directories!!! # remove fractions for comparison