import re
import unidecode

from .util import isinstance_str
from .record import Record
from .link import Link, NONE_LINK
from .record_hook import RecordHook, NONE_RECORD_HOOK
from .exceptions import FieldValidationError
from .external_file import ExternalFile

spaces_and_newlines_pattern = re.compile(r"[\n\r\s]+")
not_python_var_pattern = re.compile(r"(^[^\w]+)|([^\w\d]+)")
multiple_underscores_pattern = re.compile(r"[_]{2,}")

def var_name_to_ref(name):
    ref = re.sub(not_python_var_pattern, "_", name.lower())
    return re.sub(multiple_underscores_pattern, "_", ref)

class FieldDescriptor:
    """
    No checks implemented (idd is considered as ok).
    """
    BASIC_FIELDS = ("integer", "real", "alpha", "choice", "node", "external-list")

    def __init__(self, table_descriptor, index, field_basic_type, name=None):
        self.table_descriptor = table_descriptor
        if field_basic_type not in ("A", "N"):
            raise ValueError("Unknown field type: '%s'." % field_basic_type)
        self.index = index
        self.basic_type = field_basic_type  # A -> alphanumeric, N -> numeric
        self.name = name
        self.ref = None if name is None else var_name_to_ref(name)
        self.tags = {}

        # used for error messages on extensible fields
        self._extensible_info = None
        self._detailed_type = None

    # construct
    def append_tag(self, ref, value=None):
        if ref not in self.tags:
            self.tags[ref] = []

        # manage value
        if value is None:
            return

        self.tags[ref].append(value)

    def set_extensible_info(self, cycle_start, cycle_len, cycle_pattern):
        self._extensible_info = (cycle_start, cycle_len, cycle_pattern)

    # deserialize
    def deserialize(self, value, index):
        """
        index is used for extensible fields error messages (if given)
        """
        # -- serialize if not raw type
        # transform to string if external file
        if isinstance(value, ExternalFile):
            value = value.pointer

        # transform to string if record
        if isinstance(value, Record):
            try:
                value = value[0]
            except IndexError:
                raise ValueError("can't set given record because it does not have a name field")

        # -- prepare if string
        if isinstance(value, str):
            # change multiple spaces to mono spaces
            value = re.sub(spaces_and_newlines_pattern, lambda x: " ", value.strip())

            # see if still not empty
            if value == "":
                return None

            # make ASCII compatible
            value = unidecode.unidecode(value)

            # make lower case if not retaincase
            if "retaincase" not in self.tags:
                value = value.lower()

            # check not too big
            if len(value) >= 100:
                raise FieldValidationError(
                    f"Field has more than 100 characters which is the limit. "
                    f"{self.get_error_location_message(value, index=index)}"
                )

        # transform to external file if relevant
        if self.is_file_name:
            value = ExternalFile.deserialize(value)

        # -- deserialize

        # numeric types
        if self.detailed_type in ("integer", "real"):
            # manage none
            if value is None:
                return None

            # special values: auto-calculate, auto-size, use-weather-file
            if value in ("autocalculate", "autosize", "useweatherfile"):
                return value

            if self.detailed_type == "integer":
                try:
                    return int(value)
                except:
                    raise FieldValidationError(
                        f"Couldn't parse to integer. {self.get_error_location_message(value, index=index)}"
                    )

            try:
                return float(value)
            except:
                raise FieldValidationError(
                    f"Couldn't parse to float. {self.get_error_location_message(value, index=index)}"
                )

        # simple string types
        if self.detailed_type in ("alpha", "choice", "node", "external-list"):
            # manage none
            if value is None:
                return None

            # ensure it was str
            if not isinstance_str(value):
                raise FieldValidationError(
                    f"Value must be a string. {self.get_error_location_message(value, index=index)}"
                )
            return value

        # manage hooks (eplus reference)
        if self.detailed_type == "reference":
            # manage None
            if value is None:
                return NONE_RECORD_HOOK

            # reference class name appears in v9.0.1
            references = self.tags.get("reference", [])
            # table_name, index, value, references, class_references
            return RecordHook(references, index, value)

        # manage links (eplus object-list)
        if self.detailed_type == "object-list":
            # manage None
            if value is None:
                return NONE_LINK

            return Link(self.tags["object-list"], value, index)

        raise RuntimeError("should not be here")

    # get info
    @property
    def is_required(self):
        return "required-field" in self.tags

    @property
    def is_file_name(self):
        # we don't add this to detailed_type because can be a file name and something else (for example object-list)
        return "file_name" in self.ref

    def check_not_required(self):
        if self.is_required:
            raise FieldValidationError(f"Field is required. {self.get_error_location_message()}")

    @property
    def detailed_type(self):
        """
        Uses EPlus double approach of type ('type' tag, and/or 'key', 'object-list', 'external-list', 'reference' tags)
        to determine detailed type.

        Returns
        -------
        "integer", "real", "alpha", "choice", "reference", "object-list", "external-list", "node"
        """
        if self._detailed_type is None:
            if ("reference" in self.tags) or ("reference-class-name" in self.tags):
                self._detailed_type = "reference"
            elif "type" in self.tags:
                self._detailed_type = self.tags["type"][0].lower()  # idd is not very rigorous on case
            elif "key" in self.tags:
                self._detailed_type = "choice"
            elif "object-list" in self.tags:
                self._detailed_type = "object-list"
            elif "external-list" in self.tags:
                self._detailed_type = "external-list"
            elif self.basic_type == "A":
                self._detailed_type = "alpha"
            elif self.basic_type == "N":
                self._detailed_type = "real"
            else:
                raise ValueError("Can't find detailed type.")
        return self._detailed_type

    def get_error_location_message(self, value=None, index=None):
        # manage extensible field ref if relevant
        if (index is not None) and (self._extensible_info is not None) and (index >= self._extensible_info[0]):
            cycle_num = (index - self._extensible_info[0]) // self._extensible_info[1] + 1
            ref = self._extensible_info[2].replace(r"(\d+)", str(cycle_num))
        else:
            ref = self.ref

        return f"Table: {self.table_descriptor.table_name}, index: {self.index}, ref: {ref}" +\
               ("." if value is None else f", value: {value}.")


import os

from oplus import CONF

class ExternalFilesManager:
    def __init__(self, epm):
        self._epm = epm
        self._contents = dict()  # {ref: content_str, ...}
        self._external_files = set()

    def populate_from_json_data(self, json_data):
        """
        !! Must only be called once, when empty !!
        """
        self._contents = json_data

    @property
    def short_refs(self):
        """
        we calculate on the fly to avoid managing registrations and un-registrations

        Returns
        -------
        {ref: short_ref, ...
        """
        naive_short_refs_d = dict()  # naive_short_ref: {refs, ...}
        for ef in self._external_files:
            if ef.naive_short_ref not in naive_short_refs_d:
                naive_short_refs_d[ef.naive_short_ref] = set()
            naive_short_refs_d[ef.naive_short_ref].add(ef.ref)

        short_refs = dict()
        for naive_short_ref, refs in naive_short_refs_d.items():
            if len(refs) == 1:
                short_refs[refs.pop()] = naive_short_ref
                continue
            base, ext = os.path.splitext(naive_short_ref)
            for i, ref in enumerate(sorted(refs)):
                short_refs[ref] = f"{base}-{i}.{ext}"

        return short_refs

    def get_json_data(self):
        short_refs = self.short_refs
        return dict([(short_refs[ref], content) for (ref, content) in self._contents.items()])

    def contains(self, ref):
        return ref in self._contents

    def register(self, external_file):
        # store
        self._external_files.add(external_file)

        # leave if content is already here
        if external_file.ref in self._contents:
            return

        # prepare and store content
        self._contents[external_file.ref] = external_file._dev_prepare_content()

    def unregister(self, external_file):
        self._external_files.remove(external_file)

        # see if content is still needed
        for e in self._external_files:
            if e.ref == external_file.ref:  # still needed
                return
        # not needed
        del self._contents[external_file.ref]

    def get_content(self, ref):
        return self._contents[ref]

    def get_short_ref(self, external_file):
        naive_short_ref = external_file.naive_short_ref
        refs = tuple(sorted({e.ref for e in self._external_files if e.naive_short_ref == naive_short_ref}))
        if len(refs) == 1:
            return naive_short_ref
        base, ext = os.path.splitext(naive_short_ref)
        return base + "-" + str(refs.index(external_file.ref)) + "." + ext

    def dump_external_files(self, target_dir_path=None):
        # leave if no external files
        if len(self._contents) == 0:
            return

        # prepare directory
        if not os.path.exists(target_dir_path):
            os.mkdir(target_dir_path)

        # dump files
        for ref, short_ref in self.short_refs.items():
            with open(os.path.join(target_dir_path, short_ref), "w", encoding=CONF.encoding) as f:
                f.write(self._contents.get(ref, "TO BE FILLED"))


import os

import pandas as pd

from oplus.configuration import CONF

class Eio:
    def __init__(self, path):
        if not os.path.isfile(path):
            raise FileNotFoundError("No file at given path: '%s'." % path)
        self._path = path

        self._tables_d = parse_eio(self._path)  # { lower_ref: EioTable(), ...

    # -------------------------------------- public api ----------------------------------------------------------------
    def get_table_refs(self):
        return self._tables_d.keys()

    def get_df(self, table_ref):
        return self._tables_d[table_ref.lower()].get_df()

    def get_value(self, table_ref, column_name_or_i, filter_column_name_or_i, filter_criterion):
        lower_ref = table_ref.lower()
        if lower_ref not in self._tables_d:
            raise KeyError("Unknown table_ref: '%s'." % table_ref)
        return self._tables_d[lower_ref].get_value(column_name_or_i, filter_column_name_or_i, filter_criterion)

def parse_eio(path):
    headers_l2 = [["<Program Version>", "Program Version ID", "YMD"]]
    content_d = {}  # {ref: data_l2, ...}
    content_header_d = {}  # {ref (istr(): header_row, ...}

    # _header_ref_pattern_ = "^([^<.]*)<([^>.]*)>(.*)$"

    with open(path, encoding=CONF.encoding) as f:
        for line_s in f:
            if line_s == "End of Data":
                break
            line_s = line_s.strip().strip(",")
            line_l = [c.strip() for c in line_s.split(",")]
            if line_s[0][0] == "!":  # header
                headers_l2.append([line_l[0][1:].strip()] + line_l[1:])
            else:  # content
                ref = line_l[0]
                if ref not in content_d:
                    content_d[ref] = []
                content_d[ref].append(line_l[1:])
                # find header if necessary
                if ref not in content_header_d:
                    for i in range(len(headers_l2)-1, -1, -1):
                        if ref in headers_l2[i][0]:
                            content_header_d[ref] = i
                            break

    # prepare data for dataframes
    tables_d = {}
    for ref, data in content_d.items():
        header_i = content_header_d.get(ref)
        if header_i is None:  # header not found
            header_columns = []
        else:
            header_columns = headers_l2[header_i][1:]
            if len(header_columns) == 1 and ("Months From Jan to Dec" in header_columns[0]):
                header_columns = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]

        # prepare columns and data
        data_columns_nb = max([len(r) for r in data])
        columns_nb = max(data_columns_nb, len(header_columns))

        _columns = header_columns + ["c%i" % i for i in range(columns_nb-len(header_columns))]
        _data = []
        for r in data:
            _data.append(r + [None]*(columns_nb-len(r)))

        tables_d[ref.lower()] = EioTable(ref, _columns, _data)

    return tables_d

class EioTable:
    def __init__(self, ref, columns, data):
        """
        Stores dataframe info without parsing types (otherwise dtypes are changed even if dtyp='object' is asked...)
        """
        # check
        col_len = len(columns)
        for i, r in enumerate(data):
            if len(r) != col_len:
                raise RuntimeError("Wrong number of columns in row %i of table %s." % (i, ref))

        self._ref = ref
        self._columns = columns
        self._data = data

    def _get_column_index(self, column_name_or_i):
        if isinstance(column_name_or_i, int) or isinstance(column_name_or_i, float):
            return column_name_or_i
        if column_name_or_i not in self._columns:
            raise KeyError("Unknown column '%s' for table '%s'." % (column_name_or_i, self._ref))
        return self._columns.index(column_name_or_i)

    def get_df(self):
        _df = pd.DataFrame(data=self._data, columns=self._columns, dtype="object")
        _df.name = self._ref
        return _df

    def get_value(self, column_name_or_i, filter_column_name_or_i, filter_criterion):
        """
        Returns first occurrence of value of filter column matching filter criterion.
        """
        # find column indexes
        column_i = self._get_column_index(column_name_or_i)
        filter_column_i = self._get_column_index(filter_column_name_or_i)

        filter_fct = {
            float: lambda x: float(x) == filter_criterion,
            int: lambda x: int(x) == filter_criterion,
            str: lambda x: x.lower() == filter_criterion.lower()
        }[type(filter_criterion)]

        for row_i, row in enumerate(self._data):
            if filter_fct(row[filter_column_i]):
                break
        else:
            raise ValueError("Filter did not return any values.")

        return self._data[row_i][column_i]


import uuid
import os
import collections

from .link import Link, NONE_LINK
from .record_hook import RecordHook, NONE_RECORD_HOOK
from .external_file import ExternalFile, NONE_EXTERNAL_FILE, get_external_files_dir_name
from .exceptions import FieldValidationError

TAB_LEN = 4
COMMENT_COLUMN_START = 35

def get_type_level(value):
    if value is None:  # lowest type
        return 0

    if isinstance(value, str):
        return 1

    if isinstance(value, (int, float)):  # highest type
        return 2

    raise AssertionError(f"type not managed: {type(value)} ({value})")

class Record:
    _initialized = False  # used by __setattr__

    def __init__(self, table, data=None):
        """
        Parameters
        ----------
        table
        data: dict, default {}
            key: index_or_ref, value: raw value or value
        """
        self._table = table  # when record is deleted, __init__ fields are set to None
        self._data = {}

        # signal initialized
        self._initialized = True

        # set data if any
        if data is not None:
            self._update_inert(data)

    def _field_key_to_index(self, ref_or_index):
        if isinstance(ref_or_index, int):
            if ref_or_index < 0:
                ref_or_index += len(self)
            if ref_or_index < 0:
                raise IndexError("index out of range")
            return ref_or_index
        return self._table._dev_descriptor.get_field_index(ref_or_index)

    def _update_inert(self, data):
        # transform keys to indexes
        data = dict([(self._field_key_to_index(k), v) for (k, v) in data.items()])

        # set values inert (must be ordered, otherwise some extensible values may be rejected by mistake)
        for k, v in sorted(data.items()):
            self._update_value_inert(k, v)

        # leave if empty required fields are tolerated
        # check that no required fields are missing
        if not self._table.get_epm()._dev_check_required:
            return

        # check required
        for i in range(len(self)):
            if i in self._data:
                continue

            # see if required field
            field_descriptor = self.get_field_descriptor(i)
            field_descriptor.check_not_required()

            # check not pk (in case idd was badly written)
            if i == 0 and not self._table._dev_auto_pk:
                raise FieldValidationError(
                    f"Field is required (it is a pk). {field_descriptor.get_error_location_message()}")

    def _update_value_inert(self, index, value):
        """
        is only called by _update_inert
        """
        # get field descriptor
        field_descriptor = self._table._dev_descriptor.get_field_descriptor(index)

        # prepare value
        value = field_descriptor.deserialize(value, index)

        # unregister previous link if relevant
        if isinstance(value, Link):
            # de-activate current link if any
            current_link = self._data.get(index)
            if current_link is not None:
                current_link.unregister()

        # unregister previous hook if relevant
        if isinstance(value, RecordHook):
            current_record_hook = self._data.get(index)
            if current_record_hook is not None:
                current_record_hook.unregister()

        # unregister previous external file if relevant
        if isinstance(value, ExternalFile):
            current_external_file = self._data.get(index)
            if current_external_file is not None:
                current_external_file._dev_unregister()

        # if None remove and leave
        if value in (None, NONE_RECORD_HOOK, NONE_LINK, NONE_EXTERNAL_FILE):
            # we don't check required, because this method is called by _update_inert which does the job
            self._dev_set_none_without_unregistering(index, check_not_required=False)
            return

        # if relevant, store current pk to signal table
        old_hook = None
        if index == 0 and not self._table._dev_auto_pk:
            old_hook = self._data.get(0)  # we use get, because record may not have a pk yet if it is being created

        # set value
        self._data[index] = value

        # signal pk update if relevant
        if old_hook is not None:
            self._table._dev_record_pk_was_updated(old_hook.target_value)

    def _dev_set_none_without_unregistering(self, index, check_not_required=True):
        # get field descriptor
        field_descriptor = self._table._dev_descriptor.get_field_descriptor(index)

        # check not required (if asked and check required mode)
        if check_not_required and self._table.get_epm()._dev_check_required:
            field_descriptor.check_not_required()

        # check not pk (in case idd was badly written)
        if not self._table._dev_auto_pk and index == 0:
            raise FieldValidationError(
                f"Field is required (it is a pk). {field_descriptor.get_error_location_message()}")

        # set none
        if index in self._data:
            del self._data[index]

    def _prepare_pop_insert_index(self, index=None):
        if not self.is_extensible():
            raise TypeError("Can't use add_fields on a non extensible record.")

        # manage None or negative index
        self_len = len(self)
        if index is None:
            index = self_len - 1
        elif index < 0:
            index = self_len + index

        # check index is >= cycle_start
        cycle_start, cycle_len, patterns = self.get_extensible_info()
        if not cycle_start <= index < self_len:
            raise TypeError("Can't use pop or insert for non extensible fields.")
        if cycle_len != 1:
            raise TypeError("Can only use pop or insert for extensible fields who's cycle length is 1.")

        return index

    def _unregister_hooks(self):
        for v in self._data.values():
            if isinstance(v, RecordHook):
                v.unregister()

    def _unregister_links(self):
        for v in self._data.values():
            if isinstance(v, Link):
                v.unregister()

    def _unregister_external_files(self):
        for v in self._data.values():
            if isinstance(v, ExternalFile):
                v._dev_unregister()

    def _dev_activate_hooks(self):
        for v in self._data.values():
            if isinstance(v, RecordHook):
                v.activate(self)

    def _dev_activate_links(self):
        for v in self._data.values():
            if isinstance(v, Link):
                v.activate(self)

    def _dev_activate_external_files(self):
        for v in self._data.values():
            if isinstance(v, ExternalFile):
                v._dev_activate(self.get_epm()._dev_external_files_manager)

    # --------------------------------------------- public api ---------------------------------------------------------
    # python magic
    def __repr__(self):
        if self._table is None:
            return "<Record (deleted)>"

        if self._table._dev_auto_pk:
            return f"<Record {self.get_table()._dev_descriptor.table_name}>"

        return f"<Record {self.get_table().get_name()} '{self.get_pk()}'>"

    def __str__(self):
        if self._table is None:
            return repr(self)

        return self.to_idf()

    def __getitem__(self, item):
        """
        Parameters
        ----------
        item: field lowercase name or index

        Returns
        -------
        Field value
        """
        # prepare item
        item = self._field_key_to_index(item)

        # check not out of range
        if item > len(self):
            raise IndexError("index out of range")

        # get value
        value = self._data.get(item)

        # transform if hook or link
        if isinstance(value, RecordHook):
            return value.target_value
        if isinstance(value, Link):
            return value.target_record

        # return otherwise
        return value

    def __setitem__(self, key, value):
        """
        Parameters
        ----------
        key: field lowercase name or index
        value: value to set
        """
        self.update({key: value})

    def __getattr__(self, item):
        """
        Parameters
        ----------
        item: field name
        """
        index = self._table._dev_descriptor.get_field_index(item)
        return self[index]

    def __setattr__(self, name, value):
        """
        Parameters
        ----------
        name: field lowercase name
        value: value to set
        """
        if name in self.__dict__ or not self._initialized:
            super().__setattr__(name, value)
            return
        self.update({name: value})

    def __dir__(self):
        return [
            f"f{i}" if fd.ref is None else fd.ref for
            (i, fd) in enumerate(self._table._dev_descriptor.field_descriptors)
        ] + list(self.__dict__)

    def __len__(self):
        biggest_index = -1 if (len(self._data) == 0) else max(self._data)

        # manage extensible
        if self.is_extensible():
            # go to end of extensible group
            cycle_start, cycle_len, patterns = self.get_extensible_info()
            extensible_position = biggest_index - cycle_start
            last_position = (extensible_position//cycle_len+1)*cycle_len-1
            biggest_index = cycle_start + last_position

        return max(
            biggest_index+1,
            self._table._dev_descriptor.base_fields_nb
        )

    def __lt__(self, other):
        # compare tables
        self_ref = self.get_table_ref()
        other_ref = other.get_table_ref()
        if self_ref < other_ref:
            return True
        if self_ref > other_ref:
            return False

        # get lengths
        self_len = len(self)
        other_len = len(other)
        common_length = min(self_len, other_len)

        # compare field by field
        for i in range(common_length):
            # values
            self_value = self.get_serialized_value(i)
            other_value = self.get_serialized_value(other)

            # types
            self_type_level = get_type_level(self_value)
            other_type_level = get_type_level(other_value)

            # different types
            if self_type_level < other_type_level:
                return True
            if self_type_level > other_type_level:
                return False

            # different values
            if self_value < other_value:
                return True
            if self_value > other_value:
                return False

        # equality on common fields, len will settle
        return self_len <= other_len

    # get context
    def get_epm(self):
        return self._table.get_epm()

    def get_table(self):
        return self._table

    def get_table_ref(self):
        return self._table.get_ref()

    # explore specific info
    def get_pk(self):
        """
        Returns
        -------
        If record has a name, returns its name, else returns record's python id.
        """
        return id(self) if self._table._dev_auto_pk else self[0]

    def get_serialized_value(self, ref_or_index, model_name=None):
        """
        Parameters
        ----------
        ref_or_index
        external_files_mode: str, default 'path'
            'path', 'pointer'
        model_file_path: str, default None
            if external files are asked in a relative fashion, relative path will be calculated relatively to
            model_file_path if given, else current directory

        Returns
        -------
        serialized value (only basic types: string, int, float, None, ...)
        """
        index = (
            self._table._dev_descriptor.get_field_index(ref_or_index) if isinstance(ref_or_index, str)
            else ref_or_index
        )

        # get value
        value = self._data.get(index)

        # serialize
        value = value.serialize() if isinstance(value, (Link, RecordHook)) else value

        # manage file names
        if isinstance(value, ExternalFile):
            value = os.path.join(get_external_files_dir_name(model_name=model_name), value.naive_short_ref)

        return value

    def get_pointed_records(self):
        """
        Returns
        -------
        MultiTableQueryset of all records pointing on record.
        """
        return self.get_epm()._dev_relations_manager.get_pointed_from(self)

    def get_pointing_records(self):
        """
        Returns
        -------
        MultiTableQueryset of all records pointed by record.
        """
        return self.get_epm()._dev_relations_manager.get_pointing_on(self)

    def get_external_files(self):
        """
        Returns
        -------
        List of ExternalFiles instances contained by record.
        """
        return [v for v in self._data.values() if isinstance(v, ExternalFile)]

    # construct
    def update(self, data=None, **or_data):
        """
        Updates simultaneously all given fields.

        Parameters
        ----------
        data: dictionary containing field lowercase names or index as keys, and field values as values (dict syntax)
        or_data: keyword arguments containing field names as keys (kwargs syntax)
        """

        # workflow
        # --------
        # (methods belonging to create/update/delete framework:
        #     epm._dev_populate_from_json_data, table.batch_add, record.update, queryset.delete, record.delete)
        # 1. add inert
        #     * data is checked
        #     * old links are unregistered
        #     * record is stored in table (=> pk uniqueness is checked)
        # 2. activate: hooks, links, external files

        data = or_data if data is None else data

        self._update_inert(data)

        self._dev_activate_hooks()
        self._dev_activate_links()
        self._dev_activate_external_files()

    def copy(self, new_name=None):
        """
        Parameters
        ----------
        new_name: str, default None
            record's new name (if table has a name). If None although record has a name, a random uuid will be given.

        Returns
        -------
        Copied record.
        """
        # todo: check this really works, !! must not use same link, hook, external_file, ... for different records !!
        # auto pk tables can just be copied
        if self._table._dev_auto_pk:
            return self._table.add(self._data)

        # for ref pk tables, must manage name
        name = str(uuid.uuid4()) if new_name is None else new_name
        new_data = dict((k, name if k == 0 else v) for (k, v) in self._data.items())
        return self._table.add(new_data)

    def set_defaults(self):
        """
        sets all empty fields for which a default value is defined to default value
        """
        defaults = {}
        for i in range(len(self)):
            if i in self._data:
                continue
            default = self.get_field_descriptor(i).tags.get("default", [None])[0]
            if default is not None:
                defaults[i] = default

        self.update(defaults)

    # construct extensible fields
    def add_fields(self, *args):
        """
        This method only works for extensible fields. It allows to add values without precising their fields' names
        or indexes.

        Parameters
        ----------
        args: field values
        """
        if not self.is_extensible():
            raise TypeError("Can't use add_fields on a non extensible record.")

        # prepare update data
        self_len = len(self)
        data = dict([(self_len + i, args[i]) for i in range(len(args))])

        # update
        self.update(data)

    def pop(self, index=None):
        """
        This method only works for extensible fields. It allows to remove a value and shift all other values to fill
        the gap.

        Parameters
        ----------
        index: int, default None
            index of field to remove.

        Returns
        -------
        serialize value of popped field
        """
        # prepare index (will check for extensible)
        index = self._prepare_pop_insert_index(index=index)

        # get extensible info
        cycle_start, cycle_len, patterns = self.get_extensible_info()

        # remove extensible fields
        fields = self.clear_extensible_fields()

        # pop
        serialized_value = fields.pop(index-cycle_start)

        # add remaining
        self.add_fields(*fields)

        return serialized_value

    def insert(self, index, value):
        """
        This method only works for extensible fields. It allows to insert a value, and shifts all other following
        values.

        Parameters
        ----------
        index: position of insertion
        value: value to insert
        """
        # prepare index (will check for extensible)
        index = self._prepare_pop_insert_index(index=index)

        # remove extensible fields
        fields = self.clear_extensible_fields()

        # insert
        fields.insert(index, value)

        # add new list
        self.add_fields(*fields)

    def clear_extensible_fields(self):
        """
        Returns
        -------
        list of cleared fields (serialized)
        """
        if not self.is_extensible():
            raise TypeError("Can't use add_fields on a non extensible record.")
        cycle_start, cycle_len, patterns = self.get_extensible_info()
        return [self.get_serialized_value(i) for i in range(cycle_start, len(self))]

    # delete
    def delete(self):
        """
        Deletes record, and removes it from database.
        """
        # workflow
        # --------
        # (methods belonging to create/update/delete framework:
        #     epm._dev_populate_from_json_data, table.batch_add, record.update, queryset.delete, record.delete)
        # 1. unregister: links, hooks and external files
        # 3. remove from table without unregistering

        # unregister links
        self._unregister_links()

        # unregister hooks
        self._unregister_hooks()

        # unregister external files
        self._unregister_external_files()

        # tell table to remove without unregistering
        self.get_table()._dev_remove_record_without_unregistering(self)

        # make stale
        self._table = None
        self._data = None

    # get idd info
    def get_field_descriptor(self, ref_or_index):
        """
        Parameters
        ----------
        ref_or_index: str or int
            field lowercase name, or field position

        Returns
        -------
        Field descriptor (info contained in Idd)
        """
        if isinstance(ref_or_index, int):
            index = ref_or_index
        else:
            index = self._table._dev_descriptor.get_field_index(ref_or_index)
        return self._table._dev_descriptor.get_field_descriptor(index)

    def get_info(self):
        return self._table._dev_descriptor.get_info()

    def is_extensible(self):
        return self.get_extensible_info() is not None

    def get_extensible_info(self):
        """
        Returns
        -------
        cycle_start, cycle_len, patterns
        """
        return self._table._dev_descriptor.extensible_info

    # --------------------------------------------- export -------------------------------------------------------------
    def to_dict(self):
        return collections.OrderedDict(sorted(self._data.items()))

    def to_json_data(self, model_name=None):
        """
        Parameters
        ----------
        model_name: str, default None
            if given, will be used as external file directory base name

        Returns
        -------
        A dictionary of serialized data.
        """
        return collections.OrderedDict([(k, self.get_serialized_value(k, model_name=model_name )) for k in self._data])

    def to_idf(self, model_name=None):
        """
        Parameters
        ----------
         model_name: str, default None
            if given, will be used as external file directory base name

        Returns
        -------
        idf string
        """

        json_data = self.to_json_data(model_name=model_name)

        # record descriptor ref
        s = f"{self._table._dev_descriptor.table_name},\n"

        # fields
        # fields_nb: we don't use len(self) but max(self). We wan't to stop if no more values (even base fields)
        #   because some idd records are defined without extensibles (although they should used them), for example
        #   construction, and eplus does not know what to do...
        fields_nb = max(self._data)+1
        for i in range(fields_nb):
            # value
            tab = " " * TAB_LEN
            raw_value = json_data.get(i, "")
            content = f"{tab}{raw_value}{';' if i == fields_nb-1 else ','}"

            # comment
            spaces_nb = COMMENT_COLUMN_START - len(content)
            if spaces_nb < 0:
                spaces_nb = TAB_LEN

            # comment
            name = self._table._dev_descriptor.get_extended_name(i)
            comment = "" if name is None else " " * spaces_nb + f"! {name}"

            # store
            s += f"{content}{comment}\n"

        return s


import tempfile
import os

from oplus.configuration import CONF
from oplus import simulate, Eio

MAX_TESTS_NB = 500
START_FILE_NUM = 0
DEBUG_SIMUL_DIR_PATH = None  # r"C:\Users\Geoffroy\Desktop\simul_dir"  # -> set to None to bypass

def check():
    """
    Tested under EPlus 8.1.0 on Windows (Geoffroy).
    """
    # !! CAN BE VERY LONG
    epw_path = os.path.join(CONF.eplus_base_dir_path, "WeatherData",
                            "USA_VA_Sterling-Washington.Dulles.Intl.AP.724030_TMY3.epw")
    idf_dir_path = os.path.join(CONF.eplus_base_dir_path, "ExampleFiles")
    test_num = 0
    for file_num, file_name in enumerate(os.listdir(idf_dir_path)):
        if file_num < START_FILE_NUM:
            continue
        base, ext = os.path.splitext(file_name)
        if ext == ".idf":
            with tempfile.TemporaryDirectory() as simulation_dir_path:
                s = simulate(os.path.join(idf_dir_path, file_name), epw_path,
                             simulation_dir_path if DEBUG_SIMUL_DIR_PATH is None else
                             DEBUG_SIMUL_DIR_PATH)
                if s.exists("eio"):
                    eio = Eio(s.get_file_path("eio"))  # raise error if problem
                    test_num += 1
        if test_num == MAX_TESTS_NB:
            break

if __name__ == "__main__":
    check()


import logging
import collections

from ..util import to_buffer
from .parse_eso import parse_eso, FREQUENCIES
from .switch_instants import switch_to_datetime_instants

logger = logging.getLogger(__name__)

class StandardOutput:
    def __init__(self, buffer_or_path):
        """
        Parameters
        ----------
        buffer_or_path

        Initially, standard_output will have tuple instants (using 'year', 'month', 'day', 'hour', 'minute' columns,
            depending on given frequency). It is possible to change to datetime mode later.
        """
        self._path = None
        self._path, buffer = to_buffer(buffer_or_path)
        with buffer as f:
            self._raw_environments, self._raw_variables_info, self._dfs = parse_eso(f)
        self._start_year = None

    @property
    def has_tuple_instants(self):
        return self._start_year is None

    def switch_to_datetime_instants(self, start_year):
        # leave if not relevant
        if self._start_year == start_year:
            return

        # check not a year switch
        if (self._start_year is not None) and (self._start_year != start_year):
            raise ValueError("Can't change start_year when already datetime instants, switch to tuple instants first.")

        # switch all dataframes
        new_dfs = {}
        for env_title, env_data in self._dfs.items():
            env_dfs = {}
            new_dfs[env_title] = env_dfs
            for eplus_frequency, df in env_data.items():
                env_dfs[eplus_frequency] = None if df is None else switch_to_datetime_instants(
                    df, start_year, eplus_frequency)

        # store result and start year
        self._dfs = new_dfs
        self._start_year = start_year

    def get_data(self, environment_title_or_num=-1, frequency=None):
        """
        Parameters
        ----------
        environment_title_or_num
        frequency: 'str', default None
            'timestep', 'hourly', 'daily', 'monthly', 'annual', 'run_period'
            If None, will look for the smallest frequency of environment.
        """

        # manage environment num
        if isinstance(environment_title_or_num, int):
            environment_title = tuple(self._raw_environments.keys())[environment_title_or_num]
        else:
            environment_title = environment_title_or_num

        if environment_title not in self._dfs:
            raise ValueError(f"No environment named {environment_title}. Available environments: {tuple(self._dfs)}.")

        # get environment dataframes
        environment_dfs = self._dfs[environment_title]

        # find first non null frequency if not given
        if frequency is None:
            for frequency in FREQUENCIES:
                if environment_dfs[frequency] is not None:
                    break

        # check frequency
        if frequency not in FREQUENCIES:
            raise ValueError(f"Unknown frequency: {frequency}. Available frequencies: {FREQUENCIES}")

        return self._dfs[environment_title][frequency]

    def get_environments(self):
        environments = collections.OrderedDict()
        for e in self._raw_environments.values():
            # base info
            env = collections.OrderedDict((
                ("latitude", e.latitude),
                ("longitude", e.longitude),
                ("timezone_offset", e.timezone_offset),
                ("elevation", e.elevation)
            ))

            # add count info
            for freq in FREQUENCIES:
                df = self.get_data(environment_title_or_num=e.title, frequency=freq)
                env[f"nb_values_{freq}"] = 0 if df is None else len(df)

            # store
            environments[e.title] = env

        return environments

    def get_variables(self):
        _variables = []
        for v in self._raw_variables_info.values():
            # base info
            var = collections.OrderedDict((
                ("code", v.code),
                ("key_value", v.key_value),
                ("name", v.name),
                ("ref", f"{v.key_value},{v.name}"),
                ("unit", v.unit),
                ("frequency", v.frequency),
                ("info", v.info)
            ))

            # add environments info
            var["environments"] = [
                env_title for env_title in self._raw_environments if self._dfs[env_title] is not None]

            # store
            _variables.append(var)

        # sort, create dict and return
        return collections.OrderedDict(
            (f"{var['frequency']},{var['ref']}", var) for var in
            sorted(_variables, key=lambda x: (x["frequency"], x["ref"]))
        )

    def get_info(self):
        msg = "Standard output\n"
        msg += f"\tinstants: {'tuple' if self.has_tuple_instants else 'datetime'}\n"
        msg += "\tenvironments:\n"
        for env_title, env_info in self.get_environments().items():
            msg += f"\t\t'{env_title}'\n"
            for k, v in env_info.items():
                msg += f"\t\t\t{k}: {v}\n"
        msg += "\tvariables:\n"
        msg += "\t\t" + "\n\t\t".join(self.get_variables())

        return msg


from .record import Record
from .queryset import Queryset
from .exceptions import FieldValidationError

def get_documented_add(self, record_descriptors):
    """
    this hack is used to document add function
    a methods __doc__ attribute is read-only (or must use metaclasses, what I certainly don't want to do...)
    we therefore create a function (who's __doc__ attribute is read/write), and will bind it to Table in __init__
    """
    def add(data=None, **or_data):
        """
        Parameters
        ----------
        data: dictionary containing field lowercase names or index as keys, and field values as values (dict syntax)
        or_data: keyword arguments containing field names as keys (kwargs syntax)

        A lowercase name is the lowercase EnergyPlus name, for which all non alpha-numeric characters have been replaced
        by underscores. All multiple consecutive underscores are then replaced by one unique underscore.

        The two syntaxes are not meant to cohabit. The kwargs syntax is nicer, but does not enable to use indexes
        instead of names.

        Examples
        --------
        for Schedule:Compact table:

        schedule = table.add(  # kwarg syntax
            name="Heating Setpoint Schedule - new[1]",
            schedule_type_limits_name="Any Number",
            field_1="Through: 12/31",
            field_2="For: AllDays",
            field_3="Until: 24:00,20.0"
        )

        schedule = table.add({  # dict syntax, mixing names and index keys
            name="Heating Setpoint Schedule - new[1]",
            schedule_type_limits_name="Any Number",
            2="Through: 12/31",
            3="For: AllDays",
            4="Until: 24:00,20.0"
        })

        Returns
        -------
        Created Record instance
        """
        return self.batch_add([or_data if data is None else data])[0]

    add.__doc__ = "\n".join([fd.ref.lower() for fd in record_descriptors if fd.ref is not None])

    return add

class Table:
    def __init__(self, table_descriptor, epm):
        self._dev_descriptor = table_descriptor
        self._epm = epm
        self._records = dict()

        # auto pk if first field is not a required reference
        self._dev_auto_pk = not (
                (table_descriptor.field_descriptors[0].detailed_type == "reference") and
                ("required-field" in table_descriptor.field_descriptors[0].tags)
        )

        # monkey-patch add
        self.add = get_documented_add(self, self._dev_descriptor.field_descriptors)

        # register table hooks
        table_hooks_references = self._dev_descriptor.field_descriptors[0].tags.get("reference-class-name")
        if table_hooks_references is not None:
            self._epm._dev_relations_manager.register_table_hook(table_hooks_references, self)

    def _dev_record_pk_was_updated(self, old_pk):
        # remove old pk
        record = self._records.pop(old_pk)

        # check uniqueness
        new_pk = record.get_pk()
        if new_pk in self._records:
            field_descriptor = record.get_field_descriptor(0)
            raise FieldValidationError(
                f"Primary key already exists, can't create. {field_descriptor.get_error_location_message(new_pk)}")

        # store with new pk
        self._records[new_pk] = record

    def _dev_add_inert(self, records_data):
        """
        inert: hooks and links are not activated
        """
        added_records = []
        for r_data in records_data:
            # create record
            record = Record(
                self,
                data=r_data
            )

            # store
            # we don't check uniqueness here => will be done while checking hooks
            self._records[record.get_pk()] = record

            # remember record
            added_records.append(record)

        return added_records

    def _dev_remove_record_without_unregistering(self, record):
        del self._records[record.get_pk()]

    # --------------------------------------------- public api ---------------------------------------------------------
    def __repr__(self):
        return f"<Table {self.get_name()}>"

    def __str__(self):
        header = f"Table {self.get_name()} ({self.get_ref()})"
        if self._dev_auto_pk:
            return header
        return header + "\n" + "\n".join(f"  {pk}" for pk in sorted(self._records))

    def __getitem__(self, item):
        """
        Parameters
        ----------
        item: str or int
            if str: value of record name. If table does not have a name field, raises a KeyError
            if int: record position (records are ordered by their content, not by creation order)

        Returns
        -------
        Record instance
        """
        if isinstance(item, str):
            if self._dev_auto_pk:
                raise KeyError(f"table {self.get_ref()} does not have a primary key, can't use getitem syntax")
            try:
                return self._records[item]
            except KeyError:
                raise KeyError(f"table {self.get_ref()} does not contain a record who's pk is '{item}'")
        if isinstance(item, int):
            return self.select()[item]  # queryset will be ordered

        raise KeyError("item must be an int or a str")

    def __iter__(self):
        return iter(self._records.values())

    def __len__(self):
        return len(self._records)

    # get context
    def get_ref(self):
        """
        The table ref is the converted name, in order to be compatible with Python variable rules (so we can access
        table from Epm using __getattr__.
        Conversion rule: all non alphanumeric characters are transformed to underscores.
        Example: 'Schedule_Compact'
        """
        return self._dev_descriptor.table_ref

    def get_name(self):
        """
        The table name is the name given by EnergyPlus.
        Example: 'Schedule:Compact'
        """
        return self._dev_descriptor.table_name

    def get_epm(self):
        return self._epm

    # explore
    def select(self, filter_by=None):
        """
        Parameters
        ----------
        filter_by: callable, default None
            Callable must take one argument (a record of table), and return True to keep record, or False to skip it.
            Example : .select(lambda x: x.name == "my_name").
            If None, records are not filtered.

        Returns
        -------
        Queryset instance, containing all selected records.
        """
        records = self._records.values() if filter_by is None else filter(filter_by, self._records.values())
        return Queryset(self, records=records)

    def one(self, filter_by=None):
        """
        Parameters
        ----------
        filter_by: callable, default None
            Callable must take one argument (a record of table), and return True to keep record, or False to skip it.
            Example : .one(lambda x: x.name == "my_name").
            If None, records are not filtered.

        Returns
        -------
        Record instance if one and only one record is found. Else raises.

        Raises
        ------
        RecordDoesNotExistError if no record is found
        MultipleRecordsReturnedError if multiple records are found
        """
        return Queryset(self, records=self._records.values()).one(filter_by=filter_by)

    # construct
    # def add(self, data=None, **or_data):
    #     return self.batch_add([or_data if data is None else data])[0]
    # monkey-patched

    def batch_add(self, records_data):
        """
        Parameters
        ----------
        records_data: list of dictionaries containing records data. Keys of dictionary may be field names and/or field
            indexes

        Returns
        -------
        Queryset instance of added records
        """

        # workflow
        # --------
        # (methods belonging to create/update/delete framework:
        #     epm._dev_populate_from_json_data, table.batch_add, record.update, queryset.delete, record.delete)
        # 1. add inert
        #     * data is checked
        #     * old links are unregistered
        #     * record is stored in table (=> pk uniqueness is checked)
        # 2. activate: hooks, links, external files

        # add inert
        added_records = self._dev_add_inert(records_data)

        # activate hooks
        for r in added_records:
            r._dev_activate_hooks()

        # activate links and external files
        for r in added_records:
            r._dev_activate_links()
            r._dev_activate_external_files()

        return Queryset(self, records=added_records)

    # delete
    def delete(self):
        """
        Deletes all records of table.
        """
        self.select().delete()

    # get idd info
    def get_info(self):
        return self._dev_descriptor.get_info()

    # ------------------------------------------- export ---------------------------------------------------------------
    def to_json_data(self):
        """
        Returns
        -------
        A dictionary of serialized data.
        """
        return self.select().to_json_data()


from .multi_table_queryset import MultiTableQueryset
from .exceptions import FieldValidationError

class RelationsManager:
    """
    record: add hook field
        record
            create inert hook
            hook.activate => activate hook and relations_manager.register_hook
        relations_manager
            register hook

    record: add link field
        record
            create inert link
            link.activate => activate link and relations_manager.register_link
        relations_manager
            register link

    record: remove hook
        record
            hook.unregister => relations_manager.unregister_hook
            set None inert

        relations_manager
            find pointing links
                pointing_record.set_non_inert
                link.unregister

            unregister hook

    record: remove link
        record
            link.unregister => relations_manager.unregister_link
            set None inert
        relations_manager
            unregister link
    """
    def __init__(self, epm):
        self._epm = epm
        self._table_hooks = {}  # {(hook_ref, table_lower_name): table, ...}
        self._record_hooks = {}  # {(hook_ref, value): hook, ...
        self._links_by_source = {}  # {source_record_or_table: links_set, ...}
        self._links_by_target = {}  # {target_record_or_table: links_set, ...}

    def register_record_hook(self, hook):
        """
        target record must have been set
        """
        for key in hook.keys:
            if key in self._record_hooks:
                field_descriptor = hook.target_record.get_field_descriptor(hook.target_index)
                raise FieldValidationError(
                    f"Reference key already exists, can't create: {key}. "
                    f"{field_descriptor.get_error_location_message(hook.target_value, hook.target_index)}"
                )
            self._record_hooks[key] = hook

    def register_table_hook(self, references, table):
        table_lower_name = table.get_name().lower()
        for ref in references:
            self._table_hooks[(ref, table_lower_name)] = table

    def register_link(self, link):
        """
        source record and index must have been set
        """
        keys = tuple((ref, link.initial_hook_value) for ref in link.hook_references)

        # look for a record hook
        for k in keys:
            if k in self._record_hooks:
                # set link target
                link.set_target(target_record=self._record_hooks[k].target_record)
                break
        else:
            # look for a table hook
            for k in keys:
                if k in self._table_hooks:
                    # set link target
                    link.set_target(target_table=self._table_hooks[k])
                    break
            else:
                field_descriptor = link.source_record.get_field_descriptor(link.source_index)
                raise FieldValidationError(
                    f"No object found with any of given references : {keys}. "
                    f"{field_descriptor.get_error_location_message(link.initial_hook_value)}"
                )

        # store by source
        if link.source_record not in self._links_by_source:
            self._links_by_source[link.source_record] = set()
        self._links_by_source[link.source_record].add(link)

        # store by target
        if link.target not in self._links_by_target:
            self._links_by_target[link.target] = set()
        self._links_by_target[link.target].add(link)

    def unregister_record_hook(self, hook):
        # find records pointing on record hook
        for link in self._links_by_target.get(hook.target_record, set()).copy():
            # set link field to none on source record
            link.source_record._dev_set_none_without_unregistering(link.source_index)

            # unregister link
            link.unregister()

        # unregister record hook
        for key in hook.keys:
            self._record_hooks.pop(key)

    def unregister_link(self, link):
        self._links_by_target[link.target].remove(link)
        if len(self._links_by_target[link.target]) == 0:
            del self._links_by_target[link.target]

        self._links_by_source[link.source_record].remove(link)
        if len(self._links_by_source[link.source_record]) == 0:
            del self._links_by_source[link.source_record]

    def get_pointing_on(self, target_record_or_table):
        return MultiTableQueryset(
            self._epm,
            (l.source_record for l in self._links_by_target.get(target_record_or_table, set()))
        )

    def get_pointed_from(self, source_record):
        return MultiTableQueryset(
            self._epm,
            (l.target_record for l in self._links_by_source.get(source_record, set()))
        )


import datetime as dt
import logging
import subprocess
import os
import io
import sys
import threading
import contextlib
import textwrap

import pandas as pd

from oplus import __version__, CONF

logger = logging.getLogger(__name__)

def sort_df(df):
    version = tuple([int(x) for x in pd.__version__.split(".")])
    if version < (0, 20, 0):
        return df.sort()
    else:
        return df.sort_index()

def get_multi_line_copyright_message(prefix="! "):
    return textwrap.indent(
        "-"*45 + f"""\nGenerated by oplus version {__version__}
Copyright (c) {dt.datetime.now().year}, Openergy development team
http://www.openergy.fr
https://github.com/openergy/oplus\n""" + "-"*45,
        prefix,
        lambda line: True
    ) + "\n\n"

def get_mono_line_copyright_message():
    return "oplus version %s - copyright (c) %s - Openergy development team" % (__version__, dt.datetime.now().year)

def _redirect_stream(src, dst, stop_event, freq):
    while not stop_event.is_set():  # read all filled lines
        try:
            content = src.readline()
        except UnicodeDecodeError as e:
            logger.error(str(e))
            content = "unicode decode error"
        if content == "":  # empty: break
            break
        dst.write(content)
        if hasattr(dst, "flush"):
            dst.flush()

@contextlib.contextmanager
def redirect_stream(src, dst, freq=0.1):
    stop_event = threading.Event()
    t = threading.Thread(target=_redirect_stream, args=(src, dst, stop_event, freq))
    t.daemon = True
    t.start()
    try:
        yield
    finally:
        stop_event.set()
        t.join()

class LoggerStreamWriter:
    def __init__(self, logger_name, level):
        self._logger = logging.getLogger(logger_name)
        self._level = level

    def write(self, message):
        message = message.strip()
        if message != "":
            self._logger.log(self._level, message)

def run_subprocess(command, cwd=None, stdout=None, stderr=None, shell=False, beat_freq=None):
    """
    Parameters
    ----------
    command: command
    cwd: current working directory
    stdout: output info stream (must have 'write' method)
    stderr: output error stream (must have 'write' method)
    shell: see subprocess.Popen
    beat_freq: if not none, stdout will be used at least every beat_freq (in seconds)
    """
    sys.encoding = CONF.encoding
    # prepare variables
    stdout = sys.stdout if stdout is None else stdout
    stderr = sys.stderr if stderr is None else stderr

    # run subprocess
    with subprocess.Popen(
        command,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        cwd=cwd,
        shell=shell,
        universal_newlines=True
    ) as sub_p:
        # link output streams
        with redirect_stream(sub_p.stdout, stdout), redirect_stream(sub_p.stderr, stderr):
            while True:
                try:
                    sub_p.wait(timeout=beat_freq)
                    break
                except subprocess.TimeoutExpired:
                    stdout.write("subprocess is still running\n")
                    if hasattr(sys.stdout, "flush"):
                        sys.stdout.flush()
        return sub_p.returncode

def get_string_buffer(path_or_content, expected_extension):
    """
    path_or_content: path or content_str or content_bts or string_io or bytes_io

    Returns
    -------
    string_buffer, path

    path will be None if input was not a path
    """
    buffer, path = None, None

    # path or content string

    if isinstance(path_or_content, str):
        if path_or_content[-len(expected_extension)-1:] == ".%s" % expected_extension:
            if not os.path.isfile(path_or_content):
                raise FileNotFoundError("No file at given path: '%s'." % path_or_content)
            buffer, path = open(path_or_content, encoding=CONF.encoding), path_or_content
        else:
            buffer = io.StringIO(path_or_content, )

    # text io
    elif isinstance(path_or_content, io.TextIOBase):
        buffer = path_or_content

    # bytes
    elif isinstance(path_or_content, bytes):
        buffer = io.StringIO(path_or_content.decode(encoding=CONF.encoding))
    elif isinstance(path_or_content, io.BufferedIOBase):
        buffer = io.StringIO(path_or_content.read().decode(encoding=CONF.encoding))
    else:
        raise ValueError("path_or_content type could not be identified")

    return buffer, path

def multi_mode_write(buffer_writer, string_writer, buffer_or_path=None):
    # manage string mode
    if buffer_or_path is None:
        return string_writer()

    # manage buffer mode
    if isinstance(buffer_or_path, str):
        buffer = open(buffer_or_path, "w")
    else:
        buffer = buffer_or_path

    with buffer:
        buffer_writer(buffer)

def to_buffer(buffer_or_path):
    if isinstance(buffer_or_path, str):
        if not os.path.isfile(buffer_or_path):
            raise FileNotFoundError(f"no file found at given path: {buffer_or_path}")
        path = buffer_or_path
        buffer = open(buffer_or_path, encoding=CONF.encoding)
    else:
        path = None
        buffer = buffer_or_path
    return path, buffer


import os
import pandas as pd

from . import CONF

class Err:
    WARNING = "Warning"
    FATAL = "Fatal"
    SEVERE = "Severe"

    CATEGORIES = (WARNING, FATAL, SEVERE)

    def __init__(self, path):
        if not os.path.isfile(path):
            raise FileNotFoundError("No file at given path: '%s'." % path)
        self.path = path

        self._df = None  # multi-index dataframe
        self.info = {}
        self._parse()

        self._simulation_step_list = list(set(self._df.columns.levels[0]))

    @property
    def content(self):
        with open(self.path, encoding=CONF.encoding) as f:
            return f.read()

    def _parse(self):
        # todo: manage information with ahead "*************"
        # todo: manage "error flag" :
        # todo: it corresponds to error type for each error_category lines_s.split("=")[0] --> MultiIndex

        # first step: warmup
        simulation_step = "Warmup"
        max_nb = int(1e4)
        step_df = pd.DataFrame(columns=self.CATEGORIES, index=range(0, max_nb))
        category, index_nb = None, None
        with open(self.path, encoding=CONF.encoding) as f:
            for row_nb, content in enumerate(f):
                # line_nb = var[0]
                line_s = content.rstrip("\n")

                # GET GENERIC INFORMATION
                if "Program Version,EnergyPlus" in line_s:
                    self.info["EnergyPlus Simulation Version"] = str(line_s.split(",")[2].rstrip("Version "))
                    if CONF.eplus_version < (9, 0, 0):
                        # todo: manage properly in compatibility
                        self.info["Idd_Version"] = str(line_s.split("IDD_Version ")[1])
                    self.info["Idd_Version"] = None
                elif "EnergyPlus Warmup Error Summary" in line_s:
                    self.info["EnergyPlus Warmup Error Summary"] = str(line_s.split(". ")[1])
                elif "EnergyPlus Sizing Error Summary" in line_s:
                    self.info["EnergyPlus Sizing Error Summary"] = str(line_s.split(". ")[1])
                elif "EnergyPlus Completed Successfully" in line_s:
                    self.info["EnergyPlus Completed Successfully"] = str(line_s.split("--")[1])

                # PARSE AND ..
                elif "************* Beginning" in line_s:
                    # SET OUTPUT DATAFRAME
                    if self._df is None:
                        iterables = [(simulation_step,), step_df.columns]
                        columns = pd.MultiIndex.from_product(iterables)
                        self._df = pd.DataFrame(index=range(0, max_nb), columns=columns)
                        self._df[simulation_step] = step_df
                    else:
                        iterables = [(simulation_step,), list(step_df.columns)]
                        columns = pd.MultiIndex.from_product(iterables)
                        multi_step_df = pd.DataFrame(index=range(0, max_nb), columns=columns)
                        multi_step_df[simulation_step] = step_df
                        self._df = self._df.join(multi_step_df)

                    # start new simulation step
                    simulation_step = line_s.split("Beginning ")[1]
                    step_df = pd.DataFrame(columns=self.CATEGORIES, index=range(0, max_nb))
                elif "** Warning **" in line_s:
                    category = "Warning"
                    # new line (index) until next
                    series = step_df[category].dropna()
                    if len(series.index) == 0:
                        index_nb = 0
                    else:
                        index_nb = series.index[-1] + 1
                    step_df[category].loc[index_nb] = str(line_s.split("** Warning **")[1])
                elif "**  Fatal  **" in line_s:
                    category = "Fatal"
                    series = step_df[category].dropna()
                    if len(series.index) == 0:
                        index_nb = 0
                    else:
                        index_nb = series.index[-1] + 1
                    # new line (index) until next
                    step_df[category].loc[index_nb] = str(line_s.split("**  Fatal  **")[1])
                elif "** Severe  **" in line_s:
                    category = "Severe"
                    series = step_df[category].dropna()
                    if len(series.index) == 0:
                        index_nb = 0
                    else:
                        index_nb = series.index[-1] + 1
                    # new line (index) until next
                    step_df[category].loc[index_nb] = str(line_s.split("** Severe  **")[1])

                elif "**   ~~~   **" in line_s:  # if we are here, we are sure category and index_nb have been defined
                    # information to add to error
                    step_df[category].loc[index_nb] += "\n" + str(line_s.split("**   ~~~   **")[1])

            # save step_df
            iterables = [[simulation_step], step_df.columns]
            columns = pd.MultiIndex.from_product(iterables)
            multi_step_df = pd.DataFrame(index=range(0, max_nb), columns=columns)
            multi_step_df[simulation_step] = step_df
            if self._df is not None:  # can happen if never encounters "******* Beginning"
                self._df = self._df.join(multi_step_df)
            else:
                self._df = multi_step_df

            self.info = pd.Series(self.info, index=self.info.keys())

    def get_data(self, simulation_step=None, error_category=None):
        """
        Parameters
        ----------
        simulation_step: if not given, returns a raw report
        error_category: if only one argument is specified, swaps dataframe report
        """
        if simulation_step is None and error_category is None:
            return self._df.dropna(axis="rows", how="all")

        if simulation_step is not None:
            if simulation_step not in self._simulation_step_list:
                raise RuntimeError("The simulation_step '%s' is not referred in the error file." % simulation_step)

            if error_category is not None:
                if error_category not in self.CATEGORIES:
                    raise RuntimeError("The error_cat '%s' is wrong." % error_category)
                iterables = [simulation_step, error_category]
                columns = pd.MultiIndex.from_product(iterables)
                series = self._df[simulation_step][error_category].dropna(axis="rows", how="all")

                df = pd.DataFrame(index=series.index, columns=columns)
                df[simulation_step] = series
                return df

            return self._df[simulation_step].dropna(axis="rows", how="all")

        if error_category is not None:
            if error_category not in self.CATEGORIES:
                raise RuntimeError("The error_category '%s' is wrong." % error_category)
            df = self._df.copy()
            df.columns = df.columns.swaplevel(0, 1)
            return df[error_category].dropna(axis="rows", how="all")


import re
import sre_constants
import logging
from threading import Lock

LOG = logging.getLogger('padaos')

class IntentContainer:
    def __init__(self):
        self.intent_lines, self.entity_lines = {}, {}
        self.intents, self.entities = {}, {}
        self.must_compile = True
        self.i = 0
        self.compile_lock = Lock()

    def add_intent(self, name, lines):
        with self.compile_lock:
            self.must_compile = True
            self.intent_lines[name] = lines

    def remove_intent(self, name):
        with self.compile_lock:
            self.must_compile = True
            if name in self.intent_lines:
                del self.intent_lines[name]

    def add_entity(self, name, lines):
        with self.compile_lock:
            self.must_compile = True
            self.entity_lines[name] = lines

    def remove_entity(self, name):
        with self.compile_lock:
            self.must_compile = True
            if name in self.entity_lines:
                del self.entity_lines[name]

    def _create_pattern(self, line):
        for pat, rep in (
                # === Preserve Plain Parentheses ===
                (r'\(([^\|)]*)\)', r'{~(\1)~}'),  # (hi) -> {~(hi)~}

                # === Convert to regex literal ===
                (r'(\W)', r'\\\1'),
                (r' {} '.format, None),  # 'abc' -> ' abc '

                # === Unescape Chars for Convenience ===
                (r'\\ ', r' '),  # "\ " -> " "
                (r'\\{', r'{'),  # \{ -> {
                (r'\\}', r'}'),  # \} -> }
                (r'\\#', r'#'),  # \# -> #

                # === Support Parentheses Expansion ===
                (r'(?<!\\{\\~)\\\(', r'(?:'),  # \( -> (  ignoring  \{\~\(
                (r'\\\)(?!\\~\\})', r')'),  # \) -> )  ignoring  \)\~\}
                (r'\\{\\~\\\(', r'\\('),  # \{\~\( -> \(
                (r'\\\)\\~\\}', r'\\)'),  # \)\~\}  -> \)
                (r'\\\|', r'|'),  # \| -> |

                # === Support Special Symbols ===
                (r'(?<=\s)\\:0(?=\s)', r'\\w+'),
                (r'#', r'\\d'),
                (r'\d', r'\\d'),

                # === Space Word Separations ===
                (r'(?<!\\)(\w)([^\w\s}])', r'\1 \2'),  # a:b -> a :b
                (r'([^\\\w\s{])(\w)', r'\1 \2'),  # a :b -> a : b

                # === Make Symbols Optional ===
                (r'(\\[^\w ])', r'\1?'),

                # === Force 1+ Space Between Words ===
                (r'(?<=(\w|\}))(\\\s|\s)+(?=\S)', r'\\W+'),

                # === Force 0+ Space Between Everything Else ===
                (r'\s+', r'\\W*'),
        ):
            if callable(pat):
                line = pat(line)
            else:
                line = re.sub(pat, rep, line)
        return line

    def _create_intent_pattern(self, line, intent_name):
        namespace = intent_name.split(':')[0] + ':'
        line = self._create_pattern(line)
        replacements = {}
        for ent_name in set(re.findall(r'{([a-z_:]+)}', line)):
            replacements[ent_name] = r'(?P<{}__{{}}>.*?\w.*?)'.format(ent_name)
        for ent_name, ent in self.entities.items():
            ent_regex = r'(?P<{}__{{}}>{})'
            if ent_name.startswith(namespace):
                replacements[ent_name[len(namespace):]] = ent_regex.format(
                    ent_name[len(namespace):], ent
                )
            else:
                replacements[ent_name] = ent_regex.format(ent_name.replace(':', '__colon__'), ent)
        for key, value in replacements.items():
            line = line.replace('{' + key + '}', value.format(self.i), 1)
            self.i += 1
        return '^{}$'.format(line)

    def _create_regex(self, line, intent_name):
        """ Create regex and return. If error occurs returns None. """
        try:
            return re.compile(self._create_intent_pattern(line, intent_name),
                              re.IGNORECASE)
        except sre_constants.error as e:
            LOG.warning('Failed to parse the line "{}" '
                        'for {}'.format(line, intent_name))
            return None

    def create_regexes(self, lines, intent_name):
        regexes = [self._create_regex(line, intent_name)
                  for line in sorted(lines, key=len, reverse=True)
                  if line.strip()]
        # Filter out all regexes that fails
        return [r for r in regexes if r is not None]

    def compile(self):
        with self.compile_lock:
            self._compile()

    def _compile(self):
        self.entities = {
            ent_name: r'({})'.format('|'.join(
                self._create_pattern(line) for line in lines if line.strip()
            ))
            for ent_name, lines in self.entity_lines.items()
        }
        self.intents = {
            intent_name: self.create_regexes(lines, intent_name)
            for intent_name, lines in self.intent_lines.items()
        }
        self.must_compile = False

    def _calc_entities(self, query, regexes):
        for regex in regexes:
            match = regex.match(query)
            if match:
                yield {
                    k.rsplit('__', 1)[0].replace('__colon__', ':'): v.strip()
                    for k, v in match.groupdict().items() if v
                }

    def calc_intents(self, query):
        query = ' ' + query + ' '
        if self.must_compile:
            self.compile()
        for intent_name, regexes in self.intents.items():
            entities = list(self._calc_entities(query, regexes))
            if entities:
                yield {
                    'name': intent_name,
                    'entities': min(entities, key=lambda x: sum(map(len, x.values())))
                }

    def calc_intent(self, query):
        return min(
            self.calc_intents(query),
            key=lambda x: sum(map(len, x['entities'].values())),
            default={'name': None, 'entities': {}}
        )


# -*- coding: utf-8 -*-
'''
    pyca.db
    ~~~~¨~~

    Database specification for pyCA
'''

import json
import os.path
import string
from pyca.config import config
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, Text, LargeBinary, create_engine
from sqlalchemy.orm import sessionmaker
Base = declarative_base()

def init():
    '''Initialize connection to database. Additionally the basic database
    structure will be created if nonexistent.
    '''
    global engine
    engine = create_engine(config()['agent']['database'])
    Base.metadata.create_all(engine)

def get_session():
    '''Get a session for database communication. If necessary a new connection
    to the database will be established.

    :return:  Database session
    '''
    if 'engine' not in globals():
        init()
    Session = sessionmaker(bind=engine)
    return Session()

class Constants():

    @classmethod
    def str(cls, value):
        '''Convert status (id) to its string name.'''
        for k, v in cls.__dict__.items():
            if k[0] in string.ascii_uppercase and v == value:
                return k.lower().replace('_', ' ')

class Status(Constants):
    '''Event status definitions
    '''
    UPCOMING = 1
    RECORDING = 2
    FAILED_RECORDING = 3
    FINISHED_RECORDING = 4
    UPLOADING = 5
    FAILED_UPLOADING = 6
    FINISHED_UPLOADING = 7

class ServiceStatus(Constants):
    '''Service status type definitions
    '''
    STOPPED = 1
    IDLE = 2
    BUSY = 3

class Service(Constants):
    '''Service type definitions
    '''
    AGENTSTATE = 1
    CAPTURE = 2
    INGEST = 3
    SCHEDULE = 4

# Database Schema Definition
class BaseEvent():
    '''Database definition of an event.'''

    __tablename__ = 'event'

    uid = Column('uid', Text(), nullable=False, primary_key=True)
    start = Column('start', Integer(), primary_key=True)
    end = Column('end', Integer(), nullable=False)
    title = Column('title', Text())
    data = Column('data', LargeBinary(), nullable=False)
    status = Column('status', Integer(), nullable=False,
                    default=Status.UPCOMING)
    tracks = Column('tracks', LargeBinary(), nullable=True)

    def get_data(self):
        '''Load JSON data from event.
        '''
        return json.loads(self.data.decode('utf-8'))

    def set_data(self, data):
        '''Store data as JSON.
        '''
        # Python 3 wants bytes
        self.data = json.dumps(data).encode('utf-8')

    def name(self):
        '''Returns the filesystem name of this event.
        '''
        return 'recording-%i-%s' % (self.start, self.uid)

    def directory(self):
        '''Returns recording directory of this event.
        '''
        return os.path.join(config()['capture']['directory'], self.name())

    def remaining_duration(self, time):
        '''Returns the remaining duration for a recording.
        '''
        return max(0, self.end - max(self.start, time))

    def status_str(self):
        '''Return status as string.
        '''
        return Status.str(self.status)

    def get_tracks(self):
        '''Load JSON track data from event.
        '''
        if not self.tracks:
            return []
        return json.loads(self.tracks.decode('utf-8'))

    def set_tracks(self, tracks):
        '''Store track data as JSON.
        '''
        self.tracks = json.dumps(tracks).encode('utf-8')

    def __repr__(self):
        '''Return a string representation of an artist object.

        :return: String representation of object.
        '''
        return '<Event(start=%i, uid="%s")>' % (self.start, self.uid)

    def serialize(self):
        '''Serialize this object as dictionary usable for conversion to JSON.

        :return: Dictionary representing this object.
        '''
        return {
            'type': 'event',
            'id': self.uid,
            'attributes': {
                'start': self.start,
                'end': self.end,
                'uid': self.uid,
                'title': self.title,
                'data': self.get_data(),
                'status': Status.str(self.status)
                }
            }

class UpcomingEvent(Base, BaseEvent):
    '''List of upcoming events'''

    __tablename__ = 'upcoming_event'

class RecordedEvent(Base, BaseEvent):
    '''List of events pyca tried to record.'''

    __tablename__ = 'recorded_event'

    def __init__(self, event=None):
        if event:
            self.uid = event.uid
            self.start = event.start
            self.end = event.end
            self.title = event.title
            self.data = event.data
            self.status = event.status

class ServiceStates(Base):
    '''List of internal service states.'''

    __tablename__ = 'service_states'

    type = Column('type', Integer(), nullable=False, primary_key=True)
    status = Column('status', Integer(), nullable=False,
                    default=ServiceStatus.STOPPED)

    def __init__(self, service=None):
        if service:
            self.type = service.type
            self.status = service.status


# -*- coding: utf-8 -*-
'''
    python-capture-agent
    ~~~~~~~~~~~~~~~~~~~~

    :copyright: 2014-2017, Lars Kiesow <lkiesow@uos.de>
    :license: LGPL – see license.lgpl for more details.
'''

from pyca.config import config
from pyca import db
from datetime import datetime
from dateutil.tz import tzutc
import errno
import json
import logging
import os
import os.path
import pycurl
import sys
import time
if sys.version_info[0] == 2:
    from cStringIO import StringIO as bio
    from urllib import quote as urlquote
else:
    from io import BytesIO as bio
    from urllib.parse import quote as urlquote

logger = logging.getLogger(__name__)

def http_request(url, post_data=None):
    '''Make an HTTP request to a given URL with optional parameters.
    '''
    logger.debug('Requesting URL: %s' % url)
    buf = bio()
    curl = pycurl.Curl()
    curl.setopt(curl.URL, url.encode('ascii', 'ignore'))

    # Disable HTTPS verification methods if insecure is set
    if config()['server']['insecure']:
        curl.setopt(curl.SSL_VERIFYPEER, 0)
        curl.setopt(curl.SSL_VERIFYHOST, 0)

    if config()['server']['certificate']:
        # Make sure verification methods are turned on
        curl.setopt(curl.SSL_VERIFYPEER, 1)
        curl.setopt(curl.SSL_VERIFYHOST, 2)
        # Import your certificates
        curl.setopt(pycurl.CAINFO, config()['server']['certificate'])

    if post_data:
        curl.setopt(curl.HTTPPOST, post_data)
    curl.setopt(curl.WRITEFUNCTION, buf.write)
    curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST)
    curl.setopt(pycurl.USERPWD, "%s:%s" % (config()['server']['username'],
                                           config()['server']['password']))
    curl.setopt(curl.HTTPHEADER, ['X-Requested-Auth: Digest'])
    curl.setopt(curl.FAILONERROR, True)
    curl.setopt(curl.FOLLOWLOCATION, True)
    curl.perform()
    curl.close()
    result = buf.getvalue()
    buf.close()
    return result

def get_service(service_type):
    '''Get available service endpoints for a given service type from the
    Opencast ServiceRegistry.
    '''
    endpoint = '/services/available.json?serviceType=' + str(service_type)
    url = '%s%s' % (config()['server']['url'], endpoint)
    response = http_request(url).decode('utf-8')
    services = (json.loads(response).get('services') or {}).get('service', [])
    services = ensurelist(services)
    endpoints = [service['host'] + service['path'] for service in services
                 if service['online'] and service['active']]
    for endpoint in endpoints:
        logger.info(u'Endpoint for %s: %s', service_type, endpoint)
    return endpoints

def unix_ts(dtval):
    '''Convert datetime into a unix timestamp.
    This is the equivalent to Python 3's int(datetime.timestamp()).

    :param dt: datetime to convert
    '''
    epoch = datetime(1970, 1, 1, 0, 0, tzinfo=tzutc())
    delta = (dtval - epoch)
    return delta.days * 24 * 3600 + delta.seconds

def timestamp():
    '''Get current unix timestamp
    '''
    return unix_ts(datetime.now(tzutc()))

def try_mkdir(directory):
    '''Try to create a directory. Pass without error if it already exists.
    '''
    try:
        os.mkdir(directory)
    except OSError as err:
        if err.errno != errno.EEXIST:
            raise err

def configure_service(service):
    '''Get the location of a given service from Opencast and add it to the
    current configuration.
    '''
    while not config().get('service-' + service) and not terminate():
        try:
            config()['service-' + service] = \
                get_service('org.opencastproject.' + service)
        except pycurl.error as e:
            logger.error('Could not get %s endpoint: %s. Retrying in 5s' %
                         (service, e))
            time.sleep(5.0)

def ensurelist(x):
    '''Ensure an element is a list.'''
    return x if type(x) == list else [x]

def register_ca(status='idle'):
    '''Register this capture agent at the Matterhorn admin server so that it
    shows up in the admin interface.

    :param address: Address of the capture agent web ui
    :param status: Current status of the capture agent
    '''
    # If this is a backup CA we don't tell the Matterhorn core that we are
    # here.  We will just run silently in the background:
    if config()['agent']['backup_mode']:
        return
    params = [('address', config()['ui']['url']), ('state', status)]
    name = urlquote(config()['agent']['name'].encode('utf-8'), safe='')
    url = '%s/agents/%s' % (config()['service-capture.admin'][0], name)
    try:
        response = http_request(url, params).decode('utf-8')
        if response:
            logger.info(response)
    except pycurl.error as e:
        logger.warning('Could not set agent state to %s: %s' % (status, e))

def recording_state(recording_id, status):
    '''Send the state of the current recording to the Matterhorn core.

    :param recording_id: ID of the current recording
    :param status: Status of the recording
    '''
    # If this is a backup CA we do not update the recording state since the
    # actual CA does that and we want to interfere.  We will just run silently
    # in the background:
    if config()['agent']['backup_mode']:
        return
    params = [('state', status)]
    url = config()['service-capture.admin'][0]
    url += '/recordings/%s' % recording_id
    try:
        result = http_request(url, params)
        logger.info(result)
    except pycurl.error as e:
        logger.warning('Could not set recording state to %s: %s' % (status, e))

def update_event_status(event, status):
    '''Update the status of a particular event in the database.
    '''
    dbs = db.get_session()
    dbs.query(db.RecordedEvent).filter(db.RecordedEvent.start == event.start)\
                               .update({'status': status})
    event.status = status
    dbs.commit()

def set_service_status(service, status):
    '''Update the status of a particular service in the database.
    '''
    srv = db.ServiceStates()
    srv.type = service
    srv.status = status

    dbs = db.get_session()
    dbs.merge(srv)
    dbs.commit()
    dbs.close()

def set_service_status_immediate(service, status):
    '''Update the status of a particular service in the database and send an
    immediate signal to Opencast.
    '''
    set_service_status(service, status)
    update_agent_state()

def get_service_status(service):
    '''Update the status of a particular service in the database.
    '''
    dbs = db.get_session()
    srvs = dbs.query(db.ServiceStates).filter(db.ServiceStates.type == service)

    if srvs.count():
        return srvs[0].status

    return db.ServiceStatus.STOPPED

def update_agent_state():
    '''Update the current agent state in opencast.
    '''
    configure_service('capture.admin')
    status = 'idle'

    # Determine reported agent state with priority list
    if get_service_status(db.Service.SCHEDULE) == db.ServiceStatus.STOPPED:
        status = 'offline'
    elif get_service_status(db.Service.CAPTURE) == db.ServiceStatus.BUSY:
        status = 'capturing'
    elif get_service_status(db.Service.INGEST) == db.ServiceStatus.BUSY:
        status = 'uploading'

    register_ca(status=status)

def terminate(shutdown=None):
    '''Mark process as to be terminated.
    '''
    global _terminate
    if shutdown is not None:
        _terminate = shutdown
    return '_terminate' in globals() and _terminate


# -*- coding: utf-8 -*-
'''
Default configuration for pyCA.
'''

import configobj
import logging
import logging.handlers
import os
import sys
from validate import Validator

logger = logging.getLogger(__name__)

__CFG = '''
[agent]
name             = string(default='pyca')
update_frequency = integer(min=5, default=60)
cal_lookahead    = integer(min=0, default=14)
backup_mode      = boolean(default=false)
database         = string(default='sqlite:///pyca.db')

[capture]
directory        = string(default='./recordings')
command          = string(default='ffmpeg -nostats -re -f lavfi -r 25 -i testsrc -t {{time}} {{dir}}/{{name}}.webm')
flavors          = force_list(default=list('presenter/source'))
files            = force_list(default=list('{{dir}}/{{name}}.webm'))
preview_dir      = string(default='./recordings')
preview          = force_list(default=list())
sigcustom        = integer(min=1, default=2)
sigcustom_time   = integer(min=-1, default=-1)
sigterm_time     = integer(min=-1, default=-1)
sigkill_time     = integer(min=-1, default=120)
exit_code        = integer(min=0, max=255, default=0)

[server]
url              = string(default='https://develop.opencast.org')
username         = string(default='opencast_system_account')
password         = string(default='CHANGE_ME')
insecure         = boolean(default=False)
certificate      = string(default='')

[ui]
username         = string(default='admin')
password         = string(default='opencast')
refresh_rate     = integer(min=1, default=10)
url              = string(default='http://localhost:5000')

[logging]
syslog           = boolean(default=False)
stderr           = boolean(default=True)
file             = string(default='')
level            = option('debug', 'info', 'warning', 'error', default='info')
format           = string(default='[%(name)s:%(lineno)s:%(funcName)s()] [%(levelname)s] %(message)s')
'''  # noqa

cfgspec = __CFG.split('\n')

__config = None

def configuration_file(cfgfile):
    '''Find the best match for the configuration file.
    '''
    if cfgfile is not None:
        return cfgfile
    # If no file is explicitely specified, probe for the configuration file
    # location.
    cfg = './etc/pyca.conf'
    if not os.path.isfile(cfg):
        return '/etc/pyca.conf'
    return cfg

def update_configuration(cfgfile=None):
    '''Update configuration from file.

    :param cfgfile: Configuration file to load.
    '''
    configobj.DEFAULT_INTERPOLATION = 'template'
    cfgfile = configuration_file(cfgfile)
    cfg = configobj.ConfigObj(cfgfile, configspec=cfgspec, encoding='utf-8')
    validator = Validator()
    val = cfg.validate(validator)
    if val is not True:
        raise ValueError('Invalid configuration: %s' % val)
    if len(cfg['capture']['files']) != len(cfg['capture']['flavors']):
        raise ValueError('List of files and flavors do not match')
    globals()['__config'] = cfg
    logger_init()
    if cfg['server'].get('url', '').endswith('/'):
        logger.warning('Base URL ends with /. This is most likely a '
                       'configuration error. The URL should contain nothing '
                       'of the service paths.')
    logger.info('Configuration loaded from %s' % cfgfile)
    check()
    return cfg

def check():
    '''Check configuration for sanity.
    '''
    if config('server')['insecure']:
        logger.warning('HTTPS CHECKS ARE TURNED OFF. A SECURE CONNECTION IS '
                       'NOT GUARANTEED')
    if config('server')['certificate']:
        # Ensure certificate exists and is readable
        open(config('server')['certificate'], 'rb').close()
    if config('agent')['backup_mode']:
        logger.info('Agent runs in backup mode. No data will be sent to '
                    'Opencast')

def config(key=None):
    cfg = __config or update_configuration()
    return cfg[key] if key else cfg

def logger_init():
    '''Initialize logger based on configuration
    '''
    handlers = []
    logconf = config('logging')
    if logconf['syslog']:
        handlers.append(logging.handlers.SysLogHandler(address='/dev/log'))
    if logconf['stderr']:
        handlers.append(logging.StreamHandler(sys.stderr))
    if logconf['file']:
        handlers.append(logging.handlers.WatchedFileHandler(logconf['file']))
    for handler in handlers:
        handler.setFormatter(logging.Formatter(logconf['format']))
        logging.root.addHandler(handler)

    logging.root.setLevel(logconf['level'].upper())
    logger.info('Log level set to %s' % logconf['level'])


# -*- coding: utf-8 -*-
'''
Simple UI telling about the current state of the capture agent.
'''
from flask import Flask, request, send_from_directory, render_template
from pyca.config import config
from pyca.db import Service, ServiceStatus, UpcomingEvent, RecordedEvent
from pyca.db import get_session
from pyca.ui.utils import dtfmt, requires_auth
from pyca.utils import get_service_status
import os.path
app = Flask(__name__)
import pyca.ui.jsonapi  # noqa

@app.route('/')
@requires_auth
def home():
    '''Serve the status page of the capture agent.
    '''
    # Get IDs of existing preview images
    preview = config()['capture']['preview']
    previewdir = config()['capture']['preview_dir']
    preview = [p.replace('{{previewdir}}', previewdir) for p in preview]
    preview = zip(preview, range(len(preview)))
    preview = [p[1] for p in preview if os.path.isfile(p[0])]

    # Get limits for recording table
    try:
        limit_upcoming = int(request.args.get('limit_upcoming', 5))
        limit_processed = int(request.args.get('limit_processed', 15))
    except ValueError:
        limit_upcoming = 5
        limit_processed = 15

    db = get_session()
    upcoming_events = db.query(UpcomingEvent)\
                        .order_by(UpcomingEvent.start)\
                        .limit(limit_upcoming)
    recorded_events = db.query(RecordedEvent)\
                        .order_by(RecordedEvent.start.desc())\
                        .limit(limit_processed)
    recording = get_service_status(Service.CAPTURE) == ServiceStatus.BUSY
    uploading = get_service_status(Service.INGEST) == ServiceStatus.BUSY
    processed = db.query(RecordedEvent).count()
    upcoming = db.query(UpcomingEvent).count()

    return render_template('home.html', preview=preview, config=config(),
                           recorded_events=recorded_events,
                           upcoming_events=upcoming_events,
                           recording=recording, uploading=uploading,
                           processed=processed, upcoming=upcoming,
                           limit_upcoming=limit_upcoming,
                           limit_processed=limit_processed,
                           dtfmt=dtfmt)

@app.route("/img/<int:image_id>")
@requires_auth
def serve_image(image_id):
    '''Serve the preview image with the given id
    '''
    try:
        preview_dir = config()['capture']['preview_dir']
        filepath = config()['capture']['preview'][image_id]
        filepath = filepath.replace('{{previewdir}}', preview_dir)
        filepath = os.path.abspath(filepath)
        if os.path.isfile(filepath):
            directory, filename = filepath.rsplit('/', 1)
            return send_from_directory(directory, filename)
    except (IndexError, KeyError):
        pass
    return '', 404


# -*- coding: utf-8 -*-
'''
    python-capture-agent
    ~~~~~~~~~~~~~~~~~~~~

    :copyright: 2014-2017, Lars Kiesow <lkiesow@uos.de>
    :license: LGPL – see license.lgpl for more details.
'''

import getopt
import multiprocessing
import signal
import sys
from pyca import capture, config, schedule, ingest, ui, agentstate, utils

USAGE = '''
Usage %s [OPTIONS] COMMAND

COMMANDS:
  run        --  Start all pyCA components except ui (default)
  capture    --  Start pyCA capture service
  ingest     --  Start pyCA ingest service
  schedule   --  Start pyCA schedule service
  agentstate --  Start pyCA agentstate service
  ui         --  Start web based user interface

OPTIONS:
  --help, -h             -- Show this help',
  --config=FILE, -c FILE -- Specify a config file',

CONFIGURATION:
  PyCA will try to find a configuration in the following order:
   - Configuration specified on the command line
   - /etc/pyca.conf
   - ./etc/pyca.conf
'''

def usage(retval=0):
    '''Print usage information to stdout and exit.
    '''
    print(USAGE % sys.argv[0])
    sys.exit(retval)

def sigint_handler(signum, frame):
    '''Intercept sigint and terminate services gracefully.
    '''
    utils.terminate(True)

def sigterm_handler(signum, frame):
    '''Intercept sigterm and terminate all processes.
    '''
    sigint_handler(signum, frame)
    for process in multiprocessing.active_children():
        process.terminate()
    sys.exit(0)

def run_all(*modules):
    '''Start all services.
    '''
    processes = [multiprocessing.Process(target=mod.run) for mod in modules]
    for p in processes:
        p.start()
    for p in processes:
        p.join()

def main():
    # Check command line options
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'hc:', ['help', 'config='])
    except getopt.GetoptError:
        usage(1)

    cfg = None
    for opt, arg in opts:
        if opt in ("-h", "--help"):
            usage()
        if opt in ('-c', '--config'):
            cfg = arg
            break

    # Make sure we got only one command
    if len(args) > 1:
        usage(2)

    try:
        config.update_configuration(cfg)
    except ValueError as e:
        print(str(e))
        sys.exit(4)

    # Set signal handler
    signal.signal(signal.SIGINT, sigint_handler)
    signal.signal(signal.SIGTERM, sigterm_handler)

    # Get the command with `run` as default
    cmd = (args + ['run'])[0]

    if cmd == 'run':
        run_all(schedule, capture, ingest, agentstate)
    elif cmd == 'schedule':
        schedule.run()
    elif cmd == 'capture':
        capture.run()
    elif cmd == 'ingest':
        ingest.run()
    elif cmd == 'agentstate':
        agentstate.run()
    elif cmd == 'ui':
        signal.signal(signal.SIGINT, signal.default_int_handler)
        ui.app.run()
    else:
        # Invalid command
        usage(3)

if __name__ == '__main__':
    main()