"""
This file contains celery tasks for email marketing signal handler.
"""

from celery import shared_task
from celery.utils.log import get_task_logger
from sailthru.sailthru_error import SailthruClientError

from ecommerce_worker.cache import Cache
from ecommerce_worker.sailthru.v1.exceptions import SailthruError
from ecommerce_worker.sailthru.v1.utils import get_sailthru_client, get_sailthru_configuration
from ecommerce_worker.utils import get_ecommerce_client
from requests.exceptions import RequestException
from six import text_type

logger = get_task_logger(__name__)  # pylint: disable=invalid-name
cache = Cache()  # pylint: disable=invalid-name

# pylint: disable=unused-argument

def schedule_retry(self, config):
    """Schedule a retry"""
    raise self.retry(countdown=config.get('SAILTHRU_RETRY_SECONDS'),
                     max_retries=config.get('SAILTHRU_RETRY_ATTEMPTS'))

def _build_purchase_item(course_id, course_url, cost_in_cents, mode, course_data, sku):
    """Build and return Sailthru purchase item object"""

    # build item description
    item = {
        'id': "{}-{}".format(course_id, mode),
        'url': course_url,
        'price': cost_in_cents,
        'qty': 1,
    }

    # get title from course info if we don't already have it from Sailthru
    if 'title' in course_data:
        item['title'] = course_data['title']
    else:
        # can't find, just invent title
        item['title'] = 'Course {} mode: {}'.format(course_id, mode)

    if 'tags' in course_data:
        item['tags'] = course_data['tags']

    # add vars to item
    item['vars'] = dict(course_data.get('vars', {}), mode=mode, course_run_id=course_id)

    item['vars']['purchase_sku'] = sku

    return item

def _record_purchase(sailthru_client, email, item, purchase_incomplete, message_id, options):
    """Record a purchase in Sailthru

    Arguments:
        sailthru_client (object): SailthruClient
        email (str): user's email address
        item (dict): Sailthru required information about the course
        purchase_incomplete (boolean): True if adding item to shopping cart
        message_id (str): Cookie used to identify marketing campaign
        options (dict): Sailthru purchase API options (e.g. template name)

    Returns:
        False if retryable error, else True
    """
    try:
        sailthru_response = sailthru_client.purchase(email, [item],
                                                     incomplete=purchase_incomplete, message_id=message_id,
                                                     options=options)

        if not sailthru_response.is_ok():
            error = sailthru_response.get_error()
            logger.error("Error attempting to record purchase in Sailthru: %s", error.get_message())
            return not can_retry_sailthru_request(error)

    except SailthruClientError as exc:
        logger.exception("Exception attempting to record purchase for %s in Sailthru - %s", email, text_type(exc))
        return False

    return True

def _get_course_content(course_id, course_url, sailthru_client, site_code, config):
    """Get course information using the Sailthru content api or from cache.

    If there is an error, just return with an empty response.

    Arguments:
        course_id (str): course key of the course
        course_url (str): LMS url for course info page.
        sailthru_client (object): SailthruClient
        site_code (str): site code
        config (dict): config options

    Returns:
        course information from Sailthru
    """
    # check cache first
    cache_key = "{}:{}".format(site_code, course_url)
    response = cache.get(cache_key)
    if not response:
        try:
            sailthru_response = sailthru_client.api_get("content", {"id": course_url})
            if not sailthru_response.is_ok():
                response = {}
            else:
                response = sailthru_response.json
                cache.set(cache_key, response, config.get('SAILTHRU_CACHE_TTL_SECONDS'))

        except SailthruClientError:
            response = {}

        if not response:
            logger.error('Could not get course data from Sailthru on enroll/purchase event. '
                         'Calling Ecommerce Course API to get course info for enrollment confirmation email')
            response = _get_course_content_from_ecommerce(course_id, site_code=site_code)
            if response:
                cache.set(cache_key, response, config.get('SAILTHRU_CACHE_TTL_SECONDS'))

    return response

def _get_course_content_from_ecommerce(course_id, site_code=None):
    """
    Get course information using the Ecommerce course api.

    In case of error returns empty response.
    Arguments:
        course_id (str): course key of the course
        site_code (str): site code

    Returns:
        course information from Ecommerce
    """
    api = get_ecommerce_client(site_code=site_code)
    try:
        api_response = api.courses(course_id).get()
    except Exception:  # pylint: disable=broad-except
        logger.exception(
            'An error occurred while retrieving data for course run [%s] from the Catalog API.',
            course_id,
            exc_info=True
        )
        return {}

    return {
        'title': api_response.get('name'),
        'verification_deadline': api_response.get('verification_deadline')
    }

def _update_unenrolled_list(sailthru_client, email, course_url, unenroll):
    """Maintain a list of courses the user has unenrolled from in the Sailthru user record

    Arguments:
        sailthru_client (object): SailthruClient
        email (str): user's email address
        course_url (str): LMS url for course info page.
        unenroll (boolean): True if unenrolling, False if enrolling

    Returns:
        False if retryable error, else True
    """
    try:
        # get the user 'vars' values from sailthru
        sailthru_response = sailthru_client.api_get("user", {"id": email, "fields": {"vars": 1}})
        if not sailthru_response.is_ok():
            error = sailthru_response.get_error()
            logger.error("Error attempting to read user record from Sailthru: %s", error.get_message())
            return not can_retry_sailthru_request(error)

        response_json = sailthru_response.json

        unenroll_list = []
        if response_json and "vars" in response_json and response_json["vars"] \
           and "unenrolled" in response_json["vars"]:
            unenroll_list = response_json["vars"]["unenrolled"]

        changed = False
        # if unenrolling, add course to unenroll list
        if unenroll:
            if course_url not in unenroll_list:
                unenroll_list.append(course_url)
                changed = True

        # if enrolling, remove course from unenroll list
        elif course_url in unenroll_list:
            unenroll_list.remove(course_url)
            changed = True

        if changed:
            # write user record back
            sailthru_response = sailthru_client.api_post(
                'user', {'id': email, 'key': 'email', 'vars': {'unenrolled': unenroll_list}})

            if not sailthru_response.is_ok():
                error = sailthru_response.get_error()
                logger.error("Error attempting to update user record in Sailthru: %s", error.get_message())
                return not can_retry_sailthru_request(error)

        return True

    except SailthruClientError as exc:
        logger.exception("Exception attempting to update user record for %s in Sailthru - %s", email, text_type(exc))
        return False

def can_retry_sailthru_request(error):
    """ Returns True if a Sailthru request and be re-submitted after an error has occurred.

    Responses with the following codes can be retried:
         9: Internal Error
        43: Too many [type] requests this minute to /[endpoint] API

    All other errors are considered failures, that should not be retried. A complete list of error codes is available at
    https://getstarted.sailthru.com/new-for-developers-overview/api/api-response-errors/.

    Args:
        error (SailthruResponseError)

    Returns:
        bool: Indicates if the original request can be retried.
    """
    code = error.get_error_code()
    return code in (9, 43)

@shared_task(bind=True, ignore_result=True)
def update_course_enrollment(self, email, course_url, purchase_incomplete, mode, unit_cost=None, course_id=None,
                             currency=None, message_id=None, site_code=None, sku=None):
    """Adds/updates Sailthru when a user adds to cart/purchases/upgrades a course

     Args:
        email(str): The user's email address
        course_url(str): Course home page url
        purchase_incomplete(boolean): True if adding to cart
        mode(string): enroll mode (audit, verification, ...)
        unit_cost(decimal): cost if purchase event
        course_id(CourseKey): course id
        currency(str): currency if purchase event - currently ignored since Sailthru only supports USD
        message_id(str): value from Sailthru marketing campaign cookie
        site_code(str): site code

    Returns:
        None
    """
    # Get configuration
    config = get_sailthru_configuration(site_code)

    try:
        sailthru_client = get_sailthru_client(site_code)
    except SailthruError:
        # NOTE: We rely on the function to log the error for us
        return

    # Use event type to figure out processing required
    new_enroll = False
    send_template = None

    if not purchase_incomplete:
        if mode == 'verified':
            # upgrade complete
            send_template = config.get('SAILTHRU_UPGRADE_TEMPLATE')
        elif mode == 'audit' or mode == 'honor':
            # free enroll
            new_enroll = True
            send_template = config.get('SAILTHRU_ENROLL_TEMPLATE')
        else:
            # paid course purchase complete
            new_enroll = True
            send_template = config.get('SAILTHRU_PURCHASE_TEMPLATE')

    # calc price in pennies for Sailthru
    #  https://getstarted.sailthru.com/new-for-developers-overview/advanced-features/purchase/
    cost_in_cents = int(unit_cost * 100)

    # update the "unenrolled" course array in the user record on Sailthru if new enroll or unenroll
    if new_enroll:
        if not _update_unenrolled_list(sailthru_client, email, course_url, False):
            schedule_retry(self, config)

    # Get course data from Sailthru content library or cache
    course_data = _get_course_content(course_id, course_url, sailthru_client, site_code, config)

    # build item description
    item = _build_purchase_item(course_id, course_url, cost_in_cents, mode, course_data, sku)

    # build purchase api options list
    options = {}
    if purchase_incomplete and config.get('SAILTHRU_ABANDONED_CART_TEMPLATE'):
        options['reminder_template'] = config.get('SAILTHRU_ABANDONED_CART_TEMPLATE')
        # Sailthru reminder time format is '+n time unit'
        options['reminder_time'] = "+{} minutes".format(config.get('SAILTHRU_ABANDONED_CART_DELAY'))

    # add appropriate send template
    if send_template:
        options['send_template'] = send_template

    if not _record_purchase(sailthru_client, email, item, purchase_incomplete, message_id, options):
        schedule_retry(self, config)

@shared_task(bind=True, ignore_result=True)
def send_course_refund_email(self, email, refund_id, amount, course_name, order_number, order_url, site_code=None):
    """ Sends the course refund email.

    Args:
        self: Ignore.
        email (str): Recipient's email address.
        refund_id (int): ID of the refund that initiated this task.
        amount (str): Formatted amount of the refund.
        course_name (str): Name of the course for which payment was refunded.
        order_number (str): Order number of the order that was refunded.
        order_url (str): Receipt URL of the refunded order.
        site_code (str): Identifier of the site sending the email.
    """
    config = get_sailthru_configuration(site_code)

    try:
        sailthru_client = get_sailthru_client(site_code)
    except SailthruError:
        # NOTE: We rely on the function to log the error for us
        return

    email_vars = {
        'amount': amount,
        'course_name': course_name,
        'order_number': order_number,
        'order_url': order_url,
    }

    try:
        response = sailthru_client.send(
            template=config['templates']['course_refund'],
            email=email,
            _vars=email_vars
        )
    except SailthruClientError:
        logger.exception(
            'A client error occurred while attempting to send a course refund notification for refund [%d].',
            refund_id
        )
        return

    if response.is_ok():
        logger.info('Course refund notification sent for refund %d.', refund_id)
    else:
        error = response.get_error()
        logger.error(
            'An error occurred while attempting to send a course refund notification for refund [%d]: %d - %s',
            refund_id, error.get_error_code(), error.get_message()
        )

        if can_retry_sailthru_request(error):
            logger.info(
                'An attempt will be made again to send a course refund notification for refund [%d].',
                refund_id
            )
            schedule_retry(self, config)
        else:
            logger.warning(
                'No further attempts will be made to send a course refund notification for refund [%d].',
                refund_id
            )

@shared_task(bind=True, ignore_result=True)
def send_offer_assignment_email(self, user_email, offer_assignment_id, subject, email_body, site_code=None):
    """ Sends the offer assignment email.
    Args:
        self: Ignore.
        user_email (str): Recipient's email address.
        offer_assignment_id (str): Key of the entry in the offer_assignment model.
        subject (str): Email subject.
        email_body (str): The body of the email.
        site_code (str): Identifier of the site sending the email.
    """
    config = get_sailthru_configuration(site_code)
    response = _send_offer_assignment_notification_email(config, user_email, subject, email_body, site_code, self)
    if response and response.is_ok():
        send_id = response.get_body().get('send_id')  # pylint: disable=no-member
        if _update_assignment_email_status(offer_assignment_id, send_id, 'success'):
            logger.info('[Offer Assignment] Offer assignment notification sent with message --- {message}'.format(
                message=email_body))
        else:
            logger.exception(
                '[Offer Assignment] An error occurred while updating email status data for '
                'offer {token_offer} and email {token_email} via the ecommerce API.'.format(
                    token_offer=offer_assignment_id,
                    token_email=user_email,
                )
            )

def _send_offer_assignment_notification_email(config, user_email, subject, email_body, site_code, task):
    """Handles sending offer assignment notification emails and retrying failed emails when appropriate."""
    try:
        sailthru_client = get_sailthru_client(site_code)
    except SailthruError:
        logger.exception(
            '[Offer Assignment] A client error occurred while attempting to send a offer assignment notification.'
            ' Message: {message}'.format(message=email_body)
        )
        return None
    email_vars = {
        'subject': subject,
        'email_body': email_body,
    }
    try:
        response = sailthru_client.send(
            template=config['templates']['assignment_email'],
            email=user_email,
            _vars=email_vars
        )
    except SailthruClientError:
        logger.exception(
            '[Offer Assignment] A client error occurred while attempting to send a offer assignment notification.'
            ' Message: {message}'.format(message=email_body)
        )
        return None

    if not response.is_ok():
        error = response.get_error()
        logger.error(
            '[Offer Assignment] A {token_error_code} - {token_error_message} error occurred'
            ' while attempting to send a offer assignment notification.'
            ' Message: {message}'.format(
                message=email_body,
                token_error_code=error.get_error_code(),
                token_error_message=error.get_message()
            )
        )
        if can_retry_sailthru_request(error):
            logger.info(
                '[Offer Assignment] An attempt will be made to resend the offer assignment notification.'
                ' Message: {message}'.format(message=email_body)
            )
            schedule_retry(task, config)
        else:
            logger.warning(
                '[Offer Assignment] No further attempts will be made to send the offer assignment notification.'
                ' Failed Message: {message}'.format(message=email_body)
            )

    return response

def _update_assignment_email_status(offer_assignment_id, send_id, status, site_code=None):
    """
    Update the offer_assignment and offer_assignment_email model using the Ecommerce assignmentemail api.
    Arguments:
        offer_assignment_id (str): Key of the entry in the offer_assignment model.
        send_id (str): Unique message id from Sailthru
        status (str): status to be sent to the api
        site_code (str): site code
    Returns:
        True or False based on model update status from Ecommerce api
    """
    api = get_ecommerce_client(url_postfix='assignment-email/', site_code=site_code)
    post_data = {
        'offer_assignment_id': offer_assignment_id,
        'send_id': send_id,
        'status': status,
    }
    try:
        api_response = api.status().post(post_data)
    except RequestException:
        logger.exception(
            '[Offer Assignment] An error occurred while updating offer assignment email status for '
            'offer id {token_offer} and message id {token_send_id} via the Ecommerce API.'.format(
                token_offer=offer_assignment_id,
                token_send_id=send_id
            )
        )
        return False
    return True if api_response.get('status') == 'updated' else False

@shared_task(bind=True, ignore_result=True)
def send_offer_update_email(self, user_email, subject, email_body, site_code=None):
    """ Sends the offer emails after assignment, either for revoking or reminding.
    Args:
        self: Ignore.
        user_email (str): Recipient's email address.
        subject (str): Email subject.
        email_body (str): The body of the email.
        site_code (str): Identifier of the site sending the email.
    """
    config = get_sailthru_configuration(site_code)
    _send_offer_assignment_notification_email(config, user_email, subject, email_body, site_code, self)


"""Logging configuration"""
from logging.handlers import SysLogHandler
import os
import platform
import sys

def get_logger_config(log_dir='/var/tmp',
                      logging_env='no_env',
                      edx_filename='edx.log',
                      dev_env=False,
                      debug=False,
                      local_loglevel='INFO',
                      service_variant='ecomworker'):

    """
    Returns a dictionary containing logging configuration.

    If dev_env is True, logging will not be done via local rsyslogd.
    Instead, application logs will be dropped into log_dir. 'edx_filename'
    is ignored unless dev_env is True.
    """
    # Revert to INFO if an invalid string is passed in
    if local_loglevel not in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']:
        local_loglevel = 'INFO'

    hostname = platform.node().split('.')[0]
    syslog_format = (
        '[service_variant={service_variant}]'
        '[%(name)s][env:{logging_env}] %(levelname)s '
        '[{hostname}  %(process)d] [%(filename)s:%(lineno)d] '
        '- %(message)s'
    ).format(
        service_variant=service_variant,
        logging_env=logging_env, hostname=hostname
    )

    if debug:
        handlers = ['console']
    else:
        handlers = ['local']

    logger_config = {
        'version': 1,
        'disable_existing_loggers': False,
        'formatters': {
            'standard': {
                'format': '%(asctime)s %(levelname)s %(process)d '
                          '[%(name)s] %(filename)s:%(lineno)d - %(message)s',
            },
            'syslog_format': {'format': syslog_format},
            'raw': {'format': '%(message)s'},
        },
        'handlers': {
            'console': {
                'level': 'DEBUG' if debug else 'INFO',
                'class': 'logging.StreamHandler',
                'formatter': 'standard',
                'stream': sys.stdout,
            },
        },
        'loggers': {
            'requests': {
                'handlers': handlers,
                'level': 'WARNING',
                'propagate': True
            },
            '': {
                'handlers': handlers,
                'level': 'DEBUG',
                'propagate': False
            },
        }
    }

    if dev_env:
        edx_file_loc = os.path.join(log_dir, edx_filename)
        logger_config['handlers'].update({
            'local': {
                'class': 'logging.handlers.RotatingFileHandler',
                'level': local_loglevel,
                'formatter': 'standard',
                'filename': edx_file_loc,
                'maxBytes': 1024 * 1024 * 2,
                'backupCount': 5,
            },
        })
    else:
        logger_config['handlers'].update({
            'local': {
                'level': local_loglevel,
                'class': 'logging.handlers.SysLogHandler',
                # Use a different address for Mac OS X
                'address': '/var/run/syslog' if sys.platform == 'darwin' else '/dev/log',
                'formatter': 'syslog_format',
                'facility': SysLogHandler.LOG_LOCAL0,
            },
        })

    return logger_config


"""Order fulfillment tasks."""
from ssl import SSLError

from celery import shared_task
from celery.exceptions import Ignore
from celery.utils.log import get_task_logger
from edx_rest_api_client import exceptions

from ecommerce_worker.utils import get_configuration, get_ecommerce_client

logger = get_task_logger(__name__)  # pylint: disable=invalid-name

def _retry_order(self, exception, max_fulfillment_retries, order_number):
    """
    Retry with exponential backoff until fulfillment
    succeeds or the retry limit is reached. If the retry limit is exceeded,
    the exception is re-raised.
    """
    retries = self.request.retries
    if retries == max_fulfillment_retries:
        logger.exception('Fulfillment of order [%s] failed. Giving up.', order_number)
    else:
        logger.warning('Fulfillment of order [%s] failed. Retrying.', order_number)

    countdown = 2 ** retries
    raise self.retry(exc=exception, countdown=countdown, max_retries=max_fulfillment_retries)

@shared_task(bind=True, ignore_result=True)
def fulfill_order(self, order_number, site_code=None, email_opt_in=False):
    """Fulfills an order.

    Arguments:
        order_number (str): Order number indicating which order to fulfill.

    Returns:
        None
    """
    max_fulfillment_retries = get_configuration('MAX_FULFILLMENT_RETRIES', site_code=site_code)
    api = get_ecommerce_client(site_code=site_code)
    try:
        logger.info('Requesting fulfillment of order [%s].', order_number)
        api.orders(order_number).fulfill.put(email_opt_in=email_opt_in)
    except exceptions.HttpClientError as exc:
        status_code = exc.response.status_code  # pylint: disable=no-member
        if status_code == 406:
            # The order is not fulfillable. Therefore, it must be complete.
            logger.info('Order [%s] has already been fulfilled. Ignoring.', order_number)
            raise Ignore()
        else:
            # Unknown client error. Let's retry to resolve it.
            logger.warning(
                'Fulfillment of order [%s] failed because of HttpClientError. Retrying',
                order_number,
                exc_info=True
            )
            _retry_order(self, exc, max_fulfillment_retries, order_number)

    except (exceptions.HttpServerError, exceptions.Timeout, SSLError) as exc:
        # Fulfillment failed, retry
        _retry_order(self, exc, max_fulfillment_retries, order_number)


""" Utility functions. """

from celery.utils.log import get_task_logger
from sailthru.sailthru_client import SailthruClient

from ecommerce_worker.sailthru.v1.exceptions import ConfigurationError, SailthruNotEnabled
from ecommerce_worker.utils import get_configuration

log = get_task_logger(__name__)

def get_sailthru_configuration(site_code):
    """ Returns the Sailthru configuration for the specified site. """
    config = get_configuration('SAILTHRU', site_code=site_code)
    return config

def get_sailthru_client(site_code):
    """
    Returns a Sailthru client for the specified site.

    Args:
        site_code (str): Site for which the client should be configured.

    Returns:
        SailthruClient

    Raises:
        SailthruNotEnabled: If Sailthru is not enabled for the specified site.
        ConfigurationError: If either the Sailthru API key or secret are not set for the site.
    """
    # Get configuration
    config = get_sailthru_configuration(site_code)

    # Return if Sailthru integration disabled
    if not config.get('SAILTHRU_ENABLE'):
        msg = 'Sailthru is not enabled for site {}'.format(site_code)
        log.debug(msg)
        raise SailthruNotEnabled(msg)

    # Make sure key and secret configured
    key = config.get('SAILTHRU_KEY')
    secret = config.get('SAILTHRU_SECRET')

    if not (key and secret):
        msg = 'Both key and secret are required for site {}'.format(site_code)
        log.error(msg)
        raise ConfigurationError(msg)

    return SailthruClient(key, secret)


"""
This file contains a primitive cache
"""
import threading
import time

lock = threading.Lock()  # pylint: disable=invalid-name

class CacheObject(object):
    """Object saved in cache"""
    def __init__(self, value, duration):
        self.value = value
        self.expire = time.time() + duration

class Cache(dict):
    """
    Primitive key/value cache.  Entries are kept in a dict with an expiration.
    When a get of an expired entry is done, the cache is cleaned of all expired entries.
    Locking is used for thread safety
    """
    def get(self, key):
        """Get an object from the cache

        Arguments:
            key (str): Cache key

        Returns:
            Cached object
        """
        lock.acquire()
        try:
            if key not in self:
                return None

            current_time = time.time()
            if self[key].expire > current_time:
                return self[key].value

            # expired key, clean out all expired keys
            deletes = []
            for k, val in self.items():
                if val.expire <= current_time:
                    deletes.append(k)
            for k in deletes:
                del self[k]

            return None
        finally:
            lock.release()

    def set(self, key, value, duration):
        """Save an object in the cache

        Arguments:
            key (str): Cache key
            value (object): object to cache
            duration (int): time in seconds to keep object in cache

        """
        lock.acquire()
        try:
            self[key] = CacheObject(value, duration)
        finally:
            lock.release()


"""Helper functions."""
import os
import sys

from edx_rest_api_client.client import EdxRestApiClient

from ecommerce_worker.configuration import CONFIGURATION_MODULE

def get_configuration(variable, site_code=None):
    """
    Get a value from configuration.

    Retrieves the value corresponding to the given variable from the configuration module
    currently in use by the app.  Specify a site_code value to check for a site-specific override.

    Arguments:
        variable (str): The name of a variable from the configuration module.

    Keyword Arguments:
        site_code (str): The SITE_OVERRIDES key to inspect for site-specific values

    Returns:
        The value corresponding to the variable, or None if the variable is not found.
    """
    name = os.environ.get(CONFIGURATION_MODULE)

    # __import__ performs a full import, but only returns the top-level
    # package, not the targeted module. sys.modules is a dictionary
    # mapping module names to loaded modules.
    __import__(name)
    module = sys.modules[name]

    # Locate the setting in the specified module, then attempt to apply a site-specific override
    setting_value = getattr(module, variable, None)
    site_overrides = getattr(module, 'SITE_OVERRIDES', None)
    if site_overrides and site_code is not None:
        site_specific_overrides = site_overrides.get(site_code)
        if site_specific_overrides:
            override_value = site_specific_overrides.get(variable)
            if override_value:
                setting_value = override_value

    if setting_value is None:
        raise RuntimeError('Worker is improperly configured: {} is unset in {}.'.format(variable, module))
    return setting_value

def get_ecommerce_client(url_postfix='', site_code=None):
    """
    Get client for fetching data from ecommerce API.
    Arguments:
        site_code (str): (Optional) The SITE_OVERRIDES key to inspect for site-specific values
        url_postfix (str): (Optional) The URL postfix value to append to the ECOMMERCE_API_ROOT value.

    Returns:
        EdxRestApiClient object
    """
    ecommerce_api_root = get_configuration('ECOMMERCE_API_ROOT', site_code=site_code)
    signing_key = get_configuration('JWT_SECRET_KEY', site_code=site_code)
    issuer = get_configuration('JWT_ISSUER', site_code=site_code)
    service_username = get_configuration('ECOMMERCE_SERVICE_USERNAME', site_code=site_code)
    return EdxRestApiClient(
        ecommerce_api_root + url_postfix, signing_key=signing_key, issuer=issuer, username=service_username)


import os

# Environment variable indicating which configuration module to use
# when running the worker.
CONFIGURATION_MODULE = 'WORKER_CONFIGURATION_MODULE'

def get_overrides_filename(variable):
    """
    Get the name of the file containing configuration overrides
    from the provided environment variable.
    """
    filename = os.environ.get(variable)

    if filename is None:
        msg = 'Please set the {} environment variable.'.format(variable)
        raise EnvironmentError(msg)

    return filename


from .util import make_enum, get_value_by_version, OS_NAME

OUTPUT_FILES_LAYOUTS = make_enum(
    "eplusout",  # {simulation_dir_path}/eplusout.{extension}
    "simu",  # {simulation_dir_path}/{simulation_base_name}.{extension}
    "output_simu",  # {simulation_dir_path}/Output/{simulation_base_name}.{extension}
    "simu_table",  # {simulation_dir_path}/{simulation_base_name}Table.csv
    "output_simu_table",  # {simulation_dir_path}/Output/{simulation_base_name}Table.csv
    "eplustbl",  # {simulation_dir_path}/eplusout.csv
)

_layouts_matrix = {
    "windows": {
        "inputs": {
            (0, 0):  "simu",
            (8, 2): "eplusout"
        },
        "table": {
            (0, 0): "simu_table",
            (8, 2): "eplustbl",
        },
        "other": {
            (0, 0): "simu",
            (8, 2): "eplusout"
        }
    },
    "osx": {
        "inputs": {
            (0, 0): "output_simu",
            (8, 2): "simu"

        },
        "table": {
            (0, 0): "output_simu_table",
            (8, 2): "eplustbl",
        },
        "other": {
            (0, 0): "output_simu",
            (8, 2): "eplusout"
        }
    },
    "linux": {
        "inputs": {
            (0, 0): "output_simu",
            (8, 5): "eplusout"

        },
        "table": {
            (0, 0): "output_simu_table",
            (8, 5): "eplustbl",
        },
        "other": {
            (0, 0): "output_simu",
            (8, 5): "eplusout"
        }
    }
}

def get_output_files_layout(output_category):
    """
    Parameters
    ----------
    output_category: str
        inputs: epw, idf
        table: summary table
        other: other
    """
    # check  category
    if output_category not in ("inputs", "table", "other"):
        raise RuntimeError(f"unknown {output_category}")

    # get version dict
    layouts = _layouts_matrix[OS_NAME][output_category]

    # get version
    return get_value_by_version(layouts)


import collections
import platform
import re

SYS_NAME = platform.system()
if SYS_NAME in ("Windows",):  # windows
    OS_NAME = "windows"
elif SYS_NAME in ("Darwin",):  # mac osx
    OS_NAME = "osx"
elif SYS_NAME in ("Linux",):  # linux
    OS_NAME = "linux"
else:
    raise RuntimeError("Unknown platform.system(): '%s'." % SYS_NAME)

if OS_NAME == "windows":
    APPS_DIR_PATH, EPLUS_DIR_PATTERN = r"C:\\", re.compile("EnergyPlusV(\d*)-(\d*)-(\d*)")
elif OS_NAME == "osx":  # mac osx
    APPS_DIR_PATH, EPLUS_DIR_PATTERN = "/Applications", re.compile("[eE]nergy[pP]lus-(\d*)[-\.](\d*)[-\.](\d*)")
elif OS_NAME == "linux":  # linux
    # became lowercase and dots for >= 9.0.1
    APPS_DIR_PATH, EPLUS_DIR_PATTERN = "/usr/local", re.compile("[eE]nergy[pP]lus-(\d*)[-\.](\d*)[-\.](\d*)")
else:
    raise RuntimeError("Unknown os_name: '%s'" % OS_NAME)

def get_value_by_version(d):
    """
    Finds the value depending in current eplus version.

    Parameters
    ----------
    d: dict
        {(0, 0): value, (x, x): value, ...}
        for current version (cv), current value is the value of version v such as v <= cv < v+1
    """
    from oplus import CONF  # touchy import

    cv = CONF.eplus_version[:2]
    for v, value in sorted(d.items(), reverse=True):
        if cv >= v:
            return value

def make_enum(*keys):
    Enum = collections.namedtuple("Enum", keys)
    return Enum(**dict([(k, k) for k in keys]))


import datetime as dt
import pandas as pd
from pandas.util.testing import assert_index_equal

from .parse_eso import TIMESTEP, HOURLY, DAILY, MONTHLY, ANNUAL, RUN_PERIOD

def switch_to_datetime_instants(df, start_year, eplus_frequency):
    """
    works inplace
    """
    # timestep -> monthly
    if eplus_frequency in (TIMESTEP, DAILY, HOURLY, MONTHLY):
        # prepare year switch
        if eplus_frequency in (TIMESTEP, HOURLY, DAILY):
            # print((df[["month", "day"]] - df[["month", "day"]].shift()) == pd.Series([-12, -31]))
            year_counter = (
                    (df[["month", "day"]] - df[["month", "day"]].shift()) ==
                    pd.Series(dict(month=12, day=-31))
            ).all(axis=1).cumsum()
        else:
            year_counter = ((df["month"] - df["month"].shift()) == -12).cumsum()

        # add year columns
        df["year"] = year_counter + start_year

        # create index
        columns = {
            TIMESTEP: ("year", "month", "day", "hour", "minute"),
            HOURLY: ("year", "month", "day", "hour"),
            DAILY: ("year", "month", "day"),
            MONTHLY: ("year", "month")
        }[eplus_frequency]
        if eplus_frequency == MONTHLY:
            df.index = df.apply(  # apply transforms ints to floats, we need to re-cast
                lambda x: dt.datetime(*(tuple(int(x[k]) for k in columns) + (1,))),
                axis=1
            )
        else:
            df.index = df.apply(lambda x: dt.datetime(*(int(x[k]) for k in columns)), axis=1)

        # drop old columns
        df.drop(columns=list(columns), inplace=True)

        # force frequency
        if eplus_frequency == TIMESTEP:
            # find freq
            ts = df.index[1] - df.index[0]
            # force
            forced_df = df.asfreq(ts)
        else:
            forced_df = df.asfreq({
                HOURLY: "H",
                DAILY: "D",
                MONTHLY: "MS"
            }[eplus_frequency])

        # if timestep, hourly or daily, check did not change (only those can suffer from leap year problems)
        if eplus_frequency in (TIMESTEP, HOURLY, DAILY):
            try:
                assert_index_equal(df.index, forced_df.index)
            except AssertionError:
                raise ValueError(
                    f"Couldn't convert to datetime instants (frequency: {eplus_frequency}). Probable cause : "
                    f"given start year ({start_year}) is incorrect and data can't match because of leap year issues."
                ) from None

        return forced_df

    # annual
    if eplus_frequency == ANNUAL:
        # check first year
        if df["year"].iloc[0] != start_year:
            raise ValueError(
                f"Given start year ({start_year}) differs from annual output data first year ({df['year'].iloc[0]}),"
                f"can't switch to datetime instants.")
        df.index = df["year"].map(lambda x: dt.datetime(x, 1, 1))
        del df["year"]

        # force freq
        df = df.asfreq("YS")

        return df

    # run period
    if eplus_frequency == RUN_PERIOD:
        return df

    raise AssertionError("should not be here")


import os

from .compatibility import APPS_DIR_PATH, EPLUS_DIR_PATTERN

# ------------------------------------------------- DEFAULT VARIABLES --------------------------------------------------
class _Conf:
    encoding = "latin-1"  # even needed for example files...
    default_model_name = "oplus"
    external_files_suffix = "-external"

    def __init__(self):
        self.eplus_available_versions = {}  # {version: base_dir_path, ...
        self._eplus_version = None  # (i1, i2, i3)

        # set available versions
        self._set_available_versions()

    def _set_available_versions(self):
        # find most recent version of EnergyPlus
        for file_name in os.listdir(APPS_DIR_PATH):
            match = EPLUS_DIR_PATTERN.search(file_name)
            if match is not None:
                version_tuple = (int(match.group(1)), int(match.group(2)), int(match.group(3)))
                self.eplus_available_versions[version_tuple] = os.path.join(APPS_DIR_PATH, file_name)

    @property
    def eplus_version(self):
        """
        if _eplus_version is defined => _eplus_version
        else most recent eplus available version
        """
        # check energy plus is installed
        if len(self.eplus_available_versions) == 0:
            raise RuntimeError("Energy plus is not install, can't use oplus package.")

        # see if version is defined
        if self._eplus_version is not None:
            return self._eplus_version

        # return most recent version
        return sorted(self.eplus_available_versions.keys(), reverse=True)[0]

    @eplus_version.setter
    def eplus_version(self, value):
        # check version is available
        if value not in self.eplus_available_versions:
            raise RuntimeError(f"Eplus version {value} was not found in available versions.")

        # set
        self._eplus_version = value

    @property
    def eplus_base_dir_path(self):
        return self.eplus_available_versions[self.eplus_version]

CONF = _Conf()


import collections
import datetime as dt

import pandas as pd
from pandas.util.testing import assert_index_equal

from ..util import multi_mode_write, get_mono_line_copyright_message, to_buffer

WEEK_DAYS = ("Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday")

INSTANTS_COLUMNS = (
    "year",
    "month",
    "day",
    "hour",
    "minute"
)

WEATHER_SERIES_DEFAULTS = collections.OrderedDict((  # if None: mandatory field, else optional
    ("datasource", ""),
    ("drybulb", None),
    ("dewpoint", None),
    ("relhum", None),
    ("atmos_pressure", None),
    ("exthorrad", 9999),
    ("extdirrad", 9999),
    ("horirsky", 9999),
    ("glohorrad", None),
    ("dirnorrad", None),
    ("difhorrad", None),
    ("glohorillum", 999999),
    ("dirnorillum", 999999),
    ("difhorillum", 999999),
    ("zenlum", 9999),
    ("winddir", None),
    ("windspd", None),
    ("totskycvr", 99),
    ("opaqskycvr", 99),
    ("visibility", 9999),
    ("ceiling_hgt", 99999),
    ("presweathobs", 999),
    ("presweathcodes", 999),
    ("precip_wtr", 999),
    ("aerosol_opt_depth", 999),
    ("snowdepth", 999),
    ("days_last_snow", 99),
    ("Albedo", 999),
    ("liq_precip_depth", 999),
    ("liq_precip_rate", 99)
))

mandatory_columns = tuple(k for k, v in WEATHER_SERIES_DEFAULTS.items() if v is None)  # we use tuple for immutability

def to_str(value):
    return "" if value is None else str(value)

class WeatherData:
    def __init__(
            self,
            weather_series,  # dataframe
            latitude,
            longitude,
            timezone_offset,
            elevation,
            city=None,
            state_province_region=None,
            country=None,
            source=None,
            wmo=None,
            design_condition_source=None,
            design_conditions=None,  # list of design conditions
            typical_extreme_periods=None,  # list of typical/extreme periods
            ground_temperatures=None,  # list of ground temperatures
            leap_year_observed="no",
            daylight_savings_start_day=0,
            daylight_savings_end_day=0,
            holidays=None,  # [(name, day), ...]
            comments_1="",
            comments_2="",
            start_day_of_week=None
    ):
        """
        For more information on following concepts, see EnergyPlus documentation :
            AuxiliaryPrograms.pdf: Weather Converter Program/EnergyPlus Weather File (EPW) Data Dictionary

        Parameters
        ----------
        weather_series: dataframe
            Must at least contain mandatory columns: 'drybulb', 'dewpoint', 'relhum', 'atmos_pressure', 'glohorrad',
            'dirnorrad', 'difhorrad', 'winddir',  'windspd'.

            Two instants modes are available:
             - datetime instants mode: index must be a hourly datetime index
             - tuple mode: index does not matter, and dataframe must have following instants columns: 'year', 'month',
                 'day', 'hour', 'minute'. 'hour' is not written in a epw fashion ([0, 23], not [1, 24] as epw).

            Note that it will be possible, after weather data is created, to switch from datetime to tuple instants (or
            vice versa).
        latitude
        longitude
        timezone_offset
        elevation
        city
        state_province_region
        country
        source
        wmo
        design_condition_source
        design_conditions
        typical_extreme_periods
        ground_temperatures
        leap_year_observed
        daylight_savings_start_day
        daylight_savings_end_day
        holidays
        comments_1
        comments_2
        start_day_of_week
        """
        # instants may be datetimes or tuples
        self._weather_series = _check_and_sanitize_weather_series(weather_series)

        # manage start day of week (only relevant if tuples datetimes)
        self._start_day_of_week = None
        if self.has_tuple_instants and start_day_of_week is not None:
            self._start_day_of_week = start_day_of_week
        else:
            self._set_start_day_of_week()

        # todo: check headers
        self._headers = dict(
            # mandatory location
            latitude=latitude,
            longitude=longitude,
            timezone_offset=timezone_offset,
            elevation=elevation,
            # optional location
            city=city,
            state_province_region=state_province_region,
            country=country,
            source=source,
            wmo=wmo,
            # design conditions
            design_condition_source=design_condition_source,
            design_conditions=[] if design_conditions is None else design_conditions,
            # typical/extreme periods
            typical_extreme_periods=[] if typical_extreme_periods is None else typical_extreme_periods,
            # ground temperatures
            ground_temperatures=[] if ground_temperatures is None else ground_temperatures,
            # holidays/daylight savings
            leap_year_observed=leap_year_observed,
            daylight_savings_start_day=daylight_savings_start_day,
            daylight_savings_end_day=daylight_savings_end_day,
            holidays=[] if holidays is None else holidays,  # [(name, day), ...]
            # comments
            comments_1=comments_1,
            comments_2=comments_2
        )

    def _headers_to_epw(self):
        location = [
            "LOCATION",
            to_str(self._headers["city"]),
            to_str(self._headers["state_province_region"]),
            to_str(self._headers["country"]),
            to_str(self._headers["source"]),
            to_str(self._headers["wmo"]),
            to_str(self._headers["latitude"]),
            to_str(self._headers["longitude"]),
            to_str(self._headers["timezone_offset"]),
            to_str(self._headers["elevation"])
        ]

        # design conditions
        design_conditions = ["DESIGN CONDITIONS", len(self._headers["design_conditions"])]
        for dc in self._headers["design_conditions"]:
            design_conditions.extend([dc.name] + dc.values)

        # typical/extreme periods
        typical_extreme_periods = ["TYPICAL/EXTREME PERIODS", len(self._headers["typical_extreme_periods"])
                                   ]
        for tep in self._headers["typical_extreme_periods"]:
            typical_extreme_periods.extend([tep.name, tep.period_type, tep.start_day, tep.end_day])

        # ground temperatures
        ground_temperatures = ["GROUND TEMPERATURES", len(self._headers["ground_temperatures"])]
        for gt in self._headers["ground_temperatures"]:
            ground_temperatures.extend(
                [
                    gt.depth,
                    gt.soil_conductivity,
                    gt.soil_density,
                    gt.soil_specific_heat
                ] +
                gt.monthly_average_ground_temperatures
            )

        # holidays/daylight savings
        holidays_daylight_savings = [
            "HOLIDAYS/DAYLIGHT SAVINGS",
            self._headers["leap_year_observed"],
            self._headers["daylight_savings_start_day"],
            self._headers["daylight_savings_end_day"],
            len(self._headers["holidays"])
        ]
        for name, day in self._headers["holidays"]:
            holidays_daylight_savings.extend([name, day])

        # comments
        comments_1 = get_mono_line_copyright_message()
        if (comments_1 not in self._headers["comments_1"]) and (self._headers["comments_1"] != ""):
            comments_1 += " ; " + self._headers["comments_1"]
        comments_1 = ["COMMENTS 1", comments_1]
        comments_2 = ["COMMENTS 2", to_str(self._headers["comments_2"])]

        # data periods
        if self.has_datetime_instants:
            start_timestamp = self._weather_series.index[0]
            end_timestamp = self._weather_series.index[-1]
            data_periods = [
                "DATA PERIODS",
                1,
                1,
                WEEK_DAYS[start_timestamp.weekday()],
                f"{start_timestamp.month}/{start_timestamp.day}",
                f"{end_timestamp.month}/{end_timestamp.day}"
            ]
        else:
            start_month, start_day = self._weather_series["month"].iloc[0], self._weather_series["day"].iloc[0]
            end_month, end_day = self._weather_series["month"].iloc[-1], self._weather_series["day"].iloc[-1]
            data_periods = [
                "DATA PERIODS",
                1,
                1,
                self._start_day_of_week,
                f"{start_month}/{start_day}",
                f"{end_month}/{end_day}"
            ]

        return "\n".join([",".join([str(cell) for cell in row]) for row in (
            location,
            design_conditions,
            typical_extreme_periods,
            ground_temperatures,
            holidays_daylight_savings,
            comments_1,
            comments_2,
            data_periods
        )]) + "\n"

    def _set_start_day_of_week(self):
        if self.has_datetime_instants:
            date = self._weather_series.index[0]
        else:
            date = dt.date(
                self._weather_series["year"].iloc[0],
                self._weather_series["month"].iloc[0],
                self._weather_series["day"].iloc[0]
            )

        self._start_day_of_week = WEEK_DAYS[date.weekday()]

    # ------------------------------------------------ public api ------------------------------------------------------
    @property
    def has_datetime_instants(self):
        return isinstance(self._weather_series.index, pd.DatetimeIndex)

    @property
    def has_tuple_instants(self):
        return not self.has_datetime_instants

    def switch_to_tuple_instants(self):
        # don't switch if not relevant
        if self.has_tuple_instants:
            return
        # remove datetime index
        self._weather_series.index = range(len(self._weather_series))

    def switch_to_datetime_instants(self):
        # don't switch if not relevant
        if self.has_datetime_instants:
            return

        # create index
        index = pd.DatetimeIndex(self._weather_series.apply(
            lambda x: dt.datetime(x.year, x.month, x.day, x.hour, x.minute),
            axis=1
        ))

        # set new index
        self._weather_series.index = index

        # check and sanitize
        self._weather_series = _check_and_sanitize_datetime_instants(self._weather_series)

        # remove old start day of week
        self._start_day_of_week = None

        # set new start day of week
        self._set_start_day_of_week()

    def get_weather_series(self):
        """
        Returns
        -------
        weather series dataframe, which contains all the timeseries data. Will be a datetime series or a tuple instants
        series depending on current mode.
        """
        return self._weather_series.copy()

    def get_bounds(self):
        """
        Returns
        -------
        (start, end)

        Datetime instants of beginning and end of data. If no data, will be: (None, None).
        """
        start, end = None, None
        if len(self._weather_series) == 0:
            return start, end

        for i in (0, -1):
            # create or find instant
            if self.has_tuple_instants:
                row = self._weather_series.iloc[i, :]
                instant = dt.datetime(row["year"], row["month"], row["day"], row["hour"], row["minute"])
            else:
                instant = self._weather_series.index[i].to_pydatetime()

            # store
            if i == 0:
                start = instant
            else:
                end = instant

        return start, end

    def get_info(self):
        start, end = self.get_bounds()
        if start is None:
            start, end = "no data", "no data"

        msg = "WeatherData\n"
        msg += f"\tinstants: {'tuple' if self.has_tuple_instants else 'datetime'}\n"
        for k in ("latitude", "longitude", "timezone_offset", "elevation"):
            msg += f"\t{k}: {self._headers[k]}\n"
        msg += f"\tdata period: {start.isoformat()}, {end.isoformat()}"
        return msg

    # ------------------------------------------------- load -----------------------------------------------------------
    @classmethod
    def from_epw(cls, buffer_or_path):
        """
        Parameters
        ----------
        buffer_or_path: buffer or path containing epw format.

        Returns
        -------
        WeatherData instance.
        """
        from .epw_parse import parse_epw
        _, buffer = to_buffer(buffer_or_path)
        with buffer as f:
            return parse_epw(f)

    # ----------------------------------------------- export -----------------------------------------------------------
    def to_epw(self, buffer_or_path=None):
        """
        Parameters
        ----------
        buffer_or_path: buffer or path, default None
            Buffer or path to write into. If None, will return a string containing epw info.

        Returns
        -------
        None or a string if buffer_or_path is None.
        """
        # copy and change hours convention [0, 23] -> [1, 24]
        df = self._weather_series.copy()
        df["hour"] += 1
        epw_content = self._headers_to_epw() + df.to_csv(header=False, index=False, line_terminator="\n")
        return multi_mode_write(
            lambda buffer: buffer.write(epw_content),
            lambda: epw_content,
            buffer_or_path=buffer_or_path
        )

def _check_and_sanitize_weather_series(df):
    # check dataframe
    if not isinstance(df, pd.DataFrame):
        raise TypeError("Weather series must be a pandas DataFrame.")

    # check data columns
    given_columns = set(df.columns)
    diff = set(mandatory_columns).difference(given_columns)
    if len(diff) != 0:
        raise ValueError(f"Missing mandatory columns: {diff}.")

    # prepare data container
    data = collections.OrderedDict()

    # check index info
    is_datetime_index = False
    if isinstance(df.index, pd.DatetimeIndex):  # datetime index mode
        is_datetime_index = True

        # check and sanitize index
        df = _check_and_sanitize_datetime_instants(df)

        # prepare instant columns
        data.update((
            ("year", df.index.year),
            ("month", df.index.month),
            ("day", df.index.day),
            ("hour", df.index.hour),
            ("minute", 0)
        ))

    else:
        # check instant columns
        diff = set(INSTANTS_COLUMNS).difference(given_columns)
        if len(diff) != 0:
            raise ValueError(f"Missing mandatory columns: {diff}.")

        # prepare instant columns
        data.update((k, df[k]) for k in ("year", "month", "day", "hour", "minute"))

    # add data columns
    data.update(collections.OrderedDict(
            (k, df[k] if k in given_columns else v) for k, v in WEATHER_SERIES_DEFAULTS.items())
    )

    # create
    sanitized_df = pd.DataFrame(data, index=df.index if is_datetime_index else None)

    # check no empty values
    nan_columns = set(sanitized_df.columns[sanitized_df.isnull().sum() > 0])
    if len(nan_columns) > 0:
        raise ValueError(f"Some columns contain null values: {tuple(sorted(nan_columns))}")

    # prepare
    return sanitized_df

def _check_and_sanitize_datetime_instants(df):
    """
    Parameters
    ----------
    df

    Returns
    -------
    sanitized df
    """
    # leave if not relevant
    if df is None or len(df) == 0:
        return df

    # check datetime index
    if not isinstance(df.index, pd.DatetimeIndex):
        raise ValueError("df index must be a datetime index.")

    # force frequency if needed
    if df.index.freq != "H":
        forced_df = df.asfreq("H")
        # check no change
        try:
            assert_index_equal(df.index, forced_df.index)
        except AssertionError:
            raise ValueError(
                f"Couldn't convert to hourly datetime instants. Probable cause : "
                f"given start instant ({df.index[0]}) is incorrect and data can't match because of leap year issues."
            ) from None
        # replace old variable
        df = forced_df

    # check first minute is 0
    if df.index[0].minute != 0:
        raise ValueError("Minutes must be 0.")

    return df


from ..util import get_multi_line_copyright_message
from .util import table_name_to_ref

def parse_idf(file_like):
    """
    Records are created from string.
    They are not attached to idf yet.
    in idf: header comment, chapter comments, records
    in record: head comment, field comments, tail comment
    """
    tables_data = {}
    head_comment = ""
    record_data = None
    make_new_record = True

    copyright_list = get_multi_line_copyright_message().split("\n")

    for i, raw_line in enumerate(file_like):
        # manage if copyright
        try:
            copyright_line = copyright_list[i]
            if raw_line.strip() == copyright_line:
                # skip copyright line
                continue
        except IndexError:
            pass

        # GET LINE CONTENT AND COMMENT
        split_line = raw_line.split("!")

        # no "!" in the raw_line
        if len(split_line) == 1:
            # this is an empty line
            if len(split_line[0].strip()) == 0:
                content, comment = None, None
            # this is a record line with no comments
            else:
                content, comment = split_line[0].strip(), None
        # there is at least one "!" in the raw_line
        else:
            # this is a comment line
            if len(split_line[0].strip()) == 0:
                content, comment = None, "!".join(split_line[1:])
            # this is a record line with a comment
            else:
                content, comment = split_line[0].strip(), "!".join(split_line[1:])

        # SKIP CURRENT LINE IF VOID
        if (content, comment) == (None, None):
            continue

        # NO CONTENT
        if not content:
            if record_data is None:  # we only manage head idf comment
                head_comment += comment.strip() + "\n"
            continue

        # CONTENT
        # check if record end and prepare
        record_end = content[-1] == ";"
        content = content[:-1]  # we tear comma or semi-colon
        content_l = [text.strip() for text in content.split(",")]

        # record creation if needed
        if make_new_record:
            # get table ref
            table_ref = table_name_to_ref(content_l[0].strip())

            # skip if special table
            if table_ref.lower() in (
                    "lead input",
                    "end lead input",
                    "simulation data",
                    "end simulation data"
            ):
                continue

            # declare table if necessary
            if table_ref not in tables_data:
                tables_data[table_ref] = []

            # create and store record
            record_data = dict()
            tables_data[table_ref].append(record_data)

            # prepare in case fields on the same line
            content_l = content_l[1:]
            make_new_record = False

        # fields
        for value_s in content_l:
            field_index = len(record_data)
            record_data[field_index] = value_s

        # signal that new record must be created
        if record_end:
            make_new_record = True

    # add comment key
    tables_data["_comment"] = head_comment
    return tables_data


import os
import shutil
import stat
import logging
import collections

from oplus.configuration import CONF
from oplus.util import run_subprocess, LoggerStreamWriter
from oplus import Epm, WeatherData
from oplus.epm.idd import Idd
from oplus.standard_output.standard_output import StandardOutput
from oplus.mtd import Mtd
from oplus.eio import Eio
from oplus.err import Err
from oplus.summary_table import SummaryTable

from .compatibility import OUTPUT_FILES_LAYOUTS, SIMULATION_INPUT_COMMAND_STYLES, SIMULATION_COMMAND_STYLES, \
    get_output_files_layout, get_simulated_epw_path, get_simulation_base_command, get_simulation_input_command_style, \
    get_simulation_command_style

DEFAULT_SERVER_PERMS = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP

_refs = (
    "idf",
    "epw",
    "eio",
    "eso",
    "mtr",
    "mtd",
    "mdd",
    "err",
    "summary_table"
)
_FileRefEnum = collections.namedtuple("_FileRefEnum", _refs)

FILE_REFS = _FileRefEnum(**dict([(k, k) for k in _refs]))

class FileInfo:
    def __init__(self, constructor, get_path):
        self.constructor = constructor
        self.get_path = get_path

def get_input_file_path(dir_path, file_ref):
    if file_ref not in (FILE_REFS.idf, FILE_REFS.epw):
        raise ValueError("'%s' file ref is not an input file")
    return os.path.join(dir_path, "%s.%s" % (CONF.default_model_name, file_ref))

def get_output_file_path(dir_path, file_ref):
    # set category
    if file_ref in (
        FILE_REFS.idf,
        FILE_REFS.epw
    ):
        output_category = "inputs"

    elif file_ref == FILE_REFS.summary_table:
        output_category = "table"

    elif file_ref in (FILE_REFS.eio, FILE_REFS.eso, FILE_REFS.mtr, FILE_REFS.mtd, FILE_REFS.mdd, FILE_REFS.err):
        output_category = "other"
    else:
        raise ValueError(f"unknown file_ref: {file_ref}")

    # get layout
    layout = get_output_files_layout(output_category)

    # return path
    if layout == OUTPUT_FILES_LAYOUTS.eplusout:
        return os.path.join(dir_path, "eplusout.%s" % file_ref)

    if layout == OUTPUT_FILES_LAYOUTS.simu:
        return os.path.join(dir_path, "%s.%s" % (CONF.default_model_name, file_ref))

    if layout == OUTPUT_FILES_LAYOUTS.output_simu:
        return os.path.join(dir_path, "Output", "%s.%s" % (CONF.default_model_name, file_ref))

    if layout == OUTPUT_FILES_LAYOUTS.simu_table:
        return os.path.join(dir_path, "%sTable.csv" % CONF.default_model_name)

    if layout == OUTPUT_FILES_LAYOUTS.output_simu_table:
        return os.path.join(dir_path, "Output", "%sTable.csv" % CONF.default_model_name)

    if layout == OUTPUT_FILES_LAYOUTS.eplustbl:
        return os.path.join(dir_path, "eplustbl.csv")

    raise RuntimeError("unknown file_ref")

def _copy_without_read_only(src, dst):
    shutil.copy2(src, dst)
    # ensure not read only
    os.chmod(dst, DEFAULT_SERVER_PERMS)

class Simulation:
    # for subclassing
    _idd_cls = Idd
    _epm_cls = Epm
    _weather_data_cls = WeatherData
    _standard_output_cls = StandardOutput
    _mtd_cls = Mtd
    _eio_cls = Eio
    _summary_table_cls = SummaryTable
    _err_cls = Err

    @classmethod
    def simulate(
            cls,
            epm_or_path,
            weather_data_or_path,
            base_dir_path,
            simulation_name=None,
            stdout=None,
            stderr=None,
            beat_freq=None
    ):
        """
        Parameters
        ----------
        epm_or_path
        weather_data_or_path
        base_dir_path: simulation dir path
        simulation_name: str, default None
            if provided, simulation will be done in {base_dir_path}/{simulation_name}
            else, simulation will be done in {base_dir_path}
        stdout: stream, default logger.info
            stream where EnergyPlus standard output is redirected
        stderr: stream, default logger.error
            stream where EnergyPlus standard error is redirected
        beat_freq: float, default None
            if provided, subprocess in which EnergyPlus is run will write at given frequency in standard output. May
            be used to monitor subprocess state.

        Returns
        -------
        Simulation instance
        """
        # manage simulation dir path
        if not os.path.isdir(base_dir_path):
            raise NotADirectoryError("Base dir path not found: '%s'" % base_dir_path)
        simulation_dir_path = base_dir_path if simulation_name is None else os.path.join(base_dir_path, simulation_name)

        # make directory if does not exist
        if not os.path.exists(simulation_dir_path):
            os.mkdir(simulation_dir_path)

        # run simulation
        stdout = LoggerStreamWriter(logger_name=__name__, level=logging.INFO) if stdout is None else stdout
        stderr = LoggerStreamWriter(logger_name=__name__, level=logging.ERROR) if stderr is None else stderr
        run_eplus(
            epm_or_path,
            weather_data_or_path,
            simulation_dir_path,
            stdout=stdout,
            stderr=stderr,
            beat_freq=beat_freq
        )

        # return simulation object
        return cls(
            base_dir_path,
            simulation_name=simulation_name
        )

    def __init__(
            self,
            base_dir_path,
            simulation_name=None
    ):
        """

        Parameters
        ----------
        base_dir_path: simulation dir path
        simulation_name: str, default None
            if provided, simulation will be looked for in {base_dir_path}/{simulation_name}
            else, simulation will be looked for in {base_dir_path}

        A simulation is not characterized by it's input files but by it's base_dir_path. This approach makes it
        possible to load an already simulated directory without having to define it's idf or epw.
        """
        self._dir_path = base_dir_path if simulation_name is None else os.path.join(base_dir_path, simulation_name)
        self._prepared_file_refs = None
        self._outputs_cache = dict()

        # check simulation directory path exists
        if not os.path.isdir(self._dir_path):
            raise NotADirectoryError("Simulation directory does not exist: '%s'." % self._dir_path)

    def _check_file_ref(self, file_ref):
        if file_ref not in self._prepared_file_refs:
            raise ValueError("Unknown extension: '%s'." % file_ref)

    def _path(self, file_ref):
        return self._file_refs[file_ref].get_path()

    @property
    def _file_refs(self):
        """
        Defined here so that we can use the class variables, in order to subclass in oplusplus
        """
        if self._prepared_file_refs is None:
            self._prepared_file_refs = {
                FILE_REFS.idf: FileInfo(
                    constructor=lambda path: self._epm_cls.from_idf(path, idd_or_buffer_or_path=self._idd),
                    get_path=lambda: get_input_file_path(self.dir_path, FILE_REFS.idf)
                ),
                FILE_REFS.epw: FileInfo(
                    constructor=lambda path: self._weather_data_cls.from_epw(path),
                    get_path=lambda: get_input_file_path(self.dir_path, FILE_REFS.epw)
                ),
                FILE_REFS.eio: FileInfo(
                    constructor=lambda path: self._eio_cls(path),
                    get_path=lambda: get_output_file_path(self.dir_path, FILE_REFS.eio)
                ),
                FILE_REFS.eso: FileInfo(
                    constructor=lambda path: self._standard_output_cls(path),
                    get_path=lambda: get_output_file_path(
                        self.dir_path,
                        FILE_REFS.eso
                    )
                ),
                FILE_REFS.mtr: FileInfo(
                    constructor=lambda path: self._standard_output_cls(path),
                    get_path=lambda: get_output_file_path(self.dir_path, FILE_REFS.mtr)
                ),
                FILE_REFS.mtd: FileInfo(
                    constructor=lambda path: self._mtd_cls(path),
                    get_path=lambda: get_output_file_path(self.dir_path, FILE_REFS.mtd)
                ),
                FILE_REFS.mdd: FileInfo(
                    constructor=lambda path: open(path).read(),
                    get_path=lambda: get_output_file_path(self.dir_path, FILE_REFS.mdd)

                ),
                FILE_REFS.err: FileInfo(
                    constructor=lambda path: self._err_cls(path),
                    get_path=lambda: get_output_file_path(self.dir_path, FILE_REFS.err)
                ),
                FILE_REFS.summary_table: FileInfo(
                    constructor=lambda path: self._summary_table_cls(path),
                    get_path=lambda: get_output_file_path(self.dir_path, FILE_REFS.summary_table)
                )
            }
        return self._prepared_file_refs

    # ------------------------------------ public api ------------------------------------------------------------------
    # python magic
    def __getattr__(self, item):
        if item not in FILE_REFS:
            raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, item))

        if item not in self._outputs_cache:
            self._outputs_cache[item] = self._file_refs[item].constructor(self.get_file_path(item))

        return self._outputs_cache[item]

    def __dir__(self):
        return list(_refs) + list(self.__dict__)

    @property
    def dir_path(self):
        """
        Returns
        -------
        simulation directory path
        """
        return self._dir_path

    def exists(self, file_ref):
        """
        Parameters
        ----------
        file_ref: str
            reference of file.
            Available references:  'idf', 'epw', 'eio', 'eso', 'mtr', 'mtd', 'mdd', 'err', 'summary_table'
            See EnergyPlus documentation for more information.

        Returns
        -------
        Boolean
        """
        if file_ref not in FILE_REFS:
            raise ValueError("Unknown file_ref: '%s'. Available: '%s'." % (file_ref, list(sorted(FILE_REFS._fields))))
        return os.path.isfile(self._path(file_ref))

    def get_file_path(self, file_ref):
        """
        Parameters
        ----------
        file_ref: str
            reference of file.
            Available references:  'idf', 'epw', 'eio', 'eso', 'mtr', 'mtd', 'mdd', 'err', 'summary_table'
            See EnergyPlus documentation for more information.

        Returns
        -------
        Instance of required output.
        """
        if not self.exists(file_ref):
            raise FileNotFoundError("File '%s' not found in simulation '%s'." % (file_ref, self._path(file_ref)))
        return self._path(file_ref)

simulate = Simulation.simulate

def run_eplus(epm_or_idf_path, weather_data_or_epw_path, simulation_dir_path, stdout=None, stderr=None, beat_freq=None):
    """
    Parameters
    ----------
    epm_or_idf_path:
    weather_data_or_epw_path
    simulation_dir_path
    stdout: default sys.stdout
    stderr: default sys.stderr
    beat_freq: if not none, stdout will be used at least every beat_freq (in seconds)
    """
    # work with absolute paths
    simulation_dir_path = os.path.abspath(simulation_dir_path)

    # check dir path
    if not os.path.isdir(simulation_dir_path):
        raise NotADirectoryError("Simulation directory does not exist: '%s'." % simulation_dir_path)

    # epm
    if not isinstance(epm_or_idf_path, Epm):
        # we don't copy file directly because we want to manage it's external files
        # could be optimized (use _copy_without_read_only)
        epm = Epm.from_idf(epm_or_idf_path)
    else:
        epm = epm_or_idf_path

    # create idf
    simulation_idf_path = os.path.join(simulation_dir_path, CONF.default_model_name + ".idf")
    epm.to_idf(simulation_idf_path)

    # weather data
    simulation_epw_path = os.path.join(simulation_dir_path, CONF.default_model_name + ".epw")
    if isinstance(weather_data_or_epw_path, WeatherData):
        weather_data_or_epw_path.to_epw(simulation_epw_path)
    else:
        # no need to load: we copy directly
        _copy_without_read_only(weather_data_or_epw_path, simulation_epw_path)

    # copy epw if needed (depends on os/eplus version)
    temp_epw_path = get_simulated_epw_path()
    if temp_epw_path is not None:
        _copy_without_read_only(simulation_epw_path, temp_epw_path)

    # prepare command
    eplus_relative_cmd = get_simulation_base_command()
    eplus_cmd = os.path.join(CONF.eplus_base_dir_path, eplus_relative_cmd)

    # idf
    idf_command_style = get_simulation_input_command_style("idf")
    if idf_command_style == SIMULATION_INPUT_COMMAND_STYLES.simu_dir:
        idf_file_cmd = os.path.join(simulation_dir_path, CONF.default_model_name)
    elif idf_command_style == SIMULATION_INPUT_COMMAND_STYLES.file_path:
        idf_file_cmd = simulation_idf_path
    else:
        raise AssertionError("should not be here")

    # epw
    epw_command_style = get_simulation_input_command_style("epw")
    if epw_command_style == SIMULATION_INPUT_COMMAND_STYLES.simu_dir:
        epw_file_cmd = os.path.join(simulation_dir_path, CONF.default_model_name)
    elif epw_command_style == SIMULATION_INPUT_COMMAND_STYLES.file_path:
        epw_file_cmd = simulation_epw_path
    else:
        raise AssertionError("should not be here")

    # command list
    simulation_command_style = get_simulation_command_style()
    if simulation_command_style == SIMULATION_COMMAND_STYLES.args:
        cmd_l = [eplus_cmd, idf_file_cmd, epw_file_cmd]
    elif simulation_command_style == SIMULATION_COMMAND_STYLES.kwargs:
        cmd_l = [eplus_cmd, "-w", epw_file_cmd, "-r", idf_file_cmd]
    else:
        raise RuntimeError("should not be here")

    # launch calculation
    run_subprocess(
        cmd_l,
        cwd=simulation_dir_path,
        stdout=stdout,
        stderr=stderr,
        beat_freq=beat_freq
    )

    # if needed, we delete temp weather data (only on Windows, see above)
    if (temp_epw_path is not None) and os.path.isfile(temp_epw_path):
        os.remove(os.path.join(temp_epw_path))


"""
create/update/delete framework methods (see methods documentation):
 - epm._dev_populate_from_json_data
 - table.batch_add
 - record.update
 - queryset.delete
 - record.delete
"""

import os
import collections
import textwrap
import json

from .. import CONF
from ..util import get_multi_line_copyright_message, to_buffer
from .idd import Idd
from .table import Table
from .record import Record
from .relations_manager import RelationsManager
from .external_files_manager import ExternalFilesManager
from .external_file import get_external_files_dir_name
from .parse_idf import parse_idf
from .util import json_data_to_json, multi_mode_write

def default_external_files_dir_name(model_name):
    """
    Parameters
    ----------
    model_name: with or without extension
    """
    name, ext = os.path.splitext(model_name)
    return name + CONF.external_files_suffix

class Epm:
    """
    Energyplus model
    """
    _dev_record_cls = Record  # for subclassing
    _dev_table_cls = Table  # for subclassing
    _dev_idd_cls = Idd  # for subclassing

    def __init__(self, check_required=True, idd_or_buffer_or_path=None):
        """
        An Epm is an Energy Plus Model.
        It can come from and idf, a epjson (not coded yet), or a json.
        It can be transformed in an idf, an epjson (not coded yet) or a json.

        Parameters
        ----------
        idd_or_buffer_or_path: (expert) if you wan't to use a specific idd.
        check_required: boolean, default True
            If True, will raise an exception if a required field is missing. If False, not not perform any checks.
        """
        self._dev_idd = (
            idd_or_buffer_or_path if isinstance(idd_or_buffer_or_path, Idd) else
            self._dev_idd_cls(idd_or_buffer_or_path)
        )
        # !! relations manager must be defined before table creation because table creation will trigger
        # hook registering
        self._dev_relations_manager = RelationsManager(self)

        # external files manager
        self._dev_external_files_manager = ExternalFilesManager(self)

        self._tables = collections.OrderedDict(sorted([  # {lower_ref: table, ...}
            (table_descriptor.table_ref.lower(), Table(table_descriptor, self))
            for table_descriptor in self._dev_idd.table_descriptors.values()
        ]))

        self._dev_check_required = check_required
        self._comment = ""

    # ------------------------------------------ private ---------------------------------------------------------------
    @classmethod
    def _create_from_buffer_or_path(
            cls,
            parse_fct,
            buffer_or_path,
            idd_or_buffer_or_path=None,
            check_required=True
    ):
        # prepare buffer
        _source_file_path, buffer = to_buffer(buffer_or_path)

        # create json data
        with buffer as f:
            json_data = parse_fct(f)

        # create and return epm
        return cls.from_json_data(
            json_data,
            idd_or_buffer_or_path=idd_or_buffer_or_path,
            check_required=check_required
        )

    # ------------------------------------------ dev api ---------------------------------------------------------------
    def _dev_populate_from_json_data(self, json_data):
        """
        !! Must only be called once, when empty !!
        """

        # workflow
        # --------
        # (methods belonging to create/update/delete framework:
        #     epm._dev_populate_from_json_data, table.batch_add, record.update, queryset.delete, record.delete)
        # 1. add inert
        #     * data is checked
        #     * old links are unregistered
        #     * record is stored in table (=> pk uniqueness is checked)
        # 2. activate: hooks, links, external files

        # manage comment if any
        comment = json_data.pop("_comment", None)
        if comment is not None:
            self._comment = comment

        # populate external files
        external_files_data = json_data.pop("_external_files", dict())
        self._dev_external_files_manager.populate_from_json_data(external_files_data)

        # manage records
        added_records = []
        for table_ref, json_data_records in json_data.items():
            # find table
            table = getattr(self, table_ref)

            # create record (inert)
            records = table._dev_add_inert(json_data_records)

            # add records (inert)
            added_records.extend(records)

        # activate hooks
        for r in added_records:
            r._dev_activate_hooks()

        # activate links and external files
        for r in added_records:
            r._dev_activate_links()
            r._dev_activate_external_files()

    # --------------------------------------------- public api ---------------------------------------------------------
    # python magic
    def __repr__(self):
        return "<Epm>"

    def __str__(self):
        s = "Epm\n"

        for table in self._tables.values():
            records_nb = len(table)
            if records_nb == 0:
                continue
            plural = "" if records_nb == 1 else "s"
            s += f"  {table.get_name()}: {records_nb} record{plural}\n"

        return s

    def __getattr__(self, item):
        try:
            return self._tables[item.lower()]
        except KeyError:
            raise AttributeError(f"No table with reference '{item}'.")

    def __eq__(self, other):
        return self.to_json_data() != other.to_json_data()

    def __iter__(self):
        return iter(self._tables.values())

    def __dir__(self):
        return [t.get_ref() for t in self._tables.values()] + list(self.__dict__)

    # get info
    def get_comment(self):
        return self._comment

    # get idd info
    def get_info(self):
        return "Energy plus model\n" + "\n".join(
            f"  {table._dev_descriptor.table_ref}" for table in self._tables.values()
        )

    def get_external_files(self):
        """
        An external file manages file paths.
        """
        external_files = []
        for table in self._tables.values():
            for r in table:
                external_files.extend([ef for ef in r.get_external_files()])
        return external_files

    # construct
    def set_comment(self, comment):
        self._comment = str(comment)

    def set_defaults(self):
        """
        All fields of Epm with a default value and that are null will be set to their default value.
        """
        for table in self._tables.values():
            for r in table:
                r.set_defaults()

    def dump_external_files(self, target_dir_path=None):
        """
        Parameters
        ----------
        target_dir_path
        """
        self._dev_external_files_manager.dump_external_files(target_dir_path=target_dir_path)

    # ------------------------------------------- load -----------------------------------------------------------------
    @classmethod
    def from_json_data(cls, json_data, check_required=True, idd_or_buffer_or_path=None):
        """
        Parameters
        ----------
        json_data: dict
            Dictionary of serialized data (text, floats, ints, ...). For more information on data structure, create an
            Epm and use to_json_data or to_json.
        check_required: boolean, default True
            If True, will raise an exception if a required field is missing. If False, not not perform any checks.
        idd_or_buffer_or_path: (expert) to load using a custom idd

        Returns
        -------
        An Epm instance.
        """
        epm = cls(
            idd_or_buffer_or_path=idd_or_buffer_or_path,
            check_required=check_required
        )

        epm._dev_populate_from_json_data(json_data)
        return epm

    @classmethod
    def from_idf(cls, buffer_or_path, check_required=True, idd_or_buffer_or_path=None):
        """
        Parameters
        ----------
        buffer_or_path: idf buffer or path
        check_required: boolean, default True
            If True, will raise an exception if a required field is missing. If False, not not perform any checks.
        idd_or_buffer_or_path: (expert) to load using a custom idd

        Returns
        -------
        An Epm instance.
        """
        # todo: add geometry only (or equivalent)
        return cls._create_from_buffer_or_path(
            parse_idf,
            buffer_or_path,
            idd_or_buffer_or_path=idd_or_buffer_or_path,
            check_required=check_required
        )

    @classmethod
    def from_json(cls, buffer_or_path, check_required=True, idd_or_buffer_or_path=None):
        """
        Parameters
        ----------
        buffer_or_path: json buffer or path
        check_required: boolean, default True
            If True, will raise an exception if a required field is missing. If False, not not perform any checks.
        idd_or_buffer_or_path: (expert) to load using a custom idd

        Returns
        -------
        An Epm instance.
        """
        return cls._create_from_buffer_or_path(
            json.load,
            buffer_or_path,
            idd_or_buffer_or_path=idd_or_buffer_or_path,
            check_required=check_required
        )

    # ----------------------------------------- export -----------------------------------------------------------------
    def to_json_data(self):
        """
        Returns
        -------
        A dictionary of serialized data.
        """
        # create data
        d = collections.OrderedDict((t.get_ref(), t.to_json_data()) for t in self._tables.values())
        d["_comment"] = self._comment
        d.move_to_end("_comment", last=False)
        d["_external_files"] = self._dev_external_files_manager
        return d

    def to_json(self, buffer_or_path=None, indent=2):
        """
        Parameters
        ----------
        buffer_or_path: buffer or path, default None
            output to write into. If None, will return a json string.
        indent: int, default 2
            Defines the indentation of the json

        Returns
        -------
        None, or a json string (if buffer_or_path is None).
        """
        # return json
        return json_data_to_json(
            self.to_json_data(),
            buffer_or_path=buffer_or_path,
            indent=indent
        )

    def to_idf(self, buffer_or_path=None, dump_external_files=True):
        """
        Parameters
        ----------
        buffer_or_path: buffer or path, default None
            output to write into. If None, will return a json string.
        dump_external_files: boolean, default True
            if True, external files will be dumped in external files directory

        Returns
        -------
        None, or an idf string (if buffer_or_path is None).
        """
        # prepare comment
        comment = get_multi_line_copyright_message()
        if self._comment != "":
            comment += textwrap.indent(self._comment, "! ", lambda line: True)
        comment += "\n\n"

        # prepare external files dir path if file path
        if isinstance(buffer_or_path, str):
            dir_path, file_name = os.path.split(buffer_or_path)
            model_name, _ = os.path.splitext(file_name)
        else:
            model_name, dir_path = None, os.path.curdir

        # dump files if asked
        if dump_external_files:
            self.dump_external_files(
                target_dir_path=os.path.join(dir_path, get_external_files_dir_name(model_name=model_name))
            )

        # prepare body
        formatted_records = []
        for table_ref, table in self._tables.items():  # self._tables is already sorted
            formatted_records.extend([r.to_idf(model_name=model_name) for r in sorted(table)])
        body = "\n\n".join(formatted_records)

        # return
        content = comment + body
        return multi_mode_write(
            lambda f: f.write(content),
            lambda: content,
            buffer_or_path
        )


from itertools import filterfalse

from .exceptions import RecordDoesNotExistError, MultipleRecordsReturnedError

def unique_ever_seen(iterable, key=None):
    """
    https://docs.python.org/3.6/library/itertools.html#itertools-recipes
    List unique elements, preserving order. Remember all elements ever seen.

    unique_ever_seen('AAAABBBCCDAABBB') --> A B C D
    unique_ever_seen('ABBCcAD', str.lower) --> A B C D
    """
    seen = set()
    seen_add = seen.add
    if key is None:
        for element in filterfalse(seen.__contains__, iterable):
            seen_add(element)
            yield element
    else:
        for element in iterable:
            k = key(element)
            if k not in seen:
                seen_add(k)
                yield element

class Queryset:
    """
    Contains record, and enables filtering or other operations.
    A queryset must be immutable (not a Python sense, but list must never be modified). OR CACHE SYSTEM WILL FAIL.
    We only use lists (and not iterators), to avoid problems (could be done if optimization was needed):
        - exhaustion
        - iterator underlying list modification

    Optimization can probably be performed using iterators.
    """
    def __init__(self, table, records=None):
        self._table = table

        # manage empty
        if records is None:
            records = ()

        # ensure unique, sort, make un-mutable
        self._records = tuple(sorted(unique_ever_seen(records)))

        # ensure correct table
        if len({r.get_table() for r in self._records}.difference({self._table})) > 0:
            raise RuntimeError(
                f"queryset contains records that belong to other table than {self.get_table_ref()}"
            )

    # python magic
    def __repr__(self):
        return "<Queryset of %s: %s records>" % (self.get_table_ref(), str(len(self._records)))

    def __getitem__(self, item):
        if isinstance(item, str):
            if self._table._dev_auto_pk:
                raise KeyError(f"table {self._table.get_ref()} does not have a primary key, can't use getitem syntax")
            for r in self._records:  # todo: we could store records in an ordered dict with pk keys
                if r.get_pk() == item:
                    return r
            else:
                raise KeyError(f"queryset does not contain a record who's pk is '{item}'")
        if isinstance(item, int):
            return self._records[item]

        raise KeyError("item must be an int or a str")

    def __iter__(self):
        return iter(self._records)

    def __len__(self):
        return len(self._records)

    def __add__(self, other):
        """
        Add new query set to query set (only new records will be added since uniqueness is ensured in __init__).
        """
        return Queryset(self._table, list(self) + list(other))

    def __eq__(self, other):
        return set(self) == set(other)

    # get info
    def get_table(self):
        return self._table

    def get_table_ref(self):
        return self._table.get_ref()

    def select(self, filter_by=None):
        """
        Parameters
        ----------
        filter_by: callable, default None
            Callable must take one argument (a record of queryset), and return True to keep record, or False to skip it.
            Example : .select(lambda x: x.name == "my_name").
            If None, records are not filtered.

        Returns
        -------
        Queryset instance, containing all selected records.
        """
        iterator = self._records if filter_by is None else filter(filter_by, self._records)
        return Queryset(self._table, iterator)

    def one(self, filter_by=None):
        """
        Parameters
        ----------
        filter_by: callable, default None
            Callable must take one argument (a record of table), and return True to keep record, or False to skip it.
            Example : .one(lambda x: x.name == "my_name").
            If None, records are not filtered.

        Returns
        -------
        Record instance if one and only one record is found. Else raises.

        Raises
        ------
        RecordDoesNotExistError if no record is found
        MultipleRecordsReturnedError if multiple records are found
        """
        # filter if needed
        qs = self if filter_by is None else self.select(filter_by=filter_by)

        # check one and only one
        if len(qs) == 0:
            raise RecordDoesNotExistError("Queryset set contains no value.")
        if len(qs) > 1:
            raise MultipleRecordsReturnedError("Queryset contains more than one value.")

        # return record
        return qs[0]

    # delete
    def delete(self):
        """
        Deletes all records of queryset.
        """
        # workflow
        # --------
        # delete each record and remove from queryset

        # delete each record
        for r in self:
            r.delete()

        # clear content
        self._records = ()

    # ------------------------------------------- export ---------------------------------------------------------------
    def to_json_data(self):
        """
        Returns
        -------
        A dictionary of serialized data.
        """
        # records are already sorted
        return [r.to_json_data() for r in self._records]


import os

from .util import OS_NAME

def get_simulated_epw_path():
    """
    Returns
    -------
    None if epw can be anywhere
    """
    from oplus import CONF  # touchy imports

    if OS_NAME == "windows":
        return os.path.join(CONF.eplus_base_dir_path, "WeatherData", "%s.epw" % CONF.default_model_name)

    #  on linux or osx, epw may remain in current directory


import logging
import re

from .field_descriptor import FieldDescriptor
from .util import table_name_to_ref

logger = logging.getLogger(__name__)

class TableDescriptor:
    """
    Describes a EPlus record (see idd).
    """
    def __init__(self, table_name, group_name=None):
        self.table_name = table_name
        self.table_ref = table_name_to_ref(table_name)
        self.group_name = group_name
        # we use list (and not dict) because some field descriptors do not have a name (including non extensible tables)
        self._field_descriptors = []
        self._tags = {}

        # extensible management
        # (cycle_start, cycle_len, patterns) where patterns is (var_a_(\d+)_ref, var_b_(\d+)_ref, ...)
        self.extensible_info = None

    @property
    def field_descriptors(self):
        return self._field_descriptors

    @property
    def tags(self):
        return self._tags

    def add_tag(self, tag_ref, value=None):
        if tag_ref not in self._tags:
            self._tags[tag_ref] = []
        if value is not None:
            self._tags[tag_ref].append(value)

    def add_field_descriptor(self, fieldd_type, name=None):
        # create
        field_descriptor = FieldDescriptor(self, len(self._field_descriptors), fieldd_type, name=name)

        # append
        self._field_descriptors.append(field_descriptor)

        return field_descriptor

    def prepare_extensible(self):
        """
        This function finishes initialization, must be called once all field descriptors and tag have been filled.
        """
        # see if extensible and store cycle len
        for k in self._tags:
            if "extensible" in k:
                cycle_len = int(k.split(":")[1])
                break
        else:
            # not extensible
            return

        # find cycle start and prepare patterns
        cycle_start = None
        cycle_patterns = []
        for i, field_descriptor in enumerate(self._field_descriptors):
            # quit if finished
            if (cycle_start is not None) and (i >= (cycle_start + cycle_len)):
                break

            # set cycle start if not set yet
            if (cycle_start is None) and ("begin-extensible" in field_descriptor.tags):
                cycle_start = i

            # leave if cycle start not reached yet
            if cycle_start is None:
                continue

            # store pattern
            cycle_patterns.append(field_descriptor.ref.replace("1", r"(\d+)"))
        else:
            raise RuntimeError("cycle start not found")

        # detach unnecessary field descriptors
        self._field_descriptors = self._field_descriptors[:cycle_start + cycle_len]

        # store cycle info
        self.extensible_info = (cycle_start, cycle_len, tuple(cycle_patterns))

        # set field descriptor cycle_start index (for error messages while serialization)
        for i, fd in enumerate(self._field_descriptors[cycle_start:]):
            fd.set_extensible_info(cycle_start, cycle_len, cycle_patterns[i])

    @property
    def base_fields_nb(self):
        """
        base fields: without extensible
        """
        return len(self._field_descriptors) if self.extensible_info is None else self.extensible_info[0]

    def get_field_index(self, ref):
        # general case
        for pattern_num in range(self.base_fields_nb):
            field_descriptor = self._field_descriptors[pattern_num]
            if field_descriptor.ref is None:  # can happen
                continue
            if field_descriptor.ref == ref:
                return pattern_num

        # extensible
        ext_info = self.extensible_info
        if ext_info is not None:
            cycle_start, cycle_len, patterns = ext_info
            for pattern_num, pat in enumerate(patterns):
                match = re.fullmatch(pat, ref)
                if match is None:  # not found
                    continue
                # we found cycle
                cycle_num = int(match.group(1))

                # calculate and return index
                return cycle_start + (cycle_num-1)*cycle_len + pattern_num

        raise AttributeError("No field of '%s' has ref '%s'." % (self.table_name, ref))

    def get_field_reduced_index(self, index):
        """
        reduced index: modulo of extensible has been applied
        """
        # return index if not extensible
        if self.extensible_info is None:
            return index

        # manage extensible
        cycle_start, cycle_len, _ = self.extensible_info

        # base field
        if index < cycle_start:
            return index

        # extensible field
        return cycle_start + ((index - cycle_start) % cycle_len)

    def get_field_descriptor(self, index):
        return self._field_descriptors[self.get_field_reduced_index(index)]

    def get_extended_name(self, index):
        """
        manages extensible names
        """
        field_descriptor = self.get_field_descriptor(index)
        if self.extensible_info is None:
            return field_descriptor.name
        cycle_start, cycle_len, _ = self.extensible_info
        cycle_num = (index - cycle_start) // cycle_len
        return None if field_descriptor.name is None else field_descriptor.name.replace("1", str(cycle_num))

    def get_info(self):
        header = f"{self.table_name} ({self.table_ref})"

        msg = f"{header}\n"
        for i, field_descriptor in enumerate(self.field_descriptors):
            msg += f" {i}" + (
                "\n" if field_descriptor.name is None else f": {field_descriptor.name} ({field_descriptor.ref})\n"
            )

            for k, v in sorted(field_descriptor.tags.items()):
                if k == "begin-extensible":  # we indicate cycle len
                    v = [f"cycle length {self.extensible_info[1]}"]
                msg += f"    * {k}: {'; '.join(v)}\n"

        return msg