PY2 = sys.version_info[0] == 2
def with_metaclass(meta, *bases):
        def __new__(cls, name, this_bases, d):
def csv_open(filename, mode='r'):
    class metaclass(meta): def get_default_array(length, dtype, default_values=None):
    fill_value = get_default_record(dtype, default_values)
    return np.full(length, fill_value, dtype=dtype) def validate_dict(d, target, context=''):
    """
    Examples
    --------

    >>> validate_dict({'testkey': 'good'}, {'testkey': str})
    >>> validate_dict({'testkey': 'good'}, {'testkey': 'good'})
    >>> validate_dict({'testkey': 'bad'}, {'testkey': 'good'})
    Traceback (most recent call last):
    ...
    Exception: invalid structure for 'testkey': it should be 'good' but it is 'bad'
    >>> target1 = {'#reqkey': str, 'optkey': str}
    >>> validate_dict({'reqkey': 'good'}, target1)
    >>> validate_dict({'reqkey': 'good', 'optkey': 'good'}, target1)
    >>> validate_dict({'reqkey': 'good', 'extrakey': 'good'}, target1)
    Traceback (most recent call last):
    ...
    SyntaxError: invalid keyword(s): 'extrakey'
    >>> target2 = {'#reqkey': str, 'optkey': 'good', '*': str}
    >>> validate_dict({'reqkey': 'good'}, target2)
    >>> validate_dict({'reqkey': 'good', 'optkey': 'good'}, target2)
    >>> validate_dict({'reqkey': 'good', 'optkey': 'bad'}, target2)
    Traceback (most recent call last):
    ...
    Exception: invalid structure for 'optkey': it should be 'good' but it is 'bad'
    >>> validate_dict({'reqkey': 'good', 'extrakey': 'good'}, target2)
    """
    assert isinstance(target, dict)
    if not isinstance(d, dict):
        raise Exception("invalid structure for '%s': it should be a map and it is a %s" % (context, typename(d)))
    target_keys = target.keys()
    required = set(k[1:] for k in target_keys if k.startswith('#'))
    optional = set(k for k in target_keys if not k.startswith('#'))
    anykey = '*' in optional
    if anykey:
        optional.remove('*')
    # in case we have a * in there, we should only make sure that required keys
    # are present, otherwise we have to also check if provided keys are valid
    validate_dict_keys(d, required, optional, context, extra_allowed=anykey)
    for k, v in d.items():
        if k in required:
            section_def = target['#' + k]
        elif k in optional:
            section_def = target[k]
        elif anykey:
            section_def = target['*']
        else:
            # this shouldn't happen unless target is an empty dictionary
            assert not target
            raise KeyError('empty section def at: %s' % context)
        subcontext = context + ' -> ' + k if context else k
        validate_value(v, section_def, subcontext) def merge_dicts(*args, **kwargs):
    """
    Returns a new dictionary which is the result of recursively merging all
    the dictionaries passed as arguments and the keyword arguments.
    Later dictionaries overwrite earlier ones. kwargs overwrite all.
    """
    result = args[0].copy()
    for arg in args[1:] + (kwargs,):
        for k, v in arg.items():
            if isinstance(v, dict) and k in result:
                v = merge_dicts(result[k], v)
            result[k] = v
    return result def merge_items(item_sequences, merge_order='first_to_last', value_priority='first'):
    """
    Returns a list of (key, value) pairs which is the result of merging all sequences of (key, value) pairs passed as
    arguments.
    Order of keys is preserved on a "first" merged basis.
    If merge_order is 'first_to_last', items will be merged from first to last, appending if not present, otherwise the
    merge happens from last to first.

    value_priority determines which sequence items will have priority over the others. 'first' means the value of
    a key will be from the earliest sequence (relative to the order given in item_sequences, independently of the
    merge order).
    """
    assert merge_order in {'first_to_last', 'last_to_first'}
    assert value_priority in {'first', 'last'}

    keep_first = value_priority == 'first'
    if merge_order == 'last_to_first':
        item_sequences = item_sequences[::-1]
        keep_first = not keep_first

    result_keys = []
    values = {}
    for items in item_sequences:
        if keep_first:
            for k, v in items:
                if k not in values:
                    result_keys.append(k)
                    values[k] = v
        else:
            for k, v in items:
                if k not in values:
                    result_keys.append(k)
                values[k] = v

    return [(k, values[k]) for k in result_keys] def invert_dict(d):
    return dict((v, k) for k, v in d.items()) def countlines(filepath):
    with open(filepath) as f:
        return sum(1 for _ in f) def skip_comment_cells(lines):
    def notacomment(v):
        return not v.startswith('#')
    for line in lines:
        stripped_line = list(itertools.takewhile(notacomment, line))
        if stripped_line:
            yield stripped_line def strip_rows(lines):
    """
    returns an iterator of lines with leading and trailing blank (empty or
    which contain only space) cells.
    """
    def isblank(s):
        return s == '' or s.isspace()
    for line in lines:
        leading_dropped = list(itertools.dropwhile(isblank, line))
        rev_line = list(itertools.dropwhile(isblank,
                                            reversed(leading_dropped)))
        yield list(reversed(rev_line)) class PrettyTable(object):
    def __init__(self, data, missing=None):
        assert isinstance(data, (tuple, list))
        self.data = data
        self.missing = missing

    def __iter__(self):
        if self.missing is not None:
            return iter(self.convert_nans())
        else:
            return iter(self.data)

    def __len__(self):
        return len(self.data)

    def convert_nans(self):
        missing = self.missing
        for row in self.data:
            formatted = [missing if value != value else value
                         for value in row]
            yield formatted

    def __repr__(self):
        missing = self.missing
        if missing is None:
            missing = 'nan'
        else:
            missing = str(missing)
        return '\n' + table2str(self.data, missing) + '\n' def unique(iterable):
    """
    List all elements once, preserving order. Remember all elements ever seen.
    """
    # unique('AAAABBBCCDAABBB') --> A B C D
    seen = set()
    seen_add = seen.add
    for element in iterable:
        if element not in seen:
            seen_add(element)
            yield element def duplicates(iterable):
    """
    List duplicated elements once, preserving order. Remember all elements ever
    seen.
    """
    # duplicates('AAAABBBCCDAABBB') --> A B C
    counts = defaultdict(int)
    for element in iterable:
        counts[element] += 1
        if counts[element] == 2:
            yield element def unique_duplicate(iterable):
    """
    List all elements once and duplicated elements, preserving order.
    Remember all elements ever seen.
    """
    # unique_duplicate('AAAABBBCCDAABBB') --> [A, B, C, D], [A, B, C]
    counts = {}
    uniques = []
    dupes = []
    append_uniques = uniques.append
    append_dupes = dupes.append
    for element in iterable:
        count = counts[element] = counts.get(element, 0) + 1
        if count == 2:
            append_dupes(element)
        elif count == 1:
            append_uniques(element)
    return uniques, dupes def prod(values):
    res = 1
    for value in values:
        res *= value
    return res def field_str_to_type(str_type, context):
    """
    Converts a (field) string type to its Python type.
    """
    if str_type not in str_to_type:
        raise SyntaxError("'%s' is not a valid type for %s."
                          % (str_type, context))
    return str_to_type[str_type] def fields_yaml_to_type(dict_fields_list):
    """
    Transform a list of (one item) dict with str types to a list of tuple with
    Python types
    """
    return fields_str_to_type([list(d.items())[0] for d in dict_fields_list]) def fromiter(iterable, dtype, count=-1):
    if count == -1:
        return np.fromiter(iterable, dtype)
    else:
        buf = np.empty(count, dtype=dtype)
        i = 0
        for e in iterable:
            buf[i] = e
            i += 1
            if i == count:
                break
        if i < count:
            raise ValueError("iterator too short")
        return buf MB = 2 ** 20
import csv
import os.path
import re
import numpy as np
    import bcolz
import tables
import yaml
import larray as la
from itertools import islice, chain
from liam2.compat import basestring, csv_open
from liam2.expr import get_default_array
from liam2.utils import (validate_dict, merge_dicts, merge_items, invert_dict, countlines, skip_comment_cells,
                         strip_rows, PrettyTable, unique, duplicates, unique_duplicate, prod, field_str_to_type,
                         fields_yaml_to_type, fromiter, MB)
    from liam2.simulation import handle_imports
        # types, so it is more portable this way
        attrs.dimensions = np.array(array.axes.names)
        # attrs.dim0_pvalues = array([a, b, c])
        # attrs.dim1_pvalues = array([d, e])
        # ...
        for i, axis in enumerate(array.axes):
            setattr(attrs, 'dim%d_pvalues' % i, axis.labels)
    return disk_array


def union1d(arrays):
    """arrays is an iterable returning arrays"""
    result = next(arrays)
    for array in arrays:
        result = np.union1d(result, array)
    return result


def interpolate(target, arrays, id_periods, fields):
    print(" * indexing...")
    periods = np.unique(id_periods['period'])
    max_id = np.max(id_periods['id'])

    row_for_id = {}
    for period in periods:
        # this might seem very wasteful but when compressed through
        # bcolz it is much smaller than an (id, rownum) dict, while
        # being only a bit slower
        row_for_id[period] = np.full(max_id + 1, -1, dtype=int)

    numrows = len(id_periods)
    lastrow_for_id = {}

    # compressing this with bcolz yield interesting compression but
    # is really too slow to use afterwards because access is
    # not sequential at all.
    nextrow_for_id = np.full(numrows, -1, dtype=int)
    for rownum, (period, record_id) in enumerate(id_periods):
        row_for_id[period][record_id] = rownum

        # this assumes id_periods are ordered by period, which
        # is implicitly the case because of union1d
        lastrow = lastrow_for_id.get(record_id)
        if lastrow is not None:
            nextrow_for_id[lastrow] = rownum
        lastrow_for_id[record_id] = rownum
    del lastrow_for_id

    size = sum(row_for_id[period].nbytes for period in periods)

    if bcolz is not None:
        print(" * compressing index (%.2f Mb)..." % (size / MB), end=' ')
        for period in periods:
            row_for_id[period] = bcolz.carray(row_for_id[period])
        csize = sum(row_for_id[period].cbytes for period in periods)
        print("done. (%.2f Mb)" % (csize / MB))
    else:
        print('bcolz package not found (bcolz is required to use compression '
              'during interpolate)')

    print(" * interpolating...")
    for values in arrays:
        # sort it by id, then period
        values.sort(order=('id', 'period'))

        input_stream = iter(values)

        # eg we get:
        # 10, 2001, 5324.0
        # 10, 2004, 6200.0
        # 10, 2005, 7300.0
        prev_row = next(input_stream)
        fields_to_set = \
            [name for name in prev_row.dtype.names
             if name in target.dtype.names and name not in ('id', 'period')]
        fields_to_interpolate = [name for name in fields_to_set
                                 if name in fields]
        fields_to_copy = [name for name in fields_to_set
                          if name not in fields]
        # compute row in target array
        # 10, 2001, 5324.0 -> rowtofill = 154
        # 10, 2004, 6200.0
        # 10, 2005, 7300.0
        rowtofill = row_for_id[prev_row['period']][prev_row['id']]
        for row in input_stream:
            target_row = target[rowtofill]
            for fname in fields_to_copy:
                target_row[fname] = prev_row[fname]

            # 10, 2001, 5324.0 -> rowtofill = 154
            # 10, 2004, 6200.0 -> rownum = 180
            # 10, 2005, 7300.0
            rownum = row_for_id[row['period']][row['id']]
            # fill the row corresponding to the previous data point at
            # id, period (rowtofill) *and* all the rows (in the target
            # array) in between that point and this one (rownum)
            while rowtofill != -1 and rowtofill != rownum:
                target_row = target[rowtofill]
                for fname in fields_to_interpolate:
                    target_row[fname] = prev_row[fname]
                rowtofill = nextrow_for_id[rowtofill]

            rowtofill = rownum
            prev_row = row

        while rowtofill != -1:
            target_row = target[rowtofill]
            for fname in fields_to_interpolate:
                target_row[fname] = prev_row[fname]
            rowtofill = nextrow_for_id[rowtofill]


def load_ndarray(fpath, celltype=None):
    print(" - reading", fpath)
    # FIXME: implement celltype
    a = la.read_csv(fpath, dialect='liam2')
    # print(a.info)
    return a
    with open(fpath, "rb") as f:
        reader = csv.reader(f)

        line_stream = skip_comment_cells(strip_rows(reader))