PY2 = sys.version_info[0] == 2
def with_metaclass(meta, *bases):
        def __new__(cls, name, this_bases, d):
def csv_open(filename, mode='r'):
    class metaclass(meta): def get_default_array(length, dtype, default_values=None):
    fill_value = get_default_record(dtype, default_values)
    return np.full(length, fill_value, dtype=dtype) def validate_dict(d, target, context=''):
    """
    Examples
    --------

    >>> validate_dict({'testkey': 'good'}, {'testkey': str})
    >>> validate_dict({'testkey': 'good'}, {'testkey': 'good'})
    >>> validate_dict({'testkey': 'bad'}, {'testkey': 'good'})
    Traceback (most recent call last):
    ...
    Exception: invalid structure for 'testkey': it should be 'good' but it is 'bad'
    >>> target1 = {'#reqkey': str, 'optkey': str}
    >>> validate_dict({'reqkey': 'good'}, target1)
    >>> validate_dict({'reqkey': 'good', 'optkey': 'good'}, target1)
    >>> validate_dict({'reqkey': 'good', 'extrakey': 'good'}, target1)
    Traceback (most recent call last):
    ...
    SyntaxError: invalid keyword(s): 'extrakey'
    >>> target2 = {'#reqkey': str, 'optkey': 'good', '*': str}
    >>> validate_dict({'reqkey': 'good'}, target2)
    >>> validate_dict({'reqkey': 'good', 'optkey': 'good'}, target2)
    >>> validate_dict({'reqkey': 'good', 'optkey': 'bad'}, target2)
    Traceback (most recent call last):
    ...
    Exception: invalid structure for 'optkey': it should be 'good' but it is 'bad'
    >>> validate_dict({'reqkey': 'good', 'extrakey': 'good'}, target2)
    """
    assert isinstance(target, dict)
    if not isinstance(d, dict):
        raise Exception("invalid structure for '%s': it should be a map and it is a %s" % (context, typename(d)))
    target_keys = target.keys()
    required = set(k[1:] for k in target_keys if k.startswith('#'))
    optional = set(k for k in target_keys if not k.startswith('#'))
    anykey = '*' in optional
    if anykey:
        optional.remove('*')
    # in case we have a * in there, we should only make sure that required keys
    # are present, otherwise we have to also check if provided keys are valid
    validate_dict_keys(d, required, optional, context, extra_allowed=anykey)
    for k, v in d.items():
        if k in required:
            section_def = target['#' + k]
        elif k in optional:
            section_def = target[k]
        elif anykey:
            section_def = target['*']
        else:
            # this shouldn't happen unless target is an empty dictionary
            assert not target
            raise KeyError('empty section def at: %s' % context)
        subcontext = context + ' -> ' + k if context else k
        validate_value(v, section_def, subcontext) def merge_dicts(*args, **kwargs):
    """
    Returns a new dictionary which is the result of recursively merging all
    the dictionaries passed as arguments and the keyword arguments.
    Later dictionaries overwrite earlier ones. kwargs overwrite all.
    """
    result = args[0].copy()
    for arg in args[1:] + (kwargs,):
        for k, v in arg.items():
            if isinstance(v, dict) and k in result:
                v = merge_dicts(result[k], v)
            result[k] = v
    return result def merge_items(item_sequences, merge_order='first_to_last', value_priority='first'):
    """
    Returns a list of (key, value) pairs which is the result of merging all sequences of (key, value) pairs passed as
    arguments.
    Order of keys is preserved on a "first" merged basis.
    If merge_order is 'first_to_last', items will be merged from first to last, appending if not present, otherwise the
    merge happens from last to first.

    value_priority determines which sequence items will have priority over the others. 'first' means the value of
    a key will be from the earliest sequence (relative to the order given in item_sequences, independently of the
    merge order).
    """
    assert merge_order in {'first_to_last', 'last_to_first'}
    assert value_priority in {'first', 'last'}

    keep_first = value_priority == 'first'
    if merge_order == 'last_to_first':
        item_sequences = item_sequences[::-1]
        keep_first = not keep_first

    result_keys = []
    values = {}
    for items in item_sequences:
        if keep_first:
            for k, v in items:
                if k not in values:
                    result_keys.append(k)
                    values[k] = v
        else:
            for k, v in items:
                if k not in values:
                    result_keys.append(k)
                values[k] = v

    return [(k, values[k]) for k in result_keys] def invert_dict(d):
    return dict((v, k) for k, v in d.items()) def countlines(filepath):
    with open(filepath) as f:
        return sum(1 for _ in f) def skip_comment_cells(lines):
    def notacomment(v):
        return not v.startswith('#')
    for line in lines:
        stripped_line = list(itertools.takewhile(notacomment, line))
        if stripped_line:
            yield stripped_line def strip_rows(lines):
    """
    returns an iterator of lines with leading and trailing blank (empty or
    which contain only space) cells.
    """
    def isblank(s):
        return s == '' or s.isspace()
    for line in lines:
        leading_dropped = list(itertools.dropwhile(isblank, line))
        rev_line = list(itertools.dropwhile(isblank,
                                            reversed(leading_dropped)))
        yield list(reversed(rev_line)) class PrettyTable(object):
    def __init__(self, data, missing=None):
        assert isinstance(data, (tuple, list))
        self.data = data
        self.missing = missing

    def __iter__(self):
        if self.missing is not None:
            return iter(self.convert_nans())
        else:
            return iter(self.data)

    def __len__(self):
        return len(self.data)

    def convert_nans(self):
        missing = self.missing
        for row in self.data:
            formatted = [missing if value != value else value
                         for value in row]
            yield formatted

    def __repr__(self):
        missing = self.missing
        if missing is None:
            missing = 'nan'
        else:
            missing = str(missing)
        return '\n' + table2str(self.data, missing) + '\n' def unique(iterable):
    """
    List all elements once, preserving order. Remember all elements ever seen.
    """
    # unique('AAAABBBCCDAABBB') --> A B C D
    seen = set()
    seen_add = seen.add
    for element in iterable:
        if element not in seen:
            seen_add(element)
            yield element def duplicates(iterable):
    """
    List duplicated elements once, preserving order. Remember all elements ever
    seen.
    """
    # duplicates('AAAABBBCCDAABBB') --> A B C
    counts = defaultdict(int)
    for element in iterable:
        counts[element] += 1
        if counts[element] == 2:
            yield element def unique_duplicate(iterable):
    """
    List all elements once and duplicated elements, preserving order.
    Remember all elements ever seen.
    """
    # unique_duplicate('AAAABBBCCDAABBB') --> [A, B, C, D], [A, B, C]
    counts = {}
    uniques = []
    dupes = []
    append_uniques = uniques.append
    append_dupes = dupes.append
    for element in iterable:
        count = counts[element] = counts.get(element, 0) + 1
        if count == 2:
            append_dupes(element)
        elif count == 1:
            append_uniques(element)
    return uniques, dupes def prod(values):
    res = 1
    for value in values:
        res *= value
    return res def field_str_to_type(str_type, context):
    """
    Converts a (field) string type to its Python type.
    """
    if str_type not in str_to_type:
        raise SyntaxError("'%s' is not a valid type for %s."
                          % (str_type, context))
    return str_to_type[str_type] def fields_yaml_to_type(dict_fields_list):
    """
    Transform a list of (one item) dict with str types to a list of tuple with
    Python types
    """
    return fields_str_to_type([list(d.items())[0] for d in dict_fields_list]) def fromiter(iterable, dtype, count=-1):
    if count == -1:
        return np.fromiter(iterable, dtype)
    else:
        buf = np.empty(count, dtype=dtype)
        i = 0
        for e in iterable:
            buf[i] = e
            i += 1
            if i == count:
                break
        if i < count:
            raise ValueError("iterator too short")
        return buf MB = 2 ** 20
import csv
import os.path
import re
import numpy as np
    import bcolz
import tables
import yaml
import larray as la
from itertools import islice, chain
from liam2.compat import basestring, csv_open
from liam2.expr import get_default_array
from liam2.utils import (validate_dict, merge_dicts, merge_items, invert_dict, countlines, skip_comment_cells,
                         strip_rows, PrettyTable, unique, duplicates, unique_duplicate, prod, field_str_to_type,
                         fields_yaml_to_type, fromiter, MB)
    from liam2.simulation import handle_imports
    if ndim > 1:
        # having duplicate values is normal when there are more than 2
        # dimensions but we need to test whether there are duplicates of
        # combinations.
        dupe_combos = list(duplicates(zip(*headers)))
        if dupe_combos:
            print(("Duplicate row header value(s) in '%s':" % fpath))
            print((PrettyTable(dupe_combos)))
            raise Exception("bad alignment data in '%s': found %d "
                            "duplicate row header value(s)"
                            % (fpath, len(dupe_combos)))

    possible_values = [np.array(list(unique(pvalues))) for pvalues in headers]
    possible_values.append(np.array(unique_last_d))

    shape = tuple(len(values) for values in possible_values)
    num_possible_values = prod(shape)

    # transform the 2d table into a 1d list
    str_table = list(chain.from_iterable(str_table))
    if len(str_table) != num_possible_values:
        raise Exception("incoherent data in '%s': %d data cells "
                        "found while it should be %d based on the number "
                        "of possible values in headers (%s)"
                        % (fpath,
                           len(str_table),
                           num_possible_values,
                           ' * '.join(str(len(values))
                                      for values in possible_values)))

    # TODO: compare time with numpy built-in conversion:
    # if dtype is None, numpy tries to detect the best type itself
    # which it does a good job of if the values are already numeric values
    # if dtype is provided, numpy does a good job to convert from string
    # values.
    if celltype is None:
        celltype = detect_column_type(str_table)
    data = convert_1darray(str_table, celltype)
    array = np.array(data, dtype=celltype)
    return la.LArray(array.reshape(shape), header, possible_values)


def load_table(fpath, fields=None, newnames=None, delimiter=None, transpose=False):
    csv_file = CSV(fpath, newnames=newnames, delimiter=delimiter, transpose=transpose)
    array = csv_file.as_array(fields)
    csv_file.close()
    return array


def load_def(localdir, ent_name, section_def, required_fields):
    if 'type' in section_def and 'fields' in section_def:
        raise Exception("invalid structure for '%s': "
                        "type and fields sections are mutually exclusive"
                        % ent_name)

    if 'type' in section_def:
        csv_filename = section_def.get('path', ent_name + ".csv")
        csv_filepath = complete_path(localdir, csv_filename)
        str_type = section_def['type']
        if isinstance(str_type, basestring):
            celltype = field_str_to_type(str_type, "array '%s'" % ent_name)
        else:
            assert isinstance(str_type, type)
            celltype = str_type
        return 'ndarray', load_ndarray(csv_filepath, celltype)

    fields_def = section_def.get('fields')
    if fields_def is not None:
        for fdef in fields_def:
            if isinstance(fdef, basestring):
                raise SyntaxError("invalid field declaration: '%s', you are "
                                  "probably missing a ':'" % fdef)
        if all(isinstance(fdef, dict) for fdef in fields_def):
            fields = fields_yaml_to_type(fields_def)
        else:
            assert all(isinstance(fdef, tuple) for fdef in fields_def)
            fields = fields_def
        fnames = {name for name, _ in fields}
        for reqname, reqtype in required_fields[::-1]:
            if reqname not in fnames:
                fields.insert(0, (reqname, reqtype))
    else:
        fields = None
    newnames = merge_dicts(invert_dict(section_def.get('oldnames', {})),
                           section_def.get('newnames', {}))
    transpose = section_def.get('transposed', False)

    interpolate_def = section_def.get('interpolate')
    files_def = section_def.get('files')
    if files_def is None:
        # XXX: it might be cleaner to use the same code path than for the
        # multi-file case (however, that would loose the "import any file
        # size" feature that I'm fond of.

        # we can simply return the stream as-is
        # FIXME: stream is not sorted
        # csv file is assumed to be in the correct order (ie by period then id)
        csv_filename = section_def.get('path', ent_name + ".csv")
        csv_filepath = complete_path(localdir, csv_filename)
        csv_file = CSV(csv_filepath, newnames,
                       delimiter=',', transpose=transpose)
        stream = csv_file.read(fields)
        if fields is None:
            fields = csv_file.fields
        if interpolate_def is not None:
            raise Exception('interpolate is currently only supported with '
                            'multiple files')
        return 'table', (fields, csv_file.numlines, stream, csv_file)
    else:
        # we have to load all files, merge them and return a stream out of that
        print(" * computing number of rows...")

        # 1) only load required fields
        default_args = dict(newnames=newnames, transpose=transpose)
        if isinstance(files_def, dict):
            files_items = files_def.items()
        elif isinstance(files_def, list) and files_def:
            if isinstance(files_def[0], dict):
                # handle YAML ordered dict structure
                files_items = [list(d.items())[0] for d in files_def]

            elif isinstance(files_def[0], basestring):