PY2 = sys.version_info[0] == 2
def with_metaclass(meta, *bases):
        def __new__(cls, name, this_bases, d):
def csv_open(filename, mode='r'):
    class metaclass(meta):     def make_global_context():
    def evaluate(expr, globals_dict, locals_dict=None, **kwargs):
def normalize_type(type_):
def get_default_value(column, default_value=None):
def get_default_vector(num, dtype, default_value=None):
def get_default_record(dtype, default_values=None):
def get_default_array(length, dtype, default_values=None):
def hasvalue(column):
def coerce_types(context, *args):
def as_simple_expr(expr, context):
def as_string(expr):
def traverse_expr(expr):
def gettype(value):
def getdtype(expr, context):
def always(type_):
    def dtype(self, context):
def firstarg_dtype(self, context):
def ispresent(values):
def collect_variables(expr):
def expr_eval(expr, context):
def binop(opname, kind='binary', reverse=False):
    def op(self, other):
    def __init__(self):
    def children(self):
    def value(self):
    def __nonzero__(self):
    def evaluate(self, context):
    def as_simple_expr(self, context):
    def as_string(self):
    def __getitem__(self, key):
    def __getattr__(self, key):
    def traverse(self):
    def all_of(self, node_type):
    def collect_variables(self):
    def __eq__(self, other):
        def strict_equal(a, b):
    def __hash__(self):
    def __contains__(self, expr):
    def get_tmp_varname(self, context):
    def add_tmp_var(self, context, result):
    def evaluate(self, context):
    def as_simple_expr(self, context):
def non_scalar_array(a):
    def __init__(self, expr, key):
    def __repr__(self):
    def evaluate(self, context):
            def fixkey(orig_key, filter_value):
    def __init__(self, expr, key):
    def __repr__(self):
    def evaluate(self, context):
    def __call__(self, *args, **kwargs):
    def __getattr__(self, key):
    def __init__(cls, name, bases, dct):
    def __init__(cls, name, bases, dct):
    def get_compute_func(cls):
    def __init__(self, *args, **kwargs):
    def format_args_str(args, kwargs):
    def format(funcname, args, kwargs):
    def __repr__(self):
    def get_compute_func(cls):
    def _eval_args(self, context):
    def compute(self, context, *args, **kwargs):
    def evaluate(self, context):
    def compute(self, context, func, *args, **kwargs):
    def __init__(self, *args, **kwargs):
    def compute(self, context, func, *args, **kwargs):
    def _original_args(self):
    def funcname(self):
    def __init__(self, op, expr):
    def as_simple_expr(self, context):
    def as_string(self):
    def dtype(self, context):
    def __repr__(self):
    def __init__(self, op, expr1, expr2):
    def as_simple_expr(self, context):
    def as_string(self):
    def dtype(self, context):
    def __repr__(self):
    def assertbool(self, expr, context):
    def dtype(self, context):
    def dtype(self, context):
    def __init__(self, entity, name, dtype=None):
    def __repr__(self):
    def __str__(self):
    def as_simple_expr(self, context):
    def dtype(self, context):
    def __init__(self, tablename, name, dtype):
    def __repr__(self):
    def get_tmp_varname(self, context):
    def _eval_key(self, context):
    def evaluate(self, context):
    def __getitem__(self, key):
    def dtype(self, context):
    def __init__(self, tablename, name, key, dtype):
    def __repr__(self):
    def _eval_key(self, context):
def index_array_by_variables(array, context, axes):
    def __init__(self, name, dtype=None, autoindex=None):
    def as_simple_expr(self, context):
    def __init__(self, name, fields):
    def __getattr__(self, key):
    def traverse(self, context):
    def __repr__(self):
    def __init__(self, entity, name, args, kwargs):
    def evaluate(self, context):
    def __repr__(self):
    def __call__(self, *args, **kwargs):
    def __init__(self, name, entity):
    def __call__(self, *args, **kwargs):
    def __repr__(self):
    def __str__(self):
    def __init__(self):
class Expr(object):
class EvaluableExpression(Expr):
class SubscriptedExpr(EvaluableExpression):
class ExprAttribute(EvaluableExpression):
class FillFuncNameMeta(ExplainTypeError):
class FillArgSpecMeta(FillFuncNameMeta):
class AbstractFunction(with_metaclass(FillFuncNameMeta, Expr)):
class FunctionExpr(with_metaclass(FillArgSpecMeta, EvaluableExpression, AbstractFunction)):
class GenericFunctionCall(FunctionExpr):
class DynamicFunctionCall(GenericFunctionCall):
class UnaryOp(Expr):
class BinaryOp(Expr):
class DivisionOp(BinaryOp):
class LogicalOp(BinaryOp):
class ComparisonOp(BinaryOp):
class Variable(Expr):
class ShortLivedVariable(Variable):
class GlobalVariable(EvaluableExpression):
class SubscriptedGlobal(GlobalVariable):
class GlobalArray(Variable):
class GlobalTable(object):
class MethodCall(EvaluableExpression):
class VariableMethodHybrid(Variable):
class MethodSymbol(object):
class NotHashable(Expr): class FilteredExpression(FunctionExpr):
    @staticmethod
    def _getfilter(context, filter):
        ctx_filter = context.filter_expr
        # FIXME: this is a hack and shows that the not_hashable filter_expr in
        #  context is not really a good solution. We should rather add a flag
        # in the context "ishardsubset" or something like that.
        if filter is not_hashable:
            filter_expr = ctx_filter
        elif ctx_filter is not_hashable:
            filter_expr = filter
        elif filter is not None and ctx_filter is not None:
            filter_expr = LogicalOp('&', ctx_filter, filter)
        elif filter is not None:
            filter_expr = filter
        elif ctx_filter is not None:
            filter_expr = ctx_filter
        else:
            filter_expr = None
        if filter_expr is not None and \
                getdtype(filter_expr, context) is not bool:
            raise Exception("filter must be a boolean expression")
        return filter_expr class CompoundExpression(with_metaclass(FillArgSpecMeta, AbstractFunction)):
    """
    function expression written in terms of other expressions
    """
    kwonlyargs = {}

    @classmethod
    def get_compute_func(cls):
        return cls.build_expr

    def as_simple_expr(self, context):
        # This will effectively trigger evaluation of expressions arguments
        # which are not handled by numexpr functions such has all expressions
        # inheriting from EvaluableExpression (e.g, uniform()) and their result
        # will be stored as a temporary variables in the context. The subtlety
        # to remember is that if a CompoundExpression "duplicates" arguments
        # (such as Logit), those must be either duplicate-safe or
        # EvaluableExpression. For example, if numexpr someday supports random
        # generators, we will be in trouble if we use it as-is. This means we
        # cannot keep the "compiled" expression, because the "temporary
        # variables" would only have a value in the first period, when the
        # expr is "compiled". This would tick the balance in favor of keeping a
        # build_context method.
        args = [as_simple_expr(arg, context) for arg in self.args]
        kwargs = {name: as_simple_expr(arg, context)
                  for name, arg in self.kwargs}
        expr = self.build_expr(context, *args, **kwargs)
        # We need this because self.build_expr returns an Expr which can
        # contain CompoundExpressions
        return expr.as_simple_expr(context)

    def build_expr(self, context, *args, **kwargs):
        raise NotImplementedError() class NumexprFunction(AbstractFunction):
    """For functions which are present as-is in numexpr"""
    # argspec need to be given manually for each function
    argspec = None

    def as_simple_expr(self, context):
        args, kwargs = as_simple_expr((self.args, self.kwargs), context)
        return self.__class__(*args, **dict(kwargs))

    def as_string(self):
        args, kwargs = as_string((self.args, self.kwargs))
        return '%s(%s)' % (self.funcname, self.format_args_str(args, kwargs)) class TableExpression(FunctionExpr):
    pass class NumpyChangeArray(NumpyFunction):
    def __init__(self, *args, **kwargs):
        # the first argument should be the array to work on ('a')
        assert self.argspec.args[0] == 'a'
        NumpyFunction.__init__(self, *args, **kwargs)

    def compute(self, context, *args, **kwargs):
        filter_value = kwargs.pop('filter', None)

        func = self.get_compute_func()
        new_values = func(*args, **kwargs)

        if filter_value is None:
            return new_values
        else:
            # we cannot do this yet because dtype() currently requires
            # context (and I don't want to change the signature of compute
            # just for that) assert dtype(old_values) == dtype(new_values)
            old_values = args[0]
            return np.where(filter_value, new_values, old_values) def context_length(ctx):
    if hasattr(ctx, 'length'):
        return ctx.length()
    elif '__len__' in ctx:
        return ctx['__len__']
    else:
        usual_len = None
        for k, value in ctx.items():
            if isinstance(value, np.ndarray):
                if usual_len is not None and len(value) != usual_len:
                    raise Exception('incoherent array lengths: %s''s is %d '
                                    'while the len of others is %d' %
                                    (k, len(value), usual_len))
                usual_len = len(value)
        return usual_len def load_ndarray(fpath, celltype=None):
    print(" - reading", fpath)
    # FIXME: implement celltype
    a = la.read_csv(fpath, dialect='liam2')
    # print(a.info)
    return a
    with open(fpath, "rb") as f:
        reader = csv.reader(f)
        line_stream = skip_comment_cells(strip_rows(reader))
        header = line_stream.next()
        str_table = []
        for line in line_stream:
            if any(value == '' for value in line):
                raise Exception("empty cell found in %s" % fpath)
            str_table.append(line)
    ndim = len(header)

    # handle last dimension header (horizontal values)
    last_d_header = str_table.pop(0)
    # auto-detect type of values for the last d and convert them
    last_d_pvalues = convert_1darray(last_d_header)

    unique_last_d, dupe_last_d = unique_duplicate(last_d_pvalues)
    if dupe_last_d:
        print(("Duplicate column header value(s) (for '%s') in '%s': %s"
              % (header[-1], fpath,
                 ", ".join(str(v) for v in dupe_last_d))))
        raise Exception("bad data in '%s': found %d "
                        "duplicate column header value(s)"
                        % (fpath, len(dupe_last_d)))

    # handle other dimensions header

    # strip the ndim-1 first columns
    headers = [[line.pop(0) for line in str_table]
               for _ in range(ndim - 1)]
    headers = [convert_1darray(pvalues_str) for pvalues_str in headers]
    if ndim > 1:
        # having duplicate values is normal when there are more than 2
        # dimensions but we need to test whether there are duplicates of
        # combinations.
        dupe_combos = list(duplicates(zip(*headers)))
        if dupe_combos:
            print(("Duplicate row header value(s) in '%s':" % fpath))
            print((PrettyTable(dupe_combos)))
            raise Exception("bad alignment data in '%s': found %d "
                            "duplicate row header value(s)"
                            % (fpath, len(dupe_combos)))

    possible_values = [np.array(list(unique(pvalues))) for pvalues in headers]
    possible_values.append(np.array(unique_last_d))

    shape = tuple(len(values) for values in possible_values)
    num_possible_values = prod(shape)

    # transform the 2d table into a 1d list
    str_table = list(chain.from_iterable(str_table))
    if len(str_table) != num_possible_values:
        raise Exception("incoherent data in '%s': %d data cells "
                        "found while it should be %d based on the number "
                        "of possible values in headers (%s)"
                        % (fpath,
                           len(str_table),
                           num_possible_values,
                           ' * '.join(str(len(values))
                                      for values in possible_values)))

    # TODO: compare time with numpy built-in conversion:
    # if dtype is None, numpy tries to detect the best type itself
    # which it does a good job of if the values are already numeric values
    # if dtype is provided, numpy does a good job to convert from string
    # values.
    if celltype is None:
        celltype = detect_column_type(str_table)
    data = convert_1darray(str_table, celltype)
    array = np.array(data, dtype=celltype)
    return la.LArray(array.reshape(shape), header, possible_values) def load_table(fpath, fields=None, newnames=None, delimiter=None, transpose=False):
    csv_file = CSV(fpath, newnames=newnames, delimiter=delimiter, transpose=transpose)
    array = csv_file.as_array(fields)
    csv_file.close()
    return array class PrettyTable(object):
    def __init__(self, data, missing=None):
        assert isinstance(data, (tuple, list))
        self.data = data
        self.missing = missing

    def __iter__(self):
        if self.missing is not None:
            return iter(self.convert_nans())
        else:
            return iter(self.data)

    def __len__(self):
        return len(self.data)

    def convert_nans(self):
        missing = self.missing
        for row in self.data:
            formatted = [missing if value != value else value
                         for value in row]
            yield formatted

    def __repr__(self):
        missing = self.missing
        if missing is None:
            missing = 'nan'
        else:
            missing = str(missing)
        return '\n' + table2str(self.data, missing) + '\n' def argspec(*args, **kwonlyargs):
    """
    >>> argspec('a, b, c=1, *d, **e, f=None')
    ... # doctest: +NORMALIZE_WHITESPACE
    FullArgSpec(args=['a', 'b', 'c'], varargs='d', varkw='e', defaults=(1,),
                kwonlyargs=['f'], kwonlydefaults={'f': None}, annotations={})
    >>> a = argspec('*, f=None', g=None)
    >>> a.kwonlyargs
    ['f', 'g']
    >>> sorted(a.kwonlydefaults.items())
    [('f', None), ('g', None)]
    >>> argspec('a, *, **b, c=None')
    ... # doctest: +NORMALIZE_WHITESPACE
    FullArgSpec(args=['a'], varargs=None, varkw='b', defaults=None,
                kwonlyargs=['c'], kwonlydefaults={'c': None}, annotations={})
    >>> argspec('a, b, c=1, d=None')
    ... # doctest: +NORMALIZE_WHITESPACE
    FullArgSpec(args=['a', 'b', 'c', 'd'], varargs=None, varkw=None,
                defaults=(1, None), kwonlyargs=[], kwonlydefaults={},
                annotations={})
    >>> argspec('')
    ... # doctest: +NORMALIZE_WHITESPACE
    FullArgSpec(args=[], varargs=None, varkw=None, defaults=None, kwonlyargs=[],
                kwonlydefaults={}, annotations={})
    >>> str(argspec('a, b, c=1, *d, **e, f=None'))
    'a, b, c=1, *d, **e, f=None'
    """
    if len(args) == 1 and isinstance(args[0], basestring):
        str_args = args[0].split(',') if args[0] else []
        str_args = [a.strip().split('=') for a in str_args]
        args = [(a[0], ast.literal_eval(a[1])) if len(a) > 1 else a[0]
                for a in str_args]

        def star(a):
            return isinstance(a, basestring) and '*' in a
        if any(star(a) for a in args):
            while not star(args[-1]):
                k, v = args.pop()
                assert k not in kwonlyargs, "several kwonlyargs named %s" % k
                kwonlyargs[k] = v
    return _argspec(*args, **kwonlyargs)
from itertools import chain
from liam2 import config
from liam2.compat import zip, basestring, long
from liam2.expr import (Variable, UnaryOp, BinaryOp, ComparisonOp, DivisionOp, LogicalOp, getdtype, coerce_types,
                        expr_eval, as_simple_expr, as_string, collect_variables, get_default_array, get_default_vector,
                        FunctionExpr, always, firstarg_dtype, expr_cache, index_array_by_variables)
from liam2.exprbases import FilteredExpression, CompoundExpression, NumexprFunction, TableExpression, NumpyChangeArray
from liam2.context import context_length
from liam2.importer import load_ndarray, load_table
from liam2.utils import PrettyTable, argspec
import os
import random
import numpy as np
    import scipy
    import scipy.special as special
import larray as la
            # working with columns of structured arrays.
            # See https://github.com/numpy/numpy/issues/2462
            result[to_give_birth] = children['id']
            return result
        else:
            return None

    dtype = always(int)


class Clone(New):
    def __init__(self, filter=None, **kwargs):
        New.__init__(self, None, filter, None, **kwargs)

    def _initial_values(self, array, to_give_birth, num_birth, default_values):
        return array[to_give_birth]


class Dump(TableExpression):
    no_eval = ('args',)
    kwonlyargs = {'filter': None, 'missing': None, 'header': True,
                  'limit': None}

    def compute(self, context, *args, **kwargs):
        filter_value = kwargs.pop('filter', None)
        missing = kwargs.pop('missing', None)
        # periods = kwargs.pop('periods', None)
        header = kwargs.pop('header', True)
        limit = kwargs.pop('limit', None)
        entity = context.entity

        if args:
            expressions = list(args)
        else:
            # extra=False because we don't want globals nor "system" variables
            # (nan, period, __xxx__)
            # FIXME: we should also somehow "traverse" expressions in this case
            # too (args is ()) => all keys in the current context
            expressions = [Variable(entity, name)
                           for name in context.keys(extra=False)]

        str_expressions = [str(e) for e in expressions]
        if 'id' not in str_expressions:
            str_expressions.insert(0, 'id')
            expressions.insert(0, Variable(entity, 'id'))
            id_pos = 0
        else:
            id_pos = str_expressions.index('id')

        #        if (self.periods is not None and len(self.periods) and
        #            'period' not in str_expressions):
        #            str_expressions.insert(0, 'period')
        #            expressions.insert(0, Variable('period'))
        #            id_pos += 1

        columns = []
        for expr in expressions:
            if filter_value is False:
                # dtype does not matter much
                expr_value = np.empty(0)
            else:
                # TODO: set filter before evaluating expressions
                expr_value = expr_eval(expr, context)
                if (filter_value is not None and
                        isinstance(expr_value, np.ndarray) and
                        expr_value.shape):
                    expr_value = expr_value[filter_value]
            columns.append(expr_value)

        ids = columns[id_pos]
        if isinstance(ids, np.ndarray) and ids.shape:
            numrows = len(ids)
        else:
            # FIXME: we need a test for this case (no idea how this can happen)
            numrows = 1

        # expand scalar columns to full columns in memory
        for idx, col in enumerate(columns):
            dtype = None
            if not isinstance(col, np.ndarray):
                dtype = type(col)
            elif not col.shape:
                dtype = col.dtype.type

            if dtype is not None:
                # TODO: try using itertools.repeat instead as it seems to be a
                # bit faster and would consume less memory (however, it might
                # not play very well with Pandas.to_csv)
                newcol = np.full(numrows, col, dtype=dtype)
                columns[idx] = newcol
            elif col.ndim > 1:
                # move last axis (should be id axis) first
                # np.moveaxis requires numpy >= 1.11
                # columns[idx] = np.moveaxis(col, -1, 0)
                columns[idx] = col.transpose((-1,) + tuple(range(col.ndim - 1)))

        assert all(isinstance(col, np.ndarray) for col in columns)
        bad_lengths = {str_expr: col.shape for col, str_expr in zip(columns, str_expressions)
                       if col.shape[0] != numrows}
        if bad_lengths:
            raise ValueError("first dimension of some columns are not the same length as the id column (%d): %s"
                             % (numrows, str(bad_lengths)))

        if limit is not None:
            assert isinstance(limit, (int, long))
            columns = [col[:limit] for col in columns]

        # Transform to Python lists of normal Python types (ie no numpy types).
        # on py2, csv.writer uses repr(value) for float and str(value) for other but
        # on py3 since str(float) == repr(float), they switched to str(value) for everything
        # but str(np.float64) does not have full precision (truncated at the 12th decimal)
        # besides, this seems to be faster (but probably takes more memory).
        # Also on python2, converting produces nicer/shorter float strings (see issue #225).
        columns = [c.tolist() for c in columns]
        data = zip(*columns)
        if header:
            table = [str_expressions]
            table.extend(data)
        else:
            table = list(data)

        return PrettyTable(table, missing)