# -*- coding: utf-8 -*-

'''
mint - small, fast and simple template engine.
'''

import os
import re
import ast
import mmap
import time
import fnmatch
import logging
import weakref
import itertools
import htmlentitydefs
from ast import Load, Store, Param
from StringIO import StringIO
from functools import partial
from collections import deque
from xml.etree.ElementTree import TreeBuilder as _TreeBuilder, Element

############# LEXER

class BaseToken(object):
    pass

class TokenWrapper(BaseToken):
    '''
    Objects of this class reprezents tokens
    '''

    def __init__(self, token, value=None, regex_str=None):
        assert value or regex_str, 'Provide token text value or regex'
        self.token = intern(token)
        if regex_str is not None:
            self.regex = re.compile(regex_str, re.U)
        else:
            self.regex = re.compile(r'%s' % re.escape(value), re.U)

    def __str__(self):
        return self.token

    __repr__ = __str__

class TextToken(BaseToken):
    'Special token for text'
    def __str__(self):
        return 'text'
    __repr__ = __str__

class TokenIndent(BaseToken):
    def __str__(self):
        return 'indent'
    __repr__ = __str__

class TokenUnindent(BaseToken):
    def __str__(self):
        return 'unindent'
    __repr__ = __str__

class EOF(BaseToken):
    'Special token'
    def __str__(self):
        return 'eof'
    __repr__ = __str__

# constants
TAG_CHAR = '@'
STMT_CHAR = '#'
COMMENT_CHAR = '--'

# Tokens
TOKEN_TAG_START = TokenWrapper('tag_start', value=TAG_CHAR)
TOKEN_TAG_ATTR_SET = TokenWrapper('tag_attr_set', value='%s.' % TAG_CHAR)
TOKEN_TAG_ATTR_APPEND = TokenWrapper('tag_attr_append', value='%s+' % TAG_CHAR)
TOKEN_BASE_TEMPLATE = TokenWrapper('base_template', value='%sbase: ' % STMT_CHAR)
TOKEN_STATEMENT_IF = TokenWrapper('statement_if', value='%sif ' % STMT_CHAR)
TOKEN_STATEMENT_ELIF = TokenWrapper('statement_elif', regex_str=r'(%selif |%selse if )' % (
    re.escape(STMT_CHAR), re.escape(STMT_CHAR)))
TOKEN_STATEMENT_ELSE = TokenWrapper('statement_else', value='%selse:' % STMT_CHAR)
TOKEN_STATEMENT_FOR = TokenWrapper('statement_for', value='%sfor ' % STMT_CHAR)
TOKEN_SLOT_DEF = TokenWrapper('slot_def', regex_str=r'(%sdef |%sfunction )' % (re.escape(STMT_CHAR),
                                                                               re.escape(STMT_CHAR)))
TOKEN_STMT_CHAR = TokenWrapper('hash', value=STMT_CHAR)
TOKEN_COMMENT = TokenWrapper('comment', value=COMMENT_CHAR)
TOKEN_BACKSLASH = TokenWrapper('backslash', value='\\')
TOKEN_DOT = TokenWrapper('dot', value='.')
TOKEN_PLUS = TokenWrapper('plus', value='+')
TOKEN_MINUS = TokenWrapper('minus', value='-')
TOKEN_COLON = TokenWrapper('colon', value=':')
TOKEN_PARENTHESES_OPEN = TokenWrapper('parentheses_open', value='(')
TOKEN_PARENTHESES_CLOSE = TokenWrapper('parentheses_close', value=')')
TOKEN_EXPRESSION_START = TokenWrapper('expression_start', value='{{')
TOKEN_EXPRESSION_END = TokenWrapper('expression_end', value='}}')
TOKEN_WHITESPACE = TokenWrapper('whitespace', regex_str=r'\s+')
TOKEN_NEWLINE = TokenWrapper('newline', regex_str=r'(\r\n|\r|\n)')
TOKEN_EOF = EOF()
TOKEN_TEXT = TextToken()
TOKEN_INDENT = TokenIndent()
TOKEN_UNINDENT = TokenUnindent()

tokens = (
    TOKEN_TAG_ATTR_SET,
    TOKEN_TAG_ATTR_APPEND,
    TOKEN_TAG_START,
    TOKEN_BASE_TEMPLATE,
    TOKEN_STATEMENT_IF,
    TOKEN_STATEMENT_ELIF,
    TOKEN_STATEMENT_ELSE,
    TOKEN_STATEMENT_FOR,
    TOKEN_SLOT_DEF,
    TOKEN_STMT_CHAR,
    TOKEN_COMMENT,
    TOKEN_BACKSLASH,
    TOKEN_DOT,
    TOKEN_PLUS,
    TOKEN_MINUS,
    TOKEN_PARENTHESES_OPEN,
    TOKEN_PARENTHESES_CLOSE,
    TOKEN_EXPRESSION_START,
    TOKEN_EXPRESSION_END,
    TOKEN_COLON,
    TOKEN_WHITESPACE,
    TOKEN_NEWLINE,
)

all_tokens = list(tokens) + [TOKEN_EOF, TOKEN_TEXT, TOKEN_INDENT, TOKEN_UNINDENT]
all_except = lambda *t: filter(lambda x: x not in t, all_tokens)
re_comment = re.compile(r'\s*//')

def base_tokenizer(fp):
    'Tokenizer. Generates tokens stream from text'
    if isinstance(fp, StringIO):
        template_file = fp
        size = template_file.len
    else:
        #empty file check
        if os.fstat(fp.fileno()).st_size == 0:
            yield TOKEN_EOF, 'EOF', 0, 0
            return
        template_file = mmap.mmap(fp.fileno(), 0, access=mmap.ACCESS_READ)
        size = template_file.size()
    lineno = 0
    while 1:
        lineno += 1
        pos = 1

        # end of file
        if template_file.tell() == size:
            yield TOKEN_EOF, 'EOF', lineno, 0
            break

        # now we tokinize line by line
        line = template_file.readline().decode('utf-8')
        line = line.replace('\r\n', '')
        line = line.replace('\n', '')
        # ignoring non XML comments
        if re_comment.match(line):
            continue

        last_text = deque()
        while line:
            line_len = len(line)
            for token in tokens:
                m = token.regex.match(line)
                if m:
                    if last_text:
                        yield TOKEN_TEXT, ''.join(last_text), lineno, pos
                        pos += len(last_text)
                        last_text.clear()
                    offset, value = m.end(), m.group()
                    line = line[offset:]
                    yield token, value, lineno, pos
                    pos += offset
                    break

            # we did not get right in tokens list, so next char is text
            if line_len == len(line):
                last_text.append(line[0])
                line = line[1:]

        if last_text:
            yield TOKEN_TEXT, ''.join(last_text), lineno, pos
            pos += len(last_text)
            last_text.clear()
        yield TOKEN_NEWLINE, '\n', lineno, pos

    # all work is done
    template_file.close()

def indent_tokenizer(tokens_stream):
    current_indent = 0
    indent = 0
    for tok in tokens_stream:
        token, value, lineno, pos = tok
        # backslashed line transfer
        if token is TOKEN_BACKSLASH:
            next_tok = tokens_stream.next()
            next_token, next_value, next_lineno, next_pos = next_tok
            if next_token is TOKEN_NEWLINE:
                next_tok = tokens_stream.next()
                while next_tok[0] in (TOKEN_WHITESPACE, TOKEN_NEWLINE):
                    next_tok = tokens_stream.next()
                # first not newline or whitespace token
                yield next_tok
                continue
            yield tok
            tok = next_tok
            token, value, lineno, pos = next_tok
        # indenting and unindenting
        if token is TOKEN_NEWLINE or (token is TOKEN_WHITESPACE and (lineno, pos) == (1, 1)):
            if token is TOKEN_NEWLINE:
                yield tok
                next_tok = tokens_stream.next()
                while next_tok[0] is TOKEN_NEWLINE:
                    next_tok = tokens_stream.next()
            else:
                next_tok = tok
            next_token, next_value, next_lineno, next_pos = next_tok
            if next_token is TOKEN_WHITESPACE:
                ws_count = len(next_value)
                if indent == 0:
                    indent = ws_count
                if ws_count >= indent:
                    times = ws_count/indent
                    rest = ws_count % indent
                    range_ = times - current_indent
                    if range_ > 0:
                        # indenting
                        tmp_curr_indent = current_indent
                        for i in range(range_):
                            yield TOKEN_INDENT, ' '*indent, next_lineno, (i+tmp_curr_indent)*indent+1
                            current_indent += 1
                    elif range_ < 0:
                        # unindenting
                        for i in range(abs(range_)):
                            yield TOKEN_UNINDENT, ' '*indent, next_lineno, next_pos
                            current_indent -= 1
                    if rest:
                        yield TOKEN_WHITESPACE, ' '*rest, next_lineno, times*indent+1
                    continue
            # next token is the whitespace lighter than indent or any other
            # token, so unindenting to zero level
            for i in range(current_indent):
                yield TOKEN_UNINDENT, ' '*indent, lineno, pos
            current_indent = 0
            yield next_tok
            # we do not yielding newline tokens
            continue
        yield tok

def tokenizer(fileobj):
    return indent_tokenizer(base_tokenizer(fileobj))

############# LEXER END

UNSAFE_CHARS = '&<>"'
CHARS_ENTITIES = dict([(v, '&%s;' % k) for k, v in htmlentitydefs.entitydefs.items()])
UNSAFE_CHARS_ENTITIES = [(k, CHARS_ENTITIES[k]) for k in UNSAFE_CHARS]
UNSAFE_CHARS_ENTITIES_IN_ATTR = [(k, CHARS_ENTITIES[k]) for k in '<>"']
UNSAFE_CHARS_ENTITIES.append(("'",'''))
UNSAFE_CHARS_ENTITIES_IN_ATTR.append(("'",'''))
UNSAFE_CHARS_ENTITIES_REVERSED = [(v,k) for k,v in UNSAFE_CHARS_ENTITIES]

def escape(obj, ctx='tag'):
    if hasattr(obj, '__html__'):
        safe_markup = obj.__html__()
        if ctx == 'tag':
            return safe_markup
        else:
            for k, v in UNSAFE_CHARS_ENTITIES_IN_ATTR:
                safe_markup = safe_markup.replace(k, v)
            return safe_markup
    obj = unicode(obj)
    for k, v in UNSAFE_CHARS_ENTITIES:
        obj = obj.replace(k, v)
    return obj

def unescape(obj):
    text = unicode(obj)
    for k, v in UNSAFE_CHARS_ENTITIES_REVERSED:
        text = text.replace(k, v)
    return text

class TemplateError(Exception): pass
class WrongToken(Exception): pass

# variables names (we do not want to override user variables and vise versa)
TREE_BUILDER = '__MINT_TREE_BUILDER__'
TREE_FACTORY = '__MINT_TREE_FACTORY__'
MAIN_FUNCTION = '__MINT_MAIN__'
TAG_START = '__MINT_TAG_START__'
TAG_END = '__MINT_TAG_END__'
DATA = '__MINT_DATA__'
ESCAPE_HELLPER = '__MINT_ESCAPE__'
CURRENT_NODE = '__MINT_CURRENT_NODE__'

##### MINT NODES

class Node(ast.AST):
    def __repr__(self):
        return '%s' % self.__class__.__name__

class MintTemplate(Node):
    def __init__(self, body=None):
        self.body = body or []

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.body==other.body
        return False

    def __repr__(self):
        return '%s(body=%r)' % (self.__class__.__name__, self.body)

class BaseTemplate(Node):
    def __init__(self, name):
        self.name = name

    def to_ast(self):
        return self

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.name == other.name
        return False

class TextNode(Node):
    def __init__(self, text, lineno=None, col_offset=None):
        self.text = text
        self.lineno = lineno
        self.col_offset = col_offset

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.text==other.text and self.lineno==other.lineno \
                   and self.col_offset==other.col_offset
        return False

    def __repr__(self):
        return '%s(%r, lineno=%d, col_offset=%d)' % (self.__class__.__name__, self.text,
                                                     self.lineno, self.col_offset)

class ExpressionNode(Node):
    def __init__(self, text, lineno=None, col_offset=None):
        self.text = text.strip()
        self.lineno = lineno
        self.col_offset = col_offset

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.text==other.text and self.lineno==other.lineno \
                   and self.col_offset==other.col_offset
        return False

    def __repr__(self):
        return '%s(%r, lineno=%d, col_offset=%d)' % (self.__class__.__name__, self.text,
                                                     self.lineno, self.col_offset)

class TagAttrNode(Node):
    def __init__(self, name, value=None, lineno=None, col_offset=None):
        self.name = escape(name, ctx='attr')
        self.value = value or []
        self.lineno = lineno
        self.col_offset = col_offset

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.name==other.name and self.value==other.value and self.lineno==other.lineno \
                   and self.col_offset==other.col_offset
        return False

    def __repr__(self):
        return '%s(%r, value=%r, lineno=%d, col_offset=%d)' % (self.__class__.__name__, self.name,
                                                               self.value, self.lineno, self.col_offset)

class SetAttrNode(Node):
    def __init__(self, attr_node):
        self.attr = attr_node

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.attr==other.attr
        return False

class AppendAttrNode(Node):
    def __init__(self, attr_node):
        self.attr = attr_node

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.attr==other.attr
        return False

class TagNode(Node):
    def __init__(self, name, attrs=None, body=None, lineno=None, col_offset=None):
        self.name = name
        self.attrs = attrs or []
        self.body = body or []
        self.lineno = lineno
        self.col_offset = col_offset

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.name==other.name and self.body==other.body and self.attrs==other.attrs\
                   and self.lineno==other.lineno and self.col_offset==other.col_offset
        return False

    def __repr__(self):
        return '%s(%r, attrs=%r, body=%r, lineno=%d, col_offset=%d)' % (self.__class__.__name__, self.name,
            self.attrs, self.body, self.lineno, self.col_offset)

class ForStmtNode(Node):
    def __init__(self, text, body=None, lineno=None, col_offset=None):
        self.text = text.strip()
        self.body = body or []
        self.lineno = lineno
        self.col_offset = col_offset

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.text==other.text and self.body==other.body and self.lineno==other.lineno \
                   and self.col_offset==other.col_offset
        return False

    def __repr__(self):
        return '%s(%r, body=%r, lineno=%d, col_offset=%d)' % (self.__class__.__name__, self.text,
                                                              self.body, self.lineno, self.col_offset)

class IfStmtNode(Node):
    def __init__(self, text, body=None, orelse=None, lineno=None, col_offset=None):
        self.text = text
        self.body = body or []
        self.orelse = orelse or []
        self.lineno = lineno
        self.col_offset = col_offset

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.text==other.text and self.body==other.body and self.orelse==other.orelse\
                   and self.lineno==other.lineno and self.col_offset==other.col_offset
        return False

    def __repr__(self):
        return '%s(%r, body=%r, orelse=%r, lineno=%d, col_offset=%d)' % (self.__class__.__name__,
                                                self.text, self.body,
                                                self.orelse, self.lineno, self.col_offset)

class ElseStmtNode(Node):
    def __init__(self, body=None, lineno=None, col_offset=None):
        self.body = body or []
        self.lineno = lineno
        self.col_offset = col_offset

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.body==other.body and self.lineno==other.lineno \
                   and self.col_offset==other.col_offset
        return False

    def __repr__(self):
        return '%s(body=%r, lineno=%d, col_offset=%d)' % (self.__class__.__name__, self.body,
                                                          self.lineno, self.col_offset)

class SlotDefNode(Node):
    def __init__(self, text, body=None, lineno=None, col_offset=None):
        self.text = text.strip()
        self.body = body or []
        self.lineno = lineno
        self.col_offset = col_offset

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.text==other.text and self.body==other.body and self.lineno==other.lineno \
                   and self.col_offset==other.col_offset
        return False

    def __repr__(self):
        return '%s(%r, body=%r, lineno=%d, col_offset=%d)' % (self.__class__.__name__, self.text,
                                                              self.body, self.lineno, self.col_offset)

class SlotCallNode(Node):
    def __init__(self, text, lineno=None, col_offset=None):
        self.text = text.strip()
        self.lineno = lineno
        self.col_offset = col_offset

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.text==other.text and self.lineno==other.lineno \
                   and self.col_offset==other.col_offset
        return False

    def __repr__(self):
        return '%s(%r, lineno=%d, col_offset=%d)' % (self.__class__.__name__, self.text,
                                                     self.lineno, self.col_offset)

##### NODES END

##### PARSER

class RecursiveStack(object):
    'Stack of stacks'
    def __init__(self):
        self.stacks = [[]]

    @property
    def stack(self):
        return self.stacks[-1]

    @property
    def current(self):
        return self.stack and self.stack[-1] or None

    def push(self, item):
        self.stack.append(item)
        return True

    def pop(self):
        return self.stack.pop()
        return True

    def push_stack(self, new_stack):
        self.stacks.append(new_stack)

    def pop_stack(self):
        return self.stacks.pop()

    def __nonzero__(self):
        return len(self.stacks)

    def __repr__(self):
        return repr(self.stacks)

    def __iter__(self):
        return reversed(self.stack[:])

class Parser(object):
    def __init__(self, states):
        self.states = dict(states)

    def parse(self, tokens_stream, stack):
        current_state = 'start'
        variantes = self.states[current_state]
        for tok in tokens_stream:
            token, tok_value, lineno, pos = tok

            # accept new token
            new_state = None
            for item in variantes:
                variante, state, callback = item
                # tokens sequence
                if isinstance(variante, basestring):
                    variante = globals().get(variante)
                if isinstance(variante, (list, tuple)):
                    if token in variante:
                        new_state = state
                        break
                elif variante is token:
                    new_state = state
                    break
                elif isinstance(variante, Parser):
                    variante.parse(itertools.chain([tok], tokens_stream), stack)
                    new_state = state
                    #NOTE: tok still points to first token

            if new_state is None:
                raise WrongToken('[%s] Unexpected token "%s(%r)" at line %d, pos %d' \
                        % (current_state, token, tok_value, lineno, pos))
            # process of new_state
            elif new_state != current_state:
                if new_state == 'end':
                    #print current_state, '%s(%r)' % (token, tok_value), new_state
                    callback(tok, stack)
                    #_print_stack(stack)
                    break
                current_state = new_state
                variantes = self.states[current_state]
            # state callback
            #print current_state, '%s(%r)' % (token, tok_value), new_state
            callback(tok, stack)
            #_print_stack(stack)

def _print_stack(s):
    print '[stack]'
    for i in s:
        print ' '*4, i
    print '[end of stack]\n'

# utils functions
def get_tokens(s):
    my_tokens = []
    while s.current and isinstance(s.current, (list, tuple)):
        my_tokens.append(s.pop())
    my_tokens.reverse()
    return my_tokens

#NOTE: Callbacks are functions that takes token and stack
skip = lambda t, s: None
push = lambda t, s: s.push(t)
pop_stack = lambda t, s: s.pop_stack()
def push_stack(t, s):
    if isinstance(s.current, ElseStmtNode):
        stmt = s.pop()
        s.push_stack(stmt.body)
    elif isinstance(s.current, IfStmtNode) and s.current.orelse:
        s.push_stack(s.current.orelse[-1].body)
    else:
        if not hasattr(s.current, 'body'):
            raise SyntaxError('Unexpected indent at line %d' % t[2])
        s.push_stack(s.current.body)

# text data and inline python expressions
def py_expr(t, s):
    my_tokens = get_tokens(s)
    lineno, col_offset = my_tokens[0][2], my_tokens[0][3] - 2
    s.push(ExpressionNode(u''.join([t[1] for t in my_tokens]),
                                lineno=lineno, col_offset=col_offset))

def text_value(t, s):
    my_tokens = get_tokens(s)
    if my_tokens:
        lineno, col_offset = my_tokens[0][2], my_tokens[0][3]
        s.push(TextNode(u''.join([t[1] for t in my_tokens]),
                        lineno=lineno, col_offset=col_offset))

def text_value_with_last(t, s):
    s.push(t)
    text_value(t, s)

# parser of attribute value
attr_data_parser = Parser((
    # state name
    ('start', (
        # variantes (token, new_state, callback)
        #           ((token, token,...), new_state, callback)
        #           (other_parser, new_state, callback)
        #           ('other_parser', new_state, callback)
        (TOKEN_EXPRESSION_START, 'expr', text_value),
        (TOKEN_PARENTHESES_CLOSE, 'end', text_value),
        (all_except(TOKEN_NEWLINE), 'start', push),
        )),
    ('expr', (
        (TOKEN_EXPRESSION_END, 'start', py_expr),
        (all_tokens, 'expr', push),
        )),
))

# parser of text data and inline python expressions
data_parser = Parser((
    ('start', (
        (TOKEN_EXPRESSION_START, 'expr', text_value),
        (TOKEN_NEWLINE, 'end', text_value_with_last),
        (all_except(TOKEN_INDENT), 'start', push),
        )),
    ('expr', (
        (TOKEN_EXPRESSION_END, 'start', py_expr),
        (all_tokens, 'expr', push),
        )),
))

# tag and tag attributes callbacks
def tag_name(t, s):
    #if isinstance(s.current, (list, tuple)):
    my_tokens = get_tokens(s)
    if my_tokens:
        lineno, col_offset = my_tokens[0][2], my_tokens[0][3] - 1
        s.push(TagNode(u''.join([t[1] for t in my_tokens]),
                       lineno=lineno, col_offset=col_offset))

def tag_attr_name(t, s):
    my_tokens = get_tokens(s)
    lineno, col_offset = my_tokens[0][2], my_tokens[0][3]
    s.push(TagAttrNode(u''.join([t[1] for t in my_tokens]),
                       lineno=lineno, col_offset=col_offset))

def tag_attr_value(t, s):
    nodes = []
    while not isinstance(s.current, TagAttrNode):
        nodes.append(s.pop())
    attr = s.current
    nodes.reverse()
    attr.value = nodes

def set_attr(t, s):
    nodes = []
    while not isinstance(s.current, TagAttrNode):
        nodes.append(s.pop())
    attr = s.pop()
    nodes.reverse()
    attr.value = nodes
    s.push(SetAttrNode(attr))

def append_attr(t, s):
    nodes = []
    while not isinstance(s.current, TagAttrNode):
        nodes.append(s.pop())
    attr = s.pop()
    nodes.reverse()
    attr.value = nodes
    s.push(AppendAttrNode(attr))

def tag_node(t, s):
    attrs = []
    while isinstance(s.current, TagAttrNode):
        attrs.append(s.pop())
    tag = s.pop()
    # if there were no attrs
    if isinstance(tag, (list, tuple)):
        my_tokens = get_tokens(s)
        my_tokens.append(tag)
        lineno, col_offset = my_tokens[0][2], my_tokens[0][3] - 1
        tag = TagNode(u''.join([t[1] for t in my_tokens]),
                      lineno=lineno, col_offset=col_offset)
    if attrs:
        tag.attrs = attrs
    s.push(tag)

def tag_node_with_data(t, s):
    tag_node(t, s)
    push_stack(t, s)

# tag parser
tag_parser = Parser((
    ('start', (
        (TOKEN_TEXT, 'start', push),
        (TOKEN_MINUS, 'start', push),
        (TOKEN_COLON, 'start', push),
        (TOKEN_DOT, 'attr', tag_name),
        (TOKEN_WHITESPACE, 'continue', tag_node_with_data),
        (TOKEN_NEWLINE, 'end', tag_node),
        )),
    ('attr', (
        (TOKEN_TEXT, 'attr', push),
        (TOKEN_MINUS, 'attr', push),
        (TOKEN_COLON, 'attr', push),
        (TOKEN_PARENTHESES_OPEN, 'attr_value', tag_attr_name),
        )),
    ('attr_value', (
        (attr_data_parser, 'start', tag_attr_value),
        )),
    ('continue', (
        (TOKEN_TAG_START, 'nested_tag', skip),
        (TOKEN_NEWLINE, 'end', pop_stack),
        (data_parser, 'end', pop_stack),
        )),
    ('nested_tag', (
        ('nested_tag_parser', 'end', pop_stack),
        )),
))

# this is modified tag parser, supports inline tags with data
nested_tag_parser = Parser(dict(tag_parser.states, start=(
        (TOKEN_TEXT, 'start', push),
        (TOKEN_MINUS, 'start', push),
        (TOKEN_COLON, 'start', push),
        (TOKEN_DOT, 'attr', tag_name),
        (TOKEN_WHITESPACE, 'continue', tag_node_with_data),
        (TOKEN_NEWLINE, 'end', tag_node),
        )
).iteritems())

# base parser callbacks
def base_template(t, s):
    my_tokens = get_tokens(s)
    lineno, col_offset = my_tokens[0][2], my_tokens[0][3]
    s.push(BaseTemplate(u''.join([t[1] for t in my_tokens])))

def html_comment(t, s):
    my_tokens = get_tokens(s)
    lineno, col_offset = my_tokens[0][2], my_tokens[0][3]
    s.push(TextNode(Markup(u'<!-- %s -->' % (u''.join([t[1] for t in my_tokens])).strip()),
                       lineno=lineno, col_offset=col_offset))

def for_stmt(t, s):
    my_tokens = get_tokens(s)
    lineno, col_offset = my_tokens[0][2], my_tokens[0][3]
    s.push(ForStmtNode(u''.join([t[1] for t in my_tokens]),
                       lineno=lineno, col_offset=col_offset))

def if_stmt(t, s):
    my_tokens = get_tokens(s)
    lineno, col_offset = my_tokens[0][2], my_tokens[0][3]
    s.push(IfStmtNode(u''.join([t[1] for t in my_tokens]),
                       lineno=lineno, col_offset=col_offset))

def elif_stmt(t, s):
    if not isinstance(s.current, IfStmtNode):
        pass
        #XXX: raise TemplateError
    my_tokens = get_tokens(s)
    lineno, col_offset = my_tokens[0][2], my_tokens[0][3]
    stmt = IfStmtNode(u''.join([t[1] for t in my_tokens]),
                       lineno=lineno, col_offset=col_offset)
    s.current.orelse.append(stmt)

def else_stmt(t, s):
    lineno, col_offset = t[2], t[3] - 6
    if not isinstance(s.current, IfStmtNode):
        pass
        #XXX: raise TemplateError
    stmt = ElseStmtNode(lineno=lineno, col_offset=col_offset)
    # elif
    if s.current.orelse:
        s.current.orelse[-1].orelse.append(stmt)
    # just else
    else:
        s.current.orelse.append(stmt)
    s.push(stmt)

def slot_def(t, s):
    my_tokens = get_tokens(s)
    lineno, col_offset = my_tokens[0][2], my_tokens[0][3]
    s.push(SlotDefNode(u''.join([t[1] for t in my_tokens]),
                       lineno=lineno, col_offset=col_offset))

def slot_call(t, s):
    my_tokens = get_tokens(s)
    lineno, col_offset = my_tokens[0][2], my_tokens[0][3]
    s.push(SlotCallNode(u''.join([t[1] for t in my_tokens]),
                       lineno=lineno, col_offset=col_offset))

# base parser (MAIN PARSER)
block_parser = Parser((
    # start is always the start of a new line
    ('start', (
        (TOKEN_TEXT, 'text', push),
        (TOKEN_EXPRESSION_START, 'expr', skip),
        (TOKEN_TAG_ATTR_SET, 'set_attr', skip),
        (TOKEN_TAG_ATTR_APPEND, 'append_attr', skip),
        (TOKEN_TAG_START, 'tag', skip),
        (TOKEN_STATEMENT_FOR, 'for_stmt', push),
        (TOKEN_STATEMENT_IF, 'if_stmt', push),
        (TOKEN_STATEMENT_ELIF, 'elif_stmt', push),
        (TOKEN_STATEMENT_ELSE, 'else_stmt', skip),
        (TOKEN_SLOT_DEF, 'slot_def', push),
        (TOKEN_BASE_TEMPLATE, 'base', skip),
        (TOKEN_STMT_CHAR, 'slot_call', skip),
        (TOKEN_COMMENT, 'comment', skip),
        (TOKEN_BACKSLASH, 'escaped_text', skip),
        (TOKEN_INDENT, 'indent', push_stack),
        (TOKEN_UNINDENT, 'start', pop_stack),
        (TOKEN_NEWLINE, 'start', skip),
        (TOKEN_EOF, 'end', skip),
        (all_tokens, 'text', push),
        )),
    # to prevent multiple indentions in a row
    ('indent', (
        (TOKEN_TEXT, 'text', push),
        (TOKEN_EXPRESSION_START, 'expr', skip),
        (TOKEN_TAG_ATTR_APPEND, 'append_attr', skip),
        (TOKEN_TAG_ATTR_SET, 'set_attr', skip),
        (TOKEN_TAG_START, 'tag', skip),
        (TOKEN_STATEMENT_FOR, 'for_stmt', push),
        (TOKEN_STATEMENT_IF, 'if_stmt', push),
        (TOKEN_STATEMENT_ELIF, 'elif_stmt', push),
        (TOKEN_STATEMENT_ELSE, 'else_stmt', skip),
        (TOKEN_SLOT_DEF, 'slot_def', push),
        (TOKEN_STMT_CHAR, 'slot_call', skip),
        (TOKEN_COMMENT, 'comment', skip),
        (TOKEN_BACKSLASH, 'escaped_text', skip),
        (TOKEN_NEWLINE, 'start', skip),
        (TOKEN_UNINDENT, 'start', pop_stack),
        )),
    ('base', (
        (TOKEN_NEWLINE, 'start', base_template),
        (all_tokens, 'base', push),
        )),
    ('text', (
        (TOKEN_EXPRESSION_START, 'expr', text_value),
        (TOKEN_NEWLINE, 'start', text_value_with_last),
        (all_except(TOKEN_INDENT), 'text', push),
        )),
    ('expr', (
        (TOKEN_EXPRESSION_END, 'text', py_expr),
        (all_tokens, 'expr', push),
        )),
    ('escaped_text', (
        (TOKEN_NEWLINE, 'start', text_value_with_last),
        (all_except(TOKEN_INDENT), 'escaped_text', push),
        )),
    ('tag', (
        (tag_parser, 'start', skip),
        )),
    ('comment', (
        (TOKEN_NEWLINE, 'start', html_comment),
        (all_tokens, 'comment', push),
        )),
    ('set_attr', (
        (TOKEN_TEXT, 'set_attr', push),
        (TOKEN_MINUS, 'set_attr', push),
        (TOKEN_COLON, 'set_attr', push),
        (TOKEN_PARENTHESES_OPEN, 'set_attr_value', tag_attr_name),
        )),
    ('set_attr_value', (
        (attr_data_parser, 'start', set_attr),
        )),
    ('append_attr', (
        (TOKEN_TEXT, 'append_attr', push),
        (TOKEN_MINUS, 'append_attr', push),
        (TOKEN_COLON, 'append_attr', push),
        (TOKEN_PARENTHESES_OPEN, 'append_attr_value', tag_attr_name),
        )),
    ('append_attr_value', (
        (attr_data_parser, 'start', append_attr),
        )),
    ('for_stmt', (
        (TOKEN_NEWLINE, 'start', for_stmt),
        (all_tokens, 'for_stmt', push),
        )),
    ('if_stmt', (
        (TOKEN_NEWLINE, 'start', if_stmt),
        (all_tokens, 'if_stmt', push),
        )),
    ('elif_stmt', (
        (TOKEN_NEWLINE, 'start', elif_stmt),
        (all_tokens, 'elif_stmt', push),
        )),
    ('else_stmt', (
        (TOKEN_NEWLINE, 'start', else_stmt),
        #(all_tokens, 'else_stmt', push),
        )),
    ('slot_def', (
        (TOKEN_NEWLINE, 'start', slot_def),
        (all_tokens, 'slot_def', push),
        )),
    ('slot_call', (
        (TOKEN_NEWLINE, 'start', slot_call),
        (all_tokens, 'slot_call', push),
        )),
))

############# PARSER END

class AstWrapper(object):
    def __init__(self, lineno, col_offset):
        assert lineno is not None and col_offset is not None
        self.lineno = lineno
        self.col_offset = col_offset
    def __getattr__(self, name):
        attr = getattr(ast, name)
        return partial(attr, lineno=self.lineno, col_offset=self.col_offset, ctx=Load())

class MintToPythonTransformer(ast.NodeTransformer):

    def visit_MintTemplate(self, node):
        ast_ = AstWrapper(1,1)
        module = ast_.Module(body=[
            ast_.FunctionDef(name=MAIN_FUNCTION,
                             body=[],
                             args=ast_.arguments(args=[], vararg=None, kwargs=None, defaults=[]),
                             decorator_list=[])])
        body = module.body[0].body
        for n in node.body:
            result = self.visit(n)
            if isinstance(result, (list, tuple)):
                for i in result:
                    body.append(i)
            else:
                body.append(result)
        return module

    def visit_TextNode(self, node):
        ast_ = AstWrapper(node.lineno, node.col_offset)
        return ast_.Expr(value=ast_.Call(func=ast_.Name(id=DATA),
                                         args=[self.get_value(node, ast_)],
                                         keywords=[], starargs=None, kwargs=None))

    def visit_ExpressionNode(self, node):
        ast_ = AstWrapper(node.lineno, node.col_offset)
        return ast_.Expr(value=ast_.Call(func=ast_.Name(id=DATA),
                                         args=[self.get_value(node, ast_)],
                                         keywords=[], starargs=None, kwargs=None))

    def visit_SetAttrNode(self, node):
        ast_ = AstWrapper(node.attr.lineno, node.attr.col_offset)
        key, value = self.get_value(node.attr, ast_)
        return ast_.Expr(value=ast_.Call(func=ast_.Attribute(value=ast_.Name(id=CURRENT_NODE),
                                                            attr='set'),
                                        args=[key, value],
                                        keywords=[],
                                        starargs=None, kwargs=None))

    def visit_AppendAttrNode(self, node):
        ast_ = AstWrapper(node.attr.lineno, node.attr.col_offset)
        key, value = self.get_value(node.attr, ast_)
        value = ast_.BinOp(
            left=ast_.BoolOp(
                values=[ast_.Call(
                    func=ast_.Attribute(value=ast_.Name(id=CURRENT_NODE),
                                        attr='get'),
                    args=[key],
                    keywords=[],
                    starargs=None, kwargs=None), ast_.Str(s=u'')],
                op=ast.Or()),
            op=ast.Add(),
            right=value)
        return ast_.Expr(value=ast_.Call(func=ast_.Attribute(value=ast_.Name(id=CURRENT_NODE),
                                                            attr='set'),
                                        args=[key, value],
                                        keywords=[],
                                        starargs=None, kwargs=None))

    def visit_TagNode(self, node):
        ast_ = AstWrapper(node.lineno, node.col_offset)
        name = CURRENT_NODE
        attrs = ast_.Dict(keys=[], values=[])
        for a in node.attrs:
            k, v = self.get_value(a, ast_)
            attrs.keys.append(k)
            attrs.values.append(v)
        nodes = []
        # tag start
        node_start = ast_.Assign(targets=[ast_.Name(id=name, ctx=Store())],
                           value=ast_.Call(func=ast_.Name(id=TAG_START),
                                           args=[ast_.Str(s=escape(node.name)), attrs],
                                           keywords=[], starargs=None, kwargs=None))
        nodes.append(node_start)
        for n in node.body:
            result = self.visit(n)
            if isinstance(result, (list, tuple)):
                for i in result:
                    nodes.append(i)
            else:
                nodes.append(result)
        # tag end
        node_end = ast_.Expr(value=ast_.Call(func=ast_.Name(id=TAG_END),
                                             args=[ast_.Str(s=escape(node.name))],
                                             keywords=[], starargs=None, kwargs=None))
        nodes.append(node_end)
        return nodes

    def visit_ForStmtNode(self, node):
        ast_ = AstWrapper(node.lineno, node.col_offset)
        result = []
        expr = node.text[1:]
        if not expr.endswith(':'):
            expr += ':'
        expr += 'pass'
        value = ast.parse(expr).body[0]
        for n in node.body:
            result = self.visit(n)
            if isinstance(result, (list, tuple)):
                for i in result:
                    value.body.append(i)
            else:
                value.body.append(result)
        value.lineno = ast_.lineno
        value.col_offset = ast_.col_offset
        return value

    def visit_IfStmtNode(self, node):
        ast_ = AstWrapper(node.lineno, node.col_offset)
        result = []
        expr = node.text[1:]
        if not expr.endswith(':'):
            expr += ':'
        expr += 'pass'
        if expr.startswith('el'):
            expr = expr[2:]
        value = ast.parse(expr).body[0]
        value.body = []
        value.lineno = ast_.lineno
        value.col_offset = ast_.col_offset
        #XXX: if nodes is empty list raise TemplateError
        for n in node.body:
            result = self.visit(n)
            if isinstance(result, (list, tuple)):
                for i in result:
                    value.body.append(i)
            else:
                value.body.append(result)
        for n in node.orelse:
            result = self.visit(n)
            if isinstance(result, (list, tuple)):
                for i in result:
                    value.orelse.append(i)
            else:
                value.orelse.append(result)
        return value

    def visit_ElseStmtNode(self, node):
        value = []
        for n in node.body:
            result = self.visit(n)
            if isinstance(result, (list, tuple)):
                for i in result:
                    value.append(i)
            else:
                value.append(result)
        return value

    def visit_SlotDefNode(self, node):
        ast_ = AstWrapper(node.lineno, node.col_offset)
        result = []
        expr = node.text[1:]
        if not expr.endswith(':'):
            expr += ':'
        expr += 'pass'
        value = ast.parse(expr).body[0]
        value.lineno = ast_.lineno
        value.col_offset = ast_.col_offset
        #XXX: if self.nodes is empty list raise TemplateError
        for n in node.body:
            result = self.visit(n)
            if isinstance(result, (list, tuple)):
                for i in result:
                    value.body.append(i)
            else:
                value.body.append(result)
        return value

    def visit_SlotCallNode(self, node):
        ast_ = AstWrapper(node.lineno, node.col_offset)
        expr = node.text
        value = ast.parse(expr).body[0].value
        value.lineno = ast_.lineno
        value.col_offset = ast_.col_offset
        return ast_.Expr(value=ast_.Call(func=ast_.Name(id=DATA),
                                         args=[value], keywords=[]))

    def get_value(self, node, ast_, ctx='tag'):
        if isinstance(node, TextNode):
            return ast_.Str(s=escape(node.text, ctx=ctx))
        elif isinstance(node, ExpressionNode):
            expr = ast.parse(node.text).body[0].value
            return ast_.Call(func=ast_.Name(id=ESCAPE_HELLPER),
                             args=[expr],
                             keywords=[ast.keyword(arg='ctx', value=ast_.Str(s=ctx))],
                             starargs=None, kwargs=None)
        elif isinstance(node, TagAttrNode):
            key = ast_.Str(s=node.name)
            value = ast_.Str(s=u'')
            nodes = list(node.value)
            if nodes:
                value = ast_.Call(func=ast_.Attribute(value=ast_.Str(s=u''),
                                                      attr='join'),
                                  args=[ast_.Tuple(elts=[self.get_value(n, ast_, ctx='attr') for n in nodes])],
                                  keywords=[], starargs=None, kwargs=None)
            return key, value

class SlotsGetter(ast.NodeTransformer):
    'Node transformer, collects slots'
    def __init__(self):
        self.slots = {}
        self.base = None
    def visit_FunctionDef(self, node):
        ast_ = AstWrapper(node.lineno, node.col_offset)
        new_tree_call = ast_.Assign(targets=[ast_.Tuple(elts=[
                                                         ast_.Name(id=TREE_BUILDER, ctx=Store()),
                                                         ast_.Name(id=TAG_START, ctx=Store()),
                                                         ast_.Name(id=TAG_END, ctx=Store()),
                                                         ast_.Name(id=DATA, ctx=Store())],
                                                       ctx=Store())],
                                    value=ast_.Call(func=ast_.Name(id=TREE_FACTORY),
                                                    args=[],
                                                    keywords=[], starargs=None, kwargs=None))
        tree_to_unicode_call = ast_.Return(value=ast_.Call(func=ast_.Attribute(
                                                value=ast_.Name(id=TREE_BUILDER),
                                                attr='to_unicode'),
                                           args=[],
                                           keywords=[]))
        node.body.insert(0, new_tree_call)
        node.body.append(tree_to_unicode_call)
        if node.name == MAIN_FUNCTION:
            _nones = []
            for n in node.body:
                v = self.visit(n)
                if v is None:
                    _nones.append(n)
            for n in _nones:
                node.body.remove(n)
            return node
        self.slots[node.name] = node
        node.name = 'slot_' + os.urandom(5).encode('hex')
    def visit_BaseTemplate(self, node):
        self.base = node.name

def _correct_inheritance(new_slots, old_slots):
    slots = {}
    for k, value in new_slots.items():
        if k in old_slots:
            name = '__base__'
            old_value = old_slots[k]
            ast_ = AstWrapper(old_value.lineno + 1, old_value.col_offset)
            value.body.insert(0, ast_.Assign(targets=[ast_.Name(id=name, ctx=Store())],
                                             value=ast_.Name(id=old_value.name)))
            del old_slots[k]
            # this slot is overrided in child template
            old_slots[k+'__overrided'] = old_value
        slots[k] = value
    slots.update(old_slots)
    return slots

def get_mint_tree(tokens_stream):
    '''
    This function is wrapper to normal parsers (tag_parser, block_parser, etc.).
    Returns mint tree.
    '''
    smart_stack = RecursiveStack()
    block_parser.parse(tokens_stream, smart_stack)
    return MintTemplate(body=smart_stack.stack)

############# API
class TemplateNotFound(Exception):
    pass

class TreeBuilder(_TreeBuilder):
    'Tree with root element already set'
    def __init__(self, *args, **kwargs):
        _TreeBuilder.__init__(self, *args, **kwargs)
        self.start('root', {})

    def to_unicode(self):
        class dummy: pass
        data = []
        out = dummy()
        out.write = data.append
        # out - fast writable object
        self.end('root')
        root = self.close()
        if root.text:
            out.write(root.text)
        for node in root:
            self._node_to_unicode(out, node)
        if root.tail:
            out.write(root.tail)
        return Markup(u''.join(data))

    def _node_to_unicode(self, out, node):
        #NOTE: all data must be escaped during tree building
        tag = node.tag
        items = node.items()
        selfclosed = ['link', 'input', 'br', 'hr', 'img', 'meta']
        out.write(u'<' + tag)
        if items:
            items.sort() # lexical order
            for k, v in items:
                out.write(u' %s="%s"' % (k, v))
        if tag in selfclosed:
            out.write(u' />')
        else:
            out.write(u'>')
            if node.text or len(node):
                if node.text:
                    out.write(node.text)
                for n in node:
                    self._node_to_unicode(out, n)
            out.write(u'</' + tag + '>')
            if node.tail:
                out.write(node.tail)

class PprintTreeBuilder(_TreeBuilder):
    'Tree with root element already set'
    def __init__(self, *args, **kwargs):
        _TreeBuilder.__init__(self, *args, **kwargs)
        self.start('root', {})
        self._level = -1

    @property
    def indention(self):
        return self._level > 0 and '  '*self._level or ''

    def to_unicode(self):
        class dummy: pass
        data = []
        out = dummy()
        out.write = data.append
        # out - fast writable object
        self.end('root')
        root = self.close()
        if root.text:
            out.write(self.indent_text(root.text))
            out.write('\n')
        for node in root:
            self._node_to_unicode(out, node)
        if root.tail:
            out.write(self.indent_text(root.tail))
        return Markup(u''.join(data))

    def _node_to_unicode(self, out, node):
        #NOTE: all data must be escaped during tree building
        self.indent()
        tag = node.tag
        items = node.items()
        selfclosed = ['link', 'input', 'br', 'hr', 'img', 'meta']
        children = list(node)
        text = node.text
        tail = node.tail
        out.write(self.indention)
        out.write(u'<' + tag)
        if items:
            items.sort() # lexical order
            for k, v in items:
                out.write(u' %s="%s"' % (k, v))
        if tag in selfclosed:
            out.write(u' />')
        else:
            out.write(u'>')
            if text:
                if text.endswith('\n'):
                    text = text[:-1]
                self.indent()
                out.write('\n')
                out.write(self.indent_text(text))
                out.write('\n')
                self.unindent()
            if children:
                out.write('\n')
                for n in children:
                    self._node_to_unicode(out, n)

            if children or text:
                out.write(self.indention)
            out.write(u'</' + tag + '>')
            if node.tail:
                out.write('\n')
                tail = node.tail
                if tail.endswith('\n'):
                    tail = tail[:-1]
                out.write(self.indent_text(tail))
        out.write('\n')
        self.unindent()

    def indent_text(self, text):
        return '\n'.join([self.indention+t for t in text.split('\n')])

    def indent(self):
        self._level += 1
    def unindent(self):
        self._level -= 1

def new_tree(pprint):
    def wrapper():
        tree = pprint and PprintTreeBuilder() or TreeBuilder()
        return tree, tree.start, tree.end, tree.data
    return wrapper

class Template(object):

    def __init__(self, source, filename=None, loader=None, globals=None, pprint=False):
        assert source or filename, 'Please provide source code or filename'
        self.source = source
        self.filename = filename if filename else '<string>'
        self._loader = loader
        self.compiled_code = compile(self.tree(), self.filename, 'exec')
        self.globals = globals or {}
        self.pprint = pprint

    def tree(self, slots=None):
        slots = slots or {}
        source = StringIO(self.source) if self.source else open(self.filename, 'r')
        mint_tree = get_mint_tree(tokenizer(source))
        tree = MintToPythonTransformer().visit(mint_tree)
        slots_getter = SlotsGetter()
        slots_getter.visit(tree.body[0])
        _slots, base_template_name = slots_getter.slots, slots_getter.base
        # we do not want to override slot's names,
        # so prefixing existing slots with underscore
        slots = _correct_inheritance(slots, _slots)
        if base_template_name:
            base_template = self._loader.get_template(base_template_name)
            tree = base_template.tree(slots=slots)
        elif slots is not None:
            # insert implementation of slots
            # def slot_bb13e100d5(): ...
            # and insert assings of slots
            # real_slot_name = slot_bb13e100d5
            for k,v in slots.items():
                if not k.endswith('__overrided'):
                    ast_ = AstWrapper(v.lineno, v.col_offset)
                    tree.body.insert(0, ast_.Assign(targets=[ast_.Name(id=k, ctx=Store())],
                                                     value=ast_.Name(id=v.name)))
                tree.body.insert(0, v)
        # tree already has slots definitions and ready to be compiled
        return tree

    def render(self, **kwargs):
        ns = {
            'utils':utils,
            ESCAPE_HELLPER:escape,
            TREE_FACTORY:new_tree(self.pprint),
        }
        ns.update(self.globals)
        ns.update(kwargs)
        exec self.compiled_code in ns
        # execute template main function
        return ns[MAIN_FUNCTION]()

    def slot(self, name, **kwargs):
        ns = {
            'utils':utils,
            ESCAPE_HELLPER:escape,
            TREE_FACTORY:new_tree(self.pprint),
        }
        ns.update(self.globals)
        ns.update(kwargs)
        exec self.compiled_code in ns
        return ns[name]

class Loader(object):

    def __init__(self, *dirs, **kwargs):
        self.dirs = []
        # dirs - list of directories. Order matters
        for d in dirs:
            self.dirs.append(os.path.abspath(d))
        self.cache = kwargs.get('cache', False)
        self._templates_cache = {}
        self.globals = kwargs.get('globals', {})
        self.pprint = kwargs.get('pprint', 0)

    def get_template(self, template):
        if template in self._templates_cache:
            return self._templates_cache[template]
        for dir in self.dirs:
            location = os.path.join(dir, template)
            if os.path.exists(location) and os.path.isfile(location):
                with open(location, 'r') as f:
                    tmpl = Template(source=f.read(), filename=f.name,
                                    loader=self, globals=self.globals, pprint=self.pprint)
                    if self.cache:
                        self._templates_cache[template] = tmpl
                    return tmpl
        raise TemplateNotFound(template)

    def __add__(self, other):
        dirs = self.dirs + other.dirs
        return self.__class__(cache=self.cache, globals=self.globals,*dirs)

#NOTE: Taken from jinja2
class Markup(unicode):

    def __new__(cls, obj=u'', **kwargs):
        if hasattr(obj, '__html__'):
            obj = obj.__html__()
        return super(Markup, cls).__new__(cls, obj, **kwargs)

    def __html__(self):
        return self

    def __add__(self, other):
        if hasattr(other, '__html__') or isinstance(other, basestring):
            return self.__class__(unicode(self) + unicode(escape(other)))
        return NotImplemented

    def __radd__(self, other):
        if hasattr(other, '__html__') or isinstance(other, basestring):
            return self.__class__(unicode(escape(other)) + unicode(self))
        return NotImplemented

    def __mul__(self, num):
        if isinstance(num, (int, long)):
            return self.__class__(unicode.__mul__(self, num))
        return NotImplemented
    __rmul__ = __mul__

    def join(self, seq):
        return self.__class__(unicode.join(self, itertools.imap(escape, seq)))
    join.__doc__ = unicode.join.__doc__

    def split(self, *args, **kwargs):
        return map(self.__class__, unicode.split(self, *args, **kwargs))
    split.__doc__ = unicode.split.__doc__

    def rsplit(self, *args, **kwargs):
        return map(self.__class__, unicode.rsplit(self, *args, **kwargs))
    rsplit.__doc__ = unicode.rsplit.__doc__

    def splitlines(self, *args, **kwargs):
        return map(self.__class__, unicode.splitlines(self, *args, **kwargs))
    splitlines.__doc__ = unicode.splitlines.__doc__

    def __repr__(self):
        return 'Markup(%s)' % super(Markup, self).__repr__()

class utils(object):

    class doctype:
        html_strict = Markup('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" '
                                '"http://www.w3.org/TR/html4/strict.dtd">')
        html_transitional = Markup('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 '
                          'Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">')
        xhtml_strict = Markup('<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" '
                                 '"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">')
        xhtml_transitional = Markup('<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 '
        'Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">')
        html5 = Markup('<!DOCTYPE html>')

    markup = Markup

    @staticmethod
    def loop(iterable):
        return Looper(iterable)

    @staticmethod
    def entity(char):
        return Markup(CHARS_ENTITIES.get(char, char))

    @staticmethod
    def script(src=None, data=None, type='text/javascript'):
        if src:
            return Markup('<script type="%s" src="%s"></script>' % (type, src))
        elif data:
            return Markup('<script type="%s">%s</script>' % (type, data))
        return ''

    @staticmethod
    def scripts(*args, **kwargs):
        result = []
        for name in args:
            result.append(utils.script(name, **kwargs))
        return ''.join(result)

    @staticmethod
    def link(href, rel='stylesheet', type='text/css'):
        return Markup('<link rel="%s" type="%s" href="%s" />' % (rel, type, href))

class Looper:
    'Cool class taken from PPA project'
    class _Item:
        def __init__(self, index, has_next):
            self.index = index
            self.has_next = has_next
            self.last = not has_next
            self.first = not index
        @property
        def odd(self):
            return self.index % 2
        @property
        def even(self):
            return not self.index % 2
        def cycle(self, *args):
            'Magic method (adopted ;)'
            return args[self.index % len(args)]

    def __init__(self, iterable):
        self._iterator = iter(iterable)

    def _shift(self):
        try:
            self._next = self._iterator.next()
        except StopIteration:
            self._has_next = False
        else:
            self._has_next = True

    def __iter__(self):
        self._shift()
        index = 0
        while self._has_next:
            value = self._next
            self._shift()
            yield value, self._Item(index, self._has_next)
            index += 1

############# API END

class Printer(ast.NodeVisitor):
    'AST printer'

    def __init__(self):
        self._indent = 0
        self._indent_tab = '    '
        self.src = StringIO()
        self.write = self.src.write
        self._in_args = False
        self._in_if = False

    def make_tab(self):
        self.src.write(self._indent*self._indent_tab)

    def visit_FunctionDef(self, node):
        self.make_tab()
        self.src.write('def %s(' % node.name)
        self._in_args = True
        total_args = len(node.args.args)
        default_args_len = len(node.args.defaults) if node.args.defaults else 0
        for i, arg in enumerate(node.args.args):
            if i != 0:
                self.src.write(', ')
            self.visit(arg)
            if default_args_len > 0 and i >= (total_args - default_args_len):
                self.src.write('=')
                y = (total_args - default_args_len) - i
                self.visit(node.args.defaults[y])
        self._in_args = False
        self.src.write('):\n')
        self._indent += 1
        for n in node.body:
            self.visit(n)
        self._indent -= 1

    def visit_Return(self, node):
        self.make_tab()
        self.src.write('return ')
        self.visit(node.value)

    def visit_Name(self, node):
        self.src.write(node.id)

    def visit_Str(self, node):
        self.src.write('%r' % node.s)

    def visit_Num(self, node):
        self.src.write('%d' % node.n)

    def visit_Pass(self, node):
        self.make_tab()
        self.src.write('pass\n')

    def visit_If(self, node):
        self.make_tab()
        if self._in_if:
            self.src.write('elif ')
        else:
            self.src.write('if ')
        self.visit(node.test)
        self.src.write(':\n')
        self._indent += 1
        for n in node.body:
            self.visit(n)
        self._indent -= 1
        if node.orelse:
            self._in_if = True
            if not isinstance(node.orelse[0], ast.If):
                self.make_tab()
                self.src.write('else:\n')
                self._indent += 1
            for orelse in node.orelse:
                self.visit(orelse)
            if not isinstance(node.orelse[0], ast.If):
                self._indent -= 1
            self._in_if = False

    def visit_Compare(self, node):
        self.visit(node.left)
        self.src.write(' ')
        for op in node.ops:
            self.visit(op)
        self.src.write(' ')
        for comp in node.comparators:
            self.visit(comp)

    def visit_For(self, node):
        self.make_tab()
        self.write('for ')
        self.visit(node.target)
        self.write(' in ')
        self._in_args = True
        self.visit(node.iter)
        self._in_args = False
        self.write(':\n')
        self._indent += 1
        for n in node.body:
            self.visit(n)
        self._indent -= 1

    def visit_Tuple(self, node):
        self.src.write('(')
        for i,el in enumerate(node.elts):
            if i != 0:
                self.src.write(', ')
            self.visit(el)
        self.src.write(')')

    def visit_List(self, node):
        self.src.write('[')
        for i,el in enumerate(node.elts):
            if i != 0:
                self.src.write(', ')
            self.visit(el)
        self.src.write(']')

    def visit_Dict(self, node):
        self.src.write('{')
        total_keys = len(node.keys)
        for i in range(total_keys):
            if i != 0:
                self.src.write(', ')
            self.visit(node.keys[i])
            self.src.write(': ')
            self.visit(node.values[i])
        self.src.write('}')

    def visit_Assign(self, node):
        self.make_tab()
        for i, target in enumerate(node.targets):
            if i != 0:
                self.src.write(', ')
            self.visit(target)
        self.src.write(' = ')
        self._in_args = True
        self.visit(node.value)
        self._in_args = False
        self.src.write('\n')

    def visit_Call(self, node):
        if self._in_args:
            self.visit(node.func)
            self.src.write('(')
            for i, arg in enumerate(node.args):
                if i != 0:
                    self.src.write(', ')
                self.visit(arg)
            self.src.write(')')
        else:
            self.make_tab()
            self.visit(node.func)
            self.src.write('(')
            self._in_args = True
            for i, arg in enumerate(node.args):
                if i != 0:
                    self.src.write(', ')
                self.visit(arg)
            self.src.write(')')
            self._in_args = False
            self.src.write('\n')

    def visit_Attribute(self, node):
        self.visit(node.value)
        self.src.write('.')
        self.src.write(node.attr)

    def visit_BinOp(self, node):
        self.visit(node.left)
        self.src.write(' ')
        self.visit(node.op)
        self.src.write(' ')
        self.visit(node.right)

    def visit_BoolOp(self, node):
        for i, n in enumerate(node.values):
            self.visit(n)
            if not i:
                self.src.write(' ')
                self.visit(node.op)
                self.src.write(' ')

    # Operators
    def visit_Add(self, node):
        self.src.write('+')
    def visit_Mod(self, node):
        self.src.write('%')
    def visit_Eq(self, node):
        self.src.write('==')
    def visit_NotEq(self, node):
        self.src.write('!=')
    def visit_Lt(self, node):
        self.src.write('<=')
    def visit_Gt(self, node):
        self.src.write('>=')
    def visit_Or(self, node):
        self.src.write('or')

def all_files_by_mask(mask):
    for root, dirs, files in os.walk('.'):
        for basename in files:
            if fnmatch.fnmatch(basename, mask):
                filename = os.path.join(root, basename)
                yield filename

def render_templates(*templates, **kw):
    loader = kw['loader']
    for template_name in templates:
        result = loader.get_template(template_name).render()
        if result:
            open(template_name[:-4]+'html', 'w').write(result)

def iter_changed(interval=1):
    mtimes = {}
    while 1:
        for filename in all_files_by_mask('*.mint'):
            try:
                mtime = os.stat(filename).st_mtime
            except OSError:
                continue
            old_time = mtimes.get(filename)
            if old_time is None:
                mtimes[filename] = mtime
                continue
            elif mtime > old_time:
                mtimes[filename] = mtime
                yield filename
        time.sleep(interval)

if __name__ == '__main__':
    import datetime
    from optparse import OptionParser
    parser = OptionParser()
    parser.add_option('-c', '--code', dest='code', action='store_true',
                      default=False,
                      help='Show only python code of compiled template.')
    parser.add_option('-t', '--tokenize', dest='tokenize', action='store_true',
                      default=False,
                      help='Show tokens stream of template.')
    parser.add_option('-r', '--repeat', dest='repeat',
                      default=0, metavar='N', type='int',
                      help='Try to render template N times and display average time result.')
    parser.add_option('-p', '--pprint', dest='pprint', action='store_true',
                      default=False,
                      help='Turn pretty print on.')
    parser.add_option('-m', '--monitor', dest='monitor', action='store_true',
                      default=False,
                      help='Monitor current directory and subdirectories for changes in mint files. '
                           'And render corresponding html files.')
    (options, args) = parser.parse_args()
    loader = Loader('.', pprint=options.pprint)
    if len(args) > 0:
        template_name = args[0]
        template = loader.get_template(template_name)
        if options.code:
            printer = Printer()
            printer.visit(template.tree())
            print printer.src.getvalue()
        elif options.tokenize:
            for t in tokenizer(StringIO(template.source)):
                print t
        else:
            print template.render()
        if options.repeat > 0:
            now = datetime.datetime.now
            results = []
            for i in range(options.repeat):
                start = now()
                template.render()
                results.append(now() - start)
            print 'Total time (%d repeats): ' % options.repeat, reduce(lambda a,b: a+b, results)
            print 'Average:                 ', reduce(lambda a,b: a+b, results)/len(results)
    elif options.monitor:
        curdir = os.path.abspath(os.getcwd())
        try:
            render_templates(*all_files_by_mask('*.mint'), loader=loader)
            print 'Monitoring for file changes...'
            for changed_file in iter_changed():
                print 'Changes in file: ', changed_file, datetime.datetime.now().strftime('%H:%M:%S')
                render_templates(changed_file, loader=loader)
        except KeyboardInterrupt:
            pass
    else:
        print 'Try --help'


#!/usr/bin/python

import argparse
import time
import logging
import os
import socket
import sys
import lxml.etree
from boto.route53 import Route53Connection

R53_API_VERSION = '2013-04-01'
R53_XMLNS = 'https://route53.amazonaws.com/doc/%s/' % R53_API_VERSION
XML_PARSER = lxml.etree.XMLParser(remove_blank_text=True)
XSLT_STRIPSPACE = lxml.etree.XSLT(lxml.etree.XML('''
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
  <xsl:strip-space elements="*"/>
</xsl:stylesheet>''', parser=XML_PARSER))

log = logging.getLogger('route53client')
log.setLevel(logging.DEBUG)

class ZoneNotFoundError(Exception):
  """Raised when unable to resolve a zone to its ID."""

def lookup_zone(conn, zone):
  """Look up a zone ID for a zone string.

  Args: conn: boto.route53.Route53Connection
        zone: string eg. foursquare.com
  Returns: zone ID eg. ZE2DYFZDWGSL4.
  Raises: ZoneNotFoundError if zone not found."""
  all_zones = conn.get_all_hosted_zones()
  for resp in all_zones['ListHostedZonesResponse']['HostedZones']:
    if resp['Name'].rstrip('.') == zone.rstrip('.'):
      return resp['Id'].replace('/hostedzone/', '')
  raise ZoneNotFoundError('zone %s not found in response' % zone)

def fetch_config(zone, conn):
  """Fetch all pieces of a Route 53 config from Amazon.

  Args: zone: string, hosted zone id.
        conn: boto.route53.Route53Connection
  Returns: list of ElementTrees, one for each piece of config."""
  more_to_fetch = True
  cfg_chunks = []
  next_name = None
  next_type = None
  next_identifier = None
  while more_to_fetch == True:
    more_to_fetch = False
    getstr = '/%s/hostedzone/%s/rrset' % (R53_API_VERSION, zone)
    if next_name is not None:
      getstr += '?name=%s&type=%s' % (next_name, next_type)
      if next_identifier is not None:
        getstr += '&identifier=%s' % next_identifier
    log.debug('requesting %s' % getstr)
    resp = conn.make_request('GET', getstr)
    etree = lxml.etree.parse(resp)
    cfg_chunks.append(etree)
    root = etree.getroot()
    truncated = root.find('{%s}IsTruncated' % R53_XMLNS)
    if truncated is not None and truncated.text == 'true':
      more_to_fetch = True
      next_name = root.find('{%s}NextRecordName' % R53_XMLNS).text
      next_type = root.find('{%s}NextRecordType' % R53_XMLNS).text
      try:
        next_identifier = root.find('{%s}NextRecordIdentifier' % R53_XMLNS).text
      except AttributeError:  # may not have next_identifier
        next_identifier = None
  return cfg_chunks

def merge_config(cfg_chunks):
  """Merge a set of fetched Route 53 config Etrees into a canonical form.

  Args: cfg_chunks: [ lxml.etree.ETree ]
  Returns: lxml.etree.Element"""
  root = lxml.etree.XML('<ResourceRecordSets xmlns="%s"></ResourceRecordSets>' % R53_XMLNS, parser=XML_PARSER)
  for chunk in cfg_chunks:
    for rrset in chunk.iterfind('.//{%s}ResourceRecordSet' % R53_XMLNS):
      root.append(rrset)
  return root

class InvalidArgumentException(Exception):
  pass

def normalize_rrs(rrsets):
  """Lexically sort the order of every ResourceRecord in a ResourceRecords
  element so we don't generate spurious changes: ordering of e.g. NS records
  is irrelevant to the DNS line protocol, but XML sees it differently.

  Also rewrite any wildcard records to use the ascii hex code: somewhere deep
  inside route53 is something that used to look like tinydns, and amazon's
  API will always display wildcard records as "\052.example.com".

  Args: rrsest: lxml.etree.Element (<ResourceRecordSets>) """
  for rrset in rrsets:
    if rrset.tag == '{%s}ResourceRecordSet' % R53_XMLNS:
      for rrs in rrset:
        # preformat wildcard records
        if rrs.tag == '{%s}Name' % R53_XMLNS:
          if rrs.text.startswith('*.'):
            old_text = rrs.text
            new_text = '\\052.%s' % old_text[2:]
            print 'Found wildcard record, rewriting to %s' % new_text
            rrs.text = rrs.text.replace(old_text, new_text)
        # sort ResourceRecord elements by Value
        if rrs.tag == '{%s}ResourceRecords' % R53_XMLNS:
          # 0th value of ResourceRecord is always the Value element
          sorted_rrs = sorted(rrs, key=lambda x: x[0].text)
          rrs[:] = sorted_rrs
  return rrsets

def generate_changeset(old, new, comment=None):
  """Diff two XML configs and return an object with changes to be written.

  Args: old, new: lxml.etree.Element (<ResourceRecordSets>).
  Returns: lxml.etree.ETree (<ChangeResourceRecordSetsRequest>) or None"""
  rrsets_tag = '{%s}ResourceRecordSets' % R53_XMLNS
  if rrsets_tag not in (old.tag, new.tag):
    log.error('both configs must be ResourceRecordSets tags. old: %s, new: %s' % (old.tag, new.tag))
    raise InvalidArgumentException()
  if comment is None:
    comment = 'Generated by %s for %s@%s at %s.' % (
        __file__,
        os.environ['USER'],
        socket.gethostname(),
        time.strftime('%Y-%m-%d %H:%M:%S'))
  root = lxml.etree.XML("""<ChangeResourceRecordSetsRequest xmlns="%s">
                        <ChangeBatch>
                          <Comment>%s</Comment>
                          <Changes/>
                        </ChangeBatch>
                        </ChangeResourceRecordSetsRequest>""" % (
      R53_XMLNS, comment), parser=XML_PARSER)
  changesroot = root.find('.//{%s}Changes' % R53_XMLNS)
  old = normalize_rrs(old)
  new = normalize_rrs(new)
  oldset = set([lxml.etree.tostring(x).rstrip() for x in old])
  newset = set([lxml.etree.tostring(x).rstrip() for x in new])
  if oldset == newset:
      return None
  # look for removed elements
  for rrs in old:
    rrsst = lxml.etree.tostring(rrs).rstrip()
    if rrsst not in newset:
      log.debug("REMOVED:")
      log.debug(rrsst)
      change = lxml.etree.XML('<Change xmlns="%s"><Action>DELETE</Action></Change>' % R53_XMLNS, parser=XML_PARSER)
      change.append(rrs)
      changesroot.append(change)
  # look for added elements
  for rrs in new:
    rrsst = lxml.etree.tostring(rrs).rstrip()
    if rrsst not in oldset:
      log.debug("ADDED:")
      log.debug(rrsst)
      change = lxml.etree.XML('<Change xmlns="%s"><Action>CREATE</Action></Change>' % R53_XMLNS, parser=XML_PARSER)
      change.append(rrs)
      changesroot.append(change)
  return root

def validate_changeset(changeset):
  """Validate a changeset is compatible with Amazon's API spec.

  Args: changeset: lxml.etree.Element (<ChangeResourceRecordSetsRequest>)
  Returns: [ errors ] list of error strings or []."""
  errors = []
  changes = changeset.findall('.//{%s}Change' % R53_XMLNS)
  num_changes = len(changes)
  if num_changes == 0:
    errors.append('changeset must have at least one <Change> element')
  if num_changes > 100:
    errors.append('changeset has %d <Change> elements: max is 100' % num_changes)
  rrs = changeset.findall('.//{%s}ResourceRecord' % R53_XMLNS)
  num_rrs = len(rrs)
  if num_rrs > 1000:
    errors.append('changeset has %d ResourceRecord elements: max is 1000' % num_rrs)
  values = changeset.findall('.//{%s}Value' % R53_XMLNS)
  num_chars = 0
  for value in values:
    num_chars += len(value.text)
  if num_chars > 10000:
    errors.append('changeset has %d chars in <Value> text: max is 10000' % num_chars)
  return errors

def normalize_xml(xml):
  """Normalize an XML object. Right now this only strips whitespace.

  Args: xml: lxml.tree.Element. Mutated by this function."""
  XSLT_STRIPSPACE(xml)

def main():
  parser = argparse.ArgumentParser(description='Push/pull Amazon Route 53 configs.')
  parser.add_argument('--push', metavar='file_to_push.xml', help="Push the config in this file to R53.")
  parser.add_argument('--pull', action='store_true', help="Dump current R53 config to stdout.")
  parser.add_argument('--confirm', action='store_true', help="Do not prompt before push.")
  parser.add_argument('--dryrun', action='store_true', help="Do not actually apply changes.")
  parser.add_argument('--verbose', action='store_true')
  parser.add_argument('--zone', required=True, metavar="foursquare.com", help="Zone to push/pull.")
  args = parser.parse_args()

  ch = logging.StreamHandler()
  if args.verbose:
    ch.setLevel(logging.DEBUG)
  else:
    ch.setLevel(logging.INFO)
  log.addHandler(ch)

  if args.push == args.pull:
    print "You must specify either --push or --pull."
    sys.exit(1)

  # confirm wants stdin to itself
  if args.push == '-':
    args.confirm = True

  conn = Route53Connection()

  log.info('looking up zone for %s' % args.zone)
  zone_id = lookup_zone(conn, args.zone)
  log.info('fetching live config for zone %s' % zone_id)
  live_config = merge_config(fetch_config(zone_id, conn))

  if args.pull:
    print lxml.etree.tostring(live_config, pretty_print=True)

  if args.push:
    if args.push == '-':
        args.push = sys.stdin
    new_config = lxml.etree.parse(args.push)
    normalize_xml(live_config)
    normalize_xml(new_config.getroot())
    changeset = generate_changeset(live_config, new_config.getroot())
    if changeset is None:
        print "No changes found; exiting"
        sys.exit(0)
    changesetstr = lxml.etree.tostring(changeset, pretty_print=True)
    print "==CHANGESET=="
    print changesetstr
    errs = validate_changeset(changeset)
    if len(errs) > 0:
      print "changeset invalid. errors:"
      print '\n'.join(errs)
      print "exiting"
      sys.exit(1)
    if not args.confirm:
      ans = raw_input("Push y/N? ")
      if ans not in ['y', 'Y']:
        print "Confirmation failed; exiting"
        sys.exit(0)
    if args.dryrun:
      print "Dry run mode: exiting without applying changes"
      sys.exit(0)
    else:
      conn.change_rrsets(zone_id, changesetstr)

if __name__ == '__main__':
    main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# selection_functions.py (0.6.0)
#
# Developed in 2019 by Travis Kessler <travis.j.kessler@gmail.com>
#

def minimize_best_n(Members):
    '''
    Orders population members from lowest fitness to highest fitness

    Args:
        Members (list): list of PyGenetics Member objects

    Returns:
        lsit: ordered lsit of Members, from highest fitness to lowest fitness
    '''

    return(list(reversed(sorted(
        Members, key=lambda Member: Member.fitness_score
    ))))


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# ga_core.py (0.6.0)
#
# Developed in 2019 by Travis Kessler <travis.j.kessler@gmail.com>
#

# Stdlib imports
from random import choice, randint, uniform
from operator import add, sub
from multiprocessing import Pool

# Third party open source packages
from numpy import logspace, median, random as nrandom

# PyGenetics library imports
from pygenetics.selection_functions import minimize_best_n

# Supported parameter types and functions to generate random initial values
SUPPORTED_DTYPES = {
    int: randint,
    float: uniform
}

class Parameter:

    def __init__(self, name, min_val, max_val):
        '''Parameter object

        Args:
            name (str): name of the parameter
            min_val (int or float): minimum allowed value for the parameter
            max_val (int or float): maximum allowed value for the parameter
        '''

        self.name = name
        self.min_val = min_val
        self.max_val = max_val
        if type(min_val) != type(max_val):
            raise ValueError('Supplied min_val is not the same type as\
                             supplied max_val: {}, {}'.format(
                                 type(min_val),
                                 type(max_val))
                             )
        self.dtype = type(min_val + max_val)
        if self.dtype not in SUPPORTED_DTYPES:
            raise ValueError('Unsupported data type: use {}'
                             .format(SUPPORTED_DTYPES))

class Member:

    def __init__(self, parameters, cost_fn_val):
        '''Member object

        Args:
            parameters (dictionary): dictionary of parameter names and values
            cost_fn_val (float): value returned by cost function using params
        '''

        self.parameters = parameters
        self.cost_fn_val = cost_fn_val
        self.fitness_score = self.__calc_fitness_score(cost_fn_val)

    @staticmethod
    def __calc_fitness_score(cost_fn_val):
        '''Derives fitness score from cost function return value

        Args:
            cost_fn_val (float): value returned by cost function

        Returns:
            float: standardized fitness score
        '''

        if cost_fn_val >= 0:
            return 1 / (cost_fn_val + 1)
        else:
            return 1 + abs(cost_fn_val)

class Population:

    def __init__(self, size, cost_fn, cost_fn_args=None, num_processes=1,
                 select_fn=minimize_best_n):
        '''Population object: tunes specified parameters by measuring the
        performance of population members

        Args:
            size (int): number of Members in the population
            cost_fn (callable): function used to evaluate member fitness
            cost_fn_args (iterable or dict): additional user-specified
                arguments to pass to cost_fn
            num_processes (int): number of concurrent processes to run for
                Member generation/evaluation
            select_fn (callable): function used to sort members for parent
                based on member fitness score
        '''

        if size <= 0:
            raise ValueError('Population *size* cannot be <= 0')
        self.__pop_size = size
        if not callable(cost_fn):
            raise ValueError('Supplied *cost_fn* is not callable')
        self.__cost_fn = cost_fn
        if not callable(select_fn):
            raise ValueError('Supplied *select_fn* is not callable')
        self.__cost_fn_args = cost_fn_args
        self.__select_fn = select_fn
        self.__parameters = []
        self.__members = []
        self.__num_processes = num_processes

        self.__best_fitness = None
        self.__best_cost_fn_val = None
        self.__best_parameters = None

    def __len__(self):
        '''len(Population) == population size'''

        return self.__pop_size

    @property
    def fitness(self):
        '''Population fitness == average member fitness score'''

        if len(self.__members) != 0:
            if self.__num_processes > 1:
                members = [m.get() for m in self.__members]
            else:
                members = self.__members
            return sum(m.fitness_score for m in members) / len(members)
        else:
            return None

    @property
    def best_fitness(self):
        '''Fitness score of best performing population member so far'''

        return self.__best_fitness

    @property
    def ave_cost_fn_val(self):
        '''Returns average cost function return value for all members'''

        if len(self.__members) != 0:
            if self.__num_processes > 1:
                members = [m.get() for m in self.__members]
            else:
                members = self.__members
            return sum(m.cost_fn_val for m in members) / len(members)
        else:
            return None

    @property
    def med_cost_fn_val(self):
        '''Returns median cost function return value for all members'''

        if len(self.__members) != 0:
            if self.__num_processes > 1:
                members = [m.get() for m in self.__members]
            else:
                members = self.__members
            return median([m.cost_fn_val for m in members])
        else:
            return None

    @property
    def best_cost_fn_val(self):

        return self.__best_cost_fn_val

    @property
    def parameters(self):
        '''Population parameter vals == average member parameter vals'''

        if len(self.__members) != 0:
            if self.__num_processes > 1:
                members = [m.get() for m in self.__members]
            else:
                members = self.__members
            params = {}
            for p in self.__parameters:
                params[p.name] = sum(
                    m.parameters[p.name] for m in members
                ) / len(members)
            return params
        else:
            return None

    @property
    def best_parameters(self):
        '''Parameter values of best performing population member so far'''

        return self.__best_parameters

    @property
    def members(self):
        '''Returns Member objects of population'''

        if self.__num_processes > 1:
            return [m.get() for m in self.__members]
        else:
            return self.__members

    def add_parameter(self, name, min_val, max_val):
        '''Adds a paramber to the Population

        Args:
            name (str): name of the parameter
            min_val (int or float): minimum value for the parameter
            max_val (int or float): maximum value for the parameter
        '''

        self.__parameters.append(Parameter(name, min_val, max_val))

    def generate_population(self):
        '''Generates self.__pop_size Members with randomly initialized values
        for each parameter added with add_parameter(), evaluates their fitness
        '''

        if self.__num_processes > 1:
            process_pool = Pool(processes=self.__num_processes)
        self.__members = []

        for _ in range(self.__pop_size):
            feed_dict = {}
            for param in self.__parameters:
                feed_dict[param.name] = self.__random_param_val(
                    param.min_val,
                    param.max_val,
                    param.dtype
                )
            if self.__num_processes > 1:
                self.__members.append(process_pool.apply_async(
                    self._start_process,
                    [self.__cost_fn, feed_dict, self.__cost_fn_args])
                )
            else:
                self.__members.append(
                    Member(
                        feed_dict,
                        self.__cost_fn(feed_dict, self.__cost_fn_args)
                    )
                )

        if self.__num_processes > 1:
            process_pool.close()
            process_pool.join()

        self.__determine_best_member()

    def next_generation(self, mut_rate=0, max_mut_amt=0, log_base=10):
        '''Generates the next population from a previously evaluated generation

        Args:
            mut_rate (float): mutation rate for new members (0.0 - 1.0)
            max_mut_amt (float): how much the member is allowed to mutate
                (0.0 - 1.0, proportion change of mutated parameter)
            log_base (int): the higher this number, the more likely the first
                Members (chosen with supplied selection function) are chosen
                as parents for the next generation
        '''

        if self.__num_processes > 1:
            process_pool = Pool(processes=self.__num_processes)
            members = [m.get() for m in self.__members]
        else:
            members = self.__members

        if len(members) == 0:
            raise Exception(
                'Generation 0 not found: use generate_population() first'
            )

        selected_members = self.__select_fn(members)
        reproduction_probs = list(reversed(logspace(0.0, 1.0,
                                  num=len(selected_members), base=log_base)))
        reproduction_probs = reproduction_probs / sum(reproduction_probs)

        self.__members = []

        for _ in range(self.__pop_size):
            parent_1 = nrandom.choice(selected_members, p=reproduction_probs)
            parent_2 = nrandom.choice(selected_members, p=reproduction_probs)

            feed_dict = {}
            for param in self.__parameters:
                which_parent = uniform(0, 1)
                if which_parent < 0.5:
                    feed_dict[param.name] = parent_1.parameters[param.name]
                else:
                    feed_dict[param.name] = parent_2.parameters[param.name]
                feed_dict[param.name] = self.__mutate_parameter(
                    feed_dict[param.name], param, mut_rate, max_mut_amt
                )

            if self.__num_processes > 1:
                self.__members.append(process_pool.apply_async(
                    self._start_process,
                    [self.__cost_fn, feed_dict, self.__cost_fn_args])
                )
            else:
                self.__members.append(
                    Member(
                        feed_dict,
                        self.__cost_fn(feed_dict, self.__cost_fn_args)
                    )
                )

        if self.__num_processes > 1:
            process_pool.close()
            process_pool.join()

        self.__determine_best_member()

    @staticmethod
    def _start_process(cost_fn, feed_dict, cost_fn_args):
        '''Static method: starts a process to generate (evaluate) a new Member

        Args:
            cost_fn (function): cost function supplied to the Population
            feed_dict (dictionary): dictionary of Parameter objects
            cost_fn_args (iterable or dict): user-supplied args for cost_fn

        Returns:
            Evaluated Member object
        '''

        return Member(feed_dict, cost_fn(feed_dict, cost_fn_args))

    @staticmethod
    def __random_param_val(min_val, max_val, dtype):
        '''
        Private, static method: returns a random value

        Args:
            min_val (int or float): minimum value for random value
            max_Val (int or float): maximum value for random vlaue
            dtype (type): type of random value, int or float

        Returns:
            int or float: random value
        '''

        return SUPPORTED_DTYPES[dtype](min_val, max_val)

    @staticmethod
    def __mutate_parameter(value, param, mut_rate, max_mut_amt):
        '''Private, static method: mutates parameter

        Args:
            value (int or float): current value for Member's parameter
            param (Parameter): parameter object
            mut_rate (float): mutation rate of the value
            max_mut_amt (float): maximum mutation amount of the value

        Returns:
            int or float: mutated value
        '''

        if uniform(0, 1) < mut_rate:
            mut_amt = uniform(0, max_mut_amt)
            op = choice((add, sub))
            new_val = op(value, param.dtype(
                (param.max_val - param.min_val) * mut_amt
            ))
            if new_val > param.max_val:
                return param.max_val
            elif new_val < param.min_val:
                return param.min_val
            else:
                return new_val
        else:
            return value

    def __determine_best_member(self):
        '''Private method: determines if any current population members have a
        fitness score better than the current best
        '''

        if self.__num_processes > 1:
            members = [m.get() for m in self.__members]
        else:
            members = self.__members

        if self.__best_fitness is None:
            self.__best_fitness = members[0].fitness_score
            self.__best_cost_fn_val = members[0].cost_fn_val
            self.__best_parameters = {}
            for p in self.__parameters:
                self.__best_parameters[p.name] = members[0].parameters[p.name]

        for m_id, member in enumerate(members):
            if member.fitness_score > self.__best_fitness:
                self.__best_fitness = member.fitness_score
                self.__best_cost_fn_val = member.cost_fn_val
                self.__best_parameters = {}
                for p in self.__parameters:
                    self.__best_parameters[p.name] = member.parameters[p.name]


from __future__ import unicode_literals

from jaraco import context

def throws_exception(callable, *exceptions):
	"""
	Return True if the callable throws the specified exception

	>>> throws_exception(lambda: int('3'))
	False
	>>> throws_exception(lambda: int('a'))
	True
	>>> throws_exception(lambda: int('a'), KeyError)
	False
	"""
	with context.ExceptionTrap():
		with context.ExceptionTrap(*exceptions) as exc:
			callable()
	return bool(exc)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals, division, print_function, absolute_import
from unittest.util import strclass
from unittest import TestCase
import itertools
import logging
import inspect
from collections import namedtuple

def testpath(test, method_name=""):
    """Returns the full classpath (eg prefix.module.Class.method) for a passed in
    test

    :param test: TestCase, the test case instance
    :returns: string, the full testpath
    """
    if not method_name:
        method_name = test._testMethodName

    return "{}.{}".format(classpath(test), method_name)

def classpath(v):
    """given a class/instance return the full class path (eg, prefix.module.Classname)

    :param v: class or instance
    :returns: string, the full classpath of v
    """
    if isinstance(v, type):
        ret = strclass(v)
    else:
        ret = strclass(v.__class__)
    return ret

def chain(*sequences):
    """wrapper around itertools.chain

    :param *sequences: one or more sequences (eg, list, tuple, iterator) that you
        want to iterate through one right after another, if you pass in one sequence
        it will be assumed to be a list of sequences
    :returns: iterator, a generator that will work through each passed sequence
    """
    if len(sequences) == 1:
        sequences = sequences[0]
    return itertools.chain(*sequences)

def modname():
    """Returns the main module name (so this should return "pyt")"""
    name = __name__.split(".")[0]
    return name

def loghandler_members():
    """iterate through the attributes of every logger's handler

    this is used to switch out stderr and stdout in tests when buffer is True

    :returns: generator of tuples, each tuple has (name, handler, member_name, member_val)
    """
    Members = namedtuple("Members", ["name", "handler", "member_name", "member"])
    log_manager = logging.Logger.manager
    loggers = []
    ignore = set([modname()])
    if log_manager.root:
        loggers = list(log_manager.loggerDict.items())
        loggers.append(("root", log_manager.root))

    for logger_name, logger in loggers:
        if logger_name in ignore: continue

        for handler in getattr(logger, "handlers", []):
            members = inspect.getmembers(handler)
            for member_name, member in members:
                yield Members(logger_name, handler, member_name, member)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals, division, print_function, absolute_import
from unittest import TestCase # to allow from pyt import TestCase
from unittest import ( # so pyt can be a one-stop shop for skipping
    SkipTest,
    skip,
    skipIf,
    skipIf as skip_if,
    skipUnless,
    skipUnless as skip_unless
)
import os
import logging
import sys

from .tester import main # so you can do pyt.main like unittest.main

__version__ = '1.0.4'

def get_counts():
    """return test counts that are set via pyt environment variables when pyt
    runs the test

    :returns: dict, 3 keys (classes, tests, modules) and how many tests of each
        were found by pyt
    """
    counts = {}
    ks = [
        ('PYT_TEST_CLASS_COUNT', "classes"),
        ('PYT_TEST_COUNT', "tests"),
        ('PYT_TEST_MODULE_COUNT', "modules"),
    ]

    for ek, cn in ks:
        counts[cn] = int(os.environ.get(ek, 0))

    return counts

def is_single_class():
    """Returns True if only a single class is being run or some tests within a single class"""
    ret = False
    counts = get_counts()
    if counts["classes"] < 1 and counts["modules"] < 1:
        ret = counts["tests"] > 0
    else:
        ret = counts["classes"] <= 1 and counts["modules"] <= 1
    return ret

def skip_multi_class(msg=""):
    """will raise msg if multiple classes are being ran"""
    if not is_single_class():
        raise SkipTest(msg)

def is_single_module():
    """Returns True if only a module is being run"""
    ret = False
    counts = get_counts()
    if counts["modules"] == 1:
        ret = True

    elif counts["modules"] < 1:
        ret = is_single_class()

    return ret

def skip_multi_module(msg=""):
    """Will raise msg if multiple modules are being ran"""
    if not is_single_module():
        raise SkipTest(msg)

def is_single_test():
    """Returns True if only a single function is being run"""
    counts = get_counts()
    return counts["tests"] == 1

def skip_multi_test(msg=""):
    """Skip this test if there are multiple tests that are being run"""
    counts = get_counts()
    if not is_single_test():
        raise SkipTest(msg)


"""
    service_factory.validation
    ~~~~~~~~~~~~~~~~~~~~~~~~~~

    This module implement JSON-RPC request validation.

    :copyright: (c) 2015-2016 by Artem Malyshev.
    :license: GPL3, see LICENSE for more details.
"""

from __future__ import (
    absolute_import, unicode_literals, division, print_function,
)

from .compat import string_types

def validate_version(request):
    """Validate request version."""

    correct_version = request['jsonrpc'] == '2.0'
    error = 'Incorrect version of the JSON-RPC protocol'
    assert correct_version, error

def validate_method(request):
    """Validate request method."""

    correct_method = isinstance(request['method'], string_types)
    error = 'Incorrect name of the method to be invoked'
    assert correct_method, error

def validate_params(request):
    """Validate request params."""

    if 'params' in request:
        correct_params = isinstance(request['params'], (list, dict))
        error = 'Incorrect parameter values'
        assert correct_params, error

def validate_id(request):
    """Validate request id."""

    if 'id' in request:
        correct_id = isinstance(
            request['id'],
            (string_types, int, None),
        )
        error = 'Incorrect identifier'
        assert correct_id, error