from __future__ import absolute_import, division, print_function

from HTMLParser import HTMLParser
from itertools import izip
import logging

from streamcorpus import InvalidXpathError, Offset, OffsetType, XpathRange

logger = logging.getLogger(__name__)

# In HTML5, void elements are technically distinct from self-closing elements.
# Void elements have a "start" but no end tag, while self-closing elements
# have cause both the "start" and "end" events to fire.
#
# In our HTML parser, we need to be careful with void element handling,
# otherwise our state gets corrupt (because we cannot assume every start
# tag has an end tag).
VOID_ELEMENTS = set(['area', 'base', 'br', 'col', 'embed', 'hr', 'img',
                     'input', 'keygen', 'link', 'meta', 'param', 'source',
                     'track', 'wbr'])

# To be written transform.
class xpath_offsets(object):
    '''Add XPath offsets to all tokens.

    This is needed for specific display flows that try to present the
    document in a browser and need to highlight tokens.

    This stage has one configuration option, `validate`.  Computing
    the XPath expressions involves parsing HTML and this may encounter
    issues.  If `validate` is set to :const:`True`, after the XPath
    expressions are generated, this stage runs an additional check
    that the XPath expression range selects the original token text.
    A failure generates a log message and drops the stream item from
    the pipeline.  Validation is disabled by default.

    '''
    config_name = 'xpath_offsets'
    default_config = {
        'validate': False,
    }

    def __init__(self, config):
        self.validate = config.get('validate',
                                   self.default_config['validate'])

    def __call__(self, si, context=None):
        if not si.body or not si.body.clean_html or not si.body.clean_visible:
            logger.warning('stream item %s: has no clean_{html,visible}, '
                           'so we will not generate Xpath offsets',
                           si.stream_id)
            return si
        add_xpaths_to_stream_item(si)
        if self.validate:
            try:
                stream_item_roundtrip_xpaths(si, quick=False)
            except XpathMismatchError:
                logger.warning('stream item %s: Failed xpath roundtrip test, '
                               'dropping', si.stream_id, exc_info=True)
                return None
        return si

class TextElement(object):
    '''Marker that the most recent element is a text node.'''

class DepthStackEntry(object):
    __slots__ = ['last_tag', 'tags']

    def __init__(self):
        #: Most recent element seen at this depth.  Can be :const:`None`
        #: on startup, or :class:`TextElement` (as in the class object
        #: itself) if the most recent element was a text node.
        self.last_tag = None
        #: Dictionary mapping element name to number of preceding elements
        #: with that name, plus one.
        self.tags = {}

    def add_element(self, tag):
        '''Record that `tag` has been seen at this depth.

        If `tag` is :class:`TextElement`, it records a text node.

        '''
        # Collapse adjacent text nodes
        if tag is TextElement and self.last_tag is TextElement:
            return
        self.last_tag = tag
        if tag not in self.tags:
            self.tags[tag] = 1
        else:
            self.tags[tag] += 1

    def xpath_piece(self):
        '''Get an XPath fragment for this location.

        It is of the form ``tag[n]`` where `tag` is the most recent
        element added and n is its position.

        '''
        if self.last_tag is TextElement:
            return 'text()[{count}]'.format(count=self.text_index())
        else:
            return '{tag}[{count}]'.format(tag=self.last_tag,
                                           count=self.tags[self.last_tag])

    def text_index(self):
        '''Returns the one-based index of the current text node.'''
        # This is the number of text nodes we've seen so far.
        # If we are currently in a text node, great; if not then add
        # one for the text node that's about to begin.
        i = self.tags.get(TextElement, 0)
        if self.last_tag is not TextElement:
            i += 1
        return i

class XpathTextCollector(HTMLParser):
    '''Collects an HTML parse into an xpath.

    An instance of this class *statefully* constructs an xpath
    that corresponds to the HTML that has been parsed thus far.
    Basic usage of this class is to:

        1. Create the parser.
        2. Feed it some HTML.
        3. Ask for the xpath pointing to the location at which
           the aforementioned HTML ended.
        4. Go back to step 2 until all HTML has been consumed.

    This process is useful when you have a contiguous sequence
    of HTML fragments that, when concatenated, form a single
    HTML document. In particular, this enables one to compute
    *addresses* for each fragment.

    To that end, the xpaths generated by this class have a few
    very important properties:

        1. All xpaths *uniquely* identify a *single* text node
           in the HTML document.
        2. The offsets generated are actually of the form
           `(xpath, char offset)`, where the character offset
           indicates where the text starts in the text node
           addressed by `xpath`.
        3. The combination of two offsets forms a *range*, which
           is isomorphic to `Range` objects found in Javascript's
           standard library (which are used to represent user
           selections of text).
    '''
    def __init__(self):
        HTMLParser.__init__(self)  # old-style class :-/

        #: The index at which the most recent data node ends. The index is
        #: relative to the data node. (It is reset on every start tag.)
        self.data_start = 0
        #: XPath state stack.  This is a list of :class:`DepthStackEntry`.
        #: This is guaranteed to be non-empty, where the first item of
        #: the list is the document root and the last is the current
        #: location in the parse.
        self.depth_stack = [DepthStackEntry()]

    # This is the one public method!
    def xpath_offset(self):
        '''Returns a tuple of ``(xpath, character offset)``.

        The ``xpath`` returned *uniquely* identifies the end of the
        text node most recently inserted. The character offsets
        indicates where the text inside the node ends. (When the text
        node is empty, the offset returned is `0`.)
        '''
        datai = self.depth_stack[-1].text_index()
        xpath = (u'/' +
                 u'/'.join(dse.xpath_piece()
                           for dse in self.depth_stack[:-1]) +
                 (u'/text()[%d]' % datai))
        return (xpath, self.data_start)

    # Some hacks to track whether the parser moved to the next state.
    def feed(self, s):
        # print('FEED', s)
        HTMLParser.feed(self, s[0:len(s)-1])
        self.made_progress = False
        HTMLParser.feed(self, s[len(s)-1:len(s)])

    def progressor(meth):
        def _(self, *args, **kwargs):
            # print(meth.__name__, args[0])
            self.made_progress = True
            return meth(self, *args, **kwargs)
        return _

    # Satisfy the `HTMLParser` interface.
    @progressor
    def handle_starttag(self, tag, attrs):
        self.data_start = 0
        self.depth_stack[-1].add_element(tag)
        if tag in VOID_ELEMENTS:
            # Void elements are special. We effectively want to ignore them
            # completely, although we do need to make sure they affect node
            # indexing.
            return
        self.depth_stack.append(DepthStackEntry())

    @progressor
    def handle_endtag(self, tag):
        if tag in VOID_ELEMENTS:
            # We don't "descend" into void elements, so we can just ignore
            # them completely.
            return
        self.depth_stack.pop()
        self.data_start = 0
        assert self.depth_stack[-1].last_tag == tag, \
                '%s not at end of %r' % (tag, self.depth_stack)

    @progressor
    def handle_startendtag(self, tag, attrs):
        # This is *only* called for self-closing elements, e.g., `<br />`.
        # It is NOT called for void elements, e.g., `<br>`.
        self.depth_stack[-1].add_element(tag)
        self.data_start = 0

    # The following methods handle data. They are responsible for recording
    # enough state such that:
    #
    #   1) The index of the current text node can be computed.
    #   2) The character offset of the current position in the current
    #      text node can be computed.
    #
    # (1) is achieved by adding `None` elements to the xpath stack. These
    # `None` elements are collapsed in the `text_index` method.
    #
    # (2) is achieved by keeping track of the length of data (in terms of
    # Unicode codepoints) in the current text node. This length is reset
    # whenever a new tag is opened.

    @progressor
    def handle_data(self, data):
        # There is a bug in `HTMLParser` where the data yielded can be
        # hard-coded in certain state transitions. This means it could yield
        # data that is `bytes` even though we gave it a Unicode string. ---AG
        data = uni(data)
        self.depth_stack[-1].add_element(TextElement)
        self.data_start += len(data)

    @progressor
    def handle_entityref(self, name):
        self.depth_stack[-1].add_element(TextElement)
        # The `2` is for the `&` and `;` in, e.g., `&`.
        self.data_start += 2 + len(name)

    @progressor
    def handle_charref(self, name):
        self.depth_stack[-1].add_element(TextElement)
        # The `3` is for the `&`, `#` and `;` in, e.g., `$`.
        # This also applies to hex codes as `name` will contain the
        # preceding `x`.
        self.data_start += 3 + len(name)

    # TODO: There are some other callbacks defined in the `HTMLParser`
    # interface. We probably need to do something with them for complete
    # correctness. (If they can't be handled for whatever reason, simply
    # declare those states as not making progress---`char_offsets_to_xpaths`
    # will handle it gracefully by not computing xpaths.) ---AG

class XpathMismatchError(Exception):
    '''Raised when an Xpath offset is wrong.

    This occurs when slicing ``clean_html`` with xpaths does not
    produce the exact same string as slicing ``clean_visible``
    with character offsets.

    This class has four instance variables:

    * ``clean_html`` and ``clean_visible`` are a ``unicode`` strings.
    * ``xp_range`` is a :class:`streamcorpus.XpathRange`.
    * ``char_range`` is a ``(int, int)`` (half-open character range).
    '''
    def __init__(self, html, cleanvis, xp_range, char_range):
        self.clean_html = html
        self.clean_visible = cleanvis
        self.xp_range = xp_range
        self.char_range = char_range

def add_xpaths_to_stream_item(si):
    '''Mutably tag tokens with xpath offsets.

    Given some stream item, this will tag all tokens from all taggings
    in the document that contain character offsets. Note that some
    tokens may not have computable xpath offsets, so an xpath offset
    for those tokens will not be set. (See the documentation and
    comments for ``char_offsets_to_xpaths`` for what it means for a
    token to have a computable xpath.)

    If a token can have its xpath offset computed, it is added to its
    set of offsets with a ``OffsetType.XPATH_CHARS`` key.
    '''
    def sentences_to_xpaths(sentences):
        tokens = sentences_to_char_tokens(sentences)
        offsets = char_tokens_to_char_offsets(tokens)
        return char_offsets_to_xpaths(html, offsets)

    def xprange_to_offset(xprange):
        return Offset(type=OffsetType.XPATH_CHARS,
                      first=xprange.start_offset, length=0,
                      xpath=xprange.start_xpath,
                      content_form='clean_html', value=None,
                      xpath_end=xprange.end_xpath,
                      xpath_end_offset=xprange.end_offset)

    html = unicode(si.body.clean_html, 'utf-8')
    for sentences in si.body.sentences.itervalues():
        tokens = sentences_to_char_tokens(sentences)
        for token, xprange in izip(tokens, sentences_to_xpaths(sentences)):
            if xprange is None:
                continue
            offset = xprange_to_offset(xprange)
            token.offsets[OffsetType.XPATH_CHARS] = offset

def sentences_to_char_tokens(si_sentences):
    '''Convert stream item sentences to character ``Offset``s.'''
    for sentence in si_sentences:
        for token in sentence.tokens:
            if OffsetType.CHARS in token.offsets:
                yield token

def char_tokens_to_char_offsets(si_tokens):
    '''Convert character ``Offset``s to character ranges.'''
    for token in si_tokens:
        offset = token.offsets[OffsetType.CHARS]
        yield offset.first, offset.first + offset.length

def char_offsets_to_xpaths(html, char_offsets):
    '''Converts HTML and a sequence of char offsets to xpath offsets.

    Returns a generator of :class:`streamcorpus.XpathRange` objects
    in correspondences with the sequence of ``char_offsets`` given.
    Namely, each ``XpathRange`` should address precisely the same text
    as that ``char_offsets`` (sans the HTML).

    Depending on how ``char_offsets`` was tokenized, it's possible that
    some tokens cannot have their xpaths generated reliably. In this
    case, a ``None`` value is yielded instead of a ``XpathRange``.

    ``char_offsets`` must be a sorted and non-overlapping sequence of
    character ranges. They do not have to be contiguous.
    '''
    html = uni(html)
    parser = XpathTextCollector()
    prev_end = 0
    prev_progress = True
    for start, end in char_offsets:
        if start == end:
            # Zero length tokens shall have no quarter!
            # Note that this is a special case. If we let zero-length tokens
            # be handled normally, then it will be recorded as if the parser
            # did not make any progress. But of course, there is no progress
            # to be had!
            yield None
            continue
        # If we didn't make any progress on the previous token, then we'll
        # need to try and make progress before we can start tracking offsets
        # again. Otherwise the parser will report incorrect offset info.
        #
        # (The parser can fail to make progress when tokens are split at
        # weird boundaries, e.g., `&amp` followed by `;`. The parser won't
        # make progress after `&amp` but will once `;` is given.)
        #
        # Here, we feed the parser one character at a time between where the
        # last token ended and where the next token will start. In most cases,
        # this will be enough to nudge the parser along. Once done, we can pick
        # up where we left off and start handing out offsets again.
        #
        # If this still doesn't let us make progress, then we'll have to skip
        # this token too.
        if not prev_progress:
            for i in xrange(prev_end, start):
                parser.feed(html[i])
                prev_end += 1
                if parser.made_progress:
                    break
            if not parser.made_progress:
                yield None
                continue
        # Hand the parser everything from the end of the last token to the
        # start of this one. Then ask for the Xpath, which should be at the
        # start of `char_offsets`.
        if prev_end < start:
            parser.feed(html[prev_end:start])
            if not parser.made_progress:
                parser.feed(html[start:end])
                prev_progress = parser.made_progress
                prev_end = end
                yield None
                continue
        xstart = parser.xpath_offset()
        # print('START', xstart)
        # Hand it the actual token and ask for the ending offset.
        parser.feed(html[start:end])
        xend = parser.xpath_offset()
        # print('END', xend)
        prev_end = end

        # If we couldn't make progress then the xpaths generated are probably
        # incorrect. (If the parser doesn't make progress, then we can't rely
        # on the callbacks to have been called, which means we may not have
        # captured all state correctly.)
        #
        # Therefore, we simply give up and claim this token is not addressable.
        if not parser.made_progress:
            prev_progress = False
            yield None
        else:
            prev_progress = True
            yield XpathRange(xstart[0], xstart[1], xend[0], xend[1])
    parser.feed(html[prev_end:])
    parser.close()

def uni(s):
    if not isinstance(s, unicode):
        return unicode(s, 'utf-8')
    else:
        return s

def stream_item_roundtrip_xpaths(si, quick=False):
    '''Roundtrip all Xpath offsets in the given stream item.

    For every token that has both ``CHARS`` and ``XPATH_CHARS``
    offsets, slice the ``clean_html`` with the ``XPATH_CHARS`` offset
    and check that it matches slicing ``clean_visible`` with the
    ``CHARS`` offset.

    If this passes without triggering an assertion, then we're
    guaranteed that all ``XPATH_CHARS`` offsets in the stream item are
    correct. (Note that does not check for completeness. On occasion, a
    token's ``XPATH_CHARS`` offset cannot be computed.)

    There is copious debugging output to help make potential bugs
    easier to track down.

    This is used in tests in addition to the actual transform. It's
    expensive to run, but not running it means silent and hard to
    debug bugs.
    '''
    def debug(s):
        logger.warning(s)

    def print_window(token, size=200):
        coffset = token.offsets[OffsetType.CHARS]
        start = max(0, coffset.first - size)
        end = min(len(html), coffset.first + coffset.length + size)
        debug('-' * 49)
        debug(coffset)
        debug('window size: %d' % size)
        debug(html[start:end])
        debug('-' * 49)

    def debug_all(token, xprange, expected, err=None, got=None):
        debug('-' * 79)
        if err is not None:
            debug(err)
        debug(xprange)
        debug('expected: "%s"' % expected)
        if got is not None:
            debug('got: "%s"' % got)
        debug('token value: "%s"' % unicode(token.token, 'utf-8'))
        print_window(token, size=10)
        print_window(token, size=30)
        print_window(token, size=100)
        print_window(token, size=200)
        debug('-' * 79)

    def slice_clean_visible(token):
        coffset = token.offsets[OffsetType.CHARS]
        return cleanvis[coffset.first:coffset.first + coffset.length]

    def test_token(token):
        coffset = token.offsets.get(OffsetType.CHARS)
        if coffset is None:
            return False
        xoffset = token.offsets.get(OffsetType.XPATH_CHARS)
        if xoffset is None:
            return False
        crange = (coffset.first, coffset.first + coffset.length)
        xprange = XpathRange.from_offset(xoffset)
        expected = slice_clean_visible(token)
        if expected != unicode(token.token, 'utf-8'):
            # Yeah, apparently this can happen. Maybe it's a bug
            # in Basis? I'm trying to hustle, and this only happens
            # in two instances for the `random` document, so I'm not
            # going to try to reproduce a minimal counter-example.
            # ---AG
            return False
        try:
            got = xprange.slice_node(html_root)
        except InvalidXpathError as err:
            debug_all(token, xprange, expected, err=err)
            raise XpathMismatchError(html, cleanvis, xprange, crange)
        if expected != got:
            debug_all(token, xprange, expected, got=got)
            raise XpathMismatchError(html, cleanvis, xprange, crange)
        return True

    cleanvis = unicode(si.body.clean_visible, 'utf-8')
    html = unicode(si.body.clean_html, 'utf-8')
    html_root = XpathRange.html_node(html)
    total, has_valid_xpath = 0, 0
    for sentences in si.body.sentences.itervalues():
        for sentence in sentences:
            if quick:
                for i in xrange(len(sentence.tokens) - 1, -1, -1):
                    if test_token(sentence.tokens[i]):
                        break
            else:
                # Exhaustive test.
                for token in sentence.tokens:
                    total += 1
                    if test_token(token):
                        has_valid_xpath += 1
    if not quick:
        # This is nonsense if we have quick checking enabled.
        logger.info('stream item %s: %d/%d tokens with valid xpaths',
                    si.stream_id, has_valid_xpath, total)


# amara3.uxml.treeutil
'''
Various utilities related to amara3's lightweight tree implementation, includes
some operations widely associated with DOM, but in the form of utility functions
rather than methods.
'''

import itertools
from amara3.uxml.tree import *

def descendants(elem):
    '''
    Yields all the elements descendant of elem in document order
    '''
    for child in elem.xml_children:
        if isinstance(child, element):
            yield child
            yield from descendants(child)

def select_elements(source):
    '''
    Yields all the elements from the source
    source - if an element, yields all child elements in order; if any other iterator yields the elements from that iterator
    '''
    if isinstance(source, element):
        source = source.xml_children
    return filter(lambda x: isinstance(x, element), source)

def select_name(source, name):
    '''
    Yields all the elements with the given name
    source - if an element, starts with all child elements in order; can also be any other iterator
    name - will yield only elements with this name
    '''
    return filter(lambda x: x.xml_name == name, select_elements(source))

def select_name_pattern(source, pat):
    '''
    Yields elements from the source whose name matches the given regular expression pattern
    source - if an element, starts with all child elements in order; can also be any other iterator
    pat - re.pattern object
    '''
    return filter(lambda x: pat.match(x.xml_name) is not None, select_elements(source))

def select_value(source, val):
    '''
    Yields elements from the source with the given value (accumulated child text)
    source - if an element, starts with all child elements in order; can also be any other iterator
    val - string value to match
    '''
    if isinstance(source, element):
        source = source.xml_children
    return filter(lambda x: x.xml_value == val, source)

def select_attribute(source, name, val=None):
    '''
    Yields elements from the source having the given attrivute, optionally with the given attribute value
    source - if an element, starts with all child elements in order; can also be any other iterator
    name - attribute name to check
    val - if None check only for the existence of the attribute, otherwise compare the given value as well
    '''
    def check(x):
        if val is None:
            return name in x.xml_attributes
        else:
            return name in x.xml_attributes and x.xml_attributes[name] == val
    return filter(check, select_elements(source))

def following_siblings(elem):
    '''
    Yields elements and text which have the same parent as elem, but come afterward in document order
    '''
    it = itertools.dropwhile(lambda x: x != elem, elem.xml_parent.xml_children)
    next(it) #Skip the element itself
    return it

#

MATCHED_STATE = object()

def attr_test(next, name):
    def _attr_test(node):
        if isinstance(node, element) and name in node.xml_attributes:
            return next
    return _attr_test

def name_test(next, name):
    def _name_test(node):
        if isinstance(node, element) and node.xml_name == name:
            return next
    return _name_test

def elem_test(next):
    def _elem_test(node):
        if isinstance(node, element):
            return next
    return _elem_test

def any_until(next):
    def _any_until(node):
        if isinstance(node, element):
            next_next = next(node)
            if next_next is not None:
                return next_next
        return _any_until #Return this same state until we can match the next
    return _any_until

def any_(next, funcs):
    def _any(node):
        if any( (func(node) for func in funcs) ):
            return next
    return _any

def _prep_pattern(pattern):
    next_state = MATCHED_STATE
    #Work from the end of the pattern back to the beginning
    for i in range(len(pattern)):
        stage = pattern[-i-1] #reverse indexing
        if isinstance(stage, str):
            if stage == '*':
                next_state = elem_test(next_state)
            elif stage == '**':
                next_state = any_until(next_state)
            else:
                next_state = name_test(next_state, stage)
        elif isinstance(stage, tuple):
            new_tuple = tuple(( name_test(substage) if isinstance(substage, str) else substage for substage in stage ))
            next_state = any_(next_state, new_tuple)
        else:
            raise ValueError('Cannot interpret pattern component {0}'.format(repr(stage)))
    return next_state

def select_pattern(node, pattern, state=None):
    '''
    Yield descendant nodes matching the given pattern specification
    pattern - tuple of steps, each of which matches an element by name, with "*" acting like a wildcard, descending the tree in tuple order
                sort of like a subset of XPath in Python tuple form
    state - for internal use only

    pattern examples:

    ("a", "b", "c") - all c elements whose parent is a b element whose parent is an a element whose parent is node
    ("*", "*") - any "grandchild" of node
    ("*", "*", "*") - any "great grandchild" of node
    ("**", "a") - any a descendant of node

    >>> from amara3.uxml import tree
    >>> from amara3.uxml.treeutil import *
    >>>
    >>> tb = tree.treebuilder()
    >>> DOC = '<x>1</x><c><x>2</x><d><x>3</x></d></c><x>4</x><y>5</y>'
    >>> root = tb.parse(DOC)
    >>> results = [ e.xml_value for e in select_pattern(root, ('**', 'x')) ]
    >>> results
    ['1', '2', '3', '4']
    '''
    if state is None:
        state = _prep_pattern(pattern)
    #for child in select_elements(elem):
    if isinstance(node, element):
        for child in node.xml_children:
            new_state = state(child)
            if new_state == MATCHED_STATE:
                yield child
            elif new_state is not None:
                yield from select_pattern(child, None, state=new_state)
    return

def make_pretty(elem, depth=0, indent='  '):
    '''
    Add text nodes as possible to all descendants of an element for spacing & indentation
    to make the MicroXML as printed easier for people to read. Will not modify the
    value of any text node which is not already entirely whitespace.

    Warning: even though this operaton avoids molesting text nodes which already have
    whitespace, it still makes changes which alter the text. Not all whitespace in XML is
    ignorable. In XML cues from the DTD indicate which whitespace can be ignored.
    No such cues are available for MicroXML, so use this function with care. That said,
    in many real world applications of XML and MicroXML, this function causes no problems.

    elem - target element whose descendant nodes are to be modified.
    returns - the same element, which has been updated in place

    >>> from amara3.uxml import tree
    >>> from amara3.uxml.treeutil import *
    >>> DOC = '<x>1</x><c><x>2</x><d><x>3</x></d></c><x>4</x><y>5</y>'
    >>> tb = tree.treebuilder()
    >>> root = tb.parse(DOC)
    >>> len(root.xml_children)
    4
    >>> make_pretty(root)
    <uxml.element (8763373718343) "a" with 9 children>
    >>> len(root.xml_children)
    9
    >>> root.xml_encode()
    '\n  \n    <x>1</x>\n  \n  <c>\n    <x>2</x>\n    <d>\n      <x>3</x>\n    </d>\n  </c>\n  <x>4</x>\n  <y>5</y>\n'
    '''
    depth += 1
    updated_child_list = []
    updated_child_ix = 0
    for child in elem.xml_children:
        if isinstance(child, element):
            if updated_child_ix % 2:
                updated_child_list.append(child)
                updated_child_ix += 1
            else:
                #It's the turn for text, but we have an element
                new_text = text('\n' + indent*depth, elem)
                updated_child_list.append(new_text)
                updated_child_list.append(child)
                updated_child_ix += 2
            make_pretty(child, depth)
        else:
            if child.xml_value.strip():
                #More to it than whitespace, so leave alone
                #Note: if only whitespace entities are used, will still be left alone
                updated_child_list.append(child)
                updated_child_ix += 1
            else:
                #Only whitespace, so replace with proper indentation
                new_text = text('\n' + indent*depth, elem)
                updated_child_list.append(new_text)
                updated_child_ix += 1
    #Trailing indentation might be needed
    if not(updated_child_ix % 2):
        new_text = text('\n' + indent*(depth-1), elem)
        updated_child_list.append(new_text)
        #updated_child_ix += 1 #About to be done, so not really needed
    elem.xml_children = updated_child_list
    return elem


# coding=utf-8
# -------------------------------------------------------------------------------
# Author: Alexandre Manhaes Savio <alexsavio@gmail.com>
# Grupo de Inteligencia Computational <www.ehu.es/ccwintco>
# Universidad del Pais Vasco UPV/EHU
#
# 2015, Alexandre Manhaes Savio
# Use this at your own risk!
# -------------------------------------------------------------------------------

import os
import logging

from docstamp.config import get_inkscape_binpath
from docstamp.commands import call_command
from docstamp.svg_utils import rsvg_export

log = logging.getLogger(__name__)

def call_inkscape(args_strings, inkscape_binpath=None):
    """Call inkscape CLI with arguments and returns its return value.

    Parameters
    ----------
    args_string: list of str

    inkscape_binpath: str

    Returns
    -------
    return_value
        Inkscape command CLI call return value.
    """
    log.debug('Looking for the binary file for inkscape.')

    if inkscape_binpath is None:
        inkscape_binpath = get_inkscape_binpath()

    if inkscape_binpath is None or not os.path.exists(inkscape_binpath):
        raise IOError(
            'Inkscape binary has not been found. Please check configuration.'
        )

    return call_command(inkscape_binpath, args_strings)

def inkscape_export(input_file, output_file, export_flag="-A", dpi=90, inkscape_binpath=None):
    """ Call Inkscape to export the input_file to output_file using the
    specific export argument flag for the output file type.

    Parameters
    ----------

    input_file: str
        Path to the input file

    output_file: str
        Path to the output file

    export_flag: str
        Inkscape CLI flag to indicate the type of the output file

    Returns
    -------
    return_value
        Command call return value

    """
    if not os.path.exists(input_file):
        log.error('File {} not found.'.format(input_file))
        raise IOError((0, 'File not found.', input_file))

    if '=' not in export_flag:
        export_flag += ' '

    arg_strings = []
    arg_strings += ['--without-gui']
    arg_strings += ['--export-text-to-path']
    arg_strings += ['{}"{}"'.format(export_flag, output_file)]
    arg_strings += ['--export-dpi={}'.format(dpi)]
    arg_strings += ['"{}"'.format(input_file)]

    return call_inkscape(arg_strings, inkscape_binpath=inkscape_binpath)

def svg2pdf(svg_file_path, pdf_file_path, dpi=150, command_binpath=None, support_unicode=False):
    """ Transform SVG file to PDF file
    """

    if support_unicode:
        return rsvg_export(svg_file_path, pdf_file_path, dpi=dpi, rsvg_binpath=command_binpath)

    return inkscape_export(svg_file_path, pdf_file_path, export_flag="-A",
                           dpi=dpi, inkscape_binpath=command_binpath)

def svg2png(svg_file_path, png_file_path, dpi=150, inkscape_binpath=None):
    """ Transform SVG file to PNG file
    """
    return inkscape_export(svg_file_path, png_file_path, export_flag="-e",
                           dpi=dpi, inkscape_binpath=inkscape_binpath)


# coding=utf-8
# -------------------------------------------------------------------------------
# Author: Alexandre Manhaes Savio <alexsavio@gmail.com>
# Grupo de Inteligencia Computational <www.ehu.es/ccwintco>
# Universidad del Pais Vasco UPV/EHU
#
# 2015, Alexandre Manhaes Savio
# Use this at your own risk!
# -------------------------------------------------------------------------------

import os
import shutil
import logging

from jinja2 import Environment, FileSystemLoader

from .inkscape import svg2pdf, svg2png
from .pdflatex import tex2pdf, xetex2pdf
from .file_utils import get_tempfile, write_to_file
from .svg_utils import replace_chars_for_svg_code

log = logging.getLogger(__name__)

def get_environment_for(file_path):
    """Return a Jinja2 environment for where file_path is.

    Parameters
    ----------
    file_path: str

    Returns
    -------
    jinja_env: Jinja2.Environment

    """
    work_dir = os.path.dirname(os.path.abspath(file_path))

    if not os.path.exists(work_dir):
        raise IOError('Could not find folder for dirname of file {}.'.format(file_path))

    try:
        jinja_env = Environment(loader=FileSystemLoader(work_dir))
    except:
        raise
    else:
        return jinja_env

def get_doctype_by_extension(extension):
    if 'txt' in extension:
        doc_type = TextDocument
    elif 'svg' in extension:
        doc_type = SVGDocument
    elif 'tex' in extension:
        doc_type = LateXDocument
    else:
        raise ValueError('Could not identify the `doc_type` for `extension` {}.'.format(extension))

    return doc_type

def get_doctype_by_command(command):
    if not command:
        doc_type = TextDocument
    elif command == 'inkscape':
        doc_type = SVGDocument
    elif command == 'pdflatex':
        doc_type = PDFLateXDocument
    elif command == 'xelatex':
        doc_type = XeLateXDocument
    else:
        raise ValueError('Could not identify the `doc_type` for `command` {}.'.format(command))

    return doc_type

class TextDocument(object):
    """ A plain text document model.

    Parameters
    ----------
    template_file_path: str
        Document template file path.

    doc_contents: dict
        Dictionary with content values for the template to be filled.
    """

    def __init__(self, template_file_path, doc_contents=None):
        if not os.path.exists(template_file_path):
            raise IOError('Could not find template file {}.'.format(template_file_path))

        self._setup_template_file(template_file_path)

        if doc_contents is not None:
            self.file_content_ = self.fill(doc_contents)

    def _setup_template_file(self, template_file_path):
        """ Setup self.template

        Parameters
        ----------
        template_file_path: str
            Document template file path.
        """
        try:
            template_file = template_file_path
            template_env = get_environment_for(template_file_path)
            template = template_env.get_template(os.path.basename(template_file))
        except:
            raise
        else:
            self._template_file = template_file
            self._template_env = template_env
            self.template = template

    def fill(self, doc_contents):
        """ Fill the content of the document with the information in doc_contents.

        Parameters
        ----------
        doc_contents: dict
            Set of values to set the template document.

        Returns
        -------
        filled_doc: str
            The content of the document with the template information filled.
        """
        try:
            filled_doc = self.template.render(**doc_contents)
        except:
            log.exception('Error rendering Document '
                          'for {}.'.format(doc_contents))
            raise
        else:
            self.file_content_ = filled_doc
            return filled_doc

    def save_content(self, file_path, encoding='utf-8'):
        """ Save the content of the .txt file in a text file.

        Parameters
        ----------
        file_path: str
            Path to the output file.
        """
        if self.file_content_ is None:
            msg = 'Template content has not been updated. \
                   Please fill the template before rendering it.'
            log.exception(msg)
            raise ValueError(msg)

        try:
            write_to_file(file_path, content=self.file_content_,
                          encoding=encoding)
        except Exception as exc:
            msg = 'Document of type {} got an error when \
                   writing content.'.format(self.__class__)
            log.exception(msg)
            raise Exception(msg) from exc

    def render(self, file_path, **kwargs):
        """ See self.save_content """
        return self.save_content(file_path)

    @classmethod
    def from_template_file(cls, template_file_path, command=None):
        """ Factory function to create a specific document of the
        class given by the `command` or the extension of `template_file_path`.

        See get_doctype_by_command and get_doctype_by_extension.

        Parameters
        ----------
        template_file_path: str

        command: str

        Returns
        -------
        doc

        """
        # get template file extension
        ext = os.path.basename(template_file_path).split('.')[-1]

        try:
            doc_type = get_doctype_by_command(command)
        except ValueError:
            doc_type = get_doctype_by_extension(ext)
        except:
            raise
        else:
            return doc_type(template_file_path)

class SVGDocument(TextDocument):
    """ A .svg template document model. See GenericDocument. """
    _template_file = 'badge_template.svg'

    def fill(self, doc_contents):
        """ Fill the content of the document with the information in doc_contents.
        This is different from the TextDocument fill function, because this will
        check for symbools in the values of `doc_content` and replace them
        to good XML codes before filling the template.

        Parameters
        ----------
        doc_contents: dict
            Set of values to set the template document.

        Returns
        -------
        filled_doc: str
            The content of the document with the template information filled.
        """
        for key, content in doc_contents.items():
            doc_contents[key] = replace_chars_for_svg_code(content)

        return super(SVGDocument, self).fill(doc_contents=doc_contents)

    def render(self, file_path, **kwargs):
        """ Save the content of the .svg file in the chosen rendered format.

        Parameters
        ----------
        file_path: str
            Path to the output file.

        Kwargs
        ------
        file_type: str
            Choices: 'png', 'pdf', 'svg'
            Default: 'pdf'

        dpi: int
            Dots-per-inch for the png and pdf.
            Default: 150

        support_unicode: bool
            Whether to allow unicode to be encoded in the PDF.
            Default: False
        """
        temp = get_tempfile(suffix='.svg')
        self.save_content(temp.name)

        file_type = kwargs.get('file_type', 'pdf')
        dpi = kwargs.get('dpi', 150)
        support_unicode = kwargs.get('support_unicode', False)
        try:
            if file_type == 'svg':
                shutil.copyfile(temp.name, file_path)
            elif file_type == 'png':
                svg2png(temp.name, file_path, dpi=dpi)
            elif file_type == 'pdf':
                svg2pdf(temp.name, file_path, dpi=dpi, support_unicode=support_unicode)
        except:
            log.exception(
                'Error exporting file {} to {}'.format(file_path, file_type)
            )
            raise

class LateXDocument(TextDocument):
    """ A .tex template document model. See GenericDocument. """

    _render_function = staticmethod(tex2pdf)

    def render(self, file_path, **kwargs):
        """ Save the content of the .text file in the PDF.

        Parameters
        ----------
        file_path: str
            Path to the output file.
        """
        temp = get_tempfile(suffix='.tex')
        self.save_content(temp.name)

        try:
            self._render_function(temp.name, file_path, output_format='pdf')
        except:
            log.exception('Error exporting file {} to PDF.'.format(file_path))
            raise

class PDFLateXDocument(LateXDocument):
    pass

class XeLateXDocument(LateXDocument):
    _render_function = staticmethod(xetex2pdf)


# amara3.uxml.xml

#import asyncio
from asyncio import coroutine
import xml.parsers.expat
from xml.sax.saxutils import escape #also quoteattr?

from amara3.uxml import tree
from amara3.uxml.parser import parse, parser, parsefrags, event

class expat_callbacks(object):
    def __init__(self, handler, asyncio_based_handler=True):
        self._handler = handler
        self._elem_stack = []
        if asyncio_based_handler:
            next(self._handler) #Start the coroutine running
        return

    def start_element(self, name, attrs):
        #print('Start element:', name, attrs)
        local = name.split()[-1]
        #print(attrs, name)
        new_attrs = {}
        for aname, aval in attrs.items():
            new_attrs[aname.split()[-1]] = aval
        self._handler.send((event.start_element, local, new_attrs, self._elem_stack.copy()))
        self._elem_stack.append(local)

    def end_element(self, name):
        #print('End element:', name)
        local = name.split()[1] if ' ' in name else name
        self._elem_stack.pop()
        self._handler.send((event.end_element, local, self._elem_stack.copy()))

    def char_data(self, data):
        #print('Character data:', repr(data))
        self._handler.send((event.characters, data))

    def start_namespace(self, prefix, ns):
        pass

    def end_namespace(self, prefix):
        pass

class ns_expat_callbacks(expat_callbacks):
    def __init__(self, handler, asyncio_based_handler=True, stream=False):
        expat_callbacks.__init__(self, handler, asyncio_based_handler=asyncio_based_handler)
        #Namespace mappings encountered through the document, updated dynamically as the document is traversed
        self.prefixes = {}
        self.prefixes_rev = {}
        #If True, state of NS mappings will be maintained dynamically, as suitable to streaming mode parse operation
        #If false, best effort will be made to maintain a cumulative mapping of namespaces declared
        #For best results always use namespace normal form:
        # http://www.ibm.com/developerworks/library/x-namcar/
        self._stream = stream
        #Heuristic structure suggesting output namespaces to generate
        #element: (prefix, namespace)
        self.ns_portfolio = {}
        return

    def start_namespace(self, prefix, ns):
        self.prefixes[prefix] = ns
        self.prefixes_rev[ns] = prefix

    def end_namespace(self, prefix):
        if self._stream: del self.prefixes[prefix]

    def start_element(self, name, attrs):
        #print('Start element:', name, attrs)
        ns, local = name.split() if ' ' in name else (None, name)
        if local not in self.ns_portfolio:
            self.ns_portfolio[local] = (ns, self.prefixes_rev[ns])
        for aname, aval in attrs.items():
            ans, alocal = aname.split() if ' ' in aname else (None, aname)
            self.ns_portfolio['@' + alocal] = (ans, self.prefixes_rev.get(ans, ''))
        expat_callbacks.start_element(self, name, attrs)

def parse(source, handler):
    '''
    Convert XML 1.0 to MicroXML

    source - XML 1.0 input
    handler - MicroXML events handler

    Returns uxml, extras

    uxml - MicroXML element extracted from the source
    extras - information to be preserved but not part of MicroXML, e.g. namespaces
    '''
    h = expat_callbacks(handler)
    p = xml.parsers.expat.ParserCreate(namespace_separator=' ')

    p.StartElementHandler = h.start_element
    p.EndElementHandler = h.end_element
    p.CharacterDataHandler = h.char_data
    p.StartNamespaceDeclHandler = h.start_namespace
    p.EndNamespaceDeclHandler = h.end_namespace
    p.Parse(source)
    return p

@coroutine
def buffer_handler(accumulator):
    while True:
        event = yield
        accumulator.append(event)
    return

#Tree-based tools

class treebuilder(tree.treebuilder):
    '''
    from amara3.uxml import xml

    b = xml.treebuilder()
    root = b.parse('<spam/>')
    root
    '''
    def parse(self, source):
        self.handler = expat_callbacks(self._handler())
        self.expat_parser = xml.parsers.expat.ParserCreate(namespace_separator=' ')

        self.expat_parser.StartElementHandler = self.handler.start_element
        self.expat_parser.EndElementHandler = self.handler.end_element
        self.expat_parser.CharacterDataHandler = self.handler.char_data
        self.expat_parser.StartNamespaceDeclHandler = self.handler.start_namespace
        self.expat_parser.EndNamespaceDeclHandler = self.handler.end_namespace
        self.expat_parser.Parse(source)
        return self._root

'''
from asyncio import coroutine
from amara3.uxml import xml
@coroutine
def sink(accumulator):
    while True:
        e = yield
        accumulator.append(e.xml_value)

values = []
ts = xml.treesequence(('a', 'b'), sink(values))
ts.parse('123')
values

----

from asyncio import coroutine
from amara3.uxml import tree
from amara3.uxml.treeutil import *

def ppath(start, path):
    print((start, path))
    if not path: return None
    if len(path) == 1:
        yield from select_name(start, path[0])
    else:
        for e in select_name(start, path[0]):
            yield from ppath(e, path[1:])

ts = tree.treebuilder()
root = ts.parse('<c>1</c>23')
pathresults = ppath(root, ('b', 'c'))
print(list(pathresults))
'''

class treesequence(tree.treesequence):
    '''
    >>> from asyncio import coroutine
    >>> from amara3.uxml import xml
    >>> @coroutine
    ... def sink(accumulator):
    ...     while True:
    ...         e = yield
    ...         accumulator.append(e.xml_value)
    ...
    >>> values = []
    >>> ts = xml.treesequence(('a', 'b'), sink(values))
    >>> ts.parse('123')
    >>> values
    ['1', '2', '3']
    '''
    def __init__(self, pattern, sink, callbacks=expat_callbacks):
        super(treesequence, self).__init__(pattern, sink)
        self.handler = callbacks(self._handler())
        self.expat_parser = xml.parsers.expat.ParserCreate(namespace_separator=' ')

        self.expat_parser.StartElementHandler = self.handler.start_element
        self.expat_parser.EndElementHandler = self.handler.end_element
        self.expat_parser.CharacterDataHandler = self.handler.char_data
        self.expat_parser.StartNamespaceDeclHandler = self.handler.start_namespace
        self.expat_parser.EndNamespaceDeclHandler = self.handler.end_namespace
        return

    def parse(self, source):
        self.expat_parser.Parse(source)
        return

    def parse_file(self, fp):
        self.expat_parser.ParseFile(fp)
        return


# BSD 3-Clause License
#
# Copyright (c) 2017, Science and Technology Facilities Council
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# Author R. Ford STFC Daresbury Lab.
#
'''melody is a simple generic tuning library primarily designed for
the optimisation of software on hpc architectures'''

class Melody(object):
    ''' The main class '''

    def __init__(self, function=None, method=None, state=None, inputs=None):
        self._function = function
        self._method = method
        self._state = state
        self._inputs = inputs

    @property
    def function(self):
        '''Return the function associated with this instance of melody'''
        return self._function

    @function.setter
    def function(self, function):
        '''Set the function associated with this instance of melody'''
        self._function = function

    @property
    def method(self):
        '''Return the optimisation/search method associated with this instance
        of melody'''
        return self._method

    @method.setter
    def method(self, method):
        '''Set the optimisation/search method associated with this instance of
melody'''
        self._method = method

    @property
    def inputs(self):
        '''Return the input search parameters for the associated function'''
        return self._inputs

    @inputs.setter
    def inputs(self, inputs):
        '''Set the input search parameters for the associated function'''
        self._inputs = inputs

    def search(self):
        '''Start the optimisation/search using the supplied optimisation
        method with the supplied inputs for the supplied function'''
        search = self._method(inputs=self._inputs, function=self._function,
                              state=self._state)
        search.run()


########################################################################
# amara3.uxml.html
"""

"""

__all__ = [
'node', 'treebuilder', 'comment', 'element',
]

import copy
import itertools
import weakref
#from functools import *
#from itertools import *

from amara3.uxml import tree
from amara3.uxml.treeutil import *

try:
    import html5lib
    from html5lib.treebuilders.base import TreeBuilder, Node
except ImportError:
    raise

class node(Node):
    parent = tree.element.xml_parent
    value = tree.text.xml_value

    def appendChild(self, node):
        self.xml_append(node)

    def removeChild(self, node):
        self.xml_children.remove(node)

    def insertText(self, data, insertBefore=None):
        """Insert data as text in the current node, positioned before the
        start of node insertBefore or to the end of the node's text.
        """
        if insertBefore:
            self.insertBefore(tree.text(data), insertBefore)
        else:
            self.xml_append(tree.text(data))

    def insertBefore(self, node, refNode):
        """Insert node as a child of the current node, before refNode in the
        list of child nodes. Raises ValueError if refNode is not a child of
        the current node"""
        offset = self.xml_children.index(refNode)
        self.xml_insert(node, offset)

    def cloneNode(self):
        """Return a shallow copy of the current node i.e. a node with the same
        name and attributes but with no parent or child nodes
        """
        raise NotImplementedError

    def hasContent(self):
        """Return true if the node has children or text, false otherwise
        """
        return bool(self.xml_children)

def qname_to_local(qname):
    return qname.split(':')[-1]

class element(tree.element, node):
    '''
    attributes - a dict holding name, value pairs for attributes of the node
    childNodes - a list of child nodes of the current node. This must
    include all elements but not necessarily other node types
    _flags - A list of miscellaneous flags that can be set on the node
    '''
    #name = nodes.element_base.xml_qname
    #namespace = nodes.element_base.xml_namespace
    #xml_exclude_pnames = frozenset(('name', 'parent', 'appendChild', 'removeChild', 'removeChild', 'value', 'attributes', 'childNodes'))
    @property
    def name(self):
        return getattr(self, 'xml_html5lib_name', self.xml_name)

    @property
    def namespace(self):
        return getattr(self, 'xml_html5lib_namespace', None)

    @property
    def nameTuple(self):
        name = getattr(self, 'xml_html5lib_name', self.xml_name)
        namespace = getattr(self, 'xml_html5lib_namespace', None)
        return namespace, name
        #return XHTML_NAMESPACE, self.xml_name

    def xml_get_childNodes_(self):
        return self.xml_children

    def xml_set_childNodes_(self, nodelist):
        self.xml_children = []
        for node in nodelist:
            self.xml_append(node)
        return

    childNodes = property(xml_get_childNodes_, xml_set_childNodes_, None, "html5lib uses this property to manage HTML element children")

    def __init__(self, name, attrs=None):
        tree.element.__init__(self, name, attrs)
        self._flags = []
        return

    def xml_set_attributes_(self, attrs):
        for key, val in attrs.items():
            if isinstance(key, tuple):
                self.xml_attributes[qname_to_local(key[1])] = val
            elif key.startswith(u'xmlns'):
                continue
            else:
                self.xml_attributes[key] = val
        return

    def xml_get_attributes_(self):
        return self.xml_attributes

    attributes = property(xml_get_attributes_, xml_set_attributes_, None, "html5lib uses this property to manage HTML element attrs")

    def cloneNode(self):
        """Return a shallow copy of the current node i.e. a node with the same
        name and attributes but with no parent or child nodes
        """
        attrs = self.xml_attributes.copy()
        return element(self.xml_name, attrs=attrs)

#class comment(tree.comment):
class comment(object):
    type = 6
    value = tree.text.xml_value
    def __init__(self, data):
        self.data = data
        return

    def toxml(self):
        return "" % self.data

#Note: element = 6

class document(object):
    type = 9
    def __init__(self):
        self.root_nodes = []
        return

    def appendChild(self, node):
        self.root_nodes.append(node)

    def toxml(self):
        return "" % self.data

class doctype(object):
    type = 10
    def __init__(self, name, publicId, systemId):
        self.name = name
        self.publicId = publicId
        self.systemId = systemId
        return

    def toxml(self):
        return "" % self.data

MARKER = object()

BOGUS_NAMESPACE = u'urn:bogus:x'
NAME_FOR_ELEMENTS_UNNAMED_BY_HTML5LIB = u'UNNAMED_BY_HTML5LIB'
XHTML_NAMESPACE = 'http://www.w3.org/1999/xhtml'

class treebuilder(TreeBuilder):
    commentClass = comment
    documentClass = document
    #def __init__(self, entity_factory=None, use_xhtml_ns=False):
    #    self.entity = entity_factory()
    #def documentClass(self):
    #    return weakref.proxy(self)

    def doctypeClass(self, name, publicId, systemId):
        attrs = {'html5.publicId': publicId, 'html5.systemId': systemId}
        return tree.element(name, attrs=attrs)

    def __init__(self, use_xhtml_ns=False):
        self.root_nodes = []
        #html5lib.treebuilders._base.TreeBuilder breaks if you do not pass in True for namespaceHTMLElements
        #We'll take care of that ourselves with the if not use_xhtml_ns... below
        TreeBuilder.__init__(self, True)
        def eclass(name, namespace):
            xml_html5lib_name, xml_html5lib_namespace = MARKER, MARKER
            if not use_xhtml_ns and namespace == XHTML_NAMESPACE:
                #html5lib feints support for HTML5 elements kept in the null namespace
                #But in reality this support is broken.  We have to in effect keep
                #Two namespaces for each element, the real one from an amara perspective
                #And another that is always XHTML for HTML5 elements, so html5lib doesn't break
                xml_html5lib_namespace = namespace
                namespace = None
            #For some reason html5lib sometimes sends None as name
            if not name:
                xml_html5lib_name = name
                name = NAME_FOR_ELEMENTS_UNNAMED_BY_HTML5LIB
            #import sys; print >> sys.stderr, (namespace, name, use_xhtml_ns)
            #Deal with some broken HTML that uses bogus colons in tag names
            if (u":" in name and not namespace):
                xml_html5lib_namespace = namespace
                namespace = BOGUS_NAMESPACE
            elem = element(qname_to_local(name))
            if xml_html5lib_namespace != MARKER:
                elem.xml_html5lib_namespace = xml_html5lib_namespace
            if xml_html5lib_name != MARKER:
                elem.xml_html5lib_name = xml_html5lib_name
            return elem
        self.elementClass = eclass
        self.fragmentClass = eclass

'''
from amara3.uxml import html5
import urllib.request
with urllib.request.urlopen('http://uche.ogbuji.net/') as response:
    e = html5.parse(response)
'''

def parse(source, prefixes=None, model=None, encoding=None, use_xhtml_ns=False):
    '''
    Parse an input source with HTML text into an Amara 3 tree

    >>> from amara3.uxml import html5
    >>> import urllib.request
    >>> with urllib.request.urlopen('http://uche.ogbuji.net/') as response:
    ...     html5.parse(response)

    #Warning: if you pass a string, you must make sure it's a byte string, not a Unicode object.  You might also want to wrap it with amara.lib.inputsource.text if it's not obviously XML or HTML (for example it could be confused with a file name)
    '''
    def get_tree_instance(namespaceHTMLElements, use_xhtml_ns=use_xhtml_ns):
        #use_xhtml_ns is a boolean, whether or not to use http://www.w3.org/1999/xhtml
        return treebuilder(use_xhtml_ns)
    parser = html5lib.HTMLParser(tree=get_tree_instance)
    #doc = parser.parse(inputsource(source, None).stream, encoding=encoding)
    #doc = parser.parse(source, encoding=encoding)
    doc = parser.parse(source)
    first_element = next((e for e in doc.root_nodes if isinstance(e, element)), None)
    return first_element

def markup_fragment(source, encoding=None):
    '''
    Parse a fragment if markup in HTML mode, and return a bindery node

    Warning: if you pass a string, you must make sure it's a byte string, not a Unicode object.  You might also want to wrap it with amara.lib.inputsource.text if it's not obviously XML or HTML (for example it could be confused with a file name)

    from amara.lib import inputsource
    from amara.bindery import html
    doc = html.markup_fragment(inputsource.text('XXX
Spam!
Eggs!YYY'))

    See also: http://wiki.xml3k.org/Amara2/Tagsoup
    '''
    doc = parse(source, encoding=encoding)
    frag = doc.html.body
    return frag

class treesequence(tree.treesequence):
    '''
    >>> from asyncio import coroutine
    >>> from amara3.uxml import xml
    >>> @coroutine
    ... def sink(accumulator):
    ...     while True:
    ...         e = yield
    ...         accumulator.append(e.xml_value)
    ...
    >>> values = []
    >>> ts = xml.treesequence(('html', 'body', 'ul', 'li'), sink(values))
    >>> ts.parse('<title>

1

2

3

')
    >>> values
    ['1', '2', '3']
    '''
    #def __init__(self, pattern, sink):
    #    super(treesequence, self).__init__(pattern, sink)
    #    return

    def parse(self, source):
        self.expat_parser.Parse(source)
        return

    def parse_file(self, fp):
        self.expat_parser.ParseFile(fp)
        return


# BSD 3-Clause License
#
# Copyright (c) 2017, Science and Technology Facilities Council
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# Author R. Ford STFC Daresbury Lab.
#
'''An example use of melody in which we search a set of compiler
flags to find the best runtime for the shallow benchmark'''

from melody.inputs import create_input

def execute(option):
    '''A script that melody calls with each valid set of options. This
    script runs the required code and returns the results.'''

    namelist_option = []
    makefile_option = []
    flags = ""
    for entry in option:
        key = entry.keys()[0]
        if key == "Problem Size":
            namelist_option.append({"SIZE": entry[key]})
        elif key == "F90":
            makefile_option.append(entry)
        else:
            flags += entry[key] + " "
    makefile_option.append({"F90FLAGS": flags})

    namelist = create_input(namelist_option, "namelist",
                            template_location="templates")

    makefile_include = create_input(makefile_option, "Makefile.include",
                                    template_location="templates")

    benchmark_base = "shallow"

    # save the input files in the appropriate place
    location = benchmark_base + "/original/namelist"
    my_file = open(location, 'w')
    my_file.write(namelist)
    my_file.flush()

    location = benchmark_base + "/common/Makefile.include"
    my_file = open(location, 'w')
    my_file.write(makefile_include)
    my_file.flush()

    # compile shallow if required
    base_path = benchmark_base + "/original"
    import subprocess
    make_process = subprocess.Popen(["make", "clean"], cwd=base_path,
                                    stderr=subprocess.PIPE,
                                    stdout=subprocess.PIPE)
    if make_process.wait() != 0:
        return False, []

    make_process = subprocess.Popen(["make"], cwd=base_path,
                                    stderr=subprocess.PIPE,
                                    stdout=subprocess.PIPE)
    if make_process.wait() != 0:
        return False, []

    # run shallow
    make_process = subprocess.Popen(["./shallow_base"], cwd=base_path,
                                    stderr=subprocess.PIPE,
                                    stdout=subprocess.PIPE)
    if make_process.wait() != 0:
        return False, []
    # _ = make_process.stderr.read()
    stdout = make_process.stdout.read()

    # determine if the results are correct. We will need to look at
    # the results from stdout but for the moment we assume they are
    # correct

    # extract the required outputs
    for line in stdout.split("\n"):
        if "Time-stepping" in line:
            total_time = line.split()[2]

    return True, total_time


"""
Function helpers to create vcard formats.
"""

def create_vcard3_str(name, surname, displayname, email='', org='', title='', url='', note=''):
    """ Create a vCard3.0 string with the given parameters.
    Reference: http://www.evenx.com/vcard-3-0-format-specification
    """
    vcard = []
    vcard += ['BEGIN:VCARD']
    vcard += ['VERSION:3.0']

    if name and surname:
        name = name.strip()
        vcard += ['N:{};{};;;'.format(name, surname)]

    if not displayname:
        displayname = '{} {}'.format(name, surname)

    vcard += ['FN:{}'.format(displayname)]

    if email:
        vcard += ['EMAIL:{}'.format(email)]

    if org:
        vcard += ['ORG:{}'.format(org)]

    if title:
        vcard += ['TITLE:{}'.format(title)]

    if url:
        vcard += ['URL:{}'.format(url)]

    if note:
        vcard += ['NOTE:{}'.format(note)]

    vcard += ['END:VCARD']

    return '\n'.join([field.strip() for field in vcard])


# -----------------------------------------------------------------------------
# amara3.uxml.tree
#
# Basic tree implementation for MicroXML
#
# -----------------------------------------------------------------------------

#See also: http://www.w3.org/community/microxml/wiki/MicroLarkApi

import weakref
from asyncio import coroutine

from amara3.uxml.parser import parse, parser, parsefrags, event

#NO_PARENT = object()

class node(object):
    def __init__(self, parent=None):
        self._xml_parent = weakref.ref(parent) if parent is not None else None
        #self._xml_parent = weakref.ref(parent or NO_PARENT)

    @property
    def xml_parent(self):
        return self._xml_parent() if self._xml_parent else None
        #p = self._xml_parent()
        #return None if p is NO_PARENT else p

    def xml_encode(self):
        raise NotImplementedError

    def xml_write(self, fp):
        fp.write(self.xml_encode())

class element(node):
    '''
    Note: Meant to be bare bones & Pythonic. Does no integrity checking of direct manipulations, such as adding an integer to xml_children, or '1' as an attribute name
    '''
    def __init__(self, name, attrs=None, parent=None):#, ancestors=None):
        self.xml_name = name
        self.xml_attributes = attrs or {}
        self.xml_children = []
        node.__init__(self, parent)
        return

    def xml_encode(self, indent=None, depth=0):
        strbits = ['<', self.xml_name]
        for aname, aval in self.xml_attributes.items():
            strbits.extend([' ', aname, '="', aval, '"'])
        strbits.append('>')
        if indent:
            strbits.append('\n')
            strbits.append(indent*depth)
        for child in self.xml_children:
            if isinstance(child, element):
                strbits.append(child.xml_encode(indent=indent, depth=depth+1))
                if indent:
                    strbits.append('\n')
                    strbits.append(indent*depth)
            else:
                strbits.append(child)
        strbits.extend(['</', self.xml_name, '>'])
        return ''.join(strbits)

    @property
    def xml_value(self):
        '''
        '''
        return ''.join(map(lambda x: x.xml_value, self.xml_children))

    #Really just an alias that forbids specifying position
    def xml_append(self, child):
        '''
        Append a node as the last child

        child - the child to append. If a string, convert to a text node, for convenience
        '''
        self.xml_insert(child)
        return

    def xml_insert(self, child, index=-1):
        '''
        Append a node as the last child

        child - the child to append. If a string, convert to a text node, for convenience
        '''
        if isinstance(child, str):
            child = text(child, parent=self)
        else:
            child._xml_parent = weakref.ref(self)
        if index == -1:
            self.xml_children.append(child)
        else:
            self.xml_children.insert(index, child)
        return

    def __repr__(self):
        return u'{{uxml.element ({0}) "{1}" with {2} children}}'.format(hash(self), self.xml_name, len(self.xml_children))

    #def unparse(self):
    #    return '<' + self.name.encode('utf-8') + unparse_attrmap(self.attrmap) + '>'

class text(node, str):
    def __new__(cls, value, parent=None):
        self = super(text, cls).__new__(cls, value)
        return self

    def __init__(self, value, parent=None):#, ancestors=None):
        node.__init__(self, parent)
        return

    def __repr__(self):
        return u'{{uxml.text "{}"...}}'.format(str(self)[:10])

    def xml_encode(self, indent=None, depth=0):
        return str(self)

    @property
    def xml_value(self):
        return str(self)

    @property
    def xml_children(self):
        return []

    @property
    def xml_name(self):
        return '#text'

    #def unparse(self):
    #    return '<' + self.name.encode('utf-8') + unparse_attrmap(self.attrmap) + '>'

def strval(node, outermost=True):
    '''
    XPath-like string value of node
    '''
    if not isinstance(node, element):
        return node.xml_value if outermost else [node.xml_value]
    accumulator = []
    for child in node.xml_children:
        if isinstance(child, text):
            accumulator.append(child.xml_value)
        elif isinstance(child, element):
            accumulator.extend(strval(child, outermost=False))
    if outermost: accumulator = ''.join(accumulator)
    return accumulator

class treebuilder(object):
    def __init__(self):
        self._root = None
        self._parent = None

    @coroutine
    def _handler(self):
        while True:
            ev = yield
            if ev[0] == event.start_element:
                new_element = element(ev[1], ev[2], self._parent)
                #Note: not using weakrefs here because these refs are not circular
                if self._parent: self._parent.xml_children.append(new_element)
                self._parent = new_element
                #Hold a reference to the top element of the subtree being built,
                #or it will be garbage collected as the builder moves down the tree
                if not self._root: self._root = new_element
            elif ev[0] == event.characters:
                new_text = text(ev[1], self._parent)
                if self._parent: self._parent.xml_children.append(new_text)
            elif ev[0] == event.end_element:
                if self._parent:
                    self._parent = self._parent.xml_parent
        return

    def parse(self, doc):
        #reset
        self._root = None
        self._parent = None
        h = self._handler()
        p = parser(h)
        p.send((doc, False))
        p.send(('', True)) #Wrap it up
        return self._root

def name_test(name):
    def _name_test(ev):
        return ev[0] == event.start_element and ev[1] == name
    return _name_test

def elem_test():
    def _elem_test(ev):
        return ev[0] == event.start_element
    return _elem_test

MATCHED_STATE = object()

class treesequence(object):
    '''
    >>> from amara3.uxml import tree
    >>> from asyncio import coroutine
    >>> @coroutine
    ... def sink(accumulator):
    ...     while True:
    ...         e = yield
    ...         accumulator.append(e.xml_value)
    ...
    >>> values = []
    >>> ts = tree.treesequence(('a', 'b'), sink(values))
    >>> ts.parse('123')
    >>> values
    ['1', '2', '3']
    '''
    def __init__(self, pattern, sink):
        self._root = None
        self._parent = None
        self._pattern = pattern
        self._states = None
        self._evstack = []
        self._building_depth = 0
        self._sink = sink
        next(sink) #Prime the coroutine
        self._current = None
        self._prep_pattern()

    def _only_name(self, next, name):
        def _only_name_func(ev):
            if ev[0] == event.start_element and ev[1] == name:
                return next
        return _only_name_func

    def _any_name(self, next):
        def _any_name_func(ev):
            if ev[0] == event.start_element:
                return next
        return _any_name_func

    def _any_until(self, next):
        def _any_until_func(ev):
            if ev[0] == event.start_element:
                next_next = next(ev)
                if next_next is not None:
                    return next_next
            return _any_until_func
        return _any_until_func

    def _any(self, next, funcs):
        def _any_func(ev):
            if any( (func(ev) for func in funcs) ):
                return next
        return _any_func

    def _prep_pattern(self):
        next_state = MATCHED_STATE
        for i in range(len(self._pattern)):
            stage = self._pattern[-i-1]
            if isinstance(stage, str):
                if stage == '*':
                    next_state = self._any_name(next_state)
                elif stage == '**':
                    next_state = self._any_until(next_state)
                else:
                    next_state = self._only_name(next_state, stage)
            elif isinstance(stage, tuple):
                new_tuple = tuple(( name_test(substage) if isinstance(substage, str) else substage for substage in stage ))
                next_state = self._any(next_state, new_tuple)
            else:
                raise ValueError('Cannot interpret pattern component {0}'.format(repr(stage)))
        self._states = next_state
        return

    def _match_state(self):
        new_state = self._states
        for depth, ev in enumerate(self._evstack):
            new_state = new_state(ev)
            if new_state == MATCHED_STATE:
                return True
            elif new_state is None:
                return False
        return False

    @coroutine
    def _handler(self):
        while True:
            ev = yield
            if ev[0] == event.start_element:
                self._evstack.append(ev)
                #Keep track of the depth while we're building elements. When we ge back to 0 depth, we're done for this subtree
                if self._building_depth:
                    self._building_depth += 1
                elif self._match_state():
                    self._building_depth = 1
                if self._building_depth:
                    new_element = element(ev[1], ev[2], self._parent)
                    #if self._parent: self._parent().xml_children.append(weakref.ref(new_element))
                    #Note: not using weakrefs here because these refs are not circular
                    if self._parent: self._parent.xml_children.append(new_element)
                    self._parent = new_element
                    #Hold a reference to the top element of the subtree being built,
                    #or it will be garbage collected as the builder moves down the tree
                    if self._building_depth == 1: self._root = new_element
            elif ev[0] == event.characters:
                if self._building_depth:
                    new_text = text(ev[1], self._parent)
                    if self._parent: self._parent.xml_children.append(new_text)
            elif ev[0] == event.end_element:
                self._evstack.pop()
                if self._building_depth:
                    self._building_depth -= 1
                    #Done with this subtree
                    if not self._building_depth:
                        self._sink.send(self._parent)
                    #Pop back up in element ancestry
                    if self._parent:
                        self._parent = self._parent.xml_parent

            #print(ev, self._building_depth, self._evstack)
        return

    def parse(self, doc):
        h = self._handler()
        p = parser(h)
        p.send((doc, False))
        p.send(('', True)) #Wrap it up
        return

def parse(doc):
    return treebuilder().parse(doc)


#!/usr/bin/env python

# -*- coding: utf-8 -*-

# nagios-notification-google-calendar
# notification_google_calendar.py

# Copyright (c) 2013-2015 Alexei Andrushievich <vint21h@vint21h.pp.ua>
# Notifications via Google Calendar Nagios plugin [https://github.com/vint21h/nagios-notification-google-calendar]
#
# This file is part of nagios-notification-google-calendar.
#
# nagios-notification-google-calendar is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals
import sys

try:
    import os
    from optparse import OptionParser
    import ConfigParser
    import datetime
    import httplib2
    from oauth2client.client import flow_from_clientsecrets
    from oauth2client.file import Storage
    from apiclient.discovery import build
except (ImportError, ), err:
    sys.stderr.write("ERROR: Couldn't load module. {err}\n".format(err=err))
    sys.exit(-1)

__all__ = ["main", ]

# metadata
VERSION = (0, 3, 0)
__version__ = ".".join(map(str, VERSION))

# global variables
SCOPE = "https://www.googleapis.com/auth/calendar"
DT_FORMAT = "%Y-%m-%dT%H:%M:00.000"

def parse_options():
    """
    Commandline options arguments parsing.
    """

    # build options and help
    version = "%%prog {version}".format(version=__version__)
    parser = OptionParser(version=version)
    parser.add_option(
        "-u", "--username", action="store", dest="username",
        type="string", default="", metavar="RECIPIENT", help="user"
    )
    parser.add_option(
        "-C", "--calendar", metavar="CALENDAR", action="store",
        type="string", dest="calendar", default="", help="google calendar ID"
    )
    parser.add_option(
        "-t", "--timezone", metavar="TIMEZONE", action="store",
        type="string", dest="timezone", default="", help="user timezone"
    )
    parser.add_option(
        "-m", "--message", metavar="MESSAGE", action="store",
        type="string", dest="message", default="", help="message text"
    )
    parser.add_option(
        "-c", "--config", metavar="CONFIG", action="store",
        type="string", dest="config", help="path to config file",
        default="/etc/nagios/notification_google_calendar.ini")
    parser.add_option(
        "-q", "--quiet", metavar="QUIET", action="store_true",
        default=False, dest="quiet", help="be quiet"
    )
    parser.add_option(
        "-g", "--get-google-credentials", metavar="GET-GOOGLE-CREDENTIALS",
        action="store_true", default=False, dest="get_google_credentials",
        help="get google API credentials for user"
    )

    options = parser.parse_args(sys.argv)[0]

    mandatories = ["username", ]
    # check mandatory command line options supplied
    if not options.get_google_credentials:
        mandatories.append("calendar")  # set calendar option required when sending message
        mandatories.append("message")  # set message option required when sending message
        mandatories.append("timezone")  # set timezone option required when sending message
    if not all(options.__dict__[mandatory] for mandatory in mandatories):
        parser.error("Required command line option missing\n")

    return options

def parse_config(options):
    """
    Get settings from config file.
    """

    if os.path.exists(options.config):
        config = ConfigParser.ConfigParser()
        try:
            config.read(options.config)
        except Exception, err:
            if not options.quiet:
                sys.stderr.write("ERROR: Config file read {config} error. {err}".format(config=options.config, err=err))
            sys.exit(-1)

        try:
            configdata = {
                "secrets": config.get("GOOGLE", "secrets"),
                "credentials": config.get("nagios-notification-google-calendar", "credentials"),
                "start": config.get("nagios-notification-google-calendar", "start"),
                "end": config.get("nagios-notification-google-calendar", "end"),
                "message": config.get("nagios-notification-google-calendar", "message"),
            }
        except ConfigParser.NoOptionError, err:
            if not options.quiet:
                sys.stderr.write("ERROR: Config file missing option error. {err}\n".format(err=err))
            sys.exit(-1)

        # check mandatory config options supplied
        mandatories = ["secrets", "credentials", "start", "end", "message", ]
        if not all(configdata[mandatory] for mandatory in mandatories):
            if not options.quiet:
                sys.stdout.write("Mandatory config option missing\n")
            sys.exit(0)

        return configdata
    else:
        if not options.quiet:
            sys.stderr.write("ERROR: Config file {config} does not exist\n".format(config=options.config))
        sys.exit(0)

def get_google_credentials(options, config):
    """
    Get google API credentials for user.
    """

    try:
        if options.get_google_credentials:
            flow = flow_from_clientsecrets(config["secrets"], scope=SCOPE, redirect_uri="oob")
            sys.stdout.write("Follow this URL: {url} and grant access to calendar.\n".format(url=flow.step1_get_authorize_url()))
            token = raw_input("Enter token:")
            credentials = flow.step2_exchange(token)
            storage = Storage(os.path.join(config["credentials"], "{username}.json".format(username=options.username)))
            storage.put(credentials)
            credentials.set_store(storage)
        else:
            storage = Storage(os.path.join(config["credentials"], "{username}.json".format(username=options.username)))
            credentials = storage.get()
    except Exception, err:
        if not options.quiet:
            sys.stderr.write("ERROR: Getting google API credentials error. {err}\n".format(err=err))
        sys.exit(-1)

    return credentials

def create_event_datetimes(options, config):
    """
    Create event start and end datetimes.
    """

    now = datetime.datetime.now()

    return {
        "start": {
            "dateTime": (now + datetime.timedelta(minutes=int(config["start"]))).strftime(DT_FORMAT),
            "timeZone": options.timezone,
        },
        "end": {
            "dateTime": (now + datetime.timedelta(minutes=int(config["end"]))).strftime(DT_FORMAT),
            "timeZone": options.timezone,
        },
    }

def create_event(options, config, credentials):
    """
    Create event in calendar with sms reminder.
    """

    try:
        http = credentials.authorize(httplib2.Http())
        service = build("calendar", "v3", http=http)

        event = {
            "summary": options.message,
            "location": "",
            "reminders": {
                "useDefault": False,
                "overrides": [
                    {
                        "method": "sms",
                        "minutes": config["message"],
                    },
                ],
            }
        }
        event.update(create_event_datetimes(options, config))

        service.events().insert(calendarId=options.calendar, sendNotifications=True, body=event).execute()
    except Exception, err:
        if not options.quiet:
            sys.stderr.write("ERROR: Creating google calendar event error. {err}\n".format(err=err))
        sys.exit(-1)

def main():
    """
    Processing notification call main function.
    """

    # getting info for creating event
    options = parse_options()
    config = parse_config(options)
    credentials = get_google_credentials(options, config)

    if not options.get_google_credentials:
        create_event(options, config, credentials)

if __name__ == "__main__":

    main()


#!python
import click

import os
import math
import logging

from docstamp.file_utils import get_extension
from docstamp.template import TextDocument
from docstamp.config import LOGGING_LVL

from docstamp.cli.utils import (
    CONTEXT_SETTINGS,
    verbose_switch,
    get_items_from_csv,
    ExistingFilePath,
    DirPath
)

ACCEPTED_DOCS = "Inkscape (.svg), PDFLatex (.tex), XeLatex (.tex)"

# declare the CLI group
@click.group(context_settings=CONTEXT_SETTINGS)
def cli():
    pass

@cli.command(context_settings=CONTEXT_SETTINGS)
@click.option('-i', '--input', type=ExistingFilePath, required=False,
              help='Path to the CSV file with the data elements to be used to '
                   'fill the template. This file must have the same fields as '
                   'the template file.')
@click.option('-t', '--template', type=ExistingFilePath, required=True,
              help='Template file path. The extension of this file will be '
                   'used to determine what software to use to render the '
                   'documents: \n' + ACCEPTED_DOCS)
@click.option('-f', '--field', type=str, multiple=True,
              help='The field or fields that will be used to name '
                   'the output files. Use many of this to declare many values. '
                   'Otherwise files will be numbered.')
@click.option('-o', '--outdir', type=DirPath, default='stamped',
              show_default=True, help='Output folder path.')
@click.option('-p', '--prefix', type=str,
              help='Output files prefix. Default: Template file name.')
@click.option('-d', '--otype', type=click.Choice(['pdf', 'png', 'svg']),
              default='pdf', show_default=True,
              help='Output file type.')
@click.option('-c', '--command', type=click.Choice(['inkscape', 'pdflatex', 'xelatex']),
              default='inkscape', show_default=True,
              help='The rendering command to be used in case file name '
                   'extension is not specific.')
@click.option('--index', default=[],
              help='Index/es of the CSV file that you want to create the '
                   'document from. Note that the samples numbers start from 0 '
                   'and the empty ones do not count.')
@click.option('--dpi', type=int, default=150, help='Output file resolution')
@click.option('-v', '--verbose', is_flag=True,
              help='Output debug logs.')
@click.option('-u', '--unicode_support', is_flag=True, default=False,
              help='Allows unicode characters to be correctly encoded in the PDF.')
def create(input, template, field, outdir, prefix, otype, command, index,
           dpi, verbose, unicode_support):
    """Use docstamp to create documents from the content of a CSV file or
    a Google Spreadsheet.

    Examples: \n
    docstamp create -i badge.csv -t badge_template.svg -o badges
    docstamp create -i badge.csv -t badge_template.svg -o ./badges -d pdf
    """
    logging.basicConfig(level=LOGGING_LVL)
    log = logging.getLogger(__name__)

    # setup verbose mode
    verbose_switch(verbose)

    input_file = input
    fields = field

    # init set of template contents
    log.debug('Reading CSV elements from {}.'.format(input_file))
    items, fieldnames = get_items_from_csv(input_file)

    # check if got any item
    if len(items) == 0:
        click.echo('Quiting because found 0 items.')
        exit(-1)

    if not fields:
        # set the number of zeros that the files will have
        n_zeros = int(math.floor(math.log10(len(items))) + 1)
    else:
        # check that fields has all valid fields
        for field_name in fields:
            if field_name not in fieldnames:
                raise ValueError('Field name {} not found in input file '
                                 ' header.'.format(field_name))

    # filter the items if index
    if index:
        myitems = {int(idx): items[int(idx)] for idx in index}
        items = myitems
        log.debug('Using the elements with index {} of the input '
                  'file.'.format(index))

    # make output folder
    if not os.path.exists(outdir):
        os.mkdir(outdir)

    # create template document model
    log.debug('Creating the template object using the file {}.'.format(template))
    template_doc = TextDocument.from_template_file(template, command)
    log.debug('Created an object of type {}.'.format(type(template_doc)))

    # let's stamp them!
    for idx in items:
        item = items[idx]

        if not len(fields):
            file_name = str(idx).zfill(n_zeros)
        else:
            field_values = []
            try:
                for field_name in fields:
                    field_values.append(item[field_name].replace(' ', ''))
            except:
                log.exception('Could not get field {} value from'
                              ' {}'.format(field_name, item))
                exit(-1)
            else:
                file_name = '_'.join(field_values)

        log.debug('Filling template {} with values of item {}.'.format(file_name, idx))
        try:
            template_doc.fill(item)
        except:
            log.exception('Error filling document for {}th item'.format(idx))
            continue

        # set output file path
        file_extension = get_extension(template)
        if prefix is None:
            basename = os.path.basename(template).replace(file_extension, '')

        file_name = basename + '_' + file_name
        file_path = os.path.join(outdir, file_name + '.' + otype)

        kwargs = {'file_type': otype,
                  'dpi': dpi,
                  'support_unicode': unicode_support}

        log.debug('Rendering file {}.'.format(file_path))
        try:
            template_doc.render(file_path, **kwargs)
        except:
            log.exception('Error creating {} for {}.'.format(file_path, item))
            exit(-1)
        else:
            log.debug('Successfully rendered {}.'.format(file_path))


# coding=utf-8
# -------------------------------------------------------------------------------
# Author: Alexandre Manhaes Savio <alexsavio@gmail.com>
# Grupo de Inteligencia Computational <www.ehu.es/ccwintco>
# Universidad del Pais Vasco UPV/EHU
#
# 2015, Alexandre Manhaes Savio
# Use this at your own risk!
# -------------------------------------------------------------------------------

import os
import tempfile
import logging
from glob import glob

from docstamp.config import get_temp_dir

log = logging.getLogger(__name__)

def get_extension(filepath, check_if_exists=False):
    """Return the extension of fpath.

    Parameters
    ----------
    fpath: string
    File name or path

    check_if_exists: bool

    Returns
    -------
    str
    The extension of the file name or path
    """
    if check_if_exists:
        if not os.path.exists(filepath):
            err = 'File not found: ' + filepath
            log.error(err)
            raise IOError(err)

    try:
        rest, ext = os.path.splitext(filepath)
    except:
        raise
    else:
        return ext

def add_extension_if_needed(filepath, ext, check_if_exists=False):
    """Add the extension ext to fpath if it doesn't have it.

    Parameters
    ----------
    filepath: str
    File name or path

    ext: str
    File extension

    check_if_exists: bool

    Returns
    -------
    File name or path with extension added, if needed.
    """
    if not filepath.endswith(ext):
        filepath += ext

    if check_if_exists:
        if not os.path.exists(filepath):
            err = 'File not found: ' + filepath
            log.error(err)
            raise IOError(err)

    return filepath

def remove_ext(filepath):
    """Removes the extension of the file.

    Parameters
    ----------
    filepath: str
        File path or name

    Returns
    -------
    str
        File path or name without extension
    """
    return filepath[:filepath.rindex(get_extension(filepath))]

def get_tempfile(suffix='.txt', dirpath=None):
    """ Return a temporary file with the given suffix within dirpath.
    If dirpath is None, will look for a temporary folder in your system.

    Parameters
    ----------
    suffix: str
        Temporary file name suffix

    dirpath: str
        Folder path where create the temporary file

    Returns
    -------
    temp_filepath: str
        The path to the temporary path
    """
    if dirpath is None:
        dirpath = get_temp_dir()

    return tempfile.NamedTemporaryFile(suffix=suffix, dir=dirpath)

def cleanup(workdir, extension):
    """ Remove the files in workdir that have the given extension.

    Parameters
    ----------
    workdir:
        Folder path from where to clean the files.

    extension: str
        File extension without the dot, e.g., 'txt'
    """
    [os.remove(f) for f in glob(os.path.join(workdir, '*.' + extension))]

def mkdir(dirpath):
    """Create a folder in `dirpath` if it does'nt exist."""
    if not os.path.exists(dirpath):
        os.mkdir(dirpath)

def csv_to_json(csv_filepath, json_filepath, fieldnames, ignore_first_line=True):
    """ Convert a CSV file in `csv_filepath` into a JSON file in `json_filepath`.

    Parameters
    ----------
    csv_filepath: str
        Path to the input CSV file.

    json_filepath: str
        Path to the output JSON file. Will be overwritten if exists.

    fieldnames: List[str]
        Names of the fields in the CSV file.

    ignore_first_line: bool
    """
    import csv
    import json

    csvfile = open(csv_filepath, 'r')
    jsonfile = open(json_filepath, 'w')

    reader = csv.DictReader(csvfile, fieldnames)
    rows = []
    if ignore_first_line:
        next(reader)

    for row in reader:
        rows.append(row)

    json.dump(rows, jsonfile)
    jsonfile.close()
    csvfile.close()

def write_to_file(file_path, content, encoding=None):
    """ Write `content` inside the file in `file_path` with the given encoding.
    Parameters
    ----------
    file_path: str
        Path to the output file. Will be overwritten if exists.

    content: str
        The content you want in the file.

    encoding: str
        The name of the encoding.
    """
    try:
        # TODO: check if in Python2 this should be this way
        # it's possible that we have to make this function more complex
        # to check type(content) and depending on that set 'w' without enconde
        # or 'wb' with encode.
        with open(file_path, "wb") as f:
            f.write(content.encode(encoding))
    except:
        log.exception('Error writing to file in {}'.format(file_path))
        raise

def replace_file_content(filepath, old, new, max=1):
    """ Modify the content of `filepath`, replacing `old` for `new`.

    Parameters
    ----------
    filepath: str
        Path to the file to be modified. It will be overwritten.

    old: str
        This is old substring to be replaced.

    new: str
        This is new substring, which would replace old substring.

    max: int
        If larger than 0, Only the first `max` occurrences are replaced.
    """
    with open(filepath, 'r') as f:
        content = f.read()

    content = content.replace(old, new, max)
    with open(filepath, 'w') as f:
        f.write(content)

def cleanup_docstamp_output(output_dir=''):
    """ Remove the 'tmp*.aux', 'tmp*.out' and 'tmp*.log' files in `output_dir`.
    :param output_dir:
    """
    suffixes = ['aux', 'out', 'log']
    files = [f for suf in suffixes for f in glob(os.path.join(output_dir, 'tmp*.{}'.format(suf)))]
    [os.remove(file) for file in files]