"""This codec implements a simpler repeating group logic where the first tag is seen as a marker
for repetition in a repeating group (rather than relying on the order of the tags) """
import sys
import re

from collections import deque

from pyfixmsg import RepeatingGroup
from pyfixmsg.reference import HEADER_SORT_MAP
from pyfixmsg.util import int_or_str

SEPARATOR = '\1'
"""
Standard separator for the StringFIX codec between tag value pairs.
However, the codec supports an arbitrary separator as using | or ; or "<SOH>" is
common.
"""

DELIMITER = '='
FIX_REGEX_STRING = r'([^{s}{d}]*)[{d}](.*?){s}(?!\w+{s})'
FIX_REGEX = re.compile(FIX_REGEX_STRING.format(d=DELIMITER, s=SEPARATOR).encode('UTF-8'), re.DOTALL)
MICROSECONDS = 0
MILLISECONDS = 1

if sys.version_info.major >= 3:
    unicode = str  # pylint: disable=W0622,C0103


class Codec(object):
    """
    FIX codec.  initialise with a :py:class:`~pyfixmsg.reference.FixSpec` to support
    repeating groups.

    This class is used to transform the serialised FIX message into an instance of ``fragment_class``, default ``dict``
    Tags are assumed to be all of type ``int``, repeating groups are lists of ``fragment_class``

    Values can either bytes or unicode or a mix thereof, depending on the constructor arguments.

    """

    def __init__(self, spec=None, no_groups=False, fragment_class=dict, decode_as=None, decode_all_as_347=False):
        """
        :param spec: the :py:class:`~pyfixmsg.reference.FixSpec` instance to use to parse messages.
            If spec is not defined repeating groups will not be parsed correctly, and the logic to handle encoded
            tags will not be functional.
        :param no_groups: set to ``True`` to ignore repeating groups
        :param fragment_class: Which dict-like object to return when parsing messages. Also sets the type of
          members of repeating groups
        :param decode_as: what encoding to decode all tags. Defaults to None, which returns the raw byte strings.
           setting to a non-None value makes both non-numerical tags and values to be unicode,
           using this value for decode.
        :param decode_all_as_347: whether to trust tag 347 to decode all other tags or only the Encoded* ones.
          If set to False, use 347 normally for Encoded* tags, respect ``decode_as`` for all other tags. If 347 is
          not present on the message, the values are left encoded.

          """
        self.encoding = decode_as
        self.decode_all_as_347 = decode_all_as_347
        self.spec = spec
        if spec is None:
            self._no_groups = True
        else:
            self._no_groups = no_groups
        self._frg_class = fragment_class

    def parse(self, buff, delimiter=DELIMITER, separator=SEPARATOR):
        """
        Parse a FIX message. The FIX message is expected to be a bytestring and the output
        is a dictionary-like object which type is determined by the ``fragment_class`` constructor argument
        and which keys are ``int`` and values ``unicode``. Note that if there is a non-int tag in the message,
        this will be stored as a key in the original format (i.e. bytestring)

        :param buff: Buffer to parse
        :type buff: ``bytestr``
        :param delimiter: A character that separate key and values inside the FIX message. Generally '='. Note the type:
          because of the way the buffer is tokenised, this needs to be unicode (or ``str`` in python 2.7*).
        :type delimiter: ``unicode``
        :param separator: A character that separate key+value pairs inside the FIX message. Generally '\1'. See type
          observations above.
        :type separator: ``unicode``
        """
        def pushback_generator(iterator):
            """
            Generator which allows to push back a previously picked item
            for example:
            gen = pushback_generator(range(10))
            print next(gen)
            print next(gen)
            v = next(gen)
            print v
            gen.send(v)
            print next(gen)
            :param iterator:
            :return:
            """
            for value in iterator:
                back = yield value
                if back is not None:
                    yield back
                    yield back

        custom_r = re.compile(FIX_REGEX_STRING.format(d=re.escape(delimiter),
                                                      s=re.escape(separator)).encode('UTF-8'),
                              re.DOTALL)
        tagvals = custom_r.findall(buff)

        msg_type = None
        if not self._no_groups and self.spec is not None:
            for i in range(4):
                if tagvals[i][0] == b'35':
                    msg_type = self.spec.msg_types.get(tagvals[i][1])

        if self.encoding is not None:
            tagvals = ((int_or_str(tval[0], self.encoding), tval[1].decode(self.encoding)) for tval in tagvals)
        elif self.decode_all_as_347:
            tagvals = [(int_or_str(tval[0]), tval[1]) for tval in tagvals]
            encoding = None
            for tag, val in tagvals:
                if tag == 347:
                    encoding = val.decode('UTF-8')
                    break
            if encoding is not None:
                tagvals = ((t[0], t[1].decode(encoding)) for t in tagvals)
        else:
            tagvals = ((int_or_str(tval[0]), tval[1]) for tval in tagvals)
        # Need to add logic to parse Encoded* tags according to 347
        if self._no_groups or self.spec is None or msg_type is None:
            # no groups can be found without a spec, so no point looking up the msg type.
            return self._frg_class(tagvals)
        msg = self._frg_class()
        groups = msg_type.groups
        tagvals = pushback_generator(tagvals)
        for tag, value in tagvals:
            if tag not in groups:
                msg[tag] = value
            else:
                if value == '0':
                    msg[tag] = RepeatingGroup.create_repeating_group(tag)
                else:
                    contents, last_tagval = self._process_group(tag, tagvals,
                                                               msg_type=msg_type,
                                                               group=groups[tag])
                    msg[tag] = contents
                    if last_tagval:
                        tagvals.send(last_tagval)
        return msg

    def _process_group(self, identifying_tag, enumerator, msg_type, group):
        """
        Recursively process a group
        Returns ``(count_tag, [{}, {}])``
        """
        rep_group = RepeatingGroup()
        rep_group.number_tag = identifying_tag
        member = self._frg_class()
        first_tag = None
        inner_groups = group.groups
        valid_tags = group.tags
        for tag, value in enumerator:
            if first_tag is None:
                # handle first tag: we expect all the members of the group to start with this tag
                first_tag = tag
                rep_group.first_tag = tag
                member[tag] = value
            elif first_tag == tag:
                # we start a new group, replace the current member by an empty one and add the current tag
                rep_group.append(member)
                member = self._frg_class()
                member[tag] = value
            elif tag in valid_tags:
                # tag is a member, we just add
                member[tag] = value
            elif tag in inner_groups:
                # tag is starting a new sub group, we recurse
                contents, last_tagval = self._process_group(tag, enumerator, msg_type, group.groups[tag])
                member[tag] = contents
                if last_tagval:
                    # we are not at the end of the message.
                    tag, val = last_tagval
                    if tag == first_tag:
                        # the embedded group finished this member
                        rep_group.append(member)
                        member = self._frg_class()
                        member[tag] = val
                    elif tag in group.tags:
                        # didn't finish this member
                        member[tag] = val
                    else:
                        # didn't finish the message but finished the current group
                        rep_group.append(member)
                        return rep_group, (tag, val)
            else:
                # we're out of the group.
                rep_group.append(member)
                return rep_group, (tag, value)
        # we are reaching the end of the message, so complete, no further tags to pass on
        rep_group.append(member)
        return rep_group, None

    def _unmap(self, msg):
        """
        Create a tag,value sequence from a FixMessage (dict-type interface).
        This will leverage the spec to order the tags the same way they are defined in the spec.
        If tags are present on the message that are not in the spec they are added in order at the end
        of the message before the tail (tag 10).
        """

        def sort_values(msg, spec):
            """ Sort {tag:value} map into an iterable  """
            tvals = list(msg.items())
            get_sorting_key = lambda x: spec.sorting_key.get(x[0], int(10e8 + x[0]))
            tvals.sort(key=get_sorting_key)
            #  using a deque for this already-sorted data structure yields a ~10% speed improvement on serialisation
            expanded = deque()
            for tag, val in tvals:
                if isinstance(val, list):  # Repeating groups are also lists, so we only need one type here
                    downspec = spec.groups[tag]
                    expanded.append((tag, len(val)))
                    for member in val:
                        expanded.extend(sort_values(member, downspec))
                else:
                    expanded.append((tag, val))
            return expanded

        if self.spec is None:
            #  No spec, let's just get reasonable header order, and 10 at the end.
            tag_vals = list(msg.items())
            tag_vals.sort(key=lambda x: HEADER_SORT_MAP.get(x[0], int(10e8 + x[0])))
            return tag_vals
        else:
            return sort_values(msg, self.spec.msg_types[msg[35]])

    def serialise(self, msg, separator=SEPARATOR, delimiter=DELIMITER, encoding=None):
        """
        Serialise a message into a bytestring.

        :param msg: the message to serialse
        :type msg: ``dict``-like interface
        :param delimiter: as in ``parse()``
        :param separator: as in ``parse()``
        :param encoding: as in ``parse()``
        """
        tag_vals = self._unmap(msg)
        output = deque()
        for tag, value in tag_vals:
            if isinstance(tag, int):
                output.append(str(tag).encode('ascii'))
            elif isinstance(tag, bytes):
                output.append(tag)
            elif isinstance(tag, unicode):
                output.append(tag.encode('ascii'))
            else:
                output.append(str(tag).encode('ascii'))
            output.append(delimiter.encode('ascii'))
            if isinstance(value, bytes):
                output.append(value)
            else:
                if not isinstance(value, unicode):
                    value = unicode(value)
                if encoding is not None:
                    output.append(value.encode(encoding))
                elif self.encoding is not None:
                    output.append(value.encode(self.encoding))
                else:
                    output.append(value.encode('UTF-8'))
            output.append(separator.encode('ascii'))
        return b"".join(output)

"""This codec implements a simpler repeating group logic where the first tag is seen as a marker
for repetition in a repeating group (rather than relying on the order of the tags) """
import sys
import re

from collections import deque

from pyfixmsg import RepeatingGroup
from pyfixmsg.reference import HEADER_SORT_MAP
from pyfixmsg.util import int_or_str

SEPARATOR = '\1'
"""
Standard separator for the StringFIX codec between tag value pairs.
However, the codec supports an arbitrary separator as using | or ; or "<SOH>" is
common.
"""

DELIMITER = '='
FIX_REGEX_STRING = r'([^{s}{d}]*)[{d}](.*?){s}(?!\w+{s})'
FIX_REGEX = re.compile(FIX_REGEX_STRING.format(d=DELIMITER, s=SEPARATOR).encode('UTF-8'), re.DOTALL)
MICROSECONDS = 0
MILLISECONDS = 1

if sys.version_info.major >= 3:
    unicode = str  # pylint: disable=W0622,C0103


class Codec(object):
    """
    FIX codec.  initialise with a :py:class:`~pyfixmsg.reference.FixSpec` to support
    repeating groups.

    This class is used to transform the serialised FIX message into an instance of ``fragment_class``, default ``dict``
    Tags are assumed to be all of type ``int``, repeating groups are lists of ``fragment_class``

    Values can either bytes or unicode or a mix thereof, depending on the constructor arguments.

    """

    def __init__(self, spec=None, no_groups=False, fragment_class=dict, decode_as=None, decode_all_as_347=False):
        """
        :param spec: the :py:class:`~pyfixmsg.reference.FixSpec` instance to use to parse messages.
            If spec is not defined repeating groups will not be parsed correctly, and the logic to handle encoded
            tags will not be functional.
        :param no_groups: set to ``True`` to ignore repeating groups
        :param fragment_class: Which dict-like object to return when parsing messages. Also sets the type of
          members of repeating groups
        :param decode_as: what encoding to decode all tags. Defaults to None, which returns the raw byte strings.
           setting to a non-None value makes both non-numerical tags and values to be unicode,
           using this value for decode.
        :param decode_all_as_347: whether to trust tag 347 to decode all other tags or only the Encoded* ones.
          If set to False, use 347 normally for Encoded* tags, respect ``decode_as`` for all other tags. If 347 is
          not present on the message, the values are left encoded.

          """
        self.encoding = decode_as
        self.decode_all_as_347 = decode_all_as_347
        self.spec = spec
        if spec is None:
            self._no_groups = True
        else:
            self._no_groups = no_groups
        self._frg_class = fragment_class

    def parse(self, buff, delimiter=DELIMITER, separator=SEPARATOR):
        """
        Parse a FIX message. The FIX message is expected to be a bytestring and the output
        is a dictionary-like object which type is determined by the ``fragment_class`` constructor argument
        and which keys are ``int`` and values ``unicode``. Note that if there is a non-int tag in the message,
        this will be stored as a key in the original format (i.e. bytestring)

        :param buff: Buffer to parse
        :type buff: ``bytestr``
        :param delimiter: A character that separate key and values inside the FIX message. Generally '='. Note the type:
          because of the way the buffer is tokenised, this needs to be unicode (or ``str`` in python 2.7*).
        :type delimiter: ``unicode``
        :param separator: A character that separate key+value pairs inside the FIX message. Generally '\1'. See type
          observations above.
        :type separator: ``unicode``
        """
        def pushback_generator(iterator):
            """
            Generator which allows to push back a previously picked item
            for example:
            gen = pushback_generator(range(10))
            print next(gen)
            print next(gen)
            v = next(gen)
            print v
            gen.send(v)
            print next(gen)
            :param iterator:
            :return:
            """
            for value in iterator:
                back = yield value
                if back is not None:
                    yield back
                    yield back

        custom_r = re.compile(FIX_REGEX_STRING.format(d=re.escape(delimiter),
                                                      s=re.escape(separator)).encode('UTF-8'),
                              re.DOTALL)
        tagvals = custom_r.findall(buff)

        msg_type = None
        if not self._no_groups and self.spec is not None:
            for i in range(4):
                if tagvals[i][0] == b'35':
                    msg_type = self.spec.msg_types.get(tagvals[i][1])

        if self.encoding is not None:
            tagvals = ((int_or_str(tval[0], self.encoding), tval[1].decode(self.encoding)) for tval in tagvals)
        elif self.decode_all_as_347:
            tagvals = [(int_or_str(tval[0]), tval[1]) for tval in tagvals]
            encoding = None
            for tag, val in tagvals:
                if tag == 347:
                    encoding = val.decode('UTF-8')
                    break
            if encoding is not None:
                tagvals = ((t[0], t[1].decode(encoding)) for t in tagvals)
        else:
            tagvals = ((int_or_str(tval[0]), tval[1]) for tval in tagvals)
        # Need to add logic to parse Encoded* tags according to 347
        if self._no_groups or self.spec is None or msg_type is None:
            # no groups can be found without a spec, so no point looking up the msg type.
            return self._frg_class(tagvals)
        msg = self._frg_class()
        groups = msg_type.groups
        tagvals = pushback_generator(tagvals)
        for tag, value in tagvals:
            if tag not in groups:
                msg[tag] = value
            else:
                if value == '0':
                    msg[tag] = RepeatingGroup.create_repeating_group(tag)
                else:
                    contents, last_tagval = self._process_group(tag, tagvals,
                                                               msg_type=msg_type,
                                                               group=groups[tag])
                    msg[tag] = contents
                    if last_tagval:
                        tagvals.send(last_tagval)
        return msg

    def _process_group(self, identifying_tag, enumerator, msg_type, group):
        """
        Recursively process a group
        Returns ``(count_tag, [{}, {}])``
        """
        rep_group = RepeatingGroup()
        rep_group.number_tag = identifying_tag
        member = self._frg_class()
        first_tag = None
        inner_groups = group.groups
        valid_tags = group.tags
        for tag, value in enumerator:
            if first_tag is None:
                # handle first tag: we expect all the members of the group to start with this tag
                first_tag = tag
                rep_group.first_tag = tag
                member[tag] = value
            elif first_tag == tag:
                # we start a new group, replace the current member by an empty one and add the current tag
                rep_group.append(member)
                member = self._frg_class()
                member[tag] = value
            elif tag in valid_tags:
                # tag is a member, we just add
                member[tag] = value
            elif tag in inner_groups:
                # tag is starting a new sub group, we recurse
                contents, last_tagval = self._process_group(tag, enumerator, msg_type, group.groups[tag])
                member[tag] = contents
                if last_tagval:
                    # we are not at the end of the message.
                    tag, val = last_tagval
                    if tag == first_tag:
                        # the embedded group finished this member
                        rep_group.append(member)
                        member = self._frg_class()
                        member[tag] = val
                    elif tag in group.tags:
                        # didn't finish this member
                        member[tag] = val
                    else:
                        # didn't finish the message but finished the current group
                        rep_group.append(member)
                        return rep_group, (tag, val)
            else:
                # we're out of the group.
                rep_group.append(member)
                return rep_group, (tag, value)
        # we are reaching the end of the message, so complete, no further tags to pass on
        rep_group.append(member)
        return rep_group, None

    def _unmap(self, msg):
        """
        Create a tag,value sequence from a FixMessage (dict-type interface).
        This will leverage the spec to order the tags the same way they are defined in the spec.
        If tags are present on the message that are not in the spec they are added in order at the end
        of the message before the tail (tag 10).
        """

        def sort_values(msg, spec):
            """ Sort {tag:value} map into an iterable  """
            tvals = list(msg.items())
            get_sorting_key = lambda x: spec.sorting_key.get(x[0], int(10e8 + x[0]))
            tvals.sort(key=get_sorting_key)
            #  using a deque for this already-sorted data structure yields a ~10% speed improvement on serialisation
            expanded = deque()
            for tag, val in tvals:
                if isinstance(val, list):  # Repeating groups are also lists, so we only need one type here
                    downspec = spec.groups[tag]
                    expanded.append((tag, len(val)))
                    for member in val:
                        expanded.extend(sort_values(member, downspec))
                else:
                    expanded.append((tag, val))
            return expanded

        if self.spec is None:
            #  No spec, let's just get reasonable header order, and 10 at the end.
            tag_vals = list(msg.items())
            tag_vals.sort(key=lambda x: HEADER_SORT_MAP.get(x[0], int(10e8 + x[0])))
            return tag_vals
        else:
            return sort_values(msg, self.spec.msg_types[msg[35]])

    def serialise(self, msg, separator=SEPARATOR, delimiter=DELIMITER, encoding=None):
        """
        Serialise a message into a bytestring.

        :param msg: the message to serialse
        :type msg: ``dict``-like interface
        :param delimiter: as in ``parse()``
        :param separator: as in ``parse()``
        :param encoding: as in ``parse()``
        """
        tag_vals = self._unmap(msg)
        output = deque()
        for tag, value in tag_vals:
            if isinstance(tag, int):
                output.append(str(tag).encode('ascii'))
            elif isinstance(tag, bytes):
                output.append(tag)
            elif isinstance(tag, unicode):
                output.append(tag.encode('ascii'))
            else:
                output.append(str(tag).encode('ascii'))
            output.append(delimiter.encode('ascii'))
            if isinstance(value, bytes):
                output.append(value)
            else:
                if not isinstance(value, unicode):
                    value = unicode(value)
                if encoding is not None:
                    output.append(value.encode(encoding))
                elif self.encoding is not None:
                    output.append(value.encode(self.encoding))
                else:
                    output.append(value.encode('UTF-8'))
            output.append(separator.encode('ascii'))
        return b"".join(output)

This codec implements a simpler repeating group logic where the first tag is seen as a marker for repetition in a repeating group (rather than relying on the order of the tags) Standard separator for the StringFIX codec between tag value pairs. However, the codec supports an arbitrary separator as using | or ; or "<SOH>" is common. # pylint: disable=W0622,C0103 FIX codec.  initialise with a :py:class:`~pyfixmsg.reference.FixSpec` to support     repeating groups.      This class is used to transform the serialised FIX message into an instance of ``fragment_class``, default ``dict``     Tags are assumed to be all of type ``int``, repeating groups are lists of ``fragment_class``      Values can either bytes or unicode or a mix thereof, depending on the constructor arguments. :param spec: the :py:class:`~pyfixmsg.reference.FixSpec` instance to use to parse messages.             If spec is not defined repeating groups will not be parsed correctly, and the logic to handle encoded             tags will not be functional.         :param no_groups: set to ``True`` to ignore repeating groups         :param fragment_class: Which dict-like object to return when parsing messages. Also sets the type of           members of repeating groups         :param decode_as: what encoding to decode all tags. Defaults to None, which returns the raw byte strings.            setting to a non-None value makes both non-numerical tags and values to be unicode,            using this value for decode.         :param decode_all_as_347: whether to trust tag 347 to decode all other tags or only the Encoded* ones.           If set to False, use 347 normally for Encoded* tags, respect ``decode_as`` for all other tags. If 347 is           not present on the message, the values are left encoded. Parse a FIX message. The FIX message is expected to be a bytestring and the output         is a dictionary-like object which type is determined by the ``fragment_class`` constructor argument         and which keys are ``int`` and values ``unicode``. Note that if there is a non-int tag in the message,         this will be stored as a key in the original format (i.e. bytestring)          :param buff: Buffer to parse         :type buff: ``bytestr``         :param delimiter: A character that separate key and values inside the FIX message. Generally '='. Note the type:           because of the way the buffer is tokenised, this needs to be unicode (or ``str`` in python 2.7*).         :type delimiter: ``unicode``         :param separator: A character that separate key+value pairs inside the FIX message. Generally '\1'. See type           observations above.         :type separator: ``unicode`` Generator which allows to push back a previously picked item             for example:             gen = pushback_generator(range(10))             print next(gen)             print next(gen)             v = next(gen)             print v             gen.send(v)             print next(gen)             :param iterator:             :return: # Need to add logic to parse Encoded* tags according to 347 # no groups can be found without a spec, so no point looking up the msg type. Recursively process a group         Returns ``(count_tag, [{}, {}])`` # handle first tag: we expect all the members of the group to start with this tag # we start a new group, replace the current member by an empty one and add the current tag # tag is a member, we just add # tag is starting a new sub group, we recurse # we are not at the end of the message. # the embedded group finished this member # didn't finish this member # didn't finish the message but finished the current group # we're out of the group. # we are reaching the end of the message, so complete, no further tags to pass on Create a tag,value sequence from a FixMessage (dict-type interface).         This will leverage the spec to order the tags the same way they are defined in the spec.         If tags are present on the message that are not in the spec they are added in order at the end         of the message before the tail (tag 10). Sort {tag:value} map into an iterable #  using a deque for this already-sorted data structure yields a ~10% speed improvement on serialisation # Repeating groups are also lists, so we only need one type here #  No spec, let's just get reasonable header order, and 10 at the end. Serialise a message into a bytestring.          :param msg: the message to serialse         :type msg: ``dict``-like interface         :param delimiter: as in ``parse()``         :param separator: as in ``parse()``         :param encoding: as in ``parse()``