class TxBlockLexer(BlockLexer):

    md_stringset = []

    # Overwritten to not drop `>` character from quote block
    def parse_block_quote(self, m):
        self.tokens.append({'type': 'block_quote_start'})
        self.tokens.append({'type': 'block_quote_end'})

    def parse(self, text, rules=None):
        text = text.rstrip('\n')
        parser_rules = ('list_block', 'def_footnotes')
        table_rules = ('table', 'nptable')

        if not rules:
            rules = self.default_rules

        def manipulate(text):
            for key in rules:
                rule = getattr(self.rules, key)
                m = rule.match(text)
                if not m:
                    continue
                getattr(self, 'parse_%s' % key)(m)
                return key, m
            return False  # pragma: no cover
        while text:
            m = manipulate(text)
            if isinstance(m, tuple):
                key, m = m
            else:
                key = False
            if m is not False:

                # The following parser rules called by `manipulate` call
                # `self.parse` recursively. We don't catch matches for such
                # methods to avoid getting duplicated parts of the markdown
                # content in the `self.md_stringset` because of the recursion.
                if key and key in table_rules:
                    table_token = self.tokens[-1]
                    keys = list(six.iterkeys(table_token))
                    if 'header' in keys and 'cells' in keys:
                        self.md_stringset.extend(
                            [(h, 'header') for h in table_token['header']])
                        self.md_stringset.extend(
                            [(cell, 'cell') for row in table_token['cells']
                                for cell in row]
                        )
                elif key and key not in parser_rules:
                    # Grab md string match and put in a md_stringset list.
                    self.md_stringset.append((m.group(0), key))

                text = text[len(m.group(0)):]
                continue
            if text:  # pragma: no cover
                raise RuntimeError('Infinite loop at: %s' % text)

        return self.tokens def string_handler(token, template):
    """
    Extra checks and manipulation of extracted string from markdown file.
    Parameters:
    token: Tuple of (string, string_type) where string_type refers to the
           type of markdown element this string belongs to. string_type
           can be None.
    template: the template of the resource

    returns: the manipulated string or None in case the manipulated string
             is not valid anymore e.g. empty string
    """

    # Drop new lines around string.
    string, key = token
    string = string.strip('\n')
    # for code blocks we need to maintain the exact indentation as in
    # the source file both for matching the string and replacing it in the
    # template and for producing a valid markdown on compilation
    if key == 'block_code':
        lines = string.split('\n')
        line = lines[0]
        spaces = re.findall(
            ensure_unicode(r'\n( *){}').format(re.escape(line)),
            template
        )[0]
        if spaces:
            string = ''
            for line in lines:
                line = u'{}{}'.format(spaces, line)
                string += '\n'
                string += line

    # Line is a liquid template tag, ignore.
    if string.startswith('{%') and string.endswith('%}'):
        return

    # Drop # chars from beginning of the string
    match_header_line = re.search(ensure_unicode(r'^#+\s'), string)
    if match_header_line:
        return string.replace(match_header_line.group(), '')

    # Extract Text from `[Text]: link` or `"[Text]: link"` lines
    match_reference = re.search(ensure_unicode(r'^"?\[([^\[^\]]+)\]:.+"?$'), string)
    if match_reference:
        try:
            int(match_reference.groups()[0])
        except ValueError:
            # Get content between brackets if it's not an integer number
            return match_reference.groups()[0]
        return

    # exclude numeric values from stringset
    try:
        float(string)
        return
    except ValueError:
        pass

    return string class YamlHandler(Handler):
    name = "Yaml"
    extension = "yml"
    EXTRACTS_RAW = False

    # number of spaces used for indentation in YAML content
    indent = 2

    # the levels of indentation of a string can be extracted by doing
    # `len(keys.split('.'))`. For files that we ignore the root key
    # (e.g ruby i18n files) we need to set an extra level of indentation
    # `extra_indent = 1`
    extra_indent = 0

    # When compiling for a mode that needs to completely remove
    # an entry from the compiled file we cannot use the template for
    # compilation but we need to construct the YAML file from scratch
    # only with the subset of included entries.
    should_use_template = True

    language_code = None

    def parse(self, content, **kwargs):
        """ Parses the given YAML content to create stringset and template

        Steps are:
            1. Load yaml content using our custom loader TxYamlLoader that
               in addition to the value for each key notes the `start` and
               `end` index of each node in the file and some metadata.
            2. Flattens the output of the loader to be a list of the form:
               ```
               [{
                   'key': 'string_key1',
                   'value': 'string1',
                   'end': <end_index_of_node>,
                   'start': <start_index_value>,
                   'style': '|, >, ...'
                },
                ...
               ]
               ```
            3. Iterates over the flattened list and for each entry creates an
               OpenString object, appends it to stringset and replace its value
               with the template_replacement in the template.
            4. Returns the (template, stringset) tuple.
        """
        template = []
        stringset = []
        # The first argument of the add_constructor method is the tag you want
        # to handle. If you provide None as an argument, all the unknown tags
        # will be handled by the constructor specified in the second argument.
        # We need this in order parse all unknown custom-tagged values as
        # strings.
        SafeConstructor.add_constructor(None,
                                        SafeConstructor.construct_yaml_str)
        yaml_data = self._load_yaml(content, loader=TxYamlLoader)
        yaml_data = self._get_yaml_data_to_parse(yaml_data)
        # Helper to store the processed data while parsing the file
        self._parsed_data = []
        self._parse_yaml_data(yaml_data, '', '')
        self._parsed_data = sorted(self._parsed_data,
                                   key=lambda node: node.get('start'))

        end = 0
        order = 0
        for node in self._parsed_data:
            start = node.get('start')
            end_ = node.get('end')
            key = node.get('key')
            tag = node.get('tag')
            value = node.get('value')
            style = node.get('style')
            if not value:
                continue
            if isinstance(value, dict) and not all(six.itervalues(value)):
                continue
            string_object = OpenString(
                key, value, context=tag or '', flags=style, order=order,
            )
            stringset.append(string_object)
            order += 1
            template.append(u"{}{}".format(content[end:start],
                                           string_object.template_replacement))
            comment = self._find_comment(content, end, start)
            string_object.developer_comment = comment
            end = end_

        template.append(content[end:])
        template = u''.join(template)
        return template, stringset

    def compile(self, template, stringset, **kwargs):
        """
        Dump YAML content for a resource translation taking
        into account the mode used for compilation.

        Args:
            template: Template content for the resource.
            stringset: The resource's stringset.

        Returns:
            A unicode, dumped YAML content.
        """
        self.indent = self._get_indent(template)
        if self.should_use_template:
            return self._compile_from_template(template, stringset)
        else:
            return self._compile_without_template(stringset)

    def _load_yaml(self, content, loader):
        """
        Loads a YAML stream and returns a dictionary
        representation for YAML data

        Args:
            content: A string, YAML content
            loader: YAML Loader class or None

        Returns:
            A dictionary
        """
        try:
            return yaml.load(content, Loader=loader)
        except yaml.scanner.ScannerError as e:
            raise ParseError(six.text_type(e))
        except Exception as e:
            raise ParseError(six.text_type(e))

    def _parse_leaf_node(self, node, parent_key, style=[],
                         pluralized=False):
        """Parse a leaf node in yaml_dict.
        Args:
            node: A tuple of the form (string, start, end, style, tag)
            parent_key: A string of keys concatenated by '.' to
                reach this node
            style: A list of YAML node styles from root node to
                   immediate parent node of the current YAML node.

        Returns:
            A dictionary representing the parsed leaf node
        """
        style.append(node.style or '')
        return {
            'start': node.start,
            'end': node.end,
            'key': parent_key,
            'value': node.value,
            'tag': node.tag,
            'style': ':'.join(style or []),
            'pluralized': pluralized,
        }

    def _parse_pluralized_leaf_node(self, yaml_data, parent_key, style=[],
                                    pluralized=False):
        """Parse a leaf node in yaml_dict.
        Args:
            yaml_data: A tuple of the form
                      (plurals_dict, start, end, style, tag) that describes a
                      pluralized node `plurals_dict` example:
                          {
                              "one": (string, start, end, style, tag),
                              "other": (string, start, end, style, tag)
                          }
            parent_key: A string of keys concatenated by '.' to
                        reach this node
            style: A list of YAML node styles from root node to
                   immediate parent node of the current YAML node.

        Returns:
            A dictionary representing the parsed leaf node
        """
        raise NotImplementedError

    def _get_key_for_node(self, key, parent_key):
        """
        Get key string for the current node.

        Args:
            key: A string, current key for the node
            parent_key: A string, containing all the keys
                (traversed before reaching the current key node)
                separated by '.'

        Returns:
            A string similar to parent_key with key appended to it.
        """
        if not isinstance(key, (six.binary_type, six.text_type)):
            # Int keys are stored in < and > around it, so it can be reverted
            # on int again on compiling.
            key = u'<{}>'.format(key)
        if '.' in key:
            key = self.escape_dots(key)
        if parent_key == '':
            parent_key += key
        else:
            parent_key += '.' + key
        return parent_key

    def _parse_yaml_data(self, yaml_data, parent_key, context="",
                         parent_style=None):
        """
        Parse data returned by YAML loader

        yaml_data can be either a dict like:
            {"key1": node1, "key2": node2, ...}
        or a list like:
            [node1, node2, ...]

        Node objects above are instances of the Node namedtuple with valid
        keys ['value', 'start', 'end', 'style']. node.value can be either
        another `yaml_data` object or a string.

        Args:
            yaml_data: The output of yaml.loads()
            parent_key: A string of keys concatenated by '.' to
                        reach this node,
            context: A string
            parent_style: A list of YAML node styles for each parent node.

        Returns:
            A list of dictionaries, where each dictionary maps a node
            key to its value
        """
        parent_style = parent_style or []

        if isinstance(yaml_data, dict):
            for key, node in six.iteritems(yaml_data):
                node_key = self._get_key_for_node(key, parent_key)
                # Copy style for each node to avoid getting affected from the
                # previous loops
                node_style = copy.copy(parent_style)
                # Case of dictionary that represents a plural rule
                if (isinstance(node.value, dict) and
                        self.is_pluralized(node.value)):
                    self._parsed_data.append(
                        self._parse_pluralized_leaf_node(
                            node, node_key,
                            style=node_style,
                            pluralized=True
                        )
                    )
                # Handle dictionaries and lists
                elif isinstance(node.value, (dict, list)):
                    node_style.append(node.style or '')
                    self._parse_yaml_data(node.value, node_key, context,
                                          parent_style=node_style)
                # Otherwise handle the node as a leaf
                else:
                    self._parsed_data.append(
                        self._parse_leaf_node(
                            node, node_key, style=node_style
                        )
                    )
        elif (isinstance(yaml_data, list)):
            # If list add each dict element as an entry
            # using the position (index) of it as parent key using
            # brackets around it. I.e.: 'foo.[0].bar'.
            for i, node in enumerate(yaml_data):
                node_key = self._get_key_for_node('[%s]' % (i), parent_key)
                # Copy style for each node to avoid getting affected from the
                # previous loops
                node_style = copy.copy(parent_style)
                if isinstance(node.value, (dict, list)):
                    node_style.append(node.style or '')
                    self._parse_yaml_data(node.value, node_key, context,
                                          parent_style=node_style)
                else:
                    self._parsed_data.append(
                        self._parse_leaf_node(
                            node, node_key, style=node_style
                        )
                    )

    def _find_comment(self, content, start, end):
        """ Finds comment lines that precede a part of the YAML structure """
        lines = [
            line.strip()
            for line in content[start:end].split('\n')
            if line.strip() != ""
        ]
        lines.reverse()

        # remove non-comment line just before the string to be translated
        if lines and not lines[0].startswith('#'):
            del lines[0]

        returned_lines = []
        for line in lines:
            if not line.startswith('#'):
                break
            returned_lines.append(line[1:].strip())

        if returned_lines:
            returned_lines.reverse()
            return " ".join(returned_lines)
        else:
            return ''

    def _get_yaml_data_to_parse(self, yaml_data):
        """ Returns the part of the yaml_data that actually need to be parsed

        In some cases only a part of the YAML structure needs to be processed
        for strings to be extracted. e.g. when the root key is the language
        code we need to exclude this from the rest of the processing.

        Subclasses can override the default behavior, by returning a subset
        of the whole structure.
        """
        return yaml_data

    def _write_styled_literal(self, string):
        """ Produce a properly formatted YAML string

        Properly format translation string based on string's style
        on the original source file using yaml.emitter.Emitter class.

        Args:
            string: An OpenString instance

        Returns:
            The formatted string.
        """
        if string.flags is None:
            return string.string

        stream = StringIO()
        emitter = Emitter(stream, allow_unicode=True)
        indent = self.indent * (len(string.key.split('.')) + self.extra_indent)
        emitter.indent = indent
        # set best_width to `float(inf)` so that long strings are not broken
        # into multiple lines
        emitter.best_width = float('inf')

        analysis = emitter.analyze_scalar(string.string)

        style = string.flags.split(':')[-1]

        if style == '"':
            emitter.write_double_quoted(string.string)
        elif style == '\'':
            if analysis.allow_single_quoted:
                emitter.write_single_quoted(string.string)
            else:
                emitter.write_double_quoted(string.string)
        elif style == '':
            if analysis.allow_block_plain and analysis.allow_flow_plain:
                emitter.write_plain(string.string)
            else:
                emitter.write_double_quoted(string.string)
        elif style == '|':
            emitter.write_literal(string.string)
        elif style == '>':
            emitter.write_folded(string.string)

        translation = emitter.stream.getvalue() or string.string
        if translation.startswith(">-") and not translation.endswith("\n"):
            translation += "\n"
        emitter.stream.close()
        return translation

    def _compile_from_template(self, template, stringset, **kwargs):
        """ Compiles translation file from template

        Iterates over the stringset and for each strings replaces
        template replacement in the template with the actual translation.

        Returns:
            The compiled file content.
        """
        transcriber = Transcriber(template)
        template = transcriber.source

        for string in stringset:
            if string.pluralized:
                translation = self._compile_pluralized(string)
            else:
                translation = self._write_styled_literal(string)
            hash_position = template.index(string.template_replacement)
            transcriber.copy_until(hash_position)
            # The context contains custom tags. If it exists, we must prepend
            # it and apply a space afterwards so it doesn't get merged with the
            # string
            if string.context:
                transcriber.add(string.context)
                transcriber.add(' ')
            transcriber.add(translation)
            transcriber.skip(len(string.template_replacement))

        transcriber.copy_until(len(template))
        compiled = transcriber.get_destination()

        return compiled

    def _compile_without_template(self, stringset):
        yg = YamlGenerator(self)
        yaml_dict = yg.generate_yaml_dict(stringset)
        if self.language_code:
            yaml_dict = self._wrap_yaml_dict(yaml_dict, self.language_code)
        return_value = yaml.dump(yaml_dict, width=float('inf'),
                                 Dumper=TxYamlDumper, allow_unicode=True,
                                 indent=self.indent)
        if isinstance(return_value, six.binary_type):
            return_value = return_value.decode("utf-8")
        return return_value

    @staticmethod
    def unescape_dots(k):
        """ Replace <TX_DOT> placeholder with a dot. """
        return k.replace('<TX_DOT>', '.')

    @staticmethod
    def escape_dots(k):
        """ We use dots to construct the strings key, so we need to
        escape dots that are part of an actual YAML key """
        return k.replace('.', '<TX_DOT>')

    def _get_indent(self, template):
        """
        Use a regular expression to figure out how many spaces are used
        for indentation in the original file.

        Args:
            template: The saved template
        Returns:
            The number of spaces.
        """
        # Match all whitespace characters after first `:` (end of first  key).
        # Stops on first non whitespace character.
        indent_pattern = re.compile(
            ensure_unicode(r':\r?\n(?P<indent>[ \t\n]+)')
        )
        m = indent_pattern.search(template)
        indent = m.groups('indent')[0] if m else ' ' * 2
        # keep only last line
        indent = indent.splitlines()[-1]
        indent = indent.replace('\t', ' ' * 4)
        return len(indent)

    def _compile_pluralized(self, string):
        """ Prepare a pluralized string to be added to the template
        """
        raise NotImplementedError

    def parse_pluralized_value(self, value):
        """ Creates a dictionary of the form:
        ```
        {rule_number: rule_translation, ...}
        ```
        based on a YAML node """
        raise NotImplementedError

    def is_pluralized(self, val):
        """ Checks if given YAML node should be handled as pluralized

        This method by default returns False because no entry in a
        generic YML file should be handled as pluralized.
        Should be overriden in any subclass that should handle pluralized
        strings """
        return False

    def _wrap_yaml_dict(self, yaml_dict, lang_code=None):
        """
        Update YAML dictionary with *language code* as the
        root of the dictionary.

        Args:
            yaml_dict: A Dictionary/OrderedDict instance.
            lang_code: A language code string.

        Returns:
            A Dictionary instance.
        """
        if lang_code:
            yaml_dict = {lang_code: yaml_dict}
        return yaml_dict def ensure_unicode(pattern):
    if isinstance(pattern, six.binary_type):
        pattern = pattern.decode('utf-8')
    return pattern class Handler(object):
    """
    This class defines the interface you need to implement in order to create a
    handler. Both the `parse` and `compile` methods must be implemented.
    """

    name = None
    extension = None

    _RULES_ATOI = {
        'zero': 0,
        'one': 1,
        'two': 2,
        'few': 3,
        'many': 4,
        'other': 5
    }

    EXTRACTS_RAW = True

    # Use this flag for handlers whose the content should be processed as
    # binary and not be converted to unicode
    PROCESSES_BINARY = False

    _RULES_ITOA = {value: key for key, value in six.iteritems(_RULES_ATOI)}

    _RULE_ERROR_MSG = (
        '{attempted} is not a valid rule value. Valid choices are {valid}'
    )

    @classmethod
    def get_rule_number(cls, string_value):
        try:
            return cls._RULES_ATOI[string_value]
        except KeyError:
            msg = cls._RULE_ERROR_MSG.format(
                attempted=string_value,
                valid=list(six.iterkeys(cls._RULES_ATOI))
            )
            raise RuleError(msg)

    @classmethod
    def get_rule_string(cls, number_value):
        try:
            return cls._RULES_ITOA[number_value]
        except KeyError:
            msg = cls._RULE_ERROR_MSG.format(
                attempted=number_value,
                valid=list(six.iterkeys(cls._RULES_ITOA))
            )
            raise RuleError(msg)

    def parse(self, content, is_source=False):
        """
        Parses the content, extracts translatable strings into a stringset,
        replaces them with hashes and returns a tuple of the template with the
        stringset

        Typically this is done in the following way:

        * Use a library or your own code to segment (deserialize) the content
          into translatable entities.
        * Choose a key to uniquely identify the entity.
        * Create an ``OpenString`` object representing the entity.
        * Create a hash to replace the original content with.
        * Create a stringset with the content.
        * Use library or own code to serialize stringset back into a template.
        """

        raise NotImplementedError('Abstract method')  # pragma: no cover

    def compile(self, template, stringset, **kwargs):
        """
        Parses the template, finds the hashes, replaces them with strings from
        the stringset and returns the compiled file. If a hash in the template
        isn't found in the stringset, it's a good practice to remove the whole
        string section surrounding it

        Typically this is done in the following way:

        * Use a library or own code to segment (deserialize) the template into
          translatable entities, as if assuming that the hashes are the
          translatable entities.
        * Make sure the hash matches the first string in the stringset.
        * Replace the hash with the string.
        * Use library or own code to serialize stringset back into a compiled
          file.

        You can safely assume that the stringset will have strings in the
        correct order for the above process and thus you will probably be able
        to perform the whole compilation in a single pass.
        """

        raise NotImplementedError('Abstract method')  # pragma: no cover class OpenString(object):
    """
    This class will abstract away the business of generating hashes out of your
    strings and will serve as a place to get translations from when compiling.
    Several OpenStrings in our process define a *Stringset*, which is simply a
    python list of OpenStrings. To create an OpenString, you need 2 arguments:

    * The 'key'

      Something in your source file that uniquely identifies the section that
      the source string originated from. It might be helpful for your
      compiler to use something that appears in the same form in language
      files as well.

    * The 'string' or 'plural forms of the string':

      If the file format you're working with does not support plural forms,
      or if the string in question is not pluralized, you can just supply the
      string itself as the second argument. If you string is pluralized
      however, you have to supply all plural forms in a dictionary with the
      rule numbers as keys. For example::

          OpenString("UNREAD MESSAGES",
                     {1: "You have %s unread message",
                      5: "You have %s unread messages"})

    * There are a number of optional keyword arguments to `OpenString`:

      context, order, character_limit, occurrences, developer_comment, flags,
      fuzzy, obsolete

    Their main purpose is to provide context to the translators so that they
    can achieve higher quality. Two of them however, though optional, are
    highly recommended:

    * Context

      This is also taken into account when producing the hash, so if you
      can't ensure that your keys aren't unique within the source file, you
      can still get away with ensuring that the `(key, context)` pair is.

    * Order

      If you provide an order (integer), Transifex will save it in the
      database and then, when you try to compile a template against a
      stringset fetched from Transifex, it will already be ordered, even if
      it contains translations. This can allow you to optimize the
      compilation process as the order that the hashes appear in the template
      will be the same as the order of strings in the stringset.

      Another valuable outcome is that the order will be preserved when the
      strings are shown to translators which can provide context and thus
      improve translation quality.

    Once you have created an OpenString, you can get it's hash using the
    `template_replacement` property
    """

    DEFAULTS = {
        'context': "",
        'order': None,
        'character_limit': None,
        'occurrences': None,
        'developer_comment': "",
        'flags': "",
        'fuzzy': False,
        'obsolete': False,
        'tags': None
    }

    def __init__(self, key, string_or_strings, **kwargs):
        self.key = key
        if isinstance(string_or_strings, dict):
            self.pluralized = len(string_or_strings) > 1
            self._strings = {
                key: value
                for key, value in six.iteritems(string_or_strings)
            }
        else:
            self.pluralized = False
            self._strings = {5: string_or_strings}

        for key, value in six.iteritems(self.DEFAULTS):
            setattr(self, key, kwargs.get(key, value))

        if 'pluralized' in kwargs:
            self.pluralized = kwargs['pluralized']

        self._string_hash = None

    def __hash__(self):
        return hash((self.key, self.context, self._strings[5]))

    def __repr__(self):
        return next(
            (
                self._strings.get(i) for i in six.moves.xrange(5, -1, -1)
                if self._strings.get(i)
            ), 'Invalid string'
        ).encode('utf-8')

    @property
    def string(self):
        if self.pluralized:
            return self._strings
        else:
            return self._strings[5]

    @property
    def strings(self):
        return self._strings

    def _get_string_hash(self):
        keys = [self.key, self.context or '']
        return md5(':'.join(keys).encode('utf-8')).hexdigest()

    @property
    def string_hash(self):
        if self._string_hash is None:
            self._string_hash = self._get_string_hash()
        return self._string_hash

    @property
    def template_replacement(self):
        suffix = 'pl' if self.pluralized else 'tr'
        return "{hash}_{suffix}".format(hash=self.string_hash, suffix=suffix) class Transcriber(object):
    r"""
        This class helps with creating a template from an imported file or
        compile an output file from a template.

        **Main functionality**

        This class will help with both creating a template from an imported
        file and with compiling a file from a template. It provides functions
        for copying text. It depends on 3 things, the source content
        (self.source), the target content (self.destination) which initially
        will contain an empty string and a pointer (self.ptr) which will
        indicate which parts of 'source' have already been copied to
        'destination' (and will be initialized to 0).

        Transcriber detects and remembers the newline type (DOS, ``'\r\n'`` or
        UNIX ``'\n'``) of 'source'. It then converts 'source' to UNIX-like
        newlines and works on this. When returning the destination, the initial
        newline type will be used. Because 'source' is being potentially
        edited, it's a good idea to save Transcriber's source back on top of
        the original one:

            >>> def parse(self, source):
            ...     self.transcriber = Transcriber(source)
            ...     self.source = self.transcriber.source
            ...     # ...

        The main methods provided are demonstrated below::

            >>> transcriber = Transcriber(source)

            source:      <string name="foo">hello world</string>
            ptr:         ^ (0)
            destination: []

            >>> transcriber.copy_until(source.index('>') + 1)

            source:      <string name="foo">hello world</string>
            ptr:                            ^
            destination: ['<string name="foo">']

            >>> transcriber.add("aee8cc2abd5abd5a87cd784be_tr")

            source:      <string name="foo">hello world</string>
            ptr:                            ^
            destination: ['<string name="foo">', 'ee8cc2abd5abd5a87cd784be_tr']

            >>> transcriber.skip(len("hello world"))

            source:      <string name="foo">hello world</string>
            ptr:                                       ^
            destination: ['<string name="foo">', 'ee8cc2abd5abd5a87cd784be_tr']

            >>> transcriber.copy_until(source.index("</string>") +
            ...                        len("</string>"))

            source:      <string name="foo">hello world</string>
            ptr:                                                ^
            destination: ['<string name="foo">', 'ee8cc2abd5abd5a87cd784be_tr',
                          '</string>']

            >>> print transcriber.get_destination()

            <string name="foo">aee8cc2abd5abd5a87cd784be_tr</string>
    """

    class SectionStart:
        pass

    class SectionEnd:
        pass

    def __init__(self, source):
        self.source = source
        self.destination = []
        self.ptr = 0

        self.newline_count = 0

        # Handle newlines
        self.newline_type = find_newline_type(self.source)
        if self.newline_type == 'DOS':
            self.source = force_newline_type(self.source, 'UNIX')

    def copy(self, offset):
        chunk = self.source[self.ptr:self.ptr + offset]
        self.destination.append(chunk)
        self.ptr += offset

        self.newline_count += chunk.count('\n')

    def copy_until(self, end):
        chunk = self.source[self.ptr:end]
        self.destination.append(chunk)
        self.ptr = end

        self.newline_count += chunk.count('\n')

    def copy_to_end(self):
        self.copy_until(len(self.source))

    def add(self, text):
        self.destination.append(text)

    def skip(self, offset):
        chunk = self.source[self.ptr:self.ptr + offset]
        self.newline_count += chunk.count('\n')

        self.ptr += offset

    def skip_until(self, end):
        chunk = self.source[self.ptr:end]
        self.newline_count += chunk.count('\n')

        self.ptr = end

    def mark_section_start(self):
        self.destination.append(self.SectionStart)

    def mark_section_end(self):
        self.destination.append(self.SectionEnd)

    def remove_section(self, place=0):
        """
        You can mark sections in the target file and optionally remove them.
        Insert the section-start and section-end bookmarks wherever you want to
        mark a section. Then you can remove a section with `remove_section()`.
        For example::

            >>> transcriber = Transcriber(source)

            source:      <keep><remove>
            ptr:         ^ (0)
            destination: []

            >>> start = 0

            >>> transcriber.mark_section_start()
            >>> transcriber.copy_until(start + 1)  # copy until first '<'
            >>> string = source[start + 1:source.index('>', start)]
            >>> transcriber.add("asdf")  # add the hash
            >>> transcriber.skip(len(string))
            >>> transcriber.copy_until(source.index('>', start) + 1)
            >>> transcriber.mark_section_end()

            source:      <keep><remove>
            ptr:               ^
            destination: [SectionStart, '<', 'asdf', '>', SectionEnd]

            >>> if string == "remove":
            ...     transcriber.remove_section()

            (nothing happens)

            >>> start = source.index('>') + 1

            >>> # Same deal as before, mostly
            >>> transcriber.mark_section_start()
            >>> transcriber.copy_until(start + 1)  # copy until second '<'
            >>> string = source[start + 1:source.index('>', start)]
            >>> transcriber.add("fdsa")  # add the hash
            >>> transcriber.skip(len(string))
            >>> transcriber.copy_until(source.index('>', start) + 1)
            >>> transcriber.mark_section_end()

            source:      <keep><remove>
            ptr:                       ^
            destination: [SectionStart, '<', 'asdf', '>', SectionEnd,
                          SectionStart, '<', 'fdsa', '>', SectionEnd]

            >>> if string == "remove":
            ...     transcriber.remove_section()

            source:      <keep><remove>
            ptr:                       ^
            destination: [SectionStart,  '<', 'asdf', '>',  SectionEnd,
                          None        , None, None  , None, None      ]

            (The last section was replaced with Nones)

            Now, when you try to get the result with `get_destination()`, the
            Nones, SectionStarts and SectionEnds will be ommited:

            >>> transcriber.get_destination()

            <asdf>
        """
        section_start_position = self._find_last_section_start(place)
        try:
            section_end_position = self.destination.index(
                self.SectionEnd, section_start_position
            )
        except ValueError:
            section_end_position = len(self.destination) - 1
        for i in six.moves.xrange(section_start_position,
                                  section_end_position + 1):
            self.destination[i] = None

    def _find_last_section_start(self, place=0):
        count = place
        for i, segment in enumerate(self.destination[::-1], start=1):
            if segment == self.SectionStart:
                if count == 0:
                    return len(self.destination) - i
                else:
                    count -= 1

    @property
    def line_number(self):
        r"""
        The transcriber remembers how many newlines it has went over on the
        source, both when copying and skipping content. This allows you to
        pinpoint the line-number a parse-error has occured. For example::

            source:
                first line
                second line
                third line with error
                fourth line

            >>> transcriber = Transcriber(source)
            >>> for line in source.split("\n"):
            >>>     if "error" not in line:
            >>>         # include the newline too
            >>>         transcriber.copy(len(line) + 1)
            >>>     else:
            >>>         raise ParseError(
            >>>             "Error on line {line_no}: '{line}'".format(
            >>>                 line_no=transcriber.line_number,
            >>>                 line=line
            >>>             )
            >>>         )

            This will raise a::

            >>> ParseError("Error on line 3: 'third line with error'")
        """
        return self.newline_count + 1

    def get_destination(self, enforce_newline_type=None):
        return "".join([self.edit_newlines(chunk, enforce_newline_type)
                        for chunk in self.destination
                        if chunk not in (self.SectionStart, self.SectionEnd,
                                         None)])

    def edit_newlines(self, chunk, enforce_newline_type=None):
        r"""
        This is the part that renders the newlines to their correct type when
        returning the final result. You have the option to enforce the newline
        type if you want to.

            >>> source = "hello\r\nworld"
            >>> t = Transcriber(source)
            >>> t.source

            >>> "hello\nworld"

            >>> source = trascriber.source
            >>> # Work as if source was UNIX-type
            >>> t.copy_until(source.index('\n') + 1)  # include the '\n'
            >>> t.add("fellas")
            >>> t.get_destination()

            >>> "hello\r\nfellas"  # <- it remembered newline type from source

            >>> t.get_destination(enforce_newline_type="UNIX")

            >>> "hello\nfellas"
        """

        if ((enforce_newline_type is None and self.newline_type == "DOS") or
                enforce_newline_type == "DOS"):
            return force_newline_type(chunk, 'DOS')
        else:
            return chunk class OrderedCompilerMixin(object):
    SPACE_PAT = r'^\s*$'

    def compile(self, template, stringset, **kwargs):
        # Fix regex encoding
        space_pattern = re.compile(ensure_unicode(self.SPACE_PAT))

        # assume stringset is ordered within the template
        transcriber = Transcriber(template)
        template = transcriber.source

        for string in stringset:
            hash_position = template.index(string.template_replacement)
            if not string.pluralized:
                transcriber.copy_until(hash_position)
                transcriber.add(string.string)
                transcriber.skip(len(string.template_replacement))
            else:
                # if the hash is on its own on a line with only spaces, we have
                # to remember it's indent
                indent_length = template[hash_position::-1].index('\n') - 1
                indent = template[hash_position - indent_length:hash_position]
                tail_length = template[
                    hash_position + len(string.template_replacement):
                ].index('\n')
                tail = template[
                    hash_position + len(string.template_replacement):
                    hash_position + len(string.template_replacement) +
                    tail_length
                ]
                if (space_pattern.search(indent) and
                        space_pattern.search(tail)):
                    transcriber.copy_until(hash_position - indent_length)
                    for rule, value in six.iteritems(string.string):
                        transcriber.add(
                            indent + self.plural_template.format(
                                rule=self.RULES_ITOA[rule], string=value
                            ) + tail + '\n'
                        )
                    transcriber.skip(indent_length +
                                     len(string.template_replacement) +
                                     tail_length + 1)
                else:
                    # string is not on its own, simply replace hash with all
                    # plural forms
                    transcriber.copy_until(hash_position)
                    for rule, value in six.iteritems(string.string):
                        transcriber.add(self.plural_template.format(
                            rule=self.RULES_ITOA[rule], string=value
                        ))
                    transcriber.skip(len(string.template_replacement))

        transcriber.copy_until(len(template))
        compiled = transcriber.get_destination()

        return compiled def find_newline_type(content):
    if isinstance(content, six.text_type):
        NEWLINE = "\n"
        CARRIAGE_RETURN = "\r"
    else:
        NEWLINE = b"\n"
        CARRIAGE_RETURN = b"\r"

    try:
        first_newline_pos = content.index(NEWLINE)
    except ValueError:
        return 'UNIX'
    else:
        if (first_newline_pos > 0 and
                content[first_newline_pos - 1] == CARRIAGE_RETURN):
            return 'DOS'
        else:
            return 'UNIX' def force_newline_type(content, newline_type):
    if isinstance(content, six.text_type):
        NEWLINE = "\n"
        CARRIAGE_RETURN_NEWLINE = "\r\n"
    else:
        NEWLINE = b"\n"
        CARRIAGE_RETURN_NEWLINE = b"\r\n"

    new_content = content.replace(CARRIAGE_RETURN_NEWLINE, NEWLINE)
    if newline_type == 'DOS':
        new_content = new_content.replace(NEWLINE, CARRIAGE_RETURN_NEWLINE)
    return new_content
import re
import six
from mistune import Markdown
from yaml.reader import Reader
from openformats.formats.github_markdown import TxBlockLexer, string_handler
from openformats.formats.yaml import YamlHandler
from openformats.utils.compat import ensure_unicode
from ..handlers import Handler
from ..strings import OpenString
from ..transcribers import Transcriber
from ..utils.compilers import OrderedCompilerMixin
from ..utils.newlines import find_newline_type, force_newline_type
from __future__ import absolute_import, unicode_literals







class GithubMarkdownHandlerV2(OrderedCompilerMixin, Handler):
    name = "Github_Markdown_v2"
    extension = "md"
    EXTRACTS_RAW = False

    BACKSLASH = '\\'
    DOUBLE_QUOTES = '"'
    SINGLE_QUOTE = u"'"
    NEWLINE = '\n'
    COLON = ':'
    ASTERISK = '*'
    AMPERSAND = '&'
    DASH = '-'
    HASHTAG = '#'
    AT_SIGN = '@'  # reserved character in the YAML spec
    BRACKET_LEFT = '['
    BRACKET_RIGHT = ']'

    # A string that looks like '\u0008'
    ESCAPED_UNICODE = re.compile(r'\\u[a-fA-F0-9]{4}')
    # ... or '\x2D'
    ESCAPED_UNICODE_HEX = re.compile(r'\\x[a-fA-F0-9]{2}')

    def compile(self, template, stringset, **kwargs):
        # assume stringset is ordered within the template
        transcriber = Transcriber(template)
        template = transcriber.source

        for openstring in stringset:
            tr_string = openstring.string
            if self._is_yaml_string(openstring):
                self._escape_invalid_chars(openstring)
                tr_string = self._transform_yaml_string(openstring)

            hash_position = template.index(openstring.template_replacement)
            transcriber.copy_until(hash_position)
            transcriber.add(tr_string)
            transcriber.skip(len(openstring.template_replacement))

        transcriber.copy_until(len(template))
        compiled = transcriber.get_destination()

        return compiled

    def parse(self, content, **kwargs):
        newline_type = find_newline_type(content)
        if newline_type == 'DOS':
            content = force_newline_type(content, 'UNIX')

        # mistune expands tabs to 4 spaces and trims trailing spaces, so we
        # need to do the same in order to be able to match the substrings
        template = content.expandtabs(4)
        pattern = re.compile(ensure_unicode(r'^ +$'), re.M)
        content = pattern.sub('', template)

        stringset = []

        yml_header = re.match(
            ensure_unicode(r'^(---\s+)([\s\S]*?[^`]\s*)(\n---\s+)(?!-)'),
            content
        )
        yaml_header_content = ''
        yaml_stringset = []
        yaml_template = ''
        seperator = ''

        if yml_header:
            yaml_header_content = ''.join(yml_header.group(1, 2))
            seperator = yml_header.group(3)
            md_content = content[len(yaml_header_content + seperator):]
            yaml_template, yaml_stringset = YamlHandler().parse(
                yaml_header_content
            )
            for openstring in yaml_stringset:
                self._unescape_non_printable(openstring)
        else:
            md_content = content

        md_template = md_content

        block = TxBlockLexer()
        markdown = Markdown(block=block)

        # Making sure stringset is empty because of recursive inside `markdown`
        block.md_stringset = []

        # Command that populates block.stringset var
        markdown(md_content)

        stringset.extend(yaml_stringset)
        order = len(stringset)
        curr_pos = 0
        for string in block.md_stringset:

            string = string_handler(string, md_template)