# -*- coding: utf-8 -*-
"""
Utilities for handling unicode and other repetitive bits
"""
import six

def u(text, encoding='utf-8'):
    "Return unicode text, no matter what"

    if isinstance(text, six.binary_type):
        text = text.decode(encoding)

    # it's already unicode
    text = text.replace('\r\n', '\n')
    return text


# -*- coding: utf-8 -*-
"""
Python Frontmatter: Parse and manage posts with YAML frontmatter
"""
from __future__ import unicode_literals

import codecs
import re

import six

from .util import u
from .default_handlers import YAMLHandler, JSONHandler, TOMLHandler

__all__ = ['parse', 'load', 'loads', 'dump', 'dumps']

POST_TEMPLATE = """\
{start_delimiter}
{metadata}
{end_delimiter}

{content}
"""

# global handlers
handlers = {
    Handler.FM_BOUNDARY: Handler()
    for Handler in [YAMLHandler, JSONHandler, TOMLHandler]
    if Handler is not None
}

def detect_format(text, handlers):
    """
    Figure out which handler to use, based on metadata.
    Returns a handler instance or None.

    ``text`` should be unicode text about to be parsed.

    ``handlers`` is a dictionary where keys are opening delimiters
    and values are handler instances.
    """
    for pattern, handler in handlers.items():
        if pattern.match(text):
            return handler

    # nothing matched, give nothing back
    return None

def parse(text, encoding='utf-8', handler=None, **defaults):
    """
    Parse text with frontmatter, return metadata and content.
    Pass in optional metadata defaults as keyword args.

    If frontmatter is not found, returns an empty metadata dictionary
    (or defaults) and original text content.

    ::

        >>> with open('tests/hello-world.markdown') as f:
        ...     metadata, content = frontmatter.parse(f.read())
        >>> print(metadata['title'])
        Hello, world!

    """
    # ensure unicode first
    text = u(text, encoding).strip()

    # metadata starts with defaults
    metadata = defaults.copy()

    # this will only run if a handler hasn't been set higher up
    handler = handler or detect_format(text, handlers)
    if handler is None:
        return metadata, text

    # split on the delimiters
    try:
        fm, content = handler.split(text)
    except ValueError:
        # if we can't split, bail
        return metadata, text

    # parse, now that we have frontmatter
    fm = handler.load(fm)
    if isinstance(fm, dict):
        metadata.update(fm)

    return metadata, content.strip()

def load(fd, encoding='utf-8', handler=None, **defaults):
    """
    Load and parse a file-like object or filename,
    return a :py:class:`post <frontmatter.Post>`.

    ::

        >>> post = frontmatter.load('tests/hello-world.markdown')
        >>> with open('tests/hello-world.markdown') as f:
        ...     post = frontmatter.load(f)

    """
    if hasattr(fd, 'read'):
        text = fd.read()

    else:
        with codecs.open(fd, 'r', encoding) as f:
            text = f.read()

    handler = handler or detect_format(text, handlers)
    return loads(text, encoding, handler, **defaults)

def loads(text, encoding='utf-8', handler=None, **defaults):
    """
    Parse text (binary or unicode) and return a :py:class:`post <frontmatter.Post>`.

    ::

        >>> with open('tests/hello-world.markdown') as f:
        ...     post = frontmatter.loads(f.read())

    """
    text = u(text, encoding)
    handler = handler or detect_format(text, handlers)
    metadata, content = parse(text, encoding, handler, **defaults)
    return Post(content, handler, **metadata)

def dump(post, fd, encoding='utf-8', handler=None, **kwargs):
    """
    Serialize :py:class:`post <frontmatter.Post>` to a string and write to a file-like object.
    Text will be encoded on the way out (utf-8 by default).

    ::

        >>> from io import BytesIO
        >>> f = BytesIO()
        >>> frontmatter.dump(post, f)
        >>> print(f.getvalue())
        ---
        excerpt: tl;dr
        layout: post
        title: Hello, world!
        ---
        Well, hello there, world.

    """
    content = dumps(post, handler, **kwargs)
    if hasattr(fd, 'write'):
        fd.write(content.encode(encoding))

    else:
        with codecs.open(fd, 'w', encoding) as f:
            f.write(content)

def dumps(post, handler=None, **kwargs):
    """
    Serialize a :py:class:`post <frontmatter.Post>` to a string and return text.
    This always returns unicode text, which can then be encoded.

    Passing ``handler`` will change how metadata is turned into text. A handler
    passed as an argument will override ``post.handler``, with
    :py:class:`YAMLHandler <frontmatter.default_handlers.YAMLHandler>` used as
    a default.
    ::

        >>> print(frontmatter.dumps(post))
        ---
        excerpt: tl;dr
        layout: post
        title: Hello, world!
        ---
        Well, hello there, world.

    """
    if handler is None:
        handler = getattr(post, 'handler', None) or YAMLHandler()

    start_delimiter = kwargs.pop('start_delimiter', handler.START_DELIMITER)
    end_delimiter = kwargs.pop('end_delimiter', handler.END_DELIMITER)

    metadata = handler.export(post.metadata, **kwargs)

    return POST_TEMPLATE.format(
        metadata=metadata, content=post.content,
        start_delimiter=start_delimiter,
        end_delimiter=end_delimiter).strip()

class Post(object):
    """
    A post contains content and metadata from Front Matter. This is what gets
    returned by :py:func:`load <frontmatter.load>` and :py:func:`loads <frontmatter.loads>`.
    Passing this to :py:func:`dump <frontmatter.dump>` or :py:func:`dumps <frontmatter.dumps>`
    will turn it back into text.

    For convenience, metadata values are available as proxied item lookups.
    """
    def __init__(self, content, handler=None, **metadata):
        self.content = u(content)
        self.metadata = metadata
        self.handler = handler

    def __getitem__(self, name):
        "Get metadata key"
        return self.metadata[name]

    def __contains__(self, item):
        "Check metadata contains key"
        return item in self.metadata

    def __setitem__(self, name, value):
        "Set a metadata key"
        self.metadata[name] = value

    def __delitem__(self, name):
        "Delete a metadata key"
        del self.metadata[name]

    def __bytes__(self):
        return self.content.encode('utf-8')

    def __str__(self):
        if six.PY2:
            return self.__bytes__()
        return self.content

    def __unicode__(self):
        return self.content

    def get(self, key, default=None):
        "Get a key, fallback to default"
        return self.metadata.get(key, default)

    def keys(self):
        "Return metadata keys"
        return self.metadata.keys()

    def values(self):
        "Return metadata values"
        return self.metadata.values()

    def to_dict(self):
        "Post as a dict, for serializing"
        d = self.metadata.copy()
        d['content'] = self.content
        return d


# -*- coding: utf-8 -*-
"""
By default, ``frontmatter`` reads and writes YAML metadata. But maybe
you don't like YAML. Maybe enjoy writing metadata in JSON, or TOML, or
some other exotic markup not yet invented. For this, there are handlers.

This module includes handlers for YAML, JSON and TOML, as well as a
:py:class:`BaseHandler <frontmatter.default_handlers.BaseHandler>` that
outlines the basic API and can be subclassed to deal with new formats.

Handlers
--------

Handlers do most of the underlying work parsing and exporting front matter.
When you call :py:func:`frontmatter.loads <frontmatter.loads>`, frontmatter first needs to figure out the
best handler for the format you're using (YAML, JSON, TOML, etc), then call
methods to read or write metadata.

A handler needs to do four things:

- detect whether it can parse the given piece of text
- split front matter from content, returning both as a two-tuple
- parse front matter into a Python dictionary
- export a dictionary back into text

An example:

Calling :py:func:`frontmatter.load <frontmatter.load>` (or :py:func:`loads <frontmatter.loads>`)
with the ``handler`` argument tells frontmatter which handler to use.
The handler instance gets saved as an attribute on the returned post
object. By default, calling :py:func:`frontmatter.dumps <frontmatter.dumps>`
on the post will use the attached handler.

::

    >>> import frontmatter
    >>> from frontmatter.default_handlers import YAMLHandler, TOMLHandler
    >>> post = frontmatter.load('tests/hello-toml.markdown', handler=TOMLHandler())
    >>> post.handler #doctest: +ELLIPSIS
    <frontmatter.default_handlers.TOMLHandler object at 0x...>

    >>> print(frontmatter.dumps(post)) # doctest: +SKIP
    +++
    test = "tester"
    something = "else"
    author = "bob"
    +++
    <BLANKLINE>
    Title
    =====
    <BLANKLINE>
    title2
    ------
    <BLANKLINE>
    Hello.
    <BLANKLINE>
    Just need three dashes
    ---
    <BLANKLINE>
    And this shouldn't break.

Passing a new handler to :py:func:`frontmatter.dumps <frontmatter.dumps>`
(or :py:func:`dump <frontmatter.dump>`) changes the export format:

::

    >>> print(frontmatter.dumps(post, handler=YAMLHandler())) # doctest: +NORMALIZE_WHITESPACE
    ---
    author: bob
    something: else
    test: tester
    ---
    <BLANKLINE>
    Title
    =====
    <BLANKLINE>
    title2
    ------
    <BLANKLINE>
    Hello.
    <BLANKLINE>
    Just need three dashes
    ---
    <BLANKLINE>
    And this shouldn't break.

Changing the attached ``handler`` on a post has the same effect. Setting ``handler``
to ``None`` will default the post back to :py:class:`YAMLHandler <frontmatter.default_handlers.YAMLHandler>`.
These three variations will produce the same export:

::

    # set YAML format when dumping, but the old handler attached
    >>> t1 = frontmatter.dumps(post, handler=YAMLHandler())

    # set a new handler, changing all future exports
    >>> post.handler = YAMLHandler()
    >>> t2 = frontmatter.dumps(post)

    # remove handler, defaulting back to YAML
    >>> post.handler = None
    >>> t3 = frontmatter.dumps(post)

    >>> t1 == t2 == t3
    True

All handlers use the interface defined on ``BaseHandler``. Each handler needs to know how to:

- split metadata and content, based on a boundary pattern (``handler.split``)
- parse plain text metadata into a Python dictionary (``handler.load``)
- export a dictionary back into plain text (``handler.export``)

"""
from __future__ import unicode_literals

import json
import re
import yaml
try:
    from yaml import CSafeDumper as SafeDumper
    from yaml import CSafeLoader as SafeLoader
except ImportError:
    from yaml import SafeDumper
    from yaml import SafeLoader

try:
    import toml
except ImportError:
    toml = None

from .util import u

__all__ = ['BaseHandler', 'YAMLHandler', 'JSONHandler']

if toml:
    __all__.append('TOMLHandler')

class BaseHandler(object):
    """
    BaseHandler lays out all the steps to detecting, splitting, parsing and
    exporting front matter metadata.

    All default handlers are subclassed from BaseHandler.
    """

    FM_BOUNDARY = None
    START_DELIMITER = None
    END_DELIMITER = None

    def __init__(self, fm_boundary=None, start_delimiter=None, end_delimiter=None):
        self.FM_BOUNDARY = fm_boundary or self.FM_BOUNDARY
        self.START_DELIMITER = start_delimiter or self.START_DELIMITER
        self.END_DELIMITER = end_delimiter or self.END_DELIMITER

        if self.FM_BOUNDARY is None:
            raise NotImplementedError('No frontmatter boundary defined. '
                'Please set {}.FM_BOUNDARY to a regular expression'.format(self.__class__.__name__))

    def detect(self, text):
        """
        Decide whether this handler can parse the given ``text``,
        and return True or False.

        Note that this is *not* called when passing a handler instance to
        :py:func:`frontmatter.load <frontmatter.load>` or :py:func:`loads <frontmatter.loads>`.
        """
        if self.FM_BOUNDARY.match(text):
            return True
        return False

    def split(self, text):
        """
        Split text into frontmatter and content
        """
        _, fm, content = self.FM_BOUNDARY.split(text, 2)
        return fm, content

    def load(self, fm):
        """
        Parse frontmatter and return a dict
        """
        raise NotImplementedError

    def export(self, metadata, **kwargs):
        """
        Turn metadata back into text
        """
        raise NotImplementedError

class YAMLHandler(BaseHandler):
    """
    Load and export YAML metadata. By default, this handler uses YAML's
    "safe" mode, though it's possible to override that.
    """
    FM_BOUNDARY = re.compile(r'^-{3,}$', re.MULTILINE)
    START_DELIMITER = END_DELIMITER = "---"

    def load(self, fm, **kwargs):
        """
        Parse YAML front matter. This uses yaml.SafeLoader by default.
        """
        kwargs.setdefault('Loader', SafeLoader)
        return yaml.load(fm, **kwargs)

    def export(self, metadata, **kwargs):
        """
        Export metadata as YAML. This uses yaml.SafeDumper by default.
        """
        kwargs.setdefault('Dumper', SafeDumper)
        kwargs.setdefault('default_flow_style', False)
        kwargs.setdefault('allow_unicode', True)

        metadata = yaml.dump(metadata, **kwargs).strip()
        return u(metadata) # ensure unicode

class JSONHandler(BaseHandler):
    """
    Load and export JSON metadata.

    Note that changing ``START_DELIMITER`` or ``END_DELIMITER`` may break JSON parsing.
    """
    FM_BOUNDARY = re.compile(r'^(?:{|})$', re.MULTILINE)
    START_DELIMITER = ""
    END_DELIMITER = ""

    def split(self, text):
        _, fm, content = self.FM_BOUNDARY.split(text, 2)
        return "{" + fm + "}", content

    def load(self, fm, **kwargs):
        return json.loads(fm, **kwargs)

    def export(self, metadata, **kwargs):
        "Turn metadata into JSON"
        kwargs.setdefault('indent', 4)
        metadata = json.dumps(metadata, **kwargs)
        return u(metadata)

if toml:
    class TOMLHandler(BaseHandler):
        """
        Load and export TOML metadata.

        By default, split based on ``+++``.
        """
        FM_BOUNDARY = re.compile(r'^\+{3,}$', re.MULTILINE)
        START_DELIMITER = END_DELIMITER = "+++"

        def load(self, fm, **kwargs):
            return toml.loads(fm, **kwargs)

        def export(self, metadata, **kwargs):
            "Turn metadata into TOML"
            metadata = toml.dumps(metadata)
            return u(metadata)

else:
    TOMLHandler = None


"""Define the class for List objects."""

from typing import List, Union, Tuple, Dict, MutableSequence, Match

from regex import escape, fullmatch, MULTILINE

from ._wikitext import SubWikiText

SUBLIST_PATTERN = rb'(?>^(?<pattern>{pattern})[:;#*].*+(?>\n|\Z))*+'
LIST_PATTERN_FORMAT = (
    rb'(?<fullitem>'
    rb'^'
    rb'(?<pattern>{pattern})'
    rb'(?(?<=;\s*+)'
    # mark inline definition as an item
    rb'(?<item>[^:\n]*+)(?<fullitem>:(?<item>.*+))?+'
    rb'(?>\n|\Z)' + SUBLIST_PATTERN +
    rb'|'
    # non-definition
    rb'(?<item>.*+)'
    rb'(?>\n|\Z)' + SUBLIST_PATTERN +
    rb')'
    rb')++'
)

class WikiList(SubWikiText):

    """Class to represent ordered, unordered, and definition lists."""

    def __init__(
        self,
        string: Union[str, MutableSequence[str]],
        pattern: str,
        _match: Match = None,
        _type_to_spans: Dict[str, List[List[int]]] = None,
        _span: List[int] = None,
        _type: str = None,
    ) -> None:
        super().__init__(string, _type_to_spans, _span, _type)
        self.pattern = pattern
        if _match:
            self._match_cache = _match, self.string
        else:
            self._match_cache = fullmatch(
                LIST_PATTERN_FORMAT.replace(b'{pattern}', pattern.encode()),
                self._shadow,
                MULTILINE,
            ), self.string

    @property
    def _match(self):
        """Return the match object for the current list."""
        cache_match, cache_string = self._match_cache
        string = self.string
        if cache_string == string:
            return cache_match
        cache_match = fullmatch(
            LIST_PATTERN_FORMAT.replace(b'{pattern}', self.pattern.encode()),
            self._shadow,
            MULTILINE,
        )
        self._match_cache = cache_match, string
        return cache_match

    @property
    def items(self) -> List[str]:
        """Return items as a list of strings.

        Don't include sub-items and the start pattern.
        """
        items = []  # type: List[str]
        append = items.append
        string = self.string
        match = self._match
        ms = match.start()
        for s, e in match.spans('item'):
            append(string[s - ms:e - ms])
        return items

    @property
    def fullitems(self) -> List[str]:
        """Return list of item strings. Includes their start and sub-items."""
        fullitems = []  # type: List[str]
        append = fullitems.append
        string = self.string
        match = self._match
        ms = match.start()
        for s, e in match.spans('fullitem'):
            append(string[s - ms:e - ms])
        return fullitems

    @property
    def level(self) -> int:
        """Return level of nesting for the current list.

        Level is a one-based index, for example the level for `* a` will be 1.
        """
        return len(self._match['pattern'])

    def sublists(
        self, i: int = None, pattern: str = None
    ) -> List['WikiList']:
        """Return the Lists inside the item with the given index.

        :param i: The index if the item which its sub-lists are desired.
            The performance is likely to be better if `i` is None.

        :param pattern: The starting symbol for the desired sub-lists.
            The `pattern` of the current list will be automatically added
            as prefix.
            Although this parameter is optional, but specifying it can improve
            the performance.
        """
        patterns = (r'\#', r'\*', '[:;]') if pattern is None \
            else (pattern,)  # type: Tuple[str, ...]
        self_pattern = self.pattern
        lists = self.lists
        sublists = []  # type: List['WikiList']
        sublists_append = sublists.append
        if i is None:
            # Any sublist is acceptable
            for pattern in patterns:
                for lst in lists(self_pattern + pattern):
                    sublists_append(lst)
            return sublists
        # Only return sub-lists that are within the given item
        match = self._match
        fullitem_spans = match.spans('fullitem')
        ss = self._span[0]
        ms = match.start()
        s, e = fullitem_spans[i]
        e -= ms - ss
        s -= ms - ss
        for pattern in patterns:
            for lst in lists(self_pattern + pattern):
                # noinspection PyProtectedMember
                ls, le = lst._span
                if s < ls and le <= e:
                    sublists_append(lst)
        return sublists

    def convert(self, newstart: str) -> None:
        """Convert to another list type by replacing starting pattern."""
        match = self._match
        ms = match.start()
        for s, e in reversed(match.spans('pattern')):
            self[s - ms:e - ms] = newstart
        self.pattern = escape(newstart)


"""Define the ParserFunction class."""
from bisect import insort
from typing import List

import regex

from ._wikitext import SubWikiText
from ._argument import Argument
from ._wikilist import WikiList

BAR_SPLITS_FULLMATCH = regex.compile(
    rb'{{'
    rb'[^:|}]*+'  # name
    rb'(?<arg>:[^|}]*+)?+(?<arg>\|[^|}]*+)*+'
    rb'}}'
).fullmatch

class SubWikiTextWithArgs(SubWikiText):

    """Define common attributes for `Template` and `ParserFunction`."""

    _args_matcher = NotImplemented
    _first_arg_sep = 0

    @property
    def arguments(self) -> List[Argument]:
        """Parse template content. Create self.name and self.arguments."""
        shadow = self._shadow
        split_spans = self._args_matcher(shadow).spans('arg')
        if not split_spans:
            return []
        arguments = []
        arguments_append = arguments.append
        type_to_spans = self._type_to_spans
        ss, se = span = self._span
        type_ = id(span)
        lststr = self._lststr
        string = lststr[0]
        arg_spans = type_to_spans.setdefault(type_, [])
        span_tuple_to_span_get = {(s[0], s[1]): s for s in arg_spans}.get
        for arg_self_start, arg_self_end in split_spans:
            s, e = arg_span = [ss + arg_self_start, ss + arg_self_end]
            old_span = span_tuple_to_span_get((s, e))
            if old_span is None:
                insort(arg_spans, arg_span)
            else:
                arg_span = old_span
            arg = Argument(lststr, type_to_spans, arg_span, type_)
            arg._shadow_cache = (
                string[s:e], shadow[arg_self_start:arg_self_end])
            arguments_append(arg)
        return arguments

    def lists(self, pattern: str = None) -> List[WikiList]:
        """Return the lists in all arguments.

        For performance reasons it is usually preferred to get a specific
        Argument and use the `lists` method of that argument instead.
        """
        return [
            lst for arg in self.arguments for lst in arg.lists(pattern) if lst]

    @property
    def name(self) -> str:
        """Return template's name (includes whitespace)."""
        h = self._atomic_partition(self._first_arg_sep)[0]
        if len(h) == len(self.string):
            return h[2:-2]
        return h[2:]

    @name.setter
    def name(self, newname: str) -> None:
        """Set the new name."""
        self[2:2 + len(self.name)] = newname

class ParserFunction(SubWikiTextWithArgs):

    """Create a new ParserFunction object."""

    _args_matcher = BAR_SPLITS_FULLMATCH
    _first_arg_sep = 58


"""Utilities to override default configurations."""

from collections import defaultdict as _defaultdict
from typing import List as _List

def _plant_trie(strings: _List[str]) -> dict:
    """Create a Trie out of a list of words and return an atomic regex pattern.

    The corresponding Regex should match much faster than a simple Regex union.
    """
    # plant the trie
    trie = {}
    for string in strings:
        d = trie
        for char in string:
            d[char] = char in d and d[char] or {}
            d = d[char]
        d[''] = None  # EOS
    return trie

def _pattern(trie: dict) -> str:
    """Convert a trie to a regex pattern."""
    if '' in trie:
        if len(trie) == 1:
            return ''
        optional = True
        del trie['']
    else:
        optional = False

    subpattern_to_chars = _defaultdict(list)

    for char, sub_trie in trie.items():
        subpattern = _pattern(sub_trie)
        subpattern_to_chars[subpattern].append(char)

    alts = []
    for subpattern, chars in subpattern_to_chars.items():
        if len(chars) == 1:
            alts.append(chars[0] + subpattern)
        else:
            chars.sort(reverse=True)
            alts.append('[' + ''.join(chars) + ']' + subpattern)

    if len(alts) == 1:
        result = alts[0]
        if optional:
            if len(result) == 1:
                result += '?+'
            else:  # more than one character in alts[0]
                result = '(?:' + result + ')?+'
    else:
        alts.sort(reverse=True)
        result = '(?>' + '|'.join(alts) + ')'
        if optional:
            result += '?+'
    return result

def regex_pattern(words: _List[str]) -> str:
    """Convert words to a regex pattern that matches any of them."""
    return _pattern(_plant_trie(words))

# Contents of the some of the extension tags can be parsed as wikitext.
# For example, templates are valid inside the poem tag:
#    <poem>{{text|Hi!}}</poem>
# But not within math or source or ...
# for more information about the <categorytree> tag see:
# https://www.mediawiki.org/wiki/Extension:CategoryTree#
#    The_.7B.7B.23categorytree.7D.7D_parser_function
_parsable_tag_extensions = {
    'categorytree',
    'gallery',
    'imagemap',
    'includeonly',
    'indicator',
    'inputbox',
    'poem',
    'ref',
    'references',
    'section',
}
# For a complete list of extension tags on your wiki, see the
# "Parser extension tags" section at the end of [[Special:Version]].
# <templatedata> and <includeonly> were manually added to the  following lists.
# A simple trick to find out if a tag should be listed here or not is as
# follows:
# Create the {{text}} template in your wiki (You can copy the source code from
# English Wikipedia). Then save the following in a test page:
# {{text|0<tagname>1}}2</tagname>3}}4
# If the ending braces in the rendered result appear between 3 and 4, then
# `tagname` is not an extension tag (e.g. <small>). Otherwise, i.e. if those
# braces appear between 1 and 2 or completely don't show up, `tagname` is
# a tag extension (e.g.: <pre>).
_unparsable_tag_extensions = {
    'charinsert',
    'graph',
    'hiero',
    'math',
    'nowiki',
    'pre',
    'score',
    'source',
    'syntaxhighlight',
    'templatedata',
    'timeline',
}
_tag_extensions = _parsable_tag_extensions | _unparsable_tag_extensions

# Copied from DefaultSettings.php
# https://phabricator.wikimedia.org/source/mediawiki/browse/master/includes/DefaultSettings.php
# See also: https://www.mediawiki.org/wiki/Help:Links#External_links
_bare_external_link_schemes = {
    'bitcoin:', 'ftp://', 'ftps://', 'geo:', 'git://', 'gopher://', 'http://',
    'https://', 'irc://', 'ircs://', 'magnet:', 'mailto:', 'mms://', 'news:',
    'nntp://', 'redis://', 'sftp://', 'sip:', 'sips:', 'sms:', 'ssh://',
    'svn://', 'tel:', 'telnet://', 'urn:', 'worldwind://', 'xmpp:',  # '//'
}

_parser_functions = {
    'ARTICLEPAGENAME',
    'ARTICLEPAGENAMEE',
    'ARTICLESPACE',
    'ARTICLESPACEE',
    'BASEPAGENAME',
    'BASEPAGENAMEE',
    'CASCADINGSOURCES',
    'DEFAULTCATEGORYSORT',
    'DEFAULTSORT',
    'DEFAULTSORTKEY',
    'DISPLAYTITLE',
    'FULLPAGENAME',
    'FULLPAGENAMEE',
    'NAMESPACE',
    'NAMESPACEE',
    'NAMESPACENUMBER',
    'NUMBERINGROUP',
    'NUMBEROFACTIVEUSERS',
    'NUMBEROFADMINS',
    'NUMBEROFARTICLES',
    'NUMBEROFEDITS',
    'NUMBEROFFILES',
    'NUMBEROFPAGES',
    'NUMBEROFUSERS',
    'NUMBEROFVIEWS',
    'NUMINGROUP',
    'PAGEID',
    'PAGENAME',
    'PAGENAMEE',
    'PAGESINCAT',
    'PAGESINCATEGORY',
    'PAGESINNAMESPACE',
    'PAGESINNS',
    'PAGESIZE',
    'PROTECTIONEXPIRY',
    'PROTECTIONLEVEL',
    'REVISIONDAY',
    'REVISIONDAY2',
    'REVISIONID',
    'REVISIONMONTH',
    'REVISIONMONTH1',
    'REVISIONTIMESTAMP',
    'REVISIONUSER',
    'REVISIONYEAR',
    'ROOTPAGENAME',
    'ROOTPAGENAMEE',
    'SUBJECTPAGENAME',
    'SUBJECTPAGENAMEE',
    'SUBJECTSPACE',
    'SUBJECTSPACEE',
    'SUBPAGENAME',
    'SUBPAGENAMEE',
    'TALKPAGENAME',
    'TALKPAGENAMEE',
    'TALKSPACE',
    'TALKSPACEE',
    'anchorencode',
    'canonicalurl',
    'filepath',
    'formatnum',
    'fullurl',
    'gender',
    'grammar',
    'int',
    'lc',
    'lcfirst',
    'localurl',
    'msg',
    'msgnw',
    'ns',
    'nse',
    'padleft',
    'padright',
    'plural',
    'raw',
    'safesubst',
    'subst',
    'uc',
    'ucfirst',
    'urlencode',
}


"""Define the WikiText and SubWikiText classes."""

# Todo: consider using a tree structure (interval or segment tree).
# Todo: Consider using separate strings for each node.

from bisect import bisect, insort
from copy import deepcopy
from operator import attrgetter
from typing import (
    MutableSequence, Dict, List, Tuple, Union, Generator, Any, Optional)
from warnings import warn

from regex import VERBOSE, DOTALL, MULTILINE, IGNORECASE, search, finditer
from regex import compile as regex_compile
from wcwidth import wcswidth

# noinspection PyProtectedMember
from ._config import _tag_extensions
from ._spans import (
    parse_to_spans,
    INVALID_EXTLINK_CHARS,
    BARE_EXTERNAL_LINK,
    EXTERNAL_LINK_URL_TAIL)

# External links
# _config.regex_pattern(_config._bare_external_link_schemes | {'//'})
BRACKET_EXTERNAL_LINK_SCHEMES = (
    rb'(?>xmpp:|worldwind://|urn:|tel(?>net://|:)|s(?>vn://|sh://|ms:|ip(?>s:|'
    rb':)|ftp://)|redis://|n(?>ntp://|ews:)|m(?>ms://|a(?>ilto:|gnet:))|irc(?>'
    rb's://|://)|http(?>s://|://)|g(?>opher://|it://|eo:)|ftp(?>s://|://)|bitc'
    rb'oin:|//)')
BRACKET_EXTERNAL_LINK_URL = (
    BRACKET_EXTERNAL_LINK_SCHEMES + EXTERNAL_LINK_URL_TAIL)
BRACKET_EXTERNAL_LINK = (
    rb'\[' + BRACKET_EXTERNAL_LINK_URL + rb'[^\]\n]*+\]')
EXTERNAL_LINK = \
    rb'(?>' + BARE_EXTERNAL_LINK + rb'|' + BRACKET_EXTERNAL_LINK + rb')'
EXTERNAL_LINK_FINDITER = regex_compile(EXTERNAL_LINK, IGNORECASE).finditer
INVALID_EXT_CHARS_SUB = regex_compile(
    rb'[' + INVALID_EXTLINK_CHARS + rb'{}]'
).sub

# Sections
SECTION_HEADING = rb'^(?<equals>={1,6})[^\n]+?(?P=equals)[ \t]*+$'
SECTIONS_FULLMATCH = regex_compile(
    rb'(?<section>(?<equals>).*?)'  # lead section
    rb'(?<section>'
    + SECTION_HEADING +  # heading
    rb'.*?'  # section content
    rb')*',  # Todo: why can't be made possessive?
    DOTALL | MULTILINE | VERBOSE,
).fullmatch

# Tables
TABLE_FINDITER = regex_compile(
    rb"""
    # Table-start
    # Always starts on a new line with optional leading spaces or indentation.
    ^
    # Group the leading spaces or colons so that we can ignore them later.
    ([ :]*+)
    {\| # Table contents
    (?:
        # Any character, as long as it is not indicating another table-start
        (?!^\ *+\{\|).
    )*?
    # Table-end
    \n\s*+
    (?> \|} | \Z )
    """,
    DOTALL | MULTILINE | VERBOSE
).finditer

# Types which are detected by parse_to_spans
SPAN_PARSER_TYPES = {
    'Template', 'ParserFunction', 'WikiLink', 'Comment', 'Parameter',
    'ExtensionTag'}

WS = '\r\n\t '

class WikiText:

    """The WikiText class."""

    # In subclasses of WikiText _type is used as the key for _type_to_spans
    # Therefore: self._span can be found in self._type_to_spans[self._type].
    # The following acts as a default value.
    _type = 'WikiText'

    def __init__(
        self,
        string: Union[MutableSequence[str], str],
        _type_to_spans: Dict[str, List[List[int]]] = None,
    ) -> None:
        """Initialize the object.

        Set the initial values for self._lststr, self._type_to_spans.

        :param string: The string to be parsed or a list containing the string
            of the parent object.
        :param _type_to_spans: If the lststr is already parsed, pass its
            _type_to_spans property as _type_to_spans to avoid parsing it
            again.
        """
        if _type_to_spans is not None:
            self._type_to_spans = _type_to_spans
            self._lststr = string  # type: MutableSequence[str]
            return
        self._lststr = [string]
        span = self._span = [0, len(string)]
        byte_array = bytearray(string, 'ascii', 'replace')
        _type = self._type
        if _type not in SPAN_PARSER_TYPES:
            type_to_spans = self._type_to_spans = parse_to_spans(byte_array)
            type_to_spans[_type] = [span]
            self._shadow_cache = string, byte_array
        else:
            # In SPAN_PARSER_TYPES, we can't pass the original byte_array to
            # parser to generate the shadow because it will replace the whole
            # string with '_'. Also, we can't just modify it before passing
            # because the generated _type_to_spans will lack self._span.
            # As a workaround we can add the missed span after parsing.
            head = byte_array[:2]
            tail = byte_array[-2:]
            byte_array[-2:] = byte_array[:2] = b'__'
            type_to_spans = parse_to_spans(byte_array)
            self._shadow_cache = string, byte_array
            type_to_spans[_type].insert(0, span)
            self._type_to_spans = type_to_spans
            byte_array[:2] = head
            byte_array[-2:] = tail

    def __str__(self) -> str:
        """Return self-object as a string."""
        return self.string

    def __repr__(self) -> str:
        """Return the string representation of self."""
        return '{0}({1})'.format(type(self).__name__, repr(self.string))

    def __contains__(self, value: Union[str, 'WikiText']) -> bool:
        """Return True if parsed_wikitext is inside self. False otherwise.

        Also self and parsed_wikitext should belong to the same parsed
        wikitext object for this function to return True.
        """
        # Is it useful (and a good practice) to also accepts str inputs
        # and check if self.string contains it?
        if isinstance(value, str):
            return value in self.string
        # isinstance(value, WikiText)
        if self._lststr is not value._lststr:
            return False
        ps, pe = value._span
        ss, se = self._span
        if ss <= ps and se >= pe:
            return True
        return False

    def __len__(self):
        s, e = self._span
        return e - s

    def __getitem__(self, key: Union[slice, int]) -> str:
        """Return self.string[item]."""
        return self.string[key]

    def _check_index(self, key: Union[slice, int]) -> (int, int):
        """Return adjusted start and stop index as tuple.

        Used in  __setitem__ and __delitem__.
        """
        ss, se = self._span
        if isinstance(key, int):
            if key < 0:
                key += se - ss
                if key < 0:
                    raise IndexError('index out of range')
            elif key >= se - ss:
                raise IndexError('index out of range')
            start = ss + key
            return start, start + 1
        # isinstance(key, slice)
        if key.step is not None:
            raise NotImplementedError(
                'step is not implemented for string setter.')
        start, stop = key.start or 0, key.stop
        if start < 0:
            start += se - ss
            if start < 0:
                raise IndexError('start index out of range')
        if stop is None:
            stop = se - ss
        elif stop < 0:
            stop += se - ss
        if start > stop:
            raise IndexError(
                'stop index out of range or start is after the stop')
        return start + ss, stop + ss

    def __setitem__(
        self,
        key: Union[slice, int],
        value: str,
    ) -> None:
        """Set a new string for the given slice or character index.

        Use this method instead of calling `insert` and `del` consecutively.
        By doing so only one of the `_insert_update` and
        `_shrink_update` functions will be called and the performance
        will improve.
        """
        start, stop = self._check_index(key)
        # Update lststr
        lststr = self._lststr
        lststr0 = lststr[0]
        lststr[0] = lststr0[:start] + value + lststr0[stop:]
        # Set the length of all subspans to zero because
        # they are all being replaced.
        self._close_subspans(start, stop)
        # Update the other spans according to the new length.
        len_change = len(value) + start - stop
        if len_change > 0:
            self._insert_update(start, len_change)
        elif len_change < 0:
            self._shrink_update(
                rmstart=stop + len_change,  # new stop
                rmstop=stop)  # old stop
        # Add the newly added spans contained in the value.
        type_to_spans = self._type_to_spans
        for type_, spans in parse_to_spans(
            bytearray(value, 'ascii', 'replace')
        ).items():
            for s, e in spans:
                insort(type_to_spans[type_], [s + start, e + start])

    def __delitem__(self, key: Union[slice, int]) -> None:
        """Remove the specified range or character from self.string.

        Note: If an operation involves both insertion and deletion, it'll be
        safer to use the `insert` function first. Otherwise there is a
        possibility of insertion into the wrong spans.
        """
        start, stop = self._check_index(key)
        lststr = self._lststr
        lststr0 = lststr[0]
        # Update lststr
        lststr[0] = lststr0[:start] + lststr0[stop:]
        # Update spans
        self._shrink_update(start, stop)

    # Todo: def __add__(self, other) and __radd__(self, other)

    def insert(self, index: int, string: str) -> None:
        """Insert the given string before the specified index.

        This method has the same effect as ``self[index:index] = string``;
        it only avoids some condition checks as it rules out the possibility
        of the key being an slice, or the need to shrink any of the sub-spans.

        If parse is False, don't parse the inserted string.
        """
        ss, se = self._span
        lststr = self._lststr
        lststr0 = lststr[0]
        if index < 0:
            index += se - ss
            if index < 0:
                index = 0
        elif index > se - ss:  # Note that it is not >=. Index can be new.
            index = se - ss
        index += ss
        # Update lststr
        lststr[0] = lststr0[:index] + string + lststr0[index:]
        string_len = len(string)
        # Update spans
        self._insert_update(
            index=index,
            length=string_len)
        # Remember newly added spans by the string.
        type_to_spans = self._type_to_spans
        for type_, spans in parse_to_spans(
            bytearray(string, 'ascii', 'replace')
        ).items():
            for s, e in spans:
                insort(type_to_spans[type_], [index + s, index + e])

    @property
    def span(self) -> tuple:
        """Return the span of self relative to the start of the root node."""
        return tuple(self._span)

    @property
    def string(self) -> str:
        """Return str(self)."""
        start, end = self._span
        return self._lststr[0][start:end]

    @string.setter
    def string(self, newstring: str) -> None:
        """Set a new string for this object. Note the old data will be lost."""
        self[:] = newstring

    @string.deleter
    def string(self) -> None:
        """Set a new string for this object. Note the old data will be lost."""
        del self[:]

    def _atomic_partition(self, char: int) -> Tuple[str, str, str]:
        """Partition self.string where `char`'s not in atomic sub-spans."""
        s, e = self._span
        index = self._shadow.find(char)
        if index == -1:
            return self._lststr[0][s:e], '', ''
        lststr0 = self._lststr[0]
        return lststr0[s:s + index], chr(char), lststr0[s + index + 1:e]

    def _subspans(self, type_: str) -> List[List[int]]:
        """Return all the sub-span including self._span."""
        return self._type_to_spans[type_]

    def _close_subspans(self, start: int, stop: int) -> None:
        """Close all sub-spans of (start, stop)."""
        ss, se = self._span
        for spans in self._type_to_spans.values():
            b = bisect(spans, [start])
            for i, (s, e) in enumerate(spans[b:bisect(spans, [stop], b)]):
                if e <= stop:
                    if ss != s or se != e:
                        spans.pop(i + b)[:] = -1, -1
                        b -= 1

    def _shrink_update(self, rmstart: int, rmstop: int) -> None:
        """Update self._type_to_spans according to the removed span.

        Warning: If an operation involves both _shrink_update and
        _insert_update, you might wanna consider doing the
        _insert_update before the _shrink_update as this function
        can cause data loss in self._type_to_spans.
        """
        # Note: The following algorithm won't work correctly if spans
        # are not sorted.
        # Note: No span should be removed from _type_to_spans.
        for spans in self._type_to_spans.values():
            i = len(spans) - 1
            while i >= 0:
                s, e = span = spans[i]
                if rmstop <= s:
                    # rmstart <= rmstop <= s <= e
                    rmlength = rmstop - rmstart
                    span[:] = s - rmlength, e - rmlength
                    i -= 1
                    continue
                break
            else:
                continue
            while True:
                if rmstart <= s:
                    if rmstop < e:
                        # rmstart < s <= rmstop < e
                        span[:] = rmstart, e + rmstart - rmstop
                        i -= 1
                        if i < 0:
                            break
                        s, e = span = spans[i]
                        continue
                    # rmstart <= s <= e < rmstop
                    spans.pop(i)[:] = -1, -1
                    i -= 1
                    if i < 0:
                        break
                    s, e = span = spans[i]
                    continue
                break
            while i >= 0:
                if e <= rmstart:
                    # s <= e <= rmstart <= rmstop
                    i -= 1
                    if i < 0:
                        break
                    s, e = span = spans[i]
                    continue
                # s <= rmstart <= rmstop <= e
                span[1] -= rmstop - rmstart
                i -= 1
                if i < 0:
                    break
                s, e = span = spans[i]
                continue

    def _insert_update(self, index: int, length: int) -> None:
        """Update self._type_to_spans according to the added length."""
        ss, se = self._span
        for spans in self._type_to_spans.values():
            for span in spans:
                if index < span[1] or span[1] == index == se:
                    span[1] += length
                    # index is before s, or at s but not on self_span
                    if index < span[0] or span[0] == index != ss:
                        span[0] += length

    @property
    def nesting_level(self) -> int:
        """Return the nesting level of self.

        The minimum nesting_level is 0. Being part of any Template or
        ParserFunction increases the level by one.
        """
        ss, se = self._span
        level = 0
        type_to_spans = self._type_to_spans
        for type_ in ('Template', 'ParserFunction'):
            spans = type_to_spans[type_]
            for s, e in spans[:bisect(spans, [ss + 1])]:
                if se <= e:
                    level += 1
        return level

    @property
    def _shadow(self) -> bytearray:
        """Return a copy of self.string with specific sub-spans replaced.

        Comments blocks are replaced by spaces. Other sub-spans are replaced
        by underscores.

        The replaced sub-spans are: (
            'Template', 'WikiLink', 'ParserFunction', 'ExtensionTag',
            'Comment',
        )

        This function is called upon extracting tables or extracting the data
        inside them.
        """
        ss, se = self._span
        string = self._lststr[0][ss:se]
        cached_string, shadow = getattr(
            self, '_shadow_cache', (None, None))
        if cached_string == string:
            return shadow
        # In the old method the existing spans were used to create the shadow.
        # But it was slow because there can be thousands of spans and iterating
        # over them to find the relevant sub-spans could take a significant
        # amount of time. The new method tries to parse the self.string which
        # is usually much more faster because there are usually far less
        # sub-spans for individual objects.
        shadow = bytearray(string, 'ascii', 'replace')
        if self._type in SPAN_PARSER_TYPES:
            head = shadow[:2]
            tail = shadow[-2:]
            shadow[:2] = shadow[-2:] = b'__'
            parse_to_spans(shadow)
            shadow[:2] = head
            shadow[-2:] = tail
        else:
            parse_to_spans(shadow)
        self._shadow_cache = string, shadow
        return shadow

    @property
    def _ext_link_shadow(self):
        """Replace the invalid chars of SPAN_PARSER_TYPES with b'_'.

        For comments, all characters are replaced, but for ('Template',
        'ParserFunction', 'Parameter') only invalid characters are replaced.
        """
        ss, se = self._span
        string = self._lststr[0][ss:se]
        byte_array = bytearray(string, 'ascii', 'replace')
        subspans = self._subspans
        for type_ in 'Template', 'ParserFunction', 'Parameter':
            for s, e in subspans(type_):
                byte_array[s:e] = b'  ' + INVALID_EXT_CHARS_SUB(
                    b' ', byte_array[s + 2:e - 2]) + b'  '
        for s, e in subspans('Comment'):
            byte_array[s:e] = (e - s) * b'_'
        return byte_array

    def _pp_type_to_spans(self) -> Dict[str, List[List[int]]]:
        """Create the arguments for the parse function used in pformat method.

        Only return sub-spans and change the them to fit the new scope, i.e
        self.string.
        """
        ss, se = self._span
        if ss == 0 and se == len(self._lststr[0]):
            return deepcopy(self._type_to_spans)
        return {
            type_: [
                [s - ss, e - ss] for s, e in spans[bisect(spans, [ss]):]
                if e <= se
            ] for type_, spans in self._type_to_spans.items()}

    def pprint(self, indent: str = '    ', remove_comments=False):
        """Deprecated, use self.pformat instead."""
        warn(
            'pprint method is deprecated, use pformat instead.',
            DeprecationWarning,
        )
        return self.pformat(indent, remove_comments)

    def pformat(self, indent: str = '    ', remove_comments=False) -> str:
        """Return a pretty-print of self.string as string.

        Try to organize templates and parser functions by indenting, aligning
        at the equal signs, and adding space where appropriate.

        Note that this function will not mutate self.
        """
        ws = WS
        # Do not try to do inplace pformat. It will overwrite on some spans.
        string = self.string
        parsed = WikiText([string], self._pp_type_to_spans())
        # Since _type_to_spans arg of WikiText has been used, parsed._span
        # is not set yet.
        span = [0, len(string)]
        parsed._span = span
        parsed._type_to_spans['WikiText'] = [span]
        if remove_comments:
            for c in parsed.comments:
                del c[:]
        else:
            # Only remove comments that contain whitespace.
            for c in parsed.comments:
                if not c.contents.strip(ws):
                    del c[:]
        # First remove all current spacings.
        for template in reversed(parsed.templates):
            stripped_tl_name = template.name.strip(ws)
            template.name = (
                ' ' + stripped_tl_name + ' '
                if stripped_tl_name[0] == '{' else stripped_tl_name
            )
            args = template.arguments
            if not args:
                continue
            if ':' in stripped_tl_name:
                # Don't use False because we don't know for sure.
                not_a_parser_function = None
            else:
                not_a_parser_function = True
            # Required for alignment
            arg_stripped_names = [a.name.strip(ws) for a in args]
            arg_positionalities = [a.positional for a in args]
            arg_name_lengths = [
                wcswidth(n.replace('لا', '?'))
                if not p else 0
                for n, p in zip(arg_stripped_names, arg_positionalities)
            ]
            max_name_len = max(arg_name_lengths)
            # Format template.name.
            level = template.nesting_level
            newline_indent = '\n' + indent * level
            template.name += newline_indent
            if level == 1:
                last_comment_indent = '<!--\n-->'
            else:
                last_comment_indent = '<!--\n' + indent * (level - 2) + ' -->'
            # Special formatting for the last argument.
            last_arg = args.pop()
            last_is_positional = arg_positionalities.pop()
            last_value = last_arg.value
            last_stripped_value = last_value.strip(ws)
            if last_is_positional and last_value != last_stripped_value:
                stop_conversion = True
                if not last_value.endswith('\n' + indent * (level - 1)):
                    last_arg.value = last_value + last_comment_indent
            elif not_a_parser_function:
                stop_conversion = False
                last_arg.name = (
                    ' ' + arg_stripped_names.pop() + ' ' +
                    ' ' * (max_name_len - arg_name_lengths.pop()))
                last_arg.value = (
                    ' ' + last_stripped_value + '\n' + indent * (level - 1))
            elif last_is_positional:
                # (last_value == last_stripped_value
                # and not_a_parser_function is not True)
                stop_conversion = True
                # Can't strip or adjust the position of the value
                # because this could be a positional argument in a template.
                last_arg.value = last_value + last_comment_indent
            else:
                stop_conversion = True
                # This is either a parser function or a keyword
                # argument in a template. In both cases the name
                # can be lstripped and the value can be rstripped.
                last_arg.name = ' ' + last_arg.name.lstrip(ws)
                if not last_value.endswith('\n' + indent * (level - 1)):
                    last_arg.value = (
                        last_value.rstrip(ws) + ' ' + last_comment_indent)
            if not args:
                continue
            comment_indent = '<!--\n' + indent * (level - 1) + ' -->'
            for arg, stripped_name, positional, arg_name_len in zip(
                reversed(args),
                reversed(arg_stripped_names),
                reversed(arg_positionalities),
                reversed(arg_name_lengths),
            ):
                value = arg.value
                stripped_value = value.strip(ws)
                # Positional arguments of templates are sensitive to
                # whitespace. See:
                # https://meta.wikimedia.org/wiki/Help:Newlines_and_spaces
                if stop_conversion:
                    if not value.endswith(newline_indent):
                        arg.value += comment_indent
                elif positional and value != stripped_value:
                        stop_conversion = True
                        if not value.endswith(newline_indent):
                            arg.value += comment_indent
                elif not_a_parser_function:
                    arg.name = (
                        ' ' + stripped_name + ' ' +
                        ' ' * (max_name_len - arg_name_len))
                    arg.value = ' ' + stripped_value + newline_indent
        i = 0
        functions = parsed.parser_functions
        while i < len(functions):
            func = functions[i]
            i += 1
            name = func.name
            ls_name = name.lstrip(ws)
            lws = len(name) - len(ls_name)
            if lws:
                del func[2:lws + 2]
            if ls_name.lower() in ('#tag', '#invoke', ''):
                # The 2nd argument of `tag` parser function is an exception
                # and cannot be stripped.
                # So in `{{#tag:tagname|arg1|...}}`, no whitespace should be
                # added/removed to/from arg1.
                # See: [[mw:Help:Extension:ParserFunctions#Miscellaneous]]
                # All args of #invoke are also whitespace-sensitive.
                # Todo: Instead use comments to indent.
                continue
            args = func.arguments
            # Whitespace, including newlines, tabs, and spaces is stripped
            # from the beginning and end of all the parameters of
            # parser functions. See:
            # www.mediawiki.org/wiki/Help:Extension:ParserFunctions#
            #    Stripping_whitespace
            level = func.nesting_level
            short_indent = '\n' + indent * (level - 1)
            newline_indent = short_indent + indent
            if len(args) == 1:
                arg = args[0]
                # the first arg is both the first and last argument
                if arg.positional:
                    arg.value = (
                        newline_indent + arg.value.strip(ws) + short_indent)
                else:
                    # Note that we don't add spaces before and after the
                    # '=' in parser functions because it could be part of
                    # an ordinary string.
                    arg.name = newline_indent + arg.name.lstrip(ws)
                    arg.value = arg.value.rstrip(ws) + short_indent
                functions = parsed.parser_functions
                continue
            # Special formatting for the first argument
            arg = args[0]
            if arg.positional:
                arg.value = \
                    newline_indent + arg.value.strip(ws) + newline_indent
            else:
                arg.name = newline_indent + arg.name.lstrip(ws)
                arg.value = arg.value.rstrip(ws) + newline_indent
            # Formatting the middle arguments
            for arg in args[1:-1]:
                if arg.positional:
                    arg.value = ' ' + arg.value.strip(ws) + newline_indent
                else:
                    arg.name = ' ' + arg.name.lstrip(ws)
                    arg.value = arg.value.rstrip(ws) + newline_indent
            # Special formatting for the last argument
            arg = args[-1]
            if arg.positional:
                arg.value = ' ' + arg.value.strip(ws) + short_indent
            else:
                arg.name = ' ' + arg.name.lstrip(ws)
                arg.value = arg.value.rstrip(ws) + short_indent
            functions = parsed.parser_functions
        return parsed.string

    @property
    def parameters(self) -> List['Parameter']:
        """Return a list of parameter objects."""
        _lststr = self._lststr
        _type_to_spans = self._type_to_spans
        return [
            Parameter(_lststr, _type_to_spans, span, 'Parameter')
            for span in self._subspans('Parameter')]

    @property
    def parser_functions(self) -> List['ParserFunction']:
        """Return a list of parser function objects."""
        _lststr = self._lststr
        _type_to_spans = self._type_to_spans
        return [
            ParserFunction(_lststr, _type_to_spans, span, 'ParserFunction')
            for span in self._subspans('ParserFunction')]

    @property
    def templates(self) -> List['Template']:
        """Return a list of templates as template objects."""
        _lststr = self._lststr
        _type_to_spans = self._type_to_spans
        return [
            Template(_lststr, _type_to_spans, span, 'Template')
            for span in self._subspans('Template')]

    @property
    def wikilinks(self) -> List['WikiLink']:
        """Return a list of wikilink objects."""
        _lststr = self._lststr
        _type_to_spans = self._type_to_spans
        return [
            WikiLink(_lststr, _type_to_spans, span, 'WikiLink')
            for span in self._subspans('WikiLink')]

    @property
    def comments(self) -> List['Comment']:
        """Return a list of comment objects."""
        _lststr = self._lststr
        _type_to_spans = self._type_to_spans
        return [
            Comment(_lststr, _type_to_spans, span, 'Comment')
            for span in self._subspans('Comment')]

    @property
    def external_links(self) -> List['ExternalLink']:
        """Return a list of found external link objects.

        Note:
            Templates adjacent to external links are considered part of the
            link. In reality, this depends on the contents of the template:

            >>> WikiText(
            ...    'http://example.com{{dead link}}'
            ...).external_links[0].url
            'http://example.com{{dead link}}'

            >>> WikiText(
            ...    '[http://example.com{{space template}} text]'
            ...).external_links[0].url
            'http://example.com{{space template}}'
        """
        external_links = []  # type: List['ExternalLink']
        external_links_append = external_links.append
        type_to_spans = self._type_to_spans
        lststr = self._lststr
        ss, se = self._span
        spans = type_to_spans.setdefault('ExternalLink', [])
        if not spans:
            # All the added spans will be new.
            spans_append = spans.append
            for m in EXTERNAL_LINK_FINDITER(self._ext_link_shadow):
                s, e = m.span()
                span = [ss + s, ss + e]
                spans_append(span)
                external_links_append(
                    ExternalLink(lststr, type_to_spans, span, 'ExternalLink'))
            return external_links
        # There are already some ExternalLink spans. Use the already existing
        # ones when the detected span is one of those.
        span_tuple_to_span_get = {(s[0], s[1]): s for s in spans}.get
        for m in EXTERNAL_LINK_FINDITER(self._ext_link_shadow):
            s, e = m.span()
            span = s, e = [s + ss, e + ss]
            old_span = span_tuple_to_span_get((s, e))
            if old_span is None:
                insort(spans, span)
            else:
                span = old_span
            external_links_append(
                ExternalLink(lststr, type_to_spans, span, 'ExternalLink'))
        return external_links

    @property
    def sections(self) -> List['Section']:
        """Return a list of section in current wikitext.

        The first section will always be the lead section, even if it is an
        empty string.
        """
        sections = []  # type: List['Section']
        sections_append = sections.append
        type_to_spans = self._type_to_spans
        lststr = self._lststr
        ss, se = _span = self._span
        type_spans = type_to_spans.setdefault('Section', [])
        full_match = SECTIONS_FULLMATCH(self._shadow)
        section_spans = full_match.spans('section')
        levels = [len(eq) for eq in full_match.captures('equals')]
        if not type_spans:
            # All spans are new
            spans_append = type_spans.append
            for current_index, (current_level, (s, e)) in enumerate(
                zip(levels, section_spans), 1
            ):
                # Add text of the current_section to any parent section.
                # Note that section 0 is not a parent for any subsection.
                for section_index, section_level in enumerate(
                    levels[current_index:], current_index
                ):
                    if current_level and section_level > current_level:
                        e = section_spans[section_index][1]
                    else:
                        break
                span = [ss + s, ss + e]
                spans_append(span)
                sections_append(
                    Section(lststr, type_to_spans, span, 'Section'))
            return sections
        # There are already some spans. Instead of appending new spans
        # use them when the detected span already exists.
        span_tuple_to_span = {(s[0], s[1]): s for s in type_spans}.get
        for current_index, (current_level, (s, e)) in enumerate(
            zip(levels, section_spans), 1
        ):
            # Add text of the current_section to any parent section.
            # Note that section 0 is not a parent for any subsection.
            for section_index, section_level in enumerate(
                levels[current_index:], current_index
            ):
                if current_level and section_level > current_level:
                    e = section_spans[section_index][1]
                else:
                    break
            s, e = ss + s, ss + e
            old_span = span_tuple_to_span((s, e))
            if old_span is None:
                span = [s, e]
                insort(type_spans, span)
            else:
                span = old_span
            sections_append(Section(lststr, type_to_spans, span, 'Section'))
        return sections

    @property
    def tables(self) -> List['Table']:
        """Return a list of found table objects."""
        tables = []  # type: List['Table']
        tables_append = tables.append
        type_to_spans = self._type_to_spans
        lststr = self._lststr
        shadow = self._shadow[:]
        ss, se = self._span
        spans = type_to_spans.setdefault('Table', [])
        if not spans:
            # All the added spans will be new.
            m = True  # type: Any
            while m:
                m = False
                for m in TABLE_FINDITER(shadow):
                    ms, me = m.span()
                    # Ignore leading whitespace using len(m[1]).
                    span = [ss + ms + len(m[1]), ss + me]
                    spans.append(span)
                    tables_append(Table(lststr, type_to_spans, span, 'Table'))
                    shadow[ms:me] = b'_' * (me - ms)
            return tables
        # There are already exists some spans. Try to use the already existing
        # before appending new spans.
        span_tuple_to_span_get = {(s[0], s[1]): s for s in spans}.get
        m = True
        while m:
            m = False
            for m in TABLE_FINDITER(shadow):
                ms, me = m.span()
                # Ignore leading whitespace using len(m[1]).
                s, e = ss + ms + len(m[1]), ss + me
                old_span = span_tuple_to_span_get((s, e))
                if old_span is None:
                    span = [s, e]
                    insort(spans, span)
                else:
                    span = old_span
                tables_append(Table(lststr, type_to_spans, span, 'Table'))
                shadow[ms:me] = b'_' * (me - ms)
        return tables

    @property
    def _lists_shadow_ss(self) -> Tuple[bytearray, int]:
        """Return appropriate shadow and its offset to be used by `lists`."""
        return self._shadow, self._span[0]

    def lists(self, pattern: str = None) -> List['WikiList']:
        r"""Return a list of WikiList objects.

        :param pattern: The starting pattern for list items.
            Return all types of lists (ol, ul, and dl) if pattern is None.
            If pattern is not None, it will be passed to the regex engine,
            remember to escape the `*` character. Examples:

                - `\#` means top-level ordered lists
                - `\#\*` means unordred lists inside an ordered one
                - Currently definition lists are not well supported, but you
                    can use `[:;]` as their pattern.

            Tips and tricks:

                Be careful when using the following patterns as they will
                probably cause malfunction in the `sublists` method of the
                resultant List. (However don't worry about them if you are
                not going to use the `sublists` method.)

                - Use `\*+` as a pattern and nested unordered lists will be
                    treated as flat.
                - Use `\*\s*` as pattern to rtstrip `items` of the list.

                Although the pattern parameter is optional, but specifying it
                can improve the performance.
        """
        lists = []
        lists_append = lists.append
        lststr = self._lststr
        type_to_spans = self._type_to_spans
        spans = type_to_spans.setdefault('WikiList', [])
        span_tuple_to_span_get = {(s[0], s[1]): s for s in spans}.get
        shadow, ss = self._lists_shadow_ss
        for pattern in \
                (r'\#', r'\*', '[:;]') if pattern is None else (pattern,):
            for m in finditer(
                LIST_PATTERN_FORMAT.replace(b'{pattern}', pattern.encode()),
                shadow, MULTILINE
            ):
                ms, me = m.span()
                s, e = ss + ms, ss + me
                old_span = span_tuple_to_span_get((s, e))
                if old_span is None:
                    span = [s, e]
                    insort(spans, span)
                else:
                    span = old_span
                lists_append(WikiList(
                    lststr, pattern, m, type_to_spans, span, 'WikiList'))
        return lists

    def tags(self, name=None) -> List['Tag']:
        """Return all tags with the given name."""
        lststr = self._lststr
        type_to_spans = self._type_to_spans
        if name:
            if name in _tag_extensions:
                string = lststr[0]
                return [
                    Tag(lststr, type_to_spans, span, 'ExtensionTag')
                    for span in type_to_spans['ExtensionTag']
                    if string.startswith('<' + name, span[0])]
            tags = []  # type: List['Tag']
        else:
            # There is no name, add all extension tags. Before using shadow.
            tags = [
                Tag(lststr, type_to_spans, span, 'ExtensionTag')
                for span in type_to_spans['ExtensionTag']]
        tags_append = tags.append
        # Get the left-most start tag, match it to right-most end tag
        # and so on.
        ss = self._span[0]
        shadow = self._shadow
        if name:
            # There is a name but it is not in TAG_EXTENSIONS.
            reversed_start_matches = reversed([m for m in regex_compile(
                START_TAG_PATTERN.replace(
                    rb'{name}', rb'(?P<name>' + name.encode() + rb')')
            ).finditer(shadow)])
            end_search = regex_compile(END_TAG_PATTERN .replace(
                b'{name}', name.encode())).search
        else:
            reversed_start_matches = reversed(
                [m for m in START_TAG_FINDITER(shadow)])
        shadow_copy = shadow[:]
        spans = type_to_spans.setdefault('Tag', [])
        span_tuple_to_span_get = {(s[0], s[1]): s for s in spans}.get
        spans_append = spans.append
        for start_match in reversed_start_matches:
            if start_match['self_closing']:
                # Don't look for the end tag
                s, e = start_match.span()
                span = [ss + s, ss + e]
            else:
                # look for the end-tag
                if name:
                    # the end_search is already available
                    # noinspection PyUnboundLocalVariable
                    end_match = end_search(shadow_copy, start_match.end())
                else:
                    # build end_search according to start tag name
                    end_match = search(
                        END_TAG_PATTERN.replace(
                            b'{name}', start_match['name']),
                        shadow_copy)
                if end_match:
                    s, e = end_match.span()
                    shadow_copy[s:e] = b'_' * (e - s)
                    span = [ss + start_match.start(), ss + e]
                else:
                    # Assume start-only tag.
                    s, e = start_match.span()
                    span = [ss + s, ss + e]
            old_span = span_tuple_to_span_get((span[0], span[1]))
            if old_span is None:
                spans_append(span)
            else:
                span = old_span
            tags_append(Tag(lststr, type_to_spans, span, 'Tag'))
        return sorted(tags, key=attrgetter('_span'))

    @staticmethod
    def parent(type_: Optional[str] = None) -> None:
        """Return None (The parent of the root node is None)."""
        return None

    @staticmethod
    def ancestors(type_: Optional[str] = None) -> list:
        """Return [] (the root node has no ancestors)."""
        return []

class SubWikiText(WikiText):
    """Define a class to be inherited by some subclasses of WikiText.

    Allow to focus on a particular part of WikiText.
    """

    def __init__(
        self,
        string: Union[str, MutableSequence[str]],
        _type_to_spans: Optional[Dict[str, List[List[int]]]] = None,
        _span: Optional[List[int]] = None,
        _type: Optional[Union[str, int]] = None,
    ) -> None:
        """Initialize the object."""
        if _type is None:
            # assert _span is None
            # assert _type_to_spans is None
            self._type = _type = type(self).__name__
            super().__init__(string)
            if _type not in SPAN_PARSER_TYPES:
                span = [0, len(string)]
                self._type_to_spans[_type] = [span]
                self._span = span
        else:
            # assert _span is not None
            # assert _type_to_spans is not None
            self._type = _type
            super().__init__(string, _type_to_spans)
            self._span = _span

    def _subspans(self, _type: str) -> Generator[int, None, None]:
        """Yield all the sub-span indices excluding self._span."""
        ss, se = self._span
        spans = self._type_to_spans[_type]
        # Do not yield self._span by bisecting for s < ss.
        # The second bisect is an optimization and should be on [se + 1],
        # but empty spans are not desired thus [se] is used.
        b = bisect(spans, [ss])
        for span in spans[b:bisect(spans, [se], b)]:
            if span[1] <= se:
                yield span

    # noinspection PyProtectedMember
    def ancestors(self, type_: Optional[str] = None) -> List['WikiText']:
        """Return the ancestors of the current node.

        :param type_: the type of the desired ancestors as a string.
            Currently the following types are supported: {Template,
            ParserFunction, WikiLink, Comment, Parameter, ExtensionTag}.
            The default is None and means all the ancestors of any type above.
        """
        if type_ is None:
            types = SPAN_PARSER_TYPES
        else:
            types = type_,
        lststr = self._lststr
        type_to_spans = self._type_to_spans
        ss, se = self._span
        ancestors = []
        ancestors_append = ancestors.append
        for type_ in types:
            cls = globals()[type_]
            spans = type_to_spans[type_]
            for span in spans[:bisect(spans, [ss])]:
                if se < span[1]:
                    ancestors_append(cls(lststr, type_to_spans, span, type_))
        return sorted(ancestors, key=lambda i: ss - i._span[0])

    def parent(self, type_: Optional[str] = None) -> Optional['WikiText']:
        """Return the parent node of the current object.

        :param type_: the type of the desired parent object.
            Currently the following types are supported: {Template,
            ParserFunction, WikiLink, Comment, Parameter, ExtensionTag}.
            The default is None and means the first parent, of any type above.
        :return: parent WikiText object or None if no parent with the desired
            `type_` is found.
        """
        ancestors = self.ancestors(type_)
        if ancestors:
            return ancestors[0]
        return None

if __name__ == '__main__':
    # To make PyCharm happy! http://stackoverflow.com/questions/41524090
    from ._tag import (
        Tag, START_TAG_PATTERN, END_TAG_PATTERN, START_TAG_FINDITER)
    from ._parser_function import ParserFunction
    from ._template import Template
    from ._wikilink import WikiLink
    from ._comment import Comment
    from ._externallink import ExternalLink
    from ._section import Section
    from ._wikilist import WikiList, LIST_PATTERN_FORMAT
    from ._table import Table
    from ._parameter import Parameter


"""Define the Template class."""

from typing import List, Optional, TypeVar, Iterable, Dict, Tuple
from warnings import warn

from regex import compile as regex_compile, REVERSE

from ._argument import Argument
from ._parser_function import SubWikiTextWithArgs
from ._spans import COMMENT_PATTERN
from ._wikitext import WS

COMMENT_SUB = regex_compile(COMMENT_PATTERN).sub

BAR_SPLITS_FULLMATCH = regex_compile(
    rb'{{'
    rb'[^|}]*+'  # name
    rb'(?<arg>\|[^|}]*+)*+'
    rb'}}'
).fullmatch
STARTING_WS_MATCH = regex_compile(r'\s*+').match
ENDING_WS_MATCH = regex_compile(r'(?>\n[ \t]*)*+', REVERSE).match
SPACE_AFTER_SEARCH = regex_compile(r'\s*+(?=\|)').search

T = TypeVar('T')

class Template(SubWikiTextWithArgs):

    """Convert strings to Template objects.

    The string should start with {{ and end with }}.
    """

    _args_matcher = BAR_SPLITS_FULLMATCH
    _first_arg_sep = 124

    def normal_name(
        self,
        rm_namespaces=('Template',),
        capital_links=False,
        _code: str = None,
        *,
        code: str = None,
        capitalize=False
    ) -> str:
        """Return normal form of self.name.

        - Remove comments.
        - Remove language code.
        - Remove namespace ("template:" or any of `localized_namespaces`.
        - Use space instead of underscore.
        - Remove consecutive spaces.
        - Use uppercase for the first letter if `capitalize`.
        - Remove #anchor.

        :param rm_namespaces: is used to provide additional localized
            namespaces for the template namespace. They will be removed from
            the result. Default is ('Template',).
        :param capitalize: If True, convert the first letter of the
            template's name to a capital letter. See
            [[mw:Manual:$wgCapitalLinks]] for more info.
        :param code: is the language code.
        :param capital_links: deprecated.
        :param _code: deprecated.

        Example:
            >>> Template(
            ...     '{{ eN : tEmPlAtE : <!-- c --> t_1 # b | a }}'
            ... ).normal_name(code='en')
            'T 1'
        """
        if capital_links:
            warn('`capital_links` argument is deprecated,'
                 ' use `capitalize` instead', DeprecationWarning)
            capitalize = capital_links
        if _code:
            warn('`positional_code` argument is deprecated,'
                 ' use `code` instead', DeprecationWarning)
            code = _code
        # Remove comments
        name = COMMENT_SUB('', self.name).strip(WS)
        # Remove code
        if code:
            head, sep, tail = name.partition(':')
            if not head and sep:
                name = tail.strip(' ')
                head, sep, tail = name.partition(':')
            if code.lower() == head.strip(' ').lower():
                name = tail.strip(' ')
        # Remove namespace
        head, sep, tail = name.partition(':')
        if not head and sep:
            name = tail.strip(' ')
            head, sep, tail = name.partition(':')
        if head:
            ns = head.strip(' ').lower()
            for namespace in rm_namespaces:
                if namespace.lower() == ns:
                    name = tail.strip(' ')
                    break
        # Use space instead of underscore
        name = name.replace('_', ' ')
        if capitalize:
            # Use uppercase for the first letter
            n0 = name[0]
            if n0.islower():
                name = n0.upper() + name[1:]
        # Remove #anchor
        name, sep, tail = name.partition('#')
        return ' '.join(name.split())

    def rm_first_of_dup_args(self) -> None:
        """Eliminate duplicate arguments by removing the first occurrences.

        Remove the first occurrences of duplicate arguments, regardless of
        their value. Result of the rendered wikitext should remain the same.
        Warning: Some meaningful data may be removed from wikitext.

        Also see `rm_dup_args_safe` function.
        """
        names = set()  # type: set
        for a in reversed(self.arguments):
            name = a.name.strip(WS)
            if name in names:
                del a[:len(a.string)]
            else:
                names.add(name)

    def rm_dup_args_safe(self, tag: str = None) -> None:
        """Remove duplicate arguments in a safe manner.

        Remove the duplicate arguments only in the following situations:
            1. Both arguments have the same name AND value. (Remove one of
                them.)
            2. Arguments have the same name and one of them is empty. (Remove
                the empty one.)

        Warning: Although this is considered to be safe and no meaningful data
            is removed from wikitext, but the result of the rendered wikitext
            may actually change if the second arg is empty and removed but
            the first had had a value.

        If `tag` is defined, it should be a string that will be appended to
        the value of the remaining duplicate arguments.

        Also see `rm_first_of_dup_args` function.
        """
        name_to_lastarg_vals = {} \
            # type: Dict[str, Tuple[Argument, List[str]]]
        # Removing positional args affects their name. By reversing the list
        # we avoid encountering those kind of args.
        for arg in reversed(self.arguments):
            name = arg.name.strip(WS)
            if arg.positional:
                # Value of keyword arguments is automatically stripped by MW.
                val = arg.value
            else:
                # But it's not OK to strip whitespace in positional arguments.
                val = arg.value.strip(WS)
            if name in name_to_lastarg_vals:
                # This is a duplicate argument.
                if not val:
                    # This duplicate argument is empty. It's safe to remove it.
                    del arg[0:len(arg.string)]
                else:
                    # Try to remove any of the detected duplicates of this
                    # that are empty or their value equals to this one.
                    lastarg, dup_vals = name_to_lastarg_vals[name]
                    if val in dup_vals:
                        del arg[0:len(arg.string)]
                    elif '' in dup_vals:
                        # This happens only if the last occurrence of name has
                        # been an empty string; other empty values will
                        # be removed as they are seen.
                        # In other words index of the empty argument in
                        # dup_vals is always 0.
                        del lastarg[0:len(lastarg.string)]
                        dup_vals.pop(0)
                    else:
                        # It was not possible to remove any of the duplicates.
                        dup_vals.append(val)
                        if tag:
                            arg.value += tag
            else:
                name_to_lastarg_vals[name] = (arg, [val])

    def set_arg(
        self, name: str,
        value: str,
        positional: bool = None,
        before: str = None,
        after: str = None,
        preserve_spacing: bool = True
    ) -> None:
        """Set the value for `name` argument. Add it if it doesn't exist.

        - Use `positional`, `before` and `after` keyword arguments only when
          adding a new argument.
        - If `before` is given, ignore `after`.
        - If neither `before` nor `after` are given and it's needed to add a
          new argument, then append the new argument to the end.
        - If `positional` is True, try to add the given value as a positional
          argument. Ignore `preserve_spacing` if positional is True.
          If it's None, do what seems more appropriate.
        """
        args = list(reversed(self.arguments))
        arg = get_arg(name, args)
        # Updating an existing argument.
        if arg:
            if positional:
                arg.positional = positional
            if preserve_spacing:
                val = arg.value
                arg.value = val.replace(val.strip(WS), value)
            else:
                arg.value = value
            return
        # Adding a new argument
        if not name and positional is None:
            positional = True
        # Calculate the whitespace needed before arg-name and after arg-value.
        if not positional and preserve_spacing and args:
            before_names = []
            name_lengths = []
            before_values = []
            after_values = []
            for arg in args:
                aname = arg.name
                name_len = len(aname)
                name_lengths.append(name_len)
                before_names.append(STARTING_WS_MATCH(aname)[0])
                arg_value = arg.value
                before_values.append(STARTING_WS_MATCH(arg_value)[0])
                after_values.append(ENDING_WS_MATCH(arg_value)[0])
            pre_name_ws_mode = mode(before_names)
            name_length_mode = mode(name_lengths)
            post_value_ws_mode = mode(
                [SPACE_AFTER_SEARCH(self.string)[0]] + after_values[1:]
            )
            pre_value_ws_mode = mode(before_values)
        else:
            preserve_spacing = False
        # Calculate the string that needs to be added to the Template.
        if positional:
            # Ignore preserve_spacing for positional args.
            addstring = '|' + value
        else:
            if preserve_spacing:
                # noinspection PyUnboundLocalVariable
                addstring = (
                    '|' + (pre_name_ws_mode + name.strip(WS)).
                    ljust(name_length_mode) +
                    '=' + pre_value_ws_mode + value + post_value_ws_mode
                )
            else:
                addstring = '|' + name + '=' + value
        # Place the addstring in the right position.
        if before:
            arg = get_arg(before, args)
            arg.insert(0, addstring)
        elif after:
            arg = get_arg(after, args)
            arg.insert(len(arg.string), addstring)
        else:
            if args and not positional:
                arg = args[0]
                arg_string = arg.string
                if preserve_spacing:
                    # Insert after the last argument.
                    # The addstring needs to be recalculated because we don't
                    # want to change the the whitespace before final braces.
                    # noinspection PyUnboundLocalVariable
                    arg[0:len(arg_string)] = (
                        arg.string.rstrip(WS) + post_value_ws_mode +
                        addstring.rstrip(WS) + after_values[0]
                    )
                else:
                    arg.insert(len(arg_string), addstring)
            else:
                # The template has no arguments or the new arg is
                # positional AND is to be added at the end of the template.
                self.insert(-2, addstring)

    def get_arg(self, name: str) -> Optional[Argument]:
        """Return the last argument with the given name.

        Return None if no argument with that name is found.
        """
        return get_arg(name, reversed(self.arguments))

    def has_arg(self, name: str, value: str = None) -> bool:
        """Return true if the is an arg named `name`.

        Also check equality of values if `value` is provided.

        Note: If you just need to get an argument and you want to LBYL, it's
            better to get_arg directly and then check if the returned value
            is None.
        """
        for arg in reversed(self.arguments):
            if arg.name.strip(WS) == name.strip(WS):
                if value:
                    if arg.positional:
                        if arg.value == value:
                            return True
                        return False
                    if arg.value.strip(WS) == value.strip(WS):
                        return True
                    return False
                return True
        return False

    def del_arg(self, name: str) -> None:
        """Delete all arguments with the given then."""
        for arg in reversed(self.arguments):
            if arg.name.strip(WS) == name.strip(WS):
                del arg[:]

def mode(list_: List[T]) -> T:
    """Return the most common item in the list.

    Return the first one if there are more than one most common items.

    Example:

    >>> mode([1,1,2,2,])
    1
    >>> mode([1,2,2])
    2
    >>> mode([])
    ...
    ValueError: max() arg is an empty sequence
    """
    return max(set(list_), key=list_.count)

def get_arg(name: str, args: Iterable[Argument]) -> Optional[Argument]:
    """Return the first argument in the args that has the given name.

    Return None if no such argument is found.

    As the computation of self.arguments is a little costly, this
    function was created so that other methods that have already computed
    the arguments use it instead of calling self.get_arg directly.
    """
    for arg in args:
        if arg.name.strip(WS) == name.strip(WS):
            return arg
    return None


"""
Misc utility functions related to crs formats and online services.
"""

try:
    import urllib.request as urllib2
except ImportError:
    import urllib2
import re

def build_crs_table(savepath):
    """
    Build crs table of all equivalent format variations by scraping spatialreference.org.
    Saves table as tab-delimited text file.
    NOTE: Might take a while.

    Arguments:

    - *savepath*: The absolute or relative filepath to which to save the crs table, including the ".txt" extension.
    """
    # create table
    outfile = open(savepath, "wb")

    # create fields
    fields = ["codetype", "code", "proj4", "ogcwkt", "esriwkt"]
    outfile.write("\t".join(fields) + "\n")

    # make table from url requests
    for codetype in ("epsg", "esri", "sr-org"):
        print(codetype)

        # collect existing proj list
        print("fetching list of available codes")
        codelist = []
        page = 1
        while True:
            try:
                link = 'http://spatialreference.org/ref/%s/?page=%s' %(codetype,page)
                html = urllib2.urlopen(link).read()
                codes = [match.groups()[0] for match in re.finditer(r'/ref/'+codetype+'/(\d+)', html) ]
                if not codes: break
                print("page",page)
                codelist.extend(codes)
                page += 1
            except:
                break

        print("fetching string formats for each projection")
        for i,code in enumerate(codelist):

            # check if code exists
            link = 'http://spatialreference.org/ref/%s/%s/' %(codetype,code)
            urllib2.urlopen(link)

            # collect each projection format in a table row
            row = [codetype, code]
            for resulttype in ("proj4", "ogcwkt", "esriwkt"):
                try:
                    link = 'http://spatialreference.org/ref/%s/%s/%s/' %(codetype,code,resulttype)
                    result = urllib2.urlopen(link).read()
                    row.append(result)
                except:
                    pass

            print("projection %i of %i added" %(i,len(codelist)) )
            outfile.write("\t".join(row) + "\n")

    # close the file
    outfile.close()

def crscode_to_string(codetype, code, format):
    """
    Lookup crscode on spatialreference.org and return in specified format.

    Arguments:

    - *codetype*: "epsg", "esri", or "sr-org".
    - *code*: The code.
    - *format*: The crs format of the returned string. One of "ogcwkt", "esriwkt", or "proj4", but also several others...

    Returns:

    - Crs string in the specified format.
    """
    link = 'http://spatialreference.org/ref/%s/%s/%s/' %(codetype,code,format)
    result = urllib2.urlopen(link).read()
    if not isinstance(result, str):
        result = result.decode()
    return result

##def crsstring_to_string(string, newformat):
##    """
##    Search unknown crs string for a match on spatialreference.org.
##    Not very reliable, the search engine does not properly lookup all text.
##    If string is correct there should only be one correct match.
##    Warning: Not finished yet.
##    """
##    link = 'http://spatialreference.org/ref/?search=%s' %string
##    searchresults = urllib2.urlopen(link).read()
##    # pick the first result
##    # ...regex...
##    # go to its url, with extension for the newformat
##    link = 'http://spatialreference.org/ref/%s/%s/%s/' %(codetype,code,newformat)
##    result = urllib2.urlopen(link).read()
##    return result


"""
Coordinate system (CS) classes of different types.
"""

from . import directions
from . import datums
from . import ellipsoids

#BASE
class CS:
    """
    Base class for all CS classes.
    Mostly just for basic type checking.
    """
    pass

#GEOGCS
class GeogCS(CS):
    """
    A geographic coordinate system where the coordinates are in the latitude-longitude space.

    Attributes:

        - **cs_type**: Type of CRS, always "Geographic".
        - **name**: An arbitrary name given to this geographic coordinate system, to represent its unique
                    configuration of datum, prime meridian, angular unit, and twin axes. The actual name
                    is just for human readability, and does not actually have any implication.
        - **datum**: A pycrs.elements.datums.Datum instance, representing the shape of the earth.
        - **prime_mer**: A pycrs.elements.parameters.PrimeMeridian instance, representing the prime meridian
                    coordinate where the longitude is considered to be 0.
        - **angunit**: A pycrs.elements.parameters.AngularUnit instance, representing the angular unit in which
                    coordinates are measured.
        - **twin_ax**: A pair of pycrs.elements.directions.North/South/East/West instances, one for each axis,
                    representing the compass direction in which each axis increases. Defaults to East and North.

    """
    ogc_wkt = "GEOGCS"
    esri_wkt = "GEOGCS"

    def __init__(self, name, datum, prime_mer, angunit, twin_ax=None):
        """
        Arguments:

        - **name**: An arbitrary name given to this geographic coordinate system, to represent its unique
                    configuration of datum, prime meridian, angular unit, and twin axes. The actual name
                    is just for human readability, and does not actually have any implication.
        - **datum**: A pycrs.elements.datums.Datum instance, representing the shape of the earth.
        - **prime_mer**: A pycrs.elements.parameters.PrimeMeridian instance, representing the prime meridian
                    coordinate where the longitude is considered to be 0.
        - **angunit**: A pycrs.elements.parameters.AngularUnit instance, representing the angular unit in which
                    coordinates are measured.
        - **twin_ax**: A pair of pycrs.elements.directions.North/South/East/West instances, one for each axis,
                    representing the compass direction in which each axis increases. Defaults to East and North.
        """
        self.cs_type = "Geographic"

        self.name = name
        self.datum = datum
        self.prime_mer = prime_mer
        self.angunit = angunit
        if twin_ax == None:
            # default axes
            twin_ax = directions.East(), directions.North()
        self.twin_ax = twin_ax

    def to_proj4(self, as_dict=False, toplevel=True):
        """
        Returns the CS as a proj4 formatted string or dict.

        Arguments:

        - **as_dict** (optional): If True, returns the proj4 string as a dict (defaults to False).
        - **toplevel** (optional): If True, treats this CS as the final toplevel CS and adds the necessary proj4 elements (defaults to True).
        """
        # dont parse axis to proj4, because in proj4, axis only applies to the cs, ie the projcs (not the geogcs, where wkt can specify with axis)
        # also proj4 cannot specify angular units
        if toplevel:
            string = "+proj=longlat %s %s +nodef" % (self.datum.to_proj4(), self.prime_mer.to_proj4())
        else:
            string = "%s %s" % (self.datum.to_proj4(), self.prime_mer.to_proj4())
        if as_dict:
            return dict([
                        entry.lstrip('+').split('=')
                        for entry in string.split()
                        if entry != "+no_defs"
                         ])
        else:
            return string

    def to_ogc_wkt(self):
        """
        Returns the CS as a OGC WKT formatted string.
        """
        return 'GEOGCS["%s", %s, %s, %s, AXIS["Lon", %s], AXIS["Lat", %s]]' % (self.name, self.datum.to_ogc_wkt(), self.prime_mer.to_ogc_wkt(), self.angunit.to_ogc_wkt(), self.twin_ax[0].ogc_wkt, self.twin_ax[1].ogc_wkt )

    def to_esri_wkt(self):
        """
        Returns the CS as a ESRI WKT formatted string.
        """
        return 'GEOGCS["%s", %s, %s, %s, AXIS["Lon", %s], AXIS["Lat", %s]]' % (self.name, self.datum.to_esri_wkt(), self.prime_mer.to_esri_wkt(), self.angunit.to_esri_wkt(), self.twin_ax[0].esri_wkt, self.twin_ax[1].esri_wkt )

#PROJCS
class ProjCS(CS):
    """
    A projected coordinate system where the coordinates are projected to euclidean x,y space.

    Attributes:

    - **cs_type**: Type of CRS, always "Projected".
    - **name**: Arbitrary name of the projected coordinate system.
    - **geogcs**: A pycrs.elements.cs.GeogCS instance.
    - **proj**: A pycrs.elements.projections.Projection instance.
    - **params**: A list of custom parameters from the pycrs.elements.parameters module.
    - **unit**: A pycrs.elements.parameters.Unit instance, representing the angular unit in which
                coordinates are measured.
    - **twin_ax**: A pair of pycrs.elements.directions.North/South/East/West instances, one for each axis,
                representing the compass direction in which each axis increases. Defaults to East and North.
    """

    ogc_wkt = "PROJCS"
    esri_wkt = "PROJCS"

    def __init__(self, name, geogcs, proj, params, unit, twin_ax=None):
        """
        Arguments:

        - **name**: Arbitrary name of the projected coordinate system.
        - **geogcs**: A pycrs.elements.cs.GeogCS instance.
        - **proj**: A pycrs.elements.projections.Projection instance.
        - **params**: A list of custom parameters from the pycrs.elements.parameters module.
        - **unit**: A pycrs.elements.parameters.Unit instance, representing the angular unit in which
                    coordinates are measured.
        - **twin_ax**: A pair of pycrs.elements.directions.North/South/East/West instances, one for each axis,
                    representing the compass direction in which each axis increases. Defaults to East and North.
        """
        self.cs_type = "Projected"
        self.name = name
        self.geogcs = geogcs
        self.proj = proj
        self.params = params
        self.unit = unit
        if twin_ax == None:
            # default axes
            twin_ax = directions.East(), directions.North()
        self.twin_ax = twin_ax

    def to_proj4(self, as_dict=False):
        """
        Returns the CS as a proj4 formatted string or dict.

        Arguments:

        - **as_dict** (optional): If True, returns the proj4 string as a dict (defaults to False).
        """
        string = "%s" % self.proj.to_proj4()
        string += " %s" % self.geogcs.to_proj4(toplevel=False)
        string += " " + " ".join(param.to_proj4() for param in self.params)
        string += " %s" % self.unit.to_proj4()
        string += " +axis=" + self.twin_ax[0].proj4 + self.twin_ax[1].proj4 + "u" # up set as default because only proj4 can set it I think...
        string += " +no_defs"

        if as_dict:
            return dict([
                        entry.lstrip('+').split('=')
                        for entry in string.split()
                        if entry != "+no_defs"
                         ])
        else:
            return string

    def to_ogc_wkt(self):
        """
        Returns the CS as a OGC WKT formatted string.
        """
        string = 'PROJCS["%s", %s, %s, ' % (self.name, self.geogcs.to_ogc_wkt(), self.proj.to_ogc_wkt() )
        string += ", ".join(param.to_ogc_wkt() for param in self.params)
        string += ', %s' % self.unit.to_ogc_wkt()
        string += ', AXIS["X", %s], AXIS["Y", %s]]' % (self.twin_ax[0].ogc_wkt, self.twin_ax[1].ogc_wkt )
        return string

    def to_esri_wkt(self):
        """
        Returns the CS as a ESRI WKT formatted string.
        """
        string = 'PROJCS["%s", %s, %s, ' % (self.name, self.geogcs.to_esri_wkt(), self.proj.to_esri_wkt() )
        string += ", ".join(param.to_esri_wkt() for param in self.params)
        string += ', %s' % self.unit.to_esri_wkt()
        string += ', AXIS["X", %s], AXIS["Y", %s]]' % (self.twin_ax[0].esri_wkt, self.twin_ax[1].esri_wkt )
        return string