r""" Naively tags lines from manuscript with: code, natural, heading, etc.

    Returns:
        list of tuples  [(tag, line), ...]

    >>> ' '.join(sorted(VALID_TAGS))
    'anchor attribute blank_line block_header caption code code_end code_header code_start
     comment comment_end comment_start image_link latexmath latexmath_end latexmath_start
     natural natural_asside natural_asside_end natural_asside_start natural_end
     natural_heading1 natural_heading2 natural_heading3 natural_heading4 natural_heading5
     natural_quote natural_quote_end natural_quote_start
     natural_sidenote natural_sidenote_end natural_sidenote_start natural_start'
    >>> list(tag_lines('|= Title| :chapter: 0|Hello|cruel world|==Heading Level 2| \t| [source,bash]|====|$ grep this|====|'\
    ...                .split('|')))
    [('blank_line', ''), ('natural_heading1', '= Title'), ('attribute', ' :chapter: 0'),
     ('natural', 'Hello'), ('natural', 'cruel world'), ('natural_heading2', '==Heading Level 2'),
     ('blank_line', ' \t'), ('code_header', ' [source,bash]'),
     ('code_start', '===='), ('code', '$ grep this'), ('code_end', '===='), ('blank_line', '')]

def tag_lines(lines, include_tags=None):
    r""" Naively tags lines from manuscript with: code, natural, heading, etc.

    Returns:
        list of tuples  [(tag, line), ...]

    >>> ' '.join(sorted(VALID_TAGS))
    'anchor attribute blank_line block_header caption code code_end code_header code_start
     comment comment_end comment_start image_link latexmath latexmath_end latexmath_start
     natural natural_asside natural_asside_end natural_asside_start natural_end
     natural_heading1 natural_heading2 natural_heading3 natural_heading4 natural_heading5
     natural_quote natural_quote_end natural_quote_start
     natural_sidenote natural_sidenote_end natural_sidenote_start natural_start'
    >>> list(tag_lines('|= Title| :chapter: 0|Hello|cruel world|==Heading Level 2| \t| [source,bash]|====|$ grep this|====|'\
    ...                .split('|')))
    [('blank_line', ''), ('natural_heading1', '= Title'), ('attribute', ' :chapter: 0'),
     ('natural', 'Hello'), ('natural', 'cruel world'), ('natural_heading2', '==Heading Level 2'),
     ('blank_line', ' \t'), ('code_header', ' [source,bash]'),
     ('code_start', '===='), ('code', '$ grep this'), ('code_end', '===='), ('blank_line', '')]
    """
    current_block_type = None
    block_terminator = None
    tag = ''
    tagged_lines = []
    for idx, line in enumerate(lines):
        normalized_line = line.lower().strip().replace(" ", "")
        # [source,...] with or without any following "----" block delimiter
        # TODO: make this a regex that classifies among the different types (source, glossary, tip, etc)
        header_type = next((HEADER_TYPES[i] for i in range(len(HEADER_TYPES)) if
                            normalized_line.startswith('[') and normalized_line[1:].startswith(HEADER_TYPES[i][0])),
                           None)
        if header_type:
            current_block_type = header_type[1]
            tag = current_block_type + '_header'
            block_terminator = None
        elif normalized_line[:4] in BLOCK_HEADERS4:
            current_block_type = BLOCK_HEADERS4[normalized_line[:4]]
            tag = current_block_type + '_header'  # BLOCK_HEADERS[normalized_line]
            block_terminator = None
        elif (
                CRE_BLOCK_DELIMITER.match(normalized_line) and
                normalized_line[:2] in BLOCK_DELIMITERS):  # or (tag in set('caption anchor'.split()))):
            if (not idx or not current_block_type or not block_terminator):
                current_block_type = (current_block_type or BLOCK_DELIMITERS[normalized_line[:2]])
                tag = current_block_type + '_start'
                block_terminator = normalized_line
            elif block_terminator and line.rstrip() == block_terminator:
                tag = current_block_type + '_end'
                current_block_type = None
                block_terminator = None
            else:
                tag = current_block_type
        elif current_block_type and (line.rstrip() == block_terminator or 
                                     (not block_terminator and not normalized_line)):
            tag = current_block_type + '_end'
            current_block_type = None
            block_terminator = None
        elif current_block_type:
            tag = current_block_type
        elif not normalized_line:
            tag = 'blank_line'
        elif normalized_line.startswith(r'//'):
            tag = 'comment'
        elif normalized_line.startswith(r':'):
            tag = 'attribute'
        elif normalized_line.startswith('='):
            tag = 'natural_heading'
            tag += str(len([c for c in normalized_line[:6].split()[0] if c == '=']))
        elif normalized_line.startswith('.'):
            tag = 'caption'
        elif normalized_line.startswith('image:'):
            tag = 'image_link'
        elif normalized_line.startswith('[['):
            tag = 'anchor'
        else:
            tag = 'natural'
            current_block_type = None

        tagged_lines.append((tag, line))

    return filter_tagged_lines(tagged_lines, include_tags=include_tags)