ciseau/wiki_markup_processing.py
def to_raw_text(text, keep_whitespace=False, normalize_ascii=True):
    """
    A generator to convert raw text segments, with xml, and other
    non-textual content to a list of words without any markup.
    Additionally dates are replaced by `7777` for normalization.

    Arguments
    ---------
       text: str, input text to tokenize, strip of markup.
       keep_whitespace : bool, should the output retain the
          whitespace of the input (so that char offsets in the
          output correspond to those in the input).

    Returns
    -------
        generator<list<list<str>>>, a generator for sentences, with
            within each sentence a list of the words separated.
    """
    out = text
    out = _remove_urls(text)
    out = _remove_mvar(out)
    out = _remove_squiggly_bracket(out)
    out = _remove_table(out)
    out = _remove_brackets(out)
    out = remove_remaining_double_brackets(out)
    out = remove_markup(out)
    out = remove_wikipedia_link.sub(anchor_replacer, out)
    out = remove_bullets_nbsps.sub(empty_space, out)
    out = remove_dates(out)
    out = remove_math_sections(out)
    out = remove_html(out)
    out = sent_tokenize(out, keep_whitespace, normalize_ascii)
    return out
def to_raw_text(text, keep_whitespace=False, normalize_ascii=True):
    out = text
    out = _remove_urls(text)
    out = _remove_mvar(out)
    out = _remove_squiggly_bracket(out)
    out = _remove_table(out)
    out = _remove_brackets(out)
    out = remove_remaining_double_brackets(out)
    out = remove_markup(out)
    out = remove_wikipedia_link.sub(anchor_replacer, out)
    out = remove_bullets_nbsps.sub(empty_space, out)
    out = remove_dates(out)
    out = remove_math_sections(out)
    out = remove_html(out)
    out = sent_tokenize(out, keep_whitespace, normalize_ascii)
    return out
def to_raw_text ( text , keep_whitespace = False , normalize_ascii = True ) : out = text out = _remove_urls ( text ) out = _remove_mvar ( out ) out = _remove_squiggly_bracket ( out ) out = _remove_table ( out ) out = _remove_brackets ( out ) out = remove_remaining_double_brackets ( out ) out = remove_markup ( out ) out = remove_wikipedia_link . sub ( anchor_replacer , out ) out = remove_bullets_nbsps . sub ( empty_space , out ) out = remove_dates ( out ) out = remove_math_sections ( out ) out = remove_html ( out ) out = sent_tokenize ( out , keep_whitespace , normalize_ascii ) return out
A generator to convert raw text segments, with xml, and other
    non-textual content to a list of words without any markup.
    Additionally dates are replaced by `7777` for normalization.

    Arguments
    ---------
       text: str, input text to tokenize, strip of markup.
       keep_whitespace : bool, should the output retain the
          whitespace of the input (so that char offsets in the
          output correspond to those in the input).

    Returns
    -------
        generator<list<list<str>>>, a generator for sentences, with
            within each sentence a list of the words separated.
A generator to convert raw text segments with xml and other non - textual content to a list of words without any markup . Additionally dates are replaced by 7777 for normalization .