streamcorpus_pipeline/_clean_html.py
def make_clean_html(raw, stream_item=None, encoding=None):
    '''Get a clean text representation of presumed HTML.

    Treat `raw` as though it is HTML, even if we have no idea what it
    really is, and attempt to get a properly formatted HTML document
    with all HTML-escaped characters converted to their unicode.

    This is called below by the `clean_html` transform stage, which
    interprets MIME-type.  If `character_encoding` is not provided,
    and `stream_item` is provided, then this falles back to
    :attr:`streamcorpus.StreamItem.body.encoding`.

    :param str raw: raw text to clean up
    :param stream_item: optional stream item with encoding metadata
    :type stream_item: :class:`streamcorpus.StreamItem`
    :returns: UTF-8-encoded byte string of cleaned HTML text
    :returntype: :class:`str`

    '''
    # Fix emails by protecting the <,> from HTML
    raw = fix_emails(raw)
    raw_decoded = nice_decode(raw, stream_item=stream_item, encoding=encoding)
    if raw_decoded is None:
        # give up on decoding it... maybe this should use force_unicode
        raw_decoded = raw

    # default attempt uses vanilla lxml.html
    try:
        root = lxml.html.document_fromstring(raw_decoded)
    except ValueError, exc:
        if 'with encoding declaration' in str(exc):
            root = lxml.html.document_fromstring(raw)
        else:
            raise

    # While we have the document parsed as a DOM, let's strip attributes.
    # (The HTML cleaner seems to only support whitelisting attributes.
    # As of now, we just want to blacklist a few.)
    lxml.etree.strip_attributes(root, 'class', 'id')

    # if that worked, then we will be able to generate a
    # valid HTML string
    fixed_html = lxml.html.tostring(root, encoding=unicode)

    # remove any ^M characters
    fixed_html = string.replace(fixed_html, '\r', ' ')

    # We drop utf8 characters that are above 0xFFFF as
    # Lingpipe seems to be doing the wrong thing with them.
    fixed_html = drop_invalid_and_upper_utf8_chars(fixed_html)

    # construct a Cleaner that removes any ``<script>`` tags,
    # Javascript, like an ``onclick`` attribute, comments, style
    # tags or attributes, ``<link>`` tags
    cleaner = lxml.html.clean.Cleaner(
        scripts=True, javascript=True,
        comments=True,
        # do not remove <html> <head> <title> etc
        page_structure=False,
        remove_tags=['base'],
        style=True, links=True)

    # now get the really sanitized HTML
    _clean_html = cleaner.clean_html(fixed_html)

    # generate pretty HTML in utf-8
    _clean_html = lxml.html.tostring(
        lxml.html.document_fromstring(_clean_html),
        method='html', encoding='utf-8',
        pretty_print=True,
        # include_meta_content_type=True
        )

    return uniform_html(_clean_html)
def make_clean_html(raw, stream_item=None, encoding=None):
    '''Get a clean text representation of presumed HTML.

    Treat `raw` as though it is HTML, even if we have no idea what it
    really is, and attempt to get a properly formatted HTML document
    with all HTML-escaped characters converted to their unicode.

    This is called below by the `clean_html` transform stage, which
    interprets MIME-type.  If `character_encoding` is not provided,
    and `stream_item` is provided, then this falles back to
    :attr:`streamcorpus.StreamItem.body.encoding`.

    :param str raw: raw text to clean up
    :param stream_item: optional stream item with encoding metadata
    :type stream_item: :class:`streamcorpus.StreamItem`
    :returns: UTF-8-encoded byte string of cleaned HTML text
    :returntype: :class:`str`

    '''
    # Fix emails by protecting the <,> from HTML
    raw = fix_emails(raw)
    raw_decoded = nice_decode(raw, stream_item=stream_item, encoding=encoding)
    if raw_decoded is None:
        # give up on decoding it... maybe this should use force_unicode
        raw_decoded = raw

    # default attempt uses vanilla lxml.html
    try:
        root = lxml.html.document_fromstring(raw_decoded)
    except ValueError, exc:
        if 'with encoding declaration' in str(exc):
            root = lxml.html.document_fromstring(raw)
        else:
            raise

    # While we have the document parsed as a DOM, let's strip attributes.
    # (The HTML cleaner seems to only support whitelisting attributes.
    # As of now, we just want to blacklist a few.)
    lxml.etree.strip_attributes(root, 'class', 'id')

    # if that worked, then we will be able to generate a
    # valid HTML string
    fixed_html = lxml.html.tostring(root, encoding=unicode)

    # remove any ^M characters
    fixed_html = string.replace(fixed_html, '\r', ' ')

    # We drop utf8 characters that are above 0xFFFF as
    # Lingpipe seems to be doing the wrong thing with them.
    fixed_html = drop_invalid_and_upper_utf8_chars(fixed_html)

    # construct a Cleaner that removes any ``<script>`` tags,
    # Javascript, like an ``onclick`` attribute, comments, style
    # tags or attributes, ``<link>`` tags
    cleaner = lxml.html.clean.Cleaner(
        scripts=True, javascript=True,
        comments=True,
        # do not remove <html> <head> <title> etc
        page_structure=False,
        remove_tags=['base'],
        style=True, links=True)

    # now get the really sanitized HTML
    _clean_html = cleaner.clean_html(fixed_html)

    # generate pretty HTML in utf-8
    _clean_html = lxml.html.tostring(
        lxml.html.document_fromstring(_clean_html),
        method='html', encoding='utf-8',
        pretty_print=True,
        # include_meta_content_type=True
        )

    return uniform_html(_clean_html)
def make_clean_html ( raw , stream_item = None , encoding = None ) : # Fix emails by protecting the <,> from HTML raw = fix_emails ( raw ) raw_decoded = nice_decode ( raw , stream_item = stream_item , encoding = encoding ) if raw_decoded is None : # give up on decoding it... maybe this should use force_unicode raw_decoded = raw # default attempt uses vanilla lxml.html try : root = lxml . html . document_fromstring ( raw_decoded ) except ValueError , exc : if 'with encoding declaration' in str ( exc ) : root = lxml . html . document_fromstring ( raw ) else : raise # While we have the document parsed as a DOM, let's strip attributes. # (The HTML cleaner seems to only support whitelisting attributes. # As of now, we just want to blacklist a few.) lxml . etree . strip_attributes ( root , 'class' , 'id' ) # if that worked, then we will be able to generate a # valid HTML string fixed_html = lxml . html . tostring ( root , encoding = unicode ) # remove any ^M characters fixed_html = string . replace ( fixed_html , '\r' , ' ' ) # We drop utf8 characters that are above 0xFFFF as # Lingpipe seems to be doing the wrong thing with them. fixed_html = drop_invalid_and_upper_utf8_chars ( fixed_html ) # construct a Cleaner that removes any ``<script>`` tags, # Javascript, like an ``onclick`` attribute, comments, style # tags or attributes, ``<link>`` tags cleaner = lxml . html . clean . Cleaner ( scripts = True , javascript = True , comments = True , # do not remove <html> <head> <title> etc page_structure = False , remove_tags = [ 'base' ] , style = True , links = True ) # now get the really sanitized HTML _clean_html = cleaner . clean_html ( fixed_html ) # generate pretty HTML in utf-8 _clean_html = lxml . html . tostring ( lxml . html . document_fromstring ( _clean_html ) , method = 'html' , encoding = 'utf-8' , pretty_print = True , # include_meta_content_type=True ) return uniform_html ( _clean_html )
Get a clean text representation of presumed HTML.

    Treat `raw` as though it is HTML, even if we have no idea what it
    really is, and attempt to get a properly formatted HTML document
    with all HTML-escaped characters converted to their unicode.

    This is called below by the `clean_html` transform stage, which
    interprets MIME-type.  If `character_encoding` is not provided,
    and `stream_item` is provided, then this falles back to
    :attr:`streamcorpus.StreamItem.body.encoding`.

    :param str raw: raw text to clean up
    :param stream_item: optional stream item with encoding metadata
    :type stream_item: :class:`streamcorpus.StreamItem`
    :returns: UTF-8-encoded byte string of cleaned HTML text
    :returntype: :class:`str`
Get a clean text representation of presumed HTML .