streamcorpus_pipeline/convert_entities.py
def html_entities_to_unicode(text, space_padding=False, safe_only=False):
    '''
    Convert any HTML, XML, or numeric entities in the attribute values.
    For example '&amp;' becomes '&'.

    This is adapted from BeautifulSoup, which should be able to do the
    same thing when called like this --- but this fails to convert
    everything for some bug.

    text = unicode(BeautifulStoneSoup(text, convertEntities=BeautifulStoneSoup.XML_ENTITIES))
    '''
    def convert_entities(match):
        '''
        comes from BeautifulSoup.Tag._convertEntities
        '''
        x = match.group(1)

        if safe_only and x not in ENTITIES_THAT_ARE_SAFE_TO_STRING_PAD:
            return u'&%s;' % x

        if x in name2codepoint:
            ## handles most cases
            return unichr(name2codepoint[x])

        elif x in XML_ENTITIES_TO_SPECIAL_CHARS:
            return XML_ENTITIES_TO_SPECIAL_CHARS[x]

        elif len(x) > 0 and x[0] == '#':
            # Handle numeric entities
            if len(x) > 1 and x[1] == 'x':
                return unichr(int(x[2:], 16))
            else:
                return unichr(int(x[1:]))
        else:
            ## uh oh, failed to anything
            return u'&%s;' % x

    def convert_to_padded_entitites(match):
        converted_string = convert_entities(match)
        num_spaces_needed = len(match.group(0)) - len(converted_string)
        assert num_spaces_needed >= 0, \
            'len(%r) !<= len(%r)' % (converted_string, match.group(0))
        ## Where to put the spaces?  Before, after, symmetric?
        # Let's do symmetric.
        ## cast to int in prep for python3
        num_left = int(num_spaces_needed / 2)
        num_right = num_spaces_needed - num_left
        return (' ' * num_left) + converted_string + (' ' * num_right)

    ## brute force regex through all the characters...
    if space_padding:
        return tags.sub(
                      convert_to_padded_entitites,
                      text)
    else:
        return tags.sub(
                      convert_entities,
                      text)
def html_entities_to_unicode(text, space_padding=False, safe_only=False):
    '''
    Convert any HTML, XML, or numeric entities in the attribute values.
    For example '&amp;' becomes '&'.

    This is adapted from BeautifulSoup, which should be able to do the
    same thing when called like this --- but this fails to convert
    everything for some bug.

    text = unicode(BeautifulStoneSoup(text, convertEntities=BeautifulStoneSoup.XML_ENTITIES))
    '''
    def convert_entities(match):
        '''
        comes from BeautifulSoup.Tag._convertEntities
        '''
        x = match.group(1)

        if safe_only and x not in ENTITIES_THAT_ARE_SAFE_TO_STRING_PAD:
            return u'&%s;' % x

        if x in name2codepoint:
            ## handles most cases
            return unichr(name2codepoint[x])

        elif x in XML_ENTITIES_TO_SPECIAL_CHARS:
            return XML_ENTITIES_TO_SPECIAL_CHARS[x]

        elif len(x) > 0 and x[0] == '#':
            # Handle numeric entities
            if len(x) > 1 and x[1] == 'x':
                return unichr(int(x[2:], 16))
            else:
                return unichr(int(x[1:]))
        else:
            ## uh oh, failed to anything
            return u'&%s;' % x

    def convert_to_padded_entitites(match):
        converted_string = convert_entities(match)
        num_spaces_needed = len(match.group(0)) - len(converted_string)
        assert num_spaces_needed >= 0, \
            'len(%r) !<= len(%r)' % (converted_string, match.group(0))
        ## Where to put the spaces?  Before, after, symmetric?
        # Let's do symmetric.
        ## cast to int in prep for python3
        num_left = int(num_spaces_needed / 2)
        num_right = num_spaces_needed - num_left
        return (' ' * num_left) + converted_string + (' ' * num_right)

    ## brute force regex through all the characters...
    if space_padding:
        return tags.sub(
                      convert_to_padded_entitites,
                      text)
    else:
        return tags.sub(
                      convert_entities,
                      text)
def html_entities_to_unicode ( text , space_padding = False , safe_only = False ) : def convert_entities ( match ) : '''
        comes from BeautifulSoup.Tag._convertEntities
        ''' x = match . group ( 1 ) if safe_only and x not in ENTITIES_THAT_ARE_SAFE_TO_STRING_PAD : return u'&%s;' % x if x in name2codepoint : ## handles most cases return unichr ( name2codepoint [ x ] ) elif x in XML_ENTITIES_TO_SPECIAL_CHARS : return XML_ENTITIES_TO_SPECIAL_CHARS [ x ] elif len ( x ) > 0 and x [ 0 ] == '#' : # Handle numeric entities if len ( x ) > 1 and x [ 1 ] == 'x' : return unichr ( int ( x [ 2 : ] , 16 ) ) else : return unichr ( int ( x [ 1 : ] ) ) else : ## uh oh, failed to anything return u'&%s;' % x def convert_to_padded_entitites ( match ) : converted_string = convert_entities ( match ) num_spaces_needed = len ( match . group ( 0 ) ) - len ( converted_string ) assert num_spaces_needed >= 0 , 'len(%r) !<= len(%r)' % ( converted_string , match . group ( 0 ) ) ## Where to put the spaces?  Before, after, symmetric? # Let's do symmetric. ## cast to int in prep for python3 num_left = int ( num_spaces_needed / 2 ) num_right = num_spaces_needed - num_left return ( ' ' * num_left ) + converted_string + ( ' ' * num_right ) ## brute force regex through all the characters... if space_padding : return tags . sub ( convert_to_padded_entitites , text ) else : return tags . sub ( convert_entities , text )
Convert any HTML, XML, or numeric entities in the attribute values.
    For example '&amp;' becomes '&'.

    This is adapted from BeautifulSoup, which should be able to do the
    same thing when called like this --- but this fails to convert
    everything for some bug.

    text = unicode(BeautifulStoneSoup(text, convertEntities=BeautifulStoneSoup.XML_ENTITIES))
Convert any HTML XML or numeric entities in the attribute values . For example &amp ; becomes & .