def clean_conditionally(node):
    """Remove the clean_el if it looks like bad content based on rules."""
    if node.tag not in ('form', 'table', 'ul', 'div', 'p'):
        return  # this is not the tag we are looking for

    weight = get_class_weight(node)
    # content_score = LOOK up the content score for this node we found
    # before else default to 0
    content_score = 0

    if weight + content_score < 0:
        logger.debug('Dropping conditional node')
        logger.debug('Weight + score < 0')
        return True

    commas_count = node.text_content().count(',')
    if commas_count < 10:
        logger.debug(
            "There are %d commas so we're processing more.", commas_count)

        # If there are not very many commas, and the number of
        # non-paragraph elements is more than paragraphs or other ominous
        # signs, remove the element.
        p = len(node.findall('.//p'))
        img = len(node.findall('.//img'))
        li = len(node.findall('.//li')) - 100
        inputs = len(node.findall('.//input'))

        embed = 0
        embeds = node.findall('.//embed')
        for e in embeds:
            if ok_embedded_video(e):
                embed += 1
        link_density = get_link_density(node)
        content_length = len(node.text_content())

        remove_node = False

        if li > p and node.tag != 'ul' and node.tag != 'ol':
            logger.debug('Conditional drop: li > p and not ul/ol')
            remove_node = True
        elif inputs > p / 3.0:
            logger.debug('Conditional drop: inputs > p/3.0')
            remove_node = True
        elif content_length < 25 and (img == 0 or img > 2):
            logger.debug('Conditional drop: len < 25 and 0/>2 images')
            remove_node = True
        elif weight < 25 and link_density > 0.2:
            logger.debug('Conditional drop: weight small (%f) and link is dense (%f)', weight, link_density)
            remove_node = True
        elif weight >= 25 and link_density > 0.5:
            logger.debug('Conditional drop: weight big but link heavy')
            remove_node = True
        elif (embed == 1 and content_length < 75) or embed > 1:
            logger.debug(
                'Conditional drop: embed w/o much content or many embed')
            remove_node = True

        if remove_node:
            logger.debug('Node will be removed: %s %r %s', node.tag, node.attrib, node.text_content()[:30])

        return remove_node

    return False
def clean_conditionally ( node ) : if node . tag not in ( 'form' , 'table' , 'ul' , 'div' , 'p' ) : return weight = get_class_weight ( node ) content_score = 0 if weight + content_score < 0 : logger . debug ( 'Dropping conditional node' ) logger . debug ( 'Weight + score < 0' ) return True commas_count = node . text_content ( ) . count ( ',' ) if commas_count < 10 : logger . debug ( "There are %d commas so we're processing more." , commas_count ) p = len ( node . findall ( './/p' ) ) img = len ( node . findall ( './/img' ) ) li = len ( node . findall ( './/li' ) ) - 100 inputs = len ( node . findall ( './/input' ) ) embed = 0 embeds = node . findall ( './/embed' ) for e in embeds : if ok_embedded_video ( e ) : embed += 1 link_density = get_link_density ( node ) content_length = len ( node . text_content ( ) ) remove_node = False if li > p and node . tag != 'ul' and node . tag != 'ol' : logger . debug ( 'Conditional drop: li > p and not ul/ol' ) remove_node = True elif inputs > p / 3.0 : logger . debug ( 'Conditional drop: inputs > p/3.0' ) remove_node = True elif content_length < 25 and ( img == 0 or img > 2 ) : logger . debug ( 'Conditional drop: len < 25 and 0/>2 images' ) remove_node = True elif weight < 25 and link_density > 0.2 : logger . debug ( 'Conditional drop: weight small (%f) and link is dense (%f)' , weight , link_density ) remove_node = True elif weight >= 25 and link_density > 0.5 : logger . debug ( 'Conditional drop: weight big but link heavy' ) remove_node = True elif ( embed == 1 and content_length < 75 ) or embed > 1 : logger . debug ( 'Conditional drop: embed w/o much content or many embed' ) remove_node = True if remove_node : logger . debug ( 'Node will be removed: %s %r %s' , node . tag , node . attrib , node . text_content ( ) [ : 30 ] ) return remove_node return False
Remove the clean_el if it looks like bad content based on rules.