def score_candidates(nodes):
    """Given a list of potential nodes, find some initial scores to start"""
    MIN_HIT_LENTH = 25
    candidates = {}

    for node in nodes:
        logger.debug("* Scoring candidate %s %r", node.tag, node.attrib)

        # if the node has no parent it knows of then it ends up creating a
        # body & html tag to parent the html fragment
        parent = node.getparent()
        if parent is None:
            logger.debug("Skipping candidate - parent node is 'None'.")
            continue

        grand = parent.getparent()
        if grand is None:
            logger.debug("Skipping candidate - grand parent node is 'None'.")
            continue

        # if paragraph is < `MIN_HIT_LENTH` characters don't even count it
        inner_text = node.text_content().strip()
        if len(inner_text) < MIN_HIT_LENTH:
            logger.debug(
                "Skipping candidate - inner text < %d characters.",
                MIN_HIT_LENTH)
            continue

        # initialize readability data for the parent
        # add parent node if it isn't in the candidate list
        if parent not in candidates:
            candidates[parent] = ScoredNode(parent)

        if grand not in candidates:
            candidates[grand] = ScoredNode(grand)

        # add a point for the paragraph itself as a base
        content_score = 1

        if inner_text:
            # add 0.25 points for any commas within this paragraph
            commas_count = inner_text.count(",")
            content_score += commas_count * 0.25
            logger.debug("Bonus points for %d commas.", commas_count)

            # subtract 0.5 points for each double quote within this paragraph
            double_quotes_count = inner_text.count('"')
            content_score += double_quotes_count * -0.5
            logger.debug(
                "Penalty points for %d double-quotes.", double_quotes_count)

            # for every 100 characters in this paragraph, add another point
            # up to 3 points
            length_points = len(inner_text) / 100
            content_score += min(length_points, 3.0)
            logger.debug("Bonus points for length of text: %f", length_points)

        # add the score to the parent
        logger.debug(
            "Bonus points for parent %s %r with score %f: %f",
            parent.tag, parent.attrib, candidates[parent].content_score,
            content_score)
        candidates[parent].content_score += content_score
        # the grand node gets half
        logger.debug(
            "Bonus points for grand %s %r with score %f: %f",
            grand.tag, grand.attrib, candidates[grand].content_score,
            content_score / 2.0)
        candidates[grand].content_score += content_score / 2.0

        if node not in candidates:
            candidates[node] = ScoredNode(node)
        candidates[node].content_score += content_score

    for candidate in candidates.values():
        adjustment = 1.0 - get_link_density(candidate.node)
        candidate.content_score *= adjustment
        logger.debug(
            "Link density adjustment for %s %r: %f",
            candidate.node.tag, candidate.node.attrib, adjustment)

    return candidates
def score_candidates ( nodes ) : MIN_HIT_LENTH = 25 candidates = { } for node in nodes : logger . debug ( "* Scoring candidate %s %r" , node . tag , node . attrib ) parent = node . getparent ( ) if parent is None : logger . debug ( "Skipping candidate - parent node is 'None'." ) continue grand = parent . getparent ( ) if grand is None : logger . debug ( "Skipping candidate - grand parent node is 'None'." ) continue inner_text = node . text_content ( ) . strip ( ) if len ( inner_text ) < MIN_HIT_LENTH : logger . debug ( "Skipping candidate - inner text < %d characters." , MIN_HIT_LENTH ) continue if parent not in candidates : candidates [ parent ] = ScoredNode ( parent ) if grand not in candidates : candidates [ grand ] = ScoredNode ( grand ) content_score = 1 if inner_text : commas_count = inner_text . count ( "," ) content_score += commas_count * 0.25 logger . debug ( "Bonus points for %d commas." , commas_count ) double_quotes_count = inner_text . count ( '"' ) content_score += double_quotes_count * - 0.5 logger . debug ( "Penalty points for %d double-quotes." , double_quotes_count ) length_points = len ( inner_text ) / 100 content_score += min ( length_points , 3.0 ) logger . debug ( "Bonus points for length of text: %f" , length_points ) logger . debug ( "Bonus points for parent %s %r with score %f: %f" , parent . tag , parent . attrib , candidates [ parent ] . content_score , content_score ) candidates [ parent ] . content_score += content_score logger . debug ( "Bonus points for grand %s %r with score %f: %f" , grand . tag , grand . attrib , candidates [ grand ] . content_score , content_score / 2.0 ) candidates [ grand ] . content_score += content_score / 2.0 if node not in candidates : candidates [ node ] = ScoredNode ( node ) candidates [ node ] . content_score += content_score for candidate in candidates . values ( ) : adjustment = 1.0 - get_link_density ( candidate . node ) candidate . content_score *= adjustment logger . debug ( "Link density adjustment for %s %r: %f" , candidate . node . tag , candidate . node . attrib , adjustment ) return candidates
Given a list of potential nodes, find some initial scores to start