def prepare_data(data_dir, fileroot, block_pct_tokens_thresh=0.1):
    """
    Prepare data for a single HTML + gold standard blocks example, uniquely
    identified by ``fileroot``.

    Args:
        data_dir (str)
        fileroot (str)
        block_pct_tokens_thresh (float): must be in [0.0, 1.0]

    Returns:
        Tuple[str, Tuple[np.array[int], np.array[int], List[str]], Tuple[np.array[int], np.array[int], List[str]]]:
            The first element is simply the raw html as a string. The second and
            third elements are 3-tuples for content and comments, respectively,
            where the first element is a numpy array of 1s and 0s whose values
            correspond to whether or not a given block is considered non-content
            or not; the second element is a numpy integer array whose values are
            the total number of tokens in each block; and the third element is
            a flat list of content or comment tokens as strings, concatenated
            from all blocks.

    See Also:
        :func:`prepare_all_data`
    """
    if not 0.0 <= block_pct_tokens_thresh <= 1.0:
        raise ValueError('block_pct_tokens_thresh must be in the range [0.0, 1.0]')

    html = read_html_file(data_dir, fileroot)
    blocks = read_gold_standard_blocks_file(data_dir, fileroot, split_blocks=True)

    content_blocks = []
    comments_blocks = []
    for block in blocks:
        block_split = block.split('\t')
        num_block_tokens = len(block_split[2].split())
        # total number of tokens in block is used as weights
        content_blocks.append(
            (float(block_split[0]), num_block_tokens, block_split[3].split()))
        comments_blocks.append(
            (float(block_split[1]), num_block_tokens, block_split[4].split()))

    parsed_content_blocks = _parse_content_or_comments_blocks(
        content_blocks, block_pct_tokens_thresh)
    parsed_comments_blocks = _parse_content_or_comments_blocks(
        comments_blocks, block_pct_tokens_thresh)

    return (html, parsed_content_blocks, parsed_comments_blocks)
def prepare_data ( data_dir , fileroot , block_pct_tokens_thresh = 0.1 ) : if not 0.0 <= block_pct_tokens_thresh <= 1.0 : raise ValueError ( 'block_pct_tokens_thresh must be in the range [0.0, 1.0]' ) html = read_html_file ( data_dir , fileroot ) blocks = read_gold_standard_blocks_file ( data_dir , fileroot , split_blocks = True ) content_blocks = [ ] comments_blocks = [ ] for block in blocks : block_split = block . split ( '\t' ) num_block_tokens = len ( block_split [ 2 ] . split ( ) ) content_blocks . append ( ( float ( block_split [ 0 ] ) , num_block_tokens , block_split [ 3 ] . split ( ) ) ) comments_blocks . append ( ( float ( block_split [ 1 ] ) , num_block_tokens , block_split [ 4 ] . split ( ) ) ) parsed_content_blocks = _parse_content_or_comments_blocks ( content_blocks , block_pct_tokens_thresh ) parsed_comments_blocks = _parse_content_or_comments_blocks ( comments_blocks , block_pct_tokens_thresh ) return ( html , parsed_content_blocks , parsed_comments_blocks )
Prepare data for a single HTML + gold standard blocks example, uniquely
    identified by ``fileroot``.

    Args:
        data_dir (str)
        fileroot (str)
        block_pct_tokens_thresh (float): must be in [0.0, 1.0]

    Returns:
        Tuple[str, Tuple[np.array[int], np.array[int], List[str]], Tuple[np.array[int], np.array[int], List[str]]]:
            The first element is simply the raw html as a string. The second and
            third elements are 3-tuples for content and comments, respectively,
            where the first element is a numpy array of 1s and 0s whose values
            correspond to whether or not a given block is considered non-content
            or not; the second element is a numpy integer array whose values are
            the total number of tokens in each block; and the third element is
            a flat list of content or comment tokens as strings, concatenated
            from all blocks.

    See Also:
        :func:`prepare_all_data`