<s> """<STR_LIT>""" <EOL> from __future__ import absolute_import , division , print_function , unicode_literals <EOL> import collections <EOL> import itertools <EOL> import numpy as np <EOL> import scipy . sparse as sp <EOL> from sklearn . preprocessing import binarize as binarize_mat <EOL> from sklearn . preprocessing import normalize as normalize_mat <EOL> from spacy . strings import StringStore <EOL> def build_doc_term_matrix ( terms_lists , <EOL> weighting = '<STR_LIT>' , <EOL> normalize = False , sublinear_tf = False , smooth_idf = True , <EOL> min_df = <NUM_LIT:1> , max_df = <NUM_LIT:1.0> , min_ic = <NUM_LIT:0.0> , max_n_terms = None ) : <EOL> """<STR_LIT>""" <EOL> stringstore = StringStore ( ) <EOL> data = [ ] ; rows = [ ] ; cols = [ ] <EOL> for row_idx , terms_list in enumerate ( terms_lists ) : <EOL> bow = tuple ( ( stringstore [ term ] - <NUM_LIT:1> , count ) <EOL> for term , count in collections . Counter ( terms_list ) . items ( ) <EOL> if term ) <EOL> data . extend ( count for _ , count in bow ) <EOL> cols . extend ( term_id for term_id , _ in bow ) <EOL> rows . extend ( itertools . repeat ( row_idx , times = len ( bow ) ) ) <EOL> doc_term_matrix = sp . coo_matrix ( ( data , ( rows , cols ) ) , dtype = int ) . tocsr ( ) <EOL> id_to_term = { term_id - <NUM_LIT:1> : term for term_id , term in enumerate ( stringstore ) <EOL> if term_id != <NUM_LIT:0> } <EOL> if max_df != <NUM_LIT:1.0> or min_df != <NUM_LIT:1> or max_n_terms is not None : <EOL> doc_term_matrix , id_to_term = filter_terms_by_df ( <EOL> doc_term_matrix , id_to_term , <EOL> max_df = max_df , min_df = min_df , max_n_terms = max_n_terms ) <EOL> if min_ic != <NUM_LIT:0.0> : <EOL> doc_term_matrix , id_to_term = filter_terms_by_ic ( <EOL> doc_term_matrix , id_to_term , <EOL> min_ic = min_ic , max_n_terms = max_n_terms ) <EOL> if weighting == '<STR_LIT>' : <EOL> doc_term_matrix = binarize_mat ( doc_term_matrix , threshold = <NUM_LIT:0.0> , copy = False ) <EOL> else : <EOL> if sublinear_tf is True : <EOL> doc_term_matrix = doc_term_matrix . astype ( np . float64 ) <EOL> np . log ( doc_term_matrix . data , doc_term_matrix . data ) <EOL> doc_term_matrix . data += <NUM_LIT:1> <EOL> if weighting == '<STR_LIT>' : <EOL> doc_term_matrix = apply_idf_weighting ( doc_term_matrix , <EOL> smooth_idf = smooth_idf ) <EOL> if normalize is True : <EOL> doc_term_matrix = normalize_mat ( doc_term_matrix , <EOL> norm = '<STR_LIT>' , axis = <NUM_LIT:1> , copy = False ) <EOL> return ( doc_term_matrix , id_to_term ) <EOL> def apply_idf_weighting ( doc_term_matrix , smooth_idf = True ) : <EOL> """<STR_LIT>""" <EOL> dfs = get_doc_freqs ( doc_term_matrix , normalized = False ) <EOL> n_docs = doc_term_matrix . shape [ <NUM_LIT:0> ] <EOL> if smooth_idf is True : <EOL> n_docs += <NUM_LIT:1> <EOL> dfs += <NUM_LIT:1> <EOL> idfs = np . log ( n_docs / dfs ) + <NUM_LIT:1.0> <EOL> return doc_term_matrix . dot ( sp . diags ( idfs , <NUM_LIT:0> ) ) <EOL> def get_term_freqs ( doc_term_matrix , normalized = True ) : <EOL> """<STR_LIT>""" <EOL> if doc_term_matrix . nnz == <NUM_LIT:0> : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> _ , n_terms = doc_term_matrix . shape <EOL> tfs = np . asarray ( doc_term_matrix . sum ( axis = <NUM_LIT:0> ) ) . ravel ( ) <EOL> if normalized is True : <EOL> return tfs / n_terms <EOL> else : <EOL> return tfs <EOL> def get_doc_freqs ( doc_term_matrix , normalized = True ) : <EOL> """<STR_LIT>""" <EOL> if doc_term_matrix . nnz == <NUM_LIT:0> : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> n_docs , n_terms = doc_term_matrix . shape <EOL> dfs = np . bincount ( doc_term_matrix . indices , minlength = n_terms ) <EOL> if normalized is True : <EOL> return dfs / n_docs <EOL> else : <EOL> return dfs <EOL> def get_information_content ( doc_term_matrix ) : <EOL> """<STR_LIT>""" <EOL> dfs = get_doc_freqs ( doc_term_matrix , normalized = True ) <EOL> ics = - dfs * np . log2 ( dfs ) - ( <NUM_LIT:1> - dfs ) * np . log2 ( <NUM_LIT:1> - dfs ) <EOL> ics [ np . isnan ( ics ) ] = <NUM_LIT:0.0> <EOL> return ics <EOL> def filter_terms_by_df ( doc_term_matrix , id_to_term , <EOL> max_df = <NUM_LIT:1.0> , min_df = <NUM_LIT:1> , max_n_terms = None ) : <EOL> """<STR_LIT>""" <EOL> if max_df == <NUM_LIT:1.0> and min_df == <NUM_LIT:1> and max_n_terms is None : <EOL> return doc_term_matrix , id_to_term <EOL> if max_df < <NUM_LIT:0> or min_df < <NUM_LIT:0> or ( max_n_terms is not None and max_n_terms < <NUM_LIT:0> ) : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> n_docs , n_terms = doc_term_matrix . shape <EOL> max_doc_count = max_df if isinstance ( max_df , int ) else int ( max_df * n_docs ) <EOL> min_doc_count = min_df if isinstance ( min_df , int ) else int ( min_df * n_docs ) <EOL> if max_doc_count < min_doc_count : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> dfs = get_doc_freqs ( doc_term_matrix , normalized = False ) <EOL> mask = np . ones ( n_terms , dtype = bool ) <EOL> if max_doc_count < n_docs : <EOL> mask &= dfs <= max_doc_count <EOL> if min_doc_count > <NUM_LIT:1> : <EOL> mask &= dfs >= min_doc_count <EOL> if max_n_terms is not None and mask . sum ( ) > max_n_terms : <EOL> tfs = get_term_freqs ( doc_term_matrix , normalized = False ) <EOL> top_mask_inds = ( tfs [ mask ] ) . argsort ( ) [ : : - <NUM_LIT:1> ] [ : max_n_terms ] <EOL> new_mask = np . zeros ( n_terms , dtype = bool ) <EOL> new_mask [ np . where ( mask ) [ <NUM_LIT:0> ] [ top_mask_inds ] ] = True <EOL> mask = new_mask <EOL> new_indices = np . cumsum ( mask ) - <NUM_LIT:1> <EOL> id_to_term = { new_indices [ old_index ] : term <EOL> for old_index , term in id_to_term . items ( ) <EOL> if mask [ old_index ] } <EOL> kept_indices = np . where ( mask ) [ <NUM_LIT:0> ] <EOL> if len ( kept_indices ) == <NUM_LIT:0> : <EOL> msg = '<STR_LIT>' <EOL> raise ValueError ( msg ) <EOL> return ( doc_term_matrix [ : , kept_indices ] , id_to_term ) <EOL> def filter_terms_by_ic ( doc_term_matrix , id_to_term , <EOL> min_ic = <NUM_LIT:0.0> , max_n_terms = None ) : <EOL> """<STR_LIT>""" <EOL> if min_ic == <NUM_LIT:0.0> and max_n_terms is None : <EOL> return doc_term_matrix , id_to_term <EOL> if min_ic < <NUM_LIT:0.0> or min_ic > <NUM_LIT:1.0> : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> if max_n_terms is not None and max_n_terms < <NUM_LIT:0> : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> _ , n_terms = doc_term_matrix . shape <EOL> ics = get_information_content ( doc_term_matrix ) <EOL> mask = np . ones ( n_terms , dtype = bool ) <EOL> if min_ic > <NUM_LIT:0.0> : <EOL> mask &= ics >= min_ic <EOL> if max_n_terms is not None and mask . sum ( ) > max_n_terms : <EOL> top_mask_inds = ( ics [ mask ] ) . argsort ( ) [ : : - <NUM_LIT:1> ] [ : max_n_terms ] <EOL> new_mask = np . zeros ( n_terms , dtype = bool ) <EOL> new_mask [ np . where ( mask ) [ <NUM_LIT:0> ] [ top_mask_inds ] ] = True <EOL> mask = new_mask <EOL> new_indices = np . cumsum ( mask ) - <NUM_LIT:1> <EOL> id_to_term = { new_indices [ old_index ] : term <EOL> for old_index , term in id_to_term . items ( ) <EOL> if mask [ old_index ] } <EOL> kept_indices = np . where ( mask ) [ <NUM_LIT:0> ] <EOL> if len ( kept_indices ) == <NUM_LIT:0> : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> return ( doc_term_matrix [ : , kept_indices ] , id_to_term ) </s>
