ktext/preprocess.py
def generate_doc_length_stats(self):
        """Analyze document length statistics for padding strategy"""
        heuristic = self.heuristic_pct
        histdf = (pd.DataFrame([(a, b) for a, b in self.document_length_histogram.items()],
                               columns=['bin', 'doc_count'])
                  .sort_values(by='bin'))
        histdf['cumsum_pct'] = histdf.doc_count.cumsum() / histdf.doc_count.sum()

        self.document_length_stats = histdf
        self.doc_length_huerestic = histdf.query(f'cumsum_pct >= {heuristic}').bin.head(1).values[0]
        logging.warning(' '.join(["Setting maximum document length to",
                                  f'{self.doc_length_huerestic} based upon',
                                  f'heuristic of {heuristic} percentile.\n',
                                  'See full histogram by insepecting the',
                                  "`document_length_stats` attribute."]))
        self.padding_maxlen = self.doc_length_huerestic
def generate_doc_length_stats(self):
        heuristic = self.heuristic_pct
        histdf = (pd.DataFrame([(a, b) for a, b in self.document_length_histogram.items()],
                               columns=['bin', 'doc_count'])
                  .sort_values(by='bin'))
        histdf['cumsum_pct'] = histdf.doc_count.cumsum() / histdf.doc_count.sum()

        self.document_length_stats = histdf
        self.doc_length_huerestic = histdf.query(f'cumsum_pct >= {heuristic}').bin.head(1).values[0]
        logging.warning(' '.join(["Setting maximum document length to",
                                  f'{self.doc_length_huerestic} based upon',
                                  f'heuristic of {heuristic} percentile.\n',
                                  'See full histogram by insepecting the',
                                  "`document_length_stats` attribute."]))
        self.padding_maxlen = self.doc_length_huerestic
def generate_doc_length_stats ( self ) : heuristic = self . heuristic_pct histdf = ( pd . DataFrame ( [ ( a , b ) for a , b in self . document_length_histogram . items ( ) ] , columns = [ 'bin' , 'doc_count' ] ) . sort_values ( by = 'bin' ) ) histdf [ 'cumsum_pct' ] = histdf . doc_count . cumsum ( ) / histdf . doc_count . sum ( ) self . document_length_stats = histdf self . doc_length_huerestic = histdf . query ( f'cumsum_pct >= {heuristic}' ) . bin . head ( 1 ) . values [ 0 ] logging . warning ( ' ' . join ( [ "Setting maximum document length to" , f'{self.doc_length_huerestic} based upon' , f'heuristic of {heuristic} percentile.\n' , 'See full histogram by insepecting the' , "`document_length_stats` attribute." ] ) ) self . padding_maxlen = self . doc_length_huerestic
Analyze document length statistics for padding strategy
Analyze document length statistics for padding strategy