def generate_data(path, tokenizer, char_vcb, word_vcb, is_training=False):
    '''
    Generate data
    '''
    global root_path
    qp_pairs = data.load_from_file(path=path, is_training=is_training)

    tokenized_sent = 0
    # qp_pairs = qp_pairs[:1000]1
    for qp_pair in qp_pairs:
        tokenized_sent += 1
        data.tokenize(qp_pair, tokenizer, is_training)
        for word in qp_pair['question_tokens']:
            word_vcb.add(word['word'])
            for char in word['word']:
                char_vcb.add(char)
        for word in qp_pair['passage_tokens']:
            word_vcb.add(word['word'])
            for char in word['word']:
                char_vcb.add(char)

    max_query_length = max(len(x['question_tokens']) for x in qp_pairs)
    max_passage_length = max(len(x['passage_tokens']) for x in qp_pairs)
    #min_passage_length = min(len(x['passage_tokens']) for x in qp_pairs)
    cfg.max_query_length = max_query_length
    cfg.max_passage_length = max_passage_length

    return qp_pairs
def generate_data ( path , tokenizer , char_vcb , word_vcb , is_training = False ) : global root_path qp_pairs = data . load_from_file ( path = path , is_training = is_training ) tokenized_sent = 0 for qp_pair in qp_pairs : tokenized_sent += 1 data . tokenize ( qp_pair , tokenizer , is_training ) for word in qp_pair [ 'question_tokens' ] : word_vcb . add ( word [ 'word' ] ) for char in word [ 'word' ] : char_vcb . add ( char ) for word in qp_pair [ 'passage_tokens' ] : word_vcb . add ( word [ 'word' ] ) for char in word [ 'word' ] : char_vcb . add ( char ) max_query_length = max ( len ( x [ 'question_tokens' ] ) for x in qp_pairs ) max_passage_length = max ( len ( x [ 'passage_tokens' ] ) for x in qp_pairs ) cfg . max_query_length = max_query_length cfg . max_passage_length = max_passage_length return qp_pairs
Generate data