def create_training_instances(x):
    """Create `TrainingInstance`s from raw text."""
    (input_files, out, tokenizer, max_seq_length, dupe_factor,
     short_seq_prob, masked_lm_prob, max_predictions_per_seq, rng) = x
    time_start = time.time()
    logging.info('Processing %s', input_files)
    all_documents = [[]]

    # Input file format:
    # (1) One sentence per line. These should ideally be actual sentences, not
    # entire paragraphs or arbitrary spans of text. (Because we use the
    # sentence boundaries for the "next sentence prediction" task).
    # (2) Blank lines between documents. Document boundaries are needed so
    # that the "next sentence prediction" task doesn't span between documents.
    for input_file in input_files:
        with io.open(input_file, 'r', encoding='UTF-8') as reader:
            while True:
                line = reader.readline()
                if not line:
                    break
                line = line.strip()

                # Empty lines are used as document delimiters
                if not line:
                    all_documents.append([])
                tokens = tokenizer(line) if not args.tokenized else line.split(' ')
                if tokens:
                    all_documents[-1].append(tokens)

    # Remove empty documents
    all_documents = [x for x in all_documents if x]
    rng.shuffle(all_documents)

    vocab_words = tokenizer.vocab.idx_to_token
    instances = []
    for _ in range(dupe_factor):
        for document_index in range(len(all_documents)):
            instances.extend(
                create_instances_from_document(
                    all_documents, document_index, max_seq_length,
                    short_seq_prob, masked_lm_prob, max_predictions_per_seq,
                    vocab_words, rng))

    rng.shuffle(instances)
    input_ids = []
    segment_ids = []
    masked_lm_positions = []
    masked_lm_ids = []
    masked_lm_weights = []
    next_sentence_labels = []
    valid_lengths = []

    for inst_index, instance in enumerate(instances):
        feature = transform(instance, tokenizer, max_seq_length, max_predictions_per_seq, False)
        input_ids.append(
            np.ascontiguousarray(feature['input_ids'], dtype='int32'))
        segment_ids.append(
            np.ascontiguousarray(feature['segment_ids'], dtype='int32'))
        masked_lm_positions.append(
            np.ascontiguousarray(feature['masked_lm_positions'], dtype='int32'))
        masked_lm_ids.append(np.ascontiguousarray(feature['masked_lm_ids'], dtype='int32'))
        masked_lm_weights.append(
            np.ascontiguousarray(feature['masked_lm_weights'], dtype='float32'))
        next_sentence_labels.append(feature['next_sentence_labels'][0])
        valid_lengths.append(feature['valid_lengths'][0])
        if inst_index < 20:
            print_example(instance, feature)

    features = (input_ids, segment_ids, masked_lm_positions, masked_lm_ids, \
                masked_lm_weights, next_sentence_labels, valid_lengths)

    logging.info('*** Writing to output file %s ***', out)
    if args.format == 'numpy':
        write_to_files_np(features, tokenizer, args.max_seq_length,
                          args.max_predictions_per_seq, [out])
    elif args.format == 'recordio':
        write_to_files_rec(instances, tokenizer, args.max_seq_length,
                           args.max_predictions_per_seq, [out])
    else:
        raise ValueError('unsupported format: %s'%args.format)

    time_end = time.time()
    logging.info('Process %d files took %.1f s', len(input_files), time_end - time_start)
def create_training_instances ( x ) : ( input_files , out , tokenizer , max_seq_length , dupe_factor , short_seq_prob , masked_lm_prob , max_predictions_per_seq , rng ) = x time_start = time . time ( ) logging . info ( 'Processing %s' , input_files ) all_documents = [ [ ] ] for input_file in input_files : with io . open ( input_file , 'r' , encoding = 'UTF-8' ) as reader : while True : line = reader . readline ( ) if not line : break line = line . strip ( ) if not line : all_documents . append ( [ ] ) tokens = tokenizer ( line ) if not args . tokenized else line . split ( ' ' ) if tokens : all_documents [ - 1 ] . append ( tokens ) all_documents = [ x for x in all_documents if x ] rng . shuffle ( all_documents ) vocab_words = tokenizer . vocab . idx_to_token instances = [ ] for _ in range ( dupe_factor ) : for document_index in range ( len ( all_documents ) ) : instances . extend ( create_instances_from_document ( all_documents , document_index , max_seq_length , short_seq_prob , masked_lm_prob , max_predictions_per_seq , vocab_words , rng ) ) rng . shuffle ( instances ) input_ids = [ ] segment_ids = [ ] masked_lm_positions = [ ] masked_lm_ids = [ ] masked_lm_weights = [ ] next_sentence_labels = [ ] valid_lengths = [ ] for inst_index , instance in enumerate ( instances ) : feature = transform ( instance , tokenizer , max_seq_length , max_predictions_per_seq , False ) input_ids . append ( np . ascontiguousarray ( feature [ 'input_ids' ] , dtype = 'int32' ) ) segment_ids . append ( np . ascontiguousarray ( feature [ 'segment_ids' ] , dtype = 'int32' ) ) masked_lm_positions . append ( np . ascontiguousarray ( feature [ 'masked_lm_positions' ] , dtype = 'int32' ) ) masked_lm_ids . append ( np . ascontiguousarray ( feature [ 'masked_lm_ids' ] , dtype = 'int32' ) ) masked_lm_weights . append ( np . ascontiguousarray ( feature [ 'masked_lm_weights' ] , dtype = 'float32' ) ) next_sentence_labels . append ( feature [ 'next_sentence_labels' ] [ 0 ] ) valid_lengths . append ( feature [ 'valid_lengths' ] [ 0 ] ) if inst_index < 20 : print_example ( instance , feature ) features = ( input_ids , segment_ids , masked_lm_positions , masked_lm_ids , masked_lm_weights , next_sentence_labels , valid_lengths ) logging . info ( '*** Writing to output file %s ***' , out ) if args . format == 'numpy' : write_to_files_np ( features , tokenizer , args . max_seq_length , args . max_predictions_per_seq , [ out ] ) elif args . format == 'recordio' : write_to_files_rec ( instances , tokenizer , args . max_seq_length , args . max_predictions_per_seq , [ out ] ) else : raise ValueError ( 'unsupported format: %s' % args . format ) time_end = time . time ( ) logging . info ( 'Process %d files took %.1f s' , len ( input_files ) , time_end - time_start )
Create `TrainingInstance`s from raw text.