def create_instances_from_document(
        doc_database, doc_idx, max_seq_length, short_seq_prob,
        masked_lm_prob, max_predictions_per_seq, vocab_list):
    """This code is mostly a duplicate of the equivalent function from Google BERT's repo.
    However, we make some changes and improvements. Sampling is improved and no longer requires a loop in this function.
    Also, documents are sampled proportionally to the number of sentences they contain, which means each sentence
    (rather than each document) has an equal chance of being sampled as a false example for the NextSentence task."""
    document = doc_database[doc_idx]
    # Account for [CLS], [SEP], [SEP]
    max_num_tokens = max_seq_length - 3

    # We *usually* want to fill up the entire sequence since we are padding
    # to `max_seq_length` anyways, so short sequences are generally wasted
    # computation. However, we *sometimes*
    # (i.e., short_seq_prob == 0.1 == 10% of the time) want to use shorter
    # sequences to minimize the mismatch between pre-training and fine-tuning.
    # The `target_seq_length` is just a rough target however, whereas
    # `max_seq_length` is a hard limit.
    target_seq_length = max_num_tokens
    if random() < short_seq_prob:
        target_seq_length = randint(2, max_num_tokens)

    # We DON'T just concatenate all of the tokens from a document into a long
    # sequence and choose an arbitrary split point because this would make the
    # next sentence prediction task too easy. Instead, we split the input into
    # segments "A" and "B" based on the actual "sentences" provided by the user
    # input.
    instances = []
    current_chunk = []
    current_length = 0
    i = 0
    while i < len(document):
        segment = document[i]
        current_chunk.append(segment)
        current_length += len(segment)
        if i == len(document) - 1 or current_length >= target_seq_length:
            if current_chunk:
                # `a_end` is how many segments from `current_chunk` go into the `A`
                # (first) sentence.
                a_end = 1
                if len(current_chunk) >= 2:
                    a_end = randrange(1, len(current_chunk))

                tokens_a = []
                for j in range(a_end):
                    tokens_a.extend(current_chunk[j])

                tokens_b = []

                # Random next
                if len(current_chunk) == 1 or random() < 0.5:
                    is_random_next = True
                    target_b_length = target_seq_length - len(tokens_a)

                    # Sample a random document, with longer docs being sampled more frequently
                    random_document = doc_database.sample_doc(current_idx=doc_idx, sentence_weighted=True)

                    random_start = randrange(0, len(random_document))
                    for j in range(random_start, len(random_document)):
                        tokens_b.extend(random_document[j])
                        if len(tokens_b) >= target_b_length:
                            break
                    # We didn't actually use these segments so we "put them back" so
                    # they don't go to waste.
                    num_unused_segments = len(current_chunk) - a_end
                    i -= num_unused_segments
                # Actual next
                else:
                    is_random_next = False
                    for j in range(a_end, len(current_chunk)):
                        tokens_b.extend(current_chunk[j])
                truncate_seq_pair(tokens_a, tokens_b, max_num_tokens)

                assert len(tokens_a) >= 1
                assert len(tokens_b) >= 1

                tokens = ["[CLS]"] + tokens_a + ["[SEP]"] + tokens_b + ["[SEP]"]
                # The segment IDs are 0 for the [CLS] token, the A tokens and the first [SEP]
                # They are 1 for the B tokens and the final [SEP]
                segment_ids = [0 for _ in range(len(tokens_a) + 2)] + [1 for _ in range(len(tokens_b) + 1)]

                tokens, masked_lm_positions, masked_lm_labels = create_masked_lm_predictions(
                    tokens, masked_lm_prob, max_predictions_per_seq, vocab_list)

                instance = {
                    "tokens": tokens,
                    "segment_ids": segment_ids,
                    "is_random_next": is_random_next,
                    "masked_lm_positions": masked_lm_positions,
                    "masked_lm_labels": masked_lm_labels}
                instances.append(instance)
            current_chunk = []
            current_length = 0
        i += 1

    return instances
def create_instances_from_document ( doc_database , doc_idx , max_seq_length , short_seq_prob , masked_lm_prob , max_predictions_per_seq , vocab_list ) : document = doc_database [ doc_idx ] max_num_tokens = max_seq_length - 3 target_seq_length = max_num_tokens if random ( ) < short_seq_prob : target_seq_length = randint ( 2 , max_num_tokens ) instances = [ ] current_chunk = [ ] current_length = 0 i = 0 while i < len ( document ) : segment = document [ i ] current_chunk . append ( segment ) current_length += len ( segment ) if i == len ( document ) - 1 or current_length >= target_seq_length : if current_chunk : a_end = 1 if len ( current_chunk ) >= 2 : a_end = randrange ( 1 , len ( current_chunk ) ) tokens_a = [ ] for j in range ( a_end ) : tokens_a . extend ( current_chunk [ j ] ) tokens_b = [ ] if len ( current_chunk ) == 1 or random ( ) < 0.5 : is_random_next = True target_b_length = target_seq_length - len ( tokens_a ) random_document = doc_database . sample_doc ( current_idx = doc_idx , sentence_weighted = True ) random_start = randrange ( 0 , len ( random_document ) ) for j in range ( random_start , len ( random_document ) ) : tokens_b . extend ( random_document [ j ] ) if len ( tokens_b ) >= target_b_length : break num_unused_segments = len ( current_chunk ) - a_end i -= num_unused_segments else : is_random_next = False for j in range ( a_end , len ( current_chunk ) ) : tokens_b . extend ( current_chunk [ j ] ) truncate_seq_pair ( tokens_a , tokens_b , max_num_tokens ) assert len ( tokens_a ) >= 1 assert len ( tokens_b ) >= 1 tokens = [ "[CLS]" ] + tokens_a + [ "[SEP]" ] + tokens_b + [ "[SEP]" ] segment_ids = [ 0 for _ in range ( len ( tokens_a ) + 2 ) ] + [ 1 for _ in range ( len ( tokens_b ) + 1 ) ] tokens , masked_lm_positions , masked_lm_labels = create_masked_lm_predictions ( tokens , masked_lm_prob , max_predictions_per_seq , vocab_list ) instance = { "tokens" : tokens , "segment_ids" : segment_ids , "is_random_next" : is_random_next , "masked_lm_positions" : masked_lm_positions , "masked_lm_labels" : masked_lm_labels } instances . append ( instance ) current_chunk = [ ] current_length = 0 i += 1 return instances
This code is mostly a duplicate of the equivalent function from Google BERT's repo.
    However, we make some changes and improvements. Sampling is improved and no longer requires a loop in this function.
    Also, documents are sampled proportionally to the number of sentences they contain, which means each sentence
    (rather than each document) has an equal chance of being sampled as a false example for the NextSentence task.