def predictions(dev_dataset,
                all_results,
                tokenizer,
                max_answer_length=64,
                null_score_diff_threshold=0.0,
                n_best_size=10,
                version_2=False):
    """Get prediction results

    Parameters
    ----------
    dev_dataset: dataset
        Examples of transform.
    all_results: dict
        A dictionary containing model prediction results.
    tokenizer: callable
        Tokenizer function.
    max_answer_length: int, default 64
        Maximum length of the answer tokens.
    null_score_diff_threshold: float, default 0.0
        If null_score - best_non_null is greater than the threshold predict null.
    n_best_size: int, default 10
        The total number of n-best predictions.
    version_2: bool, default False
        If true, the SQuAD examples contain some that do not have an answer.
    Returns
    -------
    all_predictions: dict
        All final predictions.
    all_nbest_json: dict
        All n-best predictions.
    scores_diff_json: dict
        Record the difference between null score and the score of best non-null.
        when version_2 is True.
    """

    _PrelimPrediction = namedtuple('PrelimPrediction',
                                   ['feature_index', 'start_index', 'end_index',
                                    'start_logit', 'end_logit'])

    _NbestPrediction = namedtuple(
        'NbestPrediction', ['text', 'start_logit', 'end_logit'])

    all_predictions = OrderedDict()
    all_nbest_json = OrderedDict()
    scores_diff_json = OrderedDict()

    for features in dev_dataset:
        results = all_results[features[0].example_id]
        example_qas_id = features[0].qas_id
        prelim_predictions = []

        score_null = 1000000  # large and positive
        min_null_feature_index = 0  # the paragraph slice with min mull score
        null_start_logit = 0  # the start logit at the slice with min null score
        null_end_logit = 0  # the end logit at the slice with min null score

        for features_id, (result, feature) in enumerate(zip(results, features)):
            start_indexes = _get_best_indexes(result.start_logits, n_best_size)
            end_indexes = _get_best_indexes(result.end_logits, n_best_size)

            if version_2:
                feature_null_score = result.start_logits[0] + \
                    result.end_logits[0]
                if feature_null_score < score_null:
                    score_null = feature_null_score
                    min_null_feature_index = features_id
                    null_start_logit = result.start_logits[0]
                    null_end_logit = result.end_logits[0]

            for start_index in start_indexes:
                for end_index in end_indexes:
                    # We could hypothetically create invalid predictions, e.g., predict
                    # that the start of the span is in the question. We throw out all
                    # invalid predictions.
                    if start_index >= len(feature.tokens):
                        continue
                    if end_index >= len(feature.tokens):
                        continue
                    if start_index not in feature.token_to_orig_map:
                        continue
                    if end_index not in feature.token_to_orig_map:
                        continue
                    if not feature.token_is_max_context.get(start_index, False):
                        continue
                    if end_index < start_index:
                        continue
                    length = end_index - start_index + 1
                    if length > max_answer_length:
                        continue
                    prelim_predictions.append(
                        _PrelimPrediction(
                            feature_index=features_id,
                            start_index=start_index,
                            end_index=end_index,
                            start_logit=result.start_logits[start_index],
                            end_logit=result.end_logits[end_index]))

        if version_2:
            prelim_predictions.append(
                _PrelimPrediction(
                    feature_index=min_null_feature_index,
                    start_index=0,
                    end_index=0,
                    start_logit=null_start_logit,
                    end_logit=null_end_logit))

        prelim_predictions = sorted(
            prelim_predictions,
            key=lambda x: (x.start_logit + x.end_logit),
            reverse=True)

        seen_predictions = {}
        nbest = []
        for pred in prelim_predictions:
            if len(nbest) >= n_best_size:
                break
            feature = features[pred.feature_index]
            if pred.start_index > 0:  # this is a non-null prediction
                tok_tokens = feature.tokens[pred.start_index:(
                    pred.end_index + 1)]
                orig_doc_start = feature.token_to_orig_map[pred.start_index]
                orig_doc_end = feature.token_to_orig_map[pred.end_index]
                orig_tokens = feature.doc_tokens[orig_doc_start:(
                    orig_doc_end + 1)]
                tok_text = ' '.join(tok_tokens)

                # De-tokenize WordPieces that have been split off.
                tok_text = tok_text.replace(' ##', '')
                tok_text = tok_text.replace('##', '')

                # Clean whitespace
                tok_text = tok_text.strip()
                tok_text = ' '.join(tok_text.split())
                orig_text = ' '.join(orig_tokens)

                final_text = get_final_text(tok_text, orig_text, tokenizer)
                if final_text in seen_predictions:
                    continue

                seen_predictions[final_text] = True
            else:
                final_text = ''
                seen_predictions[final_text] = True

            nbest.append(
                _NbestPrediction(
                    text=final_text,
                    start_logit=pred.start_logit,
                    end_logit=pred.end_logit))

        # if we didn't inlude the empty option in the n-best, inlcude it
        if version_2:
            if '' not in seen_predictions:
                nbest.append(
                    _NbestPrediction(
                        text='',
                        start_logit=null_start_logit,
                        end_logit=null_end_logit))
        # In very rare edge cases we could have no valid predictions. So we
        # just create a nonce prediction in this case to avoid failure.
        if not nbest:
            nbest.append(
                _NbestPrediction(text='empty', start_logit=0.0, end_logit=0.0))

        assert len(nbest) >= 1

        total_scores = []
        best_non_null_entry = None
        for entry in nbest:
            total_scores.append(entry.start_logit + entry.end_logit)
            if not best_non_null_entry:
                if entry.text:
                    best_non_null_entry = entry

        probs = nd.softmax(nd.array(total_scores)).asnumpy()

        nbest_json = []
        for (i, entry) in enumerate(nbest):
            output = OrderedDict()
            output['text'] = entry.text
            output['probability'] = float(probs[i])
            output['start_logit'] = entry.start_logit
            output['end_logit'] = entry.end_logit
            nbest_json.append(output)

        if not version_2:
            all_predictions[example_qas_id] = nbest_json[0]['text']
        else:
            # predict '' iff the null score - the score of best non-null > threshold
            score_diff = score_null - best_non_null_entry.start_logit - \
                best_non_null_entry.end_logit

            scores_diff_json[example_qas_id] = score_diff
            if score_diff > null_score_diff_threshold:
                all_predictions[example_qas_id] = ''
            else:
                all_predictions[example_qas_id] = best_non_null_entry.text

        all_nbest_json[example_qas_id] = nbest_json
    return all_predictions, all_nbest_json, scores_diff_json
def predictions ( dev_dataset , all_results , tokenizer , max_answer_length = 64 , null_score_diff_threshold = 0.0 , n_best_size = 10 , version_2 = False ) : _PrelimPrediction = namedtuple ( 'PrelimPrediction' , [ 'feature_index' , 'start_index' , 'end_index' , 'start_logit' , 'end_logit' ] ) _NbestPrediction = namedtuple ( 'NbestPrediction' , [ 'text' , 'start_logit' , 'end_logit' ] ) all_predictions = OrderedDict ( ) all_nbest_json = OrderedDict ( ) scores_diff_json = OrderedDict ( ) for features in dev_dataset : results = all_results [ features [ 0 ] . example_id ] example_qas_id = features [ 0 ] . qas_id prelim_predictions = [ ] score_null = 1000000 min_null_feature_index = 0 null_start_logit = 0 null_end_logit = 0 for features_id , ( result , feature ) in enumerate ( zip ( results , features ) ) : start_indexes = _get_best_indexes ( result . start_logits , n_best_size ) end_indexes = _get_best_indexes ( result . end_logits , n_best_size ) if version_2 : feature_null_score = result . start_logits [ 0 ] + result . end_logits [ 0 ] if feature_null_score < score_null : score_null = feature_null_score min_null_feature_index = features_id null_start_logit = result . start_logits [ 0 ] null_end_logit = result . end_logits [ 0 ] for start_index in start_indexes : for end_index in end_indexes : if start_index >= len ( feature . tokens ) : continue if end_index >= len ( feature . tokens ) : continue if start_index not in feature . token_to_orig_map : continue if end_index not in feature . token_to_orig_map : continue if not feature . token_is_max_context . get ( start_index , False ) : continue if end_index < start_index : continue length = end_index - start_index + 1 if length > max_answer_length : continue prelim_predictions . append ( _PrelimPrediction ( feature_index = features_id , start_index = start_index , end_index = end_index , start_logit = result . start_logits [ start_index ] , end_logit = result . end_logits [ end_index ] ) ) if version_2 : prelim_predictions . append ( _PrelimPrediction ( feature_index = min_null_feature_index , start_index = 0 , end_index = 0 , start_logit = null_start_logit , end_logit = null_end_logit ) ) prelim_predictions = sorted ( prelim_predictions , key = lambda x : ( x . start_logit + x . end_logit ) , reverse = True ) seen_predictions = { } nbest = [ ] for pred in prelim_predictions : if len ( nbest ) >= n_best_size : break feature = features [ pred . feature_index ] if pred . start_index > 0 : tok_tokens = feature . tokens [ pred . start_index : ( pred . end_index + 1 ) ] orig_doc_start = feature . token_to_orig_map [ pred . start_index ] orig_doc_end = feature . token_to_orig_map [ pred . end_index ] orig_tokens = feature . doc_tokens [ orig_doc_start : ( orig_doc_end + 1 ) ] tok_text = ' ' . join ( tok_tokens ) tok_text = tok_text . replace ( ' ##' , '' ) tok_text = tok_text . replace ( '##' , '' ) tok_text = tok_text . strip ( ) tok_text = ' ' . join ( tok_text . split ( ) ) orig_text = ' ' . join ( orig_tokens ) final_text = get_final_text ( tok_text , orig_text , tokenizer ) if final_text in seen_predictions : continue seen_predictions [ final_text ] = True else : final_text = '' seen_predictions [ final_text ] = True nbest . append ( _NbestPrediction ( text = final_text , start_logit = pred . start_logit , end_logit = pred . end_logit ) ) if version_2 : if '' not in seen_predictions : nbest . append ( _NbestPrediction ( text = '' , start_logit = null_start_logit , end_logit = null_end_logit ) ) if not nbest : nbest . append ( _NbestPrediction ( text = 'empty' , start_logit = 0.0 , end_logit = 0.0 ) ) assert len ( nbest ) >= 1 total_scores = [ ] best_non_null_entry = None for entry in nbest : total_scores . append ( entry . start_logit + entry . end_logit ) if not best_non_null_entry : if entry . text : best_non_null_entry = entry probs = nd . softmax ( nd . array ( total_scores ) ) . asnumpy ( ) nbest_json = [ ] for ( i , entry ) in enumerate ( nbest ) : output = OrderedDict ( ) output [ 'text' ] = entry . text output [ 'probability' ] = float ( probs [ i ] ) output [ 'start_logit' ] = entry . start_logit output [ 'end_logit' ] = entry . end_logit nbest_json . append ( output ) if not version_2 : all_predictions [ example_qas_id ] = nbest_json [ 0 ] [ 'text' ] else : score_diff = score_null - best_non_null_entry . start_logit - best_non_null_entry . end_logit scores_diff_json [ example_qas_id ] = score_diff if score_diff > null_score_diff_threshold : all_predictions [ example_qas_id ] = '' else : all_predictions [ example_qas_id ] = best_non_null_entry . text all_nbest_json [ example_qas_id ] = nbest_json return all_predictions , all_nbest_json , scores_diff_json
Get prediction results

    Parameters
    ----------
    dev_dataset: dataset
        Examples of transform.
    all_results: dict
        A dictionary containing model prediction results.
    tokenizer: callable
        Tokenizer function.
    max_answer_length: int, default 64
        Maximum length of the answer tokens.
    null_score_diff_threshold: float, default 0.0
        If null_score - best_non_null is greater than the threshold predict null.
    n_best_size: int, default 10
        The total number of n-best predictions.
    version_2: bool, default False
        If true, the SQuAD examples contain some that do not have an answer.
    Returns
    -------
    all_predictions: dict
        All final predictions.
    all_nbest_json: dict
        All n-best predictions.
    scores_diff_json: dict
        Record the difference between null score and the score of best non-null.
        when version_2 is True.