The model is revaluated for each test sample with the non-important features set to resample background values.

def keep_resample(nkeep, X_train, y_train, X_test, y_test, attr_test, model_generator, metric, trained_model, random_state):
    """ The model is revaluated for each test sample with the non-important features set to resample background values.
    """ # why broken? overwriting?

    X_train, X_test = to_array(X_train, X_test)

    # how many features to mask
    assert X_train.shape[1] == X_test.shape[1]

    # how many samples to take
    nsamples = 100

    # keep nkeep top features for each test explanation
    N,M = X_test.shape
    X_test_tmp = np.tile(X_test, [1, nsamples]).reshape(nsamples * N, M)
    tie_breaking_noise = const_rand(M) * 1e-6
    inds = sklearn.utils.resample(np.arange(N), n_samples=nsamples, random_state=random_state)
    for i in range(N):
        if nkeep[i] < M:
            ordering = np.argsort(-attr_test[i,:] + tie_breaking_noise)
            X_test_tmp[i*nsamples:(i+1)*nsamples, ordering[nkeep[i]:]] = X_train[inds, :][:, ordering[nkeep[i]:]]

    yp_masked_test = trained_model.predict(X_test_tmp)
    yp_masked_test = np.reshape(yp_masked_test, (N, nsamples)).mean(1) # take the mean output over all samples

    return metric(y_test, yp_masked_test)