def __split_batches(batches, buffer_size):

    batch_size = 0
    batch_data = []

    # First, pull off all the candidate batches
    while batches and (buffer_size is None or
                       batch_size < buffer_size):
        batch_data.append(batches.pop(0))
        batch_size += batch_length(batch_data[-1])

    # Merge the batches
    batch = dict()
    residual = dict()

    has_residual = False
    has_data = False

    for key in batch_data[0].keys():
        batch[key] = np.concatenate([data[key] for data in batch_data])

        residual[key] = batch[key][buffer_size:]

        if len(residual[key]):
            has_residual = True

        # Clip to the appropriate size
        batch[key] = batch[key][:buffer_size]

        if len(batch[key]):
            has_data = True

    if has_residual:
        batches.insert(0, residual)

    if not has_data:
        batch = None

    return batch, batches
Load Core result rows using fetchmany/streaming.
dataset/ETHPy150Open bmcfee/pescador/pescador/util.py/__split_batches--crossover with--dataset/ETHPy150Open zzzeek/sqlalchemy/examples/performance/large_resultsets.py/test_core_fetchmany_w_streaming