    def __init__(
        self,
        data: List[torch.Tensor],  # ordered list of items
        bsz_per_shard,  # number of items processed per GPUs per forward
        shard_id,  # current GPU ID
        num_shards,  # number of GPUs
    ):
        super().__init__()
        self.data = data

        def batchify(data, bsz):
            # Work out how cleanly we can divide the dataset into bsz parts.
            nbatch = data.size(0) // bsz
            # Trim off any extra elements that wouldn't cleanly fit (remainders).
            data = data.narrow(0, 0, nbatch * bsz)
            # Evenly divide the data across the bsz batches.
            data = data.view(bsz, -1).contiguous()
            return data

        # total number of sequences processed by all GPUs in each forward pass
        global_batch_size = bsz_per_shard * num_shards

        """
        With a 16 item dataset, bsz_per_shard=2 and num_shards=3,
        *indices* might look like:

            indices = [[0, 1],
                       [2, 3],
                       [4, 5],
                       [6, 7],
                       [8, 9],
                       [10, 11]]

        The size of the TruncatedBPTTDataset instance will be 2,
        and shard 1 will see items:

            [(0, [data[4], data[6]]),
             (1, [data[5], data[7]])]
        """
        indices = batchify(torch.arange(len(data)), global_batch_size)
        assert indices.size(0) == global_batch_size

        self.my_indices = indices[
            shard_id * bsz_per_shard : (shard_id + 1) * bsz_per_shard
        ]
        assert self.my_indices.size(0) == bsz_per_shard
With a 16 item dataset, bsz_per_shard=2 and num_shards=3,
*indices* might look like:

indices = [[0, 1],
[2, 3],
[4, 5],
[6, 7],
[8, 9],
[10, 11]]

The size of the TruncatedBPTTDataset instance will be 2,
and shard 1 will see items:

[(0, [data[4], data[6]]),
(1, [data[5], data[7]])]
examples/truncated_bptt/truncated_bptt_lm_task.py