def fill_up(self,
                bucket_batch_sizes: List[BucketBatchSize],
                seed: int = 42) -> 'ParallelDataSet':
        """
        Returns a new dataset with buckets filled up.

        :param bucket_batch_sizes: Bucket batch sizes.
        :param seed: The random seed used for sampling sentences to fill up.
        :return: New dataset with buckets filled up to the next multiple of batch size
        """
        source = list(self.source)
        target = list(self.target)
        label = list(self.label)

        rs = np.random.RandomState(seed)

        for bucket_idx in range(len(self)):
            bucket = bucket_batch_sizes[bucket_idx].bucket
            bucket_batch_size = bucket_batch_sizes[bucket_idx].batch_size
            bucket_source = self.source[bucket_idx]
            bucket_target = self.target[bucket_idx]
            bucket_label = self.label[bucket_idx]
            num_samples = bucket_source.shape[0]

            # Fill up the last batch by randomly sampling from the extant items.
            if num_samples % bucket_batch_size != 0:
                rest = bucket_batch_size - num_samples % bucket_batch_size
                desired_indices_np = rs.randint(num_samples, size=rest)
                desired_indices = mx.nd.array(desired_indices_np)

                if isinstance(source[bucket_idx], np.ndarray):
                    source[bucket_idx] = np.concatenate((bucket_source, bucket_source.take(desired_indices_np)), axis=0)
                else:
                    source[bucket_idx] = mx.nd.concat(bucket_source, bucket_source.take(desired_indices), dim=0)
                target[bucket_idx] = mx.nd.concat(bucket_target, bucket_target.take(desired_indices), dim=0)
                label[bucket_idx] = mx.nd.concat(bucket_label, bucket_label.take(desired_indices), dim=0)

        return ParallelDataSet(source, target, label)
def fill_up ( self , bucket_batch_sizes : List [ BucketBatchSize ] , seed : int = 42 ) -> 'ParallelDataSet' : source = list ( self . source ) target = list ( self . target ) label = list ( self . label ) rs = np . random . RandomState ( seed ) for bucket_idx in range ( len ( self ) ) : bucket = bucket_batch_sizes [ bucket_idx ] . bucket bucket_batch_size = bucket_batch_sizes [ bucket_idx ] . batch_size bucket_source = self . source [ bucket_idx ] bucket_target = self . target [ bucket_idx ] bucket_label = self . label [ bucket_idx ] num_samples = bucket_source . shape [ 0 ] if num_samples % bucket_batch_size != 0 : rest = bucket_batch_size - num_samples % bucket_batch_size desired_indices_np = rs . randint ( num_samples , size = rest ) desired_indices = mx . nd . array ( desired_indices_np ) if isinstance ( source [ bucket_idx ] , np . ndarray ) : source [ bucket_idx ] = np . concatenate ( ( bucket_source , bucket_source . take ( desired_indices_np ) ) , axis = 0 ) else : source [ bucket_idx ] = mx . nd . concat ( bucket_source , bucket_source . take ( desired_indices ) , dim = 0 ) target [ bucket_idx ] = mx . nd . concat ( bucket_target , bucket_target . take ( desired_indices ) , dim = 0 ) label [ bucket_idx ] = mx . nd . concat ( bucket_label , bucket_label . take ( desired_indices ) , dim = 0 ) return ParallelDataSet ( source , target , label )
Returns a new dataset with buckets filled up.

        :param bucket_batch_sizes: Bucket batch sizes.
        :param seed: The random seed used for sampling sentences to fill up.
        :return: New dataset with buckets filled up to the next multiple of batch size