def make_data_iter_plan(self):
        "make a random data iteration plan"
        # truncate each bucket into multiple of batch-size
        bucket_n_batches = []
        for i in range(len(self.data)):
            bucket_n_batches.append(np.floor((self.data[i]) / self.batch_size))
            self.data[i] = self.data[i][:int(bucket_n_batches[i]*self.batch_size)]

        bucket_plan = np.hstack([np.zeros(n, int)+i for i, n in enumerate(bucket_n_batches)])
        np.random.shuffle(bucket_plan)

        bucket_idx_all = [np.random.permutation(len(x)) for x in self.data]

        self.bucket_plan = bucket_plan
        self.bucket_idx_all = bucket_idx_all
        self.bucket_curr_idx = [0 for x in self.data]

        self.data_buffer = []
        self.label_buffer = []
        for i_bucket in range(len(self.data)):
            if not self.model_parallel:
                data = np.zeros((self.batch_size, self.buckets[i_bucket]))
                label = np.zeros((self.batch_size, self.buckets[i_bucket]))
                self.data_buffer.append(data)
                self.label_buffer.append(label)
            else:
                data = np.zeros((self.buckets[i_bucket], self.batch_size))
                self.data_buffer.append(data)

        if self.model_parallel:
            # Transpose data if model parallel
            for i in range(len(self.data)):
                bucket_data = self.data[i]
                self.data[i] = np.transpose(bucket_data)
def make_data_iter_plan ( self ) : bucket_n_batches = [ ] for i in range ( len ( self . data ) ) : bucket_n_batches . append ( np . floor ( ( self . data [ i ] ) / self . batch_size ) ) self . data [ i ] = self . data [ i ] [ : int ( bucket_n_batches [ i ] * self . batch_size ) ] bucket_plan = np . hstack ( [ np . zeros ( n , int ) + i for i , n in enumerate ( bucket_n_batches ) ] ) np . random . shuffle ( bucket_plan ) bucket_idx_all = [ np . random . permutation ( len ( x ) ) for x in self . data ] self . bucket_plan = bucket_plan self . bucket_idx_all = bucket_idx_all self . bucket_curr_idx = [ 0 for x in self . data ] self . data_buffer = [ ] self . label_buffer = [ ] for i_bucket in range ( len ( self . data ) ) : if not self . model_parallel : data = np . zeros ( ( self . batch_size , self . buckets [ i_bucket ] ) ) label = np . zeros ( ( self . batch_size , self . buckets [ i_bucket ] ) ) self . data_buffer . append ( data ) self . label_buffer . append ( label ) else : data = np . zeros ( ( self . buckets [ i_bucket ] , self . batch_size ) ) self . data_buffer . append ( data ) if self . model_parallel : for i in range ( len ( self . data ) ) : bucket_data = self . data [ i ] self . data [ i ] = np . transpose ( bucket_data )
make a random data iteration plan