def allocate_buffers(self):
        "Create the ragged array that will be filled when we ask for items."
        if self.ite_len is None: len(self)
        self.idx   = LanguageModelPreLoader.CircularIndex(len(self.dataset.x.items), not self.backwards)
        self.batch = np.zeros((self.bs, self.bptt+1), dtype=np.int64)
        self.batch_x, self.batch_y = self.batch[:,0:self.bptt], self.batch[:,1:self.bptt+1]
        #ro: index of the text we're at inside our datasets for the various batches
        self.ro    = np.zeros(self.bs, dtype=np.int64)
        #ri: index of the token we're at inside our current text for the various batches
        self.ri    = np.zeros(self.bs, dtype=np.int)
def allocate_buffers ( self ) : if self . ite_len is None : len ( self ) self . idx = LanguageModelPreLoader . CircularIndex ( len ( self . dataset . x . items ) , not self . backwards ) self . batch = np . zeros ( ( self . bs , self . bptt + 1 ) , dtype = np . int64 ) self . batch_x , self . batch_y = self . batch [ : , 0 : self . bptt ] , self . batch [ : , 1 : self . bptt + 1 ] self . ro = np . zeros ( self . bs , dtype = np . int64 ) self . ri = np . zeros ( self . bs , dtype = np . int )
Create the ragged array that will be filled when we ask for items.