original_docstring: Load a given dataset split.
  
          Args:
              split (str): name of the split (e.g., train, valid, test)
          
docstring: Load a given dataset split.
docstring_tokens: Load a given dataset split .
code: def load_dataset(self, split, combine=False):
          """Load a given dataset split.
  
          Args:
              split (str): name of the split (e.g., train, valid, test)
          """
  
          loaded_datasets = [[], []]
          loaded_labels = []
          stop = False
  
          for k in itertools.count():
              split_k = split + (str(k) if k > 0 else '')
              base_path = os.path.join(self.args.data, split_k)
              path1 = os.path.join(base_path + '_s1')
              path2 = os.path.join(base_path + '_s2')
  
              for path, datasets in zip([path1, path2], loaded_datasets):
                  if self.args.raw_text and IndexedRawTextDataset.exists(path):
                      ds = IndexedRawTextDataset(path, self.dictionary)
                  elif not self.args.raw_text and IndexedInMemoryDataset.exists(path):
                      ds = IndexedDataset(path, fix_lua_indexing=True)
                  else:
                      if k > 0:
                          stop = True
                          break
                      else:
                          raise FileNotFoundError('Dataset not found: {} ({})'.format(split, self.args.data))
  
                  datasets.append(
                      TokenBlockDataset(
                          ds, 0, pad=self.dictionary.pad(), eos=self.dictionary.eos(),
                          break_mode='eos', include_targets=False,
                      ))
  
              if stop:
                  break
              with open(base_path + '.lbl', 'r') as lbl_f:
                  lines = lbl_f.readlines()
                  cast = int if self.num_labels > 0 else float
                  loaded_labels.extend(cast(l) for l in lines)
  
              print('| {} {} {} examples'.format(self.args.data, split_k, len(loaded_datasets[0][-1])))
  
              if not combine:
                  break
  
          if self.num_labels == 2:
              loaded_labels = [l if l == 1 else 0 for l in loaded_labels]
  
          if len(loaded_datasets[0]) == 1:
              dataset1 = loaded_datasets[0][0]
              dataset2 = loaded_datasets[1][0]
              sizes1 = dataset1.sizes
              sizes2 = dataset2.sizes
          else:
              dataset1 = ConcatDataset(loaded_datasets[0])
              dataset2 = ConcatDataset(loaded_datasets[1])
              sizes1 = np.concatenate([ds.sizes for ds in loaded_datasets[0]])
              sizes2 = np.concatenate([ds.sizes for ds in loaded_datasets[1]])
  
          self.datasets[split] = SentencePairClassificationDataset(
              dataset1, dataset2, loaded_labels, sizes1, sizes2, self.dictionary, self.concat_sentences_mode,
              use_bos=self.use_bos
          )
code_tokens: def load_dataset ( self , split , combine = False ) : loaded_datasets = [ [ ] , [ ] ] loaded_labels = [ ] stop = False for k in itertools . count ( ) : split_k = split + ( str ( k ) if k > 0 else '' ) base_path = os . path . join ( self . args . data , split_k ) path1 = os . path . join ( base_path + '_s1' ) path2 = os . path . join ( base_path + '_s2' ) for path , datasets in zip ( [ path1 , path2 ] , loaded_datasets ) : if self . args . raw_text and IndexedRawTextDataset . exists ( path ) : ds = IndexedRawTextDataset ( path , self . dictionary ) elif not self . args . raw_text and IndexedInMemoryDataset . exists ( path ) : ds = IndexedDataset ( path , fix_lua_indexing = True ) else : if k > 0 : stop = True break else : raise FileNotFoundError ( 'Dataset not found: {} ({})' . format ( split , self . args . data ) ) datasets . append ( TokenBlockDataset ( ds , 0 , pad = self . dictionary . pad ( ) , eos = self . dictionary . eos ( ) , break_mode = 'eos' , include_targets = False , ) ) if stop : break with open ( base_path + '.lbl' , 'r' ) as lbl_f : lines = lbl_f . readlines ( ) cast = int if self . num_labels > 0 else float loaded_labels . extend ( cast ( l ) for l in lines ) print ( '| {} {} {} examples' . format ( self . args . data , split_k , len ( loaded_datasets [ 0 ] [ - 1 ] ) ) ) if not combine : break if self . num_labels == 2 : loaded_labels = [ l if l == 1 else 0 for l in loaded_labels ] if len ( loaded_datasets [ 0 ] ) == 1 : dataset1 = loaded_datasets [ 0 ] [ 0 ] dataset2 = loaded_datasets [ 1 ] [ 0 ] sizes1 = dataset1 . sizes sizes2 = dataset2 . sizes else : dataset1 = ConcatDataset ( loaded_datasets [ 0 ] ) dataset2 = ConcatDataset ( loaded_datasets [ 1 ] ) sizes1 = np . concatenate ( [ ds . sizes for ds in loaded_datasets [ 0 ] ] ) sizes2 = np . concatenate ( [ ds . sizes for ds in loaded_datasets [ 1 ] ] ) self . datasets [ split ] = SentencePairClassificationDataset ( dataset1 , dataset2 , loaded_labels , sizes1 , sizes2 , self . dictionary , self . concat_sentences_mode , use_bos = self . use_bos )
short_docstring: Load a given dataset split.
short_docstring_tokens: Load a given dataset split .
comment: """Load a given dataset split.
  
          Args:
              split (str): name of the split (e.g., train, valid, test)
          """