original_docstring: This function prepares the datasets to be used in the brain class.
      It also defines the data processing pipeline through user-defined functions.
docstring: This function prepares the datasets to be used in the brain class.
  It also defines the data processing pipeline through user-defined functions.
docstring_tokens: This function prepares the datasets to be used in the brain class . It also defines the data processing pipeline through user - defined functions .
code: def dataio_prep(hparams):
      """This function prepares the datasets to be used in the brain class.
      It also defines the data processing pipeline through user-defined functions."""
  
      data_folder = hparams["data_folder"]
  
      # 1. Declarations:
      train_data = sb.dataio.dataset.DynamicItemDataset.from_json(
          json_path=hparams["train_annotation"],
          replacements={"data_root": data_folder},
      )
  
      if hparams["sorting"] == "ascending":
          # we sort training data to speed up training and get better results.
          train_data = train_data.filtered_sorted(sort_key="duration")
          # when sorting do not shuffle in dataloader ! otherwise is pointless
          hparams["dataloader_options"]["shuffle"] = False
  
      elif hparams["sorting"] == "descending":
          train_data = train_data.filtered_sorted(
              sort_key="duration", reverse=True
          )
          # when sorting do not shuffle in dataloader ! otherwise is pointless
          hparams["dataloader_options"]["shuffle"] = False
  
      elif hparams["sorting"] == "random":
          pass
  
      else:
          raise NotImplementedError(
              "sorting must be random, ascending or descending"
          )
  
      valid_data = sb.dataio.dataset.DynamicItemDataset.from_json(
          json_path=hparams["valid_annotation"],
          replacements={"data_root": data_folder},
      )
  
      test_data = sb.dataio.dataset.DynamicItemDataset.from_json(
          json_path=hparams["test_annotation"],
          replacements={"data_root": data_folder},
      )
  
      datasets = [train_data, valid_data, test_data]
      label_encoder = sb.dataio.encoder.TextEncoder()
  
      # 2. Define audio pipeline:
      @sb.utils.data_pipeline.takes("wav")
      @sb.utils.data_pipeline.provides("sig")
      def audio_pipeline(wav):
          sig = sb.dataio.dataio.read_audio(wav)
          return sig
  
      sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline)
  
      # 3. Define text pipeline:
      @sb.utils.data_pipeline.takes("phn")
      @sb.utils.data_pipeline.provides("phn_list", "phn_encoded")
      def text_pipeline(phn):
          phn_list = phn.strip().split()
          yield phn_list
          phn_encoded = label_encoder.encode_sequence_torch(phn_list)
          yield phn_encoded
  
      sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline)
  
      # 4. Define end sample pipeline
      # (end sample is used to retrieve the golden alignment )
  
      @sb.utils.data_pipeline.takes("ground_truth_phn_ends")
      @sb.utils.data_pipeline.provides("phn_ends")
      def phn_ends_pipeline(ground_truth_phn_ends):
          phn_ends = ground_truth_phn_ends.strip().split()
          phn_ends = [int(i) for i in phn_ends]
          phn_ends = torch.Tensor(phn_ends)
          return phn_ends
  
      sb.dataio.dataset.add_dynamic_item(datasets, phn_ends_pipeline)
  
      # 3. Fit encoder:
      # Load or compute the label encoder
      label_encoder_file = os.path.join(
          hparams["save_folder"], "label_encoder.txt"
      )
      if os.path.exists(label_encoder_file):
          label_encoder.load(label_encoder_file)
      else:
          label_encoder.update_from_didataset(train_data, output_key="phn_list")
          label_encoder.save(
              os.path.join(hparams["save_folder"], "label_encoder.txt")
          )
  
      # 6. Set output:
      sb.dataio.dataset.set_output_keys(
          datasets, ["id", "sig", "phn_encoded", "phn_ends"]
      )
  
      return train_data, valid_data, test_data, label_encoder
code_tokens: def dataio_prep ( hparams ) : data_folder = hparams [ "data_folder" ] train_data = sb . dataio . dataset . DynamicItemDataset . from_json ( json_path = hparams [ "train_annotation" ] , replacements = { "data_root" : data_folder } , ) if hparams [ "sorting" ] == "ascending" : train_data = train_data . filtered_sorted ( sort_key = "duration" ) hparams [ "dataloader_options" ] [ "shuffle" ] = False elif hparams [ "sorting" ] == "descending" : train_data = train_data . filtered_sorted ( sort_key = "duration" , reverse = True ) hparams [ "dataloader_options" ] [ "shuffle" ] = False elif hparams [ "sorting" ] == "random" : pass else : raise NotImplementedError ( "sorting must be random, ascending or descending" ) valid_data = sb . dataio . dataset . DynamicItemDataset . from_json ( json_path = hparams [ "valid_annotation" ] , replacements = { "data_root" : data_folder } , ) test_data = sb . dataio . dataset . DynamicItemDataset . from_json ( json_path = hparams [ "test_annotation" ] , replacements = { "data_root" : data_folder } , ) datasets = [ train_data , valid_data , test_data ] label_encoder = sb . dataio . encoder . TextEncoder ( ) @ sb . utils . data_pipeline . takes ( "wav" ) @ sb . utils . data_pipeline . provides ( "sig" ) def audio_pipeline ( wav ) : sig = sb . dataio . dataio . read_audio ( wav ) return sig sb . dataio . dataset . add_dynamic_item ( datasets , audio_pipeline ) @ sb . utils . data_pipeline . takes ( "phn" ) @ sb . utils . data_pipeline . provides ( "phn_list" , "phn_encoded" ) def text_pipeline ( phn ) : phn_list = phn . strip ( ) . split ( ) yield phn_list phn_encoded = label_encoder . encode_sequence_torch ( phn_list ) yield phn_encoded sb . dataio . dataset . add_dynamic_item ( datasets , text_pipeline ) @ sb . utils . data_pipeline . takes ( "ground_truth_phn_ends" ) @ sb . utils . data_pipeline . provides ( "phn_ends" ) def phn_ends_pipeline ( ground_truth_phn_ends ) : phn_ends = ground_truth_phn_ends . strip ( ) . split ( ) phn_ends = [ int ( i ) for i in phn_ends ] phn_ends = torch . Tensor ( phn_ends ) return phn_ends sb . dataio . dataset . add_dynamic_item ( datasets , phn_ends_pipeline ) label_encoder_file = os . path . join ( hparams [ "save_folder" ] , "label_encoder.txt" ) if os . path . exists ( label_encoder_file ) : label_encoder . load ( label_encoder_file ) else : label_encoder . update_from_didataset ( train_data , output_key = "phn_list" ) label_encoder . save ( os . path . join ( hparams [ "save_folder" ] , "label_encoder.txt" ) ) sb . dataio . dataset . set_output_keys ( datasets , [ "id" , "sig" , "phn_encoded" , "phn_ends" ] ) return train_data , valid_data , test_data , label_encoder
short_docstring: This function prepares the datasets to be used in the brain class.
short_docstring_tokens: This function prepares the datasets to be used in the brain class .
comment: """This function prepares the datasets to be used in the brain class.
      It also defines the data processing pipeline through user-defined functions."""$SEP$# 1. Declarations:$SEP$# we sort training data to speed up training and get better results.$SEP$# when sorting do not shuffle in dataloader ! otherwise is pointless$SEP$# when sorting do not shuffle in dataloader ! otherwise is pointless$SEP$# 2. Define audio pipeline:$SEP$# 3. Define text pipeline:$SEP$# 4. Define end sample pipeline$SEP$# (end sample is used to retrieve the golden alignment )$SEP$# 3. Fit encoder:$SEP$# Load or compute the label encoder$SEP$# 6. Set output: