def load_dataset_raw_to_waveforms(
    file_name,
    dataset_size=None,
    need_waveform=True,
    sample_rate=16000,
    read_using_soundfile=False,
):
    """Load raw dataset from w2v tsv file. Optionally get waveforms"""
    data = []
    with open(file_name, "r") as fp:
        lines = fp.readlines()
        data = [
            os.path.join(lines[0].strip(), line.strip().split("\t")[0])
            for line in lines[1:]
        ]

    if dataset_size:
        data = data[:dataset_size]

    if not need_waveform:
        return data

    features = []
    if read_using_soundfile:
        for _i, d in enumerate(data):
            wav = sf.read(d)[0]
            if wav.ndim == 2:
                wav = wav.mean(-1)
            features.append(torch.from_numpy(wav).float().view(1, -1))
    else:
        for i, d in enumerate(data):
            _path, slice_ptr = parse_path(d)
            if len(slice_ptr) == 0:
                feat = get_waveform(
                    _path, always_2d=True, output_sample_rate=sample_rate
                )[0]
                features.append(
                    {
                        "id": i,
                        "net_input": {
                            "src_tokens": torch.tensor(feat),
                            "src_lengths": torch.tensor([feat.shape[1]]),
                        },
                    }
                )
            else:
                raise Exception("Currently unsupported data format")
    return features
Load raw dataset from w2v tsv file. Optionally get waveforms
examples/speech_to_speech/benchmarking/data_utils.py