src/data/utils/chunk_dataset.py

from torch.utils.data.dataset import IterableDataset
from torch.utils.data.dataloader import DataLoader


# This is a pseudo class that constructs a dataset from chunks
class ChunkedDataset(IterableDataset):
    raise NotImplementedError("Please implement your own dataloading logic")


# This is a pseudo class that loads data in chunks from HDFS
class ChunkedDataLoader(DataLoader):
    raise NotImplementedError("Please implement your own dataloading logic")