from torch.utils.data import Dataset
from config import DATA_BASEDIR

brats_paths(DATA_BASEDIR)
def brats_paths(data_path):

    train_test_split = 0.8 # float between 0 and 1.
    fold_split = 5 # float between 0 and 1.

    parser.add_argument('--data', help='training data path', default="/data/dataset/BRATS2018/training/")
    parser.add_argument('--out', help="output path", default="./5fold")
    parser.add_argument('--fraction', help="precentage of validation data", default=10)
    args = parser.parse_args()
    data = {}
    
    HGG_filenames = glob.glob(os.path.join(data_path, "HGG", "*"))
    LGG_filenames = glob.glob(os.path.join(data_path, "LGG", "*"))
    np.random.shuffle(HGG_filenames)
    np.random.shuffle(LGG_filenames)
    
    val_length_HGG = len(HGG_filenames) // args.fraction
    val_length_LGG = len(LGG_filenames) // args.fraction

    print(len(HGG_filenames), len(LGG_filenames))

    for fold in range(5):
        data[f'fold{fold}'] = {}

        data['fold{}'.format(fold)]['val'] = HGG_filenames[0:val_length_HGG] + LGG_filenames[0:val_length_LGG]

        data['fold{}'.format(fold)]['training'] =  HGG_filenames[val_length_HGG:] + LGG_filenames[val_length_LGG:]

    with open(args.out+".pkl", 'wb') as f:
        pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)



class FaceLandmarksDataset(Dataset):

    def __init__(self, basedir, mode):
        """
        basedir="/data/dataset/BRATS2018/{mode}/{HGG/LGG}/patient_id/{flair/t1/t1ce/t2/seg}"
        mode: training/val/test
        """
        self.basedir = os.path.join(basedir, mode)
        self.mode = mode
    

    def load_5fold(self):
        with open(config.CROSS_VALIDATION_PATH, 'rb') as f:
            data = pickle.load(f)
        imgs = data["fold{}".format(config.FOLD)][self.mode]
        patient_ids = [x.split("/")[-1] for x in imgs]
        ret = []
        for idx, file_name in enumerate(imgs):
            data = {}
            data['image_data'] = {}
            data['file_name'] = file_name
            data['id'] = patient_ids[idx]
            # read modality
            mod = glob.glob(file_name+"/*.nii*")
            assert len(mod) >= 4  # 4mod +1gt
            for m in mod:
                if 'seg' in m:
                    data['gt'] = m
                else:
                    _m = m.split("/")[-1].split(".")[0].split("_")[-1]
                    data['image_data'][_m] = m
            if 'gt' in data:
                data['preprocessed'] = crop_brain_region(data['image_data'], data['gt'])
                del data['image_data']
                del data['gt']
            ret.append(data)
        return ret

    def load_3d(self):
        """
        dataset_mode: HGG/LGG/ALL
        return list(dict[patient_id][modality] = filename.nii.gz)
        """
        print("Data Folder: ", self.basedir)

        modalities = ['flair', 't1ce', 't1.', 't2']
        
        if 'training' in self.basedir:
            img_HGG = glob.glob(self.basedir+"/HGG/*")
            img_LGG = glob.glob(self.basedir+"/LGG/*")
            imgs = img_HGG + img_LGG
        else:
            imgs = glob.glob(self.basedir+"/*")
        imgs = [x for x in imgs if 'survival_evaluation.csv' not in x]
        
        patient_ids = [x.split("/")[-1] for x in imgs]
        ret = []
        print("Preprocessing Data ...")
        for idx, file_name in tqdm(enumerate(imgs), total=len(imgs)):
            data = {}
            data['image_data'] = {}
            data['file_name'] = file_name
            data['id'] = patient_ids[idx]
            # read modality
            mod = glob.glob(file_name+"/*.nii*")
            assert len(mod) >= 4, '{}'.format(file_name)  # 4mod +1gt
            for m in mod:
                if 'seg' in m:
                    data['gt'] = m
                else:
                    _m = m.split("/")[-1].split(".")[0].split("_")[-1]
                    data['image_data'][_m] = m
            
            if 'gt' in data:
                if not config.NO_CACHE and not 'training' in self.basedir:
                    data['preprocessed'] = crop_brain_region(data['image_data'], data['gt'])
                    del data['image_data']
                    del data['gt']
            else:
                data['preprocessed'] = crop_brain_region(data['image_data'], None, with_gt=False)
                del data['image_data']

            ret.append(data)
        return ret

    @staticmethod
    def load_from_file(basedir, names):
        brats = BRATS_SEG(basedir, names)
        return  brats.load_5fold()

    @staticmethod
    def load_many(basedir,names, add_gt=True, add_mask=False):
        """
        Load and merges several instance files together.
        """
        if not isinstance(names, (list, tuple)):
            names = [names]
        ret = []
        for n in names:
            brats = BRATS_SEG(basedir, n)
            ret.extend(brats.load_3d())
        return ret

    




    def __len__(self):
        return len(self.landmarks_frame)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        img_name = os.path.join(self.root_dir,
                                self.landmarks_frame.iloc[idx, 0])
        image = io.imread(img_name)
        landmarks = self.landmarks_frame.iloc[idx, 1:]
        landmarks = np.array([landmarks])
        landmarks = landmarks.astype('float').reshape(-1, 2)
        sample = {'image': image, 'landmarks': landmarks}

        if self.transform:
            sample = self.transform(sample)

        return sample












#!/usr/bin/env python
# -*- coding: utf-8 -*-
# File: data_loader.py

import numpy as np
import os
from termcolor import colored
from tabulate import tabulate

from tensorpack.utils import logger
# from tensorpack.utils.rect import FloatBox
from tensorpack.utils.timer import timed_operation
from tensorpack.utils.argtools import log_once

import random
import pickle
import glob
from tqdm import tqdm
import config
import cv2
import skimage
import nibabel
from utils import crop_brain_region

class BRATS_SEG(object):
    def __init__(self, basedir, mode):
        """
        basedir="/data/dataset/BRATS2018/{mode}/{HGG/LGG}/patient_id/{flair/t1/t1ce/t2/seg}"
        mode: training/val/test
        """
        self.basedir = os.path.join(basedir, mode)
        self.mode = mode
    
    def load_5fold(self):
        with open(config.CROSS_VALIDATION_PATH, 'rb') as f:
            data = pickle.load(f)
        imgs = data["fold{}".format(config.FOLD)][self.mode]
        patient_ids = [x.split("/")[-1] for x in imgs]
        ret = []
        for idx, file_name in enumerate(imgs):
            data = {}
            data['image_data'] = {}
            data['file_name'] = file_name
            data['id'] = patient_ids[idx]
            # read modality
            mod = glob.glob(file_name+"/*.nii*")
            assert len(mod) >= 4  # 4mod +1gt
            for m in mod:
                if 'seg' in m:
                    data['gt'] = m
                else:
                    _m = m.split("/")[-1].split(".")[0].split("_")[-1]
                    data['image_data'][_m] = m
            if 'gt' in data:
                data['preprocessed'] = crop_brain_region(data['image_data'], data['gt'])
                del data['image_data']
                del data['gt']
            ret.append(data)
        return ret

    def load_3d(self):
        """
        dataset_mode: HGG/LGG/ALL
        return list(dict[patient_id][modality] = filename.nii.gz)
        """
        print("Data Folder: ", self.basedir)

        modalities = ['flair', 't1ce', 't1.', 't2']
        
        if 'training' in self.basedir:
            img_HGG = glob.glob(self.basedir+"/HGG/*")
            img_LGG = glob.glob(self.basedir+"/LGG/*")
            imgs = img_HGG + img_LGG
        else:
            imgs = glob.glob(self.basedir+"/*")
        imgs = [x for x in imgs if 'survival_evaluation.csv' not in x]
        
        patient_ids = [x.split("/")[-1] for x in imgs]
        ret = []
        print("Preprocessing Data ...")
        for idx, file_name in tqdm(enumerate(imgs), total=len(imgs)):
            data = {}
            data['image_data'] = {}
            data['file_name'] = file_name
            data['id'] = patient_ids[idx]
            # read modality
            mod = glob.glob(file_name+"/*.nii*")
            assert len(mod) >= 4, '{}'.format(file_name)  # 4mod +1gt
            for m in mod:
                if 'seg' in m:
                    data['gt'] = m
                else:
                    _m = m.split("/")[-1].split(".")[0].split("_")[-1]
                    data['image_data'][_m] = m
            
            if 'gt' in data:
                if not config.NO_CACHE and not 'training' in self.basedir:
                    data['preprocessed'] = crop_brain_region(data['image_data'], data['gt'])
                    del data['image_data']
                    del data['gt']
            else:
                data['preprocessed'] = crop_brain_region(data['image_data'], None, with_gt=False)
                del data['image_data']

            ret.append(data)
        return ret

    @staticmethod
    def load_from_file(basedir, names):
        brats = BRATS_SEG(basedir, names)
        return  brats.load_5fold()

    @staticmethod
    def load_many(basedir,names, add_gt=True, add_mask=False):
        """
        Load and merges several instance files together.
        """
        if not isinstance(names, (list, tuple)):
            names = [names]
        ret = []
        for n in names:
            brats = BRATS_SEG(basedir, n)
            ret.extend(brats.load_3d())
        return ret

if __name__ == '__main__':
    brats2018 = BRATS_SEG("/data/dataset/BRATS2018/", "training")
    brats2018 = brats2018.load_3d()
    print(len(brats2018))
    print(brats2018[0])



""" TODO: rewrite this with pytorch dataloaders """


#####
# some functions are borrowed from https://github.com/taigw/brats17/
#####

import cv2
import numpy as np
import copy
import os, sys
from tqdm import tqdm
# from tensorpack.utils.argtools import memoized, log_once
# from tensorpack.dataflow import (ProxyDataFlow,
#     MapData, TestDataSpeed, MultiProcessMapData,
#     MapDataComponent, DataFromList, PrefetchDataZMQ)
from  data_loader import BRATS_SEG
import config
from tensorpack.dataflow import RNGDataFlow
import nibabel
import random
import time
from utils import *
import six

class BatchData(ProxyDataFlow):
    """
    Stack datapoints into batches.
    It produces datapoints of the same number of components as ``ds``, but
    each component has one new extra dimension of size ``batch_size``.
    The batch can be either a list of original components, or (by default)
    a numpy array of original components.
    """

    def __init__(self, ds, batch_size, remainder=False, use_list=False):
        """
        Args:
            ds (DataFlow): When ``use_list=False``, the components of ``ds``
                must be either scalars or :class:`np.ndarray`, and have to be consistent in shapes.
            batch_size(int): batch size
            remainder (bool): When the remaining datapoints in ``ds`` is not
                enough to form a batch, whether or not to also produce the remaining
                data as a smaller batch.
                If set to False, all produced datapoints are guaranteed to have the same batch size.
                If set to True, `ds.size()` must be accurate.
            use_list (bool): if True, each component will contain a list
                of datapoints instead of an numpy array of an extra dimension.
        """
        super(BatchData, self).__init__(ds)
        if not remainder:
            try:
                assert batch_size <= ds.size()
            except NotImplementedError:
                pass
        self.batch_size = int(batch_size)
        self.remainder = remainder
        self.use_list = use_list

    def size(self):
        ds_size = self.ds.size()
        div = ds_size // self.batch_size
        rem = ds_size % self.batch_size
        if rem == 0:
            return div
        return div + int(self.remainder)

    def get_data(self):
        """
        Yields:
            Batched data by stacking each component on an extra 0th dimension.
        """
        holder = []
        for data in self.ds.get_data():
            if config.DATA_SAMPLING == 'all_positive':
                if data[2].sum() == 0:
                    continue
            elif config.DATA_SAMPLING == 'one_positive':
                if len(holder) == self.batch_size - 1:
                    # force to contain label
                    t = [x[2].sum() for x in holder]
                    if sum(t) == 0 and data[2].sum() == 0:
                        continue
            else:
                pass
            holder.append(data)
            if len(holder) == self.batch_size:
                yield BatchData._aggregate_batch(holder, self.use_list)
                del holder[:]
        if self.remainder and len(holder) > 0:
            yield BatchData._aggregate_batch(holder, self.use_list)

    @staticmethod
    def _aggregate_batch(data_holder, use_list=False):
        size = len(data_holder[0])
        result = []
        for k in range(size):
            if use_list:
                result.append(
                    [x[k] for x in data_holder])
            else:
                dt = data_holder[0][k]
                if type(dt) in list(six.integer_types) + [bool]:
                    tp = 'int32'
                elif type(dt) == float:
                    tp = 'float32'
                else:
                    try:
                        tp = dt.dtype
                    except AttributeError:
                        raise TypeError("Unsupported type to batch: {}".format(type(dt)))
                try:
                    result.append(
                        np.asarray([x[k] for x in data_holder], dtype=tp))
                except Exception as e:  # noqa
                    logger.exception("Cannot batch data. Perhaps they are of inconsistent shape?")
                    if isinstance(dt, np.ndarray):
                        s = pprint.pformat([x[k].shape for x in data_holder])
                        logger.error("Shape of all arrays to be batched: " + s)
                    try:
                        # open an ipython shell if possible
                        import IPython as IP; IP.embed()    # noqa
                    except ImportError:
                        pass
        return result

class DataFromListOfDict(RNGDataFlow):
    def __init__(self, lst, keys, shuffle=False):
        self._lst = lst
        self._keys = keys
        self._shuffle = shuffle
        self._size = len(lst)

    def size(self):
        return self._size

    def get_data(self):
        if self._shuffle:
            self.rng.shuffle(self._lst)
        for dic in self._lst:
            dp = [dic[k] for k in self._keys]
            yield dp

def sampler3d(volume_list, label, weight):
    """
    sample 3d volume to size (depth, h, w, 4)
    """
    margin = 5
    volume_list = transpose_volumes(volume_list, config.DIRECTION)
    data_shape = config.PATCH_SIZE
    label_shape = config.PATCH_SIZE
    data_slice_number = data_shape[0]
    label_slice_number = label_shape[0]
    volume_shape = volume_list[0].shape
    sub_data_shape = [data_slice_number, data_shape[1], data_shape[2]]
    sub_label_shape =[label_slice_number, label_shape[1], label_shape[2]]

    label = transpose_volumes(label, config.DIRECTION)
    center_point = get_random_roi_sampling_center(volume_shape, sub_label_shape, "full", None)
    sub_label = extract_roi_from_volume(label,
                                        center_point,
                                        sub_label_shape,
                                            fill = 'zero')
    sub_data = []
    flip = False
    for moda in range(len(volume_list)):
        sub_data_moda = extract_roi_from_volume(volume_list[moda],center_point,sub_data_shape)
        if(flip):
            sub_data_moda = np.flip(sub_data_moda, -1)
        sub_data.append(sub_data_moda)
    sub_data = np.array(sub_data) #4, depth, h, w

    weight = transpose_volumes(weight, config.DIRECTION)
    sub_weight = extract_roi_from_volume(weight,
                                            center_point,
                                            sub_label_shape,
                                            fill = 'zero')
    if(flip):
        sub_weight = np.flip(sub_weight, -1)
    
    if(flip):
        sub_label = np.flip(sub_label, -1)
    batch = {}
    axis = [1,2,3,0] #[1,2,3,0] [d, h, w, modalities]
    batch['images']  = np.transpose(sub_data, axis)
    batch['weights'] = np.transpose(sub_weight[np.newaxis, ...], axis)
    batch['labels']  = np.transpose(sub_label[np.newaxis, ...], axis)
    # other meta info ?
    return batch

def sampler3d_whole(volume_list, label, weight, original_shape, bbox):
    """
    mods = sorted(im.keys())
    volume_list = []
    for mod_idx, mod in enumerate(mods):
        filename = im[mod]
        volume = load_nifty_volume_as_array(filename, with_header=False)
        if mod_idx == 0:
            # contain whole tumor
            margin = 5 # small padding value
            original_shape = volume.shape
            bbmin, bbmax = get_none_zero_region(volume, margin)
        volume = crop_ND_volume_with_bounding_box(volume, bbmin, bbmax)
        if mod_idx == 0:
            weight = np.asarray(volume > 0, np.float32)
        if config.INTENSITY_NORM == 'modality':
            volume = itensity_normalize_one_volume(volume)        
        volume_list.append(volume)
    """
    sub_data = np.array(volume_list)
    batch = {}
    axis = [1,2,3,0] #[1,2,3,0] [d, h, w, modalities]
    batch['images']  = np.transpose(sub_data, axis)
    batch['weights'] = np.transpose(weight[np.newaxis, ...], axis)
    batch['original_shape'] = original_shape
    batch['bbox'] = bbox
    
    return batch

def get_train_dataflow(add_mask=True):
    """
    
    """
    if config.CROSS_VALIDATION:
        imgs = BRATS_SEG.load_from_file(config.BASEDIR, config.TRAIN_DATASET)
    else:
        imgs = BRATS_SEG.load_many(
            config.BASEDIR, config.TRAIN_DATASET, add_gt=False, add_mask=add_mask)
    # no filter for training
    imgs = list(imgs) 

    ds = DataFromList(imgs, shuffle=True)
    
    def preprocess(data):
        if config.NO_CACHE:
            fname, gt, im = data['file_name'], data['gt'], data['image_data']
            volume_list, label, weight, _, _ = crop_brain_region(im, gt)
            batch = sampler3d(volume_list, label, weight)
        else:
            volume_list, label, weight, _, _ = data['preprocessed']
            batch = sampler3d(volume_list, label, weight)
        return [batch['images'], batch['weights'], batch['labels']]
        
    ds = BatchData(MapData(ds, preprocess), config.BATCH_SIZE)
    ds = PrefetchDataZMQ(ds, 6)
    return ds

def get_eval_dataflow():
    #if config.CROSS_VALIDATION:
    imgs = BRATS_SEG.load_from_file(config.BASEDIR, config.VAL_DATASET)
    # no filter for training
    ds = DataFromListOfDict(imgs, ['file_name', 'id', 'preprocessed'])

    def f(data):
        volume_list, label, weight, original_shape, bbox = data
        batch = sampler3d_whole(volume_list, label, weight, original_shape, bbox)
        return batch
    ds = MapDataComponent(ds, f, 2)
    ds = PrefetchDataZMQ(ds, 1)
    return ds

def get_test_dataflow():
    imgs = BRATS_SEG.load_many(config.BASEDIR, config.TEST_DATASET, add_gt=False)
    # no filter for training
    ds = DataFromListOfDict(imgs, ['file_name', 'id', 'preprocessed'])

    def f(data):
        volume_list, label, weight, original_shape, bbox = data
        batch = sampler3d_whole(volume_list, label, weight, original_shape, bbox)
        return batch
    ds = MapDataComponent(ds, f, 2)
    ds = PrefetchDataZMQ(ds, 1)
    return ds

if __name__ == "__main__":
    df = get_train_dataflow()
    df.reset_state()
    for i in tqdm(df.get_data()):
        pass
        """
        for k in i:
            arr = i[k][0]
            arr = np.rot90(arr, k=2, axes= (1,2))
            OUTPUT_AFFINE = np.array(
                    [[0, 0, 1, 0],
                    [0, 1, 0, 0],
                    [1, 0, 0, 0],
                    [0, 0, 0, 1]])
            if k == 'labels':
                img = nibabel.Nifti1Image(arr.astype(np.int32), OUTPUT_AFFINE)
            else:
                img = nibabel.Nifti1Image(arr.astype(np.float32), OUTPUT_AFFINE)
            #nibabel.save(img, "./debug_output/{}.nii".format(k))
        break
        """

from torch.utils.data import Dataset
from config import DATA_BASEDIR

brats_paths(DATA_BASEDIR)
def brats_paths(data_path):

    train_test_split = 0.8 # float between 0 and 1.
    fold_split = 5 # float between 0 and 1.

    parser.add_argument('--data', help='training data path', default="/data/dataset/BRATS2018/training/")
    parser.add_argument('--out', help="output path", default="./5fold")
    parser.add_argument('--fraction', help="precentage of validation data", default=10)
    args = parser.parse_args()
    data = {}
    
    HGG_filenames = glob.glob(os.path.join(data_path, "HGG", "*"))
    LGG_filenames = glob.glob(os.path.join(data_path, "LGG", "*"))
    np.random.shuffle(HGG_filenames)
    np.random.shuffle(LGG_filenames)
    
    val_length_HGG = len(HGG_filenames) // args.fraction
    val_length_LGG = len(LGG_filenames) // args.fraction

    print(len(HGG_filenames), len(LGG_filenames))

    for fold in range(5):
        data[f'fold{fold}'] = {}

        data['fold{}'.format(fold)]['val'] = HGG_filenames[0:val_length_HGG] + LGG_filenames[0:val_length_LGG]

        data['fold{}'.format(fold)]['training'] =  HGG_filenames[val_length_HGG:] + LGG_filenames[val_length_LGG:]

    with open(args.out+".pkl", 'wb') as f:
        pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)



class FaceLandmarksDataset(Dataset):

    def __init__(self, basedir, mode):
        """
        basedir="/data/dataset/BRATS2018/{mode}/{HGG/LGG}/patient_id/{flair/t1/t1ce/t2/seg}"
        mode: training/val/test
        """
        self.basedir = os.path.join(basedir, mode)
        self.mode = mode
    

    def load_5fold(self):
        with open(config.CROSS_VALIDATION_PATH, 'rb') as f:
            data = pickle.load(f)
        imgs = data["fold{}".format(config.FOLD)][self.mode]
        patient_ids = [x.split("/")[-1] for x in imgs]
        ret = []
        for idx, file_name in enumerate(imgs):
            data = {}
            data['image_data'] = {}
            data['file_name'] = file_name
            data['id'] = patient_ids[idx]
            # read modality
            mod = glob.glob(file_name+"/*.nii*")
            assert len(mod) >= 4  # 4mod +1gt
            for m in mod:
                if 'seg' in m:
                    data['gt'] = m
                else:
                    _m = m.split("/")[-1].split(".")[0].split("_")[-1]
                    data['image_data'][_m] = m
            if 'gt' in data:
                data['preprocessed'] = crop_brain_region(data['image_data'], data['gt'])
                del data['image_data']
                del data['gt']
            ret.append(data)
        return ret

    def load_3d(self):
        """
        dataset_mode: HGG/LGG/ALL
        return list(dict[patient_id][modality] = filename.nii.gz)
        """
        print("Data Folder: ", self.basedir)

        modalities = ['flair', 't1ce', 't1.', 't2']
        
        if 'training' in self.basedir:
            img_HGG = glob.glob(self.basedir+"/HGG/*")
            img_LGG = glob.glob(self.basedir+"/LGG/*")
            imgs = img_HGG + img_LGG
        else:
            imgs = glob.glob(self.basedir+"/*")
        imgs = [x for x in imgs if 'survival_evaluation.csv' not in x]
        
        patient_ids = [x.split("/")[-1] for x in imgs]
        ret = []
        print("Preprocessing Data ...")
        for idx, file_name in tqdm(enumerate(imgs), total=len(imgs)):
            data = {}
            data['image_data'] = {}
            data['file_name'] = file_name
            data['id'] = patient_ids[idx]
            # read modality
            mod = glob.glob(file_name+"/*.nii*")
            assert len(mod) >= 4, '{}'.format(file_name)  # 4mod +1gt
            for m in mod:
                if 'seg' in m:
                    data['gt'] = m
                else:
                    _m = m.split("/")[-1].split(".")[0].split("_")[-1]
                    data['image_data'][_m] = m
            
            if 'gt' in data:
                if not config.NO_CACHE and not 'training' in self.basedir:
                    data['preprocessed'] = crop_brain_region(data['image_data'], data['gt'])
                    del data['image_data']
                    del data['gt']
            else:
                data['preprocessed'] = crop_brain_region(data['image_data'], None, with_gt=False)
                del data['image_data']

            ret.append(data)
        return ret

    @staticmethod
    def load_from_file(basedir, names):
        brats = BRATS_SEG(basedir, names)
        return  brats.load_5fold()

    @staticmethod
    def load_many(basedir,names, add_gt=True, add_mask=False):
        """
        Load and merges several instance files together.
        """
        if not isinstance(names, (list, tuple)):
            names = [names]
        ret = []
        for n in names:
            brats = BRATS_SEG(basedir, n)
            ret.extend(brats.load_3d())
        return ret

    




    def __len__(self):
        return len(self.landmarks_frame)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        img_name = os.path.join(self.root_dir,
                                self.landmarks_frame.iloc[idx, 0])
        image = io.imread(img_name)
        landmarks = self.landmarks_frame.iloc[idx, 1:]
        landmarks = np.array([landmarks])
        landmarks = landmarks.astype('float').reshape(-1, 2)
        sample = {'image': image, 'landmarks': landmarks}

        if self.transform:
            sample = self.transform(sample)

        return sample












#!/usr/bin/env python
# -*- coding: utf-8 -*-
# File: data_loader.py

import numpy as np
import os
from termcolor import colored
from tabulate import tabulate

from tensorpack.utils import logger
# from tensorpack.utils.rect import FloatBox
from tensorpack.utils.timer import timed_operation
from tensorpack.utils.argtools import log_once

import random
import pickle
import glob
from tqdm import tqdm
import config
import cv2
import skimage
import nibabel
from utils import crop_brain_region

class BRATS_SEG(object):
    def __init__(self, basedir, mode):
        """
        basedir="/data/dataset/BRATS2018/{mode}/{HGG/LGG}/patient_id/{flair/t1/t1ce/t2/seg}"
        mode: training/val/test
        """
        self.basedir = os.path.join(basedir, mode)
        self.mode = mode
    
    def load_5fold(self):
        with open(config.CROSS_VALIDATION_PATH, 'rb') as f:
            data = pickle.load(f)
        imgs = data["fold{}".format(config.FOLD)][self.mode]
        patient_ids = [x.split("/")[-1] for x in imgs]
        ret = []
        for idx, file_name in enumerate(imgs):
            data = {}
            data['image_data'] = {}
            data['file_name'] = file_name
            data['id'] = patient_ids[idx]
            # read modality
            mod = glob.glob(file_name+"/*.nii*")
            assert len(mod) >= 4  # 4mod +1gt
            for m in mod:
                if 'seg' in m:
                    data['gt'] = m
                else:
                    _m = m.split("/")[-1].split(".")[0].split("_")[-1]
                    data['image_data'][_m] = m
            if 'gt' in data:
                data['preprocessed'] = crop_brain_region(data['image_data'], data['gt'])
                del data['image_data']
                del data['gt']
            ret.append(data)
        return ret

    def load_3d(self):
        """
        dataset_mode: HGG/LGG/ALL
        return list(dict[patient_id][modality] = filename.nii.gz)
        """
        print("Data Folder: ", self.basedir)

        modalities = ['flair', 't1ce', 't1.', 't2']
        
        if 'training' in self.basedir:
            img_HGG = glob.glob(self.basedir+"/HGG/*")
            img_LGG = glob.glob(self.basedir+"/LGG/*")
            imgs = img_HGG + img_LGG
        else:
            imgs = glob.glob(self.basedir+"/*")
        imgs = [x for x in imgs if 'survival_evaluation.csv' not in x]
        
        patient_ids = [x.split("/")[-1] for x in imgs]
        ret = []
        print("Preprocessing Data ...")
        for idx, file_name in tqdm(enumerate(imgs), total=len(imgs)):
            data = {}
            data['image_data'] = {}
            data['file_name'] = file_name
            data['id'] = patient_ids[idx]
            # read modality
            mod = glob.glob(file_name+"/*.nii*")
            assert len(mod) >= 4, '{}'.format(file_name)  # 4mod +1gt
            for m in mod:
                if 'seg' in m:
                    data['gt'] = m
                else:
                    _m = m.split("/")[-1].split(".")[0].split("_")[-1]
                    data['image_data'][_m] = m
            
            if 'gt' in data:
                if not config.NO_CACHE and not 'training' in self.basedir:
                    data['preprocessed'] = crop_brain_region(data['image_data'], data['gt'])
                    del data['image_data']
                    del data['gt']
            else:
                data['preprocessed'] = crop_brain_region(data['image_data'], None, with_gt=False)
                del data['image_data']

            ret.append(data)
        return ret

    @staticmethod
    def load_from_file(basedir, names):
        brats = BRATS_SEG(basedir, names)
        return  brats.load_5fold()

    @staticmethod
    def load_many(basedir,names, add_gt=True, add_mask=False):
        """
        Load and merges several instance files together.
        """
        if not isinstance(names, (list, tuple)):
            names = [names]
        ret = []
        for n in names:
            brats = BRATS_SEG(basedir, n)
            ret.extend(brats.load_3d())
        return ret

if __name__ == '__main__':
    brats2018 = BRATS_SEG("/data/dataset/BRATS2018/", "training")
    brats2018 = brats2018.load_3d()
    print(len(brats2018))
    print(brats2018[0])



""" TODO: rewrite this with pytorch dataloaders """


#####
# some functions are borrowed from https://github.com/taigw/brats17/
#####

import cv2
import numpy as np
import copy
import os, sys
from tqdm import tqdm
# from tensorpack.utils.argtools import memoized, log_once
# from tensorpack.dataflow import (ProxyDataFlow,
#     MapData, TestDataSpeed, MultiProcessMapData,
#     MapDataComponent, DataFromList, PrefetchDataZMQ)
from  data_loader import BRATS_SEG
import config
from tensorpack.dataflow import RNGDataFlow
import nibabel
import random
import time
from utils import *
import six

class BatchData(ProxyDataFlow):
    """
    Stack datapoints into batches.
    It produces datapoints of the same number of components as ``ds``, but
    each component has one new extra dimension of size ``batch_size``.
    The batch can be either a list of original components, or (by default)
    a numpy array of original components.
    """

    def __init__(self, ds, batch_size, remainder=False, use_list=False):
        """
        Args:
            ds (DataFlow): When ``use_list=False``, the components of ``ds``
                must be either scalars or :class:`np.ndarray`, and have to be consistent in shapes.
            batch_size(int): batch size
            remainder (bool): When the remaining datapoints in ``ds`` is not
                enough to form a batch, whether or not to also produce the remaining
                data as a smaller batch.
                If set to False, all produced datapoints are guaranteed to have the same batch size.
                If set to True, `ds.size()` must be accurate.
            use_list (bool): if True, each component will contain a list
                of datapoints instead of an numpy array of an extra dimension.
        """
        super(BatchData, self).__init__(ds)
        if not remainder:
            try:
                assert batch_size <= ds.size()
            except NotImplementedError:
                pass
        self.batch_size = int(batch_size)
        self.remainder = remainder
        self.use_list = use_list

    def size(self):
        ds_size = self.ds.size()
        div = ds_size // self.batch_size
        rem = ds_size % self.batch_size
        if rem == 0:
            return div
        return div + int(self.remainder)

    def get_data(self):
        """
        Yields:
            Batched data by stacking each component on an extra 0th dimension.
        """
        holder = []
        for data in self.ds.get_data():
            if config.DATA_SAMPLING == 'all_positive':
                if data[2].sum() == 0:
                    continue
            elif config.DATA_SAMPLING == 'one_positive':
                if len(holder) == self.batch_size - 1:
                    # force to contain label
                    t = [x[2].sum() for x in holder]
                    if sum(t) == 0 and data[2].sum() == 0:
                        continue
            else:
                pass
            holder.append(data)
            if len(holder) == self.batch_size:
                yield BatchData._aggregate_batch(holder, self.use_list)
                del holder[:]
        if self.remainder and len(holder) > 0:
            yield BatchData._aggregate_batch(holder, self.use_list)

    @staticmethod
    def _aggregate_batch(data_holder, use_list=False):
        size = len(data_holder[0])
        result = []
        for k in range(size):
            if use_list:
                result.append(
                    [x[k] for x in data_holder])
            else:
                dt = data_holder[0][k]
                if type(dt) in list(six.integer_types) + [bool]:
                    tp = 'int32'
                elif type(dt) == float:
                    tp = 'float32'
                else:
                    try:
                        tp = dt.dtype
                    except AttributeError:
                        raise TypeError("Unsupported type to batch: {}".format(type(dt)))
                try:
                    result.append(
                        np.asarray([x[k] for x in data_holder], dtype=tp))
                except Exception as e:  # noqa
                    logger.exception("Cannot batch data. Perhaps they are of inconsistent shape?")
                    if isinstance(dt, np.ndarray):
                        s = pprint.pformat([x[k].shape for x in data_holder])
                        logger.error("Shape of all arrays to be batched: " + s)
                    try:
                        # open an ipython shell if possible
                        import IPython as IP; IP.embed()    # noqa
                    except ImportError:
                        pass
        return result

class DataFromListOfDict(RNGDataFlow):
    def __init__(self, lst, keys, shuffle=False):
        self._lst = lst
        self._keys = keys
        self._shuffle = shuffle
        self._size = len(lst)

    def size(self):
        return self._size

    def get_data(self):
        if self._shuffle:
            self.rng.shuffle(self._lst)
        for dic in self._lst:
            dp = [dic[k] for k in self._keys]
            yield dp

def sampler3d(volume_list, label, weight):
    """
    sample 3d volume to size (depth, h, w, 4)
    """
    margin = 5
    volume_list = transpose_volumes(volume_list, config.DIRECTION)
    data_shape = config.PATCH_SIZE
    label_shape = config.PATCH_SIZE
    data_slice_number = data_shape[0]
    label_slice_number = label_shape[0]
    volume_shape = volume_list[0].shape
    sub_data_shape = [data_slice_number, data_shape[1], data_shape[2]]
    sub_label_shape =[label_slice_number, label_shape[1], label_shape[2]]

    label = transpose_volumes(label, config.DIRECTION)
    center_point = get_random_roi_sampling_center(volume_shape, sub_label_shape, "full", None)
    sub_label = extract_roi_from_volume(label,
                                        center_point,
                                        sub_label_shape,
                                            fill = 'zero')
    sub_data = []
    flip = False
    for moda in range(len(volume_list)):
        sub_data_moda = extract_roi_from_volume(volume_list[moda],center_point,sub_data_shape)
        if(flip):
            sub_data_moda = np.flip(sub_data_moda, -1)
        sub_data.append(sub_data_moda)
    sub_data = np.array(sub_data) #4, depth, h, w

    weight = transpose_volumes(weight, config.DIRECTION)
    sub_weight = extract_roi_from_volume(weight,
                                            center_point,
                                            sub_label_shape,
                                            fill = 'zero')
    if(flip):
        sub_weight = np.flip(sub_weight, -1)
    
    if(flip):
        sub_label = np.flip(sub_label, -1)
    batch = {}
    axis = [1,2,3,0] #[1,2,3,0] [d, h, w, modalities]
    batch['images']  = np.transpose(sub_data, axis)
    batch['weights'] = np.transpose(sub_weight[np.newaxis, ...], axis)
    batch['labels']  = np.transpose(sub_label[np.newaxis, ...], axis)
    # other meta info ?
    return batch

def sampler3d_whole(volume_list, label, weight, original_shape, bbox):
    """
    mods = sorted(im.keys())
    volume_list = []
    for mod_idx, mod in enumerate(mods):
        filename = im[mod]
        volume = load_nifty_volume_as_array(filename, with_header=False)
        if mod_idx == 0:
            # contain whole tumor
            margin = 5 # small padding value
            original_shape = volume.shape
            bbmin, bbmax = get_none_zero_region(volume, margin)
        volume = crop_ND_volume_with_bounding_box(volume, bbmin, bbmax)
        if mod_idx == 0:
            weight = np.asarray(volume > 0, np.float32)
        if config.INTENSITY_NORM == 'modality':
            volume = itensity_normalize_one_volume(volume)        
        volume_list.append(volume)
    """
    sub_data = np.array(volume_list)
    batch = {}
    axis = [1,2,3,0] #[1,2,3,0] [d, h, w, modalities]
    batch['images']  = np.transpose(sub_data, axis)
    batch['weights'] = np.transpose(weight[np.newaxis, ...], axis)
    batch['original_shape'] = original_shape
    batch['bbox'] = bbox
    
    return batch

def get_train_dataflow(add_mask=True):
    """
    
    """
    if config.CROSS_VALIDATION:
        imgs = BRATS_SEG.load_from_file(config.BASEDIR, config.TRAIN_DATASET)
    else:
        imgs = BRATS_SEG.load_many(
            config.BASEDIR, config.TRAIN_DATASET, add_gt=False, add_mask=add_mask)
    # no filter for training
    imgs = list(imgs) 

    ds = DataFromList(imgs, shuffle=True)
    
    def preprocess(data):
        if config.NO_CACHE:
            fname, gt, im = data['file_name'], data['gt'], data['image_data']
            volume_list, label, weight, _, _ = crop_brain_region(im, gt)
            batch = sampler3d(volume_list, label, weight)
        else:
            volume_list, label, weight, _, _ = data['preprocessed']
            batch = sampler3d(volume_list, label, weight)
        return [batch['images'], batch['weights'], batch['labels']]
        
    ds = BatchData(MapData(ds, preprocess), config.BATCH_SIZE)
    ds = PrefetchDataZMQ(ds, 6)
    return ds

def get_eval_dataflow():
    #if config.CROSS_VALIDATION:
    imgs = BRATS_SEG.load_from_file(config.BASEDIR, config.VAL_DATASET)
    # no filter for training
    ds = DataFromListOfDict(imgs, ['file_name', 'id', 'preprocessed'])

    def f(data):
        volume_list, label, weight, original_shape, bbox = data
        batch = sampler3d_whole(volume_list, label, weight, original_shape, bbox)
        return batch
    ds = MapDataComponent(ds, f, 2)
    ds = PrefetchDataZMQ(ds, 1)
    return ds

def get_test_dataflow():
    imgs = BRATS_SEG.load_many(config.BASEDIR, config.TEST_DATASET, add_gt=False)
    # no filter for training
    ds = DataFromListOfDict(imgs, ['file_name', 'id', 'preprocessed'])

    def f(data):
        volume_list, label, weight, original_shape, bbox = data
        batch = sampler3d_whole(volume_list, label, weight, original_shape, bbox)
        return batch
    ds = MapDataComponent(ds, f, 2)
    ds = PrefetchDataZMQ(ds, 1)
    return ds

if __name__ == "__main__":
    df = get_train_dataflow()
    df.reset_state()
    for i in tqdm(df.get_data()):
        pass
        """
        for k in i:
            arr = i[k][0]
            arr = np.rot90(arr, k=2, axes= (1,2))
            OUTPUT_AFFINE = np.array(
                    [[0, 0, 1, 0],
                    [0, 1, 0, 0],
                    [1, 0, 0, 0],
                    [0, 0, 0, 1]])
            if k == 'labels':
                img = nibabel.Nifti1Image(arr.astype(np.int32), OUTPUT_AFFINE)
            else:
                img = nibabel.Nifti1Image(arr.astype(np.float32), OUTPUT_AFFINE)
            #nibabel.save(img, "./debug_output/{}.nii".format(k))
        break
        """

# float between 0 and 1. # float between 0 and 1. basedir="/data/dataset/BRATS2018/{mode}/{HGG/LGG}/patient_id/{flair/t1/t1ce/t2/seg}"         mode: training/val/test # read modality # 4mod +1gt dataset_mode: HGG/LGG/ALL         return list(dict[patient_id][modality] = filename.nii.gz) # read modality # 4mod +1gt Load and merges several instance files together. #!/usr/bin/env python # -*- coding: utf-8 -*- # File: data_loader.py # from tensorpack.utils.rect import FloatBox basedir="/data/dataset/BRATS2018/{mode}/{HGG/LGG}/patient_id/{flair/t1/t1ce/t2/seg}"         mode: training/val/test # read modality # 4mod +1gt dataset_mode: HGG/LGG/ALL         return list(dict[patient_id][modality] = filename.nii.gz) # read modality # 4mod +1gt Load and merges several instance files together. TODO: rewrite this with pytorch dataloaders ##### # some functions are borrowed from https://github.com/taigw/brats17/ ##### # from tensorpack.utils.argtools import memoized, log_once # from tensorpack.dataflow import (ProxyDataFlow, #     MapData, TestDataSpeed, MultiProcessMapData, #     MapDataComponent, DataFromList, PrefetchDataZMQ) Stack datapoints into batches.     It produces datapoints of the same number of components as ``ds``, but     each component has one new extra dimension of size ``batch_size``.     The batch can be either a list of original components, or (by default)     a numpy array of original components. Args:             ds (DataFlow): When ``use_list=False``, the components of ``ds``                 must be either scalars or :class:`np.ndarray`, and have to be consistent in shapes.             batch_size(int): batch size             remainder (bool): When the remaining datapoints in ``ds`` is not                 enough to form a batch, whether or not to also produce the remaining                 data as a smaller batch.                 If set to False, all produced datapoints are guaranteed to have the same batch size.                 If set to True, `ds.size()` must be accurate.             use_list (bool): if True, each component will contain a list                 of datapoints instead of an numpy array of an extra dimension. Yields:             Batched data by stacking each component on an extra 0th dimension. # force to contain label # noqa # open an ipython shell if possible # noqa sample 3d volume to size (depth, h, w, 4) #4, depth, h, w #[1,2,3,0] [d, h, w, modalities] # other meta info ? mods = sorted(im.keys())     volume_list = []     for mod_idx, mod in enumerate(mods):         filename = im[mod]         volume = load_nifty_volume_as_array(filename, with_header=False)         if mod_idx == 0:             # contain whole tumor             margin = 5 # small padding value             original_shape = volume.shape             bbmin, bbmax = get_none_zero_region(volume, margin)         volume = crop_ND_volume_with_bounding_box(volume, bbmin, bbmax)         if mod_idx == 0:             weight = np.asarray(volume > 0, np.float32)         if config.INTENSITY_NORM == 'modality':             volume = itensity_normalize_one_volume(volume)                 volume_list.append(volume) #[1,2,3,0] [d, h, w, modalities] # no filter for training #if config.CROSS_VALIDATION: # no filter for training # no filter for training for k in i:             arr = i[k][0]             arr = np.rot90(arr, k=2, axes= (1,2))             OUTPUT_AFFINE = np.array(                     [[0, 0, 1, 0],                     [0, 1, 0, 0],                     [1, 0, 0, 0],                     [0, 0, 0, 1]])             if k == 'labels':                 img = nibabel.Nifti1Image(arr.astype(np.int32), OUTPUT_AFFINE)             else:                 img = nibabel.Nifti1Image(arr.astype(np.float32), OUTPUT_AFFINE)             #nibabel.save(img, "./debug_output/{}.nii".format(k))         break