###########################################################################
# Created by: <NAME>
# Email: <EMAIL>
# Copyright (c) 2018
###########################################################################

import os
import cv2
import sys
import random
import numpy as np
import pandas as pd
from tqdm import tqdm, trange
from PIL import Image, ImageOps, ImageFilter

import torch
import torch.utils.data as data
import torchvision.transforms as transform

from encoding.datasets.base import BaseDataset

class SteelSegmentation(BaseDataset):
    NUM_CLASS = 4
    def __init__(self, root=os.path.expanduser('~/.encoding/data'), source=None, split=None,
                 mode=None, transform=None, target_transform=None, **kwargs):
        super(SteelSegmentation, self).__init__(
            root, split, mode, transform, target_transform, **kwargs)
        # #self.root = os.path.join(root, self.BASE_DIR)
        # self.images, self.mask_paths = get_city_pairs(self.root, self.split)
        # assert (len(self.images) == len(self.mask_paths))
        # if len(self.images) == 0:
        #     raise RuntimeError("Found 0 images in subfolders of: \
        #         " + self.root + "\n")
        # self._indices = np.array(range(-1, 19))
        # self._classes = np.array([0, 7, 8, 11, 12, 13, 17, 19, 20, 21, 22,
        #                           23, 24, 25, 26, 27, 28, 31, 32, 33])
        # self._key = np.array([-1, -1, -1, -1, -1, -1,
        #                       -1, -1,  0,  1, -1, -1,
        #                       2,   3,  4, -1, -1, -1,
        #                       5,  -1,  6,  7,  8,  9,
        #                       10, 11, 12, 13, 14, 15,
        #                       -1, -1, 16, 17, 18])
        # self._mapping = np.array(range(-1, len(self._key)-1)).astype('int32')

        self.root = root
        self.source = source
        self.split = split
        self.mode = mode
        self.transform = transform
        self.target_transform = target_transform

        self.uid = list(np.concatenate([np.load(root + '/splited/%s' % f, allow_pickle=True) for f in self.split]))
        if mode == 'train' or self.mode == 'val':
            df = pd.concat([pd.read_csv(root + '/%s' % f) for f in self.source])
            # df.fillna('', inplace=True)
            df = df_loc_by_list(df, 'ImageId_ClassId',
                                [u.split('/')[-1] + '_%d' % c for u in self.uid for c in [1, 2, 3, 4]])
            df['ImageId'], df['ClassId'] = zip(*df['ImageId_ClassId'].str.split('_'))
            df['ClassId'] = df['ClassId'].astype(int)
            df = df.pivot(index='ImageId', columns='ClassId', values='EncodedPixels')
            df['defects'] = df.count(axis=1)
            self.df = df

    # def _class_to_index(self, mask):
    #     # assert the values
    #     values = np.unique(mask)
    #     for i in range(len(values)):
    #         assert(values[i] in self._mapping)
    #     index = np.digitize(mask.ravel(), self._mapping, right=True)
    #     return self._key[index].reshape(mask.shape)
    #
    # def _preprocess(self, mask_file):
    #     if os.path.exists(mask_file):
    #         masks = torch.load(mask_file)
    #         return masks
    #     masks = []
    #     print("Preprocessing mask, this will take a while." + \
    #         "But don't worry, it only run once for each split.")
    #     tbar = tqdm(self.mask_paths)
    #     for fname in tbar:
    #         tbar.set_description("Preprocessing masks {}".format(fname))
    #         mask = Image.fromarray(self._class_to_index(
    #             np.array(Image.open(fname))).astype('int8'))
    #         masks.append(mask)
    #     torch.save(masks, mask_file)
    #     return masks


    def __getitem__(self, index):
        # img = Image.open(self.images[index]).convert('RGB')
        # if self.mode == 'test':
        #     if self.transform is not None:
        #         img = self.transform(img)
        #     return img, os.path.basename(self.images[index])
        # #mask = self.masks[index]
        # mask = Image.open(self.mask_paths[index])
        # # synchrosized transform
        # if self.mode == 'train':
        #     img, mask = self._sync_transform(img, mask)
        # elif self.mode == 'val':
        #     img, mask = self._val_sync_transform(img, mask)
        # else:
        #     assert self.mode == 'testval'
        #     mask = self._mask_transform(mask)
        # # general resize, normalize and toTensor
        # if self.transform is not None:
        #     img = self.transform(img)
        # if self.target_transform is not None:
        #     mask = self.target_transform(mask)
        # return img, mask

        if self.mode == 'train' or self.mode == 'val':
            image_id, mask = make_mask(index, self.df)
            image_path = os.path.join(self.root, 'train', image_id)
            # read data
            image = cv2.imread(image_path)
            # image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
            # image = Image.open(image_path).convert('RGB')
            # visualize(image_path=image)

            # apply augmentations
            if self.transform:
                sample = self.transform(image=image, mask=mask)
                image, mask = sample['image'], sample['mask']
            # if self.transform:
            #     image, mask = self.transform(image, mask)

            # apply preprocessing
            # if self.preprocessing:
            #     sample = self.preprocessing(image=image, mask=mask)
            #     image, mask = sample['image'], sample['mask']

            return image, mask

        elif self.mode == 'test':
            image_id = self.uid[index]
            image_path = os.path.join(self.root, image_id)
            # read data
            image = cv2.imread(image_path)
            # image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
            # image = Image.open(image_path).convert('RGB')
            # visualize(image_path=image)

            # apply augmentations
            if self.transform:
                image = self.transform(image=image)['image']

            # # apply preprocessing
            # if self.preprocessing:
            #     image = self.preprocessing(image=image)['image']

            return image_id.split('/')[-1], image


    # def _mask_transform(self, mask):
    #     #target = np.array(mask).astype('int32') - 1
    #     target = self._class_to_index(np.array(mask).astype('int32'))
    #     return torch.from_numpy(target).long()

    def __len__(self):
        # return len(self.images)
        return len(self.uid)

    # def make_pred(self, mask):
    #     values = np.unique(mask)
    #     for i in range(len(values)):
    #         assert(values[i] in self._indices)
    #     index = np.digitize(mask.ravel(), self._indices, right=True)
    #     return self._classes[index].reshape(mask.shape)


# def get_city_pairs(folder, split='train'):
#     def get_path_pairs(img_folder, mask_folder):
#         img_paths = []
#         mask_paths = []
#         for root, directories, files in os.walk(img_folder):
#             for filename in files:
#                 if filename.endswith(".png"):
#                     imgpath = os.path.join(root, filename)
#                     foldername = os.path.basename(os.path.dirname(imgpath))
#                     maskname = filename.replace('leftImg8bit','gtFine_labelIds')
#                     maskpath = os.path.join(mask_folder, foldername, maskname)
#                     if os.path.isfile(imgpath) and os.path.isfile(maskpath):
#                         img_paths.append(imgpath)
#                         mask_paths.append(maskpath)
#                     else:
#                         print('cannot find the mask or image:', imgpath, maskpath)
#         print('Found {} images in the folder {}'.format(len(img_paths), img_folder))
#         return img_paths, mask_paths
#
#     if split == 'train' or split == 'val' or split == 'test':
#         img_folder = os.path.join(folder, 'leftImg8bit/' + split)
#         mask_folder = os.path.join(folder, 'gtFine/'+ split)
#         img_paths, mask_paths = get_path_pairs(img_folder, mask_folder)
#         return img_paths, mask_paths
#     else:
#         assert split == 'trainval'
#         print('trainval set')
#         train_img_folder = os.path.join(folder, 'leftImg8bit/train')
#         train_mask_folder = os.path.join(folder, 'gtFine/train')
#         val_img_folder = os.path.join(folder, 'leftImg8bit/val')
#         val_mask_folder = os.path.join(folder, 'gtFine/val')
#         train_img_paths, train_mask_paths = get_path_pairs(train_img_folder, train_mask_folder)
#         val_img_paths, val_mask_paths = get_path_pairs(val_img_folder, val_mask_folder)
#         img_paths = train_img_paths + val_img_paths
#         mask_paths = train_mask_paths + val_mask_paths
#     return img_paths, mask_paths


def make_mask(row_id, df):
    '''Given a row index, return image_id and mask(256, 1600, 4)'''
    fname = df.iloc[row_id].name
    labels = df.iloc[row_id][:4]
    masks = np.zeros((256, 1600, 4), dtype=np.float32)

    for idx, label in enumerate(labels.values):
        if label is not np.nan:
            label = label.split(" ")
            positions = map(int, label[0::2])
            length = map(int, label[1::2])
            mask = np.zeros(256 * 1600, dtype=np.uint8)
            for pos, le in zip(positions, length):
                mask[pos:(pos+le)] = 1
            masks[:, :, idx] = mask.reshape(256, 1600, order='F')

    return fname, masks


def df_loc_by_list(df, key, values):
    df = df.loc[df[key].isin(values)]
    df = df.assign(sort = pd.Categorical(df[key], categories=values, ordered=True))
    df = df.sort_values('sort')
    #df = df.reset_index()
    df = df.drop('sort', axis=1)
    return df


###########################################################################
# Created by: <NAME>
# Email: <EMAIL>
# Copyright (c) 2018
###########################################################################

import os
import cv2
import sys
import random
import numpy as np
import pandas as pd
from tqdm import tqdm, trange
from PIL import Image, ImageOps, ImageFilter

import torch
import torch.utils.data as data
import torchvision.transforms as transform

from encoding.datasets.base import BaseDataset

class SteelSegmentation(BaseDataset):
    NUM_CLASS = 4
    def __init__(self, root=os.path.expanduser('~/.encoding/data'), source=None, split=None,
                 mode=None, transform=None, target_transform=None, **kwargs):
        super(SteelSegmentation, self).__init__(
            root, split, mode, transform, target_transform, **kwargs)
        # #self.root = os.path.join(root, self.BASE_DIR)
        # self.images, self.mask_paths = get_city_pairs(self.root, self.split)
        # assert (len(self.images) == len(self.mask_paths))
        # if len(self.images) == 0:
        #     raise RuntimeError("Found 0 images in subfolders of: \
        #         " + self.root + "\n")
        # self._indices = np.array(range(-1, 19))
        # self._classes = np.array([0, 7, 8, 11, 12, 13, 17, 19, 20, 21, 22,
        #                           23, 24, 25, 26, 27, 28, 31, 32, 33])
        # self._key = np.array([-1, -1, -1, -1, -1, -1,
        #                       -1, -1,  0,  1, -1, -1,
        #                       2,   3,  4, -1, -1, -1,
        #                       5,  -1,  6,  7,  8,  9,
        #                       10, 11, 12, 13, 14, 15,
        #                       -1, -1, 16, 17, 18])
        # self._mapping = np.array(range(-1, len(self._key)-1)).astype('int32')

        self.root = root
        self.source = source
        self.split = split
        self.mode = mode
        self.transform = transform
        self.target_transform = target_transform

        self.uid = list(np.concatenate([np.load(root + '/splited/%s' % f, allow_pickle=True) for f in self.split]))
        if mode == 'train' or self.mode == 'val':
            df = pd.concat([pd.read_csv(root + '/%s' % f) for f in self.source])
            # df.fillna('', inplace=True)
            df = df_loc_by_list(df, 'ImageId_ClassId',
                                [u.split('/')[-1] + '_%d' % c for u in self.uid for c in [1, 2, 3, 4]])
            df['ImageId'], df['ClassId'] = zip(*df['ImageId_ClassId'].str.split('_'))
            df['ClassId'] = df['ClassId'].astype(int)
            df = df.pivot(index='ImageId', columns='ClassId', values='EncodedPixels')
            df['defects'] = df.count(axis=1)
            self.df = df

    # def _class_to_index(self, mask):
    #     # assert the values
    #     values = np.unique(mask)
    #     for i in range(len(values)):
    #         assert(values[i] in self._mapping)
    #     index = np.digitize(mask.ravel(), self._mapping, right=True)
    #     return self._key[index].reshape(mask.shape)
    #
    # def _preprocess(self, mask_file):
    #     if os.path.exists(mask_file):
    #         masks = torch.load(mask_file)
    #         return masks
    #     masks = []
    #     print("Preprocessing mask, this will take a while." + \
    #         "But don't worry, it only run once for each split.")
    #     tbar = tqdm(self.mask_paths)
    #     for fname in tbar:
    #         tbar.set_description("Preprocessing masks {}".format(fname))
    #         mask = Image.fromarray(self._class_to_index(
    #             np.array(Image.open(fname))).astype('int8'))
    #         masks.append(mask)
    #     torch.save(masks, mask_file)
    #     return masks


    def __getitem__(self, index):
        # img = Image.open(self.images[index]).convert('RGB')
        # if self.mode == 'test':
        #     if self.transform is not None:
        #         img = self.transform(img)
        #     return img, os.path.basename(self.images[index])
        # #mask = self.masks[index]
        # mask = Image.open(self.mask_paths[index])
        # # synchrosized transform
        # if self.mode == 'train':
        #     img, mask = self._sync_transform(img, mask)
        # elif self.mode == 'val':
        #     img, mask = self._val_sync_transform(img, mask)
        # else:
        #     assert self.mode == 'testval'
        #     mask = self._mask_transform(mask)
        # # general resize, normalize and toTensor
        # if self.transform is not None:
        #     img = self.transform(img)
        # if self.target_transform is not None:
        #     mask = self.target_transform(mask)
        # return img, mask

        if self.mode == 'train' or self.mode == 'val':
            image_id, mask = make_mask(index, self.df)
            image_path = os.path.join(self.root, 'train', image_id)
            # read data
            image = cv2.imread(image_path)
            # image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
            # image = Image.open(image_path).convert('RGB')
            # visualize(image_path=image)

            # apply augmentations
            if self.transform:
                sample = self.transform(image=image, mask=mask)
                image, mask = sample['image'], sample['mask']
            # if self.transform:
            #     image, mask = self.transform(image, mask)

            # apply preprocessing
            # if self.preprocessing:
            #     sample = self.preprocessing(image=image, mask=mask)
            #     image, mask = sample['image'], sample['mask']

            return image, mask

        elif self.mode == 'test':
            image_id = self.uid[index]
            image_path = os.path.join(self.root, image_id)
            # read data
            image = cv2.imread(image_path)
            # image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
            # image = Image.open(image_path).convert('RGB')
            # visualize(image_path=image)

            # apply augmentations
            if self.transform:
                image = self.transform(image=image)['image']

            # # apply preprocessing
            # if self.preprocessing:
            #     image = self.preprocessing(image=image)['image']

            return image_id.split('/')[-1], image


    # def _mask_transform(self, mask):
    #     #target = np.array(mask).astype('int32') - 1
    #     target = self._class_to_index(np.array(mask).astype('int32'))
    #     return torch.from_numpy(target).long()

    def __len__(self):
        # return len(self.images)
        return len(self.uid)

    # def make_pred(self, mask):
    #     values = np.unique(mask)
    #     for i in range(len(values)):
    #         assert(values[i] in self._indices)
    #     index = np.digitize(mask.ravel(), self._indices, right=True)
    #     return self._classes[index].reshape(mask.shape)


# def get_city_pairs(folder, split='train'):
#     def get_path_pairs(img_folder, mask_folder):
#         img_paths = []
#         mask_paths = []
#         for root, directories, files in os.walk(img_folder):
#             for filename in files:
#                 if filename.endswith(".png"):
#                     imgpath = os.path.join(root, filename)
#                     foldername = os.path.basename(os.path.dirname(imgpath))
#                     maskname = filename.replace('leftImg8bit','gtFine_labelIds')
#                     maskpath = os.path.join(mask_folder, foldername, maskname)
#                     if os.path.isfile(imgpath) and os.path.isfile(maskpath):
#                         img_paths.append(imgpath)
#                         mask_paths.append(maskpath)
#                     else:
#                         print('cannot find the mask or image:', imgpath, maskpath)
#         print('Found {} images in the folder {}'.format(len(img_paths), img_folder))
#         return img_paths, mask_paths
#
#     if split == 'train' or split == 'val' or split == 'test':
#         img_folder = os.path.join(folder, 'leftImg8bit/' + split)
#         mask_folder = os.path.join(folder, 'gtFine/'+ split)
#         img_paths, mask_paths = get_path_pairs(img_folder, mask_folder)
#         return img_paths, mask_paths
#     else:
#         assert split == 'trainval'
#         print('trainval set')
#         train_img_folder = os.path.join(folder, 'leftImg8bit/train')
#         train_mask_folder = os.path.join(folder, 'gtFine/train')
#         val_img_folder = os.path.join(folder, 'leftImg8bit/val')
#         val_mask_folder = os.path.join(folder, 'gtFine/val')
#         train_img_paths, train_mask_paths = get_path_pairs(train_img_folder, train_mask_folder)
#         val_img_paths, val_mask_paths = get_path_pairs(val_img_folder, val_mask_folder)
#         img_paths = train_img_paths + val_img_paths
#         mask_paths = train_mask_paths + val_mask_paths
#     return img_paths, mask_paths


def make_mask(row_id, df):
    '''Given a row index, return image_id and mask(256, 1600, 4)'''
    fname = df.iloc[row_id].name
    labels = df.iloc[row_id][:4]
    masks = np.zeros((256, 1600, 4), dtype=np.float32)

    for idx, label in enumerate(labels.values):
        if label is not np.nan:
            label = label.split(" ")
            positions = map(int, label[0::2])
            length = map(int, label[1::2])
            mask = np.zeros(256 * 1600, dtype=np.uint8)
            for pos, le in zip(positions, length):
                mask[pos:(pos+le)] = 1
            masks[:, :, idx] = mask.reshape(256, 1600, order='F')

    return fname, masks


def df_loc_by_list(df, key, values):
    df = df.loc[df[key].isin(values)]
    df = df.assign(sort = pd.Categorical(df[key], categories=values, ordered=True))
    df = df.sort_values('sort')
    #df = df.reset_index()
    df = df.drop('sort', axis=1)
    return df


########################################################################### # Created by: <NAME> # Email: <EMAIL> # Copyright (c) 2018 ########################################################################### # #self.root = os.path.join(root, self.BASE_DIR) # self.images, self.mask_paths = get_city_pairs(self.root, self.split) # assert (len(self.images) == len(self.mask_paths)) # if len(self.images) == 0: #     raise RuntimeError("Found 0 images in subfolders of: \ #         " + self.root + "\n") # self._indices = np.array(range(-1, 19)) # self._classes = np.array([0, 7, 8, 11, 12, 13, 17, 19, 20, 21, 22, #                           23, 24, 25, 26, 27, 28, 31, 32, 33]) # self._key = np.array([-1, -1, -1, -1, -1, -1, #                       -1, -1,  0,  1, -1, -1, #                       2,   3,  4, -1, -1, -1, #                       5,  -1,  6,  7,  8,  9, #                       10, 11, 12, 13, 14, 15, #                       -1, -1, 16, 17, 18]) # self._mapping = np.array(range(-1, len(self._key)-1)).astype('int32') # df.fillna('', inplace=True) # def _class_to_index(self, mask): #     # assert the values #     values = np.unique(mask) #     for i in range(len(values)): #         assert(values[i] in self._mapping) #     index = np.digitize(mask.ravel(), self._mapping, right=True) #     return self._key[index].reshape(mask.shape) # # def _preprocess(self, mask_file): #     if os.path.exists(mask_file): #         masks = torch.load(mask_file) #         return masks #     masks = [] #     print("Preprocessing mask, this will take a while." + \ #         "But don't worry, it only run once for each split.") #     tbar = tqdm(self.mask_paths) #     for fname in tbar: #         tbar.set_description("Preprocessing masks {}".format(fname)) #         mask = Image.fromarray(self._class_to_index( #             np.array(Image.open(fname))).astype('int8')) #         masks.append(mask) #     torch.save(masks, mask_file) #     return masks # img = Image.open(self.images[index]).convert('RGB') # if self.mode == 'test': #     if self.transform is not None: #         img = self.transform(img) #     return img, os.path.basename(self.images[index]) # #mask = self.masks[index] # mask = Image.open(self.mask_paths[index]) # # synchrosized transform # if self.mode == 'train': #     img, mask = self._sync_transform(img, mask) # elif self.mode == 'val': #     img, mask = self._val_sync_transform(img, mask) # else: #     assert self.mode == 'testval' #     mask = self._mask_transform(mask) # # general resize, normalize and toTensor # if self.transform is not None: #     img = self.transform(img) # if self.target_transform is not None: #     mask = self.target_transform(mask) # return img, mask # read data # image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # image = Image.open(image_path).convert('RGB') # visualize(image_path=image) # apply augmentations # if self.transform: #     image, mask = self.transform(image, mask) # apply preprocessing # if self.preprocessing: #     sample = self.preprocessing(image=image, mask=mask) #     image, mask = sample['image'], sample['mask'] # read data # image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # image = Image.open(image_path).convert('RGB') # visualize(image_path=image) # apply augmentations # # apply preprocessing # if self.preprocessing: #     image = self.preprocessing(image=image)['image'] # def _mask_transform(self, mask): #     #target = np.array(mask).astype('int32') - 1 #     target = self._class_to_index(np.array(mask).astype('int32')) #     return torch.from_numpy(target).long() # return len(self.images) # def make_pred(self, mask): #     values = np.unique(mask) #     for i in range(len(values)): #         assert(values[i] in self._indices) #     index = np.digitize(mask.ravel(), self._indices, right=True) #     return self._classes[index].reshape(mask.shape) # def get_city_pairs(folder, split='train'): #     def get_path_pairs(img_folder, mask_folder): #         img_paths = [] #         mask_paths = [] #         for root, directories, files in os.walk(img_folder): #             for filename in files: #                 if filename.endswith(".png"): #                     imgpath = os.path.join(root, filename) #                     foldername = os.path.basename(os.path.dirname(imgpath)) #                     maskname = filename.replace('leftImg8bit','gtFine_labelIds') #                     maskpath = os.path.join(mask_folder, foldername, maskname) #                     if os.path.isfile(imgpath) and os.path.isfile(maskpath): #                         img_paths.append(imgpath) #                         mask_paths.append(maskpath) #                     else: #                         print('cannot find the mask or image:', imgpath, maskpath) #         print('Found {} images in the folder {}'.format(len(img_paths), img_folder)) #         return img_paths, mask_paths # #     if split == 'train' or split == 'val' or split == 'test': #         img_folder = os.path.join(folder, 'leftImg8bit/' + split) #         mask_folder = os.path.join(folder, 'gtFine/'+ split) #         img_paths, mask_paths = get_path_pairs(img_folder, mask_folder) #         return img_paths, mask_paths #     else: #         assert split == 'trainval' #         print('trainval set') #         train_img_folder = os.path.join(folder, 'leftImg8bit/train') #         train_mask_folder = os.path.join(folder, 'gtFine/train') #         val_img_folder = os.path.join(folder, 'leftImg8bit/val') #         val_mask_folder = os.path.join(folder, 'gtFine/val') #         train_img_paths, train_mask_paths = get_path_pairs(train_img_folder, train_mask_folder) #         val_img_paths, val_mask_paths = get_path_pairs(val_img_folder, val_mask_folder) #         img_paths = train_img_paths + val_img_paths #         mask_paths = train_mask_paths + val_mask_paths #     return img_paths, mask_paths Given a row index, return image_id and mask(256, 1600, 4) #df = df.reset_index()