#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function, absolute_import
from six import iteritems

import logging
import os
import sys
import random
import numpy
import tempfile
import copy
import importlib
from tqdm import tqdm

from dcase_util.containers import DictContainer, ListDictContainer, TextContainer, MetaDataContainer
from dcase_util.files import RemoteFile, RemotePackage, File
from dcase_util.utils import get_byte_string, setup_logging, Path
from dcase_util.utils import get_parameter_hash, get_class_inheritors
from dcase_util.ui import FancyLogger


def dataset_list(data_path='data', group=None):
    """List of datasets available

    Parameters
    ----------
    data_path : str
        Base path for the datasets
        Default value 'data'

    group : str
        Group label for the datasets, currently supported ['scene', 'event', 'tag']

    Returns
    -------
    str
        Multi line string containing dataset table

    """
    line = '  {class_name:<42s} | {group:5s} | {remote_size:10s} | {local_present:6s} | {files:5s} | {scene:6s} | {event:6s} | {tag:4s}\n'
    output = ''
    output += '  Dataset list\n'
    output += line.format(
        class_name='Class Name',
        group='Group',
        remote_size='Remote',
        local_present='Local',
        files='Audio',
        scene='Scenes',
        event='Events',
        tag='Tags'
    )
    output += line.format(
        class_name='-' * 42,
        group='-' * 5,
        remote_size='-' * 10,
        local_present='-' * 6,
        files='-' * 5,
        scene='-' * 6,
        event='-' * 6,
        tag='-' * 4
    )

    def get_empty_row():
        return line.format(
            class_name='',
            group='',
            remote_size='',
            local_present='',
            files='',
            scene='',
            event=''
        )

    def get_row(data):
        file_count = 0
        if data.meta_container.exists():
            file_count = len(data.meta)

        return line.format(
            class_name=data.__class__.__name__,
            group=data.dataset_group,
            remote_size=data.dataset_size_string(),
            local_present=data.dataset_size_on_disk(),
            files=str(file_count) if file_count else '',
            scene=str(data.scene_label_count()) if data.scene_label_count() else '',
            event=str(data.event_label_count()) if data.event_label_count() else '',
            tag=str(data.tag_count()) if data.tag_count() else '',
        )

    if not group or group == 'scene':
        class_list = get_class_inheritors(AcousticSceneDataset)
        class_list.sort(key=lambda x: x.__name__, reverse=False)

        for dataset_class in class_list:
            d = dataset_class(data_path=data_path)
            if d.dataset_group != 'base class':
                output += get_row(d)

    if not group or group == 'event':
        class_list = get_class_inheritors(SoundEventDataset)
        class_list.sort(key=lambda x: x.__name__, reverse=False)

        for dataset_class in class_list:
            d = dataset_class(data_path=data_path)
            if d.dataset_group != 'base class':
                output += get_row(d)

    if not group or group == 'tag':
        class_list = get_class_inheritors(AudioTaggingDataset)
        class_list.sort(key=lambda x: x.__name__, reverse=False)

        for dataset_class in class_list:
            d = dataset_class(data_path=data_path)
            if d.dataset_group != 'base class':
                output += get_row(d)

    return output


def dataset_factory(dataset_class_name, **kwargs):
    """Factory to get correct dataset class based on name

    Parameters
    ----------
    dataset_class_name : str
        Class name

    Raises
    ------
    NameError
        Class does not exists

    Returns
    -------
    Dataset class

    """

    try:
        dataset_class = getattr(
            importlib.import_module("dcase_util.datasets"),
            dataset_class_name
        )

    except AttributeError:
        message = 'Invalid Dataset class given [{class_name}].'.format(
            class_name=dataset_class_name
        )
        logger = logging.getLogger(__name__)
        if not logger.handlers:
            setup_logging()

        logger.exception(message)
        raise AttributeError(message)

    return dataset_class(**dict(kwargs))


class Dataset(object):
    """Dataset base class

    The specific dataset classes are inherited from this class, and only needed methods are reimplemented.

    """

    def __init__(self,
                 name='dataset',
                 storage_name='dataset',
                 data_path=None,
                 local_path=None,

                 show_progress_in_console=True,
                 log_system_progress=True,
                 use_ascii_progress_bar=True,

                 dataset_group='base class',
                 dataset_meta=None,

                 evaluation_setup_folder='evaluation_setup',
                 meta_filename='meta.txt',
                 error_meta_filename='error.txt',
                 filelisthash_filename='filelist.python.hash',
                 filelisthash_exclude_dirs=None,
                 crossvalidation_folds=None,
                 package_list=None,
                 included_content_types=None,
                 audio_paths=None,
                 default_audio_extension='wav',
                 reference_data_present=True,
                 **kwargs):
        """Constructor

        Parameters
        ----------
        name : str
            Dataset name

        storage_name : str
            Name to be used when storing dataset on disk

        data_path : str
            Root path where the dataset is stored. If None, os.path.join(tempfile.gettempdir(), 'dcase_util_datasets')
            is used.

        local_path : str
            Direct storage path setup for the dataset. If None, data_path and storage_name are used to create one.

        show_progress_in_console : bool
            Show progress in console.

        log_system_progress : bool
            Show progress in log.

        use_ascii_progress_bar : bool
            Show progress bar using ASCII characters. Use this if your console does not support UTF-8 characters.

        dataset_group : str
            Dataset group label, one of ['scene', 'event', 'tag']

        dataset_meta : dict
            Dictionary containing metadata about the dataset, e.g., collecting device information, dataset authors.

        evaluation_setup_folder : str
            Directory name where evaluation setup files are stores

        meta_filename : str
            Filename to be used for main meta file (contains all files with their reference data) of the dataset.

        error_meta_filename : str
            Filename for the error annotation file.

        filelisthash_filename : str
            Filename for filelist hash file.

        filelisthash_exclude_dirs : str
            Directories to be excluded from filelist hash calculation.

        crossvalidation_folds : int
            Count fo cross-validation folds. Indexing starts from one.

        package_list : list of dict
            Package list, remote files associated to the dataset.

        included_content_types : list of str or str
            Indicates what content type should be processed. One or multiple from ['all', 'audio', 'meta', 'code',
            'documentation']. If None given, ['all'] is used. Parameter can be also comma separated string.

        audio_paths : list of str
            List of paths to include audio material associated to the dataset. If None given, ['audio'] is used.

        default_audio_extension : str
            Default audio extension

        reference_data_present : bool
            Reference data is delivered with the dataset.

        """

        self.disable_progress_bar = not show_progress_in_console
        self.log_system_progress = log_system_progress
        self.use_ascii_progress_bar = use_ascii_progress_bar

        # Dataset name
        self.name = name

        # Dataset group
        self.dataset_group = dataset_group

        # Dataset meta
        if dataset_meta is None:
            dataset_meta = {}
        self.dataset_meta = DictContainer(dataset_meta)

        # Folder name for dataset
        self.storage_name = storage_name

        # Path to the dataset
        if data_path is None:
            data_path = os.path.join(tempfile.gettempdir(), 'dcase_util_datasets')

        if local_path is None:
            self.local_path = os.path.join(data_path, self.storage_name)

        else:
            self.local_path = local_path

        # Evaluation setup folder
        self.evaluation_setup_folder = evaluation_setup_folder

        # Path to the folder containing evaluation setup files
        self.evaluation_setup_path = os.path.join(self.local_path, self.evaluation_setup_folder)

        # Meta data file, csv-format
        self.meta_filename = meta_filename
        self.meta_file = os.path.join(self.local_path, self.meta_filename)

        # Error meta data file, csv-format
        self.error_meta_filename = error_meta_filename

        # Path to error meta data file
        self.error_meta_file = os.path.join(self.local_path, self.error_meta_filename)

        # Hash file to detect removed or added files
        self.filelisthash_filename = filelisthash_filename

        # Dirs to be excluded when calculating filelist hash
        if filelisthash_exclude_dirs is None:
            filelisthash_exclude_dirs = []

        self.filelisthash_exclude_dirs = filelisthash_exclude_dirs

        # Number of evaluation folds
        self.crossvalidation_folds = crossvalidation_folds

        # List containing dataset package items
        # Define this in the inherited dataset specific class.
        # Format:
        # {
        #    'content_type': 'documentation',  # Possible values ['meta', 'documentation', 'audio', 'code', 'features']
        #    'remote_file': 'https://zenodo.org/record/45759/files/TUT-sound-events-2016-development.doc.zip', # URL
        #    'remote_bytes': 70918,            # Size of remote file in bytes
        #    'remote_md5': '33fd26a895530aef607a07b08704eacd',  # MD5 hash of remote file
        #    'filename': 'TUT-sound-events-2016-development.doc.zip' # filename relative to self.local_path always
        # }
        if package_list is None:
            package_list = []
        self.package_list = ListDictContainer(package_list)

        # Expand local filenames to be related to local path
        for item in self.package_list:
            item['filename'] = os.path.join(self.local_path, item['filename'])

        # What content type should be processed. Use this for example to access only the meta data, and exclude usually
        # large and time consuming audio material downloading. Leave to "all" to include all content types.
        if included_content_types is None:
            included_content_types = ['all']

        if isinstance(included_content_types, str):
            # Split string to list if given
            included_content_types = included_content_types.split(',')

        self.included_content_types = included_content_types

        # Inject all included content types, there might be packages containing multiple content types.
        if 'all' not in self.included_content_types:
            for package_data in self.package_list:
                package_item = RemoteFile(**package_data)
                if self.included_content_types is None or package_item.is_content_type(
                        content_type=self.included_content_types
                ):
                    self.included_content_types = list(set().union(
                        package_item.content_type, self.included_content_types)
                    )

        # List of directories to contain the audio material
        if audio_paths is None:
            audio_paths = ['audio']
        self.audio_paths = audio_paths

        # Expand local filenames to be related to local path
        for path_id, path in enumerate(self.audio_paths):
            self.audio_paths[path_id] = os.path.join(self.local_path, path)

        # List of audio files
        self.files = None

        # Recognized audio extensions
        self.audio_extensions = ['wav', 'flac']

        self.default_audio_extension = default_audio_extension

        # Reference data presence flag, by default dataset should have reference data present.
        # However, some evaluation dataset might not have
        self.reference_data_present = reference_data_present

        # Initialize meta data container
        self.meta_container = MetaDataContainer()

        # List of audio error meta data dict
        self.error_meta_data = None

        # Cross-validation data
        self.crossvalidation_data = DictContainer({
            'train': {},
            'test': {},
            'evaluate': {},
        })

        # Training meta data for folds
        self.crossvalidation_data_train = {}

        # Testing meta data for folds
        self.crossvalidation_data_test = {}

        # Evaluation meta data for folds
        self.crossvalidation_data_eval = {}

        # Load meta and cross-validation data in
        self.load()

    @property
    def logger(self):
        logger = logging.getLogger(__name__)
        if not logger.handlers:
            setup_logging()

        return logger

    def load(self):
        """Load dataset meta data and cross-validation sets into the container."""
        self.load_meta()
        self.load_crossvalidation_data()

    def load_meta(self):
        """Load meta data into the container."""
        # Initialize meta container
        self.meta_container = MetaDataContainer(
            filename=os.path.join(self.local_path, self.meta_filename)
        )

        # Load from disk only if file exists
        if self.meta_container.exists():
            # Load meta data
            self.meta_container.load()

            # Mark reference data being present
            self.reference_data_present = True

        return self

    def load_crossvalidation_data(self):
        """Load cross-validation into the container.

        Returns
        -------
        self

        """

        # Reset cross validation data and insert 'all_data'
        self.crossvalidation_data = DictContainer({
            'train': {
                'all_data': self.meta_container
            },
            'test': {
                'all_data': self.meta_container
            },
            'evaluate': {
                'all_data': self.meta_container
            },
        })

        for crossvalidation_set in list(self.crossvalidation_data.keys()):
            for item in self.crossvalidation_data[crossvalidation_set]['all_data']:
                self.process_meta_item(item=item)

        # Load cross validation folds
        for fold in self.folds():
            # Get filenames
            train_filename = self.evaluation_setup_filename(
                setup_part='train',
                fold=fold
            )

            test_filename = self.evaluation_setup_filename(
                setup_part='test',
                fold=fold
            )

            evaluate_filename = self.evaluation_setup_filename(
                setup_part='evaluate',
                fold=fold
            )

            if os.path.isfile(train_filename):
                # Training data for fold exists, load and process it
                self.crossvalidation_data['train'][fold] = MetaDataContainer(filename=train_filename).load()

                # Process items
                for item in self.crossvalidation_data['train'][fold]:
                    self.process_meta_item(item=item)

            else:
                # Initialize data
                self.crossvalidation_data['train'][fold] = MetaDataContainer()

            if os.path.isfile(test_filename):
                # Testing data for fold exists, load and process it
                self.crossvalidation_data['test'][fold] = MetaDataContainer(filename=test_filename).load()

                # Process items
                for item in self.crossvalidation_data['test'][fold]:
                    self.process_meta_item(item=item)

            else:
                # Initialize data
                self.crossvalidation_data['test'][fold] = MetaDataContainer()

            if os.path.isfile(evaluate_filename):
                # Evaluation data for fold exists, load and process it
                self.crossvalidation_data['evaluate'][fold] = MetaDataContainer(filename=evaluate_filename).load()

                # Process items
                for item in self.crossvalidation_data['evaluate'][fold]:
                    self.process_meta_item(item=item)

            else:
                # Initialize data
                self.crossvalidation_data['evaluate'][fold] = MetaDataContainer()

        return self

    def __getitem__(self, i):
        """Getting meta data item

        Parameters
        ----------
        i : int
            item id

        Returns
        -------
        meta_data : dict
            Meta data item

        """

        if i < len(self.meta_container):
            return self.meta_container[i]

        else:
            return None

    def __iter__(self):
        """Iterator for meta data items
        """

        i = 0
        meta = self[i]

        # yield window while it's valid
        while meta is not None:
            yield meta
            # get next item
            i += 1
            meta = self[i]

    def initialize(self):
        """Initialize the dataset, download, extract files and prepare the dataset for the usage.

        Returns
        -------
        self

        """

        # Create the dataset path if does not exist
        Path().makedirs(path=self.local_path)

        # Check file changes
        if not self.check_filelist():
            # Download packages
            self.download_packages()

            # Extract content from packages
            self.extract_packages()

            # Load meta data in
            self.load()

            # Prepare meta data for the dataset class.
            self.prepare()

            # Save new filelist hash to monitor change in the dataset.
            self._save_filelist_hash()

        return self

    def show(self):
        """Print dataset information."""
        DictContainer(self.dataset_meta).show()
        self.meta_container.show(
            show_data=False,
            show_stats=True
        )

    def log(self):
        """Log dataset information."""
        DictContainer(self.dataset_meta).log()
        self.meta_container.log(
            show_data=False,
            show_stats=True
        )

    @property
    def audio_files(self):
        """Get all audio files in the dataset

        Returns
        -------
        list
            File list with absolute paths

        """

        if self.files is None:
            self.files = []
            for path in self.audio_paths:
                if path and os.path.exists(path):
                    dir_list = os.listdir(path)
                    for f in dir_list:
                        file_name, file_extension = os.path.splitext(f)
                        if file_extension[1:] in self.audio_extensions:
                            if os.path.abspath(os.path.join(path, f)) not in self.files:
                                self.files.append(os.path.abspath(os.path.join(path, f)))

            self.files.sort()
        return self.files

    @property
    def audio_file_count(self):
        """Get number of audio files in dataset

        Returns
        -------
        int
            Number of audio files

        """

        return len(self.audio_files)

    @property
    def meta(self):
        """Get meta data for dataset. If not already read from disk, data is read and returned.

        Raises
        ------
        IOError
            meta file not found.

        Returns
        -------
        list
            List containing meta data as dict.

        """

        if self.meta_container.empty():
            if self.meta_container.exists():
                self.meta_container.load()

            else:
                message = '{name}: Meta file not found [{filename}]'.format(
                    name=self.__class__.__name__,
                    filename=self.meta_container.filename
                )

                self.logger.exception(message)
                raise IOError(message)

        return copy.deepcopy(self.meta_container)

    @property
    def meta_count(self):
        """Number of meta data items.

        Returns
        -------
        int
            Meta data item count

        """

        return len(self.meta_container)

    @property
    def error_meta(self):
        """Get audio error meta data for dataset. If not already read from disk, data is read and returned.

        Raises
        ------
        IOError:
            audio error meta file not found.

        Returns
        -------
        MetaDataContainer
            List containing audio error meta data as dict.

        """

        if self.error_meta_data is None:
            self.error_meta_data = MetaDataContainer(filename=self.error_meta_file)
            if self.error_meta_data.exists():
                self.error_meta_data.load()

            else:
                message = '{name}: Error meta file not found [{filename}]'.format(
                    name=self.__class__.__name__,
                    filename=self.error_meta_file
                )

                self.logger.exception(message)
                raise IOError(message)

        return copy.deepcopy(self.error_meta_data)

    @property
    def error_meta_count(self):
        """Number of error meta data items.

        Returns
        -------
        int
            Meta data item count

        """

        return len(self.error_meta)

    @property
    def fold_count(self):
        """Number of fold in the evaluation setup.

        Returns
        -------
        int
            Number of folds

        """

        return self.crossvalidation_folds

    def scene_labels(self):
        """List of unique scene labels in the meta data.

        Returns
        -------
        list
            List of scene labels in alphabetical order.

        """

        return self.meta_container.unique_scene_labels

    def scene_label_count(self):
        """Number of unique scene labels in the meta data.

        Returns
        -------
        int
            Number of unique scene labels.

        """

        return self.meta_container.scene_label_count

    def event_labels(self, **kwargs):
        """List of unique event labels in the meta data.

        Returns
        -------
        list
            List of event labels in alphabetical order.

        """

        return self.meta_container.unique_event_labels

    def event_label_count(self, **kwargs):
        """Number of unique event labels in the meta data.

        Returns
        -------
        int
            Number of unique event labels

        """

        return self.meta_container.event_label_count

    def tags(self):
        """List of unique audio tags in the meta data.

        Returns
        -------
        list
            List of audio tags in alphabetical order.

        """

        return self.meta_container.unique_tags

    def tag_count(self):
        """Number of unique audio tags in the meta data.

        Returns
        -------
        int
            Number of unique audio tags

        """

        return len(self.tags())

    def debug_packages(self):
        """Debug remote packages associated to the dataset.
        Use this to check remote file size and md5 hash when developing dataset class.

        Returns
        -------
        self

        """

        log = FancyLogger()
        log.sub_header('Debug packages')
        log.row('package', 'remote_md5', 'remote_bytes', widths=[70, 36, 15])
        log.row('-', '-', '-')
        for item in self.package_list:
            remote_file = RemoteFile(**item)
            if self.included_content_types is None or remote_file.is_content_type(
                    content_type=self.included_content_types
            ):
                remote_file.remote_info()

                log.row(
                    os.path.split(item['remote_file'])[-1],
                    remote_file.remote_md5 if item['remote_md5'] != remote_file.remote_md5 else 'OK',
                    remote_file.remote_bytes if item['remote_bytes'] != remote_file.remote_bytes else 'OK'
                )

        return self

    def download_packages(self):
        """Download dataset packages over the internet to the local path

        Raises
        ------
        IOError
            Download failed.

        Returns
        -------
        self

        """

        # Create the dataset path if does not exist
        Path().makedirs(path=self.local_path)

        item_progress = tqdm(
            self.package_list,
            desc="{0: <25s}".format('Download packages'),
            file=sys.stdout,
            leave=False,
            disable=self.disable_progress_bar,
            ascii=self.use_ascii_progress_bar
        )

        for item in item_progress:
            if 'remote_file' in item:
                # Download if remote file is set
                remote_file = RemoteFile(**item)
                if self.included_content_types is None or remote_file.is_content_type(
                        content_type=self.included_content_types
                ):
                    remote_file.download()

        return self

    def extract_packages(self):
        """Extract the dataset packages

        Raises
        ------
        IOError
            Local package was not found.

        Returns
        -------
        self

        """

        item_progress = tqdm(
            self.package_list,
            desc="{0: <25s}".format('Extract packages'),
            file=sys.stdout,
            leave=False,
            disable=self.disable_progress_bar,
            ascii=self.use_ascii_progress_bar
        )

        for item_id, item in enumerate(item_progress):
            if File().is_package(filename=item['filename']):
                # Process only files which are packages
                remote_package = RemotePackage(**item)

                if self.included_content_types is None or remote_package.is_content_type(
                        content_type=self.included_content_types
                ):
                    if remote_package.local_exists():
                        remote_package.extract(
                            omit_first_level=True
                        )

                    else:
                        # Local file not present
                        message = '{name}: Local package does not exists [{filename}]'.format(
                            name=self.__class__.__name__,
                            filename=remote_package.filename,
                        )
                        self.logger.exception(message)
                        raise IOError(message)

        return self

    def prepare(self):
        """Prepare dataset for the usage.

        Returns
        -------
        self

        """

        return self

    def check_filelist(self):
        """Generates hash from file list and check does it matches with one saved in filelist.hash.
        If some files have been deleted or added, checking will result False.

        Returns
        -------
        bool
            Result

        """

        if os.path.isfile(os.path.join(self.local_path, self.filelisthash_filename)):
            old_hash_value = TextContainer(filename=os.path.join(self.local_path, self.filelisthash_filename)).load()[0]

            file_list = self._get_filelist(exclude_dirs=self.filelisthash_exclude_dirs)
            new_hash_value = get_parameter_hash(file_list)

            if self.included_content_types:
                new_hash_value += ';'+','.join(self.included_content_types)

            if old_hash_value != new_hash_value:
                return False

            else:
                return True

        else:
            return False

    def process_meta_item(self, item, absolute_path=True, **kwargs):
        """Process single meta data item

        Parameters
        ----------
        item :  MetaDataItem
            Meta data item

        absolute_path : bool
            Path format for the returned meta item, if True path is absolute, False path is relative to
            the dataset root.
            Default value True

        """

        if absolute_path:
            # Make sure filename has absolute path
            item.filename = self.relative_to_absolute_path(item.filename)

        else:
            # Make sure filename has relative path
            item.filename = self.absolute_to_relative_path(item.filename)

    def evaluation_setup_filename(self, setup_part='train', fold=None, scene_label=None, file_extension='txt'):
        """Evaluation setup filename generation.

        Parameters
        ----------
        setup_part :  str
            Setup part 'train', 'test', 'evaluate'
            Default value 'train'

        fold : int
            Fold number
            Default value None

        scene_label : str
            Scene label
            Default value None

        file_extension : str
            File extension
            Default value 'txt'

        Raises
        ------
        ValueError
            Unknown setup part.

        Returns
        -------
        str
            Filename with full path.

        """

        if fold == 'all_data':
            fold = None

        parts = []
        if scene_label:
            parts.append(scene_label)

        if fold:
            parts.append('fold' + str(fold))

        if setup_part == 'train':
            parts.append('train')

        elif setup_part == 'test':
            parts.append('test')

        elif setup_part == 'evaluate':
            parts.append('evaluate')

        else:
            message = '{name}: Unknown setup_part [{setup_part}]'.format(
                name=self.__class__.__name__,
                setup_part=setup_part
            )

            self.logger.exception(message)
            raise ValueError(message)

        return os.path.join(self.evaluation_setup_path, '_'.join(parts) + '.' + file_extension)

    def train(self, fold=None, absolute_paths=True, **kwargs):
        """List of training items.

        Parameters
        ----------
        fold : int [scalar]
            Fold id, if None all meta data is returned.

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        Returns
        -------
        MetaDataContainer
            List containing all meta data assigned to training set for given fold.

        """

        if fold is None or fold == 0:
            fold = 'all_data'

        data = copy.deepcopy(self.crossvalidation_data['train'][fold])

        # Go through items and make sure path are in correct form.
        for item in data:
            if absolute_paths:
                item.filename = self.relative_to_absolute_path(item.filename)

            else:
                item.filename = self.absolute_to_relative_path(item.filename)

        return data

    def test(self, fold=None, absolute_paths=True, **kwargs):
        """List of testing items.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        Returns
        -------
        MetaDataContainer
            List containing all meta data assigned to testing set for given fold.

        """

        if fold is None or fold == 0:
            fold = 'all_data'

        data = copy.deepcopy(self.crossvalidation_data['test'][fold])

        # Go through items and make sure path are in correct form.
        for item in data:
            if absolute_paths:
                item.filename = self.relative_to_absolute_path(item.filename)

            else:
                item.filename = self.absolute_to_relative_path(item.filename)

        return data

    def eval(self, fold=None, absolute_paths=True, **kwargs):
        """List of evaluation items.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        Returns
        -------
        MetaDataContainer
            List containing all meta data assigned to testing set for given fold.

        """

        if fold is None or fold == 0:
            fold = 'all_data'

        data = copy.deepcopy(self.crossvalidation_data['evaluate'][fold])

        # Go through items and make sure path are in correct form.
        for item in data:
            if absolute_paths:
                item.filename = self.relative_to_absolute_path(item.filename)

            else:
                item.filename = self.absolute_to_relative_path(item.filename)

        return data

    def train_files(self, fold=None, absolute_paths=True, **kwargs):
        """List of training files.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        Returns
        -------
        list of str
            List containing all files assigned to training set for given fold.

        """

        return self.train(fold=fold, absolute_paths=absolute_paths).unique_files

    def test_files(self, fold=None, absolute_paths=True, **kwargs):
        """List of testing files.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        Returns
        -------
        list of str
            List containing all files assigned to testing set for given fold.

        """

        return self.test(fold=fold, absolute_paths=absolute_paths).unique_files

    def eval_files(self, fold=None, absolute_paths=True, **kwargs):
        """List of evaluation files.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        Returns
        -------
        list of str
            List containing all files assigned to testing set for given fold.

        """

        return self.eval(fold=fold, absolute_paths=absolute_paths).unique_files

    def validation_split(self,
                         fold=None, split_type='balanced', validation_amount=None,
                         seed=0, verbose=False, scene_label=None, iterations=100,
                         **kwargs):
        """List of validation files. Validation files are always subset of training files.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        split_type : str
            Split type [dataset, random, balanced]
            Default value 'balanced'

        validation_amount : float
            Amount of training files to be assigned for validation
            Default value None

        seed : int
            Randomization seed
            Default value 0

        verbose : bool
            Show information about the validation set.
            Default value False

        scene_label : str
            Scene label of the validation set. If None, all training material used.
            Default value None

        iterations : int
            Randomization iterations done when finding balanced set before selecting best matched set.
            Default value 100

        Returns
        -------
        list of str
            List containing all files assigned to training set for given fold.

        list of str
            List containing all files assigned to validation set for given fold.

        """

        kwargs.update({
            'fold': fold,
            'validation_amount': validation_amount,
            'seed': seed,
            'verbose': verbose,
            'scene_label': scene_label,
            'iterations': iterations,
        })

        if split_type == 'dataset':
            validation_files = self.validation_files_dataset(**kwargs)

        elif split_type == 'random':
            validation_files = self.validation_files_random(**kwargs)

        elif split_type == 'balanced':
            validation_files = self.validation_files_balanced(**kwargs)

        else:
            message = '{name}: Unknown split type [{split_type}]'.format(
                name=self.__class__.__name__,
                split_type=split_type
            )

            self.logger.exception(message)
            raise IOError(message)

        training_files = sorted(list(set(self.train(fold=fold).unique_files) - set(validation_files)))

        return training_files, validation_files

    def validation_files_dataset(self, fold=None, verbose=False, **kwargs):
        """List of validation files delivered by the dataset.
        """

        message = '{name}: Dataset does not have fixed validation sets, use validation set generation to get sets'.format(
            name=self.__class__.__name__,
        )

        self.logger.exception(message)
        raise ValueError(message)

    def validation_files_random(self, fold=None, validation_amount=0.3, seed=0, verbose=False, **kwargs):
        """List of validation files selected randomly from the training material.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        validation_amount : float
            Amount of training material to be assigned for validation.
            Default value 0.3

        seed : int
            Randomization seed
            Default value 0

        verbose : bool
            Show information about the validation set.
            Default value False

        Returns
        -------
        list of str
            List containing all files assigned for validation

        """

        random.seed(seed)

        training_meta = self.train(fold=fold)
        training_files = training_meta.unique_files
        random.shuffle(training_files, random.random)

        validation_split_index = int(numpy.ceil(validation_amount * len(training_files)))
        validation_files = training_files[0:validation_split_index]

        if verbose:
            validation_meta = self.train(fold=fold).filter(file_list=validation_files)

            validation_amounts = numpy.zeros((len(self.scene_labels())+1, 3))
            for scene_id, scene_label in enumerate(self.scene_labels()):
                validation_scene_meta = validation_meta.filter(scene_label=scene_label)
                validation_amounts[scene_id, 0] = len(validation_scene_meta.unique_identifiers)
                validation_amounts[scene_id, 1] = len(validation_scene_meta.unique_files)
                validation_amounts[scene_id, 2] = len(validation_scene_meta.unique_files) / float(len(training_meta.filter(scene_label=scene_label).unique_files)) * 100
            validation_amounts[-1, 0] = numpy.sum(validation_amounts[0:-1, 0])
            validation_amounts[-1, 1] = numpy.sum(validation_amounts[0:-1, 1])
            validation_amounts[-1, 2] = numpy.mean(validation_amounts[0:-1, 2])

            labels = self.scene_labels()
            labels.append('Overall')

            FancyLogger().sub_header('Validation set for fold [{fold}] / random'.format(fold=fold), indent=2)
            FancyLogger().table(
                cell_data=[labels] + validation_amounts.T.tolist(),
                column_headers=['Scene', 'Identifiers', 'Files', 'Amount (%)'],
                column_types=['str20', 'int', 'int', 'float1'],
                column_separators=[0, 1],
                row_separators=[len(labels) - 1],
                indent=2
            )

        return validation_files

    def validation_files_balanced(self, fold=None, validation_amount=0.3, seed=0, verbose=False, **kwargs):
        """List of validation files randomly selecting while maintaining data balance.
        """
        message = '{name}: Balanced validation set generation has not been implemented for dataset class.'.format(
            name=self.__class__.__name__,
        )

        self.logger.exception(message)
        raise ValueError(message)

    def folds(self, mode='folds'):
        """List of fold ids

        Parameters
        ----------
        mode : str {'folds','full'}
            Fold setup type, possible values are 'folds' and 'full'. In 'full' mode fold number is set 0 and
            all data is used for training.
            Default value 'folds'

        Returns
        -------
        list of int
            Fold ids

        """

        if mode == 'folds':
            if self.crossvalidation_folds is not None:
                return list(range(1, self.crossvalidation_folds+1))
            else:
                return ['all_data']

        elif mode == 'full':
            return ['all_data']

    def file_meta(self, filename):
        """Meta data for given file

        Parameters
        ----------
        filename : str
            File name

        Returns
        -------
        list of dict
            List containing all meta data related to given file.

        """

        # Try first with raw filename
        file_meta = self.meta_container.filter(
            filename=filename
        )

        if not file_meta:
            # Try with absolute filename
            file_meta = self.meta_container.filter(
                filename=self.absolute_to_relative_path(filename)
            )

        if not file_meta:
            # Try with relative filename
            file_meta = self.meta_container.filter(
                filename=self.relative_to_absolute_path(filename)
            )

        return file_meta

    def file_error_meta(self, filename):
        """Error meta data for given file

        Parameters
        ----------
        filename : str
            File name

        Returns
        -------
        list of dict
            List containing all error meta data related to given file.

        """
        error_meta = self.error_meta.filter(
            filename=filename
        )

        if not error_meta:
            error_meta = self.error_meta.filter(
                filename=self.absolute_to_relative_path(filename)
            )

        if not error_meta:
            error_meta = self.error_meta.filter(
                filename=self.relative_to_absolute_path(filename)
            )

        return error_meta

    def file_features(self, filename):
        """Pre-calculated acoustic features for given file

        Parameters
        ----------
        filename : str
            File name

        Returns
        -------
        numpy.ndarray
            Matrix containing acoustic features

        """

        pass

    def relative_to_absolute_path(self, path):
        """Converts relative path into absolute path.

        Parameters
        ----------
        path : str
            Relative path

        Returns
        -------
        str
            Absolute path

        """

        return os.path.abspath(os.path.expanduser(os.path.join(self.local_path, path)))

    def absolute_to_relative_path(self, path):
        """Converts absolute path into relative path.

        Parameters
        ----------
        path : str
            Absolute path

        Returns
        -------
        str
            Relative path

        """

        if path.startswith(os.path.abspath(self.local_path)):
            return os.path.relpath(path, self.local_path)

        else:
            return path

    def dataset_bytes(self):
        """Total download size of the dataset in bytes.

        Returns
        -------
        int
            Total bytes

        """

        total_bytes = 0
        for item_id, item in enumerate(self.package_list):
            remote_package = RemoteFile(**item)

            if remote_package.local_exists():
                total_bytes += remote_package.local_bytes

            else:
                total_bytes += remote_package.remote_bytes

        return total_bytes

    def dataset_size_string(self):
        """Total download size of the dataset in a string.

        Returns
        -------
        str
            Total size

        """

        return get_byte_string(self.dataset_bytes(), show_bytes=False)

    def dataset_size_on_disk(self):
        """Total size of the dataset currently stored locally.

        Returns
        -------
        str
            Total size

        """

        total_size = 0
        for dir_path, dir_names, filenames in os.walk(self.local_path):
            for f in filenames:
                filename = os.path.join(dir_path, f)
                total_size += os.path.getsize(filename)

        return get_byte_string(total_size, show_bytes=False)

    def _get_filelist(self, exclude_dirs=None):
        """List of files under local_path

        Parameters
        ----------
        exclude_dirs : list of str
            List of directories to be excluded
            Default value None

        Returns
        -------
        list of str
            File list

        """

        if exclude_dirs is None:
            exclude_dirs = []

        filelist = []
        for path, sub_directory, files in os.walk(self.local_path):
            for name in files:
                if os.path.splitext(name)[1] != os.path.splitext(self.filelisthash_filename)[1] and os.path.split(path)[1] not in exclude_dirs:
                    filelist.append(os.path.join(path, name))

        return sorted(filelist)

    def _save_filelist_hash(self):
        """Generates file list hash, and saves it as filelist.hash under local_path.

        Parameters
        ----------

        Returns
        -------
        None

        """

        filelist = self._get_filelist()

        hash_value = get_parameter_hash(filelist)
        if self.included_content_types:
            hash_value += ';'+','.join(self.included_content_types)

        TextContainer([hash_value]).save(
            filename=os.path.join(self.local_path, self.filelisthash_filename)
        )


class AcousticSceneDataset(Dataset):
    """Acoustic scene dataset baseclass """
    def __init__(self, *args, **kwargs):
        super(AcousticSceneDataset, self).__init__(*args, **kwargs)

    def validation_files_balanced(self,
                                  fold=None, validation_amount=0.3, seed=0, verbose=False, iterations=100,
                                  **kwargs):
        """List of validation files randomly selecting while maintaining data balance.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        validation_amount : float
            Amount of training material to be assigned for validation.
            Default value 0.3

        seed : int
            Randomization seed
            Default value 0

        verbose : bool
            Show information about the validation set.
            Default value False

        iterations : int
            How many randomization iterations will be done before selecting best matched.
            Default value 100

        Returns
        -------
        list of str
            List containing all files assigned for validation

        """

        random.seed(seed)
        training_meta = self.train(fold=fold)

        # Check do we have location/source identifier present
        identifier_present = False
        for item in training_meta:
            if item.identifier:
                identifier_present = True
                break

        training_files = []
        validation_files = []
        validation_amounts = numpy.zeros((len(self.scene_labels())+1, 4))

        if identifier_present:
            # Do the balance based on scene class and identifier
            for scene_id, scene_label in enumerate(self.scene_labels()):
                scene_meta = training_meta.filter(scene_label=scene_label)

                data = {}
                for identifier in scene_meta.unique_identifiers:
                    data[identifier] = scene_meta.filter(identifier=identifier).unique_files

                current_scene_validation_amount = []
                sets_candidates = []

                iteration_progress = tqdm(
                    range(0, iterations),
                    desc="{0: <25s}".format('Generate validation split candidates'),
                    file=sys.stdout,
                    leave=False,
                    disable=self.disable_progress_bar,
                    ascii=self.use_ascii_progress_bar
                )

                for i in iteration_progress:
                    current_locations = list(data.keys())
                    random.shuffle(current_locations, random.random)
                    validation_split_index = int(numpy.ceil(validation_amount * len(data)))
                    current_validation_identifiers = current_locations[0:validation_split_index]
                    current_training_identifiers = current_locations[validation_split_index:]

                    # Collect validation files
                    current_validation_files = []
                    for identifier in current_validation_identifiers:
                        current_validation_files += data[identifier]

                    # Collect training files
                    current_training_files = []
                    for identifier in current_training_identifiers:
                        current_training_files += data[identifier]

                    current_scene_validation_amount.append(
                        len(current_validation_files) / float(len(current_validation_files) + len(current_training_files))
                    )

                    sets_candidates.append({
                        'validation': current_validation_files,
                        'training': current_training_files,
                    })

                best_set_id = numpy.argmin(numpy.abs(numpy.array(current_scene_validation_amount) - validation_amount))
                validation_files += sets_candidates[best_set_id]['validation']
                training_files += sets_candidates[best_set_id]['training']
                validation_amounts[scene_id, 0] = len(scene_meta.unique_identifiers)
                validation_amounts[scene_id, 1] = len(scene_meta.unique_files)
                validation_amounts[scene_id, 2] = len(sets_candidates[best_set_id]['validation'])
                validation_amounts[scene_id, 3] = current_scene_validation_amount[best_set_id] * 100

        else:
            # Do the balance based on scene class only
            for scene_id, scene_label in enumerate(self.scene_labels()):
                scene_files = training_meta.filter(scene_label=scene_label).unique_files

                random.shuffle(scene_files, random.random)
                validation_split_index = int(numpy.ceil(validation_amount * len(scene_files)))
                current_validation_files = scene_files[0:validation_split_index]
                current_training_files = scene_files[validation_split_index:]

                validation_files += current_validation_files
                training_files += current_training_files
                validation_amounts[scene_id, 1] = len(scene_files)
                validation_amounts[scene_id, 2] = len(current_validation_files)
                validation_amounts[scene_id, 3] = len(current_validation_files) / float(len(current_validation_files) + len(current_training_files)) * 100

        if verbose:
            validation_amounts[-1, 0] = numpy.sum(validation_amounts[0:-1, 0])
            validation_amounts[-1, 1] = numpy.sum(validation_amounts[0:-1, 1])
            validation_amounts[-1, 2] = numpy.sum(validation_amounts[0:-1, 2])
            validation_amounts[-1, 3] = numpy.mean(validation_amounts[0:-1, 3])

            logger = FancyLogger()
            logger.sub_header('Validation set for fold [{fold}] / balanced'.format(fold=fold), indent=2)
            logger.row(
                '', 'Full training set', 'Selected validation subset',
                widths=[20, 25, 35],
                types=['str', 'str', 'str'],
                separators=[True, True],
                indent=2
            )
            logger.row(
                'Scene label', 'Identifiers', 'Files', 'Files', 'Amount (%)',
                widths=[20, 15, 10, 10, 15],
                types=['str20', 'int', 'int', 'int', 'float1_percentage'],
                separators=[True, False, True, False]
            )
            logger.row('-', '-', '-', '-', '-')
            for scene_id, scene_label in enumerate(self.scene_labels()):
                logger.row(
                    scene_label,
                    validation_amounts[scene_id, 0],
                    validation_amounts[scene_id, 1],
                    validation_amounts[scene_id, 2],
                    validation_amounts[scene_id, 3],
                )
            logger.row('-', '-', '-', '-', '-')
            logger.row(
                'Overall',
                validation_amounts[-1, 0],
                validation_amounts[-1, 1],
                validation_amounts[-1, 2],
                validation_amounts[-1, 3]
            )
        logger.line()

        return validation_files


class SoundEventDataset(Dataset):
    """Sound event dataset baseclass"""
    def __init__(self, *args, **kwargs):
        super(SoundEventDataset, self).__init__(*args, **kwargs)

    def load_crossvalidation_data(self):
        """Load cross-validation into the container.

        Returns
        -------
        self

        """

        # Reset cross validation data and insert 'all_data'
        self.crossvalidation_data = DictContainer({
            'train': {
                'all_data': self.meta_container
            },
            'test': {
                'all_data': self.meta_container
            },
            'evaluate': {
                'all_data': self.meta_container
            },
        })

        for crossvalidation_set in list(self.crossvalidation_data.keys()):
            for item in self.crossvalidation_data[crossvalidation_set]['all_data']:
                self.process_meta_item(item=item)

        # Load cross validation folds
        for fold in self.folds():
            # Initialize data
            self.crossvalidation_data['train'][fold] = MetaDataContainer()
            self.crossvalidation_data['test'][fold] = MetaDataContainer()
            self.crossvalidation_data['evaluate'][fold] = MetaDataContainer()

            for scene_label in self.scene_labels():
                # Get filenames
                train_filename = self.evaluation_setup_filename(
                    setup_part='train',
                    fold=fold,
                    scene_label=scene_label
                )

                test_filename = self.evaluation_setup_filename(
                    setup_part='test',
                    fold=fold,
                    scene_label=scene_label
                )

                evaluate_filename = self.evaluation_setup_filename(
                    setup_part='evaluate',
                    fold=fold,
                    scene_label=scene_label
                )

                if os.path.isfile(train_filename):
                    # Training data for fold exists, load and process it
                    self.crossvalidation_data['train'][fold] += MetaDataContainer(filename=train_filename).load()

                if os.path.isfile(test_filename):
                    # Testing data for fold exists, load and process it
                    self.crossvalidation_data['test'][fold] += MetaDataContainer(filename=test_filename).load()

                if os.path.isfile(evaluate_filename):
                    # Evaluation data for fold exists, load and process it
                    self.crossvalidation_data['evaluate'][fold] += MetaDataContainer(filename=evaluate_filename).load()

            # Process items
            for item in self.crossvalidation_data['train'][fold]:
                self.process_meta_item(item=item)

            for item in self.crossvalidation_data['test'][fold]:
                self.process_meta_item(item=item)

            for item in self.crossvalidation_data['evaluate'][fold]:
                self.process_meta_item(item=item)

        return self

    def event_label_count(self, scene_label=None):
        """Number of unique scene labels in the meta data.

        Parameters
        ----------
        scene_label : str
            Scene label
            Default value None

        Returns
        -------
        int
            Number of unique scene labels.

        """

        return len(self.event_labels(scene_label=scene_label))

    def event_labels(self, scene_label=None):
        """List of unique event labels in the meta data.

        Parameters
        ----------
        scene_label : str
            Scene label
            Default value None

        Returns
        -------
        list of str
            List of event labels in alphabetical order.

        """

        if scene_label is not None:
            labels = self.meta_container.filter(scene_label=scene_label).unique_event_labels

        else:
            labels = self.meta_container.unique_event_labels

        labels.sort()
        return labels

    def train(self, fold=None, absolute_paths=True, scene_label=None, event_label=None, **kwargs):
        """List of training items.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        scene_label : str
            Scene label
            Default value None

        event_label : str
            Event label
            Default value None

        Returns
        -------
        list of dict
            List containing all meta data assigned to training set for given fold.

        """

        if fold is None or fold == 0:
            fold = 'all_data'

        data = copy.deepcopy(self.crossvalidation_data['train'][fold])

        # Go through items and make sure path are in correct form.
        for item in data:
            if absolute_paths:
                item.filename = self.relative_to_absolute_path(item.filename)
            else:
                item.filename = self.absolute_to_relative_path(item.filename)

        if scene_label:
            data = data.filter(
                scene_label=scene_label
            )

        if event_label:
            data = data.filter(
                event_label=event_label
            )

        return data

    def test(self, fold=None, absolute_paths=True, scene_label=None, event_label=None, **kwargs):
        """List of testing items.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        scene_label : str
            Scene label
            Default value None

        event_label : str
            Event label
            Default value None

        Returns
        -------
        list of dict
            List containing all meta data assigned to testing set for given fold.

        """

        if fold is None or fold == 0:
            fold = 'all_data'

        data = copy.deepcopy(self.crossvalidation_data['test'][fold])

        # Go through items and make sure path are in correct form.
        for item in data:
            if absolute_paths:
                item.filename = self.relative_to_absolute_path(item.filename)
            else:
                item.filename = self.absolute_to_relative_path(item.filename)

        if scene_label:
            data = data.filter(
                scene_label=scene_label
            )

        if event_label:
            data = data.filter(
                event_label=event_label
            )

        return data

    def eval(self, fold=None, absolute_paths=True, scene_label=None, event_label=None, **kwargs):
        """List of evaluation items.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        scene_label : str
            Scene label
            Default value None

        event_label : str
            Event label
            Default value None

        Returns
        -------
        list of dict
            List containing all meta data assigned to testing set for given fold.

        """

        if fold is None or fold == 0:
            fold = 'all_data'

        data = copy.deepcopy(self.crossvalidation_data['evaluate'][fold])

        # Go through items and make sure path are in correct form.
        for item in data:
            if absolute_paths:
                item.filename = self.relative_to_absolute_path(item.filename)
            else:
                item.filename = self.absolute_to_relative_path(item.filename)

        if scene_label:
            data = data.filter(
                scene_label=scene_label
            )

        if event_label:
            data = data.filter(
                event_label=event_label
            )

        return data

    def train_files(self, fold=None, absolute_paths=True, scene_label=None, event_label=None, **kwargs):
        """List of training files.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        scene_label : str
            Scene label
            Default value None

        event_label : str
            Event label
            Default value None

        Returns
        -------
        list of str
            List containing all files assigned to training set for given fold.

        """

        return self.train(
            fold=fold,
            absolute_paths=absolute_paths,
            scene_label=scene_label,
            event_label=event_label
        ).unique_files

    def test_files(self, fold=None, absolute_paths=True, scene_label=None, event_label=None, **kwargs):
        """List of testing files.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        scene_label : str
            Scene label
            Default value None

        event_label : str
            Event label
            Default value None

        Returns
        -------
        list of str
            List containing all files assigned to testing set for given fold.

        """

        return self.test(
            fold=fold,
            absolute_paths=absolute_paths,
            scene_label=scene_label,
            event_label=event_label
        ).unique_files

    def eval_files(self, fold=None, absolute_paths=True, scene_label=None, event_label=None, **kwargs):
        """List of evaluation files.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        scene_label : str
            Scene label
            Default value None

        event_label : str
            Event label
            Default value None

        Returns
        -------
        list of str
            List containing all files assigned to testing set for given fold.

        """

        return self.eval(
            fold=fold,
            absolute_paths=absolute_paths,
            scene_label=scene_label,
            event_label=event_label
        ).unique_files

    def validation_files_random(self,
                                fold=None, validation_amount=0.3, seed=0, verbose=False, scene_label=None,
                                **kwargs):
        """List of validation files selected randomly from the training material.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        validation_amount : float
            Amount of training material to be assigned for validation.
            Default value 0.3

        seed : int
            Randomization seed
            Default value 0

        verbose : bool
            Show information about the validation set.
            Default value False

        scene_label : str
            Scene label of the validation set. If None, all training material used.
            Default value None

        Returns
        -------
        list of str
            List containing all files assigned for validation

        """

        random.seed(seed)
        training_meta = self.train(
            fold=fold,
            scene_label=scene_label
        )

        if scene_label:
            scene_labels = [scene_label]
        else:
            scene_labels = self.scene_labels()

        training_files = training_meta.unique_files
        random.shuffle(training_files, random.random)

        validation_split_index = int(numpy.ceil(validation_amount * len(training_files)))
        validation_files = training_files[0:validation_split_index]

        if verbose:
            logger = FancyLogger()
            logger.sub_header('Validation set for fold [{fold}] / random'.format(fold=fold), indent=2)
            for scene_id, scene_label in enumerate(scene_labels):
                all_stats = training_meta.filter(scene_label=scene_label).event_stat_counts()
                validation_stats = training_meta.filter(
                    scene_label=scene_label,
                    file_list=validation_files
                ).event_stat_counts()

                training_files = sorted(list(set(self.train(fold=fold).unique_files) - set(validation_files)))
                training_stats = training_meta.filter(
                    scene_label=scene_label,
                    file_list=training_files
                ).event_stat_counts()

                cell_data = numpy.zeros((len(list(all_stats.keys())) + 1, 4))
                for event_id, event_label in enumerate(list(all_stats.keys())):
                    cell_data[event_id, 0] = all_stats[event_label]
                    if event_label in training_stats:
                        cell_data[event_id, 1] = training_stats[event_label]

                    if event_label in validation_stats:
                        cell_data[event_id, 2] = validation_stats[event_label]

                    cell_data[event_id, 3] = cell_data[event_id, 2] / float(cell_data[event_id, 0]) * 100

                cell_data[-1, 0] = numpy.sum(list(all_stats.values()))
                cell_data[-1, 1] = numpy.sum(list(training_stats.values()))
                cell_data[-1, 2] = numpy.sum(list(validation_stats.values()))
                cell_data[-1, 3] = cell_data[-1, 2] / float(cell_data[-1, 0]) * 100

                labels = list(all_stats.keys())
                labels.append('Overall')
                logger.line(scene_label, indent=4)
                logger.table(
                    cell_data=[labels] + cell_data.T.tolist(),
                    column_headers=['Event', 'All', 'Training', 'Validation', 'Val amount (%)'],
                    column_types=['str20', 'int', 'int', 'int', 'float1'],
                    column_separators=[0, 1, 3],
                    row_separators=[len(labels)-1],
                    indent=6
                )

        return validation_files

    def validation_files_balanced(self,
                                  fold=None, validation_amount=0.3, seed=0,
                                  verbose=False, scene_label=None, iterations=100,
                                  **kwargs):
        """List of validation files randomly selecting while maintaining data balance.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        validation_amount : float
            Amount of training material to be assigned for validation.
            Default value 0.3

        seed : int
            Randomization seed
            Default value 0

        verbose : bool
            Show information about the validation set.
            Default value False

        scene_label : str
            Scene label of the validation set. If None, all training material used.
            Default value None

        iterations : int
            How many randomization iterations will be done before selecting best matched.
            Default value 100

        Returns
        -------
        list of str
            List containing all files assigned for validation

        """

        from sklearn.metrics import mean_absolute_error

        random.seed(seed)
        training_meta = self.train(
            fold=fold,
            scene_label=scene_label
        )

        if scene_label:
            scene_labels = [scene_label]
        else:
            scene_labels = self.scene_labels()

        # Check do we have location/source identifier present
        identifier_present = False
        for item in training_meta:
            if item.identifier:
                identifier_present = True
                break

        if identifier_present:
            # Do the balance based on scene class, identifier and event class
            data = {}
            for scene_id, scene_label in enumerate(scene_labels):
                scene_meta = training_meta.filter(scene_label=scene_label)
                data[scene_label] = {}
                for identifier in scene_meta.unique_identifiers:
                    data[scene_label][identifier] = scene_meta.filter(identifier=identifier).unique_files

            # Get event amounts per class
            event_amounts = {}
            for scene_id, scene_label in enumerate(scene_labels):
                event_amounts[scene_label] = {}
                for identifier in list(data[scene_label].keys()):
                    for audio_filename in data[scene_label][identifier]:
                        current_event_amounts = training_meta.filter(filename=audio_filename).event_stat_counts()

                        for event_label, count in iteritems(current_event_amounts):
                            if event_label not in event_amounts[scene_label]:
                                event_amounts[scene_label][event_label] = 0

                            event_amounts[scene_label][event_label] += count

            validation_files = []
            for scene_id, scene_label in enumerate(scene_labels):
                # Optimize scene sets separately
                validation_set_candidates = []
                validation_set_mae = []
                validation_set_event_amounts = []
                training_set_event_amounts = []

                iteration_progress = tqdm(
                    range(0, iterations),
                    desc="{0: <25s}".format('Generate validation split candidates'),
                    file=sys.stdout,
                    leave=False,
                    disable=self.disable_progress_bar,
                    ascii=self.use_ascii_progress_bar
                )

                for i in iteration_progress:
                    identifiers = list(data[scene_label].keys())
                    random.shuffle(identifiers, random.random)

                    valid_percentage_index = int(numpy.ceil(validation_amount * len(identifiers)))

                    current_validation_files = []
                    for loc_id in identifiers[0:valid_percentage_index]:
                        current_validation_files += data[scene_label][loc_id]

                    current_training_files = []
                    for loc_id in identifiers[valid_percentage_index:]:
                        current_training_files += data[scene_label][loc_id]

                    # Event count in training set candidate
                    training_set_event_counts = numpy.zeros(len(event_amounts[scene_label]))
                    for audio_filename in current_training_files:
                        current_event_amounts = training_meta.filter(filename=audio_filename).event_stat_counts()
                        for event_label_id, event_label in enumerate(event_amounts[scene_label]):
                            if event_label in current_event_amounts:
                                training_set_event_counts[event_label_id] += current_event_amounts[event_label]

                    # Accept only sets which leave at least one example for training
                    if numpy.all(training_set_event_counts > 0):
                        # Event counts in validation set candidate
                        validation_set_event_counts = numpy.zeros(len(event_amounts[scene_label]))
                        for audio_filename in current_validation_files:
                            current_event_amounts = training_meta.filter(filename=audio_filename).event_stat_counts()

                            for event_label_id, event_label in enumerate(event_amounts[scene_label]):
                                if event_label in current_event_amounts:
                                    validation_set_event_counts[event_label_id] += current_event_amounts[event_label]

                        # Accept only sets which have examples from each event class
                        if numpy.all(validation_set_event_counts > 0):
                            current_validation_amount = validation_set_event_counts / (validation_set_event_counts + training_set_event_counts)
                            validation_set_candidates.append(current_validation_files)
                            validation_set_mae.append(
                                mean_absolute_error(
                                    numpy.ones(len(current_validation_amount)) * validation_amount,
                                    current_validation_amount)
                            )
                            validation_set_event_amounts.append(validation_set_event_counts)
                            training_set_event_amounts.append(training_set_event_counts)

                # Generate balance validation set
                # Selection done based on event counts (per scene class)
                # Target count specified percentage of training event count
                if validation_set_mae:
                    best_set_id = numpy.argmin(validation_set_mae)
                    validation_files += validation_set_candidates[best_set_id]

                else:
                    message = '{name}: Validation setup creation was not successful! Could not find a set with ' \
                              'examples for each event class in both training and validation.'.format(
                                name=self.__class__.__name__
                              )

                    self.logger.exception(message)
                    raise AssertionError(message)

        else:
            # Do the balance based on scene class, identifier and event class
            data = {}
            for scene_id, scene_label in enumerate(scene_labels):
                data[scene_label] = training_meta.filter(scene_label=scene_label).unique_files

            # Get event amounts per class
            event_amounts = {}
            for scene_id, scene_label in enumerate(scene_labels):
                event_amounts[scene_label] = {}
                for audio_filename in data[scene_label]:
                    current_event_amounts = training_meta.filter(filename=audio_filename).event_stat_counts()

                    for event_label, count in iteritems(current_event_amounts):
                        if event_label not in event_amounts[scene_label]:
                            event_amounts[scene_label][event_label] = 0

                        event_amounts[scene_label][event_label] += count

            validation_files = []
            for scene_id, scene_label in enumerate(scene_labels):
                # Optimize scene sets separately
                validation_set_candidates = []
                validation_set_mae = []
                validation_set_event_amounts = []
                training_set_event_amounts = []

                iteration_progress = tqdm(
                    range(0, iterations),
                    desc="{0: <25s}".format('Generate validation split candidates'),
                    file=sys.stdout,
                    leave=False,
                    disable=self.disable_progress_bar,
                    ascii=self.use_ascii_progress_bar
                )

                for i in iteration_progress:
                    item_ids = list(range(0, len(data[scene_label])))
                    random.shuffle(item_ids, random.random)

                    valid_percentage_index = int(numpy.ceil(validation_amount * len(item_ids)))

                    current_validation_files = []
                    for loc_id in item_ids[0:valid_percentage_index]:
                        current_validation_files.append(data[scene_label][loc_id])

                    current_training_files = []
                    for loc_id in item_ids[valid_percentage_index:]:
                        current_training_files.append(data[scene_label][loc_id])

                    # Event count in training set candidate
                    training_set_event_counts = numpy.zeros(len(event_amounts[scene_label]))
                    for audio_filename in current_training_files:
                        current_event_amounts = training_meta.filter(filename=audio_filename).event_stat_counts()
                        for event_label_id, event_label in enumerate(event_amounts[scene_label]):
                            if event_label in current_event_amounts:
                                training_set_event_counts[event_label_id] += current_event_amounts[event_label]

                    # Accept only sets which leave at least one example for training
                    if numpy.all(training_set_event_counts > 0):
                        # Event counts in validation set candidate
                        validation_set_event_counts = numpy.zeros(len(event_amounts[scene_label]))
                        for audio_filename in current_validation_files:
                            current_event_amounts = training_meta.filter(filename=audio_filename).event_stat_counts()

                            for event_label_id, event_label in enumerate(event_amounts[scene_label]):
                                if event_label in current_event_amounts:
                                    validation_set_event_counts[event_label_id] += current_event_amounts[event_label]

                        # Accept only sets which have examples from each event class
                        if numpy.all(validation_set_event_counts > 0):
                            current_validation_amount = validation_set_event_counts / (validation_set_event_counts + training_set_event_counts)
                            validation_set_candidates.append(current_validation_files)
                            validation_set_mae.append(
                                mean_absolute_error(
                                    numpy.ones(len(current_validation_amount)) * validation_amount,
                                    current_validation_amount)
                            )
                            validation_set_event_amounts.append(validation_set_event_counts)
                            training_set_event_amounts.append(training_set_event_counts)

                # Generate balance validation set
                # Selection done based on event counts (per scene class)
                # Target count specified percentage of training event count
                if validation_set_mae:
                    best_set_id = numpy.argmin(validation_set_mae)
                    validation_files += validation_set_candidates[best_set_id]

                else:
                    message = '{name}: Validation setup creation was not successful! Could not find a set with ' \
                              'examples for each event class in both training and validation.'.format(
                        name=self.__class__.__name__
                    )

                    self.logger.exception(message)
                    raise AssertionError(message)

        if verbose:
            logger = FancyLogger()
            logger.sub_header('Validation set for fold [{fold}] / balanced'.format(fold=fold), indent=2)
            for scene_id, scene_label in enumerate(scene_labels):
                all_stats = training_meta.filter(scene_label=scene_label).event_stat_counts()
                validation_stats = training_meta.filter(
                    scene_label=scene_label,
                    file_list=validation_files
                ).event_stat_counts()

                training_files = sorted(list(set(self.train(fold=fold).unique_files) - set(validation_files)))
                training_stats = training_meta.filter(
                    scene_label=scene_label,
                    file_list=training_files
                ).event_stat_counts()

                cell_data = numpy.zeros((len(list(all_stats.keys())) + 1, 4))
                for event_id, event_label in enumerate(list(all_stats.keys())):
                    cell_data[event_id, 0] = all_stats[event_label]
                    if event_label in training_stats:
                        cell_data[event_id, 1] = training_stats[event_label]

                    if event_label in validation_stats:
                        cell_data[event_id, 2] = validation_stats[event_label]

                    cell_data[event_id, 3] = cell_data[event_id, 2] / float(cell_data[event_id, 0]) * 100

                cell_data[-1, 0] = numpy.sum(list(all_stats.values()))
                cell_data[-1, 1] = numpy.sum(list(training_stats.values()))
                cell_data[-1, 2] = numpy.sum(list(validation_stats.values()))
                cell_data[-1, 3] = cell_data[-1, 2] / float(cell_data[-1, 0]) * 100

                labels = list(all_stats.keys())
                labels.append('Overall')
                logger.line(scene_label, indent=4)
                logger.table(
                    cell_data=[labels] + cell_data.T.tolist(),
                    column_headers=['Event', 'All', 'Training', 'Validation', 'Val amount (%)'],
                    column_types=['str20', 'int', 'int', 'int', 'float1'],
                    column_separators=[0, 1, 3],
                    row_separators=[len(labels)-1],
                    indent=6
                )

        return validation_files


class SyntheticSoundEventDataset(SoundEventDataset):
    """Synthetic sound event dataset baseclass"""
    def __init__(self, *args, **kwargs):
        super(SyntheticSoundEventDataset, self).__init__(*args, **kwargs)

    def initialize(self):
        """Initialize the dataset, download, extract files and prepare the dataset for the usage.

        Returns
        -------
        self

        """

        # Create the dataset path if does not exist
        Path().makedirs(path=self.local_path)

        if not self.check_filelist():
            self.download_packages()
            self.extract_packages()
            self.load()
            self.prepare()
            self._save_filelist_hash()

        self.synthesize()

        return self

    def synthesize(self):
        pass


class AudioTaggingDataset(Dataset):
    """Audio tag dataset baseclass"""
    def __init__(self, *args, **kwargs):
        super(AudioTaggingDataset, self).__init__(*args, **kwargs)

    def validation_files_random(self,
                                fold=None, validation_amount=0.3, seed=0, verbose=False,
                                **kwargs):
        """List of validation files selected randomly from the training material.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        validation_amount : float
            Amount of training material to be assigned for validation.
            Default value 0.3

        seed : int
            Randomization seed
            Default value 0

        verbose : bool
            Show information about the validation set.
            Default value False

        Returns
        -------
        list of str
            List containing all files assigned for validation

        """

        random.seed(seed)
        training_meta = self.train(fold=fold)

        scene_labels = self.scene_labels()

        training_files = training_meta.unique_files
        random.shuffle(training_files, random.random)

        validation_split_index = int(numpy.ceil(validation_amount * len(training_files)))
        validation_files = training_files[0:validation_split_index]

        if verbose:
            logger = FancyLogger()
            logger.sub_header('Validation set for fold [{fold}] / random'.format(fold=fold), indent=2)
            for scene_id, scene_label in enumerate(scene_labels):
                all_stats = training_meta.filter(scene_label=scene_label).tag_stat_counts()
                validation_stats = training_meta.filter(
                    scene_label=scene_label,
                    file_list=validation_files
                ).tag_stat_counts()

                training_files = sorted(list(set(self.train(fold=fold).unique_files) - set(validation_files)))
                training_stats = training_meta.filter(
                    scene_label=scene_label,
                    file_list=training_files
                ).tag_stat_counts()

                cell_data = numpy.zeros((len(list(all_stats.keys())) + 1, 4))
                for event_id, event_label in enumerate(list(all_stats.keys())):
                    cell_data[event_id, 0] = all_stats[event_label]
                    if event_label in training_stats:
                        cell_data[event_id, 1] = training_stats[event_label]

                    if event_label in validation_stats:
                        cell_data[event_id, 2] = validation_stats[event_label]

                    cell_data[event_id, 3] = cell_data[event_id, 2] / float(cell_data[event_id, 0]) * 100

                cell_data[-1, 0] = numpy.sum(list(all_stats.values()))
                cell_data[-1, 1] = numpy.sum(list(training_stats.values()))
                cell_data[-1, 2] = numpy.sum(list(validation_stats.values()))
                cell_data[-1, 3] = cell_data[-1, 2] / float(cell_data[-1, 0]) * 100

                labels = list(all_stats.keys())
                labels.append('Overall')
                logger.line(scene_label, indent=4)
                logger.table(
                    cell_data=[labels] + cell_data.T.tolist(),
                    column_headers=['Tag', 'All', 'Training', 'Validation', 'Val amount (%)'],
                    column_types=['str20', 'int', 'int', 'int', 'float1'],
                    column_separators=[0, 1, 3],
                    row_separators=[len(labels)-1],
                    indent=6
                )

        return validation_files

    def validation_files_balanced(self,
                                  fold=None, validation_amount=0.3, seed=0, verbose=False, iterations=100,
                                  **kwargs):
        """List of validation files randomly selecting while maintaining data balance.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        validation_amount : float
            Amount of training material to be assigned for validation.
            Default value 0.3

        seed : int
            Randomization seed
            Default value 0

        verbose : bool
            Show information about the validation set.
            Default value False

        iterations : int
            How many randomization iterations will be done before selecting best matched.
            Default value 100

        Returns
        -------
        list of str
            List containing all files assigned for validation

        """

        from sklearn.metrics import mean_absolute_error

        random.seed(seed)
        training_meta = self.train(fold=fold)

        # Check do we have location/source identifier present
        identifier_present = False
        for item in training_meta:
            if item.identifier:
                identifier_present = True
                break

        scene_labels = self.scene_labels()

        if identifier_present:
            # Do the balance based on scene class, identifier and event class
            data = {}
            for scene_id, scene_label in enumerate(scene_labels):
                scene_meta = training_meta.filter(scene_label=scene_label)
                data[scene_label] = {}
                for identifier in scene_meta.unique_identifiers:
                    data[scene_label][identifier] = scene_meta.filter(identifier=identifier).unique_files

            # Get tag amounts per class
            tag_amounts = {}
            for scene_id, scene_label in enumerate(scene_labels):
                tag_amounts[scene_label] = {}
                for identifier in list(data[scene_label].keys()):
                    for audio_filename in data[scene_label][identifier]:
                        current_tag_amounts = training_meta.filter(filename=audio_filename).tag_stat_counts()

                        for tag_label, count in iteritems(current_tag_amounts):
                            if tag_label not in tag_amounts[scene_label]:
                                tag_amounts[scene_label][tag_label] = 0

                            tag_amounts[scene_label][tag_label] += count

            validation_files = []
            for scene_id, scene_label in enumerate(scene_labels):
                # Optimize scene sets separately
                validation_set_candidates = []
                validation_set_mae = []
                validation_set_tag_amounts = []
                training_set_tag_amounts = []

                iteration_progress = tqdm(
                    range(0, iterations),
                    desc="{0: <25s}".format('Generate validation split candidates'),
                    file=sys.stdout,
                    leave=False,
                    disable=self.disable_progress_bar,
                    ascii=self.use_ascii_progress_bar
                )

                for i in iteration_progress:
                    identifiers = list(data[scene_label].keys())
                    random.shuffle(identifiers, random.random)

                    valid_percentage_index = int(numpy.ceil(validation_amount * len(identifiers)))

                    current_validation_files = []
                    for loc_id in identifiers[0:valid_percentage_index]:
                        current_validation_files += data[scene_label][loc_id]

                    current_training_files = []
                    for loc_id in identifiers[valid_percentage_index:]:
                        current_training_files += data[scene_label][loc_id]

                    # Tag count in training set candidate
                    training_set_tag_counts = numpy.zeros(len(tag_amounts[scene_label]))
                    for audio_filename in current_training_files:
                        current_tag_amounts = training_meta.filter(filename=audio_filename).tag_stat_counts()
                        for tag_label_id, tag_label in enumerate(tag_amounts[scene_label]):
                            if tag_label in current_tag_amounts:
                                training_set_tag_counts[tag_label_id] += current_tag_amounts[tag_label]

                    # Accept only sets which leave at least one example for training
                    if numpy.all(training_set_tag_counts > 0):
                        # Tag counts in validation set candidate
                        validation_set_tag_counts = numpy.zeros(len(tag_amounts[scene_label]))
                        for audio_filename in current_validation_files:
                            current_tag_amounts = training_meta.filter(filename=audio_filename).tag_stat_counts()

                            for tag_label_id, tag_label in enumerate(tag_amounts[scene_label]):
                                if tag_label in current_tag_amounts:
                                    validation_set_tag_counts[tag_label_id] += current_tag_amounts[tag_label]

                        # Accept only sets which have examples from each tag class
                        if numpy.all(validation_set_tag_counts > 0):
                            current_validation_amount = validation_set_tag_counts / (validation_set_tag_counts + training_set_tag_counts)
                            validation_set_candidates.append(current_validation_files)
                            validation_set_mae.append(
                                mean_absolute_error(
                                    numpy.ones(len(current_validation_amount)) * validation_amount,
                                    current_validation_amount)
                            )
                            validation_set_tag_amounts.append(validation_set_tag_counts)
                            training_set_tag_amounts.append(training_set_tag_counts)

                # Generate balance validation set
                # Selection done based on event counts (per scene class)
                # Target count specified percentage of training event count
                if validation_set_mae:
                    best_set_id = numpy.argmin(validation_set_mae)
                    validation_files += validation_set_candidates[best_set_id]

                else:
                    message = '{name}: Validation setup creation was not successful! Could not find a set with ' \
                              'examples for each tag class in both training and validation.'.format(
                        name=self.__class__.__name__
                    )

                    self.logger.exception(message)
                    raise AssertionError(message)
        else:
            # Do the balance based on scene class, identifier and event class
            data = {}
            for scene_id, scene_label in enumerate(scene_labels):
                data[scene_label] = training_meta.filter(scene_label=scene_label).unique_files

            # Get tag amounts per class
            tag_amounts = {}
            for scene_id, scene_label in enumerate(scene_labels):
                tag_amounts[scene_label] = {}
                for audio_filename in data[scene_label]:
                    current_tag_amounts = training_meta.filter(filename=audio_filename).tag_stat_counts()

                    for tag_label, count in iteritems(current_tag_amounts):
                        if tag_label not in tag_amounts[scene_label]:
                            tag_amounts[scene_label][tag_label] = 0

                        tag_amounts[scene_label][tag_label] += count

            validation_files = []
            for scene_id, scene_label in enumerate(scene_labels):
                # Optimize scene sets separately
                validation_set_candidates = []
                validation_set_mae = []
                validation_set_tag_amounts = []
                training_set_tag_amounts = []

                iteration_progress = tqdm(
                    range(0, iterations),
                    desc="{0: <25s}".format('Generate validation split candidates'),
                    file=sys.stdout,
                    leave=False,
                    disable=self.disable_progress_bar,
                    ascii=self.use_ascii_progress_bar
                )

                for i in iteration_progress:
                    items_id = list(range(0, len(data[scene_label])))
                    random.shuffle(items_id, random.random)

                    valid_percentage_index = int(numpy.ceil(validation_amount * len(items_id)))

                    current_validation_files = []
                    for loc_id in items_id[0:valid_percentage_index]:
                        current_validation_files.append(data[scene_label][loc_id])

                    current_training_files = []
                    for loc_id in items_id[valid_percentage_index:]:
                        current_training_files.append(data[scene_label][loc_id])

                    # Tag count in training set candidate
                    training_set_tag_counts = numpy.zeros(len(tag_amounts[scene_label]))
                    for audio_filename in current_training_files:
                        current_tag_amounts = training_meta.filter(filename=audio_filename).tag_stat_counts()
                        for tag_label_id, tag_label in enumerate(tag_amounts[scene_label]):
                            if tag_label in current_tag_amounts:
                                training_set_tag_counts[tag_label_id] += current_tag_amounts[tag_label]

                    # Accept only sets which leave at least one example for training
                    if numpy.all(training_set_tag_counts > 0):
                        # Tag counts in validation set candidate
                        validation_set_tag_counts = numpy.zeros(len(tag_amounts[scene_label]))
                        for audio_filename in current_validation_files:
                            current_tag_amounts = training_meta.filter(filename=audio_filename).tag_stat_counts()

                            for tag_label_id, tag_label in enumerate(tag_amounts[scene_label]):
                                if tag_label in current_tag_amounts:
                                    validation_set_tag_counts[tag_label_id] += current_tag_amounts[tag_label]

                        # Accept only sets which have examples from each tag class
                        if numpy.all(validation_set_tag_counts > 0):
                            current_validation_amount = validation_set_tag_counts / (
                                validation_set_tag_counts + training_set_tag_counts)
                            validation_set_candidates.append(current_validation_files)
                            validation_set_mae.append(
                                mean_absolute_error(
                                    numpy.ones(len(current_validation_amount)) * validation_amount,
                                    current_validation_amount)
                            )
                            validation_set_tag_amounts.append(validation_set_tag_counts)
                            training_set_tag_amounts.append(training_set_tag_counts)

                # Generate balance validation set
                # Selection done based on event counts (per scene class)
                # Target count specified percentage of training event count
                if validation_set_mae:
                    best_set_id = numpy.argmin(validation_set_mae)
                    validation_files += validation_set_candidates[best_set_id]

                else:
                    message = '{name}: Validation setup creation was not successful! Could not find a set with ' \
                              'examples for each tag class in both training and validation.'.format(
                        name=self.__class__.__name__
                    )

                    self.logger.exception(message)
                    raise AssertionError(message)

        if verbose:
            logger = FancyLogger()
            logger.sub_header('Validation set for fold [{fold}] / balanced'.format(fold=fold), indent=2)
            for scene_id, scene_label in enumerate(scene_labels):
                all_stats = training_meta.filter(scene_label=scene_label).tag_stat_counts()
                validation_stats = training_meta.filter(
                    scene_label=scene_label,
                    file_list=validation_files
                ).tag_stat_counts()

                training_files = sorted(list(set(self.train(fold=fold).unique_files) - set(validation_files)))
                training_stats = training_meta.filter(
                    scene_label=scene_label,
                    file_list=training_files
                ).tag_stat_counts()

                cell_data = numpy.zeros((len(list(all_stats.keys())) + 1, 4))
                for tag_id, tag_label in enumerate(list(all_stats.keys())):
                    cell_data[tag_id, 0] = all_stats[tag_label]
                    if tag_label in training_stats:
                        cell_data[tag_id, 1] = training_stats[tag_label]

                    if tag_label in validation_stats:
                        cell_data[tag_id, 2] = validation_stats[tag_label]

                    cell_data[tag_id, 3] = cell_data[tag_id, 2] / float(cell_data[tag_id, 0]) * 100

                cell_data[-1, 0] = numpy.sum(list(all_stats.values()))
                cell_data[-1, 1] = numpy.sum(list(training_stats.values()))
                cell_data[-1, 2] = numpy.sum(list(validation_stats.values()))
                cell_data[-1, 3] = cell_data[-1, 2] / float(cell_data[-1, 0]) * 100

                labels = list(all_stats.keys())
                labels.append('Overall')
                logger.line(scene_label, indent=4)
                logger.table(
                    cell_data=[labels] + cell_data.T.tolist(),
                    column_headers=['Tag', 'All', 'Training', 'Validation', 'Val amount (%)'],
                    column_types=['str20', 'int', 'int', 'int', 'float1'],
                    column_separators=[0, 1, 3],
                    row_separators=[len(labels)-1],
                    indent=6
                )

        return validation_files

#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function, absolute_import
from six import iteritems

import logging
import os
import sys
import random
import numpy
import tempfile
import copy
import importlib
from tqdm import tqdm

from dcase_util.containers import DictContainer, ListDictContainer, TextContainer, MetaDataContainer
from dcase_util.files import RemoteFile, RemotePackage, File
from dcase_util.utils import get_byte_string, setup_logging, Path
from dcase_util.utils import get_parameter_hash, get_class_inheritors
from dcase_util.ui import FancyLogger


def dataset_list(data_path='data', group=None):
    """List of datasets available

    Parameters
    ----------
    data_path : str
        Base path for the datasets
        Default value 'data'

    group : str
        Group label for the datasets, currently supported ['scene', 'event', 'tag']

    Returns
    -------
    str
        Multi line string containing dataset table

    """
    line = '  {class_name:<42s} | {group:5s} | {remote_size:10s} | {local_present:6s} | {files:5s} | {scene:6s} | {event:6s} | {tag:4s}\n'
    output = ''
    output += '  Dataset list\n'
    output += line.format(
        class_name='Class Name',
        group='Group',
        remote_size='Remote',
        local_present='Local',
        files='Audio',
        scene='Scenes',
        event='Events',
        tag='Tags'
    )
    output += line.format(
        class_name='-' * 42,
        group='-' * 5,
        remote_size='-' * 10,
        local_present='-' * 6,
        files='-' * 5,
        scene='-' * 6,
        event='-' * 6,
        tag='-' * 4
    )

    def get_empty_row():
        return line.format(
            class_name='',
            group='',
            remote_size='',
            local_present='',
            files='',
            scene='',
            event=''
        )

    def get_row(data):
        file_count = 0
        if data.meta_container.exists():
            file_count = len(data.meta)

        return line.format(
            class_name=data.__class__.__name__,
            group=data.dataset_group,
            remote_size=data.dataset_size_string(),
            local_present=data.dataset_size_on_disk(),
            files=str(file_count) if file_count else '',
            scene=str(data.scene_label_count()) if data.scene_label_count() else '',
            event=str(data.event_label_count()) if data.event_label_count() else '',
            tag=str(data.tag_count()) if data.tag_count() else '',
        )

    if not group or group == 'scene':
        class_list = get_class_inheritors(AcousticSceneDataset)
        class_list.sort(key=lambda x: x.__name__, reverse=False)

        for dataset_class in class_list:
            d = dataset_class(data_path=data_path)
            if d.dataset_group != 'base class':
                output += get_row(d)

    if not group or group == 'event':
        class_list = get_class_inheritors(SoundEventDataset)
        class_list.sort(key=lambda x: x.__name__, reverse=False)

        for dataset_class in class_list:
            d = dataset_class(data_path=data_path)
            if d.dataset_group != 'base class':
                output += get_row(d)

    if not group or group == 'tag':
        class_list = get_class_inheritors(AudioTaggingDataset)
        class_list.sort(key=lambda x: x.__name__, reverse=False)

        for dataset_class in class_list:
            d = dataset_class(data_path=data_path)
            if d.dataset_group != 'base class':
                output += get_row(d)

    return output


def dataset_factory(dataset_class_name, **kwargs):
    """Factory to get correct dataset class based on name

    Parameters
    ----------
    dataset_class_name : str
        Class name

    Raises
    ------
    NameError
        Class does not exists

    Returns
    -------
    Dataset class

    """

    try:
        dataset_class = getattr(
            importlib.import_module("dcase_util.datasets"),
            dataset_class_name
        )

    except AttributeError:
        message = 'Invalid Dataset class given [{class_name}].'.format(
            class_name=dataset_class_name
        )
        logger = logging.getLogger(__name__)
        if not logger.handlers:
            setup_logging()

        logger.exception(message)
        raise AttributeError(message)

    return dataset_class(**dict(kwargs))


class Dataset(object):
    """Dataset base class

    The specific dataset classes are inherited from this class, and only needed methods are reimplemented.

    """

    def __init__(self,
                 name='dataset',
                 storage_name='dataset',
                 data_path=None,
                 local_path=None,

                 show_progress_in_console=True,
                 log_system_progress=True,
                 use_ascii_progress_bar=True,

                 dataset_group='base class',
                 dataset_meta=None,

                 evaluation_setup_folder='evaluation_setup',
                 meta_filename='meta.txt',
                 error_meta_filename='error.txt',
                 filelisthash_filename='filelist.python.hash',
                 filelisthash_exclude_dirs=None,
                 crossvalidation_folds=None,
                 package_list=None,
                 included_content_types=None,
                 audio_paths=None,
                 default_audio_extension='wav',
                 reference_data_present=True,
                 **kwargs):
        """Constructor

        Parameters
        ----------
        name : str
            Dataset name

        storage_name : str
            Name to be used when storing dataset on disk

        data_path : str
            Root path where the dataset is stored. If None, os.path.join(tempfile.gettempdir(), 'dcase_util_datasets')
            is used.

        local_path : str
            Direct storage path setup for the dataset. If None, data_path and storage_name are used to create one.

        show_progress_in_console : bool
            Show progress in console.

        log_system_progress : bool
            Show progress in log.

        use_ascii_progress_bar : bool
            Show progress bar using ASCII characters. Use this if your console does not support UTF-8 characters.

        dataset_group : str
            Dataset group label, one of ['scene', 'event', 'tag']

        dataset_meta : dict
            Dictionary containing metadata about the dataset, e.g., collecting device information, dataset authors.

        evaluation_setup_folder : str
            Directory name where evaluation setup files are stores

        meta_filename : str
            Filename to be used for main meta file (contains all files with their reference data) of the dataset.

        error_meta_filename : str
            Filename for the error annotation file.

        filelisthash_filename : str
            Filename for filelist hash file.

        filelisthash_exclude_dirs : str
            Directories to be excluded from filelist hash calculation.

        crossvalidation_folds : int
            Count fo cross-validation folds. Indexing starts from one.

        package_list : list of dict
            Package list, remote files associated to the dataset.

        included_content_types : list of str or str
            Indicates what content type should be processed. One or multiple from ['all', 'audio', 'meta', 'code',
            'documentation']. If None given, ['all'] is used. Parameter can be also comma separated string.

        audio_paths : list of str
            List of paths to include audio material associated to the dataset. If None given, ['audio'] is used.

        default_audio_extension : str
            Default audio extension

        reference_data_present : bool
            Reference data is delivered with the dataset.

        """

        self.disable_progress_bar = not show_progress_in_console
        self.log_system_progress = log_system_progress
        self.use_ascii_progress_bar = use_ascii_progress_bar

        # Dataset name
        self.name = name

        # Dataset group
        self.dataset_group = dataset_group

        # Dataset meta
        if dataset_meta is None:
            dataset_meta = {}
        self.dataset_meta = DictContainer(dataset_meta)

        # Folder name for dataset
        self.storage_name = storage_name

        # Path to the dataset
        if data_path is None:
            data_path = os.path.join(tempfile.gettempdir(), 'dcase_util_datasets')

        if local_path is None:
            self.local_path = os.path.join(data_path, self.storage_name)

        else:
            self.local_path = local_path

        # Evaluation setup folder
        self.evaluation_setup_folder = evaluation_setup_folder

        # Path to the folder containing evaluation setup files
        self.evaluation_setup_path = os.path.join(self.local_path, self.evaluation_setup_folder)

        # Meta data file, csv-format
        self.meta_filename = meta_filename
        self.meta_file = os.path.join(self.local_path, self.meta_filename)

        # Error meta data file, csv-format
        self.error_meta_filename = error_meta_filename

        # Path to error meta data file
        self.error_meta_file = os.path.join(self.local_path, self.error_meta_filename)

        # Hash file to detect removed or added files
        self.filelisthash_filename = filelisthash_filename

        # Dirs to be excluded when calculating filelist hash
        if filelisthash_exclude_dirs is None:
            filelisthash_exclude_dirs = []

        self.filelisthash_exclude_dirs = filelisthash_exclude_dirs

        # Number of evaluation folds
        self.crossvalidation_folds = crossvalidation_folds

        # List containing dataset package items
        # Define this in the inherited dataset specific class.
        # Format:
        # {
        #    'content_type': 'documentation',  # Possible values ['meta', 'documentation', 'audio', 'code', 'features']
        #    'remote_file': 'https://zenodo.org/record/45759/files/TUT-sound-events-2016-development.doc.zip', # URL
        #    'remote_bytes': 70918,            # Size of remote file in bytes
        #    'remote_md5': '33fd26a895530aef607a07b08704eacd',  # MD5 hash of remote file
        #    'filename': 'TUT-sound-events-2016-development.doc.zip' # filename relative to self.local_path always
        # }
        if package_list is None:
            package_list = []
        self.package_list = ListDictContainer(package_list)

        # Expand local filenames to be related to local path
        for item in self.package_list:
            item['filename'] = os.path.join(self.local_path, item['filename'])

        # What content type should be processed. Use this for example to access only the meta data, and exclude usually
        # large and time consuming audio material downloading. Leave to "all" to include all content types.
        if included_content_types is None:
            included_content_types = ['all']

        if isinstance(included_content_types, str):
            # Split string to list if given
            included_content_types = included_content_types.split(',')

        self.included_content_types = included_content_types

        # Inject all included content types, there might be packages containing multiple content types.
        if 'all' not in self.included_content_types:
            for package_data in self.package_list:
                package_item = RemoteFile(**package_data)
                if self.included_content_types is None or package_item.is_content_type(
                        content_type=self.included_content_types
                ):
                    self.included_content_types = list(set().union(
                        package_item.content_type, self.included_content_types)
                    )

        # List of directories to contain the audio material
        if audio_paths is None:
            audio_paths = ['audio']
        self.audio_paths = audio_paths

        # Expand local filenames to be related to local path
        for path_id, path in enumerate(self.audio_paths):
            self.audio_paths[path_id] = os.path.join(self.local_path, path)

        # List of audio files
        self.files = None

        # Recognized audio extensions
        self.audio_extensions = ['wav', 'flac']

        self.default_audio_extension = default_audio_extension

        # Reference data presence flag, by default dataset should have reference data present.
        # However, some evaluation dataset might not have
        self.reference_data_present = reference_data_present

        # Initialize meta data container
        self.meta_container = MetaDataContainer()

        # List of audio error meta data dict
        self.error_meta_data = None

        # Cross-validation data
        self.crossvalidation_data = DictContainer({
            'train': {},
            'test': {},
            'evaluate': {},
        })

        # Training meta data for folds
        self.crossvalidation_data_train = {}

        # Testing meta data for folds
        self.crossvalidation_data_test = {}

        # Evaluation meta data for folds
        self.crossvalidation_data_eval = {}

        # Load meta and cross-validation data in
        self.load()

    @property
    def logger(self):
        logger = logging.getLogger(__name__)
        if not logger.handlers:
            setup_logging()

        return logger

    def load(self):
        """Load dataset meta data and cross-validation sets into the container."""
        self.load_meta()
        self.load_crossvalidation_data()

    def load_meta(self):
        """Load meta data into the container."""
        # Initialize meta container
        self.meta_container = MetaDataContainer(
            filename=os.path.join(self.local_path, self.meta_filename)
        )

        # Load from disk only if file exists
        if self.meta_container.exists():
            # Load meta data
            self.meta_container.load()

            # Mark reference data being present
            self.reference_data_present = True

        return self

    def load_crossvalidation_data(self):
        """Load cross-validation into the container.

        Returns
        -------
        self

        """

        # Reset cross validation data and insert 'all_data'
        self.crossvalidation_data = DictContainer({
            'train': {
                'all_data': self.meta_container
            },
            'test': {
                'all_data': self.meta_container
            },
            'evaluate': {
                'all_data': self.meta_container
            },
        })

        for crossvalidation_set in list(self.crossvalidation_data.keys()):
            for item in self.crossvalidation_data[crossvalidation_set]['all_data']:
                self.process_meta_item(item=item)

        # Load cross validation folds
        for fold in self.folds():
            # Get filenames
            train_filename = self.evaluation_setup_filename(
                setup_part='train',
                fold=fold
            )

            test_filename = self.evaluation_setup_filename(
                setup_part='test',
                fold=fold
            )

            evaluate_filename = self.evaluation_setup_filename(
                setup_part='evaluate',
                fold=fold
            )

            if os.path.isfile(train_filename):
                # Training data for fold exists, load and process it
                self.crossvalidation_data['train'][fold] = MetaDataContainer(filename=train_filename).load()

                # Process items
                for item in self.crossvalidation_data['train'][fold]:
                    self.process_meta_item(item=item)

            else:
                # Initialize data
                self.crossvalidation_data['train'][fold] = MetaDataContainer()

            if os.path.isfile(test_filename):
                # Testing data for fold exists, load and process it
                self.crossvalidation_data['test'][fold] = MetaDataContainer(filename=test_filename).load()

                # Process items
                for item in self.crossvalidation_data['test'][fold]:
                    self.process_meta_item(item=item)

            else:
                # Initialize data
                self.crossvalidation_data['test'][fold] = MetaDataContainer()

            if os.path.isfile(evaluate_filename):
                # Evaluation data for fold exists, load and process it
                self.crossvalidation_data['evaluate'][fold] = MetaDataContainer(filename=evaluate_filename).load()

                # Process items
                for item in self.crossvalidation_data['evaluate'][fold]:
                    self.process_meta_item(item=item)

            else:
                # Initialize data
                self.crossvalidation_data['evaluate'][fold] = MetaDataContainer()

        return self

    def __getitem__(self, i):
        """Getting meta data item

        Parameters
        ----------
        i : int
            item id

        Returns
        -------
        meta_data : dict
            Meta data item

        """

        if i < len(self.meta_container):
            return self.meta_container[i]

        else:
            return None

    def __iter__(self):
        """Iterator for meta data items
        """

        i = 0
        meta = self[i]

        # yield window while it's valid
        while meta is not None:
            yield meta
            # get next item
            i += 1
            meta = self[i]

    def initialize(self):
        """Initialize the dataset, download, extract files and prepare the dataset for the usage.

        Returns
        -------
        self

        """

        # Create the dataset path if does not exist
        Path().makedirs(path=self.local_path)

        # Check file changes
        if not self.check_filelist():
            # Download packages
            self.download_packages()

            # Extract content from packages
            self.extract_packages()

            # Load meta data in
            self.load()

            # Prepare meta data for the dataset class.
            self.prepare()

            # Save new filelist hash to monitor change in the dataset.
            self._save_filelist_hash()

        return self

    def show(self):
        """Print dataset information."""
        DictContainer(self.dataset_meta).show()
        self.meta_container.show(
            show_data=False,
            show_stats=True
        )

    def log(self):
        """Log dataset information."""
        DictContainer(self.dataset_meta).log()
        self.meta_container.log(
            show_data=False,
            show_stats=True
        )

    @property
    def audio_files(self):
        """Get all audio files in the dataset

        Returns
        -------
        list
            File list with absolute paths

        """

        if self.files is None:
            self.files = []
            for path in self.audio_paths:
                if path and os.path.exists(path):
                    dir_list = os.listdir(path)
                    for f in dir_list:
                        file_name, file_extension = os.path.splitext(f)
                        if file_extension[1:] in self.audio_extensions:
                            if os.path.abspath(os.path.join(path, f)) not in self.files:
                                self.files.append(os.path.abspath(os.path.join(path, f)))

            self.files.sort()
        return self.files

    @property
    def audio_file_count(self):
        """Get number of audio files in dataset

        Returns
        -------
        int
            Number of audio files

        """

        return len(self.audio_files)

    @property
    def meta(self):
        """Get meta data for dataset. If not already read from disk, data is read and returned.

        Raises
        ------
        IOError
            meta file not found.

        Returns
        -------
        list
            List containing meta data as dict.

        """

        if self.meta_container.empty():
            if self.meta_container.exists():
                self.meta_container.load()

            else:
                message = '{name}: Meta file not found [{filename}]'.format(
                    name=self.__class__.__name__,
                    filename=self.meta_container.filename
                )

                self.logger.exception(message)
                raise IOError(message)

        return copy.deepcopy(self.meta_container)

    @property
    def meta_count(self):
        """Number of meta data items.

        Returns
        -------
        int
            Meta data item count

        """

        return len(self.meta_container)

    @property
    def error_meta(self):
        """Get audio error meta data for dataset. If not already read from disk, data is read and returned.

        Raises
        ------
        IOError:
            audio error meta file not found.

        Returns
        -------
        MetaDataContainer
            List containing audio error meta data as dict.

        """

        if self.error_meta_data is None:
            self.error_meta_data = MetaDataContainer(filename=self.error_meta_file)
            if self.error_meta_data.exists():
                self.error_meta_data.load()

            else:
                message = '{name}: Error meta file not found [{filename}]'.format(
                    name=self.__class__.__name__,
                    filename=self.error_meta_file
                )

                self.logger.exception(message)
                raise IOError(message)

        return copy.deepcopy(self.error_meta_data)

    @property
    def error_meta_count(self):
        """Number of error meta data items.

        Returns
        -------
        int
            Meta data item count

        """

        return len(self.error_meta)

    @property
    def fold_count(self):
        """Number of fold in the evaluation setup.

        Returns
        -------
        int
            Number of folds

        """

        return self.crossvalidation_folds

    def scene_labels(self):
        """List of unique scene labels in the meta data.

        Returns
        -------
        list
            List of scene labels in alphabetical order.

        """

        return self.meta_container.unique_scene_labels

    def scene_label_count(self):
        """Number of unique scene labels in the meta data.

        Returns
        -------
        int
            Number of unique scene labels.

        """

        return self.meta_container.scene_label_count

    def event_labels(self, **kwargs):
        """List of unique event labels in the meta data.

        Returns
        -------
        list
            List of event labels in alphabetical order.

        """

        return self.meta_container.unique_event_labels

    def event_label_count(self, **kwargs):
        """Number of unique event labels in the meta data.

        Returns
        -------
        int
            Number of unique event labels

        """

        return self.meta_container.event_label_count

    def tags(self):
        """List of unique audio tags in the meta data.

        Returns
        -------
        list
            List of audio tags in alphabetical order.

        """

        return self.meta_container.unique_tags

    def tag_count(self):
        """Number of unique audio tags in the meta data.

        Returns
        -------
        int
            Number of unique audio tags

        """

        return len(self.tags())

    def debug_packages(self):
        """Debug remote packages associated to the dataset.
        Use this to check remote file size and md5 hash when developing dataset class.

        Returns
        -------
        self

        """

        log = FancyLogger()
        log.sub_header('Debug packages')
        log.row('package', 'remote_md5', 'remote_bytes', widths=[70, 36, 15])
        log.row('-', '-', '-')
        for item in self.package_list:
            remote_file = RemoteFile(**item)
            if self.included_content_types is None or remote_file.is_content_type(
                    content_type=self.included_content_types
            ):
                remote_file.remote_info()

                log.row(
                    os.path.split(item['remote_file'])[-1],
                    remote_file.remote_md5 if item['remote_md5'] != remote_file.remote_md5 else 'OK',
                    remote_file.remote_bytes if item['remote_bytes'] != remote_file.remote_bytes else 'OK'
                )

        return self

    def download_packages(self):
        """Download dataset packages over the internet to the local path

        Raises
        ------
        IOError
            Download failed.

        Returns
        -------
        self

        """

        # Create the dataset path if does not exist
        Path().makedirs(path=self.local_path)

        item_progress = tqdm(
            self.package_list,
            desc="{0: <25s}".format('Download packages'),
            file=sys.stdout,
            leave=False,
            disable=self.disable_progress_bar,
            ascii=self.use_ascii_progress_bar
        )

        for item in item_progress:
            if 'remote_file' in item:
                # Download if remote file is set
                remote_file = RemoteFile(**item)
                if self.included_content_types is None or remote_file.is_content_type(
                        content_type=self.included_content_types
                ):
                    remote_file.download()

        return self

    def extract_packages(self):
        """Extract the dataset packages

        Raises
        ------
        IOError
            Local package was not found.

        Returns
        -------
        self

        """

        item_progress = tqdm(
            self.package_list,
            desc="{0: <25s}".format('Extract packages'),
            file=sys.stdout,
            leave=False,
            disable=self.disable_progress_bar,
            ascii=self.use_ascii_progress_bar
        )

        for item_id, item in enumerate(item_progress):
            if File().is_package(filename=item['filename']):
                # Process only files which are packages
                remote_package = RemotePackage(**item)

                if self.included_content_types is None or remote_package.is_content_type(
                        content_type=self.included_content_types
                ):
                    if remote_package.local_exists():
                        remote_package.extract(
                            omit_first_level=True
                        )

                    else:
                        # Local file not present
                        message = '{name}: Local package does not exists [{filename}]'.format(
                            name=self.__class__.__name__,
                            filename=remote_package.filename,
                        )
                        self.logger.exception(message)
                        raise IOError(message)

        return self

    def prepare(self):
        """Prepare dataset for the usage.

        Returns
        -------
        self

        """

        return self

    def check_filelist(self):
        """Generates hash from file list and check does it matches with one saved in filelist.hash.
        If some files have been deleted or added, checking will result False.

        Returns
        -------
        bool
            Result

        """

        if os.path.isfile(os.path.join(self.local_path, self.filelisthash_filename)):
            old_hash_value = TextContainer(filename=os.path.join(self.local_path, self.filelisthash_filename)).load()[0]

            file_list = self._get_filelist(exclude_dirs=self.filelisthash_exclude_dirs)
            new_hash_value = get_parameter_hash(file_list)

            if self.included_content_types:
                new_hash_value += ';'+','.join(self.included_content_types)

            if old_hash_value != new_hash_value:
                return False

            else:
                return True

        else:
            return False

    def process_meta_item(self, item, absolute_path=True, **kwargs):
        """Process single meta data item

        Parameters
        ----------
        item :  MetaDataItem
            Meta data item

        absolute_path : bool
            Path format for the returned meta item, if True path is absolute, False path is relative to
            the dataset root.
            Default value True

        """

        if absolute_path:
            # Make sure filename has absolute path
            item.filename = self.relative_to_absolute_path(item.filename)

        else:
            # Make sure filename has relative path
            item.filename = self.absolute_to_relative_path(item.filename)

    def evaluation_setup_filename(self, setup_part='train', fold=None, scene_label=None, file_extension='txt'):
        """Evaluation setup filename generation.

        Parameters
        ----------
        setup_part :  str
            Setup part 'train', 'test', 'evaluate'
            Default value 'train'

        fold : int
            Fold number
            Default value None

        scene_label : str
            Scene label
            Default value None

        file_extension : str
            File extension
            Default value 'txt'

        Raises
        ------
        ValueError
            Unknown setup part.

        Returns
        -------
        str
            Filename with full path.

        """

        if fold == 'all_data':
            fold = None

        parts = []
        if scene_label:
            parts.append(scene_label)

        if fold:
            parts.append('fold' + str(fold))

        if setup_part == 'train':
            parts.append('train')

        elif setup_part == 'test':
            parts.append('test')

        elif setup_part == 'evaluate':
            parts.append('evaluate')

        else:
            message = '{name}: Unknown setup_part [{setup_part}]'.format(
                name=self.__class__.__name__,
                setup_part=setup_part
            )

            self.logger.exception(message)
            raise ValueError(message)

        return os.path.join(self.evaluation_setup_path, '_'.join(parts) + '.' + file_extension)

    def train(self, fold=None, absolute_paths=True, **kwargs):
        """List of training items.

        Parameters
        ----------
        fold : int [scalar]
            Fold id, if None all meta data is returned.

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        Returns
        -------
        MetaDataContainer
            List containing all meta data assigned to training set for given fold.

        """

        if fold is None or fold == 0:
            fold = 'all_data'

        data = copy.deepcopy(self.crossvalidation_data['train'][fold])

        # Go through items and make sure path are in correct form.
        for item in data:
            if absolute_paths:
                item.filename = self.relative_to_absolute_path(item.filename)

            else:
                item.filename = self.absolute_to_relative_path(item.filename)

        return data

    def test(self, fold=None, absolute_paths=True, **kwargs):
        """List of testing items.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        Returns
        -------
        MetaDataContainer
            List containing all meta data assigned to testing set for given fold.

        """

        if fold is None or fold == 0:
            fold = 'all_data'

        data = copy.deepcopy(self.crossvalidation_data['test'][fold])

        # Go through items and make sure path are in correct form.
        for item in data:
            if absolute_paths:
                item.filename = self.relative_to_absolute_path(item.filename)

            else:
                item.filename = self.absolute_to_relative_path(item.filename)

        return data

    def eval(self, fold=None, absolute_paths=True, **kwargs):
        """List of evaluation items.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        Returns
        -------
        MetaDataContainer
            List containing all meta data assigned to testing set for given fold.

        """

        if fold is None or fold == 0:
            fold = 'all_data'

        data = copy.deepcopy(self.crossvalidation_data['evaluate'][fold])

        # Go through items and make sure path are in correct form.
        for item in data:
            if absolute_paths:
                item.filename = self.relative_to_absolute_path(item.filename)

            else:
                item.filename = self.absolute_to_relative_path(item.filename)

        return data

    def train_files(self, fold=None, absolute_paths=True, **kwargs):
        """List of training files.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        Returns
        -------
        list of str
            List containing all files assigned to training set for given fold.

        """

        return self.train(fold=fold, absolute_paths=absolute_paths).unique_files

    def test_files(self, fold=None, absolute_paths=True, **kwargs):
        """List of testing files.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        Returns
        -------
        list of str
            List containing all files assigned to testing set for given fold.

        """

        return self.test(fold=fold, absolute_paths=absolute_paths).unique_files

    def eval_files(self, fold=None, absolute_paths=True, **kwargs):
        """List of evaluation files.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        Returns
        -------
        list of str
            List containing all files assigned to testing set for given fold.

        """

        return self.eval(fold=fold, absolute_paths=absolute_paths).unique_files

    def validation_split(self,
                         fold=None, split_type='balanced', validation_amount=None,
                         seed=0, verbose=False, scene_label=None, iterations=100,
                         **kwargs):
        """List of validation files. Validation files are always subset of training files.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        split_type : str
            Split type [dataset, random, balanced]
            Default value 'balanced'

        validation_amount : float
            Amount of training files to be assigned for validation
            Default value None

        seed : int
            Randomization seed
            Default value 0

        verbose : bool
            Show information about the validation set.
            Default value False

        scene_label : str
            Scene label of the validation set. If None, all training material used.
            Default value None

        iterations : int
            Randomization iterations done when finding balanced set before selecting best matched set.
            Default value 100

        Returns
        -------
        list of str
            List containing all files assigned to training set for given fold.

        list of str
            List containing all files assigned to validation set for given fold.

        """

        kwargs.update({
            'fold': fold,
            'validation_amount': validation_amount,
            'seed': seed,
            'verbose': verbose,
            'scene_label': scene_label,
            'iterations': iterations,
        })

        if split_type == 'dataset':
            validation_files = self.validation_files_dataset(**kwargs)

        elif split_type == 'random':
            validation_files = self.validation_files_random(**kwargs)

        elif split_type == 'balanced':
            validation_files = self.validation_files_balanced(**kwargs)

        else:
            message = '{name}: Unknown split type [{split_type}]'.format(
                name=self.__class__.__name__,
                split_type=split_type
            )

            self.logger.exception(message)
            raise IOError(message)

        training_files = sorted(list(set(self.train(fold=fold).unique_files) - set(validation_files)))

        return training_files, validation_files

    def validation_files_dataset(self, fold=None, verbose=False, **kwargs):
        """List of validation files delivered by the dataset.
        """

        message = '{name}: Dataset does not have fixed validation sets, use validation set generation to get sets'.format(
            name=self.__class__.__name__,
        )

        self.logger.exception(message)
        raise ValueError(message)

    def validation_files_random(self, fold=None, validation_amount=0.3, seed=0, verbose=False, **kwargs):
        """List of validation files selected randomly from the training material.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        validation_amount : float
            Amount of training material to be assigned for validation.
            Default value 0.3

        seed : int
            Randomization seed
            Default value 0

        verbose : bool
            Show information about the validation set.
            Default value False

        Returns
        -------
        list of str
            List containing all files assigned for validation

        """

        random.seed(seed)

        training_meta = self.train(fold=fold)
        training_files = training_meta.unique_files
        random.shuffle(training_files, random.random)

        validation_split_index = int(numpy.ceil(validation_amount * len(training_files)))
        validation_files = training_files[0:validation_split_index]

        if verbose:
            validation_meta = self.train(fold=fold).filter(file_list=validation_files)

            validation_amounts = numpy.zeros((len(self.scene_labels())+1, 3))
            for scene_id, scene_label in enumerate(self.scene_labels()):
                validation_scene_meta = validation_meta.filter(scene_label=scene_label)
                validation_amounts[scene_id, 0] = len(validation_scene_meta.unique_identifiers)
                validation_amounts[scene_id, 1] = len(validation_scene_meta.unique_files)
                validation_amounts[scene_id, 2] = len(validation_scene_meta.unique_files) / float(len(training_meta.filter(scene_label=scene_label).unique_files)) * 100
            validation_amounts[-1, 0] = numpy.sum(validation_amounts[0:-1, 0])
            validation_amounts[-1, 1] = numpy.sum(validation_amounts[0:-1, 1])
            validation_amounts[-1, 2] = numpy.mean(validation_amounts[0:-1, 2])

            labels = self.scene_labels()
            labels.append('Overall')

            FancyLogger().sub_header('Validation set for fold [{fold}] / random'.format(fold=fold), indent=2)
            FancyLogger().table(
                cell_data=[labels] + validation_amounts.T.tolist(),
                column_headers=['Scene', 'Identifiers', 'Files', 'Amount (%)'],
                column_types=['str20', 'int', 'int', 'float1'],
                column_separators=[0, 1],
                row_separators=[len(labels) - 1],
                indent=2
            )

        return validation_files

    def validation_files_balanced(self, fold=None, validation_amount=0.3, seed=0, verbose=False, **kwargs):
        """List of validation files randomly selecting while maintaining data balance.
        """
        message = '{name}: Balanced validation set generation has not been implemented for dataset class.'.format(
            name=self.__class__.__name__,
        )

        self.logger.exception(message)
        raise ValueError(message)

    def folds(self, mode='folds'):
        """List of fold ids

        Parameters
        ----------
        mode : str {'folds','full'}
            Fold setup type, possible values are 'folds' and 'full'. In 'full' mode fold number is set 0 and
            all data is used for training.
            Default value 'folds'

        Returns
        -------
        list of int
            Fold ids

        """

        if mode == 'folds':
            if self.crossvalidation_folds is not None:
                return list(range(1, self.crossvalidation_folds+1))
            else:
                return ['all_data']

        elif mode == 'full':
            return ['all_data']

    def file_meta(self, filename):
        """Meta data for given file

        Parameters
        ----------
        filename : str
            File name

        Returns
        -------
        list of dict
            List containing all meta data related to given file.

        """

        # Try first with raw filename
        file_meta = self.meta_container.filter(
            filename=filename
        )

        if not file_meta:
            # Try with absolute filename
            file_meta = self.meta_container.filter(
                filename=self.absolute_to_relative_path(filename)
            )

        if not file_meta:
            # Try with relative filename
            file_meta = self.meta_container.filter(
                filename=self.relative_to_absolute_path(filename)
            )

        return file_meta

    def file_error_meta(self, filename):
        """Error meta data for given file

        Parameters
        ----------
        filename : str
            File name

        Returns
        -------
        list of dict
            List containing all error meta data related to given file.

        """
        error_meta = self.error_meta.filter(
            filename=filename
        )

        if not error_meta:
            error_meta = self.error_meta.filter(
                filename=self.absolute_to_relative_path(filename)
            )

        if not error_meta:
            error_meta = self.error_meta.filter(
                filename=self.relative_to_absolute_path(filename)
            )

        return error_meta

    def file_features(self, filename):
        """Pre-calculated acoustic features for given file

        Parameters
        ----------
        filename : str
            File name

        Returns
        -------
        numpy.ndarray
            Matrix containing acoustic features

        """

        pass

    def relative_to_absolute_path(self, path):
        """Converts relative path into absolute path.

        Parameters
        ----------
        path : str
            Relative path

        Returns
        -------
        str
            Absolute path

        """

        return os.path.abspath(os.path.expanduser(os.path.join(self.local_path, path)))

    def absolute_to_relative_path(self, path):
        """Converts absolute path into relative path.

        Parameters
        ----------
        path : str
            Absolute path

        Returns
        -------
        str
            Relative path

        """

        if path.startswith(os.path.abspath(self.local_path)):
            return os.path.relpath(path, self.local_path)

        else:
            return path

    def dataset_bytes(self):
        """Total download size of the dataset in bytes.

        Returns
        -------
        int
            Total bytes

        """

        total_bytes = 0
        for item_id, item in enumerate(self.package_list):
            remote_package = RemoteFile(**item)

            if remote_package.local_exists():
                total_bytes += remote_package.local_bytes

            else:
                total_bytes += remote_package.remote_bytes

        return total_bytes

    def dataset_size_string(self):
        """Total download size of the dataset in a string.

        Returns
        -------
        str
            Total size

        """

        return get_byte_string(self.dataset_bytes(), show_bytes=False)

    def dataset_size_on_disk(self):
        """Total size of the dataset currently stored locally.

        Returns
        -------
        str
            Total size

        """

        total_size = 0
        for dir_path, dir_names, filenames in os.walk(self.local_path):
            for f in filenames:
                filename = os.path.join(dir_path, f)
                total_size += os.path.getsize(filename)

        return get_byte_string(total_size, show_bytes=False)

    def _get_filelist(self, exclude_dirs=None):
        """List of files under local_path

        Parameters
        ----------
        exclude_dirs : list of str
            List of directories to be excluded
            Default value None

        Returns
        -------
        list of str
            File list

        """

        if exclude_dirs is None:
            exclude_dirs = []

        filelist = []
        for path, sub_directory, files in os.walk(self.local_path):
            for name in files:
                if os.path.splitext(name)[1] != os.path.splitext(self.filelisthash_filename)[1] and os.path.split(path)[1] not in exclude_dirs:
                    filelist.append(os.path.join(path, name))

        return sorted(filelist)

    def _save_filelist_hash(self):
        """Generates file list hash, and saves it as filelist.hash under local_path.

        Parameters
        ----------

        Returns
        -------
        None

        """

        filelist = self._get_filelist()

        hash_value = get_parameter_hash(filelist)
        if self.included_content_types:
            hash_value += ';'+','.join(self.included_content_types)

        TextContainer([hash_value]).save(
            filename=os.path.join(self.local_path, self.filelisthash_filename)
        )


class AcousticSceneDataset(Dataset):
    """Acoustic scene dataset baseclass """
    def __init__(self, *args, **kwargs):
        super(AcousticSceneDataset, self).__init__(*args, **kwargs)

    def validation_files_balanced(self,
                                  fold=None, validation_amount=0.3, seed=0, verbose=False, iterations=100,
                                  **kwargs):
        """List of validation files randomly selecting while maintaining data balance.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        validation_amount : float
            Amount of training material to be assigned for validation.
            Default value 0.3

        seed : int
            Randomization seed
            Default value 0

        verbose : bool
            Show information about the validation set.
            Default value False

        iterations : int
            How many randomization iterations will be done before selecting best matched.
            Default value 100

        Returns
        -------
        list of str
            List containing all files assigned for validation

        """

        random.seed(seed)
        training_meta = self.train(fold=fold)

        # Check do we have location/source identifier present
        identifier_present = False
        for item in training_meta:
            if item.identifier:
                identifier_present = True
                break

        training_files = []
        validation_files = []
        validation_amounts = numpy.zeros((len(self.scene_labels())+1, 4))

        if identifier_present:
            # Do the balance based on scene class and identifier
            for scene_id, scene_label in enumerate(self.scene_labels()):
                scene_meta = training_meta.filter(scene_label=scene_label)

                data = {}
                for identifier in scene_meta.unique_identifiers:
                    data[identifier] = scene_meta.filter(identifier=identifier).unique_files

                current_scene_validation_amount = []
                sets_candidates = []

                iteration_progress = tqdm(
                    range(0, iterations),
                    desc="{0: <25s}".format('Generate validation split candidates'),
                    file=sys.stdout,
                    leave=False,
                    disable=self.disable_progress_bar,
                    ascii=self.use_ascii_progress_bar
                )

                for i in iteration_progress:
                    current_locations = list(data.keys())
                    random.shuffle(current_locations, random.random)
                    validation_split_index = int(numpy.ceil(validation_amount * len(data)))
                    current_validation_identifiers = current_locations[0:validation_split_index]
                    current_training_identifiers = current_locations[validation_split_index:]

                    # Collect validation files
                    current_validation_files = []
                    for identifier in current_validation_identifiers:
                        current_validation_files += data[identifier]

                    # Collect training files
                    current_training_files = []
                    for identifier in current_training_identifiers:
                        current_training_files += data[identifier]

                    current_scene_validation_amount.append(
                        len(current_validation_files) / float(len(current_validation_files) + len(current_training_files))
                    )

                    sets_candidates.append({
                        'validation': current_validation_files,
                        'training': current_training_files,
                    })

                best_set_id = numpy.argmin(numpy.abs(numpy.array(current_scene_validation_amount) - validation_amount))
                validation_files += sets_candidates[best_set_id]['validation']
                training_files += sets_candidates[best_set_id]['training']
                validation_amounts[scene_id, 0] = len(scene_meta.unique_identifiers)
                validation_amounts[scene_id, 1] = len(scene_meta.unique_files)
                validation_amounts[scene_id, 2] = len(sets_candidates[best_set_id]['validation'])
                validation_amounts[scene_id, 3] = current_scene_validation_amount[best_set_id] * 100

        else:
            # Do the balance based on scene class only
            for scene_id, scene_label in enumerate(self.scene_labels()):
                scene_files = training_meta.filter(scene_label=scene_label).unique_files

                random.shuffle(scene_files, random.random)
                validation_split_index = int(numpy.ceil(validation_amount * len(scene_files)))
                current_validation_files = scene_files[0:validation_split_index]
                current_training_files = scene_files[validation_split_index:]

                validation_files += current_validation_files
                training_files += current_training_files
                validation_amounts[scene_id, 1] = len(scene_files)
                validation_amounts[scene_id, 2] = len(current_validation_files)
                validation_amounts[scene_id, 3] = len(current_validation_files) / float(len(current_validation_files) + len(current_training_files)) * 100

        if verbose:
            validation_amounts[-1, 0] = numpy.sum(validation_amounts[0:-1, 0])
            validation_amounts[-1, 1] = numpy.sum(validation_amounts[0:-1, 1])
            validation_amounts[-1, 2] = numpy.sum(validation_amounts[0:-1, 2])
            validation_amounts[-1, 3] = numpy.mean(validation_amounts[0:-1, 3])

            logger = FancyLogger()
            logger.sub_header('Validation set for fold [{fold}] / balanced'.format(fold=fold), indent=2)
            logger.row(
                '', 'Full training set', 'Selected validation subset',
                widths=[20, 25, 35],
                types=['str', 'str', 'str'],
                separators=[True, True],
                indent=2
            )
            logger.row(
                'Scene label', 'Identifiers', 'Files', 'Files', 'Amount (%)',
                widths=[20, 15, 10, 10, 15],
                types=['str20', 'int', 'int', 'int', 'float1_percentage'],
                separators=[True, False, True, False]
            )
            logger.row('-', '-', '-', '-', '-')
            for scene_id, scene_label in enumerate(self.scene_labels()):
                logger.row(
                    scene_label,
                    validation_amounts[scene_id, 0],
                    validation_amounts[scene_id, 1],
                    validation_amounts[scene_id, 2],
                    validation_amounts[scene_id, 3],
                )
            logger.row('-', '-', '-', '-', '-')
            logger.row(
                'Overall',
                validation_amounts[-1, 0],
                validation_amounts[-1, 1],
                validation_amounts[-1, 2],
                validation_amounts[-1, 3]
            )
        logger.line()

        return validation_files


class SoundEventDataset(Dataset):
    """Sound event dataset baseclass"""
    def __init__(self, *args, **kwargs):
        super(SoundEventDataset, self).__init__(*args, **kwargs)

    def load_crossvalidation_data(self):
        """Load cross-validation into the container.

        Returns
        -------
        self

        """

        # Reset cross validation data and insert 'all_data'
        self.crossvalidation_data = DictContainer({
            'train': {
                'all_data': self.meta_container
            },
            'test': {
                'all_data': self.meta_container
            },
            'evaluate': {
                'all_data': self.meta_container
            },
        })

        for crossvalidation_set in list(self.crossvalidation_data.keys()):
            for item in self.crossvalidation_data[crossvalidation_set]['all_data']:
                self.process_meta_item(item=item)

        # Load cross validation folds
        for fold in self.folds():
            # Initialize data
            self.crossvalidation_data['train'][fold] = MetaDataContainer()
            self.crossvalidation_data['test'][fold] = MetaDataContainer()
            self.crossvalidation_data['evaluate'][fold] = MetaDataContainer()

            for scene_label in self.scene_labels():
                # Get filenames
                train_filename = self.evaluation_setup_filename(
                    setup_part='train',
                    fold=fold,
                    scene_label=scene_label
                )

                test_filename = self.evaluation_setup_filename(
                    setup_part='test',
                    fold=fold,
                    scene_label=scene_label
                )

                evaluate_filename = self.evaluation_setup_filename(
                    setup_part='evaluate',
                    fold=fold,
                    scene_label=scene_label
                )

                if os.path.isfile(train_filename):
                    # Training data for fold exists, load and process it
                    self.crossvalidation_data['train'][fold] += MetaDataContainer(filename=train_filename).load()

                if os.path.isfile(test_filename):
                    # Testing data for fold exists, load and process it
                    self.crossvalidation_data['test'][fold] += MetaDataContainer(filename=test_filename).load()

                if os.path.isfile(evaluate_filename):
                    # Evaluation data for fold exists, load and process it
                    self.crossvalidation_data['evaluate'][fold] += MetaDataContainer(filename=evaluate_filename).load()

            # Process items
            for item in self.crossvalidation_data['train'][fold]:
                self.process_meta_item(item=item)

            for item in self.crossvalidation_data['test'][fold]:
                self.process_meta_item(item=item)

            for item in self.crossvalidation_data['evaluate'][fold]:
                self.process_meta_item(item=item)

        return self

    def event_label_count(self, scene_label=None):
        """Number of unique scene labels in the meta data.

        Parameters
        ----------
        scene_label : str
            Scene label
            Default value None

        Returns
        -------
        int
            Number of unique scene labels.

        """

        return len(self.event_labels(scene_label=scene_label))

    def event_labels(self, scene_label=None):
        """List of unique event labels in the meta data.

        Parameters
        ----------
        scene_label : str
            Scene label
            Default value None

        Returns
        -------
        list of str
            List of event labels in alphabetical order.

        """

        if scene_label is not None:
            labels = self.meta_container.filter(scene_label=scene_label).unique_event_labels

        else:
            labels = self.meta_container.unique_event_labels

        labels.sort()
        return labels

    def train(self, fold=None, absolute_paths=True, scene_label=None, event_label=None, **kwargs):
        """List of training items.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        scene_label : str
            Scene label
            Default value None

        event_label : str
            Event label
            Default value None

        Returns
        -------
        list of dict
            List containing all meta data assigned to training set for given fold.

        """

        if fold is None or fold == 0:
            fold = 'all_data'

        data = copy.deepcopy(self.crossvalidation_data['train'][fold])

        # Go through items and make sure path are in correct form.
        for item in data:
            if absolute_paths:
                item.filename = self.relative_to_absolute_path(item.filename)
            else:
                item.filename = self.absolute_to_relative_path(item.filename)

        if scene_label:
            data = data.filter(
                scene_label=scene_label
            )

        if event_label:
            data = data.filter(
                event_label=event_label
            )

        return data

    def test(self, fold=None, absolute_paths=True, scene_label=None, event_label=None, **kwargs):
        """List of testing items.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        scene_label : str
            Scene label
            Default value None

        event_label : str
            Event label
            Default value None

        Returns
        -------
        list of dict
            List containing all meta data assigned to testing set for given fold.

        """

        if fold is None or fold == 0:
            fold = 'all_data'

        data = copy.deepcopy(self.crossvalidation_data['test'][fold])

        # Go through items and make sure path are in correct form.
        for item in data:
            if absolute_paths:
                item.filename = self.relative_to_absolute_path(item.filename)
            else:
                item.filename = self.absolute_to_relative_path(item.filename)

        if scene_label:
            data = data.filter(
                scene_label=scene_label
            )

        if event_label:
            data = data.filter(
                event_label=event_label
            )

        return data

    def eval(self, fold=None, absolute_paths=True, scene_label=None, event_label=None, **kwargs):
        """List of evaluation items.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        scene_label : str
            Scene label
            Default value None

        event_label : str
            Event label
            Default value None

        Returns
        -------
        list of dict
            List containing all meta data assigned to testing set for given fold.

        """

        if fold is None or fold == 0:
            fold = 'all_data'

        data = copy.deepcopy(self.crossvalidation_data['evaluate'][fold])

        # Go through items and make sure path are in correct form.
        for item in data:
            if absolute_paths:
                item.filename = self.relative_to_absolute_path(item.filename)
            else:
                item.filename = self.absolute_to_relative_path(item.filename)

        if scene_label:
            data = data.filter(
                scene_label=scene_label
            )

        if event_label:
            data = data.filter(
                event_label=event_label
            )

        return data

    def train_files(self, fold=None, absolute_paths=True, scene_label=None, event_label=None, **kwargs):
        """List of training files.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        scene_label : str
            Scene label
            Default value None

        event_label : str
            Event label
            Default value None

        Returns
        -------
        list of str
            List containing all files assigned to training set for given fold.

        """

        return self.train(
            fold=fold,
            absolute_paths=absolute_paths,
            scene_label=scene_label,
            event_label=event_label
        ).unique_files

    def test_files(self, fold=None, absolute_paths=True, scene_label=None, event_label=None, **kwargs):
        """List of testing files.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        scene_label : str
            Scene label
            Default value None

        event_label : str
            Event label
            Default value None

        Returns
        -------
        list of str
            List containing all files assigned to testing set for given fold.

        """

        return self.test(
            fold=fold,
            absolute_paths=absolute_paths,
            scene_label=scene_label,
            event_label=event_label
        ).unique_files

    def eval_files(self, fold=None, absolute_paths=True, scene_label=None, event_label=None, **kwargs):
        """List of evaluation files.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        absolute_paths : bool
            Path format for the returned meta items, if True paths are absolute, False paths are relative to
            the dataset root.
            Default value True

        scene_label : str
            Scene label
            Default value None

        event_label : str
            Event label
            Default value None

        Returns
        -------
        list of str
            List containing all files assigned to testing set for given fold.

        """

        return self.eval(
            fold=fold,
            absolute_paths=absolute_paths,
            scene_label=scene_label,
            event_label=event_label
        ).unique_files

    def validation_files_random(self,
                                fold=None, validation_amount=0.3, seed=0, verbose=False, scene_label=None,
                                **kwargs):
        """List of validation files selected randomly from the training material.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        validation_amount : float
            Amount of training material to be assigned for validation.
            Default value 0.3

        seed : int
            Randomization seed
            Default value 0

        verbose : bool
            Show information about the validation set.
            Default value False

        scene_label : str
            Scene label of the validation set. If None, all training material used.
            Default value None

        Returns
        -------
        list of str
            List containing all files assigned for validation

        """

        random.seed(seed)
        training_meta = self.train(
            fold=fold,
            scene_label=scene_label
        )

        if scene_label:
            scene_labels = [scene_label]
        else:
            scene_labels = self.scene_labels()

        training_files = training_meta.unique_files
        random.shuffle(training_files, random.random)

        validation_split_index = int(numpy.ceil(validation_amount * len(training_files)))
        validation_files = training_files[0:validation_split_index]

        if verbose:
            logger = FancyLogger()
            logger.sub_header('Validation set for fold [{fold}] / random'.format(fold=fold), indent=2)
            for scene_id, scene_label in enumerate(scene_labels):
                all_stats = training_meta.filter(scene_label=scene_label).event_stat_counts()
                validation_stats = training_meta.filter(
                    scene_label=scene_label,
                    file_list=validation_files
                ).event_stat_counts()

                training_files = sorted(list(set(self.train(fold=fold).unique_files) - set(validation_files)))
                training_stats = training_meta.filter(
                    scene_label=scene_label,
                    file_list=training_files
                ).event_stat_counts()

                cell_data = numpy.zeros((len(list(all_stats.keys())) + 1, 4))
                for event_id, event_label in enumerate(list(all_stats.keys())):
                    cell_data[event_id, 0] = all_stats[event_label]
                    if event_label in training_stats:
                        cell_data[event_id, 1] = training_stats[event_label]

                    if event_label in validation_stats:
                        cell_data[event_id, 2] = validation_stats[event_label]

                    cell_data[event_id, 3] = cell_data[event_id, 2] / float(cell_data[event_id, 0]) * 100

                cell_data[-1, 0] = numpy.sum(list(all_stats.values()))
                cell_data[-1, 1] = numpy.sum(list(training_stats.values()))
                cell_data[-1, 2] = numpy.sum(list(validation_stats.values()))
                cell_data[-1, 3] = cell_data[-1, 2] / float(cell_data[-1, 0]) * 100

                labels = list(all_stats.keys())
                labels.append('Overall')
                logger.line(scene_label, indent=4)
                logger.table(
                    cell_data=[labels] + cell_data.T.tolist(),
                    column_headers=['Event', 'All', 'Training', 'Validation', 'Val amount (%)'],
                    column_types=['str20', 'int', 'int', 'int', 'float1'],
                    column_separators=[0, 1, 3],
                    row_separators=[len(labels)-1],
                    indent=6
                )

        return validation_files

    def validation_files_balanced(self,
                                  fold=None, validation_amount=0.3, seed=0,
                                  verbose=False, scene_label=None, iterations=100,
                                  **kwargs):
        """List of validation files randomly selecting while maintaining data balance.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        validation_amount : float
            Amount of training material to be assigned for validation.
            Default value 0.3

        seed : int
            Randomization seed
            Default value 0

        verbose : bool
            Show information about the validation set.
            Default value False

        scene_label : str
            Scene label of the validation set. If None, all training material used.
            Default value None

        iterations : int
            How many randomization iterations will be done before selecting best matched.
            Default value 100

        Returns
        -------
        list of str
            List containing all files assigned for validation

        """

        from sklearn.metrics import mean_absolute_error

        random.seed(seed)
        training_meta = self.train(
            fold=fold,
            scene_label=scene_label
        )

        if scene_label:
            scene_labels = [scene_label]
        else:
            scene_labels = self.scene_labels()

        # Check do we have location/source identifier present
        identifier_present = False
        for item in training_meta:
            if item.identifier:
                identifier_present = True
                break

        if identifier_present:
            # Do the balance based on scene class, identifier and event class
            data = {}
            for scene_id, scene_label in enumerate(scene_labels):
                scene_meta = training_meta.filter(scene_label=scene_label)
                data[scene_label] = {}
                for identifier in scene_meta.unique_identifiers:
                    data[scene_label][identifier] = scene_meta.filter(identifier=identifier).unique_files

            # Get event amounts per class
            event_amounts = {}
            for scene_id, scene_label in enumerate(scene_labels):
                event_amounts[scene_label] = {}
                for identifier in list(data[scene_label].keys()):
                    for audio_filename in data[scene_label][identifier]:
                        current_event_amounts = training_meta.filter(filename=audio_filename).event_stat_counts()

                        for event_label, count in iteritems(current_event_amounts):
                            if event_label not in event_amounts[scene_label]:
                                event_amounts[scene_label][event_label] = 0

                            event_amounts[scene_label][event_label] += count

            validation_files = []
            for scene_id, scene_label in enumerate(scene_labels):
                # Optimize scene sets separately
                validation_set_candidates = []
                validation_set_mae = []
                validation_set_event_amounts = []
                training_set_event_amounts = []

                iteration_progress = tqdm(
                    range(0, iterations),
                    desc="{0: <25s}".format('Generate validation split candidates'),
                    file=sys.stdout,
                    leave=False,
                    disable=self.disable_progress_bar,
                    ascii=self.use_ascii_progress_bar
                )

                for i in iteration_progress:
                    identifiers = list(data[scene_label].keys())
                    random.shuffle(identifiers, random.random)

                    valid_percentage_index = int(numpy.ceil(validation_amount * len(identifiers)))

                    current_validation_files = []
                    for loc_id in identifiers[0:valid_percentage_index]:
                        current_validation_files += data[scene_label][loc_id]

                    current_training_files = []
                    for loc_id in identifiers[valid_percentage_index:]:
                        current_training_files += data[scene_label][loc_id]

                    # Event count in training set candidate
                    training_set_event_counts = numpy.zeros(len(event_amounts[scene_label]))
                    for audio_filename in current_training_files:
                        current_event_amounts = training_meta.filter(filename=audio_filename).event_stat_counts()
                        for event_label_id, event_label in enumerate(event_amounts[scene_label]):
                            if event_label in current_event_amounts:
                                training_set_event_counts[event_label_id] += current_event_amounts[event_label]

                    # Accept only sets which leave at least one example for training
                    if numpy.all(training_set_event_counts > 0):
                        # Event counts in validation set candidate
                        validation_set_event_counts = numpy.zeros(len(event_amounts[scene_label]))
                        for audio_filename in current_validation_files:
                            current_event_amounts = training_meta.filter(filename=audio_filename).event_stat_counts()

                            for event_label_id, event_label in enumerate(event_amounts[scene_label]):
                                if event_label in current_event_amounts:
                                    validation_set_event_counts[event_label_id] += current_event_amounts[event_label]

                        # Accept only sets which have examples from each event class
                        if numpy.all(validation_set_event_counts > 0):
                            current_validation_amount = validation_set_event_counts / (validation_set_event_counts + training_set_event_counts)
                            validation_set_candidates.append(current_validation_files)
                            validation_set_mae.append(
                                mean_absolute_error(
                                    numpy.ones(len(current_validation_amount)) * validation_amount,
                                    current_validation_amount)
                            )
                            validation_set_event_amounts.append(validation_set_event_counts)
                            training_set_event_amounts.append(training_set_event_counts)

                # Generate balance validation set
                # Selection done based on event counts (per scene class)
                # Target count specified percentage of training event count
                if validation_set_mae:
                    best_set_id = numpy.argmin(validation_set_mae)
                    validation_files += validation_set_candidates[best_set_id]

                else:
                    message = '{name}: Validation setup creation was not successful! Could not find a set with ' \
                              'examples for each event class in both training and validation.'.format(
                                name=self.__class__.__name__
                              )

                    self.logger.exception(message)
                    raise AssertionError(message)

        else:
            # Do the balance based on scene class, identifier and event class
            data = {}
            for scene_id, scene_label in enumerate(scene_labels):
                data[scene_label] = training_meta.filter(scene_label=scene_label).unique_files

            # Get event amounts per class
            event_amounts = {}
            for scene_id, scene_label in enumerate(scene_labels):
                event_amounts[scene_label] = {}
                for audio_filename in data[scene_label]:
                    current_event_amounts = training_meta.filter(filename=audio_filename).event_stat_counts()

                    for event_label, count in iteritems(current_event_amounts):
                        if event_label not in event_amounts[scene_label]:
                            event_amounts[scene_label][event_label] = 0

                        event_amounts[scene_label][event_label] += count

            validation_files = []
            for scene_id, scene_label in enumerate(scene_labels):
                # Optimize scene sets separately
                validation_set_candidates = []
                validation_set_mae = []
                validation_set_event_amounts = []
                training_set_event_amounts = []

                iteration_progress = tqdm(
                    range(0, iterations),
                    desc="{0: <25s}".format('Generate validation split candidates'),
                    file=sys.stdout,
                    leave=False,
                    disable=self.disable_progress_bar,
                    ascii=self.use_ascii_progress_bar
                )

                for i in iteration_progress:
                    item_ids = list(range(0, len(data[scene_label])))
                    random.shuffle(item_ids, random.random)

                    valid_percentage_index = int(numpy.ceil(validation_amount * len(item_ids)))

                    current_validation_files = []
                    for loc_id in item_ids[0:valid_percentage_index]:
                        current_validation_files.append(data[scene_label][loc_id])

                    current_training_files = []
                    for loc_id in item_ids[valid_percentage_index:]:
                        current_training_files.append(data[scene_label][loc_id])

                    # Event count in training set candidate
                    training_set_event_counts = numpy.zeros(len(event_amounts[scene_label]))
                    for audio_filename in current_training_files:
                        current_event_amounts = training_meta.filter(filename=audio_filename).event_stat_counts()
                        for event_label_id, event_label in enumerate(event_amounts[scene_label]):
                            if event_label in current_event_amounts:
                                training_set_event_counts[event_label_id] += current_event_amounts[event_label]

                    # Accept only sets which leave at least one example for training
                    if numpy.all(training_set_event_counts > 0):
                        # Event counts in validation set candidate
                        validation_set_event_counts = numpy.zeros(len(event_amounts[scene_label]))
                        for audio_filename in current_validation_files:
                            current_event_amounts = training_meta.filter(filename=audio_filename).event_stat_counts()

                            for event_label_id, event_label in enumerate(event_amounts[scene_label]):
                                if event_label in current_event_amounts:
                                    validation_set_event_counts[event_label_id] += current_event_amounts[event_label]

                        # Accept only sets which have examples from each event class
                        if numpy.all(validation_set_event_counts > 0):
                            current_validation_amount = validation_set_event_counts / (validation_set_event_counts + training_set_event_counts)
                            validation_set_candidates.append(current_validation_files)
                            validation_set_mae.append(
                                mean_absolute_error(
                                    numpy.ones(len(current_validation_amount)) * validation_amount,
                                    current_validation_amount)
                            )
                            validation_set_event_amounts.append(validation_set_event_counts)
                            training_set_event_amounts.append(training_set_event_counts)

                # Generate balance validation set
                # Selection done based on event counts (per scene class)
                # Target count specified percentage of training event count
                if validation_set_mae:
                    best_set_id = numpy.argmin(validation_set_mae)
                    validation_files += validation_set_candidates[best_set_id]

                else:
                    message = '{name}: Validation setup creation was not successful! Could not find a set with ' \
                              'examples for each event class in both training and validation.'.format(
                        name=self.__class__.__name__
                    )

                    self.logger.exception(message)
                    raise AssertionError(message)

        if verbose:
            logger = FancyLogger()
            logger.sub_header('Validation set for fold [{fold}] / balanced'.format(fold=fold), indent=2)
            for scene_id, scene_label in enumerate(scene_labels):
                all_stats = training_meta.filter(scene_label=scene_label).event_stat_counts()
                validation_stats = training_meta.filter(
                    scene_label=scene_label,
                    file_list=validation_files
                ).event_stat_counts()

                training_files = sorted(list(set(self.train(fold=fold).unique_files) - set(validation_files)))
                training_stats = training_meta.filter(
                    scene_label=scene_label,
                    file_list=training_files
                ).event_stat_counts()

                cell_data = numpy.zeros((len(list(all_stats.keys())) + 1, 4))
                for event_id, event_label in enumerate(list(all_stats.keys())):
                    cell_data[event_id, 0] = all_stats[event_label]
                    if event_label in training_stats:
                        cell_data[event_id, 1] = training_stats[event_label]

                    if event_label in validation_stats:
                        cell_data[event_id, 2] = validation_stats[event_label]

                    cell_data[event_id, 3] = cell_data[event_id, 2] / float(cell_data[event_id, 0]) * 100

                cell_data[-1, 0] = numpy.sum(list(all_stats.values()))
                cell_data[-1, 1] = numpy.sum(list(training_stats.values()))
                cell_data[-1, 2] = numpy.sum(list(validation_stats.values()))
                cell_data[-1, 3] = cell_data[-1, 2] / float(cell_data[-1, 0]) * 100

                labels = list(all_stats.keys())
                labels.append('Overall')
                logger.line(scene_label, indent=4)
                logger.table(
                    cell_data=[labels] + cell_data.T.tolist(),
                    column_headers=['Event', 'All', 'Training', 'Validation', 'Val amount (%)'],
                    column_types=['str20', 'int', 'int', 'int', 'float1'],
                    column_separators=[0, 1, 3],
                    row_separators=[len(labels)-1],
                    indent=6
                )

        return validation_files


class SyntheticSoundEventDataset(SoundEventDataset):
    """Synthetic sound event dataset baseclass"""
    def __init__(self, *args, **kwargs):
        super(SyntheticSoundEventDataset, self).__init__(*args, **kwargs)

    def initialize(self):
        """Initialize the dataset, download, extract files and prepare the dataset for the usage.

        Returns
        -------
        self

        """

        # Create the dataset path if does not exist
        Path().makedirs(path=self.local_path)

        if not self.check_filelist():
            self.download_packages()
            self.extract_packages()
            self.load()
            self.prepare()
            self._save_filelist_hash()

        self.synthesize()

        return self

    def synthesize(self):
        pass


class AudioTaggingDataset(Dataset):
    """Audio tag dataset baseclass"""
    def __init__(self, *args, **kwargs):
        super(AudioTaggingDataset, self).__init__(*args, **kwargs)

    def validation_files_random(self,
                                fold=None, validation_amount=0.3, seed=0, verbose=False,
                                **kwargs):
        """List of validation files selected randomly from the training material.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        validation_amount : float
            Amount of training material to be assigned for validation.
            Default value 0.3

        seed : int
            Randomization seed
            Default value 0

        verbose : bool
            Show information about the validation set.
            Default value False

        Returns
        -------
        list of str
            List containing all files assigned for validation

        """

        random.seed(seed)
        training_meta = self.train(fold=fold)

        scene_labels = self.scene_labels()

        training_files = training_meta.unique_files
        random.shuffle(training_files, random.random)

        validation_split_index = int(numpy.ceil(validation_amount * len(training_files)))
        validation_files = training_files[0:validation_split_index]

        if verbose:
            logger = FancyLogger()
            logger.sub_header('Validation set for fold [{fold}] / random'.format(fold=fold), indent=2)
            for scene_id, scene_label in enumerate(scene_labels):
                all_stats = training_meta.filter(scene_label=scene_label).tag_stat_counts()
                validation_stats = training_meta.filter(
                    scene_label=scene_label,
                    file_list=validation_files
                ).tag_stat_counts()

                training_files = sorted(list(set(self.train(fold=fold).unique_files) - set(validation_files)))
                training_stats = training_meta.filter(
                    scene_label=scene_label,
                    file_list=training_files
                ).tag_stat_counts()

                cell_data = numpy.zeros((len(list(all_stats.keys())) + 1, 4))
                for event_id, event_label in enumerate(list(all_stats.keys())):
                    cell_data[event_id, 0] = all_stats[event_label]
                    if event_label in training_stats:
                        cell_data[event_id, 1] = training_stats[event_label]

                    if event_label in validation_stats:
                        cell_data[event_id, 2] = validation_stats[event_label]

                    cell_data[event_id, 3] = cell_data[event_id, 2] / float(cell_data[event_id, 0]) * 100

                cell_data[-1, 0] = numpy.sum(list(all_stats.values()))
                cell_data[-1, 1] = numpy.sum(list(training_stats.values()))
                cell_data[-1, 2] = numpy.sum(list(validation_stats.values()))
                cell_data[-1, 3] = cell_data[-1, 2] / float(cell_data[-1, 0]) * 100

                labels = list(all_stats.keys())
                labels.append('Overall')
                logger.line(scene_label, indent=4)
                logger.table(
                    cell_data=[labels] + cell_data.T.tolist(),
                    column_headers=['Tag', 'All', 'Training', 'Validation', 'Val amount (%)'],
                    column_types=['str20', 'int', 'int', 'int', 'float1'],
                    column_separators=[0, 1, 3],
                    row_separators=[len(labels)-1],
                    indent=6
                )

        return validation_files

    def validation_files_balanced(self,
                                  fold=None, validation_amount=0.3, seed=0, verbose=False, iterations=100,
                                  **kwargs):
        """List of validation files randomly selecting while maintaining data balance.

        Parameters
        ----------
        fold : int
            Fold id, if None all meta data is returned.
            Default value None

        validation_amount : float
            Amount of training material to be assigned for validation.
            Default value 0.3

        seed : int
            Randomization seed
            Default value 0

        verbose : bool
            Show information about the validation set.
            Default value False

        iterations : int
            How many randomization iterations will be done before selecting best matched.
            Default value 100

        Returns
        -------
        list of str
            List containing all files assigned for validation

        """

        from sklearn.metrics import mean_absolute_error

        random.seed(seed)
        training_meta = self.train(fold=fold)

        # Check do we have location/source identifier present
        identifier_present = False
        for item in training_meta:
            if item.identifier:
                identifier_present = True
                break

        scene_labels = self.scene_labels()

        if identifier_present:
            # Do the balance based on scene class, identifier and event class
            data = {}
            for scene_id, scene_label in enumerate(scene_labels):
                scene_meta = training_meta.filter(scene_label=scene_label)
                data[scene_label] = {}
                for identifier in scene_meta.unique_identifiers:
                    data[scene_label][identifier] = scene_meta.filter(identifier=identifier).unique_files

            # Get tag amounts per class
            tag_amounts = {}
            for scene_id, scene_label in enumerate(scene_labels):
                tag_amounts[scene_label] = {}
                for identifier in list(data[scene_label].keys()):
                    for audio_filename in data[scene_label][identifier]:
                        current_tag_amounts = training_meta.filter(filename=audio_filename).tag_stat_counts()

                        for tag_label, count in iteritems(current_tag_amounts):
                            if tag_label not in tag_amounts[scene_label]:
                                tag_amounts[scene_label][tag_label] = 0

                            tag_amounts[scene_label][tag_label] += count

            validation_files = []
            for scene_id, scene_label in enumerate(scene_labels):
                # Optimize scene sets separately
                validation_set_candidates = []
                validation_set_mae = []
                validation_set_tag_amounts = []
                training_set_tag_amounts = []

                iteration_progress = tqdm(
                    range(0, iterations),
                    desc="{0: <25s}".format('Generate validation split candidates'),
                    file=sys.stdout,
                    leave=False,
                    disable=self.disable_progress_bar,
                    ascii=self.use_ascii_progress_bar
                )

                for i in iteration_progress:
                    identifiers = list(data[scene_label].keys())
                    random.shuffle(identifiers, random.random)

                    valid_percentage_index = int(numpy.ceil(validation_amount * len(identifiers)))

                    current_validation_files = []
                    for loc_id in identifiers[0:valid_percentage_index]:
                        current_validation_files += data[scene_label][loc_id]

                    current_training_files = []
                    for loc_id in identifiers[valid_percentage_index:]:
                        current_training_files += data[scene_label][loc_id]

                    # Tag count in training set candidate
                    training_set_tag_counts = numpy.zeros(len(tag_amounts[scene_label]))
                    for audio_filename in current_training_files:
                        current_tag_amounts = training_meta.filter(filename=audio_filename).tag_stat_counts()
                        for tag_label_id, tag_label in enumerate(tag_amounts[scene_label]):
                            if tag_label in current_tag_amounts:
                                training_set_tag_counts[tag_label_id] += current_tag_amounts[tag_label]

                    # Accept only sets which leave at least one example for training
                    if numpy.all(training_set_tag_counts > 0):
                        # Tag counts in validation set candidate
                        validation_set_tag_counts = numpy.zeros(len(tag_amounts[scene_label]))
                        for audio_filename in current_validation_files:
                            current_tag_amounts = training_meta.filter(filename=audio_filename).tag_stat_counts()

                            for tag_label_id, tag_label in enumerate(tag_amounts[scene_label]):
                                if tag_label in current_tag_amounts:
                                    validation_set_tag_counts[tag_label_id] += current_tag_amounts[tag_label]

                        # Accept only sets which have examples from each tag class
                        if numpy.all(validation_set_tag_counts > 0):
                            current_validation_amount = validation_set_tag_counts / (validation_set_tag_counts + training_set_tag_counts)
                            validation_set_candidates.append(current_validation_files)
                            validation_set_mae.append(
                                mean_absolute_error(
                                    numpy.ones(len(current_validation_amount)) * validation_amount,
                                    current_validation_amount)
                            )
                            validation_set_tag_amounts.append(validation_set_tag_counts)
                            training_set_tag_amounts.append(training_set_tag_counts)

                # Generate balance validation set
                # Selection done based on event counts (per scene class)
                # Target count specified percentage of training event count
                if validation_set_mae:
                    best_set_id = numpy.argmin(validation_set_mae)
                    validation_files += validation_set_candidates[best_set_id]

                else:
                    message = '{name}: Validation setup creation was not successful! Could not find a set with ' \
                              'examples for each tag class in both training and validation.'.format(
                        name=self.__class__.__name__
                    )

                    self.logger.exception(message)
                    raise AssertionError(message)
        else:
            # Do the balance based on scene class, identifier and event class
            data = {}
            for scene_id, scene_label in enumerate(scene_labels):
                data[scene_label] = training_meta.filter(scene_label=scene_label).unique_files

            # Get tag amounts per class
            tag_amounts = {}
            for scene_id, scene_label in enumerate(scene_labels):
                tag_amounts[scene_label] = {}
                for audio_filename in data[scene_label]:
                    current_tag_amounts = training_meta.filter(filename=audio_filename).tag_stat_counts()

                    for tag_label, count in iteritems(current_tag_amounts):
                        if tag_label not in tag_amounts[scene_label]:
                            tag_amounts[scene_label][tag_label] = 0

                        tag_amounts[scene_label][tag_label] += count

            validation_files = []
            for scene_id, scene_label in enumerate(scene_labels):
                # Optimize scene sets separately
                validation_set_candidates = []
                validation_set_mae = []
                validation_set_tag_amounts = []
                training_set_tag_amounts = []

                iteration_progress = tqdm(
                    range(0, iterations),
                    desc="{0: <25s}".format('Generate validation split candidates'),
                    file=sys.stdout,
                    leave=False,
                    disable=self.disable_progress_bar,
                    ascii=self.use_ascii_progress_bar
                )

                for i in iteration_progress:
                    items_id = list(range(0, len(data[scene_label])))
                    random.shuffle(items_id, random.random)

                    valid_percentage_index = int(numpy.ceil(validation_amount * len(items_id)))

                    current_validation_files = []
                    for loc_id in items_id[0:valid_percentage_index]:
                        current_validation_files.append(data[scene_label][loc_id])

                    current_training_files = []
                    for loc_id in items_id[valid_percentage_index:]:
                        current_training_files.append(data[scene_label][loc_id])

                    # Tag count in training set candidate
                    training_set_tag_counts = numpy.zeros(len(tag_amounts[scene_label]))
                    for audio_filename in current_training_files:
                        current_tag_amounts = training_meta.filter(filename=audio_filename).tag_stat_counts()
                        for tag_label_id, tag_label in enumerate(tag_amounts[scene_label]):
                            if tag_label in current_tag_amounts:
                                training_set_tag_counts[tag_label_id] += current_tag_amounts[tag_label]

                    # Accept only sets which leave at least one example for training
                    if numpy.all(training_set_tag_counts > 0):
                        # Tag counts in validation set candidate
                        validation_set_tag_counts = numpy.zeros(len(tag_amounts[scene_label]))
                        for audio_filename in current_validation_files:
                            current_tag_amounts = training_meta.filter(filename=audio_filename).tag_stat_counts()

                            for tag_label_id, tag_label in enumerate(tag_amounts[scene_label]):
                                if tag_label in current_tag_amounts:
                                    validation_set_tag_counts[tag_label_id] += current_tag_amounts[tag_label]

                        # Accept only sets which have examples from each tag class
                        if numpy.all(validation_set_tag_counts > 0):
                            current_validation_amount = validation_set_tag_counts / (
                                validation_set_tag_counts + training_set_tag_counts)
                            validation_set_candidates.append(current_validation_files)
                            validation_set_mae.append(
                                mean_absolute_error(
                                    numpy.ones(len(current_validation_amount)) * validation_amount,
                                    current_validation_amount)
                            )
                            validation_set_tag_amounts.append(validation_set_tag_counts)
                            training_set_tag_amounts.append(training_set_tag_counts)

                # Generate balance validation set
                # Selection done based on event counts (per scene class)
                # Target count specified percentage of training event count
                if validation_set_mae:
                    best_set_id = numpy.argmin(validation_set_mae)
                    validation_files += validation_set_candidates[best_set_id]

                else:
                    message = '{name}: Validation setup creation was not successful! Could not find a set with ' \
                              'examples for each tag class in both training and validation.'.format(
                        name=self.__class__.__name__
                    )

                    self.logger.exception(message)
                    raise AssertionError(message)

        if verbose:
            logger = FancyLogger()
            logger.sub_header('Validation set for fold [{fold}] / balanced'.format(fold=fold), indent=2)
            for scene_id, scene_label in enumerate(scene_labels):
                all_stats = training_meta.filter(scene_label=scene_label).tag_stat_counts()
                validation_stats = training_meta.filter(
                    scene_label=scene_label,
                    file_list=validation_files
                ).tag_stat_counts()

                training_files = sorted(list(set(self.train(fold=fold).unique_files) - set(validation_files)))
                training_stats = training_meta.filter(
                    scene_label=scene_label,
                    file_list=training_files
                ).tag_stat_counts()

                cell_data = numpy.zeros((len(list(all_stats.keys())) + 1, 4))
                for tag_id, tag_label in enumerate(list(all_stats.keys())):
                    cell_data[tag_id, 0] = all_stats[tag_label]
                    if tag_label in training_stats:
                        cell_data[tag_id, 1] = training_stats[tag_label]

                    if tag_label in validation_stats:
                        cell_data[tag_id, 2] = validation_stats[tag_label]

                    cell_data[tag_id, 3] = cell_data[tag_id, 2] / float(cell_data[tag_id, 0]) * 100

                cell_data[-1, 0] = numpy.sum(list(all_stats.values()))
                cell_data[-1, 1] = numpy.sum(list(training_stats.values()))
                cell_data[-1, 2] = numpy.sum(list(validation_stats.values()))
                cell_data[-1, 3] = cell_data[-1, 2] / float(cell_data[-1, 0]) * 100

                labels = list(all_stats.keys())
                labels.append('Overall')
                logger.line(scene_label, indent=4)
                logger.table(
                    cell_data=[labels] + cell_data.T.tolist(),
                    column_headers=['Tag', 'All', 'Training', 'Validation', 'Val amount (%)'],
                    column_types=['str20', 'int', 'int', 'int', 'float1'],
                    column_separators=[0, 1, 3],
                    row_separators=[len(labels)-1],
                    indent=6
                )

        return validation_files

#!/usr/bin/env python # -*- coding: utf-8 -*- List of datasets available      Parameters     ----------     data_path : str         Base path for the datasets         Default value 'data'      group : str         Group label for the datasets, currently supported ['scene', 'event', 'tag']      Returns     -------     str         Multi line string containing dataset table Factory to get correct dataset class based on name      Parameters     ----------     dataset_class_name : str         Class name      Raises     ------     NameError         Class does not exists      Returns     -------     Dataset class Dataset base class      The specific dataset classes are inherited from this class, and only needed methods are reimplemented. Constructor          Parameters         ----------         name : str             Dataset name          storage_name : str             Name to be used when storing dataset on disk          data_path : str             Root path where the dataset is stored. If None, os.path.join(tempfile.gettempdir(), 'dcase_util_datasets')             is used.          local_path : str             Direct storage path setup for the dataset. If None, data_path and storage_name are used to create one.          show_progress_in_console : bool             Show progress in console.          log_system_progress : bool             Show progress in log.          use_ascii_progress_bar : bool             Show progress bar using ASCII characters. Use this if your console does not support UTF-8 characters.          dataset_group : str             Dataset group label, one of ['scene', 'event', 'tag']          dataset_meta : dict             Dictionary containing metadata about the dataset, e.g., collecting device information, dataset authors.          evaluation_setup_folder : str             Directory name where evaluation setup files are stores          meta_filename : str             Filename to be used for main meta file (contains all files with their reference data) of the dataset.          error_meta_filename : str             Filename for the error annotation file.          filelisthash_filename : str             Filename for filelist hash file.          filelisthash_exclude_dirs : str             Directories to be excluded from filelist hash calculation.          crossvalidation_folds : int             Count fo cross-validation folds. Indexing starts from one.          package_list : list of dict             Package list, remote files associated to the dataset.          included_content_types : list of str or str             Indicates what content type should be processed. One or multiple from ['all', 'audio', 'meta', 'code',             'documentation']. If None given, ['all'] is used. Parameter can be also comma separated string.          audio_paths : list of str             List of paths to include audio material associated to the dataset. If None given, ['audio'] is used.          default_audio_extension : str             Default audio extension          reference_data_present : bool             Reference data is delivered with the dataset. # Dataset name # Dataset group # Dataset meta # Folder name for dataset # Path to the dataset # Evaluation setup folder # Path to the folder containing evaluation setup files # Meta data file, csv-format # Error meta data file, csv-format # Path to error meta data file # Hash file to detect removed or added files # Dirs to be excluded when calculating filelist hash # Number of evaluation folds # List containing dataset package items # Define this in the inherited dataset specific class. # Format: # { #    'content_type': 'documentation',  # Possible values ['meta', 'documentation', 'audio', 'code', 'features'] #    'remote_file': 'https://zenodo.org/record/45759/files/TUT-sound-events-2016-development.doc.zip', # URL #    'remote_bytes': 70918,            # Size of remote file in bytes #    'remote_md5': '33fd26a895530aef607a07b08704eacd',  # MD5 hash of remote file #    'filename': 'TUT-sound-events-2016-development.doc.zip' # filename relative to self.local_path always # } # Expand local filenames to be related to local path # What content type should be processed. Use this for example to access only the meta data, and exclude usually # large and time consuming audio material downloading. Leave to "all" to include all content types. # Split string to list if given # Inject all included content types, there might be packages containing multiple content types. # List of directories to contain the audio material # Expand local filenames to be related to local path # List of audio files # Recognized audio extensions # Reference data presence flag, by default dataset should have reference data present. # However, some evaluation dataset might not have # Initialize meta data container # List of audio error meta data dict # Cross-validation data # Training meta data for folds # Testing meta data for folds # Evaluation meta data for folds # Load meta and cross-validation data in Load dataset meta data and cross-validation sets into the container. Load meta data into the container. # Initialize meta container # Load from disk only if file exists # Load meta data # Mark reference data being present Load cross-validation into the container.          Returns         -------         self # Reset cross validation data and insert 'all_data' # Load cross validation folds # Get filenames # Training data for fold exists, load and process it # Process items # Initialize data # Testing data for fold exists, load and process it # Process items # Initialize data # Evaluation data for fold exists, load and process it # Process items # Initialize data Getting meta data item          Parameters         ----------         i : int             item id          Returns         -------         meta_data : dict             Meta data item Iterator for meta data items # yield window while it's valid # get next item Initialize the dataset, download, extract files and prepare the dataset for the usage.          Returns         -------         self # Create the dataset path if does not exist # Check file changes # Download packages # Extract content from packages # Load meta data in # Prepare meta data for the dataset class. # Save new filelist hash to monitor change in the dataset. Print dataset information. Log dataset information. Get all audio files in the dataset          Returns         -------         list             File list with absolute paths Get number of audio files in dataset          Returns         -------         int             Number of audio files Get meta data for dataset. If not already read from disk, data is read and returned.          Raises         ------         IOError             meta file not found.          Returns         -------         list             List containing meta data as dict. Number of meta data items.          Returns         -------         int             Meta data item count Get audio error meta data for dataset. If not already read from disk, data is read and returned.          Raises         ------         IOError:             audio error meta file not found.          Returns         -------         MetaDataContainer             List containing audio error meta data as dict. Number of error meta data items.          Returns         -------         int             Meta data item count Number of fold in the evaluation setup.          Returns         -------         int             Number of folds List of unique scene labels in the meta data.          Returns         -------         list             List of scene labels in alphabetical order. Number of unique scene labels in the meta data.          Returns         -------         int             Number of unique scene labels. List of unique event labels in the meta data.          Returns         -------         list             List of event labels in alphabetical order. Number of unique event labels in the meta data.          Returns         -------         int             Number of unique event labels List of unique audio tags in the meta data.          Returns         -------         list             List of audio tags in alphabetical order. Number of unique audio tags in the meta data.          Returns         -------         int             Number of unique audio tags Debug remote packages associated to the dataset.         Use this to check remote file size and md5 hash when developing dataset class.          Returns         -------         self Download dataset packages over the internet to the local path          Raises         ------         IOError             Download failed.          Returns         -------         self # Create the dataset path if does not exist # Download if remote file is set Extract the dataset packages          Raises         ------         IOError             Local package was not found.          Returns         -------         self # Process only files which are packages # Local file not present Prepare dataset for the usage.          Returns         -------         self Generates hash from file list and check does it matches with one saved in filelist.hash.         If some files have been deleted or added, checking will result False.          Returns         -------         bool             Result Process single meta data item          Parameters         ----------         item :  MetaDataItem             Meta data item          absolute_path : bool             Path format for the returned meta item, if True path is absolute, False path is relative to             the dataset root.             Default value True # Make sure filename has absolute path # Make sure filename has relative path Evaluation setup filename generation.          Parameters         ----------         setup_part :  str             Setup part 'train', 'test', 'evaluate'             Default value 'train'          fold : int             Fold number             Default value None          scene_label : str             Scene label             Default value None          file_extension : str             File extension             Default value 'txt'          Raises         ------         ValueError             Unknown setup part.          Returns         -------         str             Filename with full path. List of training items.          Parameters         ----------         fold : int [scalar]             Fold id, if None all meta data is returned.          absolute_paths : bool             Path format for the returned meta items, if True paths are absolute, False paths are relative to             the dataset root.             Default value True          Returns         -------         MetaDataContainer             List containing all meta data assigned to training set for given fold. # Go through items and make sure path are in correct form. List of testing items.          Parameters         ----------         fold : int             Fold id, if None all meta data is returned.          absolute_paths : bool             Path format for the returned meta items, if True paths are absolute, False paths are relative to             the dataset root.             Default value True          Returns         -------         MetaDataContainer             List containing all meta data assigned to testing set for given fold. # Go through items and make sure path are in correct form. List of evaluation items.          Parameters         ----------         fold : int             Fold id, if None all meta data is returned.          absolute_paths : bool             Path format for the returned meta items, if True paths are absolute, False paths are relative to             the dataset root.             Default value True          Returns         -------         MetaDataContainer             List containing all meta data assigned to testing set for given fold. # Go through items and make sure path are in correct form. List of training files.          Parameters         ----------         fold : int             Fold id, if None all meta data is returned.             Default value None          absolute_paths : bool             Path format for the returned meta items, if True paths are absolute, False paths are relative to             the dataset root.             Default value True          Returns         -------         list of str             List containing all files assigned to training set for given fold. List of testing files.          Parameters         ----------         fold : int             Fold id, if None all meta data is returned.             Default value None          absolute_paths : bool             Path format for the returned meta items, if True paths are absolute, False paths are relative to             the dataset root.             Default value True          Returns         -------         list of str             List containing all files assigned to testing set for given fold. List of evaluation files.          Parameters         ----------         fold : int             Fold id, if None all meta data is returned.             Default value None          absolute_paths : bool             Path format for the returned meta items, if True paths are absolute, False paths are relative to             the dataset root.             Default value True          Returns         -------         list of str             List containing all files assigned to testing set for given fold. List of validation files. Validation files are always subset of training files.          Parameters         ----------         fold : int             Fold id, if None all meta data is returned.             Default value None          split_type : str             Split type [dataset, random, balanced]             Default value 'balanced'          validation_amount : float             Amount of training files to be assigned for validation             Default value None          seed : int             Randomization seed             Default value 0          verbose : bool             Show information about the validation set.             Default value False          scene_label : str             Scene label of the validation set. If None, all training material used.             Default value None          iterations : int             Randomization iterations done when finding balanced set before selecting best matched set.             Default value 100          Returns         -------         list of str             List containing all files assigned to training set for given fold.          list of str             List containing all files assigned to validation set for given fold. List of validation files delivered by the dataset. List of validation files selected randomly from the training material.          Parameters         ----------         fold : int             Fold id, if None all meta data is returned.             Default value None          validation_amount : float             Amount of training material to be assigned for validation.             Default value 0.3          seed : int             Randomization seed             Default value 0          verbose : bool             Show information about the validation set.             Default value False          Returns         -------         list of str             List containing all files assigned for validation List of validation files randomly selecting while maintaining data balance. List of fold ids          Parameters         ----------         mode : str {'folds','full'}             Fold setup type, possible values are 'folds' and 'full'. In 'full' mode fold number is set 0 and             all data is used for training.             Default value 'folds'          Returns         -------         list of int             Fold ids Meta data for given file          Parameters         ----------         filename : str             File name          Returns         -------         list of dict             List containing all meta data related to given file. # Try first with raw filename # Try with absolute filename # Try with relative filename Error meta data for given file          Parameters         ----------         filename : str             File name          Returns         -------         list of dict             List containing all error meta data related to given file. Pre-calculated acoustic features for given file          Parameters         ----------         filename : str             File name          Returns         -------         numpy.ndarray             Matrix containing acoustic features Converts relative path into absolute path.          Parameters         ----------         path : str             Relative path          Returns         -------         str             Absolute path Converts absolute path into relative path.          Parameters         ----------         path : str             Absolute path          Returns         -------         str             Relative path Total download size of the dataset in bytes.          Returns         -------         int             Total bytes Total download size of the dataset in a string.          Returns         -------         str             Total size Total size of the dataset currently stored locally.          Returns         -------         str             Total size List of files under local_path          Parameters         ----------         exclude_dirs : list of str             List of directories to be excluded             Default value None          Returns         -------         list of str             File list Generates file list hash, and saves it as filelist.hash under local_path.          Parameters         ----------          Returns         -------         None Acoustic scene dataset baseclass List of validation files randomly selecting while maintaining data balance.          Parameters         ----------         fold : int             Fold id, if None all meta data is returned.             Default value None          validation_amount : float             Amount of training material to be assigned for validation.             Default value 0.3          seed : int             Randomization seed             Default value 0          verbose : bool             Show information about the validation set.             Default value False          iterations : int             How many randomization iterations will be done before selecting best matched.             Default value 100          Returns         -------         list of str             List containing all files assigned for validation # Check do we have location/source identifier present # Do the balance based on scene class and identifier # Collect validation files # Collect training files # Do the balance based on scene class only Sound event dataset baseclass Load cross-validation into the container.          Returns         -------         self # Reset cross validation data and insert 'all_data' # Load cross validation folds # Initialize data # Get filenames # Training data for fold exists, load and process it # Testing data for fold exists, load and process it # Evaluation data for fold exists, load and process it # Process items Number of unique scene labels in the meta data.          Parameters         ----------         scene_label : str             Scene label             Default value None          Returns         -------         int             Number of unique scene labels. List of unique event labels in the meta data.          Parameters         ----------         scene_label : str             Scene label             Default value None          Returns         -------         list of str             List of event labels in alphabetical order. List of training items.          Parameters         ----------         fold : int             Fold id, if None all meta data is returned.             Default value None          absolute_paths : bool             Path format for the returned meta items, if True paths are absolute, False paths are relative to             the dataset root.             Default value True          scene_label : str             Scene label             Default value None          event_label : str             Event label             Default value None          Returns         -------         list of dict             List containing all meta data assigned to training set for given fold. # Go through items and make sure path are in correct form. List of testing items.          Parameters         ----------         fold : int             Fold id, if None all meta data is returned.             Default value None          absolute_paths : bool             Path format for the returned meta items, if True paths are absolute, False paths are relative to             the dataset root.             Default value True          scene_label : str             Scene label             Default value None          event_label : str             Event label             Default value None          Returns         -------         list of dict             List containing all meta data assigned to testing set for given fold. # Go through items and make sure path are in correct form. List of evaluation items.          Parameters         ----------         fold : int             Fold id, if None all meta data is returned.             Default value None          absolute_paths : bool             Path format for the returned meta items, if True paths are absolute, False paths are relative to             the dataset root.             Default value True          scene_label : str             Scene label             Default value None          event_label : str             Event label             Default value None          Returns         -------         list of dict             List containing all meta data assigned to testing set for given fold. # Go through items and make sure path are in correct form. List of training files.          Parameters         ----------         fold : int             Fold id, if None all meta data is returned.             Default value None          absolute_paths : bool             Path format for the returned meta items, if True paths are absolute, False paths are relative to             the dataset root.             Default value True          scene_label : str             Scene label             Default value None          event_label : str             Event label             Default value None          Returns         -------         list of str             List containing all files assigned to training set for given fold. List of testing files.          Parameters         ----------         fold : int             Fold id, if None all meta data is returned.             Default value None          absolute_paths : bool             Path format for the returned meta items, if True paths are absolute, False paths are relative to             the dataset root.             Default value True          scene_label : str             Scene label             Default value None          event_label : str             Event label             Default value None          Returns         -------         list of str             List containing all files assigned to testing set for given fold. List of evaluation files.          Parameters         ----------         fold : int             Fold id, if None all meta data is returned.             Default value None          absolute_paths : bool             Path format for the returned meta items, if True paths are absolute, False paths are relative to             the dataset root.             Default value True          scene_label : str             Scene label             Default value None          event_label : str             Event label             Default value None          Returns         -------         list of str             List containing all files assigned to testing set for given fold. List of validation files selected randomly from the training material.          Parameters         ----------         fold : int             Fold id, if None all meta data is returned.             Default value None          validation_amount : float             Amount of training material to be assigned for validation.             Default value 0.3          seed : int             Randomization seed             Default value 0          verbose : bool             Show information about the validation set.             Default value False          scene_label : str             Scene label of the validation set. If None, all training material used.             Default value None          Returns         -------         list of str             List containing all files assigned for validation List of validation files randomly selecting while maintaining data balance.          Parameters         ----------         fold : int             Fold id, if None all meta data is returned.             Default value None          validation_amount : float             Amount of training material to be assigned for validation.             Default value 0.3          seed : int             Randomization seed             Default value 0          verbose : bool             Show information about the validation set.             Default value False          scene_label : str             Scene label of the validation set. If None, all training material used.             Default value None          iterations : int             How many randomization iterations will be done before selecting best matched.             Default value 100          Returns         -------         list of str             List containing all files assigned for validation # Check do we have location/source identifier present # Do the balance based on scene class, identifier and event class # Get event amounts per class # Optimize scene sets separately # Event count in training set candidate # Accept only sets which leave at least one example for training # Event counts in validation set candidate # Accept only sets which have examples from each event class # Generate balance validation set # Selection done based on event counts (per scene class) # Target count specified percentage of training event count # Do the balance based on scene class, identifier and event class # Get event amounts per class # Optimize scene sets separately # Event count in training set candidate # Accept only sets which leave at least one example for training # Event counts in validation set candidate # Accept only sets which have examples from each event class # Generate balance validation set # Selection done based on event counts (per scene class) # Target count specified percentage of training event count Synthetic sound event dataset baseclass Initialize the dataset, download, extract files and prepare the dataset for the usage.          Returns         -------         self # Create the dataset path if does not exist Audio tag dataset baseclass List of validation files selected randomly from the training material.          Parameters         ----------         fold : int             Fold id, if None all meta data is returned.             Default value None          validation_amount : float             Amount of training material to be assigned for validation.             Default value 0.3          seed : int             Randomization seed             Default value 0          verbose : bool             Show information about the validation set.             Default value False          Returns         -------         list of str             List containing all files assigned for validation List of validation files randomly selecting while maintaining data balance.          Parameters         ----------         fold : int             Fold id, if None all meta data is returned.             Default value None          validation_amount : float             Amount of training material to be assigned for validation.             Default value 0.3          seed : int             Randomization seed             Default value 0          verbose : bool             Show information about the validation set.             Default value False          iterations : int             How many randomization iterations will be done before selecting best matched.             Default value 100          Returns         -------         list of str             List containing all files assigned for validation # Check do we have location/source identifier present # Do the balance based on scene class, identifier and event class # Get tag amounts per class # Optimize scene sets separately # Tag count in training set candidate # Accept only sets which leave at least one example for training # Tag counts in validation set candidate # Accept only sets which have examples from each tag class # Generate balance validation set # Selection done based on event counts (per scene class) # Target count specified percentage of training event count # Do the balance based on scene class, identifier and event class # Get tag amounts per class # Optimize scene sets separately # Tag count in training set candidate # Accept only sets which leave at least one example for training # Tag counts in validation set candidate # Accept only sets which have examples from each tag class # Generate balance validation set # Selection done based on event counts (per scene class) # Target count specified percentage of training event count