class ExperimentParameters(object):

    def __init__(self, use_default_values=True):
        self.parameters = deepcopy(reference_parameters)
        self._load_custom_definitions()
        if use_default_values:
            self.parameters = self._init_default_values(self.parameters)

    def _init_default_values(self, node):
        if isinstance(node, Parameter):
            if node.default is not None or node.optional:
                return node.default
        elif isinstance(node, dict):
            for name, value in node.items():
                node[name] = self._init_default_values(value)
        return node

    def _load_custom_definitions(self):
        for name, definition in get_custom_layers().items():
            self.layer_parameter(name, definition[1])
            self.add_layer_type(name, definition[2])

        for name in get_custom_activations().keys():
            self.add_activation(name)

    def get_parameter(self, *path):
        node = self.parameters
        path = expand_param_path(path)
        for elem in path:
            if not elem in node:
                return None
            node = node[elem]
        return node

    def _set_parameter(self, _id, value):
        self.parameters = self._set_node_parameter(
            self.parameters,
            _id,
            value)

    def _set_node_parameter(self, node, path, value):
        node = node or dict()
        path = expand_param_path(path)
        if len(path) > 1:
            node[path[0]] = self._set_node_parameter(
                node[path[0]],
                path[1:],
                value)
        else:
            node[path[0]] = value
        return node

    def layout_parameter(self, name, value):
        return self._set_parameter(
            ['layout', name],
            value)

    def get_layout_parameter(self, name):
        return self.get_parameter('layout', name)

    def layer_parameter(self, name, value):
        return self._set_parameter(
            ['layers', name],
            value)

    def add_activation(self, name):
        for _layer_name, parameters in self.get_parameter('layers').items():
            for param_name, param_value in parameters.items():
                if 'activation' == param_name and isinstance(param_value, Parameter):
                    param_value.values.append(name)

    def get_layer_types(self):
        param = self.get_layout_parameter('layer.type')
        if isinstance(param, list):
            return param
        elif isinstance(param, str):
            return [param]
        return self.get_layout_parameter('layer.type').values

    def layer_types(self, layer_types):
        self.layout_parameter('layer.type', layer_types)

    def add_layer_type(self, layer_type, stackable=False):
        layer_type_param = self.get_layout_parameter('layer.type')
        layer_type_param.values = list(set(layer_type_param.values + [layer_type]))
        if stackable:
            layer_stackable_param = self.get_layout_parameter('layer.stackable')
            layer_stackable_param.values = list(set(layer_stackable_param.values + [layer_type]))

    def get_layer_parameter(self, name):
        return self.get_parameter('layers', name)

    def get_layer_parameters(self, layer):
        return self.get_parameter('layers', str_param_name(layer))

    def optimizer_parameter(self, name, value):
        return self._set_parameter(
            ['optimizers', name],
            value)

    def get_optimizer_parameters(self, name):
        return self.get_parameter('optimizers', name)

    def get_optimizers_parameters(self):
        return self.get_parameter('optimizers') class Experiment(object):

    def __init__(self, label, layout=None, training=None,
                 batch_iterator=None, test_batch_iterator=None,
                 environment=None, settings=None, parameters=None, resume=False):
        self.label = label
        self.layout = layout
        self.training = training
        self.batch_iterator = batch_iterator
        self.test_batch_iterator = test_batch_iterator
        self.environment = environment or Environment()
        self.settings = settings or ExperimentSettings()
        self.parameters = parameters or ExperimentParameters()

    def get_experiment_data_dir(self):
        return join(
            self.environment.data_dir,
            self.label)

    def get_log_filename(self):
        return path.join(
            self.get_experiment_data_dir(),
            'experiment.log')

    def get_step_log_filename(self, step):
        return path.join(
            self.get_experiment_data_dir(),
            'experiment.step.%d.log' % step)

    def get_step_data_filename(self, step):
        return path.join(
            self.get_experiment_data_dir(),
            'experiment.step.%d.data' % step)

    def get_checkpoint_filename(self):
        return path.join(
            self.get_experiment_data_dir(),
            'experiment.chkpt')

    def evaluate(self, blueprints):
        from minos.train.trainer import MultiProcessModelTrainer
        model_trainer = MultiProcessModelTrainer(
            self.batch_iterator,
            self.test_batch_iterator,
            self.environment)
        return [
            [result[1]]
            for result in model_trainer.build_and_train_models(blueprints)] def _assert_valid_training_parameters(experiment):
    if not experiment.training:
        raise InvalidParametersException(
            'No training parameters specified')
    if not isinstance(
            experiment.training.stopping,
            AccuracyDecreaseStoppingCondition):
        return
    training = experiment.training
    if training.metric.metric != training.stopping.metric:
        raise InvalidParametersException(
            'The same metric must be used for training and early stopping, '
            + '%s != %s' % (
                training.metric.metric,
                training.stopping.metric)) class Training(object):

    def __init__(self, objective, optimizer, metric,
                 stopping, batch_size, class_weight=None):
        self.objective = objective
        self.optimizer = optimizer
        self.metric = metric
        self.stopping = stopping
        self.batch_size = batch_size
        self.class_weight = class_weight

    def todict(self):
        return {
            'objective': self.objective.todict(),
            'optimizer': self.optimizer.todict(),
            'metric': self.metric.todict(),
            'stopping': self.stopping.todict(),
            'batch_size': self.batch_size,
            'class_weight': self.class_weight} class EpochStoppingCondition(object):
    """ Stop training after a fixed number of epochs
        # Arguments
        epoch: number of epochs after which training will be stopped.
    """

    def __init__(self, epoch):
        self.epoch = epoch

    def todict(self):
        return dict(vars(self)) class AccuracyDecreaseStoppingCondition(object):
    """ Stop training when the metric observed has stopped improving.
        # Arguments
        noprogress_count: number of epochs with no improvement
            after which training will be stopped.
        min_epoch: minimum number of epochs.
        max_epoch: maximum number of epochs.
    """

    def __init__(self, max_epoch, metric='accuracy',
                 validation_metric=True, noprogress_count=3, min_epoch=0):
        self.metric = metric
        self.validation_metric = validation_metric
        self.noprogress_count = noprogress_count
        self.min_epoch = min_epoch
        self.max_epoch = max_epoch
        self.epoch = 0

    def get_monitor_metric(self):
        if self.validation_metric:
            return get_associated_validation_metric(self.metric)
        return self.metric

    def is_min_epoch_defined(self):
        return self.min_epoch and self.min_epoch > 0

    def is_max_epoch_defined(self):
        return self.max_epoch and self.max_epoch > 0

    def is_at_least_min_epoch(self):
        return self.is_min_epoch_defined()\
            and self.epoch >= self.min_epoch

    def is_at_most_max_epoch(self):
        return self.is_max_epoch_defined()\
            and self.epoch <= self.max_epoch

    def todict(self):
        return dict(vars(self)) class ModelBuilder(object):

    def __init__(self):
        pass

    def build(self, blueprint, device, compile_model=True):
        model = _build_model(blueprint, device)
        if compile_model:
            model.compile(
                optimizer=_build_optimizer(blueprint.training),
                loss=blueprint.training.objective.objective,
                metrics=[blueprint.training.metric.metric])
        return model def create_random_blueprint(experiment):
    return Blueprint(
        _random_layout(
            experiment.layout,
            experiment.settings,
            experiment.parameters,
            mutation_std=experiment.settings.ga['mutation_std']),
        _random_training(experiment)) class Layout(object):

    def __init__(self, input_size, output_size,
                 output_activation, block=None, block_input=None, rows=None):
        self.input_size = int(input_size)
        self.output_size = int(output_size)
        self.output_activation = output_activation
        self.block = block
        self.block_input = block_input
        self.rows = rows or list()

    def get_rows(self):
        return self.rows

    def get_blocks(self):
        return [
            block
            for row in self.rows
            for block in row.get_blocks()]

    def get_layers(self):
        return [
            layer
            for row in self.rows
            for layer in row.get_layers()]

    def todict(self):
        dict_repr = dict(vars(self))
        del dict_repr['block']
        dict_repr['rows'] = [row.todict() for row in self.rows]
        return dict_repr class Objective(object):

    def __init__(self, objective):
        self.objective = objective

    def todict(self):
        return dict(vars(self)) class Optimizer(object):

    def __init__(self, optimizer=None, parameters=None):
        self.optimizer = optimizer
        self.parameters = parameters or dict()

    def todict(self):
        return dict(vars(self)) class Metric(object):

    def __init__(self, metric):
        self.metric = metric

    def todict(self):
        return dict(vars(self)) def default_device():
    gpus = get_available_gpus()
    if len(gpus) > 0:
        return gpus[1]
    return cpu_device() def cpu_device():
    return '/cpu:0' class ModelTrainer(object):

    def __init__(self, batch_iterator, test_batch_iterator):
        from minos.model.build import ModelBuilder
        self.model_builder = ModelBuilder()
        self.batch_iterator = batch_iterator
        self.test_batch_iterator = test_batch_iterator

    def train(self, blueprint, device,
              save_best_model=False, model_filename=None, class_weight=None):
        try:
            model = self.model_builder.build(
                blueprint,
                device)
            class_weight = class_weight or blueprint.training.class_weight
            setup_tf_session(device)
            nb_epoch, callbacks = self._get_stopping_parameters(blueprint)
            if save_best_model:
                callbacks.append(self._get_model_save_callback(
                    model_filename,
                    blueprint.training.metric.metric))
            start = time()
            history = model.fit_generator(
                self.batch_iterator,
                self.batch_iterator.samples_per_epoch,
                nb_epoch,
                callbacks=callbacks,
                validation_data=self.test_batch_iterator,
                nb_val_samples=self.test_batch_iterator.sample_count,
                class_weight=class_weight)
            if save_best_model:
                del model
                model = load_keras_model(model_filename)
            return model, history, (time() - start)
        except Exception as ex:
            logging.debug(ex)
            logging.debug(traceback.format_exc())
        try:
            from keras import backend
            backend.clear_session()
        except:
            logging.debug(ex)
            logging.debug(traceback.format_exc())
        return None, None, 0

    def _get_model_save_callback(self, model_filename, metric):
        checkpoint = ModelCheckpoint(
            model_filename,
            monitor=metric,
            save_best_only=True)
        return checkpoint

    def _get_stopping_parameters(self, blueprint):
        if isinstance(blueprint.training.stopping, EpochStoppingCondition):
            nb_epoch = blueprint.training.stopping.epoch
            stopping_callbacks = []
        if isinstance(blueprint.training.stopping, AccuracyDecreaseStoppingCondition):
            nb_epoch = max(
                1,
                blueprint.training.stopping.min_epoch,
                blueprint.training.stopping.max_epoch)
            stopping_callbacks = [
                StoppingConditionWrapper(blueprint.training.stopping)]
        return nb_epoch, stopping_callbacks class CpuEnvironment(Environment):

    def __init__(self, n_jobs=1, data_dir=None,
                 tf_logging_level=tf.logging.ERROR):
        super().__init__(
            ['/cpu:0'],
            n_jobs,
            data_dir=data_dir,
            tf_logging_level=tf_logging_level) def disable_sysout():
    sys.stdout.write = lambda s: s def get_reuters_dataset(batch_size, max_words):
    (X_train, y_train), (X_test, y_test) = reuters.load_data(nb_words=max_words, test_split=0.2)

    tokenizer = Tokenizer(nb_words=max_words)
    X_train = tokenizer.sequences_to_matrix(X_train, mode='binary')
    X_test = tokenizer.sequences_to_matrix(X_test, mode='binary')

    nb_classes = numpy.max(y_train) + 1
    y_train = np_utils.to_categorical(y_train, nb_classes)
    y_test = np_utils.to_categorical(y_test, nb_classes)

    batch_iterator = SimpleBatchIterator(
        X_train,
        y_train,
        batch_size,
        autoloop=True)
    test_batch_iterator = SimpleBatchIterator(
        X_test,
        y_test,
        len(X_test),
        autoloop=True)
    return batch_iterator, test_batch_iterator, nb_classes
from genericpath import isfile
from os.path import join
from minos.experiment.experiment import ExperimentParameters, Experiment,\
    _assert_valid_training_parameters
from minos.experiment.training import Training, EpochStoppingCondition,\
    AccuracyDecreaseStoppingCondition
from minos.model.build import ModelBuilder
from minos.model.design import create_random_blueprint
from minos.model.model import Layout, Objective, Optimizer, Metric
from minos.tf_utils import default_device, cpu_device
from minos.train.trainer import ModelTrainer
from minos.train.utils import CpuEnvironment
from minos.utils import disable_sysout
from tests.fixtures import get_reuters_dataset
import tempfile
import unittest

    def test_train(self):
        disable_sysout()
        with tempfile.TemporaryDirectory() as tmp_dir:
            batch_size = 50
            batch_iterator, test_batch_iterator, nb_classes = get_reuters_dataset(batch_size, 1000)
            layout = Layout(
                input_size=1000,
                output_size=nb_classes,
                output_activation='softmax')
            training = Training(
                objective=Objective('categorical_crossentropy'),
                optimizer=Optimizer(optimizer='Adam'),
                metric=Metric('categorical_accuracy'),
                stopping=EpochStoppingCondition(10),
                batch_size=batch_size)
            experiment_parameters = ExperimentParameters(use_default_values=True)
            experiment_parameters.layout_parameter('rows', 1)
            experiment_parameters.layout_parameter('blocks', 1)
            experiment_parameters.layout_parameter('layers', 1)
            experiment = Experiment(
                'test__reuters_experiment',
                layout,
                training,
                batch_iterator,
                test_batch_iterator,
                CpuEnvironment(n_jobs=1, data_dir=tmp_dir),
                parameters=experiment_parameters)

            blueprint = create_random_blueprint(experiment)
            model = ModelBuilder().build(blueprint, default_device())
            result = model.fit_generator(
                generator=batch_iterator,
                samples_per_epoch=batch_iterator.samples_per_epoch,
                nb_epoch=10,
                validation_data=test_batch_iterator,
                nb_val_samples=test_batch_iterator.sample_count)
            self.assertIsNotNone(
                result,
                'should have fit the model')
            score = model.evaluate_generator(
                test_batch_iterator,
                val_samples=test_batch_iterator.sample_count)
            self.assertIsNotNone(
                score,
                'should have evaluated the model')

    def test_model_trainer(self):
        disable_sysout()
        with tempfile.TemporaryDirectory() as tmp_dir:
            batch_size = 50
            batch_iterator, test_batch_iterator, nb_classes = get_reuters_dataset(batch_size, 1000)
            layout = Layout(
                input_size=1000,
                output_size=nb_classes,
                output_activation='softmax')
            training = Training(
                objective=Objective('categorical_crossentropy'),
                optimizer=Optimizer(optimizer='Adam'),
                metric=Metric('categorical_accuracy'),
                stopping=EpochStoppingCondition(10),
                batch_size=batch_size)
            experiment_parameters = ExperimentParameters(use_default_values=True)
            experiment_parameters.layout_parameter('rows', 1)
            experiment_parameters.layout_parameter('blocks', 1)
            experiment_parameters.layout_parameter('layers', 1)
            experiment = Experiment(
                'test__reuters_experiment',
                layout,
                training,
                batch_iterator,
                test_batch_iterator,
                CpuEnvironment(n_jobs=1, data_dir=tmp_dir),
                parameters=experiment_parameters)

            blueprint = create_random_blueprint(experiment)
            trainer = ModelTrainer(batch_iterator, test_batch_iterator)
            model_filename = join(tmp_dir, 'model')
            model, history, _duration = trainer.train(
                blueprint,
                cpu_device(),
                save_best_model=True,
                model_filename=model_filename)
            self.assertIsNotNone(
                model,
                'should have fit the model')
            self.assertTrue(isfile(model_filename), 'Should have saved the model')
            self.assertIsNotNone(history, 'Should have the training history')

    def test_early_stopping_condition_test(self):
        disable_sysout()
        with tempfile.TemporaryDirectory() as tmp_dir:
            batch_size = 50
            min_epoch = 10
            max_epoch = 15
            batch_iterator, test_batch_iterator, nb_classes = get_reuters_dataset(batch_size, 1000)
            layout = Layout(
                input_size=1000,
                output_size=nb_classes,
                output_activation='softmax')
            training = Training(
                objective=Objective('categorical_crossentropy'),
                optimizer=Optimizer(optimizer='Adam'),
                metric=Metric('categorical_accuracy'),
                stopping=AccuracyDecreaseStoppingCondition(
                    metric='categorical_accuracy',
                    noprogress_count=2,
                    min_epoch=min_epoch,
                    max_epoch=max_epoch),
                batch_size=batch_size)
            experiment_parameters = ExperimentParameters(use_default_values=True)
            experiment_parameters.layout_parameter('rows', 1)
            experiment_parameters.layout_parameter('blocks', 1)
            experiment_parameters.layout_parameter('layers', 1)
            experiment = Experiment(
                'test__reuters_experiment',
                layout,
                training,
                batch_iterator,
                test_batch_iterator,

                CpuEnvironment(n_jobs=1, data_dir=tmp_dir),