def get_reuters_dataset(batch_size, max_words):
    (X_train, y_train), (X_test, y_test) = reuters.load_data(nb_words=max_words, test_split=0.2)
    nb_classes = np.max(y_train) + 1
    tokenizer = Tokenizer(nb_words=max_words)
    X_train = tokenizer.sequences_to_matrix(X_train, mode='binary')
    X_test = tokenizer.sequences_to_matrix(X_test, mode='binary')
    y_train = np_utils.to_categorical(y_train, nb_classes)
    y_test = np_utils.to_categorical(y_test, nb_classes)

    batch_iterator = SimpleBatchIterator(
        X_train,
        y_train,
        batch_size,
        autoloop=True)
    test_batch_iterator = SimpleBatchIterator(
        X_test,
        y_test,
        len(X_test),
        autoloop=True)
    return batch_iterator, test_batch_iterator, nb_classes class Experiment(object):

    def __init__(self, label, layout=None, training=None,
                 batch_iterator=None, test_batch_iterator=None,
                 environment=None, settings=None, parameters=None, resume=False):
        self.label = label
        self.layout = layout
        self.training = training
        self.batch_iterator = batch_iterator
        self.test_batch_iterator = test_batch_iterator
        self.environment = environment or Environment()
        self.settings = settings or ExperimentSettings()
        self.parameters = parameters or ExperimentParameters()

    def get_experiment_data_dir(self):
        return join(
            self.environment.data_dir,
            self.label)

    def get_log_filename(self):
        return path.join(
            self.get_experiment_data_dir(),
            'experiment.log')

    def get_step_log_filename(self, step):
        return path.join(
            self.get_experiment_data_dir(),
            'experiment.step.%d.log' % step)

    def get_step_data_filename(self, step):
        return path.join(
            self.get_experiment_data_dir(),
            'experiment.step.%d.data' % step)

    def get_checkpoint_filename(self):
        return path.join(
            self.get_experiment_data_dir(),
            'experiment.chkpt')

    def evaluate(self, blueprints):
        from minos.train.trainer import MultiProcessModelTrainer
        model_trainer = MultiProcessModelTrainer(
            self.batch_iterator,
            self.test_batch_iterator,
            self.environment)
        return [
            [result[1]]
            for result in model_trainer.build_and_train_models(blueprints)] class ExperimentParameters(object):

    def __init__(self, use_default_values=True):
        self.parameters = deepcopy(reference_parameters)
        self._load_custom_definitions()
        if use_default_values:
            self.parameters = self._init_default_values(self.parameters)

    def _init_default_values(self, node):
        if isinstance(node, Parameter):
            if node.default is not None or node.optional:
                return node.default
        elif isinstance(node, dict):
            for name, value in node.items():
                node[name] = self._init_default_values(value)
        return node

    def _load_custom_definitions(self):
        for name, definition in get_custom_layers().items():
            self.layer_parameter(name, definition[1])
            self.add_layer_type(name, definition[2])

        for name in get_custom_activations().keys():
            self.add_activation(name)

    def get_parameter(self, *path):
        node = self.parameters
        path = expand_param_path(path)
        for elem in path:
            if not elem in node:
                return None
            node = node[elem]
        return node

    def _set_parameter(self, _id, value):
        self.parameters = self._set_node_parameter(
            self.parameters,
            _id,
            value)

    def _set_node_parameter(self, node, path, value):
        node = node or dict()
        path = expand_param_path(path)
        if len(path) > 1:
            node[path[0]] = self._set_node_parameter(
                node[path[0]],
                path[1:],
                value)
        else:
            node[path[0]] = value
        return node

    def layout_parameter(self, name, value):
        return self._set_parameter(
            ['layout', name],
            value)

    def get_layout_parameter(self, name):
        return self.get_parameter('layout', name)

    def layer_parameter(self, name, value):
        return self._set_parameter(
            ['layers', name],
            value)

    def add_activation(self, name):
        for _layer_name, parameters in self.get_parameter('layers').items():
            for param_name, param_value in parameters.items():
                if 'activation' == param_name and isinstance(param_value, Parameter):
                    param_value.values.append(name)

    def get_layer_types(self):
        param = self.get_layout_parameter('layer.type')
        if isinstance(param, list):
            return param
        elif isinstance(param, str):
            return [param]
        return self.get_layout_parameter('layer.type').values

    def layer_types(self, layer_types):
        self.layout_parameter('layer.type', layer_types)

    def add_layer_type(self, layer_type, stackable=False):
        layer_type_param = self.get_layout_parameter('layer.type')
        layer_type_param.values = list(set(layer_type_param.values + [layer_type]))
        if stackable:
            layer_stackable_param = self.get_layout_parameter('layer.stackable')
            layer_stackable_param.values = list(set(layer_stackable_param.values + [layer_type]))

    def get_layer_parameter(self, name):
        return self.get_parameter('layers', name)

    def get_layer_parameters(self, layer):
        return self.get_parameter('layers', str_param_name(layer))

    def optimizer_parameter(self, name, value):
        return self._set_parameter(
            ['optimizers', name],
            value)

    def get_optimizer_parameters(self, name):
        return self.get_parameter('optimizers', name)

    def get_optimizers_parameters(self):
        return self.get_parameter('optimizers') def load_experiment_blueprints(experiment_label, step, environment=Environment()):
    experiment = Experiment(experiment_label, environment=environment)
    data_filename = experiment.get_step_data_filename(step)
    with open(data_filename, 'rb') as data_file:
        return pickle.load(data_file) class ExperimentSettings(object):

    def __init__(self):
        self.search = self._search_settings()
        self.ga = self._ga_settings()

    def _search_settings(self):
        return {
            'layout': True,
            'parameters': True,
            'optimizer': True}

    def _ga_settings(self):
        return {
            'population_size': 50,
            'generations': 100,
            'p_offspring': 0.5,
            'offsprings': 2,
            'p_mutation': 0.25,
            'mutation_std': 0.25,
            'p_aliens': 0.25,
            'aliens': 0.1}

    def is_layout_search(self):
        return self.search['layout']

    def is_parameters_search(self):
        return self.search['parameters']

    def is_optimizer_search(self):
        return self.search['optimizer'] def run_ga_search_experiment(experiment, resume=False, log_level='INFO'):
    run_experiment(
        GaModelExperiment(experiment),
        search,
        resume=resume,
        log_level=log_level) class Training(object):

    def __init__(self, objective, optimizer, metric,
                 stopping, batch_size, class_weight=None):
        self.objective = objective
        self.optimizer = optimizer
        self.metric = metric
        self.stopping = stopping
        self.batch_size = batch_size
        self.class_weight = class_weight

    def todict(self):
        return {
            'objective': self.objective.todict(),
            'optimizer': self.optimizer.todict(),
            'metric': self.metric.todict(),
            'stopping': self.stopping.todict(),
            'batch_size': self.batch_size,
            'class_weight': self.class_weight} class AccuracyDecreaseStoppingCondition(object):
    """ Stop training when the metric observed has stopped improving.
        # Arguments
        noprogress_count: number of epochs with no improvement
            after which training will be stopped.
        min_epoch: minimum number of epochs.
        max_epoch: maximum number of epochs.
    """

    def __init__(self, max_epoch, metric='accuracy',
                 validation_metric=True, noprogress_count=3, min_epoch=0):
        self.metric = metric
        self.validation_metric = validation_metric
        self.noprogress_count = noprogress_count
        self.min_epoch = min_epoch
        self.max_epoch = max_epoch
        self.epoch = 0

    def get_monitor_metric(self):
        if self.validation_metric:
            return get_associated_validation_metric(self.metric)
        return self.metric

    def is_min_epoch_defined(self):
        return self.min_epoch and self.min_epoch > 0

    def is_max_epoch_defined(self):
        return self.max_epoch and self.max_epoch > 0

    def is_at_least_min_epoch(self):
        return self.is_min_epoch_defined()\
            and self.epoch >= self.min_epoch

    def is_at_most_max_epoch(self):
        return self.is_max_epoch_defined()\
            and self.epoch <= self.max_epoch

    def todict(self):
        return dict(vars(self)) class EpochStoppingCondition(object):
    """ Stop training after a fixed number of epochs
        # Arguments
        epoch: number of epochs after which training will be stopped.
    """

    def __init__(self, epoch):
        self.epoch = epoch

    def todict(self):
        return dict(vars(self)) class ModelBuilder(object):

    def __init__(self):
        pass

    def build(self, blueprint, device, compile_model=True):
        model = _build_model(blueprint, device)
        if compile_model:
            model.compile(
                optimizer=_build_optimizer(blueprint.training),
                loss=blueprint.training.objective.objective,
                metrics=[blueprint.training.metric.metric])
        return model class Objective(object):

    def __init__(self, objective):
        self.objective = objective

    def todict(self):
        return dict(vars(self)) class Optimizer(object):

    def __init__(self, optimizer=None, parameters=None):
        self.optimizer = optimizer
        self.parameters = parameters or dict()

    def todict(self):
        return dict(vars(self)) class Metric(object):

    def __init__(self, metric):
        self.metric = metric

    def todict(self):
        return dict(vars(self)) class Layout(object):

    def __init__(self, input_size, output_size,
                 output_activation, block=None, block_input=None, rows=None):
        self.input_size = int(input_size)
        self.output_size = int(output_size)
        self.output_activation = output_activation
        self.block = block
        self.block_input = block_input
        self.rows = rows or list()

    def get_rows(self):
        return self.rows

    def get_blocks(self):
        return [
            block
            for row in self.rows
            for block in row.get_blocks()]

    def get_layers(self):
        return [
            layer
            for row in self.rows
            for layer in row.get_layers()]

    def todict(self):
        dict_repr = dict(vars(self))
        del dict_repr['block']
        dict_repr['rows'] = [row.todict() for row in self.rows]
        return dict_repr def int_param(lo=0, hi=100, default=None, values=None, optional=False):
    return Parameter(
        int,
        lo=lo,
        hi=hi,
        values=values,
        default=default,
        optional=optional) def string_param(values, default=None, optional=False, mutable=True):
    return Parameter(
        str,
        values=values,
        default=default,
        optional=optional,
        mutable=mutable) def float_param(lo=0., hi=1., default=None, values=None, optional=False):
    return Parameter(
        float,
        lo=lo,
        hi=hi,
        values=values,
        default=default,
        optional=optional) def cpu_device():
    return '/cpu:0' class CpuEnvironment(Environment):

    def __init__(self, n_jobs=1, data_dir=None,
                 tf_logging_level=tf.logging.ERROR):
        super().__init__(
            ['/cpu:0'],
            n_jobs,
            data_dir=data_dir,
            tf_logging_level=tf_logging_level) class Environment(object):

    def __init__(self, devices=None, n_jobs=None,
                 data_dir=None, tf_logging_level=tf.logging.ERROR):
        self.devices = devices
        self.n_jobs = n_jobs
        if devices and n_jobs and not isinstance(n_jobs, list):
            self.n_jobs = [n_jobs for _ in devices]
        self.data_dir = data_dir or self._init_minos_dir()
        self.tf_logging_level = tf_logging_level

    def _init_minos_dir(self):
        base_dir = path.expanduser('~')
        if not access(base_dir, W_OK):
            base_dir = path.join('/tmp')
        minos_dir = join(base_dir, 'minos')
        if not path.exists(minos_dir):
            makedirs(minos_dir)
        return minos_dir
from examples.ga.dataset import get_reuters_dataset
from minos.experiment.experiment import Experiment, ExperimentParameters,\
    load_experiment_blueprints, ExperimentSettings
from minos.experiment.ga import run_ga_search_experiment
from minos.experiment.training import Training, AccuracyDecreaseStoppingCondition,\
    EpochStoppingCondition
from minos.model.build import ModelBuilder
from minos.model.model import Objective, Optimizer, Metric, Layout
from minos.model.parameter import int_param, string_param, float_param
from minos.tf_utils import cpu_device
from minos.train.utils import CpuEnvironment, Environment
import numpy as np
@author: julien
'''


np.random.seed(1337)
max_words = 1000


def build_layout(input_size, output_size):
    """ Here we define a layout with a predefined architecture.
    We also fix some parameters that we don't want to evolve. Other layer parameters
    will be tested with random values.
    """
    return Layout(
        input_size=input_size,
        output_size=output_size,
        output_activation='softmax',
        block=[
            ('Dense', {'activation': 'relu'}),
            'Dropout',
            ('Dense', {'output_dim': 100})])


def experiment_settings():
    experiment_settings = ExperimentSettings()
    experiment_settings.search['layout'] = False
    experiment_settings.ga['population_size'] = 25
    experiment_settings.ga['generations'] = 50
    experiment_settings.ga['p_offspring'] = 0.75
    experiment_settings.ga['offsprings'] = 2
    experiment_settings.ga['p_mutation'] = 0.5
    experiment_settings.ga['mutation_std'] = 0.25


def custom_experiment_parameters():
    """ Here we define the experiment parameters.
    We are using use_default_values=True, which will initialize
    all the parameters with their default values. These parameters are then fixed
    for the duration of the experiment and won't evolve.
    That means that we need to manually specify which parametres we want to test,
    and the possible values, either intervals or lists of values.

    If we want to test all the parameters and possible values, we can
    set use_default_values to False. In that case, random values will be generated
    and tested during the experiment. We can redefine some parameters if we want to
    fix their values.
    Reference parameters and default values are defined in minos.model.parameters

    We set the rows, blocks and layers parameters to 1 as we have specified a fixed layout.
    We also set the 'layout' search parameter to False to disable the layout search
    """
    experiment_parameters = ExperimentParameters(use_default_values=True)
    experiment_parameters.layout_parameter('blocks', 1)
    experiment_parameters.layout_parameter('blocks', 1)
    experiment_parameters.layout_parameter('layers', 1)
    experiment_parameters.layer_parameter('Dense.output_dim', int_param(10, 500))
    experiment_parameters.layer_parameter('Dense.activation', string_param(['relu', 'tanh']))
    experiment_parameters.layer_parameter('Dropout.p', float_param(0.1, 0.9))
    return experiment_parameters


def accuracy_decrease_stopping_condition():
    """ This stopping condition lets the training continue as long as the
    accuracy improves and stops if it doesn't improve for 5 epochs
    """
    return AccuracyDecreaseStoppingCondition(
        min_epoch=2,
        max_epoch=10,
        noprogress_count=5)


def epoch_stopping_condition():
    """ This stopping condition lets the training run for 10 epochs
    """
    return EpochStoppingCondition(epoch=10)


def search_model(experiment_label, steps, batch_size=32):
    """ This is where we put everythin together.
    We get the dataset, build the Training and Experiment objects, and run the experiment.
    The experiments logs are generated in ~/minos/experiment_label
    We use the CpuEnvironment to have the experiment run on the cpu, with 2 parralel processes.
    We could use GpuEnvironment to use GPUs, and specify which GPUs to use, and how many tasks
    per GPU
    """
    batch_iterator, test_batch_iterator, nb_classes = get_reuters_dataset(batch_size, max_words)
    layout = build_layout(max_words, nb_classes)
    training = Training(
        Objective('categorical_crossentropy'),
        Optimizer(optimizer='Adam'),
        Metric('categorical_accuracy'),
        epoch_stopping_condition(),
        batch_size)
    settings = experiment_settings()
    parameters = custom_experiment_parameters()
    experiment = Experiment(
        experiment_label,
        layout,
        training,
        batch_iterator,
        test_batch_iterator,
        CpuEnvironment(n_jobs=2),
        settings=settings,
        parameters=parameters)
    run_ga_search_experiment(
        experiment,
        resume=False,
        log_level='DEBUG')


def load_best_model(experiment_label, step):
    """ Here we load the blueprints generated during an experiment
    and create the Keras model from the top scoring blueprint
    """
    blueprints = load_experiment_blueprints(
        experiment_label,
        step,
        Environment())
    return ModelBuilder().build(
        blueprints[0],

        cpu_device(),