def relu():

    def compile_fn(di, dh):

        def forward_fn(di, is_training=True):
            return {'out': tf.nn.relu(di['in'])}

        return forward_fn

    return siso_tensorflow_eager_module('ReLU', compile_fn, {}) def batch_normalization():

    def compile_fn(di, dh):
        bn = tf.layers.BatchNormalization(momentum=.9, epsilon=1e-5)

        def forward_fn(di, is_training):
            return {'out': bn(di['in'], training=is_training)}

        return forward_fn

    return siso_tensorflow_eager_module('BatchNormalization', compile_fn, {}) def conv2d(h_num_filters,
           h_filter_width,
           h_stride=1,
           h_dilation_rate=1,
           h_use_bias=True,
           h_padding='SAME'):

    def compile_fn(di, dh):
        conv = tf.layers.Conv2D(dh['num_filters'],
                                dh['filter_width'],
                                dh['stride'],
                                use_bias=dh['use_bias'],
                                dilation_rate=dh['dilation_rate'],
                                padding=dh['padding'])

        def forward_fn(di, is_training=True):
            return {'out': conv(di['in'])}

        return forward_fn

    return siso_tensorflow_eager_module(
        'Conv2D', compile_fn, {
            'num_filters': h_num_filters,
            'filter_width': h_filter_width,
            'stride': h_stride,
            'dilation_rate': h_dilation_rate,
            'use_bias': h_use_bias,
            'padding': h_padding
        }) def separable_conv2d(h_num_filters,
                     h_filter_width,
                     h_stride=1,
                     h_dilation_rate=1,
                     h_depth_multiplier=1,
                     h_use_bias=True,
                     h_padding='SAME'):

    def compile_fn(di, dh):

        conv_op = tf.layers.SeparableConv2D(
            dh['num_filters'],
            dh['filter_width'],
            strides=dh['stride'],
            dilation_rate=dh['dilation_rate'],
            depth_multiplier=dh['depth_multiplier'],
            use_bias=dh['use_bias'],
            padding=dh['padding'])

        def fn(di, is_training=True):
            return {'out': conv_op(di['in'])}

        return fn

    return siso_tensorflow_eager_module(
        'SeparableConv2D', compile_fn, {
            'num_filters': h_num_filters,
            'filter_width': h_filter_width,
            'stride': h_stride,
            'use_bias': h_use_bias,
            'dilation_rate': h_dilation_rate,
            'depth_multiplier': h_depth_multiplier,
            'padding': h_padding
        }) def avg_pool2d(h_kernel_size, h_stride=1, h_padding='SAME'):

    def compile_fn(di, dh):
        pool = tf.layers.AveragePooling2D(dh['kernel_size'],
                                          dh['stride'],
                                          padding=dh['padding'])

        def forward_fn(di, is_training=True):
            return {'out': pool(di['in'])}

        return forward_fn

    return siso_tensorflow_eager_module('MaxPool2D', compile_fn, {
        'kernel_size': h_kernel_size,
        'stride': h_stride,
        'padding': h_padding
    }) def max_pool2d(h_kernel_size, h_stride=1, h_padding='SAME'):

    def compile_fn(di, dh):
        pool = tf.layers.MaxPooling2D(dh['kernel_size'],
                                      dh['stride'],
                                      padding=dh['padding'])

        def forward_fn(di, is_training=True):
            return {'out': pool(di['in'])}

        return forward_fn

    return siso_tensorflow_eager_module('MaxPool2D', compile_fn, {
        'kernel_size': h_kernel_size,
        'stride': h_stride,
        'padding': h_padding
    }) def min_pool2d(h_kernel_size, h_stride=1, h_padding='SAME'):

    def compile_fn(di, dh):
        pool = tf.layers.MaxPooling2D(dh['kernel_size'],
                                      dh['stride'],
                                      padding=dh['padding'])

        def forward_fn(di, is_training=True):
            return {'out': -1 * pool(-1 * di['in'])}

        return forward_fn

    return siso_tensorflow_eager_module('MinPool2D', compile_fn, {
        'kernel_size': h_kernel_size,
        'stride': h_stride,
        'padding': h_padding
    }) def fc_layer(h_num_units):

    def compile_fn(di, dh):
        fc = tf.layers.Dense(dh['num_units'])

        def forward_fn(di, is_training=True):
            return {'out': fc(di['in'])}

        return forward_fn

    return siso_tensorflow_eager_module('FCLayer', compile_fn,
                                        {'num_units': h_num_units}) def global_pool2d():

    def compile_fn(di, dh):

        def forward_fn(di, is_training=True):
            return {'out': tf.reduce_mean(di['in'], [1, 2])}

        return forward_fn

    return siso_tensorflow_eager_module('GlobalAveragePool', compile_fn, {}) def dropout(h_keep_prob):

    def compile_fn(di, dh):

        def forward_fn(di, is_training=True):
            if is_training:
                out = tf.nn.dropout(di['in'], dh['keep_prob'])
            else:
                out = di['in']
            return {'out': out}

        return forward_fn

    return siso_tensorflow_eager_module('Dropout', compile_fn,
                                        {'keep_prob': h_keep_prob}) class Discrete(co.Hyperparameter):
    """List valued hyperparameter.

    This type of hyperparameter has a finite number of possible values.

    Args:
        vs (list[object]): List of possible parameter values that the
            hyperparameter can take.
        scope (deep_architect.core.Scope, optional): The scope in which to register the
            hyperparameter in.
        name (str, optional): Name from which the name of the hyperparameter
            in the scope is derived.
    """

    def __init__(self, vs, scope=None, name=None):
        assert len(vs) > 0
        co.Hyperparameter.__init__(self, scope, name)
        self.vs = vs

    def _check_value(self, val):
        """Checks if the chosen values is in the list of valid values.

        Asserts ``False`` if the value is not in the list.
        """
        if val not in self.vs:
            print(self.get_name())
            print(self.vs)
            print(val)
        assert val in self.vs class Bool(Discrete):

    def __init__(self, scope=None, name=None):
        Discrete.__init__(self, [0, 1], scope, name)
import itertools
import tensorflow as tf
import deep_architect.helpers.tensorflow_eager_support as htfe
import deep_architect.modules as mo
import deep_architect.contrib.misc.search_spaces.tensorflow.cnn2d as cnn2d
from collections import OrderedDict
from deep_architect.contrib.deep_learning_backend.tensorflow_eager_ops import (
    relu, batch_normalization, conv2d, separable_conv2d, avg_pool2d, max_pool2d,
    min_pool2d, fc_layer, global_pool2d, dropout)
from deep_architect.hyperparameters import Discrete as D
from deep_architect.hyperparameters import Bool



def cell(input_fn, node_fn, output_fn, h_connections, num_nodes, channels):

    def substitution_fn(dh):
        num_ins = [
            sum([dh['%d_%d' % (in_id, out_id)]
                 for in_id in range(out_id)])
            for out_id in range(num_nodes + 2)
        ]
        num_outs = [
            sum([
                dh['%d_%d' % (in_id, out_id)]
                for out_id in range(in_id + 1, num_nodes + 2)
            ])
            for in_id in range(num_nodes + 2)
        ]

        for i in range(1, num_nodes + 1):
            if num_outs[i] > 0 and num_ins[i] == 0:
                raise ValueError('Node exists with no path to input')
            if num_ins[i] > 0 and num_outs[i] == 0:
                raise ValueError('Node exists with no path to output')
        if num_ins[-1] == 0:
            raise ValueError('No path exists between input and output')

        if sum(num_ins) > 9:
            raise ValueError('More than 9 edges')

        if dh['%d_%d' % (0, num_nodes + 1)]:
            num_ins[-1] -= 1
        # Compute the number of channels that each vertex outputs
        int_channels = channels // num_ins[-1]
        correction = channels % num_ins[-1]

        vertex_channels = [0] * (num_nodes + 2)
        vertex_channels[-1] = channels
        # Distribute channels as evenly as possible among vertices connected
        # to output vertex
        for in_id in range(1, num_nodes + 1):
            if dh['%d_%d' % (in_id, num_nodes + 1)]:
                vertex_channels[in_id] = int_channels
                if correction > 0:
                    vertex_channels[in_id] += 1
                    correction -= 1
        # For all other vertices, the number of channels they output is the max
        # of the channels outputed by all vertices they flow into
        for in_id in range(num_nodes - 1, 0, -1):
            if not dh['%d_%d' % (in_id, num_nodes + 1)]:
                for out_id in range(in_id + 1, num_nodes + 1):
                    if dh['%d_%d' % (in_id, out_id)]:
                        vertex_channels[in_id] = max(vertex_channels[in_id],
                                                     vertex_channels[out_id])

        nodes = [mo.identity()]
        nodes += [
            node_fn(num_ins[i], i -
                    1, vertex_channels[i]) if num_outs[i] > 0 else None
            for i in range(1, num_nodes + 1)
        ]
        nodes.append(output_fn(num_ins[num_nodes + 1]))
        num_connected = [0] * (num_nodes + 2)
        # Project input vertex to correct dimensions if used
        for out_id in range(1, num_nodes + 1):
            if dh['%d_%d' % (0, out_id)]:
                proj_in, proj_out = input_fn(vertex_channels[out_id])
                nodes[0][1]['out'].connect(proj_in['in'])
                proj_out['out'].connect(
                    nodes[out_id][0]['in' + str(num_connected[out_id])])
                num_connected[out_id] += 1
        for in_id in range(1, num_nodes + 1):
            for out_id in range(in_id + 1, num_nodes + 2):
                if dh['%d_%d' % (in_id, out_id)]:
                    nodes[in_id][1]['out'].connect(
                        nodes[out_id][0]['in' + str(num_connected[out_id])])
                    num_connected[out_id] += 1
        if dh['%d_%d' % (0, num_nodes + 1)]:
            proj_in, proj_out = input_fn(channels)
            add_in, add_out = add(2)
            nodes[0][1]['out'].connect(proj_in['in'])
            proj_out['out'].connect(add_in['in0'])
            nodes[-1][1]['out'].connect(add_in['in1'])
            nodes[-1] = (add_in, add_out)

        return nodes[0][0], nodes[-1][1]

    name_to_hparam = {}
    for ix, hparam in enumerate(itertools.combinations(range(num_nodes + 2),
                                                       2)):
        name_to_hparam['%d_%d' % hparam] = h_connections[ix]
    return mo.substitution_module('NasbenchCell',
                                  substitution_fn,
                                  name_to_hparam, ['in'], ['out'],
                                  scope=None)


def add(num_inputs):

    def compile_fn(di, dh):
        in_channels = [tf.shape(di[inp])[-1] for inp in di]
        final_channels = tf.reduce_min(in_channels)

        def forward_fn(di, is_training=True):
            out = [di[inp][:, :, :, :final_channels] for inp in di]
            return {'out': tf.add_n(out)}

        return forward_fn

    return htfe.TensorflowEagerModule(
        'Add', compile_fn, {}, ['in' + str(i) for i in range(num_inputs)],
        ['out']).get_io()


def intermediate_node_fn(num_inputs, node_id, filters, cell_ops):
    return mo.siso_sequential([
        add(num_inputs),
        mo.siso_or(
            {
                'conv1': lambda: conv2d(D([filters]), D([1])),

                'conv3': lambda: conv2d(D([filters]), D([3])),