<filename>tests/chainer_tests/links_tests/normalization_tests/test_layer_normalization.py<gh_stars>0
import unittest

import numpy

import chainer
from chainer import cuda
from chainer import gradient_check
from chainer import links
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition
from chainer.utils import type_check


def _create_ln(*args, **kwargs):
    flag = chainer.disable_experimental_feature_warning
    chainer.disable_experimental_feature_warning = True
    try:
        return links.LayerNormalization(*args, **kwargs)
    finally:
        chainer.disable_experimental_feature_warning = flag


@testing.parameterize(*(testing.product({
    'batchsize': [1, 5],
    'size': [10, 20],
    'dtype': [numpy.float32],
})))
class LayerNormalizationTest(unittest.TestCase):

    def setUp(self):
        self.link = _create_ln()
        self.link.cleargrads()

        self.shape = (self.batchsize, self.size)
        self.x = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)
        self.gy = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)

        self.check_forward_optionss = {'atol': 1e-4, 'rtol': 1e-3}
        self.check_backward_optionss = {'atol': 1e-4, 'rtol': 1e-3}
        if self.dtype == numpy.float16:
            self.check_forward_optionss = {'atol': 1e-3, 'rtol': 1e-2}
            self.check_backward_optionss = {'atol': 5e-1, 'rtol': 1e-1}

    def check_forward(self, x_data):
        y = self.link(x_data)
        self.assertEqual(y.data.dtype, self.dtype)

        unbatched_concat_y = chainer.functions.concat(
            [self.link(one_x[None, ]) for one_x in x_data], axis=0)

        testing.assert_allclose(
            y.data, unbatched_concat_y.data, **self.check_forward_optionss)

    @condition.retry(3)
    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.link.to_gpu()
        self.check_forward(cuda.to_gpu(self.x))

    @attr.cudnn
    def test_forward_gpu_without_cudnn(self):
        self.link.use_cudnn = False
        self.test_forward_gpu()

    @attr.multi_gpu(2)
    @condition.retry(3)
    def test_forward_multi_gpu(self):
        with cuda.get_device_from_id(1):
            self.link.to_gpu()
            x = cuda.to_gpu(self.x)
        with cuda.get_device_from_id(0):
            self.check_forward(x)

    def check_backward(self, x_data, y_grad):
        gradient_check.check_backward(
            self.link, x_data, y_grad,
            (self.link.gamma, self.link.beta),
            eps=1e-2, **self.check_backward_optionss)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.link(numpy.zeros(self.shape, dtype='f'))
        self.check_backward(self.x, self.gy)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.link.to_gpu()
        self.link(cuda.cupy.zeros(self.shape, dtype='f'))
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

    @attr.cudnn
    def test_backward_gpu_without_cudnn(self):
        self.link.use_cudnn = False
        self.link(numpy.zeros(self.shape, dtype='f'))
        self.test_backward_gpu()


@testing.parameterize(*testing.product({
    'size': [3, 50],
}))
class TestInitialize(unittest.TestCase):

    def setUp(self):
        self.initial_gamma = numpy.random.uniform(-1, 1, self.size)
        self.initial_gamma = self.initial_gamma.astype(numpy.float32)
        self.initial_beta = numpy.random.uniform(-1, 1, self.size)
        self.initial_beta = self.initial_beta.astype(numpy.float32)
        self.link = _create_ln(
            initial_gamma=self.initial_gamma,
            initial_beta=self.initial_beta)
        self.shape = (1, self.size)

    @condition.retry(3)
    def test_initialize_cpu(self):
        self.link(numpy.zeros(self.shape, dtype='f'))
        testing.assert_allclose(self.initial_gamma, self.link.gamma.data)
        testing.assert_allclose(self.initial_beta, self.link.beta.data)

    @attr.gpu
    @condition.retry(3)
    def test_initialize_gpu(self):
        self.link.to_gpu()
        self.link(cuda.cupy.zeros(self.shape, dtype='f'))
        testing.assert_allclose(self.initial_gamma, self.link.gamma.data)
        testing.assert_allclose(self.initial_beta, self.link.beta.data)


class TestDefaultInitializer(unittest.TestCase):

    def setUp(self):
        self.size = 3
        self.link = _create_ln()
        self.shape = (1, self.size)

    def test_initialize_cpu(self):
        self.link(numpy.zeros(self.shape, dtype='f'))
        testing.assert_allclose(numpy.ones(self.size), self.link.gamma.data)
        testing.assert_allclose(
            numpy.zeros(self.size), self.link.beta.data)

    @attr.gpu
    def test_initialize_gpu(self):
        self.link.to_gpu()
        self.link(cuda.cupy.zeros(self.shape, dtype='f'))
        testing.assert_allclose(numpy.ones(self.size), self.link.gamma.data)
        testing.assert_allclose(
            numpy.zeros(self.size), self.link.beta.data)


class TestEmptyBatchInitialize(unittest.TestCase):

    def setUp(self):
        self.link = _create_ln()
        self.shape = (0, 3)
        self.x = numpy.random.uniform(-1, 1, self.shape).astype(numpy.float32)

    def test_empty_batch_dim(self):
        y = self.link(chainer.Variable(self.x))
        assert y.shape == self.shape


@testing.parameterize(*testing.product({
    'shape': [(2, 4, 3), (2, 5, 3, 4)],
}))
class TestInvalidInput(unittest.TestCase):

    def setUp(self):
        self.link = _create_ln()

    def test_invalid_shape_cpu(self):
        with self.assertRaises(type_check.InvalidType):
            self.link(chainer.Variable(numpy.zeros(self.shape, dtype='f')))

    @attr.gpu
    def test_invalid_shape_gpu(self):
        self.link.to_gpu()
        with self.assertRaises(type_check.InvalidType):
            self.link(chainer.Variable(cuda.cupy.zeros(self.shape, dtype='f')))


class TestInvalidInitialize(unittest.TestCase):

    def test_invalid_type(self):
        with self.assertRaises(TypeError):
            self.link = _create_ln(None, 1e-6, {})
            self.link(chainer.Variable(numpy.zeros((1, 5), dtype='f')))


testing.run_module(__name__, __file__)

<filename>tests/chainer_tests/links_tests/normalization_tests/test_layer_normalization.py<gh_stars>0
import unittest

import numpy

import chainer
from chainer import cuda
from chainer import gradient_check
from chainer import links
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition
from chainer.utils import type_check


def _create_ln(*args, **kwargs):
    flag = chainer.disable_experimental_feature_warning
    chainer.disable_experimental_feature_warning = True
    try:
        return links.LayerNormalization(*args, **kwargs)
    finally:
        chainer.disable_experimental_feature_warning = flag


@testing.parameterize(*(testing.product({
    'batchsize': [1, 5],
    'size': [10, 20],
    'dtype': [numpy.float32],
})))
class LayerNormalizationTest(unittest.TestCase):

    def setUp(self):
        self.link = _create_ln()
        self.link.cleargrads()

        self.shape = (self.batchsize, self.size)
        self.x = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)
        self.gy = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)

        self.check_forward_optionss = {'atol': 1e-4, 'rtol': 1e-3}
        self.check_backward_optionss = {'atol': 1e-4, 'rtol': 1e-3}
        if self.dtype == numpy.float16:
            self.check_forward_optionss = {'atol': 1e-3, 'rtol': 1e-2}
            self.check_backward_optionss = {'atol': 5e-1, 'rtol': 1e-1}

    def check_forward(self, x_data):
        y = self.link(x_data)
        self.assertEqual(y.data.dtype, self.dtype)

        unbatched_concat_y = chainer.functions.concat(
            [self.link(one_x[None, ]) for one_x in x_data], axis=0)

        testing.assert_allclose(
            y.data, unbatched_concat_y.data, **self.check_forward_optionss)

    @condition.retry(3)
    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.link.to_gpu()
        self.check_forward(cuda.to_gpu(self.x))

    @attr.cudnn
    def test_forward_gpu_without_cudnn(self):
        self.link.use_cudnn = False
        self.test_forward_gpu()

    @attr.multi_gpu(2)
    @condition.retry(3)
    def test_forward_multi_gpu(self):
        with cuda.get_device_from_id(1):
            self.link.to_gpu()
            x = cuda.to_gpu(self.x)
        with cuda.get_device_from_id(0):
            self.check_forward(x)

    def check_backward(self, x_data, y_grad):
        gradient_check.check_backward(
            self.link, x_data, y_grad,
            (self.link.gamma, self.link.beta),
            eps=1e-2, **self.check_backward_optionss)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.link(numpy.zeros(self.shape, dtype='f'))
        self.check_backward(self.x, self.gy)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.link.to_gpu()
        self.link(cuda.cupy.zeros(self.shape, dtype='f'))
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

    @attr.cudnn
    def test_backward_gpu_without_cudnn(self):
        self.link.use_cudnn = False
        self.link(numpy.zeros(self.shape, dtype='f'))
        self.test_backward_gpu()


@testing.parameterize(*testing.product({
    'size': [3, 50],
}))
class TestInitialize(unittest.TestCase):

    def setUp(self):
        self.initial_gamma = numpy.random.uniform(-1, 1, self.size)
        self.initial_gamma = self.initial_gamma.astype(numpy.float32)
        self.initial_beta = numpy.random.uniform(-1, 1, self.size)
        self.initial_beta = self.initial_beta.astype(numpy.float32)
        self.link = _create_ln(
            initial_gamma=self.initial_gamma,
            initial_beta=self.initial_beta)
        self.shape = (1, self.size)

    @condition.retry(3)
    def test_initialize_cpu(self):
        self.link(numpy.zeros(self.shape, dtype='f'))
        testing.assert_allclose(self.initial_gamma, self.link.gamma.data)
        testing.assert_allclose(self.initial_beta, self.link.beta.data)

    @attr.gpu
    @condition.retry(3)
    def test_initialize_gpu(self):
        self.link.to_gpu()
        self.link(cuda.cupy.zeros(self.shape, dtype='f'))
        testing.assert_allclose(self.initial_gamma, self.link.gamma.data)
        testing.assert_allclose(self.initial_beta, self.link.beta.data)


class TestDefaultInitializer(unittest.TestCase):

    def setUp(self):
        self.size = 3
        self.link = _create_ln()
        self.shape = (1, self.size)

    def test_initialize_cpu(self):
        self.link(numpy.zeros(self.shape, dtype='f'))
        testing.assert_allclose(numpy.ones(self.size), self.link.gamma.data)
        testing.assert_allclose(
            numpy.zeros(self.size), self.link.beta.data)

    @attr.gpu
    def test_initialize_gpu(self):
        self.link.to_gpu()
        self.link(cuda.cupy.zeros(self.shape, dtype='f'))
        testing.assert_allclose(numpy.ones(self.size), self.link.gamma.data)
        testing.assert_allclose(
            numpy.zeros(self.size), self.link.beta.data)


class TestEmptyBatchInitialize(unittest.TestCase):

    def setUp(self):
        self.link = _create_ln()
        self.shape = (0, 3)
        self.x = numpy.random.uniform(-1, 1, self.shape).astype(numpy.float32)

    def test_empty_batch_dim(self):
        y = self.link(chainer.Variable(self.x))
        assert y.shape == self.shape


@testing.parameterize(*testing.product({
    'shape': [(2, 4, 3), (2, 5, 3, 4)],
}))
class TestInvalidInput(unittest.TestCase):

    def setUp(self):
        self.link = _create_ln()

    def test_invalid_shape_cpu(self):
        with self.assertRaises(type_check.InvalidType):
            self.link(chainer.Variable(numpy.zeros(self.shape, dtype='f')))

    @attr.gpu
    def test_invalid_shape_gpu(self):
        self.link.to_gpu()
        with self.assertRaises(type_check.InvalidType):
            self.link(chainer.Variable(cuda.cupy.zeros(self.shape, dtype='f')))


class TestInvalidInitialize(unittest.TestCase):

    def test_invalid_type(self):
        with self.assertRaises(TypeError):
            self.link = _create_ln(None, 1e-6, {})
            self.link(chainer.Variable(numpy.zeros((1, 5), dtype='f')))


testing.run_module(__name__, __file__)
