<s> import logging <EOL> import operator <EOL> from collections import OrderedDict <EOL> import numpy <EOL> from numpy . testing import assert_raises , assert_equal <EOL> from six . moves import zip , cPickle <EOL> from fuel import config <EOL> from fuel . datasets import IterableDataset , IndexableDataset <EOL> from fuel . schemes import ( ConstantScheme , SequentialScheme , <EOL> SequentialExampleScheme ) <EOL> from fuel . streams import DataStream <EOL> from fuel . transformers import ( <EOL> ExpectsAxisLabels , Transformer , Mapping , SortMapping , ForceFloatX , Filter , <EOL> Cache , Batch , Padding , MultiProcessing , Unpack , Merge , <EOL> SourcewiseTransformer , Flatten , ScaleAndShift , Cast , Rename , FilterSources ) <EOL> from fuel . transformers . defaults import ToBytes <EOL> class FlagDataStream ( DataStream ) : <EOL> close_called = False <EOL> reset_called = False <EOL> next_epoch_called = False <EOL> produces_examples = True <EOL> def close ( self ) : <EOL> self . close_called = True <EOL> super ( FlagDataStream , self ) . close ( ) <EOL> def reset ( self ) : <EOL> self . reset_called = True <EOL> super ( FlagDataStream , self ) . reset ( ) <EOL> def next_epoch ( self ) : <EOL> self . next_epoch_called = True <EOL> super ( FlagDataStream , self ) . next_epoch ( ) <EOL> class TestTransformer ( object ) : <EOL> def setUp ( self ) : <EOL> self . data_stream = FlagDataStream ( IterableDataset ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> ] ) ) <EOL> self . transformer = Transformer ( <EOL> self . data_stream , produces_examples = True ) <EOL> def test_close ( self ) : <EOL> self . transformer . close ( ) <EOL> assert self . data_stream . close_called <EOL> def test_reset ( self ) : <EOL> self . transformer . reset ( ) <EOL> assert self . data_stream . reset_called <EOL> def test_next_epoch ( self ) : <EOL> self . transformer . next_epoch ( ) <EOL> assert self . data_stream . next_epoch_called <EOL> def test_value_error_on_request ( self ) : <EOL> assert_raises ( ValueError , self . transformer . get_data , [ <NUM_LIT:0> , <NUM_LIT:1> ] ) <EOL> def test_batch_to_example_and_vice_versa_not_implemented ( self ) : <EOL> self . transformer . produces_examples = False <EOL> self . transformer . get_epoch_iterator ( ) <EOL> assert_raises ( NotImplementedError , self . transformer . get_data ) <EOL> def test_transform_example_not_implemented_by_default ( self ) : <EOL> assert_raises ( <EOL> NotImplementedError , self . transformer . transform_example , None ) <EOL> def test_transform_batch_not_implemented_by_default ( self ) : <EOL> assert_raises ( <EOL> NotImplementedError , self . transformer . transform_batch , None ) <EOL> class TestMapping ( object ) : <EOL> def setUp ( self ) : <EOL> self . data = [ [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> ] , [ <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:1> ] , [ <NUM_LIT:3> , <NUM_LIT:2> , <NUM_LIT:1> ] ] <EOL> self . data_x = [ [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> ] , [ <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:1> ] , [ <NUM_LIT:3> , <NUM_LIT:2> , <NUM_LIT:1> ] ] <EOL> self . data_y = [ [ <NUM_LIT:6> , <NUM_LIT:5> , <NUM_LIT:4> ] , [ <NUM_LIT:6> , <NUM_LIT:5> , <NUM_LIT:4> ] , [ <NUM_LIT:6> , <NUM_LIT:5> , <NUM_LIT:4> ] ] <EOL> def test_mapping ( self ) : <EOL> stream = DataStream ( IterableDataset ( self . data ) ) <EOL> transformer = Mapping ( stream , lambda d : ( [ <NUM_LIT:2> * i for i in d [ <NUM_LIT:0> ] ] , ) ) <EOL> assert_equal ( list ( transformer . get_epoch_iterator ( ) ) , <EOL> list ( zip ( [ [ <NUM_LIT:2> , <NUM_LIT:4> , <NUM_LIT:6> ] , [ <NUM_LIT:4> , <NUM_LIT:6> , <NUM_LIT:2> ] , [ <NUM_LIT:6> , <NUM_LIT:4> , <NUM_LIT:2> ] ] ) ) ) <EOL> def test_add_sources ( self ) : <EOL> stream = DataStream ( IterableDataset ( self . data ) ) <EOL> transformer = Mapping ( stream , lambda d : ( [ <NUM_LIT:2> * i for i in d [ <NUM_LIT:0> ] ] , ) , <EOL> add_sources = ( '<STR_LIT>' , ) ) <EOL> assert_equal ( transformer . sources , ( '<STR_LIT:data>' , '<STR_LIT>' ) ) <EOL> assert_equal ( list ( transformer . get_epoch_iterator ( ) ) , <EOL> list ( zip ( self . data , [ [ <NUM_LIT:2> , <NUM_LIT:4> , <NUM_LIT:6> ] , [ <NUM_LIT:4> , <NUM_LIT:6> , <NUM_LIT:2> ] , [ <NUM_LIT:6> , <NUM_LIT:4> , <NUM_LIT:2> ] ] ) ) ) <EOL> def test_sort_mapping_trivial_key ( self ) : <EOL> stream = DataStream ( IterableDataset ( self . data ) ) <EOL> transformer = Mapping ( stream , SortMapping ( operator . itemgetter ( <NUM_LIT:0> ) ) ) <EOL> assert_equal ( list ( transformer . get_epoch_iterator ( ) ) , <EOL> list ( zip ( [ [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> ] ] * <NUM_LIT:3> ) ) ) <EOL> def test_sort_mapping_alternate_key ( self ) : <EOL> stream = DataStream ( IterableDataset ( self . data ) ) <EOL> transformer = Mapping ( stream , SortMapping ( lambda x : - x [ <NUM_LIT:0> ] ) ) <EOL> assert_equal ( list ( transformer . get_epoch_iterator ( ) ) , <EOL> list ( zip ( [ [ <NUM_LIT:3> , <NUM_LIT:2> , <NUM_LIT:1> ] ] * <NUM_LIT:3> ) ) ) <EOL> def test_sort_mapping_reverse ( self ) : <EOL> stream = DataStream ( IterableDataset ( self . data ) ) <EOL> transformer = Mapping ( <EOL> stream , SortMapping ( operator . itemgetter ( <NUM_LIT:0> ) , reverse = True ) ) <EOL> assert_equal ( list ( transformer . get_epoch_iterator ( ) ) , <EOL> list ( zip ( [ [ <NUM_LIT:3> , <NUM_LIT:2> , <NUM_LIT:1> ] ] * <NUM_LIT:3> ) ) ) <EOL> def test_mapping_sort_multisource_ndarrays ( self ) : <EOL> data = OrderedDict ( [ ( '<STR_LIT:x>' , numpy . array ( self . data_x ) ) , <EOL> ( '<STR_LIT:y>' , numpy . array ( self . data_y ) ) ] ) <EOL> data_sorted = [ ( numpy . array ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> ] ) , numpy . array ( [ <NUM_LIT:6> , <NUM_LIT:5> , <NUM_LIT:4> ] ) ) , <EOL> ( numpy . array ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> ] ) , numpy . array ( [ <NUM_LIT:4> , <NUM_LIT:6> , <NUM_LIT:5> ] ) ) , <EOL> ( numpy . array ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> ] ) , numpy . array ( [ <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> ] ) ) ] <EOL> stream = DataStream ( IterableDataset ( data ) ) <EOL> transformer = Mapping ( <EOL> stream , mapping = SortMapping ( operator . itemgetter ( <NUM_LIT:0> ) ) ) <EOL> assert_equal ( list ( transformer . get_epoch_iterator ( ) ) , <EOL> data_sorted ) <EOL> def test_mapping_sort_multisource ( self ) : <EOL> data = OrderedDict ( [ ( '<STR_LIT:x>' , self . data_x ) , ( '<STR_LIT:y>' , self . data_y ) ] ) <EOL> data_sorted = [ ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> ] , [ <NUM_LIT:6> , <NUM_LIT:5> , <NUM_LIT:4> ] ) , <EOL> ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> ] , [ <NUM_LIT:4> , <NUM_LIT:6> , <NUM_LIT:5> ] ) , <EOL> ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> ] , [ <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> ] ) ] <EOL> stream = DataStream ( IterableDataset ( data ) ) <EOL> transformer = Mapping ( <EOL> stream , mapping = SortMapping ( operator . itemgetter ( <NUM_LIT:0> ) ) ) <EOL> assert_equal ( list ( transformer . get_epoch_iterator ( ) ) , <EOL> data_sorted ) <EOL> def test_value_error_on_request ( self ) : <EOL> stream = DataStream ( IterableDataset ( self . data ) ) <EOL> transformer = Mapping ( stream , lambda d : ( [ <NUM_LIT:2> * i for i in d [ <NUM_LIT:0> ] ] , ) ) <EOL> assert_raises ( ValueError , transformer . get_data , [ <NUM_LIT:0> , <NUM_LIT:1> ] ) <EOL> class TestSourcewiseTransformer ( object ) : <EOL> def test_transform_source_example_not_implemented ( self ) : <EOL> transformer = SourcewiseTransformer ( <EOL> DataStream ( IterableDataset ( [ <NUM_LIT:1> , <NUM_LIT:2> ] ) ) , True ) <EOL> assert_raises ( <EOL> NotImplementedError , transformer . transform_source_example , <EOL> None , '<STR_LIT:foo>' ) <EOL> def test_transform_source_batch_not_implemented ( self ) : <EOL> transformer = SourcewiseTransformer ( <EOL> DataStream ( IterableDataset ( [ <NUM_LIT:1> , <NUM_LIT:2> ] ) ) , True ) <EOL> assert_raises ( <EOL> NotImplementedError , transformer . transform_source_batch , <EOL> None , '<STR_LIT:foo>' ) <EOL> class TestFlatten ( object ) : <EOL> def setUp ( self ) : <EOL> self . data = OrderedDict ( <EOL> [ ( '<STR_LIT>' , numpy . ones ( ( <NUM_LIT:4> , <NUM_LIT:2> , <NUM_LIT:2> ) ) ) , <EOL> ( '<STR_LIT>' , numpy . array ( [ [ <NUM_LIT:0> ] , [ <NUM_LIT:1> ] , [ <NUM_LIT:0> ] , [ <NUM_LIT:1> ] ] ) ) ] ) <EOL> def test_flatten_batches ( self ) : <EOL> wrapper = Flatten ( <EOL> DataStream ( IndexableDataset ( self . data ) , <EOL> iteration_scheme = SequentialScheme ( <NUM_LIT:4> , <NUM_LIT:2> ) ) , <EOL> which_sources = ( '<STR_LIT>' , ) ) <EOL> assert_equal ( <EOL> list ( wrapper . get_epoch_iterator ( ) ) , <EOL> [ ( numpy . ones ( ( <NUM_LIT:2> , <NUM_LIT:4> ) ) , numpy . array ( [ [ <NUM_LIT:0> ] , [ <NUM_LIT:1> ] ] ) ) , <EOL> ( numpy . ones ( ( <NUM_LIT:2> , <NUM_LIT:4> ) ) , numpy . array ( [ [ <NUM_LIT:0> ] , [ <NUM_LIT:1> ] ] ) ) ] ) <EOL> def test_axis_labels_on_flatten_batches ( self ) : <EOL> wrapper = Flatten ( <EOL> DataStream ( IndexableDataset ( self . data ) , <EOL> iteration_scheme = SequentialScheme ( <NUM_LIT:4> , <NUM_LIT:2> ) , <EOL> axis_labels = { '<STR_LIT>' : ( '<STR_LIT>' , '<STR_LIT:width>' , '<STR_LIT>' ) , <EOL> '<STR_LIT>' : ( '<STR_LIT>' , '<STR_LIT:index>' ) } ) , <EOL> which_sources = ( '<STR_LIT>' , ) ) <EOL> assert_equal ( wrapper . axis_labels , { '<STR_LIT>' : ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> '<STR_LIT>' : ( '<STR_LIT>' , '<STR_LIT:index>' ) } ) <EOL> def test_axis_labels_on_flatten_batches_with_none ( self ) : <EOL> wrapper = Flatten ( <EOL> DataStream ( IndexableDataset ( self . data ) , <EOL> iteration_scheme = SequentialScheme ( <NUM_LIT:4> , <NUM_LIT:2> ) , <EOL> axis_labels = { '<STR_LIT>' : None , <EOL> '<STR_LIT>' : ( '<STR_LIT>' , '<STR_LIT:index>' ) } ) , <EOL> which_sources = ( '<STR_LIT>' , ) ) <EOL> assert_equal ( wrapper . axis_labels , { '<STR_LIT>' : None , <EOL> '<STR_LIT>' : ( '<STR_LIT>' , '<STR_LIT:index>' ) } ) <EOL> def test_flatten_examples ( self ) : <EOL> wrapper = Flatten ( <EOL> DataStream ( IndexableDataset ( self . data ) , <EOL> iteration_scheme = SequentialExampleScheme ( <NUM_LIT:4> ) ) , <EOL> which_sources = ( '<STR_LIT>' , ) ) <EOL> assert_equal ( <EOL> list ( wrapper . get_epoch_iterator ( ) ) , <EOL> [ ( numpy . ones ( <NUM_LIT:4> ) , <NUM_LIT:0> ) , ( numpy . ones ( <NUM_LIT:4> ) , <NUM_LIT:1> ) ] * <NUM_LIT:2> ) <EOL> def test_axis_labels_on_flatten_examples ( self ) : <EOL> wrapper = Flatten ( <EOL> DataStream ( IndexableDataset ( self . data ) , <EOL> iteration_scheme = SequentialExampleScheme ( <NUM_LIT:4> ) , <EOL> axis_labels = { '<STR_LIT>' : ( '<STR_LIT>' , '<STR_LIT:width>' , '<STR_LIT>' ) , <EOL> '<STR_LIT>' : ( '<STR_LIT>' , '<STR_LIT:index>' ) } ) , <EOL> which_sources = ( '<STR_LIT>' , ) ) <EOL> assert_equal ( wrapper . axis_labels , { '<STR_LIT>' : ( '<STR_LIT>' , ) , <EOL> '<STR_LIT>' : ( '<STR_LIT:index>' , ) } ) <EOL> class TestScaleAndShift ( object ) : <EOL> def setUp ( self ) : <EOL> dataset = IterableDataset ( <EOL> OrderedDict ( [ ( '<STR_LIT>' , [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> ] ) , ( '<STR_LIT>' , [ <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:0> ] ) ] ) , <EOL> axis_labels = { '<STR_LIT>' : ( '<STR_LIT>' ) , '<STR_LIT>' : ( '<STR_LIT>' ) } ) <EOL> self . stream = DataStream ( dataset ) <EOL> self . wrapper = ScaleAndShift ( <EOL> self . stream , <NUM_LIT:2> , - <NUM_LIT:1> , which_sources = ( '<STR_LIT>' , ) ) <EOL> def test_scale_and_shift ( self ) : <EOL> assert_equal ( list ( self . wrapper . get_epoch_iterator ( ) ) , <EOL> [ ( <NUM_LIT:1> , - <NUM_LIT:1> ) , ( <NUM_LIT:2> , <NUM_LIT:1> ) , ( <NUM_LIT:3> , - <NUM_LIT:1> ) ] ) <EOL> def test_axis_labels_are_passed_through ( self ) : <EOL> assert_equal ( self . wrapper . axis_labels , self . stream . axis_labels ) <EOL> class TestCast ( object ) : <EOL> def setUp ( self ) : <EOL> dataset = IterableDataset ( <EOL> OrderedDict ( [ <EOL> ( '<STR_LIT>' , numpy . array ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> ] ) . astype ( '<STR_LIT>' ) ) , <EOL> ( '<STR_LIT>' , [ <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:0> ] ) ] ) , <EOL> axis_labels = { '<STR_LIT>' : ( '<STR_LIT>' ) , '<STR_LIT>' : ( '<STR_LIT>' ) } ) <EOL> self . stream = DataStream ( dataset ) <EOL> self . wrapper = Cast ( <EOL> self . stream , '<STR_LIT>' , which_sources = ( '<STR_LIT>' , ) ) <EOL> def test_cast ( self ) : <EOL> assert_equal ( <EOL> list ( self . wrapper . get_epoch_iterator ( ) ) , <EOL> [ ( numpy . array ( <NUM_LIT:1> ) , <NUM_LIT:0> ) , ( numpy . array ( <NUM_LIT:2> ) , <NUM_LIT:1> ) , ( numpy . array ( <NUM_LIT:3> ) , <NUM_LIT:0> ) ] ) <EOL> assert all ( <EOL> f . dtype == '<STR_LIT>' for f , t in self . wrapper . get_epoch_iterator ( ) ) <EOL> def test_axis_labels_are_passed_through ( self ) : <EOL> assert_equal ( self . wrapper . axis_labels , self . stream . axis_labels ) <EOL> class TestForceFloatX ( object ) : <EOL> def setUp ( self ) : <EOL> self . original_floatX = config . floatX <EOL> config . floatX = '<STR_LIT>' <EOL> x = numpy . arange ( <NUM_LIT:6> , dtype = '<STR_LIT>' ) . reshape ( ( <NUM_LIT:3> , <NUM_LIT:2> ) ) <EOL> y = numpy . arange ( <NUM_LIT:3> , dtype = '<STR_LIT>' ) . reshape ( ( <NUM_LIT:3> , <NUM_LIT:1> ) ) <EOL> self . dataset = IterableDataset ( OrderedDict ( [ ( '<STR_LIT:x>' , x ) , ( '<STR_LIT:y>' , y ) ] ) ) <EOL> def tearDown ( self ) : <EOL> config . floatX = self . original_floatX <EOL> def test_force_floatx ( self ) : <EOL> transformer = ForceFloatX ( DataStream ( self . dataset ) ) <EOL> data = next ( transformer . get_epoch_iterator ( ) ) <EOL> assert_equal ( str ( data [ <NUM_LIT:0> ] . dtype ) , config . floatX ) <EOL> assert_equal ( str ( data [ <NUM_LIT:1> ] . dtype ) , '<STR_LIT>' ) <EOL> def test_axis_labels_are_passed_through ( self ) : <EOL> axis_labels = { '<STR_LIT:x>' : ( '<STR_LIT>' , '<STR_LIT>' ) , '<STR_LIT:y>' : ( '<STR_LIT>' , '<STR_LIT:index>' ) } <EOL> self . dataset . axis_labels = axis_labels <EOL> stream = DataStream ( self . dataset ) <EOL> transformer = ForceFloatX ( stream ) <EOL> assert_equal ( transformer . axis_labels , stream . axis_labels ) <EOL> class TestFilter ( object ) : <EOL> def test_filter_examples ( self ) : <EOL> data = [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> ] <EOL> data_filtered = [ <NUM_LIT:1> , <NUM_LIT:3> ] <EOL> stream = DataStream ( IterableDataset ( data ) ) <EOL> wrapper = Filter ( stream , lambda d : d [ <NUM_LIT:0> ] % <NUM_LIT:2> == <NUM_LIT:1> ) <EOL> assert_equal ( list ( wrapper . get_epoch_iterator ( ) ) , <EOL> list ( zip ( data_filtered ) ) ) <EOL> def test_filter_batches ( self ) : <EOL> data = [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> ] <EOL> data_filtered = [ ( [ <NUM_LIT:3> , <NUM_LIT:4> ] , ) ] <EOL> stream = DataStream ( IndexableDataset ( data ) , <EOL> iteration_scheme = SequentialScheme ( <NUM_LIT:4> , <NUM_LIT:2> ) ) <EOL> wrapper = Filter ( stream , lambda d : d [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] % <NUM_LIT:3> == <NUM_LIT:0> ) <EOL> assert_equal ( list ( wrapper . get_epoch_iterator ( ) ) , data_filtered ) <EOL> def test_axis_labels_are_passed_through ( self ) : <EOL> stream = DataStream ( <EOL> IndexableDataset ( <EOL> { '<STR_LIT>' : [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> ] } , <EOL> axis_labels = { '<STR_LIT>' : ( '<STR_LIT>' , ) } ) , <EOL> iteration_scheme = SequentialScheme ( <NUM_LIT:4> , <NUM_LIT:2> ) ) <EOL> wrapper = Filter ( stream , lambda d : d [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] % <NUM_LIT:3> == <NUM_LIT:0> ) <EOL> assert_equal ( wrapper . axis_labels , stream . axis_labels ) <EOL> class TestCache ( object ) : <EOL> def setUp ( self ) : <EOL> self . stream = Batch ( DataStream ( IterableDataset ( range ( <NUM_LIT:100> ) ) ) , <EOL> ConstantScheme ( <NUM_LIT:11> ) ) <EOL> def test_cache_fills_correctly ( self ) : <EOL> cached_stream = Cache ( self . stream , ConstantScheme ( <NUM_LIT:7> ) ) <EOL> epoch = cached_stream . get_epoch_iterator ( ) <EOL> for ( features , ) , cache_size in zip ( epoch , [ <NUM_LIT:4> , <NUM_LIT:8> , <NUM_LIT:1> , <NUM_LIT:5> , <NUM_LIT:9> , <NUM_LIT:2> , <EOL> <NUM_LIT:6> , <NUM_LIT:10> , <NUM_LIT:3> , <NUM_LIT:7> , <NUM_LIT:0> , <NUM_LIT:4> ] ) : <EOL> assert_equal ( len ( cached_stream . cache [ <NUM_LIT:0> ] ) , cache_size ) <EOL> def test_epoch_finishes_correctly ( self ) : <EOL> cached_stream = Cache ( self . stream , ConstantScheme ( <NUM_LIT:7> ) ) <EOL> data = list ( cached_stream . get_epoch_iterator ( ) ) <EOL> assert_equal ( len ( data [ - <NUM_LIT:1> ] [ <NUM_LIT:0> ] ) , <NUM_LIT:100> % <NUM_LIT:7> ) <EOL> assert not cached_stream . cache [ <NUM_LIT:0> ] <EOL> stream = Batch ( DataStream ( IterableDataset ( range ( <NUM_LIT> ) ) ) , <EOL> ConstantScheme ( <NUM_LIT> ) ) <EOL> cached_stream = Cache ( stream , ConstantScheme ( <NUM_LIT:64> ) ) <EOL> data = list ( cached_stream . get_epoch_iterator ( ) ) <EOL> assert_equal ( len ( data [ - <NUM_LIT:1> ] [ <NUM_LIT:0> ] ) , <NUM_LIT> % <NUM_LIT:64> ) <EOL> assert not cached_stream . cache [ <NUM_LIT:0> ] <EOL> def test_epoch_transition ( self ) : <EOL> cached_stream = Cache ( self . stream , ConstantScheme ( <NUM_LIT:7> , times = <NUM_LIT:3> ) ) <EOL> for _ , epoch in zip ( range ( <NUM_LIT:2> ) , cached_stream . iterate_epochs ( ) ) : <EOL> cache_sizes = [ <NUM_LIT:4> , <NUM_LIT:8> , <NUM_LIT:1> ] <EOL> for i , ( features , ) in enumerate ( epoch ) : <EOL> assert_equal ( len ( cached_stream . cache [ <NUM_LIT:0> ] ) , cache_sizes [ i ] ) <EOL> assert_equal ( len ( features ) , <NUM_LIT:7> ) <EOL> assert_equal ( list ( range ( <NUM_LIT:100> ) ) [ i * <NUM_LIT:7> : ( i + <NUM_LIT:1> ) * <NUM_LIT:7> ] , features ) <EOL> assert_equal ( i , <NUM_LIT:2> ) <EOL> def test_value_error_on_non_batchsizescheme ( self ) : <EOL> assert_raises ( ValueError , Cache , self . stream , SequentialScheme ( <NUM_LIT:4> , <NUM_LIT:2> ) ) <EOL> def test_value_error_on_none_request ( self ) : <EOL> cached_stream = Cache ( self . stream , ConstantScheme ( <NUM_LIT:7> ) ) <EOL> cached_stream . get_epoch_iterator ( ) <EOL> assert_raises ( ValueError , cached_stream . get_data , None ) <EOL> def test_axis_labels_passed_on_by_default ( self ) : <EOL> self . stream . axis_labels = { '<STR_LIT>' : ( '<STR_LIT>' , '<STR_LIT:index>' ) } <EOL> cached_stream = Cache ( self . stream , ConstantScheme ( <NUM_LIT:7> ) ) <EOL> assert_equal ( cached_stream . axis_labels , self . stream . axis_labels ) <EOL> class TestBatch ( object ) : <EOL> def test_strictness_0 ( self ) : <EOL> stream = DataStream ( IterableDataset ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> ] ) ) <EOL> transformer = Batch ( stream , ConstantScheme ( <NUM_LIT:2> ) , strictness = <NUM_LIT:0> ) <EOL> assert_equal ( list ( transformer . get_epoch_iterator ( ) ) , <EOL> [ ( numpy . array ( [ <NUM_LIT:1> , <NUM_LIT:2> ] ) , ) , ( numpy . array ( [ <NUM_LIT:3> , <NUM_LIT:4> ] ) , ) , <EOL> ( numpy . array ( [ <NUM_LIT:5> ] ) , ) ] ) <EOL> def test_strictness_1 ( self ) : <EOL> stream = DataStream ( IterableDataset ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> ] ) ) <EOL> transformer = Batch ( stream , ConstantScheme ( <NUM_LIT:2> ) , strictness = <NUM_LIT:1> ) <EOL> assert_equal ( list ( transformer . get_epoch_iterator ( ) ) , <EOL> [ ( numpy . array ( [ <NUM_LIT:1> , <NUM_LIT:2> ] ) , ) , ( numpy . array ( [ <NUM_LIT:3> , <NUM_LIT:4> ] ) , ) ] ) <EOL> def test_strictness_2 ( self ) : <EOL> stream = DataStream ( IterableDataset ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> ] ) ) <EOL> transformer = Batch ( stream , ConstantScheme ( <NUM_LIT:2> ) , strictness = <NUM_LIT:2> ) <EOL> assert_equal ( list ( transformer . get_epoch_iterator ( ) ) , <EOL> [ ( numpy . array ( [ <NUM_LIT:1> , <NUM_LIT:2> ] ) , ) , ( numpy . array ( [ <NUM_LIT:3> , <NUM_LIT:4> ] ) , ) , <EOL> ( numpy . array ( [ <NUM_LIT:5> , <NUM_LIT:6> ] ) , ) ] ) <EOL> def test_strictness_2_error ( self ) : <EOL> stream = DataStream ( IterableDataset ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> ] ) ) <EOL> transformer = Batch ( stream , ConstantScheme ( <NUM_LIT:2> ) , strictness = <NUM_LIT:2> ) <EOL> assert_raises ( ValueError , list , transformer . get_epoch_iterator ( ) ) <EOL> def test_value_error_on_request_none ( self ) : <EOL> stream = DataStream ( IterableDataset ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> ] ) ) <EOL> transformer = Batch ( stream , ConstantScheme ( <NUM_LIT:2> ) ) <EOL> assert_raises ( ValueError , transformer . get_data , None ) <EOL> def test_adds_batch_to_axis_labels ( self ) : <EOL> stream = DataStream ( <EOL> IterableDataset ( <EOL> { '<STR_LIT>' : [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:5> ] } , <EOL> axis_labels = { '<STR_LIT>' : ( '<STR_LIT:index>' , ) } ) ) <EOL> transformer = Batch ( stream , ConstantScheme ( <NUM_LIT:2> ) , strictness = <NUM_LIT:0> ) <EOL> assert_equal ( transformer . axis_labels , { '<STR_LIT>' : ( '<STR_LIT>' , '<STR_LIT:index>' ) } ) <EOL> def test_value_error_on_batch_stream ( self ) : <EOL> stream = DataStream ( IndexableDataset ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> ] ) , <EOL> iteration_scheme = SequentialScheme ( <NUM_LIT:4> , <NUM_LIT:2> ) ) <EOL> assert_raises ( ValueError , Batch , stream , SequentialScheme ( <NUM_LIT:4> , <NUM_LIT:2> ) ) <EOL> def test_value_error_on_example_scheme ( self ) : <EOL> stream = DataStream ( IterableDataset ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> ] ) ) <EOL> assert_raises ( ValueError , Batch , stream , SequentialExampleScheme ( <NUM_LIT:4> ) ) <EOL> class TestUnpack ( object ) : <EOL> def setUp ( self ) : <EOL> data = range ( <NUM_LIT:10> ) <EOL> self . stream = Batch ( <EOL> DataStream ( IterableDataset ( data ) ) , <EOL> iteration_scheme = ConstantScheme ( <NUM_LIT:2> ) ) <EOL> data_np = numpy . arange ( <NUM_LIT:10> ) <EOL> self . stream_np = Batch ( <EOL> DataStream ( IterableDataset ( data_np ) ) , <EOL> iteration_scheme = ConstantScheme ( <NUM_LIT:2> ) ) <EOL> def test_unpack ( self ) : <EOL> wrapper = Unpack ( self . stream ) <EOL> epoch = wrapper . get_epoch_iterator ( ) <EOL> for i , v in enumerate ( epoch ) : <EOL> assert numpy . shape ( v ) [ <NUM_LIT:0> ] == <NUM_LIT:1> <EOL> assert v [ <NUM_LIT:0> ] == i <EOL> def test_unpack_picklable ( self ) : <EOL> wrapper = Unpack ( self . stream_np ) <EOL> epoch = wrapper . get_epoch_iterator ( ) <EOL> cPickle . dumps ( epoch ) <EOL> def test_value_error_on_example_stream ( self ) : <EOL> stream = DataStream ( <EOL> IterableDataset ( <EOL> dict ( features = [ [ <NUM_LIT:1> ] , [ <NUM_LIT:2> , <NUM_LIT:3> ] ] , <EOL> targets = [ [ <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> ] , [ <NUM_LIT:7> ] ] ) ) ) <EOL> assert_raises ( ValueError , Unpack , stream ) <EOL> def test_value_error_on_request ( self ) : <EOL> wrapper = Unpack ( self . stream ) <EOL> assert_raises ( ValueError , wrapper . get_data , [ <NUM_LIT:0> , <NUM_LIT:1> ] ) <EOL> def test_axis_labels_default ( self ) : <EOL> self . stream . axis_labels = { '<STR_LIT>' : ( '<STR_LIT>' , '<STR_LIT:index>' ) } <EOL> wrapper = Unpack ( self . stream ) <EOL> assert_equal ( wrapper . axis_labels , { '<STR_LIT>' : ( '<STR_LIT:index>' , ) } ) <EOL> class TestPadding ( object ) : <EOL> def test_1d_sequences ( self ) : <EOL> stream = Batch ( <EOL> DataStream ( IterableDataset ( [ [ <NUM_LIT:1> ] , [ <NUM_LIT:2> , <NUM_LIT:3> ] , [ ] , [ <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> ] , [ <NUM_LIT:7> ] ] ) ) , <EOL> ConstantScheme ( <NUM_LIT:2> ) ) <EOL> transformer = Padding ( stream ) <EOL> assert_equal ( transformer . sources , ( "<STR_LIT:data>" , "<STR_LIT>" ) ) <EOL> assert_equal ( list ( transformer . get_epoch_iterator ( ) ) , <EOL> [ ( numpy . array ( [ [ <NUM_LIT:1> , <NUM_LIT:0> ] , [ <NUM_LIT:2> , <NUM_LIT:3> ] ] ) , <EOL> numpy . array ( [ [ <NUM_LIT:1> , <NUM_LIT:0> ] , [ <NUM_LIT:1> , <NUM_LIT:1> ] ] ) ) , <EOL> ( numpy . array ( [ [ <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> ] , [ <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> ] ] ) , <EOL> numpy . array ( [ [ <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> ] , [ <NUM_LIT:1> , <NUM_LIT:1> , <NUM_LIT:1> ] ] ) ) , <EOL> ( numpy . array ( [ [ <NUM_LIT:7> ] ] ) , numpy . array ( [ [ <NUM_LIT:1> ] ] ) ) ] ) <EOL> def test_2d_sequences ( self ) : <EOL> stream = Batch ( <EOL> DataStream ( <EOL> IterableDataset ( [ numpy . ones ( ( <NUM_LIT:3> , <NUM_LIT:4> ) ) , <NUM_LIT:2> * numpy . ones ( ( <NUM_LIT:2> , <NUM_LIT:4> ) ) ] ) ) , <EOL> ConstantScheme ( <NUM_LIT:2> ) ) <EOL> it = Padding ( stream ) . get_epoch_iterator ( ) <EOL> data , mask = next ( it ) <EOL> assert data . shape == ( <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> ) <EOL> assert ( data [ <NUM_LIT:0> , : , : ] == <NUM_LIT:1> ) . all ( ) <EOL> assert ( data [ <NUM_LIT:1> , : <NUM_LIT:2> , : ] == <NUM_LIT:2> ) . all ( ) <EOL> assert ( mask == numpy . array ( [ [ <NUM_LIT:1> , <NUM_LIT:1> , <NUM_LIT:1> ] , [ <NUM_LIT:1> , <NUM_LIT:1> , <NUM_LIT:0> ] ] ) ) . all ( ) <EOL> def test_2d_sequences_error_on_unequal_shapes ( self ) : <EOL> stream = Batch ( <EOL> DataStream ( <EOL> IterableDataset ( [ numpy . ones ( ( <NUM_LIT:3> , <NUM_LIT:4> ) ) , <NUM_LIT:2> * numpy . ones ( ( <NUM_LIT:2> , <NUM_LIT:3> ) ) ] ) ) , <EOL> ConstantScheme ( <NUM_LIT:2> ) ) <EOL> assert_raises ( ValueError , next , Padding ( stream ) . get_epoch_iterator ( ) ) <EOL> def test_two_sources ( self ) : <EOL> transformer = Padding ( Batch ( <EOL> DataStream ( <EOL> IterableDataset ( <EOL> dict ( features = [ [ <NUM_LIT:1> ] , [ <NUM_LIT:2> , <NUM_LIT:3> ] ] , targets = [ [ <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> ] , [ <NUM_LIT:7> ] ] ) ) ) , <EOL> ConstantScheme ( <NUM_LIT:2> ) ) ) <EOL> assert len ( next ( transformer . get_epoch_iterator ( ) ) ) == <NUM_LIT:4> <EOL> def test_mask_dtype ( self ) : <EOL> transformer = Padding ( Batch ( <EOL> DataStream ( <EOL> IterableDataset ( <EOL> dict ( features = [ [ <NUM_LIT:1> ] , [ <NUM_LIT:2> , <NUM_LIT:3> ] ] , targets = [ [ <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> ] , [ <NUM_LIT:7> ] ] ) ) ) , <EOL> ConstantScheme ( <NUM_LIT:2> ) ) , <EOL> mask_dtype = '<STR_LIT>' ) <EOL> assert_equal ( <EOL> str ( next ( transformer . get_epoch_iterator ( ) ) [ <NUM_LIT:1> ] . dtype ) , '<STR_LIT>' ) <EOL> def test_mask_sources ( self ) : <EOL> transformer = Padding ( Batch ( <EOL> DataStream ( <EOL> IterableDataset ( <EOL> OrderedDict ( [ <EOL> ( '<STR_LIT>' , [ [ <NUM_LIT:1> ] , [ <NUM_LIT:2> , <NUM_LIT:3> ] ] ) , <EOL> ( '<STR_LIT>' , [ [ <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> ] , [ <NUM_LIT:7> ] ] ) ] ) ) ) , <EOL> ConstantScheme ( <NUM_LIT:2> ) ) , <EOL> mask_sources = ( '<STR_LIT>' , ) ) <EOL> assert_equal ( len ( next ( transformer . get_epoch_iterator ( ) ) ) , <NUM_LIT:3> ) <EOL> def test_value_error_on_request ( self ) : <EOL> transformer = Padding ( Batch ( <EOL> DataStream ( <EOL> IterableDataset ( <EOL> dict ( features = [ [ <NUM_LIT:1> ] , [ <NUM_LIT:2> , <NUM_LIT:3> ] ] , targets = [ [ <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> ] , [ <NUM_LIT:7> ] ] ) ) ) , <EOL> ConstantScheme ( <NUM_LIT:2> ) ) ) <EOL> assert_raises ( ValueError , transformer . get_data , [ <NUM_LIT:0> , <NUM_LIT:1> ] ) <EOL> def test_value_error_on_example_stream ( self ) : <EOL> stream = DataStream ( <EOL> IterableDataset ( <EOL> dict ( features = [ [ <NUM_LIT:1> ] , [ <NUM_LIT:2> , <NUM_LIT:3> ] ] , targets = [ [ <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> ] , [ <NUM_LIT:7> ] ] ) ) ) <EOL> assert_raises ( ValueError , Padding , stream ) <EOL> class TestMerge ( object ) : <EOL> def setUp ( self ) : <EOL> self . streams = ( <EOL> DataStream ( IterableDataset ( [ '<STR_LIT>' ] ) ) , <EOL> DataStream ( IterableDataset ( [ '<STR_LIT>' ] ) ) ) <EOL> self . batch_streams = ( <EOL> Batch ( DataStream ( IterableDataset ( [ '<STR_LIT>' , '<STR_LIT>' ] ) ) , <EOL> iteration_scheme = ConstantScheme ( <NUM_LIT:2> ) ) , <EOL> Batch ( DataStream ( IterableDataset ( [ '<STR_LIT>' , '<STR_LIT>' ] ) ) , <EOL> iteration_scheme = ConstantScheme ( <NUM_LIT:2> ) ) ) <EOL> self . transformer = Merge ( <EOL> self . streams , ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> self . batch_transformer = Merge ( <EOL> self . batch_streams , ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> def test_sources ( self ) : <EOL> assert_equal ( self . transformer . sources , ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> def test_merge ( self ) : <EOL> it = self . transformer . get_epoch_iterator ( ) <EOL> assert_equal ( next ( it ) , ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> assert_raises ( StopIteration , next , it ) <EOL> it = self . transformer . get_epoch_iterator ( ) <EOL> assert_equal ( next ( it ) , ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> assert_raises ( StopIteration , next , it ) <EOL> def test_batch_merge ( self ) : <EOL> it = self . batch_transformer . get_epoch_iterator ( ) <EOL> assert_equal ( next ( it ) , <EOL> ( ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) ) ) <EOL> assert_raises ( StopIteration , next , it ) <EOL> it = self . batch_transformer . get_epoch_iterator ( ) <EOL> assert_equal ( next ( it ) , <EOL> ( ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) ) ) <EOL> assert_raises ( StopIteration , next , it ) <EOL> def test_merge_batch_streams ( self ) : <EOL> it = self . transformer . get_epoch_iterator ( ) <EOL> assert_equal ( next ( it ) , ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> assert_raises ( StopIteration , next , it ) <EOL> it = self . transformer . get_epoch_iterator ( ) <EOL> assert_equal ( next ( it ) , ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> assert_raises ( StopIteration , next , it ) <EOL> def test_as_dict ( self ) : <EOL> assert_equal ( <EOL> next ( self . transformer . get_epoch_iterator ( as_dict = True ) ) , <EOL> ( { '<STR_LIT>' : '<STR_LIT>' , '<STR_LIT>' : '<STR_LIT>' } ) ) <EOL> def test_error_on_wrong_number_of_sources ( self ) : <EOL> assert_raises ( ValueError , Merge , self . streams , ( '<STR_LIT>' , ) ) <EOL> def test_value_error_on_different_stream_output_type ( self ) : <EOL> spanish_stream = DataStream ( IndexableDataset ( [ '<STR_LIT>' ] ) , <EOL> iteration_scheme = SequentialScheme ( <NUM_LIT:2> , <NUM_LIT:2> ) ) <EOL> assert_raises ( ValueError , Merge , self . streams + ( spanish_stream , ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> def test_close_calls_close_on_all_streams ( self ) : <EOL> streams = [ FlagDataStream ( IterableDataset ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> ] ) ) , <EOL> FlagDataStream ( IterableDataset ( [ <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> ] ) ) , <EOL> FlagDataStream ( IterableDataset ( [ <NUM_LIT:7> , <NUM_LIT:8> , <NUM_LIT:9> ] ) ) ] <EOL> transformer = Merge ( streams , ( '<STR_LIT:1>' , '<STR_LIT:2>' , '<STR_LIT:3>' ) ) <EOL> transformer . close ( ) <EOL> assert all ( stream . close_called for stream in streams ) <EOL> def test_next_epoch_calls_next_epoch_on_all_streams ( self ) : <EOL> streams = [ FlagDataStream ( IterableDataset ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> ] ) ) , <EOL> FlagDataStream ( IterableDataset ( [ <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> ] ) ) , <EOL> FlagDataStream ( IterableDataset ( [ <NUM_LIT:7> , <NUM_LIT:8> , <NUM_LIT:9> ] ) ) ] <EOL> transformer = Merge ( streams , ( '<STR_LIT:1>' , '<STR_LIT:2>' , '<STR_LIT:3>' ) ) <EOL> transformer . next_epoch ( ) <EOL> assert all ( stream . next_epoch_called for stream in streams ) <EOL> def test_reset_calls_reset_on_all_streams ( self ) : <EOL> streams = [ FlagDataStream ( IterableDataset ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> ] ) ) , <EOL> FlagDataStream ( IterableDataset ( [ <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT:6> ] ) ) , <EOL> FlagDataStream ( IterableDataset ( [ <NUM_LIT:7> , <NUM_LIT:8> , <NUM_LIT:9> ] ) ) ] <EOL> transformer = Merge ( streams , ( '<STR_LIT:1>' , '<STR_LIT:2>' , '<STR_LIT:3>' ) ) <EOL> transformer . reset ( ) <EOL> assert all ( stream . reset_called for stream in streams ) <EOL> class TestMultiprocessing ( object ) : <EOL> def setUp ( self ) : <EOL> stream = DataStream ( IterableDataset ( range ( <NUM_LIT:100> ) ) ) <EOL> self . transformer = Mapping ( stream , lambda x : ( x [ <NUM_LIT:0> ] + <NUM_LIT:1> , ) ) <EOL> def test_multiprocessing ( self ) : <EOL> background = MultiProcessing ( self . transformer ) <EOL> assert_equal ( list ( background . get_epoch_iterator ( ) ) , <EOL> list ( zip ( range ( <NUM_LIT:1> , <NUM_LIT> ) ) ) ) <EOL> def test_value_error_on_request ( self ) : <EOL> background = MultiProcessing ( self . transformer ) <EOL> assert_raises ( ValueError , background . get_data , [ <NUM_LIT:0> , <NUM_LIT:1> ] ) <EOL> def test_axis_labels_passed_on_by_default ( self ) : <EOL> self . transformer . axis_labels = { '<STR_LIT>' : ( '<STR_LIT>' , '<STR_LIT:index>' ) } <EOL> background = MultiProcessing ( self . transformer ) <EOL> assert_equal ( background . axis_labels , self . transformer . axis_labels ) <EOL> class TestRename ( object ) : <EOL> def setUp ( self ) : <EOL> self . stream = DataStream ( <EOL> IndexableDataset ( <EOL> OrderedDict ( [ ( '<STR_LIT:X>' , numpy . ones ( ( <NUM_LIT:4> , <NUM_LIT:2> , <NUM_LIT:2> ) ) ) , <EOL> ( '<STR_LIT:y>' , numpy . array ( [ <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:0> , <NUM_LIT:1> ] ) ) ] ) , <EOL> axis_labels = { '<STR_LIT:X>' : ( '<STR_LIT>' , '<STR_LIT:width>' , '<STR_LIT>' ) , <EOL> '<STR_LIT:y>' : ( '<STR_LIT>' , ) } ) , <EOL> iteration_scheme = SequentialScheme ( <NUM_LIT:4> , <NUM_LIT:2> ) ) <EOL> self . transformer = Rename ( <EOL> self . stream , { '<STR_LIT:X>' : '<STR_LIT>' , '<STR_LIT:y>' : '<STR_LIT>' } ) <EOL> def test_renames_sources ( self ) : <EOL> assert_equal ( self . transformer . sources , ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> def test_leaves_data_unchanged ( self ) : <EOL> assert_equal ( <EOL> list ( self . transformer . get_epoch_iterator ( ) ) , <EOL> [ ( numpy . ones ( ( <NUM_LIT:2> , <NUM_LIT:2> , <NUM_LIT:2> ) ) , numpy . array ( [ <NUM_LIT:0> , <NUM_LIT:1> ] ) ) , <EOL> ( numpy . ones ( ( <NUM_LIT:2> , <NUM_LIT:2> , <NUM_LIT:2> ) ) , numpy . array ( [ <NUM_LIT:0> , <NUM_LIT:1> ] ) ) ] ) <EOL> def test_raises_error_on_nonexistent_source_name ( self ) : <EOL> assert_raises ( KeyError , Rename , self . stream , { '<STR_LIT>' : '<STR_LIT>' } ) <EOL> def test_renames_axis_labels ( self ) : <EOL> assert_equal ( self . transformer . axis_labels , <EOL> { '<STR_LIT>' : ( '<STR_LIT>' , '<STR_LIT:width>' , '<STR_LIT>' ) , <EOL> '<STR_LIT>' : ( '<STR_LIT>' , ) } ) <EOL> class TestFilterSources ( object ) : <EOL> def setUp ( self ) : <EOL> self . stream = DataStream ( <EOL> IndexableDataset ( <EOL> OrderedDict ( [ ( '<STR_LIT>' , numpy . ones ( ( <NUM_LIT:4> , <NUM_LIT:2> , <NUM_LIT:2> ) ) ) , <EOL> ( '<STR_LIT>' , numpy . array ( [ <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:0> , <NUM_LIT:1> ] ) ) ] ) , <EOL> axis_labels = { '<STR_LIT>' : ( '<STR_LIT>' , '<STR_LIT:width>' , '<STR_LIT>' ) , <EOL> '<STR_LIT>' : ( '<STR_LIT>' , ) } ) , <EOL> iteration_scheme = SequentialScheme ( <NUM_LIT:4> , <NUM_LIT:2> ) ) <EOL> def test_works_on_sourcessubset ( self ) : <EOL> transformer = FilterSources ( self . stream , sources = ( "<STR_LIT>" , ) ) <EOL> assert_equal ( transformer . sources , ( '<STR_LIT>' , ) ) <EOL> assert_equal ( list ( transformer . get_epoch_iterator ( ) ) , <EOL> [ ( numpy . ones ( ( <NUM_LIT:2> , <NUM_LIT:2> , <NUM_LIT:2> ) ) , ) , ( numpy . ones ( ( <NUM_LIT:2> , <NUM_LIT:2> , <NUM_LIT:2> ) ) , ) ] ) <EOL> def test_works_on_all_sources ( self ) : <EOL> transformer = FilterSources ( <EOL> self . stream , sources = ( "<STR_LIT>" , "<STR_LIT>" ) ) <EOL> assert_equal ( transformer . sources , ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> assert_equal ( list ( transformer . get_epoch_iterator ( ) ) , <EOL> [ ( numpy . ones ( ( <NUM_LIT:2> , <NUM_LIT:2> , <NUM_LIT:2> ) ) , numpy . array ( [ <NUM_LIT:0> , <NUM_LIT:1> ] ) ) , <EOL> ( numpy . ones ( ( <NUM_LIT:2> , <NUM_LIT:2> , <NUM_LIT:2> ) ) , numpy . array ( [ <NUM_LIT:0> , <NUM_LIT:1> ] ) ) ] ) <EOL> def test_works_on_unsorted_sources ( self ) : <EOL> transformer = FilterSources ( <EOL> self . stream , sources = ( "<STR_LIT>" , "<STR_LIT>" ) ) <EOL> assert_equal ( transformer . sources , ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> def test_raises_value_error_on_nonexistent_sources ( self ) : <EOL> assert_raises ( <EOL> ValueError , FilterSources , self . stream , [ '<STR_LIT:error>' , '<STR_LIT>' ] ) <EOL> def test_filters_axis_labels ( self ) : <EOL> transformer = FilterSources ( self . stream , sources = ( "<STR_LIT>" , ) ) <EOL> assert_equal ( transformer . axis_labels , <EOL> { '<STR_LIT>' : ( '<STR_LIT>' , '<STR_LIT:width>' , '<STR_LIT>' ) } ) <EOL> class VerifyWarningHandler ( logging . Handler ) : <EOL> def __init__ ( self , * args , ** kwargs ) : <EOL> self . records = [ ] <EOL> super ( VerifyWarningHandler , self ) . __init__ ( * args , ** kwargs ) <EOL> def handle ( self , record ) : <EOL> self . records . append ( record ) <EOL> class TestExpectsAxisLabels ( object ) : <EOL> def setUp ( self ) : <EOL> self . obj = ExpectsAxisLabels ( ) <EOL> self . handler = VerifyWarningHandler ( ) <EOL> logging . getLogger ( ) . addHandler ( self . handler ) <EOL> def tearDown ( self ) : <EOL> root_logger = logging . getLogger ( ) <EOL> for handler in root_logger . handlers : <EOL> if isinstance ( handler , VerifyWarningHandler ) : <EOL> root_logger . handlers . remove ( handler ) <EOL> def test_warning ( self ) : <EOL> self . obj . verify_axis_labels ( ( '<STR_LIT:a>' , '<STR_LIT:b>' , '<STR_LIT:c>' ) , None , '<STR_LIT:foo>' ) <EOL> assert len ( self . handler . records ) == <NUM_LIT:1> <EOL> assert self . handler . records [ <NUM_LIT:0> ] . levelno == logging . WARNING <EOL> def test_exception ( self ) : <EOL> assert_raises ( ValueError , self . obj . verify_axis_labels , ( '<STR_LIT:a>' , '<STR_LIT:b>' , '<STR_LIT:c>' ) , <EOL> ( '<STR_LIT:b>' , '<STR_LIT:c>' , '<STR_LIT:d>' ) , '<STR_LIT:foo>' ) <EOL> class TestToBytes ( object ) : <EOL> def setUp ( self ) : <EOL> self . string_data = [ b'<STR_LIT>' , b'<STR_LIT>' ] <EOL> self . dataset = IndexableDataset ( <EOL> indexables = { '<STR_LIT>' : [ numpy . fromstring ( s , dtype = '<STR_LIT>' ) <EOL> for s in self . string_data ] } , <EOL> axis_labels = { '<STR_LIT>' : ( '<STR_LIT>' , '<STR_LIT>' ) } ) <EOL> def test_examplewise ( self ) : <EOL> stream = DataStream ( <EOL> dataset = self . dataset , iteration_scheme = SequentialExampleScheme ( <NUM_LIT:2> ) ) <EOL> decoded_stream = ToBytes ( stream ) <EOL> assert_equal ( self . string_data , <EOL> [ s for s , in decoded_stream . get_epoch_iterator ( ) ] ) <EOL> def test_batchwise ( self ) : <EOL> stream = DataStream ( <EOL> dataset = self . dataset , iteration_scheme = SequentialScheme ( <NUM_LIT:2> , <NUM_LIT:2> ) ) <EOL> decoded_stream = ToBytes ( stream ) <EOL> assert_equal ( [ self . string_data ] , <EOL> [ s for s , in decoded_stream . get_epoch_iterator ( ) ] ) </s>
