<s> import numpy as np <EOL> import theano <EOL> import theano . tensor as T <EOL> import layers <EOL> import cc_layers <EOL> import custom <EOL> import load_data <EOL> import realtime_augmentation as ra <EOL> import time <EOL> import csv <EOL> import os <EOL> import cPickle as pickle <EOL> from datetime import datetime , timedelta <EOL> BATCH_SIZE = <NUM_LIT:16> <EOL> NUM_INPUT_FEATURES = <NUM_LIT:3> <EOL> LEARNING_RATE_SCHEDULE = { <EOL> <NUM_LIT:0> : <NUM_LIT> , <EOL> <NUM_LIT> : <NUM_LIT> , <EOL> <NUM_LIT> : <NUM_LIT> , <EOL> } <EOL> MOMENTUM = <NUM_LIT> <EOL> WEIGHT_DECAY = <NUM_LIT:0.0> <EOL> CHUNK_SIZE = <NUM_LIT> <EOL> NUM_CHUNKS = <NUM_LIT> <EOL> VALIDATE_EVERY = <NUM_LIT:20> <EOL> NUM_CHUNKS_NONORM = <NUM_LIT:1> <EOL> GEN_BUFFER_SIZE = <NUM_LIT:1> <EOL> TARGET_PATH = "<STR_LIT>" <EOL> ANALYSIS_PATH = "<STR_LIT>" <EOL> print "<STR_LIT>" <EOL> input_sizes = [ ( <NUM_LIT> , <NUM_LIT> ) , ( <NUM_LIT> , <NUM_LIT> ) ] <EOL> ds_transforms = [ <EOL> ra . build_ds_transform ( <NUM_LIT> , target_size = input_sizes [ <NUM_LIT:0> ] ) , <EOL> ra . build_ds_transform ( <NUM_LIT> , target_size = input_sizes [ <NUM_LIT:1> ] ) + ra . build_augmentation_transform ( rotation = <NUM_LIT> ) <EOL> ] <EOL> num_input_representations = len ( ds_transforms ) <EOL> augmentation_params = { <EOL> '<STR_LIT>' : ( <NUM_LIT:1.0> / <NUM_LIT> , <NUM_LIT> ) , <EOL> '<STR_LIT>' : ( <NUM_LIT:0> , <NUM_LIT> ) , <EOL> '<STR_LIT>' : ( <NUM_LIT:0> , <NUM_LIT:0> ) , <EOL> '<STR_LIT>' : ( - <NUM_LIT:4> , <NUM_LIT:4> ) , <EOL> '<STR_LIT>' : True , <EOL> } <EOL> augmented_data_gen = ra . realtime_augmented_data_gen ( num_chunks = NUM_CHUNKS , chunk_size = CHUNK_SIZE , <EOL> augmentation_params = augmentation_params , ds_transforms = ds_transforms , <EOL> target_sizes = input_sizes ) <EOL> post_augmented_data_gen = ra . post_augment_brightness_gen ( augmented_data_gen , std = <NUM_LIT:0.5> ) <EOL> train_gen = load_data . buffered_gen_mp ( post_augmented_data_gen , buffer_size = GEN_BUFFER_SIZE ) <EOL> y_train = np . load ( "<STR_LIT>" ) <EOL> train_ids = load_data . train_ids <EOL> test_ids = load_data . test_ids <EOL> num_train = len ( train_ids ) <EOL> num_test = len ( test_ids ) <EOL> num_valid = num_train // <NUM_LIT:10> <EOL> num_train -= num_valid <EOL> y_valid = y_train [ num_train : ] <EOL> y_train = y_train [ : num_train ] <EOL> valid_ids = train_ids [ num_train : ] <EOL> train_ids = train_ids [ : num_train ] <EOL> train_indices = np . arange ( num_train ) <EOL> valid_indices = np . arange ( num_train , num_train + num_valid ) <EOL> test_indices = np . arange ( num_test ) <EOL> def create_train_gen ( ) : <EOL> """<STR_LIT>""" <EOL> data_gen_train = ra . realtime_fixed_augmented_data_gen ( train_indices , '<STR_LIT:train>' , <EOL> ds_transforms = ds_transforms , chunk_size = CHUNK_SIZE , target_sizes = input_sizes ) <EOL> return load_data . buffered_gen_mp ( data_gen_train , buffer_size = GEN_BUFFER_SIZE ) <EOL> def create_valid_gen ( ) : <EOL> data_gen_valid = ra . realtime_fixed_augmented_data_gen ( valid_indices , '<STR_LIT:train>' , <EOL> ds_transforms = ds_transforms , chunk_size = CHUNK_SIZE , target_sizes = input_sizes ) <EOL> return load_data . buffered_gen_mp ( data_gen_valid , buffer_size = GEN_BUFFER_SIZE ) <EOL> def create_test_gen ( ) : <EOL> data_gen_test = ra . realtime_fixed_augmented_data_gen ( test_indices , '<STR_LIT:test>' , <EOL> ds_transforms = ds_transforms , chunk_size = CHUNK_SIZE , target_sizes = input_sizes ) <EOL> return load_data . buffered_gen_mp ( data_gen_test , buffer_size = GEN_BUFFER_SIZE ) <EOL> print "<STR_LIT>" <EOL> start_time = time . time ( ) <EOL> xs_valid = [ [ ] for _ in xrange ( num_input_representations ) ] <EOL> for data , length in create_valid_gen ( ) : <EOL> for x_valid_list , x_chunk in zip ( xs_valid , data ) : <EOL> x_valid_list . append ( x_chunk [ : length ] ) <EOL> xs_valid = [ np . vstack ( x_valid ) for x_valid in xs_valid ] <EOL> xs_valid = [ x_valid . transpose ( <NUM_LIT:0> , <NUM_LIT:3> , <NUM_LIT:1> , <NUM_LIT:2> ) for x_valid in xs_valid ] <EOL> print "<STR_LIT>" % ( time . time ( ) - start_time ) <EOL> print "<STR_LIT>" <EOL> l0 = layers . Input2DLayer ( BATCH_SIZE , NUM_INPUT_FEATURES , input_sizes [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] , input_sizes [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] ) <EOL> l0_45 = layers . Input2DLayer ( BATCH_SIZE , NUM_INPUT_FEATURES , input_sizes [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] , input_sizes [ <NUM_LIT:1> ] [ <NUM_LIT:1> ] ) <EOL> l0r = layers . MultiRotSliceLayer ( [ l0 , l0_45 ] , part_size = <NUM_LIT> , include_flip = True ) <EOL> l0s = cc_layers . ShuffleBC01ToC01BLayer ( l0r ) <EOL> l1a = cc_layers . CudaConvnetConv2DLayer ( l0s , n_filters = <NUM_LIT:32> , filter_size = <NUM_LIT:6> , weights_std = <NUM_LIT> , init_bias_value = <NUM_LIT:0.1> , dropout = <NUM_LIT:0.0> , partial_sum = <NUM_LIT:1> , untie_biases = True ) <EOL> l1 = cc_layers . CudaConvnetPooling2DLayer ( l1a , pool_size = <NUM_LIT:2> ) <EOL> l2a = cc_layers . CudaConvnetConv2DLayer ( l1 , n_filters = <NUM_LIT:64> , filter_size = <NUM_LIT:5> , weights_std = <NUM_LIT> , init_bias_value = <NUM_LIT:0.1> , dropout = <NUM_LIT:0.0> , partial_sum = <NUM_LIT:1> , untie_biases = True ) <EOL> l2 = cc_layers . CudaConvnetPooling2DLayer ( l2a , pool_size = <NUM_LIT:2> ) <EOL> l3a = cc_layers . CudaConvnetConv2DLayer ( l2 , n_filters = <NUM_LIT> , filter_size = <NUM_LIT:3> , weights_std = <NUM_LIT> , init_bias_value = <NUM_LIT:0.1> , dropout = <NUM_LIT:0.0> , partial_sum = <NUM_LIT:1> , untie_biases = True ) <EOL> l3b = cc_layers . CudaConvnetConv2DLayer ( l3a , n_filters = <NUM_LIT> , filter_size = <NUM_LIT:3> , pad = <NUM_LIT:0> , weights_std = <NUM_LIT:0.1> , init_bias_value = <NUM_LIT:0.1> , dropout = <NUM_LIT:0.0> , partial_sum = <NUM_LIT:1> , untie_biases = True ) <EOL> l3 = cc_layers . CudaConvnetPooling2DLayer ( l3b , pool_size = <NUM_LIT:2> ) <EOL> l3s = cc_layers . ShuffleC01BToBC01Layer ( l3 ) <EOL> j3 = layers . MultiRotMergeLayer ( l3s , num_views = <NUM_LIT:4> ) <EOL> l4a = layers . DenseLayer ( j3 , n_outputs = <NUM_LIT> , weights_std = <NUM_LIT> , init_bias_value = <NUM_LIT> , dropout = <NUM_LIT:0.5> , nonlinearity = layers . identity ) <EOL> l4b = layers . FeatureMaxPoolingLayer ( l4a , pool_size = <NUM_LIT:2> , feature_dim = <NUM_LIT:1> , implementation = '<STR_LIT>' ) <EOL> l4c = layers . DenseLayer ( l4b , n_outputs = <NUM_LIT> , weights_std = <NUM_LIT> , init_bias_value = <NUM_LIT> , dropout = <NUM_LIT:0.5> , nonlinearity = layers . identity ) <EOL> l4 = layers . FeatureMaxPoolingLayer ( l4c , pool_size = <NUM_LIT:2> , feature_dim = <NUM_LIT:1> , implementation = '<STR_LIT>' ) <EOL> l5 = layers . DenseLayer ( l4 , n_outputs = <NUM_LIT> , weights_std = <NUM_LIT> , init_bias_value = <NUM_LIT:0.1> , dropout = <NUM_LIT:0.5> , nonlinearity = layers . identity ) <EOL> l6 = custom . OptimisedDivGalaxyOutputLayer ( l5 ) <EOL> train_loss_nonorm = l6 . error ( normalisation = False ) <EOL> train_loss = l6 . error ( ) <EOL> valid_loss = l6 . error ( dropout_active = False ) <EOL> all_parameters = layers . all_parameters ( l6 ) <EOL> all_bias_parameters = layers . all_bias_parameters ( l6 ) <EOL> xs_shared = [ theano . shared ( np . zeros ( ( <NUM_LIT:1> , <NUM_LIT:1> , <NUM_LIT:1> , <NUM_LIT:1> ) , dtype = theano . config . floatX ) ) for _ in xrange ( num_input_representations ) ] <EOL> y_shared = theano . shared ( np . zeros ( ( <NUM_LIT:1> , <NUM_LIT:1> ) , dtype = theano . config . floatX ) ) <EOL> learning_rate = theano . shared ( np . array ( LEARNING_RATE_SCHEDULE [ <NUM_LIT:0> ] , dtype = theano . config . floatX ) ) <EOL> idx = T . lscalar ( '<STR_LIT>' ) <EOL> givens = { <EOL> l0 . input_var : xs_shared [ <NUM_LIT:0> ] [ idx * BATCH_SIZE : ( idx + <NUM_LIT:1> ) * BATCH_SIZE ] , <EOL> l0_45 . input_var : xs_shared [ <NUM_LIT:1> ] [ idx * BATCH_SIZE : ( idx + <NUM_LIT:1> ) * BATCH_SIZE ] , <EOL> l6 . target_var : y_shared [ idx * BATCH_SIZE : ( idx + <NUM_LIT:1> ) * BATCH_SIZE ] , <EOL> } <EOL> updates_nonorm = layers . gen_updates_nesterov_momentum_no_bias_decay ( train_loss_nonorm , all_parameters , all_bias_parameters , learning_rate = learning_rate , momentum = MOMENTUM , weight_decay = WEIGHT_DECAY ) <EOL> updates = layers . gen_updates_nesterov_momentum_no_bias_decay ( train_loss , all_parameters , all_bias_parameters , learning_rate = learning_rate , momentum = MOMENTUM , weight_decay = WEIGHT_DECAY ) <EOL> train_nonorm = theano . function ( [ idx ] , train_loss_nonorm , givens = givens , updates = updates_nonorm ) <EOL> train_norm = theano . function ( [ idx ] , train_loss , givens = givens , updates = updates ) <EOL> compute_loss = theano . function ( [ idx ] , valid_loss , givens = givens ) <EOL> compute_output = theano . function ( [ idx ] , l6 . predictions ( dropout_active = False ) , givens = givens , on_unused_input = '<STR_LIT:ignore>' ) <EOL> compute_features = theano . function ( [ idx ] , l4 . output ( dropout_active = False ) , givens = givens , on_unused_input = '<STR_LIT:ignore>' ) <EOL> rescaling_updates = l4a . rescaling_updates ( <NUM_LIT> ) + l4c . rescaling_updates ( <NUM_LIT> ) <EOL> rescale_weights = theano . function ( [ ] , [ ] , updates = rescaling_updates ) <EOL> rescale_every = <NUM_LIT:10> <EOL> print "<STR_LIT>" <EOL> start_time = time . time ( ) <EOL> prev_time = start_time <EOL> num_batches_valid = x_valid . shape [ <NUM_LIT:0> ] // BATCH_SIZE <EOL> losses_train = [ ] <EOL> losses_valid = [ ] <EOL> param_stds = [ ] <EOL> for e in xrange ( NUM_CHUNKS ) : <EOL> print "<STR_LIT>" % ( e + <NUM_LIT:1> , NUM_CHUNKS ) <EOL> chunk_data , chunk_length = train_gen . next ( ) <EOL> y_chunk = chunk_data . pop ( ) <EOL> xs_chunk = chunk_data <EOL> xs_chunk = [ x_chunk . transpose ( <NUM_LIT:0> , <NUM_LIT:3> , <NUM_LIT:1> , <NUM_LIT:2> ) for x_chunk in xs_chunk ] <EOL> if e in LEARNING_RATE_SCHEDULE : <EOL> current_lr = LEARNING_RATE_SCHEDULE [ e ] <EOL> learning_rate . set_value ( LEARNING_RATE_SCHEDULE [ e ] ) <EOL> print "<STR_LIT>" % current_lr <EOL> if e >= NUM_CHUNKS_NONORM : <EOL> train = train_norm <EOL> else : <EOL> train = train_nonorm <EOL> print "<STR_LIT>" <EOL> for x_shared , x_chunk in zip ( xs_shared , xs_chunk ) : <EOL> x_shared . set_value ( x_chunk ) <EOL> y_shared . set_value ( y_chunk ) <EOL> num_batches_chunk = x_chunk . shape [ <NUM_LIT:0> ] // BATCH_SIZE <EOL> print "<STR_LIT>" <EOL> losses = [ ] <EOL> for b in xrange ( num_batches_chunk ) : <EOL> loss = train ( b ) <EOL> losses . append ( loss ) <EOL> if ( b + <NUM_LIT:1> ) % rescale_every == <NUM_LIT:0> : <EOL> rescale_weights ( ) <EOL> mean_train_loss = np . sqrt ( np . mean ( losses ) ) <EOL> print "<STR_LIT>" % mean_train_loss <EOL> losses_train . append ( mean_train_loss ) <EOL> param_stds . append ( [ p . std ( ) for p in layers . get_param_values ( l6 ) ] ) <EOL> if ( ( e + <NUM_LIT:1> ) % VALIDATE_EVERY ) == <NUM_LIT:0> : <EOL> print <EOL> print "<STR_LIT>" <EOL> print "<STR_LIT>" <EOL> for x_shared , x_valid in zip ( xs_shared , xs_valid ) : <EOL> x_shared . set_value ( x_valid ) <EOL> y_shared . set_value ( y_valid ) <EOL> print "<STR_LIT>" <EOL> losses = [ ] <EOL> for b in xrange ( num_batches_valid ) : <EOL> loss = compute_loss ( b ) <EOL> losses . append ( loss ) <EOL> mean_valid_loss = np . sqrt ( np . mean ( losses ) ) <EOL> print "<STR_LIT>" % mean_valid_loss <EOL> losses_valid . append ( mean_valid_loss ) <EOL> now = time . time ( ) <EOL> time_since_start = now - start_time <EOL> time_since_prev = now - prev_time <EOL> prev_time = now <EOL> est_time_left = time_since_start * ( float ( NUM_CHUNKS - ( e + <NUM_LIT:1> ) ) / float ( e + <NUM_LIT:1> ) ) <EOL> eta = datetime . now ( ) + timedelta ( seconds = est_time_left ) <EOL> eta_str = eta . strftime ( "<STR_LIT>" ) <EOL> print "<STR_LIT>" % ( load_data . hms ( time_since_start ) , time_since_prev ) <EOL> print "<STR_LIT>" % ( load_data . hms ( est_time_left ) , eta_str ) <EOL> print <EOL> del chunk_data , xs_chunk , x_chunk , y_chunk , xs_valid , x_valid <EOL> print "<STR_LIT>" <EOL> predictions_list = [ ] <EOL> for b in xrange ( num_batches_valid ) : <EOL> predictions = compute_output ( b ) <EOL> predictions_list . append ( predictions ) <EOL> all_predictions = np . vstack ( predictions_list ) <EOL> all_predictions [ all_predictions > <NUM_LIT:1> ] = <NUM_LIT:1.0> <EOL> all_predictions [ all_predictions < <NUM_LIT:0> ] = <NUM_LIT:0.0> <EOL> print "<STR_LIT>" % ANALYSIS_PATH <EOL> with open ( ANALYSIS_PATH , '<STR_LIT:w>' ) as f : <EOL> pickle . dump ( { <EOL> '<STR_LIT>' : valid_ids [ : num_batches_valid * BATCH_SIZE ] , <EOL> '<STR_LIT>' : all_predictions , <EOL> '<STR_LIT>' : y_valid , <EOL> '<STR_LIT>' : mean_train_loss , <EOL> '<STR_LIT>' : mean_valid_loss , <EOL> '<STR_LIT>' : time_since_start , <EOL> '<STR_LIT>' : losses_train , <EOL> '<STR_LIT>' : losses_valid , <EOL> '<STR_LIT>' : layers . get_param_values ( l6 ) , <EOL> '<STR_LIT>' : param_stds , <EOL> } , f , pickle . HIGHEST_PROTOCOL ) <EOL> del predictions_list , all_predictions <EOL> print "<STR_LIT>" <EOL> predictions_list = [ ] <EOL> for e , ( xs_chunk , chunk_length ) in enumerate ( create_test_gen ( ) ) : <EOL> print "<STR_LIT>" % ( e + <NUM_LIT:1> ) <EOL> xs_chunk = [ x_chunk . transpose ( <NUM_LIT:0> , <NUM_LIT:3> , <NUM_LIT:1> , <NUM_LIT:2> ) for x_chunk in xs_chunk ] <EOL> for x_shared , x_chunk in zip ( xs_shared , xs_chunk ) : <EOL> x_shared . set_value ( x_chunk ) <EOL> num_batches_chunk = int ( np . ceil ( chunk_length / float ( BATCH_SIZE ) ) ) <EOL> for b in xrange ( num_batches_chunk ) : <EOL> predictions = compute_output ( b ) <EOL> predictions_list . append ( predictions ) <EOL> all_predictions = np . vstack ( predictions_list ) <EOL> all_predictions = all_predictions [ : num_test ] <EOL> all_predictions [ all_predictions > <NUM_LIT:1> ] = <NUM_LIT:1.0> <EOL> all_predictions [ all_predictions < <NUM_LIT:0> ] = <NUM_LIT:0.0> <EOL> print "<STR_LIT>" % TARGET_PATH <EOL> with open ( TARGET_PATH , '<STR_LIT:wb>' ) as csvfile : <EOL> writer = csv . writer ( csvfile ) <EOL> writer . writerow ( [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> for k in xrange ( test_ids . shape [ <NUM_LIT:0> ] ) : <EOL> row = [ test_ids [ k ] ] + all_predictions [ k ] . tolist ( ) <EOL> writer . writerow ( row ) <EOL> print "<STR_LIT>" <EOL> os . system ( "<STR_LIT>" % ( TARGET_PATH , TARGET_PATH ) ) <EOL> del all_predictions , predictions_list , xs_chunk , x_chunk <EOL> print "<STR_LIT>" </s>
