<s> """<STR_LIT>""" <EOL> import sys <EOL> import logging <EOL> import numpy as np <EOL> import pycuda . driver as drv <EOL> from pycuda . compiler import SourceModule <EOL> from pycuda . tools import context_dependent_memoize <EOL> import kernel_specs <EOL> from neon . backends . cuda_templates import _common_round , _common_fp16_to_fp32 , _ew_types <EOL> import os . path <EOL> import shelve <EOL> from convolution import KernelGroup , _get_shuffle_kernel , _get_sm_count , _ceil_div , _magic64 , _magic32 , _flatten , _fp_convert , _closest_divisor <EOL> logger = logging . getLogger ( __name__ ) <EOL> class XpropWinograd ( KernelGroup ) : <EOL> def __init__ ( self , op , lib , dtype , <EOL> N , C , K , <EOL> H , W , P , Q , <EOL> pad_h , pad_w , <EOL> relu , bsum ) : <EOL> super ( XpropWinograd , self ) . __init__ ( lib , dtype ) <EOL> SMs = _get_sm_count ( ) <EOL> self . autotune_key = "<STR_LIT:U+0020>" . join ( str ( x ) for x in ( op + "<STR_LIT>" , <EOL> SMs , dtype . itemsize , N , C , K , H , W , P , Q ) ) <EOL> self . autotune_db_file = os . path . join ( lib . cache_dir , "<STR_LIT>" ) <EOL> self . warmup = min ( max ( int ( <NUM_LIT> / ( P * Q * K * N * C * <NUM_LIT:9> * <NUM_LIT> ) * ( SMs / <NUM_LIT> ) ) , <NUM_LIT:1> ) , <NUM_LIT:1000> ) <EOL> self . params = ( N , C , K , H , W , P , Q , pad_h , pad_w , relu , bsum ) <EOL> self . init ( self . params ) <EOL> lib . set_scratch_size ( self . trans_size ) <EOL> def init ( self , params , autotune = <NUM_LIT:0> , filter_extern = <NUM_LIT:0> ) : <EOL> R , S = <NUM_LIT:3> , <NUM_LIT:3> <EOL> N , C , K , H , W , P , Q , pad_h , pad_w , relu , bsum = self . params <EOL> if not autotune : <EOL> if N >= <NUM_LIT:32> : <EOL> if self . dtype . type is np . float32 : <EOL> filter_extern = C < <NUM_LIT> and K < <NUM_LIT> <EOL> else : <EOL> filter_extern = True <EOL> self . initialized = True <EOL> else : <EOL> autotune_db = shelve . open ( self . autotune_db_file ) <EOL> if self . autotune_key in autotune_db : <EOL> filter_extern = autotune_db [ self . autotune_key ] <EOL> self . initialized = True <EOL> else : <EOL> filter_extern = True <EOL> self . initialized = False <EOL> autotune_db . close ( ) <EOL> filter_trans = "<STR_LIT>" if filter_extern else "<STR_LIT>" <EOL> kernel_name = "<STR_LIT>" % ( self . clss , filter_trans ) <EOL> if N == <NUM_LIT:1> : <EOL> shiftN = <NUM_LIT:0> <EOL> elif N < <NUM_LIT:32> : <EOL> shiftN = len ( bin ( N - <NUM_LIT:1> ) ) - <NUM_LIT:2> <EOL> else : <EOL> shiftN = <NUM_LIT:5> <EOL> blkN = <NUM_LIT:1> << shiftN <EOL> shiftY , shiftX , superY , superX = { <EOL> <NUM_LIT:1> : ( <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT> , <NUM_LIT> ) , <EOL> <NUM_LIT:2> : ( <NUM_LIT:3> , <NUM_LIT:3> , <NUM_LIT> , <NUM_LIT> ) , <EOL> <NUM_LIT:4> : ( <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT> , <NUM_LIT> ) , <EOL> <NUM_LIT:8> : ( <NUM_LIT:2> , <NUM_LIT:2> , <NUM_LIT> , <NUM_LIT> ) , <EOL> <NUM_LIT:16> : ( <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT> , <NUM_LIT> ) , <EOL> <NUM_LIT:32> : ( <NUM_LIT:1> , <NUM_LIT:1> , <NUM_LIT> , <NUM_LIT> ) , <EOL> } . get ( blkN ) <EOL> gridK = _ceil_div ( K , <NUM_LIT:32> ) <EOL> gridY = _ceil_div ( P , <NUM_LIT:1> << shiftY ) <EOL> gridX = _ceil_div ( Q , <NUM_LIT:1> << shiftX ) <EOL> gridN = _ceil_div ( N , blkN ) <EOL> Y2 = gridY // <NUM_LIT:2> <EOL> X2 = gridX * <NUM_LIT:2> <EOL> XNp = W * N * self . dtype . itemsize <EOL> YXNp = H * W * N * self . dtype . itemsize <EOL> SKp = S * K * self . dtype . itemsize <EOL> RSKp = R * S * K * self . dtype . itemsize <EOL> C4Kp = gridK * self . dtype . itemsize * <NUM_LIT:4> * <NUM_LIT> <EOL> groupK = _closest_divisor ( gridK , <NUM_LIT:4> ) <EOL> YXGK = gridY * gridX * groupK <EOL> X2GK = X2 * groupK <EOL> magic_YXGK = _magic64 ( YXGK ) <EOL> magic_X2GK = _magic64 ( X2GK ) <EOL> magic_groupK = _magic32 ( X2GK , groupK ) <EOL> grid = ( gridK * gridY * gridX , gridN , <NUM_LIT:1> ) <EOL> block = ( <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT:1> ) <EOL> self . kernel = [ kernel_name , grid , block , None , None , None , None , None , None , None , None ] <EOL> self . kernel . extend ( _flatten ( [ <EOL> K , C , H , W , N , P , Q , W * N , H * W * N , R * S * K , Q * N , P * Q * N , <EOL> shiftY , shiftX , shiftN , superY , superX , gridX , gridK , Y2 , <EOL> YXGK , X2GK , groupK , magic_YXGK , magic_X2GK , magic_groupK , <EOL> <NUM_LIT:2> * XNp , XNp , <NUM_LIT:4> * YXNp - <NUM_LIT:3> * XNp , <NUM_LIT:2> * SKp , SKp , <NUM_LIT:4> * RSKp - SKp , C4Kp , pad_h , pad_w ] ) ) <EOL> self . flags = ( relu and <NUM_LIT:2> ) + ( bsum and <NUM_LIT:4> ) <EOL> return filter_extern <EOL> def autotune ( self , I , F , O ) : <EOL> start , stop = self . lib . get_events ( ) <EOL> if not self . lib . warmup : <EOL> self . lib . warmup = True <EOL> self . init ( self . params , autotune = <NUM_LIT:1> , filter_extern = <NUM_LIT:1> ) <EOL> self . bind_params ( I , F , O , alpha = <NUM_LIT:1.0> , beta = <NUM_LIT:0.0> , bsum = None , flags = <NUM_LIT:1> ) <EOL> self . execute ( repeat = self . warmup , unbind = False ) <EOL> results = [ ] <EOL> for external in ( <NUM_LIT:0> , <NUM_LIT:1> ) : <EOL> self . init ( self . params , autotune = <NUM_LIT:1> , filter_extern = external ) <EOL> self . bind_params ( I , F , O , alpha = <NUM_LIT:1.0> , beta = <NUM_LIT:0.0> , bsum = None , flags = <NUM_LIT:1> ) <EOL> start . record ( stream = self . lib . stream ) <EOL> self . execute ( repeat = self . warmup , unbind = False ) <EOL> stop . record ( stream = self . lib . stream ) <EOL> stop . synchronize ( ) <EOL> msecs = stop . time_since ( start ) / self . warmup <EOL> results . append ( ( msecs , external ) ) <EOL> results . sort ( ) <EOL> external = results [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] <EOL> autotune_db = shelve . open ( self . autotune_db_file ) <EOL> autotune_db [ self . autotune_key ] = external <EOL> autotune_db . close ( ) <EOL> self . init ( self . params , autotune = <NUM_LIT:0> , filter_extern = external ) <EOL> class FpropWinograd ( XpropWinograd ) : <EOL> def __init__ ( self , lib , dtype , N , C , K , H , W , P , Q , pad_h , pad_w , relu , bsum ) : <EOL> super ( FpropWinograd , self ) . __init__ ( "<STR_LIT>" , lib , dtype , <EOL> N , C , K , H , W , P , Q , pad_h , pad_w , relu , bsum ) <EOL> def init ( self , params , autotune = <NUM_LIT:0> , filter_extern = <NUM_LIT:0> ) : <EOL> filter_extern = super ( FpropWinograd , self ) . init ( params , autotune , filter_extern ) <EOL> R , S = <NUM_LIT:3> , <NUM_LIT:3> <EOL> C , K = self . params [ <NUM_LIT:1> : <NUM_LIT:3> ] <EOL> gridK = _ceil_div ( K , <NUM_LIT:32> ) <EOL> if filter_extern : <EOL> self . trans_size = C * gridK * <NUM_LIT> * self . dtype . itemsize <EOL> self . trans_shared = <NUM_LIT> * self . dtype . itemsize <EOL> self . trans_args = [ ( gridK , C , <NUM_LIT:1> ) , ( <NUM_LIT:32> , <NUM_LIT:1> , <NUM_LIT:1> ) , None , <EOL> None , None , R * S * K , S * K , S * K * <NUM_LIT:2> , K ] <EOL> else : <EOL> self . trans_size = <NUM_LIT:0> <EOL> self . trans_args = None <EOL> def bind_params ( self , I , F , O , alpha , beta , bsum , flags = <NUM_LIT:0> ) : <EOL> assert I . dtype == F . dtype == O . dtype <EOL> if not self . initialized : <EOL> self . initialized = True <EOL> self . autotune ( I , F , O ) <EOL> bsum_gpudata , flags = self . init_bsum ( bsum , flags ) <EOL> if self . trans_size : <EOL> filter_temp = self . lib . scratch_buffer ( self . trans_size ) <EOL> self . trans_args [ <NUM_LIT:2> : <NUM_LIT:5> ] = ( self . lib . stream , filter_temp , F . gpudata ) <EOL> else : <EOL> filter_temp = F . gpudata <EOL> self . kernel [ <NUM_LIT:3> : <NUM_LIT:11> ] = ( self . lib . stream , bsum_gpudata , O . gpudata , I . gpudata , filter_temp , <EOL> alpha , beta , flags ) <EOL> def execute ( self , repeat = <NUM_LIT:1> , unbind = True ) : <EOL> kernel = kernel_specs . get_kernel ( self . kernel [ <NUM_LIT:0> ] ) <EOL> if self . trans_size : <EOL> trans_kernel = _get_fprop_filter_trans_kernel ( self . dtype_str ) <EOL> for r in range ( repeat ) : <EOL> if self . trans_size : <EOL> trans_kernel . prepared_async_call ( * self . trans_args , shared_size = self . trans_shared ) <EOL> if self . bsum_zero : <EOL> drv . memset_d32_async ( * self . bsum_zero ) <EOL> kernel . prepared_async_call ( * self . kernel [ <NUM_LIT:1> : ] ) <EOL> if unbind : <EOL> self . bsum_zero = None <EOL> self . kernel [ <NUM_LIT:3> : <NUM_LIT:11> ] = ( None , ) * <NUM_LIT:8> <EOL> if self . trans_size : <EOL> self . trans_args [ <NUM_LIT:2> : <NUM_LIT:5> ] = ( None , ) * <NUM_LIT:3> <EOL> def __str__ ( self ) : <EOL> return "<STR_LIT>" + self . kernel [ <NUM_LIT:0> ] <EOL> class BpropWinograd ( XpropWinograd ) : <EOL> def __init__ ( self , lib , dtype , N , C , K , H , W , P , Q , pad_h , pad_w , relu , bsum ) : <EOL> super ( BpropWinograd , self ) . __init__ ( "<STR_LIT>" , lib , dtype , <EOL> N , K , C , P , Q , H , W , <NUM_LIT:2> - pad_h , <NUM_LIT:2> - pad_w , relu , bsum ) <EOL> def init ( self , params , autotune = <NUM_LIT:0> , filter_extern = <NUM_LIT:0> ) : <EOL> filter_extern = super ( BpropWinograd , self ) . init ( params , autotune , filter_extern ) <EOL> R , S = <NUM_LIT:3> , <NUM_LIT:3> <EOL> K , C = self . params [ <NUM_LIT:1> : <NUM_LIT:3> ] <EOL> gridC = _ceil_div ( C , <NUM_LIT:32> ) <EOL> gridK = _ceil_div ( K , <NUM_LIT:32> ) <EOL> self . filter_extern = filter_extern <EOL> if filter_extern : <EOL> cBlks = _ceil_div ( gridC * <NUM_LIT:32> , <NUM_LIT:16> ) <EOL> kBlks = _ceil_div ( K , <NUM_LIT:16> ) <EOL> self . trans_size = K * gridC * <NUM_LIT> * self . dtype . itemsize <EOL> self . trans_args = [ ( kBlks , cBlks , <NUM_LIT:1> ) , ( <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT:1> ) , None , <EOL> None , None , R * S * K , S * K , S * K * <NUM_LIT:2> , K , C , cBlks ] <EOL> else : <EOL> self . trans_size = C * R * S * K * self . dtype . itemsize <EOL> self . trans_args = [ ( gridK , gridC , R * S ) , ( <NUM_LIT:32> , <NUM_LIT:8> , <NUM_LIT:1> ) , None , None , None ] <EOL> self . trans_args . extend ( _flatten ( [ <EOL> R * S * K , R * S * K , S * K , K , <EOL> R * S * C , R * S * C , S * C , C , <EOL> R * S , <NUM_LIT:1> , R , S , _magic32 ( R * S , R * S ) , _magic32 ( R * S , S ) ] ) ) <EOL> def bind_params ( self , I , F , O , alpha , beta , bsum , flags = <NUM_LIT:0> ) : <EOL> assert I . dtype == F . dtype == O . dtype <EOL> if not self . initialized : <EOL> self . initialized = True <EOL> self . autotune ( I , F , O ) <EOL> bsum_gpudata , flags = self . init_bsum ( bsum , flags ) <EOL> filter_temp = self . lib . scratch_buffer ( self . trans_size ) <EOL> self . trans_args [ <NUM_LIT:2> : <NUM_LIT:5> ] = ( self . lib . stream , filter_temp , F . gpudata ) <EOL> self . kernel [ <NUM_LIT:3> : <NUM_LIT:11> ] = ( self . lib . stream , bsum_gpudata , O . gpudata , I . gpudata , filter_temp , <EOL> alpha , beta , flags ) <EOL> def execute ( self , repeat = <NUM_LIT:1> , unbind = True ) : <EOL> kernel = kernel_specs . get_kernel ( self . kernel [ <NUM_LIT:0> ] ) <EOL> if self . filter_extern : <EOL> trans_kernel = _get_bprop_filter_trans_kernel ( self . dtype_str ) <EOL> else : <EOL> trans_kernel = _get_shuffle_kernel ( self . dtype_str ) <EOL> for r in range ( repeat ) : <EOL> if self . trans_size : <EOL> trans_kernel . prepared_async_call ( * self . trans_args ) <EOL> if self . bsum_zero : <EOL> drv . memset_d32_async ( * self . bsum_zero ) <EOL> kernel . prepared_async_call ( * self . kernel [ <NUM_LIT:1> : ] ) <EOL> if unbind : <EOL> self . bsum_zero = None <EOL> self . kernel [ <NUM_LIT:3> : <NUM_LIT:11> ] = ( None , ) * <NUM_LIT:8> <EOL> if self . trans_size : <EOL> self . trans_args [ <NUM_LIT:2> : <NUM_LIT:5> ] = ( None , ) * <NUM_LIT:3> <EOL> def __str__ ( self ) : <EOL> return "<STR_LIT>" + self . kernel [ <NUM_LIT:0> ] <EOL> class UpdateWinograd ( KernelGroup ) : <EOL> internal_superblock = { <EOL> <NUM_LIT:1> : ( <NUM_LIT:2> , <NUM_LIT:2> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) , <EOL> <NUM_LIT:2> : ( <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) , <EOL> <NUM_LIT:4> : ( <NUM_LIT:1> , <NUM_LIT:1> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) , <EOL> } <EOL> external_superblock = { <EOL> <NUM_LIT:1> : ( <NUM_LIT:1> , <NUM_LIT:1> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) , <EOL> <NUM_LIT:2> : ( <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) , <EOL> <NUM_LIT:4> : ( <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) , <EOL> } <EOL> def __init__ ( self , lib , dtype , N , C , K , H , W , P , Q , pad_h , pad_w ) : <EOL> assert N in ( <NUM_LIT:1> , <NUM_LIT:2> ) or N % <NUM_LIT:4> == <NUM_LIT:0> <EOL> super ( UpdateWinograd , self ) . __init__ ( lib , dtype ) <EOL> SMs = _get_sm_count ( ) <EOL> self . autotune_key = [ str ( x ) for x in ( "<STR_LIT>" , <EOL> SMs , <NUM_LIT:0> , dtype . itemsize , N , C , K , H , W , P , Q ) ] <EOL> self . autotune_db_file = os . path . join ( lib . cache_dir , "<STR_LIT>" ) <EOL> self . params = ( N , C , K , H , W , P , Q , pad_h , pad_w ) <EOL> self . init ( self . params ) <EOL> self . output_size = ( self . determ_size or ( dtype . type != np . float32 and <NUM_LIT:9> * C * K ) ) * <NUM_LIT:4> <EOL> lib . set_scratch_size ( self . output_size , self . trans_size ) <EOL> self . warmup = min ( max ( int ( <NUM_LIT> / ( P * Q * K * N * C * <NUM_LIT:9> * <NUM_LIT> ) * ( SMs / <NUM_LIT> ) ) , <NUM_LIT:1> ) , <NUM_LIT:1000> ) <EOL> def init ( self , params , autotune = False ) : <EOL> N , C , K , H , W , P , Q , pad_h , pad_w = self . params <EOL> R , S = <NUM_LIT:3> , <NUM_LIT:3> <EOL> loopN = <NUM_LIT:4> if N >= <NUM_LIT:4> else N <EOL> blkN = <NUM_LIT:4> if N >= <NUM_LIT:3> else N <EOL> superI = UpdateWinograd . external_superblock . get ( blkN ) <EOL> superE = UpdateWinograd . internal_superblock . get ( blkN ) <EOL> blkXI = <NUM_LIT:1> << superI [ <NUM_LIT:1> ] <EOL> blkY = <NUM_LIT:1> << superE [ <NUM_LIT:0> ] <EOL> blkX = <NUM_LIT:1> << superE [ <NUM_LIT:1> ] <EOL> GY = _ceil_div ( P , <NUM_LIT:2> ) <EOL> GX = _ceil_div ( Q , <NUM_LIT:2> ) <EOL> GYS = self . GYS = _ceil_div ( P , blkY ) <EOL> GXS = self . GXS = _ceil_div ( Q , blkX ) <EOL> if autotune : <EOL> strideY , strideX , external = autotune <EOL> else : <EOL> autotune_db = shelve . open ( self . autotune_db_file ) <EOL> autotune_key = "<STR_LIT:U+0020>" . join ( self . autotune_key ) <EOL> if autotune_key in autotune_db : <EOL> strideY , strideX , external = autotune_db [ autotune_key ] <EOL> self . initialized = True <EOL> else : <EOL> if GYS * GXS > <NUM_LIT> : <EOL> strideY = <NUM_LIT> <EOL> strideX = <NUM_LIT:1> <EOL> else : <EOL> strideY = GYS <EOL> strideX = GXS <EOL> external = True <EOL> self . initialized = False <EOL> autotune_db . close ( ) <EOL> loopXI = N * ( strideX * blkXI - <NUM_LIT:1> ) <EOL> loopXE = N * ( strideX * blkX - <NUM_LIT:1> ) <EOL> Np = N * self . dtype . itemsize <EOL> XNp = W * N * self . dtype . itemsize <EOL> XN2p = W * N * <NUM_LIT:2> * self . dtype . itemsize <EOL> QNp = Q * N * self . dtype . itemsize <EOL> gridK = _ceil_div ( K , <NUM_LIT:32> ) <EOL> gridC = _ceil_div ( C , <NUM_LIT:32> ) <EOL> k_size = _closest_divisor ( gridK , <NUM_LIT:4> ) <EOL> c_size = _closest_divisor ( gridC , <NUM_LIT:4> ) <EOL> C_size = gridC // c_size <EOL> kc = k_size * c_size <EOL> Qkc = strideX * kc <EOL> PQkc = strideY * Qkc <EOL> CPQkc = C_size * PQkc <EOL> gridPQKC = strideY * strideX * gridK * gridC <EOL> magic_CPQkc = _magic64 ( CPQkc ) <EOL> magic_PQkc = _magic64 ( PQkc ) <EOL> magic_Qkc = _magic64 ( Qkc ) <EOL> magic_kc = _magic32 ( Qkc , kc ) <EOL> magic_c = _magic32 ( kc , c_size ) <EOL> self . blocksCK = gridK * gridC <EOL> if external : <EOL> IX = "<STR_LIT>" <EOL> WN = GX * N <EOL> HWN = GY * WN <EOL> self . image_transform ( N , C , K , H , W , P , Q , pad_h , pad_w ) <EOL> else : <EOL> IX = "<STR_LIT>" <EOL> WN = W * N <EOL> HWN = H * WN <EOL> superI = superE <EOL> self . trans_size = <NUM_LIT:0> <EOL> if strideY * strideX == <NUM_LIT:1> : <EOL> determ = "<STR_LIT:D>" <EOL> self . determ_size = <NUM_LIT:0> <EOL> self . determ_shape = False <EOL> self . zero = False <EOL> elif self . lib . deterministic : <EOL> determ = "<STR_LIT:D>" <EOL> self . determ_size = strideY * strideX * C * R * S * K <EOL> self . determ_shape = ( strideY * strideX , C * R * S * K ) <EOL> self . zero = False <EOL> else : <EOL> determ = "<STR_LIT>" <EOL> self . determ_size = <NUM_LIT:0> <EOL> self . determ_shape = False <EOL> self . zero = True <EOL> kernel_name = "<STR_LIT>" % ( self . clss , determ , IX ) <EOL> self . kernel = [ kernel_name , ( gridPQKC , <NUM_LIT:1> , <NUM_LIT:1> ) , ( <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT:1> ) , None , None , None , None , None ] <EOL> self . kernel . extend ( _flatten ( [ <EOL> H , W , P , Q , C , K , N , pad_h , pad_w , <EOL> GY , GX , GYS , GXS , superI , superE , loopXI , loopXE , loopN , strideY , strideX , <EOL> WN , HWN , Q * N , P * Q * N , S * K , R * S * K , Np , XNp , XN2p , QNp , <EOL> CPQkc , PQkc , Qkc , kc , c_size , k_size , <EOL> magic_CPQkc , magic_PQkc , magic_Qkc , magic_kc , magic_c , C * R * S * K ] ) ) <EOL> def image_transform ( self , N , C , K , H , W , P , Q , pad_h , pad_w ) : <EOL> if N == <NUM_LIT:1> : <EOL> shlN = <NUM_LIT:0> <EOL> elif N < <NUM_LIT:16> : <EOL> shlN = len ( bin ( N - <NUM_LIT:1> ) ) - <NUM_LIT:2> <EOL> else : <EOL> shlN = <NUM_LIT:4> <EOL> GY = _ceil_div ( P , <NUM_LIT:2> ) <EOL> GX = _ceil_div ( Q , <NUM_LIT:2> ) <EOL> if shlN > <NUM_LIT:1> or GX >= <NUM_LIT:32> : <EOL> shlY , shlX , maskY , shrY , maskX , shrX , maskN = { <EOL> <NUM_LIT:0> : ( <NUM_LIT:1> , <NUM_LIT:5> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> ) , <EOL> <NUM_LIT:1> : ( <NUM_LIT:1> , <NUM_LIT:4> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ) , <EOL> <NUM_LIT:2> : ( <NUM_LIT:1> , <NUM_LIT:3> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT> ) , <EOL> <NUM_LIT:3> : ( <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> , <NUM_LIT:3> , <NUM_LIT> ) , <EOL> <NUM_LIT:4> : ( <NUM_LIT:1> , <NUM_LIT:1> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> ) , <EOL> } . get ( shlN ) <EOL> elif shlN > <NUM_LIT:0> or GX >= <NUM_LIT:16> : <EOL> shlY , shlX , maskY , shrY , maskX , shrX , maskN = { <EOL> <NUM_LIT:0> : ( <NUM_LIT:2> , <NUM_LIT:4> , <NUM_LIT> , <NUM_LIT:3> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> ) , <EOL> <NUM_LIT:1> : ( <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT> , <NUM_LIT:3> , <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ) , <EOL> } . get ( shlN ) <EOL> else : <EOL> shlY , shlX , maskY , shrY , maskX , shrX , maskN = { <EOL> <NUM_LIT:0> : ( <NUM_LIT:3> , <NUM_LIT:3> , <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> ) , <EOL> } . get ( shlN ) <EOL> blkN = <NUM_LIT:1> << shlN <EOL> blkY = <NUM_LIT:1> << shlY <EOL> blkX = <NUM_LIT:1> << shlX <EOL> gridC = _ceil_div ( C , <NUM_LIT:32> ) <EOL> GC = _ceil_div ( gridC * <NUM_LIT:32> , <NUM_LIT:16> ) <EOL> GYS = _ceil_div ( P , blkY ) <EOL> GXS = _ceil_div ( Q , blkX ) <EOL> GN = _ceil_div ( N , blkN ) <EOL> GYS2 = GYS // <NUM_LIT:2> <EOL> GXS2 = GXS * <NUM_LIT:2> <EOL> groupC = <NUM_LIT:2> <EOL> groupN = <NUM_LIT:1> if GN & <NUM_LIT:1> else <NUM_LIT:2> <EOL> shift_c = groupC - <NUM_LIT:1> <EOL> shift_n = groupN - <NUM_LIT:1> <EOL> X2cn = GXS2 * groupC * groupN <EOL> magic_X2cn = _magic64 ( X2cn ) <EOL> self . trans_size = gridC * GY * GX * N * <NUM_LIT> * self . dtype . itemsize <EOL> self . trans_args = [ ( GYS * GXS * groupC * groupN , GN // groupN , GC // groupC ) , <EOL> ( <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT:1> ) , None , None , None , <EOL> C , H , W , N , pad_h , pad_w , <EOL> GY , GX , GXS , GYS2 , X2cn , <EOL> magic_X2cn [ <NUM_LIT:0> ] , magic_X2cn [ <NUM_LIT:1> ] , shift_c , shift_n , <EOL> shlY , shlX , maskY , shrY , maskX , shrX , shlN , maskN , <EOL> H * W * N , W * N , GY * GX * N * <NUM_LIT> , GX * N * <NUM_LIT> , N * <NUM_LIT> ] <EOL> def autotune ( self , I , E , O ) : <EOL> autotune_key = "<STR_LIT:U+0020>" . join ( self . autotune_key ) <EOL> start , stop = self . lib . get_events ( ) <EOL> if not self . lib . warmup : <EOL> self . lib . warmup = True <EOL> self . init ( self . params , autotune = ( self . GYS , <NUM_LIT:1> , <NUM_LIT:1> ) ) <EOL> self . bind_params ( I , E , O , <NUM_LIT:1.0> ) <EOL> self . execute ( repeat = self . warmup , unbind = False ) <EOL> block_slots = _get_sm_count ( ) <EOL> loopsN = max ( self . params [ <NUM_LIT:0> ] // <NUM_LIT:4> , <NUM_LIT:1> ) <EOL> modes = ( <NUM_LIT:0> , <NUM_LIT:1> ) if self . params [ <NUM_LIT:0> ] <= <NUM_LIT:4> else ( <NUM_LIT:1> , ) <EOL> gys = float ( self . GYS ) <EOL> gxs = float ( self . GYS ) <EOL> small_set = gys * gxs <= <NUM_LIT> <EOL> results = [ ] <EOL> sys . stdout . write ( "<STR_LIT>" + str ( self ) ) <EOL> progress = <NUM_LIT:0> <EOL> for threshold in ( True , False ) : <EOL> for external in modes : <EOL> for strideY in range ( <NUM_LIT:1> , self . GYS + <NUM_LIT:1> ) : <EOL> for strideX in range ( <NUM_LIT:1> , self . GXS + <NUM_LIT:1> ) : <EOL> if progress % <NUM_LIT:32> == <NUM_LIT:0> : <EOL> sys . stdout . write ( '<STR_LIT:.>' ) <EOL> sys . stdout . flush ( ) <EOL> progress += <NUM_LIT:1> <EOL> outputs = strideY * strideX <EOL> blocks = self . blocksCK * strideY * strideX <EOL> depth = ( gys / strideY ) * ( gxs / strideX ) * loopsN <EOL> filters = outputs <= <NUM_LIT> and blocks >= block_slots and depth >= <NUM_LIT> <EOL> if small_set or ( threshold and filters ) or ( not threshold and not filters ) : <EOL> settings = ( strideY , strideX , external ) <EOL> self . init ( self . params , autotune = settings ) <EOL> self . bind_params ( I , E , O , <NUM_LIT:1.0> ) <EOL> start . record ( stream = self . lib . stream ) <EOL> self . execute ( repeat = <NUM_LIT:2> , unbind = False ) <EOL> stop . record ( stream = self . lib . stream ) <EOL> stop . synchronize ( ) <EOL> msecs = stop . time_since ( start ) / <NUM_LIT> <EOL> results . append ( ( msecs , settings ) ) <EOL> if len ( results ) > <NUM_LIT:0> : <EOL> break <EOL> sys . stdout . write ( '<STR_LIT:\n>' ) <EOL> results . sort ( ) <EOL> settings = results [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] <EOL> autotune_db = shelve . open ( self . autotune_db_file ) <EOL> autotune_db [ autotune_key ] = settings <EOL> if settings [ <NUM_LIT:0> ] * settings [ <NUM_LIT:1> ] <= <NUM_LIT:8> : <EOL> self . autotune_key [ <NUM_LIT:2> ] = str ( <NUM_LIT:1> - int ( self . autotune_key [ <NUM_LIT:2> ] ) ) <EOL> autotune_key = "<STR_LIT:U+0020>" . join ( self . autotune_key ) <EOL> autotune_db [ autotune_key ] = settings <EOL> autotune_db . close ( ) <EOL> self . init ( self . params , autotune = settings ) <EOL> def bind_params ( self , I , E , O , alpha ) : <EOL> assert I . dtype == E . dtype <EOL> if not self . initialized : <EOL> self . initialized = True <EOL> self . autotune ( I , E , O ) <EOL> if O . dtype . type is not np . float32 or self . determ_size : <EOL> update_temp = self . lib . scratch_buffer ( self . output_size ) <EOL> self . convert_args = [ update_temp , "<STR_LIT>" , O , self . determ_shape ] <EOL> if self . trans_size : <EOL> input_temp = self . lib . scratch_buffer_offset ( self . trans_size ) <EOL> else : <EOL> update_temp = O . gpudata <EOL> self . convert_args = False <EOL> if self . trans_size : <EOL> input_temp = self . lib . scratch_buffer ( self . trans_size ) <EOL> if self . trans_size : <EOL> self . trans_args [ <NUM_LIT:2> : <NUM_LIT:5> ] = ( self . lib . stream , input_temp , I . gpudata ) <EOL> else : <EOL> input_temp = I . gpudata <EOL> if self . zero : <EOL> self . zero_args = [ update_temp , <NUM_LIT:0> , O . size , self . lib . stream ] <EOL> self . kernel [ <NUM_LIT:3> : <NUM_LIT:8> ] = ( self . lib . stream , update_temp , input_temp , E . gpudata , alpha ) <EOL> def execute ( self , repeat = <NUM_LIT:1> , unbind = True ) : <EOL> kernel = kernel_specs . get_kernel ( self . kernel [ <NUM_LIT:0> ] ) <EOL> if self . trans_size : <EOL> trans_kernel = _get_update_image_trans_kernel ( self . dtype_str ) <EOL> for r in range ( repeat ) : <EOL> if self . trans_size : <EOL> trans_kernel . prepared_async_call ( * self . trans_args ) <EOL> if self . zero : <EOL> drv . memset_d32_async ( * self . zero_args ) <EOL> kernel . prepared_async_call ( * self . kernel [ <NUM_LIT:1> : ] ) <EOL> if self . convert_args : <EOL> _fp_convert ( * self . convert_args ) <EOL> if unbind : <EOL> self . zero_args = self . convert_args = None <EOL> self . kernel [ <NUM_LIT:3> : <NUM_LIT:8> ] = ( None , ) * <NUM_LIT:5> <EOL> def __str__ ( self ) : <EOL> N , C , K , H , W , P , Q , pad_h , pad_w = self . params <EOL> return "<STR_LIT>" % ( self . kernel [ <NUM_LIT:0> ] , N , C , K , H , W ) <EOL> class XpropWinograd_4x4_3x3 ( KernelGroup ) : <EOL> def __init__ ( self , lib , dtype , <EOL> N , C , K , <EOL> H , W , P , Q , <EOL> pad_h , pad_w , <EOL> relu , bsum ) : <EOL> super ( XpropWinograd_4x4_3x3 , self ) . __init__ ( lib , dtype ) <EOL> itemsize = self . dtype . itemsize <EOL> kernel_name = "<STR_LIT>" % self . clss <EOL> if N == <NUM_LIT:1> : <EOL> shlN = <NUM_LIT:0> <EOL> elif N < <NUM_LIT:32> : <EOL> shlN = len ( bin ( N - <NUM_LIT:1> ) ) - <NUM_LIT:2> <EOL> else : <EOL> shlN = <NUM_LIT:5> <EOL> shlY , shlX , maskY , shrY , maskX , shrX , maskN , supY , supX = { <EOL> <NUM_LIT:0> : ( <NUM_LIT:4> , <NUM_LIT:5> , <NUM_LIT> , <NUM_LIT:3> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) , <EOL> <NUM_LIT:1> : ( <NUM_LIT:4> , <NUM_LIT:4> , <NUM_LIT> , <NUM_LIT:3> , <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) , <EOL> <NUM_LIT:2> : ( <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT> , <NUM_LIT:4> , <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) , <EOL> <NUM_LIT:3> : ( <NUM_LIT:2> , <NUM_LIT:4> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> , <NUM_LIT:3> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) , <EOL> <NUM_LIT:4> : ( <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> , <NUM_LIT:4> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) , <EOL> <NUM_LIT:5> : ( <NUM_LIT:2> , <NUM_LIT:2> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) , <EOL> } . get ( shlN ) <EOL> R , S = <NUM_LIT:3> , <NUM_LIT:3> <EOL> GYS = _ceil_div ( P , <NUM_LIT:1> << shlY ) <EOL> GXS = _ceil_div ( Q , <NUM_LIT:1> << shlX ) <EOL> GN = _ceil_div ( N , <NUM_LIT:1> << shlN ) <EOL> GK = _ceil_div ( K , <NUM_LIT:32> ) <EOL> GYS2 = GYS // <NUM_LIT:2> <EOL> GXS2 = GXS * <NUM_LIT:2> <EOL> k = _closest_divisor ( GK , <NUM_LIT:4> ) <EOL> Xk = GXS * k <EOL> YXk = GYS * Xk <EOL> magic_GXS2 = _magic64 ( GXS2 ) <EOL> magic_YXk = _magic64 ( YXk ) <EOL> magic_Xk = _magic64 ( Xk ) <EOL> magic_k = _magic32 ( Xk , k ) <EOL> self . image_size = itemsize * <NUM_LIT> * C * GXS * GYS * GN <EOL> self . image_args = [ <EOL> ( GN , GYS * GXS , C ) , ( <NUM_LIT:32> , <NUM_LIT:1> , <NUM_LIT:1> ) , None , None , None , <EOL> H , W , N , pad_h , pad_w , <EOL> GXS , GYS2 , GXS2 , magic_GXS2 [ <NUM_LIT:0> ] , magic_GXS2 [ <NUM_LIT:1> ] , <EOL> shlY , shlX , maskY , shrY , maskX , shrX , shlN , maskN , <EOL> H * W * N , W * N , GYS * GXS * C * <NUM_LIT> , GXS * C * <NUM_LIT> , C * <NUM_LIT> ] <EOL> self . kernel = [ <EOL> kernel_name , ( GYS * GXS * GK , GN , <NUM_LIT:1> ) , ( <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT:1> ) , None , <EOL> None , None , None , None , None , None , None ] <EOL> self . kernel . extend ( _flatten ( [ <EOL> C , K , N , YXk , Xk , k , magic_YXk , magic_Xk , magic_k , <EOL> C * <NUM_LIT> , GXS * C * <NUM_LIT> , GYS * GXS * C * <NUM_LIT> , <EOL> P , Q , Q * N , P * Q * N , N * itemsize , Q * N * itemsize , Q * N * <NUM_LIT:3> * itemsize , <EOL> max ( P * Q * N - Q * N * <NUM_LIT:3> , <NUM_LIT:0> ) * itemsize , ( P * Q * N * <NUM_LIT:15> - Q * N * <NUM_LIT:3> ) * itemsize , <EOL> maskN , shlX , shlY , supX , supY ] ) ) <EOL> lib . set_scratch_size ( self . image_size , self . filter_size ) <EOL> self . flags = ( relu and <NUM_LIT:2> ) + ( bsum and <NUM_LIT:4> ) <EOL> self . mode = "<STR_LIT>" <EOL> def bind_params ( self , I , F , O , alpha , beta , bsum , flags = <NUM_LIT:0> ) : <EOL> assert I . dtype == F . dtype == O . dtype <EOL> bsum_gpudata , flags = self . init_bsum ( bsum , flags ) <EOL> if beta : <EOL> self . mode = "<STR_LIT>" <EOL> elif bsum : <EOL> self . mode = "<STR_LIT>" <EOL> else : <EOL> self . mode = "<STR_LIT>" <EOL> image_temp = self . lib . scratch_buffer ( self . image_size ) <EOL> self . image_args [ <NUM_LIT:2> : <NUM_LIT:5> ] = ( self . lib . stream , image_temp , I . gpudata ) <EOL> filter_temp = self . lib . scratch_buffer_offset ( self . filter_size ) <EOL> self . filter_args [ <NUM_LIT:2> : <NUM_LIT:5> ] = ( self . lib . stream , filter_temp , F . gpudata ) <EOL> self . kernel [ <NUM_LIT:3> : <NUM_LIT:11> ] = ( self . lib . stream , bsum_gpudata , O . gpudata , image_temp , filter_temp , <EOL> alpha , beta , flags ) <EOL> def execute ( self , repeat = <NUM_LIT:1> , unbind = True ) : <EOL> image_kernel = _get_xprop_image_trans_4x4_kernel ( self . dtype_str ) <EOL> filter_kernel = self . filter_func ( self . dtype_str ) <EOL> kernel = kernel_specs . get_kernel ( self . kernel [ <NUM_LIT:0> ] + self . mode ) <EOL> for r in range ( repeat ) : <EOL> image_kernel . prepared_async_call ( * self . image_args ) <EOL> filter_kernel . prepared_async_call ( * self . filter_args ) <EOL> if self . bsum_zero : <EOL> drv . memset_d32_async ( * self . bsum_zero ) <EOL> kernel . prepared_async_call ( * self . kernel [ <NUM_LIT:1> : ] ) <EOL> if unbind : <EOL> self . bsum_zero = None <EOL> self . kernel [ <NUM_LIT:3> : <NUM_LIT:11> ] = ( None , ) * <NUM_LIT:8> <EOL> self . image_args [ <NUM_LIT:2> : <NUM_LIT:5> ] = ( None , ) * <NUM_LIT:3> <EOL> self . filter_args [ <NUM_LIT:2> : <NUM_LIT:5> ] = ( None , ) * <NUM_LIT:3> <EOL> class FpropWinograd_4x4_3x3 ( XpropWinograd_4x4_3x3 ) : <EOL> def __init__ ( self , lib , dtype , <EOL> N , C , K , <EOL> H , W , P , Q , <EOL> pad_h , pad_w , <EOL> relu , bsum ) : <EOL> R , S = <NUM_LIT:3> , <NUM_LIT:3> <EOL> GK = _ceil_div ( K , <NUM_LIT:32> ) <EOL> self . filter_func = _get_fprop_filter_trans_4x4_kernel <EOL> self . filter_size = dtype . itemsize * <NUM_LIT> * C * GK <EOL> self . filter_args = [ <EOL> ( GK , C , <NUM_LIT:1> ) , ( <NUM_LIT:32> , <NUM_LIT:1> , <NUM_LIT:1> ) , None , None , None , <EOL> R * S * K , S * K , S * K * <NUM_LIT:2> , K , C * <NUM_LIT> ] <EOL> super ( FpropWinograd_4x4_3x3 , self ) . __init__ ( <EOL> lib , dtype , N , C , K , H , W , P , Q , pad_h , pad_w , relu , bsum ) <EOL> def __str__ ( self ) : <EOL> return "<STR_LIT>" + self . kernel [ <NUM_LIT:0> ] <EOL> class BpropWinograd_4x4_3x3 ( XpropWinograd_4x4_3x3 ) : <EOL> def __init__ ( self , lib , dtype , <EOL> N , C , K , <EOL> H , W , P , Q , <EOL> pad_h , pad_w , <EOL> relu , bsum ) : <EOL> R , S = <NUM_LIT:3> , <NUM_LIT:3> <EOL> GC32 = _ceil_div ( C , <NUM_LIT:32> ) <EOL> GC16 = _ceil_div ( GC32 * <NUM_LIT:32> , <NUM_LIT:16> ) <EOL> GK16 = _ceil_div ( K , <NUM_LIT:16> ) <EOL> self . filter_func = _get_bprop_filter_trans_4x4_kernel <EOL> self . filter_size = dtype . itemsize * <NUM_LIT> * K * GC32 <EOL> self . filter_args = [ <EOL> ( GK16 , GC16 , <NUM_LIT:1> ) , ( <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT:1> ) , None , None , None , <EOL> R * S * K , S * K , S * K * <NUM_LIT:2> , K , C , K * <NUM_LIT> ] <EOL> super ( BpropWinograd_4x4_3x3 , self ) . __init__ ( <EOL> lib , dtype , N , K , C , P , Q , H , W , <NUM_LIT:2> - pad_h , <NUM_LIT:2> - pad_w , relu , bsum ) <EOL> def __str__ ( self ) : <EOL> return "<STR_LIT>" + self . kernel [ <NUM_LIT:0> ] <EOL> class UpdateWinograd_3x3_4x4 ( KernelGroup ) : <EOL> def __init__ ( self , lib , dtype , <EOL> N , C , K , <EOL> H , W , P , Q , <EOL> pad_h , pad_w ) : <EOL> super ( UpdateWinograd_3x3_4x4 , self ) . __init__ ( lib , dtype ) <EOL> SMs = _get_sm_count ( ) <EOL> self . autotune_key = [ str ( x ) for x in ( "<STR_LIT>" , <EOL> SMs , <NUM_LIT:0> , dtype . itemsize , N , C , K , H , W , P , Q ) ] <EOL> self . autotune_db_file = os . path . join ( lib . cache_dir , "<STR_LIT>" ) <EOL> self . params = ( N , C , K , H , W , P , Q , pad_h , pad_w ) <EOL> self . init ( self . params ) <EOL> self . output_size = ( self . determ_size or ( dtype . type != np . float32 and <NUM_LIT:9> * C * K ) ) * <NUM_LIT:4> <EOL> lib . set_scratch_size ( self . image_size , self . delta_size , self . output_size ) <EOL> self . warmup = min ( max ( int ( <NUM_LIT> / ( P * Q * K * N * C * <NUM_LIT:9> * <NUM_LIT> ) * ( SMs / <NUM_LIT> ) ) , <NUM_LIT:1> ) , <NUM_LIT:1000> ) <EOL> def init ( self , params , autotune = False ) : <EOL> N , C , K , H , W , P , Q , pad_h , pad_w = self . params <EOL> itemsize = self . dtype . itemsize <EOL> R , S = <NUM_LIT:3> , <NUM_LIT:3> <EOL> if N == <NUM_LIT:1> : <EOL> shlN = <NUM_LIT:0> <EOL> elif N < <NUM_LIT:16> : <EOL> shlN = len ( bin ( N - <NUM_LIT:1> ) ) - <NUM_LIT:2> <EOL> else : <EOL> shlN = <NUM_LIT:4> <EOL> GC32 = _ceil_div ( C , <NUM_LIT:32> ) <EOL> GK32 = _ceil_div ( K , <NUM_LIT:32> ) <EOL> GC16 = _ceil_div ( GC32 * <NUM_LIT:32> , <NUM_LIT:16> ) <EOL> GK16 = _ceil_div ( GK32 * <NUM_LIT:32> , <NUM_LIT:16> ) <EOL> GY = _ceil_div ( P , <NUM_LIT:4> ) <EOL> GX = _ceil_div ( Q , <NUM_LIT:4> ) <EOL> if shlN > <NUM_LIT:1> or GX >= <NUM_LIT:32> : <EOL> shlY , shlX , maskY , shrY , maskX , shrX , maskN = { <EOL> <NUM_LIT:0> : ( <NUM_LIT:2> , <NUM_LIT:6> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> ) , <EOL> <NUM_LIT:1> : ( <NUM_LIT:2> , <NUM_LIT:5> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ) , <EOL> <NUM_LIT:2> : ( <NUM_LIT:2> , <NUM_LIT:4> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT> ) , <EOL> <NUM_LIT:3> : ( <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> , <NUM_LIT:3> , <NUM_LIT> ) , <EOL> <NUM_LIT:4> : ( <NUM_LIT:2> , <NUM_LIT:2> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> ) , <EOL> } . get ( shlN ) <EOL> elif shlN > <NUM_LIT:0> or GX >= <NUM_LIT:16> : <EOL> shlY , shlX , maskY , shrY , maskX , shrX , maskN = { <EOL> <NUM_LIT:0> : ( <NUM_LIT:3> , <NUM_LIT:5> , <NUM_LIT> , <NUM_LIT:3> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> ) , <EOL> <NUM_LIT:1> : ( <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT> , <NUM_LIT:3> , <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT> ) , <EOL> } . get ( shlN ) <EOL> else : <EOL> shlY , shlX , maskY , shrY , maskX , shrX , maskN = { <EOL> <NUM_LIT:0> : ( <NUM_LIT:4> , <NUM_LIT:4> , <NUM_LIT> , <NUM_LIT:2> , <NUM_LIT> , <NUM_LIT:0> , <NUM_LIT> ) , <EOL> } . get ( shlN ) <EOL> GYS = _ceil_div ( P , <NUM_LIT:1> << shlY ) <EOL> GXS = _ceil_div ( Q , <NUM_LIT:1> << shlX ) <EOL> GN16 = _ceil_div ( N , <NUM_LIT:1> << shlN ) <EOL> GYS2 = GYS // <NUM_LIT:2> <EOL> GXS2 = GXS * <NUM_LIT:2> <EOL> groupC = <NUM_LIT:2> <EOL> groupK = <NUM_LIT:2> <EOL> groupN = <NUM_LIT:1> if GN16 & <NUM_LIT:1> else <NUM_LIT:2> <EOL> shift_c = groupC - <NUM_LIT:1> <EOL> shift_k = groupK - <NUM_LIT:1> <EOL> shift_n = groupN - <NUM_LIT:1> <EOL> X2cn = GXS2 * groupC * groupN <EOL> Xkn = GXS * groupK * groupN <EOL> magic_X2cn = _magic64 ( X2cn ) <EOL> magic_Xkn = _magic64 ( Xkn ) <EOL> self . image_size = GC32 * GY * GX * N * <NUM_LIT> * itemsize <EOL> self . image_args = [ <EOL> ( GYS * GXS * groupC * groupN , GN16 // groupN , GC16 // groupC ) , ( <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT:1> ) , None , None , None , <EOL> C , H , W , N , pad_h , pad_w , <EOL> GY , GX , GXS , GYS2 , X2cn , magic_X2cn [ <NUM_LIT:0> ] , magic_X2cn [ <NUM_LIT:1> ] , shift_c , shift_n , <EOL> shlY , shlX , maskY , shrY , maskX , shrX , shlN , maskN , <EOL> H * W * N , W * N , GY * GX * N * <NUM_LIT> , GX * N * <NUM_LIT> , N * <NUM_LIT> ] <EOL> self . delta_size = GK32 * GY * GX * N * <NUM_LIT> * itemsize <EOL> self . delta_args = [ <EOL> ( GYS * GXS * groupK * groupN , GN16 // groupN , GK16 // groupK ) , ( <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT:1> ) , None , None , None , <EOL> K , P , Q , N , GY , GX , <EOL> Xkn , magic_Xkn [ <NUM_LIT:0> ] , magic_Xkn [ <NUM_LIT:1> ] , shift_k , shift_n , <EOL> shlY , shlX , maskY , shrY , maskX , shrX , shlN , maskN , <EOL> P * Q * N , Q * N , GY * GX * N * <NUM_LIT> , GX * N * <NUM_LIT> , N * <NUM_LIT> ] <EOL> Gc = _closest_divisor ( GC32 , <NUM_LIT:4> ) <EOL> Gk = _closest_divisor ( GK32 , <NUM_LIT:4> ) <EOL> GC = GC32 // Gc <EOL> GK = GK32 // Gk <EOL> kc = Gk * Gc <EOL> YXN = GY * GX * N <EOL> YXN2 = self . YXN2 = _ceil_div ( YXN , <NUM_LIT:2> ) <EOL> self . maxYXN2 = max ( <NUM_LIT:1> , YXN2 // N ) <EOL> if autotune : <EOL> strideYXN = autotune <EOL> else : <EOL> autotune_db = shelve . open ( self . autotune_db_file ) <EOL> autotune_key = "<STR_LIT:U+0020>" . join ( self . autotune_key ) <EOL> if autotune_key in autotune_db : <EOL> strideYXN = autotune_db [ autotune_key ] <EOL> self . initialized = True <EOL> else : <EOL> strideYXN = self . maxYXN2 <EOL> self . initialized = False <EOL> autotune_db . close ( ) <EOL> self . blocksCK = GC32 * GK32 <EOL> magic_sYXN = _magic64 ( strideYXN ) <EOL> magic_kc = _magic64 ( kc ) <EOL> magic_c = _magic32 ( kc , Gc ) <EOL> if strideYXN == <NUM_LIT:1> : <EOL> determ = "<STR_LIT:D>" <EOL> self . determ_size = <NUM_LIT:0> <EOL> self . determ_shape = False <EOL> self . zero = False <EOL> elif self . lib . deterministic : <EOL> determ = "<STR_LIT:D>" <EOL> self . determ_size = strideYXN * C * R * S * K <EOL> self . determ_shape = ( strideYXN , C * R * S * K ) <EOL> self . zero = False <EOL> else : <EOL> determ = "<STR_LIT>" <EOL> self . determ_size = <NUM_LIT:0> <EOL> self . determ_shape = False <EOL> self . zero = True <EOL> kernel_name = "<STR_LIT>" % ( self . clss , determ ) <EOL> self . kernel = [ <EOL> kernel_name , ( strideYXN * Gk * Gc , GC , GK ) , ( <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT:1> ) , None , <EOL> None , None , None , None ] <EOL> self . kernel . extend ( _flatten ( [ <EOL> K , C , Gk , Gc , kc , magic_kc , magic_c , YXN2 , strideYXN , magic_sYXN , <EOL> strideYXN * <NUM_LIT:2> * <NUM_LIT> * itemsize , YXN , YXN * <NUM_LIT> , R * S * K , C * R * S * K , <EOL> K * <NUM_LIT:4> , S * K * <NUM_LIT:4> , ( R * S * K * <NUM_LIT:15> - S * K * <NUM_LIT:2> ) * <NUM_LIT:4> ] ) ) <EOL> def autotune ( self , I , E , O ) : <EOL> autotune_key = "<STR_LIT:U+0020>" . join ( self . autotune_key ) <EOL> if not self . lib . warmup : <EOL> self . lib . warmup = True <EOL> self . init ( self . params , autotune = self . maxYXN2 ) <EOL> self . bind_params ( I , E , O , <NUM_LIT:1.0> ) <EOL> self . execute ( repeat = self . warmup , unbind = False ) <EOL> start , stop = self . lib . get_events ( ) <EOL> block_slots = _get_sm_count ( ) <EOL> small_set = self . YXN2 < <NUM_LIT> <EOL> YXN2 = float ( self . YXN2 ) <EOL> results = [ ] <EOL> sys . stdout . write ( "<STR_LIT>" + str ( self ) ) <EOL> progress = <NUM_LIT:0> <EOL> for threshold in ( True , False ) : <EOL> for strideYXN in range ( <NUM_LIT:1> , self . maxYXN2 + <NUM_LIT:1> ) : <EOL> if progress % <NUM_LIT:32> == <NUM_LIT:0> : <EOL> sys . stdout . write ( '<STR_LIT:.>' ) <EOL> sys . stdout . flush ( ) <EOL> progress += <NUM_LIT:1> <EOL> blocks = self . blocksCK * strideYXN <EOL> depth = YXN2 / strideYXN <EOL> filters = blocks >= block_slots and blocks <= <NUM_LIT> * block_slots and depth >= <NUM_LIT> <EOL> if small_set or ( threshold and filters ) or ( not threshold and not filters ) : <EOL> self . init ( self . params , autotune = strideYXN ) <EOL> self . bind_params ( I , E , O , <NUM_LIT:1.0> ) <EOL> start . record ( stream = self . lib . stream ) <EOL> self . execute ( repeat = <NUM_LIT:2> , unbind = False ) <EOL> stop . record ( stream = self . lib . stream ) <EOL> stop . synchronize ( ) <EOL> msecs = stop . time_since ( start ) / <NUM_LIT> <EOL> results . append ( ( msecs , strideYXN ) ) <EOL> if len ( results ) > <NUM_LIT:0> : <EOL> break <EOL> sys . stdout . write ( '<STR_LIT:\n>' ) <EOL> results . sort ( ) <EOL> strideYXN = results [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] <EOL> autotune_db = shelve . open ( self . autotune_db_file ) <EOL> autotune_db [ autotune_key ] = strideYXN <EOL> if strideYXN <= <NUM_LIT:8> : <EOL> self . autotune_key [ <NUM_LIT:2> ] = str ( <NUM_LIT:1> - int ( self . autotune_key [ <NUM_LIT:2> ] ) ) <EOL> autotune_key = "<STR_LIT:U+0020>" . join ( self . autotune_key ) <EOL> autotune_db [ autotune_key ] = strideYXN <EOL> autotune_db . close ( ) <EOL> self . init ( self . params , autotune = strideYXN ) <EOL> def bind_params ( self , I , E , O , alpha ) : <EOL> assert I . dtype == E . dtype <EOL> if not self . initialized : <EOL> self . initialized = True <EOL> self . autotune ( I , E , O ) <EOL> if O . dtype . type is not np . float32 or self . determ_size : <EOL> updat_temp = self . lib . scratch_buffer ( self . output_size ) <EOL> image_temp = self . lib . scratch_buffer_offset ( self . image_size ) <EOL> delta_temp = self . lib . scratch_buffer_offset ( self . delta_size ) <EOL> self . convert_args = [ updat_temp , "<STR_LIT>" , O , self . determ_shape ] <EOL> else : <EOL> updat_temp = O . gpudata <EOL> image_temp = self . lib . scratch_buffer ( self . image_size ) <EOL> delta_temp = self . lib . scratch_buffer_offset ( self . delta_size ) <EOL> self . convert_args = False <EOL> self . image_args [ <NUM_LIT:2> : <NUM_LIT:5> ] = ( self . lib . stream , image_temp , I . gpudata ) <EOL> self . delta_args [ <NUM_LIT:2> : <NUM_LIT:5> ] = ( self . lib . stream , delta_temp , E . gpudata ) <EOL> if self . zero : <EOL> self . zero_args = [ updat_temp , <NUM_LIT:0> , O . size , self . lib . stream ] <EOL> self . kernel [ <NUM_LIT:3> : <NUM_LIT:8> ] = ( self . lib . stream , updat_temp , image_temp , delta_temp , alpha ) <EOL> def execute ( self , repeat = <NUM_LIT:1> , unbind = True ) : <EOL> kernel = kernel_specs . get_kernel ( self . kernel [ <NUM_LIT:0> ] ) <EOL> image_kernel = _get_update_image_trans_4x4_kernel ( self . dtype_str ) <EOL> delta_kernel = _get_update_delta_trans_4x4_kernel ( self . dtype_str ) <EOL> for r in range ( repeat ) : <EOL> if self . zero : <EOL> drv . memset_d32_async ( * self . zero_args ) <EOL> image_kernel . prepared_async_call ( * self . image_args ) <EOL> delta_kernel . prepared_async_call ( * self . delta_args ) <EOL> kernel . prepared_async_call ( * self . kernel [ <NUM_LIT:1> : ] ) <EOL> if self . convert_args : <EOL> _fp_convert ( * self . convert_args ) <EOL> if unbind : <EOL> self . zero_args = self . convert_args = None <EOL> self . kernel [ <NUM_LIT:3> : <NUM_LIT:8> ] = ( None , ) * <NUM_LIT:5> <EOL> self . image_args [ <NUM_LIT:2> : <NUM_LIT:5> ] = ( None , ) * <NUM_LIT:3> <EOL> self . delta_args [ <NUM_LIT:2> : <NUM_LIT:5> ] = ( None , ) * <NUM_LIT:3> <EOL> def __str__ ( self ) : <EOL> N , C , K , H , W , P , Q , pad_h , pad_w = self . params <EOL> return "<STR_LIT>" % ( self . kernel [ <NUM_LIT:0> ] , N , C , K , H , W ) <EOL> @ context_dependent_memoize <EOL> def _get_fprop_filter_trans_kernel ( dtype ) : <EOL> code = r"""<STR_LIT>""" <EOL> common = _common_round [ "<STR_LIT>" ] . get ( dtype , "<STR_LIT>" ) <EOL> if dtype == "<STR_LIT>" : <EOL> common += _common_fp16_to_fp32 <EOL> code = code % { <EOL> "<STR_LIT>" : common , <EOL> "<STR_LIT:type>" : _ew_types [ dtype ] [ "<STR_LIT:type>" ] , <EOL> "<STR_LIT>" : _ew_types [ dtype ] [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" : _ew_types [ dtype ] [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" : _ew_types [ dtype ] [ "<STR_LIT>" ] , <EOL> } <EOL> module = SourceModule ( code ) <EOL> kernel = module . get_function ( "<STR_LIT>" ) <EOL> kernel . prepare ( "<STR_LIT>" ) <EOL> return kernel <EOL> @ context_dependent_memoize <EOL> def _get_bprop_filter_trans_kernel ( dtype ) : <EOL> code = r"""<STR_LIT>""" <EOL> common = _common_round [ "<STR_LIT>" ] . get ( dtype , "<STR_LIT>" ) <EOL> if dtype == "<STR_LIT>" : <EOL> common += _common_fp16_to_fp32 <EOL> code = code % { <EOL> "<STR_LIT>" : common , <EOL> "<STR_LIT:type>" : _ew_types [ dtype ] [ "<STR_LIT:type>" ] , <EOL> "<STR_LIT>" : _ew_types [ dtype ] [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" : _ew_types [ dtype ] [ "<STR_LIT>" ] , <EOL> } <EOL> module = SourceModule ( code ) <EOL> kernel = module . get_function ( "<STR_LIT>" ) <EOL> kernel . prepare ( "<STR_LIT>" ) <EOL> return kernel <EOL> @ context_dependent_memoize <EOL> def _get_update_image_trans_kernel ( dtype ) : <EOL> code = r"""<STR_LIT>""" <EOL> common = _common_round [ "<STR_LIT>" ] . get ( dtype , "<STR_LIT>" ) <EOL> if dtype == "<STR_LIT>" : <EOL> common += _common_fp16_to_fp32 <EOL> code = code % { <EOL> "<STR_LIT>" : common , <EOL> "<STR_LIT:type>" : _ew_types [ dtype ] [ "<STR_LIT:type>" ] , <EOL> "<STR_LIT>" : _ew_types [ dtype ] [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" : _ew_types [ dtype ] [ "<STR_LIT>" ] , <EOL> } <EOL> module = SourceModule ( code ) <EOL> kernel = module . get_function ( "<STR_LIT>" ) <EOL> kernel . prepare ( "<STR_LIT>" ) <EOL> return kernel <EOL> @ context_dependent_memoize <EOL> def _get_xprop_image_trans_4x4_kernel ( dtype ) : <EOL> code = r"""<STR_LIT>""" <EOL> common = _common_round [ "<STR_LIT>" ] . get ( dtype , "<STR_LIT>" ) <EOL> if dtype == "<STR_LIT>" : <EOL> common += _common_fp16_to_fp32 <EOL> code = code % { <EOL> "<STR_LIT>" : common , <EOL> "<STR_LIT:type>" : _ew_types [ dtype ] [ "<STR_LIT:type>" ] , <EOL> "<STR_LIT>" : _ew_types [ dtype ] [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" : _ew_types [ dtype ] [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" : _ew_types [ dtype ] [ "<STR_LIT>" ] , <EOL> } <EOL> module = SourceModule ( code ) <EOL> kernel = module . get_function ( "<STR_LIT>" ) <EOL> kernel . prepare ( "<STR_LIT>" ) <EOL> return kernel <EOL> @ context_dependent_memoize <EOL> def _get_fprop_filter_trans_4x4_kernel ( dtype ) : <EOL> code = r"""<STR_LIT>""" <EOL> common = _common_round [ "<STR_LIT>" ] . get ( dtype , "<STR_LIT>" ) <EOL> if dtype == "<STR_LIT>" : <EOL> common += _common_fp16_to_fp32 <EOL> code = code % { <EOL> "<STR_LIT>" : common , <EOL> "<STR_LIT:type>" : _ew_types [ dtype ] [ "<STR_LIT:type>" ] , <EOL> "<STR_LIT>" : _ew_types [ dtype ] [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" : _ew_types [ dtype ] [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" : _ew_types [ dtype ] [ "<STR_LIT>" ] , <EOL> } <EOL> module = SourceModule ( code ) <EOL> kernel = module . get_function ( "<STR_LIT>" ) <EOL> kernel . prepare ( "<STR_LIT>" ) <EOL> return kernel <EOL> @ context_dependent_memoize <EOL> def _get_bprop_filter_trans_4x4_kernel ( dtype ) : <EOL> code = r"""<STR_LIT>""" <EOL> common = _common_round [ "<STR_LIT>" ] . get ( dtype , "<STR_LIT>" ) <EOL> if dtype == "<STR_LIT>" : <EOL> common += _common_fp16_to_fp32 <EOL> code = code % { <EOL> "<STR_LIT>" : common , <EOL> "<STR_LIT:type>" : _ew_types [ dtype ] [ "<STR_LIT:type>" ] , <EOL> "<STR_LIT>" : _ew_types [ dtype ] [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" : _ew_types [ dtype ] [ "<STR_LIT>" ] , <EOL> } <EOL> module = SourceModule ( code ) <EOL> kernel = module . get_function ( "<STR_LIT>" ) <EOL> kernel . prepare ( "<STR_LIT>" ) <EOL> return kernel <EOL> @ context_dependent_memoize <EOL> def _get_update_image_trans_4x4_kernel ( dtype ) : <EOL> code = r"""<STR_LIT>""" <EOL> common = _common_round [ "<STR_LIT>" ] . get ( dtype , "<STR_LIT>" ) <EOL> if dtype == "<STR_LIT>" : <EOL> common += _common_fp16_to_fp32 <EOL> code = code % { <EOL> "<STR_LIT>" : common , <EOL> "<STR_LIT:type>" : _ew_types [ dtype ] [ "<STR_LIT:type>" ] , <EOL> "<STR_LIT>" : _ew_types [ dtype ] [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" : _ew_types [ dtype ] [ "<STR_LIT>" ] , <EOL> } <EOL> module = SourceModule ( code ) <EOL> kernel = module . get_function ( "<STR_LIT>" ) <EOL> kernel . prepare ( "<STR_LIT>" ) <EOL> return kernel <EOL> @ context_dependent_memoize <EOL> def _get_update_delta_trans_4x4_kernel ( dtype ) : <EOL> code = r"""<STR_LIT>""" <EOL> common = _common_round [ "<STR_LIT>" ] . get ( dtype , "<STR_LIT>" ) <EOL> if dtype == "<STR_LIT>" : <EOL> common += _common_fp16_to_fp32 <EOL> code = code % { <EOL> "<STR_LIT>" : common , <EOL> "<STR_LIT:type>" : _ew_types [ dtype ] [ "<STR_LIT:type>" ] , <EOL> "<STR_LIT>" : _ew_types [ dtype ] [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" : _ew_types [ dtype ] [ "<STR_LIT>" ] , <EOL> } <EOL> module = SourceModule ( code ) <EOL> kernel = module . get_function ( "<STR_LIT>" ) <EOL> kernel . prepare ( "<STR_LIT>" ) <EOL> return kernel </s>
