<s> from __future__ import print_function <EOL> from time import time <EOL> import subprocess <EOL> import random <EOL> import numpy <EOL> STEP = <NUM_LIT:1000> * <NUM_LIT:100> <EOL> SCALE = <NUM_LIT:0.1> <EOL> NI_NTIMES = <NUM_LIT:1> <EOL> MROW = <NUM_LIT:1000> * <NUM_LIT> <EOL> COLDCACHE = <NUM_LIT:5> <EOL> WARMCACHE = <NUM_LIT:5> <EOL> READ_TIMES = <NUM_LIT:10> <EOL> rdm_cod = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> prec = <NUM_LIT:6> <EOL> def get_nrows ( nrows_str ) : <EOL> if nrows_str . endswith ( "<STR_LIT:k>" ) : <EOL> return int ( float ( nrows_str [ : - <NUM_LIT:1> ] ) * <NUM_LIT:1000> ) <EOL> elif nrows_str . endswith ( "<STR_LIT:m>" ) : <EOL> return int ( float ( nrows_str [ : - <NUM_LIT:1> ] ) * <NUM_LIT:1000> * <NUM_LIT:1000> ) <EOL> elif nrows_str . endswith ( "<STR_LIT:g>" ) : <EOL> return int ( float ( nrows_str [ : - <NUM_LIT:1> ] ) * <NUM_LIT:1000> * <NUM_LIT:1000> * <NUM_LIT:1000> ) <EOL> else : <EOL> raise ValueError ( <EOL> "<STR_LIT>" ) <EOL> class DB ( object ) : <EOL> def __init__ ( self , nrows , rng , userandom ) : <EOL> global step , scale <EOL> self . step = STEP <EOL> self . scale = SCALE <EOL> self . rng = rng <EOL> self . userandom = userandom <EOL> self . filename = '<STR_LIT:->' . join ( [ rdm_cod [ userandom ] , nrows ] ) <EOL> self . nrows = get_nrows ( nrows ) <EOL> def get_db_size ( self ) : <EOL> sout = subprocess . Popen ( "<STR_LIT>" % self . filename , shell = True , <EOL> stdout = subprocess . PIPE ) . stdout <EOL> line = [ l for l in sout ] [ <NUM_LIT:0> ] <EOL> return int ( line . split ( ) [ <NUM_LIT:0> ] ) <EOL> def print_mtime ( self , t1 , explain ) : <EOL> mtime = time ( ) - t1 <EOL> print ( "<STR_LIT>" % explain , round ( mtime , <NUM_LIT:6> ) ) <EOL> print ( "<STR_LIT>" , round ( ( self . nrows / <NUM_LIT> ) / mtime , <NUM_LIT:6> ) ) <EOL> def print_qtime ( self , colname , ltimes ) : <EOL> qtime1 = ltimes [ <NUM_LIT:0> ] <EOL> qtime2 = ltimes [ - <NUM_LIT:1> ] <EOL> print ( "<STR_LIT>" % colname , round ( qtime1 , <NUM_LIT:6> ) ) <EOL> print ( "<STR_LIT>" , round ( ( self . nrows / ( MROW ) ) / qtime1 , <NUM_LIT:6> ) ) <EOL> print ( "<STR_LIT>" % colname , round ( qtime2 , <NUM_LIT:6> ) ) <EOL> print ( "<STR_LIT>" , round ( ( self . nrows / ( MROW ) ) / qtime2 , <NUM_LIT:6> ) ) <EOL> def norm_times ( self , ltimes ) : <EOL> "<STR_LIT>" <EOL> lmean = ltimes . mean ( ) <EOL> lstd = ltimes . std ( ) <EOL> ntimes = ltimes [ ltimes < lmean + lstd ] <EOL> nmean = ntimes . mean ( ) <EOL> nstd = ntimes . std ( ) <EOL> return nmean , nstd <EOL> def print_qtime_idx ( self , colname , ltimes , repeated , verbose ) : <EOL> if repeated : <EOL> r = "<STR_LIT>" <EOL> else : <EOL> r = "<STR_LIT>" <EOL> ltimes = numpy . array ( ltimes ) <EOL> ntimes = len ( ltimes ) <EOL> qtime1 = ltimes [ <NUM_LIT:0> ] <EOL> ctimes = ltimes [ <NUM_LIT:1> : COLDCACHE ] <EOL> cmean , cstd = self . norm_times ( ctimes ) <EOL> wtimes = ltimes [ WARMCACHE : ] <EOL> wmean , wstd = self . norm_times ( wtimes ) <EOL> if verbose : <EOL> print ( "<STR_LIT>" , ctimes ) <EOL> print ( "<STR_LIT>" % <EOL> numpy . histogram ( wtimes ) ) <EOL> print ( "<STR_LIT>" % ( r , colname ) , <EOL> round ( qtime1 , prec ) ) <EOL> print ( "<STR_LIT>" % ( r , colname ) , <EOL> round ( cmean , prec ) , "<STR_LIT>" , round ( cstd , prec ) ) <EOL> print ( "<STR_LIT>" % ( r , colname ) , <EOL> round ( wmean , prec ) , "<STR_LIT>" , round ( wstd , prec ) ) <EOL> def print_db_sizes ( self , init , filled , indexed ) : <EOL> table_size = ( filled - init ) / <NUM_LIT> <EOL> indexes_size = ( indexed - filled ) / <NUM_LIT> <EOL> print ( "<STR_LIT>" , round ( table_size , <NUM_LIT:3> ) ) <EOL> print ( "<STR_LIT>" , round ( indexes_size , <NUM_LIT:3> ) ) <EOL> print ( "<STR_LIT>" , round ( table_size + indexes_size , <NUM_LIT:3> ) ) <EOL> def fill_arrays ( self , start , stop ) : <EOL> arr_f8 = numpy . arange ( start , stop , dtype = '<STR_LIT>' ) <EOL> arr_i4 = numpy . arange ( start , stop , dtype = '<STR_LIT>' ) <EOL> if self . userandom : <EOL> arr_f8 += numpy . random . normal ( <NUM_LIT:0> , stop * self . scale , <EOL> size = stop - start ) <EOL> arr_i4 = numpy . array ( arr_f8 , dtype = '<STR_LIT>' ) <EOL> return arr_i4 , arr_f8 <EOL> def create_db ( self , dtype , kind , optlevel , verbose ) : <EOL> self . con = self . open_db ( remove = <NUM_LIT:1> ) <EOL> self . create_table ( self . con ) <EOL> init_size = self . get_db_size ( ) <EOL> t1 = time ( ) <EOL> self . fill_table ( self . con ) <EOL> table_size = self . get_db_size ( ) <EOL> self . print_mtime ( t1 , '<STR_LIT>' ) <EOL> self . index_db ( dtype , kind , optlevel , verbose ) <EOL> indexes_size = self . get_db_size ( ) <EOL> self . print_db_sizes ( init_size , table_size , indexes_size ) <EOL> self . close_db ( self . con ) <EOL> def index_db ( self , dtype , kind , optlevel , verbose ) : <EOL> if dtype == "<STR_LIT:int>" : <EOL> idx_cols = [ '<STR_LIT>' ] <EOL> elif dtype == "<STR_LIT:float>" : <EOL> idx_cols = [ '<STR_LIT>' ] <EOL> else : <EOL> idx_cols = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> for colname in idx_cols : <EOL> t1 = time ( ) <EOL> self . index_col ( self . con , colname , kind , optlevel , verbose ) <EOL> self . print_mtime ( t1 , '<STR_LIT>' % colname ) <EOL> def query_db ( self , niter , dtype , onlyidxquery , onlynonidxquery , <EOL> avoidfscache , verbose , inkernel ) : <EOL> self . con = self . open_db ( ) <EOL> if dtype == "<STR_LIT:int>" : <EOL> reg_cols = [ '<STR_LIT>' ] <EOL> idx_cols = [ '<STR_LIT>' ] <EOL> elif dtype == "<STR_LIT:float>" : <EOL> reg_cols = [ '<STR_LIT>' ] <EOL> idx_cols = [ '<STR_LIT>' ] <EOL> else : <EOL> reg_cols = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> idx_cols = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> if avoidfscache : <EOL> rseed = int ( numpy . random . randint ( self . nrows ) ) <EOL> else : <EOL> rseed = <NUM_LIT> <EOL> numpy . random . seed ( rseed ) <EOL> base = numpy . random . randint ( self . nrows ) <EOL> if not onlyidxquery : <EOL> for colname in reg_cols : <EOL> ltimes = [ ] <EOL> random . seed ( rseed ) <EOL> for i in range ( NI_NTIMES ) : <EOL> t1 = time ( ) <EOL> results = self . do_query ( self . con , colname , base , inkernel ) <EOL> ltimes . append ( time ( ) - t1 ) <EOL> if verbose : <EOL> print ( "<STR_LIT>" , results ) <EOL> self . print_qtime ( colname , ltimes ) <EOL> self . close_db ( self . con ) <EOL> self . con = self . open_db ( ) <EOL> if not onlynonidxquery : <EOL> for colname in idx_cols : <EOL> ltimes = [ ] <EOL> numpy . random . seed ( rseed ) <EOL> rndbase = numpy . random . randint ( self . nrows , size = niter ) <EOL> for i in range ( niter ) : <EOL> base = rndbase [ i ] <EOL> t1 = time ( ) <EOL> results = self . do_query ( self . con , colname , base , inkernel ) <EOL> ltimes . append ( time ( ) - t1 ) <EOL> if verbose : <EOL> print ( "<STR_LIT>" , results ) <EOL> self . print_qtime_idx ( colname , ltimes , False , verbose ) <EOL> self . close_db ( self . con ) <EOL> self . con = self . open_db ( ) <EOL> ltimes = [ ] <EOL> self . close_db ( self . con ) <EOL> self . con = self . open_db ( ) <EOL> self . close_db ( self . con ) <EOL> def close_db ( self , con ) : <EOL> con . close ( ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> import sys <EOL> import getopt <EOL> try : <EOL> import psyco <EOL> psyco_imported = <NUM_LIT:1> <EOL> except : <EOL> psyco_imported = <NUM_LIT:0> <EOL> usage = """<STR_LIT>""" % sys . argv [ <NUM_LIT:0> ] <EOL> try : <EOL> opts , pargs = getopt . getopt ( <EOL> sys . argv [ <NUM_LIT:1> : ] , '<STR_LIT>' ) <EOL> except : <EOL> sys . stderr . write ( usage ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> usepytables = <NUM_LIT:0> <EOL> usepostgres = <NUM_LIT:0> <EOL> verbose = <NUM_LIT:0> <EOL> doprofile = <NUM_LIT:0> <EOL> dokprofile = <NUM_LIT:0> <EOL> usepsyco = <NUM_LIT:0> <EOL> userandom = <NUM_LIT:0> <EOL> docreate = <NUM_LIT:0> <EOL> optlevel = <NUM_LIT:0> <EOL> kind = "<STR_LIT>" <EOL> docompress = <NUM_LIT:0> <EOL> complib = "<STR_LIT>" <EOL> doquery = False <EOL> onlyidxquery = False <EOL> onlynonidxquery = False <EOL> inkernel = True <EOL> avoidfscache = <NUM_LIT:0> <EOL> rng = [ - <NUM_LIT:1000> , - <NUM_LIT:1000> ] <EOL> repeatquery = <NUM_LIT:0> <EOL> repeatvalue = <NUM_LIT:0> <EOL> krows = '<STR_LIT>' <EOL> niter = READ_TIMES <EOL> dtype = "<STR_LIT:all>" <EOL> datadir = "<STR_LIT>" <EOL> for option in opts : <EOL> if option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> usepytables = <NUM_LIT:1> <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> usepostgres = <NUM_LIT:1> <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> verbose = <NUM_LIT:1> <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> doprofile = <NUM_LIT:1> <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> dokprofile = <NUM_LIT:1> <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> usepsyco = <NUM_LIT:1> <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> userandom = <NUM_LIT:1> <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT:-c>' : <EOL> docreate = <NUM_LIT:1> <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> doquery = True <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> doquery = True <EOL> onlyidxquery = True <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> doquery = True <EOL> onlynonidxquery = True <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> doquery = True <EOL> onlynonidxquery = True <EOL> inkernel = False <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> avoidfscache = <NUM_LIT:1> <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> docompress = int ( option [ <NUM_LIT:1> ] ) <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> complib = option [ <NUM_LIT:1> ] <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> rng = [ int ( i ) for i in option [ <NUM_LIT:1> ] . split ( "<STR_LIT:U+002C>" ) ] <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> niter = int ( option [ <NUM_LIT:1> ] ) <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> krows = option [ <NUM_LIT:1> ] <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> datadir = option [ <NUM_LIT:1> ] <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> optlevel = int ( option [ <NUM_LIT:1> ] ) <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> if option [ <NUM_LIT:1> ] in ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) : <EOL> kind = option [ <NUM_LIT:1> ] <EOL> else : <EOL> print ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> if option [ <NUM_LIT:1> ] in ( '<STR_LIT:int>' , '<STR_LIT:float>' ) : <EOL> dtype = option [ <NUM_LIT:1> ] <EOL> else : <EOL> print ( "<STR_LIT>" ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> elif option [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> repeatquery = <NUM_LIT:1> <EOL> repeatvalue = int ( option [ <NUM_LIT:1> ] ) <EOL> if not usepytables and not usepostgres : <EOL> print ( "<STR_LIT>" ) <EOL> print ( "<STR_LIT>" ) <EOL> print ( "<STR_LIT>" ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> if usepytables : <EOL> from pytables_backend import PyTables_DB <EOL> db = PyTables_DB ( krows , rng , userandom , datadir , <EOL> docompress , complib , kind , optlevel ) <EOL> elif usepostgres : <EOL> from postgres_backend import Postgres_DB <EOL> db = Postgres_DB ( krows , rng , userandom ) <EOL> if not avoidfscache : <EOL> numpy . random . seed ( <NUM_LIT:20> ) <EOL> if verbose : <EOL> if userandom : <EOL> print ( "<STR_LIT>" ) <EOL> if onlyidxquery : <EOL> print ( "<STR_LIT>" ) <EOL> if psyco_imported and usepsyco : <EOL> psyco . bind ( db . create_db ) <EOL> psyco . bind ( db . query_db ) <EOL> if docreate : <EOL> if verbose : <EOL> print ( "<STR_LIT>" % krows ) <EOL> db . create_db ( dtype , kind , optlevel , verbose ) <EOL> if doquery : <EOL> print ( "<STR_LIT>" % niter ) <EOL> if doprofile : <EOL> import pstats <EOL> import cProfile as prof <EOL> prof . run ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' ) <EOL> stats = pstats . Stats ( '<STR_LIT>' ) <EOL> stats . strip_dirs ( ) <EOL> stats . sort_stats ( '<STR_LIT:time>' , '<STR_LIT>' ) <EOL> if verbose : <EOL> stats . print_stats ( ) <EOL> else : <EOL> stats . print_stats ( <NUM_LIT:20> ) <EOL> elif dokprofile : <EOL> from cProfile import Profile <EOL> import lsprofcalltree <EOL> prof = Profile ( ) <EOL> prof . run ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> kcg = lsprofcalltree . KCacheGrind ( prof ) <EOL> ofile = open ( '<STR_LIT>' , '<STR_LIT:w>' ) <EOL> kcg . output ( ofile ) <EOL> ofile . close ( ) <EOL> elif doprofile : <EOL> import hotshot <EOL> import hotshot . stats <EOL> prof = hotshot . Profile ( "<STR_LIT>" ) <EOL> benchtime , stones = prof . run ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> prof . close ( ) <EOL> stats = hotshot . stats . load ( "<STR_LIT>" ) <EOL> stats . strip_dirs ( ) <EOL> stats . sort_stats ( '<STR_LIT:time>' , '<STR_LIT>' ) <EOL> stats . print_stats ( <NUM_LIT:20> ) <EOL> else : <EOL> db . query_db ( niter , dtype , onlyidxquery , onlynonidxquery , <EOL> avoidfscache , verbose , inkernel ) <EOL> if repeatquery : <EOL> db . rng = [ <NUM_LIT:1> , <NUM_LIT:1> ] <EOL> if verbose : <EOL> print ( "<STR_LIT>" , db . rng ) <EOL> db . query_db ( niter , dtype , onlyidxquery , onlynonidxquery , <EOL> avoidfscache , verbose , inkernel ) <EOL> for i in range ( repeatvalue ) : <EOL> for j in ( <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:5> ) : <EOL> rng = j * <NUM_LIT:10> ** i <EOL> db . rng = [ - rng / <NUM_LIT:2> , rng / <NUM_LIT:2> ] <EOL> if verbose : <EOL> print ( "<STR_LIT>" , db . rng ) <EOL> db . query_db ( niter , dtype , onlyidxquery , onlynonidxquery , <EOL> avoidfscache , verbose , inkernel ) </s>
