<s> import datetime <EOL> import hashlib <EOL> import heapq <EOL> import math <EOL> import os <EOL> import random <EOL> import re <EOL> import sys <EOL> import threading <EOL> import zlib <EOL> try : <EOL> from collections import Counter <EOL> except ImportError : <EOL> Counter = None <EOL> try : <EOL> from urlparse import urlparse <EOL> except ImportError : <EOL> from urllib . parse import urlparse <EOL> try : <EOL> from vtfunc import TableFunction <EOL> except ImportError : <EOL> TableFunction = None <EOL> from peewee import binary_construct <EOL> from peewee import unicode_type <EOL> try : <EOL> from playhouse . _speedups import format_date_time_sqlite <EOL> except ImportError : <EOL> from peewee import format_date_time <EOL> from peewee import SQLITE_DATETIME_FORMATS <EOL> def format_date_time_sqlite ( date_value ) : <EOL> return format_date_time ( date_value , SQLITE_DATETIME_FORMATS ) <EOL> try : <EOL> from playhouse import _sqlite_udf as cython_udf <EOL> except ImportError : <EOL> cython_udf = None <EOL> CONTROL_FLOW = '<STR_LIT>' <EOL> DATE = '<STR_LIT:date>' <EOL> FILE = '<STR_LIT:file>' <EOL> HELPER = '<STR_LIT>' <EOL> MATH = '<STR_LIT>' <EOL> STRING = '<STR_LIT:string>' <EOL> AGGREGATE_COLLECTION = { } <EOL> TABLE_FUNCTION_COLLECTION = { } <EOL> UDF_COLLECTION = { } <EOL> class synchronized_dict ( dict ) : <EOL> def __init__ ( self , * args , ** kwargs ) : <EOL> super ( synchronized_dict , self ) . __init__ ( * args , ** kwargs ) <EOL> self . _lock = threading . Lock ( ) <EOL> def __getitem__ ( self , key ) : <EOL> with self . _lock : <EOL> return super ( synchronized_dict , self ) . __getitem__ ( key ) <EOL> def __setitem__ ( self , key , value ) : <EOL> with self . _lock : <EOL> return super ( synchronized_dict , self ) . __setitem__ ( key , value ) <EOL> def __delitem__ ( self , key ) : <EOL> with self . _lock : <EOL> return super ( synchronized_dict , self ) . __delitem__ ( key ) <EOL> STATE = synchronized_dict ( ) <EOL> SETTINGS = synchronized_dict ( ) <EOL> def aggregate ( * groups ) : <EOL> def decorator ( klass ) : <EOL> for group in groups : <EOL> AGGREGATE_COLLECTION . setdefault ( group , [ ] ) <EOL> AGGREGATE_COLLECTION [ group ] . append ( klass ) <EOL> return klass <EOL> return decorator <EOL> def table_function ( * groups ) : <EOL> def decorator ( klass ) : <EOL> for group in groups : <EOL> TABLE_FUNCTION_COLLECTION . setdefault ( group , [ ] ) <EOL> TABLE_FUNCTION_COLLECTION [ group ] . append ( klass ) <EOL> return klass <EOL> return decorator <EOL> def udf ( * groups ) : <EOL> def decorator ( fn ) : <EOL> for group in groups : <EOL> UDF_COLLECTION . setdefault ( group , [ ] ) <EOL> UDF_COLLECTION [ group ] . append ( fn ) <EOL> return fn <EOL> return decorator <EOL> def register_aggregate_groups ( conn , * groups ) : <EOL> seen = set ( ) <EOL> for group in groups : <EOL> klasses = AGGREGATE_COLLECTION [ group ] <EOL> for klass in klasses : <EOL> name = getattr ( klass , '<STR_LIT:name>' , klass . __name__ ) <EOL> if name not in seen : <EOL> seen . add ( name ) <EOL> conn . create_aggregate ( name , - <NUM_LIT:1> , klass ) <EOL> def register_table_function_groups ( conn , * groups ) : <EOL> seen = set ( ) <EOL> for group in groups : <EOL> klasses = TABLE_FUNCTION_COLLECTION [ group ] <EOL> for klass in klasses : <EOL> if klass . name not in seen : <EOL> seen . add ( klass . name ) <EOL> klass . register ( conn ) <EOL> def register_udf_groups ( conn , * groups ) : <EOL> seen = set ( ) <EOL> for group in groups : <EOL> functions = UDF_COLLECTION [ group ] <EOL> for function in functions : <EOL> name = function . __name__ <EOL> if name not in seen : <EOL> seen . add ( name ) <EOL> conn . create_function ( name , - <NUM_LIT:1> , function ) <EOL> def register_all ( conn ) : <EOL> register_aggregate_groups ( conn , * AGGREGATE_COLLECTION ) <EOL> register_table_function_groups ( conn , * TABLE_FUNCTION_COLLECTION ) <EOL> register_udf_groups ( conn , * UDF_COLLECTION ) <EOL> @ udf ( CONTROL_FLOW ) <EOL> def if_then_else ( cond , truthy , falsey = None ) : <EOL> if cond : <EOL> return truthy <EOL> return falsey <EOL> @ udf ( DATE ) <EOL> def strip_tz ( date_str ) : <EOL> date_str = date_str . replace ( '<STR_LIT:T>' , '<STR_LIT:U+0020>' ) <EOL> tz_idx1 = date_str . find ( '<STR_LIT:+>' ) <EOL> if tz_idx1 != - <NUM_LIT:1> : <EOL> return date_str [ : tz_idx1 ] <EOL> tz_idx2 = date_str . find ( '<STR_LIT:->' ) <EOL> if tz_idx2 > <NUM_LIT> : <EOL> return date_str [ : tz_idx2 ] <EOL> return date_str <EOL> @ udf ( DATE ) <EOL> def human_delta ( nseconds , glue = '<STR_LIT:U+002CU+0020>' ) : <EOL> parts = ( <EOL> ( <NUM_LIT> * <NUM_LIT> , '<STR_LIT>' ) , <EOL> ( <NUM_LIT> * <NUM_LIT:30> , '<STR_LIT>' ) , <EOL> ( <NUM_LIT> * <NUM_LIT:7> , '<STR_LIT>' ) , <EOL> ( <NUM_LIT> , '<STR_LIT>' ) , <EOL> ( <NUM_LIT> , '<STR_LIT>' ) , <EOL> ( <NUM_LIT> , '<STR_LIT>' ) , <EOL> ( <NUM_LIT:1> , '<STR_LIT>' ) , <EOL> ) <EOL> accum = [ ] <EOL> for offset , name in parts : <EOL> val , nseconds = divmod ( nseconds , offset ) <EOL> if val : <EOL> suffix = val != <NUM_LIT:1> and '<STR_LIT:s>' or '<STR_LIT>' <EOL> accum . append ( '<STR_LIT>' % ( val , name , suffix ) ) <EOL> if not accum : <EOL> return '<STR_LIT>' <EOL> return glue . join ( accum ) <EOL> @ udf ( FILE ) <EOL> def file_ext ( filename ) : <EOL> try : <EOL> res = os . path . splitext ( filename ) <EOL> except ValueError : <EOL> return None <EOL> return res [ <NUM_LIT:1> ] <EOL> @ udf ( FILE ) <EOL> def file_read ( filename ) : <EOL> try : <EOL> with open ( filename ) as fh : <EOL> return fh . read ( ) <EOL> except : <EOL> pass <EOL> if sys . version_info [ <NUM_LIT:0> ] == <NUM_LIT:2> : <EOL> @ udf ( HELPER ) <EOL> def gzip ( data , compression = <NUM_LIT:9> ) : <EOL> return binary_construct ( zlib . compress ( data , compression ) ) <EOL> @ udf ( HELPER ) <EOL> def gunzip ( data ) : <EOL> return zlib . decompress ( data ) <EOL> else : <EOL> @ udf ( HELPER ) <EOL> def gzip ( data , compression = <NUM_LIT:9> ) : <EOL> return zlib . compress ( binary_construct ( data ) , compression ) <EOL> @ udf ( HELPER ) <EOL> def gunzip ( data ) : <EOL> return zlib . decompress ( data ) . decode ( '<STR_LIT:utf-8>' ) <EOL> @ udf ( HELPER ) <EOL> def hostname ( url ) : <EOL> parse_result = urlparse ( url ) <EOL> if parse_result : <EOL> return parse_result . netloc <EOL> @ udf ( HELPER ) <EOL> def toggle ( key , on = None ) : <EOL> key = key . lower ( ) <EOL> if on is not None : <EOL> STATE [ key ] = on <EOL> else : <EOL> STATE [ key ] = on = not STATE . get ( key ) <EOL> return on <EOL> @ udf ( HELPER ) <EOL> def setting ( key , * args ) : <EOL> if not args : <EOL> return SETTINGS . get ( key ) <EOL> elif len ( args ) == <NUM_LIT:1> : <EOL> SETTINGS [ key ] = args [ <NUM_LIT:0> ] <EOL> else : <EOL> return False <EOL> @ udf ( HELPER ) <EOL> def clear_settings ( ) : <EOL> SETTINGS . clear ( ) <EOL> @ udf ( HELPER ) <EOL> def clear_toggles ( ) : <EOL> STATE . clear ( ) <EOL> @ udf ( MATH ) <EOL> def randomrange ( start , end = None , step = None ) : <EOL> if end is None : <EOL> start , end = <NUM_LIT:0> , start <EOL> elif step is None : <EOL> step = <NUM_LIT:1> <EOL> return random . randrange ( start , end , step ) <EOL> @ udf ( MATH ) <EOL> def gauss_distribution ( mean , sigma ) : <EOL> try : <EOL> return random . gauss ( mean , sigma ) <EOL> except ValueError : <EOL> return None <EOL> @ udf ( MATH ) <EOL> def sqrt ( n ) : <EOL> try : <EOL> return math . sqrt ( n ) <EOL> except ValueError : <EOL> return None <EOL> @ udf ( MATH ) <EOL> def tonumber ( s ) : <EOL> try : <EOL> return int ( s ) <EOL> except ValueError : <EOL> try : <EOL> return float ( s ) <EOL> except : <EOL> return None <EOL> @ udf ( STRING ) <EOL> def substr_count ( haystack , needle ) : <EOL> if not haystack or not needle : <EOL> return <NUM_LIT:0> <EOL> return haystack . count ( needle ) <EOL> @ udf ( STRING ) <EOL> def strip_chars ( haystack , chars ) : <EOL> return unicode_type ( haystack ) . strip ( chars ) <EOL> def _hash ( constructor , * args ) : <EOL> hash_obj = constructor ( ) <EOL> for arg in args : <EOL> hash_obj . update ( arg ) <EOL> return hash_obj . hexdigest ( ) <EOL> @ udf ( STRING ) <EOL> def md5 ( * vals ) : <EOL> return _hash ( hashlib . md5 ) <EOL> @ udf ( STRING ) <EOL> def sha1 ( * vals ) : <EOL> return _hash ( hashlib . sha1 ) <EOL> @ udf ( STRING ) <EOL> def sha256 ( * vals ) : <EOL> return _hash ( hashlib . sha256 ) <EOL> @ udf ( STRING ) <EOL> def sha512 ( * vals ) : <EOL> return _hash ( hashlib . sha512 ) <EOL> @ udf ( STRING ) <EOL> def adler32 ( s ) : <EOL> return zlib . adler32 ( s ) <EOL> @ udf ( STRING ) <EOL> def crc32 ( s ) : <EOL> return zlib . crc32 ( s ) <EOL> class _heap_agg ( object ) : <EOL> def __init__ ( self ) : <EOL> self . heap = [ ] <EOL> self . ct = <NUM_LIT:0> <EOL> def process ( self , value ) : <EOL> return value <EOL> def step ( self , value ) : <EOL> self . ct += <NUM_LIT:1> <EOL> heapq . heappush ( self . heap , self . process ( value ) ) <EOL> class _datetime_heap_agg ( _heap_agg ) : <EOL> def process ( self , value ) : <EOL> return format_date_time_sqlite ( value ) <EOL> if sys . version_info [ : <NUM_LIT:2> ] == ( <NUM_LIT:2> , <NUM_LIT:6> ) : <EOL> def total_seconds ( td ) : <EOL> return ( td . seconds + <EOL> ( td . days * <NUM_LIT> ) + <EOL> ( td . microseconds / ( <NUM_LIT> ** <NUM_LIT:6> ) ) ) <EOL> else : <EOL> total_seconds = lambda td : td . total_seconds ( ) <EOL> @ aggregate ( DATE ) <EOL> class mintdiff ( _datetime_heap_agg ) : <EOL> def finalize ( self ) : <EOL> dtp = min_diff = None <EOL> while self . heap : <EOL> if min_diff is None : <EOL> if dtp is None : <EOL> dtp = heapq . heappop ( self . heap ) <EOL> continue <EOL> dt = heapq . heappop ( self . heap ) <EOL> diff = dt - dtp <EOL> if min_diff is None or min_diff > diff : <EOL> min_diff = diff <EOL> dtp = dt <EOL> if min_diff is not None : <EOL> return total_seconds ( min_diff ) <EOL> @ aggregate ( DATE ) <EOL> class avgtdiff ( _datetime_heap_agg ) : <EOL> def finalize ( self ) : <EOL> if self . ct < <NUM_LIT:1> : <EOL> return <EOL> elif self . ct == <NUM_LIT:1> : <EOL> return <NUM_LIT:0> <EOL> total = ct = <NUM_LIT:0> <EOL> dtp = None <EOL> while self . heap : <EOL> if total == <NUM_LIT:0> : <EOL> if dtp is None : <EOL> dtp = heapq . heappop ( self . heap ) <EOL> continue <EOL> dt = heapq . heappop ( self . heap ) <EOL> diff = dt - dtp <EOL> ct += <NUM_LIT:1> <EOL> total += total_seconds ( diff ) <EOL> dtp = dt <EOL> return float ( total ) / ct <EOL> @ aggregate ( DATE ) <EOL> class duration ( object ) : <EOL> def __init__ ( self ) : <EOL> self . _min = self . _max = None <EOL> def step ( self , value ) : <EOL> dt = format_date_time_sqlite ( value ) <EOL> if self . _min is None or dt < self . _min : <EOL> self . _min = dt <EOL> if self . _max is None or dt > self . _max : <EOL> self . _max = dt <EOL> def finalize ( self ) : <EOL> if self . _min and self . _max : <EOL> td = ( self . _max - self . _min ) <EOL> return total_seconds ( td ) <EOL> return None <EOL> @ aggregate ( MATH ) <EOL> class mode ( object ) : <EOL> if Counter : <EOL> def __init__ ( self ) : <EOL> self . items = Counter ( ) <EOL> def step ( self , * args ) : <EOL> self . items . update ( args ) <EOL> def finalize ( self ) : <EOL> if self . items : <EOL> return self . items . most_common ( <NUM_LIT:1> ) [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] <EOL> else : <EOL> def __init__ ( self ) : <EOL> self . items = [ ] <EOL> def step ( self , item ) : <EOL> self . items . append ( item ) <EOL> def finalize ( self ) : <EOL> if self . items : <EOL> return max ( set ( self . items ) , key = self . items . count ) <EOL> @ aggregate ( MATH ) <EOL> class minrange ( _heap_agg ) : <EOL> def finalize ( self ) : <EOL> if self . ct == <NUM_LIT:0> : <EOL> return <EOL> elif self . ct == <NUM_LIT:1> : <EOL> return <NUM_LIT:0> <EOL> prev = min_diff = None <EOL> while self . heap : <EOL> if min_diff is None : <EOL> if prev is None : <EOL> prev = heapq . heappop ( self . heap ) <EOL> continue <EOL> curr = heapq . heappop ( self . heap ) <EOL> diff = curr - prev <EOL> if min_diff is None or min_diff > diff : <EOL> min_diff = diff <EOL> prev = curr <EOL> return min_diff <EOL> @ aggregate ( MATH ) <EOL> class avgrange ( _heap_agg ) : <EOL> def finalize ( self ) : <EOL> if self . ct == <NUM_LIT:0> : <EOL> return <EOL> elif self . ct == <NUM_LIT:1> : <EOL> return <NUM_LIT:0> <EOL> total = ct = <NUM_LIT:0> <EOL> prev = None <EOL> while self . heap : <EOL> if total == <NUM_LIT:0> : <EOL> if prev is None : <EOL> prev = heapq . heappop ( self . heap ) <EOL> continue <EOL> curr = heapq . heappop ( self . heap ) <EOL> diff = curr - prev <EOL> ct += <NUM_LIT:1> <EOL> total += diff <EOL> prev = curr <EOL> return float ( total ) / ct <EOL> @ aggregate ( MATH ) <EOL> class _range ( object ) : <EOL> name = '<STR_LIT>' <EOL> def __init__ ( self ) : <EOL> self . _min = self . _max = None <EOL> def step ( self , value ) : <EOL> if self . _min is None or value < self . _min : <EOL> self . _min = value <EOL> if self . _max is None or value > self . _max : <EOL> self . _max = value <EOL> def finalize ( self ) : <EOL> if self . _min is not None and self . _max is not None : <EOL> return self . _max - self . _min <EOL> return None <EOL> if cython_udf is not None : <EOL> damerau_levenshtein_dist = udf ( STRING ) ( cython_udf . damerau_levenshtein_dist ) <EOL> levenshtein_dist = udf ( STRING ) ( cython_udf . levenshtein_dist ) <EOL> str_dist = udf ( STRING ) ( cython_udf . str_dist ) <EOL> median = aggregate ( MATH ) ( cython_udf . median ) <EOL> if TableFunction is not None : <EOL> @ table_function ( STRING ) <EOL> class RegexSearch ( TableFunction ) : <EOL> params = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> columns = [ '<STR_LIT>' ] <EOL> name = '<STR_LIT>' <EOL> def initialize ( self , regex = None , search_string = None ) : <EOL> self . _iter = re . finditer ( regex , search_string ) <EOL> def iterate ( self , idx ) : <EOL> return ( next ( self . _iter ) . group ( <NUM_LIT:0> ) , ) <EOL> @ table_function ( DATE ) <EOL> class DateSeries ( TableFunction ) : <EOL> params = [ '<STR_LIT:start>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> columns = [ '<STR_LIT:date>' ] <EOL> name = '<STR_LIT>' <EOL> def initialize ( self , start , stop , step_seconds = <NUM_LIT> ) : <EOL> self . start = format_date_time_sqlite ( start ) <EOL> self . stop = format_date_time_sqlite ( stop ) <EOL> step_seconds = int ( step_seconds ) <EOL> self . step_seconds = datetime . timedelta ( seconds = step_seconds ) <EOL> if ( self . start . hour == <NUM_LIT:0> and <EOL> self . start . minute == <NUM_LIT:0> and <EOL> self . start . second == <NUM_LIT:0> and <EOL> step_seconds >= <NUM_LIT> ) : <EOL> self . format = '<STR_LIT>' <EOL> elif ( self . start . year == <NUM_LIT> and <EOL> self . start . month == <NUM_LIT:1> and <EOL> self . start . day == <NUM_LIT:1> and <EOL> self . stop . year == <NUM_LIT> and <EOL> self . stop . month == <NUM_LIT:1> and <EOL> self . stop . day == <NUM_LIT:1> and <EOL> step_seconds < <NUM_LIT> ) : <EOL> self . format = '<STR_LIT>' <EOL> else : <EOL> self . format = '<STR_LIT>' <EOL> def iterate ( self , idx ) : <EOL> if self . start > self . stop : <EOL> raise StopIteration <EOL> current = self . start <EOL> self . start += self . step_seconds <EOL> return ( current . strftime ( self . format ) , ) </s>
