<s> import itertools <EOL> from sqlparse3 import sql <EOL> from sqlparse3 import tokens as T <EOL> try : <EOL> next <EOL> except NameError : <EOL> next = lambda i : i . __next__ ( ) <EOL> def _group_left_right ( tlist , ttype , value , cls , <EOL> check_right = lambda t : True , <EOL> check_left = lambda t : True , <EOL> include_semicolon = False ) : <EOL> [ _group_left_right ( sgroup , ttype , value , cls , check_right , check_left , <EOL> include_semicolon ) for sgroup in tlist . get_sublists ( ) <EOL> if not isinstance ( sgroup , cls ) ] <EOL> idx = <NUM_LIT:0> <EOL> token = tlist . token_next_match ( idx , ttype , value ) <EOL> while token : <EOL> right = tlist . token_next ( tlist . token_index ( token ) ) <EOL> left = tlist . token_prev ( tlist . token_index ( token ) ) <EOL> if right is None or not check_right ( right ) : <EOL> token = tlist . token_next_match ( tlist . token_index ( token ) + <NUM_LIT:1> , <EOL> ttype , value ) <EOL> elif left is None or not check_left ( left ) : <EOL> token = tlist . token_next_match ( tlist . token_index ( token ) + <NUM_LIT:1> , <EOL> ttype , value ) <EOL> else : <EOL> if include_semicolon : <EOL> sright = tlist . token_next_match ( tlist . token_index ( right ) , <EOL> T . Punctuation , '<STR_LIT:;>' ) <EOL> if sright is not None : <EOL> right = sright <EOL> tokens = tlist . tokens_between ( left , right ) [ <NUM_LIT:1> : ] <EOL> if not isinstance ( left , cls ) : <EOL> new = cls ( [ left ] ) <EOL> new_idx = tlist . token_index ( left ) <EOL> tlist . tokens . remove ( left ) <EOL> tlist . tokens . insert ( new_idx , new ) <EOL> left = new <EOL> left . tokens . extend ( tokens ) <EOL> for t in tokens : <EOL> tlist . tokens . remove ( t ) <EOL> token = tlist . token_next_match ( tlist . token_index ( left ) + <NUM_LIT:1> , <EOL> ttype , value ) <EOL> def _group_matching ( tlist , start_ttype , start_value , end_ttype , end_value , <EOL> cls , include_semicolon = False , recurse = False ) : <EOL> def _find_matching ( i , tl , stt , sva , ett , eva ) : <EOL> depth = <NUM_LIT:1> <EOL> for n in range ( i , len ( tl . tokens ) ) : <EOL> t = tl . tokens [ n ] <EOL> if t . match ( stt , sva ) : <EOL> depth += <NUM_LIT:1> <EOL> elif t . match ( ett , eva ) : <EOL> depth -= <NUM_LIT:1> <EOL> if depth == <NUM_LIT:1> : <EOL> return t <EOL> return None <EOL> [ _group_matching ( sgroup , start_ttype , start_value , end_ttype , end_value , <EOL> cls , include_semicolon ) for sgroup in tlist . get_sublists ( ) <EOL> if recurse ] <EOL> if isinstance ( tlist , cls ) : <EOL> idx = <NUM_LIT:1> <EOL> else : <EOL> idx = <NUM_LIT:0> <EOL> token = tlist . token_next_match ( idx , start_ttype , start_value ) <EOL> while token : <EOL> tidx = tlist . token_index ( token ) <EOL> end = _find_matching ( tidx , tlist , start_ttype , start_value , <EOL> end_ttype , end_value ) <EOL> if end is None : <EOL> idx = tidx + <NUM_LIT:1> <EOL> else : <EOL> if include_semicolon : <EOL> next_ = tlist . token_next ( tlist . token_index ( end ) ) <EOL> if next_ and next_ . match ( T . Punctuation , '<STR_LIT:;>' ) : <EOL> end = next_ <EOL> group = tlist . group_tokens ( cls , tlist . tokens_between ( token , end ) ) <EOL> _group_matching ( group , start_ttype , start_value , <EOL> end_ttype , end_value , cls , include_semicolon ) <EOL> idx = tlist . token_index ( group ) + <NUM_LIT:1> <EOL> token = tlist . token_next_match ( idx , start_ttype , start_value ) <EOL> def group_if ( tlist ) : <EOL> _group_matching ( tlist , T . Keyword , '<STR_LIT>' , T . Keyword , '<STR_LIT>' , sql . If , True ) <EOL> def group_for ( tlist ) : <EOL> _group_matching ( tlist , T . Keyword , '<STR_LIT>' , T . Keyword , '<STR_LIT>' , <EOL> sql . For , True ) <EOL> def group_as ( tlist ) : <EOL> def _right_valid ( token ) : <EOL> return not token . ttype in ( T . DML , T . DDL ) <EOL> def _left_valid ( token ) : <EOL> return token . ttype is not T . Keyword <EOL> _group_left_right ( tlist , T . Keyword , '<STR_LIT>' , sql . Identifier , <EOL> check_right = _right_valid , <EOL> check_left = _left_valid ) <EOL> def group_assignment ( tlist ) : <EOL> _group_left_right ( tlist , T . Assignment , '<STR_LIT>' , sql . Assignment , <EOL> include_semicolon = True ) <EOL> def group_comparison ( tlist ) : <EOL> def _parts_valid ( token ) : <EOL> return ( token . ttype in ( T . String . Symbol , T . Name , T . Number , <EOL> T . Number . Integer , T . Literal , <EOL> T . Literal . Number . Integer ) <EOL> or isinstance ( token , ( sql . Identifier , sql . Parenthesis ) ) <EOL> or ( token . ttype is T . Keyword <EOL> and token . value . upper ( ) in [ '<STR_LIT>' , ] ) ) <EOL> _group_left_right ( tlist , T . Operator . Comparison , None , sql . Comparison , <EOL> check_left = _parts_valid , check_right = _parts_valid ) <EOL> def group_case ( tlist ) : <EOL> _group_matching ( tlist , T . Keyword , '<STR_LIT>' , T . Keyword , '<STR_LIT>' , sql . Case , <EOL> include_semicolon = True , recurse = True ) <EOL> def group_identifier ( tlist ) : <EOL> def _consume_cycle ( tl , i ) : <EOL> x = itertools . cycle ( ( <EOL> lambda y : ( y . match ( T . Punctuation , '<STR_LIT:.>' ) <EOL> or y . ttype is T . Operator <EOL> or y . ttype is T . Wildcard ) , <EOL> lambda y : ( y . ttype in ( T . String . Symbol , <EOL> T . Name , <EOL> T . Wildcard , <EOL> T . Literal . Number . Integer , <EOL> T . Literal . Number . Float ) <EOL> or isinstance ( y , ( sql . Parenthesis , sql . Function ) ) ) ) ) <EOL> for t in tl . tokens [ i : ] : <EOL> if t . ttype is T . Whitespace : <EOL> yield t <EOL> continue <EOL> if next ( x ) ( t ) : <EOL> yield t <EOL> else : <EOL> raise StopIteration <EOL> def _next_token ( tl , i ) : <EOL> t1 = tl . token_next_by_type ( <EOL> i , ( T . String . Symbol , T . Name , T . Literal . Number . Integer , <EOL> T . Literal . Number . Float ) ) <EOL> t2 = tl . token_next_by_instance ( i , ( sql . Function , sql . Parenthesis ) ) <EOL> if t1 and t2 : <EOL> i1 = tl . token_index ( t1 ) <EOL> i2 = tl . token_index ( t2 ) <EOL> if i1 > i2 : <EOL> return t2 <EOL> else : <EOL> return t1 <EOL> elif t1 : <EOL> return t1 <EOL> else : <EOL> return t2 <EOL> [ group_identifier ( sgroup ) for sgroup in tlist . get_sublists ( ) <EOL> if not isinstance ( sgroup , sql . Identifier ) ] <EOL> idx = <NUM_LIT:0> <EOL> token = _next_token ( tlist , idx ) <EOL> while token : <EOL> identifier_tokens = [ token ] + list ( <EOL> _consume_cycle ( tlist , <EOL> tlist . token_index ( token ) + <NUM_LIT:1> ) ) <EOL> if identifier_tokens and identifier_tokens [ - <NUM_LIT:1> ] . ttype is T . Whitespace : <EOL> identifier_tokens = identifier_tokens [ : - <NUM_LIT:1> ] <EOL> if not ( len ( identifier_tokens ) == <NUM_LIT:1> <EOL> and ( isinstance ( identifier_tokens [ <NUM_LIT:0> ] , ( sql . Function , sql . Parenthesis ) ) <EOL> or identifier_tokens [ <NUM_LIT:0> ] . ttype in ( T . Literal . Number . Integer , <EOL> T . Literal . Number . Float ) ) ) : <EOL> group = tlist . group_tokens ( sql . Identifier , identifier_tokens ) <EOL> idx = tlist . token_index ( group ) + <NUM_LIT:1> <EOL> else : <EOL> idx += <NUM_LIT:1> <EOL> token = _next_token ( tlist , idx ) <EOL> def group_identifier_list ( tlist ) : <EOL> [ group_identifier_list ( sgroup ) for sgroup in tlist . get_sublists ( ) <EOL> if not isinstance ( sgroup , sql . IdentifierList ) ] <EOL> idx = <NUM_LIT:0> <EOL> fend1_funcs = [ lambda t : isinstance ( t , ( sql . Identifier , sql . Function , <EOL> sql . Case ) ) , <EOL> lambda t : t . is_whitespace ( ) , <EOL> lambda t : t . ttype == T . Name , <EOL> lambda t : t . ttype == T . Wildcard , <EOL> lambda t : t . match ( T . Keyword , '<STR_LIT:null>' ) , <EOL> lambda t : t . match ( T . Keyword , '<STR_LIT>' ) , <EOL> lambda t : t . ttype == T . Number . Integer , <EOL> lambda t : t . ttype == T . String . Single , <EOL> lambda t : t . ttype == T . Name . Placeholder , <EOL> lambda t : t . ttype == T . Keyword , <EOL> lambda t : isinstance ( t , sql . Comparison ) , <EOL> lambda t : isinstance ( t , sql . Comment ) , <EOL> ] <EOL> tcomma = tlist . token_next_match ( idx , T . Punctuation , '<STR_LIT:U+002C>' ) <EOL> start = None <EOL> while tcomma is not None : <EOL> before = tlist . token_prev ( tcomma ) <EOL> after = tlist . token_next ( tcomma ) <EOL> bpassed = apassed = False <EOL> for func in fend1_funcs : <EOL> if before is not None and func ( before ) : <EOL> bpassed = True <EOL> if after is not None and func ( after ) : <EOL> apassed = True <EOL> if not bpassed or not apassed : <EOL> start = None <EOL> tcomma = tlist . token_next_match ( tlist . token_index ( tcomma ) + <NUM_LIT:1> , <EOL> T . Punctuation , '<STR_LIT:U+002C>' ) <EOL> else : <EOL> if start is None : <EOL> start = before <EOL> next_ = tlist . token_next ( after ) <EOL> if next_ is None or not next_ . match ( T . Punctuation , '<STR_LIT:U+002C>' ) : <EOL> tokens = tlist . tokens_between ( start , after ) <EOL> group = tlist . group_tokens ( sql . IdentifierList , tokens ) <EOL> start = None <EOL> tcomma = tlist . token_next_match ( tlist . token_index ( group ) + <NUM_LIT:1> , <EOL> T . Punctuation , '<STR_LIT:U+002C>' ) <EOL> else : <EOL> tcomma = next_ <EOL> def group_parenthesis ( tlist ) : <EOL> _group_matching ( tlist , T . Punctuation , '<STR_LIT:(>' , T . Punctuation , '<STR_LIT:)>' , <EOL> sql . Parenthesis ) <EOL> def group_comments ( tlist ) : <EOL> [ group_comments ( sgroup ) for sgroup in tlist . get_sublists ( ) <EOL> if not isinstance ( sgroup , sql . Comment ) ] <EOL> idx = <NUM_LIT:0> <EOL> token = tlist . token_next_by_type ( idx , T . Comment ) <EOL> while token : <EOL> tidx = tlist . token_index ( token ) <EOL> end = tlist . token_not_matching ( tidx + <NUM_LIT:1> , <EOL> [ lambda t : t . ttype in T . Comment , <EOL> lambda t : t . is_whitespace ( ) ] ) <EOL> if end is None : <EOL> idx = tidx + <NUM_LIT:1> <EOL> else : <EOL> eidx = tlist . token_index ( end ) <EOL> grp_tokens = tlist . tokens_between ( token , <EOL> tlist . token_prev ( eidx , False ) ) <EOL> group = tlist . group_tokens ( sql . Comment , grp_tokens ) <EOL> idx = tlist . token_index ( group ) <EOL> token = tlist . token_next_by_type ( idx , T . Comment ) <EOL> def group_where ( tlist ) : <EOL> [ group_where ( sgroup ) for sgroup in tlist . get_sublists ( ) <EOL> if not isinstance ( sgroup , sql . Where ) ] <EOL> idx = <NUM_LIT:0> <EOL> token = tlist . token_next_match ( idx , T . Keyword , '<STR_LIT>' ) <EOL> stopwords = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> while token : <EOL> tidx = tlist . token_index ( token ) <EOL> end = tlist . token_next_match ( tidx + <NUM_LIT:1> , T . Keyword , stopwords ) <EOL> if end is None : <EOL> end = tlist . _groupable_tokens [ - <NUM_LIT:1> ] <EOL> else : <EOL> end = tlist . tokens [ tlist . token_index ( end ) - <NUM_LIT:1> ] <EOL> group = tlist . group_tokens ( sql . Where , <EOL> tlist . tokens_between ( token , end ) , <EOL> ignore_ws = True ) <EOL> idx = tlist . token_index ( group ) <EOL> token = tlist . token_next_match ( idx , T . Keyword , '<STR_LIT>' ) <EOL> def group_aliased ( tlist ) : <EOL> clss = ( sql . Identifier , sql . Function , sql . Case ) <EOL> [ group_aliased ( sgroup ) for sgroup in tlist . get_sublists ( ) <EOL> if not isinstance ( sgroup , clss ) ] <EOL> idx = <NUM_LIT:0> <EOL> token = tlist . token_next_by_instance ( idx , clss ) <EOL> while token : <EOL> next_ = tlist . token_next ( tlist . token_index ( token ) ) <EOL> if next_ is not None and isinstance ( next_ , clss ) : <EOL> if not next_ . value . upper ( ) . startswith ( '<STR_LIT>' ) : <EOL> grp = tlist . tokens_between ( token , next_ ) [ <NUM_LIT:1> : ] <EOL> token . tokens . extend ( grp ) <EOL> for t in grp : <EOL> tlist . tokens . remove ( t ) <EOL> idx = tlist . token_index ( token ) + <NUM_LIT:1> <EOL> token = tlist . token_next_by_instance ( idx , clss ) <EOL> def group_typecasts ( tlist ) : <EOL> _group_left_right ( tlist , T . Punctuation , '<STR_LIT>' , sql . Identifier ) <EOL> def group_functions ( tlist ) : <EOL> [ group_functions ( sgroup ) for sgroup in tlist . get_sublists ( ) <EOL> if not isinstance ( sgroup , sql . Function ) ] <EOL> idx = <NUM_LIT:0> <EOL> token = tlist . token_next_by_type ( idx , T . Name ) <EOL> while token : <EOL> next_ = tlist . token_next ( token ) <EOL> if not isinstance ( next_ , sql . Parenthesis ) : <EOL> idx = tlist . token_index ( token ) + <NUM_LIT:1> <EOL> else : <EOL> func = tlist . group_tokens ( sql . Function , <EOL> tlist . tokens_between ( token , next_ ) ) <EOL> idx = tlist . token_index ( func ) + <NUM_LIT:1> <EOL> token = tlist . token_next_by_type ( idx , T . Name ) <EOL> def group_order ( tlist ) : <EOL> idx = <NUM_LIT:0> <EOL> token = tlist . token_next_by_type ( idx , T . Keyword . Order ) <EOL> while token : <EOL> prev = tlist . token_prev ( token ) <EOL> if isinstance ( prev , sql . Identifier ) : <EOL> ido = tlist . group_tokens ( sql . Identifier , <EOL> tlist . tokens_between ( prev , token ) ) <EOL> idx = tlist . token_index ( ido ) + <NUM_LIT:1> <EOL> else : <EOL> idx = tlist . token_index ( token ) + <NUM_LIT:1> <EOL> token = tlist . token_next_by_type ( idx , T . Keyword . Order ) <EOL> def group ( tlist ) : <EOL> for func in [ <EOL> group_comments , <EOL> group_parenthesis , <EOL> group_functions , <EOL> group_where , <EOL> group_case , <EOL> group_identifier , <EOL> group_order , <EOL> group_typecasts , <EOL> group_as , <EOL> group_aliased , <EOL> group_assignment , <EOL> group_comparison , <EOL> group_identifier_list , <EOL> group_if , <EOL> group_for ] : <EOL> func ( tlist ) </s>
