<s> import re <EOL> from os . path import abspath , join <EOL> from sqlparse3 import sql , tokens as T <EOL> from sqlparse3 . engine import FilterStack <EOL> from sqlparse3 . lexer import tokenize <EOL> from sqlparse3 . pipeline import Pipeline <EOL> from sqlparse3 . tokens import ( Comment , Comparison , Keyword , Name , Punctuation , <EOL> String , Whitespace ) <EOL> from sqlparse3 . utils import memoize_generator <EOL> class _CaseFilter : <EOL> ttype = None <EOL> def __init__ ( self , case = None ) : <EOL> if case is None : <EOL> case = '<STR_LIT>' <EOL> assert case in [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> self . convert = getattr ( str , case ) <EOL> def process ( self , stack , stream ) : <EOL> for ttype , value in stream : <EOL> if ttype in self . ttype : <EOL> value = self . convert ( value ) <EOL> yield ttype , value <EOL> class KeywordCaseFilter ( _CaseFilter ) : <EOL> ttype = T . Keyword <EOL> class IdentifierCaseFilter ( _CaseFilter ) : <EOL> ttype = ( T . Name , T . String . Symbol ) <EOL> def process ( self , stack , stream ) : <EOL> for ttype , value in stream : <EOL> if ttype in self . ttype and not value . strip ( ) [ <NUM_LIT:0> ] == '<STR_LIT:">' : <EOL> value = self . convert ( value ) <EOL> yield ttype , value <EOL> class TruncateStringFilter : <EOL> def __init__ ( self , width , char ) : <EOL> self . width = max ( width , <NUM_LIT:1> ) <EOL> self . char = str ( char ) <EOL> def process ( self , stack , stream ) : <EOL> for ttype , value in stream : <EOL> if ttype is T . Literal . String . Single : <EOL> if value [ : <NUM_LIT:2> ] == '<STR_LIT>' : <EOL> inner = value [ <NUM_LIT:2> : - <NUM_LIT:2> ] <EOL> quote = '<STR_LIT>' <EOL> else : <EOL> inner = value [ <NUM_LIT:1> : - <NUM_LIT:1> ] <EOL> quote = '<STR_LIT>' <EOL> if len ( inner ) > self . width : <EOL> value = '<STR_LIT>' . join ( ( quote , inner [ : self . width ] , self . char , <EOL> quote ) ) <EOL> yield ttype , value <EOL> class GetComments : <EOL> """<STR_LIT>""" <EOL> def process ( self , stack , stream ) : <EOL> for token_type , value in stream : <EOL> if token_type in Comment : <EOL> yield token_type , value <EOL> class StripComments : <EOL> """<STR_LIT>""" <EOL> def process ( self , stack , stream ) : <EOL> for token_type , value in stream : <EOL> if token_type not in Comment : <EOL> yield token_type , value <EOL> def StripWhitespace ( stream ) : <EOL> "<STR_LIT>" <EOL> last_type = None <EOL> has_space = False <EOL> ignore_group = frozenset ( ( Comparison , Punctuation ) ) <EOL> for token_type , value in stream : <EOL> if last_type : <EOL> if token_type in Whitespace : <EOL> has_space = True <EOL> continue <EOL> elif token_type in ( Whitespace , Whitespace . Newline , ignore_group ) : <EOL> continue <EOL> if has_space : <EOL> if not ignore_group . intersection ( ( last_type , token_type ) ) : <EOL> yield Whitespace , '<STR_LIT:U+0020>' <EOL> has_space = False <EOL> yield token_type , value <EOL> last_type = token_type <EOL> class IncludeStatement : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , dirpath = "<STR_LIT:.>" , maxrecursive = <NUM_LIT:10> , raiseexceptions = False ) : <EOL> if maxrecursive <= <NUM_LIT:0> : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> self . dirpath = abspath ( dirpath ) <EOL> self . maxRecursive = maxrecursive <EOL> self . raiseexceptions = raiseexceptions <EOL> self . detected = False <EOL> @ memoize_generator <EOL> def process ( self , stack , stream ) : <EOL> for token_type , value in stream : <EOL> if token_type in Name and value . upper ( ) == '<STR_LIT>' : <EOL> self . detected = True <EOL> continue <EOL> elif self . detected : <EOL> if token_type in Whitespace : <EOL> continue <EOL> if token_type in String . Symbol : <EOL> path = join ( self . dirpath , value [ <NUM_LIT:1> : - <NUM_LIT:1> ] ) <EOL> try : <EOL> f = open ( path ) <EOL> raw_sql = f . read ( ) <EOL> f . close ( ) <EOL> except IOError as err : <EOL> if self . raiseexceptions : <EOL> raise <EOL> yield Comment , '<STR_LIT>' % err <EOL> else : <EOL> try : <EOL> filtr = IncludeStatement ( self . dirpath , <EOL> self . maxRecursive - <NUM_LIT:1> , <EOL> self . raiseexceptions ) <EOL> except ValueError as err : <EOL> if self . raiseexceptions : <EOL> raise <EOL> yield Comment , '<STR_LIT>' % err <EOL> stack = FilterStack ( ) <EOL> stack . preprocess . append ( filtr ) <EOL> for tv in stack . run ( raw_sql ) : <EOL> yield tv <EOL> self . detected = False <EOL> continue <EOL> yield token_type , value <EOL> class StripCommentsFilter : <EOL> def _get_next_comment ( self , tlist ) : <EOL> token = tlist . token_next_by_instance ( <NUM_LIT:0> , sql . Comment ) <EOL> if token is None : <EOL> token = tlist . token_next_by_type ( <NUM_LIT:0> , T . Comment ) <EOL> return token <EOL> def _process ( self , tlist ) : <EOL> token = self . _get_next_comment ( tlist ) <EOL> while token : <EOL> tidx = tlist . token_index ( token ) <EOL> prev = tlist . token_prev ( tidx , False ) <EOL> next_ = tlist . token_next ( tidx , False ) <EOL> if ( prev is not None and next_ is not None <EOL> and not prev . is_whitespace ( ) and not next_ . is_whitespace ( ) <EOL> and not ( prev . match ( T . Punctuation , '<STR_LIT:(>' ) <EOL> or next_ . match ( T . Punctuation , '<STR_LIT:)>' ) ) ) : <EOL> tlist . tokens [ tidx ] = sql . Token ( T . Whitespace , '<STR_LIT:U+0020>' ) <EOL> else : <EOL> tlist . tokens . pop ( tidx ) <EOL> token = self . _get_next_comment ( tlist ) <EOL> def process ( self , stack , stmt ) : <EOL> [ self . process ( stack , sgroup ) for sgroup in stmt . get_sublists ( ) ] <EOL> self . _process ( stmt ) <EOL> class StripWhitespaceFilter : <EOL> def _stripws ( self , tlist ) : <EOL> func_name = '<STR_LIT>' % tlist . __class__ . __name__ . lower ( ) <EOL> func = getattr ( self , func_name , self . _stripws_default ) <EOL> func ( tlist ) <EOL> def _stripws_default ( self , tlist ) : <EOL> last_was_ws = False <EOL> for token in tlist . tokens : <EOL> if token . is_whitespace ( ) : <EOL> if last_was_ws : <EOL> token . value = '<STR_LIT>' <EOL> else : <EOL> token . value = '<STR_LIT:U+0020>' <EOL> last_was_ws = token . is_whitespace ( ) <EOL> def _stripws_parenthesis ( self , tlist ) : <EOL> if tlist . tokens [ <NUM_LIT:1> ] . is_whitespace ( ) : <EOL> tlist . tokens . pop ( <NUM_LIT:1> ) <EOL> if tlist . tokens [ - <NUM_LIT:2> ] . is_whitespace ( ) : <EOL> tlist . tokens . pop ( - <NUM_LIT:2> ) <EOL> self . _stripws_default ( tlist ) <EOL> def process ( self , stack , stmt , depth = <NUM_LIT:0> ) : <EOL> [ self . process ( stack , sgroup , depth + <NUM_LIT:1> ) <EOL> for sgroup in stmt . get_sublists ( ) ] <EOL> self . _stripws ( stmt ) <EOL> if depth == <NUM_LIT:0> and stmt . tokens [ - <NUM_LIT:1> ] . is_whitespace ( ) : <EOL> stmt . tokens . pop ( - <NUM_LIT:1> ) <EOL> class ReindentFilter : <EOL> def __init__ ( self , width = <NUM_LIT:2> , char = '<STR_LIT:U+0020>' , line_width = None ) : <EOL> self . width = width <EOL> self . char = char <EOL> self . indent = <NUM_LIT:0> <EOL> self . offset = <NUM_LIT:0> <EOL> self . line_width = line_width <EOL> self . _curr_stmt = None <EOL> self . _last_stmt = None <EOL> def _flatten_up_to_token ( self , token ) : <EOL> """<STR_LIT>""" <EOL> iterator = self . _curr_stmt . flatten ( ) <EOL> for t in iterator : <EOL> yield t <EOL> if t == token : <EOL> raise StopIteration <EOL> def _get_offset ( self , token ) : <EOL> raw = '<STR_LIT>' . join ( map ( str , self . _flatten_up_to_token ( token ) ) ) <EOL> line = raw . splitlines ( ) [ - <NUM_LIT:1> ] <EOL> full_offset = len ( line ) - len ( self . char * ( self . width * self . indent ) ) <EOL> return full_offset - self . offset <EOL> def nl ( self ) : <EOL> space = ( self . char * ( ( self . indent * self . width ) + self . offset ) ) <EOL> if len ( space ) > <NUM_LIT:200> : <EOL> self . indent = self . offset = <NUM_LIT:0> <EOL> space = ( self . char * ( ( self . indent * self . width ) + self . offset ) ) <EOL> ws = '<STR_LIT:\n>' + space <EOL> return sql . Token ( T . Whitespace , ws ) <EOL> def _split_kwds ( self , tlist ) : <EOL> split_words = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' ) <EOL> def _next_token ( i ) : <EOL> t = tlist . token_next_match ( i , T . Keyword , split_words , <EOL> regex = True ) <EOL> if t and t . value . upper ( ) == '<STR_LIT>' : <EOL> t = _next_token ( tlist . token_index ( t ) + <NUM_LIT:1> ) <EOL> if t and t . value . upper ( ) == '<STR_LIT>' : <EOL> t = _next_token ( tlist . token_index ( t ) + <NUM_LIT:1> ) <EOL> return t <EOL> idx = <NUM_LIT:0> <EOL> token = _next_token ( idx ) <EOL> while token : <EOL> prev = tlist . token_prev ( tlist . token_index ( token ) , False ) <EOL> offset = <NUM_LIT:1> <EOL> if prev and prev . is_whitespace ( ) : <EOL> tlist . tokens . pop ( tlist . token_index ( prev ) ) <EOL> offset += <NUM_LIT:1> <EOL> if ( prev <EOL> and isinstance ( prev , sql . Comment ) <EOL> and ( str ( prev ) . endswith ( '<STR_LIT:\n>' ) <EOL> or str ( prev ) . endswith ( '<STR_LIT:\r>' ) ) ) : <EOL> nl = tlist . token_next ( token ) <EOL> else : <EOL> nl = self . nl ( ) <EOL> tlist . insert_before ( token , nl ) <EOL> token = _next_token ( tlist . token_index ( nl ) + offset ) <EOL> def _split_statements ( self , tlist ) : <EOL> idx = <NUM_LIT:0> <EOL> token = tlist . token_next_by_type ( idx , ( T . Keyword . DDL , T . Keyword . DML ) ) <EOL> while token : <EOL> prev = tlist . token_prev ( tlist . token_index ( token ) , False ) <EOL> if prev and prev . is_whitespace ( ) : <EOL> tlist . tokens . pop ( tlist . token_index ( prev ) ) <EOL> if prev : <EOL> nl = self . nl ( ) <EOL> tlist . insert_before ( token , nl ) <EOL> token = tlist . token_next_by_type ( tlist . token_index ( token ) + <NUM_LIT:1> , <EOL> ( T . Keyword . DDL , T . Keyword . DML ) ) <EOL> def _process ( self , tlist ) : <EOL> func_name = '<STR_LIT>' % tlist . __class__ . __name__ . lower ( ) <EOL> func = getattr ( self , func_name , self . _process_default ) <EOL> func ( tlist ) <EOL> def _process_where ( self , tlist ) : <EOL> token = tlist . token_next_match ( <NUM_LIT:0> , T . Keyword , '<STR_LIT>' ) <EOL> tlist . insert_before ( token , self . nl ( ) ) <EOL> self . indent += <NUM_LIT:1> <EOL> self . _process_default ( tlist ) <EOL> self . indent -= <NUM_LIT:1> <EOL> def _process_parenthesis ( self , tlist ) : <EOL> first = tlist . token_next ( <NUM_LIT:0> ) <EOL> indented = False <EOL> if first and first . ttype in ( T . Keyword . DML , T . Keyword . DDL ) : <EOL> self . indent += <NUM_LIT:1> <EOL> tlist . tokens . insert ( <NUM_LIT:0> , self . nl ( ) ) <EOL> indented = True <EOL> num_offset = self . _get_offset ( <EOL> tlist . token_next_match ( <NUM_LIT:0> , T . Punctuation , '<STR_LIT:(>' ) ) <EOL> self . offset += num_offset <EOL> self . _process_default ( tlist , stmts = not indented ) <EOL> if indented : <EOL> self . indent -= <NUM_LIT:1> <EOL> self . offset -= num_offset <EOL> def _process_identifierlist ( self , tlist ) : <EOL> identifiers = list ( tlist . get_identifiers ( ) ) <EOL> if len ( identifiers ) > <NUM_LIT:1> and not tlist . within ( sql . Function ) : <EOL> first = list ( identifiers [ <NUM_LIT:0> ] . flatten ( ) ) [ <NUM_LIT:0> ] <EOL> num_offset = self . _get_offset ( first ) - len ( first . value ) <EOL> self . offset += num_offset <EOL> for token in identifiers [ <NUM_LIT:1> : ] : <EOL> tlist . insert_before ( token , self . nl ( ) ) <EOL> for token in tlist . tokens : <EOL> if isinstance ( token , sql . Comment ) : <EOL> tlist . insert_after ( token , self . nl ( ) ) <EOL> self . offset -= num_offset <EOL> self . _process_default ( tlist ) <EOL> def _process_case ( self , tlist ) : <EOL> is_first = True <EOL> num_offset = None <EOL> case = tlist . tokens [ <NUM_LIT:0> ] <EOL> outer_offset = self . _get_offset ( case ) - len ( case . value ) <EOL> self . offset += outer_offset <EOL> for cond , value in tlist . get_cases ( ) : <EOL> if is_first : <EOL> tcond = list ( cond [ <NUM_LIT:0> ] . flatten ( ) ) [ <NUM_LIT:0> ] <EOL> is_first = False <EOL> num_offset = self . _get_offset ( tcond ) - len ( tcond . value ) <EOL> self . offset += num_offset <EOL> continue <EOL> if cond is None : <EOL> token = value [ <NUM_LIT:0> ] <EOL> else : <EOL> token = cond [ <NUM_LIT:0> ] <EOL> tlist . insert_before ( token , self . nl ( ) ) <EOL> self . offset += <NUM_LIT:5> <EOL> self . _process_default ( tlist ) <EOL> self . offset -= <NUM_LIT:5> <EOL> if num_offset is not None : <EOL> self . offset -= num_offset <EOL> end = tlist . token_next_match ( <NUM_LIT:0> , T . Keyword , '<STR_LIT>' ) <EOL> tlist . insert_before ( end , self . nl ( ) ) <EOL> self . offset -= outer_offset <EOL> def _process_default ( self , tlist , stmts = True , kwds = True ) : <EOL> if stmts : <EOL> self . _split_statements ( tlist ) <EOL> if kwds : <EOL> self . _split_kwds ( tlist ) <EOL> [ self . _process ( sgroup ) for sgroup in tlist . get_sublists ( ) ] <EOL> def process ( self , stack , stmt ) : <EOL> if isinstance ( stmt , sql . Statement ) : <EOL> self . _curr_stmt = stmt <EOL> self . _process ( stmt ) <EOL> if isinstance ( stmt , sql . Statement ) : <EOL> if self . _last_stmt is not None : <EOL> if str ( self . _last_stmt ) . endswith ( '<STR_LIT:\n>' ) : <EOL> nl = '<STR_LIT:\n>' <EOL> else : <EOL> nl = '<STR_LIT>' <EOL> stmt . tokens . insert ( <EOL> <NUM_LIT:0> , sql . Token ( T . Whitespace , nl ) ) <EOL> if self . _last_stmt != stmt : <EOL> self . _last_stmt = stmt <EOL> class RightMarginFilter : <EOL> keep_together = ( <EOL> ) <EOL> def __init__ ( self , width = <NUM_LIT> ) : <EOL> self . width = width <EOL> self . line = '<STR_LIT>' <EOL> def _process ( self , stack , group , stream ) : <EOL> for token in stream : <EOL> if token . is_whitespace ( ) and '<STR_LIT:\n>' in token . value : <EOL> if token . value . endswith ( '<STR_LIT:\n>' ) : <EOL> self . line = '<STR_LIT>' <EOL> else : <EOL> self . line = token . value . splitlines ( ) [ - <NUM_LIT:1> ] <EOL> elif ( token . is_group ( ) <EOL> and not token . __class__ in self . keep_together ) : <EOL> token . tokens = self . _process ( stack , token , token . tokens ) <EOL> else : <EOL> val = str ( token ) <EOL> if len ( self . line ) + len ( val ) > self . width : <EOL> match = re . search ( '<STR_LIT>' , self . line ) <EOL> if match is not None : <EOL> indent = match . group ( ) <EOL> else : <EOL> indent = '<STR_LIT>' <EOL> yield sql . Token ( T . Whitespace , '<STR_LIT>' % indent ) <EOL> self . line = indent <EOL> self . line += val <EOL> yield token <EOL> def process ( self , stack , group ) : <EOL> return <EOL> group . tokens = self . _process ( stack , group , group . tokens ) <EOL> class ColumnsSelect : <EOL> """<STR_LIT>""" <EOL> def process ( self , stack , stream ) : <EOL> mode = <NUM_LIT:0> <EOL> oldValue = "<STR_LIT>" <EOL> parenthesis = <NUM_LIT:0> <EOL> for token_type , value in stream : <EOL> if token_type in Comment : <EOL> continue <EOL> if mode == <NUM_LIT:0> : <EOL> if token_type in Keyword and value == '<STR_LIT>' : <EOL> mode = <NUM_LIT:1> <EOL> elif mode == <NUM_LIT:1> : <EOL> if value == '<STR_LIT>' : <EOL> if oldValue : <EOL> yield oldValue <EOL> mode = <NUM_LIT:3> <EOL> elif value == '<STR_LIT>' : <EOL> oldValue = "<STR_LIT>" <EOL> mode = <NUM_LIT:2> <EOL> elif ( token_type == Punctuation <EOL> and value == '<STR_LIT:U+002C>' and not parenthesis ) : <EOL> if oldValue : <EOL> yield oldValue <EOL> oldValue = "<STR_LIT>" <EOL> elif token_type not in Whitespace : <EOL> if value == '<STR_LIT:(>' : <EOL> parenthesis += <NUM_LIT:1> <EOL> elif value == '<STR_LIT:)>' : <EOL> parenthesis -= <NUM_LIT:1> <EOL> oldValue += value <EOL> elif mode == <NUM_LIT:2> : <EOL> if token_type == Name or token_type == Keyword : <EOL> yield value <EOL> mode = <NUM_LIT:1> <EOL> class SerializerUnicode : <EOL> def process ( self , stack , stmt ) : <EOL> raw = str ( stmt ) <EOL> add_nl = raw . endswith ( '<STR_LIT:\n>' ) <EOL> res = '<STR_LIT:\n>' . join ( line . rstrip ( ) for line in raw . splitlines ( ) ) <EOL> if add_nl : <EOL> res += '<STR_LIT:\n>' <EOL> return res <EOL> def Tokens2Unicode ( stream ) : <EOL> result = "<STR_LIT>" <EOL> for _ , value in stream : <EOL> result += str ( value ) <EOL> return result <EOL> class OutputFilter : <EOL> varname_prefix = '<STR_LIT>' <EOL> def __init__ ( self , varname = '<STR_LIT>' ) : <EOL> self . varname = self . varname_prefix + varname <EOL> self . count = <NUM_LIT:0> <EOL> def _process ( self , stream , varname , has_nl ) : <EOL> raise NotImplementedError <EOL> def process ( self , stack , stmt ) : <EOL> self . count += <NUM_LIT:1> <EOL> if self . count > <NUM_LIT:1> : <EOL> varname = '<STR_LIT>' % ( self . varname , self . count ) <EOL> else : <EOL> varname = self . varname <EOL> has_nl = len ( str ( stmt ) . strip ( ) . splitlines ( ) ) > <NUM_LIT:1> <EOL> stmt . tokens = self . _process ( stmt . tokens , varname , has_nl ) <EOL> return stmt <EOL> class OutputPythonFilter ( OutputFilter ) : <EOL> def _process ( self , stream , varname , has_nl ) : <EOL> if self . count > <NUM_LIT:1> : <EOL> yield sql . Token ( T . Whitespace , '<STR_LIT:\n>' ) <EOL> yield sql . Token ( T . Name , varname ) <EOL> yield sql . Token ( T . Whitespace , '<STR_LIT:U+0020>' ) <EOL> yield sql . Token ( T . Operator , '<STR_LIT:=>' ) <EOL> yield sql . Token ( T . Whitespace , '<STR_LIT:U+0020>' ) <EOL> if has_nl : <EOL> yield sql . Token ( T . Operator , '<STR_LIT:(>' ) <EOL> yield sql . Token ( T . Text , "<STR_LIT:'>" ) <EOL> for token in stream : <EOL> if token . is_whitespace ( ) and '<STR_LIT:\n>' in token . value : <EOL> yield sql . Token ( T . Text , "<STR_LIT>" ) <EOL> yield sql . Token ( T . Whitespace , '<STR_LIT:\n>' ) <EOL> yield sql . Token ( T . Whitespace , '<STR_LIT:U+0020>' * ( len ( varname ) + <NUM_LIT:4> ) ) <EOL> yield sql . Token ( T . Text , "<STR_LIT:'>" ) <EOL> after_lb = token . value . split ( '<STR_LIT:\n>' , <NUM_LIT:1> ) [ <NUM_LIT:1> ] <EOL> if after_lb : <EOL> yield sql . Token ( T . Whitespace , after_lb ) <EOL> continue <EOL> elif "<STR_LIT:'>" in token . value : <EOL> token . value = token . value . replace ( "<STR_LIT:'>" , "<STR_LIT>" ) <EOL> yield sql . Token ( T . Text , token . value ) <EOL> yield sql . Token ( T . Text , "<STR_LIT:'>" ) <EOL> if has_nl : <EOL> yield sql . Token ( T . Operator , '<STR_LIT:)>' ) <EOL> class OutputPHPFilter ( OutputFilter ) : <EOL> varname_prefix = '<STR_LIT:$>' <EOL> def _process ( self , stream , varname , has_nl ) : <EOL> if self . count > <NUM_LIT:1> : <EOL> yield sql . Token ( T . Whitespace , '<STR_LIT:\n>' ) <EOL> yield sql . Token ( T . Name , varname ) <EOL> yield sql . Token ( T . Whitespace , '<STR_LIT:U+0020>' ) <EOL> if has_nl : <EOL> yield sql . Token ( T . Whitespace , '<STR_LIT:U+0020>' ) <EOL> yield sql . Token ( T . Operator , '<STR_LIT:=>' ) <EOL> yield sql . Token ( T . Whitespace , '<STR_LIT:U+0020>' ) <EOL> yield sql . Token ( T . Text , '<STR_LIT:">' ) <EOL> for token in stream : <EOL> if token . is_whitespace ( ) and '<STR_LIT:\n>' in token . value : <EOL> yield sql . Token ( T . Text , '<STR_LIT>' ) <EOL> yield sql . Token ( T . Whitespace , '<STR_LIT:\n>' ) <EOL> yield sql . Token ( T . Name , varname ) <EOL> yield sql . Token ( T . Whitespace , '<STR_LIT:U+0020>' ) <EOL> yield sql . Token ( T . Operator , '<STR_LIT>' ) <EOL> yield sql . Token ( T . Whitespace , '<STR_LIT:U+0020>' ) <EOL> yield sql . Token ( T . Text , '<STR_LIT:">' ) <EOL> after_lb = token . value . split ( '<STR_LIT:\n>' , <NUM_LIT:1> ) [ <NUM_LIT:1> ] <EOL> if after_lb : <EOL> yield sql . Token ( T . Whitespace , after_lb ) <EOL> continue <EOL> elif '<STR_LIT:">' in token . value : <EOL> token . value = token . value . replace ( '<STR_LIT:">' , '<STR_LIT>' ) <EOL> yield sql . Token ( T . Text , token . value ) <EOL> yield sql . Token ( T . Text , '<STR_LIT:">' ) <EOL> yield sql . Token ( T . Punctuation , '<STR_LIT:;>' ) <EOL> class Limit : <EOL> """<STR_LIT>""" <EOL> def process ( self , stack , stream ) : <EOL> index = <NUM_LIT:7> <EOL> stream = list ( stream ) <EOL> stream . reverse ( ) <EOL> for token_type , value in stream : <EOL> index -= <NUM_LIT:1> <EOL> if index and token_type in Keyword and value == '<STR_LIT>' : <EOL> return stream [ <NUM_LIT:4> - index ] [ <NUM_LIT:1> ] <EOL> return - <NUM_LIT:1> <EOL> def compact ( stream ) : <EOL> """<STR_LIT>""" <EOL> pipe = Pipeline ( ) <EOL> pipe . append ( StripComments ( ) ) <EOL> pipe . append ( StripWhitespace ) <EOL> return pipe ( stream ) </s>
