<s> """<STR_LIT>""" <EOL> from __future__ import unicode_literals <EOL> import collections <EOL> import heapq <EOL> import re <EOL> from lib2to3 import pytree <EOL> from lib2to3 . pgen2 import token <EOL> from yapf . yapflib import format_decision_state <EOL> from yapf . yapflib import format_token <EOL> from yapf . yapflib import line_joiner <EOL> from yapf . yapflib import pytree_utils <EOL> from yapf . yapflib import style <EOL> from yapf . yapflib import verifier <EOL> def Reformat ( uwlines , verify = True ) : <EOL> """<STR_LIT>""" <EOL> final_lines = [ ] <EOL> prev_uwline = None <EOL> for uwline in _SingleOrMergedLines ( uwlines ) : <EOL> first_token = uwline . first <EOL> _FormatFirstToken ( first_token , uwline . depth , prev_uwline , final_lines ) <EOL> indent_amt = style . Get ( '<STR_LIT>' ) * uwline . depth <EOL> state = format_decision_state . FormatDecisionState ( uwline , indent_amt ) <EOL> if not uwline . disable : <EOL> if uwline . first . is_comment : <EOL> uwline . first . node . value = uwline . first . node . value . rstrip ( ) <EOL> elif uwline . last . is_comment : <EOL> uwline . last . node . value = uwline . last . node . value . rstrip ( ) <EOL> if prev_uwline and prev_uwline . disable : <EOL> _RetainVerticalSpacingBetweenTokens ( uwline . first , prev_uwline . last ) <EOL> if any ( tok . is_comment for tok in uwline . tokens ) : <EOL> _RetainVerticalSpacingBeforeComments ( uwline ) <EOL> if ( _LineContainsI18n ( uwline ) or uwline . disable or <EOL> _LineHasContinuationMarkers ( uwline ) ) : <EOL> _RetainHorizontalSpacing ( uwline ) <EOL> _RetainVerticalSpacing ( uwline , prev_uwline ) <EOL> _EmitLineUnformatted ( state ) <EOL> elif _CanPlaceOnSingleLine ( uwline ) : <EOL> while state . next_token : <EOL> state . AddTokenToState ( newline = False , dry_run = False ) <EOL> else : <EOL> if not _AnalyzeSolutionSpace ( state , dry_run = False ) : <EOL> state = format_decision_state . FormatDecisionState ( uwline , indent_amt ) <EOL> _RetainHorizontalSpacing ( uwline ) <EOL> _RetainVerticalSpacing ( uwline , prev_uwline ) <EOL> _EmitLineUnformatted ( state ) <EOL> final_lines . append ( uwline ) <EOL> prev_uwline = uwline <EOL> formatted_code = [ ] <EOL> for line in final_lines : <EOL> formatted_line = [ ] <EOL> for tok in line . tokens : <EOL> if not tok . is_pseudo_paren : <EOL> formatted_line . append ( tok . whitespace_prefix ) <EOL> formatted_line . append ( tok . value ) <EOL> else : <EOL> if ( not tok . next_token . whitespace_prefix . startswith ( '<STR_LIT:\n>' ) and <EOL> not tok . next_token . whitespace_prefix . startswith ( '<STR_LIT:U+0020>' ) ) : <EOL> if ( tok . previous_token . value == '<STR_LIT::>' or <EOL> tok . next_token . value not in '<STR_LIT>' ) : <EOL> formatted_line . append ( '<STR_LIT:U+0020>' ) <EOL> formatted_code . append ( '<STR_LIT>' . join ( formatted_line ) ) <EOL> if verify : <EOL> verifier . VerifyCode ( formatted_code [ - <NUM_LIT:1> ] ) <EOL> return '<STR_LIT>' . join ( formatted_code ) + '<STR_LIT:\n>' <EOL> def _RetainHorizontalSpacing ( uwline ) : <EOL> """<STR_LIT>""" <EOL> for tok in uwline . tokens : <EOL> tok . RetainHorizontalSpacing ( uwline . first . column , uwline . depth ) <EOL> def _RetainVerticalSpacing ( cur_uwline , prev_uwline ) : <EOL> prev_tok = None <EOL> if prev_uwline is not None : <EOL> prev_tok = prev_uwline . last <EOL> for cur_tok in cur_uwline . tokens : <EOL> _RetainVerticalSpacingBetweenTokens ( cur_tok , prev_tok ) <EOL> prev_tok = cur_tok <EOL> def _RetainVerticalSpacingBetweenTokens ( cur_tok , prev_tok ) : <EOL> """<STR_LIT>""" <EOL> if prev_tok is None : <EOL> return <EOL> if prev_tok . is_string : <EOL> prev_lineno = prev_tok . lineno + prev_tok . value . count ( '<STR_LIT:\n>' ) <EOL> elif prev_tok . is_pseudo_paren : <EOL> prev_lineno = prev_tok . previous_token . lineno <EOL> else : <EOL> prev_lineno = prev_tok . lineno <EOL> if cur_tok . is_comment : <EOL> cur_lineno = cur_tok . lineno - cur_tok . value . count ( '<STR_LIT:\n>' ) <EOL> else : <EOL> cur_lineno = cur_tok . lineno <EOL> cur_tok . AdjustNewlinesBefore ( cur_lineno - prev_lineno ) <EOL> def _RetainVerticalSpacingBeforeComments ( uwline ) : <EOL> """<STR_LIT>""" <EOL> prev_token = None <EOL> for tok in uwline . tokens : <EOL> if tok . is_comment and prev_token : <EOL> if tok . lineno - tok . value . count ( '<STR_LIT:\n>' ) - prev_token . lineno > <NUM_LIT:1> : <EOL> tok . AdjustNewlinesBefore ( ONE_BLANK_LINE ) <EOL> prev_token = tok <EOL> def _EmitLineUnformatted ( state ) : <EOL> """<STR_LIT>""" <EOL> prev_lineno = None <EOL> while state . next_token : <EOL> previous_token = state . next_token . previous_token <EOL> previous_lineno = previous_token . lineno <EOL> if previous_token . is_multiline_string : <EOL> previous_lineno += previous_token . value . count ( '<STR_LIT:\n>' ) <EOL> if previous_token . is_continuation : <EOL> newline = False <EOL> else : <EOL> newline = ( prev_lineno is not None and <EOL> state . next_token . lineno > previous_lineno ) <EOL> prev_lineno = state . next_token . lineno <EOL> state . AddTokenToState ( newline = newline , dry_run = False ) <EOL> def _LineContainsI18n ( uwline ) : <EOL> """<STR_LIT>""" <EOL> if style . Get ( '<STR_LIT>' ) : <EOL> for tok in uwline . tokens : <EOL> if tok . is_comment and re . match ( style . Get ( '<STR_LIT>' ) , tok . value ) : <EOL> return True <EOL> if style . Get ( '<STR_LIT>' ) : <EOL> length = len ( uwline . tokens ) <EOL> index = <NUM_LIT:0> <EOL> while index < length - <NUM_LIT:1> : <EOL> if ( uwline . tokens [ index + <NUM_LIT:1> ] . value == '<STR_LIT:(>' and <EOL> uwline . tokens [ index ] . value in style . Get ( '<STR_LIT>' ) ) : <EOL> return True <EOL> index += <NUM_LIT:1> <EOL> return False <EOL> def _LineHasContinuationMarkers ( uwline ) : <EOL> """<STR_LIT>""" <EOL> return any ( tok . is_continuation for tok in uwline . tokens ) <EOL> def _CanPlaceOnSingleLine ( uwline ) : <EOL> """<STR_LIT>""" <EOL> indent_amt = style . Get ( '<STR_LIT>' ) * uwline . depth <EOL> return ( uwline . last . total_length + indent_amt <= style . Get ( '<STR_LIT>' ) and <EOL> not any ( tok . is_comment for tok in uwline . tokens [ : - <NUM_LIT:1> ] ) ) <EOL> class _StateNode ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , state , newline , previous ) : <EOL> self . state = state . Clone ( ) <EOL> self . newline = newline <EOL> self . previous = previous <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' . format ( self . state , <EOL> self . newline ) <EOL> _OrderedPenalty = collections . namedtuple ( '<STR_LIT>' , [ '<STR_LIT>' , '<STR_LIT:count>' ] ) <EOL> _QueueItem = collections . namedtuple ( '<STR_LIT>' , [ '<STR_LIT>' , <EOL> '<STR_LIT>' ] ) <EOL> def _AnalyzeSolutionSpace ( initial_state , dry_run = False ) : <EOL> """<STR_LIT>""" <EOL> count = <NUM_LIT:0> <EOL> seen = set ( ) <EOL> p_queue = [ ] <EOL> node = _StateNode ( initial_state , False , None ) <EOL> heapq . heappush ( p_queue , _QueueItem ( _OrderedPenalty ( <NUM_LIT:0> , count ) , node ) ) <EOL> count += <NUM_LIT:1> <EOL> prev_penalty = <NUM_LIT:0> <EOL> while p_queue : <EOL> item = p_queue [ <NUM_LIT:0> ] <EOL> penalty = item . ordered_penalty . penalty <EOL> node = item . state_node <EOL> if not node . state . next_token : <EOL> break <EOL> heapq . heappop ( p_queue ) <EOL> if count > <NUM_LIT> : <EOL> node . state . ignore_stack_for_comparison = True <EOL> if node . state in seen : <EOL> continue <EOL> assert penalty >= prev_penalty <EOL> prev_penalty = penalty <EOL> seen . add ( node . state ) <EOL> count = _AddNextStateToQueue ( penalty , node , False , count , p_queue ) <EOL> count = _AddNextStateToQueue ( penalty , node , True , count , p_queue ) <EOL> if not p_queue : <EOL> return False <EOL> if not dry_run : <EOL> _ReconstructPath ( initial_state , heapq . heappop ( p_queue ) . state_node ) <EOL> return True <EOL> def _AddNextStateToQueue ( penalty , previous_node , newline , count , p_queue ) : <EOL> """<STR_LIT>""" <EOL> if newline and not previous_node . state . CanSplit ( ) : <EOL> return count <EOL> if not newline and previous_node . state . MustSplit ( ) : <EOL> return count <EOL> node = _StateNode ( previous_node . state , newline , previous_node ) <EOL> penalty += node . state . AddTokenToState ( newline = newline , dry_run = True ) <EOL> heapq . heappush ( p_queue , _QueueItem ( _OrderedPenalty ( penalty , count ) , node ) ) <EOL> return count + <NUM_LIT:1> <EOL> def _ReconstructPath ( initial_state , current ) : <EOL> """<STR_LIT>""" <EOL> path = collections . deque ( ) <EOL> while current . previous : <EOL> path . appendleft ( current ) <EOL> current = current . previous <EOL> for node in path : <EOL> initial_state . AddTokenToState ( newline = node . newline , dry_run = False ) <EOL> def _FormatFirstToken ( first_token , indent_depth , prev_uwline , final_lines ) : <EOL> """<STR_LIT>""" <EOL> first_token . AddWhitespacePrefix ( <EOL> _CalculateNumberOfNewlines ( first_token , indent_depth , prev_uwline , <EOL> final_lines ) , <EOL> indent_level = indent_depth ) <EOL> NO_BLANK_LINES = <NUM_LIT:1> <EOL> ONE_BLANK_LINE = <NUM_LIT:2> <EOL> TWO_BLANK_LINES = <NUM_LIT:3> <EOL> def _CalculateNumberOfNewlines ( first_token , indent_depth , prev_uwline , <EOL> final_lines ) : <EOL> """<STR_LIT>""" <EOL> if prev_uwline is None : <EOL> if first_token . newlines is not None : <EOL> pytree_utils . SetNodeAnnotation ( first_token . GetPytreeNode ( ) , <EOL> pytree_utils . Annotation . NEWLINES , None ) <EOL> return <NUM_LIT:0> <EOL> if first_token . is_docstring : <EOL> return NO_BLANK_LINES <EOL> prev_last_token = prev_uwline . last <EOL> if prev_last_token . is_docstring : <EOL> if not indent_depth and first_token . value in { '<STR_LIT:class>' , '<STR_LIT>' , '<STR_LIT>' } : <EOL> return TWO_BLANK_LINES <EOL> if _NoBlankLinesBeforeCurrentToken ( prev_last_token . value , first_token , <EOL> prev_last_token ) : <EOL> return NO_BLANK_LINES <EOL> else : <EOL> return ONE_BLANK_LINE <EOL> if first_token . value in { '<STR_LIT:class>' , '<STR_LIT>' , '<STR_LIT:@>' } : <EOL> if not indent_depth : <EOL> is_inline_comment = prev_last_token . whitespace_prefix . count ( '<STR_LIT:\n>' ) == <NUM_LIT:0> <EOL> if prev_last_token . is_comment and not is_inline_comment : <EOL> if _NoBlankLinesBeforeCurrentToken ( prev_last_token . value , first_token , <EOL> prev_last_token ) : <EOL> index = len ( final_lines ) - <NUM_LIT:1> <EOL> while index > <NUM_LIT:0> : <EOL> if not final_lines [ index - <NUM_LIT:1> ] . is_comment : <EOL> break <EOL> index -= <NUM_LIT:1> <EOL> if final_lines [ index - <NUM_LIT:1> ] . first . value == '<STR_LIT:@>' : <EOL> final_lines [ index ] . first . AdjustNewlinesBefore ( NO_BLANK_LINES ) <EOL> else : <EOL> prev_last_token . AdjustNewlinesBefore ( TWO_BLANK_LINES ) <EOL> if first_token . newlines is not None : <EOL> pytree_utils . SetNodeAnnotation ( first_token . GetPytreeNode ( ) , <EOL> pytree_utils . Annotation . NEWLINES , <EOL> None ) <EOL> return NO_BLANK_LINES <EOL> elif prev_uwline . first . value in { '<STR_LIT:class>' , '<STR_LIT>' } : <EOL> if not style . Get ( '<STR_LIT>' ) : <EOL> pytree_utils . SetNodeAnnotation ( first_token . node , <EOL> pytree_utils . Annotation . NEWLINES , None ) <EOL> return NO_BLANK_LINES <EOL> if first_token . is_comment : <EOL> first_token_lineno = first_token . lineno - first_token . value . count ( '<STR_LIT:\n>' ) <EOL> else : <EOL> first_token_lineno = first_token . lineno <EOL> if first_token_lineno - prev_last_token . lineno > <NUM_LIT:1> : <EOL> return ONE_BLANK_LINE <EOL> return NO_BLANK_LINES <EOL> def _SingleOrMergedLines ( uwlines ) : <EOL> """<STR_LIT>""" <EOL> index = <NUM_LIT:0> <EOL> last_was_merged = False <EOL> while index < len ( uwlines ) : <EOL> if uwlines [ index ] . disable : <EOL> uwline = uwlines [ index ] <EOL> index += <NUM_LIT:1> <EOL> while index < len ( uwlines ) : <EOL> column = uwline . last . column + <NUM_LIT:2> <EOL> if uwlines [ index ] . lineno != uwline . lineno : <EOL> break <EOL> if uwline . last . value != '<STR_LIT::>' : <EOL> leaf = pytree . Leaf ( type = token . SEMI , <EOL> value = '<STR_LIT:;>' , <EOL> context = ( '<STR_LIT>' , ( uwline . lineno , column ) ) ) <EOL> uwline . AppendToken ( format_token . FormatToken ( leaf ) ) <EOL> for tok in uwlines [ index ] . tokens : <EOL> uwline . AppendToken ( tok ) <EOL> index += <NUM_LIT:1> <EOL> yield uwline <EOL> elif line_joiner . CanMergeMultipleLines ( uwlines [ index : ] , last_was_merged ) : <EOL> next_uwline = uwlines [ index + <NUM_LIT:1> ] <EOL> for tok in next_uwline . tokens : <EOL> uwlines [ index ] . AppendToken ( tok ) <EOL> if ( len ( next_uwline . tokens ) == <NUM_LIT:1> and <EOL> next_uwline . first . is_multiline_string ) : <EOL> uwlines [ index ] . disable = True <EOL> yield uwlines [ index ] <EOL> index += <NUM_LIT:2> <EOL> last_was_merged = True <EOL> else : <EOL> yield uwlines [ index ] <EOL> index += <NUM_LIT:1> <EOL> last_was_merged = False <EOL> def _NoBlankLinesBeforeCurrentToken ( text , cur_token , prev_token ) : <EOL> """<STR_LIT>""" <EOL> cur_token_lineno = cur_token . lineno <EOL> if cur_token . is_comment : <EOL> cur_token_lineno -= cur_token . value . count ( '<STR_LIT:\n>' ) <EOL> num_newlines = text . count ( '<STR_LIT:\n>' ) if not prev_token . is_comment else <NUM_LIT:0> <EOL> return prev_token . lineno + num_newlines == cur_token_lineno - <NUM_LIT:1> </s>
