<s> import difflib <EOL> import re <EOL> import textutils <EOL> re_ignore = re . compile ( "<STR_LIT>" ) <EOL> re_words = re . compile ( "<STR_LIT>" ) <EOL> re_ws = re . compile ( "<STR_LIT>" ) <EOL> re_conflict = re . compile ( "<STR_LIT>" ) <EOL> def analyzeChunk ( deletedLines , insertedLines , moved = False ) : <EOL> if not deletedLines or not insertedLines : return None <EOL> deletedLines = map ( textutils . decode , deletedLines ) <EOL> insertedLines = map ( textutils . decode , insertedLines ) <EOL> if len ( deletedLines ) * len ( insertedLines ) <= <NUM_LIT> and not moved : <EOL> analysis = analyzeChunk1 ( deletedLines , insertedLines ) <EOL> else : <EOL> deletedLinesNoWS = [ re_ws . sub ( "<STR_LIT:U+0020>" , line . strip ( ) ) for line in deletedLines ] <EOL> insertedLinesNoWS = [ re_ws . sub ( "<STR_LIT:U+0020>" , line . strip ( ) ) for line in insertedLines ] <EOL> sm = difflib . SequenceMatcher ( None , deletedLinesNoWS , insertedLinesNoWS ) <EOL> blocks = sm . get_matching_blocks ( ) <EOL> analysis = [ ] <EOL> pi = <NUM_LIT:0> <EOL> pj = <NUM_LIT:0> <EOL> for i , j , n in blocks : <EOL> if not n : continue <EOL> if i > pi and j > pj : <EOL> analysis . append ( analyzeChunk1 ( deletedLines [ pi : i ] , insertedLines [ pj : j ] , <EOL> offsetA = pi , offsetB = pj ) ) <EOL> analysis . append ( analyzeWhiteSpaceChanges ( deletedLines [ i : i + n ] , insertedLines [ j : j + n ] , <EOL> offsetA = i , offsetB = j , full = moved ) ) <EOL> pi = i + n <EOL> pj = j + n <EOL> if pi < len ( deletedLines ) and pj < len ( insertedLines ) : <EOL> analysis . append ( analyzeChunk1 ( deletedLines [ pi : ] , insertedLines [ pj : ] , <EOL> offsetA = pi , offsetB = pj ) ) <EOL> analysis = "<STR_LIT:;>" . join ( filter ( None , analysis ) ) <EOL> if analysis : return analysis <EOL> else : return None <EOL> def analyzeChunk1 ( deletedLines , insertedLines , offsetA = <NUM_LIT:0> , offsetB = <NUM_LIT:0> ) : <EOL> matches = [ ] <EOL> equals = [ ] <EOL> if len ( deletedLines ) * len ( insertedLines ) > <NUM_LIT> : return "<STR_LIT>" <EOL> def ratio ( sm , a , b , aLength , bLength ) : <EOL> matching = <NUM_LIT:0> <EOL> for i , j , n in sm . get_matching_blocks ( ) : <EOL> matching += sum ( map ( len , map ( unicode . strip , a [ i : i + n ] ) ) ) <EOL> if aLength > <NUM_LIT:5> and len ( sm . get_matching_blocks ( ) ) == <NUM_LIT:2> : <EOL> return float ( matching ) / aLength <EOL> else : <EOL> return <NUM_LIT> * matching / ( aLength + bLength ) <EOL> for deletedIndex , deleted in enumerate ( deletedLines ) : <EOL> deletedStripped = deleted . strip ( ) <EOL> deletedNoWS = re_ws . sub ( "<STR_LIT>" , deletedStripped ) <EOL> if re_conflict . match ( deleted ) : continue <EOL> if not re_ignore . match ( deleted ) : <EOL> deletedWords = re_words . findall ( deleted ) <EOL> for insertedIndex , inserted in enumerate ( insertedLines ) : <EOL> insertedStripped = inserted . strip ( ) <EOL> insertedNoWS = re_ws . sub ( "<STR_LIT>" , insertedStripped ) <EOL> if not re_ignore . match ( inserted ) : <EOL> insertedWords = re_words . findall ( inserted ) <EOL> sm = difflib . SequenceMatcher ( None , deletedWords , insertedWords ) <EOL> r = ratio ( sm , deletedWords , insertedWords , len ( deletedNoWS ) , len ( insertedNoWS ) ) <EOL> if r > <NUM_LIT:0.5> : matches . append ( ( r , deletedIndex , insertedIndex , deletedWords , insertedWords , sm ) ) <EOL> elif deletedStripped == insertedStripped : <EOL> equals . append ( ( deletedIndex , insertedIndex ) ) <EOL> else : <EOL> for insertedIndex , inserted in enumerate ( insertedLines ) : <EOL> if deletedStripped == inserted . strip ( ) : <EOL> equals . append ( ( deletedIndex , insertedIndex ) ) <EOL> if matches : <EOL> matches . sort ( key = lambda x : x [ <NUM_LIT:0> ] , reverse = True ) <EOL> final = [ ] <EOL> while matches : <EOL> r , deletedIndex , insertedIndex , deletedWords , insertedWords , sm = matches . pop ( <NUM_LIT:0> ) <EOL> final . append ( ( deletedIndex , insertedIndex , deletedWords , insertedWords , sm ) ) <EOL> matches = filter ( lambda data : data [ <NUM_LIT:1> ] != deletedIndex and data [ <NUM_LIT:2> ] != insertedIndex and ( data [ <NUM_LIT:1> ] < deletedIndex ) == ( data [ <NUM_LIT:2> ] < insertedIndex ) , matches ) <EOL> equals = filter ( lambda data : ( data [ <NUM_LIT:0> ] < deletedIndex ) == ( data [ <NUM_LIT:1> ] < insertedIndex ) , equals ) <EOL> final . sort ( ) <EOL> equals . sort ( ) <EOL> result = [ ] <EOL> previousDeletedIndex = - <NUM_LIT:1> <EOL> previousInsertedIndex = - <NUM_LIT:1> <EOL> final . append ( ( len ( deletedLines ) , len ( insertedLines ) , None , None , None ) ) <EOL> for deletedIndex , insertedIndex , deletedWords , insertedWords , sm in final : <EOL> while equals and ( equals [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] < deletedIndex or equals [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] < insertedIndex ) : <EOL> di , ii = equals . pop ( <NUM_LIT:0> ) <EOL> if previousDeletedIndex < di < deletedIndex and previousInsertedIndex < ii < insertedIndex : <EOL> deletedLine = deletedLines [ di ] <EOL> insertedLine = insertedLines [ ii ] <EOL> lineDiff = analyzeWhiteSpaceLine ( deletedLine , insertedLine ) <EOL> if lineDiff : result . append ( "<STR_LIT>" % ( di + offsetA , ii + offsetB , lineDiff ) ) <EOL> else : result . append ( "<STR_LIT>" % ( di + offsetA , ii + offsetB ) ) <EOL> previousDeletedIndex = di <EOL> previousInsertedIndex = ii <EOL> while equals and ( di == equals [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] or ii == equals [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] ) : equals . pop ( <NUM_LIT:0> ) <EOL> if sm is None : break <EOL> lineDiff = [ ] <EOL> deletedLine = deletedLines [ deletedIndex ] <EOL> insertedLine = insertedLines [ insertedIndex ] <EOL> if deletedLine != insertedLine and deletedLine . strip ( ) == insertedLine . strip ( ) : <EOL> lineDiff . append ( "<STR_LIT>" ) <EOL> lineDiff . append ( analyzeWhiteSpaceLine ( deletedLine , insertedLine ) ) <EOL> else : <EOL> for tag , i1 , i2 , j1 , j2 in sm . get_opcodes ( ) : <EOL> if tag == '<STR_LIT:replace>' : lineDiff . append ( "<STR_LIT>" % ( offsetInLine ( deletedWords , i1 ) , offsetInLine ( deletedWords , i2 ) , offsetInLine ( insertedWords , j1 ) , offsetInLine ( insertedWords , j2 ) ) ) <EOL> elif tag == '<STR_LIT>' : lineDiff . append ( "<STR_LIT>" % ( offsetInLine ( deletedWords , i1 ) , offsetInLine ( deletedWords , i2 ) ) ) <EOL> elif tag == '<STR_LIT>' : lineDiff . append ( "<STR_LIT>" % ( offsetInLine ( insertedWords , j1 ) , offsetInLine ( insertedWords , j2 ) ) ) <EOL> if lineDiff : <EOL> result . append ( "<STR_LIT>" % ( deletedIndex + offsetA , insertedIndex + offsetB , "<STR_LIT:U+002C>" . join ( lineDiff ) ) ) <EOL> else : <EOL> result . append ( "<STR_LIT>" % ( deletedIndex + offsetA , insertedIndex + offsetB ) ) <EOL> previousDeletedIndex = deletedIndex <EOL> previousInsertedIndex = insertedIndex <EOL> return "<STR_LIT:;>" . join ( result ) <EOL> elif deletedLines [ - <NUM_LIT:1> ] == insertedLines [ - <NUM_LIT:1> ] : <EOL> ndeleted = len ( deletedLines ) <EOL> ninserted = len ( insertedLines ) <EOL> result = [ ] <EOL> index = <NUM_LIT:1> <EOL> while index <= ndeleted and index <= ninserted and deletedLines [ - index ] == insertedLines [ - index ] : <EOL> result . append ( "<STR_LIT>" % ( ndeleted - index + offsetA , ninserted - index + offsetB ) ) <EOL> index += <NUM_LIT:1> <EOL> return "<STR_LIT:;>" . join ( reversed ( result ) ) <EOL> else : <EOL> return "<STR_LIT>" <EOL> def offsetInLine ( words , offset ) : <EOL> return sum ( map ( lambda word : len ( word . encode ( "<STR_LIT:utf-8>" ) ) , words [ <NUM_LIT:0> : offset ] ) ) <EOL> re_ws_words = re . compile ( "<STR_LIT>" ) <EOL> def analyzeWhiteSpaceLine ( deletedLine , insertedLine ) : <EOL> deletedLine = textutils . decode ( deletedLine ) <EOL> insertedLine = textutils . decode ( insertedLine ) <EOL> deletedWords = filter ( None , re_ws_words . findall ( deletedLine ) ) <EOL> insertedWords = filter ( None , re_ws_words . findall ( insertedLine ) ) <EOL> sm = difflib . SequenceMatcher ( None , deletedWords , insertedWords ) <EOL> lineDiff = [ ] <EOL> for tag , i1 , i2 , j1 , j2 in sm . get_opcodes ( ) : <EOL> if tag == '<STR_LIT:replace>' : <EOL> lineDiff . append ( "<STR_LIT>" % ( offsetInLine ( deletedWords , i1 ) , <EOL> offsetInLine ( deletedWords , i2 ) , <EOL> offsetInLine ( insertedWords , j1 ) , <EOL> offsetInLine ( insertedWords , j2 ) ) ) <EOL> elif tag == '<STR_LIT>' : <EOL> lineDiff . append ( "<STR_LIT>" % ( offsetInLine ( deletedWords , i1 ) , <EOL> offsetInLine ( deletedWords , i2 ) ) ) <EOL> elif tag == '<STR_LIT>' : <EOL> lineDiff . append ( "<STR_LIT>" % ( offsetInLine ( insertedWords , j1 ) , <EOL> offsetInLine ( insertedWords , j2 ) ) ) <EOL> return "<STR_LIT:U+002C>" . join ( lineDiff ) <EOL> def analyzeWhiteSpaceChanges ( deletedLines , insertedLines , at_eof = False , <EOL> offsetA = <NUM_LIT:0> , offsetB = <NUM_LIT:0> , full = False ) : <EOL> result = [ ] <EOL> for index , ( deletedLine , insertedLine ) in enumerate ( zip ( deletedLines , insertedLines ) ) : <EOL> if deletedLine != insertedLine : <EOL> result . append ( "<STR_LIT>" <EOL> % ( index + offsetA , index + offsetB , <EOL> analyzeWhiteSpaceLine ( deletedLine , insertedLine ) ) ) <EOL> elif index == len ( deletedLines ) - <NUM_LIT:1> and at_eof : <EOL> result . append ( "<STR_LIT>" % ( index + offsetA , index + offsetB ) ) <EOL> elif full : <EOL> result . append ( "<STR_LIT>" % ( index + offsetA , index + offsetB ) ) <EOL> if not result and ( offsetA or offsetB ) : <EOL> result . append ( "<STR_LIT>" % ( offsetA , offsetB ) ) <EOL> return "<STR_LIT:;>" . join ( result ) </s>
