<s> from __future__ import unicode_literals <EOL> from difflib import SequenceMatcher <EOL> from doc . models import SectionMatcher , DocDiff , Section , PageMatcher , SectionChanger , LinkChange <EOL> ABS_THRESHOLD = <NUM_LIT> <EOL> RELATIVE_THRESHOLD = <NUM_LIT:1.0> <EOL> RATIO_THRESHOLD = <NUM_LIT> <EOL> DEFAULT_DISTANCES = ( ( <NUM_LIT> , <NUM_LIT> ) , ( <NUM_LIT> , <NUM_LIT> ) , ( <NUM_LIT> , <NUM_LIT> ) ) <EOL> def get_null_matches ( from_elem , to_elems , factor ) : <EOL> mr = MatcherResult ( from_elem = from_elem , factor = factor ) <EOL> mr . to_elems = { to_elem . pk : [ to_elem , <NUM_LIT:0.0> ] for to_elem in to_elems } <EOL> return mr <EOL> def get_one_match ( from_elem , to_elems , factor , to_elem , confidence ) : <EOL> mr = get_null_matches ( from_elem , to_elems , factor ) <EOL> mr . to_elems [ to_elem . pk ] = [ to_elem , confidence ] <EOL> return mr <EOL> def get_confidence_str_distance ( text_from , text_to , distances ) : <EOL> match_ratio = SequenceMatcher ( None , text_from , text_to ) . ratio ( ) <EOL> conf = <NUM_LIT:0.0> <EOL> for ( threshold , confidence ) in distances : <EOL> if match_ratio >= threshold : <EOL> conf = confidence + ( match_ratio - threshold ) <EOL> break <EOL> return conf <EOL> def sort_match_results ( match_results ) : <EOL> matches = { } <EOL> for result in match_results : <EOL> for to_pk in result . to_elems : <EOL> if to_pk not in matches : <EOL> matches [ to_pk ] = [ result . to_elems [ to_pk ] [ <NUM_LIT:0> ] , <NUM_LIT:0.0> ] <EOL> matches [ to_pk ] [ <NUM_LIT:1> ] += result . to_elems [ to_pk ] [ <NUM_LIT:1> ] * result . factor <EOL> sorted_matches = sorted ( matches . values ( ) , key = lambda l : l [ <NUM_LIT:1> ] , reverse = True ) <EOL> return sorted_matches <EOL> def has_best_match ( sorted_results ) : <EOL> if len ( sorted_results ) <= <NUM_LIT:1> : <EOL> return True <EOL> return sorted_results [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] > ABS_THRESHOLD and ( sorted_results [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] - sorted_results [ <NUM_LIT:1> ] [ <NUM_LIT:1> ] ) > RELATIVE_THRESHOLD <EOL> def get_best_match ( match_results ) : <EOL> sorted_results = sort_match_results ( match_results ) <EOL> if has_best_match ( sorted_results ) : <EOL> return sorted_results [ <NUM_LIT:0> ] <EOL> else : <EOL> return None <EOL> class MatcherResult ( object ) : <EOL> def __init__ ( self , from_elem = None , factor = None ) : <EOL> self . from_elem = from_elem <EOL> self . factor = factor <EOL> self . to_elems = { } <EOL> class SectionNumberMatcher ( object ) : <EOL> factor = <NUM_LIT> <EOL> def match ( self , section_from , section_tos ) : <EOL> fnumber = section_from . number . strip ( ) <EOL> ftitle = section_from . title . strip ( ) <EOL> mresult = get_null_matches ( section_from , section_tos , self . factor ) <EOL> if fnumber == '<STR_LIT>' : <EOL> return mresult <EOL> for section_to in section_tos : <EOL> ttitle = section_to . title . strip ( ) <EOL> tnumber = section_to . number . strip ( ) <EOL> if fnumber == tnumber and ftitle == ttitle : <EOL> mresult . to_elems [ section_to . pk ] [ <NUM_LIT:1> ] = <NUM_LIT:1.0> <EOL> break <EOL> elif fnumber == tnumber : <EOL> conf = get_confidence_str_distance ( ftitle , ttitle , <EOL> DEFAULT_DISTANCES ) <EOL> mresult . to_elems [ section_to . pk ] [ <NUM_LIT:1> ] = conf <EOL> break <EOL> return mresult <EOL> class TitleMatcher ( object ) : <EOL> factor = <NUM_LIT> <EOL> def match ( self , from_elem , to_elems ) : <EOL> ftitle = from_elem . title . strip ( ) <EOL> mresult = MatcherResult ( from_elem , self . factor ) <EOL> for to_elem in to_elems : <EOL> ttitle = to_elem . title . strip ( ) <EOL> if ftitle == to_elem . title . strip ( ) : <EOL> mresult . to_elems [ to_elem . pk ] = [ to_elem , <NUM_LIT:1.0> ] <EOL> else : <EOL> conf = get_confidence_str_distance ( ftitle , ttitle , <EOL> DEFAULT_DISTANCES ) <EOL> mresult . to_elems [ to_elem . pk ] = [ to_elem , conf ] <EOL> return mresult <EOL> class ChildrenMatcher ( object ) : <EOL> factor = <NUM_LIT> <EOL> def _children ( self , elem ) : <EOL> children = [ ] <EOL> try : <EOL> children = list ( elem . sections . all ( ) ) <EOL> except Exception : <EOL> try : <EOL> children = list ( elem . children . all ( ) ) <EOL> except Exception : <EOL> children = [ ] <EOL> return children <EOL> def _max_ratio ( self , title , children_names ) : <EOL> max_ratio = <NUM_LIT:0.0> <EOL> for child_name in children_names : <EOL> ratio = SequenceMatcher ( None , title , child_name ) . ratio ( ) <EOL> if ratio > max_ratio : <EOL> max_ratio = ratio <EOL> return max_ratio <EOL> def match ( self , from_elem , to_elems ) : <EOL> children = self . _children ( from_elem ) <EOL> children_size = len ( children ) <EOL> mresult = get_null_matches ( from_elem , to_elems , self . factor ) <EOL> if children_size == <NUM_LIT:0> : <EOL> return mresult <EOL> for to_elem in to_elems : <EOL> to_children = self . _children ( to_elem ) <EOL> children_names = [ child . title . strip ( ) for child in to_children ] <EOL> matches = <NUM_LIT:0.0> <EOL> for child in children : <EOL> ratio = self . _max_ratio ( child . title . strip ( ) , children_names ) <EOL> if ratio >= RATIO_THRESHOLD : <EOL> matches += RATIO_THRESHOLD + ( ratio - RATIO_THRESHOLD ) <EOL> conf = matches / children_size <EOL> mresult . to_elems [ to_elem . pk ] [ <NUM_LIT:1> ] = conf <EOL> return mresult <EOL> class SectionParentMatcher ( object ) : <EOL> factor = <NUM_LIT> <EOL> def match ( self , from_elem , to_elems ) : <EOL> mresult = get_null_matches ( from_elem , to_elems , self . factor ) <EOL> if from_elem . parent is None : <EOL> return mresult <EOL> ptitle = from_elem . parent . title . strip ( ) <EOL> pnumber = from_elem . parent . number . strip ( ) <EOL> for to_elem in to_elems : <EOL> conf = <NUM_LIT:0.0> <EOL> parent = to_elem . parent <EOL> if parent is None : <EOL> continue <EOL> pttitle = parent . title . strip ( ) <EOL> ptnumber = parent . number . strip ( ) <EOL> if pnumber != '<STR_LIT>' and pnumber == ptnumber : <EOL> conf += <NUM_LIT:0.5> <EOL> ratio = SequenceMatcher ( None , ptitle , pttitle ) . ratio ( ) <EOL> if ratio >= RATIO_THRESHOLD : <EOL> new_conf = RATIO_THRESHOLD + ( ratio - RATIO_THRESHOLD ) <EOL> if pnumber == '<STR_LIT>' : <EOL> conf += new_conf <EOL> else : <EOL> conf += new_conf / <NUM_LIT> <EOL> mresult . to_elems [ to_elem . pk ] [ <NUM_LIT:1> ] = conf <EOL> return mresult <EOL> class SectionPageMatcher ( object ) : <EOL> factor = <NUM_LIT:1.0> <EOL> def match ( self , from_elem , to_elems ) : <EOL> mresult = get_null_matches ( from_elem , to_elems , self . factor ) <EOL> ptitle = from_elem . page . title . strip ( ) <EOL> for to_elem in to_elems : <EOL> pttitle = to_elem . page . title . strip ( ) <EOL> ratio = SequenceMatcher ( None , ptitle , pttitle ) . ratio ( ) <EOL> if ratio >= RATIO_THRESHOLD : <EOL> conf = RATIO_THRESHOLD + ( ratio - RATIO_THRESHOLD ) <EOL> mresult . to_elems [ to_elem . pk ] = [ to_elem , conf ] <EOL> return mresult <EOL> class DocDiffer ( object ) : <EOL> def diff_docs ( self , document_from , document_to ) : <EOL> ddiff = DocDiff ( ) <EOL> ddiff . document_from = document_from <EOL> ddiff . document_to = document_to <EOL> ddiff . pages_size_from = document_from . pages . count ( ) <EOL> ddiff . pages_size_to = document_to . pages . count ( ) <EOL> ddiff . sections_size_from = Section . objects . filter ( page__document = document_from ) . count ( ) <EOL> ddiff . sections_size_to = Section . objects . filter ( page__document = document_to ) . count ( ) <EOL> ddiff . save ( ) <EOL> self . match_pages ( ddiff ) <EOL> self . match_sections ( ddiff ) <EOL> self . compute_changes ( ddiff ) <EOL> return ddiff <EOL> def match_pages ( self , ddiff ) : <EOL> page_tos = list ( ddiff . document_to . pages . all ( ) ) <EOL> matched_tos = set ( ) <EOL> for page_from in ddiff . document_from . pages . all ( ) : <EOL> best_match = self . _match_page ( page_from , page_tos ) <EOL> if best_match is not None : <EOL> ( page_to , confidence ) = best_match <EOL> p_matcher = PageMatcher ( <EOL> page_from = page_from , <EOL> page_to = page_to , <EOL> confidence = confidence , <EOL> diff = ddiff ) <EOL> p_matcher . save ( ) <EOL> matched_tos . add ( page_to . pk ) <EOL> else : <EOL> ddiff . removed_pages . add ( page_from ) <EOL> for page_to in page_tos : <EOL> if page_to . pk not in matched_tos : <EOL> ddiff . added_pages . add ( page_to ) <EOL> print ( '<STR_LIT>' ) <EOL> def match_sections ( self , ddiff ) : <EOL> section_froms = list ( Section . objects <EOL> . filter ( page__document = ddiff . document_from ) . all ( ) ) <EOL> section_tos = list ( Section . objects <EOL> . filter ( page__document = ddiff . document_to ) . all ( ) ) <EOL> matched_tos = set ( ) <EOL> for section_from in section_froms : <EOL> best_match = self . _match_section ( section_from , section_tos ) <EOL> if best_match is not None : <EOL> ( section_to , confidence ) = best_match <EOL> s_matcher = SectionMatcher ( <EOL> section_from = section_from , <EOL> section_to = section_to , <EOL> confidence = confidence , <EOL> diff = ddiff ) <EOL> s_matcher . save ( ) <EOL> matched_tos . add ( section_to . pk ) <EOL> else : <EOL> ddiff . removed_sections . add ( section_from ) <EOL> print ( '<STR_LIT>' . format ( section_from . title ) ) <EOL> for section_to in section_tos : <EOL> if section_to . pk not in matched_tos : <EOL> ddiff . added_sections . add ( section_to ) <EOL> print ( '<STR_LIT>' ) <EOL> print ( '<STR_LIT>' ) <EOL> def compute_changes ( self , ddiff ) : <EOL> for section_matcher in ddiff . section_matches . all ( ) : <EOL> section_from = section_matcher . section_from <EOL> section_to = section_matcher . section_to <EOL> words_from = section_from . word_count <EOL> words_to = section_to . word_count <EOL> if words_from > <NUM_LIT:0> : <EOL> change_words = ( float ( words_to - words_from ) / <EOL> float ( words_from ) ) * <NUM_LIT> <EOL> else : <EOL> change_words = words_to * <NUM_LIT> <EOL> if words_from != words_to : <EOL> change = SectionChanger ( section_from = section_from , <EOL> section_to = section_to , <EOL> words_from = words_from , <EOL> words_to = words_to , <EOL> change = change_words , <EOL> diff = ddiff ) <EOL> change . save ( ) <EOL> print ( '<STR_LIT>' ) <EOL> def _match_page ( self , page_from , page_tos ) : <EOL> match_results = [ ] <EOL> match_results . append ( TitleMatcher ( ) . match ( page_from , page_tos ) ) <EOL> match_results . append ( ChildrenMatcher ( ) . match ( page_from , page_tos ) ) <EOL> best_match = get_best_match ( match_results ) <EOL> return best_match <EOL> def _match_section ( self , section_from , section_tos ) : <EOL> match_results = [ ] <EOL> match_results . append ( <EOL> SectionNumberMatcher ( ) . match ( section_from , section_tos ) ) <EOL> match_results . append ( <EOL> TitleMatcher ( ) . match ( section_from , section_tos ) ) <EOL> match_results . append ( <EOL> ChildrenMatcher ( ) . match ( section_from , section_tos ) ) <EOL> match_results . append ( <EOL> SectionParentMatcher ( ) . match ( section_from , section_tos ) ) <EOL> match_results . append ( <EOL> SectionPageMatcher ( ) . match ( section_from , section_tos ) ) <EOL> best_match = get_best_match ( match_results ) <EOL> return best_match <EOL> class DocLinkerDiffer ( object ) : <EOL> def __init__ ( self , docdiff ) : <EOL> self . docdiff = docdiff <EOL> def diff_links ( self ) : <EOL> added_links = [ ] <EOL> removed_links = [ ] <EOL> processed_sections = set ( ) <EOL> sections_from = Section . objects . filter ( page__document = self . docdiff . document_from ) . all ( ) <EOL> sections_to = Section . objects . filter ( page__document = self . docdiff . document_to ) . all ( ) <EOL> self . _process_sections_from ( added_links , removed_links , <EOL> processed_sections , sections_from ) <EOL> self . _process_sections_to ( added_links , removed_links , <EOL> processed_sections , sections_to ) <EOL> return ( added_links , removed_links ) <EOL> def _process_sections_from ( self , added_links , removed_links , <EOL> processed_sections , sections_from ) : <EOL> for section_from in sections_from : <EOL> try : <EOL> matcher = section_from . match_froms . get ( diff = self . docdiff ) <EOL> section_to = matcher . section_to <EOL> links_from = self . _get_links ( section_from ) <EOL> links_to = self . _get_links ( section_to ) <EOL> self . _diff_links ( links_from , links_to , added_links , <EOL> removed_links ) <EOL> processed_sections . add ( section_to . pk ) <EOL> except Exception : <EOL> self . _add_removed_links ( section_from , removed_links ) <EOL> def _process_sections_to ( self , added_links , removed_links , <EOL> processed_sections , sections_to ) : <EOL> for section_to in sections_to : <EOL> if section_to . pk not in processed_sections : <EOL> self . _add_added_links ( section_to , added_links ) <EOL> def _get_links ( self , section ) : <EOL> links = [ ] <EOL> for single_ref in section . code_references . all ( ) : <EOL> link = self . _get_link ( single_ref , <EOL> single_ref . project_release ) <EOL> if link is not None : <EOL> links . append ( link ) <EOL> return links <EOL> def _diff_links ( self , links_from , links_to , added_links , removed_links ) : <EOL> for link_from in links_from : <EOL> link_to = self . _pop_link ( link_from , links_to ) <EOL> if link_to is None : <EOL> link_change = LinkChange ( diff = self . docdiff , <EOL> link_from = link_from , from_matched_section = True ) <EOL> link_change . save ( ) <EOL> removed_links . append ( link_change ) <EOL> for link_to in links_to : <EOL> link_change = LinkChange ( diff = self . docdiff , <EOL> link_to = link_to , from_matched_section = True ) <EOL> link_change . save ( ) <EOL> added_links . append ( link_change ) <EOL> def _pop_link ( self , link_to_find , links ) : <EOL> index = - <NUM_LIT:1> <EOL> link = None <EOL> human_string = link_to_find . code_element . human_string ( ) <EOL> is_snippet = link_to_find . code_reference . snippet is not None <EOL> for i , temp_link in enumerate ( links ) : <EOL> temp_human = temp_link . code_element . human_string ( ) <EOL> temp_snippet = temp_link . code_reference . snippet is not None <EOL> if human_string == temp_human and is_snippet == temp_snippet : <EOL> index = i <EOL> link = temp_link <EOL> break <EOL> if index > - <NUM_LIT:1> : <EOL> del links [ index ] <EOL> return link <EOL> def _add_removed_links ( self , section , removed_links ) : <EOL> for link in self . _get_links ( section ) : <EOL> if link is None : <EOL> continue <EOL> else : <EOL> link_change = LinkChange ( <EOL> diff = self . docdiff , <EOL> link_from = link ) <EOL> link_change . save ( ) <EOL> removed_links . append ( link_change ) <EOL> def _add_added_links ( self , section , added_links ) : <EOL> for link in self . _get_links ( section ) : <EOL> if link is None : <EOL> continue <EOL> else : <EOL> link_change = LinkChange ( <EOL> diff = self . docdiff , <EOL> link_to = link ) <EOL> link_change . save ( ) <EOL> added_links . append ( link_change ) <EOL> def _get_link ( self , code_reference , project_release ) : <EOL> link = None <EOL> try : <EOL> release_links = code_reference . release_links . get ( project_release = project_release ) <EOL> link = release_links . first_link <EOL> except Exception : <EOL> pass <EOL> return link </s>
