<s> from __future__ import unicode_literals <EOL> import bz2 <EOL> import gc <EOL> import hashlib <EOL> import os <EOL> from django . db import DatabaseError , models , reset_queries , connection <EOL> from django . db . models import Count , Q <EOL> from django . db . utils import IntegrityError <EOL> from django . utils . encoding import smart_unicode <EOL> from django . utils . functional import cached_property <EOL> from django . utils . six . moves import range <EOL> from django . utils . translation import ugettext as _ <EOL> from djblets . siteconfig . models import SiteConfiguration <EOL> from reviewboard . diffviewer . differ import DiffCompatVersion <EOL> from reviewboard . diffviewer . errors import DiffTooBigError , EmptyDiffError <EOL> from reviewboard . scmtools . core import PRE_CREATION , UNKNOWN , FileNotFoundError <EOL> class FileDiffManager ( models . Manager ) : <EOL> """<STR_LIT>""" <EOL> MIGRATE_OBJECT_LIMIT = <NUM_LIT:200> <EOL> def unmigrated ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . exclude ( <EOL> ( Q ( diff64 = '<STR_LIT>' ) | Q ( diff64__isnull = True ) ) & <EOL> ( Q ( parent_diff64 = '<STR_LIT>' ) | Q ( parent_diff64__isnull = True ) ) ) <EOL> def get_migration_counts ( self ) : <EOL> """<STR_LIT>""" <EOL> from reviewboard . diffviewer . models import LegacyFileDiffData <EOL> unmigrated_filediffs_count = self . unmigrated ( ) . count ( ) <EOL> legacy_fdd_count = LegacyFileDiffData . objects . count ( ) <EOL> return { <EOL> '<STR_LIT>' : unmigrated_filediffs_count , <EOL> '<STR_LIT>' : legacy_fdd_count , <EOL> '<STR_LIT>' : unmigrated_filediffs_count + legacy_fdd_count , <EOL> } <EOL> def migrate_all ( self , batch_done_cb = None , counts = None , batch_size = <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> from reviewboard . diffviewer . models import LegacyFileDiffData <EOL> total_diffs_migrated = <NUM_LIT:0> <EOL> total_diff_size = <NUM_LIT:0> <EOL> total_bytes_saved = <NUM_LIT:0> <EOL> unmigrated_filediffs = self . unmigrated ( ) <EOL> legacy_data_items = LegacyFileDiffData . objects . all ( ) <EOL> if counts : <EOL> unmigrated_filediffs_count = counts [ '<STR_LIT>' ] <EOL> legacy_data_items_count = counts [ '<STR_LIT>' ] <EOL> total_count = counts [ '<STR_LIT>' ] <EOL> else : <EOL> unmigrated_filediffs_count = unmigrated_filediffs . count ( ) <EOL> legacy_data_items_count = legacy_data_items . count ( ) <EOL> total_count = legacy_data_items_count + unmigrated_filediffs_count <EOL> migration_tasks = ( <EOL> ( self . _migrate_filediffs , <EOL> unmigrated_filediffs , <EOL> unmigrated_filediffs_count ) , <EOL> ( self . _migrate_legacy_fdd , <EOL> legacy_data_items , <EOL> legacy_data_items_count ) , <EOL> ) <EOL> for migrate_func , queryset , count in migration_tasks : <EOL> for batch_info in migrate_func ( queryset , count , batch_size ) : <EOL> total_diffs_migrated += batch_info [ <NUM_LIT:0> ] <EOL> total_diff_size += batch_info [ <NUM_LIT:1> ] <EOL> total_bytes_saved += batch_info [ <NUM_LIT:2> ] <EOL> if callable ( batch_done_cb ) : <EOL> batch_done_cb ( total_diffs_migrated , total_count ) <EOL> return { <EOL> '<STR_LIT>' : total_diffs_migrated , <EOL> '<STR_LIT>' : total_diff_size , <EOL> '<STR_LIT>' : total_diff_size - total_bytes_saved , <EOL> '<STR_LIT>' : total_bytes_saved , <EOL> } <EOL> def _migrate_legacy_fdd ( self , legacy_data_items , count , batch_size ) : <EOL> """<STR_LIT>""" <EOL> from reviewboard . diffviewer . models import RawFileDiffData <EOL> cursor = connection . cursor ( ) <EOL> legacy_data_items = legacy_data_items . annotate ( <EOL> num_filediffs = Count ( '<STR_LIT>' ) , <EOL> num_parent_filediffs = Count ( '<STR_LIT>' ) ) <EOL> for batch in self . _iter_batches ( legacy_data_items , count , batch_size ) : <EOL> batch_total_diff_size = <NUM_LIT:0> <EOL> batch_total_bytes_saved = <NUM_LIT:0> <EOL> raw_fdds = [ ] <EOL> all_diff_hashes = [ ] <EOL> filediff_hashes = [ ] <EOL> parent_filediff_hashes = [ ] <EOL> for legacy_fdd in batch : <EOL> raw_fdd = RawFileDiffData . objects . create_from_legacy ( <EOL> legacy_fdd , save = False ) <EOL> raw_fdds . append ( raw_fdd ) <EOL> binary_hash = legacy_fdd . binary_hash <EOL> old_diff_size = len ( legacy_fdd . get_binary_base64 ( ) ) <EOL> batch_total_diff_size += old_diff_size <EOL> batch_total_bytes_saved += old_diff_size - len ( raw_fdd . binary ) <EOL> if legacy_fdd . num_filediffs > <NUM_LIT:0> : <EOL> filediff_hashes . append ( binary_hash ) <EOL> if legacy_fdd . num_parent_filediffs > <NUM_LIT:0> : <EOL> parent_filediff_hashes . append ( binary_hash ) <EOL> all_diff_hashes . append ( binary_hash ) <EOL> try : <EOL> RawFileDiffData . objects . bulk_create ( raw_fdds ) <EOL> except IntegrityError : <EOL> for raw_fdd in raw_fdds : <EOL> try : <EOL> raw_fdd . save ( ) <EOL> except IntegrityError : <EOL> raw_fdd = RawFileDiffData . objects . get ( <EOL> binary_hash = raw_fdd . binary_hash ) <EOL> if filediff_hashes : <EOL> self . _transition_hashes ( cursor , '<STR_LIT>' , filediff_hashes ) <EOL> if parent_filediff_hashes : <EOL> self . _transition_hashes ( cursor , '<STR_LIT>' , <EOL> parent_filediff_hashes ) <EOL> legacy_data_items . filter ( pk__in = all_diff_hashes ) . delete ( ) <EOL> yield ( len ( batch ) , batch_total_diff_size , <EOL> batch_total_bytes_saved , filediff_hashes , <EOL> parent_filediff_hashes , all_diff_hashes ) <EOL> def _migrate_filediffs ( self , queryset , count , batch_size ) : <EOL> """<STR_LIT>""" <EOL> for batch in self . _iter_batches ( queryset , count , batch_size ) : <EOL> batch_total_diff_size = <NUM_LIT:0> <EOL> batch_total_bytes_saved = <NUM_LIT:0> <EOL> for filediff in batch : <EOL> diff_size = len ( filediff . get_diff64_base64 ( ) ) <EOL> parent_diff_size = len ( filediff . get_parent_diff64_base64 ( ) ) <EOL> batch_total_diff_size += diff_size + parent_diff_size <EOL> diff_hash_is_new , parent_diff_hash_is_new = filediff . _migrate_diff_data ( recalculate_counts = False ) <EOL> if diff_size > <NUM_LIT:0> and not diff_hash_is_new : <EOL> batch_total_bytes_saved += diff_size <EOL> if parent_diff_size > <NUM_LIT:0> and not parent_diff_hash_is_new : <EOL> batch_total_bytes_saved += parent_diff_size <EOL> yield len ( batch ) , batch_total_diff_size , batch_total_bytes_saved <EOL> def _iter_batches ( self , queryset , count , batch_size , object_limit = <NUM_LIT:200> ) : <EOL> """<STR_LIT>""" <EOL> if count == <NUM_LIT:0> : <EOL> return <EOL> batch = [ ] <EOL> for i in range ( <NUM_LIT:0> , count , object_limit ) : <EOL> for item in queryset [ len ( batch ) : object_limit ] . iterator ( ) : <EOL> batch . append ( item ) <EOL> if len ( batch ) == batch_size : <EOL> yield batch <EOL> batch = [ ] <EOL> reset_queries ( ) <EOL> gc . collect ( ) <EOL> if batch : <EOL> yield batch <EOL> def _transition_hashes ( self , cursor , hash_field_name , diff_hashes ) : <EOL> """<STR_LIT>""" <EOL> from reviewboard . diffviewer . models import RawFileDiffData <EOL> if self . _db_supports_join_on_update : <EOL> cursor . execute ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> % { <EOL> '<STR_LIT>' : self . model . _meta . db_table , <EOL> '<STR_LIT>' : RawFileDiffData . _meta . db_table , <EOL> '<STR_LIT>' : hash_field_name , <EOL> '<STR_LIT>' : '<STR_LIT:U+002C>' . join ( [ <EOL> "<STR_LIT>" % diff_hash <EOL> for diff_hash in diff_hashes <EOL> ] ) , <EOL> } ) <EOL> else : <EOL> legacy_hash_field_name = '<STR_LIT>' % hash_field_name <EOL> legacy_hash_field_in = '<STR_LIT>' % legacy_hash_field_name <EOL> raw_fdds = RawFileDiffData . objects . filter ( <EOL> binary_hash__in = diff_hashes ) . only ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> for raw_fdd in raw_fdds : <EOL> self . filter ( ** { <EOL> legacy_hash_field_in : raw_fdd . binary_hash <EOL> } ) . update ( ** { <EOL> hash_field_name : raw_fdd . pk , <EOL> legacy_hash_field_name : None <EOL> } ) <EOL> @ cached_property <EOL> def _db_supports_join_on_update ( self ) : <EOL> """<STR_LIT>""" <EOL> cursor = connection . cursor ( ) <EOL> cursor . execute ( '<STR_LIT>' ) <EOL> cursor . execute ( '<STR_LIT>' ) <EOL> cursor . execute ( '<STR_LIT>' ) <EOL> cursor . execute ( '<STR_LIT>' ) <EOL> try : <EOL> cursor . execute ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> has_support = True <EOL> except DatabaseError : <EOL> has_support = False <EOL> cursor . execute ( '<STR_LIT>' ) <EOL> cursor . execute ( '<STR_LIT>' ) <EOL> return has_support <EOL> class RawFileDiffDataManager ( models . Manager ) : <EOL> """<STR_LIT>""" <EOL> def process_diff_data ( self , data ) : <EOL> """<STR_LIT>""" <EOL> compressed_data = bz2 . compress ( data , <NUM_LIT:9> ) <EOL> if len ( compressed_data ) < len ( data ) : <EOL> return compressed_data , self . model . COMPRESSION_BZIP2 <EOL> else : <EOL> return data , None <EOL> def get_or_create_from_data ( self , data ) : <EOL> binary_hash = self . _hash_hexdigest ( data ) <EOL> processed_data , compression = self . process_diff_data ( data ) <EOL> return self . get_or_create ( <EOL> binary_hash = binary_hash , <EOL> defaults = { <EOL> '<STR_LIT>' : processed_data , <EOL> '<STR_LIT>' : compression , <EOL> } ) <EOL> def create_from_legacy ( self , legacy , save = True ) : <EOL> processed_data , compression = self . process_diff_data ( legacy . binary ) <EOL> raw_file_diff_data = self . model ( binary_hash = legacy . binary_hash , <EOL> binary = processed_data , <EOL> compression = compression ) <EOL> raw_file_diff_data . extra_data = legacy . extra_data <EOL> if save : <EOL> raw_file_diff_data . save ( ) <EOL> return raw_file_diff_data <EOL> def _hash_hexdigest ( self , diff ) : <EOL> hasher = hashlib . sha1 ( ) <EOL> hasher . update ( diff ) <EOL> return hasher . hexdigest ( ) <EOL> class DiffSetManager ( models . Manager ) : <EOL> """<STR_LIT>""" <EOL> HEADER_EXTENSIONS = [ "<STR_LIT:h>" , "<STR_LIT:H>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] <EOL> IMPL_EXTENSIONS = [ "<STR_LIT:c>" , "<STR_LIT:C>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:m>" , "<STR_LIT>" , "<STR_LIT:M>" ] <EOL> def create_from_upload ( self , repository , diff_file , parent_diff_file , <EOL> diffset_history , basedir , request , <EOL> base_commit_id = None , save = True ) : <EOL> """<STR_LIT>""" <EOL> siteconfig = SiteConfiguration . objects . get_current ( ) <EOL> max_diff_size = siteconfig . get ( '<STR_LIT>' ) <EOL> if max_diff_size > <NUM_LIT:0> : <EOL> if diff_file . size > max_diff_size : <EOL> raise DiffTooBigError ( <EOL> _ ( '<STR_LIT>' ) , <EOL> max_diff_size = max_diff_size ) <EOL> if parent_diff_file and parent_diff_file . size > max_diff_size : <EOL> raise DiffTooBigError ( <EOL> _ ( '<STR_LIT>' ) , <EOL> max_diff_size = max_diff_size ) <EOL> if parent_diff_file : <EOL> parent_diff_file_name = parent_diff_file . name <EOL> parent_diff_file_contents = parent_diff_file . read ( ) <EOL> else : <EOL> parent_diff_file_name = None <EOL> parent_diff_file_contents = None <EOL> return self . create_from_data ( repository , <EOL> diff_file . name , <EOL> diff_file . read ( ) , <EOL> parent_diff_file_name , <EOL> parent_diff_file_contents , <EOL> diffset_history , <EOL> basedir , <EOL> request , <EOL> base_commit_id = base_commit_id , <EOL> save = save ) <EOL> def create_from_data ( self , repository , diff_file_name , diff_file_contents , <EOL> parent_diff_file_name , parent_diff_file_contents , <EOL> diffset_history , basedir , request , <EOL> base_commit_id = None , save = True ) : <EOL> """<STR_LIT>""" <EOL> from reviewboard . diffviewer . diffutils import convert_to_unicode <EOL> from reviewboard . diffviewer . models import FileDiff <EOL> tool = repository . get_scmtool ( ) <EOL> parser = tool . get_parser ( diff_file_contents ) <EOL> files = list ( self . _process_files ( <EOL> parser , <EOL> basedir , <EOL> repository , <EOL> base_commit_id , <EOL> request , <EOL> check_existence = ( not parent_diff_file_contents ) ) ) <EOL> if len ( files ) == <NUM_LIT:0> : <EOL> raise EmptyDiffError ( _ ( "<STR_LIT>" ) ) <EOL> files . sort ( cmp = self . _compare_files , key = lambda f : f . origFile ) <EOL> parent_files = { } <EOL> parent_commit_id = None <EOL> if parent_diff_file_contents : <EOL> diff_filenames = set ( [ f . origFile for f in files ] ) <EOL> parent_parser = tool . get_parser ( parent_diff_file_contents ) <EOL> for f in self . _process_files ( parent_parser , basedir , <EOL> repository , base_commit_id , request , <EOL> check_existence = True , <EOL> limit_to = diff_filenames ) : <EOL> parent_files [ f . newFile ] = f <EOL> parent_commit_id = parent_parser . get_orig_commit_id ( ) <EOL> diffset = self . model ( <EOL> name = diff_file_name , revision = <NUM_LIT:0> , <EOL> basedir = basedir , <EOL> history = diffset_history , <EOL> repository = repository , <EOL> diffcompat = DiffCompatVersion . DEFAULT , <EOL> base_commit_id = base_commit_id ) <EOL> if save : <EOL> diffset . save ( ) <EOL> encoding_list = repository . get_encoding_list ( ) <EOL> for f in files : <EOL> parent_file = None <EOL> orig_rev = None <EOL> parent_content = b'<STR_LIT>' <EOL> if f . origFile in parent_files : <EOL> parent_file = parent_files [ f . origFile ] <EOL> parent_content = parent_file . data <EOL> orig_rev = parent_file . origInfo <EOL> if not orig_rev : <EOL> if parent_commit_id and f . origInfo != PRE_CREATION : <EOL> orig_rev = parent_commit_id <EOL> else : <EOL> orig_rev = f . origInfo <EOL> enc , orig_file = convert_to_unicode ( f . origFile , encoding_list ) <EOL> enc , dest_file = convert_to_unicode ( f . newFile , encoding_list ) <EOL> if f . deleted : <EOL> status = FileDiff . DELETED <EOL> elif f . moved : <EOL> status = FileDiff . MOVED <EOL> elif f . copied : <EOL> status = FileDiff . COPIED <EOL> else : <EOL> status = FileDiff . MODIFIED <EOL> filediff = FileDiff ( <EOL> diffset = diffset , <EOL> source_file = parser . normalize_diff_filename ( orig_file ) , <EOL> dest_file = parser . normalize_diff_filename ( dest_file ) , <EOL> source_revision = smart_unicode ( orig_rev ) , <EOL> dest_detail = f . newInfo , <EOL> diff = f . data , <EOL> parent_diff = parent_content , <EOL> binary = f . binary , <EOL> status = status ) <EOL> if ( parent_file and <EOL> ( parent_file . moved or parent_file . copied ) and <EOL> parent_file . insert_count == <NUM_LIT:0> and <EOL> parent_file . delete_count == <NUM_LIT:0> ) : <EOL> filediff . extra_data = { '<STR_LIT>' : True } <EOL> filediff . set_line_counts ( raw_insert_count = f . insert_count , <EOL> raw_delete_count = f . delete_count ) <EOL> if save : <EOL> filediff . save ( ) <EOL> return diffset <EOL> def _normalize_filename ( self , filename , basedir ) : <EOL> """<STR_LIT>""" <EOL> if filename . startswith ( '<STR_LIT:/>' ) : <EOL> return filename <EOL> return os . path . join ( basedir , filename ) . replace ( '<STR_LIT:\\>' , '<STR_LIT:/>' ) <EOL> def _process_files ( self , parser , basedir , repository , base_commit_id , <EOL> request , check_existence = False , limit_to = None ) : <EOL> tool = repository . get_scmtool ( ) <EOL> for f in parser . parse ( ) : <EOL> source_filename , source_revision = tool . parse_diff_revision ( <EOL> f . origFile , <EOL> f . origInfo , <EOL> moved = f . moved , <EOL> copied = f . copied ) <EOL> dest_filename = self . _normalize_filename ( f . newFile , basedir ) <EOL> source_filename = self . _normalize_filename ( source_filename , <EOL> basedir ) <EOL> if limit_to is not None and dest_filename not in limit_to : <EOL> continue <EOL> if ( source_revision != PRE_CREATION and <EOL> source_revision != UNKNOWN and <EOL> not f . binary and <EOL> not f . deleted and <EOL> not f . moved and <EOL> not f . copied and <EOL> ( check_existence and <EOL> not repository . get_file_exists ( source_filename , <EOL> source_revision , <EOL> base_commit_id = base_commit_id , <EOL> request = request ) ) ) : <EOL> raise FileNotFoundError ( source_filename , source_revision , <EOL> base_commit_id ) <EOL> f . origFile = source_filename <EOL> f . origInfo = source_revision <EOL> f . newFile = dest_filename <EOL> yield f <EOL> def _compare_files ( self , filename1 , filename2 ) : <EOL> """<STR_LIT>""" <EOL> if filename1 . find ( '<STR_LIT:.>' ) != - <NUM_LIT:1> and filename2 . find ( '<STR_LIT:.>' ) != - <NUM_LIT:1> : <EOL> basename1 , ext1 = filename1 . rsplit ( '<STR_LIT:.>' , <NUM_LIT:1> ) <EOL> basename2 , ext2 = filename2 . rsplit ( '<STR_LIT:.>' , <NUM_LIT:1> ) <EOL> if basename1 == basename2 : <EOL> if ( ext1 in self . HEADER_EXTENSIONS and <EOL> ext2 in self . IMPL_EXTENSIONS ) : <EOL> return - <NUM_LIT:1> <EOL> elif ( ext1 in self . IMPL_EXTENSIONS and <EOL> ext2 in self . HEADER_EXTENSIONS ) : <EOL> return <NUM_LIT:1> <EOL> return cmp ( filename1 , filename2 ) </s>
