<s> """<STR_LIT>""" <EOL> import re <EOL> from cvs2svn_lib import config <EOL> from cvs2svn_lib . common import DB_OPEN_NEW <EOL> from cvs2svn_lib . common import warning_prefix <EOL> from cvs2svn_lib . common import error_prefix <EOL> from cvs2svn_lib . common import is_trunk_revision <EOL> from cvs2svn_lib . common import is_branch_revision_number <EOL> from cvs2svn_lib . log import logger <EOL> from cvs2svn_lib . context import Ctx <EOL> from cvs2svn_lib . artifact_manager import artifact_manager <EOL> from cvs2svn_lib . cvs_path import CVSFile <EOL> from cvs2svn_lib . cvs_path import CVSDirectory <EOL> from cvs2svn_lib . symbol import Symbol <EOL> from cvs2svn_lib . symbol import Trunk <EOL> from cvs2svn_lib . cvs_item import CVSRevision <EOL> from cvs2svn_lib . cvs_item import CVSBranch <EOL> from cvs2svn_lib . cvs_item import CVSTag <EOL> from cvs2svn_lib . cvs_item import cvs_revision_type_map <EOL> from cvs2svn_lib . cvs_file_items import VendorBranchError <EOL> from cvs2svn_lib . cvs_file_items import CVSFileItems <EOL> from cvs2svn_lib . key_generator import KeyGenerator <EOL> from cvs2svn_lib . cvs_item_database import NewCVSItemStore <EOL> from cvs2svn_lib . symbol_statistics import SymbolStatisticsCollector <EOL> from cvs2svn_lib . metadata_database import MetadataDatabase <EOL> from cvs2svn_lib . metadata_database import MetadataLogger <EOL> from cvs2svn_lib . rcsparser import Sink <EOL> from cvs2svn_lib . rcsparser import parse <EOL> from cvs2svn_lib . rcsparser import RCSParseError <EOL> _valid_revision_re = re . compile ( r'''<STR_LIT>''' , re . VERBOSE ) <EOL> _branch_revision_re = re . compile ( r'''<STR_LIT>''' , re . VERBOSE ) <EOL> def is_same_line_of_development ( rev1 , rev2 ) : <EOL> """<STR_LIT>""" <EOL> if rev1 is None or rev2 is None : <EOL> return False <EOL> if is_trunk_revision ( rev1 ) and is_trunk_revision ( rev2 ) : <EOL> return True <EOL> if rev1 [ <NUM_LIT:0> : rev1 . rfind ( '<STR_LIT:.>' ) ] == rev2 [ <NUM_LIT:0> : rev2 . rfind ( '<STR_LIT:.>' ) ] : <EOL> return True <EOL> return False <EOL> class _RevisionData : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , cvs_rev_id , rev , timestamp , author , state ) : <EOL> self . cvs_rev_id = cvs_rev_id <EOL> self . rev = rev <EOL> self . timestamp = timestamp <EOL> self . author = author <EOL> self . state = state <EOL> self . parent_branch_data = None <EOL> self . parent = None <EOL> self . child = None <EOL> self . branches_data = [ ] <EOL> self . branches_revs_data = [ ] <EOL> self . tags_data = [ ] <EOL> self . revision_reader_token = None <EOL> def get_first_on_branch_id ( self ) : <EOL> return self . parent_branch_data and self . parent_branch_data . id <EOL> class _SymbolData : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , id , symbol ) : <EOL> """<STR_LIT>""" <EOL> self . id = id <EOL> self . symbol = symbol <EOL> class _BranchData ( _SymbolData ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , id , symbol , branch_number ) : <EOL> _SymbolData . __init__ ( self , id , symbol ) <EOL> self . branch_number = branch_number <EOL> self . parent = self . branch_number [ : self . branch_number . rindex ( "<STR_LIT:.>" ) ] <EOL> self . child = None <EOL> class _TagData ( _SymbolData ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , id , symbol , rev ) : <EOL> _SymbolData . __init__ ( self , id , symbol ) <EOL> self . rev = rev <EOL> class _SymbolDataCollector ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , fdc , cvs_file ) : <EOL> self . fdc = fdc <EOL> self . cvs_file = cvs_file <EOL> self . pdc = self . fdc . pdc <EOL> self . collect_data = self . fdc . collect_data <EOL> self . _symbol_defs = [ ] <EOL> self . _defined_symbols = set ( ) <EOL> self . branches_data = { } <EOL> self . tags_data = { } <EOL> def _add_branch ( self , name , branch_number ) : <EOL> """<STR_LIT>""" <EOL> branch_data = self . branches_data . get ( branch_number ) <EOL> if branch_data is not None : <EOL> logger . warn ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> % ( warning_prefix , <EOL> self . cvs_file . rcs_path , branch_number , <EOL> branch_data . symbol . name , name ) <EOL> ) <EOL> return branch_data <EOL> symbol = self . pdc . get_symbol ( name ) <EOL> branch_data = _BranchData ( <EOL> self . collect_data . item_key_generator . gen_id ( ) , symbol , branch_number <EOL> ) <EOL> self . branches_data [ branch_number ] = branch_data <EOL> return branch_data <EOL> def _construct_distinct_name ( self , name , original_name ) : <EOL> """<STR_LIT>""" <EOL> if name not in self . _defined_symbols : <EOL> return name <EOL> else : <EOL> index = <NUM_LIT:1> <EOL> while True : <EOL> dup_name = '<STR_LIT>' % ( name , index , ) <EOL> if dup_name not in self . _defined_symbols : <EOL> self . collect_data . record_fatal_error ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> % ( name , self . cvs_file . rcs_path , original_name , ) <EOL> ) <EOL> return dup_name <EOL> def _add_unlabeled_branch ( self , branch_number ) : <EOL> original_name = "<STR_LIT>" + branch_number <EOL> name = self . transform_symbol ( original_name , branch_number ) <EOL> if name is None : <EOL> self . collect_data . record_fatal_error ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> % ( original_name , self . cvs_file . rcs_path , ) <EOL> ) <EOL> name = original_name <EOL> distinct_name = self . _construct_distinct_name ( name , original_name ) <EOL> self . _defined_symbols . add ( distinct_name ) <EOL> return self . _add_branch ( distinct_name , branch_number ) <EOL> def _add_tag ( self , name , revision ) : <EOL> """<STR_LIT>""" <EOL> symbol = self . pdc . get_symbol ( name ) <EOL> tag_data = _TagData ( <EOL> self . collect_data . item_key_generator . gen_id ( ) , symbol , revision <EOL> ) <EOL> self . tags_data . setdefault ( revision , [ ] ) . append ( tag_data ) <EOL> return tag_data <EOL> def transform_symbol ( self , name , revision ) : <EOL> """<STR_LIT>""" <EOL> old_name = name <EOL> name = self . cvs_file . project . transform_symbol ( <EOL> self . cvs_file , name , revision <EOL> ) <EOL> if name is None : <EOL> self . pdc . log_symbol_transform ( old_name , None ) <EOL> logger . verbose ( <EOL> "<STR_LIT>" <EOL> % ( old_name , revision , self . cvs_file . rcs_path , ) <EOL> ) <EOL> else : <EOL> if name != old_name : <EOL> self . pdc . log_symbol_transform ( old_name , name ) <EOL> logger . verbose ( <EOL> "<STR_LIT>" <EOL> % ( old_name , revision , name , self . cvs_file . rcs_path , ) <EOL> ) <EOL> return name <EOL> def define_symbol ( self , name , revision ) : <EOL> """<STR_LIT>""" <EOL> revision = _branch_revision_re . sub ( r'<STR_LIT>' , revision ) <EOL> name = self . transform_symbol ( name , revision ) <EOL> if name is not None : <EOL> if _valid_revision_re . match ( revision ) : <EOL> self . _symbol_defs . append ( ( name , revision ) ) <EOL> else : <EOL> logger . warn ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> % ( self . cvs_file . rcs_path , name , revision , ) <EOL> ) <EOL> def _eliminate_trivial_duplicate_defs ( self , symbol_defs ) : <EOL> """<STR_LIT>""" <EOL> symbol_defs = list ( symbol_defs ) <EOL> known_definitions = { } <EOL> for ( i , symbol_def ) in enumerate ( symbol_defs ) : <EOL> known_definitions . setdefault ( symbol_def , [ ] ) . append ( i ) <EOL> dup_indexes = set ( ) <EOL> for ( ( name , revision ) , indexes ) in known_definitions . iteritems ( ) : <EOL> if len ( indexes ) > <NUM_LIT:1> : <EOL> logger . verbose ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> % ( self . cvs_file . rcs_path , name , revision , ) <EOL> ) <EOL> dup_indexes . update ( indexes [ : - <NUM_LIT:1> ] ) <EOL> for ( i , symbol_def ) in enumerate ( symbol_defs ) : <EOL> if i not in dup_indexes : <EOL> yield symbol_def <EOL> def _process_duplicate_defs ( self , symbol_defs ) : <EOL> """<STR_LIT>""" <EOL> symbol_defs = list ( symbol_defs ) <EOL> known_symbols = { } <EOL> for ( i , ( name , revision ) ) in enumerate ( symbol_defs ) : <EOL> known_symbols . setdefault ( name , [ ] ) . append ( i ) <EOL> known_symbols = known_symbols . items ( ) <EOL> known_symbols . sort ( ) <EOL> dup_indexes = set ( ) <EOL> for ( name , indexes ) in known_symbols : <EOL> if len ( indexes ) > <NUM_LIT:1> : <EOL> self . collect_data . record_fatal_error ( <EOL> "<STR_LIT>" % ( <EOL> name , self . cvs_file . rcs_path , <EOL> '<STR_LIT:U+0020>' . join ( [ symbol_defs [ i ] [ <NUM_LIT:1> ] for i in indexes ] ) , <EOL> ) <EOL> ) <EOL> dup_indexes . update ( indexes [ : - <NUM_LIT:1> ] ) <EOL> for ( i , symbol_def ) in enumerate ( symbol_defs ) : <EOL> if i not in dup_indexes : <EOL> yield symbol_def <EOL> def _process_symbol ( self , name , revision ) : <EOL> """<STR_LIT>""" <EOL> if is_branch_revision_number ( revision ) : <EOL> self . _add_branch ( name , revision ) <EOL> else : <EOL> self . _add_tag ( name , revision ) <EOL> def process_symbols ( self ) : <EOL> """<STR_LIT>""" <EOL> symbol_defs = self . _symbol_defs <EOL> del self . _symbol_defs <EOL> symbol_defs = self . _eliminate_trivial_duplicate_defs ( symbol_defs ) <EOL> symbol_defs = self . _process_duplicate_defs ( symbol_defs ) <EOL> for ( name , revision ) in symbol_defs : <EOL> self . _defined_symbols . add ( name ) <EOL> self . _process_symbol ( name , revision ) <EOL> @ staticmethod <EOL> def rev_to_branch_number ( revision ) : <EOL> """<STR_LIT>""" <EOL> if is_trunk_revision ( revision ) : <EOL> return None <EOL> return revision [ : revision . rindex ( "<STR_LIT:.>" ) ] <EOL> def rev_to_branch_data ( self , revision ) : <EOL> """<STR_LIT>""" <EOL> assert not is_trunk_revision ( revision ) <EOL> return self . branches_data [ self . rev_to_branch_number ( revision ) ] <EOL> def rev_to_lod ( self , revision ) : <EOL> """<STR_LIT>""" <EOL> if is_trunk_revision ( revision ) : <EOL> return self . pdc . trunk <EOL> else : <EOL> return self . rev_to_branch_data ( revision ) . symbol <EOL> class _FileDataCollector ( Sink ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , pdc , cvs_file ) : <EOL> """<STR_LIT>""" <EOL> self . pdc = pdc <EOL> self . cvs_file = cvs_file <EOL> self . collect_data = self . pdc . collect_data <EOL> self . project = self . cvs_file . project <EOL> self . sdc = _SymbolDataCollector ( self , self . cvs_file ) <EOL> self . _rev_data = { } <EOL> self . _primary_dependencies = [ ] <EOL> self . default_branch = None <EOL> self . _file_imported = False <EOL> def _get_rev_id ( self , revision ) : <EOL> if revision is None : <EOL> return None <EOL> return self . _rev_data [ revision ] . cvs_rev_id <EOL> def set_principal_branch ( self , branch ) : <EOL> """<STR_LIT>""" <EOL> if branch . find ( '<STR_LIT:.>' ) == - <NUM_LIT:1> : <EOL> return <EOL> m = _branch_revision_re . match ( branch ) <EOL> if not m : <EOL> self . collect_data . record_fatal_error ( <EOL> '<STR_LIT>' <EOL> % ( branch , self . cvs_file . rcs_path , ) <EOL> ) <EOL> return <EOL> branch = m . group ( <NUM_LIT:1> ) + m . group ( <NUM_LIT:2> ) <EOL> if branch . count ( '<STR_LIT:.>' ) != <NUM_LIT:2> : <EOL> self . collect_data . record_fatal_error ( <EOL> '<STR_LIT>' <EOL> % ( branch , self . cvs_file . rcs_path , ) <EOL> ) <EOL> return <EOL> self . default_branch = branch <EOL> def define_tag ( self , name , revision ) : <EOL> """<STR_LIT>""" <EOL> self . sdc . define_symbol ( name , revision ) <EOL> def set_expansion ( self , mode ) : <EOL> """<STR_LIT>""" <EOL> self . cvs_file . mode = mode <EOL> def admin_completed ( self ) : <EOL> """<STR_LIT>""" <EOL> self . sdc . process_symbols ( ) <EOL> def define_revision ( self , revision , timestamp , author , state , <EOL> branches , next ) : <EOL> """<STR_LIT>""" <EOL> for branch in branches : <EOL> try : <EOL> branch_data = self . sdc . rev_to_branch_data ( branch ) <EOL> except KeyError : <EOL> branch_data = self . sdc . _add_unlabeled_branch ( <EOL> self . sdc . rev_to_branch_number ( branch ) ) <EOL> assert branch_data . child is None <EOL> branch_data . child = branch <EOL> if revision in self . _rev_data : <EOL> logger . error ( '<STR_LIT>' <EOL> % ( self . cvs_file . rcs_path , revision , ) ) <EOL> raise RuntimeError ( ) <EOL> rev_data = _RevisionData ( <EOL> self . collect_data . item_key_generator . gen_id ( ) , <EOL> revision , int ( timestamp ) , author , state ) <EOL> self . _rev_data [ revision ] = rev_data <EOL> if next : <EOL> if is_trunk_revision ( revision ) : <EOL> self . _primary_dependencies . append ( ( next , revision , ) ) <EOL> else : <EOL> self . _primary_dependencies . append ( ( revision , next , ) ) <EOL> def tree_completed ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _resolve_primary_dependencies ( ) <EOL> self . _resolve_branch_dependencies ( ) <EOL> self . _sort_branches ( ) <EOL> self . _resolve_tag_dependencies ( ) <EOL> cvs_items = [ ] <EOL> cvs_items . extend ( self . _get_cvs_revisions ( ) ) <EOL> cvs_items . extend ( self . _get_cvs_branches ( ) ) <EOL> cvs_items . extend ( self . _get_cvs_tags ( ) ) <EOL> self . _cvs_file_items = CVSFileItems ( <EOL> self . cvs_file , self . pdc . trunk , cvs_items <EOL> ) <EOL> self . _cvs_file_items . check_link_consistency ( ) <EOL> def _resolve_primary_dependencies ( self ) : <EOL> """<STR_LIT>""" <EOL> for ( parent , child , ) in self . _primary_dependencies : <EOL> parent_data = self . _rev_data [ parent ] <EOL> assert parent_data . child is None <EOL> parent_data . child = child <EOL> child_data = self . _rev_data [ child ] <EOL> assert child_data . parent is None <EOL> child_data . parent = parent <EOL> def _resolve_branch_dependencies ( self ) : <EOL> """<STR_LIT>""" <EOL> for branch_data in self . sdc . branches_data . values ( ) : <EOL> try : <EOL> parent_data = self . _rev_data [ branch_data . parent ] <EOL> except KeyError : <EOL> logger . warn ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> % ( self . cvs_file . rcs_path , branch_data . symbol . name , <EOL> branch_data . parent , ) ) <EOL> del self . sdc . branches_data [ branch_data . branch_number ] <EOL> else : <EOL> parent_data . branches_data . append ( branch_data ) <EOL> if branch_data . child is not None : <EOL> child_data = self . _rev_data [ branch_data . child ] <EOL> assert child_data . parent_branch_data is None <EOL> child_data . parent_branch_data = branch_data <EOL> assert child_data . parent is None <EOL> child_data . parent = branch_data . parent <EOL> parent_data . branches_revs_data . append ( branch_data . child ) <EOL> def _sort_branches ( self ) : <EOL> """<STR_LIT>""" <EOL> for rev_data in self . _rev_data . values ( ) : <EOL> rev_data . branches_data . sort ( lambda a , b : - cmp ( a . id , b . id ) ) <EOL> def _resolve_tag_dependencies ( self ) : <EOL> """<STR_LIT>""" <EOL> for ( rev , tag_data_list ) in self . sdc . tags_data . items ( ) : <EOL> try : <EOL> parent_data = self . _rev_data [ rev ] <EOL> except KeyError : <EOL> logger . warn ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' % ( <EOL> self . cvs_file . rcs_path , rev , <EOL> '<STR_LIT:U+002CU+0020>' . join ( [ repr ( tag_data . symbol . name ) <EOL> for tag_data in tag_data_list ] ) , ) ) <EOL> del self . sdc . tags_data [ rev ] <EOL> else : <EOL> for tag_data in tag_data_list : <EOL> assert tag_data . rev == rev <EOL> parent_data . tags_data . append ( tag_data ) <EOL> def _get_cvs_branches ( self ) : <EOL> """<STR_LIT>""" <EOL> for branch_data in self . sdc . branches_data . values ( ) : <EOL> yield CVSBranch ( <EOL> branch_data . id , self . cvs_file , branch_data . symbol , <EOL> branch_data . branch_number , <EOL> self . sdc . rev_to_lod ( branch_data . parent ) , <EOL> self . _get_rev_id ( branch_data . parent ) , <EOL> self . _get_rev_id ( branch_data . child ) , <EOL> None , <EOL> ) <EOL> def _get_cvs_tags ( self ) : <EOL> """<STR_LIT>""" <EOL> for tags_data in self . sdc . tags_data . values ( ) : <EOL> for tag_data in tags_data : <EOL> yield CVSTag ( <EOL> tag_data . id , self . cvs_file , tag_data . symbol , <EOL> self . sdc . rev_to_lod ( tag_data . rev ) , <EOL> self . _get_rev_id ( tag_data . rev ) , <EOL> None , <EOL> ) <EOL> def set_description ( self , description ) : <EOL> """<STR_LIT>""" <EOL> self . cvs_file . description = description <EOL> self . cvs_file . determine_file_properties ( Ctx ( ) . file_property_setters ) <EOL> def set_revision_info ( self , revision , log , text ) : <EOL> """<STR_LIT>""" <EOL> rev_data = self . _rev_data [ revision ] <EOL> cvs_rev = self . _cvs_file_items [ rev_data . cvs_rev_id ] <EOL> if cvs_rev . metadata_id is not None : <EOL> logger . warn ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> % ( warning_prefix , self . cvs_file . rcs_path , revision , ) <EOL> ) <EOL> return <EOL> if is_trunk_revision ( revision ) : <EOL> branch_name = None <EOL> else : <EOL> branch_name = self . sdc . rev_to_branch_data ( revision ) . symbol . name <EOL> cvs_rev . metadata_id = self . collect_data . metadata_logger . store ( <EOL> self . project , branch_name , rev_data . author , log <EOL> ) <EOL> cvs_rev . deltatext_exists = bool ( text ) <EOL> if revision == '<STR_LIT>' : <EOL> self . _file_imported = ( log == '<STR_LIT>' ) <EOL> def parse_completed ( self ) : <EOL> """<STR_LIT>""" <EOL> for cvs_item in self . _cvs_file_items . values ( ) : <EOL> if isinstance ( cvs_item , CVSRevision ) and cvs_item . metadata_id is None : <EOL> self . collect_data . record_fatal_error ( <EOL> '<STR_LIT>' <EOL> % ( self . cvs_file . rcs_path , cvs_item . rev , ) <EOL> ) <EOL> def _determine_operation ( self , rev_data ) : <EOL> prev_rev_data = self . _rev_data . get ( rev_data . parent ) <EOL> return cvs_revision_type_map [ ( <EOL> rev_data . state != '<STR_LIT>' , <EOL> prev_rev_data is not None and prev_rev_data . state != '<STR_LIT>' , <EOL> ) ] <EOL> def _get_cvs_revisions ( self ) : <EOL> """<STR_LIT>""" <EOL> for rev_data in self . _rev_data . itervalues ( ) : <EOL> yield self . _get_cvs_revision ( rev_data ) <EOL> def _get_cvs_revision ( self , rev_data ) : <EOL> """<STR_LIT>""" <EOL> branch_ids = [ <EOL> branch_data . id <EOL> for branch_data in rev_data . branches_data <EOL> ] <EOL> branch_commit_ids = [ <EOL> self . _get_rev_id ( rev ) <EOL> for rev in rev_data . branches_revs_data <EOL> ] <EOL> tag_ids = [ <EOL> tag_data . id <EOL> for tag_data in rev_data . tags_data <EOL> ] <EOL> revision_type = self . _determine_operation ( rev_data ) <EOL> return revision_type ( <EOL> self . _get_rev_id ( rev_data . rev ) , self . cvs_file , <EOL> rev_data . timestamp , None , <EOL> self . _get_rev_id ( rev_data . parent ) , <EOL> self . _get_rev_id ( rev_data . child ) , <EOL> rev_data . rev , <EOL> True , <EOL> self . sdc . rev_to_lod ( rev_data . rev ) , <EOL> rev_data . get_first_on_branch_id ( ) , <EOL> False , None , None , <EOL> tag_ids , branch_ids , branch_commit_ids , <EOL> rev_data . revision_reader_token <EOL> ) <EOL> def get_cvs_file_items ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _process_ntdbrs ( ) <EOL> del self . sdc <EOL> return self . _cvs_file_items <EOL> def _process_ntdbrs ( self ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> if self . default_branch : <EOL> try : <EOL> vendor_cvs_branch_id = self . sdc . branches_data [ self . default_branch ] . id <EOL> except KeyError : <EOL> logger . warn ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> % ( warning_prefix , self . cvs_file . rcs_path , self . default_branch , ) <EOL> ) <EOL> self . default_branch = None <EOL> return <EOL> vendor_lod_items = self . _cvs_file_items . get_lod_items ( <EOL> self . _cvs_file_items [ vendor_cvs_branch_id ] <EOL> ) <EOL> if not self . _cvs_file_items . process_live_ntdb ( vendor_lod_items ) : <EOL> return <EOL> elif self . _file_imported : <EOL> vendor_branch_data = self . sdc . branches_data . get ( '<STR_LIT>' ) <EOL> if vendor_branch_data is None : <EOL> return <EOL> else : <EOL> vendor_lod_items = self . _cvs_file_items . get_lod_items ( <EOL> self . _cvs_file_items [ vendor_branch_data . id ] <EOL> ) <EOL> if not self . _cvs_file_items . process_historical_ntdb ( <EOL> vendor_lod_items <EOL> ) : <EOL> return <EOL> else : <EOL> return <EOL> except VendorBranchError , e : <EOL> self . collect_data . record_fatal_error ( str ( e ) ) <EOL> return <EOL> if self . _file_imported : <EOL> self . _cvs_file_items . imported_remove_1_1 ( vendor_lod_items ) <EOL> self . _cvs_file_items . check_link_consistency ( ) <EOL> class _ProjectDataCollector : <EOL> def __init__ ( self , collect_data , project ) : <EOL> self . collect_data = collect_data <EOL> self . project = project <EOL> self . num_files = <NUM_LIT:0> <EOL> self . trunk = Trunk ( <EOL> self . collect_data . symbol_key_generator . gen_id ( ) , self . project <EOL> ) <EOL> self . project . trunk_id = self . trunk . id <EOL> self . collect_data . register_trunk ( self . trunk ) <EOL> self . symbols = { } <EOL> self . symbol_transform_counts = { } <EOL> def get_symbol ( self , name ) : <EOL> """<STR_LIT>""" <EOL> symbol = self . symbols . get ( name ) <EOL> if symbol is None : <EOL> symbol = Symbol ( <EOL> self . collect_data . symbol_key_generator . gen_id ( ) , <EOL> self . project , name ) <EOL> self . symbols [ name ] = symbol <EOL> return symbol <EOL> def log_symbol_transform ( self , old_name , new_name ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> self . symbol_transform_counts [ old_name , new_name ] += <NUM_LIT:1> <EOL> except KeyError : <EOL> self . symbol_transform_counts [ old_name , new_name ] = <NUM_LIT:1> <EOL> def summarize_symbol_transforms ( self ) : <EOL> if self . symbol_transform_counts and logger . is_on ( logger . NORMAL ) : <EOL> logger . normal ( '<STR_LIT>' ) <EOL> transforms = self . symbol_transform_counts . items ( ) <EOL> transforms . sort ( ) <EOL> for ( ( old_name , new_name ) , count ) in transforms : <EOL> if new_name is None : <EOL> logger . normal ( '<STR_LIT>' % ( old_name , count , ) ) <EOL> else : <EOL> logger . normal ( <EOL> '<STR_LIT>' <EOL> % ( old_name , new_name , count , ) <EOL> ) <EOL> def process_file ( self , cvs_file ) : <EOL> logger . normal ( cvs_file . rcs_path ) <EOL> fdc = _FileDataCollector ( self , cvs_file ) <EOL> try : <EOL> f = open ( cvs_file . rcs_path , '<STR_LIT:rb>' ) <EOL> try : <EOL> parse ( f , fdc ) <EOL> finally : <EOL> f . close ( ) <EOL> except ( RCSParseError , RuntimeError ) : <EOL> self . collect_data . record_fatal_error ( <EOL> "<STR_LIT>" % ( cvs_file . rcs_path , ) <EOL> ) <EOL> return <EOL> except ValueError , e : <EOL> self . collect_data . record_fatal_error ( <EOL> "<STR_LIT>" % ( cvs_file . rcs_path , str ( e ) , ) <EOL> ) <EOL> return <EOL> except : <EOL> logger . warn ( "<STR_LIT>" % cvs_file . rcs_path ) <EOL> raise <EOL> else : <EOL> self . num_files += <NUM_LIT:1> <EOL> return fdc . get_cvs_file_items ( ) <EOL> class CollectData : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , stats_keeper ) : <EOL> self . _cvs_item_store = NewCVSItemStore ( <EOL> artifact_manager . get_temp_file ( config . CVS_ITEMS_STORE ) ) <EOL> self . metadata_db = MetadataDatabase ( <EOL> artifact_manager . get_temp_file ( config . METADATA_STORE ) , <EOL> artifact_manager . get_temp_file ( config . METADATA_INDEX_TABLE ) , <EOL> DB_OPEN_NEW , <EOL> ) <EOL> self . metadata_logger = MetadataLogger ( self . metadata_db ) <EOL> self . fatal_errors = [ ] <EOL> self . num_files = <NUM_LIT:0> <EOL> self . symbol_stats = SymbolStatisticsCollector ( ) <EOL> self . stats_keeper = stats_keeper <EOL> self . item_key_generator = KeyGenerator ( ) <EOL> self . symbol_key_generator = KeyGenerator ( ) <EOL> def record_fatal_error ( self , err ) : <EOL> """<STR_LIT>""" <EOL> err = '<STR_LIT>' % ( error_prefix , err , ) <EOL> logger . error ( err + '<STR_LIT:\n>' ) <EOL> self . fatal_errors . append ( err ) <EOL> def add_cvs_directory ( self , cvs_directory ) : <EOL> """<STR_LIT>""" <EOL> Ctx ( ) . _cvs_path_db . log_path ( cvs_directory ) <EOL> def add_cvs_file_items ( self , cvs_file_items ) : <EOL> """<STR_LIT>""" <EOL> Ctx ( ) . _cvs_path_db . log_path ( cvs_file_items . cvs_file ) <EOL> self . _cvs_item_store . add ( cvs_file_items ) <EOL> self . stats_keeper . record_cvs_file ( cvs_file_items . cvs_file ) <EOL> for cvs_item in cvs_file_items . values ( ) : <EOL> self . stats_keeper . record_cvs_item ( cvs_item ) <EOL> def register_trunk ( self , trunk ) : <EOL> """<STR_LIT>""" <EOL> self . symbol_stats [ trunk ] <EOL> def _process_cvs_file_items ( self , cvs_file_items ) : <EOL> """<STR_LIT>""" <EOL> cvs_file_items . remove_unneeded_initial_trunk_delete ( self . metadata_db ) <EOL> cvs_file_items . remove_initial_branch_deletes ( self . metadata_db ) <EOL> if Ctx ( ) . trunk_only : <EOL> cvs_file_items . exclude_non_trunk ( ) <EOL> cvs_file_items . check_link_consistency ( ) <EOL> self . add_cvs_file_items ( cvs_file_items ) <EOL> self . symbol_stats . register ( cvs_file_items ) <EOL> def process_project ( self , project , cvs_paths ) : <EOL> pdc = _ProjectDataCollector ( self , project ) <EOL> found_rcs_file = False <EOL> for cvs_path in cvs_paths : <EOL> if isinstance ( cvs_path , CVSDirectory ) : <EOL> self . add_cvs_directory ( cvs_path ) <EOL> else : <EOL> cvs_file_items = pdc . process_file ( cvs_path ) <EOL> self . _process_cvs_file_items ( cvs_file_items ) <EOL> found_rcs_file = True <EOL> if not found_rcs_file : <EOL> self . record_fatal_error ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> % ( project . project_cvs_repos_path , ) <EOL> ) <EOL> pdc . summarize_symbol_transforms ( ) <EOL> self . num_files += pdc . num_files <EOL> logger . verbose ( '<STR_LIT>' , self . num_files , '<STR_LIT>' ) <EOL> def _register_empty_subdirectories ( self ) : <EOL> """<STR_LIT>""" <EOL> directories = set ( <EOL> path <EOL> for path in Ctx ( ) . _cvs_path_db . itervalues ( ) <EOL> if isinstance ( path , CVSDirectory ) <EOL> ) <EOL> for path in Ctx ( ) . _cvs_path_db . itervalues ( ) : <EOL> if isinstance ( path , CVSFile ) : <EOL> directory = path . parent_directory <EOL> while directory is not None and directory in directories : <EOL> directories . remove ( directory ) <EOL> directory = directory . parent_directory <EOL> for directory in directories : <EOL> if directory . parent_directory is not None : <EOL> directory . parent_directory . empty_subdirectory_ids . append ( directory . id ) <EOL> def close ( self ) : <EOL> """<STR_LIT>""" <EOL> self . symbol_stats . purge_ghost_symbols ( ) <EOL> self . symbol_stats . close ( ) <EOL> self . symbol_stats = None <EOL> self . metadata_logger = None <EOL> self . metadata_db . close ( ) <EOL> self . metadata_db = None <EOL> self . _cvs_item_store . close ( ) <EOL> self . _cvs_item_store = None <EOL> self . _register_empty_subdirectories ( ) <EOL> retval = self . fatal_errors <EOL> self . fatal_errors = None <EOL> return retval </s>
