<s> import collections <EOL> import errno <EOL> import logging <EOL> import math <EOL> import os <EOL> import re <EOL> import threading <EOL> import json <EOL> import time <EOL> import datetime <EOL> import dateutil . parser <EOL> try : <EOL> import ordereddict <EOL> except : <EOL> pass <EOL> import requests <EOL> import requests . utils <EOL> import six <EOL> from six . moves import queue <EOL> from six . moves . urllib import parse as urlparse <EOL> import gertty . version <EOL> from gertty import gitrepo <EOL> HIGH_PRIORITY = <NUM_LIT:0> <EOL> NORMAL_PRIORITY = <NUM_LIT:1> <EOL> LOW_PRIORITY = <NUM_LIT:2> <EOL> TIMEOUT = <NUM_LIT:30> <EOL> CLOSED_STATUSES = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> class MultiQueue ( object ) : <EOL> def __init__ ( self , priorities ) : <EOL> try : <EOL> self . queues = collections . OrderedDict ( ) <EOL> except AttributeError : <EOL> self . queues = ordereddict . OrderedDict ( ) <EOL> for key in priorities : <EOL> self . queues [ key ] = collections . deque ( ) <EOL> self . condition = threading . Condition ( ) <EOL> self . incomplete = [ ] <EOL> def qsize ( self ) : <EOL> count = <NUM_LIT:0> <EOL> self . condition . acquire ( ) <EOL> try : <EOL> for queue in self . queues . values ( ) : <EOL> count += len ( queue ) <EOL> return count + len ( self . incomplete ) <EOL> finally : <EOL> self . condition . release ( ) <EOL> def put ( self , item , priority ) : <EOL> added = False <EOL> self . condition . acquire ( ) <EOL> try : <EOL> if item not in self . queues [ priority ] : <EOL> self . queues [ priority ] . append ( item ) <EOL> added = True <EOL> self . condition . notify ( ) <EOL> finally : <EOL> self . condition . release ( ) <EOL> return added <EOL> def get ( self ) : <EOL> self . condition . acquire ( ) <EOL> try : <EOL> while True : <EOL> for queue in self . queues . values ( ) : <EOL> try : <EOL> ret = queue . popleft ( ) <EOL> self . incomplete . append ( ret ) <EOL> return ret <EOL> except IndexError : <EOL> pass <EOL> self . condition . wait ( ) <EOL> finally : <EOL> self . condition . release ( ) <EOL> def find ( self , klass , priority ) : <EOL> results = [ ] <EOL> self . condition . acquire ( ) <EOL> try : <EOL> for item in self . queues [ priority ] : <EOL> if isinstance ( item , klass ) : <EOL> results . append ( item ) <EOL> finally : <EOL> self . condition . release ( ) <EOL> return results <EOL> def complete ( self , item ) : <EOL> self . condition . acquire ( ) <EOL> try : <EOL> if item in self . incomplete : <EOL> self . incomplete . remove ( item ) <EOL> finally : <EOL> self . condition . release ( ) <EOL> class UpdateEvent ( object ) : <EOL> def updateRelatedChanges ( self , session , change ) : <EOL> related_change_keys = set ( ) <EOL> related_change_keys . add ( change . key ) <EOL> for revision in change . revisions : <EOL> parent = session . getRevisionByCommit ( revision . parent ) <EOL> if parent : <EOL> related_change_keys . add ( parent . change . key ) <EOL> for child in session . getRevisionsByParent ( revision . commit ) : <EOL> related_change_keys . add ( child . change . key ) <EOL> self . related_change_keys = related_change_keys <EOL> class ProjectAddedEvent ( UpdateEvent ) : <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( <EOL> self . project_key , ) <EOL> def __init__ ( self , project ) : <EOL> self . project_key = project . key <EOL> class ChangeAddedEvent ( UpdateEvent ) : <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( <EOL> self . project_key , self . change_key ) <EOL> def __init__ ( self , change ) : <EOL> self . project_key = change . project . key <EOL> self . change_key = change . key <EOL> self . related_change_keys = set ( ) <EOL> self . review_flag_changed = True <EOL> self . status_changed = True <EOL> self . held_changed = False <EOL> class ChangeUpdatedEvent ( UpdateEvent ) : <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( <EOL> self . project_key , self . change_key , self . review_flag_changed , self . status_changed ) <EOL> def __init__ ( self , change ) : <EOL> self . project_key = change . project . key <EOL> self . change_key = change . key <EOL> self . related_change_keys = set ( ) <EOL> self . review_flag_changed = False <EOL> self . status_changed = False <EOL> self . held_changed = False <EOL> class Task ( object ) : <EOL> def __init__ ( self , priority = NORMAL_PRIORITY ) : <EOL> self . log = logging . getLogger ( '<STR_LIT>' ) <EOL> self . priority = priority <EOL> self . succeeded = None <EOL> self . event = threading . Event ( ) <EOL> self . tasks = [ ] <EOL> self . results = [ ] <EOL> def complete ( self , success ) : <EOL> self . succeeded = success <EOL> self . event . set ( ) <EOL> def wait ( self , timeout = None ) : <EOL> self . event . wait ( timeout ) <EOL> return self . succeeded <EOL> def __eq__ ( self , other ) : <EOL> raise NotImplementedError ( ) <EOL> class SyncOwnAccountTask ( Task ) : <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' <EOL> def __eq__ ( self , other ) : <EOL> if other . __class__ == self . __class__ : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> remote = sync . get ( '<STR_LIT>' ) <EOL> sync . account_id = remote [ '<STR_LIT>' ] <EOL> with app . db . getSession ( ) as session : <EOL> session . getAccountByID ( remote [ '<STR_LIT>' ] , <EOL> remote . get ( '<STR_LIT:name>' ) , <EOL> remote . get ( '<STR_LIT:username>' ) , <EOL> remote . get ( '<STR_LIT:email>' ) ) <EOL> class GetVersionTask ( Task ) : <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' <EOL> def __eq__ ( self , other ) : <EOL> if other . __class__ == self . __class__ : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> version = sync . get ( '<STR_LIT>' ) <EOL> sync . setRemoteVersion ( version ) <EOL> class SyncProjectListTask ( Task ) : <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' <EOL> def __eq__ ( self , other ) : <EOL> if other . __class__ == self . __class__ : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> remote = sync . get ( '<STR_LIT>' ) <EOL> remote_keys = set ( remote . keys ( ) ) <EOL> with app . db . getSession ( ) as session : <EOL> local = { } <EOL> for p in session . getProjects ( ) : <EOL> local [ p . name ] = p <EOL> local_keys = set ( local . keys ( ) ) <EOL> for name in local_keys - remote_keys : <EOL> session . delete ( local [ name ] ) <EOL> for name in remote_keys - local_keys : <EOL> p = remote [ name ] <EOL> project = session . createProject ( name , <EOL> description = p . get ( '<STR_LIT:description>' , '<STR_LIT>' ) ) <EOL> self . log . info ( "<STR_LIT>" , project . name ) <EOL> self . results . append ( ProjectAddedEvent ( project ) ) <EOL> class SyncSubscribedProjectBranchesTask ( Task ) : <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' <EOL> def __eq__ ( self , other ) : <EOL> if other . __class__ == self . __class__ : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> with app . db . getSession ( ) as session : <EOL> projects = session . getProjects ( subscribed = True ) <EOL> for p in projects : <EOL> sync . submitTask ( SyncProjectBranchesTask ( p . name , self . priority ) ) <EOL> class SyncProjectBranchesTask ( Task ) : <EOL> branch_re = re . compile ( r'<STR_LIT>' ) <EOL> def __init__ ( self , project_name , priority = NORMAL_PRIORITY ) : <EOL> super ( SyncProjectBranchesTask , self ) . __init__ ( priority ) <EOL> self . project_name = project_name <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( self . project_name , ) <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ and <EOL> other . project_name == self . project_name ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> remote = sync . get ( '<STR_LIT>' % urlparse . quote_plus ( self . project_name ) ) <EOL> remote_branches = set ( ) <EOL> for x in remote : <EOL> m = self . branch_re . match ( x [ '<STR_LIT>' ] ) <EOL> if m : <EOL> remote_branches . add ( m . group ( <NUM_LIT:1> ) ) <EOL> with app . db . getSession ( ) as session : <EOL> local = { } <EOL> project = session . getProjectByName ( self . project_name ) <EOL> for branch in project . branches : <EOL> local [ branch . name ] = branch <EOL> local_branches = set ( local . keys ( ) ) <EOL> for name in local_branches - remote_branches : <EOL> session . delete ( local [ name ] ) <EOL> self . log . info ( "<STR_LIT>" , name , project . name ) <EOL> for name in remote_branches - local_branches : <EOL> project . createBranch ( name ) <EOL> self . log . info ( "<STR_LIT>" , name , project . name ) <EOL> class SyncSubscribedProjectsTask ( Task ) : <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> with app . db . getSession ( ) as session : <EOL> keys = [ p . key for p in session . getProjects ( subscribed = True ) ] <EOL> for i in range ( <NUM_LIT:0> , len ( keys ) , <NUM_LIT:10> ) : <EOL> t = SyncProjectTask ( keys [ i : i + <NUM_LIT:10> ] , self . priority ) <EOL> self . tasks . append ( t ) <EOL> sync . submitTask ( t ) <EOL> t = SyncQueriedChangesTask ( '<STR_LIT>' , '<STR_LIT>' , self . priority ) <EOL> self . tasks . append ( t ) <EOL> sync . submitTask ( t ) <EOL> t = SyncQueriedChangesTask ( '<STR_LIT>' , '<STR_LIT>' , self . priority ) <EOL> self . tasks . append ( t ) <EOL> sync . submitTask ( t ) <EOL> class SyncProjectTask ( Task ) : <EOL> def __init__ ( self , project_keys , priority = NORMAL_PRIORITY ) : <EOL> super ( SyncProjectTask , self ) . __init__ ( priority ) <EOL> if type ( project_keys ) == int : <EOL> project_keys = [ project_keys ] <EOL> self . project_keys = project_keys <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( self . project_keys , ) <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ and <EOL> other . project_keys == self . project_keys ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> now = datetime . datetime . utcnow ( ) <EOL> queries = [ ] <EOL> with app . db . getSession ( ) as session : <EOL> for project_key in self . project_keys : <EOL> project = session . getProject ( project_key ) <EOL> query = '<STR_LIT>' % project . name <EOL> if project . updated : <EOL> query += '<STR_LIT>' % ( int ( math . ceil ( ( now - project . updated ) . total_seconds ( ) ) ) + <NUM_LIT:4> , ) <EOL> else : <EOL> query += '<STR_LIT>' <EOL> queries . append ( query ) <EOL> changes = sync . query ( queries ) <EOL> change_ids = [ c [ '<STR_LIT:id>' ] for c in changes ] <EOL> with app . db . getSession ( ) as session : <EOL> change_ids = session . getChangeIDs ( change_ids ) <EOL> for c in changes : <EOL> if c [ '<STR_LIT:id>' ] in change_ids or ( c [ '<STR_LIT:status>' ] not in CLOSED_STATUSES ) : <EOL> sync . submitTask ( SyncChangeTask ( c [ '<STR_LIT:id>' ] , priority = self . priority ) ) <EOL> for key in self . project_keys : <EOL> sync . submitTask ( SetProjectUpdatedTask ( key , now , priority = self . priority ) ) <EOL> class SetProjectUpdatedTask ( Task ) : <EOL> def __init__ ( self , project_key , updated , priority = NORMAL_PRIORITY ) : <EOL> super ( SetProjectUpdatedTask , self ) . __init__ ( priority ) <EOL> self . project_key = project_key <EOL> self . updated = updated <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( self . project_key , self . updated ) <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ and <EOL> other . project_key == self . project_key and <EOL> other . updated == self . updated ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> with app . db . getSession ( ) as session : <EOL> project = session . getProject ( self . project_key ) <EOL> project . updated = self . updated <EOL> class SyncQueriedChangesTask ( Task ) : <EOL> def __init__ ( self , query_name , query , priority = NORMAL_PRIORITY ) : <EOL> super ( SyncQueriedChangesTask , self ) . __init__ ( priority ) <EOL> self . query_name = query_name <EOL> self . query = query <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % self . query_name <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ and <EOL> other . query_name == self . query_name and <EOL> other . query == self . query ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> now = datetime . datetime . utcnow ( ) <EOL> with app . db . getSession ( ) as session : <EOL> sync_query = session . getSyncQueryByName ( self . query_name ) <EOL> query = '<STR_LIT>' % self . query <EOL> if sync_query . updated : <EOL> query += '<STR_LIT>' % ( int ( math . ceil ( ( now - sync_query . updated ) . total_seconds ( ) ) ) + <NUM_LIT:4> , ) <EOL> else : <EOL> query += '<STR_LIT>' <EOL> for project in session . getProjects ( subscribed = True ) : <EOL> query += '<STR_LIT>' % project . name <EOL> changes = [ ] <EOL> sortkey = '<STR_LIT>' <EOL> done = False <EOL> while not done : <EOL> q = '<STR_LIT>' % ( sortkey , query ) <EOL> self . log . debug ( '<STR_LIT>' % ( q , ) ) <EOL> batch = sync . get ( q ) <EOL> done = True <EOL> if batch : <EOL> changes += batch <EOL> if '<STR_LIT>' in batch [ - <NUM_LIT:1> ] : <EOL> sortkey = '<STR_LIT>' % ( batch [ - <NUM_LIT:1> ] [ '<STR_LIT>' ] , ) <EOL> done = False <EOL> change_ids = [ c [ '<STR_LIT:id>' ] for c in changes ] <EOL> with app . db . getSession ( ) as session : <EOL> change_ids = session . getChangeIDs ( change_ids ) <EOL> for c in changes : <EOL> if c [ '<STR_LIT:id>' ] in change_ids or ( c [ '<STR_LIT:status>' ] not in CLOSED_STATUSES ) : <EOL> sync . submitTask ( SyncChangeTask ( c [ '<STR_LIT:id>' ] , priority = self . priority ) ) <EOL> sync . submitTask ( SetSyncQueryUpdatedTask ( self . query_name , now , priority = self . priority ) ) <EOL> class SetSyncQueryUpdatedTask ( Task ) : <EOL> def __init__ ( self , query_name , updated , priority = NORMAL_PRIORITY ) : <EOL> super ( SetSyncQueryUpdatedTask , self ) . __init__ ( priority ) <EOL> self . query_name = query_name <EOL> self . updated = updated <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( self . query_name , self . updated ) <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ and <EOL> other . query_name == self . query_name and <EOL> other . updated == self . updated ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> with app . db . getSession ( ) as session : <EOL> sync_query = session . getSyncQueryByName ( self . query_name ) <EOL> sync_query . updated = self . updated <EOL> class SyncChangesByCommitsTask ( Task ) : <EOL> def __init__ ( self , commits , priority = NORMAL_PRIORITY ) : <EOL> super ( SyncChangesByCommitsTask , self ) . __init__ ( priority ) <EOL> self . commits = commits <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( self . commits , ) <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ and <EOL> other . commits == self . commits ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> query = '<STR_LIT>' . join ( [ '<STR_LIT>' % x for x in self . commits ] ) <EOL> changes = sync . get ( '<STR_LIT>' % query ) <EOL> self . log . debug ( '<STR_LIT>' % ( query , ) ) <EOL> for c in changes : <EOL> sync . submitTask ( SyncChangeTask ( c [ '<STR_LIT:id>' ] , priority = self . priority ) ) <EOL> self . log . debug ( "<STR_LIT>" % ( c [ '<STR_LIT:id>' ] , ) ) <EOL> def addCommit ( self , commit ) : <EOL> if commit in self . commits : <EOL> return True <EOL> if len ( self . commits ) >= <NUM_LIT:100> : <EOL> return False <EOL> self . commits . append ( commit ) <EOL> return True <EOL> class SyncChangeByNumberTask ( Task ) : <EOL> def __init__ ( self , number , priority = NORMAL_PRIORITY ) : <EOL> super ( SyncChangeByNumberTask , self ) . __init__ ( priority ) <EOL> self . number = number <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( self . number , ) <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ and <EOL> other . number == self . number ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> query = '<STR_LIT:%s>' % self . number <EOL> changes = sync . get ( '<STR_LIT>' % query ) <EOL> self . log . debug ( '<STR_LIT>' % ( query , ) ) <EOL> for c in changes : <EOL> task = SyncChangeTask ( c [ '<STR_LIT:id>' ] , priority = self . priority ) <EOL> self . tasks . append ( task ) <EOL> sync . submitTask ( task ) <EOL> self . log . debug ( "<STR_LIT>" % ( c [ '<STR_LIT:id>' ] , self . number ) ) <EOL> class SyncChangeTask ( Task ) : <EOL> def __init__ ( self , change_id , force_fetch = False , priority = NORMAL_PRIORITY ) : <EOL> super ( SyncChangeTask , self ) . __init__ ( priority ) <EOL> self . change_id = change_id <EOL> self . force_fetch = force_fetch <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( self . change_id , ) <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ and <EOL> other . change_id == self . change_id and <EOL> other . force_fetch == self . force_fetch ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> start_time = time . time ( ) <EOL> app = sync . app <EOL> remote_change = sync . get ( '<STR_LIT>' % self . change_id ) <EOL> for remote_commit , remote_revision in remote_change . get ( '<STR_LIT>' , { } ) . items ( ) : <EOL> remote_comments_data = sync . get ( '<STR_LIT>' % ( self . change_id , remote_commit ) ) <EOL> remote_revision [ '<STR_LIT>' ] = remote_comments_data <EOL> remote_conflicts = sync . query ( [ '<STR_LIT>' % <EOL> remote_change [ '<STR_LIT>' ] ] ) <EOL> fetches = collections . defaultdict ( list ) <EOL> parent_commits = set ( ) <EOL> with app . db . getSession ( ) as session : <EOL> change = session . getChangeByID ( self . change_id ) <EOL> account = session . getAccountByID ( remote_change [ '<STR_LIT>' ] [ '<STR_LIT>' ] , <EOL> name = remote_change [ '<STR_LIT>' ] . get ( '<STR_LIT:name>' ) , <EOL> username = remote_change [ '<STR_LIT>' ] . get ( '<STR_LIT:username>' ) , <EOL> email = remote_change [ '<STR_LIT>' ] . get ( '<STR_LIT:email>' ) ) <EOL> if not change : <EOL> project = session . getProjectByName ( remote_change [ '<STR_LIT>' ] ) <EOL> created = dateutil . parser . parse ( remote_change [ '<STR_LIT>' ] ) <EOL> updated = dateutil . parser . parse ( remote_change [ '<STR_LIT>' ] ) <EOL> change = project . createChange ( remote_change [ '<STR_LIT:id>' ] , account , remote_change [ '<STR_LIT>' ] , <EOL> remote_change [ '<STR_LIT>' ] , remote_change [ '<STR_LIT>' ] , <EOL> remote_change [ '<STR_LIT>' ] , created , <EOL> updated , remote_change [ '<STR_LIT:status>' ] , <EOL> topic = remote_change . get ( '<STR_LIT>' ) ) <EOL> self . log . info ( "<STR_LIT>" , change . id ) <EOL> result = ChangeAddedEvent ( change ) <EOL> else : <EOL> result = ChangeUpdatedEvent ( change ) <EOL> app . project_cache . clear ( change . project ) <EOL> self . results . append ( result ) <EOL> change . owner = account <EOL> if change . status != remote_change [ '<STR_LIT:status>' ] : <EOL> change . status = remote_change [ '<STR_LIT:status>' ] <EOL> result . status_changed = True <EOL> if remote_change . get ( '<STR_LIT>' ) : <EOL> change . starred = True <EOL> else : <EOL> change . starred = False <EOL> change . subject = remote_change [ '<STR_LIT>' ] <EOL> change . updated = dateutil . parser . parse ( remote_change [ '<STR_LIT>' ] ) <EOL> change . topic = remote_change . get ( '<STR_LIT>' ) <EOL> unseen_conflicts = [ x . id for x in change . conflicts ] <EOL> for remote_conflict in remote_conflicts : <EOL> conflict_id = remote_conflict [ '<STR_LIT:id>' ] <EOL> conflict = session . getChangeByID ( conflict_id ) <EOL> if not conflict : <EOL> self . log . info ( "<STR_LIT>" , <EOL> conflict_id , change . number ) <EOL> sync . submitTask ( SyncChangeTask ( conflict_id , priority = self . priority ) ) <EOL> else : <EOL> if conflict not in change . conflicts : <EOL> self . log . info ( "<STR_LIT>" , <EOL> conflict . number , change . number ) <EOL> change . addConflict ( conflict ) <EOL> self . results . append ( ChangeUpdatedEvent ( conflict ) ) <EOL> if conflict_id in unseen_conflicts : <EOL> unseen_conflicts . remove ( conflict_id ) <EOL> for conflict_id in unseen_conflicts : <EOL> conflict = session . getChangeByID ( conflict_id ) <EOL> self . log . info ( "<STR_LIT>" , <EOL> conflict . number , change . number ) <EOL> change . delConflict ( conflict ) <EOL> self . results . append ( ChangeUpdatedEvent ( conflict ) ) <EOL> repo = gitrepo . get_repo ( change . project . name , app . config ) <EOL> new_revision = False <EOL> for remote_commit , remote_revision in remote_change . get ( '<STR_LIT>' , { } ) . items ( ) : <EOL> revision = session . getRevisionByCommit ( remote_commit ) <EOL> url = sync . app . config . git_url + change . project . name <EOL> if '<STR_LIT>' in remote_revision [ '<STR_LIT>' ] : <EOL> ref = remote_revision [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> url = remote_revision [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT:url>' ] <EOL> auth = False <EOL> elif '<STR_LIT:http>' in remote_revision [ '<STR_LIT>' ] : <EOL> auth = True <EOL> ref = remote_revision [ '<STR_LIT>' ] [ '<STR_LIT:http>' ] [ '<STR_LIT>' ] <EOL> url = list ( urlparse . urlsplit ( sync . app . config . url + change . project . name ) ) <EOL> url [ <NUM_LIT:1> ] = '<STR_LIT>' % ( <EOL> urlparse . quote_plus ( sync . app . config . username ) , <EOL> urlparse . quote_plus ( sync . app . config . password ) , url [ <NUM_LIT:1> ] ) <EOL> url = urlparse . urlunsplit ( url ) <EOL> elif '<STR_LIT>' in remote_revision [ '<STR_LIT>' ] : <EOL> ref = remote_revision [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> url = remote_revision [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT:url>' ] <EOL> auth = False <EOL> elif '<STR_LIT>' in remote_revision [ '<STR_LIT>' ] : <EOL> ref = remote_revision [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> url = remote_revision [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT:url>' ] <EOL> auth = False <EOL> else : <EOL> if len ( remote_revision [ '<STR_LIT>' ] ) : <EOL> errMessage = "<STR_LIT>" % ( <EOL> '<STR_LIT:U+002CU+0020>' . join ( remote_revision [ '<STR_LIT>' ] . keys ( ) ) ) <EOL> else : <EOL> errMessage = "<STR_LIT>" <EOL> raise Exception ( errMessage ) <EOL> if ( not revision ) or self . force_fetch : <EOL> fetches [ url ] . append ( '<STR_LIT>' % dict ( ref = ref ) ) <EOL> if not revision : <EOL> revision = change . createRevision ( remote_revision [ '<STR_LIT>' ] , <EOL> remote_revision [ '<STR_LIT>' ] [ '<STR_LIT:message>' ] , remote_commit , <EOL> remote_revision [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ <NUM_LIT:0> ] [ '<STR_LIT>' ] , <EOL> auth , ref ) <EOL> self . log . info ( "<STR_LIT>" , <EOL> revision . key , self . change_id , remote_revision [ '<STR_LIT>' ] ) <EOL> new_revision = True <EOL> revision . message = remote_revision [ '<STR_LIT>' ] [ '<STR_LIT:message>' ] <EOL> actions = remote_revision . get ( '<STR_LIT>' , { } ) <EOL> revision . can_submit = '<STR_LIT>' in actions <EOL> if revision . parent not in parent_commits : <EOL> parent_revision = session . getRevisionByCommit ( revision . parent ) <EOL> if not parent_revision and change . status not in CLOSED_STATUSES : <EOL> sync . _syncChangeByCommit ( revision . parent , self . priority ) <EOL> self . log . debug ( "<STR_LIT>" % <EOL> ( change . id , remote_revision [ '<STR_LIT>' ] , revision . parent ) ) <EOL> parent_commits . add ( revision . parent ) <EOL> result . updateRelatedChanges ( session , change ) <EOL> f = revision . getFile ( '<STR_LIT>' ) <EOL> if f is None : <EOL> f = revision . createFile ( '<STR_LIT>' , None , <EOL> None , None , None ) <EOL> for remote_path , remote_file in remote_revision [ '<STR_LIT>' ] . items ( ) : <EOL> f = revision . getFile ( remote_path ) <EOL> if f is None : <EOL> if remote_file . get ( '<STR_LIT>' ) : <EOL> inserted = deleted = None <EOL> else : <EOL> inserted = remote_file . get ( '<STR_LIT>' , <NUM_LIT:0> ) <EOL> deleted = remote_file . get ( '<STR_LIT>' , <NUM_LIT:0> ) <EOL> f = revision . createFile ( remote_path , remote_file . get ( '<STR_LIT:status>' , '<STR_LIT:M>' ) , <EOL> remote_file . get ( '<STR_LIT>' ) , <EOL> inserted , deleted ) <EOL> remote_comments_data = remote_revision [ '<STR_LIT>' ] <EOL> for remote_file , remote_comments in remote_comments_data . items ( ) : <EOL> for remote_comment in remote_comments : <EOL> account = session . getAccountByID ( remote_comment [ '<STR_LIT>' ] [ '<STR_LIT>' ] , <EOL> name = remote_comment [ '<STR_LIT>' ] . get ( '<STR_LIT:name>' ) , <EOL> username = remote_comment [ '<STR_LIT>' ] . get ( '<STR_LIT:username>' ) , <EOL> email = remote_comment [ '<STR_LIT>' ] . get ( '<STR_LIT:email>' ) ) <EOL> comment = session . getCommentByID ( remote_comment [ '<STR_LIT:id>' ] ) <EOL> if not comment : <EOL> created = dateutil . parser . parse ( remote_comment [ '<STR_LIT>' ] ) <EOL> parent = False <EOL> if remote_comment . get ( '<STR_LIT>' , '<STR_LIT>' ) == '<STR_LIT>' : <EOL> parent = True <EOL> fileobj = revision . getFile ( remote_file ) <EOL> if fileobj is None : <EOL> fileobj = revision . createFile ( remote_file , '<STR_LIT:M>' ) <EOL> comment = fileobj . createComment ( remote_comment [ '<STR_LIT:id>' ] , account , <EOL> remote_comment . get ( '<STR_LIT>' ) , <EOL> created , <EOL> parent , remote_comment . get ( '<STR_LIT>' ) , <EOL> remote_comment [ '<STR_LIT:message>' ] ) <EOL> self . log . info ( "<STR_LIT>" , <EOL> comment . key , revision . key ) <EOL> else : <EOL> if comment . author != account : <EOL> comment . author = account <EOL> new_message = False <EOL> for remote_message in remote_change . get ( '<STR_LIT>' , [ ] ) : <EOL> if '<STR_LIT>' in remote_message : <EOL> account = session . getAccountByID ( remote_message [ '<STR_LIT>' ] [ '<STR_LIT>' ] , <EOL> name = remote_message [ '<STR_LIT>' ] . get ( '<STR_LIT:name>' ) , <EOL> username = remote_message [ '<STR_LIT>' ] . get ( '<STR_LIT:username>' ) , <EOL> email = remote_message [ '<STR_LIT>' ] . get ( '<STR_LIT:email>' ) ) <EOL> if account . username != app . config . username : <EOL> new_message = True <EOL> else : <EOL> account = session . getSystemAccount ( ) <EOL> message = session . getMessageByID ( remote_message [ '<STR_LIT:id>' ] ) <EOL> if not message : <EOL> revision = session . getRevisionByNumber ( change , remote_message . get ( '<STR_LIT>' , <NUM_LIT:1> ) ) <EOL> if revision : <EOL> created = dateutil . parser . parse ( remote_message [ '<STR_LIT:date>' ] ) <EOL> message = revision . createMessage ( remote_message [ '<STR_LIT:id>' ] , account , created , <EOL> remote_message [ '<STR_LIT:message>' ] ) <EOL> self . log . info ( "<STR_LIT>" , message . key , revision . key ) <EOL> else : <EOL> self . log . info ( "<STR_LIT>" , remote_message . get ( '<STR_LIT>' ) ) <EOL> else : <EOL> if message . author != account : <EOL> message . author = account <EOL> remote_approval_entries = { } <EOL> remote_label_entries = { } <EOL> user_voted = False <EOL> for remote_label_name , remote_label_dict in remote_change . get ( '<STR_LIT>' , { } ) . items ( ) : <EOL> for remote_approval in remote_label_dict . get ( '<STR_LIT:all>' , [ ] ) : <EOL> if remote_approval . get ( '<STR_LIT:value>' ) is None : <EOL> continue <EOL> remote_approval [ '<STR_LIT>' ] = remote_label_name <EOL> key = '<STR_LIT>' % ( remote_approval [ '<STR_LIT>' ] , remote_approval [ '<STR_LIT>' ] ) <EOL> remote_approval_entries [ key ] = remote_approval <EOL> if remote_approval [ '<STR_LIT>' ] == sync . account_id and int ( remote_approval [ '<STR_LIT:value>' ] ) != <NUM_LIT:0> : <EOL> user_voted = True <EOL> for key , value in remote_label_dict . get ( '<STR_LIT>' , { } ) . items ( ) : <EOL> label = dict ( value = key , <EOL> description = value , <EOL> category = remote_label_name ) <EOL> key = '<STR_LIT>' % ( label [ '<STR_LIT>' ] , label [ '<STR_LIT:value>' ] , label [ '<STR_LIT:description>' ] ) <EOL> remote_label_entries [ key ] = label <EOL> remote_approval_keys = set ( remote_approval_entries . keys ( ) ) <EOL> remote_label_keys = set ( remote_label_entries . keys ( ) ) <EOL> local_approvals = { } <EOL> local_labels = { } <EOL> user_votes = { } <EOL> for approval in change . approvals : <EOL> if approval . draft and not new_revision : <EOL> user_votes [ approval . category ] = approval . value <EOL> user_voted = True <EOL> continue <EOL> key = '<STR_LIT>' % ( approval . category , approval . reviewer . id ) <EOL> if key in local_approvals : <EOL> session . delete ( approval ) <EOL> else : <EOL> local_approvals [ key ] = approval <EOL> local_approval_keys = set ( local_approvals . keys ( ) ) <EOL> for label in change . labels : <EOL> key = '<STR_LIT>' % ( label . category , label . value , label . description ) <EOL> local_labels [ key ] = label <EOL> local_label_keys = set ( local_labels . keys ( ) ) <EOL> for key in local_approval_keys - remote_approval_keys : <EOL> session . delete ( local_approvals [ key ] ) <EOL> for key in local_label_keys - remote_label_keys : <EOL> session . delete ( local_labels [ key ] ) <EOL> for key in remote_approval_keys - local_approval_keys : <EOL> remote_approval = remote_approval_entries [ key ] <EOL> account = session . getAccountByID ( remote_approval [ '<STR_LIT>' ] , <EOL> name = remote_approval . get ( '<STR_LIT:name>' ) , <EOL> username = remote_approval . get ( '<STR_LIT:username>' ) , <EOL> email = remote_approval . get ( '<STR_LIT:email>' ) ) <EOL> change . createApproval ( account , <EOL> remote_approval [ '<STR_LIT>' ] , <EOL> remote_approval [ '<STR_LIT:value>' ] ) <EOL> self . log . info ( "<STR_LIT>" , change . id ) <EOL> user_value = user_votes . get ( remote_approval [ '<STR_LIT>' ] , <NUM_LIT:0> ) <EOL> if user_value > <NUM_LIT:0> and remote_approval [ '<STR_LIT:value>' ] < <NUM_LIT:0> : <EOL> if not change . held : <EOL> change . held = True <EOL> result . held_changed = True <EOL> self . log . info ( "<STR_LIT>" , change . id ) <EOL> for key in remote_label_keys - local_label_keys : <EOL> remote_label = remote_label_entries [ key ] <EOL> change . createLabel ( remote_label [ '<STR_LIT>' ] , <EOL> remote_label [ '<STR_LIT:value>' ] , <EOL> remote_label [ '<STR_LIT:description>' ] ) <EOL> for key in remote_approval_keys . intersection ( local_approval_keys ) : <EOL> local_approval = local_approvals [ key ] <EOL> remote_approval = remote_approval_entries [ key ] <EOL> local_approval . value = remote_approval [ '<STR_LIT:value>' ] <EOL> account = session . getAccountByID ( remote_approval [ '<STR_LIT>' ] , <EOL> name = remote_approval . get ( '<STR_LIT:name>' ) , <EOL> username = remote_approval . get ( '<STR_LIT:username>' ) , <EOL> email = remote_approval . get ( '<STR_LIT:email>' ) ) <EOL> remote_permitted_entries = { } <EOL> for remote_label_name , remote_label_values in remote_change . get ( '<STR_LIT>' , { } ) . items ( ) : <EOL> for remote_label_value in remote_label_values : <EOL> remote_label = dict ( category = remote_label_name , <EOL> value = remote_label_value ) <EOL> key = '<STR_LIT>' % ( remote_label [ '<STR_LIT>' ] , remote_label [ '<STR_LIT:value>' ] ) <EOL> remote_permitted_entries [ key ] = remote_label <EOL> remote_permitted_keys = set ( remote_permitted_entries . keys ( ) ) <EOL> local_permitted = { } <EOL> for permitted in change . permitted_labels : <EOL> key = '<STR_LIT>' % ( permitted . category , permitted . value ) <EOL> local_permitted [ key ] = permitted <EOL> local_permitted_keys = set ( local_permitted . keys ( ) ) <EOL> for key in local_permitted_keys - remote_permitted_keys : <EOL> session . delete ( local_permitted [ key ] ) <EOL> for key in remote_permitted_keys - local_permitted_keys : <EOL> remote_permitted = remote_permitted_entries [ key ] <EOL> change . createPermittedLabel ( remote_permitted [ '<STR_LIT>' ] , <EOL> remote_permitted [ '<STR_LIT:value>' ] ) <EOL> if not user_voted : <EOL> if new_revision or new_message : <EOL> if change . reviewed : <EOL> change . reviewed = False <EOL> result . review_flag_changed = True <EOL> app . project_cache . clear ( change . project ) <EOL> for url , refs in fetches . items ( ) : <EOL> self . log . debug ( "<STR_LIT>" , url , refs ) <EOL> try : <EOL> repo . fetch ( url , refs ) <EOL> except Exception : <EOL> for ref in refs : <EOL> self . log . debug ( "<STR_LIT>" % ( url , ref ) ) <EOL> repo . fetch ( url , ref ) <EOL> end_time = time . time ( ) <EOL> total_time = end_time - start_time <EOL> self . log . info ( "<STR_LIT>" , self . change_id , total_time ) <EOL> class CheckReposTask ( Task ) : <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> with app . db . getSession ( ) as session : <EOL> projects = session . getProjects ( subscribed = True ) <EOL> for project in projects : <EOL> try : <EOL> missing = False <EOL> try : <EOL> repo = gitrepo . get_repo ( project . name , app . config ) <EOL> except gitrepo . GitCloneError : <EOL> missing = True <EOL> if missing or app . fetch_missing_refs : <EOL> sync . submitTask ( CheckRevisionsTask ( project . key , <EOL> priority = LOW_PRIORITY ) ) <EOL> except Exception : <EOL> self . log . exception ( "<STR_LIT>" % <EOL> ( project . name , ) ) <EOL> class CheckRevisionsTask ( Task ) : <EOL> def __init__ ( self , project_key , priority = NORMAL_PRIORITY ) : <EOL> super ( CheckRevisionsTask , self ) . __init__ ( priority ) <EOL> self . project_key = project_key <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( self . project_key , ) <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ and <EOL> other . project_key == self . project_key ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> to_sync = set ( ) <EOL> with app . db . getSession ( ) as session : <EOL> project = session . getProject ( self . project_key ) <EOL> repo = None <EOL> try : <EOL> repo = gitrepo . get_repo ( project . name , app . config ) <EOL> except gitrepo . GitCloneError : <EOL> pass <EOL> for change in project . open_changes : <EOL> if repo : <EOL> for revision in change . revisions : <EOL> if not ( repo . hasCommit ( revision . parent ) and <EOL> repo . hasCommit ( revision . commit ) ) : <EOL> to_sync . add ( change . id ) <EOL> else : <EOL> to_sync . add ( change . id ) <EOL> for change_id in to_sync : <EOL> sync . submitTask ( SyncChangeTask ( change_id , priority = self . priority ) ) <EOL> class UploadReviewsTask ( Task ) : <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> with app . db . getSession ( ) as session : <EOL> for c in session . getPendingTopics ( ) : <EOL> sync . submitTask ( SetTopicTask ( c . key , self . priority ) ) <EOL> for c in session . getPendingRebases ( ) : <EOL> sync . submitTask ( RebaseChangeTask ( c . key , self . priority ) ) <EOL> for c in session . getPendingStatusChanges ( ) : <EOL> sync . submitTask ( ChangeStatusTask ( c . key , self . priority ) ) <EOL> for c in session . getPendingStarred ( ) : <EOL> sync . submitTask ( ChangeStarredTask ( c . key , self . priority ) ) <EOL> for c in session . getPendingCherryPicks ( ) : <EOL> sync . submitTask ( SendCherryPickTask ( c . key , self . priority ) ) <EOL> for r in session . getPendingCommitMessages ( ) : <EOL> sync . submitTask ( ChangeCommitMessageTask ( r . key , self . priority ) ) <EOL> for m in session . getPendingMessages ( ) : <EOL> sync . submitTask ( UploadReviewTask ( m . key , self . priority ) ) <EOL> class SetTopicTask ( Task ) : <EOL> def __init__ ( self , change_key , priority = NORMAL_PRIORITY ) : <EOL> super ( SetTopicTask , self ) . __init__ ( priority ) <EOL> self . change_key = change_key <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( self . change_key , ) <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ and <EOL> other . change_key == self . change_key ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> with app . db . getSession ( ) as session : <EOL> change = session . getChange ( self . change_key ) <EOL> data = dict ( topic = change . topic ) <EOL> change . pending_topic = False <EOL> sync . put ( '<STR_LIT>' % ( change . id , ) , <EOL> data ) <EOL> sync . submitTask ( SyncChangeTask ( change . id , priority = self . priority ) ) <EOL> class RebaseChangeTask ( Task ) : <EOL> def __init__ ( self , change_key , priority = NORMAL_PRIORITY ) : <EOL> super ( RebaseChangeTask , self ) . __init__ ( priority ) <EOL> self . change_key = change_key <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( self . change_key , ) <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ and <EOL> other . change_key == self . change_key ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> with app . db . getSession ( ) as session : <EOL> change = session . getChange ( self . change_key ) <EOL> change . pending_rebase = False <EOL> sync . post ( '<STR_LIT>' % ( change . id , ) , { } ) <EOL> sync . submitTask ( SyncChangeTask ( change . id , priority = self . priority ) ) <EOL> class ChangeStarredTask ( Task ) : <EOL> def __init__ ( self , change_key , priority = NORMAL_PRIORITY ) : <EOL> super ( ChangeStarredTask , self ) . __init__ ( priority ) <EOL> self . change_key = change_key <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( self . change_key , ) <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ and <EOL> other . change_key == self . change_key ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> with app . db . getSession ( ) as session : <EOL> change = session . getChange ( self . change_key ) <EOL> if change . starred : <EOL> sync . put ( '<STR_LIT>' % ( change . id , ) , <EOL> data = { } ) <EOL> else : <EOL> sync . delete ( '<STR_LIT>' % ( change . id , ) , <EOL> data = { } ) <EOL> change . pending_starred = False <EOL> sync . submitTask ( SyncChangeTask ( change . id , priority = self . priority ) ) <EOL> class ChangeStatusTask ( Task ) : <EOL> def __init__ ( self , change_key , priority = NORMAL_PRIORITY ) : <EOL> super ( ChangeStatusTask , self ) . __init__ ( priority ) <EOL> self . change_key = change_key <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( self . change_key , ) <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ and <EOL> other . change_key == self . change_key ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> with app . db . getSession ( ) as session : <EOL> change = session . getChange ( self . change_key ) <EOL> if change . pending_status_message : <EOL> data = dict ( message = change . pending_status_message ) <EOL> else : <EOL> data = { } <EOL> change . pending_status = False <EOL> change . pending_status_message = None <EOL> if change . status == '<STR_LIT>' : <EOL> sync . post ( '<STR_LIT>' % ( change . id , ) , <EOL> data ) <EOL> elif change . status == '<STR_LIT>' : <EOL> sync . post ( '<STR_LIT>' % ( change . id , ) , <EOL> data ) <EOL> elif change . status == '<STR_LIT>' : <EOL> sync . post ( '<STR_LIT>' % ( change . id , ) , { } ) <EOL> sync . submitTask ( SyncChangeTask ( change . id , priority = self . priority ) ) <EOL> class SendCherryPickTask ( Task ) : <EOL> def __init__ ( self , cp_key , priority = NORMAL_PRIORITY ) : <EOL> super ( SendCherryPickTask , self ) . __init__ ( priority ) <EOL> self . cp_key = cp_key <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( self . cp_key , ) <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ and <EOL> other . cp_key == self . cp_key ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> with app . db . getSession ( ) as session : <EOL> cp = session . getPendingCherryPick ( self . cp_key ) <EOL> data = dict ( message = cp . message , <EOL> destination = cp . branch ) <EOL> session . delete ( cp ) <EOL> ret = sync . post ( '<STR_LIT>' % <EOL> ( cp . revision . change . id , cp . revision . commit ) , <EOL> data ) <EOL> if ret and '<STR_LIT:id>' in ret : <EOL> sync . submitTask ( SyncChangeTask ( ret [ '<STR_LIT:id>' ] , priority = self . priority ) ) <EOL> class ChangeCommitMessageTask ( Task ) : <EOL> def __init__ ( self , revision_key , priority = NORMAL_PRIORITY ) : <EOL> super ( ChangeCommitMessageTask , self ) . __init__ ( priority ) <EOL> self . revision_key = revision_key <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( self . revision_key , ) <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ and <EOL> other . revision_key == self . revision_key ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> with app . db . getSession ( ) as session : <EOL> revision = session . getRevision ( self . revision_key ) <EOL> revision . pending_message = False <EOL> data = dict ( message = revision . message ) <EOL> if sync . version < ( <NUM_LIT:2> , <NUM_LIT:11> , <NUM_LIT:0> ) : <EOL> sync . post ( '<STR_LIT>' % <EOL> ( revision . change . id , revision . commit ) , <EOL> data ) <EOL> else : <EOL> edit = sync . get ( '<STR_LIT>' % revision . change . id ) <EOL> if edit is not None : <EOL> raise Exception ( "<STR_LIT>" % <EOL> ( revision . change . number , ) ) <EOL> sync . put ( '<STR_LIT>' % ( revision . change . id , ) , data ) <EOL> sync . post ( '<STR_LIT>' % ( revision . change . id , ) , { } ) <EOL> change_id = revision . change . id <EOL> sync . submitTask ( SyncChangeTask ( change_id , priority = self . priority ) ) <EOL> class UploadReviewTask ( Task ) : <EOL> def __init__ ( self , message_key , priority = NORMAL_PRIORITY ) : <EOL> super ( UploadReviewTask , self ) . __init__ ( priority ) <EOL> self . message_key = message_key <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( self . message_key , ) <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ and <EOL> other . message_key == self . message_key ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> with app . db . getSession ( ) as session : <EOL> message = session . getMessage ( self . message_key ) <EOL> if message is None : <EOL> self . log . debug ( "<STR_LIT>" % ( <EOL> self . message_key ) ) <EOL> return <EOL> change = message . revision . change <EOL> if not change . held : <EOL> self . log . debug ( "<STR_LIT>" % ( change . id , ) ) <EOL> t = SyncChangeTask ( change . id ) <EOL> t . run ( sync ) <EOL> self . results += t . results <EOL> submit = False <EOL> change_id = None <EOL> with app . db . getSession ( ) as session : <EOL> message = session . getMessage ( self . message_key ) <EOL> revision = message . revision <EOL> change = message . revision . change <EOL> if change . held : <EOL> self . log . debug ( "<STR_LIT>" % <EOL> ( change . id , ) ) <EOL> return <EOL> change_id = change . id <EOL> current_revision = change . revisions [ - <NUM_LIT:1> ] <EOL> if change . pending_status and change . status == '<STR_LIT>' : <EOL> submit = True <EOL> data = dict ( message = message . message , <EOL> strict_labels = False ) <EOL> if revision == current_revision : <EOL> data [ '<STR_LIT>' ] = { } <EOL> for approval in change . draft_approvals : <EOL> data [ '<STR_LIT>' ] [ approval . category ] = approval . value <EOL> session . delete ( approval ) <EOL> comments = { } <EOL> for file in revision . files : <EOL> if file . draft_comments : <EOL> comment_list = [ ] <EOL> for comment in file . draft_comments : <EOL> d = dict ( line = comment . line , <EOL> message = comment . message ) <EOL> if comment . parent : <EOL> d [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> comment_list . append ( d ) <EOL> session . delete ( comment ) <EOL> comments [ file . path ] = comment_list <EOL> if comments : <EOL> data [ '<STR_LIT>' ] = comments <EOL> session . delete ( message ) <EOL> sync . post ( '<STR_LIT>' % ( change . id , revision . commit ) , <EOL> data ) <EOL> if submit : <EOL> with app . db . getSession ( ) as session : <EOL> change = session . getChangeByID ( change_id ) <EOL> change . pending_status = False <EOL> change . pending_status_message = None <EOL> sync . post ( '<STR_LIT>' % ( change_id , ) , { } ) <EOL> sync . submitTask ( SyncChangeTask ( change_id , priority = self . priority ) ) <EOL> class PruneDatabaseTask ( Task ) : <EOL> def __init__ ( self , age , priority = NORMAL_PRIORITY ) : <EOL> super ( PruneDatabaseTask , self ) . __init__ ( priority ) <EOL> self . age = age <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( self . age , ) <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ and <EOL> other . age == self . age ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> if not self . age : <EOL> return <EOL> app = sync . app <EOL> with app . db . getSession ( ) as session : <EOL> for change in session . getChanges ( '<STR_LIT>' % self . age ) : <EOL> t = PruneChangeTask ( change . key , priority = self . priority ) <EOL> self . tasks . append ( t ) <EOL> sync . submitTask ( t ) <EOL> t = VacuumDatabaseTask ( priority = self . priority ) <EOL> self . tasks . append ( t ) <EOL> sync . submitTask ( t ) <EOL> class PruneChangeTask ( Task ) : <EOL> def __init__ ( self , key , priority = NORMAL_PRIORITY ) : <EOL> super ( PruneChangeTask , self ) . __init__ ( priority ) <EOL> self . key = key <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( self . key , ) <EOL> def __eq__ ( self , other ) : <EOL> if ( other . __class__ == self . __class__ and <EOL> other . key == self . key ) : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> with app . db . getSession ( ) as session : <EOL> change = session . getChange ( self . key ) <EOL> if not change : <EOL> return <EOL> repo = gitrepo . get_repo ( change . project . name , app . config ) <EOL> self . log . info ( "<STR_LIT>" % ( <EOL> change . project . name , change . number , change . status , change . updated ) ) <EOL> change_ref = None <EOL> for revision in change . revisions : <EOL> if change_ref is None : <EOL> change_ref = '<STR_LIT:/>' . join ( revision . fetch_ref . split ( '<STR_LIT:/>' ) [ : - <NUM_LIT:1> ] ) <EOL> self . log . info ( "<STR_LIT>" % ( <EOL> change . project . name , revision . fetch_ref ) ) <EOL> repo . deleteRef ( revision . fetch_ref ) <EOL> self . log . info ( "<STR_LIT>" % ( <EOL> change . project . name , change_ref ) ) <EOL> try : <EOL> repo . deleteRef ( change_ref ) <EOL> except OSError as e : <EOL> if e . errno not in [ errno . EISDIR , errno . EPERM ] : <EOL> raise <EOL> session . delete ( change ) <EOL> class VacuumDatabaseTask ( Task ) : <EOL> def __init__ ( self , priority = NORMAL_PRIORITY ) : <EOL> super ( VacuumDatabaseTask , self ) . __init__ ( priority ) <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' <EOL> def __eq__ ( self , other ) : <EOL> if other . __class__ == self . __class__ : <EOL> return True <EOL> return False <EOL> def run ( self , sync ) : <EOL> app = sync . app <EOL> with app . db . getSession ( ) as session : <EOL> session . vacuum ( ) <EOL> class Sync ( object ) : <EOL> def __init__ ( self , app , disable_background_sync ) : <EOL> self . user_agent = '<STR_LIT>' % ( gertty . version . version_info . release_string ( ) , <EOL> requests . utils . default_user_agent ( ) ) <EOL> self . version = ( <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> ) <EOL> self . offline = False <EOL> self . account_id = None <EOL> self . app = app <EOL> self . log = logging . getLogger ( '<STR_LIT>' ) <EOL> self . queue = MultiQueue ( [ HIGH_PRIORITY , NORMAL_PRIORITY , LOW_PRIORITY ] ) <EOL> self . result_queue = queue . Queue ( ) <EOL> self . session = requests . Session ( ) <EOL> if self . app . config . auth_type == '<STR_LIT>' : <EOL> authclass = requests . auth . HTTPBasicAuth <EOL> else : <EOL> authclass = requests . auth . HTTPDigestAuth <EOL> self . auth = authclass ( <EOL> self . app . config . username , self . app . config . password ) <EOL> self . submitTask ( GetVersionTask ( HIGH_PRIORITY ) ) <EOL> self . submitTask ( SyncOwnAccountTask ( HIGH_PRIORITY ) ) <EOL> if not disable_background_sync : <EOL> self . submitTask ( CheckReposTask ( HIGH_PRIORITY ) ) <EOL> self . submitTask ( UploadReviewsTask ( HIGH_PRIORITY ) ) <EOL> self . submitTask ( SyncProjectListTask ( HIGH_PRIORITY ) ) <EOL> self . submitTask ( SyncSubscribedProjectsTask ( NORMAL_PRIORITY ) ) <EOL> self . submitTask ( SyncSubscribedProjectBranchesTask ( LOW_PRIORITY ) ) <EOL> self . submitTask ( PruneDatabaseTask ( self . app . config . expire_age , LOW_PRIORITY ) ) <EOL> self . periodic_thread = threading . Thread ( target = self . periodicSync ) <EOL> self . periodic_thread . daemon = True <EOL> self . periodic_thread . start ( ) <EOL> def periodicSync ( self ) : <EOL> hourly = time . time ( ) <EOL> while True : <EOL> try : <EOL> time . sleep ( <NUM_LIT> ) <EOL> self . syncSubscribedProjects ( ) <EOL> now = time . time ( ) <EOL> if now - hourly > <NUM_LIT> : <EOL> hourly = now <EOL> self . pruneDatabase ( ) <EOL> except Exception : <EOL> self . log . exception ( '<STR_LIT>' ) <EOL> def submitTask ( self , task ) : <EOL> if not self . offline : <EOL> if not self . queue . put ( task , task . priority ) : <EOL> task . complete ( False ) <EOL> else : <EOL> task . complete ( False ) <EOL> def run ( self , pipe ) : <EOL> task = None <EOL> while True : <EOL> task = self . _run ( pipe , task ) <EOL> def _run ( self , pipe , task = None ) : <EOL> if not task : <EOL> task = self . queue . get ( ) <EOL> self . log . debug ( '<STR_LIT>' % ( task , ) ) <EOL> try : <EOL> task . run ( self ) <EOL> task . complete ( True ) <EOL> self . queue . complete ( task ) <EOL> except requests . ConnectionError as e : <EOL> self . log . warning ( "<STR_LIT>" % ( e , ) ) <EOL> if not self . offline : <EOL> self . submitTask ( GetVersionTask ( HIGH_PRIORITY ) ) <EOL> self . submitTask ( UploadReviewsTask ( HIGH_PRIORITY ) ) <EOL> self . offline = True <EOL> self . app . status . update ( offline = True , refresh = False ) <EOL> os . write ( pipe , six . b ( '<STR_LIT>' ) ) <EOL> time . sleep ( <NUM_LIT:30> ) <EOL> return task <EOL> except Exception : <EOL> task . complete ( False ) <EOL> self . queue . complete ( task ) <EOL> self . log . exception ( '<STR_LIT>' % ( task , ) ) <EOL> self . app . status . update ( error = True , refresh = False ) <EOL> self . offline = False <EOL> self . app . status . update ( offline = False , refresh = False ) <EOL> for r in task . results : <EOL> self . result_queue . put ( r ) <EOL> os . write ( pipe , six . b ( '<STR_LIT>' ) ) <EOL> return None <EOL> def url ( self , path ) : <EOL> return self . app . config . url + '<STR_LIT>' + path <EOL> def get ( self , path ) : <EOL> url = self . url ( path ) <EOL> self . log . debug ( '<STR_LIT>' % ( url , ) ) <EOL> r = self . session . get ( url , <EOL> verify = self . app . config . verify_ssl , <EOL> auth = self . auth , timeout = TIMEOUT , <EOL> headers = { '<STR_LIT>' : '<STR_LIT:application/json>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : self . user_agent } ) <EOL> if r . status_code == <NUM_LIT:200> : <EOL> ret = json . loads ( r . text [ <NUM_LIT:4> : ] ) <EOL> if len ( ret ) : <EOL> self . log . debug ( '<STR_LIT>' % ( ret , ) ) <EOL> else : <EOL> self . log . debug ( '<STR_LIT>' ) <EOL> return ret <EOL> else : <EOL> self . log . warn ( '<STR_LIT>' , r . status_code ) <EOL> def post ( self , path , data ) : <EOL> url = self . url ( path ) <EOL> self . log . debug ( '<STR_LIT>' % ( url , ) ) <EOL> self . log . debug ( '<STR_LIT>' % ( data , ) ) <EOL> r = self . session . post ( url , data = json . dumps ( data ) . encode ( '<STR_LIT:utf8>' ) , <EOL> verify = self . app . config . verify_ssl , <EOL> auth = self . auth , timeout = TIMEOUT , <EOL> headers = { '<STR_LIT:Content-Type>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : self . user_agent } ) <EOL> self . log . debug ( '<STR_LIT>' % ( r . text , ) ) <EOL> ret = None <EOL> if r . text and len ( r . text ) > <NUM_LIT:4> : <EOL> try : <EOL> ret = json . loads ( r . text [ <NUM_LIT:4> : ] ) <EOL> except Exception : <EOL> self . log . exception ( "<STR_LIT>" % <EOL> ( r . text , url ) ) <EOL> return ret <EOL> def put ( self , path , data ) : <EOL> url = self . url ( path ) <EOL> self . log . debug ( '<STR_LIT>' % ( url , ) ) <EOL> self . log . debug ( '<STR_LIT>' % ( data , ) ) <EOL> r = self . session . put ( url , data = json . dumps ( data ) . encode ( '<STR_LIT:utf8>' ) , <EOL> verify = self . app . config . verify_ssl , <EOL> auth = self . auth , timeout = TIMEOUT , <EOL> headers = { '<STR_LIT:Content-Type>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : self . user_agent } ) <EOL> self . log . debug ( '<STR_LIT>' % ( r . text , ) ) <EOL> def delete ( self , path , data ) : <EOL> url = self . url ( path ) <EOL> self . log . debug ( '<STR_LIT>' % ( url , ) ) <EOL> self . log . debug ( '<STR_LIT>' % ( data , ) ) <EOL> r = self . session . delete ( url , data = json . dumps ( data ) . encode ( '<STR_LIT:utf8>' ) , <EOL> verify = self . app . config . verify_ssl , <EOL> auth = self . auth , timeout = TIMEOUT , <EOL> headers = { '<STR_LIT:Content-Type>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : self . user_agent } ) <EOL> self . log . debug ( '<STR_LIT>' % ( r . text , ) ) <EOL> def syncSubscribedProjects ( self ) : <EOL> task = SyncSubscribedProjectsTask ( LOW_PRIORITY ) <EOL> self . submitTask ( task ) <EOL> if task . wait ( ) : <EOL> for subtask in task . tasks : <EOL> subtask . wait ( ) <EOL> def pruneDatabase ( self ) : <EOL> task = PruneDatabaseTask ( self . app . config . expire_age , LOW_PRIORITY ) <EOL> self . submitTask ( task ) <EOL> if task . wait ( ) : <EOL> for subtask in task . tasks : <EOL> subtask . wait ( ) <EOL> def _syncChangeByCommit ( self , commit , priority ) : <EOL> task = None <EOL> for task in self . queue . find ( SyncChangesByCommitsTask , priority ) : <EOL> if task . addCommit ( commit ) : <EOL> return <EOL> task = SyncChangesByCommitsTask ( [ commit ] , priority ) <EOL> self . submitTask ( task ) <EOL> def setRemoteVersion ( self , version ) : <EOL> base = version . split ( '<STR_LIT:->' ) [ <NUM_LIT:0> ] <EOL> parts = base . split ( '<STR_LIT:.>' ) <EOL> major = minor = micro = <NUM_LIT:0> <EOL> if len ( parts ) > <NUM_LIT:0> : <EOL> major = int ( parts [ <NUM_LIT:0> ] ) <EOL> if len ( parts ) > <NUM_LIT:1> : <EOL> minor = int ( parts [ <NUM_LIT:1> ] ) <EOL> if len ( parts ) > <NUM_LIT:2> : <EOL> micro = int ( parts [ <NUM_LIT:2> ] ) <EOL> self . version = ( major , minor , micro ) <EOL> self . log . info ( "<STR_LIT>" % ( version , self . version ) ) <EOL> def query ( self , queries ) : <EOL> changes = [ ] <EOL> sortkey = '<STR_LIT>' <EOL> done = False <EOL> offset = <NUM_LIT:0> <EOL> while not done : <EOL> query = '<STR_LIT:&>' . join ( queries ) <EOL> q = '<STR_LIT>' % ( sortkey , query ) <EOL> self . log . debug ( '<STR_LIT>' % ( q , ) ) <EOL> responses = self . get ( q ) <EOL> if len ( queries ) == <NUM_LIT:1> : <EOL> responses = [ responses ] <EOL> done = True <EOL> for batch in responses : <EOL> changes += batch <EOL> if batch and '<STR_LIT>' in batch [ - <NUM_LIT:1> ] : <EOL> done = False <EOL> if '<STR_LIT>' in batch [ - <NUM_LIT:1> ] : <EOL> sortkey = '<STR_LIT>' % ( batch [ - <NUM_LIT:1> ] [ '<STR_LIT>' ] , ) <EOL> else : <EOL> offset += len ( batch ) <EOL> sortkey = '<STR_LIT>' % ( offset , ) <EOL> return changes </s>
