<s> """<STR_LIT>""" <EOL> import json <EOL> import logging <EOL> import time <EOL> from functools import partial <EOL> from tornado import gen , httpclient <EOL> from viewfinder . backend . base import message , util , retry , secrets <EOL> from viewfinder . backend . base . context_local import ContextLocal <EOL> class Version ( object ) : <EOL> """<STR_LIT>""" <EOL> _DELETED_MIGRATOR_COUNT = <NUM_LIT:16> <EOL> """<STR_LIT>""" <EOL> _version_classes = set ( ) <EOL> _rank_ordering = [ ] <EOL> _mutate_items = True <EOL> _allow_s3_queries = True <EOL> _migrate_retry_policy = retry . RetryPolicy ( max_tries = <NUM_LIT:5> , min_delay = <NUM_LIT:1> , <EOL> check_exception = lambda type , value , tb : True ) <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> self . rank = len ( Version . _rank_ordering ) + <NUM_LIT:1> + Version . _DELETED_MIGRATOR_COUNT <EOL> if Version . _rank_ordering : <EOL> assert self . rank > Version . _rank_ordering [ - <NUM_LIT:1> ] , '<STR_LIT>' % ( self . rank , Version . _rank_ordering [ - <NUM_LIT:1> ] ) <EOL> assert self . __class__ not in Version . _version_classes , '<STR_LIT>' % self . __class__ . __name__ <EOL> Version . _rank_ordering . append ( self . rank ) <EOL> Version . _version_classes . add ( self . __class__ ) <EOL> @ classmethod <EOL> def SetMutateItems ( cls , mutate ) : <EOL> """<STR_LIT>""" <EOL> Version . _mutate_items = mutate <EOL> @ classmethod <EOL> def SetAllowS3Queries ( cls , allow ) : <EOL> """<STR_LIT>""" <EOL> Version . _allow_s3_queries = allow <EOL> @ classmethod <EOL> def GetCurrentVersion ( cls ) : <EOL> """<STR_LIT>""" <EOL> if Version . _rank_ordering : <EOL> return Version . _rank_ordering [ - <NUM_LIT:1> ] <EOL> else : <EOL> return <NUM_LIT:0> <EOL> @ classmethod <EOL> def MaybeMigrate ( cls , client , original_item , versions , callback ) : <EOL> """<STR_LIT>""" <EOL> def _Migrate ( start_rank , mutate_item ) : <EOL> last_rank = <NUM_LIT:0> <EOL> for version in versions : <EOL> if version . rank < start_rank : <EOL> last_rank = version . rank <EOL> continue <EOL> assert version . rank > last_rank , '<STR_LIT>' % ( version . rank , last_rank ) <EOL> last_rank = version . rank <EOL> item_version = mutate_item . _version or <NUM_LIT:0> <EOL> if item_version < version . rank : <EOL> logging . debug ( '<STR_LIT>' % <EOL> ( type ( mutate_item ) . _table . name , version . __class__ . __name__ ) ) <EOL> mutate_item . _version = version . rank <EOL> transform_callback = partial ( _Migrate , last_rank + <NUM_LIT:1> ) <EOL> retry . CallWithRetryAsync ( Version . _migrate_retry_policy , <EOL> version . Transform , <EOL> client , <EOL> mutate_item , <EOL> callback = transform_callback ) <EOL> return <EOL> callback ( mutate_item if Version . _mutate_items else original_item ) <EOL> _Migrate ( <NUM_LIT:0> , original_item if Version . _mutate_items else original_item . _Clone ( ) ) <EOL> def _LogUpdate ( self , item ) : <EOL> """<STR_LIT>""" <EOL> mods = [ '<STR_LIT>' % ( n , getattr ( item , n ) ) for n in item . GetColNames ( ) if item . _IsModified ( n ) ] <EOL> if mods : <EOL> logging . info ( '<STR_LIT>' % ( type ( item ) . _table . name , item . GetKey ( ) , '<STR_LIT:U+002CU+0020>' . join ( mods ) ) ) <EOL> def Transform ( self , client , item , callback ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( ) <EOL> class TestVersion ( Version ) : <EOL> """<STR_LIT>""" <EOL> def Transform ( self , client , item , callback ) : <EOL> """<STR_LIT>""" <EOL> item = item . _Clone ( ) <EOL> if item . attr0 is None : <EOL> item . attr0 = <NUM_LIT:100> <EOL> self . _LogUpdate ( item ) <EOL> if Version . _mutate_items : <EOL> item . Update ( client , partial ( callback , item ) ) <EOL> else : <EOL> callback ( item ) <EOL> class TestVersion2 ( Version ) : <EOL> """<STR_LIT>""" <EOL> def Transform ( self , client , item , callback ) : <EOL> """<STR_LIT>""" <EOL> if item . attr1 is not None : <EOL> item . attr1 = None <EOL> self . _LogUpdate ( item ) <EOL> if Version . _mutate_items : <EOL> item . Update ( client , partial ( callback , item ) ) <EOL> else : <EOL> callback ( item ) <EOL> class AddViewpointSeq ( Version ) : <EOL> """<STR_LIT>""" <EOL> def Transform ( self , client , follower , callback ) : <EOL> from viewpoint import Viewpoint <EOL> def _OnQuery ( viewpoint ) : <EOL> with util . Barrier ( partial ( callback , follower ) ) as b : <EOL> follower . viewed_seq = <NUM_LIT:0> <EOL> self . _LogUpdate ( follower ) <EOL> if Version . _mutate_items : <EOL> follower . Update ( client , b . Callback ( ) ) <EOL> if viewpoint . update_seq is None : <EOL> viewpoint . _version = self . rank <EOL> viewpoint . update_seq = <NUM_LIT:0> <EOL> self . _LogUpdate ( viewpoint ) <EOL> if Version . _mutate_items : <EOL> viewpoint . Update ( client , b . Callback ( ) ) <EOL> Viewpoint . Query ( client , follower . viewpoint_id , None , _OnQuery ) <EOL> class AddActivitySeq ( Version ) : <EOL> """<STR_LIT>""" <EOL> def Transform ( self , client , activity , callback ) : <EOL> activity . update_seq = <NUM_LIT:0> <EOL> self . _LogUpdate ( activity ) <EOL> if Version . _mutate_items : <EOL> activity . Update ( client , partial ( callback , activity ) ) <EOL> else : <EOL> callback ( activity ) <EOL> class UpdateActivityShare ( Version ) : <EOL> """<STR_LIT>""" <EOL> def Transform ( self , client , viewpoint , callback ) : <EOL> from activity import Activity <EOL> from viewpoint import Viewpoint <EOL> def _OnQuery ( followers_activities ) : <EOL> ( follower_ids , last_key ) , activities = followers_activities <EOL> activities = [ activity for activity in activities if activity . name == '<STR_LIT>' ] <EOL> if len ( activities ) > <NUM_LIT:0> : <EOL> oldest_activity = None <EOL> for activity in activities : <EOL> if oldest_activity is None or activity . timestamp < oldest_activity . timestamp : <EOL> oldest_activity = activity <EOL> activity . name = '<STR_LIT>' <EOL> oldest_activity . name = '<STR_LIT>' <EOL> act_dict = json . loads ( activities [ - <NUM_LIT:1> ] . json ) <EOL> act_dict [ '<STR_LIT>' ] = [ f_id for f_id in follower_ids if f_id != viewpoint . user_id ] <EOL> oldest_activity . json = json . dumps ( act_dict ) <EOL> with util . Barrier ( partial ( callback , viewpoint ) ) as b : <EOL> for activity in activities : <EOL> self . _LogUpdate ( activity ) <EOL> if Version . _mutate_items : <EOL> activity . Update ( client , b . Callback ( ) ) <EOL> else : <EOL> b . Callback ( ) ( ) <EOL> else : <EOL> callback ( viewpoint ) <EOL> with util . ArrayBarrier ( _OnQuery ) as b : <EOL> Viewpoint . QueryFollowerIds ( client , viewpoint . viewpoint_id , b . Callback ( ) ) <EOL> Activity . RangeQuery ( client , viewpoint . viewpoint_id , None , None , None , b . Callback ( ) ) <EOL> class CreateMD5Hashes ( Version ) : <EOL> """<STR_LIT>""" <EOL> def Transform ( self , client , photo , callback ) : <EOL> from viewfinder . backend . storage . s3_object_store import S3ObjectStore <EOL> def _SetPhotoMD5Values ( md5_values ) : <EOL> tn_md5 , med_md5 , full_md5 , orig_md5 = md5_values <EOL> assert photo . tn_md5 == tn_md5 or photo . tn_md5 is None , photo <EOL> photo . tn_md5 = tn_md5 <EOL> assert photo . med_md5 == med_md5 or photo . med_md5 is None , photo <EOL> photo . med_md5 = med_md5 <EOL> assert photo . full_md5 == full_md5 or photo . full_md5 is None , photo <EOL> photo . full_md5 = full_md5 <EOL> assert photo . orig_md5 == orig_md5 or photo . orig_md5 is None , photo <EOL> photo . orig_md5 = orig_md5 <EOL> photo . placemark = photo . placemark <EOL> self . _LogUpdate ( photo ) <EOL> if Version . _mutate_items : <EOL> photo . Update ( client , partial ( callback , photo ) ) <EOL> else : <EOL> callback ( photo ) <EOL> def _OnFetchHead ( head_callback , response ) : <EOL> if response . code != <NUM_LIT> : <EOL> etag = response . headers [ '<STR_LIT>' ] [ <NUM_LIT:1> : - <NUM_LIT:1> ] <EOL> else : <EOL> etag = None <EOL> head_callback ( etag ) <EOL> def _SendHeadRequest ( photo_id , suffix , head_callback ) : <EOL> object_store = S3ObjectStore ( '<STR_LIT>' ) <EOL> url = object_store . GenerateUrl ( photo_id + suffix , method = '<STR_LIT>' ) <EOL> http_client = httpclient . AsyncHTTPClient ( ) <EOL> http_client . fetch ( url , method = '<STR_LIT>' , callback = partial ( _OnFetchHead , head_callback ) ) <EOL> client_data = photo . client_data <EOL> if client_data is None or '<STR_LIT>' not in client_data : <EOL> if not Version . _allow_s3_queries : <EOL> callback ( photo ) <EOL> return <EOL> with util . ArrayBarrier ( _SetPhotoMD5Values ) as b : <EOL> _SendHeadRequest ( photo . photo_id , '<STR_LIT>' , b . Callback ( ) ) <EOL> _SendHeadRequest ( photo . photo_id , '<STR_LIT>' , b . Callback ( ) ) <EOL> _SendHeadRequest ( photo . photo_id , '<STR_LIT>' , b . Callback ( ) ) <EOL> _SendHeadRequest ( photo . photo_id , '<STR_LIT>' , b . Callback ( ) ) <EOL> else : <EOL> _SetPhotoMD5Values ( ( client_data [ '<STR_LIT>' ] , client_data [ '<STR_LIT>' ] , <EOL> photo . full_md5 , photo . orig_md5 ) ) <EOL> class DisambiguateActivityIds ( Version ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> class MigrateMoreShares ( Version ) : <EOL> """<STR_LIT>""" <EOL> def Transform ( self , client , member , callback ) : <EOL> from episode import Episode <EOL> from member import Member <EOL> def _OnQueryMember ( root_episode , sharer_member ) : <EOL> assert Member . OWNED in sharer_member . labels , sharer_member <EOL> logging . info ( '<STR_LIT>' % ( sharer_member . user_id , member . user_id , root_episode . episode_id , <EOL> root_episode . viewpoint_id ) ) <EOL> if Version . _mutate_items : <EOL> Episode . _MigrateShare ( client , root_episode , sharer_member = sharer_member , <EOL> recipient_member = member , add_photo_ids = None , <EOL> callback = partial ( callback , member ) ) <EOL> else : <EOL> callback ( member ) <EOL> def _OnQueryEpisode ( root_episode ) : <EOL> Member . Query ( client , root_episode . user_id , root_episode . episode_id , None , <EOL> partial ( _OnQueryMember , root_episode ) ) <EOL> if member . sharing_user_id is None and Member . OWNED not in member . labels : <EOL> assert list ( member . labels ) == [ Member . SHARED ] , member <EOL> Episode . Query ( client , member . episode_id , None , _OnQueryEpisode ) <EOL> else : <EOL> callback ( member ) <EOL> class AddUserType ( Version ) : <EOL> """<STR_LIT>""" <EOL> def Transform ( self , client , user , callback ) : <EOL> from user import User <EOL> user . labels = [ User . ACTIVATED ] <EOL> user . op_id_seq = None <EOL> self . _LogUpdate ( user ) <EOL> if Version . _mutate_items : <EOL> user . Update ( client , partial ( callback , user ) ) <EOL> else : <EOL> callback ( user ) <EOL> class DisambiguateActivityIds2 ( Version ) : <EOL> """<STR_LIT>""" <EOL> _unique_activity_ids = set ( ) <EOL> def Transform ( self , client , activity , callback ) : <EOL> from activity import Activity <EOL> from asset_id import AssetIdUniquifier <EOL> from device import Device <EOL> def _OnUpdate ( new_activity ) : <EOL> """<STR_LIT>""" <EOL> if Version . _mutate_items : <EOL> activity . Delete ( client , partial ( callback , new_activity ) ) <EOL> else : <EOL> callback ( new_activity ) <EOL> timestamp , device_id , uniquifier = Activity . DeconstructActivityId ( activity . activity_id ) <EOL> if device_id == Device . SYSTEM : <EOL> if activity . activity_id in DisambiguateActivityIds2 . _unique_activity_ids : <EOL> assert uniquifier . server_id is None , ( activity , uniquifier ) <EOL> new_uniquifier = AssetIdUniquifier ( uniquifier . client_id , activity . viewpoint_id ) <EOL> new_activity_id = Activity . ConstructActivityId ( timestamp , device_id , new_uniquifier ) <EOL> new_activity_dict = activity . _asdict ( ) <EOL> new_activity_dict [ '<STR_LIT>' ] = new_activity_id <EOL> new_activity = Activity . CreateFromKeywords ( ** new_activity_dict ) <EOL> logging . info ( '<STR_LIT>' % <EOL> ( activity , activity . activity_id , timestamp , device_id , uniquifier . client_id , <EOL> new_activity_id , timestamp , device_id , new_uniquifier . client_id , new_uniquifier . server_id ) ) <EOL> if Version . _mutate_items : <EOL> new_activity . Update ( client , partial ( _OnUpdate , new_activity ) ) <EOL> else : <EOL> _OnUpdate ( new_activity ) <EOL> else : <EOL> DisambiguateActivityIds2 . _unique_activity_ids . add ( activity . activity_id ) <EOL> callback ( activity ) <EOL> else : <EOL> assert activity . activity_id not in DisambiguateActivityIds2 . _unique_activity_ids , activity <EOL> callback ( activity ) <EOL> class CopyUpdateSeq ( Version ) : <EOL> """<STR_LIT>""" <EOL> def Transform ( self , client , notification , callback ) : <EOL> from activity import Activity <EOL> from asset_id import AssetIdUniquifier <EOL> def _DoUpdate ( ) : <EOL> self . _LogUpdate ( notification ) <EOL> if Version . _mutate_items : <EOL> notification . Update ( client , partial ( callback , notification ) ) <EOL> else : <EOL> callback ( notification ) <EOL> def _OnQueryNewActivity ( activity ) : <EOL> if activity is None : <EOL> logging . warning ( '<STR_LIT>' , notification ) <EOL> notification . update_seq = <NUM_LIT:0> <EOL> self . _LogUpdate ( notification ) <EOL> _DoUpdate ( ) <EOL> else : <EOL> _OnQueryActivity ( activity ) <EOL> def _OnQueryActivity ( activity ) : <EOL> if activity is None : <EOL> timestamp , device_id , uniquifier = Activity . DeconstructActivityId ( notification . activity_id ) <EOL> new_uniquifier = AssetIdUniquifier ( uniquifier . client_id , notification . viewpoint_id ) <EOL> new_activity_id = Activity . ConstructActivityId ( timestamp , device_id , new_uniquifier ) <EOL> Activity . Query ( client , notification . viewpoint_id , new_activity_id , None , <EOL> _OnQueryNewActivity , must_exist = False ) <EOL> return <EOL> notification . activity_id = activity . activity_id <EOL> notification . update_seq = activity . update_seq <EOL> _DoUpdate ( ) <EOL> if notification . activity_id is None : <EOL> callback ( notification ) <EOL> else : <EOL> Activity . Query ( client , notification . viewpoint_id , notification . activity_id , None , <EOL> _OnQueryActivity , must_exist = False ) <EOL> class AddUserSigningKey ( Version ) : <EOL> """<STR_LIT>""" <EOL> def Transform ( self , client , user , callback ) : <EOL> user . signing_key = secrets . CreateSigningKeyset ( '<STR_LIT>' ) <EOL> self . _LogUpdate ( user ) <EOL> if Version . _mutate_items : <EOL> user . Update ( client , partial ( callback , user ) ) <EOL> else : <EOL> callback ( user ) <EOL> class UpdateUserType ( Version ) : <EOL> """<STR_LIT>""" <EOL> def Transform ( self , client , user , callback ) : <EOL> from user import User <EOL> if '<STR_LIT>' in user . labels : <EOL> labels = user . labels . combine ( ) <EOL> labels . remove ( '<STR_LIT>' ) <EOL> labels . add ( User . REGISTERED ) <EOL> user . labels = labels <EOL> self . _LogUpdate ( user ) <EOL> if Version . _mutate_items : <EOL> user . Update ( client , partial ( callback , user ) ) <EOL> else : <EOL> callback ( user ) <EOL> class AddFollowed ( Version ) : <EOL> """<STR_LIT>""" <EOL> def Transform ( self , client , viewpoint , callback ) : <EOL> from activity import Activity <EOL> from followed import Followed <EOL> from viewpoint import Viewpoint <EOL> def _OnQuery ( activities_followers ) : <EOL> activities , ( follower_ids , last_key ) = activities_followers <EOL> with util . Barrier ( partial ( callback , viewpoint ) ) as b : <EOL> old_timestamp = viewpoint . last_updated <EOL> if len ( activities ) > <NUM_LIT:0> : <EOL> new_timestamp = max ( a . timestamp for a in activities ) <EOL> else : <EOL> new_timestamp = <NUM_LIT:0> <EOL> viewpoint . last_updated = new_timestamp <EOL> self . _LogUpdate ( viewpoint ) <EOL> for follower_id in follower_ids : <EOL> logging . info ( '<STR_LIT>' % <EOL> ( follower_id , viewpoint . viewpoint_id , Followed . _TruncateToDay ( old_timestamp ) , <EOL> Followed . _TruncateToDay ( new_timestamp ) ) ) <EOL> if Version . _mutate_items : <EOL> Followed . UpdateDateUpdated ( client , follower_id , viewpoint . viewpoint_id , <EOL> old_timestamp , new_timestamp , b . Callback ( ) ) <EOL> with util . ArrayBarrier ( _OnQuery ) as b : <EOL> Activity . RangeQuery ( client , viewpoint . viewpoint_id , None , None , None , b . Callback ( ) ) <EOL> Viewpoint . QueryFollowerIds ( client , viewpoint . viewpoint_id , b . Callback ( ) ) <EOL> class InTransformContext ( ContextLocal ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> class UpdateDevices ( Version ) : <EOL> """<STR_LIT>""" <EOL> def Transform ( self , client , device , callback ) : <EOL> from tornado . web import stack_context <EOL> from viewfinder . backend . db . device import Device <EOL> def _DoUpdate ( ) : <EOL> self . _LogUpdate ( device ) <EOL> if Version . _mutate_items : <EOL> device . Update ( client , partial ( callback , device ) ) <EOL> else : <EOL> callback ( device ) <EOL> def _OnQueryByPushToken ( other_devices ) : <EOL> for other in other_devices : <EOL> if other . push_token == device . push_token and other . device_id > device . device_id : <EOL> device . alert_user_id = None <EOL> device . push_token = None <EOL> break <EOL> _DoUpdate ( ) <EOL> if InTransformContext . current ( ) is None : <EOL> with stack_context . StackContext ( InTransformContext ( ) ) : <EOL> device . timestamp = device . last_access or time . time ( ) <EOL> if device . push_token is not None : <EOL> if time . time ( ) < device . last_access + Device . _PUSH_EXPIRATION : <EOL> device . alert_user_id = device . user_id <EOL> query_expr = ( '<STR_LIT>' , { '<STR_LIT:t>' : device . push_token } ) <EOL> Device . IndexQuery ( client , query_expr , None , _OnQueryByPushToken ) <EOL> else : <EOL> _DoUpdate ( ) <EOL> else : <EOL> callback ( device ) <EOL> class FillFileSizes ( Version ) : <EOL> """<STR_LIT>""" <EOL> def Transform ( self , client , photo , callback ) : <EOL> from viewfinder . backend . storage . s3_object_store import S3ObjectStore <EOL> def _SetPhotoSizeValues ( size_values ) : <EOL> tn_size , med_size , full_size , orig_size = size_values <EOL> assert photo . tn_size == tn_size or photo . tn_size is None , photo <EOL> photo . tn_size = tn_size <EOL> assert photo . med_size == med_size or photo . med_size is None , photo <EOL> photo . med_size = med_size <EOL> assert photo . full_size == full_size or photo . full_size is None , photo <EOL> photo . full_size = full_size <EOL> assert photo . orig_size == orig_size or photo . orig_size is None , photo <EOL> photo . orig_size = orig_size <EOL> self . _LogUpdate ( photo ) <EOL> if Version . _mutate_items : <EOL> photo . Update ( client , partial ( callback , photo ) ) <EOL> else : <EOL> callback ( photo ) <EOL> def _OnListKeys ( result ) : <EOL> def _GetSizeOrNone ( files , suffix ) : <EOL> fname = photo . photo_id + '<STR_LIT:.>' + suffix <EOL> if fname in files : <EOL> return int ( files [ fname ] [ '<STR_LIT>' ] ) <EOL> else : <EOL> return None <EOL> if len ( result ) > <NUM_LIT:0> : <EOL> _SetPhotoSizeValues ( ( _GetSizeOrNone ( result , '<STR_LIT:t>' ) , _GetSizeOrNone ( result , '<STR_LIT:m>' ) , <EOL> _GetSizeOrNone ( result , '<STR_LIT:f>' ) , _GetSizeOrNone ( result , '<STR_LIT:o>' ) ) ) <EOL> else : <EOL> logging . info ( '<STR_LIT>' % photo . photo_id ) <EOL> callback ( photo ) <EOL> def _ListPhotoMetadata ( ) : <EOL> object_store = S3ObjectStore ( '<STR_LIT>' ) <EOL> object_store . ListKeyMetadata ( _OnListKeys , prefix = photo . photo_id + '<STR_LIT:.>' , <EOL> fields = [ '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> client_data = photo . client_data <EOL> if client_data is None or '<STR_LIT>' not in client_data : <EOL> if not Version . _allow_s3_queries : <EOL> callback ( photo ) <EOL> return <EOL> _ListPhotoMetadata ( ) <EOL> else : <EOL> _SetPhotoSizeValues ( ( int ( client_data [ '<STR_LIT>' ] ) , int ( client_data [ '<STR_LIT>' ] ) , <EOL> int ( client_data [ '<STR_LIT>' ] ) , int ( client_data [ '<STR_LIT>' ] ) ) ) <EOL> class RepairFacebookContacts ( Version ) : <EOL> @ gen . engine <EOL> def Transform ( self , client , identity , callback ) : <EOL> from tornado . web import stack_context <EOL> from operation import Operation <EOL> from user import User <EOL> if InTransformContext . current ( ) is None : <EOL> with stack_context . StackContext ( InTransformContext ( ) ) : <EOL> if identity . authority == '<STR_LIT>' and identity . user_id is not None : <EOL> user = yield gen . Task ( User . Query , client , identity . user_id , None ) <EOL> yield gen . Task ( Operation . CreateAndExecute , client , identity . user_id , user . webapp_dev_id , <EOL> '<STR_LIT>' , <EOL> { '<STR_LIT>' : { '<STR_LIT>' : True } , '<STR_LIT:key>' : identity . key , '<STR_LIT>' : identity . user_id } ) <EOL> callback ( identity ) <EOL> class AddAccountSettings ( Version ) : <EOL> """<STR_LIT>""" <EOL> @ gen . engine <EOL> def Transform ( self , client , user , callback ) : <EOL> from settings import AccountSettings <EOL> settings = yield gen . Task ( AccountSettings . QueryByUser , client , user . user_id , None , must_exist = False ) <EOL> if settings is None : <EOL> logging . info ( '<STR_LIT>' , user . user_id ) <EOL> settings = AccountSettings . CreateForUser ( user . user_id , <EOL> email_alerts = AccountSettings . EMAIL_NONE , <EOL> push_alerts = AccountSettings . PUSH_ALL ) <EOL> if Version . _mutate_items : <EOL> yield gen . Task ( settings . Update , client ) <EOL> callback ( user ) <EOL> class SplitUserNames ( Version ) : <EOL> """<STR_LIT>""" <EOL> @ gen . engine <EOL> def Transform ( self , client , user , callback ) : <EOL> if user . name and '<STR_LIT:@>' not in user . name and not user . given_name and not user . family_name : <EOL> match = message . FULL_NAME_RE . match ( user . name ) <EOL> if match is not None : <EOL> user . given_name = match . group ( <NUM_LIT:1> ) <EOL> if match . group ( <NUM_LIT:2> ) : <EOL> user . family_name = match . group ( <NUM_LIT:2> ) <EOL> self . _LogUpdate ( user ) <EOL> if Version . _mutate_items : <EOL> yield gen . Task ( user . Update , client ) <EOL> callback ( user ) <EOL> class ExtractAssetKeys ( Version ) : <EOL> @ gen . engine <EOL> def Transform ( self , client , photo , callback ) : <EOL> from device_photo import DevicePhoto <EOL> from photo import Photo <EOL> if photo . client_data is None : <EOL> callback ( photo ) <EOL> return <EOL> if photo . client_data . get ( '<STR_LIT>' ) : <EOL> _ , device_id , _ = Photo . DeconstructPhotoId ( photo . photo_id ) <EOL> existing = yield gen . Task ( DevicePhoto . Query , client , device_id , photo . photo_id , None , must_exist = False ) <EOL> if existing is None : <EOL> logging . info ( '<STR_LIT>' , photo . photo_id , device_id ) <EOL> device_photo = DevicePhoto . CreateFromKeywords ( photo_id = photo . photo_id , <EOL> device_id = device_id , <EOL> asset_keys = [ photo . client_data [ '<STR_LIT>' ] ] ) <EOL> else : <EOL> logging . info ( '<STR_LIT>' , photo . photo_id , device_id ) <EOL> device_photo = None <EOL> if device_photo is not None : <EOL> self . _LogUpdate ( device_photo ) <EOL> photo . client_data = None <EOL> self . _LogUpdate ( photo ) <EOL> if Version . _mutate_items : <EOL> if device_photo is not None : <EOL> yield gen . Task ( device_photo . Update , client ) <EOL> yield gen . Task ( photo . Update , client ) <EOL> callback ( photo ) <EOL> class MyOnlyFriend ( Version ) : <EOL> """<STR_LIT>""" <EOL> @ gen . engine <EOL> def Transform ( self , client , user , callback ) : <EOL> from friend import Friend <EOL> friend = Friend . CreateFromKeywords ( user_id = user . user_id , friend_id = user . user_id ) <EOL> logging . info ( '<STR_LIT>' , user . user_id ) <EOL> if Version . _mutate_items : <EOL> yield gen . Task ( friend . Update , client ) <EOL> callback ( user ) <EOL> class EraseFriendNames ( Version ) : <EOL> """<STR_LIT>""" <EOL> @ gen . engine <EOL> def Transform ( self , client , friend , callback ) : <EOL> friend . name = None <EOL> self . _LogUpdate ( friend ) <EOL> if Version . _mutate_items : <EOL> yield gen . Task ( friend . Update , client ) <EOL> callback ( friend ) <EOL> class MoveDevicePhoto ( Version ) : <EOL> _device_to_user_cache = { } <EOL> @ gen . engine <EOL> def Transform ( self , client , device_photo , callback ) : <EOL> from device import Device <EOL> from user_photo import UserPhoto <EOL> device_id = device_photo . device_id <EOL> if device_id not in MoveDevicePhoto . _device_to_user_cache : <EOL> query_expr = ( '<STR_LIT>' , { '<STR_LIT:t>' : device_id } ) <EOL> devices = yield gen . Task ( Device . IndexQuery , client , query_expr , None ) <EOL> assert len ( devices ) == <NUM_LIT:1> <EOL> MoveDevicePhoto . _device_to_user_cache [ device_id ] = devices [ <NUM_LIT:0> ] . user_id <EOL> user_id = MoveDevicePhoto . _device_to_user_cache [ device_id ] <EOL> existing = yield gen . Task ( UserPhoto . Query , client , user_id , device_photo . photo_id , None , must_exist = False ) <EOL> if existing is None : <EOL> logging . info ( '<STR_LIT>' , device_photo . photo_id , device_id , user_id ) <EOL> user_photo = UserPhoto . CreateFromKeywords ( photo_id = device_photo . photo_id , <EOL> user_id = user_id , <EOL> asset_keys = device_photo . asset_keys ) <EOL> else : <EOL> logging . info ( '<STR_LIT>' , device_photo . photo_id , device_id , user_id ) <EOL> user_photo = None <EOL> if user_photo is not None : <EOL> self . _LogUpdate ( user_photo ) <EOL> if Version . _mutate_items and user_photo is not None : <EOL> yield gen . Task ( user_photo . Update , client ) <EOL> callback ( device_photo ) <EOL> class SetCoverPhoto ( Version ) : <EOL> """<STR_LIT>""" <EOL> @ gen . engine <EOL> def Transform ( self , client , viewpoint , callback ) : <EOL> if not viewpoint . IsDefault ( ) and viewpoint . cover_photo == None : <EOL> viewpoint . cover_photo = yield gen . Task ( viewpoint . SelectCoverPhotoUsingOriginalAlgorithm , client ) <EOL> if Version . _mutate_items : <EOL> yield gen . Task ( viewpoint . Update , client ) <EOL> callback ( viewpoint ) <EOL> class RemoveAssetUrls ( Version ) : <EOL> """<STR_LIT>""" <EOL> @ gen . engine <EOL> def Transform ( self , client , user_photo , callback ) : <EOL> from user_photo import UserPhoto <EOL> asset_keys = user_photo . asset_keys . combine ( ) <EOL> changed = False <EOL> for asset_key in list ( asset_keys ) : <EOL> fingerprint = UserPhoto . AssetKeyToFingerprint ( asset_key ) <EOL> if fingerprint is None : <EOL> asset_keys . remove ( asset_key ) <EOL> changed = True <EOL> elif asset_key != fingerprint : <EOL> asset_keys . remove ( asset_key ) <EOL> asset_keys . add ( fingerprint ) <EOL> changed = True <EOL> if changed : <EOL> user_photo . asset_keys = asset_keys <EOL> self . _LogUpdate ( user_photo ) <EOL> if Version . _mutate_items : <EOL> yield gen . Task ( user_photo . Update , client ) <EOL> callback ( user_photo ) <EOL> class RemoveContactUserId ( Version ) : <EOL> """<STR_LIT>""" <EOL> @ gen . engine <EOL> def Transform ( self , client , contact , callback ) : <EOL> from contact import Contact <EOL> contact . contact_user_id = None <EOL> self . _LogUpdate ( contact ) <EOL> if Version . _mutate_items : <EOL> yield gen . Task ( contact . Update , client ) <EOL> callback ( contact ) <EOL> class UploadContactsSupport ( Version ) : <EOL> """<STR_LIT>""" <EOL> @ gen . engine <EOL> def Transform ( self , client , contact , callback ) : <EOL> from viewfinder . backend . db . contact import Contact <EOL> from viewfinder . backend . db . identity import Identity <EOL> contact_dict = contact . _asdict ( ) <EOL> assert contact . identity . startswith ( '<STR_LIT>' ) or contact . identity . startswith ( '<STR_LIT>' ) , contact <EOL> contact_dict [ '<STR_LIT>' ] = Contact . GMAIL if contact . identity . startswith ( '<STR_LIT>' ) else Contact . FACEBOOK <EOL> contact_dict [ '<STR_LIT>' ] = [ ( Identity . Canonicalize ( contact . identity ) , None ) ] <EOL> contact_dict [ '<STR_LIT>' ] = util . GetCurrentTimestamp ( ) <EOL> contact_dict . pop ( '<STR_LIT>' ) <EOL> contact_dict . pop ( '<STR_LIT>' ) <EOL> new_contact = Contact . CreateFromKeywords ( ** contact_dict ) <EOL> self . _LogUpdate ( new_contact ) <EOL> if Version . _mutate_items : <EOL> yield gen . Task ( new_contact . Update , client ) <EOL> yield gen . Task ( contact . Delete , client ) <EOL> callback ( new_contact ) <EOL> class RenameUserPostRemoved ( Version ) : <EOL> """<STR_LIT>""" <EOL> @ gen . engine <EOL> def Transform ( self , client , user_post , callback ) : <EOL> from user_post import UserPost <EOL> if UserPost . REMOVED in user_post . labels : <EOL> labels = user_post . labels . combine ( ) <EOL> labels . remove ( UserPost . REMOVED ) <EOL> labels . add ( UserPost . HIDDEN ) <EOL> user_post . labels = labels <EOL> self . _LogUpdate ( user_post ) <EOL> if Version . _mutate_items : <EOL> yield gen . Task ( user_post . Update , client ) <EOL> callback ( user_post ) <EOL> class AddRemovedToPost ( Version ) : <EOL> """<STR_LIT>""" <EOL> @ gen . engine <EOL> def Transform ( self , client , post , callback ) : <EOL> from post import Post <EOL> labels = post . labels . combine ( ) <EOL> if Post . UNSHARED in labels : <EOL> labels . add ( Post . REMOVED ) <EOL> post . labels = labels <EOL> self . _LogUpdate ( post ) <EOL> if Version . _mutate_items : <EOL> yield gen . Task ( post . Update , client ) <EOL> callback ( post ) <EOL> class RemoveHiddenPosts ( Version ) : <EOL> """<STR_LIT>""" <EOL> @ gen . engine <EOL> def Transform ( self , client , user_post , callback ) : <EOL> from episode import Episode <EOL> from post import Post <EOL> from user_post import UserPost <EOL> from viewpoint import Viewpoint <EOL> if UserPost . REMOVED in user_post . labels : <EOL> episode_id , photo_id = Post . DeconstructPostId ( user_post . post_id ) <EOL> episode = yield gen . Task ( Episode . Query , client , episode_id , None ) <EOL> viewpoint = yield gen . Task ( Viewpoint . Query , client , episode . viewpoint_id , None ) <EOL> if viewpoint . IsDefault ( ) : <EOL> post = yield gen . Task ( Post . Query , client , episode_id , photo_id , None ) <EOL> post . labels . add ( Post . REMOVED ) <EOL> logging . info ( '<STR_LIT>' , user_post . post_id , episode_id , photo_id ) <EOL> if Version . _mutate_items : <EOL> yield gen . Task ( post . Update , client ) <EOL> yield gen . Task ( user_post . Delete , client ) <EOL> callback ( user_post ) <EOL> class ReactivateAlertUser ( Version ) : <EOL> """<STR_LIT>""" <EOL> @ gen . engine <EOL> def Transform ( self , client , device , callback ) : <EOL> from device import Device <EOL> from user import User <EOL> if device . push_token is not None and device . alert_user_id is None : <EOL> user = yield gen . Task ( User . Query , client , device . user_id , None ) <EOL> if not user . IsTerminated ( ) : <EOL> device . alert_user_id = device . user_id <EOL> if Version . _mutate_items : <EOL> yield gen . Task ( device . Update , client ) <EOL> callback ( device ) <EOL> class RemoveContactIdentity ( Version ) : <EOL> """<STR_LIT>""" <EOL> @ gen . engine <EOL> def Transform ( self , client , contact , callback ) : <EOL> contact . identity = None <EOL> self . _LogUpdate ( contact ) <EOL> if Version . _mutate_items : <EOL> yield gen . Task ( contact . Update , client ) <EOL> callback ( contact ) <EOL> class RepairWelcomeConvos ( Version ) : <EOL> """<STR_LIT>""" <EOL> @ gen . engine <EOL> def Transform ( self , client , viewpoint , callback ) : <EOL> from viewpoint import Viewpoint <EOL> from viewfinder . backend . www import system_users <EOL> if viewpoint . type == Viewpoint . SYSTEM : <EOL> episodes , _ = yield gen . Task ( Viewpoint . QueryEpisodes , client , viewpoint . viewpoint_id ) <EOL> for episode in episodes : <EOL> if episode . parent_ep_id == '<STR_LIT>' : <EOL> assert viewpoint . cover_photo [ '<STR_LIT>' ] == episode . parent_ep_id or episode . episode_id , episode <EOL> assert viewpoint . cover_photo [ '<STR_LIT>' ] == '<STR_LIT>' , viewpoint . cover_photo <EOL> viewpoint . cover_photo = { '<STR_LIT>' : episode . episode_id , <EOL> '<STR_LIT>' : '<STR_LIT>' } <EOL> self . _LogUpdate ( viewpoint ) <EOL> if Version . _mutate_items : <EOL> yield gen . Task ( viewpoint . Update , client ) <EOL> callback ( viewpoint ) <EOL> class RepairWelcomeConvos2 ( Version ) : <EOL> """<STR_LIT>""" <EOL> @ gen . engine <EOL> def Transform ( self , client , viewpoint , callback ) : <EOL> from viewpoint import Viewpoint <EOL> from viewfinder . backend . www import system_users <EOL> if viewpoint . type == Viewpoint . SYSTEM : <EOL> episodes , _ = yield gen . Task ( Viewpoint . QueryEpisodes , client , viewpoint . viewpoint_id ) <EOL> if len ( episodes ) == <NUM_LIT:0> : <EOL> viewpoint . cover_photo = None <EOL> self . _LogUpdate ( viewpoint ) <EOL> if Version . _mutate_items : <EOL> yield gen . Task ( viewpoint . Update , client ) <EOL> activities , _ = yield gen . Task ( Viewpoint . QueryActivities , client , viewpoint . viewpoint_id ) <EOL> for activity in activities : <EOL> if activity . name == '<STR_LIT>' : <EOL> logging . info ( '<STR_LIT>' % activity . activity_id ) <EOL> if Version . _mutate_items : <EOL> yield gen . Task ( activity . Delete , client ) <EOL> comments , _ = yield gen . Task ( Viewpoint . QueryComments , client , viewpoint . viewpoint_id ) <EOL> for comment in comments : <EOL> if comment . asset_id is not None : <EOL> comment . asset_id = None <EOL> self . _LogUpdate ( comment ) <EOL> if Version . _mutate_items : <EOL> yield gen . Task ( comment . Update , client ) <EOL> callback ( viewpoint ) <EOL> TEST_VERSION = TestVersion ( ) <EOL> TEST_VERSION2 = TestVersion2 ( ) <EOL> ADD_VIEWPOINT_SEQ = AddViewpointSeq ( ) <EOL> ADD_ACTIVITY_SEQ = AddActivitySeq ( ) <EOL> UPDATE_ACTIVITY_SHARE = UpdateActivityShare ( ) <EOL> CREATE_MD5_HASHES = CreateMD5Hashes ( ) <EOL> DISAMBIGUATE_ACTIVITY_IDS = DisambiguateActivityIds ( ) <EOL> MIGRATE_MORE_SHARES = MigrateMoreShares ( ) <EOL> ADD_USER_TYPE = AddUserType ( ) <EOL> DISAMBIGUATE_ACTIVITY_IDS_2 = DisambiguateActivityIds2 ( ) <EOL> COPY_UPDATE_SEQ = CopyUpdateSeq ( ) <EOL> ADD_USER_SIGNING_KEY = AddUserSigningKey ( ) <EOL> UPDATE_USER_TYPE = UpdateUserType ( ) <EOL> ADD_FOLLOWED = AddFollowed ( ) <EOL> UPDATE_DEVICES = UpdateDevices ( ) <EOL> FILL_FILE_SIZES = FillFileSizes ( ) <EOL> REPAIR_FACEBOOK_CONTACTS = RepairFacebookContacts ( ) <EOL> ADD_ACCOUNT_SETTINGS = AddAccountSettings ( ) <EOL> SPLIT_USER_NAMES = SplitUserNames ( ) <EOL> EXTRACT_ASSET_KEYS = ExtractAssetKeys ( ) <EOL> MY_ONLY_FRIEND = MyOnlyFriend ( ) <EOL> ERASE_FRIEND_NAMES = EraseFriendNames ( ) <EOL> MOVE_DEVICE_PHOTO = MoveDevicePhoto ( ) <EOL> SET_COVER_PHOTO = SetCoverPhoto ( ) <EOL> REMOVE_ASSET_URLS = RemoveAssetUrls ( ) <EOL> REMOVE_CONTACT_USER_ID = RemoveContactUserId ( ) <EOL> UPLOAD_CONTACTS_SUPPORT = UploadContactsSupport ( ) <EOL> RENAME_USER_POST_REMOVED = RenameUserPostRemoved ( ) <EOL> ADD_REMOVED_TO_POST = AddRemovedToPost ( ) <EOL> REMOVE_HIDDEN_POSTS = RemoveHiddenPosts ( ) <EOL> REACTIVATE_ALERT_USER = ReactivateAlertUser ( ) <EOL> REMOVE_CONTACT_IDENTITY = RemoveContactIdentity ( ) <EOL> REPAIR_WELCOME_CONVOS = RepairWelcomeConvos ( ) <EOL> REPAIR_WELCOME_CONVOS_2 = RepairWelcomeConvos2 ( ) </s>
