<s> from okscraper . base import BaseScraper <EOL> from okscraper . sources import BaseSource <EOL> from okscraper . storages import BaseStorage <EOL> from events . models import Event <EOL> from persons . models import Person <EOL> from urlparse import urlparse , parse_qs <EOL> from django . conf import settings <EOL> from urllib import quote <EOL> from django . db . models import Max <EOL> import urllib2 <EOL> import json <EOL> import dateutil . parser <EOL> class PersonsEventsScraper ( BaseScraper ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> super ( PersonsEventsScraper , self ) . __init__ ( self ) <EOL> self . source = BaseSource ( ) <EOL> self . storage = BaseStorage ( ) <EOL> def _get_google_cal_page ( self , calendar_id , sync_token , page_token = None ) : <EOL> api_key = settings . GOOGLE_CALENDAR_API_KEY <EOL> if page_token is not None : <EOL> param = '<STR_LIT>' % quote ( page_token ) <EOL> else : <EOL> param = '<STR_LIT>' % quote ( sync_token ) if sync_token is not None else '<STR_LIT>' <EOL> url = '<STR_LIT>' % ( quote ( calendar_id ) , param , quote ( api_key ) ) <EOL> response = urllib2 . urlopen ( url ) <EOL> data = json . load ( response ) <EOL> self . _getLogger ( ) . debug ( data ) <EOL> res = { } <EOL> res [ '<STR_LIT>' ] = data [ '<STR_LIT>' ] if '<STR_LIT>' in data else None <EOL> res [ '<STR_LIT>' ] = data [ '<STR_LIT>' ] if '<STR_LIT>' in data else None <EOL> res [ '<STR_LIT>' ] = data [ '<STR_LIT>' ] if '<STR_LIT>' in data else [ ] <EOL> return res <EOL> def _process_items ( self , person , items ) : <EOL> self . _getLogger ( ) . info ( '<STR_LIT>' % len ( items ) ) <EOL> for item in items : <EOL> status = item . get ( '<STR_LIT:status>' , None ) <EOL> icaluid = item . get ( '<STR_LIT>' , None ) <EOL> start_date = dateutil . parser . parse ( item [ '<STR_LIT:start>' ] [ '<STR_LIT>' ] ) if '<STR_LIT:start>' in item and '<STR_LIT>' in item [ '<STR_LIT:start>' ] else None <EOL> end_date = dateutil . parser . parse ( item [ '<STR_LIT:end>' ] [ '<STR_LIT>' ] ) if '<STR_LIT:end>' in item and '<STR_LIT>' in item [ '<STR_LIT:end>' ] else None <EOL> if not all ( [ start_date , end_date , status , icaluid ] ) : <EOL> self . _getLogger ( ) . error ( '<STR_LIT>' ) <EOL> elif status == '<STR_LIT>' : <EOL> res = Event . objects . filter ( icaluid = icaluid ) <EOL> if res . count ( ) == <NUM_LIT:1> : <EOL> self . _getLogger ( ) . info ( '<STR_LIT>' ) <EOL> res = res [ <NUM_LIT:0> ] <EOL> res . cancelled = True <EOL> res . save ( ) <EOL> else : <EOL> self . _getLogger ( ) . info ( '<STR_LIT>' ) <EOL> else : <EOL> start_date = start_date . replace ( tzinfo = None ) <EOL> end_date = end_date . replace ( tzinfo = None ) <EOL> update_date = dateutil . parser . parse ( item [ '<STR_LIT>' ] ) . replace ( tzinfo = None ) if '<STR_LIT>' in item else None <EOL> link = item [ '<STR_LIT>' ] if '<STR_LIT>' in item else None <EOL> summary = item [ '<STR_LIT>' ] if '<STR_LIT>' in item else None <EOL> description = item [ '<STR_LIT:description>' ] if '<STR_LIT:description>' in item else None <EOL> colorId = item [ '<STR_LIT>' ] if '<STR_LIT>' in item else None <EOL> data = unicode ( item ) <EOL> kwargs = { <EOL> '<STR_LIT>' : start_date , '<STR_LIT>' : end_date , '<STR_LIT>' : link , '<STR_LIT>' : summary , '<STR_LIT>' : description , <EOL> '<STR_LIT>' : update_date , <EOL> } <EOL> res = Event . objects . filter ( icaluid = icaluid ) <EOL> event , created = Event . objects . get_or_create ( icaluid = icaluid , defaults = kwargs ) <EOL> event . who . add ( person ) <EOL> if created : <EOL> self . _getLogger ( ) . info ( '<STR_LIT>' ) <EOL> else : <EOL> self . _getLogger ( ) . info ( '<STR_LIT>' ) <EOL> def _scrape ( self ) : <EOL> for person in Person . objects . filter ( calendar_url__isnull = False ) : <EOL> qs = parse_qs ( urlparse ( person . calendar_url ) . query ) <EOL> self . _getLogger ( ) . info ( '<STR_LIT>' % person . id ) <EOL> if '<STR_LIT:src>' not in qs : <EOL> self . _getLogger ( ) . warn ( '<STR_LIT>' % ( person . id , person . calendar_url ) ) <EOL> else : <EOL> calendar_id = qs [ '<STR_LIT:src>' ] [ <NUM_LIT:0> ] <EOL> self . _getLogger ( ) . debug ( '<STR_LIT>' % calendar_id ) <EOL> res = self . _get_google_cal_page ( calendar_id , person . calendar_sync_token ) <EOL> self . _process_items ( person , res [ '<STR_LIT>' ] ) <EOL> while res [ '<STR_LIT>' ] is not None : <EOL> self . _getLogger ( ) . debug ( '<STR_LIT>' ) <EOL> res = self . _get_google_cal_page ( calendar_id , None , res [ '<STR_LIT>' ] ) <EOL> self . _process_items ( person , res [ '<STR_LIT>' ] ) <EOL> if res [ '<STR_LIT>' ] is None : <EOL> self . _getLogger ( ) . error ( '<STR_LIT>' ) <EOL> else : <EOL> person . calendar_sync_token = res [ '<STR_LIT>' ] <EOL> person . save ( ) </s>
