<s> from django . core . management import setup_environ <EOL> import settings <EOL> setup_environ ( settings ) <EOL> from django . db . models . loading import cache as model_cache <EOL> model_cache . _populate ( ) <EOL> import facebook <EOL> from django . db import connection <EOL> from django . db . models import Q <EOL> import settings <EOL> import urllib2 <EOL> import urlparse <EOL> from fabric . api import local <EOL> from org . models import Org <EOL> from miner . web . crawler import WebCrawler <EOL> from miner . jobs . org_social_media import OrgSocialMediaMapper , reducer <EOL> class FacebookMapper ( OrgSocialMediaMapper ) : <EOL> last_updated_column = '<STR_LIT>' <EOL> def get_api ( self ) : <EOL> self . api = facebook . GraphAPI ( settings . FACEBOOK_ACCESS_TOKEN ) <EOL> def has_links ( self , post ) : <EOL> return post [ '<STR_LIT:type>' ] == '<STR_LIT>' <EOL> def extract_links ( self , post ) : <EOL> return [ post [ '<STR_LIT>' ] . encode ( '<STR_LIT:utf-8>' ) ] <EOL> def default_text ( self , post ) : <EOL> return '<STR_LIT:U+0020>' . join ( [ post . get ( '<STR_LIT:name>' , '<STR_LIT>' ) , post . get ( '<STR_LIT:message>' , '<STR_LIT>' ) , post . get ( '<STR_LIT:description>' , '<STR_LIT>' ) ] ) <EOL> def get_batch ( self , org , limit = <NUM_LIT:200> , page = <NUM_LIT:1> , since = None ) : <EOL> params = dict ( limit = limit , offset = ( page - <NUM_LIT:1> ) * limit ) <EOL> if since : <EOL> params [ '<STR_LIT>' ] = since <EOL> try : <EOL> posts = self . api . get_connections ( str ( org . facebook_id ) , '<STR_LIT>' , fields = '<STR_LIT>' , ** params ) <EOL> except Exception : <EOL> return [ ] , None , None <EOL> paging = posts . get ( '<STR_LIT>' , { } ) <EOL> next_page = None <EOL> if len ( posts . get ( '<STR_LIT:data>' , [ ] ) ) == limit : <EOL> next_page = page + <NUM_LIT:1> <EOL> new_last_fetched_date = None <EOL> if page == <NUM_LIT:1> : <EOL> parsed_query_string = urlparse . parse_qs ( urlparse . urlparse ( paging . get ( '<STR_LIT>' , '<STR_LIT>' ) ) . query ) <EOL> new_last_fetched_date = parsed_query_string . get ( '<STR_LIT>' , [ None ] ) [ <NUM_LIT:0> ] <EOL> return posts [ '<STR_LIT:data>' ] , next_page , new_last_fetched_date <EOL> def runner ( job ) : <EOL> job . additer ( FacebookMapper , reducer ) <EOL> def starter ( program ) : <EOL> tempfile_path = '<STR_LIT>' <EOL> input_path = '<STR_LIT>' <EOL> output_path = '<STR_LIT>' <EOL> with open ( tempfile_path , '<STR_LIT:w>' ) as tempfile : <EOL> orgs = Org . objects . filter ( is_active = True , facebook_id__isnull = False ) [ : <NUM_LIT:10> ] <EOL> for org in orgs : <EOL> tempfile . write ( '<STR_LIT:\t>' . join ( [ str ( org . id ) , str ( org . facebook_id ) , str ( org . facebook_last_fetched or '<STR_LIT>' ) ] ) + '<STR_LIT:\n>' ) <EOL> local ( '<STR_LIT>' + input_path + '<STR_LIT>' + input_path + '<STR_LIT>' ) <EOL> local ( '<STR_LIT>' + output_path + '<STR_LIT>' + output_path + '<STR_LIT>' ) <EOL> local ( '<STR_LIT>' + tempfile_path + '<STR_LIT:U+0020>' + input_path ) <EOL> program . addopt ( '<STR_LIT:input>' , input_path ) <EOL> program . addopt ( '<STR_LIT>' , output_path ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> import dumbo <EOL> dumbo . main ( runner , starter ) </s>
