<s> import sys <EOL> import functools <EOL> import logging <EOL> from datetime import timedelta <EOL> from dateutil import parser <EOL> from datafeed . bidict import Bidict <EOL> from datafeed . exchange import * <EOL> from datafeed . quote import * <EOL> from datafeed . providers . http_fetcher import * <EOL> from datafeed . utils import json_decode <EOL> __all__ = [ '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' ] <EOL> _EXCHANGES = Bidict ( { <EOL> "<STR_LIT>" : '<STR_LIT>' , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> } ) <EOL> def currency2float ( currency ) : <EOL> """<STR_LIT>""" <EOL> if currency == '<STR_LIT>' : <EOL> return '<STR_LIT>' <EOL> if currency [ - <NUM_LIT:1> : ] == "<STR_LIT:M>" : <EOL> currency = currency [ : - <NUM_LIT:1> ] <EOL> return currency2float ( currency ) * <NUM_LIT:10> ** <NUM_LIT:6> <EOL> return float ( currency . replace ( "<STR_LIT:U+002C>" , "<STR_LIT>" ) ) <EOL> class GoogleSecurity ( Security ) : <EOL> @ property <EOL> def _abbr ( self ) : <EOL> """<STR_LIT>""" <EOL> return _EXCHANGES [ str ( self . exchange ) ] <EOL> @ classmethod <EOL> def from_string ( cls , idstr ) : <EOL> """<STR_LIT>""" <EOL> abbr , symbol = idstr . split ( '<STR_LIT::>' ) <EOL> return cls . from_abbr ( abbr , symbol ) <EOL> @ classmethod <EOL> def from_abbr ( cls , abbr , symbol ) : <EOL> """<STR_LIT>""" <EOL> exchange = cls . get_exchange_from_abbr ( abbr ) <EOL> return cls ( exchange , symbol ) <EOL> @ classmethod <EOL> def get_exchange_from_abbr ( cls , abbr ) : <EOL> """<STR_LIT>""" <EOL> ex = _EXCHANGES [ abbr ] <EOL> ex_cls = getattr ( sys . modules [ __name__ ] , ex ) <EOL> return ex_cls ( ) <EOL> class GoogleReport ( Report ) : <EOL> _TAGS_DEFINITION = { <EOL> '<STR_LIT:t>' : ( "<STR_LIT>" , str ) , <EOL> "<STR_LIT:e>" : ( "<STR_LIT>" , str ) , <EOL> '<STR_LIT>' : ( "<STR_LIT>" , currency2float ) , <EOL> '<STR_LIT>' : ( "<STR_LIT>" , currency2float ) , <EOL> '<STR_LIT>' : ( "<STR_LIT>" , currency2float ) , <EOL> '<STR_LIT>' : ( "<STR_LIT:time>" , parser . parse ) , <EOL> '<STR_LIT:l>' : ( "<STR_LIT>" , currency2float ) , <EOL> '<STR_LIT:c>' : ( "<STR_LIT>" , currency2float ) , <EOL> '<STR_LIT>' : ( "<STR_LIT>" , currency2float ) <EOL> } <EOL> _raw_data = { } <EOL> def __init__ ( self , raw_data ) : <EOL> self . assert_raw ( raw_data ) <EOL> self . _raw_data = raw_data <EOL> data = { } <EOL> for key , value in self . _TAGS_DEFINITION . iteritems ( ) : <EOL> data [ value [ <NUM_LIT:0> ] ] = value [ <NUM_LIT:1> ] ( raw_data [ key ] ) <EOL> security = GoogleSecurity . from_abbr ( data . pop ( '<STR_LIT>' ) , <EOL> data . pop ( '<STR_LIT>' ) ) <EOL> super ( GoogleReport , self ) . __init__ ( security , data ) <EOL> def assert_raw ( self , raw_data ) : <EOL> assert isinstance ( raw_data [ '<STR_LIT:t>' ] , basestring ) <EOL> assert isinstance ( raw_data [ '<STR_LIT:e>' ] , basestring ) <EOL> assert isinstance ( raw_data [ '<STR_LIT:l>' ] , basestring ) <EOL> assert isinstance ( raw_data [ '<STR_LIT>' ] , basestring ) <EOL> assert isinstance ( raw_data [ '<STR_LIT>' ] , basestring ) <EOL> def __getitem__ ( self , key ) : <EOL> """<STR_LIT>""" <EOL> return self . _raw_data [ key ] <EOL> @ property <EOL> def preclose ( self ) : <EOL> return self . price - self . change <EOL> @ staticmethod <EOL> def parse ( rawdata ) : <EOL> data = rawdata . strip ( ) [ <NUM_LIT:3> : ] . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> parsed = json_decode ( data ) <EOL> return ( GoogleReport ( x ) for x in parsed ) <EOL> class GoogleDay ( Day ) : <EOL> _DEFINITIONS = ( <EOL> ( "<STR_LIT:date>" , lambda x : parser . parse ( x ) . date ( ) ) , <EOL> ( "<STR_LIT>" , currency2float ) , <EOL> ( "<STR_LIT>" , currency2float ) , <EOL> ( "<STR_LIT>" , currency2float ) , <EOL> ( "<STR_LIT>" , currency2float ) , <EOL> ( "<STR_LIT>" , currency2float ) ) <EOL> def __init__ ( self , security , raw_data ) : <EOL> assert len ( raw_data ) == <NUM_LIT:6> <EOL> data = { } <EOL> i = <NUM_LIT:0> <EOL> for conf in self . _DEFINITIONS : <EOL> data [ conf [ <NUM_LIT:0> ] ] = conf [ <NUM_LIT:1> ] ( raw_data [ i ] ) <EOL> i += <NUM_LIT:1> <EOL> super ( GoogleDay , self ) . __init__ ( security , data ) <EOL> @ staticmethod <EOL> def parse ( security , rawdata ) : <EOL> import csv <EOL> from cStringIO import StringIO <EOL> f = StringIO ( rawdata ) <EOL> r = csv . reader ( f ) <EOL> r . next ( ) <EOL> return ( GoogleDay ( security , line ) for line in r ) <EOL> class GoogleReportFetcher ( Fetcher ) : <EOL> _MAX_REQUEST_SIZE = <NUM_LIT:100> <EOL> def __init__ ( self , base_url = '<STR_LIT>' , <EOL> time_out = <NUM_LIT:20> , max_clients = <NUM_LIT:10> , request_size = <NUM_LIT:100> ) : <EOL> assert request_size <= self . _MAX_REQUEST_SIZE <EOL> super ( GoogleReportFetcher , self ) . __init__ ( base_url , time_out , max_clients ) <EOL> self . _request_size = request_size <EOL> def _fetching_urls ( self , * args , ** kwargs ) : <EOL> gids = ( str ( s ) for s in args ) <EOL> gids = self . _slice ( gids ) <EOL> return ( self . _make_url ( filter ( lambda x : x != None , i ) ) for i in gids ) <EOL> def _make_url ( self , ids ) : <EOL> """<STR_LIT>""" <EOL> return "<STR_LIT>" % ( self . _base_url , <EOL> '<STR_LIT:U+002C>' . join ( ids ) ) <EOL> def _callback ( self , security , ** kwargs ) : <EOL> if '<STR_LIT>' in kwargs : <EOL> callback = kwargs [ '<STR_LIT>' ] <EOL> else : <EOL> callback = None <EOL> return functools . partial ( self . _handle_request , callback ) <EOL> def _handle_request ( self , callback , response ) : <EOL> try : <EOL> self . queue_len = self . queue_len - <NUM_LIT:1> <EOL> if response . error : <EOL> logging . error ( response . error ) <EOL> else : <EOL> callback ( response . body ) <EOL> finally : <EOL> self . stop ( ) <EOL> class GoogleDayFetcher ( DayFetcher ) : <EOL> def __init__ ( self , base_url = '<STR_LIT>' , <EOL> time_out = <NUM_LIT:20> , max_clients = <NUM_LIT:10> ) : <EOL> super ( GoogleDayFetcher , self ) . __init__ ( base_url , time_out , max_clients ) <EOL> def _make_url ( self , security , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> url_format = "<STR_LIT>" <EOL> return url_format % ( self . _base_url , <EOL> str ( security ) , <EOL> kwargs [ '<STR_LIT>' ] , <EOL> kwargs [ '<STR_LIT>' ] - timedelta ( days = <NUM_LIT:1> ) ) <EOL> class GoogleNewsFetcher ( Fetcher ) : <EOL> _BASE_URL = "<STR_LIT>" <EOL> _MAX_REQUEST_SIZE = <NUM_LIT:100> <EOL> def __init__ ( self , base_url = _BASE_URL , time_out = <NUM_LIT:10> , max_clients = <NUM_LIT:5> ) : <EOL> super ( GoogleNewsFetcher , self ) . __init__ ( base_url , time_out , max_clients ) <EOL> def _fetching_urls ( self , * args , ** kwargs ) : <EOL> return ( self . _make_url ( str ( security ) ) for security in args ) <EOL> def _make_url ( self , symbol ) : <EOL> """<STR_LIT>""" <EOL> return self . _base_url % symbol <EOL> def _callback ( self , security , ** kwargs ) : <EOL> if '<STR_LIT>' in kwargs : <EOL> callback = kwargs [ '<STR_LIT>' ] <EOL> else : <EOL> callback = None <EOL> return functools . partial ( self . _handle_request , <EOL> callback , <EOL> security ) <EOL> def _handle_request ( self , callback , security , response ) : <EOL> try : <EOL> self . queue_len = self . queue_len - <NUM_LIT:1> <EOL> if response . error : <EOL> logging . error ( response . error ) <EOL> else : <EOL> callback ( security , response ) <EOL> finally : <EOL> self . stop ( ) </s>
