<s> import urllib <EOL> import re <EOL> import pandas as pd <EOL> import datetime as dt <EOL> import zipfile <EOL> import StringIO <EOL> from extra import ProgressBar <EOL> import os <EOL> import yahooFinance as yf <EOL> from string import Template <EOL> import numpy as np <EOL> def fileName2date ( fName ) : <EOL> '''<STR_LIT>''' <EOL> name = os . path . splitext ( fName ) [ <NUM_LIT:0> ] <EOL> m = re . findall ( '<STR_LIT>' , name ) [ <NUM_LIT:0> ] <EOL> return dt . datetime . strptime ( m , '<STR_LIT>' ) . date ( ) <EOL> def date2fileName ( date ) : <EOL> return '<STR_LIT>' % date . strftime ( '<STR_LIT>' ) <EOL> def downloadUrl ( date ) : <EOL> s = Template ( '<STR_LIT>' ) <EOL> url = s . substitute ( fName = date2fileName ( date ) , year = date . year , month = '<STR_LIT>' % date . month ) <EOL> return url <EOL> class BATS_Data ( object ) : <EOL> def __init__ ( self , dataDir ) : <EOL> '''<STR_LIT>''' <EOL> self . dataDir = dataDir <EOL> self . shortRatio = None <EOL> self . _checkDates ( ) <EOL> def _checkDates ( self ) : <EOL> '''<STR_LIT>''' <EOL> self . dates = [ ] <EOL> for fName in os . listdir ( self . dataDir ) : <EOL> self . dates . append ( fileName2date ( fName ) ) <EOL> def _missingDates ( self ) : <EOL> '''<STR_LIT>''' <EOL> print '<STR_LIT>' , <EOL> spy = yf . getHistoricData ( '<STR_LIT>' , sDate = ( <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT:1> ) ) <EOL> busDates = [ d . date ( ) for d in spy . index ] <EOL> print '<STR_LIT>' , busDates [ <NUM_LIT:0> ] , '<STR_LIT:->' , busDates [ - <NUM_LIT:1> ] <EOL> missing = [ ] <EOL> for d in busDates : <EOL> if d not in self . dates : <EOL> missing . append ( d ) <EOL> return missing <EOL> def updateDb ( self ) : <EOL> print '<STR_LIT>' <EOL> missing = self . _missingDates ( ) <EOL> for i , date in enumerate ( missing ) : <EOL> source = downloadUrl ( date ) <EOL> dest = os . path . join ( self . dataDir , date2fileName ( date ) ) <EOL> if not os . path . exists ( dest ) : <EOL> print '<STR_LIT>' % ( i , len ( missing ) ) , source <EOL> urllib . urlretrieve ( source , dest ) <EOL> else : <EOL> print '<STR_LIT:x>' , <EOL> print '<STR_LIT>' <EOL> self . _checkDates ( ) <EOL> def loadDate ( self , date ) : <EOL> fName = os . path . join ( self . dataDir , date2fileName ( date ) ) <EOL> zipped = zipfile . ZipFile ( fName ) <EOL> lines = zipped . read ( zipped . namelist ( ) [ <NUM_LIT:0> ] ) <EOL> buf = StringIO . StringIO ( lines ) <EOL> df = pd . read_csv ( buf , sep = '<STR_LIT:|>' , index_col = <NUM_LIT:1> , parse_dates = False , dtype = { '<STR_LIT>' : object , '<STR_LIT>' : np . float32 , '<STR_LIT>' : np . float32 } ) <EOL> s = df [ '<STR_LIT>' ] / df [ '<STR_LIT>' ] <EOL> s . name = dt . datetime . strptime ( df [ '<STR_LIT>' ] [ - <NUM_LIT:1> ] , '<STR_LIT>' ) <EOL> return s <EOL> def loadData ( self ) : <EOL> '''<STR_LIT>''' <EOL> data = [ ] <EOL> pb = ProgressBar ( len ( self . dates ) - <NUM_LIT:1> ) <EOL> for idx , date in enumerate ( self . dates ) : <EOL> data . append ( self . loadDate ( date ) ) <EOL> pb . animate ( idx ) <EOL> self . shortRatio = pd . DataFrame ( data ) <EOL> return self . shortRatio </s>
