<s> import gc <EOL> import pandas <EOL> from datetime import datetime <EOL> from pandas import DataFrame <EOL> from sqlalchemy import create_engine <EOL> from sqlalchemy . sql import text <EOL> from sqlalchemy . orm import sessionmaker <EOL> from mining . utils import conf , log_it <EOL> from mining . utils . _pandas import fix_render <EOL> from mining . db import DataWarehouse <EOL> from bottle . ext . mongo import MongoPlugin <EOL> class Cube ( object ) : <EOL> def __init__ ( self , _cube ) : <EOL> log_it ( "<STR_LIT>" . format ( _cube [ '<STR_LIT>' ] ) , "<STR_LIT>" ) <EOL> self . mongo = MongoPlugin ( <EOL> uri = conf ( "<STR_LIT>" ) [ "<STR_LIT>" ] , <EOL> db = conf ( "<STR_LIT>" ) [ "<STR_LIT>" ] , <EOL> json_mongo = True ) . get_mongo ( ) <EOL> try : <EOL> del _cube [ '<STR_LIT>' ] <EOL> except KeyError : <EOL> pass <EOL> self . cube = _cube <EOL> self . slug = self . cube [ '<STR_LIT>' ] <EOL> def load ( self ) : <EOL> self . cube [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> self . mongo [ '<STR_LIT>' ] . update ( { '<STR_LIT>' : self . slug } , self . cube ) <EOL> self . cube [ '<STR_LIT>' ] = datetime . now ( ) <EOL> _sql = self . cube [ '<STR_LIT>' ] <EOL> if _sql [ - <NUM_LIT:1> ] == '<STR_LIT:;>' : <EOL> _sql = _sql [ : - <NUM_LIT:1> ] <EOL> self . sql = u"""<STR_LIT>""" . format ( _sql ) <EOL> self . connection = self . mongo [ '<STR_LIT>' ] . find_one ( { <EOL> '<STR_LIT>' : self . cube [ '<STR_LIT>' ] } ) [ '<STR_LIT>' ] <EOL> log_it ( "<STR_LIT>" . format ( self . slug ) , <EOL> "<STR_LIT>" ) <EOL> if '<STR_LIT>' in self . connection : <EOL> e = create_engine ( self . connection ) <EOL> else : <EOL> e = create_engine ( self . connection , <EOL> ** conf ( '<STR_LIT>' ) [ '<STR_LIT>' ] ) <EOL> Session = sessionmaker ( bind = e ) <EOL> session = Session ( ) <EOL> resoverall = session . execute ( text ( self . sql ) ) <EOL> self . data = resoverall . fetchall ( ) <EOL> self . keys = resoverall . keys ( ) <EOL> def environment ( self , t ) : <EOL> if t not in [ '<STR_LIT>' ] : <EOL> self . sql = t <EOL> def _data ( self , data ) : <EOL> self . data = data <EOL> def _keys ( self , keys ) : <EOL> if type ( keys ) == list : <EOL> self . keys = keys <EOL> self . keys = list ( keys ) <EOL> def frame ( self , data_type = None ) : <EOL> log_it ( "<STR_LIT>" . format ( <EOL> data_type or '<STR_LIT>' , self . slug ) , "<STR_LIT>" ) <EOL> if data_type : <EOL> self . df = getattr ( pandas , "<STR_LIT>" . format ( data_type ) ) ( self . data ) <EOL> else : <EOL> self . df = DataFrame ( self . data ) <EOL> if self . df . empty : <EOL> self . pdict = { } <EOL> log_it ( '<STR_LIT>' . format ( self . cube ) , <EOL> "<STR_LIT>" ) <EOL> return <EOL> try : <EOL> self . df . columns = self . keys <EOL> except AttributeError : <EOL> self . _keys ( self . df . columns . tolist ( ) ) <EOL> if conf ( "<STR_LIT>" ) . get ( "<STR_LIT>" ) and self . cube . get ( "<STR_LIT>" ) : <EOL> from oml import RunTime <EOL> self . df . columns = self . keys <EOL> df = RunTime ( conf ( "<STR_LIT>" ) . get ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> self . df . to_dict ( orient = '<STR_LIT>' ) , <EOL> self . cube . get ( "<STR_LIT>" ) , <EOL> conf ( "<STR_LIT>" ) . get ( "<STR_LIT:class>" , { "<STR_LIT>" : "<STR_LIT>" } ) ) <EOL> self . df = DataFrame ( df ) <EOL> self . _keys ( self . df . columns . tolist ( ) ) <EOL> self . df . head ( ) <EOL> self . pdict = map ( fix_render , self . df . to_dict ( orient = '<STR_LIT>' ) ) <EOL> def save ( self ) : <EOL> log_it ( "<STR_LIT>" . format ( self . slug ) , <EOL> "<STR_LIT>" ) <EOL> data = { '<STR_LIT:data>' : self . pdict , '<STR_LIT>' : self . keys } <EOL> DW = DataWarehouse ( ) <EOL> DW . save ( self . slug , data ) <EOL> self . cube [ '<STR_LIT:status>' ] = True <EOL> self . cube [ '<STR_LIT>' ] = datetime . now ( ) <EOL> self . cube [ '<STR_LIT>' ] = True <EOL> self . mongo [ '<STR_LIT>' ] . update ( { '<STR_LIT>' : self . cube [ '<STR_LIT>' ] } , self . cube ) <EOL> log_it ( "<STR_LIT>" . format ( self . slug ) , "<STR_LIT>" ) <EOL> gc . collect ( ) </s>
