<s> from __future__ import absolute_import , print_function , division <EOL> import itertools <EOL> from petl . compat import string_types , text_type <EOL> from petl . util . base import Table , iterpeek <EOL> from petl . io . numpy import construct_dtype <EOL> def frombcolz ( source , expression = None , outcols = None , limit = None , skip = <NUM_LIT:0> ) : <EOL> """<STR_LIT>""" <EOL> return BcolzView ( source , expression = expression , outcols = outcols , <EOL> limit = limit , skip = skip ) <EOL> class BcolzView ( Table ) : <EOL> def __init__ ( self , source , expression = None , outcols = None , limit = None , <EOL> skip = <NUM_LIT:0> ) : <EOL> self . source = source <EOL> self . expression = expression <EOL> self . outcols = outcols <EOL> self . limit = limit <EOL> self . skip = skip <EOL> def __iter__ ( self ) : <EOL> if isinstance ( self . source , string_types ) : <EOL> import bcolz <EOL> ctbl = bcolz . open ( self . source , mode = '<STR_LIT:r>' ) <EOL> else : <EOL> ctbl = self . source <EOL> if self . outcols is None : <EOL> header = tuple ( ctbl . names ) <EOL> else : <EOL> header = tuple ( self . outcols ) <EOL> assert all ( h in ctbl . names for h in header ) , '<STR_LIT>' <EOL> yield header <EOL> if self . expression is None : <EOL> it = ctbl . iter ( outcols = self . outcols , skip = self . skip , <EOL> limit = self . limit ) <EOL> else : <EOL> it = ctbl . where ( self . expression , outcols = self . outcols , skip = self . skip , <EOL> limit = self . limit ) <EOL> for row in it : <EOL> yield row <EOL> def tobcolz ( table , dtype = None , sample = <NUM_LIT:1000> , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> import bcolz <EOL> import numpy as np <EOL> it = iter ( table ) <EOL> peek , it = iterpeek ( it , sample ) <EOL> hdr = next ( it ) <EOL> it = ( tuple ( row ) for row in it ) <EOL> flds = list ( map ( text_type , hdr ) ) <EOL> dtype = construct_dtype ( flds , peek , dtype ) <EOL> kwargs . setdefault ( '<STR_LIT>' , <NUM_LIT> ) <EOL> kwargs . setdefault ( '<STR_LIT>' , '<STR_LIT:w>' ) <EOL> ctbl = bcolz . ctable ( np . array ( [ ] , dtype = dtype ) , ** kwargs ) <EOL> chunklen = sum ( ctbl . cols [ name ] . chunklen <EOL> for name in ctbl . names ) // len ( ctbl . names ) <EOL> while True : <EOL> data = list ( itertools . islice ( it , chunklen ) ) <EOL> data = np . array ( data , dtype = dtype ) <EOL> ctbl . append ( data ) <EOL> if len ( data ) < chunklen : <EOL> break <EOL> ctbl . flush ( ) <EOL> return ctbl <EOL> def appendbcolz ( table , obj , check_names = True ) : <EOL> """<STR_LIT>""" <EOL> import bcolz <EOL> import numpy as np <EOL> if isinstance ( obj , string_types ) : <EOL> ctbl = bcolz . open ( obj , mode = '<STR_LIT:a>' ) <EOL> else : <EOL> assert hasattr ( obj , '<STR_LIT>' ) and hasattr ( obj , '<STR_LIT>' ) , '<STR_LIT>' % obj <EOL> ctbl = obj <EOL> dtype = ctbl . dtype <EOL> it = iter ( table ) <EOL> hdr = next ( it ) <EOL> flds = list ( map ( text_type , hdr ) ) <EOL> if check_names : <EOL> assert tuple ( flds ) == tuple ( ctbl . names ) , '<STR_LIT>' <EOL> chunklen = sum ( ctbl . cols [ name ] . chunklen <EOL> for name in ctbl . names ) // len ( ctbl . names ) <EOL> while True : <EOL> data = list ( itertools . islice ( it , chunklen ) ) <EOL> data = np . array ( data , dtype = dtype ) <EOL> ctbl . append ( data ) <EOL> if len ( data ) < chunklen : <EOL> break <EOL> ctbl . flush ( ) <EOL> return ctbl </s>
