def read_big(fid, size=None):
 buf_size = 16777216
 if (size is None):
  if (not isinstance(fid, gzip.GzipFile)):
   size = (os.fstat(fid.fileno()).st_size - fid.tell())
 if (size is not None):
  segments = np.r_[(np.arange(0, size, buf_size), size)]
  buf = bytearray((' ' * size))
  for (start, end) in zip(segments[:(-1)], segments[1:]):
   data = fid.read(int((end - start)))
   if (len(data) != (end - start)):
    raise ValueError('Read error')
   buf[start:end] = data
  buf = bytes(buf)
 else:
  buf = ['']
  new = fid.read(buf_size)
  while (len(new) > 0):
   buf.append(new)
   new = fid.read(buf_size)
  buf = ''.join(buf)
 return buf

def read_big ( fid , size = None ) : buf_size = 16777216 if ( size is None ) : if ( not isinstance ( fid , gzip . GzipFile ) ) : size = ( os . fstat ( fid . fileno ( ) ) . st_size - fid . tell ( ) ) if ( size is not None ) : segments = np . r_ [ ( np . arange ( 0 , size , buf_size ) , size ) ] buf = bytearray ( ( ' ' * size ) ) for ( start , end ) in zip ( segments [ : ( - 1 ) ] , segments [ 1 : ] ) : data = fid . read ( int ( ( end - start ) ) ) if ( len ( data ) != ( end - start ) ) : raise ValueError ( 'Read error' ) buf [ start : end ] = data buf = bytes ( buf ) else : buf = [ '' ] new = fid . read ( buf_size ) while ( len ( new ) > 0 ) : buf . append ( new ) new = fid . read ( buf_size ) buf = '' . join ( buf ) return buf
function to read large chunks of data windows-friendly .