<s> from __future__ import division , print_function <EOL> import sqlitedict as sqd <EOL> from os . path import abspath , join , exists <EOL> import os <EOL> import sys <EOL> import shutil <EOL> import traceback <EOL> import pickle <EOL> import warnings <EOL> import random <EOL> import progress <EOL> try : <EOL> import numpy as np <EOL> _NP = True <EOL> except ImportError : <EOL> warnings . warn ( "<STR_LIT>" ) <EOL> _NP = False <EOL> if sys . version_info [ <NUM_LIT:0> ] == <NUM_LIT:2> : <EOL> os_remove = os . remove <EOL> def new_remove ( path ) : <EOL> os_remove ( path ) <EOL> os . remove = new_remove <EOL> os_rmdir = os . rmdir <EOL> def new_rmdir ( path ) : <EOL> os_rmdir ( path ) <EOL> os . rmdir = new_rmdir <EOL> MAGIC_SIGN = <NUM_LIT> <EOL> MAGIC_SIGN_NPARRAY = <NUM_LIT> <EOL> TYPE_ORD = <NUM_LIT> <EOL> TYPE_SUB = <NUM_LIT> <EOL> TYPE_NPA = <NUM_LIT> <EOL> def key_to_str ( key , max_len = <NUM_LIT:255> ) : <EOL> if isinstance ( key , ( bytearray , bytes ) ) : <EOL> return "<STR_LIT>" <EOL> s = str ( key ) <EOL> if len ( s ) > max_len : <EOL> return s [ : max_len ] + '<STR_LIT>' <EOL> else : <EOL> return s <EOL> RAND_STR_ASCII_IDX_LIST = list ( range ( <NUM_LIT> , <NUM_LIT> ) ) + list ( range ( <NUM_LIT> , <NUM_LIT> ) ) + list ( range ( <NUM_LIT> , <NUM_LIT> ) ) <EOL> def rand_str ( l = <NUM_LIT:8> ) : <EOL> s = '<STR_LIT>' <EOL> for i in range ( l ) : <EOL> s += chr ( random . choice ( RAND_STR_ASCII_IDX_LIST ) ) <EOL> return s <EOL> class PersistentDataStructure ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , name , path = "<STR_LIT>" , verbose = <NUM_LIT:1> ) : <EOL> self . _open = False <EOL> self . _name = name <EOL> self . _path = abspath ( path ) <EOL> if not exists ( self . _path ) : <EOL> print ( "<STR_LIT>" . format ( path , self . _path ) ) <EOL> print ( "<STR_LIT>" ) <EOL> os . makedirs ( self . _path ) <EOL> self . verbose = verbose <EOL> self . _dirname = join ( self . _path , "<STR_LIT>" + self . _name ) <EOL> if not exists ( self . _dirname ) : <EOL> os . mkdir ( self . _dirname ) <EOL> self . _filename = join ( self . _dirname , self . _name + '<STR_LIT>' ) <EOL> self . _l = <NUM_LIT:8> <EOL> self . open ( ) <EOL> def _repair ( self ) : <EOL> raise DeprecationWarning <EOL> def _consistency_check ( self ) : <EOL> raise DeprecationWarning <EOL> def _new_rand_file_name ( self , make_dir = False , end = '<STR_LIT>' ) : <EOL> c = <NUM_LIT:0> <EOL> while True : <EOL> fname = rand_str ( self . _l ) + end <EOL> if not make_dir : <EOL> full_name = join ( self . _dirname , fname ) <EOL> if not os . path . exists ( full_name ) : <EOL> open ( full_name , '<STR_LIT:a>' ) . close ( ) <EOL> return fname <EOL> else : <EOL> full_name = join ( self . _dirname , '<STR_LIT>' + fname ) <EOL> if not os . path . exists ( full_name ) : <EOL> os . mkdir ( full_name ) <EOL> return fname <EOL> c += <NUM_LIT:1> <EOL> if c > <NUM_LIT:10> : <EOL> self . _l += <NUM_LIT:2> <EOL> c = <NUM_LIT:0> <EOL> print ( "<STR_LIT>" , self . _l ) <EOL> def __enter__ ( self ) : <EOL> return self <EOL> def __exit__ ( self , exc_type , exc_value , traceback ) : <EOL> if self . verbose > <NUM_LIT:1> : <EOL> print ( "<STR_LIT>" . format ( self . _name , self . _dirname ) ) <EOL> self . close ( ) <EOL> def open ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . verbose > <NUM_LIT:1> : <EOL> print ( "<STR_LIT>" . format ( self . _name , self . _dirname ) ) <EOL> self . db = sqd . SqliteDict ( filename = self . _filename , autocommit = False ) <EOL> self . _open = True <EOL> def is_open ( self ) : <EOL> return self . _open <EOL> def is_closed ( self ) : <EOL> return not self . _open <EOL> def need_open ( self ) : <EOL> if self . is_closed ( ) : <EOL> raise RuntimeError ( "<STR_LIT>" ) <EOL> def close ( self ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> self . db . close ( ) <EOL> self . _open = False <EOL> if self . verbose > <NUM_LIT:1> : <EOL> print ( "<STR_LIT>" . format ( self . _name , self . _dirname ) ) <EOL> except : <EOL> if self . verbose > <NUM_LIT:1> : <EOL> print ( "<STR_LIT>" . format ( self . _name , self . _dirname ) ) <EOL> def erase ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . verbose > <NUM_LIT:1> : <EOL> print ( "<STR_LIT>" . format ( self . _name , self . _dirname ) ) <EOL> if self . is_closed ( ) : <EOL> self . open ( ) <EOL> try : <EOL> if self . verbose > <NUM_LIT:1> : <EOL> print ( "<STR_LIT>" , self . sub_data_keys ) <EOL> for key in self : <EOL> t , v = self . get_value_and_value_type ( key ) <EOL> if t == TYPE_SUB : <EOL> with self . getData ( key ) as sub_data : <EOL> sub_data . erase ( ) <EOL> elif t == TYPE_NPA : <EOL> os . remove ( os . path . join ( self . _dirname , v [ '<STR_LIT>' ] ) ) <EOL> except : <EOL> traceback . print_exc ( ) <EOL> finally : <EOL> self . close ( ) <EOL> if self . verbose > <NUM_LIT:1> : <EOL> print ( "<STR_LIT>" , self . _filename ) <EOL> os . remove ( path = self . _filename ) <EOL> try : <EOL> os . rmdir ( path = self . _dirname ) <EOL> except OSError as e : <EOL> if self . verbose > <NUM_LIT:0> : <EOL> warnings . warn ( "<STR_LIT>" . format ( e ) ) <EOL> def clear ( self ) : <EOL> """<STR_LIT>""" <EOL> self . need_open ( ) <EOL> for key in self : <EOL> t , v = self . get_value_and_value_type ( key ) <EOL> if t == TYPE_SUB : <EOL> with self . getData ( key ) as sub_data : <EOL> sub_data . erase ( ) <EOL> elif t == TYPE_NPA : <EOL> os . remove ( os . path . join ( self . _dirname , v [ '<STR_LIT>' ] ) ) <EOL> self . db . clear ( ) <EOL> def show_stat ( self , recursive = False , prepend = "<STR_LIT>" ) : <EOL> prepend += self . _name <EOL> print ( "<STR_LIT>" . format ( prepend , self . _name ) ) <EOL> print ( "<STR_LIT>" . format ( prepend , self . _dirname ) ) <EOL> print ( "<STR_LIT>" . format ( prepend , self . _filename ) ) <EOL> self . need_open ( ) <EOL> str_key = <NUM_LIT:0> <EOL> bin_key = <NUM_LIT:0> <EOL> oth_key = <NUM_LIT:0> <EOL> sub_c = <NUM_LIT:0> <EOL> npa_c = <NUM_LIT:0> <EOL> sub_data_keys = set ( ) <EOL> for k in self : <EOL> if isinstance ( k , str ) : <EOL> str_key += <NUM_LIT:1> <EOL> elif isinstance ( k , bytes ) : <EOL> bin_key += <NUM_LIT:1> <EOL> else : <EOL> oth_key += <NUM_LIT:1> <EOL> t , v = self . get_value_and_value_type ( k ) <EOL> if t == TYPE_NPA : <EOL> npa_c += <NUM_LIT:1> <EOL> elif t == TYPE_SUB : <EOL> sub_c += <NUM_LIT:1> <EOL> sub_data_keys . add ( k ) <EOL> print ( "<STR_LIT>" . format ( prepend , str_key ) ) <EOL> print ( "<STR_LIT>" . format ( prepend , bin_key ) ) <EOL> if oth_key > <NUM_LIT:0> : <EOL> print ( "<STR_LIT>" . format ( prepend , oth_key ) ) <EOL> print ( "<STR_LIT>" . format ( prepend , sub_c ) ) <EOL> print ( "<STR_LIT>" . format ( prepend , npa_c ) ) <EOL> print ( ) <EOL> sys . stdout . flush ( ) <EOL> if recursive : <EOL> for k in sub_data_keys : <EOL> print ( "<STR_LIT>" . format ( key_to_str ( k ) ) ) <EOL> sys . stdout . flush ( ) <EOL> with self . getData ( k ) as subdata : <EOL> subdata . show_stat ( recursive = recursive , <EOL> prepend = prepend + "<STR_LIT>" ) <EOL> def __is_sub_data ( self , value ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> assert value [ '<STR_LIT>' ] == MAGIC_SIGN <EOL> return True <EOL> except : <EOL> return False <EOL> def __is_nparray ( self , value ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> assert value [ '<STR_LIT>' ] == MAGIC_SIGN_NPARRAY <EOL> return True <EOL> except : <EOL> return False <EOL> def has_key ( self , key ) : <EOL> self . need_open ( ) <EOL> return ( key in self . db ) <EOL> def is_subdata ( self , key ) : <EOL> return ( key in self ) and self . __is_sub_data ( self . db [ key ] ) <EOL> def is_NPA ( self , key ) : <EOL> return ( key in self ) and self . __is_nparray ( self . db [ key ] ) <EOL> def get_value_and_value_type ( self , key ) : <EOL> v = self . db [ key ] <EOL> if self . __is_nparray ( v ) : <EOL> return TYPE_NPA , v <EOL> elif self . __is_sub_data ( v ) : <EOL> return TYPE_SUB , v <EOL> else : <EOL> return TYPE_ORD , v <EOL> def setData ( self , key , value , overwrite = False ) : <EOL> """<STR_LIT>""" <EOL> self . need_open ( ) <EOL> if overwrite : <EOL> if self . verbose > <NUM_LIT:1> : <EOL> print ( "<STR_LIT>" ) <EOL> if key in self . db : <EOL> self . __delitem__ ( key ) <EOL> if not key in self . db : <EOL> if _NP and isinstance ( value , np . ndarray ) : <EOL> if self . verbose > <NUM_LIT:1> : <EOL> print ( "<STR_LIT>" ) <EOL> return self . _setNPA ( key , nparray = value ) <EOL> else : <EOL> if self . verbose > <NUM_LIT:1> : <EOL> print ( "<STR_LIT>" ) <EOL> self . db [ key ] = value <EOL> self . db . commit ( ) <EOL> return True <EOL> else : <EOL> if overwrite : <EOL> raise RuntimeError ( "<STR_LIT>" ) <EOL> raise KeyError ( "<STR_LIT>" ) <EOL> def _setNPA ( self , key , nparray ) : <EOL> d = { '<STR_LIT>' : self . _new_rand_file_name ( end = '<STR_LIT>' ) , <EOL> '<STR_LIT>' : MAGIC_SIGN_NPARRAY } <EOL> self . db [ key ] = d <EOL> self . db . commit ( ) <EOL> full_name = os . path . join ( self . _dirname , d [ '<STR_LIT>' ] ) <EOL> np . save ( full_name , nparray ) <EOL> return True <EOL> def _loadNPA ( self , fname ) : <EOL> return np . load ( os . path . join ( self . _dirname , fname ) ) <EOL> def _getNPA ( self , key ) : <EOL> d = self . db [ key ] <EOL> assert d [ '<STR_LIT>' ] == MAGIC_SIGN_NPARRAY <EOL> fname = d [ '<STR_LIT>' ] <EOL> return self . _loadNPA ( fname ) <EOL> def newSubData ( self , key ) : <EOL> """<STR_LIT>""" <EOL> self . need_open ( ) <EOL> if not key in self . db : <EOL> d = { '<STR_LIT:name>' : self . _new_rand_file_name ( make_dir = True ) , <EOL> '<STR_LIT>' : MAGIC_SIGN } <EOL> self . db [ key ] = d <EOL> self . db . commit ( ) <EOL> return PersistentDataStructure ( name = d [ '<STR_LIT:name>' ] , path = os . path . join ( self . _dirname ) , verbose = self . verbose ) <EOL> else : <EOL> raise RuntimeError ( "<STR_LIT>" ) <EOL> def getData ( self , key , create_sub_data = False ) : <EOL> self . need_open ( ) <EOL> if key in self . db : <EOL> if self . verbose > <NUM_LIT:1> : <EOL> print ( "<STR_LIT>" ) <EOL> t , v = self . get_value_and_value_type ( key ) <EOL> if t == TYPE_SUB : <EOL> sub_db_name = v [ '<STR_LIT:name>' ] <EOL> if self . verbose > <NUM_LIT:1> : <EOL> print ( "<STR_LIT>" , key , "<STR_LIT>" , sub_db_name ) <EOL> return PersistentDataStructure ( name = sub_db_name , path = os . path . join ( self . _dirname ) , verbose = self . verbose ) <EOL> elif t == TYPE_NPA : <EOL> if self . verbose > <NUM_LIT:1> : <EOL> print ( "<STR_LIT>" ) <EOL> return self . _loadNPA ( v [ '<STR_LIT>' ] ) <EOL> else : <EOL> if self . verbose > <NUM_LIT:1> : <EOL> print ( "<STR_LIT>" ) <EOL> return v <EOL> else : <EOL> if not create_sub_data : <EOL> raise KeyError ( "<STR_LIT>" . format ( key_to_str ( key ) ) ) <EOL> else : <EOL> if self . verbose > <NUM_LIT:1> : <EOL> print ( "<STR_LIT>" ) <EOL> return self . newSubData ( key ) <EOL> def setDataFromSubData ( self , key , subData ) : <EOL> """<STR_LIT>""" <EOL> self . need_open ( ) <EOL> if self . is_subdata ( key ) : <EOL> with self [ key ] as pds : <EOL> name = pds . _name <EOL> dir_name = pds . _dirname <EOL> pds . erase ( ) <EOL> else : <EOL> with self . newSubData ( key ) as new_sub_data : <EOL> name = new_sub_data . _name <EOL> dir_name = new_sub_data . _dirname <EOL> new_sub_data . erase ( ) <EOL> shutil . copytree ( src = subData . _dirname , dst = dir_name ) <EOL> os . rename ( src = os . path . join ( dir_name , subData . _name + '<STR_LIT>' ) , dst = os . path . join ( dir_name , name + '<STR_LIT>' ) ) <EOL> def mergeOtherPDS ( self , other_db_name , other_db_path = '<STR_LIT>' , update = '<STR_LIT:error>' , status_interval = <NUM_LIT:5> ) : <EOL> """<STR_LIT>""" <EOL> error = False <EOL> if update == '<STR_LIT:error>' : <EOL> error = True <EOL> elif update == '<STR_LIT:ignore>' : <EOL> ignore = True <EOL> elif update == '<STR_LIT>' : <EOL> ignore = False <EOL> else : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> transfered = <NUM_LIT:0> <EOL> ignored = <NUM_LIT:0> <EOL> if status_interval == <NUM_LIT:0> : <EOL> PB = progress . ProgressSilentDummy <EOL> else : <EOL> PB = progress . ProgressBarFancy <EOL> with PersistentDataStructure ( name = other_db_name , <EOL> path = other_db_path , <EOL> verbose = self . verbose ) as otherData : <EOL> c = progress . UnsignedIntValue ( val = <NUM_LIT:0> ) <EOL> m = progress . UnsignedIntValue ( val = len ( otherData ) ) <EOL> with PB ( count = c , max_count = m , verbose = self . verbose , interval = status_interval ) as pb : <EOL> pb . start ( ) <EOL> for k in otherData : <EOL> if k in self : <EOL> if error : <EOL> raise KeyError ( "<STR_LIT>" ) <EOL> else : <EOL> if ignore : <EOL> if self . verbose > <NUM_LIT:1> : <EOL> print ( "<STR_LIT>" , k ) <EOL> ignored += <NUM_LIT:1> <EOL> continue <EOL> else : <EOL> if self . verbose > <NUM_LIT:1> : <EOL> print ( "<STR_LIT>" , k ) <EOL> del self [ k ] <EOL> value = otherData [ k ] <EOL> try : <EOL> self [ k ] = value <EOL> transfered += <NUM_LIT:1> <EOL> finally : <EOL> if isinstance ( value , PersistentDataStructure ) : <EOL> value . close ( ) <EOL> with c . get_lock ( ) : <EOL> c . value += <NUM_LIT:1> <EOL> sys . stdout . flush ( ) <EOL> print ( "<STR_LIT>" ) <EOL> print ( "<STR_LIT>" , transfered ) <EOL> print ( "<STR_LIT>" , ignored ) <EOL> def __len__ ( self ) : <EOL> self . need_open ( ) <EOL> return len ( self . db ) <EOL> def __iter__ ( self ) : <EOL> self . need_open ( ) <EOL> db_iter = self . db . __iter__ ( ) <EOL> while True : <EOL> next_item = db_iter . __next__ ( ) <EOL> yield next_item <EOL> def __contains__ ( self , key ) : <EOL> self . need_open ( ) <EOL> return ( key in self . db ) <EOL> def __getitem__ ( self , key ) : <EOL> self . need_open ( ) <EOL> return self . getData ( key , create_sub_data = False ) <EOL> def __setitem__ ( self , key , value ) : <EOL> if isinstance ( value , PersistentDataStructure ) : <EOL> self . setDataFromSubData ( key , value ) <EOL> else : <EOL> self . setData ( key , value , overwrite = True ) <EOL> if self . verbose > <NUM_LIT:1> : <EOL> print ( "<STR_LIT>" , key , "<STR_LIT:to>" , value , "<STR_LIT>" , self . _filename ) <EOL> def __delitem__ ( self , key ) : <EOL> self . need_open ( ) <EOL> t , v = self . get_value_and_value_type ( key ) <EOL> if t == TYPE_SUB : <EOL> with self [ key ] as pds : <EOL> pds . erase ( ) <EOL> elif t == TYPE_NPA : <EOL> fname = v [ '<STR_LIT>' ] <EOL> os . remove ( os . path . join ( self . _dirname , fname ) ) <EOL> del self . db [ key ] <EOL> self . db . commit ( ) </s>
