<s> from optparse import OptionParser <EOL> from collections import deque <EOL> from django . core . files import File <EOL> from dockit import schema <EOL> from dockit . schema . loading import get_documents , get_document <EOL> from dockit . core . serializers . python import Serializer , Deserializer <EOL> from datatap . loading import register_datatap <EOL> from datatap . datataps . base import DataTap <EOL> class FileAwareSerializer ( Serializer ) : <EOL> def start_object ( self , obj ) : <EOL> return super ( FileAwareSerializer , self ) . start_object ( obj ) <EOL> class DocumentDataTap ( DataTap ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , instream = None , track_uncommitted = True , ** kwargs ) : <EOL> if track_uncommitted : <EOL> self . deserialized_objects = deque ( ) <EOL> else : <EOL> self . deserialized_objects = None <EOL> super ( DocumentDataTap , self ) . __init__ ( instream , ** kwargs ) <EOL> def get_domain ( self ) : <EOL> if self . instream is None : <EOL> return '<STR_LIT>' <EOL> if isinstance ( self . instream , ( list , tuple ) ) : <EOL> return '<STR_LIT>' <EOL> if self . instream . domain == '<STR_LIT>' : <EOL> return '<STR_LIT>' <EOL> def get_instance_stream ( self , instream ) : <EOL> for source in instream : <EOL> try : <EOL> is_doc = issubclass ( source , schema . Document ) <EOL> is_instance = False <EOL> except TypeError : <EOL> is_doc = False <EOL> is_instance = isinstance ( source , schema . Document ) <EOL> if is_doc : <EOL> queryset = source . objects . all ( ) . iterator ( ) <EOL> elif is_instance : <EOL> queryset = [ source ] <EOL> else : <EOL> if hasattr ( source , '<STR_LIT>' ) : <EOL> queryset = source . iterator ( ) <EOL> else : <EOL> queryset = source <EOL> for item in queryset : <EOL> yield item <EOL> def get_primitive_stream ( self , instream ) : <EOL> '''<STR_LIT>''' <EOL> serializer = FileAwareSerializer ( ) <EOL> instances = self . get_instance_stream ( instream ) <EOL> return serializer . serialize ( instances ) <EOL> def get_deserialized_document_stream ( self , instream ) : <EOL> '''<STR_LIT>''' <EOL> for deserialized_object in Deserializer ( instream ) : <EOL> if self . deserialized_objects is not None : <EOL> self . deserialized_objects . append ( deserialized_object ) <EOL> yield deserialized_object <EOL> def get_document_stream ( self , instream ) : <EOL> '''<STR_LIT>''' <EOL> for item in self . get_deserialized_document_stream ( instream ) : <EOL> item . save ( ) <EOL> yield item <EOL> def commit ( self ) : <EOL> while self . deserialized_objects : <EOL> instance = self . deserialized_objects . popleft ( ) <EOL> instance . save ( ) <EOL> self . deserialized_objects = None <EOL> for instance in self : <EOL> instance . save ( ) <EOL> @ classmethod <EOL> def load_from_command_line ( cls , arglist , instream = None ) : <EOL> '''<STR_LIT>''' <EOL> parser = OptionParser ( option_list = cls . command_option_list ) <EOL> options , args = parser . parse_args ( arglist ) <EOL> document_sources = list ( ) <EOL> for arg in args : <EOL> if '<STR_LIT:.>' in arg : <EOL> document_sources . append ( get_document ( * arg . rsplit ( "<STR_LIT:.>" , <NUM_LIT:1> ) ) ) <EOL> else : <EOL> document_sources . extend ( get_documents ( arg ) ) <EOL> return cls ( instream = document_sources , ** options . __dict__ ) <EOL> register_datatap ( '<STR_LIT>' , DocumentDataTap ) </s>
