<s> import six <EOL> import json <EOL> from abc import abstractproperty , abstractmethod <EOL> from requests . structures import CaseInsensitiveDict <EOL> class BaseProcessor ( object ) : <EOL> NAME = None <EOL> def process_raw ( self , raw_doc , ** kwargs ) : <EOL> pass <EOL> def process_normalized ( self , raw_doc , normalized , ** kwargs ) : <EOL> pass <EOL> @ abstractmethod <EOL> def documents ( self , * sources ) : <EOL> '''<STR_LIT>''' <EOL> raise NotImplementedError <EOL> @ abstractmethod <EOL> def get_versions ( self , source , docID ) : <EOL> raise NotImplementedError <EOL> def different ( self , old , new ) : <EOL> try : <EOL> return not all ( [ new [ key ] == old [ key ] or ( not new [ key ] and not old [ key ] ) for key in new . keys ( ) if key != '<STR_LIT>' ] ) <EOL> except Exception : <EOL> return True <EOL> class BaseDatabaseManager ( object ) : <EOL> '''<STR_LIT>''' <EOL> @ abstractmethod <EOL> def setup ( self ) : <EOL> '''<STR_LIT>''' <EOL> raise NotImplementedError <EOL> @ abstractmethod <EOL> def tear_down ( self ) : <EOL> '''<STR_LIT>''' <EOL> raise NotImplementedError <EOL> @ abstractmethod <EOL> def clear ( self , force = False ) : <EOL> '''<STR_LIT>''' <EOL> raise NotImplementedError <EOL> @ abstractmethod <EOL> def celery_setup ( self , * args , ** kwargs ) : <EOL> '''<STR_LIT>''' <EOL> raise NotImplementedError <EOL> class BaseHarvesterResponse ( object ) : <EOL> """<STR_LIT>""" <EOL> class DoesNotExist ( Exception ) : <EOL> pass <EOL> @ abstractproperty <EOL> def method ( self ) : <EOL> raise NotImplementedError <EOL> @ abstractproperty <EOL> def url ( self ) : <EOL> raise NotImplementedError <EOL> @ abstractproperty <EOL> def ok ( self ) : <EOL> raise NotImplementedError <EOL> @ abstractproperty <EOL> def content ( self ) : <EOL> raise NotImplementedError <EOL> @ abstractproperty <EOL> def encoding ( self ) : <EOL> raise NotImplementedError <EOL> @ abstractproperty <EOL> def headers_str ( self ) : <EOL> raise NotImplementedError <EOL> @ abstractproperty <EOL> def status_code ( self ) : <EOL> raise NotImplementedError <EOL> @ abstractproperty <EOL> def time_made ( self ) : <EOL> raise NotImplementedError <EOL> @ classmethod <EOL> @ abstractmethod <EOL> def get ( self , url = None , method = None ) : <EOL> raise NotImplementedError <EOL> @ abstractmethod <EOL> def save ( self ) : <EOL> raise NotImplementedError <EOL> @ abstractmethod <EOL> def update ( self , ** kwargs ) : <EOL> raise NotImplementedError <EOL> def json ( self ) : <EOL> try : <EOL> content = self . content . decode ( '<STR_LIT:utf-8>' ) <EOL> except AttributeError : <EOL> content = self . content <EOL> return json . loads ( content ) <EOL> @ property <EOL> def headers ( self ) : <EOL> return CaseInsensitiveDict ( json . loads ( self . headers_str ) ) <EOL> @ property <EOL> def text ( self ) : <EOL> return six . u ( self . content ) </s>
