<s> from __future__ import absolute_import <EOL> import os <EOL> os . environ . setdefault ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> import copy <EOL> import logging <EOL> import django <EOL> from api . webview . models import HarvesterResponse , Document , Version <EOL> from scrapi import events <EOL> from scrapi . util import json_without_bytes <EOL> from scrapi . linter import RawDocument , NormalizedDocument <EOL> from scrapi . processing import DocumentTuple <EOL> from scrapi . processing . base import BaseProcessor , BaseHarvesterResponse , BaseDatabaseManager <EOL> django . setup ( ) <EOL> logger = logging . getLogger ( __name__ ) <EOL> class DatabaseManager ( BaseDatabaseManager ) : <EOL> '''<STR_LIT>''' <EOL> def setup ( self ) : <EOL> return True <EOL> def tear_down ( self ) : <EOL> pass <EOL> def clear ( self , force = False ) : <EOL> pass <EOL> def celery_setup ( self , * args , ** kwargs ) : <EOL> pass <EOL> def paginated ( query , page_size = <NUM_LIT:10> ) : <EOL> for offset in range ( <NUM_LIT:0> , query . count ( ) , page_size ) : <EOL> for doc in query [ offset : offset + page_size ] : <EOL> yield doc <EOL> class PostgresProcessor ( BaseProcessor ) : <EOL> NAME = '<STR_LIT>' <EOL> manager = DatabaseManager ( ) <EOL> def documents ( self , * sources ) : <EOL> q = Document . objects . all ( ) <EOL> querysets = ( q . filter ( source = source ) for source in sources ) if sources else [ q ] <EOL> for query in querysets : <EOL> for doc in paginated ( query ) : <EOL> try : <EOL> raw = RawDocument ( doc . raw , clean = False , validate = False ) <EOL> except AttributeError as e : <EOL> logger . info ( '<STR_LIT>' . format ( e ) ) <EOL> raw = None <EOL> continue <EOL> normalized = NormalizedDocument ( doc . normalized , validate = False , clean = False ) if doc . normalized else None <EOL> yield DocumentTuple ( raw , normalized ) <EOL> def get ( self , source , docID ) : <EOL> try : <EOL> document = Document . objects . get ( source = source , docID = docID ) <EOL> except Document . DoesNotExist : <EOL> return None <EOL> raw = RawDocument ( document . raw , clean = False , validate = False ) <EOL> normalized = NormalizedDocument ( document . normalized , validate = False , clean = False ) if document . normalized else None <EOL> return DocumentTuple ( raw , normalized ) <EOL> def delete ( self , source , docID ) : <EOL> doc = Document . objects . get ( source = source , docID = docID ) <EOL> doc . delete ( ) <EOL> def create ( self , attributes ) : <EOL> attributes = json_without_bytes ( attributes ) <EOL> Document . objects . create ( <EOL> source = attributes [ '<STR_LIT:source>' ] , <EOL> docID = attributes [ '<STR_LIT>' ] , <EOL> providerUpdatedDateTime = None , <EOL> raw = attributes , <EOL> normalized = None <EOL> ) . save ( ) <EOL> @ property <EOL> def HarvesterResponseModel ( self ) : <EOL> return HarvesterResponseModel <EOL> @ events . logged ( events . PROCESSING , '<STR_LIT>' ) <EOL> def process_raw ( self , raw_doc ) : <EOL> document = self . version ( raw = raw_doc ) <EOL> timestamps = raw_doc . get ( '<STR_LIT>' ) <EOL> modified_doc = copy . deepcopy ( raw_doc . attributes ) <EOL> document . raw = modified_doc <EOL> document . timestamps = timestamps <EOL> document . save ( ) <EOL> @ events . logged ( events . PROCESSING , '<STR_LIT>' ) <EOL> def process_normalized ( self , raw_doc , normalized ) : <EOL> document = self . version ( raw = raw_doc , normalized = normalized ) <EOL> timestamps = raw_doc . get ( '<STR_LIT>' ) or normalized . get ( '<STR_LIT>' ) <EOL> document . raw = raw_doc . attributes <EOL> document . timestamps = timestamps <EOL> document . normalized = normalized . attributes <EOL> document . providerUpdatedDateTime = normalized [ '<STR_LIT>' ] <EOL> document . save ( ) <EOL> def _get_by_source_id ( self , source , docID ) : <EOL> try : <EOL> return Document . objects . get ( key = Document . _make_key ( source , docID ) ) <EOL> except Document . DoesNotExist : <EOL> return None <EOL> def version ( self , raw = None , normalized = None ) : <EOL> old_doc = self . _get_by_source_id ( raw [ '<STR_LIT:source>' ] , raw [ '<STR_LIT>' ] ) <EOL> if old_doc : <EOL> raw_changed = raw and self . different ( raw . attributes , old_doc . raw ) <EOL> norm_changed = normalized and self . different ( normalized . attributes , old_doc . normalized ) <EOL> version = Version ( <EOL> key = old_doc , <EOL> source = old_doc . source , <EOL> docID = old_doc . docID , <EOL> providerUpdatedDateTime = old_doc . providerUpdatedDateTime , <EOL> raw = old_doc . raw , <EOL> normalized = old_doc . normalized , <EOL> status = old_doc . status , <EOL> timestamps = old_doc . timestamps <EOL> ) <EOL> if raw_changed or norm_changed : <EOL> version . save ( ) <EOL> return old_doc <EOL> else : <EOL> return Document . objects . create ( source = raw [ '<STR_LIT:source>' ] , docID = raw [ '<STR_LIT>' ] ) <EOL> def get_versions ( self , source , docID ) : <EOL> doc = self . _get_by_source_id ( source , docID ) <EOL> for version in doc . version_set . all ( ) . order_by ( '<STR_LIT:id>' ) : <EOL> yield DocumentTuple ( <EOL> RawDocument ( version . raw , clean = False , validate = False ) , <EOL> NormalizedDocument ( version . normalized , clean = False , validate = False ) <EOL> ) <EOL> yield DocumentTuple ( <EOL> RawDocument ( doc . raw , clean = False , validate = False ) , <EOL> NormalizedDocument ( doc . normalized , clean = False , validate = False ) <EOL> ) <EOL> class HarvesterResponseModel ( BaseHarvesterResponse ) : <EOL> response = None <EOL> def __init__ ( self , * args , ** kwargs ) : <EOL> if kwargs : <EOL> key = kwargs [ '<STR_LIT>' ] . lower ( ) + kwargs [ '<STR_LIT:url>' ] . lower ( ) <EOL> self . response = HarvesterResponse ( key = key , * args , ** kwargs ) <EOL> else : <EOL> self . response = args [ <NUM_LIT:0> ] <EOL> @ property <EOL> def method ( self ) : <EOL> return str ( self . response . method ) <EOL> @ property <EOL> def url ( self ) : <EOL> return str ( self . response . url ) <EOL> @ property <EOL> def ok ( self ) : <EOL> return bool ( self . response . ok ) <EOL> @ property <EOL> def content ( self ) : <EOL> if isinstance ( self . response . content , memoryview ) : <EOL> return self . response . content . tobytes ( ) <EOL> if isinstance ( self . response . content , bytes ) : <EOL> return self . response . content <EOL> return str ( self . response . content ) <EOL> @ property <EOL> def encoding ( self ) : <EOL> return str ( self . response . encoding ) <EOL> @ property <EOL> def headers_str ( self ) : <EOL> return str ( self . response . headers_str ) <EOL> @ property <EOL> def status_code ( self ) : <EOL> return int ( self . response . status_code ) <EOL> @ property <EOL> def time_made ( self ) : <EOL> return str ( self . response . time_made ) <EOL> def save ( self , * args , ** kwargs ) : <EOL> self . response . save ( ) <EOL> return self <EOL> def update ( self , ** kwargs ) : <EOL> for k , v in kwargs . items ( ) : <EOL> setattr ( self . response , k , v ) <EOL> return self . save ( ) <EOL> @ classmethod <EOL> def get ( cls , url = None , method = None ) : <EOL> key = method . lower ( ) + url . lower ( ) <EOL> try : <EOL> return cls ( HarvesterResponse . objects . get ( key = key ) ) <EOL> except HarvesterResponse . DoesNotExist : <EOL> raise cls . DoesNotExist </s>
