<s> import logging <EOL> from enum import Enum <EOL> logger = logging . getLogger ( __name__ ) <EOL> class PreProcessSteps ( Enum ) : <EOL> tokenization = <NUM_LIT:1> <EOL> lemmatization = <NUM_LIT:6> <EOL> sentencer = <NUM_LIT:2> <EOL> tagging = <NUM_LIT:3> <EOL> ner = <NUM_LIT:4> <EOL> segmentation = <NUM_LIT:5> <EOL> syntactic_parsing = <NUM_LIT:7> <EOL> class PreProcessPipeline ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , step_runners , documents_manager ) : <EOL> """<STR_LIT>""" <EOL> from iepy . data . db import DocumentManager <EOL> self . step_runners = step_runners <EOL> if not isinstance ( documents_manager , DocumentManager ) : <EOL> documents_manager = DocumentManager ( documents_manager ) <EOL> self . documents = documents_manager <EOL> def walk_document ( self , doc ) : <EOL> """<STR_LIT>""" <EOL> for step in self . step_runners : <EOL> step ( doc ) <EOL> return <EOL> def process_step_in_batch ( self , runner ) : <EOL> """<STR_LIT>""" <EOL> logger . info ( '<STR_LIT>' , runner ) <EOL> if hasattr ( runner , '<STR_LIT>' ) and not ( runner . override or runner . increment ) : <EOL> docs = self . documents . get_documents_lacking_preprocess ( runner . step ) <EOL> else : <EOL> docs = self . documents <EOL> for i , doc in enumerate ( docs ) : <EOL> runner ( doc ) <EOL> logger . info ( '<STR_LIT>' , i + <NUM_LIT:1> ) <EOL> def process_everything ( self ) : <EOL> """<STR_LIT>""" <EOL> for runner in self . step_runners : <EOL> self . process_step_in_batch ( runner ) <EOL> class BasePreProcessStepRunner ( object ) : <EOL> def __init__ ( self , override = False , increment = False ) : <EOL> self . override = override <EOL> self . increment = increment <EOL> def __call__ ( self , doc ) : <EOL> raise NotImplementedError </s>
