streamcorpus_pipeline/_taggers.py
def make_ner_file(self, clean_visible_path, ner_xml_path):
        '''run tagger a child process to get XML output'''
        if self.template is None:
            raise exceptions.NotImplementedError('''
Subclasses must specify a class property "template" that provides
command string format for running a tagger.  It should take
%(tagger_root_path)s as the path from the config file,
%(clean_visible_path)s as the input XML file, and %(ner_xml_path)s as
the output path to create.
''')
        tagger_config = dict(
            tagger_root_path=self.config['tagger_root_path'],
            clean_visible_path=clean_visible_path,
            ner_xml_path=ner_xml_path)
        ## get a java_heap_size or default to 1GB
        tagger_config['java_heap_size'] = self.config.get('java_heap_size', '')
        cmd = self.template % tagger_config
        start_time = time.time()
        ## make sure we are using as little memory as possible
        gc.collect()
        try:
            self._child = subprocess.Popen(cmd, stderr=subprocess.PIPE, shell=True)
        except OSError, exc:
            msg = traceback.format_exc(exc)
            msg += make_memory_info_msg(clean_visible_path, ner_xml_path)
            raise PipelineOutOfMemory(msg)

        s_out, errors = self._child.communicate()

        if not self._child.returncode == 0:
            if 'java.lang.OutOfMemoryError' in errors:
                msg = errors + make_memory_info_msg(clean_visible_path, ner_xml_path)
                raise PipelineOutOfMemory(msg)
            elif self._child.returncode == 137:
                msg = 'tagger returncode = 137\n' + errors
                msg += make_memory_info_msg(clean_visible_path, ner_xml_path)
                # maybe get a tail of /var/log/messages
                raise PipelineOutOfMemory(msg)
            elif 'Exception' in errors:
                raise PipelineBaseException(errors)
            else:
                raise PipelineBaseException('tagger exited with %r' % self._child.returncode)

        elapsed = time.time() - start_time
        logger.info('finished tagging in %.1f seconds' % elapsed)
        return elapsed
def make_ner_file(self, clean_visible_path, ner_xml_path):
        '''run tagger a child process to get XML output'''
        if self.template is None:
            raise exceptions.NotImplementedError('''
Subclasses must specify a class property "template" that provides
command string format for running a tagger.  It should take
%(tagger_root_path)s as the path from the config file,
%(clean_visible_path)s as the input XML file, and %(ner_xml_path)s as
the output path to create.
''')
        tagger_config = dict(
            tagger_root_path=self.config['tagger_root_path'],
            clean_visible_path=clean_visible_path,
            ner_xml_path=ner_xml_path)
        ## get a java_heap_size or default to 1GB
        tagger_config['java_heap_size'] = self.config.get('java_heap_size', '')
        cmd = self.template % tagger_config
        start_time = time.time()
        ## make sure we are using as little memory as possible
        gc.collect()
        try:
            self._child = subprocess.Popen(cmd, stderr=subprocess.PIPE, shell=True)
        except OSError, exc:
            msg = traceback.format_exc(exc)
            msg += make_memory_info_msg(clean_visible_path, ner_xml_path)
            raise PipelineOutOfMemory(msg)

        s_out, errors = self._child.communicate()

        if not self._child.returncode == 0:
            if 'java.lang.OutOfMemoryError' in errors:
                msg = errors + make_memory_info_msg(clean_visible_path, ner_xml_path)
                raise PipelineOutOfMemory(msg)
            elif self._child.returncode == 137:
                msg = 'tagger returncode = 137\n' + errors
                msg += make_memory_info_msg(clean_visible_path, ner_xml_path)
                # maybe get a tail of /var/log/messages
                raise PipelineOutOfMemory(msg)
            elif 'Exception' in errors:
                raise PipelineBaseException(errors)
            else:
                raise PipelineBaseException('tagger exited with %r' % self._child.returncode)

        elapsed = time.time() - start_time
        logger.info('finished tagging in %.1f seconds' % elapsed)
        return elapsed
def make_ner_file ( self , clean_visible_path , ner_xml_path ) : if self . template is None : raise exceptions . NotImplementedError ( '''
Subclasses must specify a class property "template" that provides
command string format for running a tagger.  It should take
%(tagger_root_path)s as the path from the config file,
%(clean_visible_path)s as the input XML file, and %(ner_xml_path)s as
the output path to create.
''' ) tagger_config = dict ( tagger_root_path = self . config [ 'tagger_root_path' ] , clean_visible_path = clean_visible_path , ner_xml_path = ner_xml_path ) ## get a java_heap_size or default to 1GB tagger_config [ 'java_heap_size' ] = self . config . get ( 'java_heap_size' , '' ) cmd = self . template % tagger_config start_time = time . time ( ) ## make sure we are using as little memory as possible gc . collect ( ) try : self . _child = subprocess . Popen ( cmd , stderr = subprocess . PIPE , shell = True ) except OSError , exc : msg = traceback . format_exc ( exc ) msg += make_memory_info_msg ( clean_visible_path , ner_xml_path ) raise PipelineOutOfMemory ( msg ) s_out , errors = self . _child . communicate ( ) if not self . _child . returncode == 0 : if 'java.lang.OutOfMemoryError' in errors : msg = errors + make_memory_info_msg ( clean_visible_path , ner_xml_path ) raise PipelineOutOfMemory ( msg ) elif self . _child . returncode == 137 : msg = 'tagger returncode = 137\n' + errors msg += make_memory_info_msg ( clean_visible_path , ner_xml_path ) # maybe get a tail of /var/log/messages raise PipelineOutOfMemory ( msg ) elif 'Exception' in errors : raise PipelineBaseException ( errors ) else : raise PipelineBaseException ( 'tagger exited with %r' % self . _child . returncode ) elapsed = time . time ( ) - start_time logger . info ( 'finished tagging in %.1f seconds' % elapsed ) return elapsed
run tagger a child process to get XML output
run tagger a child process to get XML output