# Copyright (c) 2017, Matt Layman and contributors

from __future__ import print_function

from httpie.cli import parser
from httpie.context import Environment
from httpie.output import streams
from requests.models import Request
from werkzeug.wrappers import Response
from werkzeug.wrappers import Request as WerkzeugRequest

from httpony import __version__

def make_app():
    """Make a WSGI app that has all the HTTPie pieces baked in."""
    env = Environment()
    # STDIN is ignored because HTTPony runs a server that doesn't care.
    # Additionally, it is needed or else pytest blows up.
    args = parser.parse_args(args=['/', '--ignore-stdin'], env=env)
    args.output_options = 'HB'  # Output only requests.
    server = 'HTTPony/{0}'.format(__version__)

    def application(environ, start_response):
        # The WSGI server puts content length and type in the environment
        # even when not provided with the request. Drop them if they are empty.
        if environ.get('CONTENT_LENGTH') == '':
            del environ['CONTENT_LENGTH']
        if environ.get('CONTENT_TYPE') == '':
            del environ['CONTENT_TYPE']

        wrequest = WerkzeugRequest(environ)
        data = wrequest.get_data()
        request = Request(
            method=wrequest.method,
            url=wrequest.url,
            headers=wrequest.headers,
            data=data,
        )
        prepared = request.prepare()

        stream = streams.build_output_stream(
            args, env, prepared, response=None,
            output_options=args.output_options)
        streams.write_stream(stream, env.stdout, env.stdout_isatty)

        # When there is data in the request, give the next one breathing room.
        if data:
            print("\n", file=env.stdout)

        # Make dreams come true.
        response = Response(headers={'Server': server})
        return response(environ, start_response)

    return application


'''
Defines TaggerBatchTransform base class for wrapping taggers that
generate Tokens with automatic annotation.

This software is released under an MIT/X11 open source license.

Copyright 2012-2013 Diffeo, Inc.
'''
from __future__ import absolute_import
import collections
import exceptions
import gc
import itertools
import logging
import os
import shutil
import subprocess
import sys
import time
import traceback

import regex as re

import xml.dom.minidom

import streamcorpus
from streamcorpus import Chunk, Tagging, Label, OffsetType, add_annotation
from streamcorpus_pipeline._clean_visible import make_clean_visible_file, \
    cleanse
from sortedcollection import SortedCollection
from streamcorpus_pipeline._exceptions import PipelineOutOfMemory, \
    PipelineBaseException, InvalidStreamItem
import streamcorpus_pipeline._memory as _memory
import streamcorpus_pipeline.stages
from yakonfig import ConfigurationError

logger = logging.getLogger(__name__)

def make_chains_with_names(sentences):
    '''
    assemble in-doc coref chains by mapping equiv_id to tokens and
    their cleansed name strings

    :param sentences: iterator over token generators
    :returns dict:
        keys are equiv_ids,
        values are tuple(concatentated name string, list of tokens)
    '''
    ## if an equiv_id is -1, then the token is classified into some
    ## entity_type but has not other tokens in its chain.  We don't
    ## want these all lumped together, so we give them distinct "fake"
    ## equiv_id other than -1 -- counting negatively to avoid
    ## collisions with "real" equiv_ids
    fake_equiv_ids = -2

    ## use a default dictionary
    equiv_ids = collections.defaultdict(lambda: (set(), set()))

    for tagger_id, sents in sentences.items():
        for sent in sents:
            for tok in sent.tokens:
                if tok.entity_type is not None:

                    ## get an appropriate equiv_id
                    if tok.equiv_id == -1:
                        eqid = fake_equiv_ids
                        fake_equiv_ids -= 1
                    else:
                        eqid = tok.equiv_id

                    ## store the name parts initially as a set
                    equiv_ids[eqid][0].add(cleanse(tok.token.decode('utf8')))
                    ## carry a *reference* to the entire Token object
                    equiv_ids[eqid][1].add(tok)

    return equiv_ids

def ALL_mentions(target_mentions, chain_mentions):
    '''
    For each name string in the target_mentions list, searches through
    all chain_mentions looking for any cleansed Token.token that
    contains the name.  Returns True only if all of the target_mention
    strings appeared as substrings of at least one cleansed
    Token.token.  Otherwise, returns False.

    :type target_mentions: list of basestring
    :type chain_mentions: list of basestring

    :returns bool:
    '''
    found_all = True
    for name in target_mentions:
        found_one = False
        for chain_ment in chain_mentions:
            if name in chain_ment:
                found_one = True
                break
        if not found_one:
            found_all = False
            break
    return found_all

def ANY_MULTI_TOKEN_mentions(multi_token_target_mentions, chain_mentions):
    '''
    For each name string (potentially consisting of multiple tokens) in the
    target_mentions list, searches through all chain_mentions looking for any
    cleansed Token.token that contains all the tokens in the name.  Returns
    True only if all of the target_mention strings appeared as substrings of at
    least one cleansed Token.token.  Otherwise, returns False.

    :type target_mentions: list of basestring
    :type chain_mentions: list of basestring

    :returns bool:
    '''
    for multi_token_name in multi_token_target_mentions:
        if ALL_mentions(multi_token_name.split(), chain_mentions):
            return True
    return False

def ANY_mentions(target_mentions, chain_mentions):
    '''
    For each name string in the target_mentions list, searches through
    all chain_mentions looking for any cleansed Token.token that
    contains the name.  Returns True if any of the target_mention
    strings appeared as substrings of any cleansed Token.token.
    Otherwise, returns False.

    :type target_mentions: list of basestring
    :type chain_mentions: list of basestring

    :returns bool:
    '''
    for name in target_mentions:
        for chain_ment in chain_mentions:
            if name in chain_ment:
                return True
    return False

_CHAIN_SELECTORS = dict(
    ALL = ALL_mentions,
    ANY = ANY_mentions,
    ANY_MULTI_TOKEN = ANY_MULTI_TOKEN_mentions,
    )

def names_in_chains(stream_item, aligner_data):
    '''
    Convert doc-level Rating object into a Label, and add that Label
    to all Token in all coref chains identified by
    aligner_data["chain_selector"]

    :param stream_item: document that has a doc-level Rating to translate into token-level Labels.
    :param aligner_data: dict containing:
      chain_selector: ALL or ANY
      annotator_id: string to find at stream_item.Ratings[i].annotator.annotator_id

    If chain_selector==ALL, then only apply Label to chains in which
    all of the Rating.mentions strings appear as substrings within at
    least one of the Token.token strings.

    If chain_selector==ANY, then apply Label to chains in which any of
    the Rating.mentions strings appear as a substring within at least
    one of the Token.token strings.

    If chain_selector==ANY_MULTI_TOKEN, then apply Label to chains in which all
    the names in any of the Rating.mentions strings appear as a substring within at least
    one of the Token.token strings.
    '''
    chain_selector = aligner_data.get('chain_selector', '')
    assert chain_selector in _CHAIN_SELECTORS, \
        'chain_selector: %r not in %r' % (chain_selector, _CHAIN_SELECTORS.keys())

    ## convert chain_selector to a function
    chain_selector = _CHAIN_SELECTORS[chain_selector]

    ## make inverted index equiv_id --> (names, tokens)
    equiv_ids = make_chains_with_names( stream_item.body.sentences )

    required_annotator_id = aligner_data.get('annotator_id')

    for annotator_id, ratings in stream_item.ratings.items():
        if (required_annotator_id is not None) and (annotator_id != required_annotator_id):
            continue
        else:
            for rating in ratings:
                label = Label(annotator=rating.annotator,
                              target=rating.target)

                for eqid, (chain_mentions, chain_tokens) in equiv_ids.items():
                    if chain_selector(rating.mentions, chain_mentions):
                        ## apply the label
                        for tok in chain_tokens:
                            add_annotation(tok, label)

                ## stream_item passed by reference, so nothing to return

def _get_tagger_id(stream_item, aligner_data):
    available_taggings = stream_item.body.sentences.keys()
    if len(available_taggings) == 1:
        default_tagger_id = available_taggings[0]
    else:
        default_tagger_id = None
    tagger_id = aligner_data.get('tagger_id', default_tagger_id)
    if not tagger_id:
        raise Exception('need tagger_id, available taggings: %r', available_taggings)
    return tagger_id

def line_offset_labels(stream_item, aligner_data):
    ## get a set of tokens -- must have OffsetType.LINES in them.
    tagger_id = _get_tagger_id(stream_item, aligner_data)
    sentences = stream_item.body.sentences.get(tagger_id, [])

    required_annotator_id = aligner_data.get('annotator_id')

    ## if labels on ContentItem, then make labels on Tokens
    for annotator_id in stream_item.body.labels:
        if (required_annotator_id is not None) and (annotator_id != required_annotator_id):
            continue
        for label in stream_item.body.labels[annotator_id]:

            ## remove the offset from the label, because we are
            ## putting it into the token
            label_off = label.offsets.pop(OffsetType.LINES)

            assert label_off.length == len(label_off.value.split('\n'))
            #print 'L: %d\t%r\t%r' % (label_off.first, label_off.value,
            #    '\n'.join(hope_original.split('\n')[label_off.first:
            #         label_off.first+label_off.length]))

            ## These next few steps are probably the most
            ## memory intensive, because they fully
            ## instantiate all the tokens.
            token_collection = SortedCollection(
                itertools.chain(*[sent.tokens for sent in sentences]),
                key=lambda tok: tok.offsets[OffsetType.LINES].first
                )

            toks = token_collection.find_range(
                    label_off.first, label_off.first + label_off.length)

            for tok in toks:
                add_annotation(tok, label)

                ## only for debugging
                if not tok.token or tok.token not in label_off.value:
                    raise InvalidStreamItem(
                        '%r not in %r' %
                        ([(t.offsets[OffsetType.LINES].first, t.token)
                          for t in toks],
                         label_off.value))

def byte_offset_labels(stream_item, aligner_data):
    _offset_labels(stream_item, aligner_data, offset_type='BYTES')

def char_offset_labels(stream_item, aligner_data):
    _offset_labels(stream_item, aligner_data, offset_type='CHARS')

def _offset_labels(stream_item, aligner_data, offset_type='BYTES'):
    ## get a set of tokens -- must have OffsetType.<offset_type> type offsets.

    otype_str = offset_type
    offset_type = OffsetType._NAMES_TO_VALUES[offset_type]

    tagger_id = _get_tagger_id(stream_item, aligner_data)
    sentences = stream_item.body.sentences.get(tagger_id, [])

    ## These next few steps are probably the most
    ## memory intensive, because they fully
    ## instantiate all the tokens.

    token_collection = SortedCollection(
        itertools.chain(*[sent.tokens for sent in sentences]),
        key=lambda tok: tok.offsets[offset_type].first
        )

    required_annotator_id = aligner_data.get('annotator_id')

    ## if labels on ContentItem, then make labels on Tokens
    for annotator_id in stream_item.body.labels:
        if (required_annotator_id is not None) and (annotator_id != aligner_data['annotator_id']):
            continue
        for label in stream_item.body.labels[annotator_id]:
            if offset_type not in label.offsets:
                # This was causing a very small number of jobs to fail in
                # the WLC dataset. Namely, the expected offset type did not
                # exist in `label.offsets`. (There was a byte offset but not
                # a char offset, even though the `char_offset_align_labels`
                # was used.) I don't understand enough about the system
                # to know why the expected offset type didn't exist, but it was
                # happening in few enough cases that I did feel comfortable
                # just ignoring it.
                # ---AG
                found_types = [OffsetType._VALUES_TO_NAMES.get(k, '???')
                               for k in label.offsets.keys()]
                logger.warn('Could not find offset type "%s" in label '
                            'offsets. Found %s types in stream item %r'
                            % (otype_str, found_types, stream_item.stream_id))
                continue

            ## remove the offset from the label, because we are
            ## putting it into the token
            label_off = label.offsets.pop( offset_type )

            if offset_type == OffsetType.CHARS:
                assert label_off.length == len(label_off.value.decode('utf8')), (label_off.length, repr(label_off.value))
            else:
                ## had better by BYTES
                assert label_off.length == len(label_off.value), (label_off.length, repr(label_off.value))

            #print 'L: %d\t%r\t%r' % (label_off.first, label_off.value,
            #                         '\n'.join(hope_original.split('\n')[label_off.first:label_off.first+label_off.length]))

            #print 'tc %d %r' % (len(token_collection), token_collection._keys)
            #print 'label_off.first=%d, length=%d, value=%r' % (label_off.first, label_off.length, label_off.value)

            toks = token_collection.find_range(
                    label_off.first, label_off.first + label_off.length)

            #print "find_le: ", token_collection.find_le(label_off.first)

            toks = list(toks)
            #print 'aligned tokens', toks

            for tok in toks:
                add_annotation(tok, label)

                ## only for debugging
                assert tok.token is not None, tok.token

                '''
                Cannot do the check below, because Serif actually *changes* the characters that it puts in Token.token....
                InvalidStreamItem: [(55701, '2002'), (55706, 'coup'), (55711, "d'etat")] not in '2002 coup d\xe2\x80\x99etat'

                if tok.token not in label_off.value and \
                   cleanse(tok.token.decode('utf8')) not in \
                   cleanse(label_off.value.decode('utf8')):
                    raise InvalidStreamItem(
                        '%r not in %r' %
                        ([(t.offsets[offset_type].first, t.token)
                          for t in toks],
                         label_off.value))
                '''

def look_ahead_match(rating, tokens):
    '''iterate through all tokens looking for matches of cleansed tokens
    or token regexes, skipping tokens left empty by cleansing and
    coping with Token objects that produce multiple space-separated
    strings when cleansed.  Yields tokens that match.

    '''
    ## this ensures that all cleansed tokens are non-zero length
    all_mregexes = []
    for m in rating.mentions:
        mregexes = []
        mpatterns = m.decode('utf8').split(' ')
        for mpat in mpatterns:
            if mpat.startswith('ur"^') and mpat.endswith('$"'): # is not regex
                ## chop out the meat of the regex so we can reconstitute it below
                mpat = mpat[4:-2]
            else:
                mpat = cleanse(mpat)
            if mpat:
                ## make a unicode raw string
                ## https://docs.python.org/2/reference/lexical_analysis.html#string-literals
                mpat = ur'^%s$' % mpat
                logger.debug('look_ahead_match compiling regex: %s', mpat)
                mregexes.append(re.compile(mpat, re.UNICODE | re.IGNORECASE))

        if not mregexes:
            logger.warn('got empty cleansed mention: %r\nrating=%r' % (m, rating))

        all_mregexes.append(mregexes)

    ## now that we have all_mregexes, go through all the tokens
    for i in range(len(tokens)):
        for mregexes in all_mregexes:
            if mregexes[0].match(tokens[i][0][0]):
                ## found the start of a possible match, so iterate
                ## through the tuples of cleansed strings for each
                ## Token while stepping through the cleansed strings
                ## for this mention.
                m_j = 1
                i_j = 0
                last_token_matched = 0
                matched = True
                while m_j < len(mregexes):
                    i_j += 1
                    if i_j == len(tokens[i + last_token_matched][0]):
                        i_j = 0
                        last_token_matched += 1
                        if i + last_token_matched == len(tokens):
                            matched = False
                            break
                    target_token = tokens[i + last_token_matched][0][i_j]
                    ## this next line is the actual string comparison
                    if mregexes[m_j].match(target_token):
                        m_j += 1
                    elif target_token == '':
                        continue
                    else:
                        matched = False
                        break
                if matched:
                    ## yield each matched token only once
                    toks = set()
                    for j in xrange(last_token_matched + 1):
                        toks.add(tokens[i + j][1])
                    for tok in toks:
                        yield tok

def multi_token_match(stream_item, aligner_data):
    '''
    iterate through tokens looking for near-exact matches to strings
    in si.ratings...mentions
    '''
    tagger_id = _get_tagger_id(stream_item, aligner_data)
    sentences = stream_item.body.sentences.get(tagger_id)
    if not sentences:
        return
    ## construct a list of tuples, where the first part of each tuple
    ## is a tuple of cleansed strings, and the second part is the
    ## Token object from which it came.
    tokens = map(lambda tok: (cleanse(tok.token.decode('utf8')).split(' '), tok),
                 itertools.chain(*[sent.tokens for sent in sentences]))
    required_annotator_id = aligner_data['annotator_id']
    for annotator_id, ratings in stream_item.ratings.items():
        if (required_annotator_id is None) or (annotator_id == required_annotator_id):
            for rating in ratings:
                label = Label(annotator=rating.annotator,
                              target=rating.target)

                num_tokens_matched = 0
                for tok in look_ahead_match(rating, tokens):
                    if aligner_data.get('update_labels'):
                        tok.labels.pop(annotator_id, None)
                    add_annotation(tok, label)
                    num_tokens_matched += 1

                if num_tokens_matched == 0:
                    logger.warning('multi_token_match didn\'t actually match '
                                   'entity %r in stream_id %r',
                                   rating.target.target_id,
                                   stream_item.stream_id)
                else:
                    logger.debug('matched %d tokens for %r in %r',
                                 num_tokens_matched, rating.target.target_id,
                                 stream_item.stream_id)

                ## stream_item passed by reference, so nothing to return

def make_memory_info_msg(clean_visible_path=None, ner_xml_path=None):
    msg = 'reporting memory while running on:\n%r\n%r' % (clean_visible_path, ner_xml_path)
    msg += 'VmSize: %d bytes' % _memory.memory()
    msg += 'VmRSS:  %d bytes' % _memory.resident()
    msg += 'VmStk:  %d bytes' % _memory.stacksize()
    msg += 'uncollectable garbage: %r' % gc.garbage
    msg += 'gc.get_count() = %r' % repr(gc.get_count())
    ## Do not do gc.get_objects, because it causes nltk to
    ## load stuff that is not actually in memory yet!
    #msg += 'current objects: %r' % gc.get_objects()
    return msg

AlignmentStrategies = {
    'names_in_chains': names_in_chains,
    #'line_offset_labels': line_offset_labels,
    'byte_offset_labels': byte_offset_labels,
    'char_offset_labels': char_offset_labels,
    'multi_token_match': multi_token_match,
    }

def align_labels(t_path1, config):
    if not ('align_labels_by' in config and config['align_labels_by']):
        return
    assert 'aligner_data' in config, 'config missing "aligner_data"'
    aligner = AlignmentStrategies[ config['align_labels_by'] ]
    _aligner_core(t_path1, aligner, config['aligner_data'])

def _aligner_core(t_path1, aligner, aligner_data):
    t_chunk1 = Chunk(t_path1, mode='rb')
    t_path2 = t_path1 + '-tmp-aligning'
    t_chunk2 = Chunk(t_path2, mode='wb')
    for si in t_chunk1:
        aligner( si, aligner_data )
        t_chunk2.add(si)
    t_chunk1.close()
    t_chunk2.close()

    if aligner_data.get('cleanup_tmp_files', True):
        os.rename(t_path2, t_path1)
    else:
        # for development, leave intermediate tmp file
        shutil.copy(t_path2, t_path1)

class _aligner_batch_transform(streamcorpus_pipeline.stages.BatchTransform, streamcorpus_pipeline.stages.IncrementalTransform):
    # aligner needs to be a single element tuple containing a function pointer,
    # because otherwise a function pointer assigned at class definition scope becomes
    # "bound" to the class and will be called with a suprious first 'self' argument.
    aligner = None

    def process_path(self, chunk_path):
        # implement BatchTransform
        _aligner_core(chunk_path, self.aligner[0], self.config)

    def process_item(self, stream_item, context):
        # implement IncrementalTransform
        aligner = self.aligner[0]
        aligner(stream_item, self.config)
        return stream_item

    def shutdown(self):
        pass

class name_align_labels(_aligner_batch_transform):
    '''
    requires config['chain_selector'] = ALL | ANY
    requires config['annotator_id'] (which person/org did manual labelling)
    '''
    config_name = 'name_align_labels'
    default_config = {
        'chain_selector': 'ALL'
    }
    @staticmethod
    def check_config(config, name):
        if 'chain_selector' not in config:
            raise ConfigurationError('{0} requires chain_selector'.format(name))
        if config['chain_selector'] not in _CHAIN_SELECTORS:
            raise ConfigurationError('chain_selector must be one of: {0!r}'.format(_CHAIN_SELECTORS.keys()))
    aligner = (names_in_chains,)

class line_offset_align_labels(_aligner_batch_transform):
    '''
    requires config['annotator_id'] (which person/org did manual labelling)
    requires config['tagger_id'] (which software did tagging to process)
    '''
    config_name = 'line_offset_align_labels'
    aligner = (line_offset_labels,)

class byte_offset_align_labels(_aligner_batch_transform):
    '''
    requires config['annotator_id'] (which person/org did manual labelling)
    requires config['tagger_id'] (which software did tagging to process)
    '''
    config_name = 'byte_offset_align_labels'
    aligner = (byte_offset_labels,)

class char_offset_align_labels(_aligner_batch_transform):
    '''
    requires config['annotator_id'] (which person/org did manual labelling)
    requires config['tagger_id'] (which software did tagging to process)
    '''
    config_name = 'char_offset_align_labels'
    aligner = (char_offset_labels,)

class multi_token_match_align_labels(_aligner_batch_transform):
    '''
    requires config['annotator_id'] (which person/org did manual labelling)
    requires config['tagger_id'] (which software did tagging to process)
    '''
    config_name = 'multi_token_match_align_labels'
    aligner = (multi_token_match,)

class TaggerBatchTransform(streamcorpus_pipeline.stages.BatchTransform):
    '''
    streamcorpus.pipeline.TaggerBatchTransform provides a structure for
    aligning a taggers output with labels and generating
    stream_item.sentences[tagger_id] = [Sentence]
    '''
    template = None

    def __init__(self, *args, **kwargs):
        super(TaggerBatchTransform, self).__init__(*args, **kwargs)
        self._child = None
        self.config['tagger_root_path'] = \
            os.path.join(self.config['third_dir_path'],
                         self.config['path_in_third'])

    def process_path(self, chunk_path):
        ## make temporary file paths based on chunk_path
        clean_visible_path = chunk_path + '-clean_visible.xml'
        ner_xml_path       = chunk_path + '-ner.xml'

        ## process the chunk's clean_visible data into xml
        i_chunk = Chunk(path=chunk_path, mode='rb')
        make_clean_visible_file(i_chunk, clean_visible_path)

        ## make sure holding nothing that consumes memory
        i_chunk = None

        ## generate an output file from the tagger
        self.make_ner_file(clean_visible_path, ner_xml_path)

        ## make a new output chunk at a temporary path
        tmp_chunk_path     = chunk_path + '_'
        o_chunk = Chunk(path=tmp_chunk_path, mode='wb')

        ## re-open i_chunk
        i_chunk = Chunk(path=chunk_path, mode='rb')

        ## fuse the output file with i_chunk to make o_chunk
        self.align_chunk_with_ner(ner_xml_path, i_chunk, o_chunk)

        ## clean up temp files
        if self.config['cleanup_tmp_files']:
            os.remove(clean_visible_path)
            os.remove(ner_xml_path)

        ## atomic rename new chunk file into place
        os.rename(tmp_chunk_path, chunk_path)

    ## gets called by self.__call__
    def make_ner_file(self, clean_visible_path, ner_xml_path):
        '''run tagger a child process to get XML output'''
        if self.template is None:
            raise exceptions.NotImplementedError('''
Subclasses must specify a class property "template" that provides
command string format for running a tagger.  It should take
%(tagger_root_path)s as the path from the config file,
%(clean_visible_path)s as the input XML file, and %(ner_xml_path)s as
the output path to create.
''')
        tagger_config = dict(
            tagger_root_path=self.config['tagger_root_path'],
            clean_visible_path=clean_visible_path,
            ner_xml_path=ner_xml_path)
        ## get a java_heap_size or default to 1GB
        tagger_config['java_heap_size'] = self.config.get('java_heap_size', '')
        cmd = self.template % tagger_config
        start_time = time.time()
        ## make sure we are using as little memory as possible
        gc.collect()
        try:
            self._child = subprocess.Popen(cmd, stderr=subprocess.PIPE, shell=True)
        except OSError, exc:
            msg = traceback.format_exc(exc)
            msg += make_memory_info_msg(clean_visible_path, ner_xml_path)
            raise PipelineOutOfMemory(msg)

        s_out, errors = self._child.communicate()

        if not self._child.returncode == 0:
            if 'java.lang.OutOfMemoryError' in errors:
                msg = errors + make_memory_info_msg(clean_visible_path, ner_xml_path)
                raise PipelineOutOfMemory(msg)
            elif self._child.returncode == 137:
                msg = 'tagger returncode = 137\n' + errors
                msg += make_memory_info_msg(clean_visible_path, ner_xml_path)
                # maybe get a tail of /var/log/messages
                raise PipelineOutOfMemory(msg)
            elif 'Exception' in errors:
                raise PipelineBaseException(errors)
            else:
                raise PipelineBaseException('tagger exited with %r' % self._child.returncode)

        elapsed = time.time() - start_time
        logger.info('finished tagging in %.1f seconds' % elapsed)
        return elapsed

        #print '%.1f sec --> %.1f StreamItems/second' % (elapsed, rate)

    ## gets called by self.__call__
    def align_chunk_with_ner(self, ner_xml_path, i_chunk, o_chunk):
        ''' iterate through ner_xml_path to fuse with i_chunk into o_chunk '''
        ## prepare to iterate over the input chunk
        input_iter = i_chunk.__iter__()

        all_ner = xml.dom.minidom.parse(open(ner_xml_path))

        ## this converts our UTF-8 data into unicode strings, so when
        ## we want to compute byte offsets or construct tokens, we
        ## must .encode('utf8')
        for ner_dom in all_ner.getElementsByTagName('FILENAME'):
        #for stream_id, raw_ner in files(open(ner_xml_path).read().decode('utf8')):

            stream_item = input_iter.next()

            ## get stream_id out of the XML
            stream_id = ner_dom.attributes.get('stream_id').value
            if stream_item.stream_id is None:
                assert not stream_id, 'out of sync: None != %r' % stream_id
                logger.critical('si.stream_id is None... ignoring')
                continue
            assert stream_id and stream_id == stream_item.stream_id, \
                '%s != %s' % (stream_id, stream_item.stream_id)

            if not stream_item.body:
                ## the XML better have had an empty clean_visible too...
                #assert not ner_dom....something
                continue

            tagging = Tagging()
            tagging.tagger_id = self.tagger_id  # pylint: disable=E1101

            '''
            ## get this one file out of its FILENAME tags
            tagged_doc_parts = list(files(ner_dom.toxml()))
            if not tagged_doc_parts:
                continue

            tagged_doc = tagged_doc_parts[0][1]

            ## hack
            hope_original = make_clean_visible(tagged_doc, '')
            open(ner_xml_path + '-clean', 'wb').write(hope_original.encode('utf-8'))
            print ner_xml_path + '-clean'
            '''

            #tagging.raw_tagging = tagged_doc
            tagging.generation_time = streamcorpus.make_stream_time()
            stream_item.body.taggings[self.tagger_id] = tagging       # pylint: disable=E1101

            ## could consume lots of memory here by instantiating everything
            sentences, relations, attributes = self.get_sentences(ner_dom)
            stream_item.body.sentences[self.tagger_id] = sentences    # pylint: disable=E1101
            stream_item.body.relations[self.tagger_id] = relations    # pylint: disable=E1101
            stream_item.body.attributes[self.tagger_id] = attributes  # pylint: disable=E1101

            logger.debug('finished aligning tokens %s' % stream_item.stream_id)

            '''
            for num, sent in enumerate(sentences):
                for tok in sent.tokens:
                    print '%d\t%d\t%s' % (num, tok.offsets[OffsetType.LINES].first, repr(tok.token))
            '''

            if 'align_labels_by' in self.config and self.config['align_labels_by']:
                assert 'aligner_data' in self.config, 'config missing "aligner_data"'
                aligner = AlignmentStrategies[ self.config['align_labels_by'] ]
                aligner( stream_item, self.config['aligner_data'] )

            ## forcibly collect dereferenced objects
            gc.collect()

            try:
                o_chunk.add(stream_item)
            except MemoryError, exc:
                msg = traceback.format_exc(exc)
                msg += make_memory_info_msg()
                logger.critical(msg)
                raise PipelineOutOfMemory(msg)

        ## all done, so close the o_chunk
        try:
            o_chunk.close()
            logger.info('finished chunk for %r' % ner_xml_path)
        except MemoryError, exc:
            msg = traceback.format_exc(exc)
            msg += make_memory_info_msg()
            logger.critical(msg)
            raise PipelineOutOfMemory(msg)

    def get_sentences(self, ner_dom):
        '''parse the sentences and tokens out of the XML'''
        raise exceptions.NotImplementedError

    def shutdown(self):
        '''
        send SIGTERM to the tagger child process
        '''
        if self._child:
            try:
                self._child.terminate()
            except OSError, exc:
                if exc.errno == 3:
                    ## child is already gone, possibly because it ran
                    ## out of memory and caused us to shutdown
                    pass


from operator import add, div, mul, neg

from _ppeg import Pattern as P

def pattprint(pattern):
    print pattern.env()
    pattern.display()

mt = P(1)
ANY = P(1)

predef = {
    'nl': P('\n'),
}

def getdef(name, defs):
    c = defs and defs[name]
    return c

def patt_error(s, i):
    msg = (len(s) < i + 20) and s[i] or s[i:i+20] + '...'
    msg = "pattern error near '%s'" % (msg, )
    raise Exception(msg)

def mult(p, n):
    """Returns a Pattern that matches exactly n repetitions of Pattern p.
    """
    np = P()
    while n >= 1:
        if n % 2:
            np = np + p
        p = p + p
        n = n // 2
    return np

def equalcap(s, i, (c,)):
    if not isinstance(c, str):
        return None
    e = len(c) + i
    if s[i: e] == c:
        return e
    else:
        return None

S = (P.Set(" \t\n") | '--' + (ANY - P.Set("\n"))**0)**0

name = P.Range("AZaz") + P.Range("AZaz09")**0
exp_follow = P('/') | P(")") | P("}") | P(":}") | P("~}") | name | -1
name = P.Cap(name)

Identifier = name + P.CapA(1)

num = (P.Cap(P.Range("09")**1) + S) / int

String = (("'" + P.Cap((ANY - "'")**0) + "'") |
          ('"' + P.Cap((ANY - '"')**0) + '"'))

def getcat(c, defs):
    cat = defs.get(c, predef.get(c))
    if not cat:
        raise Exception('name %s undefined' % (c,))
    return cat

Cat = ('%' + Identifier) / getcat

Range   = P.CapS(ANY + (P("-")/"") + (ANY-"]")) / P.Range

item    = Cat | Range | P.Cap(ANY)

def f(c, p):
    if c == "^":
        return ANY - p
    else:
        return p

Class   = ( ("[" + P.Cap(P("^")**-1) # optional complement symbol
                 + P.CapF(item + (item - "]")**0, mt.__or__)) / f
           ) + "]"

def adddef(d, k, defs, exp):
    if d.get(k):
        raise Exception("'%s' already defined as a rule" % k)
    d[k] = exp
    return d

def firstdef(n, defs, r):
    return adddef({}, k, defs, r)

def abf(a, b, f):
    return f(a, b)

def np(n, p):
    return P.CapG(p, n)

exp = P.Grammar(
    # 0 Exp
    (S + ( P.Var(6)
        | P.CapF(P.Var(1) + ('/' + S + P.Var(1))**0, mt.__or__) )),

    # 1 Seq
    (P.CapF(P.CapC(P("")) + P.Var(2)**0, mt.__add__)
    + (+exp_follow | patt_error)),

    # 2 Prefix
    ( ("&" + S + P.Var(2)) / mt.__pos__
    | ("!" + S + P.Var(2)) / mt.__neg__
    | P.Var(3)),

    # 3 Suffix
    (P.CapF(P.Var(4) + S +
           ( ( P("+") + P.CapC(1, mt.__pow__)
             | P("*") + P.CapC(0, mt.__pow__)
             | P("?") + P.CapC(-1, mt.__pow__)
             | "^" + ( P.CapG(num + P.CapC(mult))
                     | P.CapG(P.Cap(P.Set("+-") + P.Range("09")**1)
                              + P.CapC(mt.__pow__))
                     )
             | "->" + S + ( P.CapG(String + P.CapC(mt.__div__))
                          | P("{}") + P.CapC(None, P.CapT)
                          | P.CapG(Identifier / getdef + P.CapC(mt.__div__))
                          )
             | "=>" + S + P.CapG(Identifier / getdef + P.CapC(P.CapRT))
             ) + S
            )**0, abf)),

    # 4 Primary
    ("(" + P.Var(0) + ")"
    | String / P
    | Class
    | Cat
    | ("{:" + (name + ":" | P.CapC(None)) + P.Var(0) + ":}") / np
    | ("=" + name) / (lambda n: CapRT(CapB(n), equalcap))
    | P("{}") / P.CapP
    | ("{~" + P.Var(0) + "~}") / P.CapS
    | ("{"  + P.Var(0) +  "}") / P.Cap
    | P(".") + P.CapC(ANY)
    | ("<" + name + ">") / P.Var),
    # 5 Definition
    (Identifier + S + '<-' + P.Var(0)),
    # 6 Grammar
    (P.CapF(P.Var(5) / firstdef + P.CapG(P.Var(5))**0, adddef) / P.Grammar),
)
#pattprint(exp)
pattern = (S + exp) / P + (-ANY | patt_error)

def compile(p, defs=None):
    m = pattern(p, defs)
    return m.captures[0]

balanced = compile('balanced <- "(" ([^()] / <balanced>)* ")"')
if __name__ == '__main__':
    print '(hello())', balanced('(hello())').pos


'''Protect emails <user@email.com> from tag strippers.

.. This software is released under an MIT/X11 open source license.
   Copyright 2012-2015 Diffeo, Inc.

'''
import re
from uuid import uuid4

bracket_emails = re.compile(
    '''<[\w-]+(?:\.[\w-]+)*@(?:[\w-]+\.)+[a-zA-Z0-9]{2,7}>''',
    re.I)

def fix_emails(text):
    '''Replace all angle bracket emails with a unique key.'''
    emails = bracket_emails.findall(text)

    keys = []
    for email in emails:
	_email = email.replace("<","<").replace(">",">")
        text = text.replace(email, _email)

    return text


'''Pipeline transform for :mod:`nltk`.

.. This software is released under an MIT/X11 open source license.
   Copyright 2012-2015 Diffeo, Inc.

.. autoclass:: nltk_tokenizer

'''
from __future__ import absolute_import
import logging

from nltk.tokenize import WhitespaceTokenizer
from nltk.tokenize.punkt import PunktSentenceTokenizer

from sortedcollection import SortedCollection
import streamcorpus
from streamcorpus import Token, Sentence, Offset, OffsetType
from streamcorpus_pipeline.stages import IncrementalTransform

logger = logging.getLogger(__name__)

class nltk_tokenizer(IncrementalTransform):
    '''Minimal tokenizer using :mod:`nltk`.

    This is an incremental transform that creates minimal tokens and
    does minimal label alignment.  The stream item must have a
    `clean_visible` part already.  The stream item is tagged with
    ``nltk_tokenizer``.

    This tokenizer has very few dependencies, only the :mod:`nltk`
    Python package.  However, the tokens that it creates are
    extremely minimal, with no part-of-speech details or other
    information.  The tokens only contain the `token` property and
    character-offset information.

    If there are labels with character offsets attached to the stream
    item body, and this stage is configured with an `annotator_id`,
    then this stage attempts to attach those labels to tokens.

    '''
    config_name = 'nltk_tokenizer'
    tagger_id = 'nltk_tokenizer'

    def __init__(self, config):
        super(nltk_tokenizer, self).__init__(config)
        self.sentence_tokenizer = PunktSentenceTokenizer()
        self.word_tokenizer = WhitespaceTokenizer()  # PunktWordTokenizer()
        self.annotator_id = self.config.get('annotator_id', None)

    def _sentences(self, clean_visible):
        'generate strings identified as sentences'
        previous_end = 0
        clean_visible = clean_visible.decode('utf8')
        for start, end in self.sentence_tokenizer.span_tokenize(clean_visible):
            # no need to check start, because the first byte of text
            # is always first byte of first sentence, and we will
            # have already made the previous sentence longer on the
            # end if there was an overlap.
            if start < previous_end:
                start = previous_end
                if start > end:
                    # skip this sentence... because it was eaten by
                    # an earlier sentence with a label
                    continue
            try:
                label = self.label_index.find_le(end)
            except ValueError:
                label = None
            if label:
                ## avoid splitting a label
                off = label.offsets[OffsetType.CHARS]
                end = max(off.first + off.length, end)
            previous_end = end
            sent_str = clean_visible[start:end]
            yield start, end, sent_str

    def make_label_index(self, stream_item):
        'make a sortedcollection on body.labels'
        labels = stream_item.body.labels.get(self.annotator_id)
        if not labels:
            labels = []

        self.label_index = SortedCollection(
            [l for l in labels if OffsetType.CHARS in l.offsets],
            key=lambda label: label.offsets[OffsetType.CHARS].first)

    def make_sentences(self, stream_item):
        'assemble Sentence and Token objects'
        self.make_label_index(stream_item)
        sentences = []
        token_num = 0
        new_mention_id = 0
        for sent_start, sent_end, sent_str in self._sentences(
                stream_item.body.clean_visible):
            assert isinstance(sent_str, unicode)
            sent = Sentence()
            sentence_pos = 0
            for start, end in self.word_tokenizer.span_tokenize(sent_str):
                token_str = sent_str[start:end].encode('utf8')
                tok = Token(
                    token_num=token_num,
                    token=token_str,
                    sentence_pos=sentence_pos,
                )
                tok.offsets[OffsetType.CHARS] = Offset(
                    type=OffsetType.CHARS,
                    first=sent_start + start,
                    length=end - start,
                )
                # whitespace tokenizer will never get a token
                # boundary in the middle of an 'author' label
                try:
                    label = self.label_index.find_le(sent_start + start)
                except ValueError:
                    label = None
                if label:
                    off = label.offsets[OffsetType.CHARS]
                    if off.first + off.length > sent_start + start:
                        streamcorpus.add_annotation(tok, label)
                        logger.debug('adding label to tok: %r has %r',
                                     tok.token, label.target.target_id)

                        if label in self.label_to_mention_id:
                            mention_id = self.label_to_mention_id[label]
                        else:
                            mention_id = new_mention_id
                            new_mention_id += 1
                            self.label_to_mention_id[label] = mention_id

                        tok.mention_id = mention_id

                token_num += 1
                sentence_pos += 1
                sent.tokens.append(tok)
            sentences.append(sent)
        return sentences

    def process_item(self, stream_item, context=None):
        if not stream_item.body.clean_visible:
            return stream_item

        self.label_index = None
        self.label_to_mention_id = dict()
        sentences = self.make_sentences(stream_item)
        stream_item.body.sentences[self.tagger_id] = sentences

        return stream_item


#!/usr/bin/env python
'''
Converts HTML-escaped and XML-escaped characters into their simple
character equivalents.  Borrows heavily from the BeautifulSoup
library, which has an MIT/X11 License.

This software is released under an MIT/X11 open source license.

Copyright 2012-2013 Diffeo, Inc.
'''
import re
import sys
import cgi
from htmlentitydefs import name2codepoint

XML_ENTITIES_TO_SPECIAL_CHARS = { "apos" : "'",
                                  "quot" : '"',
                                  "amp" : "&",
                                  "lt" : "<",
                                  "gt" : ">" }

tags = re.compile(r"&(#\d+|#x[0-9a-fA-F]+|\w+);")

ENTITIES_THAT_ARE_SAFE_TO_STRING_PAD = [
    "nbsp", "amp", "quot", "gt", "raquo", "lt", "ldquo", "rdquo",
    "bull", "hellip", "laquo", "middot", "rsquo", "ndash", "mdash",
    "curren", "bdquo", "lsquo", "copy", "rarr", "apos", "reg"]

def html_entities_to_unicode(text, space_padding=False, safe_only=False):
    '''
    Convert any HTML, XML, or numeric entities in the attribute values.
    For example '&' becomes '&'.

    This is adapted from BeautifulSoup, which should be able to do the
    same thing when called like this --- but this fails to convert
    everything for some bug.

    text = unicode(BeautifulStoneSoup(text, convertEntities=BeautifulStoneSoup.XML_ENTITIES))
    '''
    def convert_entities(match):
        '''
        comes from BeautifulSoup.Tag._convertEntities
        '''
        x = match.group(1)

        if safe_only and x not in ENTITIES_THAT_ARE_SAFE_TO_STRING_PAD:
            return u'&%s;' % x

        if x in name2codepoint:
            ## handles most cases
            return unichr(name2codepoint[x])

        elif x in XML_ENTITIES_TO_SPECIAL_CHARS:
            return XML_ENTITIES_TO_SPECIAL_CHARS[x]

        elif len(x) > 0 and x[0] == '#':
            # Handle numeric entities
            if len(x) > 1 and x[1] == 'x':
                return unichr(int(x[2:], 16))
            else:
                return unichr(int(x[1:]))
        else:
            ## uh oh, failed to anything
            return u'&%s;' % x

    def convert_to_padded_entitites(match):
        converted_string = convert_entities(match)
        num_spaces_needed = len(match.group(0)) - len(converted_string)
        assert num_spaces_needed >= 0, \
            'len(%r) !<= len(%r)' % (converted_string, match.group(0))
        ## Where to put the spaces?  Before, after, symmetric?
        # Let's do symmetric.
        ## cast to int in prep for python3
        num_left = int(num_spaces_needed / 2)
        num_right = num_spaces_needed - num_left
        return (' ' * num_left) + converted_string + (' ' * num_right)

    ## brute force regex through all the characters...
    if space_padding:
        return tags.sub(
                      convert_to_padded_entitites,
                      text)
    else:
        return tags.sub(
                      convert_entities,
                      text)

'''
def html_entities_to_unicode(text, safe_wrap=False, space_padding=False):
    if not safe_wrap:
        return _html_entities_to_unicode(text, space_padding=space_padding)

    else:
        ## iterate byte by byte looking for possible entities
        new_text = list()
        c = 0
        p = 0
        buf = bytes()
        while c < len(text):
            if text[c] == '&':
                ## capture all bytes up to this point
                if buf:
                    ## inside of buffer that cannot be a valid entity;
                    ## accumulate buffer, there will be no replacement
                    new_text.append(buf)
                    buf = r''
                else:    # start of possible entity
                    ## get text up to this point
                    new_text.append(text[p:c])
                    ## move previous cursor forward
                    p = c
                    buf = r'&'

            elif text[c] == ';': # finished a possible entity
                buf += ';'
                try:
                    #print buf
                    replacement = _html_entities_to_unicode(
                        buf, space_padding=space_padding)
                except Exception, exc:
                    #print exc
                    #logger.warning('html_entities_to_unicode: %r --> %r' % (buf, exc))
                    ## do not replace, just accumulate what was in the buffer
                    new_text.append(buf)
                else:
                    ## accumulate the replacement
                    new_text.append(replacement)
                    ## move previous cursor forward plus one for the ';'
                    p = c + 1
                ## reset state machine
                buf = r''

            elif buf:    # inside of a possible entity
                buf += text[c]
            # not inside an entity, so continue accumulating new text
            ## continue incrementing through the input text
            c += 1
        ## all done!  return the new text
        new_text.append(buf)
        buf = bytearray()
        for portion in new_text:
            buf += portion.encode('utf8')
        return buf
'''

def unicode_to_html_entities(text):
    '''Converts unicode to HTML entities.  For example '&' becomes '&'.'''
    text = cgi.escape(text).encode('ascii', 'xmlcharrefreplace')
    return text

if __name__ == '__main__':
    '''
    Run basic tests
    '''
    text = '&, ®, <, >, ¢, £, ¥, €, §, ©'

    uni = html_entities_to_unicode(text)
    htmlent = unicode_to_html_entities(uni)

    print uni
    print htmlent
    #print html_entities_to_unicode('€', safe_wrap=True)

    ## verify inverted
    uni2 = html_entities_to_unicode(htmlent)
    assert uni == uni2, uni2

    print html_entities_to_unicode(htmlent, space_padding=True)
    ## but does not invert
    print unicode_to_html_entities(
        html_entities_to_unicode(htmlent, space_padding=True))

    # &, ®, <, >, ¢, £, ¥, €, §, ©


'''
This software is released under an MIT/X11 open source license.

Copyright 2012-2013 Diffeo, Inc.
'''
from __future__ import division
import os
import math
import hashlib
from itertools import ifilter, imap
try:
    from collections import Counter
except ImportError:
    from backport_collections import Counter
from nltk.corpus import stopwords
from _clean_visible import cleanse

def tps(text, min_token_len=2, quant_rate=0.01):
    '''
    :param text: tag-free UTF-8 string of free text
    :returns string: 32-character md5 hash in hexadecimal

    Python implementation of the TextProfileSignature provided in
    SOLR.  Unlike most other locality sensitive hashes, a TPS can be
    indexed as a searchable property of each document that does not
    require n^2 comparisons to find duplicates.

    http://wiki.apache.org/solr/TextProfileSignature
    '''

    counts = Counter(
        ifilter(lambda x: len(x) >= min_token_len,
                imap(cleanse, text.split())))

    max_freq = counts.most_common(1)[0][1]
    if max_freq <= 1:
        quant = 1
    else:
        quant = max(2, round(max_freq * quant_rate))

    to_hash = []
    for word, count in counts.most_common():
        if count <= quant:
            break
        to_hash += [word, str(int(math.floor(count / quant)))]

    to_hash = u' '.join(to_hash)
    return hashlib.md5(to_hash.encode('utf8')).hexdigest()


#!/usr/bin/python
'''
Takes a directory of streamcorpus.Chunk files as input, builds
'cleansed' for body and each other_content if needed, puts the
cleansed into XML format expected by wrappers such as the LingPipe,
the kba-stanford-corenlp wrapper, and similar, parses output from one
of those systems, puts the data into Token arrays in Sentence objects,
converts to sentence_blobs, and writes out the new Chunk.

This is designed to run in Condor, which means that it needs to be
ready to get killed and moved to a new compute slot at any time.  To
cope with this, it finishes each step in temp files before doing an
atomic move to put the final product into position.

This software is released under an MIT/X11 open source license.

Copyright 2012-2013 Diffeo, Inc.
'''

import streamcorpus
from streamcorpus import Chunk, Label, Tagging
#import lingpipe
lingpipe = None

import os
import re
import uuid
import time
import string
import hashlib
import traceback
import subprocess
import collections
import xml.dom.minidom

pipeline_cmd_templates = {
    'stanford': 'java -Xmx2048m -jar %(PIPELINE_ROOT)s/runNER.jar %(INPUT_FILE)s %(OUTPUT_FILE)s',
    'lingpipe': 'cd %(PIPELINE_ROOT)s/lingpipe-4.1.0/demos/generic/bin && cat %(INPUT_FILE)s | ./cmd_coref_en_news_muc6.sh "-contentType=text/html" "-includeElts=FILENAME" 1> %(OUTPUT_FILE)s',
}

postproc_cmd_templates = {
    'stanford': 'mv %(RAW_OUTPUT_FILE)s %(OUTPUT_FILE)s',
    'lingpipe': 'python -m kba.pipeline.lingpipe postproc %(RAW_OUTPUT_FILE)s %(OUTPUT_FILE)s',
}

stream_id_re = re.compile('<FILENAME\s+id="(.*?)"')

def make_cleansed_file(i_chunk, tmp_cleansed_path):
    '''make a temp file of cleansed text'''
    tmp_cleansed = open(tmp_cleansed_path, 'wb')
    for idx, si in enumerate(i_chunk):
        tmp_cleansed.write('<FILENAME docid="%s">\n' % si.stream_id)
        tmp_cleansed.write(si.body.cleansed)
        ## how to deal with other_content?
        tmp_cleansed.write('</FILENAME>\n')
    tmp_cleansed.close()
    ## replace this with log.info()
    print 'created %s' % tmp_cleansed_path

def make_ner_file(tagger_id, tmp_cleansed_path, tmp_ner_path, pipeline_root):
    '''run child process to get OWPL output'''
    params = dict(INPUT_FILE=tmp_cleansed_path,
                  #RAW_OUTPUT_FILE=tmp_ner_raw_path,
                  OUTPUT_FILE=tmp_ner_path,
                  PIPELINE_ROOT=pipeline_root)

    pipeline_cmd = pipeline_cmd_templates[tagger_id] % params

    print pipeline_cmd

    ## replace this with log.info()
    print 'creating %s' % tmp_ner_path
    start_time = time.time()
    gpg_child = subprocess.Popen(
        pipeline_cmd,
        stderr=subprocess.PIPE, shell=True)
    s_out, errors = gpg_child.communicate()
    assert gpg_child.returncode == 0 and 'Exception' not in errors, errors
    elapsed = time.time() - start_time
    ## replace this with log.info()
    print 'created %s in %.1f sec' % (tmp_ner_path, elapsed)

    '''
    postproc_cmd = postproc_cmd_templates[tagger_id] % params

    print postproc_cmd

    ## replace this with log.info()
    print 'creating %s' % tmp_ner_raw_path
    start_time = time.time()
    gpg_child = subprocess.Popen(
        postproc_cmd,
        stderr=subprocess.PIPE, shell=True)
    s_out, errors = gpg_child.communicate()
    assert gpg_child.returncode == 0 and 'Exception' not in errors, errors
    elapsed = time.time() - start_time

    ## replace this with log.info()
    print 'created %s in %.1f sec' % (tmp_ner_path, elapsed)
    '''

## 'cleanse'
whitespace = re.compile('''(\s|\n)+''')
strip_punctuation = string.maketrans(string.punctuation,
                                     ' ' * len(string.punctuation))

def cleanse(span):
    '''Convert a string of text into a lowercase string with no
    punctuation and only spaces for whitespace.

    :param span: string
    '''
    try:
        ## attempt to force it to utf8, which might fail
        span = span.encode('utf8', 'ignore')
    except:
        pass
    ## lowercase, strip punctuation, and shrink all whitespace
    span = span.lower()
    span = span.translate(strip_punctuation)
    span = whitespace.sub(' ', span)
    ## trim any leading or trailing whitespace
    return span.strip()

def align_chunk_with_ner(tmp_ner_path, i_chunk, tmp_done_path):
    '''
    iterate through the i_chunk and tmp_ner_path to generate a new
    Chunk with body.ner
    '''
    o_chunk = Chunk()
    input_iter = i_chunk.__iter__()
    ner = ''
    stream_id = None

    all_ner = xml.dom.minidom.parse(open(tmp_ner_path))

    for raw_ner in all_ner.getElementsByTagName('FILENAME'):

        stream_item = input_iter.next()
        ## get stream_id out of the XML
        stream_id = raw_ner.attributes.get('docid').value
        assert stream_id and stream_id == stream_item.stream_id, \
            '%s != %s\nner=%r' % (stream_id, stream_item.stream_id, ner)

        tagger_id = 'lingpipe'
        tagging = Tagging()
        tagging.tagger_id = tagger_id
        ## get this one file out of its FILENAME tags
        tagged_doc = list(lingpipe.files(raw_ner.toxml()))[0][1]
        tagging.raw_tagging = tagged_doc
        tagging.generation_time = streamcorpus.make_stream_time()
        stream_item.body.taggings[tagger_id] = tagging

        sentences = list(lingpipe.sentences(tagged_doc))

        ## make JS labels on individual tokens
        assert stream_item.ratings[0].mentions, stream_item.stream_id
        john_smith_label = Label()
        john_smith_label.annotator = stream_item.ratings[0].annotator
        john_smith_label.target_id = stream_item.ratings[0].target_id

        # first map all corefchains to their words
        equiv_ids = collections.defaultdict(lambda: set())
        for sent in sentences:
            for tok in sent.tokens:
                if tok.entity_type is not None:
                    equiv_ids[tok.equiv_id].add(cleanse(tok.token))

        ## find all the chains that are John Smith
        johnsmiths = set()
        for equiv_id, names in equiv_ids.items():
            ## detect 'smith' in 'smithye'
            _names = cleanse(' '.join(names))
            if 'john' in _names and 'smith' in _names:
                johnsmiths.add(equiv_id)

        print len(johnsmiths)
        ## now apply the label
        for sent in sentences:
            for tok in sent.tokens:
                if tok.equiv_id in johnsmiths:
                    tok.labels = [john_smith_label]

        stream_item.body.sentences[tagger_id] = sentences

        o_chunk.add(stream_item)

    ## put the o_chunk bytes into the specified file
    open(tmp_done_path, 'wb').write(str(o_chunk))
    ## replace this with log.info()
    print 'created %s' % tmp_done_path

def run_pipeline(tagger_id, input_dir, output_dir, tmp_dir='/tmp', pipeline_root='./'):
    '''
    '''
    ## make tmp_dir more
    tmp_dir = os.path.join(tmp_dir, '%s-%s' % (uuid.uuid1(), os.getpid()))
    assert not os.path.exists(tmp_dir), tmp_dir
    os.makedirs(tmp_dir)

    for fname in os.listdir(input_dir):
        if not fname.endswith('.sc'):
            ## ignore any non streamcorpus.Chunk files
            continue

        fpath = os.path.join(input_dir, fname)

        ## just need one chunk for this tiny corpus
        i_chunk = Chunk(file_obj=open(fpath))

        ## prepare to make intermediate files in tmp_dir
        tmp_cleansed_path = os.path.join(tmp_dir, fname + '.cleansed.xml')
        tmp_ner_path      = os.path.join(tmp_dir, fname + '.ner.xml')
        tmp_done_path     = os.path.join(output_dir, fname + '.done.partial')
        final_done_path   = os.path.join(output_dir, fname + '.done.sc')

        make_cleansed_file(i_chunk, tmp_cleansed_path)

        make_ner_file(tagger_id, tmp_cleansed_path, tmp_ner_path, pipeline_root)

        align_chunk_with_ner(tmp_ner_path, i_chunk, tmp_done_path)

        ## atomic rename when done
        os.rename(tmp_done_path, final_done_path)

        ## replace with log.info()
        print 'done with %s' % final_done_path

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('tagger_id',
                        metavar='stanford|lingpipe',
                        help='name of NLP pipeline to run')
    parser.add_argument('input_dir', help='directory of Chunk files')
    parser.add_argument('output_dir', help='directory to put new streamcorpus.Chunk files')
    parser.add_argument('--convert-kba', action='store_true', default=False,
                        help='Expect input_dir to have old-style KBA chunks')
    parser.add_argument('--pipeline-root', metavar='PIPELINE_ROOT', dest='pipeline_root',
                        help='file path to root dir for the particular NLP pipeline')
    parser.add_argument('--align-only', metavar='OUTPUT_PATH', dest='align_only', default=None,
                        help='produce a chunk file at OUTPUT_PATH by alignning a single input chunk file with the intermediate')
    ## could add options to delete input after verifying the output on disk?
    args = parser.parse_args()

    if args.align_only is not None:
        i_chunk = Chunk(file_obj=open(args.input_dir))
        tmp_ner_path = args.output_dir
        tmp_done_path = args.align_only
        align_chunk_with_ner(tmp_ner_path, i_chunk, tmp_done_path)

    else:
        run_pipeline(args.tagger_id, args.input_dir, args.output_dir, pipeline_root=args.pipeline_root)


#!/usr/bin/env python
''':command:`streamcorpus_pipeline` is the command-line entry point to the
pipeline.

.. This software is released under an MIT/X11 open source license.
   Copyright 2012-2014 Diffeo, Inc.

.. program:: streamcorpus_pipeline

The supplied configuration must have a :mod:`streamcorpus_pipeline`
section.  Other required configuration depends on the specific stages
in use and the rest of the operating environment; for instance, if the
:class:`~streamcorpus_pipeline._kvlayer.to_kvlayer` writer stage is in
use, then the configuration must also contain a top-level
:mod:`kvlayer` section.

This supports the standard :option:`--config <yakonfig --config>`,
:option:`--dump-config <yakonfig --dump-config>`, :option:`--verbose
<dblogger --verbose>`, :option:`--quiet <dblogger --quiet>`, and
:option:`--debug <dblogger --debug>` options.  It supports the
following additional options:

.. option:: --input <file.sc>, -i <file.sc>

   Names the input file.  If the file name is ``-`` then standard
   input is used, if this makes sense.  Some reader stages may use the
   file name specially.

.. option:: --in-glob <pattern>

   Runs the pipeline once for each file matching the shell :mod:`glob`
   `pattern`.

'''
from __future__ import absolute_import
import copy
import glob
import importlib
import itertools
import logging
import json
import os
import re
import sys
import time

import yaml

import dblogger
import kvlayer
import yakonfig
from yakonfig.toplevel import assemble_default_config

import streamcorpus_pipeline
from streamcorpus_pipeline._exceptions import ConfigurationError
from streamcorpus_pipeline._pipeline import PipelineFactory, Pipeline
from streamcorpus_pipeline.stages import PipelineStages

logger = logging.getLogger(__name__)

def make_absolute_paths(config):
    '''given a config dict with streamcorpus_pipeline as a key, find all
    keys under streamcorpus_pipeline that end with "_path" and if the
    value of that key is a relative path, convert it to an absolute
    path using the value provided by root_path
    '''
    if not 'streamcorpus_pipeline' in config:
        logger.critical('bad config: %r', config)
        raise ConfigurationError('missing "streamcorpus_pipeline" from config')
    ## remove the root_path, so it does not get extended itself
    root_path = config['streamcorpus_pipeline'].pop('root_path', None)
    if not root_path:
        root_path = os.getcwd()

    if not root_path.startswith('/'):
        root_path = os.path.join( os.getcwd(), root_path )

    def recursive_abs_path( sub_config, root_path ):
        for key, val in sub_config.items():
            if isinstance(val, basestring):
                if key.endswith('path'):
                    ## ignore URLs in *_path parameters
                    if re.match('^http.?://', val): continue
                    ## we have a path... is it already absolute?
                    if not val.startswith('/'):
                        ## make the path absolute
                        sub_config[key] = os.path.join(root_path, val)

            elif isinstance(val, dict):
                recursive_abs_path( val, root_path )

    recursive_abs_path( config, root_path )

    ## put the root_path back
    config['root_path'] = root_path

def make_hash(obj):
    '''
    Makes a hash from a dictionary, list, tuple or set to any level,
    that contains only other hashable types (including any lists,
    tuples, sets, and dictionaries).  See second answer (not the
    accepted answer):
    http://stackoverflow.com/questions/5884066/hashing-a-python-dictionary
    '''
    if isinstance(obj, (set, tuple, list)):
        return tuple([make_hash(e) for e in obj])
    elif not isinstance(obj, dict):
        return hash(obj)

    new_obj = copy.deepcopy(obj)
    for k, v in new_obj.items():
        ## call self recursively
        new_obj[k] = make_hash(v)

    return hash(tuple(frozenset(new_obj.items())))

def instantiate_config(config):
    '''setup the config and load external modules

    This updates 'config' as follows:

    * All paths are replaced with absolute paths
    * A hash and JSON dump of the config are stored in the config
    * If 'pythonpath' is in the config, it is added to sys.path
    * If 'setup_modules' is in the config, all modules named in it are loaded
    '''
    make_absolute_paths(config)

    pipeline_config = config['streamcorpus_pipeline']

    pipeline_config['config_hash'] = make_hash(config)
    pipeline_config['config_json'] = json.dumps(config)

    logger.debug('running config: {0} = {1!r}'
                 .format(pipeline_config['config_hash'], config))

    ## Load modules
    # This is a method of using settings in yaml configs to load plugins.
    die = False
    for pathstr in pipeline_config.get('pythonpath', {}).itervalues():
        if pathstr not in sys.path:
            sys.path.append(pathstr)
    for modname in pipeline_config.get('setup_modules', {}).itervalues():
        try:
            m = importlib.import_module(modname)
            if not m:
                logger.critical('could not load module %r', modname)
                die = True
                continue
            if hasattr(m, 'setup'):
                m.setup()
                logger.debug('loaded and setup %r', modname)
            else:
                logger.debug('loaded %r', modname)
        except Exception:
            logger.critical('error loading and initting module %r', modname, exc_info=True)
            die = True
    if die:
        sys.exit(1)

def pathfile_iter(path):
    if path.endswith('.gz'):
        fin = gzip.open(path, 'rb')
    else:
        fin = open(path, 'rb')
    for line in fin:
        line = line.strip()
        yield line

def main():
    import argparse
    parser = argparse.ArgumentParser(
        description='process a sequence of stream items',
        usage='streamcorpus_pipeline --config config.yaml --input file.in')
    parser.add_argument('-i', '--input', action='append',
                        help='file paths to input instead of reading from stdin')
    parser.add_argument('--in-glob', action='append', default=[], help='path glob specifying input files')
    parser.add_argument('--third-dir-path', help='path to third-party tools directory')
    parser.add_argument('--tmp-dir-path', help='path to temporary directory for scratch files, can be large')
    parser.add_argument('-f', '--file-of-paths', dest='file_of_paths', default=None, help='path to file with list of paths for input, one per line')
    parser.add_argument('--skip', type=int, default=0,
                        help='Skip the first N stream items.')

    modules = [yakonfig, kvlayer, dblogger, streamcorpus_pipeline]
    args = yakonfig.parse_args(parser, modules)
    config = yakonfig.get_global_config()

    ## this modifies the global config, passed by reference
    instantiate_config(config)

    input_paths = []
    if args.in_glob:
        for pattern in args.in_glob:
            input_paths.extend(glob.glob(pattern))
    if args.input:
        if '-' in args.input:
            if args.in_glob:
                sys.exit('cannot use "-i -" and --in-glob together')
            if len(args.input) > 1:
                sys.exit('cannot use "-i -" with multiple inputs')
            input_paths = sys.stdin
        else:
            input_paths.extend(args.input)

    if args.file_of_paths:
        input_paths = itertools.chain(input_paths, pathfile_iter(args.file_of_paths))

    scp_config = config['streamcorpus_pipeline']
    stages = PipelineStages()
    if 'external_stages_path' in scp_config:
        stages.load_external_stages(scp_config['external_stages_path'])
    if 'external_stages_modules' in scp_config:
        for mod in scp_config['external_stages_modules']:
            stages.load_module_stages(mod)
    factory = PipelineFactory(stages)
    pipeline = factory(scp_config)

    for i_str in input_paths:
        logger.info('input path %r', i_str)
        work_unit = SimpleWorkUnit(i_str.strip())
        work_unit.data['start_chunk_time'] = time.time()
        work_unit.data['start_count'] = args.skip
        pipeline._process_task(work_unit)

class SimpleWorkUnit(object):
    '''partially duck-typed coordinate.WorkUnit that wraps strings from
    stdin and provides only the methods used by the Pipeline

    '''
    def __init__(self, i_str):
        self.key = i_str
        self.data = dict()

    def update(self):
        ## a real WorkUnit would send self.data to the registry and
        ## renew the lease time
        pass

    def terminate(self):
        pass

    def fail(self, exc=None):
        logger.critical('failing SimpleWorkUnit(%r) = %r: %r', self.key, self.data, exc, exc_info=True)
        sys.exit(-1)

if __name__ == '__main__':
    main()


'''
Example of how to transform a corpus into the streamcorpus format.

This uses the John Smith corpus as a test data set for illustration.
The John Smith corpus is 197 articles from the New York Times gathered
by Amit Bagga and Breck Baldwin "Entity-Based Cross-Document
Coreferencing Using the Vector Space Model"
http://acl.ldc.upenn.edu/P/P98/P98-1012.pdf

The corpus consists of 35 directories with files inside each
directory.  The documents in each directory all refer to the same
entity named John Smith, so the directory names are document-level
labels.  First, we store these doc-level labels and then later, when
we have reader output from LingPipe or Stanford CoreNLP, we coerce
these doc-level labels into labels on individual in-doc coref chains
that contain 'john' and 'smith' as substrings.

The original data is stored in data/john-smith/original and the output
of this file is in data/john-smith/john-smith.sc

This software is released under an MIT/X11 open source license.

Copyright 2012-2013 Diffeo, Inc.
'''

## this assumes that streamcorpus has been installed
import streamcorpus
from streamcorpus_pipeline._exceptions import PipelineBaseException
from streamcorpus_pipeline.stages import Configured

import os
import hashlib

class john_smith(Configured):
    config_name = 'john_smith'
    # no other config

    def __call__(self, i_str):
        '''
        Returns a kba.pipeline "reader" that generates a single
        streamcorpus.Chunk file containing the John Smith corpus.

        :returns function:
        '''
        return generate_john_smith_chunk(i_str)

def generate_john_smith_chunk(path_to_original):
    '''
    This _looks_ like a Chunk only in that it generates StreamItem
    instances when iterated upon.
    '''
    ## Every StreamItem has a stream_time property.  It usually comes
    ## from the document creation time.  Here, we assume the JS corpus
    ## was created at one moment at the end of 1998:
    creation_time = '1998-12-31T23:59:59.999999Z'
    correct_time = 915148799

    if not os.path.isabs(path_to_original):
        path_to_original = os.path.join(os.getcwd(), path_to_original)

    ## iterate over the files in the 35 input directories
    for label_id in range(35):

        dir_path = os.path.join(path_to_original, str(label_id))
        fnames = os.listdir(dir_path)
        fnames.sort()
        for fname in fnames:

            stream_item = streamcorpus.make_stream_item(
                creation_time,
                ## make up an abs_url
                os.path.join(
                    'john-smith-corpus', str(label_id), fname))

            if int(stream_item.stream_time.epoch_ticks) != correct_time:
                raise PipelineBaseException('wrong stream_time construction: %r-->%r != %r'\
                                            % (creation_time, stream_item.stream_time.epoch_ticks,
                                               correct_time))

            ## These docs came from the authors of the paper cited above.
            stream_item.source = 'bagga-and-baldwin'

            ## build a ContentItem for the body
            body = streamcorpus.ContentItem()
            raw_string = open(os.path.join(dir_path, fname)).read()
            ## We know that this is already clean and has nothing
            ## tricky in it, because we manually cleansed it.  To
            ## illustrate how we stick all strings into thrift, we
            ## convert this to unicode (which introduces no changes)
            ## and then encode it as utf-8, which also introduces no
            ## changes.  Thrift stores strings as 8-bit character
            ## strings.
            # http://www.mail-archive.com/thrift-user@incubator.apache.org/msg00210.html
            body.clean_visible = unicode(raw_string).encode('utf8')

            ## attach the content_item to the stream_item
            stream_item.body = body

            stream_item.body.language = streamcorpus.Language(code='en', name='ENGLISH')

            ## The authors also annotated the corpus
            anno = streamcorpus.Annotator()
            anno.annotator_id = 'bagga-and-baldwin'
            anno.annotation_time = stream_item.stream_time

            ## build a Label for the doc-level label:
            rating = streamcorpus.Rating()
            rating.annotator = anno
            rating.target = streamcorpus.Target(target_id = str(label_id)) # must be string
            rating.contains_mention = True
            rating.mentions = ['john', 'smith']

            ## put this one label in the array of labels
            streamcorpus.add_annotation(stream_item, rating)

            ## provide this stream_item to the pipeline
            yield stream_item

if __name__ == '__main__':
    ## this is a simple test of this reader stage
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument(
        'input_dir',
        help='path to a directory containing the original John Smith corpus.')
    args = parser.parse_args()

    john_smith_reader_stage = john_smith({})
    for si in john_smith_reader_stage( args.input_dir ):
        print len(si.body.clean_visible), si.stream_id


'''Simple HTML to plain text conversion.

.. This software is released under an MIT/X11 open source license.
   Copyright 2012-2015 Diffeo, Inc.

.. autoclass:: clean_visible

'''
from __future__ import absolute_import
import logging
import re
import string
import traceback
from streamcorpus_pipeline.emails import fix_emails

import lxml.etree

from streamcorpus_pipeline._clean_html import drop_invalid_and_upper_utf8_chars
from streamcorpus_pipeline.stages import Configured
import yakonfig

logger = logging.getLogger(__name__)

# regex to identify all XML-like tags, including SCRIPT and STYLE tags
invisible = re.compile(
    # capture the visible text before the invisible part
    '''(?P<before>(.|\n)*?)'''
    # capture everything between SCRIPT and STYLE tags,
    '''(?P<invisible>(<script(.|\n)*?</script>|<style(.|\n)*?</style>'''
    # and also everything inside of XML-like tags, even if it
    # contains newlines
    '''|<(.|\n)*?>))''',
    # ignore case
    re.I)

def re_based_make_clean_visible(html):
    '''
    Takes an HTML-like binary string as input and returns a binary
    string of the same length with all tags replaced by whitespace.
    This also detects script and style tags, and replaces the text
    between them with whitespace.

    Pre-existing whitespace of any kind (newlines, tabs) is converted
    to single spaces ' ', which has the same byte length (and
    character length).

    Note: this does not change any characters like ’ and  ,
    so taggers operating on this text must cope with such symbols.
    Converting them to some other character would change their byte
    length, even if equivalent from a character perspective.

    This is regex based, which can occassionally just hang...
    '''
    text = ''
    # Fix emails
    html = fix_emails(html)

    for m in invisible.finditer(html):
        text += m.group('before')
        text += ' ' * len(m.group('invisible'))

    # text better be >= original
    assert len(html) >= len(text), '%d !>= %d' % (len(html), len(text))

    # capture any characters after the last tag... such as newlines
    tail = len(html) - len(text)
    text += html[-tail:]

    # now they must be equal
    assert len(html) == len(text), '%d != %d' % (len(html), len(text))

    return text

def make_clean_visible(_html, tag_replacement_char=' '):
    '''
    Takes an HTML-like Unicode string as input and returns a UTF-8
    encoded string with all tags replaced by whitespace. In particular,
    all Unicode characters inside HTML are replaced with a single
    whitespace character.

    This does not detect comments, style, script, link.  It also does
    do anything with HTML-escaped characters.  All of these are
    handled by the clean_html pre-cursor step.

    Pre-existing whitespace of any kind (newlines, tabs) is converted
    to single spaces ' ', which has the same byte length (and
    character length).

    This is a simple state machine iterator without regexes
    '''
    def non_tag_chars(html):
        n = 0
        while n < len(html):
            angle = html.find('<', n)
            if angle == -1:
                yield html[n:]
                n = len(html)
                break
            yield html[n:angle]
            n = angle

            while n < len(html):
                nl = html.find('\n', n)
                angle = html.find('>', n)
                if angle == -1:
                    yield ' ' * (len(html) - n)
                    n = len(html)
                    break
                elif nl == -1 or angle < nl:
                    yield ' ' * (angle + 1 - n)
                    n = angle + 1
                    break
                else:
                    yield ' ' * (nl - n) + '\n'
                    n = nl + 1
                    # do not break

    if not isinstance(_html, unicode):
        _html = unicode(_html, 'utf-8')

    # Protect emails by substituting with unique key
    _html = fix_emails(_html)

    #Strip tags with previous logic
    non_tag = ''.join(non_tag_chars(_html))

    return non_tag.encode('utf-8')

extended_tags = set(['!--', 'script', 'style'])
longest_extended_tag = max(map(len, extended_tags))
def non_tag_chars_from_raw(html):
    '''generator that yields clean visible as it transitions through
    states in the raw `html`

    '''
    n = 0
    while n < len(html):
        # find start of tag
        angle = html.find('<', n)
        if angle == -1:
            yield html[n:]
            n = len(html)
            break
        yield html[n:angle]
        n = angle

        # find the end of the tag string
        space = html.find(' ',  n, n + longest_extended_tag + 2)
        angle = html.find('>',  n, n + longest_extended_tag + 2)
        nl    = html.find('\n', n, n + longest_extended_tag + 2)
        tab   = html.find('\t', n, n + longest_extended_tag + 2)
        ends = filter(lambda end: end > -1, [tab, nl, space, angle])
        if ends:
            tag = html[n + 1 : min(ends)]
            if tag == '!--':
                # whiteout comment except newlines
                end = html.find('-->', n)
                while n < end:
                    nl = html.find('\n', n, end)
                    if nl != -1:
                        yield ' ' * (nl - n) + '\n'
                        n = nl + 1
                    else:
                        yield ' ' * (end - n + 3)
                        break
                n = end + 3
                continue
            is_extended = tag.lower() in extended_tags
        else:
            is_extended = False

        # find end of tag even if on a lower line
        while n < len(html):
            squote = html.find("'", n)
            dquote = html.find('"', n)
            nl = html.find('\n', n)
            angle = html.find('>', n)
            if angle == -1:
                # hits end of doc before end of tag
                yield ' ' * (len(html) - n)
                n = len(html)
                break
            elif -1 < squote < angle or -1 < dquote < angle:
                if squote != -1 and dquote != -1:
                    if squote < dquote:
                        open_quote = squote
                        quote = "'"
                    else:
                        open_quote = dquote
                        quote = '"'
                elif dquote != -1:
                    open_quote = dquote
                    quote = '"'
                else:
                    open_quote = squote
                    quote = "'"
                close_quote = html.find(quote, open_quote + 1)
                while n < close_quote:
                    nl = html.find('\n', n, close_quote)
                    if nl == -1: break
                    yield ' ' * (nl - n) + '\n'
                    n = nl + 1
                yield ' ' * (close_quote + 1 - n)
                n = close_quote + 1
                continue

            elif nl == -1 or angle < nl:
                # found close before either newline or end of doc
                yield ' ' * (angle + 1 - n)
                n = angle + 1
                if is_extended and html[angle - 1] != '/':
                    # find matching closing tag. JavaScript can
                    # include HTML *strings* within it, and in
                    # principle, that HTML could contain a closing
                    # script tag in it; ignoring for now.
                    while n < len(html):
                        nl = html.find('\n', n)
                        close = html.find('</', n)
                        close2 = html.find('</', close + 2)
                        angle = html.find('>', close + 2)
                        if nl != -1 and nl < close:
                            yield ' ' * (nl - n) + '\n'
                            n = nl + 1
                        elif close == -1 or angle == -1:
                            # end of doc before matching close tag
                            yield ' ' * (len(html) - n)
                            n = len(html)
                            break
                        elif close2 != -1 and close2 < angle:
                            # broken tag inside current tag
                            yield ' ' * (close + 2 - n)
                            n = close + 2
                        elif html[close + 2:angle].lower() == tag.lower():
                            yield ' ' * (angle + 1 - n)
                            n = angle + 1
                            break
                        else:
                            yield ' ' * (angle + 1 - n)
                            n = angle + 1
                            # do not break
                # finished with tag
                break
            else:
                # found a newline within the current tag
                yield ' ' * (nl - n) + '\n'
                n = nl + 1
                # do not break

def make_clean_visible_from_raw(_html, tag_replacement_char=' '):
    '''Takes an HTML-like Unicode (or UTF-8 encoded) string as input and
    returns a Unicode string with all tags replaced by whitespace. In
    particular, all Unicode characters inside HTML are replaced with a
    single whitespace character.

    This *does* detect comments, style, script, link tags and replaces
    them with whitespace.  This is subtle because these tags can be
    self-closing or not.

    It does do anything with HTML-escaped characters.

    Pre-existing whitespace of any kind *except* newlines (\n) and
    linefeeds (\r\n) is converted to single spaces ' ', which has the
    same byte length (and character length).  Newlines and linefeeds
    are left unchanged.

    This is a simple state machine iterator without regexes

    '''
    if not isinstance(_html, unicode):
        _html = unicode(_html, 'utf-8')

    #Strip tags with logic above
    non_tag = ''.join(non_tag_chars_from_raw(_html))

    return non_tag.encode('utf-8')

class clean_visible(Configured):
    '''Create ``body.clean_visible`` from ``body.clean_html``.

    :class:`clean_visible` maintains byte position of all visible text
    in an HTML (or XML) document and removes all of parts that are not
    visible in a web browser.  This allows taggers to operate on the
    visible-only text and any standoff annotation can refer to the
    original byte positions.

    If there is no ``clean_html``, but there is a ``raw`` property
    with a ``text/plain`` media type, use that value directly.

    This has no useful configuration options.  The configuration
    metadata will include a setting:

    .. code-block:: yaml

        require_clean_html: true

    Setting this to ``false`` will always fail.

    '''
    config_name = 'clean_visible'
    default_config = {'require_clean_html': True}

    @staticmethod
    def check_config(config, name):
        if not config['require_clean_html']:
            raise yakonfig.ConfigurationError('{0} only does clean_html'
                                              .format(name))

    def __call__(self, stream_item, context):
        if stream_item.body:
            if stream_item.body.clean_html:
                stream_item.body.clean_visible = \
                    make_clean_visible(stream_item.body.clean_html)
                logger.debug('stream item %s: '
                             'generated %d bytes of clean_visible '
                             'from %d bytes of clean_html',
                             stream_item.stream_id,
                             len(stream_item.body.clean_visible),
                             len(stream_item.body.clean_html))
            elif (stream_item.body.raw and
                  stream_item.body.media_type == 'text/plain'):
                stream_item.body.clean_visible = stream_item.body.raw
                logger.debug('stream item %s: '
                             'generated %d bytes of clean_visible '
                             'from text/plain raw content',
                             stream_item.stream_id,
                             len(stream_item.body.clean_visible))
            else:
                logger.debug('skipped stream item %s: '
                             'neither clean_html nor text/plain content',
                             stream_item.stream_id)
        return stream_item

def make_clean_visible_file(i_chunk, clean_visible_path):
    '''make a temp file of clean_visible text'''
    _clean = open(clean_visible_path, 'wb')
    _clean.write('<?xml version="1.0" encoding="UTF-8"?>')
    _clean.write('<root>')
    for idx, si in enumerate(i_chunk):
        if si.stream_id is None:
            # create the FILENAME element anyway, so the ordering
            # remains the same as the i_chunk and can be aligned.
            stream_id = ''
        else:
            stream_id = si.stream_id
        doc = lxml.etree.Element("FILENAME", stream_id=stream_id)
        if si.body and si.body.clean_visible:
            try:
                # is UTF-8, and etree wants .text to be unicode
                doc.text = si.body.clean_visible.decode('utf8')
            except ValueError:
                doc.text = drop_invalid_and_upper_utf8_chars(
                    si.body.clean_visible.decode('utf8'))
            except Exception, exc:
                # this should never ever fail, because if it does,
                # then it means that clean_visible (or more likely
                # clean_html) is not what it is supposed to be.
                # Therefore, do not take it lightly:
                logger.critical(traceback.format_exc(exc))
                logger.critical('failed on stream_id=%s to follow:',
                                si.stream_id)
                logger.critical(repr(si.body.clean_visible))
                logger.critical('above was stream_id=%s', si.stream_id)
                # [I don't know who calls this, but note that this
                # will *always* fail if clean_visible isn't valid UTF-8.]
                raise
        else:
            doc.text = ''
        _clean.write(lxml.etree.tostring(doc, encoding='UTF-8'))
    _clean.write('</root>')
    _clean.close()
    logger.info(clean_visible_path)

    '''
    ## hack to capture html for inspection
    _html = open(clean_visible_path + '-html', 'wb')
    for idx, si in enumerate(i_chunk):
        _html.write('<FILENAME docid="%s">' % si.stream_id)
        if si.body and si.body.clean_html:
            _html.write(si.body.clean_html)
        _html.write('</FILENAME>\n')
    _html.close()
    ## replace this with log.info()
    print clean_visible_path + '-html'
    '''

# used in 'cleanse' below
whitespace = re.compile('''(\s|\n)+''', re.UNICODE)
strip_punctuation = dict([(ord(c), u' ') for c in string.punctuation])
penn_treebank_brackets = re.compile('''-[RL].B-''', re.UNICODE)

def cleanse(span, lower=True):
    '''Convert a unicode string into a lowercase string with no
punctuation and only spaces for whitespace.

Replace PennTreebank escaped brackets with ' ':
-LRB- -RRB- -RSB- -RSB- -LCB- -RCB-
(The acronyms stand for (Left|Right) (Round|Square|Curly) Bracket.)
http://www.cis.upenn.edu/~treebank/tokenization.html

:param span: string
    '''
    assert isinstance(span, unicode), \
        'got non-unicode string %r' % span
    # lowercase, strip punctuation, and shrink all whitespace
    span = penn_treebank_brackets.sub(' ', span)
    if lower:
        span = span.lower()
    span = span.translate(strip_punctuation)
    span = whitespace.sub(' ', span)
    # trim any leading or trailing whitespace
    return span.strip()

def main():
    '''manual test loop for make_clean_visible_from_raw
    '''
    import argparse
    import sys
    parser = argparse.ArgumentParser()
    parser.add_argument('path')
    args = parser.parse_args()

    html = open(args.path).read()
    html = html.decode('utf8')

    cursor = 0
    for s in non_tag_chars_from_raw(html):
        for c in s:
            if c != ' ' and c != html[cursor]:
                import pdb; pdb.set_trace()
            sys.stdout.write(c.encode('utf8'))
            sys.stdout.flush()
            cursor += 1

if __name__ == '__main__':
    main()