class Annotator(object):
    def annotate(self, doc): class AnnoSpan(object):
    """
    A span of text with an annotation applied to it.
    """
    __slots__ = ["start", "end", "doc", "metadata", "label", "base_spans"]

    def __init__(self, start, end, doc, label=None, metadata=None):
        self.start = start
        self.end = end
        self.doc = doc
        self.metadata = metadata
        # Base spans is only non-empty on span groups.
        self.base_spans = EMPTY_LIST
        self.label = label

    def __repr__(self):
        return u'AnnoSpan({0}-{1}, {2})'.format(self.start, self.end, self.label or self.text)

    def __lt__(self, other):
        if self.start < other.start:
            return True
        elif self.start == other.start:
            return self.end < other.end
        else:
            return False

    def __len__(self):
        return self.end - self.start

    def distance(self, other_span):
        """
        The number of characters between this span and the other one.
        If the spans overlap the distance is the negative length of their
        overlap.

        >>> from .annotier import AnnoTier
        >>> from .annodoc import AnnoDoc
        >>> doc = AnnoDoc('one two three')
        >>> tier = AnnoTier([AnnoSpan(0, 3, doc), AnnoSpan(8, 13, doc)])
        >>> tier.spans[0].distance(tier.spans[1])
        5
        """
        if self.start < other_span.start:
            return other_span.start - self.end
        else:
            return self.start - other_span.end

    def overlaps(self, other_span):
        """
        Return true if the span overlaps other_span.
        """
        return (
            (self.start >= other_span.start and self.start < other_span.end) or
            (other_span.start >= self.start and other_span.start < self.end)
        )

    def contains(self, other_span):
        """
        Return true if the span completely contains other_span.
        """
        return self.start <= other_span.start and self.end >= other_span.end

    def adjacent_to(self, other_span, max_dist=1):
        """
        Return true if the span comes before or after other_span with at most
        max_dist charaters between them.
        """
        return (
            self.comes_before(other_span, max_dist) or
            other_span.comes_before(self, max_dist)
        )

    def comes_before(self, other_span, max_dist=1, allow_overlap=False):
        """
        Return True if the span comes before the other_span and there are
        max_dist or fewer charaters between them.

        >>> from .annotier import AnnoTier
        >>> from .annodoc import AnnoDoc
        >>> doc = AnnoDoc('one two three')
        >>> tier = AnnoTier([AnnoSpan(0, 3, doc), AnnoSpan(4, 7, doc)])
        >>> tier.spans[0].comes_before(tier.spans[1])
        True
        >>> tier.spans[1].comes_before(tier.spans[0])
        False
        """
        if allow_overlap:
            ok_start = self.start <= other_span.start
        else:
            ok_start = self.end <= other_span.start
        return ok_start and self.end >= other_span.start - max_dist

    def extended_through(self, other_span):
        """
        Create a new span that includes this one and the other span.
        """
        return SpanGroup([self, other_span], self.label)

    def trimmed(self):
        """
        Create a new AnnoSpan based on this one with the offsets adjusted
        so that there is no white space at the beginning or end.

        >>> from .annodoc import AnnoDoc
        >>> doc = AnnoDoc('one two three')
        >>> original_span = AnnoSpan(3, 8, doc)
        >>> original_span.trimmed()
        AnnoSpan(4-7, two)
        """
        start = self.start
        end = self.end
        doc_text = self.doc.text
        while start < end and doc_text[start] == " ":
            start += 1
        while start < end and doc_text[end - 1] == " ":
            end -= 1
        return AnnoSpan(start, end, self.doc, label=self.label, metadata=self.metadata)

    @property
    def text(self):
        return self.doc.text[self.start:self.end]

    def to_dict(self):
        """
        Return a json serializable dictionary.
        """
        return dict(
            label=self.label,
            textOffsets=[[self.start, self.end]]
        )

    def groupdict(self):
        """
        Return a dict with all the labeled matches.

        >>> from .annodoc import AnnoDoc
        >>> doc = AnnoDoc('one two wolf')
        >>> number_span_g = SpanGroup([AnnoSpan(0, 3, doc, 'number'),
        ...                            AnnoSpan(4, 7, doc, 'number'),
        ...                            AnnoSpan(8, 12, doc, 'animal')])
        >>> number_span_g.groupdict()['number']
        [AnnoSpan(0-3, number), AnnoSpan(4-7, number)]
        >>> number_span_g.groupdict()['animal']
        [AnnoSpan(8-12, animal)]
        """
        out = {}
        for base_span in self.base_spans:
            for key, values in base_span.groupdict().items():
                out[key] = out.get(key, []) + values
        for values in out.values():
            values.sort()
        if self.label:
            out[self.label] = [self]
        return out

    def iterate_base_spans(self):
        """
        Recursively iterate over all base_spans including base_spans of child SpanGroups.
        """
        for span in self.base_spans:
            yield span
            for span2 in span.iterate_base_spans():
                yield span2

    def iterate_leaf_base_spans(self):
        """
        Return the leaf base spans in a SpanGroup tree.
        """
        for span in self.iterate_base_spans():
            if not isinstance(span, SpanGroup):
                yield span class SpanGroup(AnnoSpan):
    """
    A AnnoSpan that extends through a group of AnnoSpans.
    """
    def __init__(self, base_spans, label=None, metadata=None):
        assert isinstance(base_spans, list)
        assert len(base_spans) > 0
        super(SpanGroup, self).__init__(
            min(s.start for s in base_spans),
            max(s.end for s in base_spans),
            base_spans[0].doc,
            label,
            metadata)
        self.base_spans = base_spans

    def __repr__(self):
        return ("SpanGroup("
                "text=" + self.text + ", "
                "label=" + str(self.label) + ", " +
                ", ".join(map(str, self.base_spans)) + ")") class CountAnnotator(Annotator):
    def annotate(self, doc):
        if 'spacy.tokens' not in doc.tiers:
            doc.add_tiers(SpacyAnnotator())
        if 'dates' not in doc.tiers:
            doc.add_tiers(DateAnnotator())
        if 'raw_numbers' not in doc.tiers:
            doc.add_tiers(RawNumberAnnotator())
        spacy_tokens = doc.tiers['spacy.tokens']
        spacy_sentences = doc.tiers['spacy.sentences']
        spacy_nes = doc.tiers['spacy.nes']
        counts = doc.tiers['raw_numbers']

        spacy_lemmas = [span.token.lemma_ for span in spacy_tokens]

        def search_lemmas(lemmas, match_name=None):
            match_spans = []
            lemmas = set(lemmas)
            for span, lemma in zip(spacy_tokens, spacy_lemmas):
                if lemma in lemmas:
                    match_spans.append(span)
            return AnnoTier(match_spans, presorted=True).label_spans(match_name)

        counts_tier = AnnoTier(AnnoSpan(count.start, count.end, doc, 'count')
                               for count in counts if is_valid_count(count.text))
        # Remove counts that overlap an age
        counts_tier = counts_tier.without_overlaps(
            spacy_tokens.search_spans('age')
            .with_following_spans_from(spacy_tokens.search_spans('of'))
            .with_following_spans_from(counts_tier))
        # Remove distances
        counts_tier = counts_tier.without_overlaps(
            counts_tier.with_following_spans_from(
                spacy_tokens.search_spans('kilometers|km|miles|mi')))
        # Add count ranges
        ranges = counts_tier.with_following_spans_from(
            spacy_tokens
            .search_spans(r'to|and|or')
            .with_following_spans_from(counts_tier)
            .label_spans('range'))
        counts_tier = (counts_tier + ranges).optimal_span_set()
        modifier_lemma_groups = [
            'average|mean',
            'annual|annually',
            'monthly',
            'weekly',
            'cumulative|total|already',
            'incremental|new|additional|recent',
            'max|less|below|under|most|maximum|up',
            'min|greater|above|over|least|minimum|down|exceeds',
            'approximate|about|near|around',
            'ongoing|active',
        ]
        count_descriptions = AnnoTier(counts_tier)
        person_and_place_nes = spacy_nes.with_label('GPE') + spacy_nes.with_label('PERSON')
        for group in modifier_lemma_groups:
            lemmas = group.split('|')
            results = search_lemmas(lemmas, match_name=lemmas[0])
            # prevent components of NEs like the "New" in New York from being
            # treated as count descriptors.
            results = results.without_overlaps(person_and_place_nes)
            count_descriptions += count_descriptions.with_nearby_spans_from(results)
        case_descriptions = AnnoTier(
            search_lemmas(
                [
                    'death',
                    'die',
                    'kill',
                    'claim',
                    'fatality',
                    'decease',
                    'deceased'
                ], 'death') +
            search_lemmas(
                [
                    'hospitalization',
                    'hospital',
                    'hospitalize'
                ], 'hospitalization') +
            search_lemmas(['recovery'], 'recovery') +
            search_lemmas(
                [
                    'case',
                    'infect',
                    'infection',
                    'strike',
                    'stricken'
                ], 'case'))
        case_statuses = (
            search_lemmas(['suspect'], 'suspected') +
            search_lemmas(['confirm'], 'confirmed'))
        case_descriptions += case_descriptions.with_nearby_spans_from(case_statuses, max_dist=1)
        person_descriptions = search_lemmas([
            'man', 'woman',
            'male', 'female',
            'adult', 'senior', 'child',
            'patient',
            'life',
            'person'], 'case')
        case_descriptions += person_descriptions.with_nearby_spans_from(case_descriptions)
        case_descriptions += person_descriptions
        case_descriptions_with_counts = case_descriptions.with_nearby_spans_from(
            count_descriptions,
            max_dist=50)
        # Add singular case reports
        singular_case_spans = []
        determiner_lemmas = set(['a', 'an', 'the', 'one'])
        for cd_span, token_group in case_descriptions.group_spans_by_containing_span(spacy_tokens):
            for t_span in token_group:
                token = t_span.token
                if token.lemma_ == 'case' and token.similarity(in_case_token) < 0.5:
                    continue
                if token.tag_ == 'NN' and any(c.lower_ in determiner_lemmas
                                              for c in token.children):
                    singular_case_spans.append(cd_span)
                    break
        # remove counts that span multiple sentences
        all_potential_counts = (
            case_descriptions_with_counts.spans +
            count_descriptions.spans +
            singular_case_spans)
        single_sentence_counts = []
        for sentence, group in spacy_sentences.group_spans_by_containing_span(all_potential_counts):
            single_sentence_counts += group
        annotated_counts = AnnoTier(single_sentence_counts, presorted=True
                                    ).optimal_span_set(prefer='num_spans_and_no_linebreaks')
        attributes = [
            # count precisions
            'approximate',
            'max',
            'min',
            'average',
            # case status
            'confirmed',
            'suspected',
            # count anchors
            'cumulative',
            'incremental',
            'ongoing',
            # count units
            'case',
            'death',
            'recovery',
            'hospitalization',
            # count periods
            'annual',
            'monthly',
            'weekly',
        ]
        count_spans = []
        for match in annotated_counts:
            match_dict = match.groupdict()
            matching_attributes = set([
                attr for attr in attributes
                if attr in match_dict
            ])
            if set(['death',
                    'hospitalization',
                    'recovery']).intersection(matching_attributes):
                matching_attributes.add('case')
            if 'count' in match_dict:
                count = utils.parse_spelled_number(match_dict['count'][0].text)
            else:
                # For single case reports a count number might not be used.
                # Ex. A new case, the index case
                count = 1
            if 'range' in match_dict:
                lower_count_match = match_dict['count'][0]
                range_match = match_dict['range'][0]
                upper_count_match = range_match.groupdict()['count'][0]
                upper_count = utils.parse_spelled_number(upper_count_match.text)
                count_spans.append(CountSpan(lower_count_match, {
                    'attributes': sorted(list(matching_attributes) + ['min']),
                    'count': count
                }))
                count_spans.append(CountSpan(upper_count_match, {
                    'attributes': sorted(list(matching_attributes) + ['max']),
                    'count': upper_count
                }))
            else:
                count_spans.append(CountSpan(match, {
                    'attributes': sorted(list(matching_attributes)),
                    'count': count
                }))
        return {'counts': AnnoTier(count_spans, presorted=True)} class DateAnnotator(Annotator):
    """
    DateAnnotator annotates and parses dates and date ranges.
    All dates are parsed as datetime ranges. For instance, a date like "11-6-87"
    would be parsed as a range from the start of the day to the end of the day,
    while a month like "December 2011" would be parsed as a range from the start
    of December 1st ending at the start of January 1st 2012.

    Args:
        include_end_date (bool): Indicates whether a date range like "Monday to
        Wednesday" should be parsed as ending at the start of Wednesday
        or the start of Thursday. If a phrase like "Monday through Wednesday"
        is used the date range will extend through Wednesday regardless of
        this argument's value.
    """
    def __init__(self, include_end_date=True):
        self.include_end_date = include_end_date

    def annotate(self, doc):
        # If no date is associated with the document, the document's date will
        # be treated as the most recent date explicitly mentioned in the
        # the document.
        detect_date = doc.date is None
        doc_date = doc.date or datetime.datetime.now()
        strict_parser = DateDataParser(['en'], settings={
            'STRICT_PARSING': True})

        def date_to_datetime_range(text,
                                   relative_base=None,
                                   prefer_dates_from='past'):
            if relative_base is None:
                relative_base = doc_date
            # Handle relative date ranges like "the past ___ days"
            relative_num_days = re.sub(relative_duration_range_re, "", text)
            if len(relative_num_days) < len(text):
                num_days_datetime_range = date_to_datetime_range(relative_num_days)
                if not num_days_datetime_range:
                    return None
                return [num_days_datetime_range[0], relative_base]
            text = clean_date_str(text)
            if len(text) < 3:
                return None
            # Handle ordinal dates like "the second month of 2006"
            match = ordinal_date_re.match(text)
            if match:
                match_dict = match.groupdict()
                if match_dict['ordinal']:
                    ordinal_number = ORDINALS.index(match_dict['ordinal']) + 1
                else:
                    ordinal_number = int(match_dict['ordinal_number'])
                unit = match_dict['unit']
                rest = match_dict['rest']
                if unit == 'day':
                    return date_to_datetime_range(
                        str(ordinal_number) + " " + rest)
                elif unit == 'week':
                    if ordinal_number > 4:
                        return
                    parsed_remainder = date_to_datetime_range("1 " + rest)
                    if not parsed_remainder:
                        return
                    week_start = parsed_remainder[0]
                    week_start = date_to_datetime_range(
                        "Sunday",
                        # A day is added because if the base date is on Sunday
                        # the prior sunday will be used.
                        relative_base=week_start + relativedelta(days=1))[0]
                    for _ in range(ordinal_number - 1):
                        week_start = date_to_datetime_range(
                            "Sunday",
                            relative_base=week_start + relativedelta(days=1),
                            prefer_dates_from='future')[0]
                    return [
                        week_start,
                        week_start + relativedelta(days=7)]
                elif unit == 'month':
                    month_name = datetime.datetime(2000,
                                                   ordinal_number,
                                                   1).strftime("%B ")
                    return date_to_datetime_range(month_name + rest)
                else:
                    raise Exception("Unknown time unit: " + unit)
            # handle dates like "1950s" since dateparser doesn't
            decade_match = re.match(r"(\d{4})s", text)
            if decade_match:
                decade = int(decade_match.groups()[0])
                return [datetime.datetime(decade, 1, 1),
                        datetime.datetime(decade + 10, 1, 1)]
            parser = DateDataParser(['en'], settings={
                'RELATIVE_BASE': relative_base or datetime.datetime.now(),
                'PREFER_DATES_FROM': prefer_dates_from
            })
            try:
                text = re.sub(r" year$", "", text)
                date_data = parser.get_date_data(text)
            except (TypeError, ValueError):
                return
            if date_data['date_obj']:
                date = date_data['date_obj']
                if date_data['period'] == 'day':
                    return [date, date + relativedelta(days=1)]
                elif date_data['period'] == 'month':
                    date = datetime.datetime(date.year, date.month, 1)
                    return [date, date + relativedelta(months=1)]
                elif date_data['period'] == 'year':
                    date = datetime.datetime(date.year, 1, 1)
                    return [date, date + relativedelta(years=1)]

        def parse_non_relative_date(text):
            result = date_to_datetime_range(
                text, relative_base=datetime.datetime(900, 1, 1))
            if result and result[0].year > 1000:
                # If the year is less than 1000 assume the year 900
                # base date was used when parsing so the date is relative.
                return result[0]

        if 'structured_data' not in doc.tiers:
            doc.add_tiers(StructuredDataAnnotator())
        if 'spacy.nes' not in doc.tiers:
            doc.add_tiers(SpacyAnnotator())
        # Create a combine tier of nes and regex dates
        date_span_tier = doc.tiers['spacy.nes'].with_label('DATE')
        # Regex for formatted dates
        regex = re.compile(
            r"\b("
            # parenthetical year
            r"((?<=[\[\(])[1-2]\d{3}(?=[\]\)]))|"
            # date MonthName yyyy
            r"(\d{1,2} [a-zA-Z]{3,} \[?\d{4})|"
            # dd-mm-yyyy
            r"(\d{1,2} ?[\/\-] ?\d{1,2} ?[\/\-] ?\d{1,4})|"
            # yyyy-MMM-dd
            r"(\d{1,4} ?[\/\-] ?[a-z]{3,4} ?[\/\-] ?\d{1,4})|"
            # yyyy-mm-dd
            r"(\d{1,4} ?[\/\-] ?\d{1,2} ?[\/\-] ?\d{1,2})"
            r")\b", re.I)
        match_tier = doc.create_regex_tier(regex)
        date_span_tier += match_tier
        # Add year components individually incase the full spans are thrown out.
        # Sometimes extra text is added to dates that makes them invalid,
        # this allows some of the date to be recovered.
        date_span_tier += date_span_tier.match_subspans(r"([1-2]\d{3})")
        # Remove spans that are probably ages.
        date_span_tier = date_span_tier.without_overlaps(
            date_span_tier.match_subspans(r"\bage\b"))
        # Group adjacent date info in case it is parsed as separate chunks.
        # ex: Friday, October 7th 2010.
        adjacent_date_spans = date_span_tier.combined_adjacent_spans(max_dist=9)
        grouped_date_spans = []

        def can_combine(text):
            if re.match(r"\d{4}", text, re.I):
                # year only date
                return True
            try:
                return strict_parser.get_date_data(text)['date_obj'] is None
            except (TypeError, ValueError):
                return True
        for date_group in adjacent_date_spans:
            date_group_spans = list(date_group.iterate_leaf_base_spans())
            if any(can_combine(span.text) for span in date_group_spans):
                if date_to_datetime_range(date_group.text) is not None:
                    grouped_date_spans.append(date_group)
        # Find date ranges by looking for joiner words between dates.
        date_range_joiners = [
            t_span for t_span in doc.tiers['spacy.tokens']
            if re.match(r"(" + DATE_RANGE_JOINERS + r"|\-)$", t_span.text, re.I)]
        date_range_tier = date_span_tier.label_spans('start')\
            .with_following_spans_from(date_range_joiners, max_dist=3)\
            .with_following_spans_from(date_span_tier.label_spans('end'), max_dist=3)\
            .label_spans('date_range')
        since_tokens = AnnoTier([
            t_span for t_span in doc.tiers['spacy.tokens']
            if 'since' == t_span.token.lemma_], presorted=True).label_spans('since_token')
        since_date_tier = (
            since_tokens.with_following_spans_from(date_span_tier, allow_overlap=True) +
            date_span_tier.with_contained_spans_from(since_tokens)
        ).label_spans('since_date')
        tier_spans = []
        all_date_spans = AnnoTier(
            date_range_tier.spans +
            grouped_date_spans +
            date_span_tier.spans +
            since_date_tier.spans)

        if detect_date:
            simple_date_spans = AnnoTier(
                grouped_date_spans +
                date_span_tier.spans).optimal_span_set(prefer='text_length')
            latest_date = None
            for span in simple_date_spans:
                if re.match(r"today|yesterday", span.text, re.I):
                    continue
                try:
                    span_date = strict_parser.get_date_data(span.text)['date_obj']
                except (TypeError, ValueError):
                    continue
                if span_date:
                    timezone_span = span_date.tzinfo
                    if span_date < datetime.datetime.now(timezone_span):
                        if not latest_date or span_date > latest_date:
                            latest_date = span_date
            if latest_date:
                doc_date = latest_date

        date_spans_without_structured_data = all_date_spans.without_overlaps(doc.tiers['structured_data'])
        date_spans_in_structured_data = []
        dates_by_structured_value = doc.tiers['structured_data.values']\
            .group_spans_by_containing_span(all_date_spans, allow_partial_containment=False)
        for value_span, date_spans in dates_by_structured_value:
            date_spans_in_structured_data += date_spans
        all_date_spans = AnnoTier(
            date_spans_without_structured_data.spans + date_spans_in_structured_data
        ).optimal_span_set(prefer='text_length')
        for date_span in all_date_spans:
            # Parse the span text into one or two components depending on
            # whether it contains multiple dates for specifying a range.
            if date_span.label == 'date_range':
                range_component_dict = date_span.groupdict()
                range_components = [
                    range_component_dict['start'][0].text,
                    range_component_dict['end'][0].text]
            else:
                range_components = re.split(r"\b(?:" + DATE_RANGE_JOINERS + r")\b",
                                            date_span.text,
                                            re.I)
                if len(range_components) == 1:
                    hyphenated_components = date_span.text.split("-")
                    if len(hyphenated_components) == 2:
                        range_components = hyphenated_components
                    elif len(hyphenated_components) == 6:
                        # Handle dote ranges like 2015-11-3 - 2015-11-6
                        range_components = [
                            '-'.join(hyphenated_components[:3]),
                            '-'.join(hyphenated_components[3:])]
            if ends_with_timeunit_re.match(date_span.text) and not relative_duration_range_re.match(date_span.text):
                # Prevent durations like "5 days" from being parsed as specific
                # dates like "5 days ago"
                continue
            elif len(range_components) == 1:
                if date_span.label == 'since_date':
                    date_str = [span for span in date_span.base_spans[0].base_spans
                                if span.label != 'since_token'][0].text
                    datetime_range = date_to_datetime_range(date_str)
                    if datetime_range is None:
                        continue
                    datetime_range = [datetime_range[0], doc_date]
                else:
                    date_str = range_components[0]
                    datetime_range = date_to_datetime_range(date_str)
                    if datetime_range is None:
                        continue
            elif len(range_components) == 2:
                # Handle partial years (e.g.: 2001-12)
                if re.match(r"\d{1,2}$", range_components[1]):
                    if re.match(r".*\d{1,2}$", range_components[0]):
                        characters_to_sub = "1"
                        if len(range_components[1]) > 1:
                            characters_to_sub = "1,2"
                        range_components[1] = re.sub(r"\d{" + characters_to_sub + "}$", range_components[1], range_components[0])
                # Check for a non-relative date in the range that can be used as
                # a relative base date the other date.
                # Example: March 3 to November 2 1984
                non_relative_dates = [
                    parse_non_relative_date(text)
                    for text in range_components]
                relative_base_date = next((x for x in non_relative_dates if x),
                                          doc_date)
                datetime_range_a = date_to_datetime_range(
                    range_components[0],
                    relative_base=relative_base_date)
                datetime_range_b = date_to_datetime_range(
                    range_components[1],
                    relative_base=relative_base_date)
                if datetime_range_a is None and datetime_range_b is None:
                    continue
                elif datetime_range_a is None:
                    datetime_range = datetime_range_b
                elif datetime_range_b is None:
                    datetime_range = datetime_range_a
                else:
                    # If include_end_date is False treat the span's daterange
                    # as ending at the start of the second date component unless
                    # a word like "through" is used in the second component.
                    if self.include_end_date or\
                       re.search(r"\bthrough\b", date_span.text) or\
                       re.search(r"\b(late|end of)\b", range_components[1]):
                        datetime_range = [
                            datetime_range_a[0],
                            datetime_range_b[1]]
                    else:
                        datetime_range = [
                            datetime_range_a[0],
                            datetime_range_b[0]]
            else:
                print("Bad date range split:", date_span.text, range_components)
                continue
            # Omit reverse ranges because they usually come from something
            # being incorrectly parsed. The main exception is relative dates
            # like 2 to 3 weeks ago.
            if datetime_range[0] <= datetime_range[1]:
                tier_spans.append(DateSpan(date_span, datetime_range))
        return {
            'dates': AnnoTier(tier_spans, presorted=True),
            # Include unparsable and non-specific dates
            'dates.all': all_date_spans
        } class SpacyAnnotator(Annotator):
    def annotate(self, doc):
        tiers = {}
        ne_spans = []
        token_spans = []
        noun_chunks = []
        # SpaCy's neural nets currently use up too much memory on large docs,
        # so the document is divided into sections before recognizing named
        # entities. Each section is composed of N sentences. Sentence parsing
        # is not memory constrained.
        # https://github.com/explosion/spaCy/issues/1636
        sentences = AnnoTier([
            SentSpan(sent, doc) for sent in custom_sentencizer(doc.text)])
        tiers['spacy.sentences'] = sentences
        group_size = 10
        for sent_group_idx in range(0, len(sentences), group_size):
            doc_offset = sentences.spans[sent_group_idx].start
            sent_group_end = sentences.spans[min(sent_group_idx + group_size, len(sentences)) - 1].end
            ne_chunk_start = None
            ne_chunk_end = None
            ne_chunk_type = None
            spacy_doc = spacy_nlp(doc.text[doc_offset:sent_group_end])
            noun_chunks.extend(SentSpan(chunk, doc, offset=doc_offset) for chunk in spacy_doc.noun_chunks)
            for token in spacy_doc:
                start = token.idx + doc_offset
                end = start + len(token)
                # White-space tokens are skipped.
                if not re.match(r"^\s", token.text):
                    token_spans.append(TokenSpan(token, doc, offset=doc_offset))
                if ne_chunk_start is not None and token.ent_iob_ != "I":
                    ne_spans.append(AnnoSpan(ne_chunk_start, ne_chunk_end,
                                             doc, label=ne_chunk_type))
                    ne_chunk_start = None
                    ne_chunk_end = None
                    ne_chunk_type = None
                if token.ent_type_:
                    if token.ent_iob_ == "B":
                        ne_chunk_start = start
                        ne_chunk_end = end
                        ne_chunk_type = token.ent_type_
                    elif token.ent_iob_ == "I":
                        ne_chunk_end = end
                    elif token.ent_iob_ == "O":
                        ne_spans.append(AnnoSpan(start, end,
                                                 doc, label=token.ent_type_))
                    else:
                        raise Exception("Unexpected IOB tag: " + str(token.ent_iob_))
            if ne_chunk_start is not None:
                ne_spans.append(AnnoSpan(ne_chunk_start, ne_chunk_end,
                                         doc, label=ne_chunk_type))

        ambiguous_year_pattern = re.compile(r'\d{1,4}$', re.I)
        for ne_span in ne_spans:
            if ne_span.label == 'DATE' and ambiguous_year_pattern.match(ne_span.text):
                # Sometimes counts like 1500 are parsed as as the year component
                # of dates. This tries to catch that mistake when the year
                # is long enough ago that it is unlikely to be a date.
                date_as_number = int(ne_span.text)
                if date_as_number < 1900:
                    ne_span.label = 'QUANTITY'

        tiers['spacy.noun_chunks'] = AnnoTier(noun_chunks, presorted=True)
        tiers['spacy.tokens'] = AnnoTier(token_spans, presorted=True)
        tiers['spacy.nes'] = AnnoTier(ne_spans, presorted=True)
        return tiers class GeonameAnnotator(Annotator):
    def __init__(self, custom_classifier=None):
        self.connection = get_database_connection()
        self.connection.row_factory = sqlite3.Row
        if custom_classifier:
            self.geoname_classifier = custom_classifier
        else:
            self.geoname_classifier = geoname_classifier

    def get_candidate_geonames(self, doc):
        """
        Returns an array of geoname dicts correponding to locations that the
        document may refer to.
        The dicts are extended with lists of associated AnnoSpans.
        """
        tokens = doc.require_tiers('spacy.tokens', via=SpacyAnnotator)
        doc.require_tiers('nes', via=NEAnnotator)
        logger.info('Named entities annotated')
        single_letter_or_empty = re.compile(r"[a-z]?$", re.I)
        numeric_or_punct_start = re.compile(r"^[0-9\(\)\[\]\.\s\,\-\"\/\|]+")
        lower_case_direction = re.compile(r"^(north|south|east|west)")

        def is_possible_geoname_text(text):
            if len(text) < 3 and text != text.upper():
                return False
            elif single_letter_or_empty.match(text) or numeric_or_punct_start.match(text):
                return False
            # We can rule out a few FPs and make the query much faster
            # by only looking at capitalized names.
            # This can fail for some cases (southern California) so directions
            # are omitted.
            elif text[0] != text[0].upper() and not lower_case_direction.match(text):
                return False
            elif text in blocklist:
                return False
            return True

        def is_possible_geoname(text, tokens):
            if not is_possible_geoname_text(text):
                return False
            elif tokens[-1].token.is_punct:
                return False
            # require a noun or foreign word token.
            elif all(not (token.token.tag_.startswith("NN") or token.token.tag_ == "FW") for token in tokens):
                return False
            return True

        all_ngrams = doc.require_tiers('ngrams', via=NgramAnnotator)
        # Associate spans with the geonames.
        # This is done up front so span information can be used in the scoring
        # function
        span_text_to_spans = defaultdict(list)
        expansions = {
            "N ": "North ",
            "S ": "South ",
            "E ": "East ",
            "W ": "West "
        }
        for span, tokens in all_ngrams.group_spans_by_containing_span(tokens):
            # Replace non-standard apostrophes
            text = span.text.replace('\u2019', "'")
            if is_possible_geoname(text, tokens):
                text = normalize_text(text)
                if is_possible_geoname_text(text):
                    for trigger, expansion in expansions.items():
                        if text.startswith(trigger):
                            expanded_text = text.replace(trigger, expansion, 1).lower()
                            span_text_to_spans[expanded_text].append(span)
                    span_text_to_spans[text.lower()].append(span)
        # Add dehyphenated variants and extended directions.
        for span_text, spans in list(span_text_to_spans.items()):
            if lower_case_direction.match(span_text):
                span_text_to_spans[re.sub(r"(north|south|east|west)\s(.+)", r"\1ern \2", span_text)].extend(spans)
        possible_geonames = list(span_text_to_spans.keys())
        possible_geonames_escaped = [x.replace("'", "''") if "'" in x else x for x in possible_geonames]
        logger.info('%s possible geoname texts' % len(possible_geonames))
        cursor = self.connection.cursor()
        geoname_results = list(cursor.execute('''
        SELECT
            geonames.*,
            count AS name_count,
            group_concat(alternatename, ";") AS names_used,
            group_concat(alternatename_lemmatized, ";") AS lemmas_used
        FROM geonames
        JOIN alternatename_counts USING ( geonameid )
        JOIN alternatenames USING ( geonameid )
        WHERE alternatename_lemmatized IN (''' + ','.join("'{0}'".format(x) for x in possible_geonames_escaped) + ''')
        GROUP BY geonameid'''))
        logger.info('%s geonames fetched' % len(geoname_results))
        geoname_results = [GeonameRow(g) for g in geoname_results]
        candidate_geonames = []
        for geoname in geoname_results:
            geoname.add_spans(span_text_to_spans)
            geoname.original_spans = set(geoname.spans)
            # In rare cases geonames may have no matching spans because
            # sqlite unicode equivalency rules match geonames that use different
            # characters the document spans used to query them.
            # These geonames are ignored.
            if len(geoname.spans) > 0:
                candidate_geonames.append(geoname)
        # Add combined spans to locations that are adjacent to a span linked to
        # an administrative division. e.g. Seattle, WA
        span_to_geonames = defaultdict(list)
        for geoname in candidate_geonames:
            for span in geoname.spans:
                span_to_geonames[span].append(geoname)
        geoname_spans = span_to_geonames.keys()
        combined_spans = AnnoTier(geoname_spans).chains(at_least=2, at_most=4, max_dist=4).label_spans('combined_span')
        for combined_span in combined_spans:
            leaf_spans = combined_span.iterate_leaf_base_spans()
            first_spans = next(leaf_spans)
            potential_geonames = {geoname: set()
                                  for geoname in span_to_geonames[first_spans]}
            for leaf_span in leaf_spans:
                leaf_span_geonames = span_to_geonames[leaf_span]
                next_potential_geonames = defaultdict(set)
                for potential_geoname, prev_containing_geonames in potential_geonames.items():
                    containing_geonames = [
                        containing_geoname
                        for containing_geoname in leaf_span_geonames
                        if location_contains(containing_geoname, potential_geoname) > 0]
                    if len(containing_geonames) > 0:
                        next_potential_geonames[potential_geoname] |= prev_containing_geonames | set(containing_geonames)
                potential_geonames = next_potential_geonames
            for geoname, containing_geonames in potential_geonames.items():
                geoname.spans.add(combined_span)
                geoname.parents |= containing_geonames
        # Replace individual spans with combined spans.
        span_to_geonames = defaultdict(list)
        for geoname in candidate_geonames:
            geoname.spans = set(AnnoTier(geoname.spans).optimal_span_set().spans)
            for span in geoname.spans:
                span_to_geonames[span].append(geoname)
        # Find locations with overlapping spans
        # Note that is is possible for two valid locations to have
        # overlapping names. For example, Harare Province has
        # Harare as an alternate name, so the city Harare is very
        # likely to be an alternate location that competes with it.
        for span, geonames in span_to_geonames.items():
            geoname_set = set(geonames)
            for geoname in geonames:
                geoname.alternate_locations |= geoname_set
        possible_geoname_tier = AnnoTier([
            AnnoSpan(span.start, span.end, span.doc, metadata=set(geonames))
            for span, geonames in span_to_geonames.items()])
        grouped_possible_geonames = possible_geoname_tier.group_spans_by_containing_span(
            possible_geoname_tier,
            allow_partial_containment=True)
        for outer_span, overlapping_spans in grouped_possible_geonames:
            for geoname in outer_span.metadata:
                for overlapping_span in overlapping_spans:
                    geoname.overlapping_locations |= overlapping_span.metadata
        for geoname in candidate_geonames:
            geoname.alternate_locations.remove(geoname)
            geoname.overlapping_locations.remove(geoname)
        logger.info('%s alternative locations found' % sum([
            len(geoname.alternate_locations)
            for geoname in candidate_geonames]))
        logger.info('%s candidate locations prepared' %
                    len(candidate_geonames))
        return candidate_geonames

    def extract_features(self, geonames, doc):
        spans_to_nes = {}
        span_to_tokens = {}
        geospan_tier = AnnoTier(
            set([span for geoname in geonames for span in geoname.spans]))
        for span, ne_spans in geospan_tier.group_spans_by_containing_span(
                doc.tiers['nes'], allow_partial_containment=True):
            spans_to_nes[span] = ne_spans
        for span, token_spans in geospan_tier.group_spans_by_containing_span(
                doc.tiers['spacy.tokens']):
            span_to_tokens[span] = token_spans
        return [GeonameFeatures(geoname, spans_to_nes, span_to_tokens)
                for geoname in geonames]

    def add_contextual_features(self, candidate_geonames, features, base_classifier_predict, base_classifier_threshold):
        """
        Extend a list of features with values that are based on the geonames
        mentioned nearby.
        """
        scores = base_classifier_predict([list(f.values()) for f in features])
        for geoname, feature, score in zip(candidate_geonames, features, scores):
            geoname.base_score = score[1]
            geoname.high_confidence = float(
                score[1]) > base_classifier_threshold

        span_to_features = defaultdict(list)
        for feature in features:
            for span in feature.geoname.spans:
                span_to_features[span].append(feature)
        geoname_span_tier = AnnoTier(list(span_to_features.keys()))

        def feature_generator(filter_fun=lambda x: True):
            for span in geoname_span_tier.spans:
                for feature in span_to_features[span]:
                    if filter_fun(feature):
                        yield span.start, feature
        # Create iterators that will cycle through all the spans returning the span
        # offset and the associated feature.
        all_feature_span_iter = feature_generator()
        resolved_feature_span_iter = feature_generator(
            lambda x: x.geoname.high_confidence)
        # boolean indicators of whether the corresponding iterator has reached
        # its end.
        afs_iter_end = False
        rfs_iter_end = False
        # The starting index of the of the current feature span or resolved
        # feature span.
        f_start = 0
        rf_start = 0
        # A ring buffer containing the recently mentioned resolved geoname
        # features.
        rf_buffer = []
        rf_buffer_idx = 0
        BUFFER_SIZE = 10
        # The number of characters to lookahead searching for nearby mentions.
        LOOKAHEAD_OFFSET = 50
        # Fill the buffer to capacity with initially mentioned resolved
        # features.
        while len(rf_buffer) < BUFFER_SIZE:
            try:
                rf_start, feature = next(resolved_feature_span_iter)
                rf_buffer.append(feature.geoname)
            except StopIteration:
                rfs_iter_end = True
                break
        # Iterate over all the feature spans and add the resolved features
        # in the ring buffer to the nearby_mentions set.
        while not afs_iter_end:
            while rfs_iter_end or f_start < rf_start - LOOKAHEAD_OFFSET:
                try:
                    f_start, feature = next(all_feature_span_iter)
                except StopIteration:
                    afs_iter_end = True
                    break
                feature.nearby_mentions.update(rf_buffer)
            try:
                rf_start, resolved_feature = next(resolved_feature_span_iter)
                rf_buffer[rf_buffer_idx %
                          BUFFER_SIZE] = resolved_feature.geoname
                rf_buffer_idx += 1
            except StopIteration:
                rfs_iter_end = True
        for feature in features:
            feature.set_contextual_features()

    def annotate(self, doc, show_features_for_geonameids=None, split_compound_geonames=False):
        logger.info('geoannotator started')
        candidate_geonames = self.get_candidate_geonames(doc)
        features = self.extract_features(candidate_geonames, doc)
        if len(features) == 0:
            doc.tiers['geonames'] = AnnoTier([])
            return doc
        logger.info('adding contextual features')
        self.add_contextual_features(
            candidate_geonames, features,
            self.geoname_classifier.predict_proba_base,
            self.geoname_classifier.HIGH_CONFIDENCE_THRESHOLD)
        scores = self.geoname_classifier.predict_proba_contextual([
            list(f.values()) for f in features])
        for geoname, score in zip(candidate_geonames, scores):
            geoname.score = float(score[1])
        if show_features_for_geonameids:
            for feature in features:
                if feature.geoname.geonameid in show_features_for_geonameids:
                    print(feature.geoname.name)
                    print(feature.geoname.score)
                    print(feature.to_dict())
        culled_geonames = [geoname
                           for geoname in candidate_geonames
                           if geoname.score > self.geoname_classifier.GEONAME_SCORE_THRESHOLD]
        cursor = self.connection.cursor()
        for geoname in culled_geonames:
            geoname_results = list(cursor.execute('''
                SELECT
                    cc.name,
                    a1.name,
                    a2.name,
                    a3.name
                FROM adminnames a3
                JOIN adminnames a2 ON (
                    a2.country_code = a3.country_code AND
                    a2.admin1_code = a3.admin1_code AND
                    a2.admin2_code = a3.admin2_code AND
                    a2.admin3_code = "" )
                JOIN adminnames a1 ON (
                    a1.country_code = a3.country_code AND
                    a1.admin1_code = a3.admin1_code AND
                    a1.admin2_code = "" AND
                    a1.admin3_code = "" )
                JOIN adminnames cc ON (
                    cc.country_code = a3.country_code AND
                    cc.admin1_code = "00" AND
                    cc.admin2_code = "" AND
                    cc.admin3_code = "" )
                WHERE (a3.country_code = ? AND a3.admin1_code = ? AND a3.admin2_code = ? AND a3.admin3_code = ?)
                ''', (
                geoname.country_code or "",
                geoname.admin1_code or "",
                geoname.admin2_code or "",
                geoname.admin3_code or "",)))
            for result in geoname_results:
                prev_val = None
                for idx, attr in enumerate(['country_name', 'admin1_name', 'admin2_name', 'admin3_name']):
                    val = result[idx]
                    if val == prev_val:
                        # Names are repeated for admin levels beyond that of
                        # the geoname.
                        break
                    setattr(geoname, attr, val)
                    prev_val = val
        logger.info('admin names added')
        geospans = []
        for geoname in culled_geonames:
            for span in geoname.spans:
                geospan = GeoSpan(span, geoname)
                geospans.append(geospan)
        culled_geospans = AnnoTier(geospans).optimal_span_set(prefer=lambda x: (len(x), x.geoname.score,))
        if split_compound_geonames:
            result = []
            for geospan in culled_geospans:
                original_span = geospan.metadata['original_span']
                if original_span.label == 'combined_span':
                    possible_geonames = geospan.metadata['geoname'].parents | set([geospan.metadata['geoname']])
                    for original_leaf_span in original_span.iterate_leaf_base_spans():
                        for geoname in possible_geonames:
                            if original_leaf_span in geoname.original_spans:
                                result.append(GeoSpan(original_leaf_span, geoname))
                else:
                    result.append(geospan)
            culled_geospans = AnnoTier(result)
        logger.info('overlapping geospans removed')
        return {'geonames': culled_geospans} class SpeciesAnnotator(Annotator):
    def annotate(self, doc):
        named_entities = doc.require_tiers('spacy.nes', via=SpacyAnnotator)
        geonames = doc.require_tiers('geonames', via=GeonameAnnotator)
        resolved_keywords = doc.require_tiers('resolved_keywords', via=ResolvedKeywordAnnotator)
        species_list = []
        for kw_span in resolved_keywords:
            first_resolution = kw_span.metadata['resolutions'][0]
            if first_resolution['entity']['type'] == 'species':
                species_list.append(AnnoSpan(kw_span.start, kw_span.end, kw_span.doc, metadata={
                    'species': {
                        'id': first_resolution['entity']['id'],
                        'label': first_resolution['entity']['label'],
                    }
                }))
        species_tier = AnnoTier(species_list, presorted=True)
        return {
            'species': species_tier.without_overlaps(geonames).without_overlaps(named_entities)
        } class DiseaseAnnotator(Annotator):
    def annotate(self, doc):
        geonames = doc.require_tiers('geonames', via=GeonameAnnotator)
        resolved_keywords = doc.require_tiers('resolved_keywords', via=ResolvedKeywordAnnotator)
        disease_list = []
        for kw_span in resolved_keywords:
            first_resolution = kw_span.metadata['resolutions'][0]
            if first_resolution['entity']['type'] == 'disease':
                disease_list.append(AnnoSpan(kw_span.start, kw_span.end, kw_span.doc, metadata={
                    'disease': {
                        'id': first_resolution['entity']['id'],
                        'label': first_resolution['entity']['label'],
                    }
                }))
        return {
            'diseases': AnnoTier(
                disease_list, presorted=True).without_overlaps(geonames)
        } class StructuredIncidentAnnotator(Annotator):
    """
    The structured incident annotator will find groupings of case counts and incidents
    """

    def annotate(self, doc):
        structured_data = doc.require_tiers('structured_data', via=StructuredDataAnnotator)
        geonames = doc.require_tiers('geonames', via=GeonameAnnotator)
        dates = doc.require_tiers('dates', via=DateAnnotator)
        resolved_keywords = doc.require_tiers('resolved_keywords', via=ResolvedKeywordAnnotator)
        spacy_tokens = doc.require_tiers('spacy.tokens', via=SpacyAnnotator)
        numbers = doc.require_tiers('raw_numbers', via=RawNumberAnnotator)
        species_list = []
        disease_list = []
        for k in resolved_keywords:
            for resolution in k.metadata.get('resolutions', []):
                if resolution['entity']['type'] == 'species':
                    species_list.append(AnnoSpan(
                        k.start,
                        k.end,
                        k.doc,
                        metadata={'species': resolution}))
                    break
            for resolution in k.metadata.get('resolutions', []):
                if resolution['entity']['type'] == 'disease':
                    disease_list.append(AnnoSpan(
                        k.start,
                        k.end,
                        k.doc,
                        metadata={'disease': resolution}))
                    break
        entities_by_type = {
            'geoname': geonames,
            'date': dates,
            'species': AnnoTier(species_list).optimal_span_set(),
            'disease': AnnoTier(disease_list, presorted=True),
            'integer': AnnoTier([
                span
                for span in numbers
                if span.metadata['number'] == int(span.metadata['number'])
            ], presorted=True),
            'incident_type': spacy_tokens.search_spans(r'(case|death)s?'),
            'incident_status': spacy_tokens.search_spans(r'suspected|confirmed'),
        }
        tables = []
        possible_titles = doc.create_regex_tier("[^\n]+\n")\
            .chains(at_most=4, max_dist=0)\
            .without_overlaps(structured_data)\
            .optimal_span_set(prefer='text_length_min_spans')
        for span in doc.tiers['structured_data'].spans:
            if span.metadata['type'] != 'table':
                continue
            # Add virtual metadata columns based on surrounding text from
            # title sentence/paragraph.
            table_title = possible_titles.span_before(span)
            if table_title:
                table_title = AnnoSpan(
                    table_title.start,
                    min(table_title.end, span.start),
                    doc)
            last_disease_mentioned = entities_by_type['disease'].span_before(span)
            last_geoname_mentioned = None
            last_date_mentioned = None
            last_species_mentioned = None
            if table_title:
                last_species_mentioned = next(iter(AnnoTier(
                    species_list,
                    presorted=True
                ).spans_contained_by_span(table_title).spans[-1:]), None)
                last_geoname_mentioned = next(iter(AnnoTier(
                    geonames,
                    presorted=True
                ).spans_contained_by_span(table_title).spans[-1:]), None)
                last_date_mentioned = next(iter(AnnoTier(
                    dates,
                    presorted=True
                ).spans_contained_by_span(table_title).spans[-1:]), None)
            rows = span.metadata['data']
            # Detect header
            first_row = AnnoTier(rows[0])
            header_entities = list(first_row.group_spans_by_containing_span(numbers))
            header_numeric_portions = [
                sum(len(s) for s in entity_spans) * 1.0 / len(header_span)
                for header_span, entity_spans in header_entities
                if len(header_span) > 0]
            if all(portion < 0.8 for portion in header_numeric_portions):
                has_header = True
            else:
                has_header = False
            if has_header:
                data_rows = rows[1:]
            else:
                data_rows = rows

            # Remove rows without the right number of columns
            median_num_cols = median(list(map(len, data_rows)))
            data_rows = [row for row in data_rows if len(row) == median_num_cols]

            # Determine column types
            table_by_column = list(zip(*data_rows))
            column_types = []
            parsed_column_entities = []
            for column_values, column_name in zip(table_by_column, first_row):
                num_non_null_rows = sum(not is_null(value.text) for value in column_values)
                column_values = AnnoTier(column_values)
                # Choose column type based on greatest percent match,
                # if under 30, choose text.
                max_score = 0
                matching_column_entities = None
                column_type = "text"
                for value_type, value_spans in entities_by_type.items():
                    filtered_value_spans = value_spans
                    column_entities = [
                        SpanGroup(contained_spans, metadata=combine_metadata(contained_spans)) if len(contained_spans) > 0 else None
                        for group_span, contained_spans in column_values.group_spans_by_containing_span(filtered_value_spans)]
                    num_matches = sum(
                        contained_spans is not None
                        for contained_spans in column_entities)
                    match_score = num_matches
                    if value_type == "integer":
                        if column_name.text.lower() in set(['cases', 'deaths', 'susceptible', 'culled']):
                            match_score += 2
                    else:
                        # Prefer other types like dates over integers if all else is equal.
                        match_score += 1
                    if num_non_null_rows > 0 and float(num_matches) / num_non_null_rows > 0.3:
                        if match_score > max_score:
                            max_score = match_score
                            matching_column_entities = column_entities
                            column_type = value_type
                    if matching_column_entities is None:
                        matching_column_entities = [[] for x in column_values]
                column_types.append(column_type)
                parsed_column_entities.append(matching_column_entities)

            column_definitions = []
            if has_header:
                for column_type, header_name in zip(column_types + len(first_row) * [None], first_row):
                    column_definitions.append({
                        'name': header_name,
                        'type': column_type
                    })
            else:
                column_definitions = [
                    {'type': column_type}
                    for column_type in column_types]
            # This is designed to detect rows that cover weekly or montly intervals
            # with some potentially missing entries.
            # Irregular intervals are not detected.
            date_period = None
            for column_def, entities in zip(column_definitions, parsed_column_entities):
                if column_def['type'] == 'date':
                    date_diffs = []
                    for entity_group in split_list(entities):
                        date_diffs += [
                            abs(d.metadata['datetime_range'][0] - next_d.metadata['datetime_range'][0])
                            for d, next_d in zip(entity_group, entity_group[1:])]
                    date_period = median([d for d in date_diffs if d < datetime.timedelta(365 * 11)])
                    break
            # Implicit metadata has to come first so values in other rows will
            # overrite it.
            parsed_column_entities = [
                len(data_rows) * [last_geoname_mentioned],
                len(data_rows) * [last_date_mentioned],
                len(data_rows) * [last_disease_mentioned],
                len(data_rows) * [last_species_mentioned],
            ] + parsed_column_entities
            column_definitions = [
                {'name': '__implicit_metadata', 'type': 'geoname'},
                {'name': '__implicit_metadata', 'type': 'date'},
                {'name': '__implicit_metadata', 'type': 'disease'},
                {'name': '__implicit_metadata', 'type': 'species'}
            ] + column_definitions
            rows = list(zip(*parsed_column_entities))
            # merge with prior table or create a new one
            if not has_header and len(tables) > 0 and len(column_definitions) == len(tables[-1].column_definitions):
                # Special case for merging detached header rows
                if len(tables[-1].rows) == 0:
                    tables[-1].rows = rows
                    tables[-1].column_definitions = [
                        {
                            'type': definition.get('type'),
                            'name': definition.get('name') or prev_definition.get('name')}
                        for definition, prev_definition in zip(column_definitions, tables[-1].column_definitions)]
                elif [d['type'] for d in column_definitions] == [d['type'] for d in tables[-1].column_definitions]:
                    tables[-1].rows += rows
                    tables[-1].column_definitions = [
                        {
                            'type': definition.get('type') or prev_definition.get('type'),
                            'name': definition.get('name') or prev_definition.get('name')}
                        for definition, prev_definition in zip(column_definitions, tables[-1].column_definitions)]
            else:
                tables.append(Table(
                    column_definitions,
                    rows,
                    metadata=dict(
                        title=table_title,
                        date_period=date_period,
                        aggregation="cumulative" if table_title and re.search("cumulative", table_title.text, re.I) else None,
                        # The default metadata will be used as a final fallback
                        # for multi-section tables. When values are not specified
                        # in section titles, the values from the table title
                        # are used.
                        default_geoname=last_geoname_mentioned,
                        default_disease=last_disease_mentioned,
                        default_species=last_species_mentioned,
                        default_date=last_date_mentioned
                    )))
        incidents = []
        for table in tables:
            logger.info("header:")
            logger.info(table.column_definitions)
            logger.info("%s rows" % len(table.rows))
            for row_idx, row in enumerate(table.rows):
                row_incident_date = table.metadata.get('default_date')
                row_incident_location = table.metadata.get('default_geoname')
                row_incident_species = table.metadata.get('default_species')
                row_incident_disease = table.metadata.get('default_disease')
                row_incident_base_type = None
                row_incident_status = None
                row_incident_aggregation = table.metadata.get('aggregation')
                for column, value in zip(table.column_definitions, row):
                    if column.get('name') == "__implicit_metadata" and not value:
                        continue
                    if not value:
                        value = CANNOT_PARSE
                    if column['type'] == 'date':
                        row_incident_date = value
                    elif column['type'] == 'geoname':
                        row_incident_location = value
                    elif column['type'] == 'species':
                        row_incident_species = value
                    elif column['type'] == 'incident_type':
                        if value == CANNOT_PARSE:
                            row_incident_base_type = value
                        elif "case" in value.text.lower():
                            row_incident_base_type = "caseCount"
                        elif "death" in value.text.lower():
                            row_incident_base_type = "deathCount"
                        else:
                            row_incident_base_type = CANNOT_PARSE
                    elif column['type'] == 'incident_status':
                        row_incident_status = value

                row_incidents = []
                logger.info('---')
                for column, value in zip(table.column_definitions, row):
                    logger.info(column)
                    logger.info(value)
                    if not value:
                        continue
                    if column['type'] == "integer":
                        column_name = column.get('name')
                        if isinstance(column_name, AnnoSpan):
                            column_name_text = column_name.text.lower()
                        else:
                            column_name_text = (column_name or '').lower()
                        incident_base_type = None
                        if row_incident_base_type:
                            incident_base_type = row_incident_base_type
                        elif re.search(r"\b(cases|ill|infections)\b", column_name_text, re.I):
                            incident_base_type = "caseCount"
                        elif "deaths" in column_name_text:
                            incident_base_type = "deathCount"

                        if row_incident_status and row_incident_status != CANNOT_PARSE:
                            count_status = row_incident_status.text
                        elif "suspect" in column_name_text or column_name_text == "reported":
                            count_status = "suspected"
                        elif "confirmed" in column_name_text:
                            count_status = "confirmed"
                        else:
                            count_status = None

                        if count_status and not incident_base_type:
                            incident_base_type = "caseCount"
                        incident_aggregation = None
                        if row_incident_aggregation is not None:
                            incident_aggregation = row_incident_aggregation
                        if "total" in column_name_text:
                            incident_aggregation = "cumulative"
                        if "new" in column_name_text:
                            incident_aggregation = "incremental"
                        incident_count = value.metadata['number']
                        incident_location = row_incident_location
                        incident_species = row_incident_species
                        incident_disease = row_incident_disease
                        incident_date = row_incident_date
                        if incident_species and incident_species != CANNOT_PARSE:
                            species_entity = incident_species.metadata['species']['entity']
                            incident_species = {
                                'id': species_entity['id'],
                                'label': species_entity['label'],
                            }
                        elif not incident_base_type and isinstance(column_name, AnnoSpan):
                            contained_spans = entities_by_type['species'].spans_contained_by_span(column_name)
                            if len(contained_spans) > 0:
                                logger.info(contained_spans[0].text)
                                incident_base_type = "caseCount"
                                entity = contained_spans[0].metadata['species']['entity']
                                incident_species = {
                                    'id': entity['id'],
                                    'label': entity['label'],
                                }

                        if incident_disease and incident_disease != CANNOT_PARSE:
                            disease_entity = incident_disease.metadata['disease']['entity']
                            incident_disease = {
                                'id': disease_entity['id'],
                                'label': disease_entity['label'],
                            }
                        elif not incident_base_type and isinstance(column_name, AnnoSpan):
                            contained_spans = entities_by_type['disease'].spans_contained_by_span(column_name)
                            if len(contained_spans) > 0:
                                incident_base_type = "caseCount"
                                entity = contained_spans[0].metadata['disease']['entity']
                                incident_disease = {
                                    'id': entity['id'],
                                    'label': entity['label'],
                                }

                        if incident_location and incident_location != CANNOT_PARSE:
                            incident_location = incident_location.metadata['geoname'].to_dict()
                            del incident_location['parents']
                        elif not incident_base_type and isinstance(column_name, AnnoSpan):
                            contained_spans = entities_by_type['geoname'].spans_contained_by_span(column_name)
                            if len(contained_spans) > 0:
                                incident_base_type = "caseCount"
                                incident_location = contained_spans[0].metadata['geoname'].to_dict()
                                del incident_location['parents']

                        if incident_date and incident_date != CANNOT_PARSE:
                            incident_date = incident_date.metadata['datetime_range']
                            if table.metadata.get('date_period'):
                                if incident_aggregation != "cumulative":
                                    incident_date = [
                                        incident_date[1] - table.metadata.get('date_period'),
                                        incident_date[1]]
                        if not incident_base_type:
                            continue
                        row_incidents.append(AnnoSpan(value.start, value.end, doc, metadata={
                            'base_type': incident_base_type,
                            'aggregation': incident_aggregation,
                            'value': incident_count,
                            'attributes': list(filter(lambda x: x, [count_status])),
                            'location': incident_location,
                            'resolvedDisease': incident_disease,
                            'dateRange': incident_date,
                            'species': incident_species
                        }))
                # If a count is marked as incremental if a count in the row greater
                # than it is cumulative.
                max_new_cases = -1
                max_new_deaths = -1
                for incident_span in row_incidents:
                    incident = incident_span.metadata
                    if incident['aggregation'] == "incremental":
                        if incident['base_type'] == 'caseCount':
                            if max_new_cases < incident['value']:
                                max_new_cases = incident['value']
                        else:
                            if max_new_deaths < incident['value']:
                                max_new_deaths = incident['value']
                for incident_span in row_incidents:
                    incident = incident_span.metadata
                    if incident['aggregation'] is None:
                        if incident['base_type'] == 'caseCount':
                            if max_new_cases >= 0 and incident['value'] > max_new_cases:
                                incident['aggregation'] = 'cumulative'
                        else:
                            if max_new_deaths >= 0 and incident['value'] > max_new_deaths:
                                incident['aggregation'] = 'cumulative'
                for incident_span in row_incidents:
                    incident = incident_span.metadata
                    if incident['aggregation'] == 'cumulative':
                        incident['type'] = "cumulative" + incident['base_type'][0].upper() + incident['base_type'][1:]
                    else:
                        incident['type'] = incident['base_type']
                    del incident['base_type']
                    del incident['aggregation']
                incidents.extend(row_incidents)
        return {'structured_incidents': AnnoTier(incidents)} CANNOT_PARSE = "Cannot parse"
from .annotator import Annotator, AnnoTier
from .annospan import AnnoSpan, SpanGroup
from .count_annotator import CountAnnotator
from .date_annotator import DateAnnotator
from .spacy_annotator import SpacyAnnotator
from .geoname_annotator import GeonameAnnotator
from .species_annotator import SpeciesAnnotator
from .disease_annotator import DiseaseAnnotator
from .structured_incident_annotator import StructuredIncidentAnnotator, CANNOT_PARSE
from collections import defaultdict
import datetime
import re
    Annotations in the same sentence are grouped.
    If sub-sentence phrase spans are provided, only the spans within each phase
    are assigned to its territory, while outside of it the usual rules apply.
    This is intended to improve associations when multiple counts for multiple
    locations appear in the same sentence.
    """
    doc = sent_spans[0].doc
    territories = []
    for sent_span, span_group in sent_spans.group_spans_by_containing_span(spans):
        if len(territories) == 0 or len(span_group) > 0:
            territories.append(AnnoSpan(
                sent_span.start, sent_span.end, doc,
                metadata=span_group))
        else:
            prev_territory = territories[-1]
            prev_single_sent_spans = [
                span for span in prev_territory.metadata
                if span.metadata.get('scope') == 'sentence']
            if len(prev_single_sent_spans) == 0:
                territories[-1] = AnnoSpan(
                    prev_territory.start, sent_span.end, doc,
                    metadata=prev_territory.metadata)
            else:
                last_doc_scope_spans = []
                for territory in reversed(territories):
                    last_doc_scope_spans = [
                        span for span in prev_territory.metadata
                        if span.metadata.get('scope') == 'document']
                    if len(last_doc_scope_spans) > 0:
                        break
                territories.append(AnnoSpan(
                    sent_span.start, sent_span.end, doc,
                    metadata=last_doc_scope_spans))
    phrase_territories = []
    for phrase_span, span_group in phrase_spans.group_spans_by_containing_span(spans, allow_partial_containment=True):
        if len(span_group) > 0:
            phrase_territories.append(AnnoSpan(
                phrase_span.start, phrase_span.end, doc,
                metadata=span_group))
    phrase_territories = AnnoTier(phrase_territories, presorted=True)
    return AnnoTier(territories).subtract_overlaps(phrase_territories) + phrase_territories


class IncidentAnnotator(Annotator):
    def annotate(self, doc, case_counts=None):
        if doc.date:
            publish_date = doc.date
        else:
            publish_date = datetime.datetime.now()
        if case_counts:
            case_counts = case_counts
        else:
            case_counts = doc.require_tiers('counts', via=CountAnnotator)
        geonames = doc.require_tiers('geonames', via=GeonameAnnotator)
        geoname_spans = [
            AnnoSpan(
                span.start,
                span.end,
                span.doc,
                metadata=dict(geoname=format_geoname(span.metadata['geoname'].to_dict())))
            for span in geonames]
        geoname_spans += [
            AnnoSpan(
                span.start,
                span.end,
                span.doc,
                metadata=dict(geoname={
                    'name': 'Earth',
                    'id': '6295630',
                    'asciiname': 'Earth',
                    'featureCode': 'AREA',
                    'countryCode': '',
                    'countryName': '',
                    'admin1Name': '',
                    'admin2Name': '',
                    'admin1Code': '',
                    'admin2Code': '',
                    'latitude': 0,
                    'longitude': 0,
                }))
            for span in doc.create_regex_tier(r"\b(global(ly)?|worldwide)\b").spans]
        geoname_spans += [
            AnnoSpan(
                span.start,
                span.end,
                span.doc,
                metadata={})
            for span in doc.create_regex_tier(r"\b(national(ly)?|nationwide)\b").spans]
        geonames = AnnoTier(geoname_spans)
        sent_spans = doc.require_tiers('spacy.sentences', via=SpacyAnnotator)
        disease_tier = doc.require_tiers('diseases', via=DiseaseAnnotator)
        species_tier = doc.require_tiers('species', via=SpeciesAnnotator)
        disease_mentions = defaultdict(lambda: 0)
        for span in disease_tier:
            disease_mentions[span.metadata['disease']['id']] += 1
        # Copy disease tier
        disease_tier = AnnoTier([
            AnnoSpan(span.start, span.end, span.doc, metadata=span.metadata)
            for span in disease_tier], presorted=True)
        # scope one off disease mentions to sentences.
        max_disease = max(disease_mentions.values()) if len(disease_mentions) > 0 else 0
        if max_disease > 5:
            for span in disease_tier:
                if disease_mentions[span.metadata['disease']['id']] == 1:
                    span.metadata['scope'] = 'sentence'
                else:
                    span.metadata['scope'] = 'document'

        species_tier = AnnoTier([
            AnnoSpan(span.start, span.end, span.doc, metadata=span.metadata)
            for span in species_tier], presorted=True)
        # scope one off species mentions to sentences.
        for span in species_tier:
            if disease_mentions[span.metadata['species']['id']] == 1:
                span.metadata['scope'] = 'sentence'
            else:
                span.metadata['scope'] = 'document'

        structured_incidents = doc.require_tiers(
            'structured_incidents', via=StructuredIncidentAnnotator)

        date_tier = doc.require_tiers('dates', via=DateAnnotator)