CATEGORY_WILDCARD = _CATEGORY_WILDCARD class Model:
    """A parser model, consisting of a set of grammar rules."""

    def __init__(self, default_restriction: Category, top_level_properties: Iterable[Property],
                 link_types: FrozenSet[LinkLabel], primary_leaf_rules: 'Iterable[leaf.LeafRule]',
                 secondary_leaf_rules: 'Iterable[leaf.LeafRule]',
                 branch_rules: 'Iterable[branch.BranchRule]',
                 tokenizer: Tokenizer, any_promoted_properties: Iterable[Property],
                 all_promoted_properties: Iterable[Property],
                 property_inheritance_rules: 'Iterable[prop_inh.PropertyInheritanceRule]',
                 language: Language, config_info: ModelConfig = None):
        self._default_restriction = default_restriction
        self._top_level_properties = make_property_set(top_level_properties)
        self._link_types = link_types
        self._primary_leaf_rules = frozenset(primary_leaf_rules)
        self._secondary_leaf_rules = frozenset(secondary_leaf_rules)
        self._branch_rules = frozenset(branch_rules)
        self._tokenizer = tokenizer
        self._any_promoted_properties = make_property_set(any_promoted_properties)
        self._all_promoted_properties = make_property_set(all_promoted_properties)
        self._property_inheritance_rules = frozenset(property_inheritance_rules)
        self._language = language
        self._config_info = config_info
        self._sequence_rules_by_link_type = {}

        for rule in self._branch_rules:
            if not isinstance(rule, sequence.SequenceRule):
                continue
            for index in range(len(rule.link_type_sets)):
                for link_type, left, right in rule.link_type_sets[index]:
                    if link_type not in self._sequence_rules_by_link_type:
                        self._sequence_rules_by_link_type[link_type] = set()
                    self._sequence_rules_by_link_type[link_type].add((rule, index))

    @property
    def default_restriction(self) -> Category:
        """The category that should be used as the restriction for most parsing operations."""
        return self._default_restriction

    @property
    def top_level_properties(self) -> FrozenSet[Property]:
        """Properties that are of relevance at the root of the parse tree."""
        return self._top_level_properties

    @property
    def link_types(self) -> FrozenSet[LinkLabel]:
        return self._link_types

    @property
    def any_promoted_properties(self) -> FrozenSet[Property]:
        """Properties that may be promoted if any element possesses them."""
        return self._any_promoted_properties

    @property
    def all_promoted_properties(self) -> FrozenSet[Property]:
        """Properties that may be promoted if all elements possess them."""
        return self._all_promoted_properties

    @property
    def tokenizer(self) -> Tokenizer:
        """The tokenizer used by this model."""
        return self._tokenizer

    @property
    def sequence_rules_by_link_type(self) \
            -> 'Mapping[LinkLabel, AbstractSet[Tuple[sequence.SequenceRule, int]]]':
        """The sequence rules of the model, organized by link type for rapid lookup."""
        return self._sequence_rules_by_link_type

    @property
    def language(self) -> Optional[Language]:
        """Get the language this parser model is designed for, if indicated."""
        return self._language

    @property
    def config_info(self) -> ModelConfig:
        """The configuration information for this model, if any."""
        return self._config_info

    @property
    def primary_leaf_rules(self) -> 'FrozenSet[leaf.LeafRule]':
        """High-confidence rules that can generate leaf nodes in the parse tree."""
        return self._primary_leaf_rules

    @property
    def secondary_leaf_rules(self) -> 'FrozenSet[leaf.LeafRule]':
        """Lower-confidence rules that can generate leaf nodes in the parse tree."""
        return self._secondary_leaf_rules

    @property
    def branch_rules(self) -> 'FrozenSet[branch.BranchRule]':
        """Rules that can generate branch nodes in a parse tree."""
        return self._branch_rules

    @property
    def property_inheritance_rules(self) -> 'FrozenSet[prop_inh.PropertyInheritanceRule]':
        """Rules for how properties are propagated upwards through the parse tree."""
        return self._property_inheritance_rules class ParseRule:
    _scoring_features: Dict[Optional[scoring.ScoringFeature], Tuple[float, float, int]]

    def __init__(self, default_score: float = None, default_accuracy: float = None):
        if default_score is None:
            default_score = .5
        if default_accuracy is None:
            default_accuracy = 0.001
        self._scoring_features = {None: (default_score, default_accuracy, 0)}

    # def __str__(self) -> str:
    #     raise NotImplementedError()

    def calculate_weighted_score(self, parse_node: 'trees.TreeNodeInterface') \
            -> Tuple[float, float]:
        default_score, default_weight, count = self._scoring_features[None]
        total_score = default_score * default_weight
        total_weight = default_weight
        for feature in self.iter_scoring_features(parse_node):
            if feature is not None and feature in self._scoring_features:
                score, weight, count = self._scoring_features[feature]
                total_score += score * weight
                total_weight += weight
        return total_score, total_weight

    def adjust_score(self, parse_node: 'trees.TreeNodeInterface', target: float) -> None:
        if not 0 <= target <= 1:
            raise ValueError("Score target must be in the interval [0, 1].")
        default_score, default_weight, count = self._scoring_features[None]
        count += 1
        error = (target - default_score) ** 2
        weight_target = 1 - error
        default_score += (target - default_score) / count
        default_weight += (weight_target - default_weight) / count
        self._scoring_features[None] = (default_score, default_weight, count)
        for feature in self.iter_scoring_features(parse_node):
            if feature in self._scoring_features:
                score, weight, count = self._scoring_features[feature]
                count += 1
            else:
                score, weight, _ = self._scoring_features[None]
                count = 2  # Default is counted as 1, plus one new measurement
            error = (target - score) ** 2
            weight_target = 1 - error
            score += (target - score) / count
            weight += (weight_target - weight) / count
            self._scoring_features[feature] = (score, weight, count)

    def get_score(self, feature: scoring.ScoringFeature) -> Tuple[float, float, int]:
        if feature in self._scoring_features:
            return self._scoring_features[feature]
        else:
            return 0, 0, 0

    def set_score(self, feature: Union[scoring.ScoringKey, scoring.ScoringFeature], score: float,
                  accuracy: float, count: int) -> None:
        if not isinstance(feature, scoring.ScoringFeature):
            feature = scoring.ScoringFeature(feature)
        score = float(score)
        accuracy = float(accuracy)
        if not 0 <= score <= 1:
            raise ValueError("Score must be in the interval [0, 1].")
        if not 0 <= accuracy <= 1:
            raise ValueError("Accuracy must be in the interval [0, 1].")
        if count < 0:
            raise ValueError("Count must be non-negative.")
        # noinspection PyTypeChecker
        self._scoring_features[feature] = (score, accuracy, count)

    def iter_all_scoring_features(self) -> Iterator[Optional[scoring.ScoringFeature]]:
        return iter(self._scoring_features)

    def iter_scoring_features(self, parse_node: 'trees.TreeNodeInterface') \
            -> Iterator[Optional[scoring.ScoringFeature]]:
        raise NotImplementedError() class GrammarParser:
    """Parsing of grammar files"""

    @staticmethod
    def parse_category(definition: str, offset: int = 1) -> Category:
        """Parse a category string, in the syntax used by grammar files."""
        definition = definition.strip()
        if '(' in definition:
            if not definition.endswith(')'):
                raise GrammarSyntaxError("Expected: ')' in category definition",
                                         offset=offset + len(definition))
            if definition.count('(') > 1:
                raise GrammarSyntaxError("Unexpected: '(' in category definition",
                                         offset=offset + definition.find("(",
                                                                         definition.find("(") + 1))
            if definition.count(')') > 1:
                raise GrammarSyntaxError("Unexpected: ')' in category definition",
                                         offset=offset + definition.find(")",
                                                                         definition.find(")") + 1))
            name, properties = definition[:-1].split('(')
            if ',' in name:
                raise GrammarSyntaxError("Unexpected: ',' in category definition",
                                         offset=offset + definition.find(","))
            if len(name.split()) > 1:
                raise GrammarSyntaxError("Unexpected: white space in category definition",
                                         offset=offset + len(name) + 1)
            properties = [prop.strip() for prop in properties.split(',')]
            for prop in properties:
                if not prop.strip():
                    if ",," in definition:
                        raise GrammarSyntaxError("Unexpected: ','",
                                                 offset=offset + definition.find(",,") + 1)
                    elif "(," in definition:
                        raise GrammarSyntaxError("Unexpected: ','",
                                                 offset=offset + definition.find("(,") + 1)
                    elif ",)" in definition:
                        raise GrammarSyntaxError("Unexpected: ')'",
                                                 offset=offset + definition.find(",)") + 1)
                    else:
                        raise GrammarSyntaxError("Unexpected: ')'",
                                                 offset=offset + definition.find("()") + 1)
            positive = [prop for prop in properties if not prop.startswith('-')]
            negative = [prop[1:] for prop in properties if prop.startswith('-')]
            for prop in negative:
                if prop.startswith('-'):
                    raise GrammarSyntaxError("Unexpected: '-'",
                                             offset=offset + definition.find('-' + prop))
                if prop in positive:
                    raise GrammarSyntaxError("Unexpected: prop is both positive and negative",
                                             offset=offset + definition.find(prop))
            return Category(name, [Property.get(n) for n in positive],
                            [Property.get(n) for n in negative])
        else:
            if ')' in definition:
                raise GrammarSyntaxError("Unexpected: ')' in category definition",
                                         offset=offset + definition.find(")"))
            if ',' in definition:
                raise GrammarSyntaxError("Unexpected: ',' in category definition",
                                         offset=offset + definition.find(","))
            if len(definition.split()) > 1:
                raise GrammarSyntaxError("Unexpected: white space in category definition",
                                         offset=offset + len(definition.split()[0]) + 1)
            if not definition:
                raise GrammarSyntaxError("Expected: category definition", offset=offset)
            return Category(definition)

    def parse_branch_rule_term(self, term: str, offset: int = 1) -> Tuple[bool, List[Category]]:
        is_head = False
        if term.startswith('*'):
            term = term[1:]
            offset += 1
            is_head = True
            if '*' in term:
                raise GrammarSyntaxError("Unexpected: '*'", offset=offset + term.find('*'))
        subcategories = []
        subcategory_definitions = term.split('|')
        for definition in subcategory_definitions:
            subcategory = self.parse_category(definition, offset=offset)
            subcategories.append(subcategory)
            offset += len(definition) + 1
        if not subcategories:
            raise GrammarSyntaxError("Expected: category", offset=offset)
        return is_head, subcategories

    @staticmethod
    def parse_branch_rule_link_type(term: str, offset: int = 1) -> Tuple[LinkLabel, bool, bool]:
        if '<' in term[1:]:
            raise GrammarSyntaxError("Unexpected: '<'",
                                     offset=offset + term.find('<', term.find('<') + 1))
        if '>' in term[:-1]:
            raise GrammarSyntaxError("Unexpected: '<'", offset=offset + term.find('>'))
        left = term.startswith('<')
        right = term.endswith('>')
        if left:
            term = term[1:]
        if right:
            term = term[:-1]
        if not term:
            raise GrammarSyntaxError("Expected: link type", offset=offset + left)
        return LinkLabel.get(term), left, right

    def parse_branch_rule(self, category: Category, definition: str,
                          offset: int = 1) -> SequenceRule:
        subcategory_sets = []
        link_types = []
        term = ''
        term_start = 0
        head_index = None
        for index in range(len(definition)):
            char = definition[index]
            if char.isspace():
                if not term:
                    continue
                if '>' in term or '<' in term:
                    if not subcategory_sets:
                        raise GrammarSyntaxError("Unexpected: link type",
                                                 offset=offset + term_start)
                    link_type, left, right = self.parse_branch_rule_link_type(term,
                                                                              offset + term_start)
                    if head_index is None:
                        if right:
                            raise GrammarSyntaxError("Unexpected: right link",
                                                     offset=offset + term_start)
                    else:
                        if left:
                            raise GrammarSyntaxError("Unexpected: left link",
                                                     offset=offset + term_start)
                    link_types[-1].add((link_type, left, right))
                else:
                    is_head, subcategories = self.parse_branch_rule_term(term,
                                                                         offset=offset + term_start)
                    if is_head:
                        if head_index is not None:
                            raise GrammarSyntaxError("Unexpected: '*'",
                                                     offset=(offset + term_start + term.find('*')))
                        head_index = len(subcategory_sets)
                    subcategory_sets.append(subcategories)
                    link_types.append(set())
                term = ''
            else:
                if not term:
                    term_start = index
                term += char
        if term:
            is_head, subcategories = self.parse_branch_rule_term(term, offset=offset + term_start)
            if is_head:
                if head_index is not None:
                    raise GrammarSyntaxError("Unexpected: '*'",
                                             offset=offset + term_start + term.find('*'))
                head_index = len(subcategory_sets)
            subcategory_sets.append(subcategories)
            link_types.append(set())
        if not subcategory_sets:
            raise GrammarSyntaxError("Expected: category", offset=offset)
        if link_types[-1]:
            raise GrammarSyntaxError("Expected: category", offset=offset + term_start + len(term))
        link_types = link_types[:-1]
        if head_index is None:
            if len(subcategory_sets) != 1:
                raise GrammarSyntaxError("Expected: '*'", offset=offset + term_start)
            head_index = 0
        return SequenceRule(category, subcategory_sets, head_index, link_types)

    def parse_grammar_definition_file(self, lines: Iterable[str],
                                      filename: str = None) -> List[SequenceRule]:
        branch_rules = []
        category = None
        sequence_found = False
        line_number = 0
        for raw_line in lines:
            line_number += 1
            try:
                line = raw_line.split('#')[0].rstrip()
                if not line:
                    continue
                if line[:1].isspace():
                    if ':' in line:
                        raise GrammarSyntaxError("Unexpected: ':'", offset=1 + line.find(':'))
                    if not category:
                        raise GrammarSyntaxError("Expected: category header",
                                                 offset=1 + line.find(line.strip()))
                    branch_rules.append(
                        self.parse_branch_rule(category, line.lstrip(),
                                               offset=1 + line.find(line.lstrip())))
                    sequence_found = True
                else:
                    if category is not None and not sequence_found:
                        raise GrammarSyntaxError("Expected: category sequence", offset=1)
                    if ':' not in line:
                        raise GrammarSyntaxError("Expected: ':'", offset=1 + len(line))
                    if line.count(':') > 1:
                        raise GrammarSyntaxError("Unexpected: ':'",
                                                 offset=1 + line.find(':', line.find(':') + 1))
                    header, sequence = line.split(':')
                    category = self.parse_category(header)
                    if sequence.strip():
                        branch_rules.append(
                            self.parse_branch_rule(category, sequence.lstrip(),
                                                   offset=1 + sequence.find(sequence.lstrip()))
                        )
                        sequence_found = True
                    else:
                        sequence_found = False
            except GrammarParserError as error:
                error.set_info(filename=filename, lineno=line_number, text=raw_line)
                raise error
            except Exception as original_exception:
                raise GrammarParserError(filename=filename,
                                         lineno=line_number, text=raw_line) from original_exception
        return branch_rules

    def parse_match_rule(self, definition: str, offset: int = 1) -> Tuple[SubtreeMatchRule, ...]:
        if not definition.startswith('['):
            raise GrammarSyntaxError("Expected: '['", offset=offset)
        if not definition.endswith(']'):
            raise GrammarSyntaxError("Expected: ']'", offset=offset + len(definition) - 1)
        generator_map = {
            'any_term': AnyTermMatchRule,
            'all_terms': AllTermsMatchRule,
            'compound': CompoundMatchRule,
            'head': HeadMatchRule,
            'one_term': OneTermMatchRule,
            'last_term': LastTermMatchRule,
        }
        rule_list = []
        for category_definition in definition[1:-1].split():
            category = self.parse_category(category_definition,
                                           offset=1 + definition.find(category_definition))
            generator = generator_map.get(str(category.name), None)
            if generator is None:
                raise GrammarSyntaxError("Unexpected: " + repr(category),
                                         offset=1 + definition.find(category_definition))
            assert callable(generator)
            rule_list.append(generator(category.positive_properties, category.negative_properties))
        if not rule_list:
            raise GrammarSyntaxError("Expected: category")
        return tuple(rule_list)

    def parse_conjunction_rule(self,
                               category: Category,
                               match_rules: List[Tuple[SubtreeMatchRule, ...]],
                               property_rules: List[Tuple[FrozenSet[Tuple[Any, bool]],
                                                    Tuple[SubtreeMatchRule, ...]]],
                               definition: str,
                               offset: int = 1) -> ConjunctionRule:
        single = False
        compound = False
        while definition[:1] in ('+', '-'):
            if definition[0] == '+':
                if compound:
                    raise GrammarSyntaxError("Unexpected: '+'", offset=offset)
                compound = True
            else:
                if single:
                    raise GrammarSyntaxError("Unexpected: '-'", offset=offset)
                single = True
            definition = definition[1:]
            offset += 1
        subcategory_sets = []
        link_types = []
        term = ''
        term_start = 0
        head_index = None
        for index in range(len(definition)):
            char = definition[index]
            if char.isspace():
                if not term:
                    continue
                if '>' in term or '<' in term:
                    if not subcategory_sets:
                        raise GrammarSyntaxError("Unexpected: link type",
                                                 offset=offset + term_start)
                    link_type, left, right = self.parse_branch_rule_link_type(term,
                                                                              offset + term_start)
                    if head_index is None:
                        if right:
                            raise GrammarSyntaxError("Unexpected: right link",
                                                     offset=offset + term_start)
                    else:
                        if left:
                            raise GrammarSyntaxError("Unexpected: left link",
                                                     offset=offset + term_start)
                    link_types[-1].add((link_type, left, right))
                else:
                    if len(subcategory_sets) >= 3:
                        raise GrammarSyntaxError("Unexpected: category",
                                                 offset=offset + term_start)
                    is_head, subcategories = self.parse_branch_rule_term(term,
                                                                         offset=offset + term_start)
                    if is_head:
                        if head_index is not None:
                            raise GrammarSyntaxError("Unexpected: '*'",
                                                     offset=offset + term_start + term.find('*'))
                        head_index = len(subcategory_sets)
                    subcategory_sets.append(subcategories)
                    link_types.append(set())
                term = ''
            else:
                if not term:
                    term_start = index
                term += char
        if term:
            if len(subcategory_sets) >= 3:
                raise GrammarSyntaxError("Unexpected: category", offset=offset + term_start)
            is_head, subcategories = self.parse_branch_rule_term(term, offset=offset + term_start)
            if is_head:
                if head_index is not None:
                    raise GrammarSyntaxError("Unexpected: '*'",
                                             offset=offset + term_start + term.find('*'))
                head_index = len(subcategory_sets)
            subcategory_sets.append(subcategories)
            link_types.append(set())
        if len(subcategory_sets) < 2:
            raise GrammarSyntaxError("Expected: category", offset=offset)
        if link_types[-1]:
            raise GrammarSyntaxError("Expected: category", offset=offset + term_start + len(term))
        link_types = link_types[:-1]
        if head_index is None:
            if len(subcategory_sets) != 1:
                raise GrammarSyntaxError("Expected: '*'", offset=offset + term_start)
            head_index = 0
        assert 2 <= len(subcategory_sets) <= 3
        if len(subcategory_sets) == 3:
            if head_index != 1:
                raise GrammarSyntaxError("Unexpected: category", offset=offset)
            leadup_cats, conjunction_cats, followup_cats = subcategory_sets
            leadup_link_types, followup_link_types = link_types
        else:
            assert len(subcategory_sets) == 2
            if head_index != 0:
                raise GrammarSyntaxError("Expected: category", offset=offset)
            leadup_cats = None
            conjunction_cats, followup_cats = subcategory_sets
            leadup_link_types = set()
            followup_link_types = link_types[0]
        return ConjunctionRule(category, match_rules, property_rules, leadup_cats, conjunction_cats,
                               followup_cats, leadup_link_types, followup_link_types, single,
                               compound)

    def parse_property_inheritance_file(self, lines: Iterable[str],
                                        filename: str = None) -> List[PropertyInheritanceRule]:
        inheritance_rules = []
        line_number = 0
        for raw_line in lines:
            try:
                line_number += 1
                line = raw_line.split('#')[0].rstrip()
                if not line:
                    continue
                if ':' not in line:
                    raise GrammarSyntaxError("Expected: ':'", offset=1 + len(line))
                if line.count(':') > 1:
                    raise GrammarSyntaxError("Unexpected: ':'",
                                             offset=1 + line.find(':', line.find(':') + 1))
                definition, additions = line.split(':')
                try:
                    category = self.parse_category(definition)
                except GrammarParserError as error:
                    error.set_info(text=line)
                    raise error
                except Exception as original_exception:
                    raise GrammarParserError(text=line) from original_exception
                additions = additions.split()
                if not additions:
                    raise GrammarSyntaxError("Expected: property", offset=1 + line.find(':') + 1)
                positive_additions = [addition for addition in additions
                                      if not addition.startswith('-')]
                negative_additions = [addition[1:] for addition in additions
                                      if addition.startswith('-')]
                # Double-negatives are not allowed
                if any(addition.startswith('-') for addition in negative_additions):
                    raise GrammarSyntaxError("Unexpected: '-'", offset=2 + line.find('--'))
                # Check that positive & negative additions don't conflict
                for addition in negative_additions:
                    if addition in positive_additions:
                        raise GrammarSyntaxError("Conflicting property signs: %s" % addition,
                                                 offset=1 + line.find(addition,
                                                                      line.find(addition) + 1))
                inheritance_rules.append(PropertyInheritanceRule(category, positive_additions,
                                                                 negative_additions))
            except GrammarParserError as error:
                if error.text is None:
                    error.set_info(text=raw_line)
                error.set_info(filename=filename, lineno=line_number)
                raise error
        return inheritance_rules

    def parse_conjunctions_file(self, lines: Iterable[str],
                                filename: str = None) -> List[ConjunctionRule]:
        """Load a conjunction grammar file, returning the conjunction rules parsed from it."""
        branch_rules = []
        category = None
        match_rules = []
        match_rules_closed = False
        property_rules = []
        property_rules_closed = False
        sequence_found = False
        line_number = 0
        for raw_line in lines:
            line_number += 1
            try:
                line = raw_line.split('#')[0].rstrip()
                if not line:
                    continue
                if line[:1].isspace():
                    if ':' in line:
                        raise GrammarSyntaxError("Unexpected: ':'", offset=1 + line.find(':'))
                    if not category:
                        raise GrammarSyntaxError("Expected: category header",
                                                 offset=1 + line.find(line.strip()))
                    if line.endswith(']') and '[' in line:
                        if line.lstrip().startswith('['):
                            if match_rules_closed:
                                raise GrammarSyntaxError("Unexpected: matching rule",
                                                         offset=1 + line.find('['))
                            match_rules.append(
                                self.parse_match_rule(line.lstrip(),
                                                      offset=1 + line.find(line.lstrip()))
                            )
                        else:
                            if property_rules_closed:
                                raise GrammarSyntaxError("Unexpected: property rule",
                                                         offset=1 + len(line) - len(line.lstrip()))
                            match_rules_closed = True
                            left_bracket_index = line.index('[')
                            property_names = line[:left_bracket_index].strip().split(',')
                            properties = set()
                            for property_name in property_names:
                                if property_name.startswith('-'):
                                    properties.add((Property.get(property_name[1:]), False))
                                else:
                                    properties.add((Property.get(property_name), True))
                            line_remainder = line[left_bracket_index:]
                            property_rules.append(
                                (frozenset(properties),
                                 self.parse_match_rule(
                                     line_remainder.lstrip(),
                                     offset=1 + line.find(line_remainder.strip()))
                                 ))
                    else:
                        match_rules_closed = True
                        property_rules_closed = True
                        if '[' in line:
                            raise GrammarSyntaxError("Unexpected: '['", offset=1 + line.find('['))
                        if ']' in line:
                            raise GrammarSyntaxError("Unexpected: ']'", offset=1 + line.find(']'))
                        branch_rules.append(
                            self.parse_conjunction_rule(category, match_rules, property_rules,
                                                        line.lstrip(),
                                                        offset=1 + line.find(line.lstrip()))
                        )
                        sequence_found = True
                else:
                    if category is not None and not sequence_found:
                        raise GrammarSyntaxError("Expected: category sequence", offset=1)
                    if ':' not in line:
                        raise GrammarSyntaxError("Expected: ':'", offset=1 + len(line))
                    if line.count(':') > 1:
                        raise GrammarSyntaxError("Unexpected: ':'",
                                                 offset=1 + line.find(':', line.find(':') + 1))
                    header, sequence = line.split(':')
                    category = self.parse_category(header)
                    match_rules = []
                    match_rules_closed = False
                    property_rules = []
                    property_rules_closed = False
                    if sequence.strip():
                        branch_rules.append(
                            self.parse_conjunction_rule(
                                category, match_rules, property_rules, sequence.lstrip(),
                                offset=1 + sequence.find(sequence.lstrip())
                            )
                        )
                        sequence_found = True
                    else:
                        sequence_found = False
            except GrammarParserError as error:
                error.set_info(filename=filename, lineno=line_number, text=raw_line)
                raise error
            except Exception as original_exception:
                raise GrammarParserError(filename=filename, lineno=line_number,
                                         text=raw_line) from original_exception
        return branch_rules

    def parse_suffix_file(self, lines: Iterable[str], filename: str = None) -> List[SuffixRule]:
        """Load a suffix grammar file, returning the suffix rules parsed from it."""
        leaf_rules = []
        line_number = 0
        for raw_line in lines:
            try:
                line_number += 1
                line = raw_line.split('#')[0].rstrip()
                if not line:
                    continue
                if ':' not in line:
                    raise GrammarSyntaxError("Expected: ':'", offset=1 + len(line))
                if line.count(':') > 1:
                    raise GrammarSyntaxError("Unexpected: ':'",
                                             offset=1 + line.find(':', line.find(':') + 1))
                definition, suffixes = line.split(':')
                category = self.parse_category(definition)
                suffixes = suffixes.split()
                if not suffixes or suffixes[0] not in ('+', '-'):
                    raise GrammarSyntaxError("Expected: '+' or '-'", offset=1 + line.find(':') + 1)
                positive = suffixes.pop(0) == '+'
                suffixes = frozenset(suffixes)
                if not suffixes:
                    suffixes = frozenset([''])
                leaf_rules.append(SuffixRule(category, suffixes, positive))
            except GrammarParserError as error:
                error.set_info(filename=filename, lineno=line_number, text=raw_line)
                raise error
            except Exception as original_exception:
                raise GrammarParserError(filename=filename, lineno=line_number,
                                         text=raw_line) from original_exception
        return leaf_rules

    def parse_special_words_file(self, lines: Iterable[str], filename: str = None) -> List[SetRule]:
        """Load a special words grammar file, returning the set rules parsed from it."""
        leaf_rules = []
        line_number = 0
        already_defined = {}
        for raw_line in lines:
            try:
                line_number += 1
                line = raw_line.split('#')[0].rstrip()
                if not line:
                    continue
                if ':' not in line:
                    raise GrammarSyntaxError("Expected: ':'", offset=1 + len(line))
                pieces = line.split(':')
                definition = pieces.pop(0)
                token_str = ':'.join(pieces)
                category = self.parse_category(definition)
                token_set = frozenset(token_str.split())
                rule = SetRule(category, token_set)
                rule_str = str(rule)
                if rule_str in already_defined:
                    # Merge them, so we don't cause a feature conflict downstream.
                    old_rule: SetRule = already_defined[rule_str]
                    token_set |= old_rule.tokens
                    leaf_rules.remove(old_rule)
                    rule = SetRule(category, token_set)
                leaf_rules.append(rule)
                already_defined[rule_str] = rule
            except GrammarParserError as error:
                error.set_info(filename=filename, lineno=line_number, text=raw_line)
                raise error
            except Exception as original_exception:
                raise GrammarParserError(filename=filename, lineno=line_number,
                                         text=raw_line) from original_exception
        return leaf_rules class Parse:
    # TODO: Should this be renamed to Forest?
    # TODO: Make sure the docstrings are up to date.
    """A finished parse. Stores the state of the parse during Parser's
    operation as a separate, first class object. Because a sentence can
    potentially be parsed in many different ways, also represents the
    collection of ParseTrees which apply to the input after parsing is
    complete."""

    def __init__(self, tokens, parse_trees: Iterable[ParseTree]):
        self._tokens = tokens
        self._parse_trees = frozenset(parse_trees)
        self._hash = None
        self._score = None
        self.update_weighted_score()

    def __hash__(self):
        if self._hash is None:
            self._hash = hash(self._tokens) ^ hash(self._parse_trees)
        return self._hash

    def __eq__(self, other):
        if not isinstance(other, Parse):
            return NotImplemented
        return self is other or (self._tokens == other._tokens and
                                 self._parse_trees == other._parse_trees)

    def __ne__(self, other):
        if not isinstance(other, Parse):
            return NotImplemented
        return not (self == other)

    def __str__(self):
        return self.to_str()

    def __repr__(self):
        return type(self).__name__ + "(" + repr(self._tokens) + ", " + repr(self._parse_trees) + ")"

    @property
    def tokens(self):
        return self._tokens

    @property
    def parse_trees(self):
        return self._parse_trees

    @property
    def coverage(self):
        """Compute the number of unique combinatoric variations of the trees in this forest."""
        return reduce(lambda a, b: a * b, (tree.coverage for tree in self._parse_trees), 1)

    def to_str(self, simplify=True):
        return '\n'.join(tree.to_str(simplify) for tree in self._parse_trees)

    def get_rank(self):
        score, weight = self.get_weighted_score()
        return self.total_gap_size(), len(self.parse_trees), -score, -weight

    def get_weighted_score(self):
        return self._score

    def adjust_score(self, target):
        for tree in self._parse_trees:
            tree.adjust_score(target)
        self.update_weighted_score()

    def update_weighted_score(self):
        total_weighted_score = 0.0
        total_weight = 0.0
        for tree in self._parse_trees:
            weighted_score, weight = tree.get_weighted_score()
            total_weighted_score += weighted_score
            total_weight += weight
        self._score = ((total_weighted_score / total_weight if total_weight else 0.0), total_weight)

    def restrict(self, categories):
        if isinstance(categories, categorization.Category):
            categories = [categories]
        trees = []
        for tree in self._parse_trees:
            for restricted in tree.restrict(categories):
                trees.append(restricted)
        return type(self)(self._tokens, trees)

    def iter_ambiguities(self):
        covered = set()
        for tree1 in self._parse_trees:
            covered.add(tree1)
            for tree2 in self._parse_trees:
                if tree2 in covered:
                    continue
                if tree1.is_ambiguous_with(tree2):
                    yield tree1, tree2

    def is_ambiguous(self):
        for _ in self.iter_ambiguities():
            return True
        return False

    def disambiguate(self):
        if len(self._parse_trees) <= 1:
            return self
        scores = {}
        for tree in self._parse_trees:
            scores[tree] = tree.get_weighted_score()
        trees = []
        for tree in sorted(scores, key=scores.get, reverse=True):
            for other_tree in trees:
                if tree.is_ambiguous_with(other_tree):
                    break
            else:
                trees.append(tree)
        return type(self)(self._tokens, trees)

    def _iter_disambiguation_tails(self, index, max_index, gaps, pieces,
                                   timeout):
        if timeout is not None and time.time() >= timeout:
            raise TimeoutError()
        if index >= len(self._tokens):
            if not gaps and not pieces:
                yield []
        elif index < max_index and pieces > 0:
            nearest_end = None
            for tree in self._parse_trees:
                if tree.token_start_index == index:
                    if nearest_end is None or tree.token_end_index < nearest_end:
                        nearest_end = tree.token_end_index
                    for tail in self._iter_disambiguation_tails(tree.token_end_index, max_index,
                                                                gaps, pieces - 1, timeout):
                        yield [tree] + tail
            if nearest_end is None:
                if gaps > 0:
                    for tail in self._iter_disambiguation_tails(index + 1, max_index, gaps - 1,
                                                                pieces, timeout):
                        yield tail
            else:
                for overlap_index in range(index + 1, nearest_end):
                    for tail in self._iter_disambiguation_tails(overlap_index, nearest_end, gaps,
                                                                pieces, timeout):
                        yield tail

    # TODO: This fails if we have a partial parse in the *middle* of the
    #       string, surrounded by gaps.
    def iter_disambiguations(self, gaps=None, pieces=None, timeout=None):
        if gaps is None:
            gaps_seq = range(self.total_gap_size(), len(self._tokens) + 1)
        else:
            gaps_seq = [gaps] if gaps >= self.total_gap_size() else []
        if pieces is None:
            pieces_seq = range(self.min_disambiguation_size(), len(self._tokens) + 1)
        else:
            pieces_seq = [pieces] if pieces >= self.min_disambiguation_size() else []
        try:
            success = False
            for gaps in gaps_seq:
                for pieces in pieces_seq:
                    for tail in self._iter_disambiguation_tails(0, len(self._tokens), gaps, pieces,
                                                                timeout):
                        yield type(self)(self._tokens, tail)
                        success = True
                    if success:
                        break
                if success:
                    break
        except TimeoutError:
            # Don't do anything; we just want to exit early if this
            # happens.
            pass

    def get_disambiguations(self, gaps=None, pieces=None, timeout=None):
        return set(self.iter_disambiguations(gaps, pieces, timeout))

    def get_ranked_disambiguations(self, gaps=None, pieces=None, timeout=None):
        ranks = {}
        for disambiguation in self.get_disambiguations(gaps, pieces, timeout):
            ranks[disambiguation] = disambiguation.get_rank()
        return ranks

    def get_sorted_disambiguations(self, gaps=None, pieces=None, timeout=None):
        ranks = self.get_ranked_disambiguations(gaps, pieces, timeout)
        return [(disambiguation, ranks[disambiguation])
                for disambiguation in sorted(ranks, key=ranks.get)]

    def iter_gaps(self):
        gap_start = None
        index = -1
        for index in range(len(self._tokens)):
            for tree in self._parse_trees:
                if tree.token_start_index <= index < tree.token_end_index:
                    if gap_start is not None:
                        yield gap_start, index
                        gap_start = None
                    break
            else:
                if gap_start is None:
                    gap_start = index
        if gap_start is not None:
            yield gap_start, index + 1

    def has_gaps(self):
        for _ in self.iter_gaps():
            return True
        return False

    def total_gap_size(self):
        size = 0
        for start, end in self.iter_gaps():
            size += end - start
        return size

    def max_tree_width(self):
        max_width = 0
        for tree in self._parse_trees:
            width = tree.token_end_index - tree.token_start_index
            if max_width is None or width > max_width:
                max_width = width
        return max_width

    def min_disambiguation_size(self):
        max_width = self.max_tree_width()
        if not max_width:
            return 0
        return int(math.floor(len(self._tokens) / float(max_width)))

    def get_parse_graphs(self):
        assert not self.is_ambiguous()
        from pyramids import traversal, parse_graph_builder
        traverser = traversal.DepthFirstTraverser()
        graph_builder = parse_graph_builder.ParseGraphBuilder()
        traverser.traverse(self, graph_builder)
        return graph_builder.get_graphs() class TreeNodeSet(TreeNodeInterface[PayloadType]):
    """A set of tree nodes that cover the same phrase with the same category."""

    def __init__(self, nodes: Union[TreeNodeInterface[PayloadType],
                                    Iterable[TreeNodeInterface[PayloadType]]]):
        self._nodes = set()  # type: Set[TreeNodeInterface[PayloadType]]
        self._parents = weakref.WeakSet()

        if isinstance(nodes, TreeNodeInterface):
            first_node = nodes
            self._best_node = first_node
            self.add(first_node)
        else:
            node_iterator = iter(nodes)
            try:
                first_node = next(node_iterator)
            except StopIteration:
                raise ValueError("ParseTreeNodeSet must contain at least one node.")
            self._best_node = first_node  # type: TreeNodeInterface

            self.add(first_node)
            for node in node_iterator:
                self.add(node)

    def __repr__(self) -> str:
        return type(self).__name__ + "(" + repr(self._nodes) + ")"

    def __iter__(self) -> Iterator[TreeNodeInterface[PayloadType]]:
        return iter(self._nodes)

    def __len__(self) -> int:
        return len(self._nodes)

    def __contains__(self, node: 'TreeNodeInterface[PayloadType]') -> bool:
        return node in self._nodes

    @property
    def best_node(self) -> TreeNodeInterface[PayloadType]:
        """Get the node in this node set with the highest score."""
        return self._best_node

    @best_node.setter
    def best_node(self, node: TreeNodeInterface[PayloadType]) -> None:
        """Get the node in this node set with the highest score."""
        assert node in self._nodes
        self._best_node = node

    @property
    def payload(self) -> PayloadType:
        """Get the payload of the node in this set with the highest score."""
        return self._best_node.payload

    @property
    def coverage(self) -> int:
        """Compute the number of unique combinatoric variations of the subtree rooted at this node
        set."""
        return sum(node.coverage for node in self._nodes)

    @property
    def components(self) -> Tuple[TreeNodeInterface[PayloadType], ...]:
        """Get the children of this tree node."""
        return self._best_node.components

    @property
    def score(self) -> Tuple[float, float]:
        return self._best_node.score

    @property
    def raw_score(self) -> Tuple[int, float, float]:
        return self._best_node.raw_score

    def recompute_weighted_score(self,
                                 affected_child: 'TreeNodeInterface[PayloadType]' = None) -> bool:
        """
        Update the parse-time score of the tree node set.

        If an affected_node is provided, the update is performed in a more efficient way that
        assumes that the affected_node is the only member of the node set whose score changed.

        If recurse is True (default) then propagate the score upward through all trees containing
        this node set.
        """
        if affected_child is None or affected_child is self._best_node:
            self._best_node = max(self._nodes, key=lambda n: n.score)
            return True
        elif self._best_node.score < affected_child.score:
            self._best_node = affected_child
            return True
        return False

    def is_leaf(self) -> bool:
        """Return a boolean indicating whether this node is a leaf."""
        return self._best_node.is_leaf()

    def has_parents(self) -> bool:
        """Return a boolean indicating whether this node set has any parents."""
        return bool(self._parents)

    def iter_parents(self) -> Iterator[TreeNodeInterface[PayloadType]]:
        """Iterate over the parents of this node set."""
        if self._parents:
            yield from self._parents

    def add(self, node: TreeNodeInterface[PayloadType]) -> None:
        """Add a new node to this node set."""
        if isinstance(node, TreeNodeSet):
            for member in node:
                self.add(member)
        else:
            if (self._best_node is not None and
                    not self._best_node.payload.is_compatible_with(node.payload)):
                raise ValueError("Node is not compatible.")
            self._nodes.add(node)
            node.add_parent(self)
            if self._best_node is None:
                self._best_node = node

    def iter_leaves(self) -> Iterator[TreeNode[PayloadType]]:
        """Iterate over the leaves of the subtree rooted at this node from left to right."""
        for node in self._nodes:
            yield from node.iter_leaves()

    def add_parent(self, parent: TreeNode[PayloadType]) -> None:
        """Add a weak reference to a parent of this node."""
        assert not parent.has_ancestor(self)
        self._parents.add(parent)

    def has_ancestor(self, ancestor: TreeNodeInterface[PayloadType]) -> bool:
        """Return a boolean indicating whether the subtree rooted at the given ancestor contains
        this node."""
        if ancestor is self:
            return True
        if not self._parents:
            return False
        return any(parent.has_ancestor(ancestor) for parent in self._parents) class ParseTreeUtils:
    """Utility methods for operating on trees at parse time."""

    def __init__(self):
        raise NotImplementedError("This class is not instantiable.")

    @staticmethod
    def make_leaf_parse_tree_node(tokens: TokenSequence, rule: ParseRule, token_index: int,
                                  category: Category) -> 'TreeNode':
        """Create a leaf tree node with a parse-time payload."""
        start = token_index
        head_spelling = tokens[start]
        end = start + 1
        payload = ParsingPayload(tokens, rule, None, category, head_spelling, start, end)
        node = TreeNode(payload, None)
        TreeUtils.update_weighted_score(node)
        return node

    @staticmethod
    def make_branch_parse_tree_node(tokens: TokenSequence, rule: ParseRule, head_index: int,
                                    category: Category,
                                    components: Sequence['TreeNodeInterface[ParsingPayload]']) \
            -> 'TreeNode':
        """Create a tree node with a parse-time payload."""
        components = tuple(components)
        if not components:
            raise ValueError("At least one component must be provided for a non-leaf node.")
        start = end = components[0].payload.token_start_index
        for component in components:
            if end != component.payload.token_start_index:
                raise ValueError("Discontinuity in component coverage.")
            end = component.payload.token_end_index
        head_spelling = components[head_index].payload.head_spelling
        payload = ParsingPayload(tokens, rule, head_index, category, head_spelling, start, end)
        node = TreeNode(payload, components)
        TreeUtils.update_weighted_score(node)
        return node

    @staticmethod
    def get_head_token_start(node: 'TreeNodeInterface[ParsingPayload]') -> int:
        """Return the starting index of the head token of the phrase for this tree node."""
        payload = node.payload
        if node.is_leaf():
            return payload.token_start_index
        return ParseTreeUtils.get_head_token_start(
            node.components[payload.head_component_index].best_node
        )

    @staticmethod
    def to_str(node: 'TreeNodeInterface[ParsingPayload]', simplify: bool = False) -> str:
        """Generate a string representation of a parse-time tree node."""
        payload = node.payload
        result = payload.category.to_str(simplify) + ':'
        if node.is_leaf():
            covered_tokens = ' '.join(
                payload.tokens[payload.token_start_index:payload.token_end_index]
            )
            result += ' ' + repr(covered_tokens) + ' ' + repr(payload.token_index_span)
            if not simplify:
                result += ' [' + str(payload.rule) + ']'
        elif len(node.components) == 1 and simplify:
            result += ' ' + ParseTreeUtils.to_str(node.components[0], simplify)
        else:
            if not simplify:
                result += ' [' + str(payload.rule) + ']'
            for component in node.components:
                result += '\n    ' + ParseTreeUtils.to_str(component, simplify).replace('\n',
                                                                                        '\n    ')
        return result class ModelBatchController:
    def _default_validator(attempt: Attempt, target: Target):
    def __init__(self, output_validator: Validator = None, threshold: float = 1):
    def run_batch(self, samples: SampleSet, attempt_generator: AttemptGenerator,
                  result_callback: ResultCallback = None,
                  failure_callback: FailureCallback = None) -> BatchTally:
    def run_one(self, input_val: Input, target: Target,
                attempt_generator: AttemptGenerator) -> IndividualTally: class ModelConfig:
    """Parser model configuration"""

    def __init__(self, config_file_path: str, defaults: Mapping[str, Any] = None):
        config_file_path = os.path.abspath(os.path.expanduser(config_file_path))

        if not os.path.isfile(config_file_path):
            raise FileNotFoundError(config_file_path)

        self._config_file_path = config_file_path

        data_folder = os.path.dirname(config_file_path)

        default_word_sets_folder = os.path.join(os.path.dirname(config_file_path), 'word_sets')

        if defaults is None:
            defaults = {}
        else:
            defaults = dict(defaults)
        for option, value in (('Tokenizer Type', 'standard'),
                              ('Discard Spaces', '1'),
                              ('Word Sets Folder', default_word_sets_folder)):
            if option not in defaults:
                defaults[option] = value

        config_parser = configparser.ConfigParser(defaults)
        config_parser.read(self._config_file_path)

        # Languages
        languages = {}
        for language_section in config_parser.sections():
            if not language_section.startswith('Language:'):
                continue
            language_header_name = language_section[9:].strip()
            language_name = config_parser.get(language_section, 'Name',
                                              fallback=language_header_name).strip()
            if not language_name:
                continue
            iso639_1 = config_parser.get(language_section, 'ISO 639-1', fallback=None)
            iso639_2 = config_parser.get(language_section, 'ISO 639-2', fallback=None)
            languages[language_header_name] = Language(language_name, iso639_1, iso639_2)

        # Model
        self._model_name = config_parser.get('Model', 'Name').strip()
        language_section_name = config_parser.get('Model', 'Language', fallback='').strip()
        self._model_language = (languages.get(language_section_name, None)
                                if language_section_name
                                else None)

        # Tokenizer
        self._tokenizer_provider = config_parser.get('Tokenizer', 'Provider').strip()
        self._tokenizer_type = config_parser.get('Tokenizer', 'Tokenizer Type').strip()
        self._discard_spaces = config_parser.getboolean('Tokenizer', 'Discard Spaces')
        language_section_name = config_parser.get('Tokenizer', 'Language', fallback='').strip()
        self._tokenizer_language = (languages.get(language_section_name, None)
                                    if language_section_name
                                    else None)

        # Properties
        self._default_restriction = config_parser.get('Properties', 'Default Restriction',
                                                      fallback='sentence')
        self._top_level_properties = frozenset(
            categorization.Property.get(prop.strip())
            for prop in config_parser.get('Properties', 'Top-Level Properties').split(';')
            if prop.strip()
        )
        self._any_promoted_properties = frozenset(
            categorization.Property.get(prop.strip())
            for prop in config_parser.get('Properties', 'Any-Promoted Properties').split(';')
            if prop.strip()
        )
        self._all_promoted_properties = frozenset(
            categorization.Property.get(prop.strip())
            for prop in config_parser.get('Properties', 'All-Promoted Properties').split(';')
            if prop.strip()
        )
        self._property_inheritance_files = tuple(
            os.path.join(data_folder, path.strip())
            for path in config_parser.get('Properties', 'Property Inheritance File').split(';')
            if path.strip()
        )

        # Grammar
        self._grammar_definition_files = tuple(
            os.path.join(data_folder, path.strip())
            for path in config_parser.get('Grammar', 'Grammar Definition File').split(';')
            if path.strip()
        )
        self._conjunction_files = tuple(
            os.path.join(data_folder, path.strip())
            for path in config_parser.get('Grammar', 'Conjunctions File').split(';')
            if path.strip()
        )
        self._word_sets_folders = tuple(
            os.path.join(data_folder, path.strip())
            for path in config_parser.get('Grammar', 'Word Sets Folder').split(';')
            if path.strip()
        )
        self._suffix_files = tuple(
            os.path.join(data_folder, path.strip())
            for path in config_parser.get('Grammar', 'Suffix File').split(';')
            if path.strip()
        )
        self._special_words_files = tuple(
            os.path.join(data_folder, path.strip())
            for path in config_parser.get('Grammar', 'Special Words File').split(';')
            if path.strip()
        )
        self._name_cases = frozenset(
            categorization.Property.get(prop_name.strip())
            for prop_name in config_parser.get('Grammar', 'Name Cases').split(';')
            if prop_name.strip()
        )

        # Scoring
        self._score_file = os.path.join(data_folder, config_parser.get('Scoring', 'Score File'))

        # Benchmarking
        self._benchmark_file = os.path.join(data_folder,
                                            config_parser.get('Benchmarking', 'Benchmark File'))

    @property
    def config_file_path(self) -> str:
        """The expanded, absolute path to the primary configuration file for the model"""
        return self._config_file_path

    @property
    def model_name(self) -> str:
        """The name of the model."""
        return self._model_name

    @property
    def model_language(self) -> Optional[Language]:
        """The language of the model."""
        return self._model_language

    @property
    def tokenizer_provider(self) -> str:
        """The plugin provider for the tokenizer."""
        return self._tokenizer_provider

    @property
    def tokenizer_type(self) -> str:
        """The type of tokenizer to use with the model"""
        return self._tokenizer_type

    @property
    def tokenizer_language(self) -> Optional[Language]:
        """The language of the tokenizer."""
        return self._tokenizer_language

    @property
    def discard_spaces(self) -> bool:
        """Whether to discard spaces when tokenizing"""
        return self._discard_spaces

    @property
    def default_restriction(self) -> str:
        """The category that should be used as the restriction for most parsing operations."""
        return self._default_restriction

    @property
    def top_level_properties(self) -> FrozenSet[categorization.Property]:
        """Properties that are of relevance at the root of the parse tree."""
        return self._top_level_properties

    @property
    def any_promoted_properties(self) -> FrozenSet[categorization.Property]:
        """Properties promoted to the head if any child has them"""
        return self._any_promoted_properties

    @property
    def all_promoted_properties(self) -> FrozenSet[categorization.Property]:
        """Properties promoted to the head if all children have them"""
        return self._all_promoted_properties

    @property
    def property_inheritance_files(self) -> Tuple[str, ...]:
        """Property inheritance rule file paths"""
        return self._property_inheritance_files

    @property
    def grammar_definition_files(self) -> Tuple[str, ...]:
        """Grammar definition file paths"""
        return self._grammar_definition_files

    @property
    def conjunction_files(self) -> Tuple[str, ...]:
        """Conjunction rule file paths"""
        return self._conjunction_files

    @property
    def word_sets_folders(self) -> Tuple[str, ...]:
        """Folder paths where word files can be found"""
        return self._word_sets_folders

    @property
    def suffix_files(self) -> Tuple[str, ...]:
        """Suffix rule file paths"""
        return self._suffix_files

    @property
    def special_words_files(self) -> Tuple[str, ...]:
        """Special word rule file paths"""
        return self._special_words_files

    @property
    def score_file(self) -> str:
        """Score file path"""
        return self._score_file

    @property
    def benchmark_file(self) -> str:
        """Benchmark file path"""
        return self._benchmark_file

    @property
    def name_cases(self) -> FrozenSet[categorization.Property]:
        """Case properties that indicate names"""
        return self._name_cases class ModelLoader:
    """Model loading & saving."""

    def __init__(self, name: str, model_path: str, verbose: bool = False):
        self.verbose = bool(verbose)
        self._name = name
        self._model_path = model_path
        self._model_config_info = self.load_model_config()
        self._grammar_parser = GrammarParser()

    @property
    def model_config_info(self) -> ModelConfig:
        """Get the configuration info for the model."""
        return self._model_config_info

    def load_tokenizer(self, config_info: ModelConfig = None) -> Tokenizer:
        """Load an appropriately configured tokenizer instance."""
        from pyramids import plugins
        if config_info is None:
            config_info = self._model_config_info

        # Tokenizer
        tokenizer = plugins.get_tokenizer(config_info.tokenizer_provider,
                                          config_info.tokenizer_type,
                                          self._model_config_info)
        if tokenizer is None:
            raise ValueError("Unrecognized tokenizer type: " + config_info.tokenizer_type +
                             "\nDo you have the required plugin package installed?")
        return tokenizer

    def load_model(self, config_info: ModelConfig = None) -> Model:
        """Load the model and return it."""
        if config_info is None:
            config_info = self._model_config_info

        if self.verbose:
            print("Loading model from", config_info.config_file_path, "...")

        tokenizer = self.load_tokenizer(config_info)
        model = self._create_model(config_info, tokenizer)
        self._init_model_scoring(config_info, model)

        if self.verbose:
            print("Done loading model from", config_info.config_file_path + ".")

        return model

    def _init_model_scoring(self, config_info: ModelConfig, model: 'Model') -> None:
        # Scoring
        if self.verbose:
            print("Loading scoring features from", config_info.score_file + "...")
        LOGGER.info("Loading scoring features from %s.", config_info.score_file)
        if os.path.isfile(config_info.score_file):
            score_map = self.load_scoring_features(model, config_info.score_file)
            self.update_model_scoring_features(model, score_map)
        else:
            self.save_scoring_features(model, config_info.score_file)

    def _create_model(self, config_info: ModelConfig, tokenizer: Tokenizer) -> 'Model':
        default_restriction = GrammarParser.parse_category(config_info.default_restriction)
        top_level_properties = config_info.top_level_properties

        # Properties
        property_inheritance_rules = []
        for path in config_info.property_inheritance_files:
            property_inheritance_rules.extend(self.load_property_inheritance_file(path))

        # Grammar
        branch_rules = []
        primary_leaf_rules = []
        secondary_leaf_rules = []
        for path in config_info.grammar_definition_files:
            branch_rules.extend(self.load_grammar_definition_file(path))
        for path in config_info.conjunction_files:
            branch_rules.extend(self.load_conjunctions_file(path))
        for folder in config_info.word_sets_folders:
            primary_leaf_rules.extend(self.load_word_sets_folder(folder))
        for path in config_info.suffix_files:
            secondary_leaf_rules.extend(self.load_suffix_file(path))
        for path in config_info.special_words_files:
            primary_leaf_rules.extend(self.load_special_words_file(path))
        for case in config_info.name_cases:
            secondary_leaf_rules.append(CaseRule(Category('name'), case))

        if LOGGER.isEnabledFor(logging.WARNING):
            covered = {}
            for rule in chain(branch_rules, primary_leaf_rules, secondary_leaf_rules):
                rule_str = str(rule)
                if rule_str in covered:
                    LOGGER.warning("Conflict for rule key: %s", rule_str)
                    LOGGER.warning("    Rule #1: %r", covered[rule_str])
                    LOGGER.warning("    Rule #2: %r", rule)
                else:
                    covered[rule_str] = rule

        model_link_types = set()
        for rule in branch_rules:
            model_link_types.update(rule.all_link_types)
        model_link_types = frozenset(model_link_types)

        return Model(default_restriction, top_level_properties, model_link_types,
                     primary_leaf_rules, secondary_leaf_rules, branch_rules, tokenizer,
                     config_info.any_promoted_properties, config_info.all_promoted_properties,
                     property_inheritance_rules, config_info.model_language, config_info)

    def load_model_config(self, path: str = None) -> ModelConfig:
        """Load the model config info and return it."""
        if path and os.path.isfile(path):
            return ModelConfig(path)
        for search_path in (path,
                            os.path.abspath('.'),
                            os.path.abspath(os.path.expanduser('~')),
                            self._model_path):
            if search_path is None:
                continue
            for file_name in 'pyramids_%s.ini' % self._name, '%s.ini' % self._name:
                file_path = os.path.join(search_path, file_name)
                if os.path.isfile(file_path):
                    return ModelConfig(file_path)
        raise FileNotFoundError(path or '%s.ini' % self._name)

    def load_word_sets_folder(self, folder: str) -> List[SetRule]:
        """Load an entire folder of word sets in one go."""
        leaf_rules = []
        for file_name in os.listdir(folder):
            file_path = os.path.join(folder, file_name)
            if os.path.splitext(file_name)[-1].lower() == '.ctg':
                leaf_rules.append(SetRule.from_word_set(file_path, verbose=self.verbose))
            elif self.verbose:
                print("Skipping file " + file_path + "...")
        return leaf_rules

    def load_property_inheritance_file(self, path: str) -> List[PropertyInheritanceRule]:
        """Load a property inheritance file as a list of property inheritance rules."""
        with open(path, encoding='utf-8') as inheritance_file:
            return self._grammar_parser.parse_property_inheritance_file(inheritance_file,
                                                                        filename=path)

    def load_grammar_definition_file(self, path: str) -> List[SequenceRule]:
        """Load a grammar definition file as a list of branch rules."""
        with open(path, encoding='utf-8') as grammar_file:
            return self._grammar_parser.parse_grammar_definition_file(grammar_file, filename=path)

    def standardize_word_set_file(self, file_path: str) -> None:
        """Rewrite a word set file, removing duplicates and sorting the contents."""
        WordSetUtils.save_word_set(file_path, WordSetUtils.load_word_set(file_path))
        folder = os.path.dirname(file_path)
        file_name = os.path.basename(file_path)
        category = self._grammar_parser.parse_category(os.path.splitext(file_name)[0])
        if str(category) + '.ctg' != file_name:
            os.rename(file_path, os.path.join(folder, str(category) + '.ctg'))

    def standardize_word_sets_folder(self, folder: str) -> None:
        """Standardize an entire folder of word set files in one go."""
        for file_name in os.listdir(folder):
            file_path = os.path.join(folder, file_name)
            if file_path.endswith('.ctg'):
                self.standardize_word_set_file(file_path)

    def standardize_model(self, config_info: ModelConfig) -> None:
        """Clean up, normalize, and otherwise standardize the parser model."""
        for folder in config_info.word_sets_folders:
            self.standardize_word_sets_folder(folder)

    def get_word_set_categories(self, config_info: ModelConfig) -> Set[Category]:
        """Return the set of categories associated with word set files."""
        categories = set()
        for folder_path in config_info.word_sets_folders:
            for file_name in os.listdir(folder_path):
                if not file_name.lower().endswith('.ctg'):
                    continue
                category = self._grammar_parser.parse_category(file_name[:-4])
                categories.add(category)
        return categories

    def find_word_set_path(self, config_info: ModelConfig, category: Category) -> str:
        """Locate the word set file associated with a particular category, and return its path."""
        for folder_path in config_info.word_sets_folders:
            for file_name in os.listdir(folder_path):
                if not file_name.lower().endswith('.ctg'):
                    continue
                file_category = self._grammar_parser.parse_category(file_name[:-4])
                if file_category == category:
                    return os.path.join(folder_path, file_name)
        for folder_path in config_info.word_sets_folders:
            return os.path.join(folder_path, str(category) + '.ctg')
        raise IOError("Could not find a word sets folder.")

    def add_words(self, config_info: ModelConfig, category: Category,
                  added: Iterable[str]) -> Set[str]:
        """Add words to the word set file associated with a particular category."""
        path = self.find_word_set_path(config_info, category)
        if os.path.isfile(path):
            known_words = WordSetUtils.load_word_set(path)
        else:
            known_words = SortedSet()
        added = set(added)
        added.difference_update(known_words)
        known_words.update(added)
        WordSetUtils.save_word_set(path, known_words)
        return added

    def remove_words(self, config_info: ModelConfig, category: Category,
                     removed: Iterable[str]) -> Set[str]:
        """Remove words from the word set file associated with a particular category."""
        path = self.find_word_set_path(config_info, category)
        if not os.path.isfile(path):
            return set()
        known_words = WordSetUtils.load_word_set(path)
        removed = set(removed)
        removed.difference_update(known_words)
        WordSetUtils.save_word_set(path, known_words)
        return removed

    @staticmethod
    def load_scoring_features(model: Model, path: str = None) -> ScoreMap:
        """Load the scoring features for a model, assigning them to the appropriate rules of the
        model."""
        if path is None:
            path = model.config_info.score_file
        scores = {}  # type: ScoreMap
        with open(path, encoding='utf-8') as save_file:
            for line in save_file:
                rule_str, feature_str, score_str, accuracy_str, count_str = line.strip().split('\t')
                if rule_str not in scores:
                    scores[rule_str] = {}
                feature = ast.literal_eval(feature_str)
                if feature in scores[rule_str]:
                    LOGGER.warning("Duplicate feature key for rule %s: %r", rule_str, feature_str)
                scores[rule_str][feature] = (float(score_str), float(accuracy_str), int(count_str))
        return scores

    @staticmethod
    def update_model_scoring_features(model: Model, scores: ScoreMap) -> None:
        """Update the mode's scoring features in place, from a score map returned by
        load_scoring_features."""
        covered = set()
        for rule in model.primary_leaf_rules | model.secondary_leaf_rules | model.branch_rules:
            rule_str = repr(str(rule))
            if rule_str not in scores:
                continue
            covered.add(rule_str)
            for feature, (score, accuracy, count) in scores[rule_str].items():
                rule.set_score(feature, score, accuracy, count)
        if LOGGER.isEnabledFor(logging.WARNING):
            for rule_str in scores.keys() - covered:
                LOGGER.warning("Unrecognized rule in score map: %s", rule_str)

    @staticmethod
    def save_scoring_features(model: Model, path: str = None) -> None:
        """Save the scoring features for a model, gathering them from the model's rules."""
        if path is None:
            path = model.config_info.score_file
        LOGGER.info("Saving scoring features to %s.", path)
        with open(path, 'w', encoding='utf-8') as save_file:
            for rule in sorted(model.primary_leaf_rules |
                               model.secondary_leaf_rules |
                               model.branch_rules,
                               key=str):
                for feature in rule.iter_all_scoring_features():
                    score, accuracy, count = rule.get_score(feature)
                    if isinstance(feature, ScoringFeature):
                        feature = feature.key
                    if not count:
                        continue
                    save_file.write('\t'.join(repr(item) for item in (str(rule), feature, score,
                                                                      accuracy, count)))
                    save_file.write('\n')

    def load_conjunctions_file(self, path: str) -> List[ConjunctionRule]:
        """Load a conjunction grammar file, returning the conjunction rules parsed from it."""
        with open(path, encoding='utf-8') as conjunctions_file:
            return self._grammar_parser.parse_conjunctions_file(conjunctions_file, filename=path)

    def load_suffix_file(self, path: str) -> List[SuffixRule]:
        """Load a suffix grammar file, returning the suffix rules parsed from it."""
        with open(path, encoding='utf-8') as suffix_file:
            return self._grammar_parser.parse_suffix_file(suffix_file, filename=path)

    def load_special_words_file(self, path: str) -> List[SetRule]:
        """Load a special words grammar file, returning the set rules parsed from it."""
        with open(path, encoding='utf-8') as word_file:
            return self._grammar_parser.parse_special_words_file(word_file, filename=path) class ParsingAlgorithm:
    """Coordinates the search for new tokens and structures in a text using
    BaseRules and BuildRules, respectively. Stores the results in a Parse
    instance, which can be passed back in if additional input is received,
    or can be queried to determine the recognized structure(s) of the
    input."""

    @staticmethod
    def new_parser_state(model):
        """Return a new parser state, for incremental parsing."""
        return ParserState(model)

    @staticmethod
    def parse(parser_state, text, fast=False, timeout=None, emergency=False):
        """Parses a piece of text, returning the results."""
        for token, start, end in parser_state.model.tokenizer.tokenize(text):
            parser_state.add_token(token, start, end)
        if fast:
            parser_state.process_necessary_nodes(timeout, emergency)
        else:
            parser_state.process_all_nodes(timeout, emergency)
        return parser_state.get_parse()

    @staticmethod
    def extract(parse, language_graph_builder):
        """Extracts a sentence network from a parse."""
        return parse.build_language_graph(language_graph_builder) class GenerationAlgorithm:

    def generate(self, model: 'Model', sentence: graphs.ParseGraph) -> Set[trees.BuildTreeNode]:
        assert isinstance(sentence, graphs.ParseGraph)
        return self._generate(model, sentence.root_index, sentence)

    def _generate(self, model: 'Model', head_node: int,
                  sentence: graphs.ParseGraph) -> Set[trees.BuildTreeNode]:
        head_spelling = sentence[head_node][1]
        head_category = sentence[head_node][3]

        # Find the subnodes of the head node
        subnodes = sentence.get_sinks(head_node)

        # Build the subtree for each subnode
        subtrees = {sink: self._generate(model, sink, sentence) for sink in subnodes}

        # Find all leaves for the head node
        subtrees[head_node] = set()
        positive_case_properties, negative_case_properties = \
            leaf.LeafRule.discover_case_properties(head_spelling)
        for rule in model.primary_leaf_rules:
            if head_spelling in rule:
                category = rule.category.promote_properties(positive_case_properties,
                                                            negative_case_properties)
                category = extend_properties(model, category)
                if category in head_category:
                    tree = trees.BuildTreeNode(rule, category, head_spelling, head_node)
                    subtrees[head_node].add(tree)
        if not subtrees[head_node]:
            for rule in model.secondary_leaf_rules:
                if head_spelling in rule:
                    category = rule.category.promote_properties(positive_case_properties,
                                                                negative_case_properties)
                    category = extend_properties(model, category)
                    if category in head_category:
                        tree = trees.BuildTreeNode(rule, category, head_spelling, head_node)
                        subtrees[head_node].add(tree)

        results = set()
        backup_results = set()
        emergency_results = set()

        # If we only have the head node, the leaves for the head node can
        # serve as results
        if len(subtrees) == 1:
            if head_node == sentence.root_index:
                for tree in subtrees[head_node]:
                    if tree.category in sentence.root_category:
                        results.add(tree)
                    else:
                        backup_results.add(tree)
            else:
                results = set(subtrees[head_node])
        else:
            results = set()

        # For each possible subtree headed by the head node, attempt to
        # iteratively expand coverage out to all subnodes via branch rules.
        # TODO: This loop only works for non-conjunction rules because it
        #       assumes the link_type_sets and head_index properties are
        #       available. Move the code into appropriate methods on the
        #       branch rule subclasses and call into them.
        # TODO: Break this up into functions so it isn't so deeply nested.
        insertion_queue = set(subtrees[head_node])
        while insertion_queue:
            head_tree = insertion_queue.pop()
            for rule in model.branch_rules:
                fits = False
                for subcategory in rule.head_category_set:
                    if head_tree.category in subcategory:
                        fits = True
                        break
                if not fits:
                    continue
                possible_components = []
                failed = False
                # TODO: AttributeError: 'ConjunctionRule' object has no
                #       attribute 'link_type_sets'
                for index in range(len(rule.link_type_sets)):
                    required_incoming = set()
                    required_outgoing = set()
                    for link_type, left, right in rule.link_type_sets[index]:
                        if (right and index < rule.head_index) or (left and
                                                                   index >= rule.head_index):
                            required_incoming.add(link_type)
                        if (left and index < rule.head_index) or (right and
                                                                  index >= rule.head_index):
                            required_outgoing.add(link_type)
                    component_candidates = self.get_component_candidates(
                        model, head_category, head_node, index, required_incoming,
                        required_outgoing, rule, sentence, subnodes, subtrees
                    )
                    if not component_candidates:
                        failed = True
                        break
                    possible_components.append(component_candidates)
                if failed:
                    continue
                possible_components.insert(rule.head_index, {head_tree})
                for component_combination in product(*possible_components):
                    covered = set()
                    for component in component_combination:
                        if component.node_coverage & covered:
                            break
                        covered |= component.node_coverage
                    else:
                        category = rule.get_category(model,
                                                     [component.category
                                                      for component in component_combination])
                        if rule.is_non_recursive(category, head_tree.category):
                            new_tree = trees.BuildTreeNode(rule, category, head_tree.head_spelling,
                                                           head_tree.head_index,
                                                           component_combination)
                            if new_tree not in results:
                                if subnodes <= new_tree.node_coverage:
                                    if (new_tree.head_index != sentence.root_index or
                                            category in sentence.root_category):
                                        results.add(new_tree)
                                    else:
                                        backup_results.add(new_tree)
                                emergency_results.add(new_tree)
                                insertion_queue.add(new_tree)
        if results:
            return results
        elif backup_results:
            return backup_results
        else:
            return emergency_results

    @staticmethod
    def get_component_candidates(model: 'Model', head_category: 'categorization.Category',
                                 head_node: int, index: int,
                                 required_incoming: 'AbstractSet[categorization.LinkLabel]',
                                 required_outgoing: 'AbstractSet[categorization.LinkLabel]',
                                 rule: 'ParseRule', sentence: 'graphs.ParseGraph',
                                 subnodes: 'Set[int]',
                                 subtrees: Mapping[int, Tuple[trees.BuildTreeNode, ...]]):
        component_head_candidates = subnodes.copy()
        for link_type in required_incoming:
            if link_type not in model.sequence_rules_by_link_type:
                continue
            component_head_candidates &= {source for source in sentence.get_sources(head_node)
                                          if link_type in sentence.get_labels(source, head_node)}
            if not component_head_candidates:
                break
        if not component_head_candidates:
            return None
        for link_type in required_outgoing:
            if link_type not in model.sequence_rules_by_link_type:
                continue
            component_head_candidates &= {sink for sink in sentence.get_sinks(head_node)
                                          if link_type in sentence.get_labels(head_node, sink)}
            if not component_head_candidates:
                break
        if not component_head_candidates:
            return None
        component_candidates = set()
        if isinstance(rule, conjunction.ConjunctionRule):
            for candidate in component_head_candidates:
                for subtree in subtrees[candidate]:
                    if subtree.category in head_category:
                        component_candidates.add(subtree)
                        break
                else:
                    for subtree in subtrees[candidate]:
                        component_candidates.add(subtree)
                        break
        else:
            assert isinstance(rule, sequence.SequenceRule)
            cat_names = {category.name
                         for category
                         in rule.subcategory_sets[index if index < rule.head_index else index + 1]}
            for candidate in component_head_candidates:
                for subtree in subtrees[candidate]:
                    if subtree.category.name in cat_names:
                        good = False
                        cat_index = (index if index < rule.head_index else index + 1)
                        for category in rule.subcategory_sets[cat_index]:
                            if subtree.category in category:
                                good = True
                                break
                        if good:
                            component_candidates.add(subtree)
        return component_candidates class SampleUtils:
    def load(file_path: str) -> SampleSet:
    def save(samples: SampleSet, file_path: str) -> None:
import cmd
import os
import sys
import time
import traceback
    import cProfile as profile
    import profile
from typing import Optional, List, Iterator, Tuple, Dict
from pyramids.categorization import Category
from pyramids.model import Model
from pyramids.rules.parse_rule import ParseRule
    from graphviz import Digraph
from pyramids.grammar import GrammarParser
from pyramids.trees import Parse, TreeNodeSet, ParseTreeUtils
from pyramids.batching import Attempt, Result, ModelBatchController, FeedbackReceiver, Failure
from pyramids.config import ModelConfig
from pyramids.loader import ModelLoader
from pyramids.parsing import ParsingAlgorithm
from pyramids.generation import GenerationAlgorithm
from pyramids.sample_utils import Input, Target, SampleSet, SampleUtils
# -*- coding: utf-8 -*-

# TODO: This thing is a beast! Refactor.



try:
except ImportError:
    Digraph = None


try:
    # noinspection PyPep8Naming
except ImportError:


__author__ = 'Aaron Hosford'
__all__ = [
    'ParserCmd',
    'repl',
]


function_to_profile = None


class ParserCmd(cmd.Cmd):

    def __init__(self, model_loader: ModelLoader):
        cmd.Cmd.__init__(self)
        self._model_loader = model_loader
        self._model = model_loader.load_model()
        self.prompt = '% '
        self._simple = True
        self._show_broken = False
        self._parser_state = None
        self._parses = []  # type: List[Parse]
        self._whole_parses = 0
        self._parse_index = 0
        self._fast = False
        self._timeout_interval = 5
        self._emergency_mode = False
        self._benchmark_path = None
        self._benchmark: Optional[SampleSet] = None
        self._benchmark_dirty = False
        self._benchmark_emergency_disambiguations = 0
        self._benchmark_parse_timeouts = 0
        self._benchmark_disambiguation_timeouts = 0
        self._benchmark_time = 0.0
        self._benchmark_tests_completed = 0
        self._benchmark_update_time = time.time()
        self._last_input_text = None
        self.do_load()

    @property

    def model(self) -> Model: