class Lexsem(object):

    STRICT = 'strict'
    FLEXIBLE = 'flexible'
    ROBUST = 'robust'
    NAIVE = 'naive'
    MODES = [STRICT, FLEXIBLE, ROBUST, NAIVE] def read_config():
    if not __app_config.config and not __app_config.locate_config():
        # need to create a config
        config_dir = FileHelper.abspath('~/.coolisf')
        if not os.path.exists(config_dir):
            os.makedirs(config_dir)
        cfg_loc = os.path.join(config_dir, 'config.json')
        default_config = read_config_template()
        getLogger().warning("CoolISF configuration file could not be found. A new configuration file will be generated at {}".format(cfg_loc))
        getLogger().debug("Default config: {}".format(default_config))
        write_file(path=cfg_loc, content=default_config)
    config = __app_config.config
    return config def _get_config_manager():
    """ Internal function for retrieving application config manager object
    Don't use this directly, use read_config() method instead
    """
    return __app_config def write_file(content, path=None):
    """ Write content to a file, or to console if no path is provided """
    if isinstance(content, str):
        mode = 'wt'
    else:
        mode = 'wb'
    if path:
        getLogger().debug("Writing content to {}".format(path))
        if path.endswith('.gz'):
            with gzip.open(path, mode) as outfile:
                outfile.write(content)
        else:
            with open(path, mode) as outfile:
                outfile.write(content)
    else:
        print(content) class Transformer(object):

    def __init__(self, ruledb_path=None):
        # read rules
        self.rules = [self.get_guard_dog(),
                      self.get_green_tea(),
                      self.get_big_bad_wolf()]
        self.loaded = set()
        self.rule_map = dd(list)
        self.rule_cache = {}
        if not ruledb_path:
            # read from config file if not provided
            self.cfg = read_config()
            if self.cfg and 'lexrule_db' in self.cfg:
                ruledb_path = self.to_path(self.cfg['lexrule_db'])
        if ruledb_path and os.path.isfile(ruledb_path):
            getLogger().info("Lexrule DB location: {}".format(ruledb_path))
            self.rdb = LexRuleDB(ruledb_path)
        else:
            getLogger().warning("Rule DB could not be found. Only manual rules will be available.")
            self.rdb = None
            self.rule_signs = {}
        # sample rules
        self.add_rule(self.get_guard_dog())
        self.add_rule(self.get_green_tea())
        self.add_rule(self.get_big_bad_wolf())

    def to_path(self, path):
        return FileHelper.abspath(path.format(data_root=self.cfg['data_root']))

    def add_rule(self, rule):
        # getLogger().debug("Adding rule {}".format(rule.sign))
        self.rules.append(rule)
        self.rule_map[rule.sign].append(rule)

    def to_hcmp_rule(self, layout, lemma, adjacent=True):
        return SimpleHeadedCompound(layout, lemma, adjacent)

    def find_rules(self, nodes, limit=None):
        applicable_rules = []
        for node in nodes:
            applicable_rules.extend(self.rule_map[node.predstr])
        if self.rdb is None:
            return applicable_rules
        with self.rdb.ctx() as ctx:
            getLogger().debug("Searching for applicable rules for {} nodes: {}".format(len(nodes), nodes))
            ruleinfos = self.rdb.find_ruleinfo(nodes, limit=limit, ctx=None)
            getLogger().debug("Found {} rules".format(len(ruleinfos)))
            for ruleinfo in ruleinfos:
                ruleinfo = self.rdb.get_rule(ruleinfo.lid, ruleinfo.rid, ctx=ctx)
                if ruleinfo is not None:
                    lemma = ruleinfo.lemma.replace(' ', '+')
                    rule = self.to_hcmp_rule(ruleinfo[0].edit(), lemma)
                    if ruleinfo.ID not in self.loaded:
                        self.loaded.add(ruleinfo.ID)
                        self.add_rule(rule)
                    # getLogger().debug("{} vs {}".format(rule.sign, node.predstr))
                    applicable_rules.append(rule)
        getLogger().debug("Found {} rules in total".format(len(applicable_rules)))
        return applicable_rules

    def get_guard_dog(self):
        # [TODO] rules should not be hardcoded
        s = Reading("""[ TOP: h0 RELS: < [ compound<0:9> LBL: h1 ARG0: e4 [ e MOOD: indicative PERF: - PROG: - SF: prop TENSE: untensed ] ARG1: x6 [ x IND: + NUM: sg PERS: 3 ] ARG2: x5 [ x IND: + PT: notpro ] ]
          [ udef_q<0:5> LBL: h2 ARG0: x5 RSTR: h7 ]
          [ _guard_n_1<0:5> LBL: h3 ARG0: x5 ]
          [ _dog_n_1<6:9> LBL: h1 ARG0: x6 ] >
  HCONS: < h0 qeq h1 h7 qeq h3 > ]""").edit()
        return Compound(s, "guard+dog", adjacent=True)

    def get_green_tea(self):
        s = Reading("""[ TOP: h0
  INDEX: e2 [ e SF: prop-or-ques ]
  RELS: < [ _green_a_2<0:5> LBL: h8 ARG0: e9 [ e SF: prop TENSE: untensed MOOD: indicative PROG: bool PERF: - ] ARG1: x4 ]
          [ _tea_n_1<6:9> LBL: h8 ARG0: x4 ] >
  HCONS: < h0 qeq h8 > ]""").edit()
        return Compound(s, "green+tea", adjacent=True)

    def get_big_bad_wolf(self):
        s = Reading("""[ TOP: h0
  RELS: < [ _big_a_1<0:3> LBL: h1 ARG0: e2 [ e MOOD: indicative PERF: - PROG: bool SF: prop TENSE: untensed ] ARG1: x4 [ x IND: + NUM: sg PERS: 3 ] ]
          [ _bad_a_at<4:7> LBL: h1 ARG0: e3 [ e SF: prop ] ARG1: x4 ]
          [ _wolf_n_1<8:12> LBL: h1 ARG0: x4 ] >
  HCONS: < h0 qeq h1 > ]""").edit()
        return Compound(s, "big+bad+wolf", adjacent=True)

    def process(self, target):
        if isinstance(target, Reading) or isinstance(target, DMRS):
            return self.process(target.edit())
        else:
            # target is a DMRSEditor
            # collapse named compound
            for node in target.nodes:
                if node.predstr == "compound":
                    arg1 = node['ARG1']
                    if arg1 and Integral.is_named(arg1):
                        # collapse compound
                        Integral.collapse(arg1)
            getLogger().debug("locating rules for nodes {}".format(target.nodes))
            applicable_rules = self.find_rules(target.nodes)
            # apply MWE rules
            getLogger().debug("There are {} applicable rules for {}".format(len(applicable_rules), target.nodes))
            for rule in applicable_rules:
                rule.apply(target)
            return target

    def apply(self, target):
        if isinstance(target, Sentence):
            for parse in target:
                self.apply(parse)
            return target
        else:
            result = self.process(target)
            result.save()
            return target class GrammarHub:

    def __init__(self):
        self.read_config()
        self.grammars = {}
        if self.cache_path:
            temp_folder = os.path.dirname(self.cache_path)
            if not os.path.exists(temp_folder):
                os.makedirs(temp_folder)
            self.cache = ISFCache(self.cache_path)
        else:
            self.cache = None
        self.preps = ProcessorManager.from_json(self.cfg["preprocessors"])
        self.posts = ProcessorManager.from_json(self.cfg["postprocessors"])

    def read_config(self):
        self.cfg = read_config()
        if not self.cfg:
            raise Exception("Application configuration could not be read")
        if 'cache' in self.cfg:
            self.cache_path = self.to_path(self.cfg['cache'])
        else:
            self.cache_path = None
        getLogger().info("ISF Cache DB: {o} => {c}".format(o=self.cfg['cache'], c=self.cache_path))
        return self.cfg

    def to_path(self, path):
        return FileHelper.abspath(path.format(data_root=self.cfg['data_root']))

    @property
    def names(self):
        return tuple(self.cfg['grammars'].keys())

    @property
    def available(self):
        """ Available grammars and their friendly names """
        return {k: g['name'] for k, g in self.cfg['grammars'].items()}

    def __getattr__(self, name):
        return self.get_grammar(name)

    def __getitem__(self, grm):
        return self.get_grammar(grm)

    def get_grammar(self, grm):
        if grm not in self.cfg['grammars']:
            raise LookupError("Unknown grammar ({})".format(grm))
        if grm not in self.grammars:
            ginfo = self.cfg['grammars'][grm]
            ace_bin = ginfo['ace'] if 'ace' in ginfo else self.cfg['ace']
            if 'acecache' in ginfo:
                cache_loc = self.to_path(ginfo['acecache'])
            elif 'acecache' in self.cfg:
                cache_loc = self.to_path(self.cfg['acecache'])
            else:
                cache_loc = None
            preps = self.lookup_preps(ginfo)
            posts = self.lookup_posts(ginfo)
            grm_path = self.to_path(ginfo['path'])
            self.grammars[grm] = Grammar(grm, grm_path, ginfo['args'], ace_bin, cache_loc, preps=preps, posts=posts)
        # done creating grammar
        return self.grammars[grm]

    def lookup_preps(self, gcfg):
        if 'preps' not in gcfg:
            return None
        else:
            return [self.preps[pname] for pname in gcfg['preps']]

    def lookup_posts(self, gcfg):
        if 'posts' not in gcfg:
            return None
        else:
            return [self.posts[pname] for pname in gcfg['posts']]

    def parse_json(self, txt, grm, pc=None, tagger=None, ignore_cache=False):
        # validation
        if not txt:
            raise ValueError('Sentence cannot be empty')
        # look up from cache first
        if not ignore_cache and self.cache:
            s = self.cache.load(txt, grm, pc, tagger)
            if s is not None:
                getLogger().debug("Retrieved {} parse(s) from cache for sent: {}".format(len(s['parses']), s['sent']))
                return s
        # else parse it ...
        sent = self.parse(txt, grm, pc, tagger, ignore_cache)
        # cache sent if possible
        if self.cache and not ignore_cache:
            self.cache.save(txt, grm, pc, tagger, sent)
        # make it JSON
        return sent2json(sent, txt, pc, tagger, grm)

    def parse(self, txt, grm, pc=None, tagger=None, ignore_cache=False, wsd=None, ctx=None):
        """ Parse a sentence using ISF """
        # validation
        if not txt:
            raise ValueError('Sentence cannot be empty')
        # Parse sentence
        getLogger().debug("Parsing sentence: {}".format(txt))
        sent = self[grm].parse(txt, parse_count=pc, ignore_cache=ignore_cache)
        if tagger:
            getLogger().debug("Sense-tagging sentence using {}".format(tagger))
            sent.tag_xml(method=tagger, wsd=wsd, ctx=ctx)
        return sent def read_ace_output(ace_output_file, top1dmrs=False):
    """ Read output file from ACE batch mode
    Sample command: ace -g grammar.dat infile.txt > outfile.txt
    Read more: http://moin.delph-in.net/AceOptions
    """
    getLogger().info("Reading parsed MRS from %s..." % (ace_output_file,))
    c = Counter()
    items = []
    doc = Document(name=FileHelper.getfilename(ace_output_file))
    skipped = []
    with open(ace_output_file, 'r') as input_mrs:
        current_sid = 0
        while True:
            current_sid += 1
            line = input_mrs.readline()
            if line.startswith('SENT'):
                mrs_line = input_mrs.readline()
                # item = [line, mrs_line]
                s = Sentence(line[5:], ID=current_sid)
                doc.add(s)
                if not top1dmrs:
                    while mrs_line.strip():
                        s.add(mrs_line)
                        mrs_line = input_mrs.readline()
                    input_mrs.readline()
                else:
                    # all lines = a single DMRS
                    dmrs_text = ''
                    while mrs_line.strip():
                        dmrs_text += mrs_line
                        mrs_line = input_mrs.readline()
                    input_mrs.readline()
                    s.add(dmrs_text)
                c.count('sent')
                c.count('total')
            elif line.startswith('SKIP'):
                s = Sentence(line[5:], ID=current_sid)
                doc.add(s)
                skipped.append(line[5:].strip())
                items.append([line])
                input_mrs.readline()
                input_mrs.readline()
                c.count('skip')
                c.count('total')
            else:
                break
    c.summarise()
    FileHelper.save(ace_output_file + '.skipped.txt', '\n'.join(skipped))
    return doc def read_tsdb(path, name=None, title=None):
    """ Read a TSDB profile in ISF format """
    prof = itsdb.ItsdbProfile(path)
    if name is None:
        name = os.path.basename(path)
    if title is None:
        title = name
    doc = Document(name=name, title=title)
    # Read all sentences
    tbl_item = prof.read_table('item')
    for row in tbl_item:
        iid = row.get('i-id')
        raw_text = row.get('i-input').strip()
        sent = Sentence(text=raw_text, ident=iid)
        doc.add(sent)
    # Read all parses
    tbl_result = prof.read_table('result')
    for row in tbl_result:
        pid = row.get('parse-id')
        mrs = row.get('mrs')
        if pid not in doc.ident_map:
            raise Exception('pid {} cannot be found in provided TSDB profile'.format(pid))
        elif mrs:
            doc.ident_map[pid].add(mrs)
        else:
            raise Exception("Invalid MRS string in provided TSDB profile")
    return doc def read_tsdb_ttl(tsdb_path, ttl_path=None, name=None, title=None, *args, **kwargs):
    """ Combine TSDB profile and TTL profile to create ISF document (shallow + deep)
    This function return an instance of coolisf.model.Document
    """
    tsdb_doc = read_tsdb(tsdb_path, name=name, title=title)
    if ttl_path is None:
        ttl_path = tsdb_path
    ttl_doc = ttl.Document.read_ttl(ttl_path)
    isf_doc = tag_doc(tsdb_doc, ttl_doc, *args, **kwargs)
    getLogger().debug("TTL doc {} contains {} sentence(s).".format(ttl_path, len(ttl_doc)))
    getLogger().debug("isf_doc size: {}".format(len(isf_doc)))
    return isf_doc def tag_doc(isf_doc, ttl_doc, use_ttl_sid=True, wsd_method=None, wsd=None, taggold=True, on_error='raise', ctx=None, **kwargs):
    """ Tag an ISF document using a TTL """
    isf_doc = match_sents(isf_doc, ttl_doc)
    if isf_doc is None:
        raise Exception("isf_doc and ttl_doc could not be matched")
    not_matched = set()
    wsd = LeLeskWSD(dbcache=LeskCache())
    ctx = PredSense.wn.ctx()
    to_remove = set()
    for sent in isf_doc:
        if use_ttl_sid and sent.shallow and sent.shallow.ID:
            sent.ident = sent.shallow.ID
        for reading in sent:
            if wsd_method:
                sent.tag(method=wsd_method, wsd=wsd, ctx=ctx)
            if taggold and sent.shallow:
                try:
                    m, n, ignored = tag_gold(reading.dmrs(), sent.shallow, sent.text, **kwargs)
                except:
                    getLogger().exception("Could not process sentence #{}: {}".format(sent.ident, sent.text))
                    if on_error == 'ignore':
                        continue
                    elif on_error == 'remove':
                        to_remove.add(sent)
                        break
                    else:
                        raise
                # getLogger().debug("Matched: {}".format(m))
                if n:
                    not_matched.add(sent.ident)
                    sent.flag = Sentence.ERROR
            # update XML
            reading.dmrs().tag_xml(method=None, update_back=True, wsd=wsd, ctx=ctx)
        sent.tag_xml()
    for sent in to_remove:
        isf_doc.remove(sent)
    if not_matched:
        getLogger().warning("TSDB/IMI mismatched sentences: {} - IDs={}".format(len(not_matched), not_matched))
    return isf_doc def generate_gold_profile():
    # doc = read_gold_sents()
    print("Reading TSDB/TTL data")
    doc = read_tsdb_ttl(GOLD_PATH, ttl_path=GOLD_TTL_PATH)
    # Process data
    print("Creating XML file ...")
    # build root XML node for data file
    isf_node = build_root_node(filename=FileHelper.getfullfilename(OUTPUT_ISF))
    # Add document nodes
    doc.to_xml_node(isf_node)
    # write to file
    print("Making it beautiful ...")
    xml_string = etree.tostring(isf_node, encoding='utf-8', pretty_print=True)
    print("Saving XML data to file ...")
    write_file(xml_string, OUTPUT_ISF)
    print("ISF gold profile has been written to %s" % (OUTPUT_ISF,))
    print("All done!") def export_to_visko(sents, doc_path, pretty_print=True, separate=True, halt_on_error=True):
    """ Export sentences to XML files """
    print("Exporting %s sentences to Visko" % (len(sents),))
    print("Visko doc path: {}".format(doc_path))
    if separate:
        if not os.path.exists(doc_path):
            os.makedirs(doc_path)
        for sent in sents:
            sentpath = os.path.join(doc_path, str(sent.ident) + '.xml.gz')
            with gzip.open(sentpath, 'w') as f:
                try:
                    f.write(etree.tostring(sent.to_xml_node(), encoding='utf-8', pretty_print=pretty_print))
                except:
                    print("ERROR")
    else:
        # write to file
        chio.write_file(doc_path, sents.to_xml_str(pretty_print=pretty_print, strict=halt_on_error))
    print("Done!") def read_gold_sents(perform_wsd=False):
    """ Read gold sentences (TSDB+TTL) """
    return read_tsdb_ttl(GOLD_PATH, name="speckled", title="The Adventure of the Speckled Band") class Document(object):

    def __init__(self, name='', corpusID=None, title='', grammar=None, tagger=None, parse_count=None, lang=None, ID=None):
        self.ID = ID
        self.name = name
        self.corpusID = corpusID
        self.title = title if title else name
        self.grammar = grammar
        self.tagger = tagger
        self.sent_count = 0
        self.parse_count = parse_count
        self.lang = lang
        self.corpus = None
        self.sentences = []
        self.ident_map = {}
        self.id_map = {}
        pass

    def add(self, sent):
        """ Add a sentence """
        sent.docID = self.ID
        sent.doc = self
        if sent.ID:
            if sent.ID in self.id_map and self.id_map[sent.ID] is not None:
                getLogger().warning("{}: Duplicate sent ID {}".format(self, sent.ID))
            self.id_map[sent.ID] = sent
        if sent.ident:
            if sent.ident in self.ident_map and self.ident_map[sent.ident] is not None:
                getLogger().warning("{}: Duplicate sent ident {}".format(self, sent.ident))
            self.ident_map[sent.ident] = sent
        self.sentences.append(sent)

    def remove(self, sent):
        if sent and sent in self.sentences:
            if sent.ID in self.id_map:
                self.id_map.pop(sent.ID)
            if sent.ident in self.ident_map:
                self.ident_map.pop(sent.ident)
            self.sentences.remove(sent)
        else:
            raise Exception("Sentence object not found: {}".format(sent))

    def new(self, text='', ident=None):
        sent = Sentence(text=text, ident=ident)
        self.add(sent)
        return sent

    def __repr__(self):
        return "Document(name={})".format(repr(self.name))

    def __str__(self):
        return repr(self)

    def __len__(self):
        return len(self.sentences)

    def __getitem__(self, idx):
        return self.sentences[idx]

    def __iter__(self):
        return iter(self.sentences)

    def get(self, ID, **kwargs):
        if ID in self.id_map:
            return self.id_map[ID]
        elif 'default' in kwargs:
            return kwargs['default']
        else:
            raise LookupError("Sentence ID#{} could not be found".format(ID))

    def by_ident(self, ident, **kwargs):
        if ident in self.ident_map:
            return self.ident_map[ident]
        elif 'default' in kwargs:
            return kwargs['default']
        else:
            raise LookupError("Sentence ident#{} could not be found".format(ident))

    def to_xml_node(self, corpus_node=None, with_raw=True, with_shallow=True, with_dmrs=True, strict=True):
        doc_node = etree.Element('document')
        if corpus_node is not None:
            corpus_node.append(doc_node)
        doc_node.set('id', str(self.ID) if self.ID else '')
        doc_node.set('name', self.name if self.name else '')
        doc_node.set('title', self.title if self.title else '')
        for sent in self.sentences:
            try:
                sent.to_xml_node(doc_node=doc_node, with_raw=with_raw, with_shallow=with_shallow, with_dmrs=with_dmrs)
            except Exception as e:
                if strict:
                    raise e
                else:
                    logger = getLogger()
                    logger.warning("Sentence {} was corrupted: {}".format(sent.ID, sent.text))
        return doc_node

    def to_xml_str(self, corpus_node=None, pretty_print=True, with_raw=True, with_shallow=True, with_dmrs=True, strict=True):
        xml_node = self.to_xml_node(corpus_node, with_raw, with_shallow, with_dmrs, strict=strict)
        return etree.tostring(xml_node, pretty_print=pretty_print, encoding="utf-8").decode("utf-8")

    @staticmethod
    def from_xml_node(doc_node, idents=None):
        """ Read sentences from an XML node object """
        logger = getLogger()
        doc = Document(name=get_attr(doc_node, 'name', ''), title=get_attr(doc_node, 'title', ''))
        doc.ident = get_attr(doc_node, 'ident', None)
        doc.ID = get_attr(doc_node, 'ID', None)
        logger.debug("Document: name={} | title={} | ident={} | ID={}".format(doc.name, doc.title, doc.ident, doc.ID))
        sent_nodes = doc_node.findall("sentence")
        getLogger().debug("Found {} sent nodes".format(len(sent_nodes)))
        for sent_node in sent_nodes:
            sent = Sentence.from_xml_node(sent_node)
            if idents and sent.ident not in idents:
                getLogger().debug("Skipped sentence #{}".format(sent.ident))
                continue
            doc.add(sent)
        return doc

    @staticmethod
    def from_xml_str(xml_str, *args, **kwargs):
        doc_root = etree.XML(xml_str, parser=etree.XMLParser(huge_tree=True))
        if doc_root.tag == 'rootisf':
            doc_root = doc_root.find('document')
        return Document.from_xml_node(doc_root, *args, **kwargs)

    @staticmethod
    def from_file(file_path, *args, **kwargs):
        getLogger().info("Reading ISF Document from: {}".format(file_path))
        doc_str = read_file(file_path)
        getLogger().debug("Raw doc string size: {}".format(len(doc_str)))
        doc = Document.from_xml_str(doc_str, *args, **kwargs)
        getLogger().debug("Doc size: {}".format(len(doc)))
        return doc class RawCollection(MetaObject):

    def __init__(self, path, name='', title='', *args, **kwargs):
        super().__init__(path, *args, **kwargs)
        self.name = name if name else self.getinfo('name', os.path.basename(self.path))
        self.title = title if title else self.getinfo('title', '')

    def get_corpuses(self):
        corpuses = RawCorpusCollection()
        corpus_names = self.getinfo('corpuses', FileHelper.get_child_folders(self.path))
        for corpus_name in corpus_names:
            corpus_path = os.path.join(self.path, corpus_name)
            corpuses.add(RawCorpus(corpus_path))
        return corpuses

    def write_meta(self, name, title, corpuses):
        self.write_folder_meta({'name': name, 'title': title, 'corpuses': corpuses}) class PredSense(object):

    wn = get_wn()

    MODAL_VERBS = ('would', 'could', 'may', 'must', 'should', 'might', 'going+to', 'ought')
    PREPOSITIONS = ['of', 'to', 'on', 'as']
    IGNORED_GPREDS = ('named', 'a_q',
                      'time_n_rel', 'person_n_rel', 'place_n_rel', 'thing_n_rel', 'manner_n_rel',
                      'reason_n_rel', 'proper_q_rel')
    REPLACE_TEMPLATES = [['-', ' '],
                         ['-', ''],
                         ['-', '_'],
                         ['-', '+'],
                         ['_', ' '],
                         ['_', ''],
                         ['+', ' '],
                         ['+', '-'],
                         ['+', ''],
                         [' ', '-'],
                         [' ', '']]
    MANUAL_MAP = {'neg_rel': ('00024073-r',),
                  '_no_a_1_rel': ('00024356-r',),
                  'no_q_rel': ('02268485-a',),
                  'little-few_a_rel': ('01552885-a', '01554510-a'),
                  'more_comp_rel': ('00099341-r',),
                  '_more_x_comp_rel': ('00099341-r',),
                  '_more_a_1_rel': ('00099712-r',),
                  '_more+than_p_rel': ('01555133-a',),
                  'superl_rel': ('00111609-r',),
                  'the+most_q_rel': ('01555732-a',),
                  '_most_q_rel': ('01557120-a',),
                  'be_v_id_rel': ('02604760-v', ' 02664769-v', '02445925-v', '02616386-v'),
                  'be_v_there_rel': ('02749904-v', '02603699-v', '02655135-v')}

    @staticmethod
    def extend_lemma(lemma):
        """ Get a set of potential lemmas """
        potential = set()
        potential.add(lemma)
        for rfrom, rto in PredSense.REPLACE_TEMPLATES:
            potential.add(lemma.replace(rfrom, rto))
        # add lower case
        potential.add(lemma.lower())
        # negations & determiners
        if lemma.startswith('not+') or lemma.startswith('the+'):
            potential.add(lemma[4:])
        getLogger().debug("Potential for `{}': {}".format(lemma, potential))
        # remove prepositions as well
        no_preps = set()
        for p in potential:
            parts = p.split()
            if parts[-1] in PredSense.PREPOSITIONS:
                parts = parts[:-1]
                if parts:
                    no_preps.add(' '.join(parts))
        potential.update(no_preps)
        return potential

    singleton_sm = None
    search_cache = dd()

    @staticmethod
    def search_sense(lemmata, pos=None, ctx=None):
        search_key = (tuple(lemmata), pos)
        if search_key in PredSense.search_cache:
            return copy.deepcopy(PredSense.search_cache[search_key])
        if ctx is None:
            with PredSense.wn.ctx() as ctx:
                return PredSense.search_sense(lemmata, pos=pos, ctx=ctx)
        # ctx is ensured to be not null
        """ Return a SynsetCollection """
        if pos and pos in ('x', 'p'):
            pos = None
        potential = SynsetCollection()
        getLogger().debug("search sense lemmas={} (pos={})".format(lemmata, pos))
        for lemma in lemmata:
            synsets = PredSense.wn.search(lemma, pos=pos, ctx=ctx)
            getLogger().debug("search_sense: {} (pos={}): {}".format(lemma, pos, synsets))
            for synset in synsets:
                if synset.ID not in potential:
                    synset.lemma = lemma
                    potential.add(synset)
        PredSense.search_cache[search_key] = potential
        return potential

    # alias
    @staticmethod
    def search_pred_string(pred_str, extend_lemma=True, ctx=None):
        if not pred_str:
            raise Exception("pred_str cannot be empty")
        if ctx is None:
            with PredSense.wn.ctx() as ctx:
                return PredSense.search_pred_string(pred_str, extend_lemma=extend_lemma, ctx=ctx)
        # ctx will never be null
        # ensure that pred_str is really a str
        pred_str = str(pred_str)
        pred = Pred.string_or_grammar_pred(pred_str)
        # use manual mapping whenever possible
        pred_str_search = pred_str if pred_str.endswith('_rel') else pred_str + '_rel'
        # if this is a known preds
        if pred_str_search in MWE_ERG_PRED_LEMMA:
            ss = PredSense.search_sense([MWE_ERG_PRED_LEMMA[pred_str_search]], pred.pos)
            if ss:
                return sorted(ss, key=lambda x: x.tagcount, reverse=True)
        return PredSense.search_pred(pred, extend_lemma)

    @staticmethod
    def get_wn_pos(ep, **kwargs):
        candidates = list(PredSense.search_ep(ep, **kwargs))
        if candidates:
            return candidates[0].ID.pos
        else:
            return ep.pred.pos if ep.pred.pos and ep.pred.pos in 'nvar' else 'x'

    @staticmethod
    def search_ep(ep, extend_lemma=True, ctx=None):
        candidates = None
        pred_str = str(ep.pred)
        if not pred_str.endswith('_rel'):
            pred_str += "_rel"
        getLogger().debug("search_ep {} | pred={}".format(ep, pred_str))
        if pred_str in PredSense.MANUAL_MAP:
            return PredSense.wn.get_synsets(PredSense.MANUAL_MAP[pred_str], ctx=ctx)
        elif ep.pred.type == Pred.GRAMMARPRED and ep.carg:
            lemmas = PredSense.extend_lemma(ep.carg) if extend_lemma else (ep.cargs,)
            pos = None
            arg0 = ep.args['ARG0'] if 'ARG0' in ep.args else ''
            if arg0.startswith('e'):
                pos = 'a'
            elif arg0.startswith('i') or arg0.startswith('x'):
                pos = 'n'
            candidates = PredSense.search_sense(lemmas, pos=pos, ctx=ctx)
            getLogger().debug("Candidate for {}: {}".format(ep.pred.string, candidates))
        elif ep.pred.pos == 'a' and 'ARG1' in ep.args:
            if extend_lemma:
                lemmas = list(chain(PredSense.extend_lemma(ep.pred.lemma), PredSense.extend_lemma(ep.pred.lemma + 'ly')))
            else:
                lemmas = (ep.pred.lemma, ep.pred.lemma + 'ly')
            arg1 = ep.args['ARG1']
            # try to guess POS first
            if arg1.startswith('h') or arg1.startswith('e'):
                pos = 'r'
            else:
                pos = 'a'
            candidates = PredSense.search_sense(lemmas, pos=pos, ctx=ctx)
            if candidates:
                return candidates
            else:
                # change to the other
                old_pos = pos
                pos = 'r' if pos != 'r' else 'a'
                getLogger().debug("Not found {}, change POS {} -> {}".format(lemmas, old_pos, pos))
                candidates = PredSense.search_sense(lemmas, pos=pos, ctx=ctx)
                getLogger().debug("Candidate for {}: {}".format(ep.pred.string, candidates))
        else:
            candidates = PredSense.search_pred_string(ep.pred.string, ctx=ctx)
            getLogger().debug("Candidates for [{} [CARG '{}']]: {}".format(ep.pred.string, ep.carg, [(c, c.lemmas) for c in candidates]))
        if candidates:
            return candidates
        # try to search by known lemmas
        if pred_str in MWE_ERG_PRED_LEMMA:
            ss = PredSense.search_sense([MWE_ERG_PRED_LEMMA[pred_str]], ep.pred.pos)
            if ss:
                sorted(ss, key=lambda x: x.tagcount, reverse=True)
        # search by preds
        return PredSense.search_pred(ep.pred, auto_expand=extend_lemma, ctx=ctx)

    @staticmethod
    def search_pred(pred, auto_expand=True, ctx=None):
        """ Retrieve suitable synsets for a given predicate, return a SynsetCollection """
        # ensure not null ctx
        if ctx is None:
            with PredSense.wn.ctx() as ctx:
                return PredSense.search_pred(pred, auto_expand=auto_expand, ctx=ctx)
        # ctx will never be null
        if not pred:
            raise Exception("Predicate cannot be empty")
        # ignore some predicates
        if pred in PredSense.IGNORED_GPREDS:
            # don't map anything
            return SynsetCollection()
        # don't map modal verbs
        if pred.lemma in PredSense.MODAL_VERBS and pred.pos == 'v' and pred.sense == 'modal':
            #    ss = PredSense.search_sense(('modal',), 'a', ctx=ctx)
            return SynsetCollection()
        pred_str = str(pred)
        if not pred_str.endswith('_rel'):
            pred_str += "_rel"
        if pred_str in PredSense.MANUAL_MAP:
            return PredSense.wn.get_synsets(PredSense.MANUAL_MAP[pred_str], ctx=ctx)
        elif pred.pos == 'x' and pred.sense == 'subord':
                return SynsetCollection()
        elif pred.pos == 'u' and pred.sense == 'unknown' and '/' in pred.lemma:
            # unknown pred handling
            unk_lemma, unk_pos = pred.lemma.split('/')
            pos = ptpos_to_wn(unk_pos)
            getLogger().debug("Unknown | lemma: {} | unk_pos: {} | pos: {}".format(unk_lemma, unk_pos, pos))
            lemmata = [unk_lemma] if not auto_expand else list(PredSense.extend_lemma(unk_lemma))
        # elif (pred.pos == 'x' and pred.sense == 'deg') or pred.pos in 'pq':
        elif pred.pos and pred.pos in 'xpq':
            lemmata = [pred.lemma] if not auto_expand else list(PredSense.extend_lemma(pred.lemma))
            ssa = PredSense.search_sense(lemmata, 'a', ctx=ctx)
            ssr = PredSense.search_sense(lemmata, 'r', ctx=ctx)
            ssa = ssa.merge(ssr)
            return ssa
        else:
            lemmata = [pred.lemma] if not auto_expand else list(PredSense.extend_lemma(pred.lemma))
            pos = pred.pos
        if pred.type == Pred.GRAMMARPRED:
            # at this point, if it's not a known gpred, kill it
            return SynsetCollection()
        ss = PredSense.search_sense(lemmata, pos, ctx=ctx)
        # hardcode: try to match noun & adj/v
        if not ss and auto_expand:
            getLogger().debug("Trying to change POS for lemmas: {}".format(lemmata))
            # hard code modal
            if pred.pos == 'a':
                ss = PredSense.search_sense(lemmata, 'r', ctx=ctx)
                if not ss:
                    pos = 'n'
                    ss = PredSense.search_sense(lemmata, pos, ctx=ctx)
            elif pred.pos == 'n':
                pos = 'a'
                ss = PredSense.search_sense(lemmata, pos, ctx=ctx)
        # Done
        ss.synsets.sort(key=lambda x: x.tagcount, reverse=True)
        # return sorted(ss.synsets, key=lambda x: x.tagcount, reverse=True)
        return ss
import os
import logging
import collections
from texttaglib.chirptext.cli import CLIApp, setup_logging
from texttaglib.chirptext import header, confirm, TextReport, FileHelper, Counter, Timer
from texttaglib.chirptext.leutile import is_number
from texttaglib.chirptext import texttaglib as ttl
from lelesk import LeLeskWSD
from lelesk import LeskCache  # WSDResources
from coolisf import __version__
from coolisf.lexsem import Lexsem
from coolisf.config import read_config, _get_config_manager
from coolisf.common import write_file
from coolisf.morph import Transformer
from coolisf.ghub import GrammarHub
from coolisf.util import read_ace_output
from coolisf.dao import read_tsdb
from coolisf.gold_extract import read_tsdb_ttl, tag_doc
from coolisf.gold_extract import generate_gold_profile
from coolisf.gold_extract import export_to_visko
from coolisf.gold_extract import read_gold_sents
from coolisf.model import Document
from coolisf.dao.textcorpus import RawCollection
from coolisf.mappings import PredSense
        sents = read_ace_output(args.file, top1dmrs=args.top1dmrs)  # e.g. data/wndefs.nokey.mrs.txt
        if args.topk:
            sents = sents[:int(args.topk)]
    else:
        if input("No MRS file provided. Proceed to export gold profile to Visko (yes/no)? ").lower() in ['y', 'yes']:
            print("Exporting gold profile to Visko")
            sents = read_gold_sents()
        else:
            print("Aborted")
            return
    export_to_visko(sents, export_path, separate=args.separate, halt_on_error=not args.forgive)


def parse_isf(cli, args):
    ''' Process raw text file '''
    # verification
    if not os.path.isfile(args.infile):
        print("Error. File does not exist. (provided: {})".format(args.infile))
    if args.output and os.path.exists(args.output):
        if not confirm("Output file exists. Do you want to continue (Y/N)? "):
            print("Program aborted.")
            return
    # process
    header("Processing file: {}".format(args.infile))
    report = TextReport(args.output)
    ghub = GrammarHub()
    lines = FileHelper.read(args.infile).splitlines()
    wsd = LeLeskWSD(dbcache=LeskCache())
    ctx = PredSense.wn.ctx()
    timer = Timer(cli.logger)
    timer.start("Parsing {} sentences".format(len(lines)))
    for idx, sent in enumerate(ghub.ERG_ISF.parse_many_iterative(lines, parse_count=args.topk, ignore_cache=args.nocache)):
        if args.max and args.max < idx:
            break
        print("Processing sentence {} of {}".format(idx + 1, len(lines)))
        sent.tag_xml(method=args.wsd, wsd=wsd, ctx=ctx)
        report.writeline(sent.to_xml_str(pretty_print=not args.compact))
        report.writeline("\n\n")
    timer.stop("Finished")


def retag_doc(cli, args):
    ''' Re-tag an ISF document '''
    # verification
    if args.output and os.path.exists(args.output) and not args.yes:
        if not confirm("Output file exists. Do you want to continue (Y/N)? "):
            print("Program aborted.")
            return
    # process
    header("Processing file: {}".format(args.path))
    doc = Document.from_file(args.path, idents=args.ident)
    print("Document size: {}".format(len(doc)))
    if not args.wsd and not args.ttl:
        print("Nothing to do")
        return
    if args.wsd:
        print("Retagging document using {}".format(args.wsd))
        wsd = LeLeskWSD(dbcache=LeskCache())
        wsd.connect()
        ctx = PredSense.wn.ctx()
        for idx, sent in enumerate(doc):
            if args.topk and idx > args.topk:
                break
            if args.ident and sent.ident not in args.ident:
                continue
            print("Tagging sentence #{}/{}".format(idx + 1, len(doc)))
            try:
                sent.tag_xml(method=args.wsd, wsd=wsd, ctx=ctx)
            except Exception as e:
                if args.forgive:
                    getLogger().warning("Could not process sentence {}".format(sent.ID))
                else:
                    raise e
        ctx.close()
        wsd.disconnect()
    if args.ttl:
        print("Tagging doc {} using TTL doc {}".format(doc.name, args.ttl))
        ttl_doc = ttl.read(args.ttl, mode=args.ttl_format)
        kwargs = {}
        if args.mode:
            kwargs['mode'] = args.mode
        tag_doc(doc, ttl_doc, taggold=not args.nogold, on_error=args.on_error, **kwargs)
    doc_xml_str = doc.to_xml_str(pretty_print=not args.compact, with_dmrs=not args.nodmrs)
    write_file(doc_xml_str, args.output)


def extract_tsdb(cli, args):
    ''' Read parsed sentences from a TSDB profile '''
    if not os.path.exists(args.path):
        print("TSDB profile does not exist (path: {})".format(args.path))
    else:
        # read document
        timer = Timer(cli.logger)
        timer.start("Reading document")
        if os.path.isdir(args.path):
            # is TSDB
            if args.ttl:
                doc = read_tsdb_ttl(args.path, ttl_path=args.ttl, name=args.name, title=args.title)
            else:
                doc = read_tsdb(args.path)
            if args.ident or args.ids:
                print("Idents: {}".format(args.ident))
                doc_new = Document(name=doc.name, title=doc.title)
                for sent in doc:
                    if (sent.ident in args.ident) or (sent.ID in args.ids):
                        doc_new.add(sent)
                doc = doc_new
        elif os.path.isfile(args.path):
            # read doc XML
            doc = Document.from_file(args.path, idents=args.ident)
            print(len(doc))
            if args.ttl:
                ttl_doc = ttl.Document.read_ttl(args.ttl)
                doc = tag_doc(doc, ttl_doc)
        # process doc
        print("Found sentences: {}".format(len(doc)))
        print("With shallow: {}".format(len(list(s for s in doc if s.shallow))))
        timer.stop("Reading document")
        if args.isf:
            print("performing ISF transformation")

            transformer = Transformer()