ARG1 = "Arg1" ARG2 = "Arg2" CHAR_SPAN = "CharacterSpanList" CONNECTIVE = "Connective" ENCODING = "utf-8" RAW_TEXT = "RawText" SENSE = "Sense" TOK_LIST = "TokenList" TOK_OFFS_IDX = 2 TYPE = "Type" DFLT_MODEL_PATH = os.path.join(MODEL_DIR, "pdtb.sense.model") DFLT_MODEL_TYPE = WANG DFLT_ECONN_PATH = os.path.join(ECONN_DIR, "ExpConn.txt") ALT_LEX = "AltLex" EXPLICIT = "Explicit" IMPLICIT = "Implicit" SVD = 1 LSTM = 2 MJR = 4 WANG = 8 XGBOOST = 16 PARSE_TREE = "parsetree" DEPS = "dependencies" WORDS = "words" SENTENCES = "sentences" SHORT2FULL = {"Expansion": "Expansion.Conjunction",
              "Comparison": "Comparison.Contrast",
              "Comparison.COntrast": "Comparison.Contrast"} class timeit(object):
    """Decorator class for measuring time performance.

    Attributes:
      msg (str): message to be printed on method invocation

    """

    def __init__(self, a_msg):
        """Class constructor.

        Args:
          a_msg (str): debug message to print

        """
        self.msg = a_msg

    def __call__(self, a_func):
        """Decorator function.

        Args:
          a_func (method): decorated method

        Returns:
          method: wrapped method

        """
        def _wrapper(*args, **kwargs):
            print(self.msg + " started", file=sys.stderr)
            start_time = datetime.utcnow()
            a_func(*args, **kwargs)
            end_time = datetime.utcnow()
            time_delta = (end_time - start_time).total_seconds()
            print(self.msg + " finished ({:.2f} sec)".format(time_delta),
                  file=sys.stderr)
        return wraps(a_func)(_wrapper)
from dsenser.constants import ARG1, ARG2, CHAR_SPAN, CONNECTIVE, ENCODING, \
    RAW_TEXT, SENSE, TOK_LIST, TOK_OFFS_IDX, TYPE, DFLT_MODEL_PATH, \
    DFLT_MODEL_TYPE, DFLT_ECONN_PATH, ALT_LEX, EXPLICIT, IMPLICIT, SVD, \
    LSTM, MJR, WANG, XGBOOST, PARSE_TREE, DEPS, WORDS, SENTENCES, SHORT2FULL
from dsenser.utils import timeit
from collections import Iterable
from cPickle import dump, load
from itertools import chain
            from dsenser.major import MajorSenser
            from dsenser.wang import WangSenser
            from dsenser.xgboost import XGBoostSenser
            from dsenser.svd import SVDSenser
            from dsenser.lstm import LSTMSenser
                from dsenser.svd import SVDSenser
                from dsenser.lstm import LSTMSenser
import codecs
import gc
import numpy as np
import os
import sys
        self.wbench = np.zeros((len(rels), len(self.cls2idx)))
        # iterate over each trained model and sum up their predictions
        for ipath in self.model_paths:
            print("ipath = {:s}".format(ipath).encode(ENCODING),
                  file=sys.stderr)
            with open(ipath, "rb") as ifile:
                imodel = load(ifile)
            imodel.batch_predict(rels, a_data, self.wbench)
            del imodel
            imodel = None
            gc.collect()
        # make final judgements
        idx = -1
        isense = None
        for i, irel in enumerate(rels):
            idx = int(np.argmax(self.wbench[i]))
            isense = self.idx2cls[idx]
            irel[SENSE].append(SHORT2FULL.get(isense, isense))
        # free memory occupied by workbench
        del self.wbench
        self.wbench = None
        gc.collect()
        # postprocess input relations
        self._postprocess_rels(rels)

    def _predict(self, a_rel, a_data):
        """Determine sense of discourse relation.

        Args:
          a_rel (dict):
            JSON instance representing discourse relation
          a_data (list):
            2-tuple(dict, dict): input rels and parses

        Returns:
          tuple(str, float): predicted label and its probability

        """
        # the best performing strategy so far is to return the highest mean
        # judgment
        x = self._prejudge(a_rel, a_data)
        x_mean = np.mean(x, axis=0)
        idx = np.argmax(x_mean)
        lbl = self.idx2cls[int(idx)]
        return (lbl, x_mean[idx])
        # earlier we were using a pre-trained judge tensor
        # idx, iprob = self.judge.predict(a_rel, self._prejudge(a_rel, a_data))
        # lbl = self.idx2cls[int(idx)]
        # return (lbl, iprob)

    def _prejudge(self, a_rel, a_data):
        """Collect judgments of single classifiers.

        Args:
          a_rel (dict):
            discourse relation whose sense should be predicted
          a_data (2-tuple(dict, dict)):
            list of input JSON data

        Returns:
          np.array: modified ``a_ret``

        """
        if self.wbench is None:
            self.wbench = np.zeros((len(self.models), len(self.cls2idx)))
        else:
            self.wbench *= 0
        for i, imodel in enumerate(self.models):
            imodel.predict(a_rel, a_data, self.wbench, i)
        return self.wbench

    def _prune_data(self, a_rels, a_parses):
        """Remove unnecessary information from data.

        Args:
        a_rels (list):
          list of input discourse relations
        a_parses (dict):
          parse trees

        Returns:
        2-tuple(list, dict):
          abridged input data

        """
        arg = None
        # clean-up relations
        for irel in a_rels:
            irel.pop("ID")
            irel[CONNECTIVE].pop(CHAR_SPAN)
            arg = irel[ARG1]
            arg.pop(CHAR_SPAN)
            arg.pop(RAW_TEXT)
            arg = irel[ARG2]
            arg.pop(CHAR_SPAN)
            arg.pop(RAW_TEXT)
        # clean-up parses
        w_attrs = None
        for isentences in a_parses.itervalues():
            for isent in isentences[SENTENCES]:
                isent.pop(PARSE_TREE)
                isent.pop(DEPS)
                for iword in isent[WORDS]:
                    iword[-1].clear()
        return (a_rels, a_parses)

    def _preprocess_rels(self, a_rels):
        """Preprocess input relations.

        Args:
        a_rels (list):
          input relations to be preprocessed

        Returns:
        (void):
          modifies ``a_rels`` in place

        """
        arg1 = arg2 = None
        for irel in a_rels:

            arg1 = irel[ARG1]