YAML_AVAILABLE = True
    YAML_AVAILABLE = False
    OKGREEN = '\033[92m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
def str2bool(v):
def run_diagnostics():
def parse_args(parser):
def _load_config_file(arg_dict, path):
def parse_param_string(param):
def get_parser():
def get_args():
def validate_args(args): def encode(src_sentence):
def decode(trg_sentence):
def initialize(args):
    def encode(self, src_sentence):
    def decode(self, trg_sentence):
    def encode(self, src_sentence):
    def decode(self, trg_sentence):
    def encode(self, src_sentence):
    def decode(self, trg_sentence):
    def encode(self, src_sentence):
    def decode(self, trg_sentence):
    def __init__(self, codes_path, separator='@@', remove_eow=False):
    def process_line(self, line):
    def segment(self, sentence):
    def segment_tokens(self, tokens):
    def get_pairs(self, word):
    def encode(self, orig):
    def __init__(self, codes_path, separator='', remove_eow=False):
    def encode(self, src_sentence):
    def decode(self, trg_sentence):
    def decode(self, trg_sentence):
def load_src_wmap(path):
def load_trg_wmap(path):
class Encoder(object):
class Decoder(object):
class IDEncoder(Encoder):
class IDDecoder(Decoder):
class WordEncoder(Encoder):
class WordDecoder(Decoder):
class CharEncoder(Encoder):
class CharDecoder(Decoder):
class BPE(object):
class BPEEncoder(Encoder):
class BPEDecoder(Decoder):
class BPEAtAtDecoder(Decoder): GO_ID = 1
EOS_ID = 2
UNK_ID = 0
NOTAPPLICABLE_ID = 3
NEG_INF = float("-inf")
INF = float("inf")
EPS_P = 0.00001
    GO_ID = 0
    EOS_ID = 2
    UNK_ID = 3
    GO_ID = 2 # Usually not used
    EOS_ID = 1
    UNK_ID = 3 # Don't rely on this: UNK not standardized in T2T
TMP_FILENAME = '/tmp/sgnmt.%s.fst' % os.getpid()
MESSAGE_TYPE_DEFAULT = 1
MESSAGE_TYPE_POSTERIOR = 2
MESSAGE_TYPE_FULL_HYPO = 3
def switch_to_fairseq_indexing():
def switch_to_t2t_indexing():
def log_sum_tropical_semiring(vals):
def log_sum_log_semiring(vals):
def oov_to_unk(seq, vocab_size, unk_idx=None):
def argmax_n(arr, n):
def argmax(arr):
def common_viewkeys(obj):
def common_iterable(obj):
def common_get(obj, key, default):
def common_contains(obj, key):
def w2f(fstweight):
def load_fst(path):
def get_path(tmpl, sub = 1):
def split_comma(s, func=None):
    def notify(self, message, message_type = MESSAGE_TYPE_DEFAULT):
    def __init__(self):
    def add_observer(self, observer):
    def notify_observers(self, message, message_type = MESSAGE_TYPE_DEFAULT):
class Observer(object):
class Observable(object): class ParsePredictor(Predictor):
    """Predictor wrapper allowing internal beam search over a representation
    which contains some pre-defined 'non-terminal' ids, which should not appear 
    in the output.
    """
    def __init__(self, slave_predictor, normalize_scores=True, beam_size=4,
                 max_internal_len=35, nonterminal_ids=None):
        """Create a new parse wrapper for a predictor

        Args:
          slave_predictor: predictor to wrap with parse wrapper
          normalize_scores (bool): whether to normalize posterior scores,
                                   e.g. after some tokens have been removed
          beam_size (int): beam size for internal beam search over non-terminals
          max_internal_len (int): number of consecutive non-terminal tokens
                            allowed in internal search before path is ignored
          nonterminal_ids: file containing non-terminal ids, one per line          
        """
        super(ParsePredictor, self).__init__()
        self.predictor = slave_predictor
        self.normalize_scores = normalize_scores
        self.beam_size = beam_size
        self.max_internal_len = max_internal_len
        self.nonterminals = load_external_ids(nonterminal_ids)
        self.nonterminals.discard(utils.EOS_ID)
        self.nonterminals.discard(utils.UNK_ID)
        self.tok_to_hypo = {}

    def get_unk_probability(self, posterior):
        """Return unk probability as determined by slave predictor
        Returns:
            float, unk prob
        """
        return self.predictor.get_unk_probability(posterior)
   

    def are_best_terminal(self, posterior):
        """Return true if most probable tokens in posterior are all terminals
        (including EOS)
        """
        best_rule_ids = utils.argmax_n(posterior, self.beam_size)
        for tok in best_rule_ids:
            if tok in self.nonterminals:
                return False
        return True


    def predict_next(self, predicting_internally=False):
        """Predict next tokens.

        Args: 
          predicting_internally: will be true if called from internal 
                                 beam search, prevents infinite loop
        """
        original_posterior = self.predictor.predict_next()
        all_keys = utils.common_viewkeys(original_posterior)
        scores = {rule_id: original_posterior[rule_id] for rule_id in all_keys}

        scores = self.finalize_posterior(
            scores, 
            use_weights=True,
            normalize_scores=self.normalize_scores)
        if not predicting_internally:
            scores = self.find_word_beam(scores)
        return scores
    
    def maybe_add_new_top_tokens(self, top_terminals, hypo, next_hypos):
        new_post = self.predict_next(predicting_internally=True)
        top_tokens = utils.argmax_n(new_post, self.beam_size)
        next_state = copy.deepcopy(self.predictor.get_state())
        for tok in top_tokens:
            score = hypo.score + new_post[tok]
            new_hypo = InternalHypo(score, new_post[tok], next_state, tok)
            if tok not in self.nonterminals:
                add_hypo = False
                found = False
                for t in top_terminals:
                    if t == tok:
                        found = True
                        if self.tok_to_hypo[tok].score < new_hypo.score:
                            add_hypo = True
                            top_terminals.remove(t)
                        break
                if not found:
                    add_hypo = True
                if add_hypo:
                    top_terminals.append(tok)
                    self.tok_to_hypo[tok] = new_hypo
            else:
                next_hypos.append(new_hypo)

    def initialize_internal_hypos(self, posterior):
        top_tokens = utils.argmax_n(posterior, self.beam_size)
        hypos = []
        top_terminals = []
        for tok in top_tokens:
            new_hypo = InternalHypo(posterior[tok],
                                    posterior[tok],
                                    copy.deepcopy(self.predictor.get_state()),
                                    tok)
            if tok not in self.nonterminals:
                self.tok_to_hypo[tok] = new_hypo
                top_terminals.append(tok)
            hypos.append(new_hypo)
        return hypos, top_terminals

    def find_word_beam(self, posterior):
        """Internal beam search over posterior until a beam of terminals
        is found
        """
        hypos, top_terminals = self.initialize_internal_hypos(posterior)
        min_score = utils.NEG_INF
        if top_terminals:
            top_terminals.sort(key=lambda h: -self.tok_to_hypo[h].score)
            min_score = self.tok_to_hypo[top_terminals[-1]].score
        hypos.sort(key=lambda h: -h.score)
        # search until we have n terminal hypos with path scores better
        # than further internal search can give us
        while hypos and hypos[0].score > min_score:
            next_hypos = []
            for hypo in hypos:
                if not hypo.word_to_consume in self.nonterminals:
                    continue
                self.predictor.set_state(copy.deepcopy(hypo.predictor_state))
                self.consume(hypo.word_to_consume, internal=True)
                self.maybe_add_new_top_tokens(top_terminals, hypo, next_hypos)
            next_hypos.sort(key=lambda h: -h.score)
            hypos = next_hypos[:self.beam_size]
            top_terminals.sort(key=lambda t: -self.tok_to_hypo[t].score)
            top_terminals = top_terminals[:self.beam_size]
            if top_terminals:
                min_score = self.tok_to_hypo[top_terminals[-1]].score
        token_scores = [self.tok_to_hypo[t].score for t in top_terminals]
        return_post = {t: s for t, s in zip(top_terminals, token_scores)}
        return return_post

    def initialize(self, src_sentence):
        """Initializes slave predictor with source sentence 
        
        Args:
            src_sentence (list)
        """
        self.predictor.initialize(src_sentence)
    
    def consume(self, word, internal=False):
        try:
            if not internal:
                self.predictor.set_state(
                    copy.deepcopy(self.tok_to_hypo[word].predictor_state))
        except KeyError:
            logging.info('Consuming {}, not in tok-to-hypo'.format(word))
        return self.predictor.consume(word) 
    
    def get_state(self):
        """Returns the current state. """
        return self.predictor.get_state(), self.tok_to_hypo
    
    def set_state(self, state):
        """Sets the current state. """
        slave_state, tok_to_hypo = state
        self.tok_to_hypo = tok_to_hypo
        self.predictor.set_state(slave_state)
            
    def initialize_heuristic(self, src_sentence):
        """Creates a matrix of shortest distances between nodes. """
        pass
    
    def is_equal(self, state1, state2):
        """Returns true if the current node is the same """
        return state1 == state2 class TokParsePredictor(ParsePredictor):
    """
    Unlike ParsePredictor, the grammar predicts tokens according to a grammar. 
    Use BPEParsePredictor if including rules to connect BPE units inside words.
    """
    
    def __init__(self, grammar_path, slave_predictor, word_out=True,
                 normalize_scores=True, norm_alpha=1.0, 
                 beam_size=1, max_internal_len=35, allow_early_eos=False,
                 consume_out_of_class=False):
        """Creates a new parse predictor wrapper.

        Args:
            grammar_path (string): Path to the grammar file
            slave_predictor: predictor to wrap
            word_out (bool): since this wrapper can be used for grammar
                             constraint, this bool determines whether we
                             also do internal beam search over non-terminals
            normalize_scores (bool): true if normalizing scores, e.g. if some
                                     are removed from the posterior
            norm_alpha (float): may be used for path weight normalization
            beam_size (int): beam size for internal beam search
            max_internal_len (int): max number of consecutive nonterminals
                                    before path is ignored by internal search
            allow_early_eos (bool): true if permitting EOS consumed even if
                                    it is not permitted by the grammar
                                    at that point
            consume_out_of_class (bool): true if permitting any tokens to be 
                                         consumed even if not allowed by the
                                         grammar at that point
        """
        super(TokParsePredictor, self).__init__(slave_predictor,
            normalize_scores, beam_size, max_internal_len, nonterminal_ids)
        self.grammar_path = grammar_path
        self.word_out = word_out
        self.stack = []
        self.norm_alpha = norm_alpha
        self.check_n_best_terminal = False
        self.current_lhs = None
        self.current_rhs = []
        self.allow_early_eos=allow_early_eos
        self.consume_ooc = consume_out_of_class
        self.prepare_grammar()
        self.tok_to_internal_state = {}
        
    def norm_hypo_score(self, hypo):
        hypo.norm_score = self.norm_score(hypo.score, hypo.beam_len)
        
    def norm_score(self, score, beam_len):
        length_penalty = (5.0 + beam_len) / 6
        if self.norm_alpha != 1.0:
            length_penalty = pow(length_penalty, self.norm_alpha)
        return score / length_penalty

    def prepare_grammar(self):
        self.lhs_to_can_follow = {}
        with open(self.grammar_path) as f:
            for line in f:
                nt, rule = line.split(':')
                nt = int(nt.strip())
                self.lhs_to_can_follow[nt] = set(
                    [int(r) for r in rule.strip().split()])
        self.last_nt_in_rule = {nt: True for nt in self.lhs_to_can_follow}
        for nt, following in self.lhs_to_can_follow.items():
            if 0 in following:
                following.remove(0)
                self.last_nt_in_rule[nt] = False
            if self.allow_early_eos and utils.UNK_ID in following:
                self.lhs_to_can_follow[nt].add(utils.EOS_ID)
        self.lhs_to_can_follow[utils.EOS_ID].add(utils.UNK_ID)
                    

    def initialize(self, src_sentence):
        self.predictor.initialize(src_sentence)
        self.current_lhs = None
        self.current_rhs = []
        self.stack = [utils.EOS_ID]
        self.consume(utils.GO_ID)

    def replace_lhs(self):
        while self.current_rhs:
            self.stack.append(self.current_rhs.pop())
        if self.stack:
            self.current_lhs = self.stack.pop()
        else:
            self.current_lhs = utils.EOS_ID
            
    def get_current_allowed(self):
        if self.current_lhs:
            return self.lhs_to_can_follow[self.current_lhs]
        return set([utils.GO_ID])


    def predict_next(self, predicting_next_word=False):
        """predict next tokens as permitted by 
        the current stack and the grammar
        """
        original_posterior = self.predictor.predict_next()
        outgoing_rules = self.lhs_to_can_follow[self.current_lhs]       
        scores = {rule_id: original_posterior[rule_id] 
                  for rule_id in outgoing_rules}
        scores = self.finalize_posterior(
            scores, 
            use_weights=True,
            normalize_scores=self.normalize_scores)
        if self.word_out and not predicting_next_word:
            scores = self.find_word(scores)
        return scores
    
    def find_word_greedy(self, posterior):
        while not self.are_best_terminal(posterior):
            best_rule_id = utils.argmax(posterior)
            self.consume(best_rule_id)
            posterior = self.predict_next(predicting_next_word=True)
        return posterior

    def find_word_beam(self, posterior):
        """
        Do an internal beam search over non-terminal functions to find 
        the next best n terminal tokens, as ranked by normalized path score
        
        Returns: posterior containing up to n terminal tokens 
                 and their normalized path score
        """
        top_tokens = utils.argmax_n(posterior, self.beam_size)
        hypos = [InternalHypo(posterior[tok], self.get_state(), tok) 
                 for tok in top_tokens if tok in self.nonterminals]
        best_hypo = InternalHypo(utils.NEG_INF, None, None)
        best_posterior = None
        while hypos and hypos[0].norm_score > best_hypo.norm_score:
            next_hypos = []
            for hypo in hypos:
                self.set_state(copy.deepcopy(hypo.predictor_state))
                self.consume(hypo.word_to_consume)
                new_post = self.predict_next(predicting_next_word=True)
                top_tokens = utils.argmax_n(new_post, self.beam_size)
                next_state = copy.deepcopy(self.get_state())
                new_norm_score = self.norm_score(
                    new_post[top_tokens[0]] + hypo.score, hypo.beam_len + 1)
                if (self.are_best_terminal(new_post) and
                    new_norm_score > best_hypo.norm_score):
                    best_hypo = copy.deepcopy(hypo)
                    best_hypo.predictor_state = next_state
                    best_hypo.norm_score = new_norm_score
                    best_posterior = new_post
                    self.norm_score(best_hypo)
                else:
                    if hypo.beam_len == self.max_internal_len:
                        logging.info('cutting off internal hypo - too long')
                        continue
                    for tok in top_tokens:
                        if tok in self.nonterminals:
                            new_hypo = copy.deepcopy(hypo)
                            new_hypo.extend(new_post[tok], next_state, tok)
                            next_hypos.append(new_hypo)

            map(self.norm_hypo_score, next_hypos)
            next_hypos.sort(key=lambda h: -h.norm_score)
            hypos = next_hypos[:self.beam_size]
        self.set_state(best_hypo.predictor_state)
        for tok in best_posterior.keys():
            best_posterior[tok] = self.norm_score(
                best_hypo.score + best_posterior[tok], best_hypo.beam_len + 1)
            if tok in self.nonterminals:
                del best_posterior[tok]
        return best_posterior

    def find_word(self, posterior):
        """Check whether rhs of best option in posterior is a terminal
        if it is, return the posterior for decoding
        if not, take the best result and follow that path until a word is found
        this follows a greedy 1best or a beam path through non-terminals
        """
        if self.beam_size <= 1:
            return self.find_word_greedy(posterior)
        else:
          if self.are_best_terminal(posterior):
              return posterior
          else:
              return self.find_word_beam(posterior)

    def consume(self, word):
        """
        Args:
            word (int): word token being consumed
        """
        change_to_unk = (
            (word == utils.UNK_ID) or
            (not self.consume_ooc and word not in self.get_current_allowed()))
        if change_to_unk:
            word = utils.UNK_ID
        self.update_stacks(word)
        return self.predictor.consume(word) 
        
    
    def update_stacks(self, word):
        if word in self.nonterminals:
            self.current_rhs.append(word)
            if self.last_nt_in_rule[word]:
                self.replace_lhs()
        else:
            self.replace_lhs()  

    def get_state(self):
        """Returns the current state, including slave predictor state """
        return (self.stack, self.current_lhs, self.current_rhs,
                self.predictor.get_state())
    
    def set_state(self, state):
        """Sets the current state """
        self.stack, self.current_lhs, self.current_rhs, slave_state = state
        self.predictor.set_state(slave_state) class BpeParsePredictor(TokParsePredictor):
    """                                                                         
    Predict over a BPE-based grammar with two possible grammar constraints:
    one between non-terminals and bpe start-of-word tokens, one over
    bpe tokens in a word
    """

    def __init__(self, grammar_path, bpe_rule_path, slave_predictor,
                 word_out=True, normalize_scores=True,
                 norm_alpha=1.0, beam_size=1, max_internal_len=35,
                 allow_early_eos=False, consume_out_of_class=False, 
                 eow_ids=None, terminal_restrict=True, terminal_ids=None,
                 internal_only_restrict=False):
        """Creates a new parse predictor wrapper which can be constrained to 2
           grammars: one over non-terminals / terminals, one internally to
           constrain BPE units within a single word
        Args: 
            grammar_path (string): Path to the grammar file
            bpe_rule_path (string): Path to file defining rules between BPEs
            slave_predictor: predictor to wrap
            word_out (bool): since this wrapper can be used for grammar
                             constraint, this bool determines whether we
                             also do internal beam search over non-terminals
            normalize_scores (bool): true if normalizing scores, e.g. if some
                                     are removed from the posterior
            norm_alpha (float): may be used for path weight normalization
            beam_size (int): beam size for internal beam search
            max_internal_len (int): max number of consecutive nonterminals
                                    before path is ignored by internal search
            allow_early_eos (bool): true if permitting EOS consumed even if
                                    it is not permitted by the grammar
                                    at that point
            consume_out_of_class (bool): true if permitting any tokens to be
                                         consumed even if not allowed by the
                                         grammar at that point
            eow_ids (string): path to file containing ids of BPEs that mark
                              the end of a word
            terminal_restrict (bool): true if applying grammar constraint over 
                                      nonterminals and terminals
            terminal_ids (string): path to file containing all terminal ids
            internal_only_restrict (bool): true if applying grammar constraint
                                           over BPE units inside words
        """
        super(BpeParsePredictor, self).__init__(grammar_path,
                                                slave_predictor,
                                                word_out,
                                                normalize_scores,
                                                norm_alpha,
                                                beam_size,
                                                max_internal_len,
                                                allow_early_eos,
                                                consume_out_of_class)
        self.internal_only_restrict = internal_only_restrict
        self.terminal_restrict = terminal_restrict
        self.eow_ids = self.get_eow_ids(eow_ids)
        self.all_terminals = self.get_all_terminals(terminal_ids)
        self.get_bpe_can_follow(bpe_rule_path)
        
    def get_eow_ids(self, eow_ids):
        eows = set()
        if eow_ids:
            with open(eow_ids) as f:
                for line in f:
                    eows.add(int(line.strip()))
        return eows
        
    def get_all_terminals(self, terminal_ids):
        all_terminals = set([utils.EOS_ID])
        if terminal_ids:
            with open(terminal_ids) as f:
                for line in f:
                    all_terminals.add(int(line.strip()))
            if not self.terminal_restrict:
                for terminal in all_terminals:
                    self.lhs_to_can_follow[terminal] = all_terminals
        return all_terminals

    def get_bpe_can_follow(self, rule_path):
        with open(rule_path) as f:
            for line in f:
                nt, following = line.split(' : ')
                nt_tuple = tuple(map(int, nt.split(',')))
                following = set([int(r) for r in following.strip().split()])
                self.lhs_to_can_follow[nt_tuple] = following
                
    def update_stacks(self, word):
        if word in self.nonterminals:
            self.current_rhs.append(word)
            if self.last_nt_in_rule[word]:
                self.replace_lhs()
        else:
            if self.terminal_restrict:
                try:
                    internal_lhs = self.current_lhs + (word,)
                except TypeError:
                    internal_lhs = (self.current_lhs, word)
            else:
                internal_lhs = word
            if not self.terminal_restrict and word in self.eow_ids:
                self.replace_lhs()
            elif (not self.terminal_restrict or 
                  internal_lhs in self.lhs_to_can_follow):
                self.current_lhs = internal_lhs
            else:
                self.replace_lhs()

    def is_nt(self, word):
        if word in self.all_terminals:
            return False
        return True

    def predict_next(self, predicting_next_word=False):
        """predict next tokens as permitted by 
        the current stack and the BPE grammar
        """
        original_posterior = self.predictor.predict_next()
        outgoing_rules = self.lhs_to_can_follow[self.current_lhs]
        scores = {rule_id: original_posterior[rule_id] 
                  for rule_id in outgoing_rules}
        if self.internal_only_restrict and self.are_best_terminal(scores):
            outgoing_rules = self.all_terminals
            scores = {rule_id: original_posterior[rule_id]
                      for rule_id in outgoing_rules}
        scores = self.finalize_posterior(
            scores, 
            use_weights=True,
            normalize_scores=self.normalize_scores)
        if self.word_out and not predicting_next_word:
            scores = self.find_word(scores)
        return scores def breakdown2score_sum(working_score, score_breakdown, full=False):
def breakdown2score_length_norm(working_score, score_breakdown, full=False):
def breakdown2score_bayesian(working_score, score_breakdown, full=False, prev_score=None):
def breakdown2score_bayesian_state_dependent(working_score, score_breakdown, 
                                             full=False, prev_score=None,
                                             lambdas=None):
def breakdown2score_bayesian_loglin(working_score, score_breakdown, full=False,
                                    prev_score=None): class AstarDecoder(Decoder):
    """This decoder implements A*. For heuristics, see the the 
    ``decoding.core`` module for interfaces and the general handling of
    heuristics, and the ``decoding.heuristics`` package for heuristic
    implementations. This A* implementation does not have a 'closed
    set', i.e. we do not keep track of already visited states. Make 
    sure that your search space is acyclic (normally it is unless you
    decode on cyclic lattices with the fst predictor.
    """
    
    def __init__(self, decoder_args):
        """Creates a new A* decoder instance. The following values are
        fetched from `decoder_args`:
        
            beam (int): Maximum number of active hypotheses.
            pure_heuristic_scores (bool): For standard A* set this to
                                          false. If set to true, partial
                                          hypo scores are ignored when
                                          scoring hypotheses.
            early_stopping (bool): If this is true, partial hypotheses
                                   with score worse than the current
                                   best complete scores are not
                                   expanded. This applies when nbest is
                                   larger than one and inadmissible
                                   heuristics are used
            nbest (int): If this is set to a positive value, we do not
                         stop decoding at the first complete path, but
                         continue search until we collected this many
                         complete hypothesis. With an admissible
                         heuristic, this will yield an exact n-best
                         list.
        
        Args:
            decoder_args (object): Decoder configuration passed through
                                   from the configuration API.
        """
        super(AstarDecoder, self).__init__(decoder_args)
        self.nbest = max(1, decoder_args.nbest)
        self.capacity = decoder_args.beam
        self.early_stopping = decoder_args.early_stopping
        self.pure_heuristic_scores = decoder_args.pure_heuristic_scores
    
    def _get_combined_score(self, hypo):
        est_score = -self.estimate_future_cost(hypo)
        if not self.pure_heuristic_scores:
            return est_score + hypo.score
        return est_score

    def decode(self, src_sentence):
        """Decodes a single source sentence using A* search. """
        self.initialize_predictors(src_sentence)
        open_set = []
        best_score = self.get_lower_score_bound()
        heappush(open_set, (0.0,
                            PartialHypothesis(self.get_predictor_states())))
        while open_set:
            c,hypo = heappop(open_set)
            if self.early_stopping and hypo.score < best_score:
                continue
            logging.debug("Expand (est=%f score=%f exp=%d best=%f): sentence: %s"
                          % (-c, 
                             hypo.score, 
                             self.apply_predictors_count, 
                             best_score, 
                             hypo.trgt_sentence))
            if hypo.get_last_word() == utils.EOS_ID: # Found best hypothesis
                if hypo.score > best_score:
                    logging.debug("New best hypo (score=%f exp=%d): %s" % (
                            hypo.score,
                            self.apply_predictors_count,
                            ' '.join([str(w) for w in hypo.trgt_sentence])))
                    best_score = hypo.score
                self.add_full_hypo(hypo.generate_full_hypothesis())
                if len(self.full_hypos) >= self.nbest: # if we have enough hypos
                    return self.get_full_hypos_sorted()
                continue
            self.set_predictor_states(copy.deepcopy(hypo.predictor_states))
            if not hypo.word_to_consume is None: # Consume if cheap expand
                self.consume(hypo.word_to_consume)
                hypo.word_to_consume = None
            posterior,score_breakdown = self.apply_predictors()
            hypo.predictor_states = self.get_predictor_states()
            for trgt_word in posterior: # Estimate future cost, add to heap
                next_hypo = hypo.cheap_expand(trgt_word, posterior[trgt_word],
                                                  score_breakdown[trgt_word])
                combined_score = -self.estimate_future_cost(next_hypo)
                if not self.pure_heuristic_scores:
                    combined_score += next_hypo.score
                heappush(open_set, (-self._get_combined_score(next_hypo),
                                    next_hypo))
            # Limit heap capacity
            if self.capacity > 0 and len(open_set) > self.capacity:
                new_open_set = []
                for _ in range(self.capacity):
                    heappush(new_open_set, heappop(open_set))
                open_set = new_open_set
        return self.get_full_hypos_sorted() class BeamDecoder(Decoder):
    """This decoder implements standard beam search and several
    variants of it such as diversity promoting beam search and beam
    search with heuristic future cost estimates. This implementation
    supports risk-free pruning and hypotheses recombination.
    """
    
    def __init__(self, decoder_args):
        """Creates a new beam decoder instance. The following values
        are fetched from `decoder_args`:
        
            hypo_recombination (bool): Activates hypo recombination 
            beam (int): Absolute beam size. A beam of 12 means
                        that we keep track of 12 active hypotheses
            sub_beam (int): Number of children per hypothesis. Set to
                            beam size if zero.
            pure_heuristic_scores (bool): Hypotheses to keep in the beam
                                          are normally selected 
                                          according the sum of partial
                                          hypo score and future cost
                                          estimates. If set to true, 
                                          partial hypo scores are 
                                          ignored.
            diversity_factor (float): If this is set to a positive 
                                      value we add diversity promoting
                                      penalization terms to the partial
                                      hypothesis scores following Li
                                      and Jurafsky, 2016
            early_stopping (bool): If true, we stop when the best
                                   scoring hypothesis ends with </S>.
                                   If false, we stop when all hypotheses
                                   end with </S>. Enable if you are
                                   only interested in the single best
                                   decoding result. If you want to 
                                   create full 12-best lists, disable

        Args:
            decoder_args (object): Decoder configuration passed through
                                   from the configuration API.
        """
        super(BeamDecoder, self).__init__(decoder_args)
        self.diversity_factor = decoder_args.decoder_diversity_factor
        self.diverse_decoding = (self.diversity_factor > 0.0)
        if self.diversity_factor > 0.0:
            logging.fatal("Diversity promoting beam search is not implemented "
                          "yet")
        self.beam_size = decoder_args.beam
        self.sub_beam_size = decoder_args.sub_beam
        if self.sub_beam_size <= 0:
            self.sub_beam_size = decoder_args.beam
        self.hypo_recombination = decoder_args.hypo_recombination
        self.maintain_best_scores = False
        if decoder_args.early_stopping:
            self.stop_criterion = self._best_eos 
            if not self.hypo_recombination:
                self.maintain_best_scores = True
                logging.debug("Risk-free beam-search pruning enabled")
        else:
            self.stop_criterion = self._all_eos
        self.pure_heuristic_scores = decoder_args.pure_heuristic_scores
    
    def _get_combined_score(self, hypo):
        """Combines hypo score with future cost estimates."""
        est_score = -self.estimate_future_cost(hypo)
        if not self.pure_heuristic_scores:
            return est_score + hypo.score
        return est_score

    def _best_eos(self, hypos):
        """Returns true if the best hypothesis ends with </S>"""
        return hypos[0].get_last_word() != utils.EOS_ID

    def _all_eos(self, hypos):
        """Returns true if the all hypotheses end with </S>"""
        for hypo in hypos:
            if hypo.get_last_word() != utils.EOS_ID:
                return True
        return False
    
    def _expand_hypo(self, hypo):
        """Get the best beam size expansions of ``hypo``.
        
        Args:
            hypo (PartialHypothesis): Hypothesis to expand
        
        Returns:
            list. List of child hypotheses
        """
        if hypo.score <= self.min_score:
            return []
        self.set_predictor_states(copy.deepcopy(hypo.predictor_states))
        if not hypo.word_to_consume is None: # Consume if cheap expand
            self.consume(hypo.word_to_consume)
            hypo.word_to_consume = None
        posterior, score_breakdown = self.apply_predictors(self.sub_beam_size)
        hypo.predictor_states = self.get_predictor_states()
        return [hypo.cheap_expand(
                        trgt_word,
                        posterior[trgt_word],
                        score_breakdown[trgt_word]) for trgt_word in posterior]
    
    def _filter_equal_hypos(self, hypos, scores):
        """Apply hypo recombination to the hypotheses in ``hypos``.
        
        Args:
            hypos (list): List of hypotheses
            scores (list): hypo scores with heuristic estimates
        
        Return:
            list. List with hypotheses in ``hypos`` after applying
            hypotheses recombination.
        """
        new_hypos = []
        for idx in reversed(np.argsort(scores)):
            candidate = hypos[idx]
            self.set_predictor_states(copy.deepcopy(candidate.predictor_states))
            if not candidate.word_to_consume is None:
                self.consume(candidate.word_to_consume)
                candidate.word_to_consume = None
                candidate.predictor_states = self.get_predictor_states()
            valid = True
            for hypo in new_hypos:
                if self.are_equal_predictor_states(
                                                hypo.predictor_states,
                                                candidate.predictor_states):
                    logging.debug("Hypo recombination: %s > %s" % (
                                                 hypo.trgt_sentence,
                                                 candidate.trgt_sentence))
                    valid = False
                    break
            if valid:
                new_hypos.append(candidate)
                if len(new_hypos) >= self.beam_size:
                    break
        return new_hypos

    def _get_next_hypos(self, all_hypos, all_scores):
        """Get hypos for the next iteration. """
        hypos = [all_hypos[idx]
                        for idx in np.argsort(all_scores)[-self.beam_size:]]
        hypos.reverse()
        return hypos
    
    def _register_score(self, score):
        """Updates best_scores and min_score. """
        if not self.maintain_best_scores:
            return
        self.best_scores.append(score)
        self.best_scores.sort(reverse=True)
        if len(self.best_scores) >= self.beam_size:
            self.best_scores = self.best_scores[:self.beam_size]
            self.min_score = self.best_scores[-1] 
    
    def _get_initial_hypos(self):
        """Get the list of initial ``PartialHypothesis``. """
        return [PartialHypothesis(self.get_predictor_states())]
    
    def decode(self, src_sentence):
        """Decodes a single source sentence using beam search. """
        self.initialize_predictors(src_sentence)
        hypos = self._get_initial_hypos()
        it = 0
        while self.stop_criterion(hypos):
            if it > self.max_len: # prevent infinite loops
                break
            it = it + 1
            next_hypos = []
            next_scores = []
            self.min_score = utils.NEG_INF
            self.best_scores = []
            for hypo in hypos:
                if hypo.get_last_word() == utils.EOS_ID:
                    next_hypos.append(hypo)
                    next_scores.append(self._get_combined_score(hypo))
                    continue 
                for next_hypo in self._expand_hypo(hypo):
                    next_score = self._get_combined_score(next_hypo)
                    if next_score > self.min_score:
                        next_hypos.append(next_hypo)
                        next_scores.append(next_score)
                        self._register_score(next_score)
            if self.hypo_recombination:
                hypos = self._filter_equal_hypos(next_hypos, next_scores)
            else:
                hypos = self._get_next_hypos(next_hypos, next_scores)
        for hypo in hypos:
            if hypo.get_last_word() == utils.EOS_ID:
                self.add_full_hypo(hypo.generate_full_hypothesis()) 
        if not self.full_hypos:
            logging.warn("No complete hypotheses found for %s" % src_sentence)
            for hypo in hypos:
                self.add_full_hypo(hypo.generate_full_hypothesis())
        return self.get_full_hypos_sorted() class BigramGreedyDecoder(Decoder):
    """The bigram greedy decoder collects bigram statistics at each
    node expansions. After each decoding pass, it constructs a new
    hypothesis to rescore by greedily selecting bigrams and gluing
    them together. Afterwards, the new hypothesis is rescored and new
    bigram statistics are collected.
    
    Note that this decoder does not support the ``max_length`` 
    parameter as it is designed for fixed length decoding problems.
    
    Also note that this decoder works only for bag-of-words problems.
    Do not use the bow predictor in combination with this decoder as
    it will hide the EOS scores which are important to estimate bigram
    scores.
    """
    
    def __init__(self, decoder_args):
        """Creates a new bigram greedy decoder. Do not use this decoder
        in combination with the bow predictor as it inherently already
        satisfies the bag-of-word constrains. The following values
        are fetched from `decoder_args`:
        
            trg_test (string): Path to a plain text file which 
                               defines the bag of words
            max_node_expansions (int): Maximum number of node expansions
                                       for inadmissible pruning.
            early_stopping (boolean): Activates admissible pruning
        
        Args:
            decoder_args (object): Decoder configuration passed through
                                   from the configuration API.
        """
        super(BigramGreedyDecoder, self).__init__(decoder_args)
        self.max_expansions_param = decoder_args.max_node_expansions
        self.early_stopping = decoder_args.early_stopping
        with open(decoder_args.trg_test) as f:
            self.lines = f.read().splitlines()
    
    def _greedy_decode(self):
        """Performs greedy decoding from the start node. Used to obtain
        initial bigram statistics.
        """
        hypo = PartialHypothesis()
        hypos = []
        posteriors = []
        score_breakdowns = []
        bag = dict(self.full_bag)
        while bag:
            posterior,score_breakdown = self.apply_predictors()
            hypo.predictor_states = copy.deepcopy(self.get_predictor_states())
            bag_posterior = {w: posterior[w] for w in self.full_bag_with_eos}
            bag_breakdown = {w: score_breakdown[w] 
                                        for w in self.full_bag_with_eos}
            posteriors.append(bag_posterior)
            score_breakdowns.append(bag_breakdown)
            hypos.append(hypo)
            best_word = utils.argmax({w: bag_posterior[w] for w in bag})
            bag[best_word] -= 1
            if bag[best_word] < 1:
                del bag[best_word]
            self.consume(best_word)
            hypo = hypo.expand(best_word,
                               None,
                               bag_posterior[best_word],
                               score_breakdown[best_word])
        posterior,score_breakdown = self.apply_predictors()
        hypo.predictor_states = copy.deepcopy(self.get_predictor_states())
        bag_posterior = {w: posterior[w] for w in self.full_bag_with_eos}
        bag_breakdown = {w: score_breakdown[w] for w in self.full_bag_with_eos}
        posteriors.append(bag_posterior)
        score_breakdowns.append(bag_breakdown)
        hypos.append(hypo)
        
        hypo = hypo.cheap_expand(utils.EOS_ID,
                                 bag_posterior[utils.EOS_ID],
                                 score_breakdown[utils.EOS_ID])
        logging.debug("Greedy hypo (%f): %s" % (
                          hypo.score,
                          ' '.join([str(w) for w in hypo.trgt_sentence])))
        self._process_new_hypos(hypos, posteriors, score_breakdowns, hypo)
    
    def _process_new_hypos(self,
                           hypos,
                           posteriors, 
                           score_breakdowns, 
                           complete_hypo = None):
        """This method is called after a decoding pass. It updates 
        bigram statistics, stores partial hypotheses for restarting
        from them later, and creates full hypotheses if a hypo
        ends with EOS
        """
        if complete_hypo:
            self.best_score = max(self.best_score, complete_hypo.score)
            self.add_full_hypo(complete_hypo.generate_full_hypothesis())
        for idx,hypo in enumerate(hypos):
            posterior = posteriors[idx]
            prefix = hypo.trgt_sentence
            self._register_bigram_scores(prefix[-1] if prefix else utils.GO_ID, 
                                         posterior)
            self.posteriors.add(prefix, posterior)
            self.score_breakdowns.add(prefix, score_breakdowns[idx])
            self.hypos.add(prefix, hypo)
        self._sort_bigram_scores()
    
    def _get_next_sentence(self):
        """Get the next sentence to rescore
        """
        bag0 = dict(self.full_bag)
        bag0[utils.GO_ID] = 1
        bag1 = dict(self.full_bag_with_eos)
        return self._get_next_sentence_recursive([], bag0, bag1)
    
    def _get_next_sentence_recursive(self,
                                     bigrams,
                                     remaining_bag0,
                                     remaining_bag1):
        """Recursive helper function for _get_next_sentence
        
        Args:
            bigrams (list): List of already selected bigrams
            remaining_bag0 (dict): Remaining words in the bag for the
                                   first word in the bigram
            remaining_bag1 (dict): Remaining words in the bag for the
                                   second word in the bigram
        
        Returns:
            Tuple. hypo, sen tuple where sen is an unexplored sentence
            and hypo corresponds to the largest explored prefix of sen.
            Returns None if no consistent sentence was found
        """
        if len(bigrams) == self.num_words + 1: # Collected enough bigrams
            sens = self._get_sentences_from_bigrams(bigrams)
            if not sens: # Bigrams are not consistent
                return None
            for sen in sens:
                hypo = self._get_largest_prefix_hypo(sen)
                if hypo and hypo.score > self.best_score:
                    return hypo, sen
            return None
        for bigram in self.sorted_bigrams:
            if remaining_bag0[bigram[0]] > 0 and remaining_bag1[bigram[1]] > 0:
                remaining_bag0[bigram[0]] -= 1
                remaining_bag1[bigram[1]] -= 1
                ret = self._get_next_sentence_recursive(bigrams + [bigram],
                                                        remaining_bag0,
                                                        remaining_bag1)
                if ret:
                    return ret
                remaining_bag0[bigram[0]] += 1
                remaining_bag1[bigram[1]] += 1
        return None
    
    def _get_largest_prefix_hypo(self, sen):
        """Get the explored hypothesis with the largest common prefix
        with ``sen``.
        """
        prefix = self.hypos.get_prefix(sen)
        if len(prefix) == len(sen): # hypo is already fully explored
            return None
        hypo = self.hypos.get(prefix)
        posterior = self.posteriors.get(prefix)
        score_breakdown = self.score_breakdowns.get(prefix)
        next_word = sen[len(prefix)]
        return hypo.cheap_expand(next_word,
                                 posterior[next_word],
                                 score_breakdown[next_word])
    
    def _get_sentences_from_bigrams(self, bigrams):
        """Constructs all full consistent sentences from a list of 
        bigrams. The search is implemented as BFS. """
        candidates = [([utils.GO_ID], bigrams)]
        for _ in range(len(bigrams)):
            next_candidates = []
            for candidate in candidates:
                # Select the next consistent bigram
                cand_sen,cand_bigrams = candidate 
                last_word = cand_sen[-1]
                for idx,bigram in enumerate(cand_bigrams):
                    if bigram[0] == last_word: # Consistent
                        new_bigrams = list(cand_bigrams)
                        del new_bigrams[idx]
                        next_candidates.append((cand_sen + [bigram[1]],
                                                new_bigrams))
            candidates = next_candidates
            if not candidates:
                break
        return [candidate[0][1:] for candidate in candidates]
    
    def _forced_decode(self, start_hypo, sen):
        """Performs forced decoding from a the node in the search tree.
        
        Args:
            start_hypo (PartialHypothesis): This is a partial hypothesis
                                            for a prefix of sen from 
                                            which we start decoding
            sen (list): Sentence to rescore
        """
        logging.debug("best=%f prefix=%s prefix_score=%f sen=%s" % (
                                                 self.best_score, 
                                                 start_hypo.trgt_sentence, 
                                                 start_hypo.score, 
                                                 sen))
        self.set_predictor_states(copy.deepcopy(start_hypo.predictor_states))
        if not start_hypo.word_to_consume is None: # Consume if cheap expand
            self.consume(start_hypo.word_to_consume)
        hypos = []
        posteriors = []
        score_breakdowns = []
        hypo = start_hypo
        cancelled = False
        for forced_w in sen[len(start_hypo.trgt_sentence):]:
            posterior,score_breakdown = self.apply_predictors()
            hypo.predictor_states = copy.deepcopy(self.get_predictor_states())
            bag_posterior = {w: posterior[w] for w in self.full_bag_with_eos}
            bag_breakdown = {w: score_breakdown[w] 
                                        for w in self.full_bag_with_eos}
            posteriors.append(bag_posterior)
            score_breakdowns.append(bag_breakdown)
            hypos.append(hypo)
            hypo = hypo.expand(forced_w,
                               None,
                               bag_posterior[forced_w],
                               score_breakdown[forced_w])
            if self.early_stopping and hypo.score < self.best_score:
                cancelled = True
                break
            self.consume(forced_w)
        self._process_new_hypos(hypos, posteriors, score_breakdowns,
                                hypo if not cancelled else None)
    
    def _load_bag(self):
        """Load the current bag of words """
        self.full_bag = {}
        for w in self.lines[self.current_sen_id].strip().split(): 
            int_w = int(w)
            self.full_bag[int_w] = self.full_bag.get(int_w, 0) + 1
        self.num_words = sum(self.full_bag.values())
        self.full_bag_with_eos = dict(self.full_bag)
        self.full_bag_with_eos[utils.EOS_ID] = 1
    
    def _register_bigram_scores(self, last_word, posterior):
        for w,score in utils.common_iterable(posterior):
            self.bigram_scores[last_word][w] = min(
                                    self.bigram_scores[last_word][w], score)
    
    def _sort_bigram_scores(self):
        self.sorted_bigrams = []
        for w1,scores in self.bigram_scores.items():
            self.sorted_bigrams.extend([(w1, w2, score)
                                        for w2,score in scores.items()])
        self.sorted_bigrams.sort(key=operator.itemgetter(2), reverse=True)
    
    def _initialize_bigram_scores(self):
        default_scores = {w: 0.0 for w in self.full_bag_with_eos}
        self.bigram_scores = {w: dict(default_scores) for w in self.full_bag}
        self.bigram_scores[utils.GO_ID] = default_scores

    def decode(self, src_sentence):
        """Decodes a single source sentence with the flip decoder """
        self.initialize_predictors(src_sentence)
        self.max_expansions = self.get_max_expansions(self.max_expansions_param,
                                                      src_sentence) 
        self._load_bag()
        self.hypos = SimpleTrie()
        self.posteriors = SimpleTrie()
        self.score_breakdowns = SimpleTrie()
        self.best_score = self.get_lower_score_bound()
        self._initialize_bigram_scores()
        self._greedy_decode()
        while self.max_expansions > self.apply_predictors_count:
            ret = self._get_next_sentence()
            if not ret:
                break
            self._forced_decode(ret[0], ret[1])
        return self.get_full_hypos_sorted() class BucketDecoder(Decoder):
    """The bucket decoder maintains separate buckets for each sentence
    length. The buckets contain partial hypotheses. In each iteration,
    the decoder selects a bucket, and expands the best hypothesis in
    this bucket by one token. The core of the bucket decoder is the
    bucket selection strategy. The following strategies are available:
    
    * 'iter': Puts all buckets in a big loop and iterates through it.
              With this strategy, the number of hypothesis expansions
              is equally distributed over the buckets
    * 'random': (with stochastic=true and bucket_selecto!=difference)
                Randomly select a non-empty bucket
    * 'difference': Similar to the heuristic used by the restarting
                    decoder. Select the bucket in which the difference
                    between best and second best hypothesis is minimal
    * 'maxdiff': Like 'iter', but filters buckets in which the 
                 difference between first and second hypo is larger
                 than epsilon. If no such buckets exist, increase 
                 epsilon
    """
    
    def __init__(self, 
                 decoder_args,
                 hypo_recombination, 
                 max_expansions=0,
                 low_memory_mode = True,
                 beam=1,
                 pure_heuristic_scores = False,
                 diversity_factor = -1.0,
                 early_stopping=True,
                 stochastic=False,
                 bucket_selector='maxscore',
                 bucket_score_strategy='difference',
                 collect_stats_strategy='best'):
        """Create a new bucket decoder
        
        Args:
            decoder_args (object): Decoder configuration passed through
                                   from the configuration API.
            hypo_recombination (boolean): Activates hypothesis 
                                          recombination. Hypos are
                                          tested only within a bucket 
            max_expansions (int): Maximum number of node expansions for
                                  inadmissible pruning.
            low_memory_mode (bool): Switch on low memory mode at cost 
                                    of some computational overhead.
                                    Limits the number of hypotheses in
                                    each bucket to the number of 
                                    remaining node expansions
            beam (int): Number of hypotheses which get expanded at once
                        after selecting a bucket.
            pure_heuristic_scores (bool): If false, hypos are scored 
                                          with partial score plus
                                          future cost estimates. If
                                          true, only the future cost
                                          estimates are used
            diversity_factor (float): If this is set to a positive 
                                      value, we reorder hypos in a 
                                      bucket by adding a term which 
                                      counts how many hypos with the
                                      same parent have been expanded
                                      already
            early_stopping (boolean): Admissible pruning (works only if
                                      scores are non-positive)
            stochastic (boolean): Stochastic bucket selection. If the 
                                  bucket selector is not 'difference',
                                  this results in random bucket 
                                  selection. If ``bucket_selector`` is
                                  set to 'difference', buckets are 
                                  randomly selected with probability
                                  according the bucket score
            bucket_selector (string): Bucket selection strategy. 'iter',
                                      'maxscore'. 'score'. See the
                                      class docstring for more info
            bucket_score_strategy (string): Determines the way the 
                                            buckets are scored.
                                            'difference' between current
                                            best word score and best 
                                            hypo in bucket, 'absolute'
                                            hypo score, 'heap' score of
                                            top scoring hypo in bucket ,
                                            'constant' score of 0.0.
            collect_stats_strategy (string): best, full, or all. Defines 
                                             how unigram estimates are 
                                             collected for heuristic
                                            
        """
        super(BucketDecoder, self).__init__(decoder_args)
        self.max_expansions_param = max_expansions
        self.low_memory_mode = low_memory_mode
        self.beam = beam
        self.pure_heuristic_scores = pure_heuristic_scores
        self.diversity_factor = diversity_factor
        self.diverse_decoding = (diversity_factor > 0.0)
        self.early_stopping = early_stopping
        if stochastic:
            if bucket_selector == 'score':
                self.get_bucket = self._get_bucket_stochastic
            else:
                self.get_bucket = self._get_bucket_random
        else:
            if 'iter' in bucket_selector:
                self.get_bucket = self._get_bucket_iter
                self.max_iter = 1000000
                if '-' in bucket_selector:
                    _,n = bucket_selector.split("-")
                    self.max_iter = int(n)
            elif bucket_selector == 'maxscore':
                self.get_bucket = self._get_bucket_maxscore
            elif bucket_selector == 'score-end':
                self.get_bucket = self._get_bucket_score_end
            elif bucket_selector == 'score':
                self.get_bucket = self._get_bucket_score
            else:
                logging.fatal("Unknown bucket selector")
        if bucket_score_strategy == 'difference':
            self.get_bucketscore = self._get_bucketscore_difference
        elif bucket_score_strategy == 'heap':
            self.get_bucketscore = self._get_bucketscore_heap
        elif bucket_score_strategy == 'absolute':
            self.get_bucketscore = self._get_bucketscore_absolute
        elif bucket_score_strategy == 'constant':
            self.get_bucketscore = self._get_bucketscore_constant
        else:
            logging.fatal("Unknown bucket score strategy")
        self.collect_stats_from_partial = (collect_stats_strategy == 'all')
        if collect_stats_strategy == 'best':
            self.collect_stats = self._collect_stats_best
        elif collect_stats_strategy == 'full':
            self.collect_stats = self._collect_stats_full
        self.hypo_recombination = hypo_recombination
        
    def _get_bucketscore_difference(self, length):
        return self.best_word_scores[length] - self.buckets[length][0][1].score
    
    def _get_bucketscore_heap(self, length):
        return self.buckets[length][0][0]
    
    def _get_bucketscore_absolute(self, length):
        return -self.buckets[length][0][1].score
    
    def _get_bucketscore_constant(self, length):
        return 0.0

    def _get_bucket_iter(self):
        """Implements the bucket selector 'iter' """
        if self.cur_iter > self.max_iter:
            if self.guaranteed_optimality:
                logging.info("max iter. Optimality not guaranteed for ID %d" %
                      (self.current_sen_id + 1))
            self.guaranteed_optimality = False
            return -1
        last_length = self.last_bucket
        for length in range(last_length+1, self.max_len):
            if self.buckets[length]:
                self.last_bucket = length
                return length
        # Restart with first bucket
        self.cur_iter += 1
        for length in range(last_length+1):
            if self.buckets[length]:
                self.last_bucket = length
                return length
        return -1

    def _get_bucket_maxscore(self):
        """Implements the bucket selector 'maxscore' """
        for max_score in range(0, 500, 5):
            length = self._get_bucket_maxscore_helper(max_score) 
            if length >= 0:
                return length
        return -1

    def _get_bucket_maxscore_helper(self, max_score):
        """Helper method for maxscore """
        last_length = self.last_bucket
        for length in range(last_length+1, self.max_len):
            if self.buckets[length]:
                score = self.get_bucketscore(length)
                if score < max_score:
                    self.last_bucket = length
                    return length
        for length in range(last_length+1):
            if self.buckets[length]:
                score = self.get_bucketscore(length)
                if score < max_score:
                    self.last_bucket = length
                    return length
        return -1

    def _get_bucket_score(self):
        """Implements the bucket selector 'score' """
        best_score = INF
        best_length = -1
        for length in range(self.max_len):
            if self.buckets[length]:
                score = self.get_bucketscore(length)
                if score <= best_score:
                    best_score = score
                    best_length = length 
        return best_length

    def _get_bucket_score_end(self):
        """Implements the bucket selector 'score-end' """
        last_length = self.last_bucket
        for length in range(last_length+1, self.max_len):
            if self.buckets[length]:
                self.last_bucket = length
                return length
        # Restart with best bucket
        best_score = INF
        best_length = -1
        for length in range(self.max_len):
            if self.buckets[length]:
                score = self.get_bucketscore(length)
                if score <= best_score:
                    best_score = score
                    best_length = length 
        self.last_bucket = best_length
        return best_length

    def _get_bucket_random(self):
        """Implements random bucket selection """
        lengths = [l for l in range(self.max_len) if self.buckets[l]]
        return np.random.choice(lengths)
        
    def _get_bucket_stochastic(self):
        """Implements the stochastic bucket selector 'difference' """
        lengths = []
        scores = []
        for length in range(self.max_len):
            if self.buckets[length]:
                score = self.get_bucketscore(length)
                if score == NEG_INF:
                    return self._get_bucket_difference()
                lengths.append(length)
                scores.append(score)
        if not lengths:
            return -1
        exps = np.exp([-d for d in scores])
        total = sum(exps)
        return np.random.choice(lengths, p=[e/total for e in exps])
    
    def _initialize_decoding(self, src_sentence):
        """Helper function for ``decode`` to which initializes all the
        class attributes
        """
        self.initialize_predictors(src_sentence)
        self.max_expansions = self.get_max_expansions(self.max_expansions_param,
                                                      src_sentence) 
        init_hypo = PartialHypothesis()
        init_hypo.predictor_states = self.get_predictor_states()
        init_hypo.scores = []
        init_hypo.parent_hypo_array_idx = 0 # point to guardian
        self.buckets = [[] for _ in range(self.max_len+1)]
        self.expanded_hypos = [[] for _ in range(self.max_len+1)]
        self.buckets[0].append((0.0, init_hypo))
        self.expand_counts = [0.0] # with guardian
        self.expand_backpointers = [0] # with guardian
        self.last_bucket = 0
        self.best_score = self.get_lower_score_bound()
        self.best_word_scores = [NEG_INF] * (self.max_len+1)
        self.compressed = [True] * (self.max_len+1)
        self.guaranteed_optimality = True
        self.cur_iter = 0
    
    def _activate_hypo(self, hypo, length, heap_score):
        """Prepares the decoder for expanding the given hypothesis. 
        This may include updating global word scores, loading the
        predictor states and consume the last word of the hypothesis
        if necessary. After this method, ``apply_predictors`` computes
        the next posterior vector
        """
        if (self.collect_stats_from_partial 
                and hypo.score > self.best_word_scores[length]):
            self.best_word_scores[length] = hypo.score
            self._update_heap_scores()
        logging.debug("Expand (best_glob=%f diff=%f (%f-%f) heap=%f exp=%d): %s"
                            % (self.best_score,
                               self.best_word_scores[length] - hypo.score,
                               self.best_word_scores[length],
                               hypo.score,
                               heap_score,
                               self.apply_predictors_count,
                               ' '.join([str(w) for w in hypo.trgt_sentence])))
        self.set_predictor_states(copy.deepcopy(hypo.predictor_states))
        if not hypo.word_to_consume is None: # Consume if cheap expand
            self.consume(hypo.word_to_consume)
            hypo.word_to_consume = None
    
    def _get_next_hypo_array_idx(self, hypo):
        """Used to assign a new hypothesis array index used in 
        ``expand_counts`` and ``expand_backpointers``. This also 
        updates the count and backpointer array. Only called if
        ``diverse_decoding`` is true.
        """
        idx = hypo.parent_hypo_array_idx
        while idx != 0:
            self.expand_counts[idx] += 1.0
            idx = self.expand_backpointers[idx]
        hypo_array_idx = len(self.expand_counts)
        self.expand_counts.append(0.0)
        self.expand_backpointers.append(hypo.parent_hypo_array_idx)
        return hypo_array_idx
    
    def _collect_stats_best(self, hypo):
        if hypo.score > self.best_score:
            self.best_score = hypo.score
            self.best_word_scores[:len(hypo.scores) ] = hypo.scores
            self._update_heap_scores()
    
    def _collect_stats_full(self, hypo):
        update = False
        for idx, score in enumerate(hypo.scores):
            if score > self.best_word_scores[idx]:
                self.best_word_scores[idx] = score
                update = True
        if update:
            self._update_heap_scores()
    
    def _register_full_hypo(self, hypo):
        """Update all class attributes according a new full hypothesis.
        ``hypo`` is a ``PartialHypothesis`` ending with EOS.
        """
        self.add_full_hypo(hypo.generate_full_hypothesis())
        if hypo.score > self.best_score:
            logging.info("New_best (ID: %d): score=%f exp=%d hypo=%s" 
                            % (self.current_sen_id + 1,
                               hypo.score, 
                               self.apply_predictors_count,
                               ' '.join([str(w) for w in hypo.trgt_sentence])))
        if not self.collect_stats_from_partial:
            self.collect_stats(hypo)
        if hypo.score > self.best_score:
            self.best_score = hypo.score
    
    def _update_heap_scores(self):
        """Called when ``best_word_scores`` has changed and heap scores
        need to be updated
        """
        for length in range(len(self.buckets)):
            self._update_heap_score(length)
    
    def _update_heap_score(self, length):
        """``_update_heap_score`` for a single bucket
        """
        new_bucket = [(-self._get_combined_score(h), h) 
                                for _,h in self.buckets[length]]
        self.buckets[length] = new_bucket
        self.buckets[length].sort(key=operator.itemgetter(0))
    
    def _get_max_bucket_size(self):
        if not self.low_memory_mode:
            return 10000000
        max_size = max(1, 1 + self.max_expansions - self.apply_predictors_count)
        if self.hypo_recombination:
            max_size = 5 * max_size
        return max_size
    
    def _add_new_hypos_to_bucket(self, length, new_hypos):
        max_size = self._get_max_bucket_size()
        if self.diverse_decoding:
            self._update_heap_score(length)
        if (not self.hypo_recombination or
                not self.guaranteed_optimality or 
                max_size >= len(new_hypos) + len(self.buckets[length])):
            self.buckets[length].extend(new_hypos)
            self.buckets[length].sort(key=operator.itemgetter(0))
            self.buckets[length] = self.buckets[length][:max_size]
            self.compressed[length] = False
        elif self.compressed[length]: # Equivalence check only for new
            logging.debug("Add %d hypos to compressed bucket of size %d" % (
                                    len(new_hypos), len(self.buckets[length])))
            new_hypos.sort(key=operator.itemgetter(0))
            new_bucket = []
            oidx = 0
            nidx = 0
            olen = len(self.buckets[length])
            nlen = len(new_hypos)
            while len(new_bucket) < max_size:
                oscore = INF if oidx >= olen else self.buckets[length][oidx][0]
                nscore = INF if nidx >= nlen else new_hypos[nidx][0]
                if oscore == INF and nscore == INF:
                    break
                if oscore < nscore: # Add hypos from old bucket without checks
                    new_bucket.append(self.buckets[length][oidx])
                    oidx += 1
                else: # Check equivalence
                    hypo = new_hypos[nidx][1]
                    self.set_predictor_states(copy.deepcopy(
                                                    hypo.predictor_states))
                    if not hypo.word_to_consume is None:
                        self.consume(hypo.word_to_consume)
                        hypo.word_to_consume = None
                    hypo.predictor_states = self.get_predictor_states()
                    valid = True
                    for other_hypo in [b for _,b in new_bucket]:
                        if other_hypo.score >= hypo.score and self.are_equal_predictor_states(
                                                hypo.predictor_states,
                                                other_hypo.predictor_states):
                            valid = False
                            logging.debug("Hypo recombination: %s > %s (compress)"
                                                  % (other_hypo.trgt_sentence,
                                                     hypo.trgt_sentence))
                            break
                    if valid:
                        new_bucket.append((nscore, hypo))
                    nidx += 1
            self.buckets[length] = new_bucket
            self.compressed[length] = True
        else: # Compress from scratch
            hypos = self.buckets[length] + new_hypos
            logging.debug("Compress bucket of size %d" % len(hypos))
            new_hypos.sort(key=operator.itemgetter(0))
            new_bucket = []
            idx = 0
            while len(new_bucket) < max_size and idx < len(hypos):
                hypo = hypos[idx][1]
                self.set_predictor_states(copy.deepcopy(
                                                    hypo.predictor_states))
                if not hypo.word_to_consume is None:
                    self.consume(hypo.word_to_consume)
                    hypo.word_to_consume = None
                hypo.predictor_states = self.get_predictor_states()
                valid = True
                for other_hypo in [b for _,b in new_bucket]:
                    if other_hypo.score >= hypo.score and self.are_equal_predictor_states(
                                                hypo.predictor_states,
                                                other_hypo.predictor_states):
                        valid = False
                        logging.debug("Hypo recombination: %s > %s"
                                                  % (other_hypo.trgt_sentence,
                                                     hypo.trgt_sentence))
                        break
                if valid:
                    new_bucket.append((hypos[idx][0], hypo))
                idx += 1
            self.buckets[length] = new_bucket
            self.compressed[length] = True
        if (self.hypo_recombination
                and self.guaranteed_optimality 
                and len(self.buckets[length]) >= max_size):
            logging.info("Shrunk bucket. Optimality not guaranteed for ID %d" %
                      (self.current_sen_id + 1))
            self.guaranteed_optimality = False

    def _get_combined_score(self, hypo):
        est_score = -self.estimate_future_cost(hypo)
        if self.diverse_decoding:
            cnt = 0.0
            idx = hypo.parent_hypo_array_idx
            while idx != 0:
                cnt += self.expand_counts[idx]
                idx = self.expand_backpointers[idx]
            est_score -= self.diversity_factor * cnt
        if not self.pure_heuristic_scores:
            est_score += hypo.score
            if self.best_score != NEG_INF:
                est_score -= self.best_score
        return est_score
    
    def _get_min_bucket_score(self, length):
        max_bucket_size = self._get_max_bucket_size() 
        if len(self.buckets[length]) >= max_bucket_size:
            return -self.buckets[length][max_bucket_size-1][0]
        return NEG_INF
    
    def _get_hypo(self, length):
        hypo = None
        while self.buckets[length] and hypo is None:
            s,hypo = self.buckets[length].pop(0)
            if self.early_stopping and hypo.score <= self.best_score:
                hypo = None
            else:
                self._activate_hypo(hypo, length, s)
                if self.hypo_recombination:
                    hypo.predictor_states = self.get_predictor_states()
                    for other_hypo in self.expanded_hypos[length]:
                        if other_hypo.score >= hypo.score and self.are_equal_predictor_states(
                                                hypo.predictor_states,
                                                other_hypo.predictor_states):
                            logging.debug("Hypo recombination: %s > %s (activate)"
                                                  % (other_hypo.trgt_sentence,
                                                     hypo.trgt_sentence))
                            hypo = None
                            break
                    if not hypo is None:
                        self.expanded_hypos[length].append(hypo)
        return hypo

    def decode(self, src_sentence):
        """Decodes a single source sentence. 
        """
        self._initialize_decoding(src_sentence)
        while self.max_expansions > self.apply_predictors_count:
            length = self.get_bucket()
            if length < 0: # No more full buckets
                break
            min_next_bucket_score = self._get_min_bucket_score(length+1)
            hypos_to_add = []
            for _ in range(self.beam): # Expand beam_size hypos in this bucket
                hypo = self._get_hypo(length)
                if hypo is None:
                    break
                posterior,score_breakdown = self.apply_predictors()
                hypo.predictor_states = self.get_predictor_states()
                if self.diverse_decoding:
                    hypo_array_idx = self._get_next_hypo_array_idx(hypo)
                for w,score in posterior.items():
                    exp_hypo = hypo.cheap_expand(w, score, score_breakdown[w])
                    exp_hypo.scores = hypo.scores + [hypo.score] 
                    if self.diverse_decoding:
                        exp_hypo.parent_hypo_array_idx = hypo_array_idx
                    combi_score = self._get_combined_score(exp_hypo) 
                    if w == utils.EOS_ID:
                        self._register_full_hypo(exp_hypo)
                    elif (combi_score >= min_next_bucket_score
                          and (exp_hypo.score > self.best_score 
                               or not self.early_stopping)):
                        hypos_to_add.append((-combi_score, exp_hypo))
            self._add_new_hypos_to_bucket(length+1, hypos_to_add)
        if self.guaranteed_optimality and self.max_expansions <= self.apply_predictors_count:
            logging.info("Reached max_node_expansions. Optimality not guaranteed for ID %d" %
                      (self.current_sen_id + 1))
        if not self.full_hypos: # Add incomplete longest hypos if no complete
            logging.warn("No complete hypotheses found for %s" % src_sentence)
            for hypo in self.buckets[self.max_len]:
                self.add_full_hypo(hypo.generate_full_hypothesis())
        return self.get_full_hypos_sorted() class Hypothesis:
class PartialHypothesis(object):
class Heuristic(Observer):
class Decoder(Observable):    
    def __init__(self, trgt_sentence, total_score, score_breakdown = []):
    def __repr__(self):
    def __init__(self, initial_states = None):
    def get_last_word(self):
    def generate_full_hypothesis(self):
    def _new_partial_hypo(self, states, word, score, score_breakdown):
    def expand(self, word, new_states, score, score_breakdown):
    def cheap_expand(self, word, score, score_breakdown):
    def __init__(self):
    def set_predictors(self, predictors):
    def initialize(self, src_sentence):
    def estimate_future_cost(self, hypo):
    def notify(self, message, message_type = MESSAGE_TYPE_DEFAULT):
    def __init__(self, decoder_args):
    def add_predictor(self, name, predictor, weight=1.0):
    def remove_predictors(self):
    def change_predictor_weights(self, new_weights):
    def set_heuristic_predictors(self, heuristic_predictors):
    def add_heuristic(self, heuristic):
    def estimate_future_cost(self, hypo):
    def has_predictors(self):
    def consume(self, word):
    def _get_non_zero_words(self, bounded_predictors, posteriors):
    def _split_restricted_posteriors(self, predictors, posteriors):
    def apply_interpolation_strategy(
            self, pred_weights, non_zero_words, posteriors, unk_probs):
    def apply_predictors(self, top_n=0):
    def _combine_posteriors_norm_none(self,
                                      non_zero_words,
                                      posteriors,
                                      unk_probs,
                                      pred_weights,
                                      top_n=0):
    def _combine_posteriors_norm_rescale_unk(self,
                                             non_zero_words,
                                             posteriors,
                                             unk_probs,
                                             pred_weights,
                                             top_n=0):
    def _combine_posteriors_norm_exact(self,
                                       non_zero_words,
                                       posteriors,
                                       unk_probs,
                                       pred_weights,
                                       top_n=0):
    def _combine_posteriors_norm_reduced(self,
                                         non_zero_words,
                                         posteriors,
                                         unk_probs,
                                         pred_weights,
                                         top_n=0):
    def _scale_combine_non_zero_scores(non_zero_word_count,
                                       posteriors,
                                       unk_probs,
                                       pred_weights,
                                       top_n=0):
    def _combine_posteriors_norm_non_zero(self,
                                          non_zero_words,
                                          posteriors,
                                          unk_probs,
                                          pred_weights,
                                          top_n=0):
    def _combine_posteriors_with_renorm(self,
                                        score_breakdown_raw,
                                        renorm_factors):
    def set_current_sen_id(self, sen_id):
    def initialize_predictors(self, src_sentence):
    def add_full_hypo(self, hypo):
    def get_full_hypos_sorted(self):
    def get_lower_score_bound(self):
    def get_max_expansions(self, max_expansions_param, src_sentence):
    def set_predictor_states(self, states):
    def get_predictor_states(self):
    def combi_arithmetic_unnormalized(x):
    def decode(self, src_sentence):
    def are_equal_predictor_states(self, states1, states2):
CLOSED_VOCAB_SCORE_NORM_NONE = 1
CLOSED_VOCAB_SCORE_NORM_EXACT = 2
CLOSED_VOCAB_SCORE_NORM_REDUCED = 3
CLOSED_VOCAB_SCORE_NORM_RESCALE_UNK = 4
CLOSED_VOCAB_SCORE_NORM_NON_ZERO = 5 class Hypothesis:
    """Complete translation hypotheses are represented by an instance
    of this class. We store the produced sentence, the combined score,
    and a score breakdown to the separate predictor scores.
    """
    
    def __init__(self, trgt_sentence, total_score, score_breakdown = []):
        """Creates a new full hypothesis.
        
        Args:
            trgt_sentence (list): List of target word ids without <S> 
                                  or </S> which make up the target 
                                  sentence
            total_score (float): combined total score of this hypo
            score_breakdown (list): Predictor score breakdown for each
                                    target token in ``trgt_sentence``
        """
        self.trgt_sentence = trgt_sentence
        self.total_score = total_score
        self.score_breakdown = score_breakdown

    def __repr__(self):
        """Returns a string representation of this hypothesis."""
        return "%s (%f)" % (' '.join(str(w) for w in self.trgt_sentence),
                            self.total_score) class DFSDecoder(Decoder):
    """This decoder implements depth first search without using
    heuristics. This is the most efficient search algorithm for 
    complete enumeration of the search space as it minimizes the 
    number of ``get_state()`` and ``set_state()`` calls. Note that
    this DFS implementation has no cycle detection, i.e. if the search
    space has cycles this decoder may run into an infinite loop.
    """
    
    def __init__(self, decoder_args):
        """Creates new DFS decoder instance. The following values are
        fetched from `decoder_args`:
        
            decoder_args (object): Decoder configuration passed through
                                   from the configuration API.
            early_stopping (bool): Enable safe (admissible) branch
                                   pruning if the accumulated score
                                   is already worse than the currently
                                   best complete score. Do not use if
                                   scores can be positive
            max_expansions (int): Maximum number of node expansions for
                                  inadmissible pruning.

        Args:
            decoder_args (object): Decoder configuration passed through
                                   from the configuration API.
        """
        super(DFSDecoder, self).__init__(decoder_args)
        self.early_stopping = decoder_args.early_stopping
        self.max_expansions_param = decoder_args.max_node_expansions
    
    def _dfs(self, partial_hypo):
        """Recursive function for doing dfs. Note that we do not keep
        track of the predictor states inside ``partial_hypo``, because 
        at each call of ``_dfs`` the current predictor states are equal
        to the hypo predictor states.
        ATTENTION: Early stopping plus DFS produces wrong results if 
        you have positive scores!
        
        Args:
            partial_hypo (PartialHypothesis): Partial hypothesis 
                                              generated so far. 
        """
        if (partial_hypo.get_last_word() == utils.EOS_ID
                or len(partial_hypo.trgt_sentence) > self.max_len):
            self.add_full_hypo(partial_hypo.generate_full_hypothesis())
            self.best_score = max(self.best_score, partial_hypo.score)
            return
        if self.apply_predictors_count > self.max_expansions: # pruning
            return
        posterior,score_breakdown = self.apply_predictors() 
        reload_states = False
        if self.early_stopping:
            worst_score = self.best_score - partial_hypo.score
            children = [i for i in posterior.items() if i[1] > worst_score]
        else:
            children = [i for i in posterior.items()]
        if len(children) > 1: # deep copy only if necessary
            pred_states = copy.deepcopy(self.get_predictor_states())
        logging.debug("Expand: best_score: %f exp: %d partial_score: "
                      "%f children: %d sentence: %s" %
                      (self.best_score,
                       self.apply_predictors_count,
                       partial_hypo.score,
                       len(children),
                       partial_hypo.trgt_sentence))
        for trgt_word,score in sorted(children,
                                      key=operator.itemgetter(1),
                                      reverse=True):
            new_hypo =  partial_hypo.expand(trgt_word,
                                            None, # Do not store states
                                            score,
                                            score_breakdown[trgt_word])
            if self.early_stopping and new_hypo.score < self.best_score:
                return
            if reload_states: # TODO: save one deepcopy (in last iteration)
                self.set_predictor_states(copy.deepcopy(pred_states))
            self.consume(trgt_word)
            self._dfs(new_hypo)
            reload_states = True
    
    def decode(self, src_sentence):
        """Decodes a single source sentence using depth first search.
        If ``max_expansions`` equals 0, this corresponds to exhaustive
        search for the globally best scoring hypothesis. Note that with
        ``early_stopping`` enabled, the returned set of hypothesis are
        not necessarily the global n-best hypotheses. To create an 
        exact n-best list, disable both ``max_expansions`` and 
        ``early_stopping`` in the constructor.
        
        Args:
            src_sentence (list): List of source word ids without <S> or
                                 </S> which make up the source sentence
        
        Returns:
            list. A list of ``Hypothesis`` instances ordered by their
            score. If ``max_expansions`` equals 0, the first element
            holds the global best scoring hypothesis
        """
        self.initialize_predictors(src_sentence)
        self.max_expansions = self.get_max_expansions(self.max_expansions_param,
                                                      src_sentence) 
        self.best_score = self.get_lower_score_bound()
        self._dfs(PartialHypothesis())
        return self.get_full_hypos_sorted() class SimpleDFSDecoder(Decoder):
    """This is a stripped down version of the DFS decoder which is
    designed to explore the entire search space. SimpleDFS is
    intended to be used with a `score_lower_bounds_file` from a
    previous beam search run which already contains good lower
    bounds. SimpleDFS verifies whether the lower bound is actually
    the global best score.

    SimpleDFS can only be used with a single predictor.

    SimpleDFS does not support max_expansions or max_len_factor.
    early_stopping cannot be disabled.
    """
    
    def __init__(self, decoder_args):
        """Creates new SimpleDFS decoder instance. 

        Args:
            decoder_args (object): Decoder configuration passed through
                                   from the configuration API.
        """
        super(SimpleDFSDecoder, self).__init__(decoder_args)
        #self._min_length_ratio = 0.25  # TODO: Make configurable
        self._min_length_ratio = -0.1
        self._min_length = -100
    
    def _dfs(self, partial_hypo):
        """Recursive function for doing dfs. Note that we do not keep
        track of the predictor states inside ``partial_hypo``, because 
        at each call of ``_dfs`` the current predictor states are equal
        to the hypo predictor states.
        
        Args:
            partial_hypo (PartialHypothesis): Partial hypothesis 
                                              generated so far. 
        """
        if partial_hypo.get_last_word() == utils.EOS_ID:
            if len(partial_hypo.trgt_sentence) >= self._min_length:
                self.add_full_hypo(partial_hypo.generate_full_hypothesis())
                if partial_hypo.score > self.best_score:
                    self.best_score = partial_hypo.score
                    logging.info("New best: score: %f exp: %d sentence: %s" %
                          (self.best_score,
                           self.apply_predictors_count,
                           partial_hypo.trgt_sentence))
            return

        self.apply_predictors_count += 1
        posterior = self.dfs_predictor.predict_next()
        logging.debug("Expand: best_score: %f exp: %d partial_score: "
                      "%f sentence: %s" %
                      (self.best_score,
                       self.apply_predictors_count,
                       partial_hypo.score,
                       partial_hypo.trgt_sentence))
        first_expansion = True
        for trgt_word, score in utils.common_iterable(posterior):
            if partial_hypo.score + score > self.best_score:
                if first_expansion:
                    pred_states = copy.deepcopy(self.get_predictor_states())
                    first_expansion = False
                else:
                    self.set_predictor_states(copy.deepcopy(pred_states))
                self.consume(trgt_word)
                self._dfs(partial_hypo.expand(trgt_word,
                                              None, # Do not store states
                                              score,
                                              [(score, 1.0)]))
    
    def decode(self, src_sentence):
        """Decodes a single source sentence exhaustively using depth 
        first search.
        
        Args:
            src_sentence (list): List of source word ids without <S> or
                                 </S> which make up the source sentence
        
        Returns:
            list. A list of ``Hypothesis`` instances ordered by their
            score.
        """
        if len(self.predictors) != 1:
            logging.fatal("SimpleDFS only works with a single predictor!")
        self.dfs_predictor = self.predictors[0][0]
        if self._min_length_ratio > 0.0:
            self._min_length = int(math.ceil(
              self._min_length_ratio * len(src_sentence))) + 1
        self.initialize_predictors(src_sentence)
        self.best_score = self.get_lower_score_bound()
        self._dfs(PartialHypothesis())
        return self.get_full_hypos_sorted() class SimpleLengthDFSDecoder(Decoder):
    """This is a length dependent version of SimpleDFS. This
    decoder finds the global best scores for certain hypo lengths.
    The `simplelendfs_lower_bounds_file` contains lines of the form
    
      <length1>:<lower-bound> ... <lengthN>:<lower-boundN>

    that specify length dependent score lower bounds. The decoder
    will search for the optimal model scores for the specified
    lengths.

    SimpleDFS can only be used with a single predictor.

    SimpleDFS does not support max_expansions or max_len_factor.
    early_stopping cannot be disabled.
    """
    
    def __init__(self, decoder_args):
        """Creates new SimpleDFS decoder instance. 

        Args:
            decoder_args (object): Decoder configuration passed through
                                   from the configuration API.
        """
        super(SimpleLengthDFSDecoder, self).__init__(decoder_args)
        with open(decoder_args.simplelendfs_lower_bounds_file) as f:
            self.all_lower_bounds = [{tuple(el.split(":"))
                                 for el in line.strip().split()} for line in f]
    
    def _dfs(self, partial_hypo):
        """Recursive function for doing dfs. Note that we do not keep
        track of the predictor states inside ``partial_hypo``, because 
        at each call of ``_dfs`` the current predictor states are equal
        to the hypo predictor states.
        
        Args:
            partial_hypo (PartialHypothesis): Partial hypothesis 
                                              generated so far. 
        """
        partial_hypo_length = len(partial_hypo.trgt_sentence)
        self.apply_predictors_count += 1
        posterior = self.dfs_predictor.predict_next()
        logging.debug("Expand: exp: %d partial_score: "
                      "%f sentence: %s" %
                      (self.apply_predictors_count,
                       partial_hypo.score,
                       partial_hypo.trgt_sentence))
        # Check EOS
        eos_score = posterior[utils.EOS_ID]
        if (self.len_enabled[partial_hypo_length] 
               and partial_hypo.score + eos_score 
                   > self.len_lower_bounds[partial_hypo_length]):
            eos_hypo = partial_hypo.expand(utils.EOS_ID,
                                           None, # Do not store states
                                           eos_score,
                                           [(eos_score, 1.0)])
            logging.info("New best: len: %d score: %f -> %f exp: %d" %
                          (partial_hypo_length,
                           self.len_lower_bounds[partial_hypo_length],
                           eos_hypo.score,
                           self.apply_predictors_count))
            self.len_lower_bounds[partial_hypo_length] = eos_hypo.score
            self.len_best_hypos[partial_hypo_length] = eos_hypo
            self._update_min_lower_bounds()
        if partial_hypo_length >= self.max_len:
            return
        first_expansion = True
        for trgt_word, score in utils.common_iterable(posterior):
            if trgt_word != utils.EOS_ID:
                lower_bound = self.len_min_lower_bounds[partial_hypo_length+1]
                if partial_hypo.score + score > lower_bound:
                    if first_expansion:
                        pred_states = copy.deepcopy(self.get_predictor_states())
                        first_expansion = False
                    else:
                        self.set_predictor_states(copy.deepcopy(pred_states))
                    self.consume(trgt_word)
                    self._dfs(partial_hypo.expand(trgt_word,
                                                  None, # Do not store states
                                                  score,
                                                  [(score, 1.0)]))

    def _update_min_lower_bounds(self):
        min_lower_bound = np.inf
        for l in range(self.max_len, -1, -1):
            if self.len_enabled[l]:
                min_lower_bound = min(min_lower_bound, self.len_lower_bounds[l])
            self.len_min_lower_bounds[l] = min_lower_bound
        logging.info("Lower bounds: %s" % (self.len_min_lower_bounds,))
    
    def decode(self, src_sentence):
        """Decodes a single source sentence exhaustively using depth 
        first search under length constraints.
        
        Args:
            src_sentence (list): List of source word ids without <S> or
                                 </S> which make up the source sentence
        
        Returns:
            list. A list of ``Hypothesis`` instances ordered by their
            score.
        """
        if len(self.predictors) != 1:
            logging.fatal("SimpleDFS only works with a single predictor!")
        self.dfs_predictor = self.predictors[0][0]
        self.initialize_predictors(src_sentence)
        lower_bounds = self.all_lower_bounds[self.current_sen_id]
        self.max_len = max(int(el[0]) for el in lower_bounds)
        self.len_enabled = np.zeros((self.max_len + 1,), np.bool)
        self.len_lower_bounds = -np.ones((self.max_len + 1,)) * np.inf
        self.len_min_lower_bounds = np.zeros((self.max_len + 1,))
        self.len_best_hypos = [None] * (self.max_len+1)
        for el in lower_bounds:
            l = int(el[0])
            self.len_enabled[l] = True
            self.len_lower_bounds[l] = float(el[1])
        self._update_min_lower_bounds()
        self._dfs(PartialHypothesis())
        for hypo in self.len_best_hypos:
            if hypo is not None:
                self.add_full_hypo(hypo.generate_full_hypothesis())
        return self.get_full_hypos_sorted() class FlipDecoder(Decoder):
    """The flip decoder explores the search space by permutating
    already explored hypotheses with a single permutation operation. We
    support two operations: 'flip' flips the position of two target 
    tokens. 'move' moves one target token to another location in the
    sentence.
    
    Note that this decoder does not support the ``max_length`` 
    parameter as it is designed for fixed length decoding problems.
    
    Also note that this decoder works only for bag-of-words problems.
    Do not use the bow predictor in combination with this decoder as
    it will hide the EOS scores which are important to estimate bigram
    scores.
    """
    
    def __init__(self, decoder_args, always_greedy=False):
        """Creates a new flip decoder. Do not use this decoder in 
        combination with the bow predictor as it inherently already
        satisfies the bag-of-word constrains. The following values
        are fetched from `decoder_args`:
        
            trg_test(string): Path to a plain text file which 
                              defines the bag of words
            max_node_expansions (int): Maximum number of node expansions 
                                       for inadmissible pruning.
            early_stopping (boolean): Activates admissible pruning

        Args:
            decoder_args (object): Decoder configuration passed through
                                   from the configuration API.
            always_greedy (boolean): Per default, the flip decoder does
                                     forced decoding along the complete
                                     candidate sentence. Set to True to
                                     do greedy decoding from the
                                     backtraced node instead
        """
        super(FlipDecoder, self).__init__(decoder_args) 
        self.max_expansions_param = decoder_args.max_node_expansions
        self.early_stopping = decoder_args.early_stopping
        self.always_greedy = always_greedy
        with open(decoder_args.trg_test) as f:
            self.lines = f.read().splitlines()
        if decoder_args.flip_strategy == 'flip':
            self._extract_candidates = self._extract_candidates_flip
        elif decoder_args.flip_strategy == 'move':
            self._extract_candidates = self._extract_candidates_move
        else:
            logging.fatal("Unknown flip strategy!")
    
    def _greedy_decode(self):
        """Performs greedy decoding from the start node. Used to obtain
        the initial hypothesis.
        """
        hypo = PartialHypothesis()
        hypos = []
        posteriors = []
        score_breakdowns = []
        scores = []
        bag = dict(self.full_bag)
        while bag:
            posterior,score_breakdown = self.apply_predictors()
            hypo.predictor_states = copy.deepcopy(self.get_predictor_states())
            hypos.append(hypo)
            posteriors.append(posterior)
            score_breakdowns.append(score_breakdown)
            best_word = utils.argmax({w: posterior[w] for w in bag})
            bag[best_word] -= 1
            if bag[best_word] < 1:
                del bag[best_word]
            self.consume(best_word)
            hypo = hypo.expand(best_word,
                               None,
                               posterior[best_word],
                               score_breakdown[best_word])
            scores.append(posterior[best_word])
        posterior,score_breakdown = self.apply_predictors()
        hypo.predictor_states = self.get_predictor_states()
        hypos.append(hypo)
        posteriors.append(posterior)
        score_breakdowns.append(score_breakdown)
        hypo = hypo.expand(utils.EOS_ID,
                           None,
                           posterior[utils.EOS_ID],
                           score_breakdown[utils.EOS_ID])
        logging.debug("Greedy hypo (%f): %s" % (
                          hypo.score,
                          ' '.join([str(w) for w in hypo.trgt_sentence])))
        scores.append(posterior[utils.EOS_ID])
        self.best_score = hypo.score
        self.add_full_hypo(hypo.generate_full_hypothesis())
        self._process_new_hypos(FlipCandidate(hypo.trgt_sentence,
                                               scores,
                                               self._create_dummy_bigrams(),
                                               hypo.score),
                                 len(hypo.trgt_sentence),
                                 hypos,
                                 posteriors,
                                 score_breakdowns)
    
    def _create_dummy_bigrams(self):
        """Creates a dictionary of optimistic bigram scores which is to
        be updated as the decoder steps through the target sentence
        """ 
        bigram_scores = {}
        words = [utils.GO_ID, utils.EOS_ID] + [w for w in self.full_bag]
        for w in words:
            bigram_scores[w] = {w2: 0.0 for w2 in words}
        return bigram_scores

    def _is_explored(self, trgt_sentence):
        """Returns true if this target sentence has been explored
        already
        """
        prefix = self.explored.get_prefix(trgt_sentence)
        return self.explored.get(prefix)
    
    def _process_new_hypos(self, 
                           candidate, 
                           max_pos, 
                           explored_hypos, 
                           explored_posteriors, 
                           explored_score_breakdowns):
        """This method is called after a new candidate has been 
        explored. It derives new candidates by updating the bigram
        scores and delegating to ``_extract_candidates`` for permutating
        the parent target sentence.
        """
        full_hypo_len = len(candidate.trgt_sentence)
        max_pos = min(full_hypo_len-1, # -1 because we don't permit moving EOS
                      max_pos)
        self.explored.add(candidate.trgt_sentence[0:max_pos+1], True)
        if (explored_hypos 
                and explored_hypos[-1].trgt_sentence[-1] == utils.EOS_ID):
            full_hypo = explored_hypos[-1].generate_full_hypothesis()
            self.best_score = max(self.best_score, full_hypo.total_score)
            self.add_full_hypo(full_hypo)
        # Update data structures
        bigram_scores = copy.deepcopy(candidate.bigram_scores)
        for idx,hypo in enumerate(explored_hypos):
            prev_word = hypo.trgt_sentence[-1] if hypo.trgt_sentence \
                                               else utils.GO_ID
            scores = candidate.scores[0:len(hypo.trgt_sentence)]
            for w in self.full_bag:
                score = explored_posteriors[idx][w]
                # Update bigram scores
                # not quite correct if we have two occurrences of the same word
                bigram_scores[prev_word][w] = score 
                # Update tries
                exp_hypo = hypo.cheap_expand(w, 
                                             score, 
                                             explored_score_breakdowns[idx][w])
                exp_hypo.scores = scores + [score]
                self.hypos.add(exp_hypo.trgt_sentence, exp_hypo)
        # Extract candidates
        self._extract_candidates(candidate, max_pos, bigram_scores)

    def _extract_candidates_move(self, candidate, max_pos, bigram_scores):
        """Implements the traversal strategy 'move' """
        full_hypo_len = len(candidate.trgt_sentence)
        for from_pos in range(full_hypo_len-1):
            from_word = candidate.trgt_sentence[from_pos]
            stub = candidate.trgt_sentence[0:from_pos] \
                        + candidate.trgt_sentence[from_pos+1:]
            for to_pos in range(full_hypo_len-1):
                change_pos = min(from_pos, to_pos)
                if change_pos >= max_pos:
                    continue
                trgt_sentence = list(stub)
                trgt_sentence.insert(to_pos, from_word)
                if self._is_explored(trgt_sentence):
                    continue
                scores = list(candidate.scores)
                prev_word = trgt_sentence[change_pos-1] if change_pos > 0 \
                                                        else utils.GO_ID
                for idx in range(change_pos, full_hypo_len):
                    word = trgt_sentence[idx]
                    scores[idx] = bigram_scores[prev_word][word]
                    prev_word = word
                self._add_candidate(FlipCandidate(trgt_sentence, 
                                                  scores, 
                                                  bigram_scores, 
                                                  sum(scores[0:change_pos+1])))

    def _extract_candidates_flip(self, candidate, max_pos, bigram_scores):
        """Implements the traversal strategy 'flip' """
        full_hypo_len = len(candidate.trgt_sentence)
        for from_pos in range(max_pos):
            max_score = sum(candidate.scores[0:from_pos])
            from_word = candidate.trgt_sentence[from_pos]
            for to_pos in range(from_pos+1, full_hypo_len-1):
                trgt_sentence = list(candidate.trgt_sentence)
                trgt_sentence[from_pos] = trgt_sentence[to_pos]
                trgt_sentence[to_pos] = from_word
                if self._is_explored(trgt_sentence):
                    continue
                scores = list(candidate.scores)
                prev_word = trgt_sentence[from_pos-1] if from_pos > 0 \
                                                      else utils.GO_ID
                for idx in range(from_pos, full_hypo_len):
                    word = trgt_sentence[idx]
                    scores[idx] = bigram_scores[prev_word][word]
                    prev_word = word
                self._add_candidate(FlipCandidate(trgt_sentence, 
                                                  scores, 
                                                  bigram_scores, 
                                                  max_score+scores[from_pos]))
        
    def _add_candidate(self, candidate):
        """Add a candidate to the heap """
        if candidate.max_score > self.best_score:
            heappush(self.open_candidates, (-candidate.expected_score, 
                                            candidate))
    
    def _explore_candidate(self, candidate):
        """Explores a candidate, adds it to the list of full hypotheses
        (if not pruned by early stopping) and calls  
        ``_process_new_hypos`` to derive new candidates. 
        """
        prefix = self.hypos.get_prefix(candidate.trgt_sentence)
        hypo = self.hypos.get(prefix)
        self.set_predictor_states(copy.deepcopy(hypo.predictor_states))
        for pos,score in enumerate(hypo.scores): # Update candidate scores
            candidate.scores[pos] = score
        self.consume(hypo.word_to_consume) 
        hypos = []
        posteriors = []
        score_breakdowns = []
        for pos in range(len(prefix), len(candidate.trgt_sentence)):
            if self.early_stopping and hypo.score <= self.best_score:
                break # admissible pruning
            posterior,score_breakdown = self.apply_predictors()
            hypo.predictor_states = copy.deepcopy(self.get_predictor_states())
            hypos.append(hypo)
            posteriors.append(posterior)
            score_breakdowns.append(score_breakdown)
            logging.debug("Explored: %s (%f)" % (
                          ' '.join([str(w) for w in hypo.trgt_sentence]),
                          hypo.score))
            word = candidate.trgt_sentence[pos]
            score = posterior[word]
            if self.always_greedy: # change sentence st best word is at this pos
                best_pos = pos
                stub = candidate.trgt_sentence[0:pos]
                for p in range(pos+1, len(candidate.trgt_sentence)-1):
                    if (posterior[candidate.trgt_sentence[p]] > score
                           and not self._is_explored(
                                        stub + [candidate.trgt_sentence[p]])):
                        best_pos = p
                        word = candidate.trgt_sentence[p]
                        score = posterior[word]  
                if best_pos != pos:
                    del candidate.trgt_sentence[best_pos]
                    del candidate.scores[best_pos]
                    candidate.trgt_sentence.insert(pos, word)
                    candidate.scores.insert(pos, score)
            self.consume(word)
            hypo = hypo.expand(word,
                               None,
                               score,
                               score_breakdown[word])
            candidate.scores[pos] = posterior[word]
        if hypo.trgt_sentence[-1] == utils.EOS_ID:
            self.best_score = max(self.best_score, hypo.score)
            self.add_full_hypo(hypo.generate_full_hypothesis())
        acc = 0.0
        for max_pos,score in enumerate(candidate.scores):
            acc += score
            if acc <= self.best_score:
                break
        self._process_new_hypos(candidate,
                                 max_pos,
                                 hypos,
                                 posteriors,
                                 score_breakdowns)
    
    def _load_bag(self):
        """Load the current bag of words """
        self.full_bag = {}
        for w in self.lines[self.current_sen_id].strip().split(): 
            int_w = int(w)
            self.full_bag[int_w] = self.full_bag.get(int_w, 0) + 1

    def decode(self, src_sentence):
        """Decodes a single source sentence with the flip decoder """
        self.initialize_predictors(src_sentence)
        self.max_expansions = self.get_max_expansions(self.max_expansions_param,
                                                      src_sentence) 
        self._load_bag()
        self.hypos = SimpleTrie()
        self.explored = SimpleTrie()
        self.open_candidates = []
        self.best_score = self.get_lower_score_bound()
        self._greedy_decode()
        while (self.open_candidates 
                        and self.max_expansions > self.apply_predictors_count):
            _,candidate = heappop(self.open_candidates)
            if candidate.max_score <= self.best_score:
                continue
            if self._is_explored(candidate.trgt_sentence): # Already explored
                continue
            logging.debug(
                "Best: %f Expected: %f Expansions: %d Open: %d Explore %s" % (
                          self.best_score,
                          candidate.expected_score,
                          self.apply_predictors_count,
                          len(self.open_candidates),
                          ' '.join([str(w) for w in candidate.trgt_sentence])))
            self._explore_candidate(candidate)
        return self.get_full_hypos_sorted() class GreedyDecoder(Decoder):
    """The greedy decoder does not revise decisions and therefore does
    not have to maintain predictor states. Therefore, this 
    implementation is particularly simple and can be used as template
    for more complex decoders. The greedy decoder can be imitated with
    the ``BeamDecoder`` with beam size 1.
    """
    
    def __init__(self, decoder_args):
        """Initialize the greedy decoder. """
        super(GreedyDecoder, self).__init__(decoder_args)
    
    def decode(self, src_sentence):
        """Decode a single source sentence in a greedy way: Always take
        the highest scoring word as next word and proceed to the next
        position. This makes it possible to decode without using the 
        predictors ``get_state()`` and ``set_state()`` methods as we
        do not have to keep track of predictor states.
        
        Args:
            src_sentence (list): List of source word ids without <S> or
                                 </S> which make up the source sentence
        
        Returns:
            list. A list of a single best ``Hypothesis`` instance."""
        self.initialize_predictors(src_sentence)
        trgt_sentence = []
        score_breakdown = []
        trgt_word = None
        score = 0.0
        while trgt_word != utils.EOS_ID and len(trgt_sentence) <= self.max_len:
            posterior,breakdown = self.apply_predictors(1)
            trgt_word = utils.argmax(posterior)
            score += posterior[trgt_word]
            trgt_sentence.append(trgt_word)
            logging.debug("Partial hypothesis (%f): %s" % (
                    score, " ".join([str(i) for i in trgt_sentence]))) 
            score_breakdown.append(breakdown[trgt_word])
            self.consume(trgt_word)
        self.add_full_hypo(Hypothesis(trgt_sentence, score, score_breakdown))
        return self.full_hypos class GreedyHeuristic(Heuristic):
    """This heuristic performs greedy decoding to get future cost 
    estimates. This is expensive but can lead to very close estimates.
    """
    
    def __init__(self, decoder_args, cache_estimates = True):
        """Creates a new ``GreedyHeuristic`` instance. The greedy 
        heuristic performs full greedy decoding from the current
        state to get accurate cost estimates. However, this can be very
        expensive.
        
        Args:
            decoder_args (object): Decoder configuration passed through
                                   from the configuration API.
            cache_estimates (bool): Set to true to enable a cache for
                                    predictor states which have been
                                    visited during the greedy decoding.
        """
        super(GreedyHeuristic, self).__init__()
        self.cache_estimates = cache_estimates
        self.decoder = GreedyDecoder(decoder_args)
        self.cache = SimpleTrie()
        
    def set_predictors(self, predictors):
        """Override ``Decoder.set_predictors`` to redirect the 
        predictors to ``self.decoder``
        """
        self.predictors = predictors
        self.decoder.predictors = predictors
    
    def initialize(self, src_sentence):
        """Initialize the cache. """
        self.cache = SimpleTrie()
    
    def estimate_future_cost(self, hypo):
        """Estimate the future cost by full greedy decoding. If
        ``self.cache_estimates`` is enabled, check cache first
        """
        if self.cache_estimates:
            return self.estimate_future_cost_with_cache(hypo)
        else:
            return self.estimate_future_cost_without_cache(hypo)
    
    def estimate_future_cost_with_cache(self, hypo):
        """Enabled cache... """
        cached_cost = self.cache.get(hypo.trgt_sentence)
        if not cached_cost is None:
            return cached_cost
        old_states = self.decoder.get_predictor_states()
        self.decoder.set_predictor_states(copy.deepcopy(old_states))
        # Greedy decoding
        trgt_word = hypo.trgt_sentence[-1]
        scores = []
        words = []
        while trgt_word != utils.EOS_ID:
            self.decoder.consume(trgt_word)
            posterior,_ = self.decoder.apply_predictors()
            trgt_word = utils.argmax(posterior)
            scores.append(posterior[trgt_word])
            words.append(trgt_word)
        # Update cache using scores and words
        for i in range(1, len(scores)):
            self.cache.add(hypo.trgt_sentence + words[:i], -sum(scores[i:]))
        # Reset predictor states
        self.decoder.set_predictor_states(old_states)
        return -sum(scores)
    
    def estimate_future_cost_without_cache(self, hypo):
        """Disabled cache... """
        old_states = self.decoder.get_predictor_states()
        self.decoder.set_predictor_states(copy.deepcopy(old_states))
        # Greedy decoding
        trgt_word = hypo.trgt_sentence[-1]
        score = 0.0
        while trgt_word != utils.EOS_ID:
            self.decoder.consume(trgt_word)
            posterior,_ = self.decoder.apply_predictors()
            trgt_word = utils.argmax(posterior)
            score += posterior[trgt_word]
        # Reset predictor states
        self.decoder.set_predictor_states(old_states)
        return -score class PredictorHeuristic(Heuristic):
    """The predictor heuristic relies on the 
    ``estimate_future_costs()`` implementation of the predictors. Use
    this heuristic to access predictor specific future cost functions,
    e.g. shortest path for the fst predictor.
    """
    
    def estimate_future_cost(self, hypo):
        """Returns the weighted sum of predictor estimates. """
        return Decoder.combi_arithmetic_unnormalized([
                                    (pred.estimate_future_cost(hypo), w)
                                            for (pred, w) in self.predictors])
    
    def initialize(self, src_sentence):
        """Calls ``initialize_heuristic()`` on all predictors. """
        for (pred, _) in self.predictors:
            pred.initialize_heuristic(src_sentence)
    
    def notify(self, message, message_type = MESSAGE_TYPE_DEFAULT):
        """This heuristic passes through notifications to the 
        predictors.
        """
        for (pred, _) in self.predictors:
            pred.notify(message, message_type) class ScorePerWordHeuristic(Heuristic):
    """Using this heuristic results in length normalized scores instead
    of the pure sum of predictor scores for a partial hypothesis.
    Therefore, it is not a heuristic like in the classical A* sense.
    Instead, using the A* decoder with this heuristic simulates beam
    search which always keeps the hypotheses with the best per word
    scores.
    """
    
    def estimate_future_cost(self, hypo):
        """A* will put ``cost-score`` on the heap. In order to simulate
        length normalized beam search, we want to use ``-score/length``
        as partial hypothesis score. Therefore, this method returns
        ``-score/length + score``
        """
        if len(hypo.trgt_sentence) > 0:
            return hypo.score - hypo.score/len(hypo.trgt_sentence)
        return 0.0
    
    def initialize(self, src_sentence):
        """Empty method."""
        pass class StatsHeuristic(Heuristic):
    """This heuristic is based on the sum of unigram costs of consumed 
    words. Unigram statistics are collected via a ``UnigramTable``.
    """
    
    def __init__(self, heuristic_scores_file="", collect_stats_strategy='best'):
        """Creates a new ``StatsHeuristic`` instance. The constructor
        initializes the unigram table.
        
        Args:
            heuristic_scores_file (string): Path to the unigram scores 
                                            which are used if this 
                                            predictor estimates future
                                            costs
            collect_stats_strategy (string): best, full, or all. Defines 
                                             how unigram estimates are 
                                             collected for heuristic
        """
        super(StatsHeuristic, self).__init__()
        if heuristic_scores_file:
            self.estimates = FileUnigramTable(heuristic_scores_file)
        elif collect_stats_strategy == 'best':
            self.estimates = BestStatsUnigramTable()
        elif collect_stats_strategy == 'full':
            self.estimates = FullStatsUnigramTable()
        elif collect_stats_strategy == 'all':
            self.estimates = AllStatsUnigramTable()
        else:
            logging.error("Unknown statistics collection strategy")
    
    def initialize(self, src_sentence):
        """Calls ``reset`` to reset collected statistics from previous
        sentences
        
        Args:
            src_sentence (list): Not used
        """
        self.estimates.reset()
    
    def notify(self, message, message_type = MESSAGE_TYPE_DEFAULT):
        """Passing through to the unigram table ``self.estimates``.
        """
        self.estimates.notify(message, message_type)
    
    def estimate_future_cost(self, hypo):
        """Returns the sum of heuristic unigram estimates of the words
        in the translation prefix of ``hypo``. Combined with the hypo
        score, this leads to using the ratio between actual hypo score 
        and an idealistic score (product of unigrams) to discriminate
        partial hypotheses.
        """
        return sum([self.estimates.estimate(w, -1000.0)
                            for w in hypo.trgt_sentence]) class LastTokenHeuristic(Heuristic):
    """This heuristic reflects the score of the last token in the
    translation prefix only, ie. not the accumulated score. Using this
    with pure_heuristic_estimates leads to expanding the partial 
    hypothesis with the end token with the best individual score. This
    can be useful in search spaces in which bad translation prefixes
    imply low individual scores later.
    """
    
    def estimate_future_cost(self, hypo):
        """Returns the negative score of the last token in hypo."""
        return -Decoder.combi_arithmetic_unnormalized(hypo.score_breakdown[-1])
    
    def initialize(self, src_sentence):
        """Empty method."""
        pass class MultisegBeamDecoder(Decoder):
    """This is a version of beam search which can handle predictors 
    with differing tokenizations. We assume that all tokenizations are
    consistent with words, i.e. no token crosses word boundaries. The
    search simulates beam search on the word level. At each time step, 
    we keep the n best hypotheses on the word level. Predictor scores
    on finer-grained tokens are collapsed into a single score s.t. they
    can be combined with scores from other predictors. This decoder can 
    produce words without entry in the word map. In this case, words 
    are added to ``io.trg_wmap``. Consider using the ``output_chars``
    option to avoid dealing with the updated word map in the output.
    """
    
    def __init__(self,
                 decoder_args,
                 hypo_recombination,
                 beam_size,
                 tokenizations,
                 early_stopping = True,
                 max_word_len = 25):
        """Creates a new beam decoder instance for predictors with
        multiple tokenizations.
        
        Args:
            decoder_args (object): Decoder configuration passed through
                                   from the configuration API.
            hypo_recombination (bool): Activates hypo recombination 
            beam_size (int): Absolute beam size. A beam of 12 means
                             that we keep track of 12 active hypothesis
            tokenizations (string): Comma separated strings describing 
                                    the predictor tokenizations
            early_stopping (bool): If true, we stop when the best
                                   scoring hypothesis ends with </S>.
                                   If false, we stop when all hypotheses
                                   end with </S>. Enable if you are
                                   only interested in the single best
                                   decoding result. If you want to 
                                   create full 12-best lists, disable
            max_word_len (int): Maximum length of a single word
        """
        super(MultisegBeamDecoder, self).__init__(decoder_args)
        self.hypo_recombination = hypo_recombination
        self.beam_size = beam_size
        self.stop_criterion = self._best_eos if early_stopping else self._all_eos
        self.toks = []
        self.max_word_len = max_word_len
        if not tokenizations:
            logging.fatal("Specify --multiseg_tokenizations!")
        for tok_config in tokenizations.split(","):
            if tok_config[:6] == "mixed:":
                tok = MixedTokenizer(tok_config[6:])
            elif tok_config[:4] == "eow:":
                tok = EOWTokenizer(tok_config[4:])
            else:
                if tok_config[:5] == "word:":
                    tok_config = tok_config[5:]
                tok = WordTokenizer(tok_config)
            self.toks.append(tok)

    def _best_eos(self, hypos):
        """Returns true if the best hypothesis ends with </S>"""
        return hypos[0].get_last_word() != utils.EOS_ID

    def _all_eos(self, hypos):
        """Returns true if the all hypotheses end with </S>"""
        for hypo in hypos[:self.beam_size]:
            if hypo.get_last_word() != utils.EOS_ID:
                return True
        return False
    
    def _rebuild_hypo_list(self, hypos, new_hypo):
        """Add new_hypo to the list of n best complete hypos.
        Implements hypothesis recombination.
        
        Returns:
            list. Sorted list of n best hypos in hypos + new_hypo
        """
        if not self.hypo_recombination:
            hypos.append(new_hypo)
        else:
            combined = False
            for idx,hypo in list(enumerate(hypos)):
                if hypo.predictor_states and self.are_equal_predictor_states(
                                                    hypo.predictor_states,
                                                    new_hypo.predictor_states):
                    if hypo.score >= new_hypo.score: # Keep old one
                        hypo1 = hypo
                        hypo2 = new_hypo
                    else: # Discard old one
                        hypo1 = new_hypo
                        hypo2 = hypo
                        hypos[idx] = new_hypo
                    logging.debug("Hypo recombination: %s > %s" % (
                        hypo1.trgt_sentence, hypo2.trgt_sentence))
                    combined = True
                    break
            if not combined:
                hypos.append(new_hypo)
        hypos.sort(key=lambda h: h.score, reverse=True)
        return hypos[:self.beam_size]
    
    def _get_word_initial_posteriors(self, hypo):
        """Call ``predict_next`` on all predictors to get the 
        distributions over the first tokens of the next word.

        Args:
            hypo (PartialHypothesis): Defines the predictor states 

        Returns:
            list. List of posterior vectors for each predictor. The
            UNK scores are added to the vectors.
        """
        self.apply_predictors_count += 1
        self.set_predictor_states(hypo.predictor_states)
        posteriors = []
        for p, _ in self.predictors:
            posterior = p.predict_next()
            posterior[utils.UNK_ID] = p.get_unk_probability(posterior) 
            posteriors.append(posterior)
        return posteriors

    def _get_initial_stubs(self, predictor, start_posterior, min_score):
        """Get the initial predictor stubs for full word search with a 
        single predictor. 
        """
        stubs = []
        pred_state = predictor.get_state()
        for t, s in utils.common_iterable(start_posterior):
            stub = PredictorStub([t], pred_state)
            stub.score_next(s)
            if stub.score >= min_score:
                stubs.append(stub)
        stubs.sort(key=lambda s: s.score, reverse=True)
        return stubs
   
    def _best_keys_complete(self, stubs, tok):
        """Stopping criterion for single predictor full word search.
        We stop full word search if the n best stubs are complete.
        """
        return all([is_key_complete(tok.tokens2key(s.tokens)) 
                                           for s in stubs[:self.beam_size]])

    def _search_full_words(self, predictor, start_posterior, tok, min_score):
        stubs = self._get_initial_stubs(predictor, start_posterior, min_score)
        while not self._best_keys_complete(stubs, tok):
            next_stubs = []
            for stub in stubs[:self.beam_size]:
                key = tok.tokens2key(stub.tokens)
                if (not key) or len(key) > self.max_word_len:
                    continue
                if is_key_complete(key):
                    next_stubs.append(stub)
                    continue
                predictor.set_state(copy.deepcopy(stub.pred_state))
                predictor.consume(stub.tokens[-1])
                posterior = predictor.predict_next()
                pred_state = predictor.get_state()
                for t, s in utils.common_iterable(posterior):
                    if t != utils.UNK_ID and not tok.is_word_begin_token(t):
                        child_stub = stub.expand(t, s, pred_state)
                        if child_stub.score >= min_score:
                            next_stubs.append(child_stub)
            stubs = next_stubs
            stubs.sort(key=lambda s: s.score, reverse=True)
        return stubs
    
    def _get_complete_continuations(self, hypo, min_hypo_score):
        """This is a generator which yields the complete continuations 
        of ``hypo`` in descending order of score
        """
        min_score = min_hypo_score - hypo.score
        if min_score > 0.0:
            return
        
        pred_weights = list(map(lambda el: el[1], self.predictors))
        # Get initial continuations by searching with predictors separately
        start_posteriors = self._get_word_initial_posteriors(hypo)
        pred_states = self.get_predictor_states()
        keys = {}
        for pidx, (p,w) in enumerate(self.predictors):
            stubs = self._search_full_words(p,
                                            start_posteriors[pidx],
                                            self.toks[pidx],
                                            min_score / w)
            n_added = 0
            for stub in stubs:
                key = self.toks[pidx].tokens2key(stub.tokens)
                if is_key_complete(key):
                    if key in keys: # Add to existing continuation
                        prev_stub = keys[key].pred_stubs[pidx]
                        if prev_stub is None or prev_stub.score < stub.score:
                            keys[key].pred_stubs[pidx] = stub
                    elif n_added < self.beam_size: # Create new continuation
                        n_added += 1
                        stubs = [None] * len(self.predictors)
                        stubs[pidx] = stub
                        keys[key] = Continuation(hypo, stubs, key)
        # Fill in stubs which are set to None
        for cont in keys.values():
            for pidx in range(len(self.predictors)):
                if cont.pred_stubs[pidx] is None:
                    stub = PredictorStub(self.toks[pidx].key2tokens(cont.key),
                                         pred_states[pidx])
                    stub.score_next(utils.common_get(
                                         start_posteriors[pidx],
                                         stub.tokens[0],
                                         start_posteriors[pidx][utils.UNK_ID]))
                    cont.pred_stubs[pidx] = stub
        conts = [(-c.calculate_score(pred_weights), c) for c in keys.values()]
        heapq.heapify(conts)
        # Iterate through conts, expand if necessary, yield if complete
        while conts:
            s,cont = heapq.heappop(conts)
            if cont.is_complete():
                yield -s,cont
            else: # Need to rescore with sec predictors
                cont.expand(self)
                heapq.heappush(conts, (-cont.calculate_score(pred_weights), cont))
    
    def decode(self, src_sentence):
        """Decodes a single source sentence using beam search. """
        self.initialize_predictors(src_sentence)
        hypos = [PartialHypothesis(self.get_predictor_states())]
        guard_hypo = PartialHypothesis(None)
        guard_hypo.score = utils.NEG_INF
        it = 0
        while self.stop_criterion(hypos):
            if it > self.max_len: # prevent infinite loops
                break
            it = it + 1
            next_hypos = [guard_hypo]
            for hypo in hypos:
                if hypo.get_last_word() == utils.EOS_ID:
                    next_hypos = self._rebuild_hypo_list(next_hypos, hypo)
                for s, cont in self._get_complete_continuations(
                                                        hypo,
                                                        next_hypos[-1].score):
                    if hypo.score + s < next_hypos[-1].score:
                        break
                    next_hypos = self._rebuild_hypo_list(
                                            next_hypos,
                                            cont.generate_expanded_hypo(self))
            hypos = [h for h in next_hypos if h.score > utils.NEG_INF]  
        for hypo in hypos:
            if hypo.get_last_word() == utils.EOS_ID:
                self.add_full_hypo(hypo.generate_full_hypothesis()) 
        if not self.full_hypos:
            logging.warn("No complete hypotheses found for %s" % src_sentence)
            for hypo in hypos:
                self.add_full_hypo(hypo.generate_full_hypothesis())
        return self.get_full_hypos_sorted() class RestartingDecoder(Decoder):
    """This decoder first creates a path to the final node greedily.
    Then, it looks for the node on this path with the smallest 
    difference between best and second best child, and restarts greedy 
    decoding from this point. In order to do so, it maintains a 
    priority queue of all visited nodes, which is ordered by the 
    difference between the worst expanded child and the best unexpanded
    one. If this queue is empty, we have visited the best path. This 
    algorithm is similar to DFS but does not backtrace to the last call
    of the recursive function but to the one which is most promising.
    
    Note that this algorithm is exact. It tries to exploit the problem
    characteristics of NMT search: Reloading predictor states can be
    expensive, node expansion is even more expensive but for free from
    visited nodes, and there is no good admissible heuristic.
    
    Note2: Does not work properly if predictor scores can be positive 
    because of admissible pruning
    """
    
    def __init__(self, 
                 decoder_args,
                 hypo_recombination,
                 max_expansions = 0,
                 low_memory_mode = True,
                 node_cost_strategy='difference',
                 stochastic=False,
                 always_single_step=False):
        """Creates new Restarting decoder instance.
        
        Args:
            decoder_args (object): Decoder configuration passed through
                                   from the configuration API.
            hypo_recombination (bool): Activates hypo recombination 
            max_expansions (int): Maximum number of node expansions for
                                  inadmissible pruning.
            low_memory_mode (bool): Switch on low memory mode at cost 
                                    of some computational overhead as
                                    the set of open nodes is reduced
                                    after each decoding pass
            node_cost_strategy (string): How to decide which node to 
                                          restart from next
            stochastic (bool): If true, select the next node to restart
                               from randomly. If false, take the one
                               with the best node score
            always_single_step (bool): If false, do greedy decoding 
                                       when restarting. If true, expand
                                       the hypothesis only by a single
                                       token
        """
        super(RestartingDecoder, self).__init__(decoder_args)
        self.max_expansions_param = max_expansions
        self.always_single_step = always_single_step
        self.low_memory_mode = low_memory_mode
        self.hypo_recombination = hypo_recombination
        if node_cost_strategy == 'difference':
            self.get_node_cost = self._get_node_cost_difference
        elif node_cost_strategy == 'absolute':
            self.get_node_cost = self._get_node_cost_absolute
        elif node_cost_strategy == 'constant':
            self.get_node_cost = self._get_node_cost_constant
        elif node_cost_strategy == 'expansions':
            self.get_node_cost = self._get_node_cost_expansions
        else:
            logging.fatal("restarting node score strategy unknown!")
        if stochastic:
            self.select_node = self._select_node_stochastic
        else:
            self.select_node = self._select_node_max
    
    def _select_node_stochastic(self):
        """Implements stochastic node selection. """
        exps = np.exp([-c for c,_ in self.open_nodes])
        total = sum(exps)
        idx = np.random.choice(range(len(exps)),
                               p=[e/total for e in exps])
        return self.open_nodes.pop(idx)
    
    def _select_node_max(self):
        """Implements deterministic node selection. """
        return heappop(self.open_nodes)
    
    def _get_node_cost_difference(self,
                                   prev_node_cost,
                                   first_word_score, 
                                   sec_word_score):
        """Implements the node scoring function difference. """
        return first_word_score - sec_word_score
    
    def _get_node_cost_absolute(self,
                                   prev_node_cost,
                                   first_word_score, 
                                   sec_word_score):
        """Implements the node scoring function absolute. """
        return -sec_word_score
    
    def _get_node_cost_constant(self,
                                   prev_node_cost,
                                   first_word_score, 
                                   sec_word_score):
        """Implements the node scoring function constant. """
        return 0.0
    
    def _get_node_cost_expansions(self,
                                   prev_node_cost,
                                   first_word_score, 
                                   sec_word_score):
        """Implements the node scoring function constant. """
        return prev_node_cost + 1.0
    
    def greedy_decode(self, hypo):
        """Helper function for greedy decoding from a certain point in
        the search tree."""
        best_word = hypo.trgt_sentence[-1]
        prev_hypo = hypo
        remaining_exps = max(self.max_expansions - self.apply_predictors_count,
                             1)
        while (best_word != utils.EOS_ID 
               and len(prev_hypo.trgt_sentence) <= self.max_len):
            self.consume(best_word)
            posterior,score_breakdown = self.apply_predictors()
            if len(posterior) < 1:
                return
            best_word = utils.argmax(posterior)
            best_word_score = posterior[best_word]
            new_hypo = prev_hypo.expand(best_word,
                                        None,
                                        best_word_score,
                                        score_breakdown[best_word])
            if new_hypo.score < self.best_score: # Admissible pruning
                return
            logging.debug("Expanded hypo: score=%f prefix= %s" % (
                            new_hypo.score,
                            ' '.join([str(w) for w in new_hypo.trgt_sentence])))
            if len(posterior) > 1:
                if not self.always_single_step:
                    posterior.pop(best_word)
                children = sorted([RestartingChild(w,
                                                   posterior[w],
                                                   score_breakdown[w])
                    for w in posterior], key=lambda c: c.score, reverse=True)
                children = children[:remaining_exps]
                node_cost = self.get_node_cost(0.0, 
                                               best_word_score, 
                                               children[0].score)
                if node_cost <= self.max_heap_node_cost:
                    prev_hypo.predictor_states = copy.deepcopy(
                                                self.get_predictor_states())
                    heappush(self.open_nodes, (node_cost,
                                               RestartingNode(prev_hypo,
                                                              children)))
            prev_hypo = new_hypo
            if self.always_single_step:
                break
        if best_word == utils.EOS_ID:
            self.add_full_hypo(prev_hypo.generate_full_hypothesis())
            if prev_hypo.score > self.best_score: 
                logging.info("New_best (ID: %d): score=%f exp=%d hypo=%s" 
                    % (self.current_sen_id + 1,
                       prev_hypo.score, 
                       self.apply_predictors_count,
                       ' '.join([str(w) for w in prev_hypo.trgt_sentence])))
                self.best_score = prev_hypo.score
    
    def create_initial_node(self):
        """Create the root node for the search tree. """
        init_hypo = PartialHypothesis()
        posterior,score_breakdown = self.apply_predictors()
        children = sorted([RestartingChild(w, posterior[w], score_breakdown[w])
                            for w in posterior],
                          key=lambda c: c.score, reverse=True)
        init_hypo.predictor_states = self.get_predictor_states()
        heappush(self.open_nodes, (0.0, RestartingNode(init_hypo, children)))

    def decode(self, src_sentence):
        """Decodes a single source sentence using Restarting search. """
        self.initialize_predictors(src_sentence)
        self.max_expansions = self.get_max_expansions(self.max_expansions_param,
                                                      src_sentence) 
        self.open_nodes = []
        self.best_score = self.get_lower_score_bound()
        self.max_heap_node_cost = INF
        # First, create a RestartingNode object for the initial state
        self.create_initial_node()
        # Then, restart from open nodes until the heap is empty
        while self.open_nodes:
            prev_node_score,node = self.select_node()
            best_child = node.children.pop(0)
            new_hypo = node.hypo.expand(best_child.word,
                                        None,
                                        best_child.score,
                                        best_child.score_breakdown)
            if new_hypo.score > self.best_score: # Admissible pruning
                logging.debug("Restart from %s" % (
                            ' '.join([str(w) for w in new_hypo.trgt_sentence])))
                if node.children: # Still has children -> back to heap
                    node_cost = self.get_node_cost(prev_node_score, 
                                                   best_child.score, 
                                                   node.children[0].score)
                    heappush(self.open_nodes, (node_cost, node))
                    self.set_predictor_states(copy.deepcopy(
                                                node.hypo.predictor_states))
                else: # No need to copy, don't put back to heap
                    self.set_predictor_states(node.hypo.predictor_states)
                self.greedy_decode(new_hypo)
            # Reduce heap size (we don't need more nodes than remaining exps
            rest = self.max_expansions - self.apply_predictors_count
            if rest <= 0:
                break
            if self.hypo_recombination:
                new_open = []
                while len(new_open) < rest and self.open_nodes:
                    c_cost,candidate = heappop(self.open_nodes)
                    valid = True
                    for _,node in new_open:
                        if self.are_equal_predictor_states(
                                                candidate.hypo.predictor_states,
                                                node.hypo.predictor_states):
                            valid = False
                            break
                    if valid:
                        new_open.append((c_cost, candidate))
                        if len(new_open) > rest:
                            break
                self.open_nodes = new_open
                heapify(self.open_nodes) 
            elif self.low_memory_mode and len(self.open_nodes) > rest:
                new_open = [heappop(self.open_nodes) for _ in range(rest+1)]
                self.max_heap_node_cost = new_open[-1][0]
                self.open_nodes = new_open
                heapify(self.open_nodes)
        return self.get_full_hypos_sorted() class SepBeamDecoder(BeamDecoder):
    """This beam search implementation breaks with the predictor
    abstraction via the ``Decoder.apply_predictors()`` and 
    ``Decoder.consume()`` interfaces. We do not use combined scores
    of all predictors, but link single predictors to hypotheses in 
    the beam. On hypo expansion, we call ``predict_next()`` only on
    this predictor. This is suitable for approximated ensembling as
    it reduces the runtime nearly to a single system run.
    
    Note that ``PartialHypothesis.predictor_states`` holds a list
    with ``None`` objects except for one position.
    
    Also note that predictor weights are ignored for this decoding
    strategy.
    """
    
    def __init__(self, decoder_args):
        """Creates a new beam decoder instance for system level 
        combination. See the docstring of the BeamDecoder constructor
        for a description of which arguments are fetched from
        `decoder_args`.
        """
        super(SepBeamDecoder, self).__init__(decoder_args)
        if self.hypo_recombination:
            logging.warn("Hypothesis recombination is not applicable "
                         "to the sepbeam decoder.")
    
    def _get_initial_hypos(self):
        """Get the list of initial ``PartialHypothesis``. This is not
        a single empty hypo but one empty hypo for each predictor.
        """
        states = self.get_predictor_states()
        none_states = [None] * len(states)
        ret = []
        for idx, state in enumerate(states):
            pred_states = list(none_states)
            pred_states[idx] = state
            ret.append(PartialHypothesis(pred_states))
        return ret
    
    def _expand_hypo(self, hypo):
        """Expands hypothesis by calling predict_next() only on one
        single predictor.
        """
        if hypo.score <= self.min_score:
            return []
        pred_idx = 0
        for idx, s in enumerate(hypo.predictor_states):
            if not s is None:
                pred_idx = idx
                break
        self.apply_predictors_count += 1
        predictor = self.predictors[pred_idx][0]
        predictor.set_state(copy.deepcopy(hypo.predictor_states[pred_idx]))
        if not hypo.word_to_consume is None: # Consume if cheap expand
            predictor.consume(hypo.word_to_consume)
            hypo.word_to_consume = None
        posterior = predictor.predict_next()
        hypo.predictor_states = list(hypo.predictor_states)
        hypo.predictor_states[pred_idx] = predictor.get_state()
        breakdown_dummy = [(0.0, 1.0)] * len(self.predictors)
        ret = []
        for trgt_word in utils.argmax_n(posterior, self.beam_size):
            score_breakdown = list(breakdown_dummy)
            score_breakdown[pred_idx] = (posterior[trgt_word], 1.0)
            ret.append(hypo.cheap_expand(trgt_word,
                                         posterior[trgt_word],
                                         score_breakdown))
        return ret class SyntaxBeamDecoder(BeamDecoder):
    """The syntax beam search strategy is an extension of beam search 
    which ensures diversity amongst the terminals in the active
    hypotheses. The decoder clusters hypotheses by their terminal 
    history. Each cluster cannot have more than beam_size hypos, and
    the number of clusters is topped by beam_size. This means that
    the effective beam size varies between beam_size and beam_size^2,
    and there are always beam_size different terminal histories in the
    active beam.
    """
    
    def __init__(self, decoder_args):
        """Creates a new beam decoder instance for system level 
        combination. In addition to the constructor of
        `BeamDecoder`, the following values are fetched from 
        `decoder_args`:
        
            syntax_min_terminal_id (int): All IDs smaller than this are NTs
            syntax_max_terminal_id (int): All IDs larger than this are NTs
        """
        super(SyntaxBeamDecoder, self).__init__(decoder_args)
        self.min_terminal_id = decoder_args.syntax_min_terminal_id
        self.max_terminal_id = decoder_args.syntax_max_terminal_id

    def is_terminal(self, tok):
        return (tok <= self.max_terminal_id and tok >= self.min_terminal_id)

    def _get_next_hypos_diverse(self, hypos, scores):
        """Get hypotheses of the next time step.
        
        Args:
            hypos (list): List of hypotheses
            scores (list): hypo scores with heuristic estimates
        
        Return:
            list. List with hypotheses.
        """
        new_hypos = []
        terminal_history_counts = {}
        for idx in reversed(np.argsort(scores)):
            candidate = hypos[idx]
            key = " ".join([str(i) for i in candidate.trgt_sentence
                            if self.is_terminal(i)])
            cnt = terminal_history_counts.get(key, 0)
            if cnt >= self.beam_size:
                continue
            valid = True
            if self.hypo_recombination:
                self.set_predictor_states(copy.deepcopy(candidate.predictor_states))
                if not candidate.word_to_consume is None:
                    self.consume(candidate.word_to_consume)
                    candidate.word_to_consume = None
                    candidate.predictor_states = self.get_predictor_states()
                for hypo in new_hypos:
                    if self.are_equal_predictor_states(
                                                    hypo.predictor_states,
                                                    candidate.predictor_states):
                        logging.debug("Hypo recombination: %s > %s" % (
                                                     hypo.trgt_sentence,
                                                     candidate.trgt_sentence))
                        valid = False
                        break
            if valid:
                terminal_history_counts[key] = cnt + 1
                new_hypos.append(candidate)
                if len(terminal_history_counts) >= self.beam_size:
                    break
        return new_hypos
    
    def decode(self, src_sentence):
        """Decodes a single source sentence using beam search. """
        self.initialize_predictors(src_sentence)
        hypos = self._get_initial_hypos()
        it = 0
        self.min_score = utils.NEG_INF
        while self.stop_criterion(hypos):
            if it > self.max_len: # prevent infinite loops
                break
            it = it + 1
            next_hypos = []
            next_scores = []
            for hypo in hypos:
                logging.debug("Expand (it=%d score=%f): %s"
                              % (it, hypo.score, hypo.trgt_sentence))
                if hypo.get_last_word() == utils.EOS_ID:
                    next_hypos.append(hypo)
                    next_scores.append(self._get_combined_score(hypo))
                    continue 
                for next_hypo in self._expand_hypo(hypo):
                    next_score = self._get_combined_score(next_hypo)
                    next_hypos.append(next_hypo)
                    next_scores.append(next_score)
            hypos = self._get_next_hypos_diverse(next_hypos, next_scores)
        for hypo in hypos:
            if hypo.get_last_word() == utils.EOS_ID:
                self.add_full_hypo(hypo.generate_full_hypothesis()) 
        if not self.full_hypos:
            logging.warn("No complete hypotheses found for %s" % src_sentence)
            for hypo in hypos:
                self.add_full_hypo(hypo.generate_full_hypothesis())
        return self.get_full_hypos_sorted() class MBRBeamDecoder(BeamDecoder):
    """The MBR-based beam decoder does not select the n most likely
    hypotheses in each timestep. Instead, it tries to find the translation
    with the best expected BLEU. Two strategies control the behavior of 
    mbrbeam: the `evidence_strategy` and the `selection_strategy`.
    Available evidence strategies:
      'renorm': Only makes use of the n-best expansions of the hypos in the
                current beam. It renormalizes the scores, and count ngrams in
                the n^2 hypos.
      'maxent': Applies the MaxEnt rule to the evidence space. It makes use of
                all partial hypos seen so far and updates its belief about the
                probability of an ngram based on that. Following MaxEnt we 
                assume that translations outside the explored space are 
                uniformly distributed.
    Available selection strategies:
      'bleu': Select the n hypotheses with the best expected BLEU
      'oracle_bleu': Select'a subset with n elements which maximizes the 
                     expected maximum BLEU of one of the selected hypos. In 
                     other words, we optimize the oracle BLEU of the n-best 
                     list at each time step, where the n-best list consists 
                     of the active hypotheses in the beam.
    """
    
    def __init__(self, decoder_args):
        """Creates a new MBR beam decoder instance. We explicitly
        set early stopping to False since risk-free pruning is not 
        supported by the MBR-based beam decoder. The MBR-based
        decoder fetches the following fields from ``decoder_args``:

          min_ngram_order (int): Minimum n-gram order
          max_ngram_order (int): Maximum n-gram order
          mbrbeam_smooth_factor (float): Smoothing factor for evidence space
          mbrbeam_evidence_strategy (String): Evidence strategy
          mbrbeam_selection_strategy (String): Selection strategy

        Args:
            decoder_args (object): Decoder configuration passed through
                                   from the configuration API.
        """
        self.min_order = decoder_args.min_ngram_order
        self.max_order = decoder_args.max_ngram_order
        self.smooth_factor = decoder_args.mbrbeam_smooth_factor
        self.selection_strategy = decoder_args.mbrbeam_selection_strategy
        if not self.selection_strategy in ['bleu', 'oracle_bleu']:
            raise AttributeError("Unknown selection strategy '%s'"
                                 % self.selection_strategy)
        if decoder_args.mbrbeam_evidence_strategy == 'renorm':
            self._get_next_hypos_mbr = self._get_next_hypos_renorm
        elif decoder_args.mbrbeam_evidence_strategy == 'maxent':
            self._get_next_hypos_mbr = self._get_next_hypos_maxent
            if decoder_args.sub_beam <= 0:
                self.sub_beam_size = 0 # We need all expansions
        else:
            raise AttributeError("Unknown evidence strategy '%s'"
                                 % decoder_args.mbrbeam_evidence_strategy)
        super(MBRBeamDecoder, self).__init__(decoder_args)
        self.maintain_best_scores = False # Does not work with MBR

    def _compute_bleu(self, hyp_ngrams, ref_ngrams, hyp_length, ref_length):
        """Not the exact BLEU score, we do filter out multiple matches for
        the same ngram.
        """
        precisions = []
        for hyp_ngrams_order, ref_ngrams_order in zip(hyp_ngrams, ref_ngrams):
            if not hyp_ngrams_order:
                continue
            cnt = 0
            for hyp_ngram in hyp_ngrams_order:
                if hyp_ngram in ref_ngrams_order:
                    cnt += 1
            precisions.append(float(cnt) / float(len(hyp_ngrams_order)))
        weight = 1.0 / float(len(precisions))
        p = 1.0
        for precision in precisions:
            p *= precision ** weight
        bp = 1.0
        if hyp_length < ref_length:
            bp = np.exp(1.0 - float(ref_length) / float(hyp_length))
        return bp * p

    def _get_next_hypos_maxent(self, hypos, scores):
        """Get hypotheses of the next time step.
        
        Args:
            hypos (list): List of hypotheses
            scores (list): hypo scores with heuristic estimates
        
        Return:
            list. List with hypotheses.
        """
        # Update self.maxent_ngram_mass
        for hypo_score, hypo in zip(scores, hypos):
            s = hypo.trgt_sentence
            h = s[:-1]
            l = len(s)
            if l <= self.maxent_processed_length:
                continue
            # TODO: Could be more efficient by checking is_sublist for
            # all orders in one pass
            for order in range(min(len(s), self.max_order), 
                                self.min_order-1,
                                -1):
                ngram = s[-order:]
                # Do not use this ngram if it occurs before
                if is_sublist(ngram, h):
                    break # All lower order ngrams are too
                prev_mass = self.maxent_ngram_mass.get(ngram)
                if prev_mass is None:
                    updated_mass = hypo_score
                else:
                    updated_mass = max(prev_mass, hypo_score, 
                            np.log(np.exp(prev_mass)+np.exp(hypo_score)))
                self.maxent_ngram_mass.add(ngram, updated_mass)
        self.maxent_processed_length += 1
        exp_counts = []
        for hypo in hypos:
            s = hypo.trgt_sentence
            l = len(s)
            cnt = 0.0
            for order in range(self.min_order, self.max_order+1):
                for start in range(l-order+1):
                    logprob = self.maxent_ngram_mass.get(s[start:start+order])
                    # MaxEnt means that we estimate the probability of the 
                    # ngram as p + (1-p) * 0.5 ie.
                    if logprob:
                        cnt += 1.0 + np.exp(logprob)
                    else:
                        cnt += 1.0
            exp_counts.append(cnt * 0.5)
        next_hypos = []
        for idx in utils.argmax_n(exp_counts, self.beam_size):
            hypos[idx].bleu = exp_counts[idx]
            next_hypos.append(hypos[idx])
            logging.debug("Selected (score=%f expected_counts=%f): %s"
                % (scores[idx], hypos[idx].bleu, hypos[idx].trgt_sentence))
        return next_hypos

    def _get_next_hypos_renorm(self, hypos, scores):
        """Get hypotheses of the next time step.
        
        Args:
            hypos (list): List of hypotheses
            scores (list): hypo scores with heuristic estimates
        
        Return:
            list. List with hypotheses.
        """
        probs = (1.0 - self.smooth_factor) * np.exp(
            scores - utils.log_sum(scores)) \
            + self.smooth_factor / float(len(scores))
        lengths = [len(hypo.trgt_sentence) for hypo in hypos]
        logging.debug("%d candidates min_length=%d max_length=%d" % 
            (len(lengths), min(lengths), max(lengths)))
        ngrams = []
        for hypo in hypos:
            ngram_list = []
            for order in range(self.min_order, self.max_order+1):
                ngram_list.append(set([
                    " ".join(map(str, hypo.trgt_sentence[start:start+order]))
                    for start in range(len(hypo.trgt_sentence))]))
            ngrams.append(ngram_list)
        exp_bleus = []
        for hyp_ngrams, hyp_length in zip(ngrams, lengths):
            precisions = np.array([self._compute_bleu(
                    hyp_ngrams, ref_ngrams, hyp_length, ref_length)
                for ref_ngrams, ref_length in zip(ngrams, lengths)])
            exp_bleus.append(precisions * probs)
        next_hypos = []
        if self.selection_strategy == 'oracle_bleu': 
            for _ in range(min(self.beam_size, len(hypos))):
                idx = np.argmax(np.sum(exp_bleus, axis=1))
                bleu = np.sum(exp_bleus[idx])
                logging.debug("Selected (score=%f expected_bleu=%f): %s"
                        % (scores[idx], bleu, hypos[idx].trgt_sentence))
                hypos[idx].bleu = -bleu
                next_hypos.append(hypos[idx])
                gained_bleus = exp_bleus[idx]
                for update_idx in range(len(exp_bleus)):
                    exp_bleus[update_idx] = np.maximum(exp_bleus[update_idx], 
                                                       gained_bleus)
        else: # selection strategy 'bleu'
            total_exp_bleus = np.sum(exp_bleus, axis=1)
            for idx in utils.argmax_n(total_exp_bleus, self.beam_size):
                hypos[idx].bleu = total_exp_bleus[idx]
                next_hypos.append(hypos[idx])
                logging.debug("Selected (score=%f expected_bleu=%f): %s"
                    % (scores[idx], hypos[idx].bleu, hypos[idx].trgt_sentence))
        return next_hypos
    
    def decode(self, src_sentence):
        """Decodes a single source sentence using beam search. """
        self.initialize_predictors(src_sentence)
        hypos = self._get_initial_hypos()
        it = 0
        self.min_score = utils.NEG_INF
        self.maxent_ngram_mass = SimpleTrie()
        self.maxent_processed_length = 0
        while self.stop_criterion(hypos):
            if it > self.max_len: # prevent infinite loops
                break
            it = it + 1
            next_hypos = []
            next_scores = []
            for hypo in hypos:
                if hypo.get_last_word() == utils.EOS_ID:
                    next_hypos.append(hypo)
                    next_scores.append(self._get_combined_score(hypo))
                    continue 
                for next_hypo in self._expand_hypo(hypo):
                    next_score = self._get_combined_score(next_hypo)
                    if next_score > utils.NEG_INF:
                        next_hypos.append(next_hypo)
                        next_scores.append(next_score)
            hypos = self._get_next_hypos_mbr(next_hypos, next_scores)
        for hypo in hypos:
            hypo.score = hypo.bleu
            if hypo.get_last_word() == utils.EOS_ID:
                self.add_full_hypo(hypo.generate_full_hypothesis()) 
        if not self.full_hypos:
            logging.warn("No complete hypotheses found for %s" % src_sentence)
            for hypo in hypos:
                self.add_full_hypo(hypo.generate_full_hypothesis())
        return self.get_full_hypos_sorted() class LengthBeamDecoder(Decoder):
    """This beam decoder variant finds hypotheses for all lengths up
    to the maximum hypo length. At each time step, all EOS extensions
    are added to the results set.
    """

    def __init__(self, decoder_args):
        """Creates a new beam decoder instance. The following values
        are fetched from `decoder_args`:
        
            beam (int): Absolute beam size. A beam of 12 means
                        that we keep track of 12 active hypotheses
        """
        super(LengthBeamDecoder, self).__init__(decoder_args)
        self.beam_size = decoder_args.beam

    def _expand_hypo(self, hypo):
        self.set_predictor_states(copy.deepcopy(hypo.predictor_states))
        if not hypo.word_to_consume is None: # Consume if cheap expand
            self.consume(hypo.word_to_consume)
            hypo.word_to_consume = None
        posterior, score_breakdown = self.apply_predictors()
        hypo.predictor_states = self.get_predictor_states()
        top = utils.argmax_n(posterior, self.beam_size)
        # EOS hypo
        eos_hypo = hypo.cheap_expand(utils.EOS_ID,
                                     posterior[utils.EOS_ID],
                                     score_breakdown[utils.EOS_ID])
        self.add_full_hypo(eos_hypo.generate_full_hypothesis())
        # All other hypos
        return [hypo.cheap_expand(
                      trgt_word,
                      posterior[trgt_word],
                      score_breakdown[trgt_word]) for trgt_word in top 
                                                  if trgt_word != utils.EOS_ID]

    def decode(self, src_sentence):
        """Decodes a single source sentence using beam search. """
        self.initialize_predictors(src_sentence)
        hypos = [PartialHypothesis(self.get_predictor_states())]
        for _ in range(self.max_len):
            next_hypos = []
            for hypo in hypos:
                next_hypos.extend(self._expand_hypo(hypo))
            next_hypos.sort(key=lambda h: -h.score)
            hypos = next_hypos[:self.beam_size]
        return self.get_full_hypos_sorted() class SyncBeamDecoder(BeamDecoder):
    """This beam search implementation is a two level approach.
    Hypotheses are not compared after each iteration, but after
    consuming an explicit synchronization symbol. This is useful
    when SGNMT runs on the character level, but it makes more sense
    to compare hypos with same lengths in terms of number of words 
    and not characters. The end-of-word symbol </w> can be used as
    synchronization symbol.
    """
    
    def __init__(self, decoder_args):
        """Creates a new beam decoder instance with explicit
        synchronization symbol. In addition to the constructor of
        `BeamDecoder`, the following values are fetched from 
        `decoder_args`:
        
            sync_symb (int): Synchronization symbol. If negative,
                             consider a hypothesis as closed when it
                             ends with a terminal symbol (see 
                             syntax_min_terminal_id and
                             syntax_max_terminal_id)
            max_word_len (int): Maximum length of a single word
        """
        super(SyncBeamDecoder, self).__init__(decoder_args)
        if decoder_args.sync_symbol >= 0:
            self.sync_min = decoder_args.sync_symbol
            self.sync_max = decoder_args.sync_symbol
        else:
            self.sync_min = decoder_args.syntax_min_terminal_id
            self.sync_max = decoder_args.syntax_max_terminal_id
        self.max_word_len = decoder_args.max_word_len
    
    def _is_closed(self, hypo):
        """Returns true if hypo ends with </S> or </w>"""
        w = hypo.get_last_word()
        return w == utils.EOS_ID or (w >= self.sync_min and w <= self.sync_max)
    
    def _all_eos_or_eow(self, hypos):
        """Returns true if the all hypotheses end with </S> or </W>"""
        for hypo in hypos:
            if not self._is_closed(hypo):
                return True
        return False

    def _register_sub_score(self, score):
        """Updates best_scores and min_score. """
        if not self.maintain_best_scores:
            return
        self.sub_best_scores.append(score)
        self.sub_best_scores.sort(reverse=True)
        if len(self.sub_best_scores) >= self.beam_size:
            self.sub_best_scores = self.sub_best_scores[:self.beam_size]
            self.sub_min_score = self.sub_best_scores[-1] 
    
    def _expand_hypo(self, hypo):
        """Expand hypo until all of the beam size best hypotheses end 
        with ``sync_symb`` or EOS.
        
        Args:
            hypo (PartialHypothesis): Hypothesis to expand
        
        Return:
            list. List of expanded hypotheses.
        """
        # Get initial expansions
        next_hypos = []
        next_scores = []
        for next_hypo in super(SyncBeamDecoder, self)._expand_hypo(hypo):
            next_hypos.append(next_hypo)
            next_scores.append(self._get_combined_score(next_hypo))
        hypos = self._get_next_hypos(next_hypos, next_scores)
        # Expand until all hypos are closed
        it = 1
        while self._all_eos_or_eow(hypos):
            if it > self.max_word_len: # prevent infinite loops
                logging.debug("Maximum word length reached.")
                return []
            it = it + 1
            next_hypos = []
            next_scores = []
            self.sub_min_score = self.min_score
            self.sub_best_scores = []
            for h in hypos:
                if self._is_closed(h):
                    next_hypos.append(h)
                    next_scores.append(self._get_combined_score(h))
                    continue 
                if h.score > self.sub_min_score:
                    for next_hypo in super(SyncBeamDecoder, self)._expand_hypo(h):
                        next_score = self._get_combined_score(next_hypo)
                        if next_score > self.sub_min_score:
                            next_hypos.append(next_hypo)
                            next_scores.append(next_score)
                            self._register_sub_score(next_score)
            hypos = self._get_next_hypos(next_hypos, next_scores)
        ret =  [h for h in hypos if self._is_closed(h)]
        logging.debug("Expand %f: %s (%d)" % (hypo.score,
                                              hypo.trgt_sentence, 
                                              len(hypo.trgt_sentence)))
        for h in ret:
            logging.debug("-> %f: %s (%d)" % (h.score,
                                              h.trgt_sentence, 
                                              len(h.trgt_sentence)))
        return ret class FSTBeamDecoder(BeamDecoder):
    """This beam search variant synchronizes hypotheses if they share
    the same node in an FST. This is similar to the syncbeam strategy,
    but rather than using a dedicated synchronization symbol, we keep
    track of the state ID of each hypothesis in an FST. Hypotheses are
    expanded until all of them arrive at the same state id, and are
    then compared with each other to select the set of active 
    hypotheses in the next time step.
    """
    
    def __init__(self, decoder_args):
        """Creates a new beam decoder instance with FST-based
        synchronization. In addition to the constructor of
        `BeamDecoder`, the following values are fetched from 
        `decoder_args`:
        
            max_word_len (int): Maximum length of a single word
            fst_path (string): Path to the FST.
        """
        super(FSTBeamDecoder, self).__init__(decoder_args)
        self.fst_path = decoder_args.fst_path
        self.max_word_len = decoder_args.max_word_len
    
    def _register_sub_score(self, score):
        """Updates best_scores and min_score. """
        if not self.maintain_best_scores:
            return
        self.sub_best_scores.append(score)
        self.sub_best_scores.sort(reverse=True)
        if len(self.sub_best_scores) >= self.beam_size:
            self.sub_best_scores = self.sub_best_scores[:self.beam_size]
            self.sub_min_score = self.sub_best_scores[-1] 

    def _get_label2node(self, root_node):
        return {arc.olabel: arc.nextstate 
                for arc in self.cur_fst.arcs(root_node)}

    def _find_start_node(self):
        for arc in self.cur_fst.arcs(self.cur_fst.start()):
            if arc.olabel == utils.GO_ID:
                return arc.nextstate
        logging.error("Start symbol %d not found in fstbeam FST!" % utils.GO_ID)

    def _get_initial_hypos(self):
        """Get the list of initial ``PartialHypothesis``. """
        self.cur_fst = load_fst(utils.get_path(self.fst_path,
                                               self.current_sen_id+1))
        init_hypo = PartialHypothesis(self.get_predictor_states())
        init_hypo.fst_node = self._find_start_node()
        return [init_hypo]
    
    def _expand_hypo(self, hypo):
        """Expand hypo until all of the beam size best hypotheses end 
        with ``sync_symb`` or EOS.
        
        Args:
            hypo (PartialHypothesis): Hypothesis to expand
        
        Return:
            list. List of expanded hypotheses.
        """
        # Get initial expansions
        l2n = self._get_label2node(hypo.fst_node)
        deepest_node = -1
        next_hypos = []
        next_scores = []
        for next_hypo in super(FSTBeamDecoder, self)._expand_hypo(hypo):
            node_id = l2n[next_hypo.trgt_sentence[-1]]
            deepest_node = max(node_id, deepest_node)
            next_hypo.fst_node = node_id
            next_hypos.append(next_hypo)
            next_scores.append(self._get_combined_score(next_hypo))
        # Expand until all hypos are at deepest_node.
        # This assumes that the FST is topologically sorted
        open_hypos = []
        open_hypos_scores = []
        closed_hypos = []
        for next_hypo, next_score in zip(next_hypos, next_scores):
            if next_hypo.fst_node == deepest_node:
                closed_hypos.append(next_hypo)
            else:
                open_hypos.append(next_hypo)
                open_hypos_scores.append(next_score)
        open_hypos = self._get_next_hypos(open_hypos, open_hypos_scores)
        it = 1
        while open_hypos:
            if it > self.max_word_len: # prevent infinite loops
                logging.debug("Maximum word length reached.")
                break
            it = it + 1
            next_hypos = []
            next_scores = []
            self.sub_min_score = self.min_score
            self.sub_best_scores = []
            for h in open_hypos:
                if h.score > self.sub_min_score:
                    l2n = self._get_label2node(h.fst_node)
                    for next_hypo in super(FSTBeamDecoder, self)._expand_hypo(h):
                        next_score = self._get_combined_score(next_hypo)
                        if next_score > self.sub_min_score:
                            next_hypo.fst_node = l2n[next_hypo.trgt_sentence[-1]]
                            if next_hypo.fst_node < deepest_node: # Keep
                                next_hypos.append(next_hypo)
                                next_scores.append(next_score)
                                self._register_sub_score(next_score)
                            elif next_hypo.fst_node == deepest_node: # Add to closed
                                closed_hypos.append(next_hypo)
                            elif next_hypo.fst_node > deepest_node: # Log
                                logging.debug("FSTBeam: Deepest node exceeded")
            open_hypos = self._get_next_hypos(next_hypos, next_scores)
        logging.debug("Expand %f: %s (%d)" % (hypo.score,
                                              hypo.trgt_sentence, 
                                              hypo.fst_node))
        for h in closed_hypos:
            logging.debug("-> %f: %s (%d)" % (h.score,
                                              h.trgt_sentence, 
                                              h.fst_node))
        return closed_hypos class PredLimitBeamDecoder(BeamDecoder):
    """Beam search variant with explicit limits on the culmulative
    predictor scores at each node expansion.
    """
    
    def __init__(self, decoder_args):
        """Creates a new beam decoder with culmulative predictor score
        limits. In addition to the constructor of `BeamDecoder`, the 
        following values are fetched from `decoder_args`:
        
            pred_limits (string): Comma-separated list of predictor
                                  score limits.
        """
        super(PredLimitBeamDecoder, self).__init__(decoder_args)
        self.pred_limits = []
        for l in utils.split_comma(decoder_args.pred_limits):
          try:
              self.pred_limits.append(float(l))
          except:
              self.pred_limits.append(utils.NEG_INF)
        logging.info("Cumulative predictor score limits: %s" % self.pred_limits)
    
    def _expand_hypo(self, hypo):
        n_preds = len(self.pred_limits)
        all_hypos = super(PredLimitBeamDecoder, self)._expand_hypo(hypo)
        next_hypos = []
        all_accs = [0.0] * n_preds
        for hypo in sorted(all_hypos, key=lambda h: -h.score):
            valid = True
            for limit, acc, breakdown in zip(self.pred_limits,
                                             all_accs,
                                             hypo.score_breakdown[-1]):
                if acc + breakdown[0] < limit:
                    valid = False
                    break
            if valid:
                for i in range(n_preds):
                    all_accs[i] += hypo.score_breakdown[-1][i][0]
                next_hypos.append(hypo)
        return next_hypos class CombiBeamDecoder(BeamDecoder):
    """This beam search implementation is a modification to the hypo
    expansion strategy. Rather than selecting hypotheses based on
    the sum of the previous hypo scores and the current one, we
    apply combination_scheme in each time step. This makes it possible
    to use schemes like Bayesian combination on the word rather than
    the full sentence level.
    """
    
    def __init__(self, decoder_args):
        """Creates a new beam decoder instance. In addition to the 
        constructor of `BeamDecoder`, the following values are fetched 
        from `decoder_args`:
        
            combination_scheme (string): breakdown2score strategy
        """
        super(CombiBeamDecoder, self).__init__(decoder_args)
        # Whether to pass combination cached predictor weights
        self.breakdown2score_kwargs = {}
        if decoder_args.combination_scheme == 'length_norm':
            self.breakdown2score = combination.breakdown2score_length_norm
        if decoder_args.combination_scheme == 'bayesian_loglin':
            self.breakdown2score = combination.breakdown2score_bayesian_loglin
        if decoder_args.combination_scheme == 'bayesian_state_dependent':
            self.breakdown2score_kwargs['lambdas'] = self.get_domain_task_weights(
                decoder_args.bayesian_domain_task_weights)
            self.breakdown2score = combination.breakdown2score_bayesian_state_dependent
        if decoder_args.combination_scheme == 'bayesian':
            self.breakdown2score = combination.breakdown2score_bayesian
        if decoder_args.combination_scheme == 'sum':
            self.breakdown2score = combination.breakdown2score_sum
        if decoder_args.combination_scheme in ['sum', 'length_norm']:
            logging.warn("Using the %s combination strategy has no effect "
                         "under the combibeam decoder."
                         % decoder_args.combination_scheme)
        else:
            self.breakdown2score_kwargs['prev_score'] = None
        self.maintain_best_scores = False
        
    @staticmethod
    def get_domain_task_weights(w):
        """Get array of domain-task weights from string w
        Returns None if w is None or contains non-square number
                of weights (currently invalid)
                or 2D numpy float array of weights otherwise
        """
        if w is None:
            logging.critical(
                'Need bayesian_domain_task_weights for state-dependent BI')
        else:
            domain_weights = utils.split_comma(w, float)
            num_domains = int(len(domain_weights) ** 0.5)
            if len(domain_weights) == num_domains ** 2:
                weights_array = np.reshape(domain_weights,
                                           (num_domains, num_domains))
                logging.info('Using {} for Bayesian Interpolation'.format(
                    weights_array))
                return weights_array
            else:
                logging.critical(
                    'Need square number of domain-task weights, have {}'.format(
                        len(domain_weights)))

    def _get_initial_hypos(self):
        """Get list containing an initial CombiStatePartialHypothesis"""
        return [CombiStatePartialHypo(self.get_predictor_states())]


    def _expand_hypo(self, hypo):
        """Get the best beam size expansions of ``hypo``.
        
        Args:
            hypo (PartialHypothesis): Hypothesis to expand        
        Returns:
            list. List of child hypotheses
        """
        self.set_predictor_states(copy.deepcopy(hypo.predictor_states))
        if not hypo.word_to_consume is None: # Consume if cheap expand
            self.consume(hypo.word_to_consume)
            hypo.word_to_consume = None
        posterior, score_breakdown = self.apply_predictors()
        hypo.predictor_states = self.get_predictor_states()
        expanded_hypos = [hypo.cheap_expand(w, s, score_breakdown[w]) 
                          for w, s in utils.common_iterable(posterior)]
        for expanded_hypo in expanded_hypos:
            if 'prev_score' in self.breakdown2score_kwargs:
                self.breakdown2score_kwargs['prev_score'] = expanded_hypo.score_minus_last
            expanded_hypo.score = self.breakdown2score(
                expanded_hypo.score,
                expanded_hypo.score_breakdown,
                **self.breakdown2score_kwargs)
        expanded_hypos.sort(key=lambda x: -x.score)
        return expanded_hypos[:self.beam_size] class TextOutputHandler(OutputHandler):
    """Writes the first best hypotheses to a plain text file """
    
    def __init__(self, path):
        """Creates a plain text output handler to write to ``path`` """
        super(TextOutputHandler, self).__init__()
        self.path = path
        
    def write_hypos(self, all_hypos, sen_indices=None):
        """Writes the hypotheses in ``all_hypos`` to ``path`` """
        if self.f is not None:
            for hypos in all_hypos:
                self.f.write(io.decode(hypos[0].trgt_sentence))
                self.f.write("\n")
                self.f.flush()
        else:
            with codecs.open(self.path, "w", encoding='utf-8') as f:
                for hypos in all_hypos:
                    f.write(io.decode(hypos[0].trgt_sentence))
                    f.write("\n")
                    self.f.flush()

    def open_file(self):
        self.f = codecs.open(self.path, "w", encoding='utf-8')

    def close_file(self):
        self.f.close() class NBestOutputHandler(OutputHandler):
    """Produces a n-best file in Moses format. The third part of each 
    entry is used to store the separated unnormalized predictor scores.
    Note that the sentence IDs are shifted: Moses n-best files start 
    with the index 0, but in SGNMT and HiFST we usually refer to the 
    first sentence with 1 (e.g. in lattice directories or --range)
    """
    
    def __init__(self, path, predictor_names):
        """Creates a Moses n-best list output handler.
        
        Args:
            path (string):  Path to the n-best file to write
            predictor_names: Names of the predictors whose scores
                             should be included in the score breakdown
                             in the n-best list
        """
        super(NBestOutputHandler, self).__init__()
        self.path = path
        self.predictor_names = []
        name_count = {}
        for name in predictor_names:
            if not name in name_count:
                name_count[name] = 1
                final_name = name
            else:
                name_count[name] += 1
                final_name = "%s%d" % (name, name_count[name])
            self.predictor_names.append(final_name.replace("_", "0"))
        
    def write_hypos(self, all_hypos, sen_indices):
        """Writes the hypotheses in ``all_hypos`` to ``path`` """
        with codecs.open(self.path, "w", encoding='utf-8') as f:
            n_predictors = len(self.predictor_names)
            for idx, hypos in zip(sen_indices, all_hypos):
                for hypo in hypos:
                    f.write("%d ||| %s ||| %s ||| %f" %
                            (idx,
                             io.decode(hypo.trgt_sentence),
                             ' '.join("%s= %f" % (
                                  self.predictor_names[i],
                                  sum([s[i][0] for s in hypo.score_breakdown]))
                                      for i in range(n_predictors)),
                             hypo.total_score))
                    f.write("\n")
                idx += 1 class NgramOutputHandler(OutputHandler):
    """This output handler extracts MBR-style ngram posteriors from the 
    hypotheses returned by the decoder. The hypothesis scores are assumed to
    be loglikelihoods, which we renormalize to make sure that we operate on a
    valid distribution. The scores produced by the output handler are 
    probabilities of an ngram being in the translation.
    """
    
    def __init__(self, path, min_order, max_order):
        """Creates an ngram output handler.
        
        Args:
            path (string):  Path to the ngram directory to create
            min_order (int):  Minimum order of extracted ngrams
            max_order (int):  Maximum order of extracted ngrams
        """
        super(NgramOutputHandler, self).__init__()
        self.path = path
        self.min_order = min_order
        self.max_order = max_order
        self.file_pattern = path + "/%d.txt" 
      
    def write_hypos(self, all_hypos, sen_indices):
        """Writes ngram files for each sentence in ``all_hypos``.
        
        Args:
            all_hypos (list): list of nbest lists of hypotheses
            sen_indices (list): List of sentence indices (0-indexed)
        
        Raises:
            OSError. If the directory could not be created
            IOError. If something goes wrong while writing to the disk
        """
        _mkdir(self.path, "ngram")
        for sen_idx, hypos in zip(sen_indices, all_hypos):
            sen_idx += 1
            total = utils.log_sum([hypo.total_score for hypo in hypos])
            normed_scores = [hypo.total_score - total for hypo in hypos]
            ngrams = defaultdict(dict)
            # Collect ngrams
            for hypo_idx, hypo in enumerate(hypos):
                sen_eos = [utils.GO_ID] + hypo.trgt_sentence + [utils.EOS_ID]
                for pos in range(1, len(sen_eos) + 1):
                    hist = sen_eos[:pos]
                    for order in range(self.min_order, self.max_order + 1):
                        ngram = ' '.join(map(str, hist[-order:]))
                        ngrams[ngram][hypo_idx] = True
            with open(self.file_pattern % sen_idx, "w") as f:
                for ngram, hypo_indices in ngrams.items():
                    ngram_score = np.exp(utils.log_sum(
                       [normed_scores[hypo_idx] for hypo_idx in hypo_indices]))
                    f.write("%s : %f\n" % (ngram, min(1.0, ngram_score))) class TimeCSVOutputHandler(OutputHandler):
    """Produces one CSV file for each sentence. The CSV files contain
    the predictor score breakdown for each translation prefix length.
    """
    
    def __init__(self, path, predictor_names):
        """Creates a Moses n-best list output handler.
        
        Args:
            path (string):  Path to the n-best file to write
            predictor_names: Names of the predictors whose scores
                             should be included in the score breakdown
                             in the n-best list
        """
        super(TimeCSVOutputHandler, self).__init__()
        self.path = path
        self.file_pattern = path + "/%d.csv" 
        self.predictor_names = []
        name_count = {}
        for name in predictor_names:
            if not name in name_count:
                name_count[name] = 1
                final_name = name
            else:
                name_count[name] += 1
                final_name = "%s%d" % (name, name_count[name])
            self.predictor_names.append(final_name)
        
    def write_hypos(self, all_hypos, sen_indices):
        """Writes ngram files for each sentence in ``all_hypos``.
        
        Args:
            all_hypos (list): list of nbest lists of hypotheses
            sen_indices (list): List of sentence indices (0-indexed)
        
        Raises:
            OSError. If the directory could not be created
            IOError. If something goes wrong while writing to the disk
        """
        _mkdir(self.path, "TimeCSV")
        n_predictors = len(self.predictor_names)
        placeholder = "\t-" * (n_predictors*2)
        for sen_idx, hypos in zip(sen_indices, all_hypos):
            sen_idx += 1
            with open(self.file_pattern % sen_idx, "w") as f:
                hypo_count = len(hypos)
                # Headers
                f.write("Time")
                for i in range(hypo_count):
                    f.write("".join(["\t%s-%d" % (n, i+1) 
                                       for n in self.predictor_names]))
                    f.write("".join(["\t%s-%d_weight" % (n, i+1) 
                                       for n in self.predictor_names]))
                f.write("\n")
                max_len = max([len(hypo.trgt_sentence) for hypo in hypos])
                for pos in range(max_len+1):
                    f.write(str(pos))
                    for hypo in hypos:
                        if pos >= len(hypo.score_breakdown):
                            f.write(placeholder)
                        else:
                            for pred_idx in range(n_predictors):
                                acc_pred_score = sum([s[pred_idx][0] for s in hypo.score_breakdown[:pos+1]])
                                f.write("\t%f" % acc_pred_score)
                            for pred_idx in range(n_predictors):
                                f.write("\t%f" % hypo.score_breakdown[pos][pred_idx][1])
                    f.write("\n") class FSTOutputHandler(OutputHandler):
    """This output handler creates FSTs with with sparse tuple arcs 
    from the n-best lists from the decoder. The predictor scores are 
    kept separately in the sparse tuples. Note that this means that 
    the parameter ``--combination_scheme`` might not be visible in the 
    lattices because predictor scores are not combined. The order in 
    the sparse tuples corresponds to the order of the predictors in 
    the ``--predictors`` argument.
    
    Note that the created FSTs use another ID for UNK to avoid 
    confusion with the epsilon symbol used by OpenFST.
    """
    
    def __init__(self, path, unk_id):
        """Creates a sparse tuple FST output handler.
        
        Args:
            path (string):  Path to the VECLAT directory to create
            unk_id (int): Id which should be used in the FST for UNK
        """
        super(FSTOutputHandler, self).__init__()
        self.path = path
        self.unk_id = unk_id
        self.file_pattern = path + "/%d.fst" 
      
    def write_weight(self, score_breakdown):
        """Helper method to create the weight string """
        els = ['0']
        for (idx,score) in enumerate(score_breakdown):
            els.append(str(idx+1))
            # We need to take the negative here since the tropical
            # FST arc type expects negative log probs instead of log probs
            els.append(str(-score[0]))
        return ','.join(els)

    def write_hypos(self, all_hypos, sen_indices):
        """Writes FST files with sparse tuples for each sentence in 
        ``all_hypos``. The created lattices are not optimized in any
        way: We create a distinct path for each entry in 
        ``all_hypos``. We advise you to determinize/minimize them if 
        you are planning to use them for further processing.
        
        Args:
            all_hypos (list): list of nbest lists of hypotheses
            sen_indices (list): List of sentence indices (0-indexed)
        
        Raises:
            OSError. If the directory could not be created
            IOError. If something goes wrong while writing to the disk
        """
        _mkdir(self.path, "FST")
        for fst_idx, hypos in zip(sen_indices, all_hypos):
            fst_idx += 1
            c = fst.Compiler(arc_type="tropicalsparsetuple")
            # state ID 0 is start, 1 is final state
            next_free_id = 2
            for hypo in hypos:
                syms = hypo.trgt_sentence
                # Connect with start node
                c.write("0\t%d\t%d\t%d\n" % (next_free_id,
                                             utils.GO_ID,
                                             utils.GO_ID))
                next_free_id += 1
                for pos in range(len(hypo.score_breakdown)-1):
                    c.write("%d\t%d\t%d\t%d\t%s\n" % (
                            next_free_id-1, # last state id
                            next_free_id, # next state id 
                            syms[pos], syms[pos], # arc labels
                            self.write_weight(hypo.score_breakdown[pos])))
                    next_free_id += 1
                # Connect with final node
                c.write("%d\t1\t%d\t%d\t%s\n" % (
                                next_free_id-1,
                                utils.EOS_ID,
                                utils.EOS_ID,
                                self.write_weight(hypo.score_breakdown[-1])))
            c.write("1\n") # Add final node
            write_fst(c.compile(), self.file_pattern % fst_idx) class StandardFSTOutputHandler(OutputHandler):
    """This output handler creates FSTs with standard arcs. In contrast
    to ``FSTOutputHandler``, predictor scores are combined using 
    ``--combination_scheme``.
    
    Note that the created FSTs use another ID for UNK to avoid 
    confusion with the epsilon symbol used by OpenFST.
    """
    
    def __init__(self, path, unk_id):
        """Creates a standard arc FST output handler.
        
        Args:
            path (string):  Path to the fst directory to create
            unk_id (int): Id which should be used in the FST for UNK
        """
        super(StandardFSTOutputHandler, self).__init__()
        self.path = path
        self.unk_id = unk_id
        self.file_pattern = path + "/%d.fst" 
      
    def write_hypos(self, all_hypos, sen_indices):
        """Writes FST files with standard arcs for each
        sentence in ``all_hypos``. The created lattices are not 
        optimized in any way: We create a distinct path for each entry 
        in ``all_hypos``. We advise you to determinize/minimize them if
        you are planning to use them for further processing. 
        
        Args:
            all_hypos (list): list of nbest lists of hypotheses
            sen_indices (list): List of sentence indices (0-indexed)
        
        Raises:
            OSError. If the directory could not be created
            IOError. If something goes wrong while writing to the disk
        """
        _mkdir(self.path, "FST")
        for fst_idx, hypos in zip(sen_indices, all_hypos):
            fst_idx += 1
            c = fst.Compiler()
            # state ID 0 is start, 1 is final state
            next_free_id = 2
            for hypo in hypos:
                # Connect with start node
                c.write("0\t%d\t%d\t%d\t%f\n" % (next_free_id,
                                                 utils.GO_ID,
                                                 utils.GO_ID,
                                                 -hypo.total_score))
                next_free_id += 1
                for sym in hypo.trgt_sentence:
                    c.write("%d\t%d\t%d\t%d\n" % (next_free_id-1,
                                                  next_free_id,
                                                  sym, sym))
                    next_free_id += 1
                # Connect with final node
                c.write("%d\t1\t%d\t%d\n" % (next_free_id-1,
                                             utils.EOS_ID,
                                             utils.EOS_ID))
            c.write("1\n")
            write_fst(c.compile(), self.file_pattern % fst_idx) class FstPredictor(Predictor):
    """This predictor can read determinized translation lattices. The
    predictor state consists of the current node. This is unique as the
    lattices are determinized.
    """
    
    def __init__(self,
                 fst_path,
                 use_weights,
                 normalize_scores,
                 skip_bos_weight = True,
                 to_log = True):
        """Creates a new fst predictor.
        
        Args:
            fst_path (string): Path to the FST file
            use_weights (bool): If false, replace all arc weights with
                                0 (=log 1).
            normalize_scores (bool): If true, we normalize the weights
                                     on all outgoing arcs such that
                                     they sum up to 1
            skip_bos_weight (bool): Add the score at the <S> arc to the
                                    </S> arc if this is false. This results
                                    in scores consistent with 
                                    OpenFST's replace operation,
                                    as <S> scores are normally
                                    ignored by SGNMT.
            to_log (bool): SGNMT uses normal log probs (scores) while
                           arc weights in FSTs normally have cost (i.e.
                           neg. log values) semantics. Therefore, if
                           true, we multiply arc weights by -1.
        """
        super(FstPredictor, self).__init__()
        self.fst_path = fst_path
        self.weight_factor = -1.0 if to_log else 1.0
        self.use_weights = use_weights
        self.normalize_scores = normalize_scores
        self.cur_fst = None
        self.add_bos_to_eos_score = not skip_bos_weight
        self.cur_node = -1
        
    def get_unk_probability(self, posterior):
        """Returns negative infinity if UNK is not in the lattice.
        Otherwise, return UNK score.
        
        Returns:
            float. Negative infinity
        """
        return utils.common_get(posterior, utils.UNK_ID, utils.NEG_INF)
    
    def predict_next(self):
        """Uses the outgoing arcs from the current node to build up the
        scores for the next word.
        
        Returns:
            dict. Set of words on outgoing arcs from the current node
            together with their scores, or an empty set if we currently
            have no active node or fst.
        """
        if self.cur_node < 0:
            return {}
        scores = {arc.olabel: self.weight_factor*w2f(arc.weight)
                for arc in self.cur_fst.arcs(self.cur_node)}
        if utils.EOS_ID in scores and self.add_bos_to_eos_score:
            scores[utils.EOS_ID] += self.bos_score
        return self.finalize_posterior(scores,
                self.use_weights, self.normalize_scores)
    
    def initialize(self, src_sentence):
        """Loads the FST from the file system and consumes the start
        of sentence symbol. 
        
        Args:
            src_sentence (list):  Not used
        """
        self.cur_fst = load_fst(utils.get_path(self.fst_path,
                                               self.current_sen_id+1))
        self.cur_node = self.cur_fst.start() if self.cur_fst else None
        self.bos_score = self.consume(utils.GO_ID)
        if not self.bos_score: # Override None
            self.bos_score = 0.0
        if self.cur_node is None:
            logging.warn("The lattice for sentence %d does not contain any "
                         "valid path. Please double-check that the lattice "
                         "is not empty and that paths contain the begin-of-"
                         "sentence symbol %d. If you are using a different "
                         "begin-of-sentence symbol, double-check --indexing_"
                         "scheme." % (self.current_sen_id+1, utils.GO_ID))
    
    def consume(self, word):
        """Updates the current node by following the arc labelled with
        ``word``. If there is no such arc, we set ``cur_node`` to -1,
        indicating that the predictor is in an invalid state. In this
        case, all subsequent ``predict_next`` calls will return the
        empty set.
        
        Args:
            word (int): Word on an outgoing arc from the current node
        
        Returns:
            float. Weight on the traversed arc
        """
        if self.cur_node < 0:
            return
        from_state = self.cur_node
        self.cur_node = None
        unk_arc = None
        for arc in self.cur_fst.arcs(from_state):
            if arc.olabel == word:
                self.cur_node = arc.nextstate
                return self.weight_factor*w2f(arc.weight)
            elif arc.olabel == utils.UNK_ID:
                unk_arc = arc
        if unk_arc is not None:
            self.cur_node = unk_arc.nextstate
    
    def get_state(self):
        """Returns the current node. """
        return self.cur_node
    
    def set_state(self, state):
        """Sets the current node. """
        self.cur_node = state

    def initialize_heuristic(self, src_sentence):
        """Creates a matrix of shortest distances between nodes. """
        self.distances = fst.shortestdistance(self.cur_fst, reverse=True)
    
    def estimate_future_cost(self, hypo):
        """The FST predictor comes with its own heuristic function. We
        use the shortest path in the fst as future cost estimator. """
        if not self.cur_node:
            return 0.0
        last_word = hypo.trgt_sentence[-1]
        for arc in self.cur_fst.arcs(self.cur_node):
            if arc.olabel == last_word:
                return w2f(self.distances[arc.nextstate])
        return 0.0
    
    def is_equal(self, state1, state2):
        """Returns true if the current node is the same """
        return state1 == state2 class RtnPredictor(Predictor):
    """Predictor for RTNs (recurrent transition networks). This 
    predictor assumes a directory structure as produced by HiFST. You 
    can use this predictor for non-deterministic lattices too. This
    implementation supports late expansion: RTNs are only expanded as
    far as necessary to retrieve all currently reachable states.
    
    ``cur_nodes`` contains the accumulated weights from the last 
    consumed word (if ambiguous, the largest)
    
    This implementation does not maintain a list of active nodes like 
    the other automata predictors. Instead, we store the current 
    history and search for the active nodes at each expansion. This is
    more expensive, but fstreplace might change state IDs so a list of
    active nodes might get corrupted.
    
    Note that this predictor does not support FSTs in gzip format.
    """
    
    def __init__(self,
                 rtn_path,
                 use_weights,
                 normalize_scores,
                 to_log = True,
                 minimize_rtns = False,
                 rmeps = True):
        """Creates a new RTN predictor.
        
        Args:
            rtn_path (string): Path to the RTN directory
            use_weights (bool): If false, replace all arc weights with
                                0 (=log 1).
            normalize_scores (bool): If true, we normalize the weights
                                     on all outgoing arcs such that
                                     they sum up to 1
            to_log (bool): SGNMT uses normal log probs (scores) while
                           arc weights in FSTs normally have cost (i.e.
                           neg. log values) semantics. Therefore, if
                           true, we multiply arc weights by -1.
            minimize_rtns (bool): Minimize the FST after each replace
                                  operation
            rmeps (bool): Remove epsilons in the FST after each replace
                          operation 
        """
        super(RtnPredictor, self).__init__()
        self.root_path = rtn_path
        self.minimize_rtns = minimize_rtns
        self.rmeps = rmeps
        self.use_weights = use_weights
        self.normalize_scores = normalize_scores
        self.weight_factor = -1.0 if to_log else 1.0
        self.cur_fst = None # current root fst
        start_id = '1'
        try:
            with open("%s/ntmap" % self.root_path) as f:
                ntmap = dict(line.strip().split(None, 1) for line in f)
                start_id = ntmap['S']
        except:
            logging.warn("Could not find NT S in ntmap. Assuming its ID 1")
        self.root_fst_prefix = "1%s000" % start_id.zfill(3)
        
    def get_unk_probability(self, posterior):
        """Always returns negative infinity: Words outside the 
        RTN are not possible according to this predictor.
        
        Returns:
            float. Negative infinity
        """
        return utils.NEG_INF
    
    def initialize(self, src_sentence):
        """Loads the root RTN and consumes the start of sentence 
        symbol.
        
        Args:
            src_sentence (list):  Not used
        """
        try:
            file_name = "%s/%d.fst" % (self.root_path, self.current_sen_id+1)
            if not os.access(file_name, os.R_OK): # Find root FST
                search_pattern = '%s/%d/%s*.fst' % (self.root_path,
                                                    self.current_sen_id+1,
                                                    self.root_fst_prefix)
                candidates = glob.glob(search_pattern)
                if not candidates:
                    logging.error("Could not find root fst in %s" % 
                                    search_pattern)
                    self.cur_fst = None
                    return
                if len(candidates) > 1:
                    logging.warn("Ambiguous root fst for %s. Take the one "
                                 "with largest span." % search_pattern)
                    candidates = sorted(candidates)
                file_name = candidates[-1]
            self.cur_fst = fst.Fst.read(file_name) 
            logging.debug("Read (root)fst from %s" % file_name)
        except Exception as e:
            logging.error("%s error reading fst from %s: %s" %
                (sys.exc_info()[1], file_name, e))
            self.cur_fst = None
        finally:
            self.cur_history = []
            self.sub_fsts = {}
        self.consume(utils.GO_ID)
    
    def expand_rtn(self, func):
        """This method expands the RTN as far as necessary. This means
        that the RTN is expanded s.t. we can build the posterior for 
        ``cur_history``. In practice, this means that we follow all 
        epsilon edges and replaces all NT edges until all paths with 
        the prefix ``cur_history`` in the RTN have at least one more 
        terminal token. Then, we apply ``func`` to all reachable nodes.
        """
        updated = True
        while updated:
            updated = False
            label_fst_map = {}
            self.visited_nodes = {}
            self.cur_fst.arcsort(sort_type="olabel")
            self.add_to_label_fst_map_recursive(label_fst_map,
                                                {},
                                                self.cur_fst.start(), 
                                                0.0,
                                                self.cur_history, func)
            if label_fst_map:
                logging.debug("Replace %d NT arcs for history %s" % (
                                                            len(label_fst_map),
                                                            self.cur_history))
                # First in the list is the root FST and label
                replaced_fst = fst.replace(
                        [(len(label_fst_map) + 2000000000, self.cur_fst)] 
                        + [(nt_label, f) 
                            for (nt_label, f) in label_fst_map.items()],
                        epsilon_on_replace=True)
                self.cur_fst = replaced_fst
                updated = True
        if self.rmeps or self.minimize_rtns:
            self.cur_fst.rmepsilon()
        if self.minimize_rtns:
            tmp = fst.determinize(self.cur_fst.determinize)
            self.cur_fst = tmp
            self.cur_fst.minimize()
    
    def add_to_label_fst_map_recursive(self, 
                                       label_fst_map, 
                                       visited_nodes, 
                                       root_node, 
                                       acc_weight, 
                                       history, 
                                       func):
        """Adds arcs to ``label_fst_map`` if they are labeled with an
        NT symbol and reachable from ``root_node`` via ``history``.
          
        Note: visited_nodes is maintained for each history separately
        """
        if root_node in visited_nodes:
            # This introduces some error as we take the score of the first best
            # path with a certain history, not the globally best path. For now,
            # this error should not be significant
            return
        visited_nodes[root_node] = True
        for arc in self.cur_fst.arcs(root_node):
            arc_acc_weight = acc_weight + self.weight_factor*w2f(arc.weight)
            if arc.olabel == EPS_ID: # Follow epsilon edges
                self.add_to_label_fst_map_recursive(label_fst_map,
                                                    visited_nodes,
                                                    arc.nextstate,
                                                    arc_acc_weight, 
                                                    history,
                                                    func)
            elif not history:
                if self.is_nt_label(arc.olabel): # Add to label_fst_map
                    replace_label = len(label_fst_map) + 2000000000
                    label_fst_map[replace_label] = self.get_sub_fst(
                                                                    arc.olabel)
                    arc.ilabel = replace_label
                    arc.olabel = replace_label
                else: # This is a regular arc and we have no history left
                    func(arc.nextstate, arc.olabel, arc_acc_weight) # apply func
            elif arc.olabel == history[0]: # history is not empty
                self.add_to_label_fst_map_recursive(label_fst_map,
                                                    {},
                                                    arc.nextstate,
                                                    arc_acc_weight,
                                                    history[1:],
                                                    func)
            elif arc.olabel > history[0]: # FST is arc sorted, we can stop here
                break
        
    
    def is_nt_label(self, label):
        """Returns true if ``label`` is a non-terminal. """
        s = str(label)
        return len(s) == 10 and s[0] == '1'

    def get_sub_fst(self, fst_id):
        """ Load sub fst from the file system or the cache """
        if fst_id in self.sub_fsts:
            return self.sub_fsts[fst_id]
        sub_fst_path = "%s/%d/%d.fst" %  (self.root_path,
                                          self.current_sen_id+1,
                                          fst_id)
        try:
            sub_fst = fst.Fst.read(sub_fst_path)
            logging.debug("Read sub fst from %s" % sub_fst_path)
            self.sub_fsts[fst_id] = sub_fst
            return sub_fst
        except Exception as e:
            logging.error("%s error reading sub fst from %s: %s" %
                (sys.exc_info()[1], sub_fst_path, e))
        
    def _add_to_cur_posterior(self, node, label, weight):
        """Can be used as ``func`` argument in ``expand_rtn`` to build
        up the posterior for the next target token  in ``predict_next``
        """
        self.cur_posterior[label] = max(self.cur_posterior.get(label, utils.NEG_INF),
                                        weight)
    
    def predict_next(self):
        """Expands RTN as far as possible and uses the outgoing edges 
        from nodes reachable by the current history to build up
        the posterior for the next word. If there are no such nodes
        or arcs, or no root FST is loaded, return the empty set.
        """
        if not self.cur_fst:
            return {}
        self.cur_posterior = {}
        self.expand_rtn(self._add_to_cur_posterior)
        return self.finalize_posterior(self.cur_posterior,
                                       self.use_weights,
                                       self.normalize_scores)
    
    def consume(self, word):
        """Adds ``word`` to the current history. """
        self.cur_history.append(word)
    
    def get_state(self):
        """Returns the current history. """
        return self.cur_history
    
    def set_state(self, state):
        """Sets the current history. """
        self.cur_history = state class NondeterministicFstPredictor(Predictor):
    """This predictor can handle non-deterministic translation 
    lattices. In contrast to the fst predictor for deterministic
    lattices, we store a set of nodes which are all reachable from
    the start node through the current history.
    """
    
    def __init__(self, 
                 fst_path, 
                 use_weights, 
                 normalize_scores, 
                 skip_bos_weight = True, 
                 to_log = True):
        """Creates a new nfst predictor.
        
        Args:
            fst_path (string): Path to the FST file
            use_weights (bool): If false, replace all arc weights with
                                0 (=log 1).
            normalize_scores (bool): If true, we normalize the weights
                                     on all outgoing arcs such that
                                     they sum up to 1
            skip_bos_weight (bool): If true, set weights on <S> arcs
                                    to 0 (= log1)
            to_log (bool): SGNMT uses normal log probs (scores) while
                           arc weights in FSTs normally have cost (i.e.
                           neg. log values) semantics. Therefore, if
                           true, we multiply arc weights by -1.
        """
        super(NondeterministicFstPredictor, self).__init__()
        self.fst_path = fst_path
        self.weight_factor = -1.0 if to_log else 1.0
        self.score_max_func = max if to_log else min
        self.use_weights = use_weights
        self.skip_bos_weight = skip_bos_weight
        self.normalize_scores = normalize_scores
        self.cur_fst = None
        self.cur_nodes = []
        
    def get_unk_probability(self, posterior):
        """Always returns negative infinity: Words outside the 
        translation lattice are not possible according to this
        predictor.
        
        Returns:
            float. Negative infinity
        """
        return utils.NEG_INF 
    
    def predict_next(self):
        """Uses the outgoing arcs from all current node to build up the
        scores for the next word. This method does not follow epsilon
        arcs: ``consume`` updates ``cur_nodes`` such that all reachable
        arcs with word ids are connected directly with a node in
        ``cur_nodes``. If there are multiple arcs with the same word,
        we use the log sum of the arc weights as score.
        
        Returns:
            dict. Set of words on outgoing arcs from the current node
            together with their scores, or an empty set if we currently
            have no active nodes or fst.
        """
        scores = {}
        for weight,node in self.cur_nodes:
            for arc in self.cur_fst.arcs(node): 
                if arc.olabel != EPS_ID:
                    score = weight + self.weight_factor*w2f(arc.weight) 
                    if arc.olabel in scores:
                        scores[arc.olabel] = self.score_max_func(
                                        scores[arc.olabel], score)
                    else:
                        scores[arc.olabel] = score 
        return self.finalize_posterior(scores,
                self.use_weights, self.normalize_scores)
    
    def initialize(self, src_sentence):
        """Loads the FST from the file system and consumes the start
        of sentence symbol. 
        
        Args:
            src_sentence (list):  Not used
        """
        self.cur_fst = load_fst(utils.get_path(self.fst_path,
                                               self.current_sen_id+1))
        self.cur_nodes = []
        if self.cur_fst:
            self.cur_nodes = self._follow_eps({self.cur_fst.start(): 0.0})
        self.consume(utils.GO_ID)
        if not self.cur_nodes:
            logging.warn("The lattice for sentence %d does not contain any "
                         "valid path. Please double-check that the lattice "
                         "is not empty and that paths start with the begin-of-"
                         "sentence symbol." % (self.current_sen_id+1))
    
    def consume(self, word):
        """Updates the current nodes by searching for all nodes which
        are reachable from the current nodes by a path consisting of 
        any number of epsilons and exactly one ``word`` label. If there
        is no such arc, we set the predictor in an invalid state. In 
        this case, all subsequent ``predict_next`` calls will return 
        the empty set.
        
        Args:
            word (int): Word on an outgoing arc from the current node
        """
        d_unconsumed = {}
        # Collect distances to nodes reachable by word
        for weight,node in self.cur_nodes:
            for arc in self.cur_fst.arcs(node):
                if arc.olabel == word:
                    next_node = arc.nextstate
                    next_score = weight + self.weight_factor*w2f(arc.weight)
                    if d_unconsumed.get(next_node, utils.NEG_INF) < next_score:
                        d_unconsumed[next_node] = next_score
        # Subtract the word score from the last predict_next 
        consumed_score = self.score_max_func(d_unconsumed.values()) \
             if (word != utils.GO_ID or self.skip_bos_weight) else 0.0
        # Add epsilon reachable states
        self.cur_nodes = self._follow_eps({node: score - consumed_score
                    for node,score in d_unconsumed.items()})
    
    def _follow_eps(self, roots):
        """BFS to find nodes reachable from root through eps arcs. This
        traversal strategy is efficient if the triangle inquality holds 
        for weights in the graphs, i.e. for all vertices v1,v2,v3: 
        (v1,v2),(v2,v3),(v1,v3) in E => d(v1,v2)+d(v2,v3) >= d(v1,v3).
        The method still returns the correct results if the triangle
        inequality does not hold, but edges may be traversed multiple
        times which makes it more inefficient.
        """
        open_nodes = dict(roots)
        d = {}
        visited = dict(roots)
        while open_nodes:
            next_open = {}
            for node,score in open_nodes.items():
                has_noneps = False
                for arc in self.cur_fst.arcs(node):
                    if arc.olabel == EPS_ID:
                        next_node = arc.nextstate
                        next_score = score + self.weight_factor*w2f(arc.weight)
                        if visited.get(next_node, utils.NEG_INF) < next_score:
                            visited[next_node] = next_score
                            next_open[next_node] = next_score
                    else:
                        has_noneps = True
                if has_noneps:
                    d[node] = score
            open_nodes = next_open
        return [(weight, node) for node, weight in d.items()]
        
    def get_state(self):
        """Returns the set of current nodes """
        return self.cur_nodes
    
    def set_state(self, state):
        """Sets the set of current nodes """
        self.cur_nodes = state

    def initialize_heuristic(self, src_sentence):
        """Creates a matrix of shortest distances between all nodes """
        self.distances = fst.shortestdistance(self.cur_fst, reverse=True)
    
    def estimate_future_cost(self, hypo):
        """The FST predictor comes with its own heuristic function. We
        use the shortest path in the fst as future cost estimator. """
        last_word = hypo.trgt_sentence[-1]
        dists = []
        for n in self.cur_nodes:
            for arc in self.cur_fst[n].arcs:
                if arc.olabel == last_word:
                    dists.append(w2f(self.distances[arc.nextstate]))
                    break
        return 0.0 if not dists else min(dists)
    
    def is_equal(self, state1, state2):
        """Returns true if the current nodes are the same """
        return sorted([n for _,n in state1]) == sorted([n for _,n in state2]) class BagOfWordsPredictor(Predictor):
    """This predictor is similar to the forced predictor, but it does
    not enforce the word order in the reference. Therefore, it assigns
    1 to all hypotheses which have the words in the reference in any 
    order, and -inf to all other hypos.
    """
    
    def __init__(self, 
                 trg_test_file, 
                 accept_subsets=False,
                 accept_duplicates=False,
                 heuristic_scores_file="",
                 collect_stats_strategy='best',
                 heuristic_add_consumed = False, 
                 heuristic_add_remaining = True,
                 diversity_heuristic_factor = -1.0,
                 equivalence_vocab=-1):
        """Creates a new bag-of-words predictor.
        
        Args:
            trg_test_file (string): Path to the plain text file with 
                                    the target sentences. Must have the
                                    same number of lines as the number
                                    of source sentences to decode. The 
                                    word order in the target sentences
                                    is not relevant for this predictor.
            accept_subsets (bool): If true, this predictor permits
                                       EOS even if the bag is not fully
                                       consumed yet
            accept_duplicates (bool): If true, counts are not updated
                                      when a word is consumed. This
                                      means that we allow a word in a
                                      bag to appear multiple times
            heuristic_scores_file (string): Path to the unigram scores 
                                            which are used if this 
                                            predictor estimates future
                                            costs
            collect_stats_strategy (string): best, full, or all. Defines 
                                             how unigram estimates are 
                                             collected for heuristic 
            heuristic_add_consumed (bool): Set to true to add the 
                                           difference between actual
                                           partial score and unigram
                                           estimates of consumed words
                                           to the predictor heuristic
            heuristic_add_remaining (bool): Set to true to add the sum
                                            of unigram scores of words
                                            remaining in the bag to the
                                            predictor heuristic
            diversity_heuristic_factor (float): Factor for diversity
                                                heuristic which 
                                                penalizes hypotheses
                                                with the same bag as
                                                full hypos
            equivalence_vocab (int): If positive, predictor states are
                                     considered equal if the the 
                                     remaining words within that vocab
                                     and OOVs regarding this vocab are
                                     the same. Only relevant when using
                                     hypothesis recombination
        """
        super(BagOfWordsPredictor, self).__init__()
        with open(trg_test_file) as f:
            self.lines = f.read().splitlines()
        if heuristic_scores_file:
            self.estimates = FileUnigramTable(heuristic_scores_file)
        elif collect_stats_strategy == 'best':
            self.estimates = BestStatsUnigramTable()
        elif collect_stats_strategy == 'full':
            self.estimates = FullStatsUnigramTable()
        elif collect_stats_strategy == 'all':
            self.estimates = AllStatsUnigramTable()
        else:
            logging.error("Unknown statistics collection strategy")
        self.accept_subsets = accept_subsets
        self.accept_duplicates = accept_duplicates
        self.heuristic_add_consumed = heuristic_add_consumed
        self.heuristic_add_remaining = heuristic_add_remaining
        self.equivalence_vocab = equivalence_vocab
        if accept_duplicates and not accept_subsets:
            logging.error("You enabled bow_accept_duplicates but not bow_"
                          "accept_subsets. Therefore, the bow predictor will "
                          "never accept end-of-sentence and could cause "
                          "an infinite loop in the search strategy.")
        self.diversity_heuristic_factor = diversity_heuristic_factor
        self.diverse_heuristic = (diversity_heuristic_factor > 0.0)
          
    def get_unk_probability(self, posterior):
        """Returns negative infinity unconditionally: Words which are
        not in the target sentence have assigned probability 0 by
        this predictor.
        """
        return NEG_INF
    
    def predict_next(self):
        """If the bag is empty, the only allowed symbol is EOS. 
        Otherwise, return the list of keys in the bag.
        """
        if not self.bag: # Empty bag
            return {utils.EOS_ID : 0.0}
        ret = {w : 0.0 for w in self.bag}
        if self.accept_subsets:
            ret[utils.EOS_ID] = 0.0
        return ret
    
    def initialize(self, src_sentence):
        """Creates a new bag for the current target sentence..
        
        Args:
            src_sentence (list):  Not used
        """
        self.best_hypo_score = NEG_INF
        self.bag = {}
        for w in self.lines[self.current_sen_id].strip().split(): 
            int_w = int(w)
            self.bag[int_w] = self.bag.get(int_w, 0) + 1
        self.full_bag = dict(self.bag)
        
    def consume(self, word):
        """Updates the bag by deleting the consumed word.
        
        Args:
            word (int): Next word to consume
        """
        if word == utils.EOS_ID:
            self.bag = {}
            return
        if not word in self.bag:
            logging.warn("Consuming word which is not in bag-of-words!")
            return
        cnt = self.bag.pop(word)
        if cnt > 1 and not self.accept_duplicates:
            self.bag[word] = cnt - 1
    
    def get_state(self):
        """State of this predictor is the current bag """
        return self.bag
    
    def set_state(self, state):
        """State of this predictor is the current bag """
        self.bag = state

    def initialize_heuristic(self, src_sentence):
        """Calls ``reset`` of the used unigram table with estimates
        ``self.estimates`` to clear all statistics from the previous
        sentence
        
        Args:
            src_sentence (list): Not used
        """
        self.estimates.reset()
        if self.diverse_heuristic:
            self.explored_bags = SimpleTrie()
    
    def notify(self, message, message_type = MESSAGE_TYPE_DEFAULT):
        """This gets called if this predictor observes the decoder. It
        updates unigram heuristic estimates via passing through this
        message to the unigram table ``self.estimates``.
        """
        self.estimates.notify(message, message_type)
        if self.diverse_heuristic and message_type == MESSAGE_TYPE_FULL_HYPO:
            self._update_explored_bags(message)
    
    def _update_explored_bags(self, hypo):
        """This is called if diversity heuristic is enabled. It updates
        ``self.explored_bags``
        """
        sen = hypo.trgt_sentence
        for l in range(len(sen)):
            key = sen[:l]
            key.sort()
            cnt = self.explored_bags.get(key)
            if not cnt:
                cnt = 0.0
            self.explored_bags.add(key, cnt + 1.0)
                    
    def estimate_future_cost(self, hypo):
        """The bow predictor comes with its own heuristic function. We
        use the sum of scores of the remaining words as future cost 
        estimator. 
        """
        acc = 0.0
        if self.heuristic_add_remaining:
            remaining = dict(self.full_bag)
            remaining[utils.EOS_ID] = 1
            for w in hypo.trgt_sentence:
                remaining[w] -= 1
            acc -= sum([cnt*self.estimates.estimate(w) 
                            for w,cnt in remaining.items()])
        if self.diverse_heuristic:
            key = list(hypo.trgt_sentence)
            key.sort()
            cnt = self.explored_bags.get(key)
            if cnt:
                acc += cnt * self.diversity_heuristic_factor
        if self.heuristic_add_consumed:
            acc -= hypo.score - sum([self.estimates.estimate(w, -1000.0)
                            for w in hypo.trgt_sentence])
        return acc
    
    def _get_unk_bag(self, org_bag):
        if self.equivalence_vocab <= 0:
            return org_bag
        unk_bag = {}
        for word,cnt in org_bag.items():
            idx = word if word < self.equivalence_vocab else utils.UNK_ID
            unk_bag[idx] = unk_bag.get(idx, 0) + cnt
        return unk_bag
    
    def is_equal(self, state1, state2):
        """Returns true if the bag is the same """
        return self._get_unk_bag(state1) == self._get_unk_bag(state2)  class BagOfWordsSearchPredictor(BagOfWordsPredictor):
    """Combines the bag-of-words predictor with a proxy decoding pass
    which creates a skeleton translation.
    """
    
    def __init__(self,
                 main_decoder,
                 hypo_recombination,
                 trg_test_file, 
                 accept_subsets=False,
                 accept_duplicates=False,
                 heuristic_scores_file="",
                 collect_stats_strategy='best',
                 heuristic_add_consumed = False, 
                 heuristic_add_remaining = True,
                 diversity_heuristic_factor = -1.0,
                 equivalence_vocab=-1):
        """Creates a new bag-of-words predictor with pre search
        
        Args:
            main_decoder (Decoder): Reference to the main decoder
                                    instance, used to fetch the predictors
            hypo_recombination (bool): Activates hypo recombination for the
                                       pre decoder 
            trg_test_file (string): Path to the plain text file with 
                                    the target sentences. Must have the
                                    same number of lines as the number
                                    of source sentences to decode. The 
                                    word order in the target sentences
                                    is not relevant for this predictor.
            accept_subsets (bool): If true, this predictor permits
                                       EOS even if the bag is not fully
                                       consumed yet
            accept_duplicates (bool): If true, counts are not updated
                                      when a word is consumed. This
                                      means that we allow a word in a
                                      bag to appear multiple times
            heuristic_scores_file (string): Path to the unigram scores 
                                            which are used if this 
                                            predictor estimates future
                                            costs
            collect_stats_strategy (string): best, full, or all. Defines 
                                             how unigram estimates are 
                                             collected for heuristic 
            heuristic_add_consumed (bool): Set to true to add the 
                                           difference between actual
                                           partial score and unigram
                                           estimates of consumed words
                                           to the predictor heuristic
            heuristic_add_remaining (bool): Set to true to add the sum
                                            of unigram scores of words
                                            remaining in the bag to the
                                            predictor heuristic
            equivalence_vocab (int): If positive, predictor states are
                                     considered equal if the the 
                                     remaining words within that vocab
                                     and OOVs regarding this vocab are
                                     the same. Only relevant when using
                                     hypothesis recombination
        """
        self.main_decoder = main_decoder
        self.pre_decoder = BeamDecoder(CLOSED_VOCAB_SCORE_NORM_NONE,
                                       main_decoder.max_len_factor,
                                       hypo_recombination,
                                       10)
        self.pre_decoder.combine_posteriors = main_decoder.combine_posteriors 
        super(BagOfWordsSearchPredictor, self).__init__(trg_test_file, 
                                                        accept_subsets,
                                                        accept_duplicates,
                                                        heuristic_scores_file,
                                                        collect_stats_strategy,
                                                        heuristic_add_consumed, 
                                                        heuristic_add_remaining,
                                                        diversity_heuristic_factor,
                                                        equivalence_vocab)
        self.pre_mode = False
    
    def predict_next(self):
        """If in ``pre_mode``, pass through to super class. Otherwise,
        scan skeleton 
        """
        if self.pre_mode:
            return super(BagOfWordsSearchPredictor, self).predict_next()
        if not self.bag: # Empty bag
            return {utils.EOS_ID : 0.0}
        ret = {w : 0.0 for w in self.missing}
        if self.accept_subsets:
            ret[utils.EOS_ID] = 0.0
        if self.skeleton_pos < len(self.skeleton):
            ret[self.skeleton[self.skeleton_pos]] = 0.0
        return ret
    
    def initialize(self, src_sentence):
        """If in ``pre_mode``, pass through to super class. Otherwise,
        initialize skeleton. 
        """
        if self.pre_mode:
            return super(BagOfWordsSearchPredictor, self).initialize(src_sentence)
        self.pre_mode = True
        old_accept_subsets = self.accept_subsets
        old_accept_duplicates = self.accept_duplicates
        self.accept_subsets = True
        self.accept_duplicates = True
        self.pre_decoder.predictors = self.main_decoder.predictors
        self.pre_decoder.current_sen_id = self.main_decoder.current_sen_id - 1
        hypos = self.pre_decoder.decode(src_sentence)
        score = INF
        if not hypos:
            logging.warn("No hypothesis found by the pre decoder. Effectively "
                         "reducing bowsearch predictor to bow predictor.")
            self.skeleton = []
        else:
            self.skeleton = hypos[0].trgt_sentence
            score = hypos[0].total_score
            if self.skeleton and self.skeleton[-1] -- utils.EOS_ID:
                self.skeleton = self.skeleton[:-1] # Remove EOS
        self.skeleton_pos = 0
        self.accept_subsets = old_accept_subsets
        self.accept_duplicates = old_accept_duplicates
        self._set_up_full_mode()
        logging.debug("BOW Skeleton (score=%f missing=%d): %s" % (
                                          score,
                                          sum(self.missing.values()),
                                          self.skeleton))
        self.main_decoder.current_sen_id -= 1
        self.main_decoder.initialize_predictors(src_sentence)
        self.pre_mode = False
    
    def _set_up_full_mode(self):
        """This method initializes ``missing`` by using
        ``self.skeleton`` and ``self.full_bag`` and removes
        duplicates from ``self.skeleton``.
        """
        self.bag = dict(self.full_bag)
        missing = dict(self.full_bag)
        skeleton_no_duplicates = []
        for word in self.skeleton:
            if missing[word] > 0:
                missing[word] -= 1
                skeleton_no_duplicates.append(word)
        self.skeleton = skeleton_no_duplicates
        self.missing = {w: cnt for w, cnt in missing.items() if cnt > 0}
        
    def consume(self, word):
        """Calls super class ``consume``. If not in ``pre_mode``,
        update skeleton info. 
        
        Args:
            word (int): Next word to consume
        """
        super(BagOfWordsSearchPredictor, self).consume(word)
        if self.pre_mode:
            return
        if (self.skeleton_pos < len(self.skeleton) 
                 and word == self.skeleton[self.skeleton_pos]):
            self.skeleton_pos += 1
        elif word in self.missing:
            self.missing[word] -= 1
            if self.missing[word] <= 0:
                del self.missing[word]
    
    def get_state(self):
        """If in pre_mode, state of this predictor is the current bag
        Otherwise, its the bag plus skeleton state
        """
        if self.pre_mode:
            return super(BagOfWordsSearchPredictor, self).get_state()
        return self.bag, self.skeleton_pos, self.missing
    
    def set_state(self, state):
        """If in pre_mode, state of this predictor is the current bag
        Otherwise, its the bag plus skeleton state
        """
        if self.pre_mode:
            return super(BagOfWordsSearchPredictor, self).set_state(state)
        self.bag, self.skeleton_pos, self.missing = state
    
    def is_equal(self, state1, state2):
        """Returns true if the bag and the skeleton states are the same
        """
        if self.pre_mode:
            return super(BagOfWordsSearchPredictor, self).is_equal(state1, 
                                                                   state2)
        return super(BagOfWordsSearchPredictor, self).is_equal(state1[0], 
                                                               state2[0]) class ForcedPredictor(Predictor):
    """This predictor realizes forced decoding. It stores one target
    sentence for each source sentence and outputs predictive probability
    1 along this path, and 0 otherwise.
    """
    
    def __init__(self, trg_test_file, spurious_words=[]):
        """Creates a new forced decoding predictor.
        
        Args:
            trg_test_file (string): Path to the plain text file with 
                                    the target sentences. Must have the
                                    same number of lines as the number
                                    of source sentences to decode
            spurious_words (list): List of words that are permitted to
                                   occur anywhere in the sequence
        """
        super(ForcedPredictor, self).__init__()
        self.trg_sentences = []
        with open(trg_test_file) as f:
            for line in f:
                self.trg_sentences.append([int(w) 
                            for w in line.strip().split()] + [utils.EOS_ID])
        self.n_consumed = 0 
        self.spurious_words = set(spurious_words)
        
    def get_unk_probability(self, posterior):
        """Returns negative infinity unconditionally: Words which are
        not in the target sentence have assigned probability 0 by
        this predictor.
        """
        return posterior.get(utils.UNK_ID, NEG_INF)
    
    def predict_next(self):
        """Returns a dictionary with one entry and value 0 (=log 1). The
        key is either the next word in the target sentence or (if the
        target sentence has no more words) the end-of-sentence symbol.
        """
        ret = {w: 0.0 for w in self.spurious_words}
        if self.n_consumed < len(self.cur_trg_sentence):
            ret[utils.EOS_ID] = NEG_INF
            ret[self.cur_trg_sentence[self.n_consumed]] = 0.0
        else:
            ret[utils.EOS_ID] = 0.0
        return ret
    
    def initialize(self, src_sentence):
        """Fetches the corresponding target sentence and resets the
        current history.
        
        Args:
            src_sentence (list):  Not used
        """
        self.cur_trg_sentence = self.trg_sentences[self.current_sen_id] 
        self.n_consumed = 0
    
    def consume(self, word):
        """If ``word`` matches the target sentence, we increase the
        current history by one. Otherwise, we set this predictor in
        an invalid state, in which it always predicts </S>
        
        Args:
            word (int): Next word to consume
        """
        if word in self.spurious_words:
            return
        if self.n_consumed < len(self.cur_trg_sentence):
            trg_word = self.cur_trg_sentence[self.n_consumed]
            if trg_word != utils.UNK_ID and trg_word != word:
                self.cur_trg_sentence = [] # Mismatch with our target sentence
            else:
                self.n_consumed = self.n_consumed + 1
    
    def get_state(self):
        """``cur_trg_sentence`` can be changed so its part of the 
        predictor state
        """
        return self.n_consumed,self.cur_trg_sentence
    
    def set_state(self, state):
        """Set the predictor state. """
        self.n_consumed,self.cur_trg_sentence = state

    def is_equal(self, state1, state2):
        """Returns true if the state is the same """
        n1,s1 = state1
        n2,s2 = state2
        return n1 == n2 and s1 == s2 class ForcedLstPredictor(Predictor):
    """This predictor can be used for direct n-best list rescoring. In
    contrast to the ``ForcedPredictor``, it reads an n-best list in 
    Moses format and uses its scores as predictive probabilities of the
    </S> symbol. Everywhere else it gives the predictive probability 1 
    if the history corresponds to at least one n-best list entry, 0 
    otherwise. From the n-best list we use
    First column: Sentence id
    Second column: Hypothesis in integer format
    Last column: score
    
    Note: Behavior is undefined if you have duplicates in the n-best
    list
    
    TODO: Would be much more efficient to use Tries for 
    cur_trgt_sentences instead of a flat list.
    """
    
    def __init__(self, 
                 trg_test_file, 
                 use_scores=True, 
                 match_unk=False, 
                 feat_name=None):
        """Creates a new n-best rescoring predictor instance.
        
        Args:
            trg_test_file (string):  Path to the n-best list
            use_scores (bool): Whether to use the scores from the
                               n-best list. If false, use uniform
                               scores of 0 (=log 1).
            match_unk (bool): If true, allow any word where the n-best
                              list contains UNK.
            feat_name (string): Instead of the combined score in the
                                last column of the Moses n-best list,
                                we can use one of the sparse features.
                                Set this to the name of the feature
                                (denoted as <name>= in the n-best list)
                                if you wish to do that.
        """
        super(ForcedLstPredictor, self).__init__()
        self.trg_sentences = []
        self.match_unk = match_unk
        score = 0.0
        with open(trg_test_file) as f:
            for line in f:
                parts = line.split("|||")
                if len(parts) < 2:
                    logging.warn("Malformed line %s in n-best list %s" % (
                                        line.strip(),
                                        trg_test_file))
                else:
                    if use_scores:
                        score = self._get_score(parts, feat_name)
                    sen_id = int(parts[0].strip())
                    while len(self.trg_sentences) <= sen_id:
                        self.trg_sentences.append([])
                    sen = [int(w) for w in parts[1].strip().split()]
                    if sen and sen[0] == utils.GO_ID:
                        sen  = sen[1:]
                    if sen and sen[-1] == utils.EOS_ID:
                        sen = sen[:-1]
                    self.trg_sentences[sen_id].append((score, sen))
        
    def _get_score(self, parts, feat_name):
        """Get the score for a hypothesis.
        
        Args:
            parts (list): Parts of the n-best entry (separated by |||
                          in the Moses n-best format)
            feat_name (string): Name of the sparse feature which should
                                be used as score (or None to use the
                                combined score)
        """
        feat_str = "%s=" % feat_name
        if not feat_name:
            return float(parts[-1].strip()) if len(parts) > 2 else 0.0
        feat_parts = parts[-2].strip().split()
        for idx in range(len(feat_parts)-1):
            if feat_parts[idx] == feat_str:
                return float(feat_parts[idx+1])
        return 0.0

    def get_unk_probability(self, posterior):
        """Return negative infinity unconditionally - words outside the
        n-best list are not possible according to this predictor.
        """
        if self.match_unk:
            return posterior.get(utils.UNK_ID, NEG_INF)
        return NEG_INF
    
    def predict_next(self):
        """Outputs 0.0 (i.e. prob=1) for all words for which there is 
        an entry ``in cur_trg_sentences``, and the score in 
        ``cur_trg_sentences`` if the current history is by itself equal
        to an entry in ``cur_trg_sentences``.
        
        TODO: The implementation here is fairly inefficient as it scans 
        through all target sentences linearly. Would be better to 
        organize the target sentences in a Trie
        """
        scores = {}
        hist_len = len(self.history)
        for sen_score,trg_sentence in self.cur_trg_sentences:
            sen_len = len(trg_sentence)
            if sen_len < hist_len:
                continue
            if self.match_unk:
                hist = [self.history[i] 
                      if trg_sentence[i] != utils.UNK_ID else utils.UNK_ID
                      for i in range(hist_len)]
            else:
                hist = self.history 
            if trg_sentence[:hist_len] == hist:
                if sen_len == hist_len:
                    scores[utils.EOS_ID] = sen_score
                else:
                    scores[trg_sentence[hist_len]] = 0.0
        if not utils.EOS_ID in scores:
            scores[utils.EOS_ID] = NEG_INF
        return scores
    
    def initialize(self, src_sentence):
        """Resets the history and loads the n-best list entries for the
        next source sentence
        
        Args:
            src_sentence (list): Not used
        """
        self.cur_trg_sentences = self.trg_sentences[self.current_sen_id] 
        self.history = []
    
    def consume(self, word):
        """Extends the current history by ``word``. """
        self.history.append(word)
    
    def get_state(self):
        """Returns the current history. """
        return self.history
    
    def set_state(self, state):
        """Sets the current history. """
        self.history = state

    def is_equal(self, state1, state2):
        """Returns true if the history is the same """
        return state1 == state2 class RuleXtractPredictor(Predictor):
    """Predictor based on ruleXtract rules. Bins are organized 
    according the number of target words. We assume that no rule 
    produces the empty word on the source side (but possibly on the 
    target side). Hypotheses are produced iteratively s.t. the 
    following invariant holds: The bins contain a set of (partial) 
    hypotheses from which we can derive all full hypotheses which are 
    consistent with the current target prefix (i.e. the prefix of the
    target sentence which has already been translated). This set is
    updated when calling either consume_word or predict_next: consume\_
    word deletes all hypotheses which become inconsistent with the new 
    word. ``predict_next`` requires all hypotheses to have a target\_
    prefix length of at least one plus the number of consumed words. 
    Therefore, ``predict_next`` expands hypotheses as long as they are
    shorter. This fits nicely with grouping hypotheses in bins of same
    target prefix length: we expand until all low rank bins are empty. 
    We predict the next target word by using the cost of the best 
    hypothesis with the word at the right position. 
    
    Note that this predictor is similar to the decoding algorithm in
    
      Efficient Left-to-Right Hierarchical Phrase-based Translation with
      Improved Reordering. 
      Maryam Siahbani, Baskaran Sankaran and Anoop Sarkar. 
      EMNLP 2013. Oct 18-21, 2013. Seattle, USA.
    
    without cube pruning, but it is extended to an arbitrary number of
    non-terminals as produced with ruleXtract.
    """
    
    def __init__(self, ruleXtract_path, use_weights, feature_weights = None):
        """Creates a new hiero predictor.
        
        Args:
            ruleXtract_path (string): Path to the rules file
            use_weights (bool): If false, set all hypothesis scores 
                                uniformly to 0 (= log 1). If true,
                                use the rule weights to compute
                                hypothesis scores
            feature_weights (list): Rule feature weights to compute
                                    the rule scores. If this is none
                                    we use uniform weights
        """
        super(RuleXtractPredictor, self).__init__()
        self.use_weights = use_weights
        self.rules = RuleSet()
        with (gzip.open(ruleXtract_path) if ruleXtract_path[-3:] == '.gz' 
                                         else open(ruleXtract_path)) as f:
            for line in f:
                self.rules.parse(line, feature_weights)
        self.rules.update_span_len_range()
        logging.info("%d rules loaded (%d discarded because not in GNF)" %
            (self.rules.n_rules, self.rules.n_discarded))
        if not 'S' in self.rules.nt2id:
            logging.fatal("No rule with start symbol S found!") 
        self.start_nt = self.rules.nt2id['S']
        logging.debug("Grammar start symbol: S (ID: %d)" % self.start_nt)

    def get_unk_probability(self, posterior):
        """Returns negative infinity if the posterior is not empty as
        words outside the grammar are not possible according this
        predictor. If ``posterior`` is empty, return 0 (= log 1)
        """ 
        return utils.NEG_INF
    
    def predict_next(self):
        """For predicting the distribution of the next target tokens, 
        we need to empty the stack with the current history length
        by expanding all hypotheses on it. Then, all hypotheses are
        in larger bins, i.e. have a longer target prefix than the
        current history. Thus, we can look up the possible next words
        by iterating through all active hypotheses.
        """
        # If there are still partial hypotheses...
        if self.n_consumed < len(self.stacks):
            # empty stack with n_consumed trgt_prefix length
            while self.stacks[self.n_consumed]:
                hypo = self.stacks[self.n_consumed].pop()
                #print("EXPAND: %s" % hypo)
                new_hypos = self.rules.expand_hypo(hypo, self.src_seq)
                for new_hypo in new_hypos:
                    n_covered = len(new_hypo.trgt_prefix)
                    if new_hypo.is_final():
                        while len(self.finals) <= n_covered:
                            self.finals.append(Cell())
                        # Make sure that it ends with EOS
                        new_hypo.trgt_prefix[-1] = utils.EOS_ID
                        self.finals[n_covered].add(new_hypo)
                    else:
                        while len(self.stacks) <= n_covered:
                            self.stacks.append(Cell())
                        self.stacks[n_covered].add(new_hypo)
        logging.debug("Predict next (consumed: %d)" % self.n_consumed)
        for idx,c in enumerate(self.stacks):
            if c.hypos:
                logging.debug("Stack %d: %d" % (idx, len(c.hypos)))
        for idx,c in enumerate(self.finals):
            if c.hypos:
                logging.debug("Finals %d: %s" % (idx, c.hypos))
        return self.build_posterior()
    
    def build_posterior(self):
        """We need to scan all hypotheses in ``self.stacks`` and add up
        scores grouped by the symbol at the n_consumed+1-th position.
        Then, we add end-of-sentence probability by checking 
        ``self.finals[n_consumed]``
        """
        posterior = {}
        for stack_idx in range(self.n_consumed+1, len(self.stacks)):
            for hypo in self.stacks[stack_idx].hypos:
                symb = hypo.trgt_prefix[self.n_consumed]
                posterior[symb] = max(posterior.get(symb, 0), hypo.cost)
        if self.n_consumed+1 < len(self.finals) and self.finals[self.n_consumed+1]:
            posterior[utils.EOS_ID] = max([hypo.cost
                            for hypo in self.finals[self.n_consumed+1].hypos])
        return self.finalize_posterior(posterior, self.use_weights, False)
    
    def initialize(self, src_sentence):
        """Delete all bins and add the initial cell to the first bin """
        self.stacks = []
        self.n_consumed = 0
        self.src_seq = [utils.GO_ID] + src_sentence + [utils.EOS_ID]
        self.src_len = len(self.src_seq)
        span = Span([-self.start_nt], (0, self.src_len))
        init_hypo = LRHieroHypothesis([], [span], 0)
        self.stacks = [Cell(init_hypo)]
        self.finals = []
        self.predict_next()
        self.consume(utils.GO_ID)
    
    def consume(self, word):
        """Remove all hypotheses with translation prefixes which do not
        match ``word``
        """
        for stack_idx in range(self.n_consumed+1, len(self.stacks)):
            self.stacks[stack_idx].filter(self.n_consumed, word)
        for stack_idx in range(self.n_consumed+1, len(self.finals)):
            self.finals[stack_idx].filter(self.n_consumed, word)
        if self.n_consumed < len(self.finals):
            # Empty this entry, not needed anymore
            self.finals[self.n_consumed] = Cell() 
        self.n_consumed = self.n_consumed + 1
    
    def get_state(self):
        """Predictor state consists of the stacks, the completed 
        hypotheses, and the number of consumed words. """
        return self.stacks,self.finals,self.n_consumed
    
    def set_state(self, state):
        """Set the predictor state. """
        self.stacks,self.finals,self.n_consumed = state class WordCountPredictor(Predictor):
    """This predictor adds the (negative) number of words as feature.
    This means that this predictor encourages shorter hypotheses when
    used with a positive weight.
    """
    
    def __init__(self, word=-1,
                 nonterminal_penalty=False,
                 nonterminal_ids=None,
                 min_terminal_id=0,
                 max_terminal_id=30003,
                 negative_wc=True,
                 vocab_size=30003):
        """Creates a new word count predictor instance.
        
        Args:
            word (int): If this is non-negative we count only the
                        number of the specified word. If its
                        negative, count all words
            nonterminal_penalty (bool): If true, apply penalty only to 
                        tokens in a range  (the range *outside* 
                        min/max terminal id)
            nonterminal_ids: file containing ids of nonterminal tokens
            min_terminal_id: lower bound of tokens *not* to penalize,
                              if nonterminal_penalty selected
            max_terminal_id: upper bound of tokens *not* to penalize,
                             if nonterminal_penalty selected
            negative_wc: If true, the score of this predictor is the 
                         negative word count.
            vocab_size: upper bound of tokens, used to find nonterminal range

        """
        super(WordCountPredictor, self).__init__()
        val = 1.0
        if negative_wc:
          val = -1.0
        if nonterminal_penalty:
            if nonterminal_ids:
                nts = load_external_ids(nonterminal_ids)
            else:
                min_nt_range = range(0, min_terminal_id)
                max_nt_range = range(max_terminal_id + 1, vocab_size)
                nts = list(min_nt_range) + list(max_nt_range)
            self.posterior = {nt: val for nt in nts}
            self.posterior[utils.EOS_ID] = 0.0
            self.posterior[utils.UNK_ID] = 0.0
            self.unk_prob = 0.0
        elif word < 0:
            self.posterior = {utils.EOS_ID : 0.0}
            self.unk_prob = val
        else:
            self.posterior = {word : val}
            self.unk_prob = 0.0 
        
    def get_unk_probability(self, posterior):
        return self.unk_prob
    
    def predict_next(self):
        return self.posterior
    
    def initialize(self, src_sentence):
        """Empty"""
        pass
    
    def consume(self, word):
        """Empty"""
        pass
    
    def get_state(self):
        """Returns true """
        return True
    
    def set_state(self, state):
        """Empty"""
        pass

    def is_equal(self, state1, state2):
        """Returns true """
        return True class NBLengthPredictor(Predictor):
    """This predictor assumes that target sentence lengths are 
    distributed according a negative binomial distribution with 
    parameters r,p. r is linear in features, p is the logistic of a
    linear function over the features. Weights can be trained using 
    the Matlab script ``estimate_length_model.m`` 
    
    Let w be the model_weights. All features are extracted from the
    src sentence::
    
      r = w0 * #char
      + w1 * #words
      + w2 * #punctuation
      + w3 * #char/#words
      + w4 * #punct/#words
      + w10
      
      p = logistic(w5 * #char
      + w6 * #words
      + w7 * #punctuation
      + w8 * #char/#words
      + w9 * #punct/#words
      + w11)
      
      target_length ~ NB(r,p)
      
    The biases w10 and w11 are optional.
    
    The predictor predicts EOS with NB(#consumed_words,r,p)
    """
    
    def __init__(self, text_file, model_weights, use_point_probs, offset = 0):
        """Creates a new target sentence length model predictor.
        
        Args:
            text_file (string): Path to the text file with the 
                                unindexed source sentences, i.e. not
                                using word ids
            model_weights (list): Weights w0 to w11 of the length 
                                  model. See class docstring for more
                                  information
            use_point_probs (bool): Use point estimates for EOS token,
                                    0.0 otherwise 
            offset (int): Subtract this from hypothesis length before
                          applying the NB model
        """
        super(NBLengthPredictor, self).__init__()
        self.use_point_probs = use_point_probs
        self.offset = offset
        if len(model_weights) == 2*NUM_FEATURES: # add biases
            model_weights.append(0.0)
            model_weights.append(0.0)
        if len(model_weights) != 2*NUM_FEATURES+2:
            logging.fatal("Number of length model weights has to be %d or %d"
                    % (2*NUM_FEATURES, 2*NUM_FEATURES+2))
        self.r_weights = model_weights[0:NUM_FEATURES] + [model_weights[-2]]
        self.p_weights = model_weights[NUM_FEATURES:2*NUM_FEATURES] + [model_weights[-1]]
        self.src_features = self._extract_features(text_file)
        self.n_consumed = 0 

    def _extract_features(self, file_name):
        """Extract all features from the source sentences. """
        feats = []
        with open(file_name) as f:
            for line in f:
                feats.append(self._analyse_sentence(line.strip()))
        return feats
    
    def _analyse_sentence(self, sentence):
        """Extract features for a single source sentence.
        
        Args:
            sentence (string): Source sentence string
        
        Returns:
            5-tuple of features as described in the class docstring
        """
        n_char = len(sentence) + 0.0
        n_words = len(sentence.split()) + 0.0
        n_punct = sum([sentence.count(s) for s in ",.:;-"]) + 0.0
        return [n_char, n_words, n_punct, n_char/n_words, n_punct/n_words]
        
    def get_unk_probability(self, posterior):
        """If we use point estimates, return 0 (=1). Otherwise, return
        the 1-p(EOS), with p(EOS) fetched from ``posterior``
        """
        if self.use_point_probs:
            if self.n_consumed == 0:
                return self.max_eos_prob
            return 0.0
        if self.n_consumed == 0:
            return 0.0
        return np.log(1.0 - np.exp(posterior[utils.EOS_ID]))
    
    def predict_next(self):
        """Returns a dictionary with single entry for EOS. """
        if self.n_consumed == 0:
            return {utils.EOS_ID : utils.NEG_INF}
        return {utils.EOS_ID : self._get_eos_prob()}
    
    def _get_eos_prob(self):
        """Get loglikelihood according cur_p, cur_r, and n_consumed """
        eos_point_prob = self._get_eos_point_prob(max(
                                              1, 
                                              self.n_consumed - self.offset))
        if self.use_point_probs:
            return eos_point_prob - self.max_eos_prob
        if not self.prev_eos_probs:
            self.prev_eos_probs.append(eos_point_prob)
            return eos_point_prob
        # bypass utils.log_sum because we always want to use logsumexp here 
        prev_sum = logsumexp(np.asarray([p for p in self.prev_eos_probs])) 
        self.prev_eos_probs.append(eos_point_prob)
        # Desired prob is eos_point_prob / (1-last_eos_probs_sum)
        return eos_point_prob - np.log(1.0-np.exp(prev_sum))
    
    def _get_eos_point_prob(self, n):
        return gammaln(n + self.cur_r) \
                - gammaln(n + 1) \
                - gammaln(self.cur_r) \
                + n * np.log(self.cur_p) \
                + self.cur_r * np.log(1.0-self.cur_p)
    
    def _get_max_eos_prob(self):
        """Get the maximum loglikelihood according cur_p, cur_r 
        TODO: replace this brute force impl. with something smarter
        """
        max_prob = utils.NEG_INF
        n_prob = max_prob
        n = 0
        while n_prob == max_prob:
            n += 1
            n_prob = self._get_eos_point_prob(n)
            max_prob = max(max_prob, n_prob)
        return max_prob
    
    def initialize(self, src_sentence):
        """Extract features for the source sentence. Note that this
        method does not use ``src_sentence`` as we need the string
        representation of the source sentence to extract features.
        
        Args:
            src_sentence (list): Not used
        """
        feat = self.src_features[self.current_sen_id] + [1.0]
        self.cur_r  = max(EPS_R, np.dot(feat, self.r_weights));
        p = np.dot(feat, self.p_weights)
        p = 1.0 / (1.0 + math.exp(-p))
        self.cur_p = max(utils.EPS_P, min(1.0 - utils.EPS_P, p))
        self.n_consumed = 0
        self.prev_eos_probs = []
        if self.use_point_probs:
            self.max_eos_prob = self._get_max_eos_prob()
    
    def consume(self, word):
        """Increases the current history length
        
        Args:
            word (int): Not used
        """
        self.n_consumed = self.n_consumed + 1
    
    def get_state(self):
        """State consists of the number of consumed words, and the
        accumulator for previous EOS probability estimates if we 
        don't use point estimates.
        """
        return self.n_consumed,self.prev_eos_probs
    
    def set_state(self, state):
        """Set the predictor state """
        self.n_consumed,self.prev_eos_probs = state

    def is_equal(self, state1, state2):
        """Returns true if the number of consumed words is the same """
        n1,_ = state1
        n2,_ = state2
        return n1 == n2 class ExternalLengthPredictor(Predictor):
    """This predictor loads the distribution over target sentence
    lengths from an external file. The file contains blank separated
    length:score pairs in each line which define the length 
    distribution. The predictor adds the specified scores directly
    to the EOS score.
    """
    
    def __init__(self, path):
        """Creates a external length distribution predictor.
        
        Args:
            path (string): Path to the file with target sentence length
                           distributions.
        """
        super(ExternalLengthPredictor, self).__init__()
        self.trg_lengths = load_external_lengths(path)
        
    def get_unk_probability(self, posterior):
        """Returns 0=log 1 if the partial hypothesis does not exceed
        max length. Otherwise, predict next returns an empty set,
        and we set everything else to -inf.
        """
        if self.n_consumed < self.max_length:
            return 0.0
        return utils.NEG_INF
    
    def predict_next(self):
        """Returns a dictionary with one entry and value 0 (=log 1). The
        key is either the next word in the target sentence or (if the
        target sentence has no more words) the end-of-sentence symbol.
        """
        if self.n_consumed in self.cur_scores: 
            return {utils.EOS_ID : self.cur_scores[self.n_consumed]}
        return {utils.EOS_ID : utils.NEG_INF} 
    
    def initialize(self, src_sentence):
        """Fetches the corresponding target sentence length 
        distribution and resets the word counter.
        
        Args:
            src_sentence (list):  Not used
        """
        self.cur_scores = self.trg_lengths[self.current_sen_id]
        self.max_length = max(self.cur_scores)
        self.n_consumed = 0

    def consume(self, word):
        """Increases word counter by one.
        
        Args:
            word (int): Not used
        """
        self.n_consumed = self.n_consumed + 1
    
    def get_state(self):
        """Returns the number of consumed words """
        return self.n_consumed
    
    def set_state(self, state):
        """Set the number of consumed words """
        self.n_consumed = state

    def is_equal(self, state1, state2):
        """Returns true if the number of consumed words is the same """
        return state1 == state2 class NgramCountPredictor(Predictor):
    """This predictor counts the number of n-grams in hypotheses. n-gram
    posteriors are loaded from a file. The predictor score is the sum of
    all n-gram posteriors in a hypothesis. """
    
    def __init__(self, path, order=0, discount_factor=-1.0):
        """Creates a new ngram count predictor instance.
        
        Args:
            path (string): Path to the n-gram posteriors. File format:
                           <ngram> : <score> (one ngram per line). Use
                           placeholder %d for sentence id.
            order (int): If positive, count n-grams of the specified
                         order. Otherwise, count all n-grams
            discount_factor (float): If non-negative, discount n-gram
                                     posteriors by this factor each time 
                                     they are consumed 
        """
        super(NgramCountPredictor, self).__init__()
        self.path = path 
        self.order = order
        self.discount_factor = discount_factor
        
    def get_unk_probability(self, posterior):
        """Always return 0.0 """
        return 0.0
    
    def predict_next(self):
        """Composes the posterior vector by collecting all ngrams which
        are consistent with the current history.
        """
        posterior = {}
        for i in reversed(range(len(self.cur_history)+1)):
            scores = self.ngrams.get(self.cur_history[i:])
            if scores:
                factors = False
                if self.discount_factor >= 0.0:
                    factors = self.discounts.get(self.cur_history[i:])
                if not factors:
                    for w,score in scores.items():
                        posterior[w] = posterior.get(w, 0.0) + score
                else:
                    for w,score in scores.items():
                        posterior[w] = posterior.get(w, 0.0) +  \
                                       factors.get(w, 1.0) * score
        return posterior
    
    def _load_posteriors(self, path):
        """Sets up self.max_history_len and self.ngrams """
        self.max_history_len = 0
        self.ngrams = SimpleTrie()
        logging.debug("Loading n-gram scores from %s..." % path)
        with open(path) as f:
            for line in f:
                ngram,score = line.split(':')
                words = [int(w) for w in ngram.strip().split()]
                if self.order > 0 and len(words) != self.order:
                    continue
                hist = words[:-1]
                last_word = words[-1]
                if last_word == utils.GO_ID:
                    continue
                self.max_history_len = max(self.max_history_len, len(hist))
                p = self.ngrams.get(hist)
                if p:
                    p[last_word] = float(score.strip())
                else:
                    self.ngrams.add(hist, {last_word: float(score.strip())})
    
    def initialize(self, src_sentence):
        """Loads n-gram posteriors and resets history.
        
        Args:
            src_sentence (list): not used
        """
        self._load_posteriors(utils.get_path(self.path, self.current_sen_id+1))
        self.cur_history = [utils.GO_ID]
        self.discounts = SimpleTrie()
    
    def consume(self, word):
        """Adds ``word`` to the current history. Shorten if the extended
        history exceeds ``max_history_len``.
        
        Args:
            word (int): Word to add to the history.
        """
        self.cur_history.append(word)
        if len(self.cur_history) > self.max_history_len:
            self.cur_history = self.cur_history[-self.max_history_len:]
        if self.discount_factor >= 0.0:
            for i in range(len(self.cur_history)):
                key = self.cur_history[i:-1]
                factors = self.discounts.get(key)
                if not factors:
                    factors = {word: self.discount_factor}
                else:
                    factors[word] = factors.get(word, 1.0)*self.discount_factor
                self.discounts.add(key, factors)
    
    def get_state(self):
        """Current history is the predictor state """
        return self.cur_history,self.discounts
    
    def set_state(self, state):
        """Current history is the predictor state """
        self.cur_history,self.discounts = state

    def is_equal(self, state1, state2):
        """Hypothesis recombination is
        not supported if discounting is enabled.
        """
        if self.discount_factor >= 0.0:
            return False
        hist1 = state1[0]
        hist2 = state2[0]
        if hist1 == hist2: # Return true if histories match
            return True
        if len(hist1) > len(hist2):
            hist_long = hist1
            hist_short = hist2
        else:
            hist_long = hist2
            hist_short = hist1
        min_len = len(hist_short)
        for n in range(1, min_len+1): # Look up non matching in self.ngrams
            key1 = hist1[-n:]
            key2 = hist2[-n:]
            if key1 != key2:
                if self.ngrams.get(key1) or self.ngrams.get(key2):
                    return False
        for n in range(min_len+1, len(hist_long)+1):
            if self.ngrams.get(hist_long[-n:]):
                return False
        return True class UnkCountPredictor(Predictor):
    """This predictor regulates the number of UNKs in the output. We 
    assume that the number of UNKs in the target sentence is Poisson 
    distributed. This predictor is configured with n lambdas for
    0,1,...,>=n-1 UNKs in the source sentence. """
    
    def __init__(self, src_vocab_size, lambdas):
        """Initializes the UNK count predictor.

        Args:
            src_vocab_size (int): Size of source language vocabulary.
                                  Indices greater than this are 
                                  considered as UNK.
            lambdas (list): List of floats. The first entry is the 
                            lambda parameter given that the number of
                            unks in the source sentence is 0 etc. The
                            last float is lambda given that the source
                            sentence has more than n-1 unks.
        """
        self.lambdas = lambdas
        self.l = lambdas[0]
        self.src_vocab_size = src_vocab_size
        super(UnkCountPredictor, self).__init__()
        
    def get_unk_probability(self, posterior):
        """Always returns 0 (= log 1) except for the first time """
        if self.n_consumed == 0:
            return self.max_prob
        return 0.0
    
    def predict_next(self):
        """Set score for EOS to the number of consumed words """
        if self.n_consumed == 0:
            return {utils.EOS_ID : self.unk_prob}
        if self.n_unk < self.max_prob_idx:
            return {utils.EOS_ID : self.unk_prob - self.max_prob}
        return {utils.UNK_ID : self.unk_prob - self.consumed_prob}
    
    def initialize(self, src_sentence):
        """Count UNKs in ``src_sentence`` and reset counters.
        
        Args:
            src_sentence (list): Count UNKs in this list
        """
        src_n_unk = len([w for w in src_sentence if w == utils.UNK_ID 
                                                    or w > self.src_vocab_size])
        self.l = self.lambdas[min(len(self.lambdas)-1, src_n_unk)]
        self.n_consumed = 0
        self.n_unk = 0
        self.unk_prob = self._get_poisson_prob(1)
        # Mode at lambda is the maximum of the poisson function
        self.max_prob_idx = int(self.l)
        self.max_prob = self._get_poisson_prob(self.max_prob_idx)
        ceil_prob = self._get_poisson_prob(self.max_prob_idx + 1)
        if ceil_prob > self.max_prob:
            self.max_prob = ceil_prob
            self.max_prob_idx = self.max_prob_idx + 1
        self.consumed_prob = self.max_prob

    def _get_poisson_prob(self, n):
        """Get the log of the poisson probability for n events. """
        return n * np.log(self.l) - self.l - sum([np.log(i+1) for i in range(n)])
    
    def consume(self, word):
        """Increases unk counter by one if ``word`` is unk.
        
        Args:
            word (int): Increase counter if ``word`` is UNK
        """
        self.n_consumed += 1
        if word == utils.UNK_ID:
            if self.n_unk >= self.max_prob_idx:
                self.consumed_prob = self.unk_prob
            self.n_unk += 1
            self.unk_prob = self._get_poisson_prob(self.n_unk+1)
    
    def get_state(self):
        """Returns the number of consumed words """
        return self.n_unk,self.n_consumed,self.unk_prob,self.consumed_prob
    
    def set_state(self, state):
        """Set the number of consumed words """
        self.n_unk,self.n_consumed,self.unk_prob,self.consumed_prob = state

    def is_equal(self, state1, state2):
        """Returns true if the state is the same"""
        return state1 == state2 class NgramizePredictor(Predictor):
    """This wrapper extracts n-gram posteriors from a predictor which
    does not depend on the particular argument of `consume()`. In that
    case, we can build a lookup mechanism for all possible n-grams in
    a single forward pass through the predictor search space: We record
    all posteriors (predict_next() return values) of the slave
    predictor during a greedy pass in `initialize()`. The wrapper
    predictor state is the current n-gram history. We use the 
    (semiring) sum over all possible positions of the current n-gram
    history in the recorded slave predictor posteriors to form the
    n-gram scores returned by this predictor.

    Note that this wrapper does not work correctly if the slave
    predictor feeds back the selected token in the history, ie. depends
    on the particular token which is provided via `consume()`.

    TODO: Make this wrapper work with slaves which return dicts.
    """
    
    def __init__(self, min_order, max_order, max_len_factor, slave_predictor):
        """Creates a new ngramize wrapper predictor.
        
        Args:
            min_order (int): Minimum n-gram order
            max_order (int): Maximum n-gram order
            max_len_factor (int): Stop the forward pass through the 
                                  slave predictor after src_length
                                  times this factor
            slave_predictor (Predictor): Instance of the predictor which
                                         uses the source sentences in
                                         ``src_test``

        Raises:
            AttributeError if order is not positive.
        """
        super(NgramizePredictor, self).__init__()
        if max_order < 1:
             raise AttributeError("max_ngram_order must be positive.")
        if min_order > max_order:
             raise AttributeError("min_ngram_order greater than max_order.")
        self.slave_predictor = slave_predictor
        self.max_history_length = max_order - 1
        self.min_order = max(1, min_order)
        self.max_len_factor = max_len_factor
    
    def initialize(self, src_sentence):
        """Runs greedy decoding on the slave predictor to populate
        self.scores and self.unk_scores, resets the history.
        """
        self.slave_predictor.initialize(src_sentence)
        self.scores = []
        self.unk_scores = []
        trg_word = -1
        max_len = self.max_len_factor * len(src_sentence)
        l = 0
        while trg_word != utils.EOS_ID and l <= max_len:
            posterior = self.slave_predictor.predict_next()
            trg_word = utils.argmax(posterior)
            self.scores.append(posterior)
            self.unk_scores.append(self.slave_predictor.get_unk_probability(
                posterior))
            self.slave_predictor.consume(utils.UNK_ID)
            l += 1
        logging.debug("ngramize uses %d time steps." % l)
        self.history = []
        self.cur_unk_score = utils.NEG_INF
    
    def initialize_heuristic(self, src_sentence):
        """Pass through to slave predictor """
        logging.warning("ngramize does not support predictor heuristics")
        self.slave_predictor.initialize_heuristic(src_sentence)
    
    def predict_next(self):
        """Looks up ngram scores via self.scores. """
        cur_hist_length = len(self.history)
        this_scores = [[] for _ in range(cur_hist_length+1)]
        this_unk_scores = [[] for _ in range(cur_hist_length+1)]
        for pos in range(len(self.scores)):
            this_scores[0].append(self.scores[pos])
            this_unk_scores[0].append(self.unk_scores[pos])
            acc = 0.0
            for order, word in enumerate(self.history):
                if pos + order + 1 >= len(self.scores):
                    break
                acc += utils.common_get(
                    self.scores[pos + order], word, 
                    self.unk_scores[pos + order])
                this_scores[order+1].append(acc + self.scores[pos + order + 1])
                this_unk_scores[order+1].append(
                    acc + self.unk_scores[pos + order + 1])
        combined_scores = []
        combined_unk_scores = []
        for order, (scores, unk_scores) in enumerate(zip(this_scores, 
                                                         this_unk_scores)):
            if scores and order + 1 >= self.min_order:
                score_matrix = np.vstack(scores)
                combined_scores.append(logsumexp(score_matrix, axis=0))
                combined_unk_scores.append(utils.log_sum(unk_scores))
        if not combined_scores:
            self.cur_unk_score = 0.0
            return {}
        self.cur_unk_score = sum(combined_unk_scores)
        return sum(combined_scores)
        
    def get_unk_probability(self, posterior):
        return self.cur_unk_score
    
    def consume(self, word):
        """Pass through to slave predictor """
        if self.max_history_length > 0:
            self.history.append(word)
            self.history = self.history[-self.max_history_length:]
    
    def get_state(self):
        """State is the current n-gram history. """
        return self.history, self.cur_unk_score
    
    def set_state(self, state):
        """State is the current n-gram history. """
        self.history, self.cur_unk_score = state

    def set_current_sen_id(self, cur_sen_id):
        """We need to override this method to propagate current\_
        sentence_id to the slave predictor
        """
        super(NgramizePredictor, self).set_current_sen_id(cur_sen_id)
        self.slave_predictor.set_current_sen_id(cur_sen_id)
    
    def is_equal(self, state1, state2):
        """Pass through to slave predictor """
        return state1 == state2 class WeightNonTerminalPredictor(Predictor):
    """This wrapper multiplies the weight of given tokens (those outside
    the min/max terminal range) by a factor."""
    
    def __init__(self, slave_predictor,
                 penalty_factor=1.0,
                 nonterminal_ids=None,
                 min_terminal_id=0,
                 max_terminal_id=30003,
                 vocab_size=30003):
        """Creates a new id-weighting wrapper for a predictor
        
        Args:
            slave_predictor: predictor to apply penalty to.
            penalty_factor (float): factor by which to multiply tokens in range
            min_terminal_id: lower bound of tokens *not* to penalize, 
        if nonterminal_penalty selected
            max_terminal_id: upper bound of tokens *not* to penalize,
        if nonterminal_penalty selected
            vocab_size: upper bound of tokens, used to find nonterminal range

        """
        super(WeightNonTerminalPredictor, self).__init__()
        if nonterminal_ids:
            nts = load_external_ids(nonterminal_ids)
        else:
            min_nt_range = range(0, min_terminal_id)
            max_nt_range = range(max_terminal_id + 1, vocab_size)
            nts = list(min_nt_range) + list(max_nt_range)
        self.slave_predictor = slave_predictor
        self.mult = {tok: penalty_factor for tok in nts}
        self.mult[utils.EOS_ID] = 1.0
        self.mult[utils.UNK_ID] = 1.0
        
    def get_unk_probability(self, posterior):
        return self.slave_predictor.get_unk_probability(posterior)
    
    def predict_next(self):
        posterior = self.slave_predictor.predict_next()
        post_keys = set(utils.common_viewkeys(posterior))
        for tok in self.mult:
            if tok in post_keys:
                posterior[tok] *= self.mult[tok]
        return posterior
    
    def initialize(self, src_sentence):
        self.slave_predictor.initialize(src_sentence)
    
    def consume(self, word):
        return self.slave_predictor.consume(word)
    
    def get_state(self):
        return self.slave_predictor.get_state()
    
    def set_state(self, state):
        self.slave_predictor.set_state(state)
    
    def is_equal(self, state1, state2):
        return self.slave_predictor.is_equal(state1, state2) class BracketPredictor(UnboundedVocabularyPredictor):
    """This predictor constrains the output to well-formed bracket
    expressions. It also allows to specify the number of terminals with
    an external length distribution file.
    """
    
    def __init__(self, max_terminal_id, closing_bracket_id, max_depth=-1, 
                 extlength_path=""):
        """Creates a new bracket predictor.
        
        Args:
            max_terminal_id (int): All IDs greater than this are 
                brackets
            closing_bracket_id (string): All brackets except these ones are 
                opening. Comma-separated list of integers.
            max_depth (int): If positive, restrict the maximum depth
            extlength_path (string): If this is set, restrict the 
                number of terminals to the distribution specified in
                the referenced file. Terminals can be implicit: We
                count a single terminal between each adjacent opening
                and closing bracket.
        """
        super(BracketPredictor, self).__init__()
        self.max_terminal_id = max_terminal_id
        try:
            self.closing_bracket_ids = utils.split_comma(closing_bracket_id, int)
        except:
            self.closing_bracket_ids = [int(closing_bracket_id)]
        self.max_depth = max_depth if max_depth >= 0 else 1000000
        if extlength_path:
            self.length_scores = load_external_lengths(extlength_path)
        else:
            self.length_scores = None
            self.max_length = 1000000
    
    def initialize(self, src_sentence):
        """Sets the current depth to 0.
        
        Args:
            src_sentence (list): Not used
        """
        self.cur_depth = 0
        self.ends_with_opening = True
        self.n_terminals = 0
        if self.length_scores:
            self.cur_length_scores = self.length_scores[self.current_sen_id]
            self.max_length = max(self.cur_length_scores)

    def _no_closing_bracket(self):
        return {i: utils.NEG_INF for i in self.closing_bracket_ids}
    
    def predict_next(self, words):
        """If the maximum depth is reached, exclude all opening
        brackets. If history is not balanced, exclude EOS. If the
        current depth is zero, exclude closing brackets.
        
        Args:
            words (list): Set of words to score
        Returns:
            dict.
        """
        if self.cur_depth == 0:
            # Balanced: Score EOS with extlengths, supress closing bracket
            if self.ends_with_opening:  # Initial predict next call
                ret = self._no_closing_bracket()
                ret[utils.EOS_ID] = utils.NEG_INF
                return ret
            return {utils.EOS_ID: self.cur_length_scores.get(
                        self.n_terminals, utils.NEG_INF) 
                       if self.length_scores else 0.0}
        # Unbalanced: do not allow EOS
        ret = {utils.EOS_ID: utils.NEG_INF}
        if (self.cur_depth >= self.max_depth 
                or self.n_terminals >= self.max_length):
            # Do not allow opening brackets
            ret.update({w: utils.NEG_INF for w in words 
                if (w > self.max_terminal_id 
                    and not w in self.closing_bracket_ids)})
        if (self.length_scores 
                and self.cur_depth == 1 
                and self.n_terminals > 0 
                and not self.n_terminals in self.cur_length_scores):
            # Do not allow to go back to depth 0 with wrong number of terminals
            ret.update(self._no_closing_bracket())
        return ret
        
    def get_unk_probability(self, posterior):
        """Always returns 0.0"""
        if self.cur_depth == 0 and not self.ends_with_opening:
            return utils.NEG_INF 
        return 0.0
    
    def consume(self, word):
        """Updates current depth and the number of consumed terminals."""
        if word in self.closing_bracket_ids:
            if self.ends_with_opening:
                self.n_terminals += 1
            self.cur_depth -= 1
            self.ends_with_opening = False
        elif word > self.max_terminal_id:
            self.cur_depth += 1
            self.ends_with_opening = True
    
    def get_state(self):
        """Returns the current depth and number of consumed terminals"""
        return self.cur_depth, self.n_terminals, self.ends_with_opening
    
    def set_state(self, state):
        """Sets the current depth and number of consumed terminals"""
        self.cur_depth, self.n_terminals, self.ends_with_opening = state

    def is_equal(self, state1, state2):
        """Trivial implementation"""
        return state1 == state2 class OSMPredictor(Predictor):
    """This predictor applies the following constraints to an OSM output:

      - The number of POP tokens must be equal to the number of source
        tokens
      - JUMP_FWD and JUMP_BWD tokens are constraint to avoid jumping out 
        of bounds.

    The predictor supports the original OSNMT operation set (default)
    plus a number of variations that are set by the use_* arguments in
    the constructor.
    """
    
    def __init__(self, src_wmap, trg_wmap, use_jumps=True, use_auto_pop=False, 
                 use_unpop=False, use_pop2=False, use_src_eop=False,
                 use_copy=False):
        """Creates a new osm predictor.

        Args:
            src_wmap (string): Path to the source wmap. Used to grap
                               EOP id.
            trg_wmap (string): Path to the target wmap. Used to update
                               IDs of operations.
            use_jumps (bool): If true, use SET_MARKER, JUMP_FWD and
                              JUMP_BWD operations
            use_auto_pop (bool): If true, each word insertion
                                 automatically moves read head
            use_unpop (bool): If true, use SRC_UNPOP to move read head
                              to the left.
            use_pop2 (bool): If true, use two read heads to align
                             phrases
            use_src_eop (bool): If true, expect EOP tokens in the src
                                sentence
            use_copy (bool): If true, move read head at COPY operations
        """
        super(OSMPredictor, self).__init__()
        update_trg_osm_ids(trg_wmap)
        self.use_jumps = use_jumps
        self.use_auto_pop = use_auto_pop
        self.use_unpop = use_unpop
        self.use_src_eop = use_src_eop
        if use_src_eop:
            update_src_osm_ids(src_wmap)
        self.pop_ids = set([OSM_SRC_POP_ID])
        if use_pop2:
            self.pop_ids.add(OSM_SRC_POP2_ID)
        if use_copy:
            self.pop_ids.add(OSM_COPY_ID)
        self.illegal_sequences = []
        if use_jumps:
            self.illegal_sequences.extend([
                #[OSM_JUMP_FWD_ID, OSM_JUMP_BWD_ID],
                #[OSM_JUMP_BWD_ID, OSM_JUMP_FWD_ID],
                #[OSM_JUMP_FWD_ID, OSM_SET_MARKER_ID, OSM_JUMP_FWD_ID],
                #[OSM_JUMP_FWD_ID, OSM_SET_MARKER_ID, OSM_JUMP_BWD_ID],
                #[OSM_JUMP_BWD_ID, OSM_SET_MARKER_ID, OSM_JUMP_FWD_ID],
                #[OSM_JUMP_BWD_ID, OSM_SET_MARKER_ID, OSM_JUMP_BWD_ID],
                [OSM_SET_MARKER_ID, OSM_SET_MARKER_ID]
            ])
        if use_auto_pop:
            self.no_auto_pop = set()
            if use_jumps:
                self.no_auto_pop.add(OSM_JUMP_FWD_ID)
                self.no_auto_pop.add(OSM_JUMP_BWD_ID)
                self.no_auto_pop.add(OSM_SET_MARKER_ID)
            if use_unpop:
                self.no_auto_pop.add(OSM_SRC_UNPOP_ID)

    def _is_pop(self, token):
        if token in self.pop_ids:
            return True
        return self.use_auto_pop and token not in self.no_auto_pop
    
    def initialize(self, src_sentence):
        """Sets the number of source tokens.
        
        Args:
            src_sentence (list): Not used
        """
        if self.use_src_eop:
            self.src_len = src_sentence.count(OSM_EOP_ID) + 1
        else:
            self.src_len = len(src_sentence)
        self.n_holes = 0
        self.head = 0
        self.n_pop = 0
        self.history = []

    def predict_next(self):
        """Apply OSM constraints.
        
        Returns:
            dict.
        """
        ret = {}
        if self.n_pop >= self.src_len:
            return {utils.EOS_ID: 0.0} # Force EOS
        else:
            ret[utils.EOS_ID] = utils.NEG_INF
        if self.use_unpop and self.n_pop <= 0:
            ret[OSM_SRC_UNPOP_ID] = utils.NEG_INF
        if self.use_jumps:
            if self.head <= 0:
                ret[OSM_JUMP_BWD_ID] = utils.NEG_INF
            if self.head >= self.n_holes:
                ret[OSM_JUMP_FWD_ID] = utils.NEG_INF
        for seq in self.illegal_sequences:
            hist = seq[:-1]
            if self.history[-len(hist):] == hist:
                ret[seq[-1]] = utils.NEG_INF
        return ret
        
    def get_unk_probability(self, posterior):
        if self.n_pop >= self.src_len: # Force EOS
            return utils.NEG_INF
        return 0.0
    
    def consume(self, word):
        """Updates the number of holes, EOPs, and the head position."""
        if not self._is_pop(word):
            if self.use_unpop and word == OSM_SRC_UNPOP_ID:
                self.n_pop -= 1
            else:
                self.history.append(word)
        else:
            self.n_pop += 1
        if self.use_jumps:
            if word == OSM_SET_MARKER_ID:
                self.n_holes += 1
                self.head += 1
            elif word == OSM_JUMP_FWD_ID:
                self.head += 1
            elif word == OSM_JUMP_BWD_ID:
                self.head -= 1
    
    def get_state(self):
        return self.n_holes, self.head, self.n_pop
    
    def set_state(self, state):
        self.n_holes, self.head, self.n_pop = state

    def is_equal(self, state1, state2):
        """Trivial implementation"""
        return state1 == state2 class ForcedOSMPredictor(Predictor):
    """This predictor allows forced decoding with an OSM output, which
    essentially means running the OSM in alignment mode. This predictor
    assumes well-formed operation sequences. Please combine this
    predictor with the osm constraint predictor to satisfy this
    requirement. The state of this predictor is the compiled version of
    the current history. It allows terminal symbols which are 
    consistent with the reference. The end-of-sentence symbol is
    supressed until all words in the reference have been consumed.
    """
    
    def __init__(self, trg_wmap, trg_test_file):
        """Creates a new forcedosm predictor.

        Args:
            trg_wmap (string): Path to the target wmap file. Used to
                               grap OSM operation IDs.
            trg_test_file (string): Path to the plain text file with 
                                    the target sentences. Must have the
                                    same number of lines as the number
                                    of source sentences to decode
        """
        super(ForcedOSMPredictor, self).__init__()
        update_trg_osm_ids(trg_wmap)
        self.trg_sentences = []
        with open(trg_test_file) as f:
            for line in f:
                self.trg_sentences.append([int(w) 
                            for w in line.strip().split()])
    
    def initialize(self, src_sentence):
        """Resets compiled and head.
        
        Args:
            src_sentence (list): Not used
        """
        self.compiled = ["X"]
        self.head = 0
        self.cur_trg_sentence = self.trg_sentences[self.current_sen_id] 

    def _is_complete(self):
        """Returns true if the compiled sentence contains the right
        number of terminals.
        """
        n_terminals = len([s for s in self.compiled if s != "X"])
        return n_terminals == len(self.cur_trg_sentence)

    def _generate_alignments(self, align_stub=[], compiled_start_pos=0, 
                             sentence_start_pos=0):
        for pos in range(compiled_start_pos, len(self.compiled)):
            if self.compiled[pos] != 'X':
                word = int(self.compiled[pos])
                for sen_pos in range(sentence_start_pos, 
                                     len(self.cur_trg_sentence)):
                    if self.cur_trg_sentence[sen_pos] == word:
                        self._generate_alignments(
                            align_stub + [(pos, sen_pos)],
                            pos+1,
                            sen_pos+1)
                return
        self.alignments.append(align_stub)

    def _align(self):
        possible_words = [set() for _ in range(len(self.compiled))]
        self.alignments = []
        self._generate_alignments(align_stub=[])
        for alignment in self.alignments:
            alignment.append((len(self.compiled), len(self.cur_trg_sentence)))
            prev_compiled_pos = -1
            prev_sentence_pos = -1
            for compiled_pos, sentence_pos in alignment:
               section_words = set(
                   self.cur_trg_sentence[prev_sentence_pos+1:sentence_pos])
               if section_words:
                   seen_gap = False
                   for section_pos in range(prev_compiled_pos+1, 
                                            compiled_pos):
                       if self.compiled[section_pos] == "X":
                           if seen_gap:
                               possible_words[section_pos] |= section_words
                           else:
                               possible_words[section_pos].add(
                                   self.cur_trg_sentence[prev_sentence_pos 
                                                         + section_pos 
                                                         - prev_compiled_pos])
                               seen_gap = True
               prev_compiled_pos = compiled_pos
               prev_sentence_pos = sentence_pos
        return possible_words

    def predict_next(self):
        """Apply word reference constraints.
        
        Returns:
            dict.
        """
        ret = {OSM_SRC_POP_ID: 0.0}
        possible_words = self._align()
        if possible_words[self.head]:
            ret[OSM_SET_MARKER_ID] = 0.0
        if any(possible_words[:self.head]):
            ret[OSM_JUMP_BWD_ID] = 0.0
        if any(possible_words[self.head+1:]):
            ret[OSM_JUMP_FWD_ID] = 0.0
        if self._is_complete():
            ret[utils.EOS_ID] = 0.0
        for word in possible_words[self.head]:
            ret[word] = 0.0
        return ret
        
    def get_unk_probability(self, posterior):
        """Always returns -inf."""
        return utils.NEG_INF

    def _jump_op(self, step):
        self.head += step
        while self.compiled[self.head] != "X":
            self.head += step

    def _insert_op(self, op):
        self.compiled = self.compiled[:self.head] + [op] + \
                        self.compiled[self.head:]
        self.head += 1
    
    def consume(self, word):
        """Updates the compiled string and the head position."""
        if word == OSM_SET_MARKER_ID:
            self._insert_op("X")
        elif word == OSM_JUMP_FWD_ID:
            self._jump_op(1)
        elif word == OSM_JUMP_BWD_ID:
            self._jump_op(-1)
        elif word != OSM_SRC_POP_ID:
            self._insert_op(str(word))
    
    def get_state(self):
        return self.compiled, self.head
    
    def set_state(self, state):
        self.compiled, self.head = state

    def is_equal(self, state1, state2):
        """Trivial implementation"""
        return state1 == state2 class UnboundedAltsrcPredictor(AltsrcPredictor, UnboundedVocabularyPredictor):
    """This class is a version of ``AltsrcPredictor`` for unbounded 
    vocabulary predictors. This needs an adjusted ``predict_next`` 
    method to pass through the set of target words to score correctly.
    """
    
    def __init__(self, src_test, slave_predictor):
        """Pass through to ``AltsrcPredictor.__init__`` """
        super(UnboundedAltsrcPredictor, self).__init__(src_test,
                                                       slave_predictor)

    def predict_next(self, trgt_words):
        """Pass through to slave predictor """
        return self.slave_predictor.predict_next(trgt_words) class AltsrcPredictor(Predictor):
    """This wrapper loads the source sentences from an alternative 
    source file. The ``src_sentence`` arguments of ``initialize`` and
    ``initialize_heuristic`` are overridden with sentences loaded from
    the file specified via the argument ``--altsrc_test``. All other
    methods are pass through calls to the slave predictor.
    """
    
    def __init__(self, src_test, slave_predictor):
        """Creates a new altsrc wrapper predictor.
        
        Args:
            src_test (string): Path to the text file with source
                               sentences
            slave_predictor (Predictor): Instance of the predictor which
                                         uses the source sentences in
                                         ``src_test``
        """
        super(AltsrcPredictor, self).__init__()
        self.slave_predictor = slave_predictor
        self.altsens = []
        with open(src_test) as f:
            self.altsens = [io.encode(line.strip()) for line in f]
    
    def _get_current_sentence(self):
        return self.altsens[self.current_sen_id]
    
    def initialize(self, src_sentence):
        """Pass through to slave predictor but replace 
        ``src_sentence`` with a sentence from ``self.altsens``
        """
        self.slave_predictor.initialize(self._get_current_sentence())
    
    def initialize_heuristic(self, src_sentence):
        """Pass through to slave predictor but replace 
        ``src_sentence`` with a sentence from ``self.altsens``
        """
        self.slave_predictor.initialize_heuristic(self._get_current_sentence())
    
    def predict_next(self):
        """Pass through to slave predictor """
        return self.slave_predictor.predict_next()
        
    def get_unk_probability(self, posterior):
        """Pass through to slave predictor """
        return self.slave_predictor.get_unk_probability(posterior)
    
    def consume(self, word):
        """Pass through to slave predictor """
        self.slave_predictor.consume(word)
    
    def get_state(self):
        """Pass through to slave predictor """
        return self.slave_predictor.get_state()
    
    def set_state(self, state):
        """Pass through to slave predictor """
        self.slave_predictor.set_state(state)

    def estimate_future_cost(self, hypo):
        """Pass through to slave predictor """
        return self.slave_predictor.estimate_future_cost(hypo)

    def set_current_sen_id(self, cur_sen_id):
        """We need to override this method to propagate current\_
        sentence_id to the slave predictor
        """
        super(AltsrcPredictor, self).set_current_sen_id(cur_sen_id)
        self.slave_predictor.set_current_sen_id(cur_sen_id)
    
    def is_equal(self, state1, state2):
        """Pass through to slave predictor """
        return self.slave_predictor.is_equal(state1, state2) class UnboundedRankPredictor(RankPredictor, UnboundedVocabularyPredictor):
    """This class is a version of ``RankPredictor`` for unbounded 
    vocabulary predictors. This needs an adjusted ``predict_next`` 
    method to pass through the set of target words to score correctly.
    """
    
    def predict_next(self, trgt_words):
        """Return ranks instead of slave scores """
        return self.score2rank(self.slave_predictor.predict_next(trgt_words)) class RankPredictor(Predictor):
    """This wrapper converts predictor scores to (negative) ranks, i.e.
    the best word gets a score of -1, the second best of -2 and so on.

    Note: Using this predictor with UNK matching or predictor heuristics
    is not recommended.
    """
    
    def __init__(self, slave_predictor):
        """Creates a new rank wrapper predictor.
        
        Args:
            slave_predictor (Predictor): Use score of this predictor to
                                         compute ranks.
        """
        super(RankPredictor, self).__init__()
        self.slave_predictor = slave_predictor
    
    def score2rank(self, scores):
        if isinstance(scores, dict):
            l = list(utils.common_iterable(scores))
            l.sort(key=operator.itemgetter(1), reverse=True)
            return {el[0]: -(r+1) for r, el in enumerate(l)}
        # scores is a list or numpy array.
        indices = np.argsort(-scores)
        ranks = np.empty_like(scores)
        ranks[indices] =  np.arange(-1, -len(scores)-1, -1, dtype=scores.dtype)
        return ranks
    
    def initialize(self, src_sentence):
        """Pass through to slave predictor. """
        self.slave_predictor.initialize(src_sentence)
    
    def initialize_heuristic(self, src_sentence):
        """Pass through to slave predictor. """
        self.slave_predictor.initialize_heuristic(src_sentence)
    
    def predict_next(self):
        """Pass through to slave predictor """
        return self.score2rank(self.slave_predictor.predict_next())
        
    def get_unk_probability(self, posterior):
        """Pass through to slave predictor """
        return self.slave_predictor.get_unk_probability(posterior)
    
    def consume(self, word):
        """Pass through to slave predictor """
        self.slave_predictor.consume(word)
    
    def get_state(self):
        """Pass through to slave predictor """
        return self.slave_predictor.get_state()
    
    def set_state(self, state):
        """Pass through to slave predictor """
        self.slave_predictor.set_state(state)

    def estimate_future_cost(self, hypo):
        """Pass through to slave predictor """
        return self.slave_predictor.estimate_future_cost(hypo)

    def set_current_sen_id(self, cur_sen_id):
        """We need to override this method to propagate current\_
        sentence_id to the slave predictor
        """
        super(RankPredictor, self).set_current_sen_id(cur_sen_id)
        self.slave_predictor.set_current_sen_id(cur_sen_id)
    
    def is_equal(self, state1, state2):
        """Pass through to slave predictor """
        return self.slave_predictor.is_equal(state1, state2) class UnboundedGluePredictor(GluePredictor, UnboundedVocabularyPredictor):
    """This class is a version of ``GluePredictor`` for unbounded 
    vocabulary predictors.
    """

    def predict_next(self, trgt_words):
        """Pass through to slave predictor """
        return self.slave_predictor.predict_next(trgt_words) class GluePredictor(Predictor):
    """This wrapper masks sentence-level predictors when SGNMT runs on
    the document level. The SGNMT hypotheses consist of multiple
    sentences, glued together with <s>, but the wrapped predictor is
    trained on the sentence level. This predictor splits input
    sequences at <s> and feed them to the predictor one by one. The
    wrapped predictor is initialized with a new source sentence when
    the sentence boundary symbol <s> is emitted. Note that using the
    predictor heuristic of the wrapped predictor estimates the future
    cost for the current sentence, not for the entire document.
    """
    
    def __init__(self, max_len_factor, slave_predictor):
        """Creates a new glue wrapper predictor.
        
        Args:
            max_len_factor (int): Target sentences cannot be longer
                                  than this times source sentence length
            slave_predictor (Predictor): Instance of the sentence-level
                                         predictor.
        """
        super(GluePredictor, self).__init__()
        self.slave_predictor = slave_predictor
        self.max_len_factor = max_len_factor
    
    def initialize(self, src_sentence):
        """Splits ``src_sentence`` at ``utils.GO_ID``, stores all
        segments for later use, and calls ``initialize()`` of the
        slave predictor with the first segment.
        """
        self._src_sentences = []
        last_pos = 0
        for pos, word in enumerate(src_sentence):
            if word == utils.GO_ID:
                self._src_sentences.append(src_sentence[last_pos:pos])
                last_pos = pos + 1
        self._src_sentences.append(src_sentence[last_pos:])
        self._next_src_sentence_idx = 0
        self._next_src_sentence()

    def _next_src_sentence(self):
        src = self._src_sentences[self._next_src_sentence_idx]
        self.slave_predictor.initialize(src)
        self._sen_budget = self.max_len_factor * len(src)
        self._next_src_sentence_idx += 1

    def _sen_len_exceeded(self):
        return self._sen_budget < 0

    def is_last_sentence(self):
        """Returns True if the current sentence is the last sentence
        in this document - i.e. we have already consumed n-1 <s>
        symbols since the last call of ``initialize()``.
        """
        return self._next_src_sentence_idx >= len(self._src_sentences)
    
    def predict_next(self):
        """Calls predict_next() of the wrapped predictor. Replaces BOS
        scores with EOS score if we still have source sentences left.
        """
        if self._sen_len_exceeded():
            end_token = utils.EOS_ID if self.is_last_sentence() else utils.GO_ID
            logging.info("Sentence length exceeded!")
            return {end_token: 0.0}
        slave_scores = self.slave_predictor.predict_next()
        if not self.is_last_sentence():
            slave_scores[utils.GO_ID] = slave_scores[utils.EOS_ID]
            slave_scores[utils.EOS_ID] = utils.NEG_INF
        else:
            slave_scores[utils.GO_ID] = utils.NEG_INF
        return slave_scores
        
    def consume(self, word):
        """If ``word`` is <s>, initialize the slave predictor with the
        next source sentence. Otherwise, pass through ``word`` to the
        ``consume()`` method of the slave.
        """
        if word == utils.GO_ID:
            self._next_src_sentence()
        else:
            self.slave_predictor.consume(word)
            self._sen_budget -= 1
    
    def get_state(self):
        """State is the slave state plus the source sentence index."""
        return (self._next_src_sentence_idx,
                self._sen_budget,
                self.slave_predictor.get_state())
    
    def set_state(self, state):
        """State is the slave state plus the source sentence index."""
        self._next_src_sentence_idx, self._sen_budget, slave_state = state
        self.slave_predictor.set_state(slave_state)

    def initialize_heuristic(self, src_sentence):
        """Pass through to slave predictor."""
        self.slave_predictor.initialize_heuristic(self._get_current_sentence())

    def get_unk_probability(self, posterior):
        """Pass through to slave predictor """
        if self._sen_len_exceeded():
            return utils.NEG_INF
        return self.slave_predictor.get_unk_probability(posterior)

    def estimate_future_cost(self, hypo):
        """Pass through to slave predictor """
        return self.slave_predictor.estimate_future_cost(hypo)

    def set_current_sen_id(self, cur_sen_id):
        """We need to override this method to propagate current\_
        sentence_id to the slave predictor
        """
        super(GluePredictor, self).set_current_sen_id(cur_sen_id)
        self.slave_predictor.set_current_sen_id(cur_sen_id)
    
    def is_equal(self, state1, state2):
        """Pass through to slave predictor """
        return self.slave_predictor.is_equal(state1, state2) class IdxmapPredictor(Predictor):
    """This wrapper predictor can be applied to slave predictors which
    use different wmaps than SGNMT. It translates between SGNMT word 
    indices and predictors indices each time the predictor is called.
    This mapping is transparent to both the decoder and the wrapped
    slave predictor.
    """
    
    def __init__(self,
                 src_idxmap_path,
                 trgt_idxmap_path,
                 slave_predictor,
                 slave_weight):
        """Creates a new idxmap wrapper predictor. The index maps have
        to be plain text files, each line containing the mapping from
        a SGNMT word index to the slave predictor word index.
        
        Args:
            src_idxmap_path (string): Path to the source index map
            trgt_idxmap_path (string): Path to the target index map
            slave_predictor (Predictor): Instance of the predictor with
                                         a different wmap than SGNMT
            slave_weight (float): Slave predictor weight
        """
        super(IdxmapPredictor, self).__init__()
        self.slave_predictor = slave_predictor
        self.slave_weight = slave_weight
        # src_map goes from sgnmt index -> slave index for the source 
        # trgt map goes from sgnmt index -> slave index for the target 
        # trgt map_inverse goes from slave index -> sgnmt index for the target 
        self.src_map = self.load_map(src_idxmap_path, "source")
        self.trgt_map = self.load_map(trgt_idxmap_path, "target")
        self.trgt_map_inverse = {slave_idx: gnmt_idx 
                        for gnmt_idx, slave_idx in enumerate(self.trgt_map)}
    
    def load_map(self, path, name):
        """Load a index map file. Mappings should be bijections, but
        there is no sanity check in place to verify this.
        
        Args:
            path (string): Path to the mapping file
            name (string): 'source' or 'target' for error messages
        
        Returns:
            dict. Mapping from SGNMT index to slave predictor index
        """
        if not path:
            logging.info("%s-side identity mapping (no idxmap specified)" % name)
            return {}
        with open(path) as f:
            d = dict(tuple(map(int, line.strip().split(None, 1))) for line in f)
            if (d[utils.UNK_ID] != utils.UNK_ID
                    or d[utils.EOS_ID] != utils.EOS_ID
                    or d[utils.GO_ID] != utils.GO_ID):
                logging.fatal(
                   "idxmap %s contains non-identical maps for reserved indices"
                        % path)
            logging.debug("Loaded idxmap from %s" % path)
            return [d[idx] if idx in d else 0 for idx in range(max(d)+1)]
    
    def initialize(self, src_sentence):
        """Pass through to slave predictor """
        if not self.src_map:
            self.slave_predictor.initialize(src_sentence)
        else:
            self.slave_predictor.initialize([self.src_map[idx]
                                            for idx in src_sentence])
    
    def predict_next(self):
        """Pass through to slave predictor """
        if not self.trgt_map:
            return self.slave_predictor.predict_next()
        posterior = self.slave_predictor.predict_next()
        return {self.trgt_map_inverse.get(idx, utils.UNK_ID): self.slave_weight * prob 
            for idx, prob in utils.common_iterable(posterior)}
        
    def get_unk_probability(self, posterior):
        """ATTENTION: We should translate the posterior array 
        back to slave predictor indices. However, the unk_id is 
        translated to the identical index, and others normally do not
        matter when computing the UNK probability. Therefore, we 
        refrain from a complete conversion and pass through
        ``posterior`` without changing its word indices.
        """
        return self.slave_predictor.get_unk_probability(posterior)
    
    def consume(self, word):
        """Pass through to slave predictor """
        if not self.trgt_map:
            self.slave_predictor.consume(word)
        else:
            self.slave_predictor.consume(utils.common_get(
                self.trgt_map, word, utils.UNK_ID))
    
    def get_state(self):
        """Pass through to slave predictor """
        return self.slave_predictor.get_state()
    
    def set_state(self, state):
        """Pass through to slave predictor """
        self.slave_predictor.set_state(state)

    def estimate_future_cost(self, hypo):
        """Pass through to slave predictor """
        if not self.trgt_map:
            return self.slave_predictor.estimate_future_cost(hypo)
        old_sen = hypo.trgt_sentence
        hypo.trgt_sentence = [self.trgt_map[idx] for idx in old_sen]
        ret = self.slave_predictor.estimate_future_cost(hypo)
        hypo.trgt_sentence = old_sen
        return ret

    def initialize_heuristic(self, src_sentence):
        """Pass through to slave predictor """
        if self.src_map:
            self.slave_predictor.initialize_heuristic([self.src_map[idx] 
                                                    for idx in src_sentence])

    def set_current_sen_id(self, cur_sen_id):
        """We need to override this method to propagate current\_
        sentence_id to the slave predictor
        """
        super(IdxmapPredictor, self).set_current_sen_id(cur_sen_id)
        self.slave_predictor.set_current_sen_id(cur_sen_id)
    
    def is_equal(self, state1, state2):
        """Pass through to slave predictor """
        return self.slave_predictor.is_equal(state1, state2) class UnboundedIdxmapPredictor(IdxmapPredictor, UnboundedVocabularyPredictor):
    """This class is a version of ``IdxmapPredictor`` for unbounded 
    vocabulary predictors. This needs an adjusted ``predict_next`` 
    method to pass through the set of target words to score correctly.
    """
    
    def __init__(self,
                 src_idxmap_path,
                 trgt_idxmap_path,
                 slave_predictor,
                 slave_weight):
        """Pass through to ``IdxmapPredictor.__init__`` """
        super(UnboundedIdxmapPredictor, self).__init__(src_idxmap_path,
                                                       trgt_idxmap_path,
                                                       slave_predictor,
                                                       slave_weight)

    def predict_next(self, trgt_words):
        """Pass through to slave predictor """
        if not self.trgt_map:
            return self.slave_predictor.predict_next(trgt_words)
        posterior = self.slave_predictor.predict_next([self.trgt_map[w] 
                                                       for w in trgt_words])
        return {self.trgt_map_inverse.get(idx,
                                          utils.UNK_ID): self.slave_weight*prob 
                            for idx, prob in utils.common_iterable(posterior)} class MaskvocabPredictor(Predictor):
    """This wrapper predictor hides certain words in the SGNMT 
    vocabulary from the predictor. Those words are scored by the
    masked predictor with zero. The wrapper passes through consume()
    only for other words.
    """
    
    def __init__(self, vocab_spec, slave_predictor):
        """Creates a new maskvocab wrapper predictor.
        
        Args:
            vocab_spec (string): Vocabulary specification (see VocabSpec)
            slave_predictor (Predictor): Instance of the predictor with
                                         a different wmap than SGNMT
        """
        super(MaskvocabPredictor, self).__init__()
        self.vocab_spec = VocabSpec(vocab_spec)
        self.slave_predictor = slave_predictor

    def initialize(self, src_sentence):
        """Pass through to slave predictor """
        self.slave_predictor.initialize(src_sentence)
    
    def predict_next(self):
        """Pass through to slave predictor, set masked to 0.0 """
        posterior = self.slave_predictor.predict_next()
        for w in self.vocab_spec.tokens:
            posterior[w] = 0.0
        return posterior
        
    def get_unk_probability(self, posterior):
        """Pass through to slave predictor """
        return self.slave_predictor.get_unk_probability(posterior)
    
    def consume(self, word):
        """Pass through to slave predictor """
        if not self.vocab_spec.contains(word):
            self.slave_predictor.consume(word)
    
    def get_state(self):
        """Pass through to slave predictor """
        return self.slave_predictor.get_state()
    
    def set_state(self, state):
        """Pass through to slave predictor """
        self.slave_predictor.set_state(state)

    def estimate_future_cost(self, hypo):
        """Pass through to slave predictor """
        return self.slave_predictor.estimate_future_cost(hypo)

    def initialize_heuristic(self, src_sentence):
        """Pass through to slave predictor """
        self.slave_predictor.initialize_heuristic(src_sentence)

    def set_current_sen_id(self, cur_sen_id):
        """We need to override this method to propagate current\_
        sentence_id to the slave predictor
        """
        super(MaskvocabPredictor, self).set_current_sen_id(cur_sen_id)
        self.slave_predictor.set_current_sen_id(cur_sen_id)
    
    def is_equal(self, state1, state2):
        """Pass through to slave predictor """
        return self.slave_predictor.is_equal(state1, state2) class UnboundedMaskvocabPredictor(MaskvocabPredictor,
                                  UnboundedVocabularyPredictor):
    """This class is a version of ``MaskvocabPredictor`` for unbounded 
    vocabulary predictors. This needs an adjusted ``predict_next`` 
    method to pass through the set of target words to score correctly.
    """
    
    def predict_next(self, trgt_words):
        """Pass through to slave predictor, set masked to 0.0 """
        posterior = self.slave_predictor.predict_next(trgt_words)
        for w in self.words:
            if utils.common_contains(trgt_words, w):
                posterior[w] = 0.0
        return posterior class UnkvocabPredictor(Predictor):
    """If the predictor wrapped by the unkvocab wrapper produces an UNK
    with predict next, this wrapper adds explicit NEG_INF scores to all
    in-vocabulary words not in its posterior. This can control which 
    words are matched by the UNK scores of other predictors.
    """
    
    def __init__(self, trg_vocab_size, slave_predictor):
        """Creates a new unkvocab wrapper predictor.
        
        Args:
            trg_vocab_size (int): Size of the target vocabulary
        """
        super(UnkvocabPredictor, self).__init__()
        self.slave_predictor = slave_predictor
        self.trg_vocab_size = trg_vocab_size
    
    def initialize(self, src_sentence):
        """Pass through to slave predictor """
        self.slave_predictor.initialize(src_sentence)
    
    def initialize_heuristic(self, src_sentence):
        """Pass through to slave predictor """
        self.slave_predictor.initialize_heuristic(src_sentence)
 
    def predict_next(self):
        """Pass through to slave predictor. If the posterior from the
        slave predictor contains util.UNK_ID, add NEG_INF for all 
        word ids lower than trg_vocab_size that are not already
        defined """
        posterior = self.slave_predictor.predict_next()
        if utils.UNK_ID in posterior:
            for w in range(self.trg_vocab_size):
                if not w in posterior:
                    posterior[w] = utils.NEG_INF
        return posterior
        
    def get_unk_probability(self, posterior):
        """Pass through to slave predictor """
        return self.slave_predictor.get_unk_probability(posterior)
    
    def consume(self, word):
        """Pass through to slave predictor """
        self.slave_predictor.consume(word)
    
    def get_state(self):
        """Pass through to slave predictor """
        return self.slave_predictor.get_state()
    
    def set_state(self, state):
        """Pass through to slave predictor """
        self.slave_predictor.set_state(state)

    def estimate_future_cost(self, hypo):
        """Pass through to slave predictor """
        return self.slave_predictor.estimate_future_cost(hypo)

    def set_current_sen_id(self, cur_sen_id):
        """We need to override this method to propagate current\_
        sentence_id to the slave predictor
        """
        super(UnkvocabPredictor, self).set_current_sen_id(cur_sen_id)
        self.slave_predictor.set_current_sen_id(cur_sen_id)
    
    def is_equal(self, state1, state2):
        """Pass through to slave predictor """
        return self.slave_predictor.is_equal(state1, state2) class SkipvocabPredictor(Predictor):
    """This predictor wrapper masks predictors with a larger vocabulary
    than the SGNMT vocabulary. The SGNMT OOV words are not scored with 
    UNK scores from the other predictors like usual, but are hidden by 
    this wrapper. Therefore, this wrapper does not produce any word
    from the larger vocabulary, but searches internally until enough
    in-vocabulary word scores are collected from the wrapped predictor.
    """
    
    def __init__(self, vocab_spec, stop_size, beam, slave_predictor):
        """Creates a new skipvocab wrapper predictor.
        
        Args:
            vocab_spec (string): Vocabulary specification (see VocabSpec)
            stop_size (int): Stop internal beam search when the best
                             stop_size words are in-vocabulary
            beam (int): Beam size of internal beam search
            slave_predictor (Predictor): Wrapped predictor.
        """
        super(SkipvocabPredictor, self).__init__()
        self.vocab_spec = VocabSpec(vocab_spec)
        self.slave_predictor = slave_predictor
        self.stop_size = stop_size
        self.beam = beam
    
    def initialize(self, src_sentence):
        """Pass through to slave predictor """
        self.slave_predictor.initialize(src_sentence)
    
    def initialize_heuristic(self, src_sentence):
        """Pass through to slave predictor """
        self.slave_predictor.initialize_heuristic(src_sentence)

    def get_unk_probability(self, posterior):
        """Pass through to slave predictor """
        return self.slave_predictor.get_unk_probability(posterior)

    def _is_stopping_posterior(self, posterior):
        for word, _ in sorted(utils.common_iterable(posterior),
                              key=lambda h: -h[1])[:self.stop_size]:
            if self.vocab_spec.contains(word):
                return False
        return True
 
    def predict_next(self):
        """This method first performs beam search internally to update
        the slave predictor state to a point where the best stop_size 
        entries in the predict_next() return value are in-vocabulary
        (bounded by max_id). Then, it returns the slave posterior in 
        that state.
        """
        hypos = [SkipvocabInternalHypothesis(0.0, 
                                             self.slave_predictor.get_state(),
                                             None)]
        best_score = utils.NEG_INF
        best_predictor_state = None
        best_posterior = None
        while hypos and hypos[0].score > best_score:
            next_hypos = []
            for hypo in hypos:
                self.slave_predictor.set_state(copy.deepcopy(
                    hypo.predictor_state))
                if hypo.word_to_consume is not None:
                    self.slave_predictor.consume(hypo.word_to_consume)
                posterior = self.slave_predictor.predict_next()
                pred_state = copy.deepcopy(self.slave_predictor.get_state())
                if (self._is_stopping_posterior(posterior) 
                        and hypo.score > best_score):
                    # This is the new best result of the internal beam search
                    best_score = hypo.score
                    best_predictor_state = pred_state
                    best_posterior = posterior
                else:
                    # Look for ways to expand this hypo with OOV words.
                    for word, score in utils.common_iterable(posterior):
                        if self.vocab_spec.contains(word):
                            next_hypos.append(SkipvocabInternalHypothesis(
                                hypo.score + score, pred_state, word))
            next_hypos.sort(key=lambda h: -h.score)
            hypos = next_hypos[:self.beam]
        self.slave_predictor.set_state(copy.deepcopy(best_predictor_state))
        return best_posterior
        
    def consume(self, word):
        """Pass through to slave predictor """
        self.slave_predictor.consume(word)
    
    def get_state(self):
        """Pass through to slave predictor """
        return self.slave_predictor.get_state()
    
    def set_state(self, state):
        """Pass through to slave predictor """
        self.slave_predictor.set_state(state)

    def estimate_future_cost(self, hypo):
        """Pass through to slave predictor """
        return self.slave_predictor.estimate_future_cost(hypo)

    def set_current_sen_id(self, cur_sen_id):
        """We need to override this method to propagate current\_
        sentence_id to the slave predictor
        """
        super(SkipvocabPredictor, self).set_current_sen_id(cur_sen_id)
        self.slave_predictor.set_current_sen_id(cur_sen_id)
    
    def is_equal(self, state1, state2):
        """Pass through to slave predictor """
        return self.slave_predictor.is_equal(state1, state2) class KenLMPredictor(UnboundedVocabularyPredictor):
    """KenLM predictor based on
    https://github.com/kpu/kenlm 
    
    The predictor state is described by the n-gram history.
    """
    
    def __init__(self, path):
        """Creates a new n-gram language model predictor.
        
        Args:
            path (string): Path to the ARPA language model file
            
        Raises:
            NameError. If KenLM is not installed
        """
        super(KenLMPredictor, self).__init__()
        self.lm = kenlm.Model(path)
        self.lm_state2 = kenlm.State()
    
    def initialize(self, src_sentence):
        """Initializes the KenLM state.
        
        Args:
            src_sentence (list): Not used
        """
        self.history = []
        self._update_lm_state()

    def _update_lm_state(self):
        self.lm_state = kenlm.State()
        tmp_state = kenlm.State()
        self.lm.BeginSentenceWrite(self.lm_state)
        for w in self.history[-6:]:
            self.lm.BaseScore(self.lm_state, w, tmp_state)
            self.lm_state, tmp_state = tmp_state, self.lm_state
    
    def predict_next(self, words):
        return {w: self.lm.BaseScore(self.lm_state, 
                                     "</s>" if w == utils.EOS_ID else str(w),
                                     self.lm_state2)
                for w in words}
    
        
    def get_unk_probability(self, posterior):
        """Use the probability for '<unk>' in the language model """
        return self.lm.BaseScore(self.lm_state, "<unk>", self.lm_state2)
    
    def consume(self, word):
        self.lm.BaseScore(self.lm_state, str(word), self.lm_state2)
        self.lm_state, self.lm_state2 = self.lm_state2, self.lm_state
        self.history.append(str(word))
    
    def get_state(self):
        return self.lm_state.clone()
    
    def get_state(self):
        """Returns the current n-gram history """
        return self.history
    
    def set_state(self, state):
        """Sets the current n-gram history and LM state """
        self.history = state
        self._update_lm_state()

    def is_equal(self, state1, state2):
        return state1 == state2 class T2TPredictor(_BaseTensor2TensorPredictor):
    """This predictor implements scoring with Tensor2Tensor models. We
    follow the decoder implementation in T2T and do not reuse network
    states in decoding. We rather compute the full forward pass along
    the current history. Therefore, the decoder state is simply the
    the full history of consumed words.
    """

    def __init__(self,
                 src_vocab_size,
                 trg_vocab_size,
                 model_name,
                 problem_name,
                 hparams_set_name,
                 t2t_usr_dir,
                 checkpoint_dir,
                 t2t_unk_id=None,
                 n_cpu_threads=-1,
                 max_terminal_id=-1,
                 pop_id=-1):
        """Creates a new T2T predictor. The constructor prepares the
        TensorFlow session for predict_next() calls. This includes:
        - Load hyper parameters from the given set (hparams)
        - Update registry, load T2T model
        - Create TF placeholders for source sequence and target prefix
        - Create computation graph for computing log probs.
        - Create a MonitoredSession object, which also handles 
          restoring checkpoints.
        
        Args:
            src_vocab_size (int): Source vocabulary size.
            trg_vocab_size (int): Target vocabulary size.
            model_name (string): T2T model name.
            problem_name (string): T2T problem name.
            hparams_set_name (string): T2T hparams set name.
            t2t_usr_dir (string): See --t2t_usr_dir in tensor2tensor.
            checkpoint_dir (string): Path to the T2T checkpoint 
                                     directory. The predictor will load
                                     the top most checkpoint in the 
                                     `checkpoints` file.
            t2t_unk_id (int): If set, use this ID to get UNK scores. If
                              None, UNK is always scored with -inf.
            n_cpu_threads (int): Number of TensorFlow CPU threads.
            max_terminal_id (int): If positive, maximum terminal ID. Needs to
                be set for syntax-based T2T models.
            pop_id (int): If positive, ID of the POP or closing bracket symbol.
                Needs to be set for syntax-based T2T models.
        """
        super(T2TPredictor, self).__init__(t2t_usr_dir, 
                                           checkpoint_dir, 
                                           src_vocab_size,
                                           trg_vocab_size,
                                           t2t_unk_id, 
                                           n_cpu_threads,
                                           max_terminal_id,
                                           pop_id)
        if not model_name or not problem_name or not hparams_set_name:
            logging.fatal(
                "Please specify t2t_model, t2t_problem, and t2t_hparams_set!")
            raise AttributeError
        self.consumed = []
        self.src_sentence = []
        predictor_graph = tf.Graph()
        with predictor_graph.as_default() as g:
            hparams = trainer_lib.create_hparams(hparams_set_name)
            self._add_problem_hparams(hparams, problem_name)
            translate_model = registry.model(model_name)(
                hparams, tf.estimator.ModeKeys.PREDICT)
            self._inputs_var = tf.placeholder(dtype=tf.int32, shape=[None],
                                              name="sgnmt_inputs")
            self._targets_var = tf.placeholder(dtype=tf.int32, shape=[None], 
                                               name="sgnmt_targets")
            features = {"inputs": expand_input_dims_for_t2t(self._inputs_var), 
                        "targets": expand_input_dims_for_t2t(self._targets_var)}
            translate_model.prepare_features_for_infer(features)
            translate_model._fill_problem_hparams_features(features)
            logits, _ = translate_model(features)
            logits = tf.squeeze(logits, [0, 1, 2, 3])
            self._log_probs = log_prob_from_logits(logits)
            self.mon_sess = self.create_session()

                
    def predict_next(self):
        """Call the T2T model in self.mon_sess."""
        log_probs = self.mon_sess.run(self._log_probs,
            {self._inputs_var: self.src_sentence,
             self._targets_var: utils.oov_to_unk(
                 self.consumed + [text_encoder.PAD_ID],
                 self.trg_vocab_size,
                 self._t2t_unk_id)})
        log_probs[text_encoder.PAD_ID] = utils.NEG_INF
        return log_probs
    
    def initialize(self, src_sentence):
        """Set src_sentence, reset consumed."""
        self.consumed = []
        self.src_sentence = utils.oov_to_unk(
            src_sentence + [text_encoder.EOS_ID], 
            self.src_vocab_size, self._t2t_unk_id)
   
    def consume(self, word):
        """Append ``word`` to the current history."""
        self.consumed.append(word)
    
    def get_state(self):
        """The predictor state is the complete history."""
        return self.consumed
    
    def set_state(self, state):
        """The predictor state is the complete history."""
        self.consumed = state

    def is_equal(self, state1, state2):
        """Returns true if the history is the same """
        return state1 == state2 class EditT2TPredictor(_BaseTensor2TensorPredictor):
    """This predictor can be used for T2T models conditioning on the
    full target sentence. The predictor state is a full target sentence.
    The state can be changed by insertions, substitutions, and deletions
    of single tokens, whereas each operation is encoded as SGNMT token
    in the following way:

      1xxxyyyyy: Insert the token yyyyy at position xxx.
      2xxxyyyyy: Replace the xxx-th word with the token yyyyy.
      3xxx00000: Delete the xxx-th token.
    """

    INS_OFFSET = 100000000
    SUB_OFFSET = 200000000
    DEL_OFFSET = 300000000

    POS_FACTOR = 100000
    MAX_SEQ_LEN = 999

    def __init__(self,
                 src_vocab_size,
                 trg_vocab_size,
                 model_name,
                 problem_name,
                 hparams_set_name,
                 trg_test_file,
                 beam_size,
                 t2t_usr_dir,
                 checkpoint_dir,
                 t2t_unk_id=None,
                 n_cpu_threads=-1,
                 max_terminal_id=-1,
                 pop_id=-1):
        """Creates a new edit T2T predictor. This constructor is
        similar to the constructor of T2TPredictor but creates a
        different computation graph which retrieves scores at each
        target position, not only the last one.
        
        Args:
            src_vocab_size (int): Source vocabulary size.
            trg_vocab_size (int): Target vocabulary size.
            model_name (string): T2T model name.
            problem_name (string): T2T problem name.
            hparams_set_name (string): T2T hparams set name.
            trg_test_file (string): Path to a plain text file with
                initial target sentences. Can be empty.
            beam_size (int): Determines how many substitutions and
                insertions are considered at each position.
            t2t_usr_dir (string): See --t2t_usr_dir in tensor2tensor.
            checkpoint_dir (string): Path to the T2T checkpoint 
                                     directory. The predictor will load
                                     the top most checkpoint in the 
                                     `checkpoints` file.
            t2t_unk_id (int): If set, use this ID to get UNK scores. If
                              None, UNK is always scored with -inf.
            n_cpu_threads (int): Number of TensorFlow CPU threads.
            max_terminal_id (int): If positive, maximum terminal ID. Needs to
                be set for syntax-based T2T models.
            pop_id (int): If positive, ID of the POP or closing bracket symbol.
                Needs to be set for syntax-based T2T models.
        """
        super(EditT2TPredictor, self).__init__(t2t_usr_dir, 
                                               checkpoint_dir, 
                                               src_vocab_size,
                                               trg_vocab_size,
                                               t2t_unk_id, 
                                               n_cpu_threads,
                                               max_terminal_id,
                                               pop_id)
        if not model_name or not problem_name or not hparams_set_name:
            logging.fatal(
                "Please specify t2t_model, t2t_problem, and t2t_hparams_set!")
            raise AttributeError
        if trg_vocab_size >= EditT2TPredictor.POS_FACTOR:
            logging.fatal("Target vocabulary size (%d) must be less than %d!"
                          % (trg_vocab_size, EditT2TPredictor.POS_FACTOR))
            raise AttributeError
        self.beam_size = max(1, beam_size // 10) + 1
        self.batch_size = 2048 # TODO(fstahlberg): Move to config
        self.initial_trg_sentences = None
        if trg_test_file: 
            self.initial_trg_sentences = []
            with open(trg_test_file) as f:
                for line in f:
                    self.initial_trg_sentences.append(utils.oov_to_unk(
                       [int(w) for w in line.strip().split()] + [utils.EOS_ID],
                       self.trg_vocab_size, self._t2t_unk_id))
        predictor_graph = tf.Graph()
        with predictor_graph.as_default() as g:
            hparams = trainer_lib.create_hparams(hparams_set_name)
            self._add_problem_hparams(hparams, problem_name)
            translate_model = registry.model(model_name)(
                hparams, tf.estimator.ModeKeys.EVAL)
            self._inputs_var = tf.placeholder(dtype=tf.int32, shape=[None],
                                              name="sgnmt_inputs")
            self._targets_var = tf.placeholder(dtype=tf.int32, shape=[None, None], 
                                               name="sgnmt_targets")
            shp = tf.shape(self._targets_var)
            bsz = shp[0]
            inputs = tf.tile(tf.expand_dims(self._inputs_var, 0), [bsz, 1])
            features = {"inputs": expand_input_dims_for_t2t(inputs,
                                                            batched=True), 
                        "targets": expand_input_dims_for_t2t(self._targets_var,
                                                             batched=True)}
            translate_model.prepare_features_for_infer(features)
            translate_model._fill_problem_hparams_features(features)
            logits, _ = translate_model(features)
            logits = tf.squeeze(logits, [2, 3])
            self._log_probs = log_prob_from_logits(logits)
            diag_logits = gather_2d(logits, tf.expand_dims(tf.range(bsz), 1))
            self._diag_log_probs = log_prob_from_logits(diag_logits)
            no_pad = tf.cast(tf.not_equal(
                self._targets_var, text_encoder.PAD_ID), tf.float32)
            flat_bsz = shp[0] * shp[1]
            word_scores = gather_2d(
                tf.reshape(self._log_probs, [flat_bsz, -1]),
                tf.reshape(self._targets_var, [flat_bsz, 1]))
            word_scores = tf.reshape(word_scores, (shp[0], shp[1])) * no_pad
            self._sentence_scores = tf.reduce_sum(word_scores, -1)
            self.mon_sess = self.create_session()

    def _ins_op(self, pos, token):
        """Returns a copy of trg sentence after an insertion."""
        return self.trg_sentence[:pos] + [token] + self.trg_sentence[pos:]

    def _sub_op(self, pos, token):
        """Returns a copy of trg sentence after a substitution."""
        ret = list(self.trg_sentence)
        ret[pos] = token
        return ret

    def _del_op(self, pos):
        """Returns a copy of trg sentence after a deletion."""
        return self.trg_sentence[:pos] + self.trg_sentence[pos+1:]

    def _top_n(self, scores, sort=False):
        """Sorted indices of beam_size best entries along axis 1"""
        costs = -scores
        costs[:, utils.EOS_ID] = utils.INF
        top_n_indices = np.argpartition(
            costs,
            self.beam_size, 
            axis=1)[:, :self.beam_size]
        if not sort:
            return top_n_indices
        b_indices = np.expand_dims(np.arange(top_n_indices.shape[0]), axis=1)
        sorted_indices = np.argsort(costs[b_indices, top_n_indices], axis=1)
        return top_n_indices[b_indices, sorted_indices]

    def predict_next(self):
        """Call the T2T model in self.mon_sess."""
        next_sentences = {}
        logging.debug("EditT2T: Exploring score=%f sentence=%s" 
                      % (self.cur_score, " ".join(map(str, self.trg_sentence))))
        n_trg_words = len(self.trg_sentence)
        if n_trg_words > EditT2TPredictor.MAX_SEQ_LEN:
            logging.warn("EditT2T: Target sentence exceeds maximum length (%d)"
                         % EDITT2TPredictor.MAX_SEQ_LEN)
            return {utils.EOS_ID: 0.0}
        # Substitutions
        log_probs = self.mon_sess.run(self._log_probs,
            {self._inputs_var: self.src_sentence,
             self._targets_var: [self.trg_sentence]})
        top_n = self._top_n(np.squeeze(log_probs, axis=0))
        for pos, cur_token in enumerate(self.trg_sentence[:-1]):
            offset = EditT2TPredictor.SUB_OFFSET 
            offset += EditT2TPredictor.POS_FACTOR * pos
            for token in top_n[pos]:
                if token != cur_token:
                    next_sentences[offset + token] = self._sub_op(pos, token)

        # Insertions
        if n_trg_words < EditT2TPredictor.MAX_SEQ_LEN - 1:
            ins_trg_sentences = np.full((n_trg_words, n_trg_words+1), 999)
            for pos in range(n_trg_words):
                ins_trg_sentences[pos, :pos] = self.trg_sentence[:pos]
                ins_trg_sentences[pos, pos+1:] = self.trg_sentence[pos:]
            diag_log_probs = self.mon_sess.run(self._diag_log_probs,
                {self._inputs_var: self.src_sentence,
                 self._targets_var: ins_trg_sentences})
            top_n = self._top_n(np.squeeze(diag_log_probs, axis=1))
            for pos in range(n_trg_words):
                offset = EditT2TPredictor.INS_OFFSET 
                offset += EditT2TPredictor.POS_FACTOR * pos
                for token in top_n[pos]:
                    next_sentences[offset + token] = self._ins_op(pos, token)
        # Deletions
        idx = EditT2TPredictor.DEL_OFFSET
        for pos in range(n_trg_words - 1): # -1: Do not delete EOS
            next_sentences[idx] = self._del_op(pos)
            idx += EditT2TPredictor.POS_FACTOR
        abs_scores = self._score(next_sentences, n_trg_words + 1)
        rel_scores = {i: s - self.cur_score 
                      for i, s in abs_scores.items()}
        rel_scores[utils.EOS_ID] = 0.0
        return rel_scores

    def _score(self, sentences, n_trg_words=1):
        max_n_sens = max(1, self.batch_size // n_trg_words)
        scores = {}
        batch_ids = []
        batch_sens = []
        for idx, trg_sentence in sentences.items():
            score = self.cache.get(trg_sentence)
            if score is None:
                batch_ids.append(idx)
                np_sen = np.zeros(n_trg_words, dtype=np.int)
                np_sen[:len(trg_sentence)] = trg_sentence
                batch_sens.append(np_sen)
                if len(batch_ids) >= max_n_sens:
                    self._score_single_batch(scores, batch_ids, batch_sens)
                    batch_ids = []
                    batch_sens = []
            else:
                scores[idx] = score
        self._score_single_batch(scores, batch_ids, batch_sens)
        return scores 

    def _score_single_batch(self, scores, ids, trg_sentences):
        "Score sentences and add them to scores and the cache."""
        if not ids:
            return
        batch_scores = self.mon_sess.run(self._sentence_scores,
            {self._inputs_var: self.src_sentence,
             self._targets_var: np.stack(trg_sentences)})
        for idx, sen, score in zip(ids, trg_sentences, batch_scores):
            self.cache.add(sen, score)
            scores[idx] = score

    def _update_cur_score(self):
        self.cur_score = self.cache.get(self.trg_sentence)
        if self.cur_score is None:
            scores = self._score({1: self.trg_sentence}, len(self.trg_sentence))
            self.cur_score = scores[1]
            self.cache.add(self.trg_sentence, self.cur_score)
    
    def initialize(self, src_sentence):
        """Set src_sentence, reset consumed."""
        if self.initial_trg_sentences is None:
            self.trg_sentence = [text_encoder.EOS_ID]
        else:
            self.trg_sentence = self.initial_trg_sentences[self.current_sen_id]
        self.src_sentence = utils.oov_to_unk(
            src_sentence + [text_encoder.EOS_ID], 
            self.src_vocab_size, self._t2t_unk_id)
        self.cache = SimpleTrie()
        self._update_cur_score()
        logging.debug("Initial score: %f" % self.cur_score)
   
    def consume(self, word):
        """Append ``word`` to the current history."""
        if word == utils.EOS_ID:
            return
        pos = (word // EditT2TPredictor.POS_FACTOR) \
              % (EditT2TPredictor.MAX_SEQ_LEN + 1)
        token = word % EditT2TPredictor.POS_FACTOR
        # TODO(fstahlberg): Do not hard code the following section
        op = word // 100000000  
        if op == 1:  # Insertion
            self.trg_sentence = self._ins_op(pos, token)
        elif op == 2:  # Substitution
            self.trg_sentence = self._sub_op(pos, token)
        elif op == 3:  # Deletion
            self.trg_sentence = self._del_op(pos)
        else:
            logging.warn("Invalid edit descriptor %d. Ignoring..." % word)
        self._update_cur_score()
        self.cache.add(self.trg_sentence, utils.NEG_INF)
    
    def get_state(self):
        """The predictor state is the complete target sentence."""
        return self.trg_sentence, self.cur_score
    
    def set_state(self, state):
        """The predictor state is the complete target sentence."""
        self.trg_sentence, self.cur_score = state

    def is_equal(self, state1, state2):
        """Returns true if the target sentence is the same """
        return state1[0] == state2[0] class SegT2TPredictor(_BaseTensor2TensorPredictor):
    """This predictor is designed for document-level T2T models. It 
    differs from the normal t2t predictor in the following ways:

    - In addition to `input` and `targets`, it generates the features
      `inputs_seg`. `targets_seg`, `inputs_pos`, `targets_pos` which
      are used in glue models and the contextual Transformer.
    - The history is pruned when it exceeds a maximum number of <s> 
      symbols. This can be used to reduce complexity for document-level
      models on very long documents. When the maximum number is reached,
      we start removing sentences from ``self.consumed``, starting with
      the sentence which is `begin_margin` away from the document start
      and `end_margin` sentences away from the current sentence.
    """

    def __init__(self,
                 src_vocab_size,
                 trg_vocab_size,
                 model_name,
                 problem_name,
                 hparams_set_name,
                 t2t_usr_dir,
                 checkpoint_dir,
                 t2t_unk_id=None,
                 n_cpu_threads=-1,
                 max_terminal_id=-1,
                 pop_id=-1):
        """Creates a new document-level T2T predictor. See
        T2TPredictor.__init__().
        
        Args:
            src_vocab_size (int): Source vocabulary size.
            trg_vocab_size (int): Target vocabulary size.
            model_name (string): T2T model name.
            problem_name (string): T2T problem name.
            hparams_set_name (string): T2T hparams set name.
            t2t_usr_dir (string): See --t2t_usr_dir in tensor2tensor.
            checkpoint_dir (string): Path to the T2T checkpoint 
                                     directory. The predictor will load
                                     the top most checkpoint in the 
                                     `checkpoints` file.
            t2t_unk_id (int): If set, use this ID to get UNK scores. If
                              None, UNK is always scored with -inf.
            n_cpu_threads (int): Number of TensorFlow CPU threads.
            max_terminal_id (int): If positive, maximum terminal ID. Needs to
                be set for syntax-based T2T models.
            pop_id (int): If positive, ID of the POP or closing bracket symbol.
                Needs to be set for syntax-based T2T models.
        """
        super(SegT2TPredictor, self).__init__(t2t_usr_dir, 
                                             checkpoint_dir, 
                                             src_vocab_size,
                                             trg_vocab_size,
                                             t2t_unk_id, 
                                             n_cpu_threads,
                                             max_terminal_id,
                                             pop_id)
        if not model_name or not problem_name or not hparams_set_name:
            logging.fatal(
                "Please specify t2t_model, t2t_problem, and t2t_hparams_set!")
            raise AttributeError
        self.begin_margin = 3
        self.end_margin = 3
        self.max_sentences = self.begin_margin + self.end_margin
        self.max_sentences = 10000  # TODO: Make configurable. Default disabled
        #self.max_sentences = 20
        #self.max_sentences = 25
        predictor_graph = tf.Graph()
        with predictor_graph.as_default() as g:
            hparams = trainer_lib.create_hparams(hparams_set_name)
            self._add_problem_hparams(hparams, problem_name)
            translate_model = registry.model(model_name)(
                hparams, tf.estimator.ModeKeys.PREDICT)
            self._inputs_var = tf.placeholder(dtype=tf.int32, shape=[None],
                                              name="sgnmt_inputs")
            self._targets_var = tf.placeholder(dtype=tf.int32, shape=[None], 
                                               name="sgnmt_targets")
            self._inputs_seg_var = tf.placeholder(dtype=tf.int32, shape=[None],
                                                  name="sgnmt_inputs_seg")
            self._targets_seg_var = tf.placeholder(dtype=tf.int32, shape=[None], 
                                                   name="sgnmt_targets_seg")
            self._inputs_pos_var = tf.placeholder(dtype=tf.int32, shape=[None],
                                                  name="sgnmt_inputs_pos")
            self._targets_pos_var = tf.placeholder(dtype=tf.int32, shape=[None],
                                                   name="sgnmt_targets_pos")
            features = {
                "inputs": expand_input_dims_for_t2t(self._inputs_var), 
                "targets": expand_input_dims_for_t2t(self._targets_var),
                "inputs_seg": tf.expand_dims(self._inputs_seg_var, 0),
                "targets_seg": tf.expand_dims(self._targets_seg_var, 0),
                "inputs_pos": tf.expand_dims(self._inputs_pos_var, 0), 
                "targets_pos": tf.expand_dims(self._targets_pos_var, 0)
            }
            translate_model.prepare_features_for_infer(features)
            translate_model._fill_problem_hparams_features(features)
            logits, _ = translate_model(features)
            logits = tf.squeeze(logits, [0, 1, 2, 3])
            self._log_probs = log_prob_from_logits(logits)
            self.mon_sess = self.create_session()

    def initialize(self, src_sentence):
        self.consumed = []
        self.src_sentence = utils.oov_to_unk(
            src_sentence + [text_encoder.EOS_ID], 
            self.src_vocab_size, self._t2t_unk_id)
        self.src_seg, self.src_pos = self._gen_seg_and_pos(self.src_sentence)
        self.history_sentences = [[]]

    def _gen_seg_and_pos(self, glued, trg=False):
        seg = []
        pos = []
        cur_seg = 1
        cur_pos = 0
        for w in glued:
            seg.append(cur_seg)
            pos.append(cur_pos)
            if w == utils.GO_ID:
                cur_seg += 1
                cur_pos = 0
            else:
                cur_pos += 1
        if trg:
            seg.append(cur_seg)
            pos.append(cur_pos)
        return seg, pos
   
    def consume(self, word):
        self.history_sentences[-1].append(
            word if word < self.trg_vocab_size else self._t2t_unk_id)
        if word == utils.GO_ID:
            if False and len(self.history_sentences) > self.max_sentences:
                logging.debug("Pruning document level history...")
                self.history_sentences = (
                    self.history_sentences[:self.begin_margin] 
                    + self.history_sentences[-self.end_margin:])
            self.history_sentences.append([])
    
    def get_state(self):
        return self.history_sentences
    
    def set_state(self, state):
        self.history_sentences = state

    def predict_next(self):
        """Call the T2T model in self.mon_sess."""
        if len(self.history_sentences) > self.max_sentences:
            return {}
        consumed = [w for s in self.history_sentences for w in s]
        trg_seg, trg_pos = self._gen_seg_and_pos(consumed, trg=True)
        log_probs = self.mon_sess.run(self._log_probs,
            {self._inputs_var: self.src_sentence,
             self._inputs_seg_var: self.src_seg,
             self._inputs_pos_var: self.src_pos,
             self._targets_var: consumed + [text_encoder.PAD_ID],
             self._targets_seg_var: trg_seg,
             self._targets_pos_var: trg_pos})
        log_probs[text_encoder.PAD_ID] = utils.NEG_INF
        return log_probs
    
    def is_equal(self, state1, state2):
        """Returns true if the (pruned) history is the same """
        return state1 == state2

    def get_unk_probability(self, posterior):
        """Fetch posterior[t2t_unk_id]"""
        if len(self.history_sentences) > self.max_sentences:
            return 0.0
        return utils.common_get(posterior, self._t2t_unk_id, utils.NEG_INF) class FertilityT2TPredictor(T2TPredictor):
    """Use this predictor to integrate fertility models trained with 
    T2T. Fertility models output the fertility for each source word
    instead of target words. We define the fertility of the i-th
    source word in a hypothesis as the number of tokens between the 
    (i-1)-th and the i-th POP token.

    TODO: This is not SOLID (violates substitution principle)
    """

    def _update_scores(self):
        """Call the T2T model in self.mon_sess to update pop_scores
        and other_scores.
        """
        log_probs = self.mon_sess.run(self._log_probs,
           {self._inputs_var: self.src_sentence,
            self._targets_var: self.fertility_history + [text_encoder.PAD_ID]})
        fert_log_probs = [p for p in log_probs[4:]] + [log_probs[utils.UNK_ID]]
        fert_log_probs = fert_log_probs[:10]
        prev_max = utils.NEG_INF
        best_future = []
        for f in fert_log_probs[:0:-1]:
            prev_max = max(prev_max, f)
            best_future.append(prev_max)
        best_future.reverse()
        self.pop_scores = []
        self.other_scores = []
        acc = 0.0
        for score, best_future_score in zip(fert_log_probs[:-1], best_future):
            self.pop_scores.append(score - acc)
            self.other_scores.append(best_future_score - acc)
            acc = best_future_score
    
    def initialize(self, src_sentence):
        """Set src_sentence, compute fertilities for first src word."""
        self.fertility_history = []
        self.n_aligned_words = 0
        self.src_sentence = utils.oov_to_unk(
            src_sentence + [text_encoder.EOS_ID], 
            self.src_vocab_size)
        self._update_scores()

    def predict_next(self):
        """Returns self.pop_scores[n_aligned_words] for POP and EOS."""
        score = utils.common_get(self.pop_scores, self.n_aligned_words, 0.0)
        return {self.pop_id: score, utils.EOS_ID: score, 6: 0.0, 7: 0.0}
   
    def consume(self, word):
        if word == self.pop_id:
            target = 4 + self.n_aligned_words
            if target >= self.trg_vocab_size:
                target = utils.UNK_ID
            self.fertility_history.append(target)
            self.n_aligned_words = 0
            self._update_scores()
        elif word != 6 and word != 7: 
            self.n_aligned_words += 1
    
    def get_state(self):
        return (self.fertility_history, 
                self.n_aligned_words, 
                self.pop_scores, 
                self.other_scores)
    
    def set_state(self, state):
        (self.fertility_history, 
         self.n_aligned_words, 
         self.pop_scores, 
         self.other_scores) = state

    def is_equal(self, state1, state2):
        """Returns true if the history is the same """
        return state1 == state2

    def get_unk_probability(self, posterior):
        """Returns self.other_scores[n_aligned_words]."""
        return utils.common_get(self.other_scores, self.n_aligned_words, 0.0) class NizzaPredictor(BaseNizzaPredictor):
    """This predictor uses Nizza alignment models to derive a posterior over
    the target vocabulary for the next position. It mainly relies on the
    predict_next_word() implementation of Nizza models.
    """

    def predict_next(self):
        """Call the T2T model in self.mon_sess."""
        log_probs = self.mon_sess.run(self.log_probs,
            {self._inputs_var: self.src_sentence,
             self._targets_var: self.consumed + [common_utils.PAD_ID]})
        log_probs[common_utils.PAD_ID] = utils.NEG_INF  # Mask padding
        return log_probs
    
    def initialize(self, src_sentence):
        """Set src_sentence, reset consumed."""
        self.consumed = []
        self.src_sentence = src_sentence + [utils.EOS_ID]
    
    def consume(self, word):
        """Append ``word`` to the current history."""
        self.consumed.append(word)
    
    def get_state(self):
        """The predictor state is the complete history."""
        return self.consumed
    
    def set_state(self, state):
        """The predictor state is the complete history."""
        self.consumed = state

    def is_equal(self, state1, state2):
        """Returns true if the history is the same """
        return state1 == state2 class LexNizzaPredictor(BaseNizzaPredictor):
    """This predictor is only compatible to Model1-like Nizza models
    which return lexical translation probabilities in precompute(). The
    predictor keeps a list of the same length as the source sentence
    and initializes it with zeros. At each timestep it updates this list
    by the lexical scores Model1 assigned to the last consumed token.
    The predictor score aims to bring up all entries in the list, and 
    thus serves as a coverage mechanism over the source sentence.
    """

    def __init__(self, src_vocab_size, trg_vocab_size, model_name, 
                 hparams_set_name, checkpoint_dir, single_cpu_thread,
                 alpha, beta, shortlist_strategies,
                 trg2src_model_name="", trg2src_hparams_set_name="",
                 trg2src_checkpoint_dir="",
                 max_shortlist_length=0,
                 min_id=0,
                 nizza_unk_id=None):
        """Initializes a nizza predictor.

        Args:
            src_vocab_size (int): Source vocabulary size (called inputs_vocab_size
                in nizza)
            trg_vocab_size (int): Target vocabulary size (called targets_vocab_size
                in nizza)
            model_name (string): Name of the nizza model
            hparams_set_name (string): Name of the nizza hyper-parameter set
            checkpoint_dir (string): Path to the Nizza checkpoint directory. The 
                                     predictor will load the top most checkpoint in 
                                     the `checkpoints` file.
            single_cpu_thread (bool): If true, prevent tensorflow from
                                      doing multithreading.
            alpha (float): Score for each matching word
            beta (float): Penalty for each uncovered word at the end
            shortlist_strategies (string): Comma-separated list of shortlist
                strategies.
            trg2src_model_name (string): Name of the target2source nizza model
            trg2src_hparams_set_name (string): Name of the nizza hyper-parameter set
                                     for the target2source model
            trg2src_checkpoint_dir (string): Path to the Nizza checkpoint directory
                                     for the target2source model. The 
                                     predictor will load the top most checkpoint in 
                                     the `checkpoints` file.
            max_shortlist_length (int): If a shortlist exceeds this limit,
                initialize the initial coverage with 1 at this position. If
                zero, do not apply any limit
            min_id (int): Do not use IDs below this threshold (filters out most
                frequent words).
            nizza_unk_id (int): If set, use this as UNK id. Otherwise, the
                nizza is assumed to have no UNKs

        Raises:
            IOError if checkpoint file not found.
        """
        super(LexNizzaPredictor, self).__init__(
                src_vocab_size, trg_vocab_size, model_name, hparams_set_name, 
                checkpoint_dir, single_cpu_thread, nizza_unk_id=nizza_unk_id)
        self.alpha = alpha
        self.alpha_is_zero = alpha == 0.0
        self.beta = beta
        self.shortlist_strategies = utils.split_comma(shortlist_strategies)
        self.max_shortlist_length = max_shortlist_length
        self.min_id = min_id
        if trg2src_checkpoint_dir:
            self.use_trg2src = True
            predictor_graph = tf.Graph()
            with predictor_graph.as_default() as g:
                hparams = registry.get_registered_hparams_set(trg2src_hparams_set_name)
                hparams.add_hparam("inputs_vocab_size", trg_vocab_size)
                hparams.add_hparam("targets_vocab_size", src_vocab_size)
                run_config = tf.contrib.learn.RunConfig()
                run_config = run_config.replace(model_dir=trg2src_checkpoint_dir)
                model = registry.get_registered_model(trg2src_model_name, hparams, run_config)
                features = {"inputs": tf.expand_dims(tf.range(trg_vocab_size), 0)}
                mode = tf.estimator.ModeKeys.PREDICT
                trg2src_lex_logits = model.precompute(features, mode, hparams)
                # Precompute trg2src partitions
                partitions = tf.reduce_logsumexp(trg2src_lex_logits, axis=-1)
                self._trg2src_src_words_var = tf.placeholder(dtype=tf.int32, shape=[None],
                                                  name="sgnmt_trg2src_src_words")
                # trg2src_lex_logits has shape [1, trg_vocab_size, src_vocab_size]
                self.trg2src_logits = tf.gather(tf.transpose(trg2src_lex_logits[0, :, :]), self._trg2src_src_words_var)
                # trg2src_logits has shape [len(src_words), trg_vocab_size]
                self.trg2src_mon_sess = self.create_session(trg2src_checkpoint_dir)
                logging.debug("Precomputing lexnizza trg2src partitions...")
                self.trg2src_partitions = self.trg2src_mon_sess.run(partitions)
        else:
            self.use_trg2src = False
            logging.warn("No target-to-source model specified for lexnizza.")

    def get_unk_probability(self, posterior):
        if self.alpha_is_zero:
            return 0.0 
        if self._nizza_unk_id is None:
            return utils.NEG_INF
        return posterior[self._nizza_unk_id]

    def predict_next(self):
        """Predict record scores."""
        if self.alpha_is_zero:
            n_uncovered = self.coverage.count("0")
            return {utils.EOS_ID: -float(n_uncovered) * self.beta}
        uncovered_scores = [self.short_list_scores[src_pos]
             for src_pos, is_covered in enumerate(self.coverage)
             if is_covered == "0"]
        if not uncovered_scores:
            return np.zeros(self.trg_vocab_size)
        scores = np.max(uncovered_scores, axis=0)
        scores[utils.EOS_ID] = -len(uncovered_scores) * self.beta
        return scores
    
    def initialize(self, src_sentence):
        """Set src_sentence, reset consumed."""
        self.filt_src_sentence = [w for w in src_sentence if w >= self.min_id]
        scores = self.mon_sess.run(self.precomputed,
            {self._inputs_var: self.filt_src_sentence})
        scores = scores[0, :, :]
        # scores has shape [src_sentence_len, trg_vocab_size]
        self.trg_vocab_size = scores.shape[1]
        if self.use_trg2src:
            trg2src_logits = self.trg2src_mon_sess.run(self.trg2src_logits,
                {self._trg2src_src_words_var: self.filt_src_sentence})
            src2trg_logits = scores
            src2trg_partitions = logsumexp(src2trg_logits, axis=1, keepdims=True)
            trg2src_logprobs = trg2src_logits - self.trg2src_partitions
            src2trg_logprobs = src2trg_logits - src2trg_partitions
            scores = src2trg_logprobs + trg2src_logprobs
        src_len = len(self.filt_src_sentence)
        is_covered = []
        self.short_lists = []
        self.short_list_scores = []
        for src_pos in range(src_len):
            shortlist = self._create_short_list(scores[src_pos, :])
            if (self.max_shortlist_length > 0 
                      and len(shortlist) > self.max_shortlist_length):
                is_covered.append("1")
                shortlist = set([])
            else:
                is_covered.append("0")
            self.short_lists.append(shortlist)
            if not self.alpha_is_zero:
                alpha_scores = np.zeros(self.trg_vocab_size)
                for w in shortlist:
                    alpha_scores[w] = self.alpha
                self.short_list_scores.append(alpha_scores)
        self.coverage = "".join(is_covered)
        logging.debug("Short list sizes: %s" % ", ".join([
                str(len(l)) for l in self.short_lists]))
        logging.debug("Initial coverage: %s" % self.coverage)
        #print("SHORT LISTS")
        #for w, l in zip(self.filt_src_sentence, self.short_lists):
        #    print("\n\n%d" % w)
        #    if len(l) < 40:
        #        print(" ".join(map(str, l)))
              
    def consume(self, word):
        """Update coverage."""
        new_coverage = []
        for src_pos, is_covered in enumerate(self.coverage):
            if is_covered == "0" and word in self.short_lists[src_pos]:
                is_covered = "1"
            new_coverage.append(is_covered)
        self.coverage = "".join(new_coverage)
        #logging.debug("Partial: %s" % " ".join([str(self.filt_src_sentence[idx])  for idx, c in enumerate(self.coverage) if c == "1"]))
        #logging.debug(self.coverage)

    def _create_short_list(self, logits):
        """Creates a set of tokens which are likely translations."""
        words = set()
        filt_logits = logits[self.min_id:]
        for strat in self.shortlist_strategies:
            if strat[:3] == "top":
                n = int(strat[3:])
                words.update(utils.argmax_n(filt_logits, n))
            elif strat[:4] == "prob":
                p = float(strat[4:])
                unnorm_probs = np.exp(filt_logits)
                threshold = np.sum(unnorm_probs) * p
                acc = 0.0
                for word in np.argsort(filt_logits)[::-1]:
                    acc += unnorm_probs[word]
                    words.add(word)
                    if acc >= threshold:
                        break
            else:
                raise AttributeError("Unknown shortlist strategy '%s'" % strat)
        if self.min_id:
            words = set(w+self.min_id for w in words)
        try:
            words.remove(utils.EOS_ID)
        except KeyError:
            pass
        return words

    def estimate_future_cost(self, hypo):
        """We use the number of uncovered words times beta as heuristic
        estimate.
        """
        if hypo.trgt_sentence[:-1] == [utils.EOS_ID]:
            return 0.0
        n_uncovered = 0
        for short_list in self.short_lists:
            if not any(w in hypo.trgt_sentence for w in short_list):
                n_uncovered += 1
        return -float(n_uncovered) * self.beta * 0.1
    
    def get_state(self):
        """The predictor state is the coverage vector."""
        return self.coverage
    
    def set_state(self, state):
        """The predictor state is the coverage vector."""
        self.coverage = state class Word2charPredictor(UnboundedVocabularyPredictor):
    """This predictor wraps word level predictors when SGNMT is running
    on the character level. The mapping between word ID and character 
    ID sequence is loaded from the file system. All characters which
    do not appear in that mapping are treated as word boundary
    makers. The wrapper blocks consume and predict_next calls until a
    word boundary marker is consumed, and updates the slave predictor
    according the word between the last two word boundaries. The 
    mapping is done only on the target side, and the source sentences
    are passed through as they are. To use alternative tokenization on
    the source side, see the altsrc predictor wrapper. The word2char
    wrapper is always an ``UnboundedVocabularyPredictor``.
    """
    
    def __init__(self, map_path, slave_predictor):
        """Creates a new word2char wrapper predictor. The map_path 
        file has to be plain text files, each line containing the 
        mapping from a word index to the character index sequence
        (format: word char1 char2... charn).
        
        Args:
            map_path (string): Path to the mapping file
            slave_predictor (Predictor): Instance of the predictor with
                                         a different wmap than SGNMT
        """
        super(Word2charPredictor, self).__init__()
        self.slave_predictor = slave_predictor
        self.words = SimpleTrie()
        self.word_chars = {}
        with open(map_path) as f:
            for line in f:
                l = [int(x) for x in line.strip().split()]
                word = l[0]
                chars = l[1:]
                self.words.add(chars, word)
                for c in chars:
                    self.word_chars[c] = True   
        if isinstance(slave_predictor, UnboundedVocabularyPredictor): 
            self._get_stub_prob = self._get_stub_prob_unbounded
            self._start_new_word = self._start_new_word_unbounded
        else:
            self._get_stub_prob = self._get_stub_prob_bounded
            self._start_new_word = self._start_new_word_bounded             
    
    def initialize(self, src_sentence):
        """Pass through to slave predictor. The source sentence is not
        modified 
        """
        self.slave_predictor.initialize(src_sentence)
        self._start_new_word()
    
    def initialize_heuristic(self, src_sentence):
        """Pass through to slave predictor. The source sentence is not
        modified 
        """
        self.slave_predictor.initialize_heuristic(src_sentence)
    
    def _update_slave_vars(self, posterior):
        self.slave_unk = self.slave_predictor.get_unk_probability(posterior)
        self.slave_go = common_get(posterior, utils.GO_ID, self.slave_unk)
        self.slave_eos = common_get(posterior, utils.EOS_ID, self.slave_unk)
        
    def _start_new_word_unbounded(self):
        """start_new_word implementation for unbounded vocabulary slave
        predictors. Needs to set slave_go, slave_eos, and slave_unk
        """
        self.word_stub = []
        posterior = self.slave_predictor.predict_next([utils.UNK_ID,
                                                       utils.GO_ID,
                                                       utils.EOS_ID])
        self._update_slave_vars(posterior)
    
    def _start_new_word_bounded(self):
        """start_new_word implementation for bounded vocabulary slave
        predictors. Needs to set slave_go, slave_eos, slave_unk, and
        slave_posterior
        """
        self.word_stub = []
        self.slave_posterior = self.slave_predictor.predict_next()
        self._update_slave_vars(self.slave_posterior)
    
    def _get_stub_prob_unbounded(self):
        """get_stub_prob implementation for unbounded vocabulary slave
        predictors.
        """
        word = self.words.get(self.word_stub)
        if word:
            posterior = self.slave_predictor.predict_next([word])
            return common_get(posterior, word, self.slave_unk)
        return self.slave_unk
    
    def _get_stub_prob_bounded(self):
        """get_stub_prob implementation for bounded vocabulary slave
        predictors.
        """
        word = self.words.get(self.word_stub)
        return common_get(self.slave_posterior,
                          word if word else utils.UNK_ID,
                          self.slave_unk)
    
    def predict_next(self, trgt_words):
        posterior = {}
        stub_prob = False
        for ch in trgt_words:
            if ch in self.word_chars:
                posterior[ch] = 0.0
            else: # Word boundary marker
                if stub_prob is False:
                    stub_prob = self._get_stub_prob() if self.word_stub else 0.0
                posterior[ch] = stub_prob
        if utils.GO_ID in posterior:
            posterior[utils.GO_ID] += self.slave_go
        if utils.EOS_ID in posterior:
            posterior[utils.EOS_ID] += self.slave_eos
        return posterior
        
    def get_unk_probability(self, posterior):
        """This is about the unkown character, not word. Since the word
        level slave predictor has no notion of the unknown character, 
        we return NEG_INF unconditionally.
        """
        return NEG_INF
    
    def consume(self, word):
        """If ``word`` is a word boundary marker, truncate ``word_stub``
        and let the slave predictor consume word_stub. Otherwise, 
        extend ``word_stub`` by the character.
        """
        if word in self.word_chars:
            self.word_stub.append(word)
        elif self.word_stub:
            word = self.words.get(self.word_stub)
            self.slave_predictor.consume(word if word else utils.UNK_ID)
            self._start_new_word()
    
    def get_state(self):
        """Pass through to slave predictor """
        return self.word_stub, self.slave_predictor.get_state()
    
    def set_state(self, state):
        """Pass through to slave predictor """
        self.word_stub, slave_state = state
        self.slave_predictor.set_state(slave_state)

    def estimate_future_cost(self, hypo):
        """Not supported """
        logging.warn("Cannot use future cost estimates of predictors "
                     "wrapped by word2char")
        return 0.0

    def set_current_sen_id(self, cur_sen_id):
        """We need to override this method to propagate current\_
        sentence_id to the slave predictor
        """
        super(Word2charPredictor, self).set_current_sen_id(cur_sen_id)
        self.slave_predictor.set_current_sen_id(cur_sen_id)
    
    def is_equal(self, state1, state2):
        """Pass through to slave predictor """
        stub1, slave_state1 = state1
        stub2, slave_state2 = state2
        return (stub1 == stub2 
                and self.slave_predictor.is_equal(slave_state1, slave_state2)) class FSTTokPredictor(Predictor):
    """This wrapper can be used if the SGNMT decoder operates on the
    character level, but a predictor uses a more coarse grained 
    tokenization. The mapping is defined by an FST which transduces
    character to predictor unit sequences. This wrapper maintains a
    list of ``CombinedState`` objects which are tuples of an FST node
    and a predictor state for which holds:
    
    - The input labels on the path to the node are consistent with the
      consumed characters
    - The output labels on the path to the node are consistent with the
      predictor states
    """
    
    def __init__(self, path, fst_unk_id, max_pending_score, slave_predictor):
        """Constructor for the fsttok wrapper
        
        Args:
            path (string): Path to an FST which transduces characters 
                           to predictor tokens
            fst_unk_id (int): ID used to represent UNK in the FSTs
                              (usually 999999998)
            max_pending_score (float): Maximum pending score in a
                                       ``CombinedState`` instance.
            slave_predictor (Predictor): Wrapped predictor
        """
        super(FSTTokPredictor, self).__init__()
        self.max_pending_score = max_pending_score 
        self.fst_unk_id = fst_unk_id
        self.slave_predictor = slave_predictor
        if isinstance(slave_predictor, UnboundedVocabularyPredictor):
            logging.fatal("fsttok cannot wrap an unbounded "
                          "vocabulary predictor.")
        self.trans_fst = utils.load_fst(path)
    
    def initialize(self, src_sentence):
        """Pass through to slave predictor. The source sentence is not
        modified. ``states`` is updated to the initial FST node and
        predictor posterior and state.
        """
        self.slave_predictor.initialize(src_sentence)
        posterior = self.slave_predictor.predict_next()
        self.states = [CombinedState(self.trans_fst.start(),
                                     self.slave_predictor.get_state(),
                                     posterior)]
        self.last_prediction = {}
    
    def initialize_heuristic(self, src_sentence):
        """Pass through to slave predictor. The source sentence is not
        modified 
        """
        logging.warning("fsttok does not support predictor heuristics")
        self.slave_predictor.initialize_heuristic(src_sentence)
    
    def predict_next(self):
        self.last_prediction = {}
        for state in self.states:
            self._collect_chars(state, state.fst_node, None)
        return self.last_prediction
    
    def _collect_chars(self, state, root_node, first_olabel):
        """Recursively builds up ``last_prediction`` by traversing
        epsilon arcs in the FST from ``root_node``
        """
        for arc in self.trans_fst.arcs(root_node):
            arc_first_olabel = first_olabel if first_olabel else arc.olabel
            if arc.ilabel == EPS_ID:
                self._collect_chars(state, arc.nextstate, arc_first_olabel)
            else:
                score = state.score(arc_first_olabel, self.slave_predictor)
                if arc.ilabel in self.last_prediction: 
                    self.last_prediction[arc.ilabel] = max(
                                            self.last_prediction[arc.ilabel], 
                                            score)
                else:
                    self.last_prediction[arc.ilabel] = score

    def get_unk_probability(self, posterior):
        """Always returns negative infinity. Handling UNKs needs to be 
        realized by the FST.
        """
        return utils.NEG_INF

    def _choose_better(self, s1, s2):
        """``consume`` merges states if they have the same ``fst_node``
        This method defines which one to keep. We prefer states that
        
        1) have less unconsumed UNK tokens
        2) have higher pending_score
        """
        if s1 is None:
            return s2
        n_unk1 = len([1 for t in s1.unconsumed if t == self.fst_unk_id])
        n_unk2 = len([1 for t in s2.unconsumed if t == self.fst_unk_id])
        if n_unk1 > n_unk2:
            return s2
        if n_unk1 < n_unk2:
            return s1
        if s1.pending_score < s2.pending_score:
            return s2
        return s1
    
    def consume(self, word):
        """Update ``self.states`` to be consistent with ``word`` and 
        consumes all the predictor tokens.
        """
        next_states = []
        for state in self.states:
            next_states.extend(state.traverse_fst(self.trans_fst, word))
        consumed_score = self.last_prediction.get(word, 0.0)
        for state in next_states:
            state.pending_score -= consumed_score
            state.consume_single(self.slave_predictor)
        # if two states have the same fst_node, keep only the better one
        # Also: Remove states with too large pending_score
        uniq_states = {}
        for state in next_states:
            if state.pending_score < -self.max_pending_score:
                continue
            n = state.fst_node
            uniq_states[n] = self._choose_better(uniq_states.get(n, None),
                                                 state)
        self.states = list(uniq_states.values())
    
    def get_state(self):
        return self.states, self.last_prediction
    
    def set_state(self, state):
        self.states, self.last_prediction = state

    def estimate_future_cost(self, hypo):
        """Not implemented yet"""
        return 0.0

    def set_current_sen_id(self, cur_sen_id):
        """We need to override this method to propagate current\_
        sentence_id to the slave predictor
        """
        super(FSTTokPredictor, self).set_current_sen_id(cur_sen_id)
        self.slave_predictor.set_current_sen_id(cur_sen_id)
    
    def is_equal(self, state1, state2):
        """Not implemented yet"""
        return False class FairseqPredictor(Predictor):
    """Predictor for using fairseq models."""

    def __init__(self, model_path, user_dir, lang_pair, n_cpu_threads=-1):
        """Initializes a fairseq predictor.

        Args:
            model_path (string): Path to the fairseq model (*.pt). Like
                                 --path in fairseq-interactive.
            lang_pair (string): Language pair string (e.g. 'en-fr').
            user_dir (string): Path to fairseq user directory.
            n_cpu_threads (int): Number of CPU threads. If negative,
                                 use GPU.
        """
        super(FairseqPredictor, self).__init__()
        _initialize_fairseq(user_dir)
        self.use_cuda = torch.cuda.is_available() and n_cpu_threads < 0

        parser = options.get_generation_parser()
        input_args = ["--path", model_path, os.path.dirname(model_path)]
        if lang_pair:
            src, trg = lang_pair.split("-")
            input_args.extend(["--source-lang", src, "--target-lang", trg])
        args = options.parse_args_and_arch(parser, input_args)

        # Setup task, e.g., translation
        task = tasks.setup_task(args)
        self.src_vocab_size = len(task.source_dictionary)
        self.trg_vocab_size = len(task.target_dictionary)
        self.pad_id = task.source_dictionary.pad()

        # Load ensemble
        logging.info('Loading fairseq model(s) from {}'.format(model_path))
        self.models, _ = checkpoint_utils.load_model_ensemble(
            model_path.split(':'),
            task=task,
        )

        # Optimize ensemble for generation
        for model in self.models:
            model.make_generation_fast_(
                beamable_mm_beam_size=1,
                need_attn=False,
            )
            if self.use_cuda:
                model.cuda()
        self.model = EnsembleModel(self.models)
        self.model.eval()

    def get_unk_probability(self, posterior):
        """Fetch posterior[utils.UNK_ID]"""
        return utils.common_get(posterior, utils.UNK_ID, utils.NEG_INF)
                
    def predict_next(self):
        """Call the fairseq model."""
        lprobs, _ = self.model.forward_decoder(
            torch.LongTensor([self.consumed]), self.encoder_outs
        )
        lprobs[0, self.pad_id] = utils.NEG_INF
        return np.array(lprobs[0])
    
    def initialize(self, src_sentence):
        """Initialize source tensors, reset consumed."""
        self.consumed = []
        src_tokens = torch.LongTensor([
            utils.oov_to_unk(src_sentence + [utils.EOS_ID],
                             self.src_vocab_size)])
        src_lengths = torch.LongTensor([len(src_sentence) + 1])
        if self.use_cuda:
            src_tokens = src_tokens.cuda()
            src_lengths = src_lengths.cuda()
        self.encoder_outs = self.model.forward_encoder({
            'src_tokens': src_tokens,
            'src_lengths': src_lengths})
        self.consumed = [utils.GO_ID or utils.EOS_ID]
        # Reset incremental states
        for model in self.models:
            self.model.incremental_states[model] = {}
   
    def consume(self, word):
        """Append ``word`` to the current history."""
        self.consumed.append(word)
    
    def get_state(self):
        """The predictor state is the complete history."""
        return self.consumed, [self.model.incremental_states[m] 
                               for m in self.models]
    
    def set_state(self, state):
        """The predictor state is the complete history."""
        self.consumed, inc_states = state
        for model, inc_state in zip(self.models, inc_states):
            self.model.incremental_states[model] = inc_state

    def is_equal(self, state1, state2):
        """Returns true if the history is the same """
        return state1[0] == state2[0]
import logging
import codecs
import sys
import time
import traceback
import os
import uuid
from cam.sgnmt import ui
from cam.sgnmt import io
from cam.sgnmt import utils
from cam.sgnmt.predictors.parse import ParsePredictor, TokParsePredictor, \
                                       BpeParsePredictor
from cam.sgnmt.decoding import combination
from cam.sgnmt.decoding.astar import AstarDecoder
from cam.sgnmt.decoding.beam import BeamDecoder
from cam.sgnmt.decoding.bigramgreedy import BigramGreedyDecoder
from cam.sgnmt.decoding.bucket import BucketDecoder
from cam.sgnmt.decoding.core import UnboundedVocabularyPredictor
from cam.sgnmt.decoding.core import Hypothesis
from cam.sgnmt.decoding.dfs import DFSDecoder, \
                                   SimpleDFSDecoder, \
                                   SimpleLengthDFSDecoder
from cam.sgnmt.decoding.flip import FlipDecoder
from cam.sgnmt.decoding.greedy import GreedyDecoder
from cam.sgnmt.decoding.heuristics import GreedyHeuristic, \
                                         PredictorHeuristic, \
                                         ScorePerWordHeuristic, \
                                         StatsHeuristic, \
                                         LastTokenHeuristic
from cam.sgnmt.decoding.multisegbeam import MultisegBeamDecoder
from cam.sgnmt.decoding.restarting import RestartingDecoder
from cam.sgnmt.decoding.sepbeam import SepBeamDecoder
from cam.sgnmt.decoding.syntaxbeam import SyntaxBeamDecoder
from cam.sgnmt.decoding.mbrbeam import MBRBeamDecoder
from cam.sgnmt.decoding.lenbeam import LengthBeamDecoder
from cam.sgnmt.decoding.syncbeam import SyncBeamDecoder
from cam.sgnmt.decoding.fstbeam import FSTBeamDecoder
from cam.sgnmt.decoding.predlimitbeam import PredLimitBeamDecoder
from cam.sgnmt.decoding.combibeam import CombiBeamDecoder
from cam.sgnmt.output import TextOutputHandler, \
                             NBestOutputHandler, \
                             NgramOutputHandler, \
                             TimeCSVOutputHandler, \
                             FSTOutputHandler, \
                             StandardFSTOutputHandler
from cam.sgnmt.predictors.automata import FstPredictor, \
                                         RtnPredictor, \
                                         NondeterministicFstPredictor
from cam.sgnmt.predictors.bow import BagOfWordsPredictor, \
                                     BagOfWordsSearchPredictor
from cam.sgnmt.predictors.forced import ForcedPredictor, ForcedLstPredictor
from cam.sgnmt.predictors.grammar import RuleXtractPredictor
from cam.sgnmt.predictors.length import WordCountPredictor, NBLengthPredictor, \
    ExternalLengthPredictor, NgramCountPredictor, UnkCountPredictor, \
    NgramizePredictor, WeightNonTerminalPredictor
from cam.sgnmt.predictors.structure import BracketPredictor, OSMPredictor, \
                                           ForcedOSMPredictor
from cam.sgnmt.predictors.misc import UnboundedAltsrcPredictor, \
                                      AltsrcPredictor, \
                                      UnboundedRankPredictor, \
                                      RankPredictor, \
                                      UnboundedGluePredictor, \
                                      GluePredictor
from cam.sgnmt.predictors.vocabulary import IdxmapPredictor, \
                                            UnboundedIdxmapPredictor, \
                                            MaskvocabPredictor, \
                                            UnboundedMaskvocabPredictor, \
                                            UnkvocabPredictor, \
                                            SkipvocabPredictor
from cam.sgnmt.predictors.ngram import KenLMPredictor
from cam.sgnmt.predictors.tf_t2t import T2TPredictor, \
                                        EditT2TPredictor, \
                                        SegT2TPredictor, \
                                        FertilityT2TPredictor
from cam.sgnmt.predictors.tf_nizza import NizzaPredictor, LexNizzaPredictor
from cam.sgnmt.predictors.tokenization import Word2charPredictor, FSTTokPredictor
from cam.sgnmt.predictors.pytorch_fairseq import FairseqPredictor
                                args.heuristic_scores_file,
                                args.collect_statistics,
                                "consumed" in args.bow_heuristic_strategies,
                                "remaining" in args.bow_heuristic_strategies,
                                args.bow_diversity_heuristic_factor,
                                _get_override_args("pred_trg_vocab_size"))
            elif pred == "forcedlst":
                feat_name = _get_override_args("forcedlst_sparse_feat")
                p = ForcedLstPredictor(args.trg_test,
                                       args.use_nbest_weights,
                                       args.forcedlst_match_unk,
                                       feat_name if feat_name else None)
            elif pred == "rtn":
                p = RtnPredictor(args.rtn_path,
                                 args.use_rtn_weights,
                                 args.normalize_rtn_weights,
                                 to_log=args.fst_to_log,
                                 minimize_rtns=args.minimize_rtns,
                                 rmeps=args.remove_epsilon_in_rtns)
            elif pred == "kenlm":
                p = KenLMPredictor(args.lm_path)
            elif pred == "wc":
                p = WordCountPredictor(args.wc_word,
                                       args.wc_nonterminal_penalty,
                                       args.syntax_nonterminal_ids,
                                       args.syntax_min_terminal_id,
                                       args.syntax_max_terminal_id,
                                       args.negative_wc,
                                       _get_override_args("pred_trg_vocab_size"))
            elif pred == "ngramc":
                p = NgramCountPredictor(_get_override_args("ngramc_path"),
                                        _get_override_args("ngramc_order"),
                                        args.ngramc_discount_factor)
            elif pred == "unkc":
                p = UnkCountPredictor(
                     _get_override_args("pred_src_vocab_size"), 
                     utils.split_comma(args.unk_count_lambdas, float))
            elif pred == "length":
                length_model_weights = utils.split_comma(
                    args.length_model_weights, float)
                p = NBLengthPredictor(args.src_test_raw, 
                                      length_model_weights, 
                                      args.use_length_point_probs,
                                      args.length_model_offset)
            elif pred == "extlength":
                p = ExternalLengthPredictor(args.extlength_path)
            elif pred == "lrhiero":
                fw = None
                if args.grammar_feature_weights:
                    fw = utils.split_comma(args.grammar_feature_weights, float)
                p = RuleXtractPredictor(args.rules_path,
                                        args.use_grammar_weights,
                                        fw)
            else:
                logging.fatal("Predictor '%s' not available. Please check "
                              "--predictors for spelling errors." % pred)
                decoder.remove_predictors()
                return
            for _,wrapper in enumerate(wrappers):
                # Embed predictor ``p`` into wrapper predictors if necessary
                # TODO: Use wrapper_weights
                if wrapper == "idxmap":
                    src_path = _get_override_args("src_idxmap")
                    trg_path = _get_override_args("trg_idxmap")
                    if isinstance(p, UnboundedVocabularyPredictor): 
                        p = UnboundedIdxmapPredictor(src_path, trg_path, p, 1.0) 
                    else: # idxmap predictor for bounded predictors
                        p = IdxmapPredictor(src_path, trg_path, p, 1.0)
                elif wrapper == "maskvocab":
                    if isinstance(p, UnboundedVocabularyPredictor): 
                        p = UnboundedMaskvocabPredictor(args.maskvocab_vocab, p)
                    else: # idxmap predictor for bounded predictors
                        p = MaskvocabPredictor(args.maskvocab_vocab, p)
                elif wrapper == "weightnt":
                    p = WeightNonTerminalPredictor(
                        p, 
                        args.syntax_nonterminal_factor,
                        args.syntax_nonterminal_ids,
                        args.syntax_min_terminal_id,
                        args.syntax_max_terminal_id,
                        _get_override_args("pred_trg_vocab_size"))
                elif wrapper == "parse":
                    if args.parse_tok_grammar:
                        if args.parse_bpe_path:
                            p = BpeParsePredictor(
                                args.syntax_path,
                                args.syntax_bpe_path,
                                p,
                                args.syntax_word_out,
                                args.normalize_fst_weights,
                                norm_alpha=args.syntax_norm_alpha,
                                beam_size=args.syntax_internal_beam,
                                max_internal_len=args.syntax_max_internal_len,
                                allow_early_eos=args.syntax_allow_early_eos,
                                consume_out_of_class=args.syntax_consume_ooc,
                                terminal_restrict=args.syntax_terminal_restrict,
                                internal_only_restrict=args.syntax_internal_only,
                                eow_ids=args.syntax_eow_ids,
                                terminal_ids=args.syntax_terminal_ids)
                        else:
                            p = TokParsePredictor(
                                args.syntax_path,
                                p,
                                args.syntax_word_out,
                                args.normalize_fst_weights,
                                norm_alpha=args.syntax_norm_alpha,
                                beam_size=args.syntax_internal_beam,
                                max_internal_len=args.syntax_max_internal_len,
                                allow_early_eos=args.syntax_allow_early_eos,
                                consume_out_of_class=args.syntax_consume_ooc)
                    else:
                        p = ParsePredictor(
                            p,
                            args.normalize_fst_weights,
                            beam_size=args.syntax_internal_beam,
                            max_internal_len=args.syntax_max_internal_len,
                            nonterminal_ids=args.syntax_nonterminal_ids)
                elif wrapper == "altsrc":
                    src_test = _get_override_args("altsrc_test")
                    if isinstance(p, UnboundedVocabularyPredictor): 

                        p = UnboundedAltsrcPredictor(src_test, p)