def forward(self, word_inputs, tag_inputs, arc_targets=None, rel_targets=None):
        """Run decoding

        Parameters
        ----------
        word_inputs : mxnet.ndarray.NDArray
            word indices of seq_len x batch_size
        tag_inputs : mxnet.ndarray.NDArray
            tag indices of seq_len x batch_size
        arc_targets : mxnet.ndarray.NDArray
            gold arc indices of seq_len x batch_size
        rel_targets : mxnet.ndarray.NDArray
            gold rel indices of seq_len x batch_size
        Returns
        -------
        tuple
            (arc_accuracy, rel_accuracy, overall_accuracy, loss) when training, else if given gold target
        then return arc_accuracy, rel_accuracy, overall_accuracy, outputs, otherwise return outputs, where outputs is a
        list of (arcs, rels).
        """
        is_train = autograd.is_training()

        def flatten_numpy(ndarray):
            """Flatten nd-array to 1-d column vector

            Parameters
            ----------
            ndarray : numpy.ndarray
                input tensor

            Returns
            -------
            numpy.ndarray
                A column vector

            """
            return np.reshape(ndarray, (-1,), 'F')

        batch_size = word_inputs.shape[1]
        seq_len = word_inputs.shape[0]
        mask = np.greater(word_inputs, self._vocab.ROOT).astype(np.float32)
        num_tokens = int(np.sum(mask))  # non padding, non root token number

        if is_train or arc_targets is not None:
            mask_1D = flatten_numpy(mask)
            mask_1D_tensor = nd.array(mask_1D)

        unked_words = np.where(word_inputs < self._vocab.words_in_train, word_inputs, self._vocab.UNK)
        word_embs = self.word_embs(nd.array(unked_words, dtype='int'))
        if self.pret_word_embs:
            word_embs = word_embs + self.pret_word_embs(nd.array(word_inputs))
        tag_embs = self.tag_embs(nd.array(tag_inputs))

        # Dropout
        emb_inputs = nd.concat(word_embs, tag_embs, dim=2)  # seq_len x batch_size

        top_recur = biLSTM(self.f_lstm, self.b_lstm, emb_inputs, batch_size,
                           dropout_x=self.dropout_lstm_input if is_train else 0)
        top_recur = nd.Dropout(data=top_recur, axes=[0], p=self.dropout_mlp)

        W_dep, b_dep = self.mlp_dep_W.data(), self.mlp_dep_b.data()
        W_head, b_head = self.mlp_head_W.data(), self.mlp_head_b.data()
        dep, head = leaky_relu(nd.dot(top_recur, W_dep.T) + b_dep), leaky_relu(nd.dot(top_recur, W_head.T) + b_head)
        dep, head = nd.Dropout(data=dep, axes=[0], p=self.dropout_mlp), nd.Dropout(data=head, axes=[0],
                                                                                       p=self.dropout_mlp)
        dep, head = nd.transpose(dep, axes=[2, 0, 1]), nd.transpose(head, axes=[2, 0, 1])
        dep_arc, dep_rel = dep[:self.mlp_arc_size], dep[self.mlp_arc_size:]
        head_arc, head_rel = head[:self.mlp_arc_size], head[self.mlp_arc_size:]

        W_arc = self.arc_W.data()
        arc_logits = bilinear(dep_arc, W_arc, head_arc, self.mlp_arc_size, seq_len, batch_size, num_outputs=1,
                              bias_x=True, bias_y=False)
        # (#head x #dep) x batch_size

        flat_arc_logits = reshape_fortran(arc_logits, (seq_len, seq_len * batch_size))
        # (#head ) x (#dep x batch_size)

        arc_preds = arc_logits.argmax(0)
        # seq_len x batch_size

        if is_train or arc_targets is not None:
            correct = np.equal(arc_preds.asnumpy(), arc_targets)
            arc_correct = correct.astype(np.float32) * mask
            arc_accuracy = np.sum(arc_correct) / num_tokens
            targets_1D = flatten_numpy(arc_targets)
            losses = self.softmax_loss(flat_arc_logits, nd.array(targets_1D))
            arc_loss = nd.sum(losses * mask_1D_tensor) / num_tokens

        if not is_train:
            arc_probs = np.transpose(
                np.reshape(nd.softmax(flat_arc_logits, axis=0).asnumpy(), (seq_len, seq_len, batch_size), 'F'))
        # #batch_size x #dep x #head

        W_rel = self.rel_W.data()
        rel_logits = bilinear(dep_rel, W_rel, head_rel, self.mlp_rel_size, seq_len, batch_size,
                              num_outputs=self._vocab.rel_size, bias_x=True, bias_y=True)
        # (#head x rel_size x #dep) x batch_size

        flat_rel_logits = reshape_fortran(rel_logits, (seq_len, self._vocab.rel_size, seq_len * batch_size))
        # (#head x rel_size) x (#dep x batch_size)

        _target_vec = nd.array(targets_1D if is_train else flatten_numpy(arc_preds.asnumpy())).reshape(
            seq_len * batch_size, 1)
        _target_mat = _target_vec * nd.ones((1, self._vocab.rel_size))

        partial_rel_logits = nd.pick(flat_rel_logits, _target_mat.T, axis=0)
        # (rel_size) x (#dep x batch_size)

        if is_train or arc_targets is not None:
            rel_preds = partial_rel_logits.argmax(0)
            targets_1D = flatten_numpy(rel_targets)
            rel_correct = np.equal(rel_preds.asnumpy(), targets_1D).astype(np.float32) * mask_1D
            rel_accuracy = np.sum(rel_correct) / num_tokens
            losses = self.softmax_loss(partial_rel_logits, nd.array(targets_1D))
            rel_loss = nd.sum(losses * mask_1D_tensor) / num_tokens

        if not is_train:
            rel_probs = np.transpose(np.reshape(nd.softmax(flat_rel_logits.transpose([1, 0, 2]), axis=0).asnumpy(),
                                                (self._vocab.rel_size, seq_len, seq_len, batch_size), 'F'))
        # batch_size x #dep x #head x #nclasses

        if is_train or arc_targets is not None:
            loss = arc_loss + rel_loss
            correct = rel_correct * flatten_numpy(arc_correct)
            overall_accuracy = np.sum(correct) / num_tokens

        if is_train:
            return arc_accuracy, rel_accuracy, overall_accuracy, loss

        outputs = []

        for msk, arc_prob, rel_prob in zip(np.transpose(mask), arc_probs, rel_probs):
            # parse sentences one by one
            msk[0] = 1.
            sent_len = int(np.sum(msk))
            arc_pred = arc_argmax(arc_prob, sent_len, msk)
            rel_prob = rel_prob[np.arange(len(arc_pred)), arc_pred]
            rel_pred = rel_argmax(rel_prob, sent_len)
            outputs.append((arc_pred[1:sent_len], rel_pred[1:sent_len]))

        if arc_targets is not None:
            return arc_accuracy, rel_accuracy, overall_accuracy, outputs
        return outputs
def forward ( self , word_inputs , tag_inputs , arc_targets = None , rel_targets = None ) : is_train = autograd . is_training ( ) def flatten_numpy ( ndarray ) : return np . reshape ( ndarray , ( - 1 , ) , 'F' ) batch_size = word_inputs . shape [ 1 ] seq_len = word_inputs . shape [ 0 ] mask = np . greater ( word_inputs , self . _vocab . ROOT ) . astype ( np . float32 ) num_tokens = int ( np . sum ( mask ) ) if is_train or arc_targets is not None : mask_1D = flatten_numpy ( mask ) mask_1D_tensor = nd . array ( mask_1D ) unked_words = np . where ( word_inputs < self . _vocab . words_in_train , word_inputs , self . _vocab . UNK ) word_embs = self . word_embs ( nd . array ( unked_words , dtype = 'int' ) ) if self . pret_word_embs : word_embs = word_embs + self . pret_word_embs ( nd . array ( word_inputs ) ) tag_embs = self . tag_embs ( nd . array ( tag_inputs ) ) emb_inputs = nd . concat ( word_embs , tag_embs , dim = 2 ) top_recur = biLSTM ( self . f_lstm , self . b_lstm , emb_inputs , batch_size , dropout_x = self . dropout_lstm_input if is_train else 0 ) top_recur = nd . Dropout ( data = top_recur , axes = [ 0 ] , p = self . dropout_mlp ) W_dep , b_dep = self . mlp_dep_W . data ( ) , self . mlp_dep_b . data ( ) W_head , b_head = self . mlp_head_W . data ( ) , self . mlp_head_b . data ( ) dep , head = leaky_relu ( nd . dot ( top_recur , W_dep . T ) + b_dep ) , leaky_relu ( nd . dot ( top_recur , W_head . T ) + b_head ) dep , head = nd . Dropout ( data = dep , axes = [ 0 ] , p = self . dropout_mlp ) , nd . Dropout ( data = head , axes = [ 0 ] , p = self . dropout_mlp ) dep , head = nd . transpose ( dep , axes = [ 2 , 0 , 1 ] ) , nd . transpose ( head , axes = [ 2 , 0 , 1 ] ) dep_arc , dep_rel = dep [ : self . mlp_arc_size ] , dep [ self . mlp_arc_size : ] head_arc , head_rel = head [ : self . mlp_arc_size ] , head [ self . mlp_arc_size : ] W_arc = self . arc_W . data ( ) arc_logits = bilinear ( dep_arc , W_arc , head_arc , self . mlp_arc_size , seq_len , batch_size , num_outputs = 1 , bias_x = True , bias_y = False ) flat_arc_logits = reshape_fortran ( arc_logits , ( seq_len , seq_len * batch_size ) ) arc_preds = arc_logits . argmax ( 0 ) if is_train or arc_targets is not None : correct = np . equal ( arc_preds . asnumpy ( ) , arc_targets ) arc_correct = correct . astype ( np . float32 ) * mask arc_accuracy = np . sum ( arc_correct ) / num_tokens targets_1D = flatten_numpy ( arc_targets ) losses = self . softmax_loss ( flat_arc_logits , nd . array ( targets_1D ) ) arc_loss = nd . sum ( losses * mask_1D_tensor ) / num_tokens if not is_train : arc_probs = np . transpose ( np . reshape ( nd . softmax ( flat_arc_logits , axis = 0 ) . asnumpy ( ) , ( seq_len , seq_len , batch_size ) , 'F' ) ) W_rel = self . rel_W . data ( ) rel_logits = bilinear ( dep_rel , W_rel , head_rel , self . mlp_rel_size , seq_len , batch_size , num_outputs = self . _vocab . rel_size , bias_x = True , bias_y = True ) flat_rel_logits = reshape_fortran ( rel_logits , ( seq_len , self . _vocab . rel_size , seq_len * batch_size ) ) _target_vec = nd . array ( targets_1D if is_train else flatten_numpy ( arc_preds . asnumpy ( ) ) ) . reshape ( seq_len * batch_size , 1 ) _target_mat = _target_vec * nd . ones ( ( 1 , self . _vocab . rel_size ) ) partial_rel_logits = nd . pick ( flat_rel_logits , _target_mat . T , axis = 0 ) if is_train or arc_targets is not None : rel_preds = partial_rel_logits . argmax ( 0 ) targets_1D = flatten_numpy ( rel_targets ) rel_correct = np . equal ( rel_preds . asnumpy ( ) , targets_1D ) . astype ( np . float32 ) * mask_1D rel_accuracy = np . sum ( rel_correct ) / num_tokens losses = self . softmax_loss ( partial_rel_logits , nd . array ( targets_1D ) ) rel_loss = nd . sum ( losses * mask_1D_tensor ) / num_tokens if not is_train : rel_probs = np . transpose ( np . reshape ( nd . softmax ( flat_rel_logits . transpose ( [ 1 , 0 , 2 ] ) , axis = 0 ) . asnumpy ( ) , ( self . _vocab . rel_size , seq_len , seq_len , batch_size ) , 'F' ) ) if is_train or arc_targets is not None : loss = arc_loss + rel_loss correct = rel_correct * flatten_numpy ( arc_correct ) overall_accuracy = np . sum ( correct ) / num_tokens if is_train : return arc_accuracy , rel_accuracy , overall_accuracy , loss outputs = [ ] for msk , arc_prob , rel_prob in zip ( np . transpose ( mask ) , arc_probs , rel_probs ) : msk [ 0 ] = 1. sent_len = int ( np . sum ( msk ) ) arc_pred = arc_argmax ( arc_prob , sent_len , msk ) rel_prob = rel_prob [ np . arange ( len ( arc_pred ) ) , arc_pred ] rel_pred = rel_argmax ( rel_prob , sent_len ) outputs . append ( ( arc_pred [ 1 : sent_len ] , rel_pred [ 1 : sent_len ] ) ) if arc_targets is not None : return arc_accuracy , rel_accuracy , overall_accuracy , outputs return outputs
Run decoding

        Parameters
        ----------
        word_inputs : mxnet.ndarray.NDArray
            word indices of seq_len x batch_size
        tag_inputs : mxnet.ndarray.NDArray
            tag indices of seq_len x batch_size
        arc_targets : mxnet.ndarray.NDArray
            gold arc indices of seq_len x batch_size
        rel_targets : mxnet.ndarray.NDArray
            gold rel indices of seq_len x batch_size
        Returns
        -------
        tuple
            (arc_accuracy, rel_accuracy, overall_accuracy, loss) when training, else if given gold target
        then return arc_accuracy, rel_accuracy, overall_accuracy, outputs, otherwise return outputs, where outputs is a
        list of (arcs, rels).