    def forward(
        self,
        src_tokens,
        src_lengths=None,
        src_txt_tokens=None,
        src_txt_lengths=None,
        **kwargs
    ):
        """
        Args:
            src_tokens: padded tensor (B, T, C * feat)
            src_lengths: tensor of original lengths of input utterances (speech) (B,)
            src_txt_tokens: padded tensor (B, T)
            src_txt_lengths: tensor of original lengths of input utterances (text) (B,)
        """
        # src_tokens only: inference
        # src_tokens, src_lengths: speech only training
        # src_txt_tokens, src_txt_lengths: text only training
        # all valid: speech + text training

        if src_tokens is None and src_txt_tokens is None:
            raise ValueError(
                "src_tokens and src_txt_tokens cannot be None at the same time"
            )
        ret1 = None
        ret2 = None
        return_all_hiddens = False
        if src_tokens is not None:
            if (
                self.use_cross_attentive_loss and src_txt_tokens is not None
            ):  # remove self.training so we can get attn score during validation step
                return_all_hiddens = True
            ret1 = self.spch_encoder(
                src_tokens, src_lengths, return_all_hiddens=return_all_hiddens
            )

            if self.use_cross_attentive_loss and src_txt_tokens is not None:
                assert self.cross_attentive_loss_before_last_layer < len(
                    ret1["encoder_states"]
                )
                ret1 = self.process_attentive_loss_states(
                    ret1,
                    ret1["encoder_states"][
                        -self.cross_attentive_loss_before_last_layer - 1
                    ],
                )

        if src_txt_tokens is not None:
            ret2 = self.text_encoder(
                src_txt_tokens, src_txt_lengths, return_all_hiddens=return_all_hiddens
            )
            if return_all_hiddens:
                if self.cross_attentive_loss_before_last_layer == len(
                    self.text_encoder.layers
                ):
                    text_embedding, _ = self.text_encoder.forward_embedding(
                        src_txt_tokens
                    )
                    text_embedding = text_embedding.transpose(0, 1)
                    ret2 = self.process_attentive_loss_states(ret2, text_embedding)
                else:
                    assert self.cross_attentive_loss_before_last_layer < len(
                        self.text_encoder.layers
                    )
                    ret2 = self.process_attentive_loss_states(
                        ret2,
                        ret2["encoder_states"][
                            -self.cross_attentive_loss_before_last_layer - 1
                        ],
                    )

        def merge_output(rst1, rst2):
            if rst1 is None:
                if not (self.enc2_along_grad_mult == 1.0 or self.training):
                    rst2 = self.mult_rst_grad(rst2, self.enc2_along_grad_mult)
                return rst2
            if rst2 is None:
                return rst1
            if self.enc_grad_mult != 1.0 and self.training:
                rst1 = self.mult_rst_grad(rst1, self.enc_grad_mult)
                rst2 = self.mult_rst_grad(rst2, self.enc_grad_mult)
            rst = (rst1, rst2)
            return rst

        return merge_output(ret1, ret2)
Args:
src_tokens: padded tensor (B, T, C * feat)
src_lengths: tensor of original lengths of input utterances (speech) (B,)
src_txt_tokens: padded tensor (B, T)
src_txt_lengths: tensor of original lengths of input utterances (text) (B,)
examples/speech_text_joint_to_text/models/s2t_dualinputtransformer.py