    def forward(
        self,
        input_tensor1,
        input_tensor2,
        attention_mask1=None,
        attention_mask2=None,
        co_attention_mask=None,
    ):
        """
        # Parameters

        input_tensor1 : `torch.Tensor`
            Shape `batch_size x seq_len1 x hidden_dim1`
            where `seq_len1` can be the sequence length
            when the modality is text, or the number of
            regions when the modality is image.
        input_tensor2 : `torch.Tensor`
            Shape `batch_size x seq_len2 x hidden_dim2`
            where `seq_len2` can be the sequence length
            when the modality is text, or the number of
            regions when the modality is image.
        attention_mask1 : `torch.BoolTensor`, optional
            Shape `batch_size x seq_len1`
        attention_mask : `torch.BoolTensor`, optional
            Shape `batch_size x seq_len2`
        co_attention_mask : `torch.Tensor`, optional
            Shape `batch_size x seq_len1 x seq_len2 x all_head_size`
            This mask is for cases when you already have some prior information
            about the interaction between the two modalities. For example,
            if you know which words correspond to which regions in the image,
            this mask can be applied to limit the attention given the bias.
        """

        # for the first modality:
        mixed_query_layer1 = self.query1(input_tensor1)
        mixed_key_layer1 = self.key1(input_tensor1)
        mixed_value_layer1 = self.value1(input_tensor1)

        query_layer1 = self._transpose_for_scores(mixed_query_layer1)
        key_layer1 = self._transpose_for_scores(mixed_key_layer1)
        value_layer1 = self._transpose_for_scores(mixed_value_layer1)

        # for the second modality:
        mixed_query_layer2 = self.query2(input_tensor2)
        mixed_key_layer2 = self.key2(input_tensor2)
        mixed_value_layer2 = self.value2(input_tensor2)

        query_layer2 = self._transpose_for_scores(mixed_query_layer2)
        key_layer2 = self._transpose_for_scores(mixed_key_layer2)
        value_layer2 = self._transpose_for_scores(mixed_value_layer2)

        # Conditioning the second modality on the first one.
        attention_scores1 = self.attn1(query_layer2, key_layer1)
        if attention_mask1 is not None:
            attention_scores1 = apply_mask(attention_scores1, attention_mask1)
        if co_attention_mask is not None:
            attention_scores1 = apply_mask(attention_scores1, co_attention_mask.permute(0, 1, 3, 2))

        attention_probs1 = torch.nn.Softmax(dim=-1)(attention_scores1)

        # This is actually dropping out entire tokens to attend to, which might
        # seem a bit unusual, but is taken from the original Transformer paper.
        attention_probs1 = self.dropout1(attention_probs1)

        context_layer1 = torch.matmul(attention_probs1, value_layer1)
        context_layer1 = context_layer1.permute(0, 2, 1, 3).contiguous()
        new_context_layer_shape1 = context_layer1.size()[:-2] + (self.all_head_size,)
        context_layer1 = context_layer1.view(*new_context_layer_shape1)

        # Conditioning the first modality on the second one.
        attention_scores2 = self.attn2(query_layer1, key_layer2)
        # we can comment this line for single flow.
        if attention_mask2 is not None:
            attention_scores2 = apply_mask(attention_scores2, attention_mask2)
        if co_attention_mask is not None:
            attention_scores2 = apply_mask(attention_scores2, co_attention_mask)

        attention_probs2 = torch.nn.Softmax(dim=-1)(attention_scores2)

        # This is actually dropping out entire tokens to attend to, which might
        # seem a bit unusual, but is taken from the original Transformer paper.
        attention_probs2 = self.dropout2(attention_probs2)

        context_layer2 = torch.matmul(attention_probs2, value_layer2)
        context_layer2 = context_layer2.permute(0, 2, 1, 3).contiguous()
        new_context_layer_shape2 = context_layer2.size()[:-2] + (self.all_head_size,)
        context_layer2 = context_layer2.view(*new_context_layer_shape2)

        return context_layer1, context_layer2
# Parameters

input_tensor1 : `torch.Tensor`
Shape `batch_size x seq_len1 x hidden_dim1`
where `seq_len1` can be the sequence length
when the modality is text, or the number of
regions when the modality is image.
input_tensor2 : `torch.Tensor`
Shape `batch_size x seq_len2 x hidden_dim2`
where `seq_len2` can be the sequence length
when the modality is text, or the number of
regions when the modality is image.
attention_mask1 : `torch.BoolTensor`, optional
Shape `batch_size x seq_len1`
attention_mask : `torch.BoolTensor`, optional
Shape `batch_size x seq_len2`
co_attention_mask : `torch.Tensor`, optional
Shape `batch_size x seq_len1 x seq_len2 x all_head_size`
This mask is for cases when you already have some prior information
about the interaction between the two modalities. For example,
if you know which words correspond to which regions in the image,
this mask can be applied to limit the attention given the bias.
allennlp/modules/transformer/bimodal_attention.py