<filename>colbert/modeling/layers/position_ffn.py<gh_stars>1-10
# -*- encoding:utf-8 -*-
import torch.nn as nn
import math
import torch


class PositionwiseFeedForward(nn.Module):
    """ Feed Forward Layer """
    def __init__(self, hidden_size, feedforward_size):
        super(PositionwiseFeedForward, self).__init__()
        self.linear_1 = nn.Linear(hidden_size, feedforward_size)
        self.linear_2 = nn.Linear(feedforward_size, hidden_size)
        
    def forward(self, x):
        inter = self.gelu(self.linear_1(x))
        output = self.linear_2(inter)
        return output

    def gelu(self,x):
        return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
<filename>colbert/modeling/layers/position_ffn.py<gh_stars>1-10
# -*- encoding:utf-8 -*-
import torch.nn as nn
import math
import torch


class PositionwiseFeedForward(nn.Module):
    """ Feed Forward Layer """
    def __init__(self, hidden_size, feedforward_size):
        super(PositionwiseFeedForward, self).__init__()
        self.linear_1 = nn.Linear(hidden_size, feedforward_size)
        self.linear_2 = nn.Linear(feedforward_size, hidden_size)
        
    def forward(self, x):
        inter = self.gelu(self.linear_1(x))
        output = self.linear_2(inter)
        return output

    def gelu(self,x):
        return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
# -*- encoding:utf-8 -*- Feed Forward Layer