original_docstring: r"""
          labels (``torch.LongTensor`` of shape ``(batch_size, sequence_length)``, `optional`):
              Labels for computing the ELECTRA-style loss. Input should be a sequence of tokens (see :obj:`input_ids`
              docstring) Indices should be in ``[0, 1]``:
  
              - 0 indicates the token is an original token,
              - 1 indicates the token was replaced.
  
          Returns:
  
          Examples::
  
              >>> from transformers import FunnelTokenizer, FunnelForPreTraining
              >>> import torch
  
              >>> tokenizer = FunnelTokenizer.from_pretrained('funnel-transformer/small')
              >>> model = FunnelForPreTraining.from_pretrained('funnel-transformer/small')
  
              >>> inputs = tokenizer("Hello, my dog is cute", return_tensors= "pt")
              >>> logits = model(**inputs).logits
          
docstring: 
  
  0 indicates the token is an original token,
  1 indicates the token was replaced.
docstring_tokens: 0 indicates the token is an original token 1 indicates the token was replaced .
code: def forward(
          self,
          input_ids=None,
          attention_mask=None,
          token_type_ids=None,
          inputs_embeds=None,
          labels=None,
          output_attentions=None,
          output_hidden_states=None,
          return_dict=None,
      ):
          r"""
          labels (``torch.LongTensor`` of shape ``(batch_size, sequence_length)``, `optional`):
              Labels for computing the ELECTRA-style loss. Input should be a sequence of tokens (see :obj:`input_ids`
              docstring) Indices should be in ``[0, 1]``:
  
              - 0 indicates the token is an original token,
              - 1 indicates the token was replaced.
  
          Returns:
  
          Examples::
  
              >>> from transformers import FunnelTokenizer, FunnelForPreTraining
              >>> import torch
  
              >>> tokenizer = FunnelTokenizer.from_pretrained('funnel-transformer/small')
              >>> model = FunnelForPreTraining.from_pretrained('funnel-transformer/small')
  
              >>> inputs = tokenizer("Hello, my dog is cute", return_tensors= "pt")
              >>> logits = model(**inputs).logits
          """
          return_dict = return_dict if return_dict is not None else self.config.use_return_dict
  
          discriminator_hidden_states = self.funnel(
              input_ids,
              attention_mask=attention_mask,
              token_type_ids=token_type_ids,
              inputs_embeds=inputs_embeds,
              output_attentions=output_attentions,
              output_hidden_states=output_hidden_states,
              return_dict=return_dict,
          )
          discriminator_sequence_output = discriminator_hidden_states[0]
  
          logits = self.discriminator_predictions(discriminator_sequence_output)
  
          loss = None
          if labels is not None:
              loss_fct = nn.BCEWithLogitsLoss()
              if attention_mask is not None:
                  active_loss = attention_mask.view(-1, discriminator_sequence_output.shape[1]) == 1
                  active_logits = logits.view(-1, discriminator_sequence_output.shape[1])[active_loss]
                  active_labels = labels[active_loss]
                  loss = loss_fct(active_logits, active_labels.float())
              else:
                  loss = loss_fct(logits.view(-1, discriminator_sequence_output.shape[1]), labels.float())
  
          if not return_dict:
              output = (logits,) + discriminator_hidden_states[1:]
              return ((loss,) + output) if loss is not None else output
  
          return FunnelForPreTrainingOutput(
              loss=loss,
              logits=logits,
              hidden_states=discriminator_hidden_states.hidden_states,
              attentions=discriminator_hidden_states.attentions,
          )
code_tokens: def forward ( self , input_ids = None , attention_mask = None , token_type_ids = None , inputs_embeds = None , labels = None , output_attentions = None , output_hidden_states = None , return_dict = None , ) : return_dict = return_dict if return_dict is not None else self . config . use_return_dict discriminator_hidden_states = self . funnel ( input_ids , attention_mask = attention_mask , token_type_ids = token_type_ids , inputs_embeds = inputs_embeds , output_attentions = output_attentions , output_hidden_states = output_hidden_states , return_dict = return_dict , ) discriminator_sequence_output = discriminator_hidden_states [ 0 ] logits = self . discriminator_predictions ( discriminator_sequence_output ) loss = None if labels is not None : loss_fct = nn . BCEWithLogitsLoss ( ) if attention_mask is not None : active_loss = attention_mask . view ( - 1 , discriminator_sequence_output . shape [ 1 ] ) == 1 active_logits = logits . view ( - 1 , discriminator_sequence_output . shape [ 1 ] ) [ active_loss ] active_labels = labels [ active_loss ] loss = loss_fct ( active_logits , active_labels . float ( ) ) else : loss = loss_fct ( logits . view ( - 1 , discriminator_sequence_output . shape [ 1 ] ) , labels . float ( ) ) if not return_dict : output = ( logits , ) + discriminator_hidden_states [ 1 : ] return ( ( loss , ) + output ) if loss is not None else output return FunnelForPreTrainingOutput ( loss = loss , logits = logits , hidden_states = discriminator_hidden_states . hidden_states , attentions = discriminator_hidden_states . attentions , )
short_docstring: r"""
  labels (``torch.LongTensor`` of shape ``(batch_size, sequence_length)``, `optional`):
  Labels for computing the ELECTRA-style loss.
short_docstring_tokens: r " " " labels ( ` ` torch . LongTensor ` ` of shape ` ` ( batch_size sequence_length ) ` ` ` optional ` ) : Labels for computing the ELECTRA - style loss .
comment: r"""
          labels (``torch.LongTensor`` of shape ``(batch_size, sequence_length)``, `optional`):
              Labels for computing the ELECTRA-style loss. Input should be a sequence of tokens (see :obj:`input_ids`
              docstring) Indices should be in ``[0, 1]``:
  
              - 0 indicates the token is an original token,
              - 1 indicates the token was replaced.
  
          Returns:
  
          Examples::
  
              >>> from transformers import FunnelTokenizer, FunnelForPreTraining
              >>> import torch
  
              >>> tokenizer = FunnelTokenizer.from_pretrained('funnel-transformer/small')
              >>> model = FunnelForPreTraining.from_pretrained('funnel-transformer/small')
  
              >>> inputs = tokenizer("Hello, my dog is cute", return_tensors= "pt")
              >>> logits = model(**inputs).logits
          """