original_docstring: 
          Similar to *forward* but only return features.
  
          Includes several features from "Jointly Learning to Align and
          Translate with Transformer Models" (Garg et al., EMNLP 2019).
  
          Args:
              full_context_alignment (bool, optional): don't apply
                  auto-regressive mask to self-attention (default: False).
              alignment_layer (int, optional): return mean alignment over
                  heads at this layer (default: last layer).
              alignment_heads (int, optional): only average alignment over
                  this many heads (default: all heads).
  
          Returns:
              tuple:
                  - the decoder's features of shape `(batch, tgt_len, embed_dim)`
                  - a dictionary with any model-specific outputs
          
docstring: Similar to *forward* but only return features.
  Includes several features from "Jointly Learning to Align and
  Translate with Transformer Models" (Garg et al., EMNLP 2019).
docstring_tokens: Similar to * forward * but only return features . Includes several features from " Jointly Learning to Align and Translate with Transformer Models " ( Garg et al . EMNLP 2019 ) .
code: def extract_features_scriptable(
          self, prev_output_tokens, encoder_out = None, incremental_state = None,
          full_context_alignment = False, alignment_layer = None, alignment_heads = None,):
          """
          Similar to *forward* but only return features.
  
          Includes several features from "Jointly Learning to Align and
          Translate with Transformer Models" (Garg et al., EMNLP 2019).
  
          Args:
              full_context_alignment (bool, optional): don't apply
                  auto-regressive mask to self-attention (default: False).
              alignment_layer (int, optional): return mean alignment over
                  heads at this layer (default: last layer).
              alignment_heads (int, optional): only average alignment over
                  this many heads (default: all heads).
  
          Returns:
              tuple:
                  - the decoder's features of shape `(batch, tgt_len, embed_dim)`
                  - a dictionary with any model-specific outputs
          """
          if alignment_layer is None:
              alignment_layer = self.num_layers - 1
  
          # embed positions
          positions = (
              self.embed_positions(prev_output_tokens, incremental_state=incremental_state)
              if self.embed_positions is not None else None)
  
          if incremental_state is not None:
              prev_output_tokens = prev_output_tokens[:, -1:]
              if positions is not None:
                  positions = positions[:, -1:]
          x = self.embed_scale * self.embed_tokens(prev_output_tokens)
  
          if self.quant_noise is not None:
              x = self.quant_noise(x)
  
          if self.project_in_dim is not None:
              x = self.project_in_dim(x)
  
          if positions is not None:
              x += positions
  
          if self.layernorm_embedding is not None:
              x = self.layernorm_embedding(x)
  
          x = self.dropout_module(x)
  
          # B x T x C -> T x B x C
          x = x.transpose(0, 1)
  
          self_attn_padding_mask: Optional[Tensor] = None
          if self.cross_self_attention or prev_output_tokens.eq(self.padding_idx).any():
              self_attn_padding_mask = prev_output_tokens.eq(self.padding_idx)
  
          # decoder layers
          attn: Optional[Tensor] = None
          inner_states: List[Optional[Tensor]] = [x]
          for idx, layer in enumerate(self.layers):
              if incremental_state is None and not full_context_alignment:
                  self_attn_mask = self.buffered_future_mask(x)
              else:
                  self_attn_mask = None
              x, layer_attn, _ = layer(
                  x, encoder_out.encoder_out if encoder_out is not None else None,
                  encoder_out.encoder_padding_mask if encoder_out is not None else None,
                  incremental_state, self_attn_mask=self_attn_mask, self_attn_padding_mask=self_attn_padding_mask,
                  need_attn=bool((idx == alignment_layer)), need_head_weights=bool((idx == alignment_layer)),)
              inner_states.append(x)
              if layer_attn is not None and idx == alignment_layer:
                  attn = layer_attn.float().to(x)
  
          if attn is not None:
              if alignment_heads is not None:
                  attn = attn[:alignment_heads]
  
              # average probabilities over heads
              attn = attn.mean(dim=0)
  
          if self.layer_norm is not None:
              x = self.layer_norm(x)
  
          # T x B x C -> B x T x C
          x = x.transpose(0, 1)
  
          if self.project_out_dim is not None:
              x = self.project_out_dim(x)
  
          return x, {"attn": [attn], "inner_states": inner_states}
code_tokens: def extract_features_scriptable ( self , prev_output_tokens , encoder_out = None , incremental_state = None , full_context_alignment = False , alignment_layer = None , alignment_heads = None , ) : if alignment_layer is None : alignment_layer = self . num_layers - 1 positions = ( self . embed_positions ( prev_output_tokens , incremental_state = incremental_state ) if self . embed_positions is not None else None ) if incremental_state is not None : prev_output_tokens = prev_output_tokens [ : , - 1 : ] if positions is not None : positions = positions [ : , - 1 : ] x = self . embed_scale * self . embed_tokens ( prev_output_tokens ) if self . quant_noise is not None : x = self . quant_noise ( x ) if self . project_in_dim is not None : x = self . project_in_dim ( x ) if positions is not None : x += positions if self . layernorm_embedding is not None : x = self . layernorm_embedding ( x ) x = self . dropout_module ( x ) x = x . transpose ( 0 , 1 ) self_attn_padding_mask : Optional [ Tensor ] = None if self . cross_self_attention or prev_output_tokens . eq ( self . padding_idx ) . any ( ) : self_attn_padding_mask = prev_output_tokens . eq ( self . padding_idx ) attn : Optional [ Tensor ] = None inner_states : List [ Optional [ Tensor ] ] = [ x ] for idx , layer in enumerate ( self . layers ) : if incremental_state is None and not full_context_alignment : self_attn_mask = self . buffered_future_mask ( x ) else : self_attn_mask = None x , layer_attn , _ = layer ( x , encoder_out . encoder_out if encoder_out is not None else None , encoder_out . encoder_padding_mask if encoder_out is not None else None , incremental_state , self_attn_mask = self_attn_mask , self_attn_padding_mask = self_attn_padding_mask , need_attn = bool ( ( idx == alignment_layer ) ) , need_head_weights = bool ( ( idx == alignment_layer ) ) , ) inner_states . append ( x ) if layer_attn is not None and idx == alignment_layer : attn = layer_attn . float ( ) . to ( x ) if attn is not None : if alignment_heads is not None : attn = attn [ : alignment_heads ] attn = attn . mean ( dim = 0 ) if self . layer_norm is not None : x = self . layer_norm ( x ) x = x . transpose ( 0 , 1 ) if self . project_out_dim is not None : x = self . project_out_dim ( x ) return x , { "attn" : [ attn ] , "inner_states" : inner_states }
short_docstring: Similar to *forward* but only return features.
short_docstring_tokens: Similar to * forward * but only return features .
comment: """
          Similar to *forward* but only return features.
  
          Includes several features from "Jointly Learning to Align and
          Translate with Transformer Models" (Garg et al., EMNLP 2019).
  
          Args:
              full_context_alignment (bool, optional): don't apply
                  auto-regressive mask to self-attention (default: False).
              alignment_layer (int, optional): return mean alignment over
                  heads at this layer (default: last layer).
              alignment_heads (int, optional): only average alignment over
                  this many heads (default: all heads).
  
          Returns:
              tuple:
                  - the decoder's features of shape `(batch, tgt_len, embed_dim)`
                  - a dictionary with any model-specific outputs
          """$SEP$# embed positions$SEP$# B x T x C -> T x B x C$SEP$# decoder layers$SEP$# average probabilities over heads$SEP$# T x B x C -> B x T x C