def process_sequences(sequences, end_id=0, pad_val=0, is_shorten=True, remain_end_id=False):
 max_length = 0
 for (i_s, seq) in enumerate(sequences):
  is_end = False
  for (i_w, n) in enumerate(seq):
   if ((n == end_id) and (is_end == False)):
    is_end = True
    if (max_length < i_w):
     max_length = i_w
    if (remain_end_id is False):
     seq[i_w] = pad_val
   elif (is_end == True):
    seq[i_w] = pad_val
 if (remain_end_id is True):
  max_length += 1
 if is_shorten:
  for (i, seq) in enumerate(sequences):
   sequences[i] = seq[:max_length]
 return sequences

def process_sequences ( sequences , end_id = 0 , pad_val = 0 , is_shorten = True , remain_end_id = False ) : max_length = 0 for ( i_s , seq ) in enumerate ( sequences ) : is_end = False for ( i_w , n ) in enumerate ( seq ) : if ( ( n == end_id ) and ( is_end == False ) ) : is_end = True if ( max_length < i_w ) : max_length = i_w if ( remain_end_id is False ) : seq [ i_w ] = pad_val elif ( is_end == True ) : seq [ i_w ] = pad_val if ( remain_end_id is True ) : max_length += 1 if is_shorten : for ( i , seq ) in enumerate ( sequences ) : sequences [ i ] = seq [ : max_length ] return sequences
set all tokens after end token to the padding value .