def token_offsets(tokens):
 end_offset = 0
 previous_end_row = 0
 previous_end_column = 0
 for t in tokens:
  token_type = t[0]
  token_string = t[1]
  (start_row, start_column) = t[2]
  (end_row, end_column) = t[3]
  end_offset += start_column
  if (previous_end_row == start_row):
   end_offset -= previous_end_column
  start_offset = end_offset
  end_offset += len(token_string)
  (yield (token_type, token_string, start_offset, end_offset))
  previous_end_row = end_row
  previous_end_column = end_column

def token_offsets ( tokens ) : end_offset = 0 previous_end_row = 0 previous_end_column = 0 for t in tokens : token_type = t [ 0 ] token_string = t [ 1 ] ( start_row , start_column ) = t [ 2 ] ( end_row , end_column ) = t [ 3 ] end_offset += start_column if ( previous_end_row == start_row ) : end_offset -= previous_end_column start_offset = end_offset end_offset += len ( token_string ) ( yield ( token_type , token_string , start_offset , end_offset ) ) previous_end_row = end_row previous_end_column = end_column
yield tokens and offsets .