def _load_vocab_file(vocab_file, reserved_tokens=None):
  """Load vocabulary while ensuring reserved tokens are at the top."""
  if reserved_tokens is None:
    reserved_tokens = RESERVED_TOKENS

  subtoken_list = []
  with tf.gfile.Open(vocab_file, mode="r") as f:
    for line in f:
      subtoken = _native_to_unicode(line.strip())
      subtoken = subtoken[1:-1]  # Remove surrounding single-quotes
      if subtoken in reserved_tokens:
        continue
      subtoken_list.append(_native_to_unicode(subtoken))
  return reserved_tokens + subtoken_list
def _load_vocab_file ( vocab_file , reserved_tokens = None ) : if reserved_tokens is None : reserved_tokens = RESERVED_TOKENS subtoken_list = [ ] with tf . gfile . Open ( vocab_file , mode = "r" ) as f : for line in f : subtoken = _native_to_unicode ( line . strip ( ) ) subtoken = subtoken [ 1 : - 1 ] if subtoken in reserved_tokens : continue subtoken_list . append ( _native_to_unicode ( subtoken ) ) return reserved_tokens + subtoken_list
Load vocabulary while ensuring reserved tokens are at the top.