def data_to_token_ids(data_path, target_path, vocabulary_path, tokenizer=None, normalize_digits=True, UNK_ID=3, _DIGIT_RE=re.compile('\\d')):
 if (not gfile.Exists(target_path)):
  print ('Tokenizing data in %s' % data_path)
  (vocab, _) = initialize_vocabulary(vocabulary_path)
  with gfile.GFile(data_path, mode='rb') as data_file:
   with gfile.GFile(target_path, mode='w') as tokens_file:
    counter = 0
    for line in data_file:
     counter += 1
     if ((counter % 100000) == 0):
      print ('  tokenizing line %d' % counter)
     token_ids = sentence_to_token_ids(line, vocab, tokenizer, normalize_digits, UNK_ID=UNK_ID, _DIGIT_RE=_DIGIT_RE)
     tokens_file.write((' '.join([str(tok) for tok in token_ids]) + '\n'))
 else:
  print ('Target path %s exists' % target_path)

def data_to_token_ids ( data_path , target_path , vocabulary_path , tokenizer = None , normalize_digits = True , UNK_ID = 3 , _DIGIT_RE = re . compile ( '\\d' ) ) : if ( not gfile . Exists ( target_path ) ) : print ( 'Tokenizing data in %s' % data_path ) ( vocab , _ ) = initialize_vocabulary ( vocabulary_path ) with gfile . GFile ( data_path , mode = 'rb' ) as data_file : with gfile . GFile ( target_path , mode = 'w' ) as tokens_file : counter = 0 for line in data_file : counter += 1 if ( ( counter % 100000 ) == 0 ) : print ( '  tokenizing line %d' % counter ) token_ids = sentence_to_token_ids ( line , vocab , tokenizer , normalize_digits , UNK_ID = UNK_ID , _DIGIT_RE = _DIGIT_RE ) tokens_file . write ( ( ' ' . join ( [ str ( tok ) for tok in token_ids ] ) + '\n' ) ) else : print ( 'Target path %s exists' % target_path )
tokenize data file and turn into token-ids using given vocabulary file .