def prepare_wmt_data(data_dir, en_vocabulary_size, fr_vocabulary_size):
  # Get wmt data to the specified directory.
  train_path = get_wmt_enfr_train_set(data_dir)
  dev_path = get_wmt_enfr_dev_set(data_dir)

  # Create vocabularies of the appropriate sizes.
  fr_vocab_path = os.path.join(data_dir, "vocab%d.fr" % fr_vocabulary_size)
  en_vocab_path = os.path.join(data_dir, "vocab%d.en" % en_vocabulary_size)
  create_vocabulary(fr_vocab_path, train_path + ".fr", fr_vocabulary_size)
  create_vocabulary(en_vocab_path, train_path + ".en", en_vocabulary_size)

  # Create token ids for the training data.
  fr_train_ids_path = train_path + (".ids%d.fr" % fr_vocabulary_size)
  en_train_ids_path = train_path + (".ids%d.en" % en_vocabulary_size)
  data_to_token_ids(train_path + ".fr", fr_train_ids_path, fr_vocab_path)
  data_to_token_ids(train_path + ".en", en_train_ids_path, en_vocab_path)

  # Create token ids for the development data.
  fr_dev_ids_path = dev_path + (".ids%d.fr" % fr_vocabulary_size)
  en_dev_ids_path = dev_path + (".ids%d.en" % en_vocabulary_size)
  data_to_token_ids(dev_path + ".fr", fr_dev_ids_path, fr_vocab_path)
  data_to_token_ids(dev_path + ".en", en_dev_ids_path, en_vocab_path)

  return (en_train_ids_path, fr_train_ids_path,
          en_dev_ids_path, fr_dev_ids_path,
          en_vocab_path, fr_vocab_path)
Try to behave like cloudstack, which uses
dataset/ETHPy150Open tokestermw/tensorflow-shakespeare/tensorshake/translate/data_utils.py/prepare_wmt_data--crossover with--dataset/ETHPy150Open exoscale/cs/cs.py/cs_encode