    def add_transformer_vocab(
        self, tokenizer: PreTrainedTokenizer, namespace: str = "tokens"
    ) -> None:
        """
        Copies tokens from a transformer tokenizer's vocab into the given namespace.
        """
        try:
            vocab_items = tokenizer.get_vocab().items()
        except NotImplementedError:
            vocab_items = (
                (tokenizer.convert_ids_to_tokens(idx), idx) for idx in range(tokenizer.vocab_size)
            )

        for word, idx in vocab_items:
            self._token_to_index[namespace][word] = idx
            self._index_to_token[namespace][idx] = word

        self._non_padded_namespaces.add(namespace)
Copies tokens from a transformer tokenizer's vocab into the given namespace.
allennlp/data/vocabulary.py