    def test_from_params_valid_vocab_extension_thoroughly(self):
        """
        Tests for Valid Vocab Extension thoroughly: Vocab extension is valid
        when overlapping namespaces have same padding behaviour (padded/non-padded)
        Summary of namespace paddings in this test:
        original_vocab namespaces
            tokens0     padded
            tokens1     non-padded
            tokens2     padded
            tokens3     non-padded
        instances namespaces
            tokens0     padded
            tokens1     non-padded
            tokens4     padded
            tokens5     non-padded
        TypicalExtention example: (of tokens1 namespace)
        -> original_vocab index2token
           apple          #0->apple
           bat            #1->bat
           cat            #2->cat
        -> Token to be extended with: cat, an, apple, banana, atom, bat
        -> extended_vocab: index2token
           apple           #0->apple
           bat             #1->bat
           cat             #2->cat
           an              #3->an
           atom            #4->atom
           banana          #5->banana
        """

        vocab_dir = self.TEST_DIR / "vocab_save"
        original_vocab = Vocabulary(non_padded_namespaces=["tokens1", "tokens3"])
        original_vocab.add_token_to_namespace("apple", namespace="tokens0")  # index:2
        original_vocab.add_token_to_namespace("bat", namespace="tokens0")  # index:3
        original_vocab.add_token_to_namespace("cat", namespace="tokens0")  # index:4

        original_vocab.add_token_to_namespace("apple", namespace="tokens1")  # index:0
        original_vocab.add_token_to_namespace("bat", namespace="tokens1")  # index:1
        original_vocab.add_token_to_namespace("cat", namespace="tokens1")  # index:2

        original_vocab.add_token_to_namespace("a", namespace="tokens2")  # index:0
        original_vocab.add_token_to_namespace("b", namespace="tokens2")  # index:1
        original_vocab.add_token_to_namespace("c", namespace="tokens2")  # index:2

        original_vocab.add_token_to_namespace("p", namespace="tokens3")  # index:0
        original_vocab.add_token_to_namespace("q", namespace="tokens3")  # index:1

        original_vocab.save_to_files(vocab_dir)

        text_field0 = TextField(
            [Token(t) for t in ["cat", "an", "apple", "banana", "atom", "bat"]],
            {"tokens0": SingleIdTokenIndexer("tokens0")},
        )
        text_field1 = TextField(
            [Token(t) for t in ["cat", "an", "apple", "banana", "atom", "bat"]],
            {"tokens1": SingleIdTokenIndexer("tokens1")},
        )
        text_field4 = TextField(
            [Token(t) for t in ["l", "m", "n", "o"]], {"tokens4": SingleIdTokenIndexer("tokens4")}
        )
        text_field5 = TextField(
            [Token(t) for t in ["x", "y", "z"]], {"tokens5": SingleIdTokenIndexer("tokens5")}
        )
        instances = Batch(
            [
                Instance(
                    {
                        "text0": text_field0,
                        "text1": text_field1,
                        "text4": text_field4,
                        "text5": text_field5,
                    }
                )
            ]
        )

        params = Params(
            {
                "type": "extend",
                "directory": vocab_dir,
                "non_padded_namespaces": ["tokens1", "tokens5"],
            }
        )
        extended_vocab = Vocabulary.from_params(params, instances=instances)

        # namespaces: tokens0, tokens1 is common.
        # tokens2, tokens3 only vocab has. tokens4, tokens5 only instances
        extended_namespaces = {*extended_vocab._token_to_index}
        assert extended_namespaces == {"tokens{}".format(i) for i in range(6)}

        # # Check that _non_padded_namespaces list is consistent after extension
        assert extended_vocab._non_padded_namespaces == {"tokens1", "tokens3", "tokens5"}

        # # original_vocab["tokens1"] has 3 tokens, instances of "tokens1" ns has 5 tokens. 2 overlapping
        assert extended_vocab.get_vocab_size("tokens1") == 6
        assert extended_vocab.get_vocab_size("tokens0") == 8  # 2 extra overlapping because padded

        # namespace tokens3, tokens4 was only in original_vocab,
        # and its token count should be same in extended_vocab
        assert extended_vocab.get_vocab_size("tokens2") == original_vocab.get_vocab_size("tokens2")
        assert extended_vocab.get_vocab_size("tokens3") == original_vocab.get_vocab_size("tokens3")

        # namespace tokens2 was only in instances,
        # and its token count should be same in extended_vocab
        assert extended_vocab.get_vocab_size("tokens4") == 6  # l,m,n,o + oov + padding
        assert extended_vocab.get_vocab_size("tokens5") == 3  # x,y,z

        # Word2index mapping of all words in all namespaces of original_vocab
        # should be maintained in extended_vocab
        for namespace, token2index in original_vocab._token_to_index.items():
            for token, _ in token2index.items():
                vocab_index = original_vocab.get_token_index(token, namespace)
                extended_vocab_index = extended_vocab.get_token_index(token, namespace)
                assert vocab_index == extended_vocab_index
        # And same for Index2Word mapping
        for namespace, index2token in original_vocab._index_to_token.items():
            for index, _ in index2token.items():
                vocab_token = original_vocab.get_token_from_index(index, namespace)
                extended_vocab_token = extended_vocab.get_token_from_index(index, namespace)
                assert vocab_token == extended_vocab_token
Tests for Valid Vocab Extension thoroughly: Vocab extension is valid
when overlapping namespaces have same padding behaviour (padded/non-padded)
Summary of namespace paddings in this test:
original_vocab namespaces
tokens0     padded
tokens1     non-padded
tokens2     padded
tokens3     non-padded
instances namespaces
tokens0     padded
tokens1     non-padded
tokens4     padded
tokens5     non-padded
TypicalExtention example: (of tokens1 namespace)
-> original_vocab index2token
apple          #0->apple
bat            #1->bat
cat            #2->cat
-> Token to be extended with: cat, an, apple, banana, atom, bat
-> extended_vocab: index2token
apple           #0->apple
bat             #1->bat
cat             #2->cat
an              #3->an
atom            #4->atom
banana          #5->banana
tests/data/vocabulary_test.py