Wikipedia dump helper.

    Parameters
    ----------
    wiki_root : str
        Parameter for WikiDumpStream
    wiki_date : str
        Parameter for WikiDumpStream
    wiki_language : str
        Parameter for WikiDumpStream
    max_vocab_size : int, optional
        Specifies a maximum size for the vocabulary.

    Returns
    -------
    gluonnlp.data.DataStream
        Each sample is a valid input to
        gluonnlp.data.EmbeddingCenterContextBatchify.
    gluonnlp.Vocab
        Vocabulary of all tokens in the Wikipedia corpus as provided by
        WikiDumpStream but with maximum size max_vocab_size.
    idx_to_counts : list of int
        Mapping from token indices to their occurrence-counts in the Wikipedia
        corpus.

def wiki(wiki_root, wiki_date, wiki_language, max_vocab_size=None):
    """Wikipedia dump helper.

    Parameters
    ----------
    wiki_root : str
        Parameter for WikiDumpStream
    wiki_date : str
        Parameter for WikiDumpStream
    wiki_language : str
        Parameter for WikiDumpStream
    max_vocab_size : int, optional
        Specifies a maximum size for the vocabulary.

    Returns
    -------
    gluonnlp.data.DataStream
        Each sample is a valid input to
        gluonnlp.data.EmbeddingCenterContextBatchify.
    gluonnlp.Vocab
        Vocabulary of all tokens in the Wikipedia corpus as provided by
        WikiDumpStream but with maximum size max_vocab_size.
    idx_to_counts : list of int
        Mapping from token indices to their occurrence-counts in the Wikipedia
        corpus.

    """
    data = WikiDumpStream(
        root=os.path.expanduser(wiki_root), language=wiki_language,
        date=wiki_date)
    vocab = data.vocab
    if max_vocab_size:
        for token in vocab.idx_to_token[max_vocab_size:]:
            vocab.token_to_idx.pop(token)
        vocab.idx_to_token = vocab.idx_to_token[:max_vocab_size]
    idx_to_counts = data.idx_to_counts

    def code(shard):
        return [[vocab[token] for token in sentence if token in vocab]
                for sentence in shard]

    data = data.transform(code)
    return data, vocab, idx_to_counts