def dir_tokenize(chunks, labels, chunk_name='article', paragraphs=True,
                 verbose=1, tokenizer=word_tokenize, simple=False):
    words, chk_tokens, sent_tokens = [], [], []
    chk_break, par_break, sent_break = 0, 0 ,0
    chk_n, par_n, sent_n = 0, 0, 0
    
    if verbose == 1:
        pbar = ProgressBar(widgets=[Percentage(), Bar()], maxval=len(chunks)).start()

    if simple:
        paragraphs = False;

    if paragraphs:
        par_tokens = []
        par_n = 0
        
        for chk, label in zip(chunks, labels):
            # print 'Tokenizing', label
            pars = paragraph_tokenize(chk)

            for par in pars:
                sents = sentence_tokenize(par)
                
                for sent in sents:
                    w = tokenizer(sent)
                    words.extend(w)
                    sent_break += len(w)
                    sent_tokens.append((sent_break, label, par_n, sent_n))
                    sent_n += 1

                par_tokens.append((sent_break, label, par_n))
                par_n += 1
    
            if verbose == 1:
                pbar.update(chk_n)

            chk_tokens.append((sent_break, label))

            chk_n += 1
    else:
        for chk, label in zip(chunks, labels):
            # print 'Tokenizing', label
            if simple:
                w = tokenizer(chk)
                words.extend(w)
                chk_break += len(w)
                chk_tokens.append((chk_break, label))
  

            else:
                sents = sentence_tokenize(chk)

                for sent in sents:
                    w = tokenizer(sent)
                    words.extend(w)
                    sent_break += len(w)
                    sent_tokens.append((sent_break, label, sent_n))
                    sent_n += 1
                chk_tokens.append((sent_break, label))
    
            if verbose == 1:
                pbar.update(chk_n)

  
            chk_n += 1


    idx_dt = ('idx', np.int32)
    label_dt = (chunk_name + '_label', np.array(labels).dtype)
    if not simple:
        sent_label_dt = ('sentence_label', np.array(sent_n, np.str_).dtype)
    corpus_data = dict()
    dtype = [idx_dt, label_dt]
    corpus_data[chunk_name] = np.array(chk_tokens, dtype=dtype)

    if paragraphs:
        par_label_dt = ('paragraph_label', np.array(par_n, np.str_).dtype)
        dtype = [idx_dt, label_dt, par_label_dt]
        corpus_data['paragraph'] = np.array(par_tokens, dtype=dtype)
        dtype = [idx_dt, label_dt, par_label_dt, sent_label_dt]
        corpus_data['sentence'] = np.array(sent_tokens, dtype=dtype)
    elif not simple:
        dtype = [idx_dt, label_dt, sent_label_dt]
        corpus_data['sentence'] = np.array(sent_tokens, dtype=dtype)
    
    if verbose == 1:
        pbar.finish()

    return words, corpus_data
`dir_tokenize` is a helper function for :meth:`dir_corpus`.
dataset/ETHPy150Open inpho/vsm/vsm/extensions/corpusbuilders/corpusbuilders.py/dir_tokenize