def record_tokenize(records, record_names, verbose=1):
    words, record_tokens, book_tokens, page_tokens, sent_tokens = [], [], [], [], []
    sent_break, record_n, book_n, page_n, sent_n = 0, 0, 0, 0, 0

    if verbose == 1:
        pbar = ProgressBar(widgets=[Percentage(), Bar()], maxval=len(books)).start()

    for record, record_label, book_labels in zip(records, record_names, record_book_names):
        # TODO extract book data

        for book, book_label in zip(record, book_labels):
            # print 'Tokenizing', book_label
            for page, page_file in book:
                sents = sentence_tokenize(page)

                for sent in sents:
                    w = word_tokenize(sent)
                    words.extend(w)
                    sent_break += len(w)
                    sent_tokens.append((sent_break, sent_n, page_n, book_label, 
                                        record_label, page_file))
                    sent_n += 1
    
                page_tokens.append((sent_break, page_n, book_label,
                                    record_label, page_file))
                page_n += 1

    
            if verbose == 1:
                pbar.update(book_n)
                
            book_tokens.append((sent_break, book_label, record_label))
            book_n += 1

        record_tokens.append((sent_break, record_label))
        record_n += 1

    idx_dt = ('idx', np.int32)
    record_label_dt = ('record_label', np.array(record_names).dtype)
    book_label_dt = ('book_label', np.array(book_names).dtype)
    page_label_dt = ('page_label', np.array(page_n, np.str_).dtype)
    sent_label_dt = ('sentence_label', np.array(sent_n, np.str_).dtype)
    files = [f for (a,b,c,f) in page_tokens]
    file_dt = ('file', np.array(files, np.str_).dtype)

    corpus_data = dict()
    dtype = [idx_dt, record_label_dt]
    corpus_data['record'] = np.array(record_tokens, dtype=dtype)
    dtype = [idx_dt, book_label_dt, record_label_dt]
    corpus_data['book'] = np.array(book_tokens, dtype=dtype)
    dtype = [idx_dt, page_label_dt, book_label_dt, record_label_dt, file_dt]
    corpus_data['page'] = np.array(page_tokens, dtype=dtype)
    dtype = [idx_dt, sent_label_dt, page_label_dt, book_label_dt, record_label_dt, file_dt]
    corpus_data['sentence'] = np.array(sent_tokens, dtype=dtype)

    if verbose == 1:
        pbar.finish()

    return words, corpus_data


#TODO: This should be a whitelist not a blacklist
Returns:
dataset/ETHPy150Open inpho/vsm/vsm/extensions/corpusbuilders/corpusbuilders.py/record_tokenize--crossover with--dataset/ETHPy150Open Axelrod-Python/Axelrod/axelrod/result_set.py/ResultSet.build_score_diffs