<s> from lda2vec import preprocess <EOL> import numpy as np <EOL> import pytest <EOL> import os <EOL> on_ci = os . environ . get ( '<STR_LIT>' , False ) == '<STR_LIT:true>' <EOL> @ pytest . mark . skipif ( on_ci , reason = '<STR_LIT>' ) <EOL> def test_tokenize ( ) : <EOL> texts = [ u'<STR_LIT>' ] <EOL> texts += [ u'<STR_LIT>' ] <EOL> arr , vocab = preprocess . tokenize ( texts , <NUM_LIT:10> ) <EOL> assert arr [ <NUM_LIT:0> , <NUM_LIT:0> ] != arr [ <NUM_LIT:0> , <NUM_LIT:1> ] <EOL> assert arr . shape [ <NUM_LIT:0> ] == <NUM_LIT:2> <EOL> assert arr . shape [ <NUM_LIT:1> ] == <NUM_LIT:10> <EOL> assert arr [ <NUM_LIT:0> , - <NUM_LIT:1> ] == - <NUM_LIT:2> <EOL> assert arr . dtype == np . dtype ( '<STR_LIT>' ) <EOL> first_word = texts [ <NUM_LIT:0> ] . split ( '<STR_LIT:U+0020>' ) [ <NUM_LIT:0> ] . lower ( ) <EOL> first_lowr = preprocess . nlp . vocab [ arr [ <NUM_LIT:0> , <NUM_LIT:0> ] ] . lower_ <EOL> assert first_word == first_lowr </s>
