@pytest.mark.parametrize(u'text,expected_tokens', [(u"l'avion", [u"l'", u'avion']), (u"j'ai", [u"j'", u'ai'])])
def test_issue768(fr_tokenizer_w_infix, text, expected_tokens):
 tokens = fr_tokenizer_w_infix(text)
 assert (len(tokens) == 2)
 assert ([t.text for t in tokens] == expected_tokens)

@ pytest . mark . parametrize ( u'text,expected_tokens' , [ ( u"l'avion" , [ u"l'" , u'avion' ] ) , ( u"j'ai" , [ u"j'" , u'ai' ] ) ] ) def test_issue768 ( fr_tokenizer_w_infix , text , expected_tokens ) : tokens = fr_tokenizer_w_infix ( text ) assert ( len ( tokens ) == 2 ) assert ( [ t . text for t in tokens ] == expected_tokens )
allow zero-width infix token during the tokenization process .