<s> import ldap . schema <EOL> from ldap . schema . tokenizer import split_tokens , extract_tokens <EOL> testcases_split_tokens = ( <EOL> ( "<STR_LIT>" , [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] ) , <EOL> ( "<STR_LIT>" , [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] ) , <EOL> ( "<STR_LIT>" , [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] ) , <EOL> ( "<STR_LIT>" , [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] ) , <EOL> ( "<STR_LIT>" , [ "<STR_LIT>" , "<STR_LIT:(>" , "<STR_LIT>" , "<STR_LIT:)>" , "<STR_LIT>" ] ) , <EOL> ( "<STR_LIT>" , [ "<STR_LIT>" , "<STR_LIT:(>" , "<STR_LIT>" , "<STR_LIT:)>" ] ) , <EOL> ( "<STR_LIT>" , [ "<STR_LIT>" , "<STR_LIT:(>" , "<STR_LIT>" , "<STR_LIT:)>" ] ) , <EOL> ( "<STR_LIT>" , [ "<STR_LIT>" , "<STR_LIT>" ] ) , <EOL> ( "<STR_LIT>" , [ "<STR_LIT:(>" , "<STR_LIT>" , "<STR_LIT:(>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:)>" , "<STR_LIT:)>" ] ) , <EOL> ( "<STR_LIT>" , [ '<STR_LIT>' , "<STR_LIT:(>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:)>" ] ) , <EOL> ( "<STR_LIT>" , [ '<STR_LIT>' , "<STR_LIT:(>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:)>" ] ) , <EOL> ( "<STR_LIT>" , [ '<STR_LIT>' , "<STR_LIT:(>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:)>" ] ) , <EOL> ( "<STR_LIT>" , [ '<STR_LIT>' , "<STR_LIT:(>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:)>" ] ) , <EOL> ( "<STR_LIT>" , [ '<STR_LIT>' , "<STR_LIT>" ] ) , <EOL> ( "<STR_LIT>" , [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) , <EOL> ( "<STR_LIT>" , [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) , <EOL> ( "<STR_LIT>" , [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] ) , <EOL> ( "<STR_LIT>" , [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> ) <EOL> for t , r in testcases_split_tokens : <EOL> l = ldap . schema . tokenizer . split_tokens ( t , { '<STR_LIT>' : None } ) <EOL> if l != r : <EOL> print '<STR_LIT>' , repr ( t ) <EOL> print '<STR_LIT>' , l <EOL> print '<STR_LIT>' , r </s>
