def test_annotate_text_utf32_directly_index_into_unicode():
 test_string = u'a \xe3 \u0201 \U0001f636 b'
 result = analyze.analyze_syntax(test_string, encoding='UTF32')
 tokens = result['tokens']
 assert (tokens[0]['text']['content'] == 'a')
 offset = tokens[0]['text'].get('beginOffset', 0)
 assert (test_string[offset] == tokens[0]['text']['content'])
 assert (tokens[1]['text']['content'] == u'\xe3')
 offset = tokens[1]['text'].get('beginOffset', 0)
 assert (test_string[offset] == tokens[1]['text']['content'])
 assert (tokens[2]['text']['content'] == u'\u0201')
 offset = tokens[2]['text'].get('beginOffset', 0)
 assert (test_string[offset] == tokens[2]['text']['content'])
 assert (tokens[3]['text']['content'] == u'\U0001f636')
 offset = tokens[3]['text'].get('beginOffset', 0)
 assert (test_string[offset] == tokens[3]['text']['content'])
 assert (tokens[4]['text']['content'] == u'b')
 offset = tokens[4]['text'].get('beginOffset', 0)
 assert (test_string[offset] == tokens[4]['text']['content'])

def test_annotate_text_utf32_directly_index_into_unicode ( ) : test_string = u'a \xe3 \u0201 \U0001f636 b' result = analyze . analyze_syntax ( test_string , encoding = 'UTF32' ) tokens = result [ 'tokens' ] assert ( tokens [ 0 ] [ 'text' ] [ 'content' ] == 'a' ) offset = tokens [ 0 ] [ 'text' ] . get ( 'beginOffset' , 0 ) assert ( test_string [ offset ] == tokens [ 0 ] [ 'text' ] [ 'content' ] ) assert ( tokens [ 1 ] [ 'text' ] [ 'content' ] == u'\xe3' ) offset = tokens [ 1 ] [ 'text' ] . get ( 'beginOffset' , 0 ) assert ( test_string [ offset ] == tokens [ 1 ] [ 'text' ] [ 'content' ] ) assert ( tokens [ 2 ] [ 'text' ] [ 'content' ] == u'\u0201' ) offset = tokens [ 2 ] [ 'text' ] . get ( 'beginOffset' , 0 ) assert ( test_string [ offset ] == tokens [ 2 ] [ 'text' ] [ 'content' ] ) assert ( tokens [ 3 ] [ 'text' ] [ 'content' ] == u'\U0001f636' ) offset = tokens [ 3 ] [ 'text' ] . get ( 'beginOffset' , 0 ) assert ( test_string [ offset ] == tokens [ 3 ] [ 'text' ] [ 'content' ] ) assert ( tokens [ 4 ] [ 'text' ] [ 'content' ] == u'b' ) offset = tokens [ 4 ] [ 'text' ] . get ( 'beginOffset' , 0 ) assert ( test_string [ offset ] == tokens [ 4 ] [ 'text' ] [ 'content' ] )
demonstrate using offsets directly .