def test_analyze_syntax_utf8():
 test_string = u'a \xe3 \u0201 \U0001f636 b'
 byte_array = test_string.encode('utf8')
 result = analyze.analyze_syntax(test_string, encoding='UTF8')
 tokens = result['tokens']
 assert (tokens[0]['text']['content'] == 'a')
 offset = tokens[0]['text'].get('beginOffset', 0)
 assert (byte_array[offset:(offset + 1)].decode('utf8') == tokens[0]['text']['content'])
 assert (tokens[1]['text']['content'] == u'\xe3')
 offset = tokens[1]['text'].get('beginOffset', 0)
 assert (byte_array[offset:(offset + 2)].decode('utf8') == tokens[1]['text']['content'])
 assert (tokens[2]['text']['content'] == u'\u0201')
 offset = tokens[2]['text'].get('beginOffset', 0)
 assert (byte_array[offset:(offset + 2)].decode('utf8') == tokens[2]['text']['content'])
 assert (tokens[3]['text']['content'] == u'\U0001f636')
 offset = tokens[3]['text'].get('beginOffset', 0)
 assert (byte_array[offset:(offset + 4)].decode('utf8') == tokens[3]['text']['content'])
 assert (tokens[4]['text']['content'] == u'b')
 offset = tokens[4]['text'].get('beginOffset', 0)
 assert (byte_array[offset:(offset + 1)].decode('utf8') == tokens[4]['text']['content'])

def test_analyze_syntax_utf8 ( ) : test_string = u'a \xe3 \u0201 \U0001f636 b' byte_array = test_string . encode ( 'utf8' ) result = analyze . analyze_syntax ( test_string , encoding = 'UTF8' ) tokens = result [ 'tokens' ] assert ( tokens [ 0 ] [ 'text' ] [ 'content' ] == 'a' ) offset = tokens [ 0 ] [ 'text' ] . get ( 'beginOffset' , 0 ) assert ( byte_array [ offset : ( offset + 1 ) ] . decode ( 'utf8' ) == tokens [ 0 ] [ 'text' ] [ 'content' ] ) assert ( tokens [ 1 ] [ 'text' ] [ 'content' ] == u'\xe3' ) offset = tokens [ 1 ] [ 'text' ] . get ( 'beginOffset' , 0 ) assert ( byte_array [ offset : ( offset + 2 ) ] . decode ( 'utf8' ) == tokens [ 1 ] [ 'text' ] [ 'content' ] ) assert ( tokens [ 2 ] [ 'text' ] [ 'content' ] == u'\u0201' ) offset = tokens [ 2 ] [ 'text' ] . get ( 'beginOffset' , 0 ) assert ( byte_array [ offset : ( offset + 2 ) ] . decode ( 'utf8' ) == tokens [ 2 ] [ 'text' ] [ 'content' ] ) assert ( tokens [ 3 ] [ 'text' ] [ 'content' ] == u'\U0001f636' ) offset = tokens [ 3 ] [ 'text' ] . get ( 'beginOffset' , 0 ) assert ( byte_array [ offset : ( offset + 4 ) ] . decode ( 'utf8' ) == tokens [ 3 ] [ 'text' ] [ 'content' ] ) assert ( tokens [ 4 ] [ 'text' ] [ 'content' ] == u'b' ) offset = tokens [ 4 ] [ 'text' ] . get ( 'beginOffset' , 0 ) assert ( byte_array [ offset : ( offset + 1 ) ] . decode ( 'utf8' ) == tokens [ 4 ] [ 'text' ] [ 'content' ] )
demonstrate the interpretation of the offsets when encoding=utf8 .