<s> """<STR_LIT>""" <EOL> from __future__ import absolute_import <EOL> import unittest <EOL> from cpp import tokenize <EOL> __author__ = '<STR_LIT>' <EOL> def Syntax ( name , start , end ) : <EOL> return tokenize . Token ( tokenize . SYNTAX , name , start , end ) <EOL> def Constant ( name , start , end ) : <EOL> return tokenize . Token ( tokenize . CONSTANT , name , start , end ) <EOL> def Name ( name , start , end ) : <EOL> return tokenize . Token ( tokenize . NAME , name , start , end ) <EOL> def Preprocessor ( name , start , end ) : <EOL> return tokenize . Token ( tokenize . PREPROCESSOR , name , start , end ) <EOL> def __eq__ ( self , other ) : <EOL> assert isinstance ( other , self . __class__ ) <EOL> return ( self . token_type == other . token_type and <EOL> self . name == other . name and <EOL> self . start == other . start and <EOL> self . end == other . end ) <EOL> tokenize . Token . __eq__ = __eq__ <EOL> class TokenizeTest ( unittest . TestCase ) : <EOL> def get_tokens ( self , string ) : <EOL> return list ( tokenize . get_tokens ( string ) ) <EOL> def test_get_tokens_empty_string ( self ) : <EOL> self . assertEqual ( [ ] , self . get_tokens ( '<STR_LIT>' ) ) <EOL> def test_get_tokens_whitespace ( self ) : <EOL> self . assertEqual ( [ ] , self . get_tokens ( '<STR_LIT:U+0020>' ) ) <EOL> self . assertEqual ( [ ] , self . get_tokens ( '<STR_LIT>' ) ) <EOL> def test_get_tokens_cpp_comment ( self ) : <EOL> self . assertEqual ( [ ] , self . get_tokens ( '<STR_LIT>' ) ) <EOL> def test_get_tokens_multiline_comment ( self ) : <EOL> self . assertEqual ( [ ] , self . get_tokens ( '<STR_LIT>' ) ) <EOL> def test_get_tokens_if0 ( self ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:0> , len ( tokens ) , tokens ) <EOL> def test_get_tokens_define ( self ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:1> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Preprocessor ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:15> ) , tokens [ <NUM_LIT:0> ] ) <EOL> def test_get_tokens_binary_operators ( self ) : <EOL> for operator in '<STR_LIT>' : <EOL> tokens = self . get_tokens ( '<STR_LIT>' % operator ) <EOL> self . assertEqual ( <NUM_LIT:3> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT:5>' , <NUM_LIT:0> , <NUM_LIT:1> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( operator , <NUM_LIT:2> , <NUM_LIT:3> ) , tokens [ <NUM_LIT:1> ] ) <EOL> self . assertEqual ( Constant ( '<STR_LIT:3>' , <NUM_LIT:4> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:2> ] ) <EOL> def test_get_tokens_multi_char_binary_operators ( self ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:3> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT:5>' , <NUM_LIT:0> , <NUM_LIT:1> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT>' , <NUM_LIT:2> , <NUM_LIT:4> ) , tokens [ <NUM_LIT:1> ] ) <EOL> self . assertEqual ( Constant ( '<STR_LIT:3>' , <NUM_LIT:5> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:2> ] ) <EOL> def test_get_tokens_addition_with_comment ( self ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:3> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT:5>' , <NUM_LIT:0> , <NUM_LIT:1> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:+>' , <NUM_LIT> , <NUM_LIT> ) , tokens [ <NUM_LIT:1> ] ) <EOL> self . assertEqual ( Constant ( '<STR_LIT:3>' , <NUM_LIT> , <NUM_LIT> ) , tokens [ <NUM_LIT:2> ] ) <EOL> def test_get_tokens_logical_operators ( self ) : <EOL> for operator in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' % operator ) <EOL> self . assertEqual ( <NUM_LIT:3> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Name ( '<STR_LIT:a>' , <NUM_LIT:0> , <NUM_LIT:1> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( operator , <NUM_LIT:2> , <NUM_LIT:4> ) , tokens [ <NUM_LIT:1> ] ) <EOL> self . assertEqual ( Name ( '<STR_LIT:b>' , <NUM_LIT:5> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:2> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:!>' , <NUM_LIT:0> , <NUM_LIT:1> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Name ( '<STR_LIT>' , <NUM_LIT:1> , <NUM_LIT:4> ) , tokens [ <NUM_LIT:1> ] ) <EOL> def test_get_tokens_operators ( self ) : <EOL> for operator in ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' % operator ) <EOL> self . assertEqual ( <NUM_LIT:3> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Name ( '<STR_LIT:a>' , <NUM_LIT:0> , <NUM_LIT:1> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( operator , <NUM_LIT:2> , <NUM_LIT:4> ) , tokens [ <NUM_LIT:1> ] ) <EOL> self . assertEqual ( Name ( '<STR_LIT:b>' , <NUM_LIT:5> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:2> ] ) <EOL> def test_get_tokens_ones_complement ( self ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:1> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Name ( '<STR_LIT>' , <NUM_LIT:1> , <NUM_LIT:4> ) , tokens [ <NUM_LIT:1> ] ) <EOL> def test_get_tokens_pre_increment_operators ( self ) : <EOL> for operator in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' % operator ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Syntax ( operator , <NUM_LIT:0> , <NUM_LIT:2> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Name ( '<STR_LIT>' , <NUM_LIT:2> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' % operator ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Syntax ( operator , <NUM_LIT:0> , <NUM_LIT:2> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Name ( '<STR_LIT>' , <NUM_LIT:3> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:1> ] ) <EOL> def test_get_tokens_post_increment_operators ( self ) : <EOL> for operator in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' % operator ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Name ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:3> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( operator , <NUM_LIT:3> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' % operator ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Name ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:3> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( operator , <NUM_LIT:4> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:1> ] ) <EOL> def test_get_tokens_semicolons ( self ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:4> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Name ( '<STR_LIT:foo>' , <NUM_LIT:2> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:5> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:1> ] ) <EOL> self . assertEqual ( Name ( '<STR_LIT:bar>' , <NUM_LIT:9> , <NUM_LIT:12> ) , tokens [ <NUM_LIT:2> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:12> , <NUM_LIT> ) , tokens [ <NUM_LIT:3> ] ) <EOL> def test_get_tokens_pointers1 ( self ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:4> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Name ( '<STR_LIT:foo>' , <NUM_LIT:0> , <NUM_LIT:3> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT>' , <NUM_LIT:3> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:1> ] ) <EOL> self . assertEqual ( Name ( '<STR_LIT:bar>' , <NUM_LIT:5> , <NUM_LIT:8> ) , tokens [ <NUM_LIT:2> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:8> , <NUM_LIT:9> ) , tokens [ <NUM_LIT:3> ] ) <EOL> def test_get_tokens_pointers2 ( self ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:7> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:(>' , <NUM_LIT:0> , <NUM_LIT:1> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:*>' , <NUM_LIT:1> , <NUM_LIT:2> ) , tokens [ <NUM_LIT:1> ] ) <EOL> self . assertEqual ( Name ( '<STR_LIT:foo>' , <NUM_LIT:2> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:2> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:)>' , <NUM_LIT:5> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:3> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:.>' , <NUM_LIT:6> , <NUM_LIT:7> ) , tokens [ <NUM_LIT:4> ] ) <EOL> self . assertEqual ( Name ( '<STR_LIT:bar>' , <NUM_LIT:7> , <NUM_LIT:10> ) , tokens [ <NUM_LIT:5> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:10> , <NUM_LIT:11> ) , tokens [ <NUM_LIT:6> ] ) <EOL> def test_get_tokens_block ( self ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:4> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:{>' , <NUM_LIT:0> , <NUM_LIT:1> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Constant ( '<STR_LIT:0>' , <NUM_LIT:2> , <NUM_LIT:3> ) , tokens [ <NUM_LIT:1> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:3> , <NUM_LIT:4> ) , tokens [ <NUM_LIT:2> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:}>' , <NUM_LIT:5> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:3> ] ) <EOL> def test_get_tokens_bit_fields ( self ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:5> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Name ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:8> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Name ( '<STR_LIT:foo>' , <NUM_LIT:9> , <NUM_LIT:12> ) , tokens [ <NUM_LIT:1> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT::>' , <NUM_LIT> , <NUM_LIT> ) , tokens [ <NUM_LIT:2> ] ) <EOL> self . assertEqual ( Constant ( '<STR_LIT:1>' , <NUM_LIT:15> , <NUM_LIT:16> ) , tokens [ <NUM_LIT:3> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:16> , <NUM_LIT> ) , tokens [ <NUM_LIT:4> ] ) <EOL> def test_get_tokens_assignment ( self ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:5> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Name ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:8> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Name ( '<STR_LIT:foo>' , <NUM_LIT:9> , <NUM_LIT:12> ) , tokens [ <NUM_LIT:1> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:=>' , <NUM_LIT> , <NUM_LIT> ) , tokens [ <NUM_LIT:2> ] ) <EOL> self . assertEqual ( Constant ( '<STR_LIT:1>' , <NUM_LIT:15> , <NUM_LIT:16> ) , tokens [ <NUM_LIT:3> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:16> , <NUM_LIT> ) , tokens [ <NUM_LIT:4> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:5> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Name ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:8> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Name ( '<STR_LIT:foo>' , <NUM_LIT:9> , <NUM_LIT:12> ) , tokens [ <NUM_LIT:1> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:=>' , <NUM_LIT> , <NUM_LIT> ) , tokens [ <NUM_LIT:2> ] ) <EOL> self . assertEqual ( Constant ( '<STR_LIT:1>' , <NUM_LIT:16> , <NUM_LIT> ) , tokens [ <NUM_LIT:3> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT> , <NUM_LIT> ) , tokens [ <NUM_LIT:4> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:5> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Name ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:8> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Name ( '<STR_LIT:foo>' , <NUM_LIT:9> , <NUM_LIT:12> ) , tokens [ <NUM_LIT:1> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:=>' , <NUM_LIT> , <NUM_LIT> ) , tokens [ <NUM_LIT:2> ] ) <EOL> self . assertEqual ( Constant ( '<STR_LIT:1>' , <NUM_LIT> , <NUM_LIT> ) , tokens [ <NUM_LIT:3> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT> , <NUM_LIT> ) , tokens [ <NUM_LIT:4> ] ) <EOL> def test_get_tokens_int_constants ( self ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:3> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:3> , <NUM_LIT:4> ) , tokens [ <NUM_LIT:1> ] ) <EOL> for suffix in ( '<STR_LIT:l>' , '<STR_LIT:u>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' % suffix ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> value = '<STR_LIT>' % suffix <EOL> size = len ( value ) <EOL> self . assertEqual ( Constant ( value , <NUM_LIT:0> , size ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , size , size + <NUM_LIT:1> ) , tokens [ <NUM_LIT:1> ] ) <EOL> suffix = suffix . upper ( ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' % suffix ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> value = '<STR_LIT>' % suffix <EOL> size = len ( value ) <EOL> self . assertEqual ( Constant ( value , <NUM_LIT:0> , size ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , size , size + <NUM_LIT:1> ) , tokens [ <NUM_LIT:1> ] ) <EOL> def test_get_tokens_octal_constants ( self ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:8> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:8> , <NUM_LIT:9> ) , tokens [ <NUM_LIT:1> ] ) <EOL> for suffix in ( '<STR_LIT:l>' , '<STR_LIT:u>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' % suffix ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> value = '<STR_LIT>' % suffix <EOL> size = len ( value ) <EOL> self . assertEqual ( Constant ( value , <NUM_LIT:0> , size ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , size , size + <NUM_LIT:1> ) , tokens [ <NUM_LIT:1> ] ) <EOL> suffix = suffix . upper ( ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' % suffix ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> value = '<STR_LIT>' % suffix <EOL> size = len ( value ) <EOL> self . assertEqual ( Constant ( value , <NUM_LIT:0> , size ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , size , size + <NUM_LIT:1> ) , tokens [ <NUM_LIT:1> ] ) <EOL> def test_get_tokens_hex_constants ( self ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:10> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:10> , <NUM_LIT:11> ) , tokens [ <NUM_LIT:1> ] ) <EOL> for suffix in ( '<STR_LIT:l>' , '<STR_LIT:u>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' % suffix ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> value = '<STR_LIT>' % suffix <EOL> size = len ( value ) <EOL> self . assertEqual ( Constant ( value , <NUM_LIT:0> , size ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , size , size + <NUM_LIT:1> ) , tokens [ <NUM_LIT:1> ] ) <EOL> suffix = suffix . upper ( ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' % suffix ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> value = '<STR_LIT>' % suffix <EOL> size = len ( value ) <EOL> self . assertEqual ( Constant ( value , <NUM_LIT:0> , size ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , size , size + <NUM_LIT:1> ) , tokens [ <NUM_LIT:1> ] ) <EOL> def test_get_tokens_float_constants ( self ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:4> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:4> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:5> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:5> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:3> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:3> , <NUM_LIT:4> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:8> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:8> , <NUM_LIT:9> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:8> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:8> , <NUM_LIT:9> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:5> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:5> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:5> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:5> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:4> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:4> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:4> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:4> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:4> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:4> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:4> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:4> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:1> ] ) <EOL> def test_get_tokens_char_constants ( self ) : <EOL> tokens = self . get_tokens ( "<STR_LIT>" ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( "<STR_LIT>" , <NUM_LIT:0> , <NUM_LIT:3> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:3> , <NUM_LIT:4> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( "<STR_LIT>" ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( "<STR_LIT>" , <NUM_LIT:0> , <NUM_LIT:4> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:4> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( "<STR_LIT>" ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( "<STR_LIT>" , <NUM_LIT:0> , <NUM_LIT:4> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:4> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( "<STR_LIT>" ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( "<STR_LIT>" , <NUM_LIT:0> , <NUM_LIT:4> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:4> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( r"<STR_LIT>" ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( r"<STR_LIT>" , <NUM_LIT:0> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:6> , <NUM_LIT:7> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( r"<STR_LIT>" ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( r"<STR_LIT>" , <NUM_LIT:0> , <NUM_LIT:4> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:4> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( r"<STR_LIT>" ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( r"<STR_LIT>" , <NUM_LIT:0> , <NUM_LIT:4> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:4> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( r"<STR_LIT>" ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( r"<STR_LIT>" , <NUM_LIT:0> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:5> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:1> ] ) <EOL> def test_get_tokens_string_constants ( self ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:5> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:6> , <NUM_LIT:7> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:6> , <NUM_LIT:7> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:7> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:7> , <NUM_LIT:8> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( r'<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( r'<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:9> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:9> , <NUM_LIT:10> ) , tokens [ <NUM_LIT:1> ] ) <EOL> tokens = self . get_tokens ( r'<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:2> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Constant ( r'<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:7> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:7> , <NUM_LIT:8> ) , tokens [ <NUM_LIT:1> ] ) <EOL> def test_get_tokens_ternary_operator ( self ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:6> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Name ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:4> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:?>' , <NUM_LIT:5> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:1> ] ) <EOL> self . assertEqual ( Name ( '<STR_LIT:foo>' , <NUM_LIT:7> , <NUM_LIT:10> ) , tokens [ <NUM_LIT:2> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT::>' , <NUM_LIT:11> , <NUM_LIT:12> ) , tokens [ <NUM_LIT:3> ] ) <EOL> self . assertEqual ( Name ( '<STR_LIT:bar>' , <NUM_LIT> , <NUM_LIT:16> ) , tokens [ <NUM_LIT:4> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:16> , <NUM_LIT> ) , tokens [ <NUM_LIT:5> ] ) <EOL> def test_get_tokens_identifier ( self ) : <EOL> tokens = self . get_tokens ( '<STR_LIT>' ) <EOL> self . assertEqual ( <NUM_LIT:3> , len ( tokens ) , tokens ) <EOL> self . assertEqual ( Name ( '<STR_LIT>' , <NUM_LIT:0> , <NUM_LIT:1> ) , tokens [ <NUM_LIT:0> ] ) <EOL> self . assertEqual ( Name ( '<STR_LIT>' , <NUM_LIT:2> , <NUM_LIT:5> ) , tokens [ <NUM_LIT:1> ] ) <EOL> self . assertEqual ( Syntax ( '<STR_LIT:;>' , <NUM_LIT:5> , <NUM_LIT:6> ) , tokens [ <NUM_LIT:2> ] ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> unittest . main ( ) </s>
