def tokenize(self):
        """
        Tokenize the input text

        Scans for instances of perl tags and include directives.
        Tokenization skips line and block comments.

        Returns
        -------
        list
            List of tuples: (typ, start, end)

            Where:

            - typ is "perl" or "incl"
            - start/end mark the first/last char offset of the token
        """
        tokens = []
        token_spec = [
            ('mlc', r'/\*.*?\*/'),
            ('slc', r'//[^\r\n]*?\r?\n'),
            ('perl', r'<%.*?%>'),
            ('incl', r'`include'),
        ]
        tok_regex = '|'.join('(?P<%s>%s)' % pair for pair in token_spec)
        for m in re.finditer(tok_regex, self.text, re.DOTALL):
            if m.lastgroup in ("incl", "perl"):
                tokens.append((m.lastgroup, m.start(0), m.end(0)-1))
        return tokens
def tokenize ( self ) : tokens = [ ] token_spec = [ ( 'mlc' , r'/\*.*?\*/' ) , ( 'slc' , r'//[^\r\n]*?\r?\n' ) , ( 'perl' , r'<%.*?%>' ) , ( 'incl' , r'`include' ) , ] tok_regex = '|' . join ( '(?P<%s>%s)' % pair for pair in token_spec ) for m in re . finditer ( tok_regex , self . text , re . DOTALL ) : if m . lastgroup in ( "incl" , "perl" ) : tokens . append ( ( m . lastgroup , m . start ( 0 ) , m . end ( 0 ) - 1 ) ) return tokens
Tokenize the input text

        Scans for instances of perl tags and include directives.
        Tokenization skips line and block comments.

        Returns
        -------
        list
            List of tuples: (typ, start, end)

            Where:

            - typ is "perl" or "incl"
            - start/end mark the first/last char offset of the token