Tokenizer for the policy language.

def _parse_tokenize(self, rule):
        """Tokenizer for the policy language."""

        for token in self._TOKENIZE_RE.split(rule):
            # Skip empty tokens
            if not token or token.isspace():
                continue

            # Handle leading parens on the token
            clean = token.lstrip('(')
            for i in range(len(token) - len(clean)):
                yield '(', '('

            # If it was only parentheses, continue
            if not clean:
                continue
            else:
                token = clean

            # Handle trailing parens on the token
            clean = token.rstrip(')')
            trail = len(token) - len(clean)

            # Yield the cleaned token
            lowered = clean.lower()
            if lowered in ('and', 'or', 'not'):
                # Special tokens
                yield lowered, clean
            elif clean:
                # Not a special token, but not composed solely of ')'
                if len(token) >= 2 and ((token[0], token[-1]) in
                                        [('"', '"'), ("'", "'")]):
                    # It's a quoted string
                    yield 'string', token[1:-1]
                else:
                    yield 'check', self._parse_check(clean)

            # Yield the trailing parens
            for i in range(trail):
                yield ')', ')'