public function tokenise ($pattern, $expand = false) {
        preg_match_all('
        /
                (?<class_U_explicit>\\\U)
                \{
                    (?<class_U_repetition>[0-9]+)
                \}
            |
                (?<class_U_implicit>\\\U)
            |
                \[
                    (?<range_token_explicit>[^]]+)
                \]
                \{
                    (?<range_repetition>[0-9]+)
                \}
            |
                \[
                    (?<range_token_implicit>[^]]+)
                \]
            |
                (?<literal_string>[^\\\[]+)

        /x
        ', $pattern, $matches, \PREG_SET_ORDER);

        $tokens = [];

        foreach ($matches as $match) {
            
            if (!empty($match['class_U_explicit'])) {
                $token = [
                    'type' => 'class',
                    'class' => static::CLASS_UPPERCASE_UNAMBIGUOUS,
                    'repetition' => (int) $match['class_U_repetition']
                ];

                if ($expand) {
                    $token['haystack'] = 'ABCDEFGHKMNOPRSTUVWXYZ23456789';
                }

                $tokens[] = $token;
            } else if (!empty($match['class_U_implicit'])) {
                $token = [
                    'type' => 'class',
                    'class' => static::CLASS_UPPERCASE_UNAMBIGUOUS,
                    'repetition' => 1
                ];

                if ($expand) {
                    $token['haystack'] = 'ABCDEFGHKMNOPRSTUVWXYZ23456789';
                }

                $tokens[] = $token;
            } else if (!empty($match['range_token_explicit'])) {
                $token = [
                    'type' => 'range',
                    'token' => $match['range_token_explicit'],
                    'repetition' => (int) $match['range_repetition']
                ];

                if ($expand) {
                    $token['haystack'] = static::expandRange($match['range_token_explicit']);
                }

                $tokens[] = $token;
            } else if (!empty($match['range_token_implicit'])) {
                $token = [
                    'type' => 'range',
                    'token' => $match['range_token_implicit'],
                    'repetition' => 1
                ];

                if ($expand) {
                    $token['haystack'] = static::expandRange($match['range_token_implicit']);
                }

                $tokens[] = $token;
            } else if (!empty($match['literal_string'])) {
                $tokens[] = [
                    'type' => 'literal',
                    'string' => $match['literal_string']
                ];
            }
        }

        return $tokens;
    }
public function tokenise ($pattern, $expand = false) {
        preg_match_all('
        /
                (?<class_U_explicit>\\\U)
                \{
                    (?<class_U_repetition>[0-9]+)
                \}
            |
                (?<class_U_implicit>\\\U)
            |
                \[
                    (?<range_token_explicit>[^]]+)
                \]
                \{
                    (?<range_repetition>[0-9]+)
                \}
            |
                \[
                    (?<range_token_implicit>[^]]+)
                \]
            |
                (?<literal_string>[^\\\[]+)

        /x
        ', $pattern, $matches, \PREG_SET_ORDER);

        $tokens = [];

        foreach ($matches as $match) {
            
            if (!empty($match['class_U_explicit'])) {
                $token = [
                    'type' => 'class',
                    'class' => static::CLASS_UPPERCASE_UNAMBIGUOUS,
                    'repetition' => (int) $match['class_U_repetition']
                ];

                if ($expand) {
                    $token['haystack'] = 'ABCDEFGHKMNOPRSTUVWXYZ23456789';
                }

                $tokens[] = $token;
            } else if (!empty($match['class_U_implicit'])) {
                $token = [
                    'type' => 'class',
                    'class' => static::CLASS_UPPERCASE_UNAMBIGUOUS,
                    'repetition' => 1
                ];

                if ($expand) {
                    $token['haystack'] = 'ABCDEFGHKMNOPRSTUVWXYZ23456789';
                }

                $tokens[] = $token;
            } else if (!empty($match['range_token_explicit'])) {
                $token = [
                    'type' => 'range',
                    'token' => $match['range_token_explicit'],
                    'repetition' => (int) $match['range_repetition']
                ];

                if ($expand) {
                    $token['haystack'] = static::expandRange($match['range_token_explicit']);
                }

                $tokens[] = $token;
            } else if (!empty($match['range_token_implicit'])) {
                $token = [
                    'type' => 'range',
                    'token' => $match['range_token_implicit'],
                    'repetition' => 1
                ];

                if ($expand) {
                    $token['haystack'] = static::expandRange($match['range_token_implicit']);
                }

                $tokens[] = $token;
            } else if (!empty($match['literal_string'])) {
                $tokens[] = [
                    'type' => 'literal',
                    'string' => $match['literal_string']
                ];
            }
        }

        return $tokens;
    }
public function tokenise ( $ pattern , $ expand = false ) { preg_match_all ( '
        /
                (?<class_U_explicit>\\\U)
                \{
                    (?<class_U_repetition>[0-9]+)
                \}
            |
                (?<class_U_implicit>\\\U)
            |
                \[
                    (?<range_token_explicit>[^]]+)
                \]
                \{
                    (?<range_repetition>[0-9]+)
                \}
            |
                \[
                    (?<range_token_implicit>[^]]+)
                \]
            |
                (?<literal_string>[^\\\[]+)

        /x
        ' , $ pattern , $ matches , \ PREG_SET_ORDER ) ; $ tokens = [ ] ; foreach ( $ matches as $ match ) { if ( ! empty ( $ match [ 'class_U_explicit' ] ) ) { $ token = [ 'type' => 'class' , 'class' => static :: CLASS_UPPERCASE_UNAMBIGUOUS , 'repetition' => ( int ) $ match [ 'class_U_repetition' ] ] ; if ( $ expand ) { $ token [ 'haystack' ] = 'ABCDEFGHKMNOPRSTUVWXYZ23456789' ; } $ tokens [ ] = $ token ; } else if ( ! empty ( $ match [ 'class_U_implicit' ] ) ) { $ token = [ 'type' => 'class' , 'class' => static :: CLASS_UPPERCASE_UNAMBIGUOUS , 'repetition' => 1 ] ; if ( $ expand ) { $ token [ 'haystack' ] = 'ABCDEFGHKMNOPRSTUVWXYZ23456789' ; } $ tokens [ ] = $ token ; } else if ( ! empty ( $ match [ 'range_token_explicit' ] ) ) { $ token = [ 'type' => 'range' , 'token' => $ match [ 'range_token_explicit' ] , 'repetition' => ( int ) $ match [ 'range_repetition' ] ] ; if ( $ expand ) { $ token [ 'haystack' ] = static :: expandRange ( $ match [ 'range_token_explicit' ] ) ; } $ tokens [ ] = $ token ; } else if ( ! empty ( $ match [ 'range_token_implicit' ] ) ) { $ token = [ 'type' => 'range' , 'token' => $ match [ 'range_token_implicit' ] , 'repetition' => 1 ] ; if ( $ expand ) { $ token [ 'haystack' ] = static :: expandRange ( $ match [ 'range_token_implicit' ] ) ; } $ tokens [ ] = $ token ; } else if ( ! empty ( $ match [ 'literal_string' ] ) ) { $ tokens [ ] = [ 'type' => 'literal' , 'string' => $ match [ 'literal_string' ] ] ; } } return $ tokens ; }
Tokeniser explodes input into components describing the properties expressed in the pattern.

@param string $pattern
@param boolean $expand Augment token definition with the haystack of possible values.
@return array
Tokeniser explodes input into components describing the properties expressed in the pattern .