src/Tokenizers/GeneralTokenizer.php
/**
     * Use the php function strtok to Tokenize simple string
     * @internal
     * @return array
     */
Use the php function strtok to Tokenize simple string
@internal
@return array
Use the php function strtok to Tokenize simple string @internal @return array
protected function strTokenWrapper($string)
    {
        $token = strtok($string, $this->tokenExpression);
        $tokens = array();
        while ($token !== false) {
            if(!empty(trim($token))) { 
                $tokens[] = $token;
            }
            $token = strtok($this->tokenExpression);        
        }
        return $tokens;
    }
protected function strTokenWrapper ( $ string ) { $ token = strtok ( $ string , $ this -> tokenExpression ) ; $ tokens = array ( ) ; while ( $ token !== false ) { if ( ! empty ( trim ( $ token ) ) ) { $ tokens [ ] = $ token ; } $ token = strtok ( $ this -> tokenExpression ) ; } return $ tokens ; }
Use the php function strtok to Tokenize simple string
 @internal
 @return array
Use the php function strtok to Tokenize simple string @internal @return array
// avoid tokenizing white spaces