Predicates/SimpleParser/AbstractTree/Tokenizer.cs
/// <summary>
/// This method tokenize the string expression.
/// It is the very first step to do to parse text and produces
/// a tree structure that will be further elaborated
/// </summary>
/// <param name="expression"></param>
/// <returns></returns>
This method tokenize the string expression.
It is the very first step to do to parse text and produces
a tree structure that will be further elaborated
This method tokenize the string expression . It is the very first step to do to parse text and produces a tree structure that will be further elaborated
public IList<NodeItem> Tokenize(string expression)
        {
            _builder = new StringBuilder();
            var root = new NodeItem(null, ""); 
            _current = root;
            bool _lastParenthesisWasNotDown = false;
            Down();
            foreach (var ch in expression)
            {
                switch (ch)
                {
                    case '(':
                        if (BuilderIsEmpty())
                        {
                            Down();
                        }
                        else
                        {
                            _lastParenthesisWasNotDown = true;
                            CommitDownFunction();
                        }
                        break;
                    case ')':
                        if (_lastParenthesisWasNotDown)
                        {
                            Commit();
                            Up();
                            _lastParenthesisWasNotDown = false;
                        }
                        else
                        {
                            Commit();
                            Up();
                        }
                        break;
                    case '^':
                    case '%':
                    case '!':
                    case '~':
                    case '+':
                    case '-':
                    case '*':
                    case '/':
                    case '>':
                    case '<':
                    case '=':
                    case '&':
                    case '|':
                        Commit();
                        CommitSpecial(ch);
                        break;
                    case ',':
                        Commit();
                        CommitSpecial(ch);
                        break;
                    case ' ':
                        Commit();
                        _builder.Append(ch);
                        Commit();
                        break;
                    default:
                        _builder.Append(ch);
                        break;
                }
            }
            Commit();
            Up();
            root.RemoveEmptyChildren();
            var str = root.ToString();
            Debug.WriteLine(string.Format("Before: {0}\r\nAfter:  {1}", expression, str));
            if (_current.Parent != null)
            {
                throw new Exception("Parenthesis error");
            }
            return _current.NodeItems;
        }
public IList < NodeItem > Tokenize ( string expression ) { _builder = new StringBuilder ( ) ; var root = new NodeItem ( null , " " ) ; _current = root ; bool _lastParenthesisWasNotDown = false ; Down ( ) ; foreach ( var ch in expression ) { switch ( ch ) { case ' ' : if ( BuilderIsEmpty ( ) ) { Down ( ) ; } else { _lastParenthesisWasNotDown = true ; CommitDownFunction ( ) ; } break ; case ' ' : if ( _lastParenthesisWasNotDown ) { Commit ( ) ; Up ( ) ; _lastParenthesisWasNotDown = false ; } else { Commit ( ) ; Up ( ) ; } break ; case ' ' : case ' ' : case ' ' : case ' ' : case ' ' : case ' ' : case ' ' : case ' ' : case ' ' : case ' ' : case ' ' : case ' ' : case ' ' : Commit ( ) ; CommitSpecial ( ch ) ; break ; case ' ' : Commit ( ) ; CommitSpecial ( ch ) ; break ; case ' ' : Commit ( ) ; _builder . Append ( ch ) ; Commit ( ) ; break ; default : _builder . Append ( ch ) ; break ; } } Commit ( ) ; Up ( ) ; root . RemoveEmptyChildren ( ) ; var str = root . ToString ( ) ; Debug . WriteLine ( string . Format ( " \r \n " , expression , str ) ) ; if ( _current . Parent != null ) { throw new Exception ( " " ) ; } return _current . NodeItems ; }
This method tokenize the string expression.
This method tokenize the string expression .
// root //Commit(); //_builder.Append(ch); //_builder.Append(ch); //Commit();
None
expression
string
None