/// Lexes and returns the next `Token` from the source code.
Lexes and returns the next `Token` from the source code.
Lexes and returns the next ` Token ` from the source code .
pub fn lex(&mut self) -> LexResult {
        let chars = self.chars.deref_mut();
        let src = self.input;
        let mut pos = self.pos;
        // Skip whitespaces
        loop {
            // Note: the following lines are in their own scope to
            // limit how long 'chars' is borrowed, and in order to allow
            // it to be borrowed again in the loop by 'chars.next()'.
            {
                let ch = chars.peek();
                if ch.is_none() {
                    self.pos = pos;
                    return Ok(Token::EOF);
                }
                if !ch.unwrap().is_whitespace() {
                    break;
                }
            }
            chars.next();
            pos += 1;
        }
        let start = pos;
        let next = chars.next();
        if next.is_none() {
            return Ok(Token::EOF);
        }
        pos += 1;
        // Actually get the next token.
        let result = match next.unwrap() {
            '(' => Ok(Token::LParen),
            ')' => Ok(Token::RParen),
            ',' => Ok(Token::Comma),
            '#' => {
                // Comment
                loop {
                    let ch = chars.next();
                    pos += 1;
                    if ch == Some('\n') {
                        break;
                    }
                }
                Ok(Token::Comment)
            }
            '.' | '0'..='9' => {
                // Parse number literal
                loop {
                    let ch = match chars.peek() {
                        Some(ch) => *ch,
                        None => return Ok(Token::EOF),
                    };
                    // Parse float.
                    if ch != '.' && !ch.is_digit(16) {
                        break;
                    }
                    chars.next();
                    pos += 1;
                }
                Ok(Token::Number(src[start..pos].parse().unwrap()))
            }
            'a'..='z' | 'A'..='Z' | '_' => {
                // Parse identifier
                loop {
                    let ch = match chars.peek() {
                        Some(ch) => *ch,
                        None => return Ok(Token::EOF),
                    };
                    // A word-like identifier only contains underscores and alphanumeric characters.
                    if ch != '_' && !ch.is_alphanumeric() {
                        break;
                    }
                    chars.next();
                    pos += 1;
                }
                match &src[start..pos] {
                    "def" => Ok(Token::Def),
                    "extern" => Ok(Token::Extern),
                    "if" => Ok(Token::If),
                    "then" => Ok(Token::Then),
                    "else" => Ok(Token::Else),
                    "for" => Ok(Token::For),
                    "in" => Ok(Token::In),
                    "unary" => Ok(Token::Unary),
                    "binary" => Ok(Token::Binary),
                    "var" => Ok(Token::Var),
                    ident => Ok(Token::Ident(ident.to_string())),
                }
            }
            op => {
                // Parse operator
                Ok(Token::Op(op))
            }
        };
        // Update stored position, and return
        self.pos = pos;
        result
    }
pub fn lex ( & mut self ) -> LexResult { let chars = self . chars . deref_mut ( ) ; let src = self . input ; let mut pos = self . pos ; // Skip whitespaces loop { // Note: the following lines are in their own scope to // limit how long 'chars' is borrowed, and in order to allow // it to be borrowed again in the loop by 'chars.next()'. { let ch = chars . peek ( ) ; if ch . is_none ( ) { self . pos = pos ; return Ok ( Token :: EOF ) ; } if ! ch . unwrap ( ) . is_whitespace ( ) { break ; } } chars . next ( ) ; pos += 1 ; } let start = pos ; let next = chars . next ( ) ; if next . is_none ( ) { return Ok ( Token :: EOF ) ; } pos += 1 ; // Actually get the next token. let result = match next . unwrap ( ) { '(' => Ok ( Token :: LParen ) , ')' => Ok ( Token :: RParen ) , ',' => Ok ( Token :: Comma ) , '#' => { // Comment loop { let ch = chars . next ( ) ; pos += 1 ; if ch == Some ( '\n' ) { break ; } } Ok ( Token :: Comment ) } '.' | '0' ..= '9' => { // Parse number literal loop { let ch = match chars . peek ( ) { Some ( ch ) => * ch , None => return Ok ( Token :: EOF ) , } ; // Parse float. if ch != '.' && ! ch . is_digit ( 16 ) { break ; } chars . next ( ) ; pos += 1 ; } Ok ( Token :: Number ( src [ start .. pos ] . parse ( ) . unwrap ( ) ) ) } 'a' ..= 'z' | 'A' ..= 'Z' | '_' => { // Parse identifier loop { let ch = match chars . peek ( ) { Some ( ch ) => * ch , None => return Ok ( Token :: EOF ) , } ; // A word-like identifier only contains underscores and alphanumeric characters. if ch != '_' && ! ch . is_alphanumeric ( ) { break ; } chars . next ( ) ; pos += 1 ; } match & src [ start .. pos ] { " " => Ok ( Token :: Def ) , " " => Ok ( Token :: Extern ) , " " => Ok ( Token :: If ) , " " => Ok ( Token :: Then ) , " " => Ok ( Token :: Else ) , " " => Ok ( Token :: For ) , " " => Ok ( Token :: In ) , " " => Ok ( Token :: Unary ) , " " => Ok ( Token :: Binary ) , " " => Ok ( Token :: Var ) , ident => Ok ( Token :: Ident ( ident . to_string ( ) ) ) , } } op => { // Parse operator Ok ( Token :: Op ( op ) ) } } ; // Update stored position, and return self . pos = pos ; result }
Lexes and returns the next `Token` from the source code.
Lexes and returns the next ` Token ` from the source code .
&mut self