//todo: add support for individual chars with single quotes
//todo: add support for escaped characters (\n, \t, etc)
/// Parses a single string into a token stream.
add support for individual chars with single quotes
todo: add support for escaped characters (\n, \t, etc)
Parses a single string into a token stream.
add support for individual chars with single quotes todo : add support for escaped characters ( \ n \ t etc ) Parses a single string into a token stream .
pub fn tokenize(&mut self, code: &'a str) -> Result<Vec<Token>, AsmParseErr> {
        self.code = code.chars().peekable();
        let mut tokens = Vec::new();
        let mut buffer = String::new();
        while let Some(c) = self.code.next() {
            if c.is_whitespace() && c != ' ' {
                if !buffer.is_empty() {
                    if let Ok(num) = self.parse_as_number(&buffer) {
                        tokens.push(Token::NumLiteral(num, self.context))
                    } else {
                        tokens.push(
                            Token::Opcode(self.consume_opcode(buffer.clone()).ok_or(
                                AsmParseErr::UnexpectedToken(buffer.clone(), self.context)
                            )?, self.context)
                        );
                    }
                    buffer.clear();
                }
                if c == '\n' {
                    self.context.line += 1;
                    self.context.column = 1;
                }
                continue
            }
            match c {
                '$' => {
                    let token = self.consume_register()?;
                    self.state = Some(token.clone());
                    tokens.push(token);
                    continue
                }
                '[' => {
                    let token = self.consume_pointer()?;
                    self.state = Some(token.clone());
                    tokens.push(token);
                    continue
                }
                ':' => {
                    let token = Token::LabelDeclStart(buffer.clone(), self.context);
                    self.state = Some(token.clone());
                    tokens.push(token);
                    buffer.clear();
                    continue
                }
                '{' => {
                    if let Some(&Token::LabelDeclStart(_, _)) = tokens.last()  {
                        continue
                    } else {
                        return Err(AsmParseErr::UnexpectedToken(c.to_string(), self.context))
                    }
                }
                '}' => {
                    tokens.push(Token::LabelDeclEnd(self.context));
                    continue
                }
                '@' => {
                    tokens.push(self.consume_label_use()?);
                    continue
                }
                '"' => {
                    tokens.push(self.consume_str_lit()?);
                }
                '.' => {
                    tokens.push(self.consume_directive()?);
                }
                ' ' => {
                    if !buffer.is_empty() {
                        if let Ok(num) = self.parse_as_number(&buffer) {
                            tokens.push(Token::NumLiteral(num, self.context))
                        } else {
                            tokens.push(
                                Token::Opcode(self.consume_opcode(buffer.clone()).ok_or(
                                    AsmParseErr::UnexpectedToken(buffer.clone(), self.context)
                                )?, self.context)
                            );
                        }
                        buffer.clear();
                    }
                }
                _ => {
                    buffer.push(c);
                }
            }
            self.context.column += 1;
        }
        if !buffer.is_empty() {
            tokens.push(self.consume_last_token(buffer)?);
        }
        Ok(tokens)
    }
pub fn tokenize ( & mut self , code : & ' a str ) -> Result < Vec < Token > , AsmParseErr > { self . code = code . chars ( ) . peekable ( ) ; let mut tokens = Vec :: new ( ) ; let mut buffer = String :: new ( ) ; while let Some ( c ) = self . code . next ( ) { if c . is_whitespace ( ) && c != ' ' { if ! buffer . is_empty ( ) { if let Ok ( num ) = self . parse_as_number ( & buffer ) { tokens . push ( Token :: NumLiteral ( num , self . context ) ) } else { tokens . push ( Token :: Opcode ( self . consume_opcode ( buffer . clone ( ) ) . ok_or ( AsmParseErr :: UnexpectedToken ( buffer . clone ( ) , self . context ) ) ? , self . context ) ) ; } buffer . clear ( ) ; } if c == '\n' { self . context . line += 1 ; self . context . column = 1 ; } continue } match c { '$' => { let token = self . consume_register ( ) ? ; self . state = Some ( token . clone ( ) ) ; tokens . push ( token ) ; continue } '[' => { let token = self . consume_pointer ( ) ? ; self . state = Some ( token . clone ( ) ) ; tokens . push ( token ) ; continue } ':' => { let token = Token :: LabelDeclStart ( buffer . clone ( ) , self . context ) ; self . state = Some ( token . clone ( ) ) ; tokens . push ( token ) ; buffer . clear ( ) ; continue } '{' => { if let Some ( & Token :: LabelDeclStart ( _ , _ ) ) = tokens . last ( ) { continue } else { return Err ( AsmParseErr :: UnexpectedToken ( c . to_string ( ) , self . context ) ) } } '}' => { tokens . push ( Token :: LabelDeclEnd ( self . context ) ) ; continue } '@' => { tokens . push ( self . consume_label_use ( ) ? ) ; continue } '"' => { tokens . push ( self . consume_str_lit ( ) ? ) ; } '.' => { tokens . push ( self . consume_directive ( ) ? ) ; } ' ' => { if ! buffer . is_empty ( ) { if let Ok ( num ) = self . parse_as_number ( & buffer ) { tokens . push ( Token :: NumLiteral ( num , self . context ) ) } else { tokens . push ( Token :: Opcode ( self . consume_opcode ( buffer . clone ( ) ) . ok_or ( AsmParseErr :: UnexpectedToken ( buffer . clone ( ) , self . context ) ) ? , self . context ) ) ; } buffer . clear ( ) ; } } _ => { buffer . push ( c ) ; } } self . context . column += 1 ; } if ! buffer . is_empty ( ) { tokens . push ( self . consume_last_token ( buffer ) ? ) ; } Ok ( tokens ) }
todo: add support for individual chars with single quotes
todo: add support for escaped characters (\n, \t, etc)
Parses a single string into a token stream.
todo : add support for individual chars with single quotes todo : add support for escaped characters ( \ n \ t etc ) Parses a single string into a token stream .
&mut self
code
&'a str