/// Processes the next batch of subtokens. This might not be a single
/// subtoken since some subtokens are processed at the same time,
/// such as newlines and blanklines.
Processes the next batch of subtokens. This might not be a single
subtoken since some subtokens are processed at the same time,
such as newlines and blanklines.
Processes the next batch of subtokens . This might not be a single subtoken since some subtokens are processed at the same time such as newlines and blanklines .
fn scan_subtoken(&mut self) {
        let ch = self.advance();
        match ch {
            '.' => {
                if self.peek_match(|c| !is_whitespace(c)) {
                    self.add_subtoken(SubTokenKind::InvalidChar(ch));
                } else {
                    self.add_subtoken(SubTokenKind::FullStop);
                }
            }
            '\r' => {}
            ' ' => {
                if self.peek_match(|c| c == ' ') {
                    self.add_subtoken(SubTokenKind::InvalidChar(' '));
                    // not advancing, to catch multiple spaces in a row
                }
            }
            '\n' => self.newline(),
            ch => {
                if is_identifier_char(ch) {
                    self.identifier();
                } else {
                    self.add_subtoken(SubTokenKind::InvalidChar(ch));
                }
            }
        }
    }
fn scan_subtoken ( & mut self ) { let ch = self . advance ( ) ; match ch { '.' => { if self . peek_match ( | c | ! is_whitespace ( c ) ) { self . add_subtoken ( SubTokenKind :: InvalidChar ( ch ) ) ; } else { self . add_subtoken ( SubTokenKind :: FullStop ) ; } } '\r' => { } ' ' => { if self . peek_match ( | c | c == ' ' ) { self . add_subtoken ( SubTokenKind :: InvalidChar ( ' ' ) ) ; // not advancing, to catch multiple spaces in a row } } '\n' => self . newline ( ) , ch => { if is_identifier_char ( ch ) { self . identifier ( ) ; } else { self . add_subtoken ( SubTokenKind :: InvalidChar ( ch ) ) ; } } } }
Processes the next batch of subtokens.
Processes the next batch of subtokens .
&mut self