// Iterator returns the next Token from the lexer.
Iterator returns the next Token from the lexer.
// Iterator returns the next Token from the lexer .
func (l *LexerState) Iterator() Token { 
	for l.Pos < len(l.Text) && len(l.Stack) > 0 {
		for len(l.iteratorStack) > 0 {
			n := len(l.iteratorStack) - 1
			t := l.iteratorStack[n]()
			if t == EOF {
				l.iteratorStack = l.iteratorStack[:n]
				continue
			}
			return t
		}
		l.State = l.Stack[len(l.Stack)-1]
		if l.Lexer.trace {
			fmt.Fprintf(os.Stderr, "%s: pos=%d, text=%q\n", l.State, l.Pos, string(l.Text[l.Pos:]))
		}
		selectedRule, ok := l.Rules[l.State]
		if !ok {
			panic("unknown state " + l.State)
		}
		ruleIndex, rule, groups := matchRules(l.Text, l.Pos, selectedRule)
		if groups == nil {
			 If the RegexLexer encounters a newline that is flagged as an error token, the stack is
			 emptied and the lexer continues scanning in the 'root' state. This can help producing
			 error-tolerant highlighting for erroneous input, e.g. when a single-line string is not
			 closed.
			if l.Text[l.Pos] == '\n' && l.State != l.options.State {
				l.Stack = []string{l.options.State}
				continue
			}
			l.Pos++
			return Token{Error, string(l.Text[l.Pos-1 : l.Pos])}
		}
		l.Rule = ruleIndex
		l.Groups = groups
		l.Pos += utf8.RuneCountInString(groups[0])
		if rule.Mutator != nil {
			if err := rule.Mutator.Mutate(l); err != nil {
				panic(err)
			}
		}
		if rule.Type != nil {
			l.iteratorStack = append(l.iteratorStack, rule.Type.Emit(l.Groups, l.Lexer))
		}
	}
	 Exhaust the IteratorStack, if any.
	 Duplicate code, but eh.
	for len(l.iteratorStack) > 0 {
		n := len(l.iteratorStack) - 1
		t := l.iteratorStack[n]()
		if t == EOF {
			l.iteratorStack = l.iteratorStack[:n]
			continue
		}
		return t
	}
	 If we get to here and we still have text, return it as an error.
	if l.Pos != len(l.Text) && len(l.Stack) == 0 {
		value := string(l.Text[l.Pos:])
		l.Pos = len(l.Text)
		return Token{Type: Error, Value: value}
	}
	return EOF
}
func ( l * LexerState ) Iterator ( ) Token { for l . Pos < len ( l . Text ) && len ( l . Stack ) > 0 { for len ( l . iteratorStack ) > 0 { n := len ( l . iteratorStack ) - 1 t := l . iteratorStack [ n ] ( ) if t == EOF { l . iteratorStack = l . iteratorStack [ : n ] continue } return t } l . State = l . Stack [ len ( l . Stack ) - 1 ] if l . Lexer . trace { fmt . Fprintf ( os . Stderr , " \n " , l . State , l . Pos , string ( l . Text [ l . Pos : ] ) ) } selectedRule , ok := l . Rules [ l . State ] if ! ok { panic ( " " + l . State ) } ruleIndex , rule , groups := matchRules ( l . Text , l . Pos , selectedRule ) if groups == nil { if l . Text [ l . Pos ] == '\n' && l . State != l . options . State { l . Stack = [ ] string { l . options . State } continue } l . Pos ++ return Token { Error , string ( l . Text [ l . Pos - 1 : l . Pos ] ) } } l . Rule = ruleIndex l . Groups = groups l . Pos += utf8 . RuneCountInString ( groups [ 0 ] ) if rule . Mutator != nil { if err := rule . Mutator . Mutate ( l ) ; err != nil { panic ( err ) } } if rule . Type != nil { l . iteratorStack = append ( l . iteratorStack , rule . Type . Emit ( l . Groups , l . Lexer ) ) } } for len ( l . iteratorStack ) > 0 { n := len ( l . iteratorStack ) - 1 t := l . iteratorStack [ n ] ( ) if t == EOF { l . iteratorStack = l . iteratorStack [ : n ] continue } return t } if l . Pos != len ( l . Text ) && len ( l . Stack ) == 0 { value := string ( l . Text [ l . Pos : ] ) l . Pos = len ( l . Text ) return Token { Type : Error , Value : value } } return EOF }
Iterator returns the next Token from the lexer.
Iterator returns the next Token from the lexer .
// nolint: gocognit // Exhaust the iterator stack, if any. // No match. // From Pygments :\ // // If the RegexLexer encounters a newline that is flagged as an error token, the stack is // emptied and the lexer continues scanning in the 'root' state. This can help producing // error-tolerant highlighting for erroneous input, e.g. when a single-line string is not // closed. // Exhaust the IteratorStack, if any. // Duplicate code, but eh. // If we get to here and we still have text, return it as an error.