// Tokenise text using lexer, returning tokens as a slice.
Tokenise text using lexer, returning tokens as a slice.
// Tokenise text using lexer returning tokens as a slice .
func Tokenise(lexer Lexer, options *TokeniseOptions, text string) ([]Token, error) {
	var out []Token
	it, err := lexer.Tokenise(options, text)
	if err != nil {
		return nil, err
	}
	for t := it(); t != EOF; t = it() {
		out = append(out, t)
	}
	return out, nil
}
func Tokenise ( lexer Lexer , options * TokeniseOptions , text string ) ( [ ] Token , error ) { var out [ ] Token it , err := lexer . Tokenise ( options , text ) if err != nil { return nil , err } for t := it ( ) ; t != EOF ; t = it ( ) { out = append ( out , t ) } return out , nil }
Tokenise text using lexer, returning tokens as a slice.
Tokenise text using lexer returning tokens as a slice .