/*
 * This file contains the tests and test utilities for tokenizer
 */
This file contains the tests and test utilities for tokenizer
/ * * This file contains the tests and test utilities for tokenizer * /
func TestTokenize(t *testing.T) {
	loadTestDICT()
	time.Sleep(time.Second)
	toks, err := interpreter.Tokenize(testUser, []rune("show me the brands of with Swift cars"))
	if err != nil {
		t.Fatal("error while tokenizing the sentence", err)
	}
	if len(toks) == 0 {
		t.Error("Expected to have more than 1 tokens found. Go none")
	}
}
func TestTokenize ( t * testing . T ) { loadTestDICT ( ) time . Sleep ( time . Second ) toks , err := interpreter . Tokenize ( testUser , [ ] rune ( " " ) ) if err != nil { t . Fatal ( " " , err ) } if len ( toks ) == 0 { t . Error ( " " ) } }
This file contains the tests and test utilities for tokenizer
This file contains the tests and test utilities for tokenizer