//Tokenize will tokenize a given sentence according to the tokenizer of the given id
Tokenize will tokenize a given sentence according to the tokenizer of the given id
// Tokenize will tokenize a given sentence according to the tokenizer of the given id
func Tokenize(id string, sentence []rune) ([]FastToken, error) {
	ch, err := StartCheckingForDates(sentence)
	if err != nil {
		return nil, err
	}
	dictReq := DICTRequest{ID: id, Type: DICTPreCache, Out: make(chan DICTRequest)}
	SendDICTToChannel(DICTInputChannel, dictReq)
	<-dictReq.Out
	req := Request{
		ID:       id,
		Type:     TokenizerGet,
		Sentence: sentence,
		Out:      make(chan Request),
	}
	go SendTokenizerToChannel(TokenizerInputChannel, req)
	res := <-req.Out
	if !res.Valid {
		return nil, errors.New("couldn't tokenize the given sentence for the id " + id)
	}
	res.Matches = BuildTimeNodes(res.Matches, ch)
	res.Matches = BuildUnknowns(sentence, res.Matches)
	res.Matches = AdjustPositions(res.Matches)
	result := []FastToken{}
	for _, v := range res.Matches {
		result = append(result, v.FastToken())
	}
	return result, nil
}
func Tokenize ( id string , sentence [ ] rune ) ( [ ] FastToken , error ) { ch , err := StartCheckingForDates ( sentence ) if err != nil { return nil , err } dictReq := DICTRequest { ID : id , Type : DICTPreCache , Out : make ( chan DICTRequest ) } SendDICTToChannel ( DICTInputChannel , dictReq ) <- dictReq . Out req := Request { ID : id , Type : TokenizerGet , Sentence : sentence , Out : make ( chan Request ) , } go SendTokenizerToChannel ( TokenizerInputChannel , req ) res := <- req . Out if ! res . Valid { return nil , errors . New ( " " + id ) } res . Matches = BuildTimeNodes ( res . Matches , ch ) res . Matches = BuildUnknowns ( sentence , res . Matches ) res . Matches = AdjustPositions ( res . Matches ) result := [ ] FastToken { } for _ , v := range res . Matches { result = append ( result , v . FastToken ( ) ) } return result , nil }
Tokenize will tokenize a given sentence according to the tokenizer of the given id
Tokenize will tokenize a given sentence according to the tokenizer of the given id
/*
	 * We will initiate the datetime service
	 * Then we will prepcache the dictionary for the id
	 * We will make a request to the tokenizer to get the sentence tokenized
	 * Then we will build the unknowns
	 * Then we will adjust the date nodes
	 * Then we will adjust the postions
	 * Then we will do a fast token for all the tokens and return the same
	 */ //start checking for the dates //error while checking for the dates //precaching //requesting the tokenizer to tokenize the given sentence //adjusting the date fields //adding the date fields before adding unknowns is important as the //positions still refers to the original position in the sentence //building the unknowns //adjusting the positions of the tokens according to the position in the token list //fast tokenizing the tokens