//BuildUnknowns build unknown nodes.
//It return the tokens with unidentified words in the sentence transfomed to tokens with unknowns
BuildUnknowns build unknown nodes.
It return the tokens with unidentified words in the sentence transfomed to tokens with unknowns
// BuildUnknowns build unknown nodes . // It return the tokens with unidentified words in the sentence transfomed to tokens with unknowns
func BuildUnknowns(sentence []rune, toks []Token) []Token {
	result := []Token{}
	words := [][]rune{}
	tok := 0
	word := []rune{}
	tokenMap := map[string]Token{}
	for p := 0; p < len(sentence); p++ {
		if (len(toks) > tok && p < toks[tok].Pos) || len(toks) <= tok {
			word = append(word, sentence[p])
		} else if len(toks) > tok && toks[tok].Pos == p {
			if len(word) > 0 {
				words = append(words, word)
				word = []rune{}
			}
			tokenMap[string(toks[tok].Word)] = toks[tok]
			words = append(words, toks[tok].Word)
			p = toks[tok].Pos + len(toks[tok].Word)
			tok++
		}
	}
	if len(word) > 0 {
		words = append(words, word)
	}
	for i, w := range words {
		if len(w) == 0 {
			continue
		}
		tok := Token{
			Word: []rune(w),
		}
		oTok, ok := tokenMap[string(w)]
		if ok {
			tok.Nodes = oTok.Nodes
		} else {
			tok.Nodes = []Node{&UnknownNode{UID: fmt.Sprint("U", i), Word: []rune(w)}}
		}
		result = append(result, tok)
	}
	return result
}
func BuildUnknowns ( sentence [ ] rune , toks [ ] Token ) [ ] Token { result := [ ] Token { } words := [ ] [ ] rune { } tok := 0 word := [ ] rune { } tokenMap := map [ string ] Token { } for p := 0 ; p < len ( sentence ) ; p ++ { if ( len ( toks ) > tok && p < toks [ tok ] . Pos ) || len ( toks ) <= tok { word = append ( word , sentence [ p ] ) } else if len ( toks ) > tok && toks [ tok ] . Pos == p { if len ( word ) > 0 { words = append ( words , word ) word = [ ] rune { } } tokenMap [ string ( toks [ tok ] . Word ) ] = toks [ tok ] words = append ( words , toks [ tok ] . Word ) p = toks [ tok ] . Pos + len ( toks [ tok ] . Word ) tok ++ } } if len ( word ) > 0 { words = append ( words , word ) } for i , w := range words { if len ( w ) == 0 { continue } tok := Token { Word : [ ] rune ( w ) , } oTok , ok := tokenMap [ string ( w ) ] if ok { tok . Nodes = oTok . Nodes } else { tok . Nodes = [ ] Node { & UnknownNode { UID : fmt . Sprint ( " " , i ) , Word : [ ] rune ( w ) } } } result = append ( result , tok ) } return result }
BuildUnknowns build unknown nodes.
BuildUnknowns build unknown nodes .
/*
	 * We will iterate through the sentence
	 * Till the positions of each token is reached we will itearte through the sentence and build each word
	 * Upon reaching a position, it will add the word to the words list
	 * Once all the unknown words are recorded, we will split each word based on space
	 * The resultant words are consolidated as tokens
	 */ //iterating through the sentence //checking whether the rune is outside the token //need to append only if there exist a word //if any pending word, copy it to the words list //split the words with space // ws := strings.Split(string(w), " ") // for i, wr := range ws { //no need to take empty strings // }