//Cache is the cache for providing the tokenizer to the platform on demand
Cache is the cache for providing the tokenizer to the platform on demand
// Cache is the cache for providing the tokenizer to the platform on demand
func Cache(in chan Request) {
	dict := make(map[string]Tokenizer)
	for {
		req := <-in
		switch req.Type {
		case TokenizerAdd:
			m := new(goahocorasick.Machine)
			d := [][]rune{}
			for word := range req.Tokenizer.Map {
				if len(word) == 0 {
					continue
				}
				d = append(d, []rune(strings.ToLower(word)))
			}
			if err := m.Build(d); err != nil {
				fmt.Println(err)
				break
			}
			dict[req.ID] = Tokenizer{Map: req.Tokenizer.Map, Machine: m}
			break
		case TokenizerGet:
			t, mOk := dict[req.ID]
			if !mOk {
				req.Valid = false
				go SendTokenizerToChannel(req.Out, req)
				break
			}
			terms := t.Machine.MultiPatternSearch([]rune(strings.ToLower(string(req.Sentence))), false)
			result := []Token{}
			for _, term := range terms {
				tok, ok := t.Map[string(term.Word)]
				if ok {
					result = append(result, Token{Pos: term.Pos, Word: term.Word, Nodes: tok.Nodes})
				}
			}
			req.Matches = result
			req.Valid = true
			go SendTokenizerToChannel(req.Out, req)
			break
		case TokenizerRemove:
			delete(dict, req.ID)
		}
	}
}
func Cache ( in chan Request ) { dict := make ( map [ string ] Tokenizer ) for { req := <- in switch req . Type { case TokenizerAdd : m := new ( goahocorasick . Machine ) d := [ ] [ ] rune { } for word := range req . Tokenizer . Map { if len ( word ) == 0 { continue } d = append ( d , [ ] rune ( strings . ToLower ( word ) ) ) } if err := m . Build ( d ) ; err != nil { fmt . Println ( err ) break } dict [ req . ID ] = Tokenizer { Map : req . Tokenizer . Map , Machine : m } break case TokenizerGet : t , mOk := dict [ req . ID ] if ! mOk { req . Valid = false go SendTokenizerToChannel ( req . Out , req ) break } terms := t . Machine . MultiPatternSearch ( [ ] rune ( strings . ToLower ( string ( req . Sentence ) ) ) , false ) result := [ ] Token { } for _ , term := range terms { tok , ok := t . Map [ string ( term . Word ) ] if ok { result = append ( result , Token { Pos : term . Pos , Word : term . Word , Nodes : tok . Nodes } ) } } req . Matches = result req . Valid = true go SendTokenizerToChannel ( req . Out , req ) break case TokenizerRemove : delete ( dict , req . ID ) } } }
Cache is the cache for providing the tokenizer to the platform on demand
Cache is the cache for providing the tokenizer to the platform on demand
/*
	 * We will go into an infinte loop
	 * Will wait for the requests to come through the channel
	 * Based on the type of the request we will remove them from memory
	 */