// IndexDocs - index some Docs
// nextDoc allow you to parse your file progressivly
index some Docs
nextDoc allow you to parse your file progressivly
// IndexDocs - index some Docs // nextDoc allow you to parse your file progressivly
func IndexDocs(index bleve.Index, nextDoc func() (Indexable, error)) error {
	batch := index.NewBatch()
	batchCount := 100
	nbDone := 0
	for {
		doc, err := nextDoc()
		if err != nil {
			if err == io.EOF {
				break
			}
			return fmt.Errorf("Error while parsing file\n	==> %v\n	==> %v", err, doc)
		}
		err = batch.Index(doc.GetID(), doc)
		if err != nil {
			return fmt.Errorf("Error while adding doc to the batch\n	==> %v\nDoc:	==> %v\nBatch	==>%v", err, doc, batch)
		}
		batchCount--
		if batchCount == 0 {
			err = index.Batch(batch)
			if err != nil {
				return fmt.Errorf("Error while indexing batch:\n	Error ==> %v", err)
			}
			batch = index.NewBatch()
			batchCount = 100
		}
		nbDone++
		if nbDone%100 == 0 {
			fmt.Println("Done :", nbDone)
		}
	}
	fmt.Println("Over")
	if batchCount > 0 {
		err := index.Batch(batch)
		if err != nil {
			return fmt.Errorf("Error while indexing batch:\n	Error ==> %v", err)
		}
	}
	return nil
}
func IndexDocs ( index bleve . Index , nextDoc func ( ) ( Indexable , error ) ) error { batch := index . NewBatch ( ) batchCount := 100 nbDone := 0 for { doc , err := nextDoc ( ) if err != nil { if err == io . EOF { break } return fmt . Errorf ( " \n \n " , err , doc ) } err = batch . Index ( doc . GetID ( ) , doc ) if err != nil { return fmt . Errorf ( " \n \n \n " , err , doc , batch ) } batchCount -- if batchCount == 0 { err = index . Batch ( batch ) if err != nil { return fmt . Errorf ( " \n " , err ) } batch = index . NewBatch ( ) batchCount = 100 } nbDone ++ if nbDone % 100 == 0 { fmt . Println ( " " , nbDone ) } } fmt . Println ( " " ) if batchCount > 0 { err := index . Batch ( batch ) if err != nil { return fmt . Errorf ( " \n " , err ) } } return nil }
index some Docs
nextDoc allow you to parse your file progressivly
index some Docs nextDoc allow you to parse your file progressivly
// Loop through the Docs with a custom passed function (nextDoc) // End of the file // Other errors // Add doc to the current batch // Decrement batchCount // If batch is full, write it to the index // batch.Reset() // REMOVE - This stops the indexing at 100 Docs !! //fmt.Println("REMOVE ME") //return nil // flush the last batch