KevinHuSh commited on
Commit
5d16bca
·
1 Parent(s): ba9251b

let index be batchly. (#733)

Browse files

### What problem does this PR solve?

let index be batchly.

### Type of change


- [x] Refactoring

Files changed (1) hide show
  1. rag/svr/task_executor.py +6 -1
rag/svr/task_executor.py CHANGED
@@ -285,7 +285,12 @@ def main():
285
  init_kb(r)
286
  chunk_count = len(set([c["_id"] for c in cks]))
287
  st = timer()
288
- es_r = ELASTICSEARCH.bulk(cks, search.index_name(r["tenant_id"]))
 
 
 
 
 
289
  cron_logger.info("Indexing elapsed({}): {}".format(r["name"], timer()-st))
290
  if es_r:
291
  callback(-1, "Index failure!")
 
285
  init_kb(r)
286
  chunk_count = len(set([c["_id"] for c in cks]))
287
  st = timer()
288
+ es_r = ""
289
+ for b in range(0, len(cks), 32):
290
+ es_r = ELASTICSEARCH.bulk(cks[b:b+32], search.index_name(r["tenant_id"]))
291
+ if b % 128 == 0:
292
+ callback(prog=0.8 + 0.1 * (b + 1) / len(cks), msg="")
293
+
294
  cron_logger.info("Indexing elapsed({}): {}".format(r["name"], timer()-st))
295
  if es_r:
296
  callback(-1, "Index failure!")