indexes all the URIs defined by the query into Elasticsearch

def _index_group_with_subgroup(self, **kwargs):
        """ indexes all the URIs defined by the query into Elasticsearch """

        log.setLevel(self.log_level)
        # get a list of all the uri to index
        uri_list = kwargs.get('uri_list', self.get_uri_list())
        if not uri_list:
            log.info("0 items to index")
            return
        # results = results[:100]
        # Start processing through uri
        batch_file = os.path.join(CFG.dirs.logs, "batch_list.txt")
        # with open(batch_file, "w") as fo:
        #     fo.write("{")
        log.info("'%s' items to index", len(uri_list))
        self.time_start = datetime.datetime.now()
        batch_size = kwargs.get("batch_size", 12000)
        if len(uri_list) > batch_size:
            batch_end = batch_size
        else:
            batch_end = len(uri_list)
        batch_start = 0
        batch_num = 1
        self.batch_data = {}
        self.batch_data[batch_num] = {}
        self.batch_data[batch_num]['main'] = []
        self.batch_uris = {}
        self.batch_uris[batch_num] = []
        for name, indexer in self.other_indexers.items():
            self.batch_data[batch_num][name] = []
        end = False
        last = False
        final_list = []
        expand_index = kwargs.get("expand_index", True)
        while not end:
            log.debug("batch %s: %s-%s", batch_num, batch_start, batch_end)
            sub_batch = []
            j = 0
            for i in range(batch_start, batch_end):
            # for i, subj in enumerate(uri_list[batch_start:batch_end]):
                qry_size = kwargs.get("qry_size", 1000)
                if j < qry_size:
                    try:
                        sub_batch.append(uri_list.pop()) #subj)
                    except IndexError:
                        pass
                if j == qry_size -1 or i == batch_end - 1:
                    try:
                        sub_batch.append(uri_list.pop()) #subj)
                    except IndexError:
                        pass
                    # with open(batch_file, "a") as fo:
                    #     fo.write(json.dumps({str('%s-%s' % (batch_num, i+1)):
                    #                          [item[0].sparql
                    #                           for item in sub_batch]})[1:-1]+",\n")
                    if not kwargs.get("no_threading", False):
                        th = threading.Thread(name=batch_start + i + 1,
                                              target=self._index_sub,
                                              args=(sub_batch,
                                                    i+1,
                                                    batch_num,))
                        th.start()
                    else:
                        self._index_sub(sub_batch, i+1, batch_num)
                    j = 0
                    final_list += sub_batch
                    sub_batch = []
                else:
                    j += 1
            log.debug(datetime.datetime.now() - self.time_start)
            if not kwargs.get("no_threading", False):
                main_thread = threading.main_thread()
                for t in threading.enumerate():
                    if t is main_thread:
                        continue
                    t.join()
            action_list = []
            for key, items in self.batch_data[batch_num].items():
                if key == 'main':
                    es_worker = self.es_worker
                else:
                    es_worker = self.other_indexers[key]
                action_list += es_worker.make_action_list(items)
            result = self.es_worker.bulk_save(action_list)
            final_list += self.batch_uris[batch_num]
            self._update_triplestore(result, action_list)
            del action_list
            del self.batch_uris[batch_num]
            del self.batch_data[batch_num]
            try:
                del pyrdf.memorized
                pyrdf.memorized = {}
            except AttributeError:
                pass
            while gc.collect() > 0:
                pass
            # pdb.set_trace()
            batch_end += batch_size
            batch_start += batch_size
            if last:
                end = True
            if len(uri_list) <= batch_size:
                batch_end = len(uri_list)
                last = True
            batch_num += 1
            self.batch_uris[batch_num] = []
            self.batch_data[batch_num] = {}
            self.batch_data[batch_num]['main'] = []
            for name, indexer in self.other_indexers.items():
                self.batch_data[batch_num][name] = []
            log.debug(datetime.datetime.now() - self.time_start)