

def thread_worker(self):
    self.idle_event.clear()
    while True:
        try:
            (action, data) = self.input_queue.get(block=False)
        except Empty:
            self.idle_event.set()
            time.sleep(0.1)
            self.idle_event.clear()
        else:
            assert (action in ('load', 'save'))
            if (action == 'load'):
                (task, grab) = data
                result = None
                if self.is_cache_loading_allowed(task, grab):
                    result = self.load_from_cache(task, grab)
                if result:
                    self.result_queue.put(('network_result', result))
                else:
                    self.result_queue.put(('task', task))
            elif (action == 'save'):
                (task, grab) = data
                if self.is_cache_saving_allowed(task, grab):
                    with self.spider.timer.log_time('cache'):
                        with self.spider.timer.log_time('cache.write'):
                            self.cache.save_response(task.url, grab)

dataset/ETHPy150Open lorien/grab/grab/spider/cache_pipeline.py CachePipeline.thread_worker/original