Gather spider based stats

def get_spider_stats(self):
        '''
        Gather spider based stats
        '''
        self.logger.debug("Gathering spider stats")
        the_dict = {}
        spider_set = set()
        total_spider_count = 0

        keys = self.redis_conn.keys('stats:crawler:*:*:*')
        for key in keys:
            # we only care about the spider
            elements = key.split(":")
            spider = elements[3]

            if spider not in the_dict:
                the_dict[spider] = {}
                the_dict[spider]['count'] = 0

            if len(elements) == 6:
                # got a time based stat
                response = elements[4]
                end = elements[5]

                if response not in the_dict[spider]:
                    the_dict[spider][response] = {}

                the_dict[spider][response][end] = self._get_key_value(key, end == 'lifetime')

            elif len(elements) == 5:
                # got a spider identifier
                the_dict[spider]['count'] += 1
                total_spider_count += 1
                spider_set.add(spider)

            else:
                self.logger.warn("Unknown crawler stat key", {"key":key})

        # simple counts
        the_dict['unique_spider_count'] = len(spider_set)
        the_dict['total_spider_count'] = total_spider_count

        ret_dict = {}
        ret_dict['spiders'] = the_dict

        return ret_dict