def get_cats(self):
        '''Get top keywords categories'''
        start_url = 'http://top.taobao.com/index.php?from=tbsy'
        rs = self.fetch(start_url)
        if not rs: return None
        soup = BeautifulSoup(rs.content, convertEntities=BeautifulSoup.HTML_ENTITIES, markupMassage=hexentityMassage)
        cats = [{'id':'TR_%s'%li['id'].encode('utf-8').upper(), 'title':li.a.text.encode('utf-8').strip()} for li in soup.find('div', id='nav').findAll('li') if li['id']!='index']
        threadPool = ThreadPool(len(cats) if len(cats)<=5 else 5)
        for cat in cats:
            threadPool.run(self.get_cats_thread, callback=None, cat=cat)
        cats = threadPool.killAllWorkers(None)
        return cats
def get_cats ( self ) : start_url = 'http://top.taobao.com/index.php?from=tbsy' rs = self . fetch ( start_url ) if not rs : return None soup = BeautifulSoup ( rs . content , convertEntities = BeautifulSoup . HTML_ENTITIES , markupMassage = hexentityMassage ) cats = [ { 'id' : 'TR_%s' % li [ 'id' ] . encode ( 'utf-8' ) . upper ( ) , 'title' : li . a . text . encode ( 'utf-8' ) . strip ( ) } for li in soup . find ( 'div' , id = 'nav' ) . findAll ( 'li' ) if li [ 'id' ] != 'index' ] threadPool = ThreadPool ( len ( cats ) if len ( cats ) <= 5 else 5 ) for cat in cats : threadPool . run ( self . get_cats_thread , callback = None , cat = cat ) cats = threadPool . killAllWorkers ( None ) return cats
Get top keywords categories