获取某个菜谱分类url下的所有菜谱url

def get_class_recipes(class_url, max_page=20, sleep=0.1):
    """获取某个菜谱分类url下的所有菜谱url"""
    class_url = class_url + "?page={page}"
    recipes = dict()

    # 暴力爬取方案，每个菜谱分类请求100页
    for page in range(1, max_page):
        time.sleep(sleep)
        url = class_url.format(page=page)
        print("current url: ", url)
        response = requests.get(url, headers=get_header())
        html = BeautifulSoup(response.text, "lxml")

        # 获取本页菜谱
        menus = html.find("div", {'class': 'new-menu-list search-menu-list clearfix mt10'})
        if menus:
            print("get recipes fail: ", url)
            menus = menus.find_all('a')
            for m in menus:
                name = re.sub("\n| ", "", m.text)
                recipe_url = urljoin(HOME_URL, m['href'])
                recipes[name] = recipe_url

        # 判断是否是最后一页
        # next_page = html.find('div', {'class': 'paging mt20'}).text
        # if "下一页" in next_page:
        #     page += 1
        # else:
        #     break
    return recipes