<reponame>stevenling/crawler-collect<filename>src/crawling-zhihu-favorites.py
#!/usr/bin/env python
# encoding=utf-8

import codecs
import requests
import re
from bs4 import BeautifulSoup
from hyperlink._url import NoneType
import tomd

DOWNLOAD_URL = "https://www.zhihu.com/collection/25830319"

index = 1
digit = 0

ansTitle = []  # 问题
ansVote = []   # 赞数
ansAuthor = []  # 作者
ansAnswer = []  # 回答

# 伪装成浏览器


def download_page(url):
    return requests.get(url, headers={
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36'
    }).content


def main(html):
    global ansAnswer, ansAuthor, ansVote, ansTitle, index

    ansTitle.clear()
    ansVote.clear()
    ansAuthor.clear()
    ansAnswer.clear()

    url = DOWNLOAD_URL
    soup = BeautifulSoup(html)

    all_title = soup.find_all("h2", attrs={'class': 'zm-item-title'})
    all_vote = soup.find_all("div", attrs={'class': 'zm-item-vote'})
    all_author = soup.find_all("span", attrs={'class': 'author-link-line'})
    all_answer = soup.find_all("div", attrs={'class': 'zm-item-answer'})

    # 获取所有标题，放在ansTitle中
    for title_index in all_title:
        res_title = title_index.find("a", attrs={'': ''})
        if(res_title == None):
            continue
        ansTitle.append(res_title.getText())

    for vote_index in all_vote:
        res_vote = vote_index.find("a", attrs={'': ''})
        if(res_vote == None):
            continue
        ansVote.append(res_vote.getText())

    for author_index in all_author:
        res_author = author_index.find("a", attrs={'class': 'author-link'})
        if(res_author != None):
            ansAuthor.append(res_author.getText())

    for answer_index in all_answer:
        res_answer = answer_index.find(
            "div", attrs={'class': 'zm-item-rich-text expandable js-collapse-body'})
        if(res_answer == None):
            continue
        anstemp = res_answer.textarea.getText()
        print(anstemp)
        print("\n\n\n")
        ansAnswer.append(anstemp)

    next = soup.find('div', attrs={'class': 'zm-invite-pager'})
    next_page = next.find_all('a', attrs={'': ''})
    # 获取下一页的url
    for next_url in next_page:
        if(next_url.getText() == "下一页" and next_url != None and next_url.get("href") != None):
            res_url = DOWNLOAD_URL + next_url['href']
            print(res_url)
            return res_url


if __name__ == '__main__':
    url = DOWNLOAD_URL
    with codecs.open('answer.md', 'wb', encoding='utf-8') as fp:  # 打开answer.md文件
        # index回答的个数，digit是页面数
        while url and index < 100 and digit < 50:
            html = download_page(url)
            url = main(html)
            index = index + 1
            for i in range(min(len(ansAuthor), len(ansVote), len(ansAnswer))-1):
                digit = digit + 1
                fp.write("#### "+"".join(ansTitle[i])+"\n\n"+"□ "+"".join(
                    ansAuthor[i])+"\n\n"+"".join(ansAnswer[i])+"\n\n")
            print(digit)

#!/usr/bin/env python
# encoding=utf-8

import codecs
import requests
import re
from bs4 import BeautifulSoup
from hyperlink._url import NoneType
import tomd

DOWNLOAD_URL = "https://www.zhihu.com/collection/25830319"

index = 1
digit = 0

ansTitle = []  # 问题
ansVote = []   # 赞数
ansAuthor = []  # 作者
ansAnswer = []  # 回答

# 伪装成浏览器


def download_page(url):
    return requests.get(url, headers={
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36'
    }).content


def main(html):
    global ansAnswer, ansAuthor, ansVote, ansTitle, index

    ansTitle.clear()
    ansVote.clear()
    ansAuthor.clear()
    ansAnswer.clear()

    url = DOWNLOAD_URL
    soup = BeautifulSoup(html)

    all_title = soup.find_all("h2", attrs={'class': 'zm-item-title'})
    all_vote = soup.find_all("div", attrs={'class': 'zm-item-vote'})
    all_author = soup.find_all("span", attrs={'class': 'author-link-line'})
    all_answer = soup.find_all("div", attrs={'class': 'zm-item-answer'})

    # 获取所有标题，放在ansTitle中
    for title_index in all_title:
        res_title = title_index.find("a", attrs={'': ''})
        if(res_title == None):
            continue
        ansTitle.append(res_title.getText())

    for vote_index in all_vote:
        res_vote = vote_index.find("a", attrs={'': ''})
        if(res_vote == None):
            continue
        ansVote.append(res_vote.getText())

    for author_index in all_author:
        res_author = author_index.find("a", attrs={'class': 'author-link'})
        if(res_author != None):
            ansAuthor.append(res_author.getText())

    for answer_index in all_answer:
        res_answer = answer_index.find(
            "div", attrs={'class': 'zm-item-rich-text expandable js-collapse-body'})
        if(res_answer == None):
            continue
        anstemp = res_answer.textarea.getText()
        print(anstemp)
        print("\n\n\n")
        ansAnswer.append(anstemp)

    next = soup.find('div', attrs={'class': 'zm-invite-pager'})
    next_page = next.find_all('a', attrs={'': ''})
    # 获取下一页的url
    for next_url in next_page:
        if(next_url.getText() == "下一页" and next_url != None and next_url.get("href") != None):
            res_url = DOWNLOAD_URL + next_url['href']
            print(res_url)
            return res_url


if __name__ == '__main__':
    url = DOWNLOAD_URL
    with codecs.open('answer.md', 'wb', encoding='utf-8') as fp:  # 打开answer.md文件
        # index回答的个数，digit是页面数
        while url and index < 100 and digit < 50:
            html = download_page(url)
            url = main(html)
            index = index + 1
            for i in range(min(len(ansAuthor), len(ansVote), len(ansAnswer))-1):
                digit = digit + 1
                fp.write("#### "+"".join(ansTitle[i])+"\n\n"+"□ "+"".join(
                    ansAuthor[i])+"\n\n"+"".join(ansAnswer[i])+"\n\n")
            print(digit)
#!/usr/bin/env python # encoding=utf-8 # 问题 # 赞数 # 作者 # 回答 # 伪装成浏览器 # 获取所有标题，放在ansTitle中 # 获取下一页的url # 打开answer.md文件 # index回答的个数，digit是页面数 ### "+"".join(ansTitle[i])+"\n\n"+"□ "+"".join(