<reponame>Anthlay/FUN_TOOLS
import json
import requests
import time
from lxml import etree
import os
from urllib.parse import parse_qs

"""
爬取微博评论
"""

proxies = ["172.16.58.3:52590","192.168.3.11:80","192.168.127.12:8080","192.168.3.11:1080","172.16.17.32:3128","172.16.17.32:9999","172.16.17.32:9797","172.16.31.10:80","192.168.3.11:9999","172.16.58.3:9999","192.168.3.11:80","192.168.127.12:9999","172.16.58.3:80","192.168.3.11:9999","172.16.58.3:8888","172.16.17.32:8080","172.16.17.32:9797","172.16.31.10:9999","192.168.127.12:9999","172.16.17.32:9999","172.16.17.32:53281","192.168.127.12:8888","172.16.58.3:9999","192.168.127.12:9999","172.16.58.3:3128","172.16.31.10:80","192.168.3.11:9999","172.16.58.3:80","172.16.17.32:9999","172.16.58.3:80"]

headers = {

    "cookie":
              "_T_WM=55585044428; ALF=1583456656; SCF=AqIHTcJA3mbhrJOGdDGoFVZMfyrqTt8eAOTfwdXdssuBsNl9OuXirJNvxj5LIj-mw5DUvrQ4_zHWvu8G33ueqL0.; SUB=_2A25zPlPxDeRhGeFO7VsR-S7MwzqIHXVQwX25rDV6PUJbktAKLUPVkW1NQULhppLQkWODC7P9YtLH-k5048ZyPmMy; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WhQ-ZjgKjOJccAE6BOqDrUw5JpX5K-hUgL.FoM7So.71K571hq2dJLoI0YLxKqL1hnL1K2LxKBLB.zLB-eLxKnLB.qL1K.LxKqL1-qLBK-LxKML1-2L1hBLxK-LB.-L1hnLxKMLBoBLBKMt; SUHB=0FQbwPXwVikYFd; SSOLoginState=1580868514",

}


# 当前路径+pic
pic_file_path = os.path.join(os.path.abspath(''), 'pic')

# 下载图片
def download_pic(url, nick_name):
    if not url:
        return
    if not os.path.exists(pic_file_path):
        os.mkdir(pic_file_path)
    resp = requests.get(url)
    if resp.status_code == 200:
        with open(pic_file_path + f'/{nick_name}.jpg', 'wb') as f:
            f.write(resp.content)

# 写入留言内容
def write_comment(comment):
    comment += '\n'
    with open('comment.txt', 'a', encoding='utf-8') as f:
        f.write(comment.replace('回复', '').replace('等人', '').replace('图片评论', ''))

# 获取子评论所需参数
comment_params = {
    'ajwvr': 6,
    'more_comment': 'big',
    'is_child_comment': 'true',
    'id': '4460033192179568',
    'from': 'singleWeiBo',
    'last_child_comment_id': '',
    'root_comment_id': '',
    'root_comment_max_id': ''
}

# 获取子评论，这里只是获取了第一页的子评论信息
def get_child_comment(root_comment_id):
    comment_params['root_comment_id'] = root_comment_id
    resp = requests.get(URL, params=comment_params, headers=headers)
    resp = json.loads(resp.text)
    if resp['code'] == '100000':
        html = resp['data']['html']
        from lxml import etree
        html = etree.HTML(html)
        # 每个子评论的节点
        data = html.xpath('//div[@class="WB_text"]')
        for i in data:
            num = 1
            nick_name = ''.join(i.xpath('./a/text()')).strip().replace('\n', '')
            comment = ''.join(i.xpath('./text()')).strip().replace('\n', '')
            write_comment(comment)
            # 获取图片对应的html节点
            pic = i.xpath('.//a[@action-type="widget_photoview"]/@action-data')
            pic = pic[0] if pic else ''
            if pic:
                # 拼接另外两个必要参数
                pic = pic + 'ajwvr=6&uid=5648894345'
                # 构造出一个完整的图片url
                url = 'https://weibo.com/aj/photo/popview?' + pic
                resp = requests.get(url, headers=headers)
                resp = resp.json()
                if resp.get('code') == '100000':
                    # 从突然url中，第一个就是评论中的图
                    url = resp['data']['pic_list'][0]['clear_picSrc']
                    # 下载图片
                    download_pic(pic_url, nick_name)
        print(" 子评论抓取完毕...")




if __name__ == '__main__':
    params = {
        'ajwvr': 6,
        'id': '4456015556800686',
        'from': 'singleWeiBo',
        'root_comment_max_id': ''
    }
    URL = 'https://weibo.com/aj/v6/comment/big'
    # 爬去100页，需要代理，或者进行sleep 不然会超时。
    for num in range(101):
        print(f'=========   正在读取第 {num} 页 ====================')
        # resp = requests.get(URL, params=params, headers=headers, proxies={"http": random.choices(proxies)[0]})
        resp = requests.get(URL, params=params, headers=headers)
        resp = json.loads(resp.text)
        if resp['code'] == '100000':
            html = resp['data']['html']

            html = etree.HTML(html)
            max_id_json = html.xpath('//div[@node-type="comment_loading"]/@action-data')[0]


            node_params = parse_qs(max_id_json)
            # max_id
            max_id = node_params['root_comment_max_id'][0]
            params['root_comment_max_id'] = max_id
            # data = html.xpath('//div[@class="list_ul"]/div[@node-type="root_comment"]/div[@class="list_con"]')
            data = html.xpath('//div[@node-type="root_comment"]')
            for i in data:
                # 评论人昵称
                nick_name = i.xpath('.//div[@class="WB_text"]/a/text()')[0]
                # 评论内容。
                # test = i.xpath('.//div[@class="WB_text"]/text()')
                wb_text = i.xpath('.//div[@class="WB_text"][1]/text()')
                string = ''.join(wb_text).strip().replace('\n', '')
                write_comment(string)
                # 评论id , 用于获取评论内容
                comment_id = i.xpath('./@comment_id')[0]
                # 评论的图片地址
                pic_url = i.xpath('.//li[@class="WB_pic S_bg2 bigcursor"]/img/@src')
                pic_url = 'https:' + pic_url[0] if pic_url else ''
                download_pic(pic_url, nick_name)
                # 查看评论
                get_child_comment(root_comment_id=comment_id)
                time.sleep(8)



import json
import requests
import time
from lxml import etree
import os
from urllib.parse import parse_qs

"""
爬取微博评论
"""

proxies = ["172.16.58.3:52590","192.168.3.11:80","192.168.127.12:8080","192.168.3.11:1080","172.16.17.32:3128","172.16.17.32:9999","172.16.17.32:9797","172.16.31.10:80","192.168.3.11:9999","172.16.58.3:9999","192.168.3.11:80","192.168.127.12:9999","172.16.58.3:80","192.168.3.11:9999","172.16.58.3:8888","172.16.17.32:8080","172.16.17.32:9797","172.16.31.10:9999","192.168.127.12:9999","172.16.17.32:9999","172.16.17.32:53281","192.168.127.12:8888","172.16.58.3:9999","192.168.127.12:9999","172.16.58.3:3128","172.16.31.10:80","192.168.3.11:9999","172.16.58.3:80","172.16.17.32:9999","172.16.58.3:80"]

headers = {

    "cookie":
              "_T_WM=55585044428; ALF=1583456656; SCF=AqIHTcJA3mbhrJOGdDGoFVZMfyrqTt8eAOTfwdXdssuBsNl9OuXirJNvxj5LIj-mw5DUvrQ4_zHWvu8G33ueqL0.; SUB=_2A25zPlPxDeRhGeFO7VsR-S7MwzqIHXVQwX25rDV6PUJbktAKLUPVkW1NQULhppLQkWODC7P9YtLH-k5048ZyPmMy; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WhQ-ZjgKjOJccAE6BOqDrUw5JpX5K-hUgL.FoM7So.71K571hq2dJLoI0YLxKqL1hnL1K2LxKBLB.zLB-eLxKnLB.qL1K.LxKqL1-qLBK-LxKML1-2L1hBLxK-LB.-L1hnLxKMLBoBLBKMt; SUHB=0FQbwPXwVikYFd; SSOLoginState=1580868514",

}


# 当前路径+pic
pic_file_path = os.path.join(os.path.abspath(''), 'pic')

# 下载图片
def download_pic(url, nick_name):
    if not url:
        return
    if not os.path.exists(pic_file_path):
        os.mkdir(pic_file_path)
    resp = requests.get(url)
    if resp.status_code == 200:
        with open(pic_file_path + f'/{nick_name}.jpg', 'wb') as f:
            f.write(resp.content)

# 写入留言内容
def write_comment(comment):
    comment += '\n'
    with open('comment.txt', 'a', encoding='utf-8') as f:
        f.write(comment.replace('回复', '').replace('等人', '').replace('图片评论', ''))

# 获取子评论所需参数
comment_params = {
    'ajwvr': 6,
    'more_comment': 'big',
    'is_child_comment': 'true',
    'id': '4460033192179568',
    'from': 'singleWeiBo',
    'last_child_comment_id': '',
    'root_comment_id': '',
    'root_comment_max_id': ''
}

# 获取子评论，这里只是获取了第一页的子评论信息
def get_child_comment(root_comment_id):
    comment_params['root_comment_id'] = root_comment_id
    resp = requests.get(URL, params=comment_params, headers=headers)
    resp = json.loads(resp.text)
    if resp['code'] == '100000':
        html = resp['data']['html']
        from lxml import etree
        html = etree.HTML(html)
        # 每个子评论的节点
        data = html.xpath('//div[@class="WB_text"]')
        for i in data:
            num = 1
            nick_name = ''.join(i.xpath('./a/text()')).strip().replace('\n', '')
            comment = ''.join(i.xpath('./text()')).strip().replace('\n', '')
            write_comment(comment)
            # 获取图片对应的html节点
            pic = i.xpath('.//a[@action-type="widget_photoview"]/@action-data')
            pic = pic[0] if pic else ''
            if pic:
                # 拼接另外两个必要参数
                pic = pic + 'ajwvr=6&uid=5648894345'
                # 构造出一个完整的图片url
                url = 'https://weibo.com/aj/photo/popview?' + pic
                resp = requests.get(url, headers=headers)
                resp = resp.json()
                if resp.get('code') == '100000':
                    # 从突然url中，第一个就是评论中的图
                    url = resp['data']['pic_list'][0]['clear_picSrc']
                    # 下载图片
                    download_pic(pic_url, nick_name)
        print(" 子评论抓取完毕...")




if __name__ == '__main__':
    params = {
        'ajwvr': 6,
        'id': '4456015556800686',
        'from': 'singleWeiBo',
        'root_comment_max_id': ''
    }
    URL = 'https://weibo.com/aj/v6/comment/big'
    # 爬去100页，需要代理，或者进行sleep 不然会超时。
    for num in range(101):
        print(f'=========   正在读取第 {num} 页 ====================')
        # resp = requests.get(URL, params=params, headers=headers, proxies={"http": random.choices(proxies)[0]})
        resp = requests.get(URL, params=params, headers=headers)
        resp = json.loads(resp.text)
        if resp['code'] == '100000':
            html = resp['data']['html']

            html = etree.HTML(html)
            max_id_json = html.xpath('//div[@node-type="comment_loading"]/@action-data')[0]


            node_params = parse_qs(max_id_json)
            # max_id
            max_id = node_params['root_comment_max_id'][0]
            params['root_comment_max_id'] = max_id
            # data = html.xpath('//div[@class="list_ul"]/div[@node-type="root_comment"]/div[@class="list_con"]')
            data = html.xpath('//div[@node-type="root_comment"]')
            for i in data:
                # 评论人昵称
                nick_name = i.xpath('.//div[@class="WB_text"]/a/text()')[0]
                # 评论内容。
                # test = i.xpath('.//div[@class="WB_text"]/text()')
                wb_text = i.xpath('.//div[@class="WB_text"][1]/text()')
                string = ''.join(wb_text).strip().replace('\n', '')
                write_comment(string)
                # 评论id , 用于获取评论内容
                comment_id = i.xpath('./@comment_id')[0]
                # 评论的图片地址
                pic_url = i.xpath('.//li[@class="WB_pic S_bg2 bigcursor"]/img/@src')
                pic_url = 'https:' + pic_url[0] if pic_url else ''
                download_pic(pic_url, nick_name)
                # 查看评论
                get_child_comment(root_comment_id=comment_id)
                time.sleep(8)
爬取微博评论 # 当前路径+pic # 下载图片 # 写入留言内容 # 获取子评论所需参数 # 获取子评论，这里只是获取了第一页的子评论信息 # 每个子评论的节点 # 获取图片对应的html节点 # 拼接另外两个必要参数 # 构造出一个完整的图片url # 从突然url中，第一个就是评论中的图 # 下载图片 # 爬去100页，需要代理，或者进行sleep 不然会超时。 # resp = requests.get(URL, params=params, headers=headers, proxies={"http": random.choices(proxies)[0]}) # max_id # data = html.xpath('//div[@class="list_ul"]/div[@node-type="root_comment"]/div[@class="list_con"]') # 评论人昵称 # 评论内容。 # test = i.xpath('.//div[@class="WB_text"]/text()') # 评论id , 用于获取评论内容 # 评论的图片地址 # 查看评论