# -*- coding: utf-8 -*-
# !/usr/bin/env python
"""
-------------------------------------------------
   File Name：     utilFunction.py
   Description :  tool function
   Author :       JHao
   date：          2016/11/25
-------------------------------------------------
   Change Activity:
                   2016/11/25: 添加robustCrawl、verifyProxy、getHtmlTree
-------------------------------------------------
"""

import requests
from lxml import etree

from Util.WebRequest import WebRequest


def robustCrawl(func):
    def decorate(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except Exception as e:
            pass
            # logger.info(u"sorry, 抓取出错。错误原因:")
            # logger.info(e)

    return decorate


def verifyProxyFormat(proxy):
    """
    检查代理格式
    :param proxy:
    :return:
    """
    import re
    verify_regex = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}"
    _proxy = re.findall(verify_regex, proxy)
    return True if len(_proxy) == 1 and _proxy[0] == proxy else False


def getHtmlTree(url, **kwargs):
    """
    获取html树
    :param url:
    :param kwargs:
    :return:
    """

    header = {'Connection': 'keep-alive',
              'Cache-Control': 'max-age=0',
              'Upgrade-Insecure-Requests': '1',
              'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko)',
              'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
              'Accept-Encoding': 'gzip, deflate, sdch',
              'Accept-Language': 'zh-CN,zh;q=0.8',
              }
    # TODO 取代理服务器用代理服务器访问
    wr = WebRequest()
    html = wr.get(url=url, header=header).content
    return etree.HTML(html)


def tcpConnect(proxy):
    """
    TCP 三次握手
    :param proxy:
    :return:
    """
    from socket import socket, AF_INET, SOCK_STREAM
    s = socket(AF_INET, SOCK_STREAM)
    ip, port = proxy.split(':')
    result = s.connect_ex((ip, int(port)))
    return True if result == 0 else False


def validUsefulProxy(proxy):
    """
    检验代理是否可用
    :param proxy:
    :return:
    """
    if isinstance(proxy, bytes):
        proxy = proxy.decode("utf8")
    proxies = {"http": "http://{proxy}".format(proxy=proxy)}
    try:
        r = requests.get('http://www.baidu.com', proxies=proxies, timeout=10, verify=False)
        if r.status_code == 200:
            return True
    except Exception as e:
        pass
    return False


def validUsefulProxyUnicom(proxy, origin_ips):
    """
    检验代理是否可用
    :param proxy:
    :return:
    """
    if isinstance(proxy, bytes):
        proxy = proxy.decode("utf8")
    proxies = {"http": "http://{proxy}".format(proxy=proxy)}
    is_ok = get_anonymity(proxies, origin_ips)
    if not is_ok:
        return False
    try:
        r = requests.get('https://vip.17wo.cn', headers={
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
            'Accept-Encoding': 'gzip, deflate',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36',
        }, proxies=proxies, timeout=10, verify=False)
        if r.status_code == 200:
            return True
    except Exception as e:
        pass
    return False


def get_origin_ips():
    resp = requests.get('http://httpbin.org/get?show_env=1', timeout=10, verify=False)
    if resp.status_code == 200:
        data = resp.json()
        origin_ip_str = data.get('origin')
        if origin_ip_str:
            return [ip.strip() for ip in origin_ip_str.split(',') if ip]
    raise RuntimeError


def get_anonymity(proxies, origin_ips):
    try:
        r = requests.get('http://httpbin.org/get?show_env=1', proxies=proxies, timeout=10, verify=False)
        if r.status_code == 200:
            data = r.json()

            origin = data.get('origin')
            if origin:
                origin = {ip.strip() for ip in origin.split(',') if ip}
            headers = data.get('headers')
            x_forwarded_for = headers.get('X-Forwarded-For', None)
            x_real_ip = headers.get('X-Real-Ip', None)
            via = headers.get('Via', None)

            if (origin - origin_ips) != origin:
                anonymity = 3
            elif via is not None:
                anonymity = 2
            elif x_forwarded_for is not None and x_real_ip is not None:
                anonymity = 1
                return True
            else:
                anonymity = 0
    except Exception as e:
        pass
    return False

# -*- coding: utf-8 -*-
# !/usr/bin/env python
"""
-------------------------------------------------
   File Name：     utilFunction.py
   Description :  tool function
   Author :       JHao
   date：          2016/11/25
-------------------------------------------------
   Change Activity:
                   2016/11/25: 添加robustCrawl、verifyProxy、getHtmlTree
-------------------------------------------------
"""

import requests
from lxml import etree

from Util.WebRequest import WebRequest


def robustCrawl(func):
    def decorate(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except Exception as e:
            pass
            # logger.info(u"sorry, 抓取出错。错误原因:")
            # logger.info(e)

    return decorate


def verifyProxyFormat(proxy):
    """
    检查代理格式
    :param proxy:
    :return:
    """
    import re
    verify_regex = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}"
    _proxy = re.findall(verify_regex, proxy)
    return True if len(_proxy) == 1 and _proxy[0] == proxy else False


def getHtmlTree(url, **kwargs):
    """
    获取html树
    :param url:
    :param kwargs:
    :return:
    """

    header = {'Connection': 'keep-alive',
              'Cache-Control': 'max-age=0',
              'Upgrade-Insecure-Requests': '1',
              'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko)',
              'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
              'Accept-Encoding': 'gzip, deflate, sdch',
              'Accept-Language': 'zh-CN,zh;q=0.8',
              }
    # TODO 取代理服务器用代理服务器访问
    wr = WebRequest()
    html = wr.get(url=url, header=header).content
    return etree.HTML(html)


def tcpConnect(proxy):
    """
    TCP 三次握手
    :param proxy:
    :return:
    """
    from socket import socket, AF_INET, SOCK_STREAM
    s = socket(AF_INET, SOCK_STREAM)
    ip, port = proxy.split(':')
    result = s.connect_ex((ip, int(port)))
    return True if result == 0 else False


def validUsefulProxy(proxy):
    """
    检验代理是否可用
    :param proxy:
    :return:
    """
    if isinstance(proxy, bytes):
        proxy = proxy.decode("utf8")
    proxies = {"http": "http://{proxy}".format(proxy=proxy)}
    try:
        r = requests.get('http://www.baidu.com', proxies=proxies, timeout=10, verify=False)
        if r.status_code == 200:
            return True
    except Exception as e:
        pass
    return False


def validUsefulProxyUnicom(proxy, origin_ips):
    """
    检验代理是否可用
    :param proxy:
    :return:
    """
    if isinstance(proxy, bytes):
        proxy = proxy.decode("utf8")
    proxies = {"http": "http://{proxy}".format(proxy=proxy)}
    is_ok = get_anonymity(proxies, origin_ips)
    if not is_ok:
        return False
    try:
        r = requests.get('https://vip.17wo.cn', headers={
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
            'Accept-Encoding': 'gzip, deflate',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36',
        }, proxies=proxies, timeout=10, verify=False)
        if r.status_code == 200:
            return True
    except Exception as e:
        pass
    return False


def get_origin_ips():
    resp = requests.get('http://httpbin.org/get?show_env=1', timeout=10, verify=False)
    if resp.status_code == 200:
        data = resp.json()
        origin_ip_str = data.get('origin')
        if origin_ip_str:
            return [ip.strip() for ip in origin_ip_str.split(',') if ip]
    raise RuntimeError


def get_anonymity(proxies, origin_ips):
    try:
        r = requests.get('http://httpbin.org/get?show_env=1', proxies=proxies, timeout=10, verify=False)
        if r.status_code == 200:
            data = r.json()

            origin = data.get('origin')
            if origin:
                origin = {ip.strip() for ip in origin.split(',') if ip}
            headers = data.get('headers')
            x_forwarded_for = headers.get('X-Forwarded-For', None)
            x_real_ip = headers.get('X-Real-Ip', None)
            via = headers.get('Via', None)

            if (origin - origin_ips) != origin:
                anonymity = 3
            elif via is not None:
                anonymity = 2
            elif x_forwarded_for is not None and x_real_ip is not None:
                anonymity = 1
                return True
            else:
                anonymity = 0
    except Exception as e:
        pass
    return False

# -*- coding: utf-8 -*- # !/usr/bin/env python -------------------------------------------------    File Name：     utilFunction.py    Description :  tool function    Author :       JHao    date：          2016/11/25 -------------------------------------------------    Change Activity:                    2016/11/25: 添加robustCrawl、verifyProxy、getHtmlTree ------------------------------------------------- # logger.info(u"sorry, 抓取出错。错误原因:") # logger.info(e) 检查代理格式     :param proxy:     :return: 获取html树     :param url:     :param kwargs:     :return: # TODO 取代理服务器用代理服务器访问 TCP 三次握手     :param proxy:     :return: 检验代理是否可用     :param proxy:     :return: 检验代理是否可用     :param proxy:     :return: