<gh_stars>0
from .utils import get_page
from pyquery import PyQuery as pq
import re
from bs4 import BeautifulSoup as soup

class ProxyMetaclass(type):
    """
        元类，在FreeProxyGetter类中加入
        __CrawlFunc__和__CrawlFuncCount__
        两个参数，分别表示爬虫函数，和爬虫函数的数量。
    """
    def __new__(cls, name, bases, attrs):
        count = 0
        attrs['__CrawlFunc__'] = []
        for k, v in attrs.items():
            if 'crawl_' in k:
                attrs['__CrawlFunc__'].append(k)
                count += 1
        attrs['__CrawlFuncCount__'] = count
        return type.__new__(cls, name, bases, attrs)


class FreeProxyGetter(object, metaclass=ProxyMetaclass):
    def get_raw_proxies(self, callback):
        proxies = []
        print('Callback', callback)
        for proxy in eval("self.{}()".format(callback)):
            print('Getting', proxy, 'from', callback)
            proxies.append(proxy)
        return proxies

# 高匿
    '''
        def crawl_ip181(self):
            start_url = 'http://www.ip181.com/'
            html = get_page(start_url)
            print(html)
    
            s = soup(html, 'html.parser').find_all('tr')
            re_ip_adress=[]
            for i in soup:
                tds = i.find_all('td')
                for td in tds:
                    if td.text == "高匿":
                        ip=td.parent.find_all('td')
                        re_ip_adress.append((ip[0].text,ip[1].text))
    
    
    
            for adress,port in re_ip_adress:
                result = adress + ':' + port
    
                yield result.replace(' ', '')
    '''

#非高匿但是暂时留着或许有用
    '''def crawl_xicidaili(self):
        for page in range(1, 4):
            start_url = 'http://www.xicidaili.com/wt/{}'.format(page)
            html = get_page(start_url)
            ip_adress = re.compile('<td class="country"><img src="http://fs.xicidaili.com/images/flag/cn.png" alt="Cn" /></td>\s*<td>(.*?)</td>\s*<td>(.*?)</td>')
            # \s* 匹配空格，起到换行作用
            re_ip_adress = ip_adress.findall(html)
            for adress, port in re_ip_adress:
                result = adress+':'+ port
                yield result.replace(' ', '')
'''
# 高匿ip
    def crawl_ip3366(self):
        for page in range(1, 4):
            start_url = 'http://www.ip3366.net/free/?stype=1&page={}'.format(page)
            html = get_page(start_url)
            ip_adress = re.compile('<tr>\s*<td>(.*?)</td>\s*<td>(.*?)</td>')
            # \s * 匹配空格，起到换行作用
            re_ip_adress = ip_adress.findall(html)
            for adress, port in re_ip_adress:
                result = adress+':'+ port
                yield result.replace(' ', '')

# 高匿ip
    def crawl_daili66(self, page_count=4):
        start_url = 'http://www.66ip.cn/{}.html'
        urls = [start_url.format(page) for page in range(1, page_count + 1)]
        for url in urls:
            print('Crawling', url)
            html = get_page(url)
            if html:
                doc = pq(html)
                trs = doc('.containerbox table tr:gt(0)').items()
                for tr in trs:
                    ip = tr.find('td:nth-child(1)').text()
                    port = tr.find('td:nth-child(2)').text()
                    yield ':'.join([ip, port])

# 高匿ip
    def crawl_data5u(self):
        for i in ['gngn']:
            start_url = 'http://www.data5u.com/free/{}/index.shtml'.format(i)
            html = get_page(start_url)
            ip_adress = re.compile(' <ul class="l2">\s*<span><li>(.*?)</li></span>\s*<span style="width: 100px;"><li class=".*">(.*?)</li></span>')
            # \s * 匹配空格，起到换行作用
            re_ip_adress = ip_adress.findall(html)
            for adress, port in re_ip_adress:
                result = adress+':'+port
                yield result.replace(' ','')

#高匿ip
    def crawl_kxdaili(self):
        for i in range(1, 4):
            start_url = 'http://www.kxdaili.com/dailiip/1/{}.html#ip'.format(i)
            html = get_page(start_url)
            ip_adress = re.compile('<tr.*?>\s*<td>(.*?)</td>\s*<td>(.*?)</td>')
            # \s* 匹配空格，起到换行作用
            re_ip_adress = ip_adress.findall(html)
            for adress, port in re_ip_adress:
                result = adress + ':' + port
                yield result.replace(' ', '')




from .utils import get_page
from pyquery import PyQuery as pq
import re
from bs4 import BeautifulSoup as soup

class ProxyMetaclass(type):
    """
        元类，在FreeProxyGetter类中加入
        __CrawlFunc__和__CrawlFuncCount__
        两个参数，分别表示爬虫函数，和爬虫函数的数量。
    """
    def __new__(cls, name, bases, attrs):
        count = 0
        attrs['__CrawlFunc__'] = []
        for k, v in attrs.items():
            if 'crawl_' in k:
                attrs['__CrawlFunc__'].append(k)
                count += 1
        attrs['__CrawlFuncCount__'] = count
        return type.__new__(cls, name, bases, attrs)


class FreeProxyGetter(object, metaclass=ProxyMetaclass):
    def get_raw_proxies(self, callback):
        proxies = []
        print('Callback', callback)
        for proxy in eval("self.{}()".format(callback)):
            print('Getting', proxy, 'from', callback)
            proxies.append(proxy)
        return proxies

# 高匿
    '''
        def crawl_ip181(self):
            start_url = 'http://www.ip181.com/'
            html = get_page(start_url)
            print(html)
    
            s = soup(html, 'html.parser').find_all('tr')
            re_ip_adress=[]
            for i in soup:
                tds = i.find_all('td')
                for td in tds:
                    if td.text == "高匿":
                        ip=td.parent.find_all('td')
                        re_ip_adress.append((ip[0].text,ip[1].text))
    
    
    
            for adress,port in re_ip_adress:
                result = adress + ':' + port
    
                yield result.replace(' ', '')
    '''

#非高匿但是暂时留着或许有用
    '''def crawl_xicidaili(self):
        for page in range(1, 4):
            start_url = 'http://www.xicidaili.com/wt/{}'.format(page)
            html = get_page(start_url)
            ip_adress = re.compile('<td class="country"><img src="http://fs.xicidaili.com/images/flag/cn.png" alt="Cn" /></td>\s*<td>(.*?)</td>\s*<td>(.*?)</td>')
            # \s* 匹配空格，起到换行作用
            re_ip_adress = ip_adress.findall(html)
            for adress, port in re_ip_adress:
                result = adress+':'+ port
                yield result.replace(' ', '')
'''
# 高匿ip
    def crawl_ip3366(self):
        for page in range(1, 4):
            start_url = 'http://www.ip3366.net/free/?stype=1&page={}'.format(page)
            html = get_page(start_url)
            ip_adress = re.compile('<tr>\s*<td>(.*?)</td>\s*<td>(.*?)</td>')
            # \s * 匹配空格，起到换行作用
            re_ip_adress = ip_adress.findall(html)
            for adress, port in re_ip_adress:
                result = adress+':'+ port
                yield result.replace(' ', '')

# 高匿ip
    def crawl_daili66(self, page_count=4):
        start_url = 'http://www.66ip.cn/{}.html'
        urls = [start_url.format(page) for page in range(1, page_count + 1)]
        for url in urls:
            print('Crawling', url)
            html = get_page(url)
            if html:
                doc = pq(html)
                trs = doc('.containerbox table tr:gt(0)').items()
                for tr in trs:
                    ip = tr.find('td:nth-child(1)').text()
                    port = tr.find('td:nth-child(2)').text()
                    yield ':'.join([ip, port])

# 高匿ip
    def crawl_data5u(self):
        for i in ['gngn']:
            start_url = 'http://www.data5u.com/free/{}/index.shtml'.format(i)
            html = get_page(start_url)
            ip_adress = re.compile(' <ul class="l2">\s*<span><li>(.*?)</li></span>\s*<span style="width: 100px;"><li class=".*">(.*?)</li></span>')
            # \s * 匹配空格，起到换行作用
            re_ip_adress = ip_adress.findall(html)
            for adress, port in re_ip_adress:
                result = adress+':'+port
                yield result.replace(' ','')

#高匿ip
    def crawl_kxdaili(self):
        for i in range(1, 4):
            start_url = 'http://www.kxdaili.com/dailiip/1/{}.html#ip'.format(i)
            html = get_page(start_url)
            ip_adress = re.compile('<tr.*?>\s*<td>(.*?)</td>\s*<td>(.*?)</td>')
            # \s* 匹配空格，起到换行作用
            re_ip_adress = ip_adress.findall(html)
            for adress, port in re_ip_adress:
                result = adress + ':' + port
                yield result.replace(' ', '')
元类，在FreeProxyGetter类中加入         __CrawlFunc__和__CrawlFuncCount__         两个参数，分别表示爬虫函数，和爬虫函数的数量。 # 高匿 def crawl_ip181(self):             start_url = 'http://www.ip181.com/'             html = get_page(start_url)             print(html)                  s = soup(html, 'html.parser').find_all('tr')             re_ip_adress=[]             for i in soup:                 tds = i.find_all('td')                 for td in tds:                     if td.text == "高匿":                         ip=td.parent.find_all('td')                         re_ip_adress.append((ip[0].text,ip[1].text))                            for adress,port in re_ip_adress:                 result = adress + ':' + port                      yield result.replace(' ', '') #非高匿但是暂时留着或许有用 def crawl_xicidaili(self):         for page in range(1, 4):             start_url = 'http://www.xicidaili.com/wt/{}'.format(page)             html = get_page(start_url)             ip_adress = re.compile('<td class="country"><img src="http://fs.xicidaili.com/images/flag/cn.png" alt="Cn" /></td>\s*<td>(.*?)</td>\s*<td>(.*?)</td>')             # \s* 匹配空格，起到换行作用             re_ip_adress = ip_adress.findall(html)             for adress, port in re_ip_adress:                 result = adress+':'+ port                 yield result.replace(' ', '') # 高匿ip # \s * 匹配空格，起到换行作用 # 高匿ip # 高匿ip # \s * 匹配空格，起到换行作用 #高匿ip #ip'.format(i) # \s* 匹配空格，起到换行作用