<s> """<STR_LIT>""" <EOL> import os <EOL> import time <EOL> import random <EOL> import logging <EOL> import requests <EOL> import yaml <EOL> import redis <EOL> import gevent <EOL> from gevent . pool import Pool , Group <EOL> from gevent . timeout import Timeout <EOL> from gevent import monkey <EOL> from lxml import etree <EOL> monkey . patch_all ( thread = False , select = False ) <EOL> logging . basicConfig ( level = logging . INFO , <EOL> format = '<STR_LIT>' , <EOL> datefmt = '<STR_LIT>' ) <EOL> class ProxyPool ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , configfile = '<STR_LIT>' ) : <EOL> self . configs = self . _get_configs ( configfile ) <EOL> self . rdb = redis . StrictRedis ( db = self . configs [ '<STR_LIT>' ] ) <EOL> self . db_zproxy = self . configs [ '<STR_LIT>' ] <EOL> self . pool = Pool ( self . configs [ '<STR_LIT>' ] ) <EOL> self . group = Group ( ) <EOL> self . wrong_value = self . configs [ '<STR_LIT>' ] <EOL> self . init_value = self . configs [ '<STR_LIT>' ] <EOL> def _get_configs ( self , configfile ) : <EOL> """<STR_LIT>""" <EOL> configpath = os . path . join ( '<STR_LIT:.>' , configfile ) <EOL> with open ( configpath , '<STR_LIT:rb>' ) as fp : <EOL> configs = yaml . load ( fp ) <EOL> return configs <EOL> def get_many ( self , num = <NUM_LIT:3> , minscore = <NUM_LIT:0> , maxscore = None ) : <EOL> """<STR_LIT>""" <EOL> minscore = minscore <EOL> maxscore = maxscore or self . init_value <EOL> res = self . rdb . zrange ( self . db_zproxy , minscore , maxscore ) <EOL> if res : <EOL> random . shuffle ( res ) <EOL> if len ( res ) < num : <EOL> logging . warning ( "<STR_LIT>" <EOL> % ( num , ) ) <EOL> return [ proxy for proxy in res [ : num ] ] <EOL> else : <EOL> logging . warning ( "<STR_LIT>" <EOL> % ( minscore , maxscore ) ) <EOL> return [ ] <EOL> def get_one ( self , minscore = <NUM_LIT:0> , maxscore = None ) : <EOL> """<STR_LIT>""" <EOL> minscore = minscore <EOL> maxscore = maxscore or self . init_value <EOL> res = self . get_many ( num = <NUM_LIT:1> , minscore = minscore , maxscore = maxscore ) <EOL> if res : <EOL> return res [ <NUM_LIT:0> ] <EOL> else : <EOL> return '<STR_LIT>' <EOL> def crawl_proxies ( self ) : <EOL> """<STR_LIT>""" <EOL> statics = [ ] <EOL> self . _crawl_proxies_sites ( statics = statics ) <EOL> logging . info ( '<STR_LIT>' % ( len ( statics ) , ) ) <EOL> def _crawl_proxies_sites ( self , statics = [ ] ) : <EOL> """<STR_LIT>""" <EOL> args = ( ( url , val [ '<STR_LIT>' ] , val [ '<STR_LIT>' ] , statics ) <EOL> for url , val in self . configs [ '<STR_LIT>' ] . iteritems ( ) ) <EOL> self . pool . map ( self . _crawl_proxies_one_site , args ) <EOL> def _crawl_proxies_one_site ( self , args ) : <EOL> """<STR_LIT>""" <EOL> url = args [ <NUM_LIT:0> ] <EOL> rules = args [ <NUM_LIT:1> ] <EOL> proxies = args [ <NUM_LIT:2> ] <EOL> headers = self . configs [ '<STR_LIT>' ] <EOL> headers [ '<STR_LIT>' ] = url . split ( '<STR_LIT:/>' , <NUM_LIT:3> ) [ <NUM_LIT:2> ] <EOL> logging . info ( '<STR_LIT>' % ( url , ) ) <EOL> res = requests . get ( url , headers = headers , proxies = proxies ) <EOL> encoding = res . encoding <EOL> html = etree . HTML ( res . content ) <EOL> proxies = [ ] <EOL> len_rules = len ( rules ) <EOL> nodes = html . xpath ( rules [ <NUM_LIT:0> ] ) <EOL> if nodes : <EOL> if len_rules == <NUM_LIT:1> : <EOL> for node in nodes : <EOL> text = str ( node . text ) . encode ( encoding ) . strip ( ) <EOL> if text : <EOL> proxies . append ( '<STR_LIT>' % ( text , ) ) <EOL> elif len_rules == <NUM_LIT:2> : <EOL> rule_1 = rules [ <NUM_LIT:1> ] . split ( '<STR_LIT:U+002C>' ) <EOL> for node in nodes : <EOL> node = node . xpath ( rule_1 [ <NUM_LIT:0> ] ) <EOL> ip = str ( node [ <NUM_LIT:1> ] . text ) . encode ( encoding ) . strip ( ) <EOL> port = str ( node [ <NUM_LIT:2> ] . text ) . encode ( encoding ) . strip ( ) or '<STR_LIT>' <EOL> if ip : <EOL> proxies . append ( '<STR_LIT>' % ( ip , port ) ) <EOL> for proxy in proxies : <EOL> logging . info ( '<STR_LIT>' % ( proxy , url ) ) <EOL> args [ <NUM_LIT:3> ] . append ( proxy ) <EOL> self . rdb . zadd ( self . db_zproxy , self . init_value , proxy ) <EOL> def validate_proxies ( self ) : <EOL> """<STR_LIT>""" <EOL> maxscore = self . init_value + self . wrong_value <EOL> proxies = self . rdb . zrange ( self . db_zproxy , <NUM_LIT:0> , maxscore ) <EOL> statics_errors = [ ] <EOL> args = [ ( proxy , statics_errors ) for proxy in proxies ] <EOL> self . group . map ( self . _validate_one_proxy , args ) <EOL> logging . info ( '<STR_LIT>' <EOL> % ( len ( proxies ) , len ( statics_errors ) ) ) <EOL> def _validate_one_proxy ( self , args ) : <EOL> """<STR_LIT>""" <EOL> test_sites = self . configs [ '<STR_LIT>' ] <EOL> proxy = args [ <NUM_LIT:0> ] <EOL> time_res = [ ] <EOL> args = [ ( site , proxy , time_res , args [ <NUM_LIT:1> ] ) for site in test_sites ] <EOL> self . group . map ( self . _test_one_site , args ) <EOL> mean_time = sum ( time_res ) / len ( test_sites ) <EOL> logging . info ( '<STR_LIT>' % ( proxy , mean_time ) ) <EOL> self . rdb . zadd ( self . db_zproxy , mean_time , proxy ) <EOL> def _test_one_site ( self , args ) : <EOL> url = args [ <NUM_LIT:0> ] <EOL> headers = self . configs [ '<STR_LIT>' ] <EOL> headers [ '<STR_LIT>' ] = url . split ( '<STR_LIT:/>' , <NUM_LIT:3> ) [ <NUM_LIT:2> ] <EOL> proxy = args [ <NUM_LIT:1> ] <EOL> proxies = { '<STR_LIT:http>' : proxy , } <EOL> start_time = time . time ( ) <EOL> timeout = self . configs [ '<STR_LIT>' ] <EOL> error = False <EOL> with Timeout ( timeout , False ) : <EOL> try : <EOL> res = requests . get ( url , headers = headers , proxies = proxies ) <EOL> except Exception as e : <EOL> error = True <EOL> logging . error ( '<STR_LIT>' % ( proxy , e . message ) ) <EOL> if error : <EOL> args [ <NUM_LIT:2> ] . append ( self . configs [ '<STR_LIT>' ] ) <EOL> args [ <NUM_LIT:3> ] . append ( proxy ) <EOL> else : <EOL> args [ <NUM_LIT:2> ] . append ( time . time ( ) - start_time ) </s>
