<s> import time <EOL> import socket <EOL> import urllib <EOL> import urllib2 <EOL> import re <EOL> from urllib2 import URLError <EOL> import cloud <EOL> import os <EOL> import redis <EOL> cloud . setkey ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> set = "<STR_LIT>" <EOL> redis_host = "<STR_LIT>" <EOL> host_list = "<STR_LIT>" <EOL> chunksize = <NUM_LIT:1000> <EOL> start = <NUM_LIT:0> <EOL> timeout = <NUM_LIT:10> <EOL> def chunks ( start , size ) : <EOL> f = open ( host_list ) <EOL> patterns = [ ] <EOL> count = <NUM_LIT:0> <EOL> lines = [ ] <EOL> for line in f : <EOL> if count < start : <EOL> count = count + <NUM_LIT:1> <EOL> elif count < start + chunksize : <EOL> lines . append ( "<STR_LIT>" % line . split ( "<STR_LIT:U+002C>" ) [ <NUM_LIT:0> ] . rstrip ( ) ) <EOL> patterns . append ( '<STR_LIT>' ) <EOL> count = count + <NUM_LIT:1> <EOL> else : <EOL> break <EOL> f . close ( ) <EOL> return lines , patterns <EOL> def check_git ( url , pattern ) : <EOL> request = urllib2 . Request ( "<STR_LIT:%s>" % ( url ) ) <EOL> try : <EOL> response = urllib2 . urlopen ( request ) <EOL> except Exception , e : <EOL> return False , "<STR_LIT>" <EOL> else : <EOL> result = response . read ( ) <EOL> match = re . search ( pattern , result ) <EOL> if match is None : <EOL> return False , "<STR_LIT>" <EOL> else : <EOL> r = redis . Redis ( redis_host ) <EOL> r . sadd ( set , url ) <EOL> return True , result <EOL> try : <EOL> lines , patterns = chunks ( start , chunksize ) <EOL> print cloud . map ( check_git , lines , patterns ) <EOL> except Exception , e : <EOL> print e </s>
