<s> import os <EOL> import atexit <EOL> from uuid import uuid4 <EOL> from datetime import datetime <EOL> from threading import local <EOL> from scrapekit . config import Config <EOL> from scrapekit . tasks import TaskManager , Task <EOL> from scrapekit . http import make_session <EOL> from scrapekit . logs import make_logger <EOL> from scrapekit import reporting <EOL> class Scraper ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , name , config = None , report = False ) : <EOL> self . name = name <EOL> self . id = uuid4 ( ) <EOL> self . start_time = datetime . utcnow ( ) <EOL> self . config = Config ( self , config ) <EOL> try : <EOL> os . makedirs ( self . config . data_path ) <EOL> except : <EOL> pass <EOL> self . _task_manager = None <EOL> self . task_ctx = local ( ) <EOL> self . log = make_logger ( self ) <EOL> self . session = self . Session ( ) <EOL> if report : <EOL> atexit . register ( self . report ) <EOL> @ property <EOL> def task_manager ( self ) : <EOL> if self . _task_manager is None : <EOL> self . _task_manager = TaskManager ( threads = self . config . threads ) <EOL> return self . _task_manager <EOL> def task ( self , fn ) : <EOL> """<STR_LIT>""" <EOL> return Task ( self , fn ) <EOL> def Session ( self ) : <EOL> """<STR_LIT>""" <EOL> return make_session ( self ) <EOL> def head ( self , url , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> return self . Session ( ) . get ( url , ** kwargs ) <EOL> def get ( self , url , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> return self . Session ( ) . get ( url , ** kwargs ) <EOL> def post ( self , url , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> return self . Session ( ) . post ( url , ** kwargs ) <EOL> def put ( self , url , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> return self . Session ( ) . put ( url , ** kwargs ) <EOL> def report ( self ) : <EOL> """<STR_LIT>""" <EOL> index_file = reporting . generate ( self ) <EOL> print ( "<STR_LIT>" % index_file ) <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % self . name </s>
