<s> import os <EOL> import re <EOL> import sys <EOL> import json <EOL> import time <EOL> import click <EOL> import socket <EOL> import shutil <EOL> import hashlib <EOL> import tempfile <EOL> import requests <EOL> import subprocess <EOL> from requests . exceptions import RequestException , SSLError <EOL> try : <EOL> from selenium import webdriver <EOL> from PIL import Image <EOL> browser_available = True <EOL> except ImportError : <EOL> browser_available = False <EOL> if sys . platform . startswith ( '<STR_LIT>' ) : <EOL> from pyvirtualdisplay import Display <EOL> needs_display = True <EOL> else : <EOL> needs_display = False <EOL> from werkzeug . urls import url_parse <EOL> from werkzeug . datastructures import Headers <EOL> _kv_re = re . compile ( r'<STR_LIT>' ) <EOL> CAPTURE_SIZE = ( <NUM_LIT> , <NUM_LIT> ) <EOL> THUMBNAIL_SIZES = ( <EOL> ( '<STR_LIT:m>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ( '<STR_LIT:s>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ( '<STR_LIT>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ) <EOL> WZ_405_HASH = '<STR_LIT>' <EOL> MAX_STALE_CACHE = <NUM_LIT> * <NUM_LIT> * <NUM_LIT> * <NUM_LIT:7> <EOL> class ProbeResult ( object ) : <EOL> def __init__ ( self , site_exists = True , werkzeug_405 = False , <EOL> responds_to_head = False ) : <EOL> self . site_exists = site_exists <EOL> self . werkzeug_405 = werkzeug_405 <EOL> self . responds_to_head = responds_to_head <EOL> def to_dict ( self ) : <EOL> return dict ( self . __dict__ ) <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % '<STR_LIT:U+002CU+0020>' . join ( <EOL> '<STR_LIT>' % ( k , v ) <EOL> for k , v in sorted ( self . __dict__ . items ( ) ) , <EOL> ) <EOL> class FlaskProbeCrawler ( object ) : <EOL> def __init__ ( self ) : <EOL> self . session = requests . session ( ) <EOL> self . closed = False <EOL> def close ( self ) : <EOL> if self . closed : <EOL> return <EOL> self . session . close ( ) <EOL> self . closed = True <EOL> def __enter__ ( self ) : <EOL> return self <EOL> def __exit__ ( self , exc_type , exc_value , tb ) : <EOL> self . close ( ) <EOL> def test_website ( self , url ) : <EOL> site_exists = self . session . get ( url ) . ok <EOL> rv = self . session . request ( '<STR_LIT>' , url ) <EOL> werkzeug_405 = False <EOL> responds_to_head = False <EOL> if rv . status_code == <NUM_LIT> and hashlib . sha1 ( rv . content ) . hexdigest ( ) == WZ_405_HASH and '<STR_LIT>' in rv . headers : <EOL> werkzeug_405 = True <EOL> if site_exists : <EOL> rv = self . session . request ( '<STR_LIT>' , url ) <EOL> if rv . status_code == <NUM_LIT:200> and not rv . content : <EOL> responds_to_head = True <EOL> return ProbeResult ( <EOL> site_exists = site_exists , <EOL> werkzeug_405 = werkzeug_405 , <EOL> responds_to_head = responds_to_head , <EOL> ) <EOL> class ImageCrawler ( object ) : <EOL> def __init__ ( self , target_dir , capture_size = None , thumbnail_sizes = None ) : <EOL> self . target_dir = target_dir <EOL> if capture_size is None : <EOL> capture_size = CAPTURE_SIZE <EOL> if needs_display : <EOL> self . display = Display ( visible = <NUM_LIT:0> , size = capture_size ) <EOL> self . display . start ( ) <EOL> else : <EOL> self . display = None <EOL> self . _browser = None <EOL> self . closed = False <EOL> self . working_dir = tempfile . mkdtemp ( ) <EOL> self . capture_size = capture_size <EOL> if thumbnail_sizes is None : <EOL> thumbnail_sizes = THUMBNAIL_SIZES <EOL> self . thumbnail_sizes = thumbnail_sizes <EOL> def get_browser ( self ) : <EOL> if self . _browser is not None : <EOL> return self . _browser <EOL> self . _browser = webdriver . Firefox ( ) <EOL> self . _browser . set_window_size ( * self . capture_size ) <EOL> return self . _browser <EOL> def close ( self ) : <EOL> try : <EOL> shutil . rmtree ( self . working_dir ) <EOL> except ( IOError , OSError ) : <EOL> pass <EOL> if not self . closed : <EOL> if self . _browser is not None : <EOL> self . _browser . quit ( ) <EOL> if self . display is not None : <EOL> self . display . stop ( ) <EOL> self . closed = True <EOL> def get_filename ( self , short_name , thumb_name ) : <EOL> return os . path . join ( self . target_dir , '<STR_LIT>' % ( <EOL> short_name , <EOL> thumb_name , <EOL> ) ) <EOL> def iter_thumbnail_sizes ( self ) : <EOL> yield ( '<STR_LIT:l>' , ) + self . capture_size <EOL> for item in self . thumbnail_sizes : <EOL> yield item <EOL> def take_screenshot ( self , url , short_name = None ) : <EOL> if short_name is None : <EOL> url_info = url_parse ( url ) <EOL> short_name = url_info . ascii_host <EOL> url_path = ( url_info . path or '<STR_LIT:/>' ) . strip ( '<STR_LIT:/>' ) <EOL> if url_path : <EOL> short_name += '<STR_LIT:.>' + url_path <EOL> fn = tempfile . mktemp ( '<STR_LIT>' , prefix = '<STR_LIT>' , dir = self . working_dir ) <EOL> browser = self . get_browser ( ) <EOL> browser . get ( url ) <EOL> browser . save_screenshot ( fn ) <EOL> im = Image . open ( fn ) <EOL> for thumb_name , width , height in self . iter_thumbnail_sizes ( ) : <EOL> small = im . crop ( ( <NUM_LIT:0> , <NUM_LIT:0> , im . size [ <NUM_LIT:0> ] , <EOL> min ( im . size [ <NUM_LIT:1> ] , self . capture_size [ <NUM_LIT:1> ] ) ) ) <EOL> small . thumbnail ( ( width , height ) , Image . ANTIALIAS ) <EOL> fn = self . get_filename ( short_name , thumb_name ) <EOL> try : <EOL> os . makedirs ( os . path . dirname ( fn ) ) <EOL> except OSError : <EOL> pass <EOL> small . save ( fn ) <EOL> def __enter__ ( self ) : <EOL> return self <EOL> def __exit__ ( self , exc_type , exc_value , tb ) : <EOL> self . close ( ) <EOL> def read_mime ( f ) : <EOL> headers = [ ] <EOL> h = hashlib . sha1 ( ) <EOL> def _readline ( ) : <EOL> line = f . readline ( ) <EOL> h . update ( line ) <EOL> if not line . strip ( ) : <EOL> return u'<STR_LIT>' <EOL> return line . rstrip ( '<STR_LIT:\r\n>' ) . decode ( '<STR_LIT:utf-8>' ) <EOL> while <NUM_LIT:1> : <EOL> line = _readline ( ) <EOL> if not line : <EOL> break <EOL> match = _kv_re . match ( line ) <EOL> if match is not None : <EOL> headers . append ( match . groups ( ) ) <EOL> continue <EOL> elif line [ : <NUM_LIT:1> ] . isspace ( ) : <EOL> old_key , old_value = headers [ - <NUM_LIT:1> ] <EOL> headers [ - <NUM_LIT:1> ] = ( old_key , old_value + u'<STR_LIT:U+0020>' + line [ <NUM_LIT:1> : ] ) <EOL> else : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> payload = f . read ( ) <EOL> h . update ( payload ) <EOL> return Headers ( headers ) , payload , h . hexdigest ( ) <EOL> class RepoProcessor ( object ) : <EOL> def __init__ ( self , repo , log = False ) : <EOL> self . log = log <EOL> self . repo = repo <EOL> self . cache = { } <EOL> self . cache_filename = os . path . join ( self . repo , '<STR_LIT>' , '<STR_LIT>' ) <EOL> image_path = os . path . join ( self . repo , '<STR_LIT>' , '<STR_LIT>' ) <EOL> try : <EOL> os . makedirs ( image_path ) <EOL> except OSError : <EOL> pass <EOL> self . probe_crawler = FlaskProbeCrawler ( ) <EOL> self . image_crawler = ImageCrawler ( image_path ) <EOL> if os . path . isfile ( self . cache_filename ) : <EOL> with open ( self . cache_filename , '<STR_LIT:rb>' ) as f : <EOL> self . cache . update ( json . load ( f ) ) <EOL> def __enter__ ( self ) : <EOL> return self <EOL> def __exit__ ( self , exc_type , exc_value , tb ) : <EOL> self . close ( ) <EOL> def close ( self ) : <EOL> self . image_crawler . close ( ) <EOL> self . probe_crawler . close ( ) <EOL> def process_website ( self , domain ) : <EOL> if self . log : <EOL> click . echo ( '<STR_LIT>' % click . style ( domain , fg = '<STR_LIT>' ) ) <EOL> filename = os . path . join ( self . repo , '<STR_LIT>' , domain ) <EOL> cache_entry = self . cache . get ( domain ) <EOL> with open ( filename , '<STR_LIT:rb>' ) as f : <EOL> headers , payload , checksum = read_mime ( f ) <EOL> if not ( cache_entry is None or <EOL> cache_entry [ '<STR_LIT>' ] != checksum or <EOL> cache_entry [ '<STR_LIT>' ] < time . time ( ) - MAX_STALE_CACHE ) : <EOL> if self . log : <EOL> click . secho ( '<STR_LIT>' , fg = '<STR_LIT>' ) <EOL> return False <EOL> url = headers . get ( '<STR_LIT:url>' ) <EOL> if not url : <EOL> url = '<STR_LIT>' % domain <EOL> try : <EOL> test_result = self . probe_crawler . test_website ( url ) <EOL> except ( socket . error , SSLError , RequestException ) : <EOL> if self . log : <EOL> click . secho ( '<STR_LIT>' , fg = '<STR_LIT>' ) <EOL> return False <EOL> if test_result . site_exists : <EOL> self . image_crawler . take_screenshot ( url , short_name = domain ) <EOL> self . cache [ domain ] = { <EOL> '<STR_LIT>' : checksum , <EOL> '<STR_LIT>' : int ( time . time ( ) ) , <EOL> '<STR_LIT>' : test_result . to_dict ( ) , <EOL> '<STR_LIT:url>' : url , <EOL> '<STR_LIT:name>' : headers . get ( '<STR_LIT:name>' ) , <EOL> '<STR_LIT:source>' : headers . get ( '<STR_LIT:source>' ) , <EOL> '<STR_LIT:description>' : payload , <EOL> } <EOL> if self . log : <EOL> click . secho ( '<STR_LIT>' , fg = '<STR_LIT>' ) <EOL> return True <EOL> def process_all ( self ) : <EOL> updated = [ ] <EOL> p = os . path . join ( self . repo , '<STR_LIT>' ) <EOL> for filename in os . listdir ( p ) : <EOL> if os . path . isfile ( os . path . join ( p , filename ) ) and not filename . startswith ( ( '<STR_LIT:.>' , '<STR_LIT:_>' ) ) : <EOL> if self . process_website ( filename ) : <EOL> updated . append ( filename ) <EOL> return updated <EOL> def write_cache ( self ) : <EOL> if self . log : <EOL> click . secho ( '<STR_LIT>' , fg = '<STR_LIT>' ) <EOL> with open ( self . cache_filename , '<STR_LIT:wb>' ) as f : <EOL> body = json . dumps ( self . cache , indent = <NUM_LIT:2> , sort_keys = True ) <EOL> for line in body . splitlines ( ) : <EOL> f . write ( line . rstrip ( ) + '<STR_LIT:\n>' ) <EOL> def commit ( self , changes ) : <EOL> if not changes : <EOL> return <EOL> if self . log : <EOL> click . secho ( '<STR_LIT>' , fg = '<STR_LIT>' ) <EOL> msg = '<STR_LIT>' % '<STR_LIT:\n>' . join ( '<STR_LIT>' % x for x in changes ) <EOL> self . git ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> self . git ( '<STR_LIT>' , '<STR_LIT>' , msg ) <EOL> def pull ( self , reset = True ) : <EOL> if self . log : <EOL> click . secho ( '<STR_LIT>' , fg = '<STR_LIT>' ) <EOL> if reset : <EOL> self . git ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> self . git ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> def push ( self ) : <EOL> if self . log : <EOL> click . secho ( '<STR_LIT>' , fg = '<STR_LIT>' ) <EOL> self . git ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> self . git ( '<STR_LIT>' ) <EOL> def git ( self , * args , ** extra ) : <EOL> """<STR_LIT>""" <EOL> extra . setdefault ( '<STR_LIT>' , self . repo ) <EOL> return subprocess . Popen ( [ '<STR_LIT>' ] + list ( args ) , ** extra ) . wait ( ) <EOL> def run_processor ( repo , log = False , commit = False , push = False ) : <EOL> with RepoProcessor ( repo , log = log ) as processor : <EOL> processor . pull ( ) <EOL> changes = processor . process_all ( ) <EOL> if changes : <EOL> processor . write_cache ( ) <EOL> if commit : <EOL> processor . commit ( changes ) <EOL> if push : <EOL> processor . push ( ) <EOL> @ click . command ( ) <EOL> @ click . option ( '<STR_LIT>' , help = '<STR_LIT>' , <EOL> default = '<STR_LIT>' , type = click . Path ( ) ) <EOL> @ click . option ( '<STR_LIT>' , help = '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> default = True ) <EOL> @ click . option ( '<STR_LIT>' , help = '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> default = True ) <EOL> def cli ( repo , commit , push ) : <EOL> """<STR_LIT>""" <EOL> click . echo ( '<STR_LIT>' % repo ) <EOL> run_processor ( repo , log = True , commit = commit , push = push ) <EOL> click . echo ( '<STR_LIT>' ) </s>
