<s> import requests <EOL> import re <EOL> import os , subprocess , signal , sys <EOL> from getopt import getopt <EOL> from time import sleep <EOL> root = '<STR_LIT>' <EOL> opts , args = getopt ( sys . argv [ <NUM_LIT:1> : ] , "<STR_LIT>" , [ "<STR_LIT>" , ] ) <EOL> for opt , arg in opts : <EOL> if opt in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> root = arg + '<STR_LIT:/>' <EOL> print ( "<STR_LIT>" , root ) <EOL> def get_links_in_line ( line ) : <EOL> links = [ ] <EOL> link_re = """<STR_LIT>""" <EOL> p = re . compile ( link_re ) <EOL> for segment in line . split ( '<STR_LIT>' ) : <EOL> m = p . match ( segment ) <EOL> if m : <EOL> links . append ( m . group ( <NUM_LIT:2> ) ) <EOL> return links <EOL> def get_links_in_file ( root_dir , filename ) : <EOL> f = open ( root_dir + filename , '<STR_LIT:r>' ) <EOL> lines = f . readlines ( ) <EOL> f . close ( ) <EOL> links_to_check = [ ] <EOL> for line in lines : <EOL> if '<STR_LIT>' in line : <EOL> for link in get_links_in_line ( line ) : <EOL> if '<STR_LIT:html>' in link : <EOL> links_to_check . append ( link ) <EOL> return links_to_check <EOL> def check_links ( filename , links , bad_links , links_tested ) : <EOL> print ( "<STR_LIT>" , links ) <EOL> for link in links : <EOL> print ( "<STR_LIT>" % link ) <EOL> if link in links_tested : <EOL> continue <EOL> if '<STR_LIT:http>' in link : <EOL> url = link <EOL> else : <EOL> url = root + link <EOL> print ( '<STR_LIT>' , url ) <EOL> r = requests . get ( url ) <EOL> print ( '<STR_LIT>' , r . status_code ) <EOL> if r . status_code != <NUM_LIT:200> : <EOL> bad_links [ filename + '<STR_LIT>' + link ] = r . status_code <EOL> else : <EOL> links_tested . append ( link ) <EOL> root_dir = '<STR_LIT>' <EOL> filenames = [ ] <EOL> for filename in os . listdir ( root_dir ) : <EOL> if '<STR_LIT:html>' in filename : <EOL> filenames . append ( filename ) <EOL> print ( "<STR_LIT>" ) <EOL> cmd = '<STR_LIT>' <EOL> cmd += '<STR_LIT>' <EOL> pro = subprocess . Popen ( cmd , shell = True , preexec_fn = os . setsid ) <EOL> print ( "<STR_LIT>" ) <EOL> sleep ( <NUM_LIT:1> ) <EOL> bad_links = { } <EOL> links_tested = [ ] <EOL> num_links_checked = <NUM_LIT:0> <EOL> for filename in filenames : <EOL> links_to_check = get_links_in_file ( root_dir , filename ) <EOL> check_links ( filename , links_to_check , bad_links , links_tested ) <EOL> num_links_checked += len ( links_to_check ) <EOL> os . killpg ( pro . pid , signal . SIGTERM ) <EOL> print ( "<STR_LIT>" ) <EOL> if bad_links : <EOL> for link in bad_links : <EOL> print ( '<STR_LIT:\n>' , bad_links [ link ] , link ) <EOL> else : <EOL> print ( "<STR_LIT>" ) <EOL> print ( "<STR_LIT:\n>" ) <EOL> print ( "<STR_LIT>" % num_links_checked ) <EOL> print ( "<STR_LIT>" % len ( links_tested ) ) </s>
