<s> """<STR_LIT>""" <EOL> import argparse <EOL> import io <EOL> import re <EOL> import urllib <EOL> from xml . dom . minidom import parse <EOL> import trie <EOL> uasparser_xml_uri = "<STR_LIT>" <EOL> uasparser_xml_query = "<STR_LIT>" <EOL> download_xml_url = "<STR_LIT>" % ( uasparser_xml_uri , uasparser_xml_query ) <EOL> file_uasparser_xml = "<STR_LIT>" <EOL> file_robots_config = "<STR_LIT>" <EOL> file_user_agents = "<STR_LIT>" <EOL> others = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] <EOL> default_xml_file = "<STR_LIT>" <EOL> re_trailing_version = re . compile ( r"""<STR_LIT>""" , re . VERBOSE ) <EOL> def parser_setup ( ) : <EOL> """<STR_LIT>""" <EOL> ap = argparse . ArgumentParser ( description = __doc__ ) <EOL> ap . add_argument ( "<STR_LIT>" , "<STR_LIT>" , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> args = ap . parse_args ( ) <EOL> return args <EOL> def download_uasparser_xml ( xml_url , xml_file ) : <EOL> """<STR_LIT>""" <EOL> urllib . urlretrieve ( xml_url , xml_file ) <EOL> def add_others_to_robots ( robots , others_list ) : <EOL> """<STR_LIT>""" <EOL> for other in others_list : <EOL> robots . add ( other ) <EOL> return robots <EOL> def add_xml_names_to_robots ( robots , xml_file ) : <EOL> """<STR_LIT>""" <EOL> dom = parse ( xml_file ) <EOL> user_agents = list ( ) <EOL> for robot in dom . getElementsByTagName ( "<STR_LIT>" ) : <EOL> for node in robot . childNodes : <EOL> if node . localName == "<STR_LIT>" : <EOL> family = node . firstChild . data . strip ( ) <EOL> if node . localName == "<STR_LIT:name>" : <EOL> name = node . firstChild . data . strip ( ) <EOL> if node . localName == "<STR_LIT>" : <EOL> user_agents . append ( node . firstChild . data . strip ( ) ) <EOL> if len ( family ) > <NUM_LIT:3> : <EOL> name = family <EOL> name_new = re_trailing_version . search ( name ) <EOL> if name_new : <EOL> name = name_new . group ( "<STR_LIT>" ) <EOL> if name . endswith ( "<STR_LIT>" ) : <EOL> name = name . rstrip ( "<STR_LIT>" ) <EOL> robots . add ( name . strip ( ) ) <EOL> return robots , sorted ( user_agents ) <EOL> def create_optmized_robots_regex ( robots ) : <EOL> """<STR_LIT>""" <EOL> optimized_robots = trie . Trie ( ) <EOL> for robot in sorted ( robots ) : <EOL> optimized_robots . add ( robot ) <EOL> return optimized_robots . regexp ( ) <EOL> def main ( ) : <EOL> robots = set ( ) <EOL> args = parser_setup ( ) <EOL> if args . file : <EOL> xml_file = args . file <EOL> else : <EOL> download_uasparser_xml ( download_xml_url , file_uasparser_xml ) <EOL> xml_file = file_uasparser_xml <EOL> robots = add_others_to_robots ( robots , others ) <EOL> robots , user_agents = add_xml_names_to_robots ( robots , xml_file ) <EOL> optibots = create_optmized_robots_regex ( robots ) <EOL> with io . open ( file_robots_config , "<STR_LIT:w>" , encoding = "<STR_LIT:utf8>" ) as fh : <EOL> fh . write ( u"<STR_LIT>" ) <EOL> fh . write ( u"<STR_LIT>" % optibots ) <EOL> with io . open ( file_user_agents , "<STR_LIT:w>" , encoding = "<STR_LIT:utf8>" ) as fh : <EOL> for agent in user_agents : <EOL> fh . write ( "<STR_LIT>" % agent ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> main ( ) </s>
