<s> from lxml import etree <EOL> from cStringIO import StringIO <EOL> from urllib import urlopen <EOL> from gzip import GzipFile <EOL> import os <EOL> import re <EOL> import sys <EOL> from exceptions import * <EOL> from urlsetelement import * <EOL> class UrlSet ( object ) : <EOL> """<STR_LIT>""" <EOL> @ staticmethod <EOL> def from_url ( url , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> u = urlopen ( url ) <EOL> if u . headers . has_key ( "<STR_LIT>" ) and u . headers [ "<STR_LIT>" ] . lower ( ) == "<STR_LIT>" : <EOL> u = GzipFile ( fileobj = StringIO ( u . read ( ) ) ) <EOL> return UrlSet ( u , url , ** kwargs ) <EOL> @ staticmethod <EOL> def from_file ( file , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> return UrlSet ( open ( file ) , file , ** kwargs ) <EOL> @ staticmethod <EOL> def from_str ( str , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> return UrlSet ( StringIO ( str ) , '<STR_LIT:string>' , ** kwargs ) <EOL> @ staticmethod <EOL> def empty_container ( ) : <EOL> """<STR_LIT>""" <EOL> return UrlSet ( ) <EOL> source = property ( lambda self : self . _source ) <EOL> def __init__ ( self , handle = None , source = '<STR_LIT>' , validate = True ) : <EOL> """<STR_LIT>""" <EOL> self . _source = source <EOL> self . _handle = handle <EOL> self . _validate = validate <EOL> self . _elements = [ ] <EOL> def append ( self , urlsetelement ) : <EOL> if self . _handle : <EOL> raise Exception ( "<STR_LIT>" + "<STR_LIT>" ) <EOL> self . _elements . append ( urlsetelement ) <EOL> def get_urls ( self ) : <EOL> if not self . _handle : <EOL> return self . get_urls_from_elements ( ) <EOL> else : <EOL> return self . get_urls_from_handle ( ) <EOL> def get_urls_from_elements ( self ) : <EOL> return self . _elements <EOL> def get_urls_from_handle ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . _validate : <EOL> schema = etree . XMLSchema ( file = open ( self . get_schema_path ( ) ) ) <EOL> else : <EOL> schema = None <EOL> context = etree . iterparse ( self . _handle , events = ( '<STR_LIT:end>' , ) , schema = schema ) <EOL> element_data = { } <EOL> for action , elem in context : <EOL> tag = self . _remove_ns ( elem . tag ) <EOL> if tag == '<STR_LIT:url>' and element_data : <EOL> try : <EOL> e = UrlSetElement ( ** element_data ) <EOL> yield e <EOL> except ValueError : <EOL> element_data = { } <EOL> continue <EOL> elif tag in [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] : <EOL> element_data [ tag ] = elem . text <EOL> while elem . getprevious ( ) is not None : <EOL> del elem . getparent ( ) [ <NUM_LIT:0> ] <EOL> del context <EOL> del schema <EOL> def _remove_ns ( self , str ) : <EOL> return re . sub ( '<STR_LIT>' , '<STR_LIT>' , str ) <EOL> def get_schema_path ( self ) : <EOL> base = os . path . dirname ( os . path . abspath ( __file__ ) ) <EOL> return os . path . join ( base , '<STR_LIT>' , '<STR_LIT>' ) <EOL> def pprint ( self , out = sys . stdout ) : <EOL> """<STR_LIT>""" <EOL> if self . _handle : <EOL> raise Exception ( "<STR_LIT>" + "<STR_LIT>" ) <EOL> urlset = etree . Element ( "<STR_LIT>" , xmlns = "<STR_LIT>" ) <EOL> for url in self . _elements : <EOL> ue = etree . Element ( "<STR_LIT:url>" ) <EOL> loc = etree . Element ( "<STR_LIT>" ) <EOL> lastmod = etree . Element ( "<STR_LIT>" ) <EOL> changefreq = etree . Element ( "<STR_LIT>" ) <EOL> priority = etree . Element ( "<STR_LIT>" ) <EOL> loc . text = url . loc <EOL> ue . append ( loc ) <EOL> if url . lastmod : <EOL> lastmod . text = url . lastmod . isoformat ( ) <EOL> ue . append ( lastmod ) <EOL> if url . changefreq : <EOL> changefreq . text = url . changefreq <EOL> ue . append ( changefreq ) <EOL> if url . priority : <EOL> priority . text = str ( url . priority ) <EOL> ue . append ( priority ) <EOL> urlset . append ( ue ) <EOL> out . write ( etree . tostring ( urlset , xml_declaration = True , pretty_print = True , encoding = "<STR_LIT>" ) ) <EOL> def __iter__ ( self ) : <EOL> return iter ( self . get_urls ( ) ) </s>
