<s> import sys <EOL> import os <EOL> import re <EOL> import urllib2 <EOL> import time <EOL> from xml . dom . minidom import Document <EOL> try : <EOL> import xml . etree . ElementTree as ElementTree <EOL> except ImportError : <EOL> try : <EOL> import cElementTree as ElementTree <EOL> except ImportError : <EOL> try : <EOL> import elementtree . ElementTree as ElementTree <EOL> except ImportError : <EOL> import lxml . etree as ElementTree <EOL> missing_deps = False <EOL> try : <EOL> import simplejson <EOL> from BeautifulSoup import BeautifulSoup <EOL> except ImportError , E : <EOL> missing_deps = E <EOL> feedName = "<STR_LIT>" <EOL> feedPath = "<STR_LIT>" <EOL> def getListOfOnlineExamples ( baseUrl ) : <EOL> """<STR_LIT>""" <EOL> html = urllib2 . urlopen ( baseUrl ) <EOL> soup = BeautifulSoup ( html ) <EOL> examples = soup . findAll ( '<STR_LIT>' ) <EOL> examples = [ example . find ( '<STR_LIT:a>' ) . get ( '<STR_LIT>' ) for example in examples ] <EOL> examples = [ example for example in examples if example . endswith ( '<STR_LIT>' ) ] <EOL> examples = [ example for example in examples ] <EOL> return examples <EOL> def getListOfExamples ( relPath ) : <EOL> """<STR_LIT>""" <EOL> examples = os . listdir ( relPath ) <EOL> examples = [ example for example in examples if example . endswith ( '<STR_LIT>' ) and example != "<STR_LIT>" ] <EOL> return examples <EOL> def getExampleHtml ( location ) : <EOL> """<STR_LIT>""" <EOL> print '<STR_LIT:.>' , <EOL> if location . startswith ( '<STR_LIT:http>' ) : <EOL> return urllib2 . urlopen ( location ) . read ( ) <EOL> else : <EOL> f = open ( location ) <EOL> html = f . read ( ) <EOL> f . close ( ) <EOL> return html <EOL> def extractById ( soup , tagId , value = None ) : <EOL> """<STR_LIT>""" <EOL> beautifulTag = soup . find ( id = tagId ) <EOL> if beautifulTag : <EOL> if beautifulTag . contents : <EOL> value = str ( beautifulTag . renderContents ( ) ) . strip ( ) <EOL> value = value . replace ( '<STR_LIT:\t>' , '<STR_LIT>' ) <EOL> value = value . replace ( '<STR_LIT:\n>' , '<STR_LIT>' ) <EOL> return value <EOL> def getRelatedClasses ( html ) : <EOL> """<STR_LIT>""" <EOL> rawstr = r'''<STR_LIT>''' <EOL> return re . findall ( rawstr , html ) <EOL> def parseHtml ( html , ids ) : <EOL> """<STR_LIT>""" <EOL> soup = BeautifulSoup ( html ) <EOL> d = { } <EOL> for tagId in ids : <EOL> d [ tagId ] = extractById ( soup , tagId ) <EOL> classes = getRelatedClasses ( html ) <EOL> d [ '<STR_LIT>' ] = classes <EOL> return d <EOL> def getSvnInfo ( path ) : <EOL> h = os . popen ( "<STR_LIT>" % path ) <EOL> tree = ElementTree . fromstring ( h . read ( ) ) <EOL> h . close ( ) <EOL> d = { <EOL> '<STR_LIT:url>' : tree . findtext ( '<STR_LIT>' ) , <EOL> '<STR_LIT>' : tree . findtext ( '<STR_LIT>' ) , <EOL> '<STR_LIT:date>' : tree . findtext ( '<STR_LIT>' ) <EOL> } <EOL> return d <EOL> def createFeed ( examples ) : <EOL> doc = Document ( ) <EOL> atomuri = "<STR_LIT>" <EOL> feed = doc . createElementNS ( atomuri , "<STR_LIT>" ) <EOL> feed . setAttribute ( "<STR_LIT>" , atomuri ) <EOL> title = doc . createElementNS ( atomuri , "<STR_LIT:title>" ) <EOL> title . appendChild ( doc . createTextNode ( "<STR_LIT>" ) ) <EOL> feed . appendChild ( title ) <EOL> link = doc . createElementNS ( atomuri , "<STR_LIT>" ) <EOL> link . setAttribute ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> link . setAttribute ( "<STR_LIT>" , feedPath + feedName ) <EOL> modtime = time . strftime ( "<STR_LIT>" , time . gmtime ( ) ) <EOL> id = doc . createElementNS ( atomuri , "<STR_LIT:id>" ) <EOL> id . appendChild ( doc . createTextNode ( "<STR_LIT>" % ( feedPath , feedName , modtime ) ) ) <EOL> feed . appendChild ( id ) <EOL> updated = doc . createElementNS ( atomuri , "<STR_LIT>" ) <EOL> updated . appendChild ( doc . createTextNode ( modtime ) ) <EOL> feed . appendChild ( updated ) <EOL> examples . sort ( key = lambda x : x [ "<STR_LIT>" ] ) <EOL> for example in sorted ( examples , key = lambda x : x [ "<STR_LIT>" ] , reverse = True ) : <EOL> entry = doc . createElementNS ( atomuri , "<STR_LIT>" ) <EOL> title = doc . createElementNS ( atomuri , "<STR_LIT:title>" ) <EOL> title . appendChild ( doc . createTextNode ( example [ "<STR_LIT:title>" ] or example [ "<STR_LIT>" ] ) ) <EOL> entry . appendChild ( title ) <EOL> link = doc . createElementNS ( atomuri , "<STR_LIT>" ) <EOL> link . setAttribute ( "<STR_LIT>" , "<STR_LIT>" % ( feedPath , example [ "<STR_LIT>" ] ) ) <EOL> entry . appendChild ( link ) <EOL> summary = doc . createElementNS ( atomuri , "<STR_LIT>" ) <EOL> summary . appendChild ( doc . createTextNode ( example [ "<STR_LIT>" ] or example [ "<STR_LIT>" ] ) ) <EOL> entry . appendChild ( summary ) <EOL> updated = doc . createElementNS ( atomuri , "<STR_LIT>" ) <EOL> updated . appendChild ( doc . createTextNode ( example [ "<STR_LIT>" ] ) ) <EOL> entry . appendChild ( updated ) <EOL> author = doc . createElementNS ( atomuri , "<STR_LIT>" ) <EOL> name = doc . createElementNS ( atomuri , "<STR_LIT:name>" ) <EOL> name . appendChild ( doc . createTextNode ( example [ "<STR_LIT>" ] ) ) <EOL> author . appendChild ( name ) <EOL> entry . appendChild ( author ) <EOL> id = doc . createElementNS ( atomuri , "<STR_LIT:id>" ) <EOL> id . appendChild ( doc . createTextNode ( "<STR_LIT>" % ( feedPath , example [ "<STR_LIT>" ] , example [ "<STR_LIT>" ] ) ) ) <EOL> entry . appendChild ( id ) <EOL> feed . appendChild ( entry ) <EOL> doc . appendChild ( feed ) <EOL> return doc <EOL> def wordIndex ( examples ) : <EOL> """<STR_LIT>""" <EOL> index = { } <EOL> unword = re . compile ( "<STR_LIT>" ) <EOL> keys = [ "<STR_LIT>" , "<STR_LIT:title>" ] <EOL> for i in range ( len ( examples ) ) : <EOL> for key in keys : <EOL> text = examples [ i ] [ key ] <EOL> if text : <EOL> words = unword . split ( text ) <EOL> for word in words : <EOL> if word : <EOL> word = word . lower ( ) <EOL> if index . has_key ( word ) : <EOL> if index [ word ] . has_key ( i ) : <EOL> index [ word ] [ i ] += <NUM_LIT:1> <EOL> else : <EOL> index [ word ] [ i ] = <NUM_LIT:1> <EOL> else : <EOL> index [ word ] = { i : <NUM_LIT:1> } <EOL> return index <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> if missing_deps : <EOL> print "<STR_LIT>" % E <EOL> sys . exit ( ) <EOL> if len ( sys . argv ) > <NUM_LIT:1> : <EOL> outFile = open ( sys . argv [ <NUM_LIT:1> ] , '<STR_LIT:w>' ) <EOL> else : <EOL> outFile = open ( '<STR_LIT>' , '<STR_LIT:w>' ) <EOL> examplesLocation = '<STR_LIT>' <EOL> print '<STR_LIT>' % ( examplesLocation , outFile . name ) <EOL> exampleList = [ ] <EOL> docIds = [ '<STR_LIT:title>' , '<STR_LIT>' ] <EOL> examples = getListOfExamples ( examplesLocation ) <EOL> modtime = time . strftime ( "<STR_LIT>" , time . gmtime ( ) ) <EOL> for example in examples : <EOL> url = os . path . join ( examplesLocation , example ) <EOL> html = getExampleHtml ( url ) <EOL> tagvalues = parseHtml ( html , docIds ) <EOL> tagvalues [ '<STR_LIT>' ] = example <EOL> d = getSvnInfo ( url ) <EOL> tagvalues [ "<STR_LIT>" ] = d [ "<STR_LIT:date>" ] or modtime <EOL> tagvalues [ "<STR_LIT>" ] = d [ "<STR_LIT>" ] or "<STR_LIT>" <EOL> tagvalues [ '<STR_LIT>' ] = example <EOL> exampleList . append ( tagvalues ) <EOL> print <EOL> exampleList . sort ( key = lambda x : x [ '<STR_LIT>' ] . lower ( ) ) <EOL> index = wordIndex ( exampleList ) <EOL> json = simplejson . dumps ( { "<STR_LIT>" : exampleList , "<STR_LIT:index>" : index } ) <EOL> json = '<STR_LIT>' + json <EOL> outFile . write ( json ) <EOL> outFile . close ( ) <EOL> print "<STR_LIT>" % feedName <EOL> atom = open ( '<STR_LIT>' % feedName , '<STR_LIT:w>' ) <EOL> doc = createFeed ( exampleList ) <EOL> atom . write ( doc . toxml ( ) ) <EOL> atom . close ( ) <EOL> print '<STR_LIT>' </s>
