<s> """<STR_LIT>""" <EOL> import httplib , urllib , re <EOL> import simplejson as json <EOL> from StringIO import StringIO <EOL> PARAMS_XML = """<STR_LIT>""" <EOL> STRIP_RE = re . compile ( '<STR_LIT>' , re . IGNORECASE ) <EOL> __version__ = "<STR_LIT>" <EOL> VALID_CONTENT_TYPES = ( <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> ) <EOL> VALID_OUTPUT_TYPES = ( <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT:application/json>" , <EOL> "<STR_LIT>" , <EOL> ) <EOL> VALID_BOOLEANS = ( '<STR_LIT:true>' , '<STR_LIT:false>' , '<STR_LIT:t>' , '<STR_LIT:f>' , ) <EOL> VALID_METADATA_TYPES = ( <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> ) <EOL> class AppURLopener ( urllib . FancyURLopener ) : <EOL> version = "<STR_LIT>" <EOL> urllib . _urlopener = AppURLopener ( ) <EOL> class Calais ( ) : <EOL> """<STR_LIT>""" <EOL> api_key = None <EOL> processing_directives = { <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT:application/json>" , <EOL> "<STR_LIT>" : '<STR_LIT>' , <EOL> "<STR_LIT>" : True , <EOL> "<STR_LIT>" : '<STR_LIT>' , <EOL> "<STR_LIT>" : True , <EOL> } <EOL> user_directives = { <EOL> "<STR_LIT>" : False , <EOL> "<STR_LIT>" : False , <EOL> "<STR_LIT>" : '<STR_LIT>' , <EOL> "<STR_LIT>" : "<STR_LIT>" % __version__ , <EOL> } <EOL> external_metadata = { } <EOL> def __init__ ( self , api_key , submitter = None ) : <EOL> self . api_key = api_key <EOL> if submitter : <EOL> self . user_directives [ "<STR_LIT>" ] = submitter <EOL> def _validate_directives ( self ) : <EOL> """<STR_LIT>""" <EOL> if ( self . processing_directives [ '<STR_LIT>' ] and <EOL> self . processing_directives [ '<STR_LIT>' ] . lower ( ) not in VALID_CONTENT_TYPES ) : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> if ( self . processing_directives [ '<STR_LIT>' ] and <EOL> self . processing_directives [ '<STR_LIT>' ] . lower ( ) not in VALID_OUTPUT_TYPES ) : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> if ( self . processing_directives [ '<STR_LIT>' ] and <EOL> self . processing_directives [ '<STR_LIT>' ] . lower ( ) not in VALID_METADATA_TYPES ) : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> if ( self . processing_directives [ '<STR_LIT>' ] and <EOL> not isinstance ( self . processing_directives [ '<STR_LIT>' ] , bool ) ) : <EOL> if self . processing_directives [ '<STR_LIT>' ] . lower ( ) not in VALID_BOOLEAN_TYPES : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> if ( self . processing_directives [ '<STR_LIT>' ] and <EOL> not isinstance ( self . processing_directives [ '<STR_LIT>' ] , bool ) ) : <EOL> if self . processing_directives [ '<STR_LIT>' ] . lower ( ) not in VALID_BOOLEAN_TYPES : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> if ( self . user_directives [ '<STR_LIT>' ] and <EOL> not isinstance ( self . user_directives [ '<STR_LIT>' ] , bool ) ) : <EOL> if self . user_directives [ '<STR_LIT>' ] . lower ( ) not in VALID_BOOLEAN_TYPES : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> if ( self . user_directives [ '<STR_LIT>' ] and <EOL> not isinstance ( self . user_directives [ '<STR_LIT>' ] , bool ) ) : <EOL> if self . user_directives [ '<STR_LIT>' ] . lower ( ) not in VALID_BOOLEAN_TYPES : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> def _get_param_headers ( self ) : <EOL> headers = { } <EOL> for key , val in self . processing_directives . items ( ) : <EOL> if key == '<STR_LIT>' : <EOL> headers [ '<STR_LIT:Content-Type>' ] = val <EOL> elif key == '<STR_LIT>' : <EOL> headers [ '<STR_LIT>' ] = val <EOL> elif val : <EOL> headers [ key ] = val <EOL> for key , val in self . user_directives . items ( ) : <EOL> if val : <EOL> headers [ key ] = val <EOL> for key , val in self . external_metadata . items ( ) : <EOL> if val : <EOL> headers [ key ] = val <EOL> return headers <EOL> def rest_POST ( self , content ) : <EOL> headers = { <EOL> "<STR_LIT>" : self . api_key , <EOL> } <EOL> headers . update ( self . _get_param_headers ( ) ) <EOL> conn = httplib . HTTPConnection ( "<STR_LIT>" ) <EOL> conn . request ( "<STR_LIT:POST>" , "<STR_LIT>" , content . encode ( '<STR_LIT:utf8>' ) , headers ) <EOL> response = conn . getresponse ( ) <EOL> data = response . read ( ) <EOL> conn . close ( ) <EOL> return data <EOL> def get_random_id ( self ) : <EOL> """<STR_LIT>""" <EOL> import string <EOL> from random import choice <EOL> chars = string . letters + string . digits <EOL> np = "<STR_LIT>" <EOL> for i in range ( <NUM_LIT:10> ) : <EOL> np = np + choice ( chars ) <EOL> return np <EOL> def get_content_id ( self , text ) : <EOL> """<STR_LIT>""" <EOL> import hashlib <EOL> h = hashlib . sha1 ( ) <EOL> h . update ( text ) <EOL> return h . hexdigest ( ) <EOL> def preprocess_html ( self , html ) : <EOL> html = html . replace ( '<STR_LIT:\n>' , '<STR_LIT>' ) <EOL> html = STRIP_RE . sub ( '<STR_LIT>' , html ) <EOL> return html <EOL> def analyze ( self , content , content_type = "<STR_LIT>" , external_id = None ) : <EOL> if not ( content and len ( content . strip ( ) ) ) : <EOL> return None <EOL> self . processing_directives [ "<STR_LIT>" ] = content_type <EOL> if external_id : <EOL> self . user_directives [ "<STR_LIT>" ] = external_id <EOL> return CalaisResponse ( self . rest_POST ( content ) ) <EOL> def analyze_url ( self , url ) : <EOL> f = urllib . urlopen ( url ) <EOL> html = self . preprocess_html ( f . read ( ) ) <EOL> return self . analyze ( html , content_type = "<STR_LIT>" , external_id = url ) <EOL> def analyze_file ( self , fn ) : <EOL> import mimetypes <EOL> try : <EOL> filetype = mimetypes . guess_type ( fn ) [ <NUM_LIT:0> ] <EOL> except : <EOL> raise ValueError ( "<STR_LIT>" % fn ) <EOL> if filetype == "<STR_LIT>" : <EOL> content_type = "<STR_LIT>" <EOL> f = open ( fn ) <EOL> content = f . read ( ) <EOL> f . close ( ) <EOL> elif filetype == "<STR_LIT>" : <EOL> content_type = "<STR_LIT>" <EOL> f = open ( fn ) <EOL> content = self . preprocess_html ( f . read ( ) ) <EOL> f . close ( ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> return self . analyze ( content , content_type = content_type , external_id = fn ) <EOL> class CalaisResponse ( ) : <EOL> """<STR_LIT>""" <EOL> raw_response = None <EOL> simplified_response = None <EOL> def __init__ ( self , raw_result ) : <EOL> try : <EOL> self . raw_response = json . load ( StringIO ( raw_result ) ) <EOL> except : <EOL> raise ValueError ( raw_result ) <EOL> self . simplified_response = self . _simplify_json ( self . raw_response ) <EOL> self . __dict__ [ '<STR_LIT>' ] = self . raw_response [ '<STR_LIT>' ] <EOL> for k , v in self . simplified_response . items ( ) : <EOL> self . __dict__ [ k ] = v <EOL> def _simplify_json ( self , json ) : <EOL> result = { } <EOL> for element in json . values ( ) : <EOL> for k , v in element . items ( ) : <EOL> if isinstance ( v , unicode ) and v . startswith ( "<STR_LIT>" ) and json . has_key ( v ) : <EOL> element [ k ] = json [ v ] <EOL> for k , v in json . items ( ) : <EOL> if v . has_key ( "<STR_LIT>" ) : <EOL> group = v [ "<STR_LIT>" ] <EOL> if not result . has_key ( group ) : <EOL> result [ group ] = [ ] <EOL> del v [ "<STR_LIT>" ] <EOL> v [ "<STR_LIT>" ] = k <EOL> result [ group ] . append ( v ) <EOL> return result <EOL> def print_summary ( self ) : <EOL> if not hasattr ( self , "<STR_LIT>" ) : <EOL> return None <EOL> info = self . doc [ '<STR_LIT:info>' ] <EOL> print "<STR_LIT>" % info [ '<STR_LIT>' ] <EOL> if info . has_key ( '<STR_LIT>' ) : <EOL> print "<STR_LIT>" % info [ '<STR_LIT>' ] <EOL> if info . has_key ( '<STR_LIT>' ) : <EOL> print "<STR_LIT>" % info [ '<STR_LIT>' ] <EOL> print "<STR_LIT>" % self . doc [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> print "<STR_LIT>" <EOL> for k , v in self . simplified_response . items ( ) : <EOL> print "<STR_LIT>" % ( len ( v ) , k ) <EOL> def print_entities ( self ) : <EOL> if not hasattr ( self , "<STR_LIT>" ) : <EOL> return None <EOL> for item in self . entities : <EOL> print "<STR_LIT>" % ( item [ '<STR_LIT>' ] , item [ '<STR_LIT:name>' ] , item [ '<STR_LIT>' ] ) <EOL> def print_topics ( self ) : <EOL> if not hasattr ( self , "<STR_LIT>" ) : <EOL> return None <EOL> for topic in self . topics : <EOL> print topic [ '<STR_LIT>' ] <EOL> def print_relations ( self ) : <EOL> if not hasattr ( self , "<STR_LIT>" ) : <EOL> return None <EOL> for relation in self . relations : <EOL> print relation [ '<STR_LIT>' ] <EOL> for k , v in relation . items ( ) : <EOL> if not k . startswith ( "<STR_LIT:_>" ) : <EOL> if isinstance ( v , unicode ) : <EOL> print "<STR_LIT>" % ( k , v ) <EOL> elif isinstance ( v , dict ) and v . has_key ( '<STR_LIT:name>' ) : <EOL> print "<STR_LIT>" % ( k , v [ '<STR_LIT:name>' ] ) </s>
