<s> import requests <EOL> from bs4 import BeautifulSoup <EOL> from HTMLParser import HTMLParseError <EOL> import json <EOL> import re <EOL> from urlparse import urlparse <EOL> from urlparse import urljoin <EOL> def get_canonical ( url ) : <EOL> canonical_url = "<STR_LIT>" <EOL> response = requests . get ( url ) <EOL> if response . status_code == <NUM_LIT:200> : <EOL> soup = BeautifulSoup ( response . text ) <EOL> canonical = soup . find ( "<STR_LIT>" , rel = "<STR_LIT>" ) <EOL> if canonical : <EOL> canonical_url = canonical [ '<STR_LIT>' ] <EOL> else : <EOL> og_url = soup . find ( "<STR_LIT>" , property = "<STR_LIT>" ) <EOL> canonical_url = og_url [ '<STR_LIT:content>' ] <EOL> return canonical_url <EOL> def get_meta ( url ) : <EOL> data = { } <EOL> data [ "<STR_LIT:title>" ] = "<STR_LIT>" <EOL> data [ "<STR_LIT:description>" ] = None <EOL> data [ "<STR_LIT>" ] = None <EOL> data [ "<STR_LIT>" ] = { } <EOL> data [ "<STR_LIT>" ] = { } <EOL> try : <EOL> response = requests . get ( url , timeout = <NUM_LIT:3> ) <EOL> if response . status_code == <NUM_LIT:200> : <EOL> soup = BeautifulSoup ( response . text ) <EOL> if soup . title . string : <EOL> data [ "<STR_LIT:title>" ] = soup . title . string <EOL> canonical = soup . find ( "<STR_LIT>" , rel = "<STR_LIT>" ) <EOL> if canonical : <EOL> data [ "<STR_LIT>" ] = canonical [ '<STR_LIT>' ] <EOL> parsed_uri = urlparse ( url ) <EOL> if soup . find ( "<STR_LIT>" , rel = "<STR_LIT>" ) : <EOL> icon_rel = soup . find ( "<STR_LIT>" , rel = "<STR_LIT>" ) [ "<STR_LIT>" ] <EOL> icon_abs = urljoin ( url , icon_rel ) <EOL> data [ "<STR_LIT>" ] = icon_abs <EOL> else : <EOL> domain = '<STR_LIT>' . format ( uri = parsed_uri ) <EOL> data [ "<STR_LIT>" ] = domain + '<STR_LIT>' <EOL> if soup . find ( '<STR_LIT>' , attrs = { '<STR_LIT:name>' : '<STR_LIT:description>' } ) : <EOL> data [ "<STR_LIT:description>" ] = soup . find ( '<STR_LIT>' , attrs = { '<STR_LIT:name>' : '<STR_LIT:description>' } ) [ "<STR_LIT:content>" ] <EOL> if soup . findAll ( '<STR_LIT>' , { "<STR_LIT>" : re . compile ( "<STR_LIT>" ) } ) : <EOL> for tag in soup . findAll ( '<STR_LIT>' , { "<STR_LIT>" : re . compile ( "<STR_LIT>" ) } ) : <EOL> tag_type = tag [ '<STR_LIT>' ] <EOL> data [ "<STR_LIT>" ] [ tag_type ] = tag [ '<STR_LIT:content>' ] <EOL> if tag_type == "<STR_LIT>" and data [ "<STR_LIT:description>" ] is None : <EOL> data [ "<STR_LIT:description>" ] = tag [ "<STR_LIT:content>" ] <EOL> if soup . findAll ( '<STR_LIT>' , attrs = { '<STR_LIT:name>' : re . compile ( "<STR_LIT>" ) } ) : <EOL> for tag in soup . findAll ( '<STR_LIT>' , attrs = { '<STR_LIT:name>' : re . compile ( "<STR_LIT>" ) } ) : <EOL> tag_type = tag [ '<STR_LIT:name>' ] <EOL> if '<STR_LIT:content>' in tag . attrs : <EOL> data [ "<STR_LIT>" ] [ tag_type ] = tag [ '<STR_LIT:content>' ] <EOL> if tag_type == "<STR_LIT>" and data [ "<STR_LIT:description>" ] is None : <EOL> data [ "<STR_LIT:description>" ] = tag [ "<STR_LIT:content>" ] <EOL> if not data [ '<STR_LIT>' ] or len ( data [ '<STR_LIT>' ] ) == <NUM_LIT:0> : <EOL> if data [ '<STR_LIT>' ] . has_key ( '<STR_LIT>' ) : <EOL> data [ '<STR_LIT>' ] = data [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> if not data [ '<STR_LIT>' ] or len ( data [ '<STR_LIT>' ] ) == <NUM_LIT:0> : <EOL> data [ '<STR_LIT>' ] = url <EOL> return json . dumps ( data ) <EOL> else : <EOL> return json . dumps ( { "<STR_LIT>" : url , "<STR_LIT:error>" : "<STR_LIT>" + str ( response . status_code ) } ) <EOL> except HTMLParseError : <EOL> return json . dumps ( { "<STR_LIT>" : url , "<STR_LIT:error>" : "<STR_LIT>" } ) </s>
