<s> '''<STR_LIT>''' <EOL> from __future__ import absolute_import , division , print_function <EOL> from itertools import imap <EOL> import logging <EOL> import regex as re <EOL> import traceback <EOL> from bs4 import BeautifulSoup <EOL> import urlnorm <EOL> from urlparse import urlparse <EOL> from dossier . fc import StringCounter <EOL> logger = logging . getLogger ( __name__ ) <EOL> REGEX_PHONES = [ <EOL> r'<STR_LIT>' , <EOL> r'<STR_LIT>' , <EOL> r'<STR_LIT>' , <EOL> r'<STR_LIT>' , <EOL> r'<STR_LIT>' , <EOL> ] <EOL> REGEX_PHONE = re . compile ( '<STR_LIT:|>' . join ( REGEX_PHONES ) ) <EOL> REGEX_EMAIL = re . compile ( r'<STR_LIT>' , re . IGNORECASE ) <EOL> REGEX_SKYPE = re . compile ( ur '<STR_LIT>' <EOL> ur '<STR_LIT>' , <EOL> flags = re . UNICODE | re . IGNORECASE ) <EOL> def phones ( text ) : <EOL> '''<STR_LIT>''' <EOL> return imap ( lambda m : m . group ( <NUM_LIT:0> ) . replace ( '<STR_LIT:->' , '<STR_LIT>' ) , <EOL> REGEX_PHONE . finditer ( text ) ) <EOL> def emails ( text ) : <EOL> '''<STR_LIT>''' <EOL> return imap ( lambda m : m . group ( <NUM_LIT:0> ) . lower ( ) , REGEX_EMAIL . finditer ( text ) ) <EOL> def skypes ( text ) : <EOL> '''<STR_LIT>''' <EOL> return imap ( lambda m : m . group ( '<STR_LIT>' ) , REGEX_SKYPE . finditer ( text ) ) <EOL> def image_urls ( html ) : <EOL> soup = BeautifulSoup ( html , "<STR_LIT>" ) <EOL> for node in soup . find_all ( '<STR_LIT>' ) : <EOL> try : <EOL> src = node [ '<STR_LIT:src>' ] <EOL> except KeyError : <EOL> continue <EOL> yield norm_url ( src ) <EOL> def a_urls ( html ) : <EOL> '''<STR_LIT>''' <EOL> soup = BeautifulSoup ( html , '<STR_LIT>' ) <EOL> for node in soup . find_all ( '<STR_LIT:a>' ) : <EOL> try : <EOL> href = node [ '<STR_LIT>' ] <EOL> except KeyError : <EOL> continue <EOL> yield norm_url ( href ) <EOL> def host_names ( urls ) : <EOL> '''<STR_LIT>''' <EOL> host_names = StringCounter ( ) <EOL> for url in urls : <EOL> host_names [ urlparse ( url ) . netloc ] += urls [ url ] <EOL> return host_names <EOL> def path_dirs ( urls ) : <EOL> '''<STR_LIT>''' <EOL> path_dirs = StringCounter ( ) <EOL> for url in urls : <EOL> for path_dir in filter ( None , urlparse ( url ) . path . split ( '<STR_LIT:/>' ) ) : <EOL> path_dirs [ path_dir ] += urls [ url ] <EOL> return path_dirs <EOL> def norm_url ( url ) : <EOL> url = uni ( url ) . encode ( '<STR_LIT:utf-8>' ) <EOL> try : <EOL> return urlnorm . norm ( url ) <EOL> except urlnorm . InvalidUrl : <EOL> try : <EOL> return urlnorm . norm_path ( '<STR_LIT>' , url ) <EOL> except UnicodeDecodeError : <EOL> return url <EOL> except UnicodeDecodeError : <EOL> return url <EOL> except : <EOL> traceback . print_exc ( ) <EOL> return None <EOL> def uni ( s ) : <EOL> if not isinstance ( s , unicode ) : <EOL> try : <EOL> return s . decode ( '<STR_LIT:utf-8>' ) <EOL> except UnicodeDecodeError : <EOL> return s . decode ( '<STR_LIT>' ) <EOL> return s <EOL> REGEX_ICQ = re . compile ( ur '<STR_LIT>' , <EOL> flags = re . UNICODE | re . IGNORECASE ) <EOL> REGEX_DIGITS = re . compile ( r'<STR_LIT>' , flags = re . MULTILINE ) <EOL> def ICQs ( text ) : <EOL> icqs = [ ] <EOL> for match in REGEX_ICQ . finditer ( text ) : <EOL> sub = match . span ( '<STR_LIT>' ) <EOL> raw_string = text [ sub [ <NUM_LIT:0> ] : sub [ <NUM_LIT:1> ] ] <EOL> icq = digits_re . sub ( '<STR_LIT>' , raw_string ) <EOL> if icq : <EOL> try : <EOL> icq = int ( icq ) <EOL> except : <EOL> continue <EOL> icqs . append ( str ( icq ) ) <EOL> return icqs </s>
