<s> """<STR_LIT>""" <EOL> import os <EOL> import re <EOL> import numbers <EOL> import collections <EOL> from distutils . version import LooseVersion <EOL> import numpy as np <EOL> from pandas . io . common import ( EmptyDataError , _is_url , urlopen , <EOL> parse_url , _validate_header_arg ) <EOL> from pandas . io . parsers import TextParser <EOL> from pandas . compat import ( lrange , lmap , u , string_types , iteritems , <EOL> raise_with_traceback , binary_type ) <EOL> from pandas . core import common as com <EOL> from pandas import Series <EOL> from pandas . core . common import AbstractMethodError <EOL> from pandas . formats . printing import pprint_thing <EOL> _IMPORTS = False <EOL> _HAS_BS4 = False <EOL> _HAS_LXML = False <EOL> _HAS_HTML5LIB = False <EOL> def _importers ( ) : <EOL> global _IMPORTS <EOL> if _IMPORTS : <EOL> return <EOL> _IMPORTS = True <EOL> global _HAS_BS4 , _HAS_LXML , _HAS_HTML5LIB <EOL> try : <EOL> import bs4 <EOL> _HAS_BS4 = True <EOL> except ImportError : <EOL> pass <EOL> try : <EOL> import lxml <EOL> _HAS_LXML = True <EOL> except ImportError : <EOL> pass <EOL> try : <EOL> import html5lib <EOL> _HAS_HTML5LIB = True <EOL> except ImportError : <EOL> pass <EOL> _RE_WHITESPACE = re . compile ( r'<STR_LIT>' ) <EOL> char_types = string_types + ( binary_type , ) <EOL> def _remove_whitespace ( s , regex = _RE_WHITESPACE ) : <EOL> """<STR_LIT>""" <EOL> return regex . sub ( '<STR_LIT:U+0020>' , s . strip ( ) ) <EOL> def _get_skiprows ( skiprows ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( skiprows , slice ) : <EOL> return lrange ( skiprows . start or <NUM_LIT:0> , skiprows . stop , skiprows . step or <NUM_LIT:1> ) <EOL> elif isinstance ( skiprows , numbers . Integral ) or com . is_list_like ( skiprows ) : <EOL> return skiprows <EOL> elif skiprows is None : <EOL> return <NUM_LIT:0> <EOL> raise TypeError ( '<STR_LIT>' % <EOL> type ( skiprows ) . __name__ ) <EOL> def _read ( obj ) : <EOL> """<STR_LIT>""" <EOL> if _is_url ( obj ) : <EOL> with urlopen ( obj ) as url : <EOL> text = url . read ( ) <EOL> elif hasattr ( obj , '<STR_LIT>' ) : <EOL> text = obj . read ( ) <EOL> elif isinstance ( obj , char_types ) : <EOL> text = obj <EOL> try : <EOL> if os . path . isfile ( text ) : <EOL> with open ( text , '<STR_LIT:rb>' ) as f : <EOL> return f . read ( ) <EOL> except ( TypeError , ValueError ) : <EOL> pass <EOL> else : <EOL> raise TypeError ( "<STR_LIT>" % type ( obj ) . __name__ ) <EOL> return text <EOL> class _HtmlFrameParser ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , io , match , attrs , encoding ) : <EOL> self . io = io <EOL> self . match = match <EOL> self . attrs = attrs <EOL> self . encoding = encoding <EOL> def parse_tables ( self ) : <EOL> tables = self . _parse_tables ( self . _build_doc ( ) , self . match , self . attrs ) <EOL> return ( self . _build_table ( table ) for table in tables ) <EOL> def _parse_raw_data ( self , rows ) : <EOL> """<STR_LIT>""" <EOL> data = [ [ _remove_whitespace ( self . _text_getter ( col ) ) for col in <EOL> self . _parse_td ( row ) ] for row in rows ] <EOL> return data <EOL> def _text_getter ( self , obj ) : <EOL> """<STR_LIT>""" <EOL> raise AbstractMethodError ( self ) <EOL> def _parse_td ( self , obj ) : <EOL> """<STR_LIT>""" <EOL> raise AbstractMethodError ( self ) <EOL> def _parse_tables ( self , doc , match , attrs ) : <EOL> """<STR_LIT>""" <EOL> raise AbstractMethodError ( self ) <EOL> def _parse_tr ( self , table ) : <EOL> """<STR_LIT>""" <EOL> raise AbstractMethodError ( self ) <EOL> def _parse_thead ( self , table ) : <EOL> """<STR_LIT>""" <EOL> raise AbstractMethodError ( self ) <EOL> def _parse_tbody ( self , table ) : <EOL> """<STR_LIT>""" <EOL> raise AbstractMethodError ( self ) <EOL> def _parse_tfoot ( self , table ) : <EOL> """<STR_LIT>""" <EOL> raise AbstractMethodError ( self ) <EOL> def _build_doc ( self ) : <EOL> """<STR_LIT>""" <EOL> raise AbstractMethodError ( self ) <EOL> def _build_table ( self , table ) : <EOL> header = self . _parse_raw_thead ( table ) <EOL> body = self . _parse_raw_tbody ( table ) <EOL> footer = self . _parse_raw_tfoot ( table ) <EOL> return header , body , footer <EOL> def _parse_raw_thead ( self , table ) : <EOL> thead = self . _parse_thead ( table ) <EOL> res = [ ] <EOL> if thead : <EOL> res = lmap ( self . _text_getter , self . _parse_th ( thead [ <NUM_LIT:0> ] ) ) <EOL> return np . atleast_1d ( <EOL> np . array ( res ) . squeeze ( ) ) if res and len ( res ) == <NUM_LIT:1> else res <EOL> def _parse_raw_tfoot ( self , table ) : <EOL> tfoot = self . _parse_tfoot ( table ) <EOL> res = [ ] <EOL> if tfoot : <EOL> res = lmap ( self . _text_getter , self . _parse_td ( tfoot [ <NUM_LIT:0> ] ) ) <EOL> return np . atleast_1d ( <EOL> np . array ( res ) . squeeze ( ) ) if res and len ( res ) == <NUM_LIT:1> else res <EOL> def _parse_raw_tbody ( self , table ) : <EOL> tbody = self . _parse_tbody ( table ) <EOL> try : <EOL> res = self . _parse_tr ( tbody [ <NUM_LIT:0> ] ) <EOL> except IndexError : <EOL> res = self . _parse_tr ( table ) <EOL> return self . _parse_raw_data ( res ) <EOL> class _BeautifulSoupHtml5LibFrameParser ( _HtmlFrameParser ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , * args , ** kwargs ) : <EOL> super ( _BeautifulSoupHtml5LibFrameParser , self ) . __init__ ( * args , <EOL> ** kwargs ) <EOL> from bs4 import SoupStrainer <EOL> self . _strainer = SoupStrainer ( '<STR_LIT>' ) <EOL> def _text_getter ( self , obj ) : <EOL> return obj . text <EOL> def _parse_td ( self , row ) : <EOL> return row . find_all ( ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> def _parse_tr ( self , element ) : <EOL> return element . find_all ( '<STR_LIT>' ) <EOL> def _parse_th ( self , element ) : <EOL> return element . find_all ( '<STR_LIT>' ) <EOL> def _parse_thead ( self , table ) : <EOL> return table . find_all ( '<STR_LIT>' ) <EOL> def _parse_tbody ( self , table ) : <EOL> return table . find_all ( '<STR_LIT>' ) <EOL> def _parse_tfoot ( self , table ) : <EOL> return table . find_all ( '<STR_LIT>' ) <EOL> def _parse_tables ( self , doc , match , attrs ) : <EOL> element_name = self . _strainer . name <EOL> tables = doc . find_all ( element_name , attrs = attrs ) <EOL> if not tables : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> result = [ ] <EOL> unique_tables = set ( ) <EOL> for table in tables : <EOL> if ( table not in unique_tables and <EOL> table . find ( text = match ) is not None ) : <EOL> result . append ( table ) <EOL> unique_tables . add ( table ) <EOL> if not result : <EOL> raise ValueError ( "<STR_LIT>" % <EOL> match . pattern ) <EOL> return result <EOL> def _setup_build_doc ( self ) : <EOL> raw_text = _read ( self . io ) <EOL> if not raw_text : <EOL> raise ValueError ( '<STR_LIT>' % self . io ) <EOL> return raw_text <EOL> def _build_doc ( self ) : <EOL> from bs4 import BeautifulSoup <EOL> return BeautifulSoup ( self . _setup_build_doc ( ) , features = '<STR_LIT>' , <EOL> from_encoding = self . encoding ) <EOL> def _build_xpath_expr ( attrs ) : <EOL> """<STR_LIT>""" <EOL> if '<STR_LIT>' in attrs : <EOL> attrs [ '<STR_LIT:class>' ] = attrs . pop ( '<STR_LIT>' ) <EOL> s = [ u ( "<STR_LIT>" ) % ( k , v ) for k , v in iteritems ( attrs ) ] <EOL> return u ( '<STR_LIT>' ) % '<STR_LIT>' . join ( s ) <EOL> _re_namespace = { '<STR_LIT>' : '<STR_LIT>' } <EOL> _valid_schemes = '<STR_LIT:http>' , '<STR_LIT:file>' , '<STR_LIT>' <EOL> class _LxmlFrameParser ( _HtmlFrameParser ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , * args , ** kwargs ) : <EOL> super ( _LxmlFrameParser , self ) . __init__ ( * args , ** kwargs ) <EOL> def _text_getter ( self , obj ) : <EOL> return obj . text_content ( ) <EOL> def _parse_td ( self , row ) : <EOL> return row . xpath ( '<STR_LIT>' ) <EOL> def _parse_tr ( self , table ) : <EOL> expr = '<STR_LIT>' <EOL> return table . xpath ( expr ) <EOL> def _parse_tables ( self , doc , match , kwargs ) : <EOL> pattern = match . pattern <EOL> query = '<STR_LIT>' <EOL> xpath_expr = u ( query ) % pattern <EOL> if kwargs : <EOL> xpath_expr += _build_xpath_expr ( kwargs ) <EOL> tables = doc . xpath ( xpath_expr , namespaces = _re_namespace ) <EOL> if not tables : <EOL> raise ValueError ( "<STR_LIT>" % pattern ) <EOL> return tables <EOL> def _build_doc ( self ) : <EOL> """<STR_LIT>""" <EOL> from lxml . html import parse , fromstring , HTMLParser <EOL> from lxml . etree import XMLSyntaxError <EOL> parser = HTMLParser ( recover = False , encoding = self . encoding ) <EOL> try : <EOL> r = parse ( self . io , parser = parser ) <EOL> try : <EOL> r = r . getroot ( ) <EOL> except AttributeError : <EOL> pass <EOL> except ( UnicodeDecodeError , IOError ) : <EOL> if not _is_url ( self . io ) : <EOL> r = fromstring ( self . io , parser = parser ) <EOL> try : <EOL> r = r . getroot ( ) <EOL> except AttributeError : <EOL> pass <EOL> else : <EOL> scheme = parse_url ( self . io ) . scheme <EOL> if scheme not in _valid_schemes : <EOL> msg = ( '<STR_LIT>' <EOL> '<STR_LIT:%s>' ) % ( scheme , _valid_schemes ) <EOL> raise ValueError ( msg ) <EOL> else : <EOL> raise <EOL> else : <EOL> if not hasattr ( r , '<STR_LIT>' ) : <EOL> raise XMLSyntaxError ( "<STR_LIT>" , <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> ) <EOL> return r <EOL> def _parse_tbody ( self , table ) : <EOL> return table . xpath ( '<STR_LIT>' ) <EOL> def _parse_thead ( self , table ) : <EOL> return table . xpath ( '<STR_LIT>' ) <EOL> def _parse_tfoot ( self , table ) : <EOL> return table . xpath ( '<STR_LIT>' ) <EOL> def _parse_raw_thead ( self , table ) : <EOL> expr = '<STR_LIT>' <EOL> return [ _remove_whitespace ( x . text_content ( ) ) for x in <EOL> table . xpath ( expr ) ] <EOL> def _parse_raw_tfoot ( self , table ) : <EOL> expr = '<STR_LIT>' <EOL> return [ _remove_whitespace ( x . text_content ( ) ) for x in <EOL> table . xpath ( expr ) ] <EOL> def _expand_elements ( body ) : <EOL> lens = Series ( lmap ( len , body ) ) <EOL> lens_max = lens . max ( ) <EOL> not_max = lens [ lens != lens_max ] <EOL> empty = [ '<STR_LIT>' ] <EOL> for ind , length in iteritems ( not_max ) : <EOL> body [ ind ] += empty * ( lens_max - length ) <EOL> def _data_to_frame ( data , header , index_col , skiprows , <EOL> parse_dates , tupleize_cols , thousands ) : <EOL> head , body , foot = data <EOL> if head : <EOL> body = [ head ] + body <EOL> if header is None : <EOL> header = <NUM_LIT:0> <EOL> if foot : <EOL> body += [ foot ] <EOL> _expand_elements ( body ) <EOL> tp = TextParser ( body , header = header , index_col = index_col , <EOL> skiprows = _get_skiprows ( skiprows ) , <EOL> parse_dates = parse_dates , tupleize_cols = tupleize_cols , <EOL> thousands = thousands ) <EOL> df = tp . read ( ) <EOL> return df <EOL> _valid_parsers = { '<STR_LIT>' : _LxmlFrameParser , None : _LxmlFrameParser , <EOL> '<STR_LIT>' : _BeautifulSoupHtml5LibFrameParser , <EOL> '<STR_LIT>' : _BeautifulSoupHtml5LibFrameParser } <EOL> def _parser_dispatch ( flavor ) : <EOL> """<STR_LIT>""" <EOL> valid_parsers = list ( _valid_parsers . keys ( ) ) <EOL> if flavor not in valid_parsers : <EOL> raise ValueError ( '<STR_LIT>' % <EOL> ( flavor , valid_parsers ) ) <EOL> if flavor in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> if not _HAS_HTML5LIB : <EOL> raise ImportError ( "<STR_LIT>" ) <EOL> if not _HAS_BS4 : <EOL> raise ImportError ( <EOL> "<STR_LIT>" ) <EOL> import bs4 <EOL> if bs4 . __version__ == LooseVersion ( '<STR_LIT>' ) : <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> else : <EOL> if not _HAS_LXML : <EOL> raise ImportError ( "<STR_LIT>" ) <EOL> return _valid_parsers [ flavor ] <EOL> def _print_as_set ( s ) : <EOL> return '<STR_LIT>' % '<STR_LIT:U+002CU+0020>' . join ( [ pprint_thing ( el ) for el in s ] ) <EOL> def _validate_flavor ( flavor ) : <EOL> if flavor is None : <EOL> flavor = '<STR_LIT>' , '<STR_LIT>' <EOL> elif isinstance ( flavor , string_types ) : <EOL> flavor = flavor , <EOL> elif isinstance ( flavor , collections . Iterable ) : <EOL> if not all ( isinstance ( flav , string_types ) for flav in flavor ) : <EOL> raise TypeError ( '<STR_LIT>' % <EOL> type ( flavor ) . __name__ ) <EOL> else : <EOL> fmt = '<STR_LIT>' if isinstance ( flavor , string_types ) else '<STR_LIT>' <EOL> fmt += '<STR_LIT>' <EOL> raise ValueError ( fmt . format ( flavor ) ) <EOL> flavor = tuple ( flavor ) <EOL> valid_flavors = set ( _valid_parsers ) <EOL> flavor_set = set ( flavor ) <EOL> if not flavor_set & valid_flavors : <EOL> raise ValueError ( '<STR_LIT>' <EOL> '<STR_LIT:%s>' % ( _print_as_set ( flavor_set ) , <EOL> _print_as_set ( valid_flavors ) ) ) <EOL> return flavor <EOL> def _parse ( flavor , io , match , header , index_col , skiprows , <EOL> parse_dates , tupleize_cols , thousands , attrs , encoding ) : <EOL> flavor = _validate_flavor ( flavor ) <EOL> compiled_match = re . compile ( match ) <EOL> retained = None <EOL> for flav in flavor : <EOL> parser = _parser_dispatch ( flav ) <EOL> p = parser ( io , compiled_match , attrs , encoding ) <EOL> try : <EOL> tables = p . parse_tables ( ) <EOL> except Exception as caught : <EOL> retained = caught <EOL> else : <EOL> break <EOL> else : <EOL> raise_with_traceback ( retained ) <EOL> ret = [ ] <EOL> for table in tables : <EOL> try : <EOL> ret . append ( _data_to_frame ( data = table , <EOL> header = header , <EOL> index_col = index_col , <EOL> skiprows = skiprows , <EOL> parse_dates = parse_dates , <EOL> tupleize_cols = tupleize_cols , <EOL> thousands = thousands ) ) <EOL> except EmptyDataError : <EOL> continue <EOL> return ret <EOL> def read_html ( io , match = '<STR_LIT>' , flavor = None , header = None , index_col = None , <EOL> skiprows = None , attrs = None , parse_dates = False , <EOL> tupleize_cols = False , thousands = '<STR_LIT:U+002C>' , encoding = None ) : <EOL> r"""<STR_LIT>""" <EOL> _importers ( ) <EOL> if isinstance ( skiprows , numbers . Integral ) and skiprows < <NUM_LIT:0> : <EOL> raise ValueError ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> _validate_header_arg ( header ) <EOL> return _parse ( flavor , io , match , header , index_col , skiprows , <EOL> parse_dates , tupleize_cols , thousands , attrs , encoding ) </s>
