<s> """<STR_LIT>""" <EOL> import calendar <EOL> import logging <EOL> import re <EOL> import time <EOL> from datetime import datetime <EOL> from functools import wraps <EOL> from io import BytesIO <EOL> from mrjob . py2 import ParseResult <EOL> from mrjob . py2 import to_string <EOL> from mrjob . py2 import urlparse as urlparse_buggy <EOL> try : <EOL> import boto . utils <EOL> except ImportError : <EOL> boto = None <EOL> log = logging . getLogger ( __name__ ) <EOL> _NETLOC_RE = re . compile ( r'<STR_LIT>' ) <EOL> _WINPATH_RE = re . compile ( r"<STR_LIT>" ) <EOL> def is_windows_path ( uri ) : <EOL> """<STR_LIT>""" <EOL> if _WINPATH_RE . match ( uri ) : <EOL> return True <EOL> else : <EOL> return False <EOL> def is_uri ( uri ) : <EOL> """<STR_LIT>""" <EOL> if is_windows_path ( uri ) : <EOL> return False <EOL> return bool ( urlparse ( uri ) . scheme ) <EOL> def is_s3_uri ( uri ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> parse_s3_uri ( uri ) <EOL> return True <EOL> except ValueError : <EOL> return False <EOL> def parse_s3_uri ( uri ) : <EOL> """<STR_LIT>""" <EOL> components = urlparse ( uri ) <EOL> if ( components . scheme not in ( '<STR_LIT>' , '<STR_LIT>' ) or <EOL> '<STR_LIT:/>' not in components . path ) : <EOL> raise ValueError ( '<STR_LIT>' % uri ) <EOL> return components . netloc , components . path [ <NUM_LIT:1> : ] <EOL> @ wraps ( urlparse_buggy ) <EOL> def urlparse ( urlstring , scheme = '<STR_LIT>' , allow_fragments = True , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> ( scheme , netloc , path , params , query , fragment ) = ( <EOL> urlparse_buggy ( urlstring , scheme , allow_fragments , * args , ** kwargs ) ) <EOL> if netloc == '<STR_LIT>' and path . startswith ( '<STR_LIT>' ) : <EOL> m = _NETLOC_RE . match ( path ) <EOL> netloc = m . group ( <NUM_LIT:1> ) <EOL> path = m . group ( <NUM_LIT:2> ) <EOL> if allow_fragments and '<STR_LIT:#>' in path and not fragment : <EOL> path , fragment = path . split ( '<STR_LIT:#>' , <NUM_LIT:1> ) <EOL> return ParseResult ( scheme , netloc , path , params , query , fragment ) <EOL> def parse_port_range_list ( range_list_str ) : <EOL> """<STR_LIT>""" <EOL> all_ranges = [ ] <EOL> for range_str in range_list_str . split ( '<STR_LIT:U+002C>' ) : <EOL> if '<STR_LIT::>' in range_str : <EOL> a , b = [ int ( x ) for x in range_str . split ( '<STR_LIT::>' ) ] <EOL> all_ranges . extend ( range ( a , b + <NUM_LIT:1> ) ) <EOL> else : <EOL> all_ranges . append ( int ( range_str ) ) <EOL> return all_ranges <EOL> def parse_key_value_list ( kv_string_list , error_fmt , error_func ) : <EOL> """<STR_LIT>""" <EOL> ret = { } <EOL> for value in kv_string_list : <EOL> try : <EOL> k , v = value . split ( '<STR_LIT:=>' , <NUM_LIT:1> ) <EOL> ret [ k ] = v <EOL> except ValueError : <EOL> error_func ( error_fmt % ( value , ) ) <EOL> return ret <EOL> _COUNTER_RE = re . compile ( br'<STR_LIT>' ) <EOL> _STATUS_RE = re . compile ( br'<STR_LIT>' ) <EOL> def parse_mr_job_stderr ( stderr , counters = None ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( stderr , bytes ) : <EOL> stderr = BytesIO ( stderr ) <EOL> if counters is None : <EOL> counters = { } <EOL> statuses = [ ] <EOL> other = [ ] <EOL> for line in stderr : <EOL> m = _COUNTER_RE . match ( line . rstrip ( b'<STR_LIT:\r\n>' ) ) <EOL> if m : <EOL> group , counter , amount_str = m . groups ( ) <EOL> group = to_string ( group ) <EOL> counter = to_string ( counter ) <EOL> counters . setdefault ( group , { } ) <EOL> counters [ group ] . setdefault ( counter , <NUM_LIT:0> ) <EOL> counters [ group ] [ counter ] += int ( amount_str ) <EOL> continue <EOL> m = _STATUS_RE . match ( line . rstrip ( b'<STR_LIT:\r\n>' ) ) <EOL> if m : <EOL> statuses . append ( to_string ( m . group ( <NUM_LIT:1> ) ) ) <EOL> continue <EOL> other . append ( to_string ( line ) ) <EOL> return { '<STR_LIT>' : counters , '<STR_LIT>' : statuses , '<STR_LIT>' : other } <EOL> def _find_python_traceback ( lines ) : <EOL> """<STR_LIT>""" <EOL> all_tb_lines = [ ] <EOL> tb_lines = [ ] <EOL> non_tb_lines = [ ] <EOL> in_traceback = False <EOL> for line in lines : <EOL> line = to_string ( line ) <EOL> if in_traceback : <EOL> tb_lines . append ( line ) <EOL> if line . lstrip ( ) == line : <EOL> in_traceback = False <EOL> if line . startswith ( '<STR_LIT>' ) : <EOL> all_tb_lines += non_tb_lines <EOL> all_tb_lines += tb_lines <EOL> tb_lines = [ ] <EOL> non_tb_lines = [ ] <EOL> else : <EOL> if line . startswith ( '<STR_LIT>' ) : <EOL> tb_lines . append ( line ) <EOL> in_traceback = True <EOL> else : <EOL> non_tb_lines . append ( line ) <EOL> if all_tb_lines : <EOL> return all_tb_lines <EOL> else : <EOL> return None <EOL> _JOB_TRACKER_HTML_RE = re . compile ( br'<STR_LIT>' ) <EOL> _RESOURCE_MANAGER_JS_RE = re . compile ( <EOL> br'<STR_LIT>' ) <EOL> def _parse_progress_from_job_tracker ( html_bytes ) : <EOL> """<STR_LIT>""" <EOL> matches = _JOB_TRACKER_HTML_RE . findall ( html_bytes ) <EOL> if len ( matches ) >= <NUM_LIT:2> : <EOL> return float ( matches [ <NUM_LIT:0> ] ) , float ( matches [ <NUM_LIT:1> ] ) <EOL> else : <EOL> return None , None <EOL> def _parse_progress_from_resource_manager ( html_bytes ) : <EOL> """<STR_LIT>""" <EOL> app_id_percent_tuples = [ ] <EOL> for line in html_bytes . splitlines ( ) : <EOL> m = _RESOURCE_MANAGER_JS_RE . match ( line ) <EOL> if m : <EOL> app_id_percent_tuples . append ( ( m . group ( <NUM_LIT:1> ) , float ( m . group ( <NUM_LIT:2> ) ) ) ) <EOL> if app_id_percent_tuples : <EOL> return sorted ( app_id_percent_tuples ) [ - <NUM_LIT:1> ] [ <NUM_LIT:1> ] <EOL> else : <EOL> return None <EOL> _SUBSECOND_RE = re . compile ( '<STR_LIT>' ) <EOL> _RFC1123 = '<STR_LIT>' <EOL> def iso8601_to_timestamp ( iso8601_time ) : <EOL> iso8601_time = _SUBSECOND_RE . sub ( '<STR_LIT>' , iso8601_time ) <EOL> try : <EOL> return calendar . timegm ( time . strptime ( iso8601_time , boto . utils . ISO8601 ) ) <EOL> except ValueError : <EOL> return calendar . timegm ( time . strptime ( iso8601_time , _RFC1123 ) ) <EOL> def iso8601_to_datetime ( iso8601_time ) : <EOL> iso8601_time = _SUBSECOND_RE . sub ( '<STR_LIT>' , iso8601_time ) <EOL> try : <EOL> return datetime . strptime ( iso8601_time , boto . utils . ISO8601 ) <EOL> except ValueError : <EOL> return datetime . strptime ( iso8601_time , _RFC1123 ) </s>
