<s> from __future__ import unicode_literals <EOL> import base64 <EOL> import io <EOL> import itertools <EOL> import os <EOL> import time <EOL> import xml . etree . ElementTree as etree <EOL> from . common import FileDownloader <EOL> from . http import HttpFD <EOL> from . . compat import ( <EOL> compat_urlparse , <EOL> ) <EOL> from . . utils import ( <EOL> struct_pack , <EOL> struct_unpack , <EOL> format_bytes , <EOL> encodeFilename , <EOL> sanitize_open , <EOL> xpath_text , <EOL> ) <EOL> class FlvReader ( io . BytesIO ) : <EOL> """<STR_LIT>""" <EOL> def read_unsigned_long_long ( self ) : <EOL> return struct_unpack ( '<STR_LIT>' , self . read ( <NUM_LIT:8> ) ) [ <NUM_LIT:0> ] <EOL> def read_unsigned_int ( self ) : <EOL> return struct_unpack ( '<STR_LIT>' , self . read ( <NUM_LIT:4> ) ) [ <NUM_LIT:0> ] <EOL> def read_unsigned_char ( self ) : <EOL> return struct_unpack ( '<STR_LIT>' , self . read ( <NUM_LIT:1> ) ) [ <NUM_LIT:0> ] <EOL> def read_string ( self ) : <EOL> res = b'<STR_LIT>' <EOL> while True : <EOL> char = self . read ( <NUM_LIT:1> ) <EOL> if char == b'<STR_LIT:\x00>' : <EOL> break <EOL> res += char <EOL> return res <EOL> def read_box_info ( self ) : <EOL> """<STR_LIT>""" <EOL> real_size = size = self . read_unsigned_int ( ) <EOL> box_type = self . read ( <NUM_LIT:4> ) <EOL> header_end = <NUM_LIT:8> <EOL> if size == <NUM_LIT:1> : <EOL> real_size = self . read_unsigned_long_long ( ) <EOL> header_end = <NUM_LIT:16> <EOL> return real_size , box_type , self . read ( real_size - header_end ) <EOL> def read_asrt ( self ) : <EOL> self . read_unsigned_char ( ) <EOL> self . read ( <NUM_LIT:3> ) <EOL> quality_entry_count = self . read_unsigned_char ( ) <EOL> for i in range ( quality_entry_count ) : <EOL> self . read_string ( ) <EOL> segment_run_count = self . read_unsigned_int ( ) <EOL> segments = [ ] <EOL> for i in range ( segment_run_count ) : <EOL> first_segment = self . read_unsigned_int ( ) <EOL> fragments_per_segment = self . read_unsigned_int ( ) <EOL> segments . append ( ( first_segment , fragments_per_segment ) ) <EOL> return { <EOL> '<STR_LIT>' : segments , <EOL> } <EOL> def read_afrt ( self ) : <EOL> self . read_unsigned_char ( ) <EOL> self . read ( <NUM_LIT:3> ) <EOL> self . read_unsigned_int ( ) <EOL> quality_entry_count = self . read_unsigned_char ( ) <EOL> for i in range ( quality_entry_count ) : <EOL> self . read_string ( ) <EOL> fragments_count = self . read_unsigned_int ( ) <EOL> fragments = [ ] <EOL> for i in range ( fragments_count ) : <EOL> first = self . read_unsigned_int ( ) <EOL> first_ts = self . read_unsigned_long_long ( ) <EOL> duration = self . read_unsigned_int ( ) <EOL> if duration == <NUM_LIT:0> : <EOL> discontinuity_indicator = self . read_unsigned_char ( ) <EOL> else : <EOL> discontinuity_indicator = None <EOL> fragments . append ( { <EOL> '<STR_LIT>' : first , <EOL> '<STR_LIT>' : first_ts , <EOL> '<STR_LIT>' : duration , <EOL> '<STR_LIT>' : discontinuity_indicator , <EOL> } ) <EOL> return { <EOL> '<STR_LIT>' : fragments , <EOL> } <EOL> def read_abst ( self ) : <EOL> self . read_unsigned_char ( ) <EOL> self . read ( <NUM_LIT:3> ) <EOL> self . read_unsigned_int ( ) <EOL> self . read ( <NUM_LIT:1> ) <EOL> self . read_unsigned_int ( ) <EOL> self . read_unsigned_long_long ( ) <EOL> self . read_unsigned_long_long ( ) <EOL> self . read_string ( ) <EOL> server_count = self . read_unsigned_char ( ) <EOL> for i in range ( server_count ) : <EOL> self . read_string ( ) <EOL> quality_count = self . read_unsigned_char ( ) <EOL> for i in range ( quality_count ) : <EOL> self . read_string ( ) <EOL> self . read_string ( ) <EOL> self . read_string ( ) <EOL> segments_count = self . read_unsigned_char ( ) <EOL> segments = [ ] <EOL> for i in range ( segments_count ) : <EOL> box_size , box_type , box_data = self . read_box_info ( ) <EOL> assert box_type == b'<STR_LIT>' <EOL> segment = FlvReader ( box_data ) . read_asrt ( ) <EOL> segments . append ( segment ) <EOL> fragments_run_count = self . read_unsigned_char ( ) <EOL> fragments = [ ] <EOL> for i in range ( fragments_run_count ) : <EOL> box_size , box_type , box_data = self . read_box_info ( ) <EOL> assert box_type == b'<STR_LIT>' <EOL> fragments . append ( FlvReader ( box_data ) . read_afrt ( ) ) <EOL> return { <EOL> '<STR_LIT>' : segments , <EOL> '<STR_LIT>' : fragments , <EOL> } <EOL> def read_bootstrap_info ( self ) : <EOL> total_size , box_type , box_data = self . read_box_info ( ) <EOL> assert box_type == b'<STR_LIT>' <EOL> return FlvReader ( box_data ) . read_abst ( ) <EOL> def read_bootstrap_info ( bootstrap_bytes ) : <EOL> return FlvReader ( bootstrap_bytes ) . read_bootstrap_info ( ) <EOL> def build_fragments_list ( boot_info ) : <EOL> """<STR_LIT>""" <EOL> res = [ ] <EOL> segment_run_table = boot_info [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> segment_run_entry = segment_run_table [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> n_frags = segment_run_entry [ <NUM_LIT:1> ] <EOL> fragment_run_entry_table = boot_info [ '<STR_LIT>' ] [ <NUM_LIT:0> ] [ '<STR_LIT>' ] <EOL> first_frag_number = fragment_run_entry_table [ <NUM_LIT:0> ] [ '<STR_LIT>' ] <EOL> for ( i , frag_number ) in zip ( range ( <NUM_LIT:1> , n_frags + <NUM_LIT:1> ) , itertools . count ( first_frag_number ) ) : <EOL> res . append ( ( <NUM_LIT:1> , frag_number ) ) <EOL> return res <EOL> def write_unsigned_int ( stream , val ) : <EOL> stream . write ( struct_pack ( '<STR_LIT>' , val ) ) <EOL> def write_unsigned_int_24 ( stream , val ) : <EOL> stream . write ( struct_pack ( '<STR_LIT>' , val ) [ <NUM_LIT:1> : ] ) <EOL> def write_flv_header ( stream ) : <EOL> """<STR_LIT>""" <EOL> stream . write ( b'<STR_LIT>' ) <EOL> stream . write ( b'<STR_LIT>' ) <EOL> stream . write ( b'<STR_LIT>' ) <EOL> stream . write ( b'<STR_LIT>' ) <EOL> def write_metadata_tag ( stream , metadata ) : <EOL> """<STR_LIT>""" <EOL> SCRIPT_TAG = b'<STR_LIT>' <EOL> FLV_TAG_HEADER_LEN = <NUM_LIT:11> <EOL> if metadata : <EOL> stream . write ( SCRIPT_TAG ) <EOL> write_unsigned_int_24 ( stream , len ( metadata ) ) <EOL> stream . write ( b'<STR_LIT>' ) <EOL> stream . write ( metadata ) <EOL> write_unsigned_int ( stream , FLV_TAG_HEADER_LEN + len ( metadata ) ) <EOL> def _add_ns ( prop ) : <EOL> return '<STR_LIT>' % prop <EOL> class HttpQuietDownloader ( HttpFD ) : <EOL> def to_screen ( self , * args , ** kargs ) : <EOL> pass <EOL> class F4mFD ( FileDownloader ) : <EOL> """<STR_LIT>""" <EOL> def real_download ( self , filename , info_dict ) : <EOL> man_url = info_dict [ '<STR_LIT:url>' ] <EOL> requested_bitrate = info_dict . get ( '<STR_LIT>' ) <EOL> self . to_screen ( '<STR_LIT>' ) <EOL> manifest = self . ydl . urlopen ( man_url ) . read ( ) <EOL> self . report_destination ( filename ) <EOL> http_dl = HttpQuietDownloader ( <EOL> self . ydl , <EOL> { <EOL> '<STR_LIT>' : True , <EOL> '<STR_LIT>' : True , <EOL> '<STR_LIT>' : True , <EOL> '<STR_LIT>' : self . params . get ( '<STR_LIT>' , None ) , <EOL> '<STR_LIT:test>' : self . params . get ( '<STR_LIT:test>' , False ) , <EOL> } <EOL> ) <EOL> doc = etree . fromstring ( manifest ) <EOL> formats = [ ( int ( f . attrib . get ( '<STR_LIT>' , - <NUM_LIT:1> ) ) , f ) for f in doc . findall ( _add_ns ( '<STR_LIT>' ) ) ] <EOL> if requested_bitrate is None : <EOL> formats = sorted ( formats , key = lambda f : f [ <NUM_LIT:0> ] ) <EOL> rate , media = formats [ - <NUM_LIT:1> ] <EOL> else : <EOL> rate , media = list ( filter ( <EOL> lambda f : int ( f [ <NUM_LIT:0> ] ) == requested_bitrate , formats ) ) [ <NUM_LIT:0> ] <EOL> base_url = compat_urlparse . urljoin ( man_url , media . attrib [ '<STR_LIT:url>' ] ) <EOL> bootstrap_node = doc . find ( _add_ns ( '<STR_LIT>' ) ) <EOL> if bootstrap_node . text is None : <EOL> bootstrap_url = compat_urlparse . urljoin ( <EOL> base_url , bootstrap_node . attrib [ '<STR_LIT:url>' ] ) <EOL> bootstrap = self . ydl . urlopen ( bootstrap_url ) . read ( ) <EOL> else : <EOL> bootstrap = base64 . b64decode ( bootstrap_node . text ) <EOL> metadata_node = media . find ( _add_ns ( '<STR_LIT>' ) ) <EOL> if metadata_node is not None : <EOL> metadata = base64 . b64decode ( metadata_node . text ) <EOL> else : <EOL> metadata = None <EOL> boot_info = read_bootstrap_info ( bootstrap ) <EOL> fragments_list = build_fragments_list ( boot_info ) <EOL> if self . params . get ( '<STR_LIT:test>' , False ) : <EOL> fragments_list = fragments_list [ : <NUM_LIT:1> ] <EOL> total_frags = len ( fragments_list ) <EOL> akamai_pv = xpath_text ( doc , _add_ns ( '<STR_LIT>' ) ) <EOL> tmpfilename = self . temp_name ( filename ) <EOL> ( dest_stream , tmpfilename ) = sanitize_open ( tmpfilename , '<STR_LIT:wb>' ) <EOL> write_flv_header ( dest_stream ) <EOL> write_metadata_tag ( dest_stream , metadata ) <EOL> state = { <EOL> '<STR_LIT>' : <NUM_LIT:0> , <EOL> '<STR_LIT>' : <NUM_LIT:0> , <EOL> } <EOL> start = time . time ( ) <EOL> def frag_progress_hook ( status ) : <EOL> frag_total_bytes = status . get ( '<STR_LIT>' , <NUM_LIT:0> ) <EOL> estimated_size = ( state [ '<STR_LIT>' ] + <EOL> ( total_frags - state [ '<STR_LIT>' ] ) * frag_total_bytes ) <EOL> if status [ '<STR_LIT:status>' ] == '<STR_LIT>' : <EOL> state [ '<STR_LIT>' ] += frag_total_bytes <EOL> state [ '<STR_LIT>' ] += <NUM_LIT:1> <EOL> progress = self . calc_percent ( state [ '<STR_LIT>' ] , total_frags ) <EOL> byte_counter = state [ '<STR_LIT>' ] <EOL> else : <EOL> frag_downloaded_bytes = status [ '<STR_LIT>' ] <EOL> byte_counter = state [ '<STR_LIT>' ] + frag_downloaded_bytes <EOL> frag_progress = self . calc_percent ( frag_downloaded_bytes , <EOL> frag_total_bytes ) <EOL> progress = self . calc_percent ( state [ '<STR_LIT>' ] , total_frags ) <EOL> progress += frag_progress / float ( total_frags ) <EOL> eta = self . calc_eta ( start , time . time ( ) , estimated_size , byte_counter ) <EOL> self . report_progress ( progress , format_bytes ( estimated_size ) , <EOL> status . get ( '<STR_LIT>' ) , eta ) <EOL> http_dl . add_progress_hook ( frag_progress_hook ) <EOL> frags_filenames = [ ] <EOL> for ( seg_i , frag_i ) in fragments_list : <EOL> name = '<STR_LIT>' % ( seg_i , frag_i ) <EOL> url = base_url + name <EOL> if akamai_pv : <EOL> url += '<STR_LIT:?>' + akamai_pv . strip ( '<STR_LIT:;>' ) <EOL> frag_filename = '<STR_LIT>' % ( tmpfilename , name ) <EOL> success = http_dl . download ( frag_filename , { '<STR_LIT:url>' : url } ) <EOL> if not success : <EOL> return False <EOL> with open ( frag_filename , '<STR_LIT:rb>' ) as down : <EOL> down_data = down . read ( ) <EOL> reader = FlvReader ( down_data ) <EOL> while True : <EOL> _ , box_type , box_data = reader . read_box_info ( ) <EOL> if box_type == b'<STR_LIT>' : <EOL> dest_stream . write ( box_data ) <EOL> break <EOL> frags_filenames . append ( frag_filename ) <EOL> dest_stream . close ( ) <EOL> self . report_finish ( format_bytes ( state [ '<STR_LIT>' ] ) , time . time ( ) - start ) <EOL> self . try_rename ( tmpfilename , filename ) <EOL> for frag_file in frags_filenames : <EOL> os . remove ( frag_file ) <EOL> fsize = os . path . getsize ( encodeFilename ( filename ) ) <EOL> self . _hook_progress ( { <EOL> '<STR_LIT>' : fsize , <EOL> '<STR_LIT>' : fsize , <EOL> '<STR_LIT:filename>' : filename , <EOL> '<STR_LIT:status>' : '<STR_LIT>' , <EOL> } ) <EOL> return True </s>
