<s> """<STR_LIT>""" <EOL> import errno <EOL> import os <EOL> import re <EOL> import boto <EOL> from boto . s3 . resumable_download_handler import get_cur_file_size <EOL> from boto . s3 . resumable_download_handler import ResumableDownloadHandler <EOL> from boto . exception import ResumableTransferDisposition <EOL> from boto . exception import ResumableDownloadException <EOL> from cb_test_harness import CallbackTestHarness <EOL> from tests . integration . gs . testcase import GSTestCase <EOL> SMALL_KEY_SIZE = <NUM_LIT:2> * <NUM_LIT> <EOL> LARGE_KEY_SIZE = <NUM_LIT> * <NUM_LIT> <EOL> class ResumableDownloadTests ( GSTestCase ) : <EOL> """<STR_LIT>""" <EOL> def make_small_key ( self ) : <EOL> small_src_key_as_string = os . urandom ( SMALL_KEY_SIZE ) <EOL> small_src_key = self . _MakeKey ( data = small_src_key_as_string ) <EOL> return small_src_key_as_string , small_src_key <EOL> def make_tracker_file ( self , tmpdir = None ) : <EOL> if not tmpdir : <EOL> tmpdir = self . _MakeTempDir ( ) <EOL> tracker_file = os . path . join ( tmpdir , '<STR_LIT>' ) <EOL> return tracker_file <EOL> def make_dst_fp ( self , tmpdir = None ) : <EOL> if not tmpdir : <EOL> tmpdir = self . _MakeTempDir ( ) <EOL> dst_file = os . path . join ( tmpdir , '<STR_LIT>' ) <EOL> return open ( dst_file , '<STR_LIT:w>' ) <EOL> def test_non_resumable_download ( self ) : <EOL> """<STR_LIT>""" <EOL> dst_fp = self . make_dst_fp ( ) <EOL> small_src_key_as_string , small_src_key = self . make_small_key ( ) <EOL> small_src_key . get_contents_to_file ( dst_fp ) <EOL> self . assertEqual ( SMALL_KEY_SIZE , <EOL> get_cur_file_size ( dst_fp ) ) <EOL> self . assertEqual ( small_src_key_as_string , <EOL> small_src_key . get_contents_as_string ( ) ) <EOL> def test_download_without_persistent_tracker ( self ) : <EOL> """<STR_LIT>""" <EOL> res_download_handler = ResumableDownloadHandler ( ) <EOL> dst_fp = self . make_dst_fp ( ) <EOL> small_src_key_as_string , small_src_key = self . make_small_key ( ) <EOL> small_src_key . get_contents_to_file ( <EOL> dst_fp , res_download_handler = res_download_handler ) <EOL> self . assertEqual ( SMALL_KEY_SIZE , <EOL> get_cur_file_size ( dst_fp ) ) <EOL> self . assertEqual ( small_src_key_as_string , <EOL> small_src_key . get_contents_as_string ( ) ) <EOL> def test_failed_download_with_persistent_tracker ( self ) : <EOL> """<STR_LIT>""" <EOL> harness = CallbackTestHarness ( ) <EOL> tmpdir = self . _MakeTempDir ( ) <EOL> tracker_file_name = self . make_tracker_file ( tmpdir ) <EOL> dst_fp = self . make_dst_fp ( tmpdir ) <EOL> res_download_handler = ResumableDownloadHandler ( <EOL> tracker_file_name = tracker_file_name , num_retries = <NUM_LIT:0> ) <EOL> small_src_key_as_string , small_src_key = self . make_small_key ( ) <EOL> try : <EOL> small_src_key . get_contents_to_file ( <EOL> dst_fp , cb = harness . call , <EOL> res_download_handler = res_download_handler ) <EOL> self . fail ( '<STR_LIT>' ) <EOL> except ResumableDownloadException as e : <EOL> self . assertEqual ( e . disposition , <EOL> ResumableTransferDisposition . ABORT_CUR_PROCESS ) <EOL> self . assertTrue ( os . path . exists ( tracker_file_name ) ) <EOL> f = open ( tracker_file_name ) <EOL> etag_line = f . readline ( ) <EOL> self . assertEquals ( etag_line . rstrip ( '<STR_LIT:\n>' ) , small_src_key . etag . strip ( '<STR_LIT>' ) ) <EOL> def test_retryable_exception_recovery ( self ) : <EOL> """<STR_LIT>""" <EOL> exception = ResumableDownloadHandler . RETRYABLE_EXCEPTIONS [ <NUM_LIT:0> ] <EOL> harness = CallbackTestHarness ( exception = exception ) <EOL> res_download_handler = ResumableDownloadHandler ( num_retries = <NUM_LIT:1> ) <EOL> dst_fp = self . make_dst_fp ( ) <EOL> small_src_key_as_string , small_src_key = self . make_small_key ( ) <EOL> small_src_key . get_contents_to_file ( <EOL> dst_fp , cb = harness . call , <EOL> res_download_handler = res_download_handler ) <EOL> self . assertEqual ( SMALL_KEY_SIZE , <EOL> get_cur_file_size ( dst_fp ) ) <EOL> self . assertEqual ( small_src_key_as_string , <EOL> small_src_key . get_contents_as_string ( ) ) <EOL> def test_broken_pipe_recovery ( self ) : <EOL> """<STR_LIT>""" <EOL> exception = IOError ( errno . EPIPE , "<STR_LIT>" ) <EOL> harness = CallbackTestHarness ( exception = exception ) <EOL> res_download_handler = ResumableDownloadHandler ( num_retries = <NUM_LIT:1> ) <EOL> dst_fp = self . make_dst_fp ( ) <EOL> small_src_key_as_string , small_src_key = self . make_small_key ( ) <EOL> small_src_key . get_contents_to_file ( <EOL> dst_fp , cb = harness . call , <EOL> res_download_handler = res_download_handler ) <EOL> self . assertEqual ( SMALL_KEY_SIZE , <EOL> get_cur_file_size ( dst_fp ) ) <EOL> self . assertEqual ( small_src_key_as_string , <EOL> small_src_key . get_contents_as_string ( ) ) <EOL> def test_non_retryable_exception_handling ( self ) : <EOL> """<STR_LIT>""" <EOL> harness = CallbackTestHarness ( <EOL> exception = OSError ( errno . EACCES , '<STR_LIT>' ) ) <EOL> res_download_handler = ResumableDownloadHandler ( num_retries = <NUM_LIT:1> ) <EOL> dst_fp = self . make_dst_fp ( ) <EOL> small_src_key_as_string , small_src_key = self . make_small_key ( ) <EOL> try : <EOL> small_src_key . get_contents_to_file ( <EOL> dst_fp , cb = harness . call , <EOL> res_download_handler = res_download_handler ) <EOL> self . fail ( '<STR_LIT>' ) <EOL> except OSError as e : <EOL> self . assertEqual ( e . errno , <NUM_LIT> ) <EOL> def test_failed_and_restarted_download_with_persistent_tracker ( self ) : <EOL> """<STR_LIT>""" <EOL> harness = CallbackTestHarness ( ) <EOL> tmpdir = self . _MakeTempDir ( ) <EOL> tracker_file_name = self . make_tracker_file ( tmpdir ) <EOL> dst_fp = self . make_dst_fp ( tmpdir ) <EOL> small_src_key_as_string , small_src_key = self . make_small_key ( ) <EOL> res_download_handler = ResumableDownloadHandler ( <EOL> tracker_file_name = tracker_file_name , num_retries = <NUM_LIT:1> ) <EOL> small_src_key . get_contents_to_file ( <EOL> dst_fp , cb = harness . call , <EOL> res_download_handler = res_download_handler ) <EOL> self . assertEqual ( SMALL_KEY_SIZE , <EOL> get_cur_file_size ( dst_fp ) ) <EOL> self . assertEqual ( small_src_key_as_string , <EOL> small_src_key . get_contents_as_string ( ) ) <EOL> self . assertFalse ( os . path . exists ( tracker_file_name ) ) <EOL> def test_multiple_in_process_failures_then_succeed ( self ) : <EOL> """<STR_LIT>""" <EOL> res_download_handler = ResumableDownloadHandler ( num_retries = <NUM_LIT:3> ) <EOL> dst_fp = self . make_dst_fp ( ) <EOL> small_src_key_as_string , small_src_key = self . make_small_key ( ) <EOL> small_src_key . get_contents_to_file ( <EOL> dst_fp , res_download_handler = res_download_handler ) <EOL> self . assertEqual ( SMALL_KEY_SIZE , <EOL> get_cur_file_size ( dst_fp ) ) <EOL> self . assertEqual ( small_src_key_as_string , <EOL> small_src_key . get_contents_as_string ( ) ) <EOL> def test_multiple_in_process_failures_then_succeed_with_tracker_file ( self ) : <EOL> """<STR_LIT>""" <EOL> harness = CallbackTestHarness ( <EOL> fail_after_n_bytes = LARGE_KEY_SIZE / <NUM_LIT:2> , num_times_to_fail = <NUM_LIT:2> ) <EOL> larger_src_key_as_string = os . urandom ( LARGE_KEY_SIZE ) <EOL> larger_src_key = self . _MakeKey ( data = larger_src_key_as_string ) <EOL> tmpdir = self . _MakeTempDir ( ) <EOL> tracker_file_name = self . make_tracker_file ( tmpdir ) <EOL> dst_fp = self . make_dst_fp ( tmpdir ) <EOL> res_download_handler = ResumableDownloadHandler ( <EOL> tracker_file_name = tracker_file_name , num_retries = <NUM_LIT:0> ) <EOL> try : <EOL> larger_src_key . get_contents_to_file ( <EOL> dst_fp , cb = harness . call , <EOL> res_download_handler = res_download_handler ) <EOL> self . fail ( '<STR_LIT>' ) <EOL> except ResumableDownloadException as e : <EOL> self . assertEqual ( e . disposition , <EOL> ResumableTransferDisposition . ABORT_CUR_PROCESS ) <EOL> self . assertTrue ( os . path . exists ( tracker_file_name ) ) <EOL> larger_src_key . get_contents_to_file ( <EOL> dst_fp , cb = harness . call , <EOL> res_download_handler = res_download_handler ) <EOL> self . assertEqual ( LARGE_KEY_SIZE , <EOL> get_cur_file_size ( dst_fp ) ) <EOL> self . assertEqual ( larger_src_key_as_string , <EOL> larger_src_key . get_contents_as_string ( ) ) <EOL> self . assertFalse ( os . path . exists ( tracker_file_name ) ) <EOL> self . assertTrue ( <EOL> len ( harness . transferred_seq_before_first_failure ) > <NUM_LIT:1> and <EOL> len ( harness . transferred_seq_after_first_failure ) > <NUM_LIT:1> ) <EOL> def test_download_with_inital_partial_download_before_failure ( self ) : <EOL> """<STR_LIT>""" <EOL> harness = CallbackTestHarness ( <EOL> fail_after_n_bytes = LARGE_KEY_SIZE / <NUM_LIT:2> ) <EOL> larger_src_key_as_string = os . urandom ( LARGE_KEY_SIZE ) <EOL> larger_src_key = self . _MakeKey ( data = larger_src_key_as_string ) <EOL> res_download_handler = ResumableDownloadHandler ( num_retries = <NUM_LIT:1> ) <EOL> dst_fp = self . make_dst_fp ( ) <EOL> larger_src_key . get_contents_to_file ( <EOL> dst_fp , cb = harness . call , <EOL> res_download_handler = res_download_handler ) <EOL> self . assertEqual ( LARGE_KEY_SIZE , <EOL> get_cur_file_size ( dst_fp ) ) <EOL> self . assertEqual ( larger_src_key_as_string , <EOL> larger_src_key . get_contents_as_string ( ) ) <EOL> self . assertTrue ( <EOL> len ( harness . transferred_seq_before_first_failure ) > <NUM_LIT:1> and <EOL> len ( harness . transferred_seq_after_first_failure ) > <NUM_LIT:1> ) <EOL> def test_zero_length_object_download ( self ) : <EOL> """<STR_LIT>""" <EOL> res_download_handler = ResumableDownloadHandler ( ) <EOL> dst_fp = self . make_dst_fp ( ) <EOL> k = self . _MakeKey ( ) <EOL> k . get_contents_to_file ( dst_fp , <EOL> res_download_handler = res_download_handler ) <EOL> self . assertEqual ( <NUM_LIT:0> , get_cur_file_size ( dst_fp ) ) <EOL> def test_download_with_invalid_tracker_etag ( self ) : <EOL> """<STR_LIT>""" <EOL> tmp_dir = self . _MakeTempDir ( ) <EOL> dst_fp = self . make_dst_fp ( tmp_dir ) <EOL> small_src_key_as_string , small_src_key = self . make_small_key ( ) <EOL> invalid_etag_tracker_file_name = os . path . join ( tmp_dir , <EOL> '<STR_LIT>' ) <EOL> f = open ( invalid_etag_tracker_file_name , '<STR_LIT:w>' ) <EOL> f . write ( '<STR_LIT>' ) <EOL> f . close ( ) <EOL> res_download_handler = ResumableDownloadHandler ( <EOL> tracker_file_name = invalid_etag_tracker_file_name ) <EOL> small_src_key . get_contents_to_file ( <EOL> dst_fp , res_download_handler = res_download_handler ) <EOL> self . assertEqual ( SMALL_KEY_SIZE , get_cur_file_size ( dst_fp ) ) <EOL> self . assertEqual ( small_src_key_as_string , <EOL> small_src_key . get_contents_as_string ( ) ) <EOL> def test_download_with_inconsistent_etag_in_tracker ( self ) : <EOL> """<STR_LIT>""" <EOL> tmp_dir = self . _MakeTempDir ( ) <EOL> dst_fp = self . make_dst_fp ( tmp_dir ) <EOL> small_src_key_as_string , small_src_key = self . make_small_key ( ) <EOL> inconsistent_etag_tracker_file_name = os . path . join ( tmp_dir , <EOL> '<STR_LIT>' ) <EOL> f = open ( inconsistent_etag_tracker_file_name , '<STR_LIT:w>' ) <EOL> good_etag = small_src_key . etag . strip ( '<STR_LIT>' ) <EOL> new_val_as_list = [ ] <EOL> for c in reversed ( good_etag ) : <EOL> new_val_as_list . append ( c ) <EOL> f . write ( '<STR_LIT>' % '<STR_LIT>' . join ( new_val_as_list ) ) <EOL> f . close ( ) <EOL> res_download_handler = ResumableDownloadHandler ( <EOL> tracker_file_name = inconsistent_etag_tracker_file_name ) <EOL> small_src_key . get_contents_to_file ( <EOL> dst_fp , res_download_handler = res_download_handler ) <EOL> self . assertEqual ( SMALL_KEY_SIZE , <EOL> get_cur_file_size ( dst_fp ) ) <EOL> self . assertEqual ( small_src_key_as_string , <EOL> small_src_key . get_contents_as_string ( ) ) <EOL> def test_download_with_unwritable_tracker_file ( self ) : <EOL> """<STR_LIT>""" <EOL> tmp_dir = self . _MakeTempDir ( ) <EOL> tracker_file_name = os . path . join ( tmp_dir , '<STR_LIT>' ) <EOL> save_mod = os . stat ( tmp_dir ) . st_mode <EOL> try : <EOL> os . chmod ( tmp_dir , <NUM_LIT:0> ) <EOL> res_download_handler = ResumableDownloadHandler ( <EOL> tracker_file_name = tracker_file_name ) <EOL> except ResumableDownloadException as e : <EOL> self . assertEqual ( e . disposition , ResumableTransferDisposition . ABORT ) <EOL> self . assertNotEqual ( <EOL> e . message . find ( '<STR_LIT>' ) , - <NUM_LIT:1> ) <EOL> finally : <EOL> os . chmod ( tmp_dir , save_mod ) </s>
