<s> """<STR_LIT>""" <EOL> from __future__ import absolute_import <EOL> import os <EOL> from importlib import import_module <EOL> from twisted . trial . unittest import SkipTest <EOL> from scrapy . exceptions import NotConfigured <EOL> from scrapy . utils . boto import is_botocore <EOL> def assert_aws_environ ( ) : <EOL> """<STR_LIT>""" <EOL> skip_if_no_boto ( ) <EOL> if '<STR_LIT>' not in os . environ : <EOL> raise SkipTest ( "<STR_LIT>" ) <EOL> def skip_if_no_boto ( ) : <EOL> try : <EOL> is_botocore ( ) <EOL> except NotConfigured as e : <EOL> raise SkipTest ( e ) <EOL> def get_s3_content_and_delete ( bucket , path , with_key = False ) : <EOL> """<STR_LIT>""" <EOL> if is_botocore ( ) : <EOL> import botocore . session <EOL> session = botocore . session . get_session ( ) <EOL> client = session . create_client ( '<STR_LIT>' ) <EOL> key = client . get_object ( Bucket = bucket , Key = path ) <EOL> content = key [ '<STR_LIT>' ] . read ( ) <EOL> client . delete_object ( Bucket = bucket , Key = path ) <EOL> else : <EOL> import boto <EOL> bucket = boto . connect_s3 ( ) . get_bucket ( bucket , validate = False ) <EOL> key = bucket . get_key ( path ) <EOL> content = key . get_contents_as_string ( ) <EOL> bucket . delete_key ( path ) <EOL> return ( content , key ) if with_key else content <EOL> def get_crawler ( spidercls = None , settings_dict = None ) : <EOL> """<STR_LIT>""" <EOL> from scrapy . crawler import CrawlerRunner <EOL> from scrapy . spiders import Spider <EOL> runner = CrawlerRunner ( settings_dict ) <EOL> return runner . create_crawler ( spidercls or Spider ) <EOL> def get_pythonpath ( ) : <EOL> """<STR_LIT>""" <EOL> scrapy_path = import_module ( '<STR_LIT>' ) . __path__ [ <NUM_LIT:0> ] <EOL> return os . path . dirname ( scrapy_path ) + os . pathsep + os . environ . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> def get_testenv ( ) : <EOL> """<STR_LIT>""" <EOL> env = os . environ . copy ( ) <EOL> env [ '<STR_LIT>' ] = get_pythonpath ( ) <EOL> return env <EOL> def assert_samelines ( testcase , text1 , text2 , msg = None ) : <EOL> """<STR_LIT>""" <EOL> testcase . assertEqual ( text1 . splitlines ( ) , text2 . splitlines ( ) , msg ) </s>
