<s> import logging <EOL> from django . contrib . auth . models import User <EOL> from nose . plugins . attrib import attr <EOL> from nose . tools import assert_equal , assert_true , assert_not_equal <EOL> from hadoop import cluster , pseudo_hdfs4 <EOL> from hadoop . conf import HDFS_CLUSTERS , MR_CLUSTERS , YARN_CLUSTERS <EOL> from liboozie . submittion import Submission <EOL> from oozie . tests import OozieMockBase <EOL> from desktop . lib . test_utils import clear_sys_caches <EOL> from desktop . lib . django_test_util import make_logged_in_client <EOL> LOG = logging . getLogger ( __name__ ) <EOL> @ attr ( '<STR_LIT>' ) <EOL> def test_copy_files ( ) : <EOL> cluster = pseudo_hdfs4 . shared_cluster ( ) <EOL> try : <EOL> c = make_logged_in_client ( ) <EOL> user = User . objects . get ( username = '<STR_LIT:test>' ) <EOL> prefix = '<STR_LIT>' <EOL> if cluster . fs . exists ( prefix ) : <EOL> cluster . fs . rmtree ( prefix ) <EOL> deployment_dir = '<STR_LIT>' % prefix <EOL> external_deployment_dir = '<STR_LIT>' % prefix <EOL> jar_1 = '<STR_LIT>' % prefix <EOL> jar_2 = '<STR_LIT>' % prefix <EOL> jar_3 = '<STR_LIT>' % deployment_dir <EOL> jar_4 = '<STR_LIT>' % deployment_dir <EOL> cluster . fs . mkdir ( prefix ) <EOL> cluster . fs . create ( jar_1 ) <EOL> cluster . fs . create ( jar_2 ) <EOL> cluster . fs . create ( jar_3 ) <EOL> cluster . fs . create ( jar_4 ) <EOL> class MockNode ( ) : <EOL> def __init__ ( self , jar_path ) : <EOL> self . jar_path = jar_path <EOL> class MockJob ( ) : <EOL> def __init__ ( self ) : <EOL> self . node_list = [ <EOL> MockNode ( jar_1 ) , <EOL> MockNode ( jar_2 ) , <EOL> MockNode ( jar_3 ) , <EOL> MockNode ( jar_4 ) , <EOL> ] <EOL> def get_application_filename ( self ) : <EOL> return '<STR_LIT>' <EOL> submission = Submission ( user , job = MockJob ( ) , fs = cluster . fs , jt = cluster . jt ) <EOL> submission . _copy_files ( deployment_dir , "<STR_LIT>" ) <EOL> submission . _copy_files ( external_deployment_dir , "<STR_LIT>" ) <EOL> assert_true ( cluster . fs . exists ( jar_1 ) ) <EOL> assert_true ( cluster . fs . exists ( jar_2 ) ) <EOL> assert_true ( cluster . fs . exists ( jar_3 ) ) <EOL> assert_true ( cluster . fs . exists ( jar_4 ) ) <EOL> deployment_dir = deployment_dir + '<STR_LIT>' <EOL> external_deployment_dir = external_deployment_dir + '<STR_LIT>' <EOL> list_dir_workspace = cluster . fs . listdir ( deployment_dir ) <EOL> list_dir_deployement = cluster . fs . listdir ( external_deployment_dir ) <EOL> assert_true ( cluster . fs . exists ( deployment_dir + '<STR_LIT>' ) , list_dir_workspace ) <EOL> assert_true ( cluster . fs . exists ( deployment_dir + '<STR_LIT>' ) , list_dir_workspace ) <EOL> assert_true ( cluster . fs . exists ( deployment_dir + '<STR_LIT>' ) , list_dir_workspace ) <EOL> assert_true ( cluster . fs . exists ( deployment_dir + '<STR_LIT>' ) , list_dir_workspace ) <EOL> assert_true ( cluster . fs . exists ( external_deployment_dir + '<STR_LIT>' ) , list_dir_deployement ) <EOL> assert_true ( cluster . fs . exists ( external_deployment_dir + '<STR_LIT>' ) , list_dir_deployement ) <EOL> assert_true ( cluster . fs . exists ( external_deployment_dir + '<STR_LIT>' ) , list_dir_deployement ) <EOL> assert_true ( cluster . fs . exists ( external_deployment_dir + '<STR_LIT>' ) , list_dir_deployement ) <EOL> stats_udf1 = cluster . fs . stats ( deployment_dir + '<STR_LIT>' ) <EOL> stats_udf2 = cluster . fs . stats ( deployment_dir + '<STR_LIT>' ) <EOL> stats_udf3 = cluster . fs . stats ( deployment_dir + '<STR_LIT>' ) <EOL> stats_udf4 = cluster . fs . stats ( deployment_dir + '<STR_LIT>' ) <EOL> submission . _copy_files ( '<STR_LIT>' % prefix , "<STR_LIT>" ) <EOL> assert_not_equal ( stats_udf1 [ '<STR_LIT>' ] , cluster . fs . stats ( deployment_dir + '<STR_LIT>' ) [ '<STR_LIT>' ] ) <EOL> assert_not_equal ( stats_udf2 [ '<STR_LIT>' ] , cluster . fs . stats ( deployment_dir + '<STR_LIT>' ) [ '<STR_LIT>' ] ) <EOL> assert_not_equal ( stats_udf3 [ '<STR_LIT>' ] , cluster . fs . stats ( deployment_dir + '<STR_LIT>' ) [ '<STR_LIT>' ] ) <EOL> assert_equal ( stats_udf4 [ '<STR_LIT>' ] , cluster . fs . stats ( deployment_dir + '<STR_LIT>' ) [ '<STR_LIT>' ] ) <EOL> finally : <EOL> try : <EOL> cluster . fs . rmtree ( prefix ) <EOL> except : <EOL> LOG . exception ( '<STR_LIT>' % prefix ) <EOL> class MockFs ( ) : <EOL> def __init__ ( self , logical_name = None ) : <EOL> self . fs_defaultfs = '<STR_LIT>' <EOL> self . logical_name = logical_name if logical_name else '<STR_LIT>' <EOL> class MockJt ( ) : <EOL> def __init__ ( self , logical_name = None ) : <EOL> self . logical_name = logical_name if logical_name else '<STR_LIT>' <EOL> class TestSubmission ( OozieMockBase ) : <EOL> def test_get_properties ( self ) : <EOL> submission = Submission ( self . user , fs = MockFs ( ) ) <EOL> assert_equal ( { } , submission . properties ) <EOL> submission . _update_properties ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> assert_equal ( { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' <EOL> } , submission . properties ) <EOL> def test_get_logical_properties ( self ) : <EOL> submission = Submission ( self . user , fs = MockFs ( logical_name = '<STR_LIT>' ) , jt = MockJt ( logical_name = '<STR_LIT>' ) ) <EOL> assert_equal ( { } , submission . properties ) <EOL> submission . _update_properties ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> assert_equal ( { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' <EOL> } , submission . properties ) <EOL> def test_update_properties ( self ) : <EOL> finish = [ ] <EOL> finish . append ( MR_CLUSTERS . set_for_testing ( { '<STR_LIT:default>' : { } } ) ) <EOL> finish . append ( MR_CLUSTERS [ '<STR_LIT:default>' ] . SUBMIT_TO . set_for_testing ( True ) ) <EOL> finish . append ( YARN_CLUSTERS . set_for_testing ( { '<STR_LIT:default>' : { } } ) ) <EOL> finish . append ( YARN_CLUSTERS [ '<STR_LIT:default>' ] . SUBMIT_TO . set_for_testing ( True ) ) <EOL> try : <EOL> properties = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' <EOL> } <EOL> final_properties = properties . copy ( ) <EOL> submission = Submission ( None , properties = properties , oozie_id = '<STR_LIT:test>' , fs = MockFs ( ) ) <EOL> assert_equal ( properties , submission . properties ) <EOL> submission . _update_properties ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> assert_equal ( final_properties , submission . properties ) <EOL> clear_sys_caches ( ) <EOL> fs = cluster . get_hdfs ( ) <EOL> jt = cluster . get_next_ha_mrcluster ( ) [ <NUM_LIT:1> ] <EOL> final_properties = properties . copy ( ) <EOL> final_properties . update ( { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : fs . fs_defaultfs <EOL> } ) <EOL> submission = Submission ( None , properties = properties , oozie_id = '<STR_LIT:test>' , fs = fs , jt = jt ) <EOL> assert_equal ( properties , submission . properties ) <EOL> submission . _update_properties ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> assert_equal ( final_properties , submission . properties ) <EOL> finish . append ( HDFS_CLUSTERS [ '<STR_LIT:default>' ] . LOGICAL_NAME . set_for_testing ( '<STR_LIT>' ) ) <EOL> finish . append ( MR_CLUSTERS [ '<STR_LIT:default>' ] . LOGICAL_NAME . set_for_testing ( '<STR_LIT>' ) ) <EOL> clear_sys_caches ( ) <EOL> fs = cluster . get_hdfs ( ) <EOL> jt = cluster . get_next_ha_mrcluster ( ) [ <NUM_LIT:1> ] <EOL> final_properties = properties . copy ( ) <EOL> final_properties . update ( { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' <EOL> } ) <EOL> submission = Submission ( None , properties = properties , oozie_id = '<STR_LIT:test>' , fs = fs , jt = jt ) <EOL> assert_equal ( properties , submission . properties ) <EOL> submission . _update_properties ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> assert_equal ( final_properties , submission . properties ) <EOL> finally : <EOL> clear_sys_caches ( ) <EOL> for reset in finish : <EOL> reset ( ) <EOL> def test_get_external_parameters ( self ) : <EOL> xml = """<STR_LIT>""" <EOL> properties = """<STR_LIT>""" <EOL> parameters = Submission ( self . user ) . _get_external_parameters ( xml , properties ) <EOL> assert_equal ( { '<STR_LIT>' : '<STR_LIT:true>' , <EOL> '<STR_LIT:input>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT:default>' <EOL> } , <EOL> parameters ) </s>
