<s> """<STR_LIT>""" <EOL> import os , unittest <EOL> from hstestcase import HSTestCase <EOL> from hubstorage . jobq import DuplicateJobError <EOL> from hubstorage . utils import apipoll <EOL> EXCLUSIVE = os . environ . get ( '<STR_LIT>' ) <EOL> class JobqTest ( HSTestCase ) : <EOL> def test_push ( self ) : <EOL> jobq = self . project . jobq <EOL> qjob = jobq . push ( self . spidername ) <EOL> self . assertTrue ( '<STR_LIT:key>' in qjob , qjob ) <EOL> self . assertTrue ( '<STR_LIT>' in qjob , qjob ) <EOL> job = self . hsclient . get_job ( qjob [ '<STR_LIT:key>' ] ) <EOL> self . assertEqual ( job . metadata . get ( '<STR_LIT:state>' ) , u'<STR_LIT>' ) <EOL> self . assertEqual ( job . metadata . get ( '<STR_LIT>' ) , self . spidername ) <EOL> self . assertEqual ( job . metadata . get ( '<STR_LIT>' ) , qjob [ '<STR_LIT>' ] ) <EOL> jobq . start ( job ) <EOL> job . metadata . expire ( ) <EOL> self . assertEqual ( job . metadata . get ( '<STR_LIT:state>' ) , u'<STR_LIT>' ) <EOL> jobq . finish ( job ) <EOL> job . metadata . expire ( ) <EOL> self . assertEqual ( job . metadata . get ( '<STR_LIT:state>' ) , u'<STR_LIT>' ) <EOL> jobq . delete ( job ) <EOL> job . metadata . expire ( ) <EOL> self . assertEqual ( job . metadata . get ( '<STR_LIT:state>' ) , u'<STR_LIT>' ) <EOL> def test_push_with_extras ( self ) : <EOL> qjob = self . project . jobq . push ( self . spidername , foo = '<STR_LIT:bar>' , baz = '<STR_LIT>' ) <EOL> job = self . hsclient . get_job ( qjob [ '<STR_LIT:key>' ] ) <EOL> self . assertEqual ( job . metadata . get ( '<STR_LIT:foo>' ) , u'<STR_LIT:bar>' ) <EOL> self . assertEqual ( job . metadata . get ( '<STR_LIT>' ) , u'<STR_LIT>' ) <EOL> def test_push_with_priority ( self ) : <EOL> jobq = self . project . jobq <EOL> qjob = jobq . push ( self . spidername , priority = jobq . PRIO_HIGHEST ) <EOL> self . assertTrue ( '<STR_LIT:key>' in qjob , qjob ) <EOL> self . assertTrue ( '<STR_LIT>' in qjob , qjob ) <EOL> def test_push_with_state ( self ) : <EOL> qjob = self . project . jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> self . assertTrue ( '<STR_LIT:key>' in qjob , qjob ) <EOL> self . assertTrue ( '<STR_LIT>' in qjob , qjob ) <EOL> job = self . hsclient . get_job ( qjob [ '<STR_LIT:key>' ] ) <EOL> self . assertEqual ( job . metadata . get ( '<STR_LIT:state>' ) , u'<STR_LIT>' ) <EOL> def test_push_with_unique ( self ) : <EOL> jobq = self . project . jobq <EOL> jobq . push ( self . spidername ) <EOL> jobq . push ( self . spidername ) <EOL> jobq . push ( self . spidername , unique = None ) <EOL> jobq . push ( self . spidername , unique = None ) <EOL> q1 = jobq . push ( self . spidername , unique = '<STR_LIT>' ) <EOL> jobq . push ( self . spidername , unique = '<STR_LIT>' ) <EOL> self . assertRaises ( DuplicateJobError , jobq . push , self . spidername , unique = '<STR_LIT>' ) <EOL> jobq . finish ( q1 ) <EOL> self . assertRaises ( DuplicateJobError , jobq . push , self . spidername , unique = '<STR_LIT>' ) <EOL> jobq . push ( self . spidername , unique = '<STR_LIT>' ) <EOL> def test_startjob ( self ) : <EOL> jobq = self . project . jobq <EOL> qj = jobq . push ( self . spidername ) <EOL> nj = jobq . start ( ) <EOL> self . assertTrue ( nj . pop ( '<STR_LIT>' , None ) , nj ) <EOL> self . assertTrue ( nj . pop ( '<STR_LIT>' , None ) , nj ) <EOL> self . assertTrue ( nj . pop ( '<STR_LIT>' , None ) , nj ) <EOL> self . assertEqual ( nj [ u'<STR_LIT:key>' ] , qj [ '<STR_LIT:key>' ] ) <EOL> self . assertEqual ( nj [ u'<STR_LIT>' ] , self . spidername ) <EOL> self . assertEqual ( nj [ u'<STR_LIT:state>' ] , u'<STR_LIT>' ) <EOL> self . assertEqual ( nj [ u'<STR_LIT>' ] , jobq . PRIO_NORMAL ) <EOL> def test_startjob_with_extras ( self ) : <EOL> jobq = self . project . jobq <EOL> pushextras = { <EOL> '<STR_LIT:string>' : '<STR_LIT:foo>' , <EOL> '<STR_LIT>' : <NUM_LIT:1> , <EOL> '<STR_LIT:float>' : <NUM_LIT> , <EOL> '<STR_LIT>' : [ '<STR_LIT:b>' , <NUM_LIT:1> , None , True , False , { '<STR_LIT:k>' : '<STR_LIT:c>' } ] , <EOL> '<STR_LIT>' : [ ] , <EOL> '<STR_LIT>' : { '<STR_LIT>' : <NUM_LIT:5> , '<STR_LIT:b>' : '<STR_LIT:B>' , '<STR_LIT>' : [ ] } , <EOL> '<STR_LIT>' : { } , <EOL> '<STR_LIT:true>' : True , <EOL> '<STR_LIT:false>' : False , <EOL> '<STR_LIT>' : None , <EOL> } <EOL> qj = jobq . push ( self . spidername , ** pushextras ) <EOL> startextras = dict ( ( '<STR_LIT>' + k , v ) for k , v in pushextras . iteritems ( ) ) <EOL> nj = jobq . start ( ** startextras ) <EOL> self . assertEqual ( qj [ '<STR_LIT:key>' ] , nj [ '<STR_LIT:key>' ] ) <EOL> for k , v in dict ( pushextras , ** startextras ) . iteritems ( ) : <EOL> if type ( v ) is float : <EOL> self . assertAlmostEqual ( nj . get ( k ) , v ) <EOL> else : <EOL> self . assertEqual ( nj . get ( k ) , v ) <EOL> def test_startjob_order ( self ) : <EOL> jobq = self . project . jobq <EOL> q1 = jobq . push ( self . spidername ) <EOL> q2 = jobq . push ( self . spidername ) <EOL> q3 = jobq . push ( self . spidername ) <EOL> self . assertEqual ( jobq . start ( ) [ '<STR_LIT:key>' ] , q1 [ '<STR_LIT:key>' ] ) <EOL> self . assertEqual ( jobq . start ( ) [ '<STR_LIT:key>' ] , q2 [ '<STR_LIT:key>' ] ) <EOL> self . assertEqual ( jobq . start ( ) [ '<STR_LIT:key>' ] , q3 [ '<STR_LIT:key>' ] ) <EOL> def test_summary ( self ) : <EOL> jobq = self . project . jobq <EOL> jobq . push ( self . spidername ) <EOL> jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> summaries = dict ( ( s [ '<STR_LIT:name>' ] , s ) for s in jobq . summary ( ) ) <EOL> self . assertEqual ( set ( summaries ) , set ( [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) ) <EOL> self . assertTrue ( jobq . summary ( '<STR_LIT>' ) ) <EOL> self . assertTrue ( jobq . summary ( '<STR_LIT>' ) ) <EOL> self . assertTrue ( jobq . summary ( '<STR_LIT>' ) ) <EOL> def test_summary_jobmeta ( self ) : <EOL> jobq = self . project . jobq <EOL> jobq . push ( self . spidername , foo = '<STR_LIT:bar>' , caz = '<STR_LIT>' ) <EOL> pendings = jobq . summary ( '<STR_LIT>' , jobmeta = '<STR_LIT:foo>' ) [ '<STR_LIT>' ] <EOL> p1 = pendings [ <NUM_LIT:0> ] <EOL> self . assertEqual ( p1 . get ( '<STR_LIT:foo>' ) , '<STR_LIT:bar>' ) <EOL> self . assertFalse ( '<STR_LIT>' in p1 ) <EOL> pendings = jobq . summary ( '<STR_LIT>' , jobmeta = [ '<STR_LIT:foo>' , '<STR_LIT>' ] ) [ '<STR_LIT>' ] <EOL> p1 = pendings [ <NUM_LIT:0> ] <EOL> self . assertEqual ( p1 . get ( '<STR_LIT:foo>' ) , '<STR_LIT:bar>' ) <EOL> self . assertEqual ( p1 . get ( '<STR_LIT>' ) , '<STR_LIT>' ) <EOL> def test_summary_countstart ( self ) : <EOL> N = <NUM_LIT:6> <EOL> jobq = self . project . jobq <EOL> for state in ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) : <EOL> for idx in xrange ( N ) : <EOL> jobq . push ( self . spidername , state = state , idx = idx ) <EOL> s1 = jobq . summary ( state ) <EOL> self . assertEqual ( s1 [ '<STR_LIT:count>' ] , N ) <EOL> self . assertEqual ( len ( s1 [ '<STR_LIT>' ] ) , <NUM_LIT:5> ) <EOL> s2 = jobq . summary ( state , count = N ) <EOL> self . assertEqual ( len ( s2 [ '<STR_LIT>' ] ) , N ) <EOL> s3 = jobq . summary ( state , start = N - <NUM_LIT:6> , count = <NUM_LIT:3> ) <EOL> self . assertEqual ( [ o [ '<STR_LIT:key>' ] for o in s3 [ '<STR_LIT>' ] ] , <EOL> [ o [ '<STR_LIT:key>' ] for o in s2 [ '<STR_LIT>' ] [ - <NUM_LIT:6> : - <NUM_LIT:3> ] ] ) <EOL> def test_summaries_and_state_changes ( self ) : <EOL> jobq = self . project . jobq <EOL> j1 = jobq . push ( self . spidername ) <EOL> j2 = jobq . push ( self . spidername ) <EOL> j3 = jobq . push ( self . spidername ) <EOL> j4 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> self . _assert_queue ( '<STR_LIT>' , [ j3 , j2 , j1 ] ) <EOL> self . _assert_queue ( '<STR_LIT>' , [ j4 ] ) <EOL> self . _assert_queue ( '<STR_LIT>' , [ ] ) <EOL> jobq . start ( j1 ) <EOL> jobq . finish ( j2 ) <EOL> jobq . finish ( j4 ) <EOL> self . _assert_queue ( '<STR_LIT>' , [ j3 ] ) <EOL> self . _assert_queue ( '<STR_LIT>' , [ j1 ] ) <EOL> self . _assert_queue ( '<STR_LIT>' , [ j4 , j2 ] ) <EOL> jobq . finish ( j1 ) <EOL> jobq . finish ( j3 ) <EOL> jobq . delete ( j1 ) <EOL> jobq . delete ( j2 ) <EOL> jobq . delete ( j3 ) <EOL> jobq . delete ( j4 ) <EOL> self . _assert_queue ( '<STR_LIT>' , [ ] ) <EOL> self . _assert_queue ( '<STR_LIT>' , [ ] ) <EOL> self . _assert_queue ( '<STR_LIT>' , [ ] ) <EOL> def test_list_with_state ( self ) : <EOL> jobq = self . project . jobq <EOL> j1 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> j2 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> j3 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> j4 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> assert _keys ( jobq . list ( ) ) == _keys ( [ j4 , j1 ] ) <EOL> assert _keys ( jobq . list ( state = '<STR_LIT>' ) ) == _keys ( [ j4 , j1 ] ) <EOL> assert _keys ( jobq . list ( state = '<STR_LIT>' ) ) == _keys ( [ j2 ] ) <EOL> assert _keys ( jobq . list ( state = [ '<STR_LIT>' , '<STR_LIT>' ] ) ) == _keys ( [ j3 , j2 ] ) <EOL> def test_list_with_count ( self ) : <EOL> jobq = self . project . jobq <EOL> j1 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> j2 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> j3 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> j4 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> assert _keys ( jobq . list ( count = <NUM_LIT:2> ) ) == _keys ( [ j4 , j3 ] ) <EOL> def test_list_with_stop ( self ) : <EOL> jobq = self . project . jobq <EOL> j1 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> j2 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> j3 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> j4 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> assert _keys ( jobq . list ( stop = j1 [ '<STR_LIT:key>' ] ) ) == _keys ( [ j4 , j3 , j2 ] ) <EOL> def test_list_with_tags ( self ) : <EOL> jobq = self . project . jobq <EOL> j1 = jobq . push ( self . spidername , state = '<STR_LIT>' , tags = [ '<STR_LIT>' ] ) <EOL> j2 = jobq . push ( self . spidername , state = '<STR_LIT>' , tags = [ '<STR_LIT>' ] ) <EOL> j3 = jobq . push ( self . spidername , state = '<STR_LIT>' , tags = [ '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> j4 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> assert _keys ( jobq . list ( has_tag = '<STR_LIT>' ) ) == _keys ( [ j3 , j1 ] ) <EOL> assert _keys ( jobq . list ( has_tag = [ '<STR_LIT>' , '<STR_LIT>' ] ) ) == _keys ( [ j3 , j2 , j1 ] ) <EOL> assert _keys ( jobq . list ( has_tag = '<STR_LIT>' , lacks_tag = '<STR_LIT>' ) ) == _keys ( [ j2 ] ) <EOL> assert _keys ( jobq . list ( lacks_tag = [ '<STR_LIT>' , '<STR_LIT>' ] ) ) == _keys ( [ j4 ] ) <EOL> @ unittest . expectedFailure <EOL> def test_list_with_startts_endts ( self ) : <EOL> jobq = self . project . jobq <EOL> j1 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> j2 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> j3 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> j4 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> timestamps = [ j [ '<STR_LIT>' ] for j in jobq . list ( ) ] <EOL> jobs = jobq . list ( startts = timestamps [ <NUM_LIT:2> ] , endts = timestamps [ <NUM_LIT:0> ] ) <EOL> assert _keys ( jobs ) == _keys ( [ j3 , j2 ] ) <EOL> def _assert_queue ( self , qname , jobs ) : <EOL> summary = self . project . jobq . summary ( qname , spiderid = self . spiderid ) <EOL> self . assertEqual ( summary [ '<STR_LIT:name>' ] , qname ) <EOL> self . assertEqual ( summary [ '<STR_LIT:count>' ] , len ( jobs ) ) <EOL> self . assertEqual ( len ( summary [ '<STR_LIT>' ] ) , len ( jobs ) ) <EOL> self . assertEqual ( [ s [ '<STR_LIT:key>' ] for s in summary [ '<STR_LIT>' ] ] , <EOL> [ j [ '<STR_LIT:key>' ] for j in jobs ] ) <EOL> def test_simple_botgroups ( self ) : <EOL> self . project . settings [ '<STR_LIT>' ] = [ '<STR_LIT>' ] <EOL> self . project . settings . save ( ) <EOL> pq = self . project . jobq <EOL> hq = self . hsclient . jobq <EOL> q1 = pq . push ( self . spidername ) <EOL> self . assertEqual ( hq . start ( botgroup = '<STR_LIT>' ) , None ) <EOL> self . assertEqual ( apipoll ( hq . start , botgroup = '<STR_LIT>' ) [ '<STR_LIT:key>' ] , q1 [ '<STR_LIT:key>' ] ) <EOL> @ unittest . skipUnless ( EXCLUSIVE , "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> def test_botgroups ( self ) : <EOL> self . project . settings [ '<STR_LIT>' ] = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> self . project . settings . save ( ) <EOL> pq = self . project . jobq <EOL> hq = self . hsclient . jobq <EOL> q1 = pq . push ( self . spidername ) <EOL> q2 = pq . push ( self . spidername ) <EOL> q3 = pq . push ( self . spidername ) <EOL> self . assertEqual ( hq . start ( ) , None ) <EOL> self . assertEqual ( hq . start ( botgroup = '<STR_LIT>' ) , None ) <EOL> self . assertEqual ( apipoll ( hq . start , botgroup = '<STR_LIT>' ) [ '<STR_LIT:key>' ] , q1 [ '<STR_LIT:key>' ] ) <EOL> self . assertEqual ( apipoll ( hq . start , botgroup = '<STR_LIT>' ) [ '<STR_LIT:key>' ] , q2 [ '<STR_LIT:key>' ] ) <EOL> del self . project . settings [ '<STR_LIT>' ] <EOL> self . project . settings . save ( ) <EOL> q4 = pq . push ( self . spidername ) <EOL> self . assertEqual ( hq . start ( botgroup = '<STR_LIT>' ) , None ) <EOL> self . assertEqual ( hq . start ( botgroup = '<STR_LIT>' ) , None ) <EOL> self . assertEqual ( hq . start ( botgroup = '<STR_LIT>' ) , None ) <EOL> self . assertEqual ( hq . start ( ) [ '<STR_LIT:key>' ] , q3 [ '<STR_LIT:key>' ] ) <EOL> self . assertEqual ( hq . start ( ) [ '<STR_LIT:key>' ] , q4 [ '<STR_LIT:key>' ] ) <EOL> self . project . settings [ '<STR_LIT>' ] = [ '<STR_LIT>' ] <EOL> self . project . settings . save ( ) <EOL> def test_spider_updates ( self ) : <EOL> jobq = self . project . jobq <EOL> spiderkey = '<STR_LIT>' % ( self . projectid , self . spiderid ) <EOL> def finish_and_delete_jobs ( ) : <EOL> for job in jobq . finish ( spiderkey ) : <EOL> yield job <EOL> jobq . delete ( spiderkey ) <EOL> q1 = jobq . push ( self . spidername ) <EOL> q2 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> q3 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> q4 = jobq . push ( self . spidername , state = '<STR_LIT>' ) <EOL> r = dict ( ( x [ '<STR_LIT:key>' ] , x [ '<STR_LIT>' ] ) for x in finish_and_delete_jobs ( ) ) <EOL> self . assertEqual ( r . get ( q1 [ '<STR_LIT:key>' ] ) , '<STR_LIT>' , r ) <EOL> self . assertEqual ( r . get ( q2 [ '<STR_LIT:key>' ] ) , '<STR_LIT>' , r ) <EOL> self . assertEqual ( r . get ( q3 [ '<STR_LIT:key>' ] ) , '<STR_LIT>' , r ) <EOL> self . assertTrue ( q4 [ '<STR_LIT:key>' ] not in r ) <EOL> self . assertFalse ( list ( jobq . delete ( spiderkey ) ) ) <EOL> def test_multiple_job_update ( self ) : <EOL> jobq = self . project . jobq <EOL> q1 = jobq . push ( self . spidername ) <EOL> q2 = jobq . push ( self . spidername ) <EOL> q3 = jobq . push ( self . spidername ) <EOL> ids = [ q1 , q2 [ '<STR_LIT:key>' ] , self . project . get_job ( q3 [ '<STR_LIT:key>' ] ) ] <EOL> self . assertTrue ( [ x [ '<STR_LIT>' ] for x in jobq . start ( ids ) ] , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> self . assertTrue ( [ x [ '<STR_LIT>' ] for x in jobq . finish ( ids ) ] , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> self . assertTrue ( [ x [ '<STR_LIT>' ] for x in jobq . delete ( ids ) ] , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> def test_update ( self ) : <EOL> job = self . project . push_job ( self . spidername ) <EOL> self . assertEqual ( job . metadata [ '<STR_LIT:state>' ] , '<STR_LIT>' ) <EOL> self . project . jobq . update ( job , state = '<STR_LIT>' , foo = '<STR_LIT:bar>' ) <EOL> job = self . project . get_job ( job . key ) <EOL> self . assertEqual ( job . metadata [ '<STR_LIT:state>' ] , '<STR_LIT>' ) <EOL> self . assertEqual ( job . metadata [ '<STR_LIT:foo>' ] , '<STR_LIT:bar>' ) <EOL> def test_jobsummary ( self ) : <EOL> jobs = [ self . project . push_job ( self . spidername , foo = i ) <EOL> for i in range ( <NUM_LIT:5> ) ] <EOL> jobmetas = list ( self . project . jobq . jobsummary ( <EOL> jobkeys = [ j . key for j in jobs ] , jobmeta = [ '<STR_LIT:key>' , '<STR_LIT:foo>' ] ) ) <EOL> jobmeta_dict = { jm [ '<STR_LIT:key>' ] : jm [ '<STR_LIT:foo>' ] for jm in jobmetas } <EOL> assert jobmeta_dict == { <EOL> jobs [ i ] . key : i <EOL> for i in range ( <NUM_LIT:5> ) <EOL> } <EOL> def _keys ( lst ) : <EOL> return [ x [ '<STR_LIT:key>' ] for x in lst ] </s>
