examples/verify_kba2014.py
def rejester_run(work_unit):
     '''get a rejester.WorkUnit with KBA s3 path, fetch it, and save
     some counts about it.
     '''
     #fname = 'verify-chunks-%d-%d' % (os.getpid(), time.time())
     fname = work_unit.key.strip().split('/')[-1]
     
     output_dir_path = work_unit.data.get('output_dir_path', '/mnt')
     u = uuid.uuid3(uuid.UUID(int=0), work_unit.key.strip())
     path1 = u.hex[0]
     path2 = u.hex[1]
     fpath = os.path.join(output_dir_path, path1, path2, fname)
     if not os.path.exists(os.path.dirname(fpath)):
          os.makedirs(os.path.dirname(fpath))

     output = gzip.open(fpath + '-out.gz', 'wb')

     expected_si_count = int(fname.split('-')[1])

     max_tries = 20
     tries = 0
     while tries < max_tries:
          try:
               exc, si_count, serif_count, clean_visible_bytes, clean_visible_count, stream_ids = \
                   attempt_fetch(work_unit, fpath)
               if si_count != expected_si_count:
                    print 'retrying because si_count = %d != %d expected_si_count' % (si_count, expected_si_count)
                    sys.stdout.flush()
                    tries += 1
                    continue
               else:
                    print 'succeeded in reading si_count = %d' % (si_count,)
                    sys.stdout.flush()
               output.write( '%s\t%d\t%d\t%d\t%d\t%s\t%s\n' % (
                         exc, si_count, serif_count, clean_visible_bytes, clean_visible_count, 
                         work_unit.key.strip(), ','.join(['%s|%s' % tup for tup in stream_ids])) )
               break
          except Exception, exc:
               print 'broken?'
               print traceback.format_exc(exc)
               sys.stdout.flush()
               tries += 1
               output.write(traceback.format_exc(exc))

     output.close()
def rejester_run(work_unit):
     '''get a rejester.WorkUnit with KBA s3 path, fetch it, and save
     some counts about it.
     '''
     #fname = 'verify-chunks-%d-%d' % (os.getpid(), time.time())
     fname = work_unit.key.strip().split('/')[-1]
     
     output_dir_path = work_unit.data.get('output_dir_path', '/mnt')
     u = uuid.uuid3(uuid.UUID(int=0), work_unit.key.strip())
     path1 = u.hex[0]
     path2 = u.hex[1]
     fpath = os.path.join(output_dir_path, path1, path2, fname)
     if not os.path.exists(os.path.dirname(fpath)):
          os.makedirs(os.path.dirname(fpath))

     output = gzip.open(fpath + '-out.gz', 'wb')

     expected_si_count = int(fname.split('-')[1])

     max_tries = 20
     tries = 0
     while tries < max_tries:
          try:
               exc, si_count, serif_count, clean_visible_bytes, clean_visible_count, stream_ids = \
                   attempt_fetch(work_unit, fpath)
               if si_count != expected_si_count:
                    print 'retrying because si_count = %d != %d expected_si_count' % (si_count, expected_si_count)
                    sys.stdout.flush()
                    tries += 1
                    continue
               else:
                    print 'succeeded in reading si_count = %d' % (si_count,)
                    sys.stdout.flush()
               output.write( '%s\t%d\t%d\t%d\t%d\t%s\t%s\n' % (
                         exc, si_count, serif_count, clean_visible_bytes, clean_visible_count, 
                         work_unit.key.strip(), ','.join(['%s|%s' % tup for tup in stream_ids])) )
               break
          except Exception, exc:
               print 'broken?'
               print traceback.format_exc(exc)
               sys.stdout.flush()
               tries += 1
               output.write(traceback.format_exc(exc))

     output.close()
def rejester_run ( work_unit ) : #fname = 'verify-chunks-%d-%d' % (os.getpid(), time.time()) fname = work_unit . key . strip ( ) . split ( '/' ) [ - 1 ] output_dir_path = work_unit . data . get ( 'output_dir_path' , '/mnt' ) u = uuid . uuid3 ( uuid . UUID ( int = 0 ) , work_unit . key . strip ( ) ) path1 = u . hex [ 0 ] path2 = u . hex [ 1 ] fpath = os . path . join ( output_dir_path , path1 , path2 , fname ) if not os . path . exists ( os . path . dirname ( fpath ) ) : os . makedirs ( os . path . dirname ( fpath ) ) output = gzip . open ( fpath + '-out.gz' , 'wb' ) expected_si_count = int ( fname . split ( '-' ) [ 1 ] ) max_tries = 20 tries = 0 while tries < max_tries : try : exc , si_count , serif_count , clean_visible_bytes , clean_visible_count , stream_ids = attempt_fetch ( work_unit , fpath ) if si_count != expected_si_count : print 'retrying because si_count = %d != %d expected_si_count' % ( si_count , expected_si_count ) sys . stdout . flush ( ) tries += 1 continue else : print 'succeeded in reading si_count = %d' % ( si_count , ) sys . stdout . flush ( ) output . write ( '%s\t%d\t%d\t%d\t%d\t%s\t%s\n' % ( exc , si_count , serif_count , clean_visible_bytes , clean_visible_count , work_unit . key . strip ( ) , ',' . join ( [ '%s|%s' % tup for tup in stream_ids ] ) ) ) break except Exception , exc : print 'broken?' print traceback . format_exc ( exc ) sys . stdout . flush ( ) tries += 1 output . write ( traceback . format_exc ( exc ) ) output . close ( )
get a rejester.WorkUnit with KBA s3 path, fetch it, and save
     some counts about it.
get a rejester . WorkUnit with KBA s3 path fetch it and save some counts about it .