examples/verify_kba2014.py
def attempt_fetch(work_unit, fpath):
     '''attempt a fetch and iteration over a work_unit.key path in s3
     '''
     url = 'http://s3.amazonaws.com/aws-publicdatasets/' + work_unit.key.strip()

     ## cheapest way to iterate over the corpus is a few stages of
     ## streamed child processes.  Note that stderr needs to go
     ## separately to a file so that reading the stdin doesn't get
     ## blocked:
     cmd = '(wget -O - %s | gpg --no-permission-warning --trust-model always --output - --decrypt - | xz --decompress) 2> %s-err' % (url, fpath)
     print cmd
     child = Popen(cmd, stdout=PIPE, shell=True)
     print 'child launched'
     sys.stdout.flush()

     si_count = 0
     serif_count = 0
     exc = ''
     stream_ids = list()
     clean_visible_bytes = 0
     clean_visible_count = 0
     try:
         for si in Chunk(file_obj=child.stdout):
             print si.stream_id, si.abs_url
             if si.body.language:
                 lang = si.body.language.code
             else:
                 lang = ''
             stream_ids.append((lang, si.stream_id))
             if si.body.clean_visible:
                 clean_visible_count += 1
                 clean_visible_bytes += len(si.body.clean_visible)
             si_count += 1
             if 'serif' in si.body.sentences:
                 serif_count += 1
     except Exception, exc:
         exc = re.sub('\s+', ' ', str(exc)).strip()

     child.terminate()
     child.wait()
     child.stdout.close()
     return exc, si_count, serif_count, clean_visible_bytes, clean_visible_count, stream_ids
def attempt_fetch(work_unit, fpath):
     '''attempt a fetch and iteration over a work_unit.key path in s3
     '''
     url = 'http://s3.amazonaws.com/aws-publicdatasets/' + work_unit.key.strip()

     ## cheapest way to iterate over the corpus is a few stages of
     ## streamed child processes.  Note that stderr needs to go
     ## separately to a file so that reading the stdin doesn't get
     ## blocked:
     cmd = '(wget -O - %s | gpg --no-permission-warning --trust-model always --output - --decrypt - | xz --decompress) 2> %s-err' % (url, fpath)
     print cmd
     child = Popen(cmd, stdout=PIPE, shell=True)
     print 'child launched'
     sys.stdout.flush()

     si_count = 0
     serif_count = 0
     exc = ''
     stream_ids = list()
     clean_visible_bytes = 0
     clean_visible_count = 0
     try:
         for si in Chunk(file_obj=child.stdout):
             print si.stream_id, si.abs_url
             if si.body.language:
                 lang = si.body.language.code
             else:
                 lang = ''
             stream_ids.append((lang, si.stream_id))
             if si.body.clean_visible:
                 clean_visible_count += 1
                 clean_visible_bytes += len(si.body.clean_visible)
             si_count += 1
             if 'serif' in si.body.sentences:
                 serif_count += 1
     except Exception, exc:
         exc = re.sub('\s+', ' ', str(exc)).strip()

     child.terminate()
     child.wait()
     child.stdout.close()
     return exc, si_count, serif_count, clean_visible_bytes, clean_visible_count, stream_ids
def attempt_fetch ( work_unit , fpath ) : url = 'http://s3.amazonaws.com/aws-publicdatasets/' + work_unit . key . strip ( ) ## cheapest way to iterate over the corpus is a few stages of ## streamed child processes.  Note that stderr needs to go ## separately to a file so that reading the stdin doesn't get ## blocked: cmd = '(wget -O - %s | gpg --no-permission-warning --trust-model always --output - --decrypt - | xz --decompress) 2> %s-err' % ( url , fpath ) print cmd child = Popen ( cmd , stdout = PIPE , shell = True ) print 'child launched' sys . stdout . flush ( ) si_count = 0 serif_count = 0 exc = '' stream_ids = list ( ) clean_visible_bytes = 0 clean_visible_count = 0 try : for si in Chunk ( file_obj = child . stdout ) : print si . stream_id , si . abs_url if si . body . language : lang = si . body . language . code else : lang = '' stream_ids . append ( ( lang , si . stream_id ) ) if si . body . clean_visible : clean_visible_count += 1 clean_visible_bytes += len ( si . body . clean_visible ) si_count += 1 if 'serif' in si . body . sentences : serif_count += 1 except Exception , exc : exc = re . sub ( '\s+' , ' ' , str ( exc ) ) . strip ( ) child . terminate ( ) child . wait ( ) child . stdout . close ( ) return exc , si_count , serif_count , clean_visible_bytes , clean_visible_count , stream_ids
attempt a fetch and iteration over a work_unit.key path in s3
attempt a fetch and iteration over a work_unit . key path in s3