def process_items(r, keys, timeout, limit=0, log_every=1000, wait=0.1):
 limit = (limit or float(u'inf'))
 processed = 0
 while (processed < limit):
  ret = r.blpop(keys, timeout)
  if (ret is None):
   time.sleep(wait)
   continue
  (source, data) = ret
  try:
   item = json.loads(data)
  except Exception:
   logger.exception(u'Failed to load item:\n%r', pprint.pformat(data))
   continue
  try:
   name = (item.get(u'name') or item.get(u'title'))
   url = (item.get(u'url') or item.get(u'link'))
   logger.debug(u'[%s] Processing item: %s <%s>', source, name, url)
  except KeyError:
   logger.exception(u'[%s] Failed to process item:\n%r', source, pprint.pformat(item))
   continue
  processed += 1
  if ((processed % log_every) == 0):
   logger.info(u'Processed %s items', processed)

def process_items ( r , keys , timeout , limit = 0 , log_every = 1000 , wait = 0.1 ) : limit = ( limit or float ( u'inf' ) ) processed = 0 while ( processed < limit ) : ret = r . blpop ( keys , timeout ) if ( ret is None ) : time . sleep ( wait ) continue ( source , data ) = ret try : item = json . loads ( data ) except Exception : logger . exception ( u'Failed to load item:\n%r' , pprint . pformat ( data ) ) continue try : name = ( item . get ( u'name' ) or item . get ( u'title' ) ) url = ( item . get ( u'url' ) or item . get ( u'link' ) ) logger . debug ( u'[%s] Processing item: %s <%s>' , source , name , url ) except KeyError : logger . exception ( u'[%s] Failed to process item:\n%r' , source , pprint . pformat ( item ) ) continue processed += 1 if ( ( processed % log_every ) == 0 ) : logger . info ( u'Processed %s items' , processed )
process items from a redis queue .