def upload_chunked(server, docs, chunksize=1000, preprocess=None):
 start = 0
 for chunk in grouper(docs, chunksize):
  end = (start + len(chunk))
  logger.info(('uploading documents %i-%i' % (start, (end - 1))))
  if (preprocess is not None):
   pchunk = []
   for doc in chunk:
    doc['tokens'] = preprocess(doc['text'])
    del doc['text']
    pchunk.append(doc)
   chunk = pchunk
  server.buffer(chunk)
  start = end

def upload_chunked ( server , docs , chunksize = 1000 , preprocess = None ) : start = 0 for chunk in grouper ( docs , chunksize ) : end = ( start + len ( chunk ) ) logger . info ( ( 'uploading documents %i-%i' % ( start , ( end - 1 ) ) ) ) if ( preprocess is not None ) : pchunk = [ ] for doc in chunk : doc [ 'tokens' ] = preprocess ( doc [ 'text' ] ) del doc [ 'text' ] pchunk . append ( doc ) chunk = pchunk server . buffer ( chunk ) start = end
memory-friendly upload of documents to a simserver .