def update_items(item_seq,  batch_len=500, dry_run=True, start_batch=0, end_batch=None, ignore_errors=False, verbosity=1):
    """Given a sequence (queryset, generator, tuple, list) of dicts run the _update method on them and do bulk_update"""
    stats = collections.Counter()
    try:
        try:
            src_qs = item_seq.objects.all()
        except AttributeError:
            src_qs = item_seq.all()
        N = src_qs.count()
    except AttributeError:
        item_seq = iter(src_qs)
        print_exc()
        N = item_seq.count()

    if not N:
        if verbosity > 0:
            print 'No records found in %r' % src_qs
        return N

    # make sure there's a valid last batch number so the verbose messages will make sense
    end_batch = end_batch or int(N / float(batch_len))

    if verbosity > 0:
        print('Updating from a source queryset/model/sequence with %r records...' % N)
        widgets = [pb.Counter(), '/%d rows: ' % N or 1, pb.Percentage(), ' ', pb.RotatingMarker(), ' ', pb.Bar(),' ', pb.ETA()]
        pbar = pb.ProgressBar(widgets=widgets, maxval=N).start()

    for batch_num, obj_batch in enumerate(util.generate_batches(item_seq, batch_len=batch_len, start_batch=start_batch)):
        if start_batch + batch_num > end_batch:
            if verbosity > 1:
                print('Stopping before batch {0} because it is not between {1} and {2}'.format(batch_num, start_batch, end_batch))
            break
        for obj in obj_batch:
            if verbosity > 2:
                print(repr(obj))
            try:
                if hasattr(obj, '_update'):
                    obj._update(save=False, overwrite=False)
            except:
                if verbosity > 0:
                    print_exc()
                    print 'ERROR: Unable to update record: %r' % obj
                pass
        if verbosity and verbosity < 2:
            pbar.update(batch_num * batch_len + len(obj_batch))
        elif verbosity > 1:
            print('Writing {0} items (of type {1}) from batch {2}. Will stop at batch {3} which is records {4} - {5} ...'.format(
                len(obj_batch), src_qs.model, batch_num, end_batch, min(batch_len * (start_batch + end_batch), N)
                ))
        if not dry_run:
            try:
                bulk_update(obj_batch, ignore_errors=ignore_errors, delete_first=True, verbosity=verbosity)
            except Exception as err:
                from django.db import transaction
                transaction.rollback()
                print '%s' % err
                print 'Attempting to save objects one at a time instead of as a batch...'
                for obj in obj_batch:
                    try:
                        obj.save()
                        stats += collections.Counter(['batch_Error'])
                    except:
                        from django.db import transaction
                        transaction.rollback()
                        print str(obj)
                        print repr(obj.__dict__)
                        print_exc()
                        stats += collections.Counter(['save_Error'])
                if not ignore_errors:
                    print_exc()
                    raise
        if batch_num < end_batch:
            if len(obj_batch) != batch_len:
                stats += collections.Counter(['batch_len={0}'.format(len(obj_batch))])
            print('Retrieving {0} {1} items for the next batch, batch number {2}...'.format(
                batch_len, src_qs.model, batch_num + 1))

    if verbosity > 0:
        pbar.finish()
    return stats
def update_items ( item_seq , batch_len = 500 , dry_run = True , start_batch = 0 , end_batch = None , ignore_errors = False , verbosity = 1 ) : stats = collections . Counter ( ) try : try : src_qs = item_seq . objects . all ( ) except AttributeError : src_qs = item_seq . all ( ) N = src_qs . count ( ) except AttributeError : item_seq = iter ( src_qs ) print_exc ( ) N = item_seq . count ( ) if not N : if verbosity > 0 : print 'No records found in %r' % src_qs return N end_batch = end_batch or int ( N / float ( batch_len ) ) if verbosity > 0 : print ( 'Updating from a source queryset/model/sequence with %r records...' % N ) widgets = [ pb . Counter ( ) , '/%d rows: ' % N or 1 , pb . Percentage ( ) , ' ' , pb . RotatingMarker ( ) , ' ' , pb . Bar ( ) , ' ' , pb . ETA ( ) ] pbar = pb . ProgressBar ( widgets = widgets , maxval = N ) . start ( ) for batch_num , obj_batch in enumerate ( util . generate_batches ( item_seq , batch_len = batch_len , start_batch = start_batch ) ) : if start_batch + batch_num > end_batch : if verbosity > 1 : print ( 'Stopping before batch {0} because it is not between {1} and {2}' . format ( batch_num , start_batch , end_batch ) ) break for obj in obj_batch : if verbosity > 2 : print ( repr ( obj ) ) try : if hasattr ( obj , '_update' ) : obj . _update ( save = False , overwrite = False ) except : if verbosity > 0 : print_exc ( ) print 'ERROR: Unable to update record: %r' % obj pass if verbosity and verbosity < 2 : pbar . update ( batch_num * batch_len + len ( obj_batch ) ) elif verbosity > 1 : print ( 'Writing {0} items (of type {1}) from batch {2}. Will stop at batch {3} which is records {4} - {5} ...' . format ( len ( obj_batch ) , src_qs . model , batch_num , end_batch , min ( batch_len * ( start_batch + end_batch ) , N ) ) ) if not dry_run : try : bulk_update ( obj_batch , ignore_errors = ignore_errors , delete_first = True , verbosity = verbosity ) except Exception as err : from django . db import transaction transaction . rollback ( ) print '%s' % err print 'Attempting to save objects one at a time instead of as a batch...' for obj in obj_batch : try : obj . save ( ) stats += collections . Counter ( [ 'batch_Error' ] ) except : from django . db import transaction transaction . rollback ( ) print str ( obj ) print repr ( obj . __dict__ ) print_exc ( ) stats += collections . Counter ( [ 'save_Error' ] ) if not ignore_errors : print_exc ( ) raise if batch_num < end_batch : if len ( obj_batch ) != batch_len : stats += collections . Counter ( [ 'batch_len={0}' . format ( len ( obj_batch ) ) ] ) print ( 'Retrieving {0} {1} items for the next batch, batch number {2}...' . format ( batch_len , src_qs . model , batch_num + 1 ) ) if verbosity > 0 : pbar . finish ( ) return stats
Given a sequence (queryset, generator, tuple, list) of dicts run the _update method on them and do bulk_update