Import a sequence (queryset.values(), generator, tuple, list) of dicts into the given model

def import_items(item_seq, dest_model,  batch_len=500, 
                 clear=False, dry_run=True, 
                 start_batch=0, end_batch=None, 
                 overwrite=True,
                 run_update=False, ignore_related=True,
                 ignore_errors=False, verbosity=1):
    """Import a sequence (queryset.values(), generator, tuple, list) of dicts into the given model

    """
    if isinstance(dest_model, (djmodels.query.QuerySet, djmodels.Manager)):
        dest_qs = dest_model.all()
        dest_model = get_model(dest_qs)
    else:
        dest_qs = dest_model.objects.all()

    stats = collections.Counter()
    try:
        try:
            src_qs = item_seq.objects.all()
        except AttributeError:
            src_qs = item_seq.all()
        N = src_qs.count()
        item_seq = src_qs.values()
    except AttributeError as e:
        print_exc()
        if not ignore_errors:
            raise e
        N = len(item_seq)

    if not N:
        if verbosity > 0:
            print 'No records found in %r' % src_qs
        return N

    # make sure there's a valid last batch number so the verbose messages will make sense
    end_batch = end_batch or int(N / float(batch_len))

    if clear and not dry_run:
        if N < dest_qs.count():
            if verbosity > 0:
                print "WARNING: There are %d %r records in the destinsation queryset which is more than the %d records in the source data. So no records will be deleted/cleared in the destination!" % (dest_qs.count(), dest_model, N)

        if verbosity > 0:
            print "WARNING: Deleting %d records from %r to make room for %d new records !!!!!!!" % (dest_qs.count(), dest_model, N)
        num_deleted = delete_in_batches(dest_qs)
        if verbosity > 0:
            print "Finished deleting %d records in %r." % (num_deleted, dest_model)

    if verbosity > 0:
        print('Loading %r records from sequence provided...' % N)
        widgets = [pb.Counter(), '/%d rows: ' % N or 1, pb.Percentage(), ' ', pb.RotatingMarker(), ' ', pb.Bar(),' ', pb.ETA()]
        pbar = pb.ProgressBar(widgets=widgets, maxval=N)

    for batch_num, dict_batch in enumerate(util.generate_slices(item_seq, batch_len=batch_len, start_batch=start_batch)):
        if start_batch + batch_num > end_batch:
            if verbosity > 1:
                print('Stopping before batch {0} because it is not between {1} and {2}'.format(start_batch + batch_num, start_batch, end_batch))
            break
        if verbosity > 2:
            print '-------- dict batch ------'
            # print(repr(dict_batch))
            print(repr((start_batch + batch_num, len(dict_batch), batch_len)))
        item_batch = []
        total_len_batches = 0

        # convert an iterable of Django ORM record dictionaries into a list of Django ORM objects
        for d in dict_batch:
            if verbosity > 2:
                print '-------- dict of source obj ------'
                print(repr(d))
            obj = dest_model()
            try:
                # if the model has an import_item method then use it
                obj.import_item(d, verbosity=verbosity)
            except:
                if verbosity > 2:
                    print '------ Creating a new %r instance --------' % dest_model
                obj, row_errors = django_object_from_row(d, dest_model, ignore_related=ignore_related, verbosity=verbosity)
                if verbosity > 2:
                    print 'new obj.__dict__: %r' % obj.__dict__
            if run_update:
                try:
                    if verbosity > 2:
                        print '------ Updating FKs with overwrite=%r --------' % overwrite
                    obj._update(save=False, overwrite=overwrite)
                except:
                    if verbosity > 0:
                        print_exc()
                        print 'ERROR: Unable to update record: %r' % obj
                    pass
            item_batch += [obj]
            stats += row_errors

        del(dict_batch)
        if verbosity and verbosity < 2:
            if batch_num:
                pbar.update(batch_num * batch_len + len(item_batch))
            else:
                # don't start the progress bar until at least one batch has been loaded
                pbar.start()
        elif verbosity > 1:
            print('Writing {0} items (of type {1}) from batch {2}. Will stop at batch {3} which is record {4} ...'.format(
                len(item_batch), dest_model, start_batch + batch_num, end_batch, min(batch_len * (start_batch + end_batch), N),
                ))
        total_len_batches += len(item_batch)

        # use bulk_create to make fast DB insertions. Note: any custom save() or _update() methods will *NOT* be run
        if not dry_run:
            try:
                dest_model.objects.bulk_create(item_batch)
            except UnicodeDecodeError as err:
                from django.db import transaction
                transaction.rollback()
                if verbosity > 0:
                    print '%s' % err
                    print 'Now attempting to save objects one at a time instead of as a batch...'
                for obj in item_batch:
                    try:
                        obj.save()
                        stats += collections.Counter(['batch_UnicodeDecodeError'])
                    except:
                        from django.db import transaction
                        transaction.rollback()
                        stats += collections.Counter(['save_UnicodeDecodeError'])
                        print str(obj)
                        print repr(obj.__dict__)
                if not ignore_errors:
                    print_exc()
                    raise
            except Exception as err:
                from django.db import transaction
                transaction.rollback()
                if verbosity > 0:
                    print '%s' % err
                    print 'Now attempting to save objects one at a time instead of as a batch...'
                for obj in item_batch:
                    try:
                        obj.save()
                        stats += collections.Counter(['batch_Error'])
                    except:
                        from django.db import transaction
                        transaction.rollback()
                        print str(obj)
                        print repr(obj.__dict__)
                        print_exc()
                        stats += collections.Counter(['save_Error'])
                if not ignore_errors:
                    print_exc()
                    raise
        if batch_num < end_batch:
            if len(item_batch) != batch_len:
                stats += collections.Counter(['batch_len={0}'.format(len(item_batch))])
            print('Retrieving {0} {1} items for the next batch, batch number {2}...'.format(
                  batch_len, src_qs.model, batch_num + 1))
        del(item_batch)

    if verbosity > 0:
        pbar.finish()
    return stats