Bulk create database records from batches of rows in a csv file.

def load_csv_to_model(path, model, field_names=None, delimiter=None, batch_len=10000, 
                      dialect=None, num_header_rows=1, mode='rUb',
                      strip=True, clear=False, dry_run=True, ignore_errors=True, verbosity=2):
    '''Bulk create database records from batches of rows in a csv file.'''

    reader_kwargs = {}
    errors = collections.Counter()
    if delimiter or dialect:
        reader_kwargs['dialect'] = dialect or 'excel'
        if delimiter:
            reader_kwargs['delimiter'] = delimiter
    reader_kwargs['delimiter'] = str(reader_kwargs['delimiter'][0])
    delimiter = reader_kwargs['delimiter']

    path = path or './'
    if not delimiter:
        for d in ',', '|', '\t', ';':
            try:
                return load_csv_to_model(path=path, model=model, field_names=field_names, delimiter=d, batch_len=batch_len,
                                         dialect=dialect, num_header_rows=num_header_rows,
                                         strip=strip, clear=clear, dry_run=dry_run, ignore_errors=ignore_errors, verbosity=verbosity)
            except:
                pass
        return None

    if clear:
        clear_model(model, dry_run=dry_run, verbosity=verbosity)

    M = 0
    with open(path, mode) as f:
        reader = csv.reader(f, **reader_kwargs)
        header_rows = []
        i = 0
        while len(header_rows) < num_header_rows and i < 100:
            row = reader.next()
            i += 1
            if not row or any(compiled_regex.match(row[0]) for compiled_regex in header_rows_to_ignore):
                if verbosity > 1:
                    print 'IGNORED: %r' % row
            else:
                header_rows += [row]
        if verbosity > 2:
            print 'HEADER: %r' % header_rows
        if verbosity > 0:
            N = count_lines(path, mode) - i + 10  # + 10 fudge factor in case multiple newlines in a single csv row
            widgets = [pb.Counter(), '/%d lines: ' % N, pb.Percentage(), ' ', pb.RotatingMarker(), ' ', pb.Bar(),' ', pb.ETA()]
            i, pbar = 0, pb.ProgressBar(widgets=widgets, maxval=N).start()
        if verbosity > 3:
            print 'Generating all the batches before iterating may take a while...'
        for batch_num, batch_of_rows in enumerate(util.generate_batches(reader, batch_len)):
            if verbosity > 2:
                print 
                print i
            batch_of_objects = []
            for j, row in enumerate(batch_of_rows):
                if verbosity > 3:
                    print j, row
                if not row or all(not el for el in row):
                    if verbosity > 2:
                        print 'IGNORED: %r' % row
                    continue
                if verbosity or not ignore_errors:
                    M = M or len(row)
                    if len(row) != M:
                        print 'ERROR importing row #%d in batch_num=%d which is row #%d overall. The row had %d columns, but previous rows had %d.' % (j + 1, batch_num + 1, i + j + 1, len(row), M)
                        print 'Erroneously parsed row:'
                        print repr(row)
                        
                        if not ignore_errors:
                            raise ValueError('ERROR importing row #%d which had %d columns, but previous rows had %d.' % (i + j + 1, len(row), M))
                try:
                    obj, row_errors = django_object_from_row(row, model=model, field_names=field_names, strip=strip, verbosity=verbosity)
                    batch_of_objects += [obj]
                    errors += row_errors
                except:
                    if verbosity > 0:
                        print 'Error importing row #%d' % (i + j + 1)
                        print_exc()
                    if not ignore_errors:
                        raise
                if verbosity > 0:
                    try:
                        pbar.update(i + j)
                    except:
                        print_exc()
                        if not ignore_errors:
                            raise
            i += len(batch_of_rows)
            if not dry_run:
                model.objects.bulk_create(batch_of_objects)
            elif verbosity > 0:
                print "DRY_RUN: NOT bulk creating batch of %d records in %r" % (len(batch_of_objects), model)
        if verbosity > 0:
            pbar.finish()
    return i