Like index_model_field except uses 50x less memory and 10x more processing cycles

    Returns 2 dicts where both the keys and values are tuples:

    target_index = {(<key_fields[0]>, <key_fields[1]>, ...): (<value_fields[0]>,)} for all distinct model-serial pairs in the Sales queryset
    target_dupes = {(<key_fields[0]>, <key_fields[1]>, ...): [(<value_fields[1]>,), (<value_fields[2]>,), ...]}  with all the duplicates except the first pk already listed above

def index_model_field_batches(model_or_queryset, key_fields=['model_number', 'serial_number'], value_fields=['pk'], 
    key_formatter=lambda x: str.lstrip(str.strip(str(x or '')), '0'), 
    value_formatter=lambda x: str.strip(str(x)), batch_len=10000,
    limit=100000000, verbosity=1):
    '''Like index_model_field except uses 50x less memory and 10x more processing cycles

    Returns 2 dicts where both the keys and values are tuples:

    target_index = {(<key_fields[0]>, <key_fields[1]>, ...): (<value_fields[0]>,)} for all distinct model-serial pairs in the Sales queryset
    target_dupes = {(<key_fields[0]>, <key_fields[1]>, ...): [(<value_fields[1]>,), (<value_fields[2]>,), ...]}  with all the duplicates except the first pk already listed above
    '''

    qs = djdb.get_queryset(model_or_queryset)

    N = qs.count()
    if verbosity > 0:
        print 'Indexing %d rows (database records) to aid in finding record %r values using the field %r.' % (N, value_fields, key_fields)

    index, dupes, rownum = {}, {}, 0

    pbar, rownum = None, 0
    if verbosity and N > min(1000000, max(0, 100000**(1./verbosity))):
        widgets = [pb.Counter(), '/%d rows: ' % N, pb.Percentage(), ' ', pb.RotatingMarker(), ' ', pb.Bar(),' ', pb.ETA()]
        pbar = pb.ProgressBar(widgets=widgets, maxval=N).start()


    # to determine the type of the field value and decide whether to strip() or normalize in any way
    #obj0 = qs.filter(**{field + '__isnull': False}).all()[0]

    value_fields = util.listify(value_fields)
    key_fields = util.listify(key_fields)

    for batch in djdb.generate_queryset_batches(qs, batch_len=batch_len, verbosity=verbosity):
        for obj in batch:
            # print obj
            # normalize the key
            keys = []
            for kf in key_fields:
                k = getattr(obj, kf)
                keys += [key_formatter(k or '')]
            values = []
            keys = tuple(keys)
            for vf in value_fields:
                v = getattr(obj, vf)
                values += [value_formatter(v or '')]
            values = tuple(values)           

            if keys in index:
                dupes[keys] = dupes.get(keys, []) + [values]
            else:
                index[keys] = values
            # print rownum  / float(N)
            if pbar:
                pbar.update(rownum)
            rownum += 1
            if rownum >= limit:
                break
    if pbar:
        pbar.finish()
    if verbosity > 0:
        print 'Found %d duplicate %s values among the %d records or %g%%' % (len(dupes), key_fields, len(index), len(dupes)*100./(len(index) or 1.))
    return index, dupes