r"""
    Read a csv file from a path or file pointer, returning a dict of lists, or list of lists (according to `format`)

    filename: a directory or list of file paths
    numbers: whether to attempt to convert strings in csv to numbers

    TODO:
        merge with `nlp.util.make_dataframe` function

    Handles unquoted and quoted strings, quoted commas, quoted newlines (EOLs), complex numbers,
        times, dates, datetimes

    >>> read_csv('"name\r\n",rank,"serial\nnumber",date <BR />\t\n"McCain, John","1","123456789",9/11/2001\n' +
    ...          'Bob,big cheese,1-23,1/1/2001 12:00 GMT', format='header+values list', numbers=True)
    [['name', 'rank', 'serial\nnumber', 'date'], ['McCain, John', 1.0, 123456789.0, '9/11/2001'],
    ['Bob', 'big cheese', '1-23', '1/1/2001 12:00 GMT']]

def read_csv(csv_file, ext='.csv', format=None, delete_empty_keys=False,
             fieldnames=[], rowlimit=100000000, numbers=False, normalize_names=True, unique_names=True,
             verbosity=0):
    r"""
    Read a csv file from a path or file pointer, returning a dict of lists, or list of lists (according to `format`)

    filename: a directory or list of file paths
    numbers: whether to attempt to convert strings in csv to numbers

    TODO:
        merge with `nlp.util.make_dataframe` function

    Handles unquoted and quoted strings, quoted commas, quoted newlines (EOLs), complex numbers,
        times, dates, datetimes

    >>> read_csv('"name\r\n",rank,"serial\nnumber",date <BR />\t\n"McCain, John","1","123456789",9/11/2001\n' +
    ...          'Bob,big cheese,1-23,1/1/2001 12:00 GMT', format='header+values list', numbers=True)
    [['name', 'rank', 'serial\nnumber', 'date'], ['McCain, John', 1.0, 123456789.0, '9/11/2001'],
    ['Bob', 'big cheese', '1-23', '1/1/2001 12:00 GMT']]
    """
    if not csv_file:
        return
    if isinstance(csv_file, basestring):
        # truncate `csv_file` in case it is a string buffer containing GBs of data
        path = csv_file[:1025]
        try:
            # see http://stackoverflow.com/a/4169762/623735 before trying 'rU'
            fpin = open(path, 'rUb')  # U = universal EOL reader, b = binary
        except (IOError, FileNotFoundError):
            # truncate path more, in case path is used later as a file description:
            path = csv_file[:128]
            fpin = StringIO(str(csv_file))
    else:
        fpin = csv_file
        try:
            path = csv_file.name
        except (IndexError, ValueError, AttributeError, TypeError):
            path = 'unknown file buffer path'

    format = format or 'h'
    format = format[0].lower()

    # if fieldnames not specified then assume that first row of csv contains headings
    csvr = csv.reader(fpin, dialect=csv.excel)
    if not fieldnames:
        while not fieldnames or not any(fieldnames):
            fieldnames = strip_br([str(s).strip() for s in next(csvr)])
        if verbosity > 0:
            logger.info('Column Labels: ' + repr(fieldnames))
    if unique_names:
        norm_names = OrderedDict([(fldnm, fldnm) for fldnm in fieldnames])
    else:
        norm_names = OrderedDict([(num, fldnm) for num, fldnm in enumerate(fieldnames)])
    if normalize_names:
        norm_names = OrderedDict([(num, make_name(fldnm, **make_name.DJANGO_FIELD)) for num, fldnm in enumerate(fieldnames)])
        # required for django-formatted json files
        model_name = make_name(path, **make_name.DJANGO_MODEL)
    if format in 'c':  # columnwise dict of lists
        recs = OrderedDict((norm_name, []) for norm_name in list(norm_names.values()))
    elif format in 'vh':
        recs = [fieldnames]
    else:
        recs = []
    if verbosity > 0:
        logger.info('Field Names: ' + repr(norm_names if normalize_names else fieldnames))
    rownum = 0
    eof = False
    pbar = None
    start_seek_pos = fpin.tell() or 0
    if verbosity > 1:
        print('Starting at byte {} in file buffer.'.format(start_seek_pos))
    fpin.seek(0, os.SEEK_END)
    file_len = fpin.tell() - start_seek_pos  # os.fstat(fpin.fileno()).st_size
    fpin.seek(start_seek_pos)

    if verbosity > 1:
        print(('There appear to be {} bytes remaining in the file buffer.' +
               'Resetting (seek) to starting position in file.').format(file_len))
    # if verbosity > 0:
    #     pbar = progressbar.ProgressBar(maxval=file_len)
    #     pbar.start()
    while csvr and rownum < rowlimit and not eof:
        if pbar:
            pbar.update(fpin.tell() - start_seek_pos)
        rownum += 1
        row = []
        row_dict = OrderedDict()
        # skip rows with all empty strings as values,
        while not row or not any(len(x) for x in row):
            try:
                row = next(csvr)
                if verbosity > 1:
                    logger.info('  row content: ' + repr(row))
            except StopIteration:
                eof = True
                break
        if eof:
            break
        if len(row) and isinstance(row[-1], basestring) and len(row[-1]):
            row = strip_br(row)
        if numbers:
            # try to convert the type to a numerical scalar type (int, float etc)
            row = [tryconvert(v, desired_types=NUMBERS_AND_DATETIMES, empty=None, default=v) for v in row]
        if row:
            N = min(max(len(row), 0), len(norm_names))
            row_dict = OrderedDict(
                ((field_name, field_value) for field_name, field_value in zip(
                    list(list(norm_names.values()) if unique_names else norm_names)[:N], row[:N])
                    if (str(field_name).strip() or delete_empty_keys is False))
            )
            if format in 'dj':  # django json format
                recs += [{"pk": rownum, "model": model_name, "fields": row_dict}]
            elif format in 'vhl':  # list of lists of values, with header row (list of str)
                recs += [[value for field_name, value in viewitems(row_dict)
                          if (field_name.strip() or delete_empty_keys is False)]]
            elif format in 'c':  # columnwise dict of lists
                for field_name in row_dict:
                    recs[field_name] += [row_dict[field_name]]
                if verbosity > 2:
                    print([recs[field_name][-1] for field_name in row_dict])
            else:
                recs += [row_dict]
            if verbosity > 2 and format not in 'c':
                print(recs[-1])

    if file_len > fpin.tell():
        logger.info("Only %d of %d bytes were read and processed." % (fpin.tell(), file_len))
    if pbar:
        pbar.finish()
    fpin.close()
    if not unique_names:
        return recs, norm_names
    return recs