def read_csv(csv_file, ext='.csv', format=None, delete_empty_keys=False,
             fieldnames=[], rowlimit=100000000, numbers=False, normalize_names=True, unique_names=True,
             verbosity=0):
    r"""
    Read a csv file from a path or file pointer, returning a dict of lists, or list of lists (according to `format`)

    filename: a directory or list of file paths
    numbers: whether to attempt to convert strings in csv to numbers

    TODO:
        merge with `nlp.util.make_dataframe` function

    Handles unquoted and quoted strings, quoted commas, quoted newlines (EOLs), complex numbers,
        times, dates, datetimes

    >>> read_csv('"name\r\n",rank,"serial\nnumber",date <BR />\t\n"McCain, John","1","123456789",9/11/2001\n' +
    ...          'Bob,big cheese,1-23,1/1/2001 12:00 GMT', format='header+values list', numbers=True)
    [['name', 'rank', 'serial\nnumber', 'date'], ['McCain, John', 1.0, 123456789.0, '9/11/2001'],
    ['Bob', 'big cheese', '1-23', '1/1/2001 12:00 GMT']]
    """
    if not csv_file:
        return
    if isinstance(csv_file, basestring):
        # truncate `csv_file` in case it is a string buffer containing GBs of data
        path = csv_file[:1025]
        try:
            # see http://stackoverflow.com/a/4169762/623735 before trying 'rU'
            fpin = open(path, 'rUb')  # U = universal EOL reader, b = binary
        except (IOError, FileNotFoundError):
            # truncate path more, in case path is used later as a file description:
            path = csv_file[:128]
            fpin = StringIO(str(csv_file))
    else:
        fpin = csv_file
        try:
            path = csv_file.name
        except (IndexError, ValueError, AttributeError, TypeError):
            path = 'unknown file buffer path'

    format = format or 'h'
    format = format[0].lower()

    # if fieldnames not specified then assume that first row of csv contains headings
    csvr = csv.reader(fpin, dialect=csv.excel)
    if not fieldnames:
        while not fieldnames or not any(fieldnames):
            fieldnames = strip_br([str(s).strip() for s in next(csvr)])
        if verbosity > 0:
            logger.info('Column Labels: ' + repr(fieldnames))
    if unique_names:
        norm_names = OrderedDict([(fldnm, fldnm) for fldnm in fieldnames])
    else:
        norm_names = OrderedDict([(num, fldnm) for num, fldnm in enumerate(fieldnames)])
    if normalize_names:
        norm_names = OrderedDict([(num, make_name(fldnm, **make_name.DJANGO_FIELD)) for num, fldnm in enumerate(fieldnames)])
        # required for django-formatted json files
        model_name = make_name(path, **make_name.DJANGO_MODEL)
    if format in 'c':  # columnwise dict of lists
        recs = OrderedDict((norm_name, []) for norm_name in list(norm_names.values()))
    elif format in 'vh':
        recs = [fieldnames]
    else:
        recs = []
    if verbosity > 0:
        logger.info('Field Names: ' + repr(norm_names if normalize_names else fieldnames))
    rownum = 0
    eof = False
    pbar = None
    start_seek_pos = fpin.tell() or 0
    if verbosity > 1:
        print('Starting at byte {} in file buffer.'.format(start_seek_pos))
    fpin.seek(0, os.SEEK_END)
    file_len = fpin.tell() - start_seek_pos  # os.fstat(fpin.fileno()).st_size
    fpin.seek(start_seek_pos)

    if verbosity > 1:
        print(('There appear to be {} bytes remaining in the file buffer.' +
               'Resetting (seek) to starting position in file.').format(file_len))
    # if verbosity > 0:
    #     pbar = progressbar.ProgressBar(maxval=file_len)
    #     pbar.start()
    while csvr and rownum < rowlimit and not eof:
        if pbar:
            pbar.update(fpin.tell() - start_seek_pos)
        rownum += 1
        row = []
        row_dict = OrderedDict()
        # skip rows with all empty strings as values,
        while not row or not any(len(x) for x in row):
            try:
                row = next(csvr)
                if verbosity > 1:
                    logger.info('  row content: ' + repr(row))
            except StopIteration:
                eof = True
                break
        if eof:
            break
        if len(row) and isinstance(row[-1], basestring) and len(row[-1]):
            row = strip_br(row)
        if numbers:
            # try to convert the type to a numerical scalar type (int, float etc)
            row = [tryconvert(v, desired_types=NUMBERS_AND_DATETIMES, empty=None, default=v) for v in row]
        if row:
            N = min(max(len(row), 0), len(norm_names))
            row_dict = OrderedDict(
                ((field_name, field_value) for field_name, field_value in zip(
                    list(list(norm_names.values()) if unique_names else norm_names)[:N], row[:N])
                    if (str(field_name).strip() or delete_empty_keys is False))
            )
            if format in 'dj':  # django json format
                recs += [{"pk": rownum, "model": model_name, "fields": row_dict}]
            elif format in 'vhl':  # list of lists of values, with header row (list of str)
                recs += [[value for field_name, value in viewitems(row_dict)
                          if (field_name.strip() or delete_empty_keys is False)]]
            elif format in 'c':  # columnwise dict of lists
                for field_name in row_dict:
                    recs[field_name] += [row_dict[field_name]]
                if verbosity > 2:
                    print([recs[field_name][-1] for field_name in row_dict])
            else:
                recs += [row_dict]
            if verbosity > 2 and format not in 'c':
                print(recs[-1])

    if file_len > fpin.tell():
        logger.info("Only %d of %d bytes were read and processed." % (fpin.tell(), file_len))
    if pbar:
        pbar.finish()
    fpin.close()
    if not unique_names:
        return recs, norm_names
    return recs
def read_csv ( csv_file , ext = '.csv' , format = None , delete_empty_keys = False , fieldnames = [ ] , rowlimit = 100000000 , numbers = False , normalize_names = True , unique_names = True , verbosity = 0 ) : if not csv_file : return if isinstance ( csv_file , basestring ) : path = csv_file [ : 1025 ] try : fpin = open ( path , 'rUb' ) except ( IOError , FileNotFoundError ) : path = csv_file [ : 128 ] fpin = StringIO ( str ( csv_file ) ) else : fpin = csv_file try : path = csv_file . name except ( IndexError , ValueError , AttributeError , TypeError ) : path = 'unknown file buffer path' format = format or 'h' format = format [ 0 ] . lower ( ) csvr = csv . reader ( fpin , dialect = csv . excel ) if not fieldnames : while not fieldnames or not any ( fieldnames ) : fieldnames = strip_br ( [ str ( s ) . strip ( ) for s in next ( csvr ) ] ) if verbosity > 0 : logger . info ( 'Column Labels: ' + repr ( fieldnames ) ) if unique_names : norm_names = OrderedDict ( [ ( fldnm , fldnm ) for fldnm in fieldnames ] ) else : norm_names = OrderedDict ( [ ( num , fldnm ) for num , fldnm in enumerate ( fieldnames ) ] ) if normalize_names : norm_names = OrderedDict ( [ ( num , make_name ( fldnm , ** make_name . DJANGO_FIELD ) ) for num , fldnm in enumerate ( fieldnames ) ] ) model_name = make_name ( path , ** make_name . DJANGO_MODEL ) if format in 'c' : recs = OrderedDict ( ( norm_name , [ ] ) for norm_name in list ( norm_names . values ( ) ) ) elif format in 'vh' : recs = [ fieldnames ] else : recs = [ ] if verbosity > 0 : logger . info ( 'Field Names: ' + repr ( norm_names if normalize_names else fieldnames ) ) rownum = 0 eof = False pbar = None start_seek_pos = fpin . tell ( ) or 0 if verbosity > 1 : print ( 'Starting at byte {} in file buffer.' . format ( start_seek_pos ) ) fpin . seek ( 0 , os . SEEK_END ) file_len = fpin . tell ( ) - start_seek_pos fpin . seek ( start_seek_pos ) if verbosity > 1 : print ( ( 'There appear to be {} bytes remaining in the file buffer.' + 'Resetting (seek) to starting position in file.' ) . format ( file_len ) ) while csvr and rownum < rowlimit and not eof : if pbar : pbar . update ( fpin . tell ( ) - start_seek_pos ) rownum += 1 row = [ ] row_dict = OrderedDict ( ) while not row or not any ( len ( x ) for x in row ) : try : row = next ( csvr ) if verbosity > 1 : logger . info ( '  row content: ' + repr ( row ) ) except StopIteration : eof = True break if eof : break if len ( row ) and isinstance ( row [ - 1 ] , basestring ) and len ( row [ - 1 ] ) : row = strip_br ( row ) if numbers : row = [ tryconvert ( v , desired_types = NUMBERS_AND_DATETIMES , empty = None , default = v ) for v in row ] if row : N = min ( max ( len ( row ) , 0 ) , len ( norm_names ) ) row_dict = OrderedDict ( ( ( field_name , field_value ) for field_name , field_value in zip ( list ( list ( norm_names . values ( ) ) if unique_names else norm_names ) [ : N ] , row [ : N ] ) if ( str ( field_name ) . strip ( ) or delete_empty_keys is False ) ) ) if format in 'dj' : recs += [ { "pk" : rownum , "model" : model_name , "fields" : row_dict } ] elif format in 'vhl' : recs += [ [ value for field_name , value in viewitems ( row_dict ) if ( field_name . strip ( ) or delete_empty_keys is False ) ] ] elif format in 'c' : for field_name in row_dict : recs [ field_name ] += [ row_dict [ field_name ] ] if verbosity > 2 : print ( [ recs [ field_name ] [ - 1 ] for field_name in row_dict ] ) else : recs += [ row_dict ] if verbosity > 2 and format not in 'c' : print ( recs [ - 1 ] ) if file_len > fpin . tell ( ) : logger . info ( "Only %d of %d bytes were read and processed." % ( fpin . tell ( ) , file_len ) ) if pbar : pbar . finish ( ) fpin . close ( ) if not unique_names : return recs , norm_names return recs
r"""
    Read a csv file from a path or file pointer, returning a dict of lists, or list of lists (according to `format`)

    filename: a directory or list of file paths
    numbers: whether to attempt to convert strings in csv to numbers

    TODO:
        merge with `nlp.util.make_dataframe` function

    Handles unquoted and quoted strings, quoted commas, quoted newlines (EOLs), complex numbers,
        times, dates, datetimes

    >>> read_csv('"name\r\n",rank,"serial\nnumber",date <BR />\t\n"McCain, John","1","123456789",9/11/2001\n' +
    ...          'Bob,big cheese,1-23,1/1/2001 12:00 GMT', format='header+values list', numbers=True)
    [['name', 'rank', 'serial\nnumber', 'date'], ['McCain, John', 1.0, 123456789.0, '9/11/2001'],
    ['Bob', 'big cheese', '1-23', '1/1/2001 12:00 GMT']]