def _read_csv_impl(cls,
                       url,
                       delimiter=',',
                       header=True,
                       error_bad_lines=False,
                       comment_char='',
                       escape_char='\\',
                       double_quote=True,
                       quote_char='\"',
                       skip_initial_space=True,
                       column_type_hints=None,
                       na_values=["NA"],
                       line_terminator="\n",
                       usecols=[],
                       nrows=None,
                       skiprows=0,
                       verbose=True,
                       store_errors=True,
                       nrows_to_infer=100,
                       true_values=[],
                       false_values=[],
                       _only_raw_string_substitutions=False,
                       **kwargs):
        """
        Constructs an SFrame from a CSV file or a path to multiple CSVs, and
        returns a pair containing the SFrame and optionally
        (if store_errors=True) a dict of filenames to SArrays
        indicating for each file, what are the incorrectly parsed lines
        encountered.

        Parameters
        ----------
        store_errors : bool
            If true, the output errors dict will be filled.

        See `read_csv` for the rest of the parameters.
        """
        # Pandas argument compatibility
        if "sep" in kwargs:
            delimiter = kwargs['sep']
            del kwargs['sep']
        if "quotechar" in kwargs:
            quote_char = kwargs['quotechar']
            del kwargs['quotechar']
        if "doublequote" in kwargs:
            double_quote = kwargs['doublequote']
            del kwargs['doublequote']
        if "comment" in kwargs:
            comment_char = kwargs['comment']
            del kwargs['comment']
            if comment_char is None:
                comment_char = ''
        if "lineterminator" in kwargs:
            line_terminator = kwargs['lineterminator']
            del kwargs['lineterminator']
        if len(kwargs) > 0:
            raise TypeError("Unexpected keyword arguments " + str(kwargs.keys()))

        parsing_config = dict()
        parsing_config["delimiter"] = delimiter
        parsing_config["use_header"] = header
        parsing_config["continue_on_failure"] = not error_bad_lines
        parsing_config["comment_char"] = comment_char
        parsing_config["escape_char"] = '\0' if escape_char is None else escape_char
        parsing_config["use_escape_char"] = escape_char is None
        parsing_config["double_quote"] = double_quote
        parsing_config["quote_char"] = quote_char
        parsing_config["skip_initial_space"] = skip_initial_space
        parsing_config["store_errors"] = store_errors
        parsing_config["line_terminator"] = line_terminator
        parsing_config["output_columns"] = usecols
        parsing_config["skip_rows"] =skiprows
        parsing_config["true_values"] = true_values
        parsing_config["false_values"] = false_values
        parsing_config["only_raw_string_substitutions"] = _only_raw_string_substitutions 

        if type(na_values) is str:
          na_values = [na_values]
        if na_values is not None and len(na_values) > 0:
            parsing_config["na_values"] = na_values

        if nrows is not None:
          parsing_config["row_limit"] = nrows

        proxy = UnitySFrameProxy()
        internal_url = _make_internal_url(url)

        # Attempt to automatically detect the column types. Either produce a
        # list of types; otherwise default to all str types.
        column_type_inference_was_used = False
        if column_type_hints is None:
            try:
                # Get the first nrows_to_infer rows (using all the desired arguments).
                first_rows = SFrame.read_csv(url, nrows=nrows_to_infer,
                                 column_type_hints=type(None),
                                 header=header,
                                 delimiter=delimiter,
                                 comment_char=comment_char,
                                 escape_char=escape_char,
                                 double_quote=double_quote,
                                 quote_char=quote_char,
                                 skip_initial_space=skip_initial_space,
                                 na_values=na_values,
                                 line_terminator=line_terminator,
                                 usecols=usecols,
                                 skiprows=skiprows,
                                 verbose=verbose,
                                 true_values=true_values,
                                 false_values=false_values,
                                 _only_raw_string_substitutions=_only_raw_string_substitutions)
                column_type_hints = SFrame._infer_column_types_from_lines(first_rows)
                typelist = '[' + ','.join(t.__name__ for t in column_type_hints) + ']'
                if verbose:
                    print("------------------------------------------------------")
                    print("Inferred types from first %d line(s) of file as " % nrows_to_infer)
                    print("column_type_hints="+ typelist)
                    print("If parsing fails due to incorrect types, you can correct")
                    print("the inferred type list above and pass it to read_csv in")
                    print( "the column_type_hints argument")
                    print("------------------------------------------------------")
                column_type_inference_was_used = True
            except RuntimeError as e:
                if type(e) == RuntimeError and ("cancel" in str(e.args[0]) or "Cancel" in str(e.args[0])):
                    raise e
                # If the above fails, default back to str for all columns.
                column_type_hints = str
                if verbose:
                    print('Could not detect types. Using str for each column.')

        if type(column_type_hints) is type:
            type_hints = {'__all_columns__': column_type_hints}
        elif type(column_type_hints) is list:
            type_hints = dict(list(zip(['__X%d__' % i for i in range(len(column_type_hints))], column_type_hints)))
        elif type(column_type_hints) is dict:
            # we need to fill in a potentially incomplete dictionary
            try:
                # Get the first nrows_to_infer rows (using all the desired arguments).
                first_rows = SFrame.read_csv(url, nrows=nrows_to_infer,
                                 column_type_hints=type(None),
                                 header=header,
                                 delimiter=delimiter,
                                 comment_char=comment_char,
                                 escape_char=escape_char,
                                 double_quote=double_quote,
                                 quote_char=quote_char,
                                 skip_initial_space=skip_initial_space,
                                 na_values=na_values,
                                 line_terminator=line_terminator,
                                 usecols=usecols,
                                 skiprows=skiprows,
                                 verbose=verbose,
                                 true_values=true_values,
                                 false_values=false_values,
                                 _only_raw_string_substitutions=_only_raw_string_substitutions)
                inferred_types = SFrame._infer_column_types_from_lines(first_rows)
                # make a dict of column_name to type
                inferred_types = dict(list(zip(first_rows.column_names(), inferred_types)))
                # overwrite with the user's specified types
                for key in column_type_hints:
                    inferred_types[key] = column_type_hints[key]
                column_type_hints = inferred_types
            except RuntimeError as e:
                if type(e) == RuntimeError and ("cancel" in str(e) or "Cancel" in str(e)):
                    raise e
                # If the above fails, default back to str for unmatched columns
                if verbose:
                    print('Could not detect types. Using str for all unspecified columns.')
            type_hints = column_type_hints
        else:
            raise TypeError("Invalid type for column_type_hints. Must be a dictionary, list or a single type.")

        try:
            if (not verbose):
                glconnect.get_server().set_log_progress(False)
            with cython_context():
                errors = proxy.load_from_csvs(internal_url, parsing_config, type_hints)
        except Exception as e:
            if type(e) == RuntimeError and "CSV parsing cancelled" in str(e.args[0]):
                raise e
            if column_type_inference_was_used:
                # try again
                if verbose:
                    print("Unable to parse the file with automatic type inference.")
                    print("Defaulting to column_type_hints=str")
                type_hints = {'__all_columns__': str}
                try:
                    with cython_context():
                        errors = proxy.load_from_csvs(internal_url, parsing_config, type_hints)
                except:
                    glconnect.get_server().set_log_progress(True)
                    raise
            else:
                glconnect.get_server().set_log_progress(True)
                raise

        glconnect.get_server().set_log_progress(True)

        return (cls(_proxy=proxy), { f: SArray(_proxy = es) for (f, es) in errors.items() })
def _read_csv_impl ( cls , url , delimiter = ',' , header = True , error_bad_lines = False , comment_char = '' , escape_char = '\\' , double_quote = True , quote_char = '\"' , skip_initial_space = True , column_type_hints = None , na_values = [ "NA" ] , line_terminator = "\n" , usecols = [ ] , nrows = None , skiprows = 0 , verbose = True , store_errors = True , nrows_to_infer = 100 , true_values = [ ] , false_values = [ ] , _only_raw_string_substitutions = False , ** kwargs ) : if "sep" in kwargs : delimiter = kwargs [ 'sep' ] del kwargs [ 'sep' ] if "quotechar" in kwargs : quote_char = kwargs [ 'quotechar' ] del kwargs [ 'quotechar' ] if "doublequote" in kwargs : double_quote = kwargs [ 'doublequote' ] del kwargs [ 'doublequote' ] if "comment" in kwargs : comment_char = kwargs [ 'comment' ] del kwargs [ 'comment' ] if comment_char is None : comment_char = '' if "lineterminator" in kwargs : line_terminator = kwargs [ 'lineterminator' ] del kwargs [ 'lineterminator' ] if len ( kwargs ) > 0 : raise TypeError ( "Unexpected keyword arguments " + str ( kwargs . keys ( ) ) ) parsing_config = dict ( ) parsing_config [ "delimiter" ] = delimiter parsing_config [ "use_header" ] = header parsing_config [ "continue_on_failure" ] = not error_bad_lines parsing_config [ "comment_char" ] = comment_char parsing_config [ "escape_char" ] = '\0' if escape_char is None else escape_char parsing_config [ "use_escape_char" ] = escape_char is None parsing_config [ "double_quote" ] = double_quote parsing_config [ "quote_char" ] = quote_char parsing_config [ "skip_initial_space" ] = skip_initial_space parsing_config [ "store_errors" ] = store_errors parsing_config [ "line_terminator" ] = line_terminator parsing_config [ "output_columns" ] = usecols parsing_config [ "skip_rows" ] = skiprows parsing_config [ "true_values" ] = true_values parsing_config [ "false_values" ] = false_values parsing_config [ "only_raw_string_substitutions" ] = _only_raw_string_substitutions if type ( na_values ) is str : na_values = [ na_values ] if na_values is not None and len ( na_values ) > 0 : parsing_config [ "na_values" ] = na_values if nrows is not None : parsing_config [ "row_limit" ] = nrows proxy = UnitySFrameProxy ( ) internal_url = _make_internal_url ( url ) column_type_inference_was_used = False if column_type_hints is None : try : first_rows = SFrame . read_csv ( url , nrows = nrows_to_infer , column_type_hints = type ( None ) , header = header , delimiter = delimiter , comment_char = comment_char , escape_char = escape_char , double_quote = double_quote , quote_char = quote_char , skip_initial_space = skip_initial_space , na_values = na_values , line_terminator = line_terminator , usecols = usecols , skiprows = skiprows , verbose = verbose , true_values = true_values , false_values = false_values , _only_raw_string_substitutions = _only_raw_string_substitutions ) column_type_hints = SFrame . _infer_column_types_from_lines ( first_rows ) typelist = '[' + ',' . join ( t . __name__ for t in column_type_hints ) + ']' if verbose : print ( "------------------------------------------------------" ) print ( "Inferred types from first %d line(s) of file as " % nrows_to_infer ) print ( "column_type_hints=" + typelist ) print ( "If parsing fails due to incorrect types, you can correct" ) print ( "the inferred type list above and pass it to read_csv in" ) print ( "the column_type_hints argument" ) print ( "------------------------------------------------------" ) column_type_inference_was_used = True except RuntimeError as e : if type ( e ) == RuntimeError and ( "cancel" in str ( e . args [ 0 ] ) or "Cancel" in str ( e . args [ 0 ] ) ) : raise e column_type_hints = str if verbose : print ( 'Could not detect types. Using str for each column.' ) if type ( column_type_hints ) is type : type_hints = { '__all_columns__' : column_type_hints } elif type ( column_type_hints ) is list : type_hints = dict ( list ( zip ( [ '__X%d__' % i for i in range ( len ( column_type_hints ) ) ] , column_type_hints ) ) ) elif type ( column_type_hints ) is dict : try : first_rows = SFrame . read_csv ( url , nrows = nrows_to_infer , column_type_hints = type ( None ) , header = header , delimiter = delimiter , comment_char = comment_char , escape_char = escape_char , double_quote = double_quote , quote_char = quote_char , skip_initial_space = skip_initial_space , na_values = na_values , line_terminator = line_terminator , usecols = usecols , skiprows = skiprows , verbose = verbose , true_values = true_values , false_values = false_values , _only_raw_string_substitutions = _only_raw_string_substitutions ) inferred_types = SFrame . _infer_column_types_from_lines ( first_rows ) inferred_types = dict ( list ( zip ( first_rows . column_names ( ) , inferred_types ) ) ) for key in column_type_hints : inferred_types [ key ] = column_type_hints [ key ] column_type_hints = inferred_types except RuntimeError as e : if type ( e ) == RuntimeError and ( "cancel" in str ( e ) or "Cancel" in str ( e ) ) : raise e if verbose : print ( 'Could not detect types. Using str for all unspecified columns.' ) type_hints = column_type_hints else : raise TypeError ( "Invalid type for column_type_hints. Must be a dictionary, list or a single type." ) try : if ( not verbose ) : glconnect . get_server ( ) . set_log_progress ( False ) with cython_context ( ) : errors = proxy . load_from_csvs ( internal_url , parsing_config , type_hints ) except Exception as e : if type ( e ) == RuntimeError and "CSV parsing cancelled" in str ( e . args [ 0 ] ) : raise e if column_type_inference_was_used : if verbose : print ( "Unable to parse the file with automatic type inference." ) print ( "Defaulting to column_type_hints=str" ) type_hints = { '__all_columns__' : str } try : with cython_context ( ) : errors = proxy . load_from_csvs ( internal_url , parsing_config , type_hints ) except : glconnect . get_server ( ) . set_log_progress ( True ) raise else : glconnect . get_server ( ) . set_log_progress ( True ) raise glconnect . get_server ( ) . set_log_progress ( True ) return ( cls ( _proxy = proxy ) , { f : SArray ( _proxy = es ) for ( f , es ) in errors . items ( ) } )
Constructs an SFrame from a CSV file or a path to multiple CSVs, and
        returns a pair containing the SFrame and optionally
        (if store_errors=True) a dict of filenames to SArrays
        indicating for each file, what are the incorrectly parsed lines
        encountered.

        Parameters
        ----------
        store_errors : bool
            If true, the output errors dict will be filled.

        See `read_csv` for the rest of the parameters.