giraffez/load.py
def from_file(self, filename, table=None, delimiter='|', null='NULL',
            panic=True, quotechar='"', parse_dates=False):
        """
        Load from a file into the target table, handling each step of the
        load process.

        Can load from text files, and properly formatted giraffez archive
        files. In both cases, if Gzip compression is detected the file will be
        decompressed while reading and handled appropriately. The encoding is
        determined automatically by the contents of the file.

        It is not necessary to set the columns in use prior to loading from a file.
        In the case of a text file, the header is used to determine column names
        and their order. Valid delimiters include '|', ',', and '\\t' (tab). When
        loading an archive file, the column information is decoded alongside the data.

        :param str filename: The location of the file to be loaded
        :param str table: The name of the target table, if it was not specified
            to the constructor for the isntance
        :param str null: The string that indicates a null value in the rows being
            inserted from a file. Defaults to 'NULL'
        :param str delimiter: When loading a file, indicates that fields are
            separated by this delimiter. Defaults to :code:`None`, which causes the
            delimiter to be determined from the header of the file. In most
            cases, this behavior is sufficient
        :param str quotechar: The character used to quote fields containing special characters,
            like the delimiter.
        :param bool panic: If :code:`True`, when an error is encountered it will be
            raised. Otherwise, the error will be logged and :code:`self.error_count`
            is incremented.
        :return: The output of the call to
            :meth:`~giraffez.load.TeradataBulkLoad.finish`
        :raises `giraffez.errors.GiraffeError`: if table was not set and :code:`table`
            is :code:`None`, or if a Teradata error ocurred while retrieving table info.
        :raises `giraffez.errors.GiraffeEncodeError`: if :code:`panic` is :code:`True` and there
            are format errors in the row values.
        """
        if not self.table:
            if not table:
                raise GiraffeError("Table must be set or specified to load a file.")
            self.table = table
        if not isinstance(null, basestring):
            raise GiraffeError("Expected 'null' to be str, received {}".format(type(null)))
        with Reader(filename, delimiter=delimiter, quotechar=quotechar) as f:
            if not isinstance(f.delimiter, basestring):
                raise GiraffeError("Expected 'delimiter' to be str, received {}".format(type(delimiter)))
            self.columns = f.header
            if isinstance(f, ArchiveFileReader):
                self.mload.set_encoding(ROW_ENCODING_RAW)
                self.preprocessor = lambda s: s
            if parse_dates:
                self.preprocessor = DateHandler(self.columns)
            self._initiate()
            self.mload.set_null(null)
            self.mload.set_delimiter(delimiter)
            i = 0
            for i, line in enumerate(f, 1):
                self.put(line, panic=panic)
                if i % self.checkpoint_interval == 1:
                    log.info("\rBulkLoad", "Processed {} rows".format(i), console=True)
                    checkpoint_status = self.checkpoint()
                    self.exit_code = self._exit_code()
                    if self.exit_code != 0:
                        return self.exit_code
            log.info("\rBulkLoad", "Processed {} rows".format(i))
            return self.finish()
def from_file(self, filename, table=None, delimiter='|', null='NULL',
            panic=True, quotechar='"', parse_dates=False):
        if not self.table:
            if not table:
                raise GiraffeError("Table must be set or specified to load a file.")
            self.table = table
        if not isinstance(null, basestring):
            raise GiraffeError("Expected 'null' to be str, received {}".format(type(null)))
        with Reader(filename, delimiter=delimiter, quotechar=quotechar) as f:
            if not isinstance(f.delimiter, basestring):
                raise GiraffeError("Expected 'delimiter' to be str, received {}".format(type(delimiter)))
            self.columns = f.header
            if isinstance(f, ArchiveFileReader):
                self.mload.set_encoding(ROW_ENCODING_RAW)
                self.preprocessor = lambda s: s
            if parse_dates:
                self.preprocessor = DateHandler(self.columns)
            self._initiate()
            self.mload.set_null(null)
            self.mload.set_delimiter(delimiter)
            i = 0
            for i, line in enumerate(f, 1):
                self.put(line, panic=panic)
                if i % self.checkpoint_interval == 1:
                    log.info("\rBulkLoad", "Processed {} rows".format(i), console=True)
                    checkpoint_status = self.checkpoint()
                    self.exit_code = self._exit_code()
                    if self.exit_code != 0:
                        return self.exit_code
            log.info("\rBulkLoad", "Processed {} rows".format(i))
            return self.finish()
def from_file ( self , filename , table = None , delimiter = '|' , null = 'NULL' , panic = True , quotechar = '"' , parse_dates = False ) : if not self . table : if not table : raise GiraffeError ( "Table must be set or specified to load a file." ) self . table = table if not isinstance ( null , basestring ) : raise GiraffeError ( "Expected 'null' to be str, received {}" . format ( type ( null ) ) ) with Reader ( filename , delimiter = delimiter , quotechar = quotechar ) as f : if not isinstance ( f . delimiter , basestring ) : raise GiraffeError ( "Expected 'delimiter' to be str, received {}" . format ( type ( delimiter ) ) ) self . columns = f . header if isinstance ( f , ArchiveFileReader ) : self . mload . set_encoding ( ROW_ENCODING_RAW ) self . preprocessor = lambda s : s if parse_dates : self . preprocessor = DateHandler ( self . columns ) self . _initiate ( ) self . mload . set_null ( null ) self . mload . set_delimiter ( delimiter ) i = 0 for i , line in enumerate ( f , 1 ) : self . put ( line , panic = panic ) if i % self . checkpoint_interval == 1 : log . info ( "\rBulkLoad" , "Processed {} rows" . format ( i ) , console = True ) checkpoint_status = self . checkpoint ( ) self . exit_code = self . _exit_code ( ) if self . exit_code != 0 : return self . exit_code log . info ( "\rBulkLoad" , "Processed {} rows" . format ( i ) ) return self . finish ( )
Load from a file into the target table, handling each step of the
        load process.

        Can load from text files, and properly formatted giraffez archive
        files. In both cases, if Gzip compression is detected the file will be
        decompressed while reading and handled appropriately. The encoding is
        determined automatically by the contents of the file.

        It is not necessary to set the columns in use prior to loading from a file.
        In the case of a text file, the header is used to determine column names
        and their order. Valid delimiters include '|', ',', and '\\t' (tab). When
        loading an archive file, the column information is decoded alongside the data.

        :param str filename: The location of the file to be loaded
        :param str table: The name of the target table, if it was not specified
            to the constructor for the isntance
        :param str null: The string that indicates a null value in the rows being
            inserted from a file. Defaults to 'NULL'
        :param str delimiter: When loading a file, indicates that fields are
            separated by this delimiter. Defaults to :code:`None`, which causes the
            delimiter to be determined from the header of the file. In most
            cases, this behavior is sufficient
        :param str quotechar: The character used to quote fields containing special characters,
            like the delimiter.
        :param bool panic: If :code:`True`, when an error is encountered it will be
            raised. Otherwise, the error will be logged and :code:`self.error_count`
            is incremented.
        :return: The output of the call to
            :meth:`~giraffez.load.TeradataBulkLoad.finish`
        :raises `giraffez.errors.GiraffeError`: if table was not set and :code:`table`
            is :code:`None`, or if a Teradata error ocurred while retrieving table info.
        :raises `giraffez.errors.GiraffeEncodeError`: if :code:`panic` is :code:`True` and there
            are format errors in the row values.
Load from a file into the target table handling each step of the load process .