language-extensions/python/src/PythonDataSet.cpp
//-------------------------------------------------------------------------------------------------
// Name: PythonInputDataSet::AddRawColumnToDictionary
//
// Description:
//  Adds a raw column to the python dictionary that will be the DataFrame
//

:AddRawColumnToDictionary
Description:
Adds a raw column to the python dictionary that will be the DataFrame
: AddRawColumnToDictionary Description : Adds a raw column to the python dictionary that will be the DataFrame
void PythonInputDataSet::AddRawColumnToDictionary(
	SQLSMALLINT columnNumber,
	SQLULEN     rowsNumber,
	SQLPOINTER  data,
	SQLINTEGER  *strLen_or_Ind)
{
	LOG("PythonInputDataSet::AddRawColumnToDictionary");
	string name = m_columns[columnNumber].get()->Name();
	char *rawArray = reinterpret_cast<char*>(data);
	int length = 0;
	bp::tuple shape = bp::make_tuple(rowsNumber);
	np::ndarray nArray = np::empty(shape, np::dtype(bp::object("O")));
	for (SQLULEN row = 0; row < rowsNumber; ++row)
	{
		if (strLen_or_Ind == nullptr || strLen_or_Ind[row] == SQL_NULL_DATA)
		{
			nArray[row] = bp::object();
		}
		else
		{
			char *rawVal = rawArray + length;
			SQLINTEGER strlen = strLen_or_Ind[row] / sizeof(CHAR);
			bp::object bytesObj = bp::object(bp::handle<>(
				PyBytes_FromObject(PyMemoryView_FromMemory(rawVal, strlen, PyBUF_READ))
			));
			nArray[row] = bytesObj;
			length += strlen;
		}
	}
	m_dataDict[name] = nArray;
}
void PythonInputDataSet :: AddRawColumnToDictionary ( SQLSMALLINT columnNumber , SQLULEN rowsNumber , SQLPOINTER data , SQLINTEGER * strLen_or_Ind ) { LOG ( " " ) ; string name = m_columns [ columnNumber ] . get ( ) -> Name ( ) ; char * rawArray = reinterpret_cast < char * > ( data ) ; int length = 0 ; bp :: tuple shape = bp :: make_tuple ( rowsNumber ) ; np :: ndarray nArray = np :: empty ( shape , np :: dtype ( bp :: object ( " " ) ) ) ; for ( SQLULEN row = 0 ; row < rowsNumber ; ++ row ) { if ( strLen_or_Ind == nullptr || strLen_or_Ind [ row ] == SQL_NULL_DATA ) { nArray [ row ] = bp :: object ( ) ; } else { char * rawVal = rawArray + length ; SQLINTEGER strlen = strLen_or_Ind [ row ] / sizeof ( CHAR ) ; bp :: object bytesObj = bp :: object ( bp :: handle < > ( PyBytes_FromObject ( PyMemoryView_FromMemory ( rawVal , strlen , PyBUF_READ ) ) ) ) ; nArray [ row ] = bytesObj ; length += strlen ; } } m_dataDict [ name ] = nArray ; }
Name: PythonInputDataSet::AddRawColumnToDictionary
Description:
Adds a raw column to the python dictionary that will be the DataFrame
Name : PythonInputDataSet :: AddRawColumnToDictionary Description : Adds a raw column to the python dictionary that will be the DataFrame
// Create an empty numpy array of type python object
 //
 // If this string should be NULL, then we set it to the Python None object.
 //
 // Create a bytes PyObject from the raw bytes and strLen.
 // We create a PyMemoryViewObject (which does not copy the underlying data) then
 // convert to a Bytes object. This step DOES copy the data.
 // Then, convert to a boost object so that boost handles ref counting.
 //
 // By assigning the boost python objects into the data dictionary, the memory will not be
 // deallocated because we keep at least one reference to the PyObjects that were created
 // in the loop above.
 //
