language-extensions/python/src/PythonDataSet.cpp
//-------------------------------------------------------------------------------------------------
// Name: PythonOutputDataSet::RetrieveStringColumnFromDataFrame
//
// Description:
//  Gets string column information from the underlying DataFrame,
//  adds data to m_data and nullmap to m_columnNullMap.
//

:RetrieveStringColumnFromDataFrame
Description:
Gets string column information from the underlying DataFrame,
adds data to m_data and nullmap to m_columnNullMap.
: RetrieveStringColumnFromDataFrame Description : Gets string column information from the underlying DataFrame adds data to m_data and nullmap to m_columnNullMap .
void PythonOutputDataSet::RetrieveStringColumnFromDataFrame(
	string      columnName,
	SQLULEN     &columnSize,
	SQLSMALLINT &decimalDigits,
	SQLSMALLINT &nullable)
{
	LOG("PythonOutputDataSet::RetrieveStringColumnFromDataFrame");
	vector<char> columnData;
	SQLINTEGER *strLenOrNullMap = nullptr;
	if (m_rowsNumber > 0)
	{
		strLenOrNullMap = new SQLINTEGER[m_rowsNumber];
	}
	decimalDigits = 0;
	nullable = SQL_NO_NULLS;
	np::ndarray column = ExtractArrayFromDataFrame(columnName);
	string dType = bp::extract<string>(bp::str(column.get_dtype()));
	SQLINTEGER maxLen = sizeof(char);
	int fullSize = 0;
	for (SQLULEN row = 0; row < m_rowsNumber; ++row)
	{
		string stringToAdd = "";
		bp::object pyObj = column[row];
		bool null = true;
		if (!pyObj.is_none())
		{
			null = false;
			if (PyBytes_Check(pyObj.ptr()))
			{
				PyObject *baseObj = pyObj.ptr();
				int size = PyBytes_Size(baseObj);
				char *bytes = static_cast<char *>(
					static_cast<void *>(PyBytes_AsString(baseObj)));
				stringToAdd = string(bytes, size);
			}
			else
			{
				stringToAdd = bp::extract<string>(bp::str(pyObj).encode("utf-8"));
				if (m_isStreaming && regex_match(dType, regex(".*date.*")))
				{
					if (stringToAdd.compare("NaT") == 0)
					{
						null = true;
					}
					else 
					{
						smatch matches;
						string trimmedDate;
						if (regex_search(stringToAdd, matches, regex("((.*)([.].*?))(0+)$"))) 
						{
							trimmedDate = matches[1].str();
							if (matches[3].str() == ".") 
							{
								trimmedDate = matches[2].str();
							}
							stringToAdd = trimmedDate;
						}
					}
				}
			}
		}
		if (!null)
		{
			fullSize += stringToAdd.size();
			strLenOrNullMap[row] = stringToAdd.size();
			columnData.insert(columnData.end(), stringToAdd.begin(), stringToAdd.end());
		}
		else
		{
			strLenOrNullMap[row] = SQL_NULL_DATA;
			nullable = SQL_NULLABLE;
		}
		if (maxLen < strLenOrNullMap[row])
		{
			maxLen = strLenOrNullMap[row];
		}
	}
	unique_ptr<char[]> dataPtr(new char[fullSize]);
	memcpy(dataPtr.get(), columnData.data(), fullSize);
	columnSize = maxLen;
	if (m_rowsNumber > 0)
	{
		m_data.push_back(static_cast<SQLPOINTER>(dataPtr.release()));
	}
	else
	{
		m_data.push_back(nullptr);
	}
	m_columnNullMap.push_back(strLenOrNullMap);
}
void PythonOutputDataSet :: RetrieveStringColumnFromDataFrame ( string columnName , SQLULEN & columnSize , SQLSMALLINT & decimalDigits , SQLSMALLINT & nullable ) { LOG ( " " ) ; vector < char > columnData ; SQLINTEGER * strLenOrNullMap = nullptr ; if ( m_rowsNumber > 0 ) { strLenOrNullMap = new SQLINTEGER [ m_rowsNumber ] ; } decimalDigits = 0 ; nullable = SQL_NO_NULLS ; np :: ndarray column = ExtractArrayFromDataFrame ( columnName ) ; string dType = bp :: extract < string > ( bp :: str ( column . get_dtype ( ) ) ) ; SQLINTEGER maxLen = sizeof ( char ) ; int fullSize = 0 ; for ( SQLULEN row = 0 ; row < m_rowsNumber ; ++ row ) { string stringToAdd = " " ; bp :: object pyObj = column [ row ] ; bool null = true ; if ( ! pyObj . is_none ( ) ) { null = false ; if ( PyBytes_Check ( pyObj . ptr ( ) ) ) { PyObject * baseObj = pyObj . ptr ( ) ; int size = PyBytes_Size ( baseObj ) ; char * bytes = static_cast < char * > ( static_cast < void * > ( PyBytes_AsString ( baseObj ) ) ) ; stringToAdd = string ( bytes , size ) ; } else { stringToAdd = bp :: extract < string > ( bp :: str ( pyObj ) . encode ( " " ) ) ; if ( m_isStreaming && regex_match ( dType , regex ( " " ) ) ) { if ( stringToAdd . compare ( " " ) == 0 ) { null = true ; } else { smatch matches ; string trimmedDate ; if ( regex_search ( stringToAdd , matches , regex ( " " ) ) ) { trimmedDate = matches [ 1 ] . str ( ) ; if ( matches [ 3 ] . str ( ) == " " ) { trimmedDate = matches [ 2 ] . str ( ) ; } stringToAdd = trimmedDate ; } } } } } if ( ! null ) { fullSize += stringToAdd . size ( ) ; strLenOrNullMap [ row ] = stringToAdd . size ( ) ; columnData . insert ( columnData . end ( ) , stringToAdd . begin ( ) , stringToAdd . end ( ) ) ; } else { strLenOrNullMap [ row ] = SQL_NULL_DATA ; nullable = SQL_NULLABLE ; } if ( maxLen < strLenOrNullMap [ row ] ) { maxLen = strLenOrNullMap [ row ] ; } } unique_ptr < char [ ] > dataPtr ( new char [ fullSize ] ) ; memcpy ( dataPtr . get ( ) , columnData . data ( ) , fullSize ) ; columnSize = maxLen ; if ( m_rowsNumber > 0 ) { m_data . push_back ( static_cast < SQLPOINTER > ( dataPtr . release ( ) ) ) ; } else { m_data . push_back ( nullptr ) ; } m_columnNullMap . push_back ( strLenOrNullMap ) ; }
Name: PythonOutputDataSet::RetrieveStringColumnFromDataFrame
Description:
Gets string column information from the underlying DataFrame,
adds data to m_data and nullmap to m_columnNullMap.
Name : PythonOutputDataSet :: RetrieveStringColumnFromDataFrame Description : Gets string column information from the underlying DataFrame adds data to m_data and nullmap to m_columnNullMap .
// Store the data in a vector (columnData) while we calculate the total size we need 
 // for the contiguous char data.
 //
 // If we have a bytes object we want to take only the bytes, not the b'' around them
 //
 // Extract the size and bytes of the pyObj
 //
 // Extract a utf-8 encoded version of the string
 //
 // If we are dealing with date types in a streaming case, 
 // we have to clean up the datetime string so that SQL can convert properly
 //
 // NaT counts as None / NULL for date types
 //
 // Python Datetimes will look like this: YYYY-MM-DDThh:mm:ss.fffffffff
 // We trim trailing 0s in the fractional part of the timestamp 
 // (and the "." if the fraction is all 0) so SQL can auto-convert 
 // from string to the date type. 
 // Lower precision date types cannot handle trailing zeroes.
 //
 // "matches" will have this by index:
 // 0. YYYY-MM-DDThh:mm:ss.fffff0000
 // 1. YYYY-MM-DDThh:mm:ss.fffff
 // 2. YYYY-MM-DDThh:mm:ss
 // 3. .fffff
 // If there is no fractional second then #3 is just ".", and #1 will
 // have the trailing "." which we don't want, so we use #2 instead.
 // 
 // Store the maximum length to find the widest the column needs to be
 //
 // Create a single block of memory that will hold all the data contiguously.
 //
 // Copy our data from the vector of strings to the single chunk of memory.
 //
 // Add the memory block to the m_data and give the memory handling job to m_data.
 //
