<commit_before>/*
 * Copyright © 2014 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors:
 *     Wei Lin<wei.w.lin@intel.com>
 *     Yuting Yang<yuting.yang@intel.com>
 *     Lina Sun<lina.sun@intel.com>
 */

#include "cm_event.h"
#include "cm_device.h"
#include "cm_queue.h"
#include "cm_mem.h"
#include "cm_task.h"
#include "cm_kernel.h"
#include "cm_thread_space.h"
#include "cm_group_space.h"
#include "hal_cm.h"
#include "cm_surface_manager.h"

INT CmEvent::Create(UINT index, CmTaskInternal * pTask, INT taskDriverId,
		    CmDevice * pCmDev, BOOL isVisible, CmEvent * &pEvent)
{
	INT result = CM_SUCCESS;
	pEvent =
	    new(std::nothrow) CmEvent(index, pTask, taskDriverId, pCmDev,
				      isVisible);
	if (pEvent) {
		if (isVisible) {
			pEvent->Acquire();
		}
		result = pEvent->Initialize();
		if (result != CM_SUCCESS) {
			CmEvent::Destroy(pEvent);
		}
	} else {
		CM_ASSERT(0);
		result = CM_OUT_OF_HOST_MEMORY;
	}
	return result;
}

INT CmEvent::Destroy(CmEvent * &pEvent)
{
	long refCount = pEvent->SafeRelease();
	if (refCount == 0) {
		pEvent = NULL;
	}

	return CM_SUCCESS;
}

 CmEvent::CmEvent(UINT index, CmTaskInternal * pTask, INT taskDriverId, CmDevice * pCmDev, BOOL isVisible):
m_Index(index),
m_TaskDriverId(taskDriverId),
m_OsData(NULL),
m_Status(CM_STATUS_QUEUED),
m_Time(0),
m_pDevice(pCmDev),
m_pQueue(NULL), m_RefCount(0), isVisible(isVisible), m_pTask(pTask)
{
}

INT CmEvent::Acquire(void)
{
	++m_RefCount;
	return m_RefCount;
}

INT CmEvent::SafeRelease(void)
{
	--m_RefCount;
	if (m_RefCount == 0) {
		delete this;
		return 0;
	} else {
		return m_RefCount;
	}
}

CmEvent::~CmEvent(void)
{
	if (m_SurEntryInfoArrays.pSurfEntryInfosArray != NULL) {

		for (UINT i = 0; i < m_SurEntryInfoArrays.dwKrnNum; i++) {
			if (m_SurEntryInfoArrays.
			    pSurfEntryInfosArray[i].pSurfEntryInfos != NULL) {
				CmSafeDelete
				    (m_SurEntryInfoArrays.pSurfEntryInfosArray
				     [i].pSurfEntryInfos);
			}
			if (m_SurEntryInfoArrays.
			    pSurfEntryInfosArray[i].pGlobalSurfInfos != NULL) {
				CmSafeDelete
				    (m_SurEntryInfoArrays.pSurfEntryInfosArray
				     [i].pGlobalSurfInfos);
			}
		}
		CmSafeDelete(m_SurEntryInfoArrays.pSurfEntryInfosArray);
	}

	if (m_KernelNames != NULL) {
		for (UINT i = 0; i < m_KernelCount; i++) {
			CmSafeDeleteArray(m_KernelNames[i]);
		}
		CmSafeDeleteArray(m_KernelNames);
		CmSafeDeleteArray(m_ThreadSpace);
	}
}

INT CmEvent::Initialize(void)
{
	CmSafeMemSet(&m_SurEntryInfoArrays, 0,
		     sizeof(CM_HAL_SURFACE_ENTRY_INFO_ARRAYS));
	if (m_TaskDriverId == -1) {
		m_Status = CM_STATUS_QUEUED;
	} else {
		CM_ASSERT(0);
		return CM_FAILURE;
	}

	m_KernelNames = NULL;
	m_KernelCount = 0;

	m_pDevice->GetQueue(m_pQueue);

	return CM_SUCCESS;
}

CM_RT_API INT CmEvent::GetStatus(CM_STATUS & status)
{

	if ((m_Status == CM_STATUS_FLUSHED) || (m_Status == CM_STATUS_STARTED)) {
		Query();
	} else if (m_Status == CM_STATUS_QUEUED) {
		m_pQueue->FlushTaskWithoutSync();

	} else if (m_Status == CM_STATUS_FINISHED) {
	} else {
		CM_ASSERT(0);
	}

	status = m_Status;
	return CM_SUCCESS;
}

CM_RT_API INT CmEvent::GetExecutionTime(UINT64 & time)
{
	CM_STATUS EventStatus = CM_STATUS_QUEUED;
	m_pQueue->AcquireQueueLock();
	GetStatus(EventStatus);
	m_pQueue->ReleaseQueueLock();

	if (EventStatus == CM_STATUS_FINISHED) {
		time = m_Time;
		return CM_SUCCESS;
	} else {
		return CM_FAILURE;
	}
}

CM_RT_API INT CmEvent::GetSubmitTime(LARGE_INTEGER & time)
{

	CM_STATUS EventStatus = CM_STATUS_QUEUED;

	m_pQueue->AcquireQueueLock();
	GetStatus(EventStatus);
	m_pQueue->ReleaseQueueLock();

	if (EventStatus == CM_STATUS_FINISHED) {
		time = m_GlobalCMSubmitTime;
		return CM_SUCCESS;
	} else {
		CM_ASSERT(0);
		return CM_FAILURE;
	}
}

CM_RT_API INT CmEvent::GetHWStartTime(LARGE_INTEGER & time)
{
	CM_STATUS EventStatus = CM_STATUS_QUEUED;

	m_pQueue->AcquireQueueLock();
	GetStatus(EventStatus);
	m_pQueue->ReleaseQueueLock();

	if (EventStatus == CM_STATUS_FINISHED) {
		time.QuadPart =
		    m_GlobalCMSubmitTime.QuadPart +
		    m_HWStartTimeStamp.QuadPart - m_CMSubmitTimeStamp.QuadPart;
		return CM_SUCCESS;
	} else {
		return CM_FAILURE;
	}

}

CM_RT_API INT CmEvent::GetHWEndTime(LARGE_INTEGER & time)
{

	CM_STATUS EventStatus = CM_STATUS_QUEUED;

	m_pQueue->AcquireQueueLock();
	GetStatus(EventStatus);
	m_pQueue->ReleaseQueueLock();

	if (EventStatus == CM_STATUS_FINISHED) {
		time.QuadPart =
		    m_GlobalCMSubmitTime.QuadPart + m_HWEndTimeStamp.QuadPart -
		    m_CMSubmitTimeStamp.QuadPart;
		return CM_SUCCESS;
	} else {
		return CM_FAILURE;
	}
}

CM_RT_API INT CmEvent::GetCompleteTime(LARGE_INTEGER & time)
{

	CM_STATUS EventStatus = CM_STATUS_QUEUED;

	m_pQueue->AcquireQueueLock();
	GetStatus(EventStatus);
	m_pQueue->ReleaseQueueLock();

	if (EventStatus == CM_STATUS_FINISHED) {
		time = m_CompleteTime;
		return CM_SUCCESS;
	} else {
		return CM_FAILURE;
	}

}

CM_RT_API UINT CmEvent::GetKernelCount()
{
	return m_KernelCount;
}

CM_RT_API INT CmEvent::GetKernelName(UINT index, char *&KernelName)
{
	if (index < m_KernelCount) {
		KernelName = m_KernelNames[index];
		return CM_SUCCESS;
	}
	return CM_FAILURE;
}

CM_RT_API INT
    CmEvent::GetKernelThreadSpace(UINT index, UINT & localWidth,
				  UINT & localHeight, UINT & globalWidth,
				  UINT & globalHeight)
{
	if (index < m_KernelCount) {
		localWidth = m_ThreadSpace[4 * index];
		localHeight = m_ThreadSpace[4 * index + 1];
		globalWidth = m_ThreadSpace[4 * index + 2];
		globalHeight = m_ThreadSpace[4 * index + 3];
		return CM_SUCCESS;
	}
	return CM_FAILURE;
}

INT CmEvent::SetKernelNames(CmTask * pTask, CmThreadSpace * pThreadSpace,
			    CmThreadGroupSpace * pThreadGroupSpace)
{
	UINT i = 0;
	INT hr = CM_SUCCESS;
	CmThreadSpace *pThreadSpace_RT =
	    dynamic_cast < CmThreadSpace * >(pThreadSpace);
	UINT ThreadCount;
	m_KernelCount = pTask->GetKernelCount();

	m_KernelNames = new(std::nothrow) char *[m_KernelCount];
	m_ThreadSpace = new(std::nothrow) UINT[4 * m_KernelCount];
	CMCHK_NULL_RETURN(m_KernelNames, CM_OUT_OF_HOST_MEMORY);
	CmSafeMemSet(m_KernelNames, 0, m_KernelCount * sizeof(char *));
	CMCHK_NULL_RETURN(m_ThreadSpace, CM_OUT_OF_HOST_MEMORY);

	for (i = 0; i < m_KernelCount; i++) {
		m_KernelNames[i] =
		    new(std::nothrow) char[CM_MAX_KERNEL_NAME_SIZE_IN_BYTE];
		CMCHK_NULL_RETURN(m_KernelNames[i], CM_OUT_OF_HOST_MEMORY);
		CmKernel *pKernel = pTask->GetKernelPointer(i);
		strcpy_s(m_KernelNames[i], CM_MAX_KERNEL_NAME_SIZE_IN_BYTE,
			 pKernel->GetName());

		pKernel->GetThreadCount(ThreadCount);
		m_ThreadSpace[4 * i] = ThreadCount;
		m_ThreadSpace[4 * i + 1] = 1;
		m_ThreadSpace[4 * i + 2] = ThreadCount;
		m_ThreadSpace[4 * i + 3] = 1;
	}

	if (pThreadSpace) {
		UINT ThreadWidth, ThreadHeight;
		pThreadSpace_RT->GetThreadSpaceSize(ThreadWidth, ThreadHeight);
		m_ThreadSpace[0] = ThreadWidth;
		m_ThreadSpace[1] = ThreadHeight;
		m_ThreadSpace[2] = ThreadWidth;
		m_ThreadSpace[3] = ThreadHeight;
	} else if (pThreadGroupSpace) {
		UINT ThreadWidth, ThreadHeight, GroupWidth, GroupHeight;
		pThreadGroupSpace->GetThreadGroupSpaceSize(ThreadWidth,
							   ThreadHeight,
							   GroupWidth,
							   GroupHeight);
		m_ThreadSpace[0] = ThreadWidth;
		m_ThreadSpace[1] = ThreadHeight;
		m_ThreadSpace[2] = ThreadWidth * GroupWidth;
		m_ThreadSpace[3] = ThreadHeight * GroupHeight;
	}

 finish:
	if (hr == CM_OUT_OF_HOST_MEMORY) {
		if (m_KernelNames != NULL) {
			for (UINT j = 0; j < m_KernelCount; j++) {
				CmSafeDeleteArray(m_KernelNames[j]);
			}
		}
		CmSafeDeleteArray(m_KernelNames);
		CmSafeDeleteArray(m_ThreadSpace);
	}
	return hr;
}

INT CmEvent::GetIndex(UINT & index)
{
	index = m_Index;
	return CM_SUCCESS;
}

INT CmEvent::SetTaskDriverId(INT id)
{
	m_TaskDriverId = id;
	if (m_TaskDriverId > -1) {
		m_Status = CM_STATUS_FLUSHED;
	} else if (m_TaskDriverId == -1) {
		m_Status = CM_STATUS_QUEUED;
	} else {
		CM_ASSERT(0);
		return CM_FAILURE;
	}

	return CM_SUCCESS;
}

INT CmEvent::SetTaskOsData(PVOID data)
{
	m_OsData = data;
	return CM_SUCCESS;
}

INT CmEvent::GetTaskDriverId(INT & id)
{
	id = m_TaskDriverId;
	return CM_SUCCESS;
}

INT CmEvent::Query(void)
{
	CM_RETURN_CODE hr = CM_SUCCESS;

	if ((m_Status != CM_STATUS_FLUSHED) && (m_Status != CM_STATUS_STARTED)) {
		return CM_FAILURE;
	}

	CM_ASSERT(m_TaskDriverId > -1);
	CM_HAL_QUERY_TASK_PARAM param;
	param.iTaskId = m_TaskDriverId;

	PCM_CONTEXT pCmData = (PCM_CONTEXT) m_pDevice->GetAccelData();

	CHK_GENOSSTATUS_RETURN_CMERROR(pCmData->pCmHalState->
				       pfnQueryTask(pCmData->pCmHalState,
						    &param));

	if (param.status == CM_TASK_FINISHED) {
		CmQueue *pQueue = NULL;

		m_Time = param.iTaskDuration;
		m_Status = CM_STATUS_FINISHED;
		m_GlobalCMSubmitTime = param.iTaskGlobalCMSubmitTime;
		m_CMSubmitTimeStamp = param.iTaskCMSubmitTimeStamp;
		m_HWStartTimeStamp = param.iTaskHWStartTimeStamp;
		m_HWEndTimeStamp = param.iTaskHWEndTimeStamp;
		m_CompleteTime = param.iTaskCompleteTime;

		m_pDevice->GetQueue(pQueue);
		if (!pQueue) {
			CM_ASSERT(0);
			return CM_FAILURE;
		}

		if (m_OsData) {
			drm_intel_bo_unreference((drm_intel_bo *) m_OsData);
		}

		m_GlobalCMSubmitTime = param.iTaskGlobalCMSubmitTime;
		m_CMSubmitTimeStamp = param.iTaskCMSubmitTimeStamp;
		m_HWStartTimeStamp = param.iTaskHWStartTimeStamp;
		m_HWEndTimeStamp = param.iTaskHWEndTimeStamp;
		m_CompleteTime = param.iTaskCompleteTime;

	} else if (param.status == CM_TASK_IN_PROGRESS) {
		m_Status = CM_STATUS_STARTED;
	}

 finish:
	return hr;
}

CM_RT_API INT CmEvent::WaitForTaskFinished(DWORD dwTimeOutMs)
{
	INT result = CM_SUCCESS;
	m_pQueue->AcquireQueueLock();

	while (m_Status == CM_STATUS_QUEUED) {
		m_pQueue->FlushTaskWithoutSync();
	}

	Query();

	while (m_Status != CM_STATUS_FINISHED) {
		if (m_OsData) {
			drm_intel_bo_wait_rendering((drm_intel_bo *) m_OsData);
			drm_intel_gem_bo_clear_relocs((drm_intel_bo *) m_OsData,
						      0);
		}

		Query();
	}

	m_pQueue->ReleaseQueueLock();
	return result;
}
<commit_msg>Support timeout in WaitForTaskFinished<commit_after>/*
 * Copyright © 2014 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors:
 *     Wei Lin<wei.w.lin@intel.com>
 *     Yuting Yang<yuting.yang@intel.com>
 *     Lina Sun<lina.sun@intel.com>
 */

#include "cm_event.h"
#include "cm_device.h"
#include "cm_queue.h"
#include "cm_mem.h"
#include "cm_task.h"
#include "cm_kernel.h"
#include "cm_thread_space.h"
#include "cm_group_space.h"
#include "hal_cm.h"
#include "cm_surface_manager.h"

INT CmEvent::Create(UINT index, CmTaskInternal * pTask, INT taskDriverId,
		    CmDevice * pCmDev, BOOL isVisible, CmEvent * &pEvent)
{
	INT result = CM_SUCCESS;
	pEvent =
	    new(std::nothrow) CmEvent(index, pTask, taskDriverId, pCmDev,
				      isVisible);
	if (pEvent) {
		if (isVisible) {
			pEvent->Acquire();
		}
		result = pEvent->Initialize();
		if (result != CM_SUCCESS) {
			CmEvent::Destroy(pEvent);
		}
	} else {
		CM_ASSERT(0);
		result = CM_OUT_OF_HOST_MEMORY;
	}
	return result;
}

INT CmEvent::Destroy(CmEvent * &pEvent)
{
	long refCount = pEvent->SafeRelease();
	if (refCount == 0) {
		pEvent = NULL;
	}

	return CM_SUCCESS;
}

 CmEvent::CmEvent(UINT index, CmTaskInternal * pTask, INT taskDriverId, CmDevice * pCmDev, BOOL isVisible):
m_Index(index),
m_TaskDriverId(taskDriverId),
m_OsData(NULL),
m_Status(CM_STATUS_QUEUED),
m_Time(0),
m_pDevice(pCmDev),
m_pQueue(NULL), m_RefCount(0), isVisible(isVisible), m_pTask(pTask)
{
}

INT CmEvent::Acquire(void)
{
	++m_RefCount;
	return m_RefCount;
}

INT CmEvent::SafeRelease(void)
{
	--m_RefCount;
	if (m_RefCount == 0) {
		delete this;
		return 0;
	} else {
		return m_RefCount;
	}
}

CmEvent::~CmEvent(void)
{
	if (m_SurEntryInfoArrays.pSurfEntryInfosArray != NULL) {

		for (UINT i = 0; i < m_SurEntryInfoArrays.dwKrnNum; i++) {
			if (m_SurEntryInfoArrays.
			    pSurfEntryInfosArray[i].pSurfEntryInfos != NULL) {
				CmSafeDelete
				    (m_SurEntryInfoArrays.pSurfEntryInfosArray
				     [i].pSurfEntryInfos);
			}
			if (m_SurEntryInfoArrays.
			    pSurfEntryInfosArray[i].pGlobalSurfInfos != NULL) {
				CmSafeDelete
				    (m_SurEntryInfoArrays.pSurfEntryInfosArray
				     [i].pGlobalSurfInfos);
			}
		}
		CmSafeDelete(m_SurEntryInfoArrays.pSurfEntryInfosArray);
	}

	if (m_KernelNames != NULL) {
		for (UINT i = 0; i < m_KernelCount; i++) {
			CmSafeDeleteArray(m_KernelNames[i]);
		}
		CmSafeDeleteArray(m_KernelNames);
		CmSafeDeleteArray(m_ThreadSpace);
	}
}

INT CmEvent::Initialize(void)
{
	CmSafeMemSet(&m_SurEntryInfoArrays, 0,
		     sizeof(CM_HAL_SURFACE_ENTRY_INFO_ARRAYS));
	if (m_TaskDriverId == -1) {
		m_Status = CM_STATUS_QUEUED;
	} else {
		CM_ASSERT(0);
		return CM_FAILURE;
	}

	m_KernelNames = NULL;
	m_KernelCount = 0;

	m_pDevice->GetQueue(m_pQueue);

	return CM_SUCCESS;
}

CM_RT_API INT CmEvent::GetStatus(CM_STATUS & status)
{

	if ((m_Status == CM_STATUS_FLUSHED) || (m_Status == CM_STATUS_STARTED)) {
		Query();
	} else if (m_Status == CM_STATUS_QUEUED) {
		m_pQueue->FlushTaskWithoutSync();

	} else if (m_Status == CM_STATUS_FINISHED) {
	} else {
		CM_ASSERT(0);
	}

	status = m_Status;
	return CM_SUCCESS;
}

CM_RT_API INT CmEvent::GetExecutionTime(UINT64 & time)
{
	CM_STATUS EventStatus = CM_STATUS_QUEUED;
	m_pQueue->AcquireQueueLock();
	GetStatus(EventStatus);
	m_pQueue->ReleaseQueueLock();

	if (EventStatus == CM_STATUS_FINISHED) {
		time = m_Time;
		return CM_SUCCESS;
	} else {
		return CM_FAILURE;
	}
}

CM_RT_API INT CmEvent::GetSubmitTime(LARGE_INTEGER & time)
{

	CM_STATUS EventStatus = CM_STATUS_QUEUED;

	m_pQueue->AcquireQueueLock();
	GetStatus(EventStatus);
	m_pQueue->ReleaseQueueLock();

	if (EventStatus == CM_STATUS_FINISHED) {
		time = m_GlobalCMSubmitTime;
		return CM_SUCCESS;
	} else {
		CM_ASSERT(0);
		return CM_FAILURE;
	}
}

CM_RT_API INT CmEvent::GetHWStartTime(LARGE_INTEGER & time)
{
	CM_STATUS EventStatus = CM_STATUS_QUEUED;

	m_pQueue->AcquireQueueLock();
	GetStatus(EventStatus);
	m_pQueue->ReleaseQueueLock();

	if (EventStatus == CM_STATUS_FINISHED) {
		time.QuadPart =
		    m_GlobalCMSubmitTime.QuadPart +
		    m_HWStartTimeStamp.QuadPart - m_CMSubmitTimeStamp.QuadPart;
		return CM_SUCCESS;
	} else {
		return CM_FAILURE;
	}

}

CM_RT_API INT CmEvent::GetHWEndTime(LARGE_INTEGER & time)
{

	CM_STATUS EventStatus = CM_STATUS_QUEUED;

	m_pQueue->AcquireQueueLock();
	GetStatus(EventStatus);
	m_pQueue->ReleaseQueueLock();

	if (EventStatus == CM_STATUS_FINISHED) {
		time.QuadPart =
		    m_GlobalCMSubmitTime.QuadPart + m_HWEndTimeStamp.QuadPart -
		    m_CMSubmitTimeStamp.QuadPart;
		return CM_SUCCESS;
	} else {
		return CM_FAILURE;
	}
}

CM_RT_API INT CmEvent::GetCompleteTime(LARGE_INTEGER & time)
{

	CM_STATUS EventStatus = CM_STATUS_QUEUED;

	m_pQueue->AcquireQueueLock();
	GetStatus(EventStatus);
	m_pQueue->ReleaseQueueLock();

	if (EventStatus == CM_STATUS_FINISHED) {
		time = m_CompleteTime;
		return CM_SUCCESS;
	} else {
		return CM_FAILURE;
	}

}

CM_RT_API UINT CmEvent::GetKernelCount()
{
	return m_KernelCount;
}

CM_RT_API INT CmEvent::GetKernelName(UINT index, char *&KernelName)
{
	if (index < m_KernelCount) {
		KernelName = m_KernelNames[index];
		return CM_SUCCESS;
	}
	return CM_FAILURE;
}

CM_RT_API INT
    CmEvent::GetKernelThreadSpace(UINT index, UINT & localWidth,
				  UINT & localHeight, UINT & globalWidth,
				  UINT & globalHeight)
{
	if (index < m_KernelCount) {
		localWidth = m_ThreadSpace[4 * index];
		localHeight = m_ThreadSpace[4 * index + 1];
		globalWidth = m_ThreadSpace[4 * index + 2];
		globalHeight = m_ThreadSpace[4 * index + 3];
		return CM_SUCCESS;
	}
	return CM_FAILURE;
}

INT CmEvent::SetKernelNames(CmTask * pTask, CmThreadSpace * pThreadSpace,
			    CmThreadGroupSpace * pThreadGroupSpace)
{
	UINT i = 0;
	INT hr = CM_SUCCESS;
	CmThreadSpace *pThreadSpace_RT =
	    dynamic_cast < CmThreadSpace * >(pThreadSpace);
	UINT ThreadCount;
	m_KernelCount = pTask->GetKernelCount();

	m_KernelNames = new(std::nothrow) char *[m_KernelCount];
	m_ThreadSpace = new(std::nothrow) UINT[4 * m_KernelCount];
	CMCHK_NULL_RETURN(m_KernelNames, CM_OUT_OF_HOST_MEMORY);
	CmSafeMemSet(m_KernelNames, 0, m_KernelCount * sizeof(char *));
	CMCHK_NULL_RETURN(m_ThreadSpace, CM_OUT_OF_HOST_MEMORY);

	for (i = 0; i < m_KernelCount; i++) {
		m_KernelNames[i] =
		    new(std::nothrow) char[CM_MAX_KERNEL_NAME_SIZE_IN_BYTE];
		CMCHK_NULL_RETURN(m_KernelNames[i], CM_OUT_OF_HOST_MEMORY);
		CmKernel *pKernel = pTask->GetKernelPointer(i);
		strcpy_s(m_KernelNames[i], CM_MAX_KERNEL_NAME_SIZE_IN_BYTE,
			 pKernel->GetName());

		pKernel->GetThreadCount(ThreadCount);
		m_ThreadSpace[4 * i] = ThreadCount;
		m_ThreadSpace[4 * i + 1] = 1;
		m_ThreadSpace[4 * i + 2] = ThreadCount;
		m_ThreadSpace[4 * i + 3] = 1;
	}

	if (pThreadSpace) {
		UINT ThreadWidth, ThreadHeight;
		pThreadSpace_RT->GetThreadSpaceSize(ThreadWidth, ThreadHeight);
		m_ThreadSpace[0] = ThreadWidth;
		m_ThreadSpace[1] = ThreadHeight;
		m_ThreadSpace[2] = ThreadWidth;
		m_ThreadSpace[3] = ThreadHeight;
	} else if (pThreadGroupSpace) {
		UINT ThreadWidth, ThreadHeight, GroupWidth, GroupHeight;
		pThreadGroupSpace->GetThreadGroupSpaceSize(ThreadWidth,
							   ThreadHeight,
							   GroupWidth,
							   GroupHeight);
		m_ThreadSpace[0] = ThreadWidth;
		m_ThreadSpace[1] = ThreadHeight;
		m_ThreadSpace[2] = ThreadWidth * GroupWidth;
		m_ThreadSpace[3] = ThreadHeight * GroupHeight;
	}

 finish:
	if (hr == CM_OUT_OF_HOST_MEMORY) {
		if (m_KernelNames != NULL) {
			for (UINT j = 0; j < m_KernelCount; j++) {
				CmSafeDeleteArray(m_KernelNames[j]);
			}
		}
		CmSafeDeleteArray(m_KernelNames);
		CmSafeDeleteArray(m_ThreadSpace);
	}
	return hr;
}

INT CmEvent::GetIndex(UINT & index)
{
	index = m_Index;
	return CM_SUCCESS;
}

INT CmEvent::SetTaskDriverId(INT id)
{
	m_TaskDriverId = id;
	if (m_TaskDriverId > -1) {
		m_Status = CM_STATUS_FLUSHED;
	} else if (m_TaskDriverId == -1) {
		m_Status = CM_STATUS_QUEUED;
	} else {
		CM_ASSERT(0);
		return CM_FAILURE;
	}

	return CM_SUCCESS;
}

INT CmEvent::SetTaskOsData(PVOID data)
{
	m_OsData = data;
	return CM_SUCCESS;
}

INT CmEvent::GetTaskDriverId(INT & id)
{
	id = m_TaskDriverId;
	return CM_SUCCESS;
}

INT CmEvent::Query(void)
{
	CM_RETURN_CODE hr = CM_SUCCESS;

	if ((m_Status != CM_STATUS_FLUSHED) && (m_Status != CM_STATUS_STARTED)) {
		return CM_FAILURE;
	}

	CM_ASSERT(m_TaskDriverId > -1);
	CM_HAL_QUERY_TASK_PARAM param;
	param.iTaskId = m_TaskDriverId;

	PCM_CONTEXT pCmData = (PCM_CONTEXT) m_pDevice->GetAccelData();

	CHK_GENOSSTATUS_RETURN_CMERROR(pCmData->pCmHalState->
				       pfnQueryTask(pCmData->pCmHalState,
						    &param));

	if (param.status == CM_TASK_FINISHED) {
		CmQueue *pQueue = NULL;

		m_Time = param.iTaskDuration;
		m_Status = CM_STATUS_FINISHED;
		m_GlobalCMSubmitTime = param.iTaskGlobalCMSubmitTime;
		m_CMSubmitTimeStamp = param.iTaskCMSubmitTimeStamp;
		m_HWStartTimeStamp = param.iTaskHWStartTimeStamp;
		m_HWEndTimeStamp = param.iTaskHWEndTimeStamp;
		m_CompleteTime = param.iTaskCompleteTime;

		m_pDevice->GetQueue(pQueue);
		if (!pQueue) {
			CM_ASSERT(0);
			return CM_FAILURE;
		}

		if (m_OsData) {
			drm_intel_bo_unreference((drm_intel_bo *) m_OsData);
		}

		m_GlobalCMSubmitTime = param.iTaskGlobalCMSubmitTime;
		m_CMSubmitTimeStamp = param.iTaskCMSubmitTimeStamp;
		m_HWStartTimeStamp = param.iTaskHWStartTimeStamp;
		m_HWEndTimeStamp = param.iTaskHWEndTimeStamp;
		m_CompleteTime = param.iTaskCompleteTime;

	} else if (param.status == CM_TASK_IN_PROGRESS) {
		m_Status = CM_STATUS_STARTED;
	}

 finish:
	return hr;
}

CM_RT_API INT CmEvent::WaitForTaskFinished(DWORD dwTimeOutMs)
{
	INT result = CM_SUCCESS;
	m_pQueue->AcquireQueueLock();

	while (m_Status == CM_STATUS_QUEUED) {
		m_pQueue->FlushTaskWithoutSync();
	}

	Query();

	while (m_Status != CM_STATUS_FINISHED) {
		if (m_OsData) {
			result = drm_intel_gem_bo_wait((drm_intel_bo*)m_OsData, 1000000LL*dwTimeOutMs);
			drm_intel_gem_bo_clear_relocs((drm_intel_bo*)m_OsData, 0);
			if (result) {
				//translate the drm ecode (-ETIME).
				result = CM_EXCEED_MAX_TIMEOUT;
				break;
			}
		}

		Query();
	}

	m_pQueue->ReleaseQueueLock();
	return result;
}
<|endoftext|>


<commit_before>#include "physicscomponent.hpp"
#include <glm/gtc/type_ptr.hpp>

void PhysicsComponent::registerImGui()
{
	ImGui::DragFloat3("Velocity", glm::value_ptr(velocity), 0.1);
	ImGui::DragFloat3("Acceleration", glm::value_ptr(acceleration), 0.1);
}
<commit_msg>hgffghdfgh<commit_after><|endoftext|>


<commit_before>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include "SearchDashboardQuery.h"

#include <stx/uri.h>

using namespace stx;

namespace zbase {

SearchDashboardQuery::SearchDashboardQuery(
    AnalyticsTableScan* query,
    const Vector<RefPtr<TrafficSegment>>& segments,
    UnixTime start_time,
    UnixTime end_time,
    const AnalyticsQuery::SubQueryParams& params) :
    start_time_(start_time.unixMicros() / kMicrosPerSecond),
    end_time_(end_time.unixMicros() / kMicrosPerSecond),
    result_(new TimeseriesDrilldownResult<SearchCTRStats>()),
    segments_(segments),
    time_col_(query->fetchColumn("search_queries.time")),
    pagetype_col_(query->fetchColumn("search_queries.page_type")),
    num_items_col_(query->fetchColumn("search_queries.num_result_items")),
    num_itemclicks_col_(query->fetchColumn("search_queries.num_result_items_clicked")),
    num_adimprs_col_(query->fetchColumn("search_queries.num_ad_impressions")),
    num_adclicks_col_(query->fetchColumn("search_queries.num_ad_clicks")),
    qnum_cartitems_col_(query->fetchColumn("search_queries.num_cart_items")),
    qnum_orderitems_col_(query->fetchColumn("search_queries.num_order_items")),
    qcartvalue_col_(query->fetchColumn("search_queries.cart_value_eurcents")),
    qgmv_col_(query->fetchColumn("search_queries.gmv_eurcents")),
    new_session_(false),
    window_secs_(kSecondsPerDay) {
  query->onSession(std::bind(&SearchDashboardQuery::onSession, this));
  query->onQuery(std::bind(&SearchDashboardQuery::onQuery, this));

  for (const auto& s : segments_) {
    result_->drilldown.segment_keys.emplace_back(s->key());
    result_->drilldown.counters.emplace_back();
    result_->timeseries.segment_keys.emplace_back(s->key());
    result_->timeseries.series.emplace_back();
  }

  String drilldown_type = "time";
  URI::getParam(params.params, "drilldown", &drilldown_type);
  setDrilldownFn(drilldown_type, query);

  String window_str;
  if (URI::getParam(params.params, "window_size", &window_str)) {
    window_secs_ = std::stoull(window_str);
  }
}

void SearchDashboardQuery::onSession() {
  new_session_ = true;
}

void SearchDashboardQuery::onQuery() {
  auto time = time_col_->getUInt32();
  auto pagetype = (PageType) pagetype_col_->getUInt32();
  auto num_items = num_items_col_->getUInt32();
  auto num_clicks = num_itemclicks_col_->getUInt32();
  auto num_ad_imprs = num_adimprs_col_->getUInt32();
  auto num_ad_clicks = num_adclicks_col_->getUInt32();
  auto qnum_cartitems = qnum_cartitems_col_->getUInt32();
  auto qnum_orderitems = qnum_orderitems_col_->getUInt32();
  auto qcart_value_eurcent = qcartvalue_col_->getUInt32();
  auto qgmv_eurcent = qgmv_col_->getUInt32();

  //if (pagetype != PageType::SEARCH_PAGE) {
  //  return;
  //}

  auto drilldown_dim = drilldown_fn_();

  for (int i = 0; i < segments_.size(); ++i) {
    auto& series = result_->timeseries.series[i];
    auto& tpoint = series[(time / window_secs_) * window_secs_];
    auto& dpoint = result_->drilldown.counters[i][drilldown_dim];

    if (!segments_[i]->checkPredicates()) {
      continue;
    }

    tpoint.num_queries += 1;
    dpoint.num_queries += 1;
    tpoint.num_queries_clicked += num_clicks > 0 ? 1 : 0;
    dpoint.num_queries_clicked += num_clicks > 0 ? 1 : 0;
    tpoint.num_query_items_clicked += num_clicks;
    dpoint.num_query_items_clicked += num_clicks;
    tpoint.num_query_item_impressions += num_items;
    dpoint.num_query_item_impressions += num_items;
    tpoint.num_ad_impressions += num_ad_imprs;
    dpoint.num_ad_impressions += num_ad_imprs;
    tpoint.num_ads_clicked += num_ad_clicks;
    dpoint.num_ads_clicked += num_ad_clicks;
    tpoint.query_num_cart_items += qnum_cartitems;
    dpoint.query_num_cart_items += qnum_cartitems;
    tpoint.query_num_order_items += qnum_orderitems;
    dpoint.query_num_order_items += qnum_orderitems;
    tpoint.query_cart_value_eurcent += qcart_value_eurcent;
    dpoint.query_cart_value_eurcent += qcart_value_eurcent;
    tpoint.query_gmv_eurcent += qgmv_eurcent;
    dpoint.query_gmv_eurcent += qgmv_eurcent;
    tpoint.query_ecommerce_conversions += qgmv_eurcent > 0 ? 1 : 0;
    dpoint.query_ecommerce_conversions += qgmv_eurcent > 0 ? 1 : 0;
    tpoint.query_abandoned_carts += (qgmv_eurcent == 0 && qcart_value_eurcent > 0) ? 1 : 0;
    dpoint.query_abandoned_carts += (qgmv_eurcent == 0 && qcart_value_eurcent > 0) ? 1 : 0;

    if (new_session_) {
      ++tpoint.num_sessions;
      ++dpoint.num_sessions;
    }
  }

  new_session_ = false;
}

RefPtr<AnalyticsQueryResult::SubQueryResult> SearchDashboardQuery::result() {
  return result_.get();
}

void SearchDashboardQuery::setDrilldownFn(
    const String& type,
    AnalyticsTableScan* query) {
  if (type == "time") {
    drilldown_fn_ = [] { return String{}; };
  }

  else if (type == "device_type") {
    auto col = query->fetchColumn("search_queries.device_type");
    drilldown_fn_ = [col] {
      return String(deviceTypeToString((DeviceType) col->getUInt32()));
    };
  }

  else if (type == "ab_test_group") {
    auto col = query->fetchColumn("search_queries.ab_test_group");
    drilldown_fn_ = [col] {
      return StringUtil::toString(col->getUInt32());
    };
  }

  else if (type == "referrer_url") {
    auto col = query->fetchColumn("referrer_url");
    drilldown_fn_ = [col] () -> String {
      auto s = col->getString();
      return s.length() == 0 ? "unknown" : s;
    };
  }

  else if (type == "referrer_name") {
    auto col = query->fetchColumn("referrer_name");
    drilldown_fn_ = [col] () -> String {
      auto s = col->getString();
      return s.length() == 0 ? "unknown" : s;
    };
  }

  else if (type == "referrer_campaign") {
    auto col = query->fetchColumn("referrer_campaign");
    drilldown_fn_ = [col] () -> String {
      auto s = col->getString();
      return s.length() == 0 ? "unknown" : s;
    };
  }

  else if (type == "search_experiment") {
    auto col = query->fetchColumn("search_queries.experiment");
    drilldown_fn_ = [col] () -> String {
      auto e = col->getString();
      if (e.length() > 0) {
        auto p = e.find(';');
        if (p != String::npos) {
          e.erase(e.begin() + p, e.end());
        }
        return e;
      } else {
        return "control";
      }
    };
  }

  else if (type == "session_experiment") {
    auto col = query->fetchColumn("experiment");
    drilldown_fn_ = [col] () -> String {
      auto e = col->getString();
      if (e.length() > 0) {
        auto p = e.find(';');
        if (p != String::npos) {
          e.erase(e.begin() + p, e.end());
        }
        return e;
      } else {
        return "control";
      }
    };
  }

  else {
    RAISEF(kRuntimeError, "invalid drilldown type: '$0'", type);
  }
}

} // namespace zbase

<commit_msg>dont fetch page_type<commit_after>/**
 * Copyright (c) 2015 - The CM Authors <legal@clickmatcher.com>
 *   All Rights Reserved.
 *
 * This file is CONFIDENTIAL -- Distribution or duplication of this material or
 * the information contained herein is strictly forbidden unless prior written
 * permission is obtained.
 */
#include "SearchDashboardQuery.h"

#include <stx/uri.h>

using namespace stx;

namespace zbase {

SearchDashboardQuery::SearchDashboardQuery(
    AnalyticsTableScan* query,
    const Vector<RefPtr<TrafficSegment>>& segments,
    UnixTime start_time,
    UnixTime end_time,
    const AnalyticsQuery::SubQueryParams& params) :
    start_time_(start_time.unixMicros() / kMicrosPerSecond),
    end_time_(end_time.unixMicros() / kMicrosPerSecond),
    result_(new TimeseriesDrilldownResult<SearchCTRStats>()),
    segments_(segments),
    time_col_(query->fetchColumn("search_queries.time")),
    num_items_col_(query->fetchColumn("search_queries.num_result_items")),
    num_itemclicks_col_(query->fetchColumn("search_queries.num_result_items_clicked")),
    num_adimprs_col_(query->fetchColumn("search_queries.num_ad_impressions")),
    num_adclicks_col_(query->fetchColumn("search_queries.num_ad_clicks")),
    qnum_cartitems_col_(query->fetchColumn("search_queries.num_cart_items")),
    qnum_orderitems_col_(query->fetchColumn("search_queries.num_order_items")),
    qcartvalue_col_(query->fetchColumn("search_queries.cart_value_eurcents")),
    qgmv_col_(query->fetchColumn("search_queries.gmv_eurcents")),
    new_session_(false),
    window_secs_(kSecondsPerDay) {
  query->onSession(std::bind(&SearchDashboardQuery::onSession, this));
  query->onQuery(std::bind(&SearchDashboardQuery::onQuery, this));

  for (const auto& s : segments_) {
    result_->drilldown.segment_keys.emplace_back(s->key());
    result_->drilldown.counters.emplace_back();
    result_->timeseries.segment_keys.emplace_back(s->key());
    result_->timeseries.series.emplace_back();
  }

  String drilldown_type = "time";
  URI::getParam(params.params, "drilldown", &drilldown_type);
  setDrilldownFn(drilldown_type, query);

  String window_str;
  if (URI::getParam(params.params, "window_size", &window_str)) {
    window_secs_ = std::stoull(window_str);
  }
}

void SearchDashboardQuery::onSession() {
  new_session_ = true;
}

void SearchDashboardQuery::onQuery() {
  auto time = time_col_->getUInt32();
  auto num_items = num_items_col_->getUInt32();
  auto num_clicks = num_itemclicks_col_->getUInt32();
  auto num_ad_imprs = num_adimprs_col_->getUInt32();
  auto num_ad_clicks = num_adclicks_col_->getUInt32();
  auto qnum_cartitems = qnum_cartitems_col_->getUInt32();
  auto qnum_orderitems = qnum_orderitems_col_->getUInt32();
  auto qcart_value_eurcent = qcartvalue_col_->getUInt32();
  auto qgmv_eurcent = qgmv_col_->getUInt32();

  //if (pagetype != PageType::SEARCH_PAGE) {
  //  return;
  //}

  auto drilldown_dim = drilldown_fn_();

  for (int i = 0; i < segments_.size(); ++i) {
    auto& series = result_->timeseries.series[i];
    auto& tpoint = series[(time / window_secs_) * window_secs_];
    auto& dpoint = result_->drilldown.counters[i][drilldown_dim];

    if (!segments_[i]->checkPredicates()) {
      continue;
    }

    tpoint.num_queries += 1;
    dpoint.num_queries += 1;
    tpoint.num_queries_clicked += num_clicks > 0 ? 1 : 0;
    dpoint.num_queries_clicked += num_clicks > 0 ? 1 : 0;
    tpoint.num_query_items_clicked += num_clicks;
    dpoint.num_query_items_clicked += num_clicks;
    tpoint.num_query_item_impressions += num_items;
    dpoint.num_query_item_impressions += num_items;
    tpoint.num_ad_impressions += num_ad_imprs;
    dpoint.num_ad_impressions += num_ad_imprs;
    tpoint.num_ads_clicked += num_ad_clicks;
    dpoint.num_ads_clicked += num_ad_clicks;
    tpoint.query_num_cart_items += qnum_cartitems;
    dpoint.query_num_cart_items += qnum_cartitems;
    tpoint.query_num_order_items += qnum_orderitems;
    dpoint.query_num_order_items += qnum_orderitems;
    tpoint.query_cart_value_eurcent += qcart_value_eurcent;
    dpoint.query_cart_value_eurcent += qcart_value_eurcent;
    tpoint.query_gmv_eurcent += qgmv_eurcent;
    dpoint.query_gmv_eurcent += qgmv_eurcent;
    tpoint.query_ecommerce_conversions += qgmv_eurcent > 0 ? 1 : 0;
    dpoint.query_ecommerce_conversions += qgmv_eurcent > 0 ? 1 : 0;
    tpoint.query_abandoned_carts += (qgmv_eurcent == 0 && qcart_value_eurcent > 0) ? 1 : 0;
    dpoint.query_abandoned_carts += (qgmv_eurcent == 0 && qcart_value_eurcent > 0) ? 1 : 0;

    if (new_session_) {
      ++tpoint.num_sessions;
      ++dpoint.num_sessions;
    }
  }

  new_session_ = false;
}

RefPtr<AnalyticsQueryResult::SubQueryResult> SearchDashboardQuery::result() {
  return result_.get();
}

void SearchDashboardQuery::setDrilldownFn(
    const String& type,
    AnalyticsTableScan* query) {
  if (type == "time") {
    drilldown_fn_ = [] { return String{}; };
  }

  else if (type == "device_type") {
    auto col = query->fetchColumn("search_queries.device_type");
    drilldown_fn_ = [col] {
      return String(deviceTypeToString((DeviceType) col->getUInt32()));
    };
  }

  else if (type == "ab_test_group") {
    auto col = query->fetchColumn("search_queries.ab_test_group");
    drilldown_fn_ = [col] {
      return StringUtil::toString(col->getUInt32());
    };
  }

  else if (type == "referrer_url") {
    auto col = query->fetchColumn("referrer_url");
    drilldown_fn_ = [col] () -> String {
      auto s = col->getString();
      return s.length() == 0 ? "unknown" : s;
    };
  }

  else if (type == "referrer_name") {
    auto col = query->fetchColumn("referrer_name");
    drilldown_fn_ = [col] () -> String {
      auto s = col->getString();
      return s.length() == 0 ? "unknown" : s;
    };
  }

  else if (type == "referrer_campaign") {
    auto col = query->fetchColumn("referrer_campaign");
    drilldown_fn_ = [col] () -> String {
      auto s = col->getString();
      return s.length() == 0 ? "unknown" : s;
    };
  }

  else if (type == "search_experiment") {
    auto col = query->fetchColumn("search_queries.experiment");
    drilldown_fn_ = [col] () -> String {
      auto e = col->getString();
      if (e.length() > 0) {
        auto p = e.find(';');
        if (p != String::npos) {
          e.erase(e.begin() + p, e.end());
        }
        return e;
      } else {
        return "control";
      }
    };
  }

  else if (type == "session_experiment") {
    auto col = query->fetchColumn("experiment");
    drilldown_fn_ = [col] () -> String {
      auto e = col->getString();
      if (e.length() > 0) {
        auto p = e.find(';');
        if (p != String::npos) {
          e.erase(e.begin() + p, e.end());
        }
        return e;
      } else {
        return "control";
      }
    };
  }

  else {
    RAISEF(kRuntimeError, "invalid drilldown type: '$0'", type);
  }
}

} // namespace zbase

<|endoftext|>


<commit_before>/*
 * eos - A 3D Morphable Model fitting library written in modern C++11/14.
 *
 * File: include/eos/fitting/blendshape_fitting.hpp
 *
 * Copyright 2015 Patrik Huber
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#pragma once

#ifndef BLENDSHAPEFITTING_HPP_
#define BLENDSHAPEFITTING_HPP_

#include "eos/morphablemodel/Blendshape.hpp"

#include "opencv2/core/core.hpp"

#include <vector>
#include <cassert>

namespace eos {
	namespace fitting {

/**
 * Fits blendshape coefficients to given 2D landmarks, given a current face shape instance.
 * It's a linear, closed-form solution fitting algorithm, with regularisation (constraining the L2-norm of the coefficients).
 *
 * This algorithm is very similar to the shape fitting in fit_shape_to_landmarks_linear. Instead of the PCA basis, the blendshapes are used, and instead of
 * the mean, a current face instance is used to do the fitting from.
 *
 * @param[in] blendshapes A vector with blendshapes to estimate the coefficients for.
 * @param[in] face_instance A shape instance from which the blendshape coefficients should be estimated (i.e. the current mesh without expressions, e.g. estimated from a previous PCA-model fitting). A 3m x 1 matrix.
 * @param[in] affine_camera_matrix A 3x4 affine camera matrix from model to screen-space (should probably be of type CV_32FC1 as all our calculations are done with float).
 * @param[in] landmarks 2D landmarks from an image to fit the blendshapes to.
 * @param[in] vertex_ids The vertex ids in the model that correspond to the 2D points.
 * @param[in] lambda A regularisation parameter, constraining the L2-norm of the coefficients.
 * @return The estimated blendshape-coefficients.
 */
inline std::vector<float> fit_blendshapes_to_landmarks_linear(std::vector<eos::morphablemodel::Blendshape> blendshapes, cv::Mat face_instance, cv::Mat affine_camera_matrix, std::vector<cv::Vec2f> landmarks, std::vector<int> vertex_ids, float lambda=500.0f)
{
	using cv::Mat;
	assert(landmarks.size() == vertex_ids.size());

	int num_coeffs_to_fit = blendshapes.size();
	int num_landmarks = static_cast<int>(landmarks.size());

	// Copy all blendshapes into a "basis" matrix with each blendshape being a column:
	cv::Mat blendshapes_as_basis(blendshapes[0].deformation.rows, blendshapes.size(), CV_32FC1); // assert blendshapes.size() > 0 and all of them have same number of rows, and 1 col
	for (int i = 0; i < blendshapes.size(); ++i)
	{
		blendshapes[i].deformation.copyTo(blendshapes_as_basis.col(i));
	}

	// $\hat{V} \in R^{3N\times m-1}$, subselect the rows of the eigenvector matrix $V$ associated with the $N$ feature points
	// And we insert a row of zeros after every third row, resulting in matrix $\hat{V}_h \in R^{4N\times m-1}$:
	Mat V_hat_h = Mat::zeros(4 * num_landmarks, num_coeffs_to_fit, CV_32FC1);
	int row_index = 0;
	for (int i = 0; i < num_landmarks; ++i) {
		//Mat basis_rows = morphable_model.get_shape_model().get_normalised_pca_basis(vertex_ids[i]); // In the paper, the not-normalised basis might be used? I'm not sure, check it. It's even a mess in the paper. PH 26.5.2014: I think the normalised basis is fine/better.
		Mat basis_rows = blendshapes_as_basis.rowRange(vertex_ids[i] * 3, (vertex_ids[i] * 3) + 3);
		//basisRows.copyTo(V_hat_h.rowRange(rowIndex, rowIndex + 3));
		basis_rows.colRange(0, num_coeffs_to_fit).copyTo(V_hat_h.rowRange(row_index, row_index + 3));
		row_index += 4; // replace 3 rows and skip the 4th one, it has all zeros
	}
	// Form a block diagonal matrix $P \in R^{3N\times 4N}$ in which the camera matrix C (P_Affine, affine_camera_matrix) is placed on the diagonal:
	Mat P = Mat::zeros(3 * num_landmarks, 4 * num_landmarks, CV_32FC1);
	for (int i = 0; i < num_landmarks; ++i) {
		Mat submatrix_to_replace = P.colRange(4 * i, (4 * i) + 4).rowRange(3 * i, (3 * i) + 3);
		affine_camera_matrix.copyTo(submatrix_to_replace);
	}

	// The landmarks in matrix notation (in homogeneous coordinates), $3N\times 1$
	Mat y = Mat::ones(3 * num_landmarks, 1, CV_32FC1);
	for (int i = 0; i < num_landmarks; ++i) {
		y.at<float>(3 * i, 0) = landmarks[i][0];
		y.at<float>((3 * i) + 1, 0) = landmarks[i][1];
		//y.at<float>((3 * i) + 2, 0) = 1; // already 1, stays (homogeneous coordinate)
	}
	// The mean, with an added homogeneous coordinate (x_1, y_1, z_1, 1, x_2, ...)^t
	Mat v_bar = Mat::ones(4 * num_landmarks, 1, CV_32FC1);
	for (int i = 0; i < num_landmarks; ++i) {
		//cv::Vec4f model_mean = morphable_model.get_shape_model().get_mean_at_point(vertex_ids[i]);
		cv::Vec4f model_mean(face_instance.at<float>(vertex_ids[i]*3), face_instance.at<float>(vertex_ids[i]*3 + 1), face_instance.at<float>(vertex_ids[i]*3 + 2), 1.0f);
		v_bar.at<float>(4 * i, 0) = model_mean[0];
		v_bar.at<float>((4 * i) + 1, 0) = model_mean[1];
		v_bar.at<float>((4 * i) + 2, 0) = model_mean[2];
		//v_bar.at<float>((4 * i) + 3, 0) = 1; // already 1, stays (homogeneous coordinate)
		// note: now that a Vec4f is returned, we could use copyTo?
	}

	// Bring into standard regularised quadratic form:
	Mat A = P * V_hat_h; // camera matrix times the basis
	Mat b = P * v_bar - y; // camera matrix times the mean, minus the landmarks.

	Mat AtAReg = A.t() * A + lambda * Mat::eye(num_coeffs_to_fit, num_coeffs_to_fit, CV_32FC1);
	// Solve using OpenCV:
	Mat c_s;
	bool non_singular = cv::solve(AtAReg, -A.t() * b, c_s, cv::DECOMP_SVD); // DECOMP_SVD calculates the pseudo-inverse if the matrix is not invertible.
	// Because we're using SVD, non_singular will always be true. If we were to use e.g. Cholesky, we could return an expected<T>.

	return std::vector<float>(c_s);
};

	} /* namespace fitting */
} /* namespace eos */

#endif /* BLENDSHAPEFITTING_HPP_ */
<commit_msg>Added non-negative least squares blendshape fitting function<commit_after>/*
 * eos - A 3D Morphable Model fitting library written in modern C++11/14.
 *
 * File: include/eos/fitting/blendshape_fitting.hpp
 *
 * Copyright 2015 Patrik Huber
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#pragma once

#ifndef BLENDSHAPEFITTING_HPP_
#define BLENDSHAPEFITTING_HPP_

#include "eos/morphablemodel/Blendshape.hpp"

#include "nnls.h"

#include "opencv2/core/core.hpp"

#include <vector>
#include <cassert>

namespace eos {
	namespace fitting {

/**
 * Fits blendshape coefficients to given 2D landmarks, given a current face shape instance.
 * It's a linear, closed-form solution fitting algorithm, with regularisation (constraining
 * the L2-norm of the coefficients). However, there is no constraint on the coefficients,
 * so negative coefficients are allowed, which, with linear blendshapes (offsets), will most
 * likely not be desired. Thus, prefer the function
 * fit_blendshapes_to_landmarks_nnls(std::vector<eos::morphablemodel::Blendshape>, cv::Mat, cv::Mat, std::vector<cv::Vec2f>, std::vector<int>).
 *
 * This algorithm is very similar to the shape fitting in fit_shape_to_landmarks_linear.
 * Instead of the PCA basis, the blendshapes are used, and instead of the mean, a current
 * face instance is used to do the fitting from.
 *
 * @param[in] blendshapes A vector with blendshapes to estimate the coefficients for.
 * @param[in] face_instance A shape instance from which the blendshape coefficients should be estimated (i.e. the current mesh without expressions, e.g. estimated from a previous PCA-model fitting). A 3m x 1 matrix.
 * @param[in] affine_camera_matrix A 3x4 affine camera matrix from model to screen-space (should probably be of type CV_32FC1 as all our calculations are done with float).
 * @param[in] landmarks 2D landmarks from an image to fit the blendshapes to.
 * @param[in] vertex_ids The vertex ids in the model that correspond to the 2D points.
 * @param[in] lambda A regularisation parameter, constraining the L2-norm of the coefficients.
 * @return The estimated blendshape-coefficients.
 */
inline std::vector<float> fit_blendshapes_to_landmarks_linear(std::vector<eos::morphablemodel::Blendshape> blendshapes, cv::Mat face_instance, cv::Mat affine_camera_matrix, std::vector<cv::Vec2f> landmarks, std::vector<int> vertex_ids, float lambda=500.0f)
{
	using cv::Mat;
	assert(landmarks.size() == vertex_ids.size());

	int num_coeffs_to_fit = blendshapes.size();
	int num_landmarks = static_cast<int>(landmarks.size());

	// Copy all blendshapes into a "basis" matrix with each blendshape being a column:
	cv::Mat blendshapes_as_basis(blendshapes[0].deformation.rows, blendshapes.size(), CV_32FC1); // assert blendshapes.size() > 0 and all of them have same number of rows, and 1 col
	for (int i = 0; i < blendshapes.size(); ++i)
	{
		blendshapes[i].deformation.copyTo(blendshapes_as_basis.col(i));
	}

	// $\hat{V} \in R^{3N\times m-1}$, subselect the rows of the eigenvector matrix $V$ associated with the $N$ feature points
	// And we insert a row of zeros after every third row, resulting in matrix $\hat{V}_h \in R^{4N\times m-1}$:
	Mat V_hat_h = Mat::zeros(4 * num_landmarks, num_coeffs_to_fit, CV_32FC1);
	int row_index = 0;
	for (int i = 0; i < num_landmarks; ++i) {
		//Mat basis_rows = morphable_model.get_shape_model().get_normalised_pca_basis(vertex_ids[i]); // In the paper, the not-normalised basis might be used? I'm not sure, check it. It's even a mess in the paper. PH 26.5.2014: I think the normalised basis is fine/better.
		Mat basis_rows = blendshapes_as_basis.rowRange(vertex_ids[i] * 3, (vertex_ids[i] * 3) + 3);
		//basisRows.copyTo(V_hat_h.rowRange(rowIndex, rowIndex + 3));
		basis_rows.colRange(0, num_coeffs_to_fit).copyTo(V_hat_h.rowRange(row_index, row_index + 3));
		row_index += 4; // replace 3 rows and skip the 4th one, it has all zeros
	}
	// Form a block diagonal matrix $P \in R^{3N\times 4N}$ in which the camera matrix C (P_Affine, affine_camera_matrix) is placed on the diagonal:
	Mat P = Mat::zeros(3 * num_landmarks, 4 * num_landmarks, CV_32FC1);
	for (int i = 0; i < num_landmarks; ++i) {
		Mat submatrix_to_replace = P.colRange(4 * i, (4 * i) + 4).rowRange(3 * i, (3 * i) + 3);
		affine_camera_matrix.copyTo(submatrix_to_replace);
	}

	// The landmarks in matrix notation (in homogeneous coordinates), $3N\times 1$
	Mat y = Mat::ones(3 * num_landmarks, 1, CV_32FC1);
	for (int i = 0; i < num_landmarks; ++i) {
		y.at<float>(3 * i, 0) = landmarks[i][0];
		y.at<float>((3 * i) + 1, 0) = landmarks[i][1];
		//y.at<float>((3 * i) + 2, 0) = 1; // already 1, stays (homogeneous coordinate)
	}
	// The mean, with an added homogeneous coordinate (x_1, y_1, z_1, 1, x_2, ...)^t
	Mat v_bar = Mat::ones(4 * num_landmarks, 1, CV_32FC1);
	for (int i = 0; i < num_landmarks; ++i) {
		//cv::Vec4f model_mean = morphable_model.get_shape_model().get_mean_at_point(vertex_ids[i]);
		cv::Vec4f model_mean(face_instance.at<float>(vertex_ids[i]*3), face_instance.at<float>(vertex_ids[i]*3 + 1), face_instance.at<float>(vertex_ids[i]*3 + 2), 1.0f);
		v_bar.at<float>(4 * i, 0) = model_mean[0];
		v_bar.at<float>((4 * i) + 1, 0) = model_mean[1];
		v_bar.at<float>((4 * i) + 2, 0) = model_mean[2];
		//v_bar.at<float>((4 * i) + 3, 0) = 1; // already 1, stays (homogeneous coordinate)
		// note: now that a Vec4f is returned, we could use copyTo?
	}

	// Bring into standard regularised quadratic form:
	Mat A = P * V_hat_h; // camera matrix times the basis
	Mat b = P * v_bar - y; // camera matrix times the mean, minus the landmarks.

	Mat AtAReg = A.t() * A + lambda * Mat::eye(num_coeffs_to_fit, num_coeffs_to_fit, CV_32FC1);
	// Solve using OpenCV:
	Mat c_s;
	bool non_singular = cv::solve(AtAReg, -A.t() * b, c_s, cv::DECOMP_SVD); // DECOMP_SVD calculates the pseudo-inverse if the matrix is not invertible.
	// Because we're using SVD, non_singular will always be true. If we were to use e.g. Cholesky, we could return an expected<T>.

	return std::vector<float>(c_s);
};

/**
 * Fits blendshape coefficients to given 2D landmarks, given a current face shape instance.
 * Uses non-negative least-squares (NNLS) to solve for the coefficients. The NNLS algorithm
 * used doesn't support any regularisation.
 *
 * This algorithm is very similar to the shape fitting in fit_shape_to_landmarks_linear.
 * Instead of the PCA basis, the blendshapes are used, and instead of the mean, a current
 * face instance is used to do the fitting from.
 *
 * @param[in] blendshapes A vector with blendshapes to estimate the coefficients for.
 * @param[in] face_instance A shape instance from which the blendshape coefficients should be estimated (i.e. the current mesh without expressions, e.g. estimated from a previous PCA-model fitting). A 3m x 1 matrix.
 * @param[in] affine_camera_matrix A 3x4 affine camera matrix from model to screen-space (should probably be of type CV_32FC1 as all our calculations are done with float).
 * @param[in] landmarks 2D landmarks from an image to fit the blendshapes to.
 * @param[in] vertex_ids The vertex ids in the model that correspond to the 2D points.
 * @return The estimated blendshape-coefficients.
 */
inline std::vector<float> fit_blendshapes_to_landmarks_nnls(std::vector<eos::morphablemodel::Blendshape> blendshapes, cv::Mat face_instance, cv::Mat affine_camera_matrix, std::vector<cv::Vec2f> landmarks, std::vector<int> vertex_ids)
{
	using cv::Mat;
	assert(landmarks.size() == vertex_ids.size());

	int num_coeffs_to_fit = blendshapes.size();
	int num_landmarks = static_cast<int>(landmarks.size());

	// Copy all blendshapes into a "basis" matrix with each blendshape being a column:
	cv::Mat blendshapes_as_basis(blendshapes[0].deformation.rows, blendshapes.size(), CV_32FC1); // assert blendshapes.size() > 0 and all of them have same number of rows, and 1 col
	for (int i = 0; i < blendshapes.size(); ++i)
	{
		blendshapes[i].deformation.copyTo(blendshapes_as_basis.col(i));
	}

	// $\hat{V} \in R^{3N\times m-1}$, subselect the rows of the eigenvector matrix $V$ associated with the $N$ feature points
	// And we insert a row of zeros after every third row, resulting in matrix $\hat{V}_h \in R^{4N\times m-1}$:
	Mat V_hat_h = Mat::zeros(4 * num_landmarks, num_coeffs_to_fit, CV_32FC1);
	int row_index = 0;
	for (int i = 0; i < num_landmarks; ++i) {
		Mat basis_rows = blendshapes_as_basis.rowRange(vertex_ids[i] * 3, (vertex_ids[i] * 3) + 3);
		basis_rows.colRange(0, num_coeffs_to_fit).copyTo(V_hat_h.rowRange(row_index, row_index + 3)); // Todo: I think we can remove colRange here, as we always want to use all given blendshapes
		row_index += 4; // replace 3 rows and skip the 4th one, it has all zeros
	}
	// Form a block diagonal matrix $P \in R^{3N\times 4N}$ in which the camera matrix C (P_Affine, affine_camera_matrix) is placed on the diagonal:
	Mat P = Mat::zeros(3 * num_landmarks, 4 * num_landmarks, CV_32FC1);
	for (int i = 0; i < num_landmarks; ++i) {
		Mat submatrix_to_replace = P.colRange(4 * i, (4 * i) + 4).rowRange(3 * i, (3 * i) + 3);
		affine_camera_matrix.copyTo(submatrix_to_replace);
	}

	// The landmarks in matrix notation (in homogeneous coordinates), $3N\times 1$
	Mat y = Mat::ones(3 * num_landmarks, 1, CV_32FC1);
	for (int i = 0; i < num_landmarks; ++i) {
		y.at<float>(3 * i, 0) = landmarks[i][0];
		y.at<float>((3 * i) + 1, 0) = landmarks[i][1];
		//y.at<float>((3 * i) + 2, 0) = 1; // already 1, stays (homogeneous coordinate)
	}
	// The mean, with an added homogeneous coordinate (x_1, y_1, z_1, 1, x_2, ...)^t
	Mat v_bar = Mat::ones(4 * num_landmarks, 1, CV_32FC1);
	for (int i = 0; i < num_landmarks; ++i) {
		cv::Vec4f model_mean(face_instance.at<float>(vertex_ids[i]*3), face_instance.at<float>(vertex_ids[i]*3 + 1), face_instance.at<float>(vertex_ids[i]*3 + 2), 1.0f);
		v_bar.at<float>(4 * i, 0) = model_mean[0];
		v_bar.at<float>((4 * i) + 1, 0) = model_mean[1];
		v_bar.at<float>((4 * i) + 2, 0) = model_mean[2];
		//v_bar.at<float>((4 * i) + 3, 0) = 1; // already 1, stays (homogeneous coordinate)
		// note: now that a Vec4f is returned, we could use copyTo?
	}

	// Bring into standard regularised quadratic form:
	Mat A = P * V_hat_h; // camera matrix times the basis
	Mat b = P * v_bar - y; // camera matrix times the mean, minus the landmarks.

	// Solve using NNLS:
	using RowMajorMatrixXf = Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>;
	Eigen::Map<RowMajorMatrixXf> A_Eigen(A.ptr<float>(), A.rows, A.cols);
	Eigen::Map<RowMajorMatrixXf> b_Eigen(b.ptr<float>(), b.rows, b.cols);

	Eigen::VectorXf x;
	bool non_singular = Eigen::NNLS<Eigen::MatrixXf>::solve(A_Eigen, -b_Eigen, x);
	Mat c_s(x.rows(), x.cols(), CV_32FC1, x.data()); // create an OpenCV Mat header for the Eigen data

	return std::vector<float>(c_s);
};

	} /* namespace fitting */
} /* namespace eos */

#endif /* BLENDSHAPEFITTING_HPP_ */
<|endoftext|>


<commit_before>#ifndef STAN_MATH_OPENCL_REV_ARENA_MATRIX_CL_HPP
#define STAN_MATH_OPENCL_REV_ARENA_MATRIX_CL_HPP
#ifdef STAN_OPENCL

#include <stan/math/rev/core/chainable_alloc.hpp>
#include <stan/math/opencl/kernel_generator/is_kernel_expression.hpp>
#include <stan/math/opencl/matrix_cl.hpp>
#include <stan/math/opencl/matrix_cl_view.hpp>
#include <utility>

namespace stan {
namespace math {
namespace internal {
template <typename T>
class arena_matrix_cl_impl : public chainable_alloc, public matrix_cl<T> {
 public:
  using Scalar = typename matrix_cl<T>::Scalar;
  using type = typename matrix_cl<T>::type;

  template <typename... Args>
  explicit arena_matrix_cl_impl(Args&&... args)
      : chainable_alloc(), matrix_cl<T>(std::forward<Args>(args)...) {}

  arena_matrix_cl_impl(const arena_matrix_cl_impl&) = default;
  arena_matrix_cl_impl(arena_matrix_cl_impl&) = default;
  arena_matrix_cl_impl(arena_matrix_cl_impl&&) = default;
};

}  // namespace internal

/**
 * A variant of `matrix_cl` that schedules its destructor to be called, so it
 * can be used on the AD stack.
 */
template <typename T>
class arena_matrix_cl {
 private:
  internal::arena_matrix_cl_impl<T>* impl_;

 public:
  using Scalar = typename matrix_cl<T>::Scalar;
  using type = typename matrix_cl<T>::type;

  /**
   * General constructor forwards arguments to various `matrix_cl` constructors.
   * @tparam Args argument types
   * @param args arguments
   */
  template <typename... Args>
  explicit arena_matrix_cl(Args&&... args)
      : impl_(new internal::arena_matrix_cl_impl<T>(
            std::forward<Args>(args)...)) {}

  arena_matrix_cl(const arena_matrix_cl&) = default;
  arena_matrix_cl(arena_matrix_cl&) = default;
  arena_matrix_cl(arena_matrix_cl&&) = default;

  /**
   * Constructor from a kernel generator expression.
   * @tparam Expr expression type
   * @param expression expression
   */
  // we need this as a separate overload, because the general constructor is
  // explicit
  template <typename Expr,
            require_all_kernel_expressions_and_none_scalar_t<Expr>* = nullptr>
  arena_matrix_cl(Expr&& expression)  // NOLINT(runtime/explicit)
      : impl_(new internal::arena_matrix_cl_impl<T>(
            std::forward<Expr>(expression))) {}

  /**
   * Implicit conversion operator to `matrix_cl`.
   * @return `matrix_cl` equivalent to `*this`
   */
  operator matrix_cl<T>() const& { return *impl_; }  // NOLINT(runtime/explicit)
  operator matrix_cl<T>() && {                       // NOLINT(runtime/explicit)
    return std::move(*impl_);
  }

  /**
   * Evaluates `this`.
   * @return `matrix_cl` equivalent to `*this`
   */
  matrix_cl<T> eval() const& { return *impl_; }
  matrix_cl<T> eval() && { return std::move(*impl_); }

  // Wrapers to functions with explicit template parameters are implemented
  // without macros.
  template <matrix_cl_view matrix_view = matrix_cl_view::Entire>
  inline void zeros() {
    impl_->template zeros<matrix_view>();
  }
  template <matrix_cl_view matrix_view = matrix_cl_view::Entire>
  inline void zeros_strict_tri() {
    impl_->template zeros_strict_tri<matrix_view>();
  }
  template <TriangularMapCL triangular_map = TriangularMapCL::LowerToUpper>
  inline void triangular_transpose() {
    impl_->template triangular_transpose<triangular_map>();
  }

/**
 * Implements a wrapper for a non-const function in `matrix_cl`.
 * @param function_name name of the function to wrap
 */
#define ARENA_MATRIX_CL_FUNCTION_WRAPPER(function_name)       \
  template <typename... Args>                                 \
  inline decltype(auto) function_name(Args&&... args) {       \
    return impl_->function_name(std::forward<Args>(args)...); \
  }

  /**
   * Implements a wrapper for a const function in `matrix_cl`.
   * @param function_name name of the function to wrap
   */
#define ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(function_name) \
  template <typename... Args>                                 \
  inline decltype(auto) function_name(Args&&... args) const { \
    return impl_->function_name(std::forward<Args>(args)...); \
  }

  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(rows)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(cols)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(size)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(view)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(clear_write_events)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(clear_read_events)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(clear_read_write_events)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(write_events)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(read_events)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(read_write_events)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(add_read_event)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(add_write_event)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(add_read_write_event)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(wait_for_write_events)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(wait_for_read_events)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(wait_for_read_write_events)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(buffer)
  ARENA_MATRIX_CL_FUNCTION_WRAPPER(buffer)
  ARENA_MATRIX_CL_FUNCTION_WRAPPER(sub_block)
  ARENA_MATRIX_CL_FUNCTION_WRAPPER(operator=)

#undef ARENA_MATRIX_CL_FUNCTION_WRAPPER
#undef ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER
};

}  // namespace math
}  // namespace stan

#endif
#endif
<commit_msg>bugfix maybe<commit_after>#ifndef STAN_MATH_OPENCL_REV_ARENA_MATRIX_CL_HPP
#define STAN_MATH_OPENCL_REV_ARENA_MATRIX_CL_HPP
#ifdef STAN_OPENCL

#include <stan/math/rev/core/chainable_alloc.hpp>
#include <stan/math/opencl/kernel_generator/is_kernel_expression.hpp>
#include <stan/math/opencl/matrix_cl.hpp>
#include <stan/math/opencl/matrix_cl_view.hpp>
#include <utility>

namespace stan {
namespace math {
namespace internal {
template <typename T>
class arena_matrix_cl_impl : public chainable_alloc, public matrix_cl<T> {
 public:
  using Scalar = typename matrix_cl<T>::Scalar;
  using type = typename matrix_cl<T>::type;

  template <typename... Args>
  explicit arena_matrix_cl_impl(Args&&... args)
      : chainable_alloc(), matrix_cl<T>(std::forward<Args>(args)...) {}

  arena_matrix_cl_impl(const arena_matrix_cl_impl<T>&) = default;
  arena_matrix_cl_impl(arena_matrix_cl_impl<T>&) = default;
  arena_matrix_cl_impl(arena_matrix_cl_impl<T>&&) = default;
  arena_matrix_cl_impl<T>& operator=(const arena_matrix_cl_impl<T>&) = default;
  arena_matrix_cl_impl<T>& operator=(arena_matrix_cl_impl<T>&&) = default;
};

}  // namespace internal

/**
 * A variant of `matrix_cl` that schedules its destructor to be called, so it
 * can be used on the AD stack.
 */
template <typename T>
class arena_matrix_cl {
 private:
  internal::arena_matrix_cl_impl<T>* impl_;

 public:
  using Scalar = typename matrix_cl<T>::Scalar;
  using type = typename matrix_cl<T>::type;

  /**
   * General constructor forwards arguments to various `matrix_cl` constructors.
   * @tparam Args argument types
   * @param args arguments
   */
  template <typename... Args>
  explicit arena_matrix_cl(Args&&... args)
      : impl_(new internal::arena_matrix_cl_impl<T>(
            std::forward<Args>(args)...)) {}

  arena_matrix_cl(const arena_matrix_cl<T>&) = default;
  arena_matrix_cl(arena_matrix_cl<T>&) = default;
  arena_matrix_cl(arena_matrix_cl<T>&&) = default;
  arena_matrix_cl<T>& operator=(const arena_matrix_cl<T>&) = default;
  arena_matrix_cl<T>& operator=(arena_matrix_cl<T>&&) = default;

  /**
   * Constructor from a kernel generator expression.
   * @tparam Expr expression type
   * @param expression expression
   */
  // we need this as a separate overload, because the general constructor is
  // explicit
  template <typename Expr,
            require_all_kernel_expressions_and_none_scalar_t<Expr>* = nullptr>
  arena_matrix_cl(Expr&& expression)  // NOLINT(runtime/explicit)
      : impl_(new internal::arena_matrix_cl_impl<T>(
            std::forward<Expr>(expression))) {}

  /**
   * Implicit conversion operator to `matrix_cl`.
   * @return `matrix_cl` equivalent to `*this`
   */
  operator matrix_cl<T>() const& { return *impl_; }  // NOLINT(runtime/explicit)
  operator matrix_cl<T>() && {                       // NOLINT(runtime/explicit)
    return std::move(*impl_);
  }

  /**
   * Evaluates `this`.
   * @return `matrix_cl` equivalent to `*this`
   */
  matrix_cl<T> eval() const& { return *impl_; }
  matrix_cl<T> eval() && { return std::move(*impl_); }

  // Wrapers to functions with explicit template parameters are implemented
  // without macros.
  template <matrix_cl_view matrix_view = matrix_cl_view::Entire>
  inline void zeros() {
    impl_->template zeros<matrix_view>();
  }
  template <matrix_cl_view matrix_view = matrix_cl_view::Entire>
  inline void zeros_strict_tri() {
    impl_->template zeros_strict_tri<matrix_view>();
  }
  template <TriangularMapCL triangular_map = TriangularMapCL::LowerToUpper>
  inline void triangular_transpose() {
    impl_->template triangular_transpose<triangular_map>();
  }

/**
 * Implements a wrapper for a non-const function in `matrix_cl`.
 * @param function_name name of the function to wrap
 */
#define ARENA_MATRIX_CL_FUNCTION_WRAPPER(function_name)       \
  template <typename... Args>                                 \
  inline decltype(auto) function_name(Args&&... args) {       \
    return impl_->function_name(std::forward<Args>(args)...); \
  }

  /**
   * Implements a wrapper for a const function in `matrix_cl`.
   * @param function_name name of the function to wrap
   */
#define ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(function_name) \
  template <typename... Args>                                 \
  inline decltype(auto) function_name(Args&&... args) const { \
    return impl_->function_name(std::forward<Args>(args)...); \
  }

  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(rows)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(cols)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(size)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(view)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(clear_write_events)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(clear_read_events)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(clear_read_write_events)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(write_events)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(read_events)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(read_write_events)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(add_read_event)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(add_write_event)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(add_read_write_event)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(wait_for_write_events)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(wait_for_read_events)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(wait_for_read_write_events)
  ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER(buffer)
  ARENA_MATRIX_CL_FUNCTION_WRAPPER(buffer)
  ARENA_MATRIX_CL_FUNCTION_WRAPPER(sub_block)
  ARENA_MATRIX_CL_FUNCTION_WRAPPER(operator=)

#undef ARENA_MATRIX_CL_FUNCTION_WRAPPER
#undef ARENA_MATRIX_CL_CONST_FUNCTION_WRAPPER
};

}  // namespace math
}  // namespace stan

#endif
#endif
<|endoftext|>


<commit_before><commit_msg>Multithread cal3d core mesh loading.  Shaves 20ms off of avatar index requests.<commit_after><|endoftext|>


<commit_before>// Copyright 2010-2012 University of Washington. All Rights Reserved.
// LICENSE_PLACEHOLDER
// This software was created with Government support under DE
// AC05-76RL01830 awarded by the United States Department of
// Energy. The Government has certain rights in the software.

#ifndef WORKER_HPP
#define WORKER_HPP

#ifdef VTRACE
#include <vt_user.h>
#endif

#include "stack.h"
#include "StateTimer.hpp"
#include "PerformanceTools.hpp"
#include <sys/mman.h> // mprotect

#ifdef ENABLE_VALGRIND
#include <valgrind/valgrind.h>
#endif

#ifdef CORO_PROTECT_UNUSED_STACK
// for assertions: might as well not include if not using mprotect
#include <glog/logging.h>
#endif

class Scheduler;
typedef uint32_t threadid_t;
#ifdef GRAPPA_TRACE
// keeps track of last id assigned
extern int thread_last_tau_taskid;
#endif

/// Size in bytes of the stack allocated for every Thread
const size_t STACK_SIZE = 1L<<19;

class Worker;

/// A queue of threads
class ThreadQueue {
    private:
        Worker * head;
        Worker * tail;
        int len;

        std::ostream& dump( std::ostream& o ) const {
            return o << "[length:" << len
                     << "; head:" << (void*)head
                     << "; tail:" << (void*)tail << "]";
        }

    public:
        ThreadQueue ( )
            : head ( NULL )
            , tail ( NULL )
            , len ( 0 ) { }

        void enqueue(Worker * t);
        Worker * dequeue();
        int length() {
            return len;
        }

        bool empty() {
            return (head==NULL);
        }

        friend std::ostream& operator<< ( std::ostream& o, const ThreadQueue& tq );
};

/// Worker/coroutine
class Worker {
  //TODO
  //enum State { RUNNING, SUSPENDED };

  public: // TODO: privatize some members
    Worker() {}

  /* used on a context switch */
  // current stack pointer.  Since stack grows down in x86,
  // stack >= base (hopefully!)
  void * stack;

  /* used on synchronization objects (and maybe switch?) */
  // next thread field for wait queues
  Worker * next;

  /* use for just assertions? */
  // status flag
  //TODO State runstate;
  bool running;
  bool suspended;
  bool idle;

  /* used at startup and shutdown */
  Scheduler * sched;
  bool done;

  /* used less often */
  // start of the stack
  void * base;
  // size of the stack
  size_t ssize;
  threadid_t id;

  /* debugging state */
#ifdef CORO_PROTECT_UNUSED_STACK
  void *guess;
#endif

#ifdef ENABLE_VALGRIND
  // valgrind
  unsigned valgrind_stack_id;
#endif

  // pointers for tracking all coroutines
  Worker * tracking_prev;
  Worker * tracking_next;

#ifdef GRAPPA_TRACE
  int state;
  int tau_taskid;
#endif

  public:
  inline intptr_t stack_remaining() {
    register long rsp asm("rsp");
    int64_t remain = static_cast<int64_t>(rsp) - reinterpret_cast<int64_t>(this->base) - 4096;
    DCHECK_LT(remain, static_cast<int64_t>(STACK_SIZE)) << "rsp = " << reinterpret_cast<void*>(rsp) << ", base = " << base << ", STACK_SIZE = " << STACK_SIZE;
    DCHECK_GE(remain, 0) << "rsp = " << reinterpret_cast<void*>(rsp) << ", base = " << base << ", STACK_SIZE = " << STACK_SIZE;

    return remain;
  }

};

typedef void (*thread_func)(Worker *, void *arg);

/// Turns the current (system) thread into a Grappa Thread.
/// This is simply necessary so that a Scheduler can
/// cause a context switch back to the system thread when
/// the Scheduler is done
Worker * convert_to_master( Worker * me = NULL );

/// spawn a new coroutine, creating a stack and everything, but
/// doesn't run until scheduled
void coro_spawn(Worker * me, Worker * c, coro_func f, size_t ssize);

/// pass control to <to> (giving it <val>, either as an argument for a
/// new coro or the return value of its last invoke.)
static inline void * coro_invoke(Worker * me, Worker * to, void * val) {
#ifdef CORO_PROTECT_UNUSED_STACK
  if( to->base != NULL ) CHECK( 0 == mprotect( (void*)((intptr_t)to->base + 4096), to->ssize, PROT_READ | PROT_WRITE ) ) << "mprotect failed; check errno";

  // compute expected stack pointer

  // get current stack pointer
  void * rsp = NULL;
  asm volatile("mov %%rsp, %0" : "=r"(rsp) );

  // adjust by register count
  // TODO: couple this with save/restore strategies in stack.S
  int num_registers_to_save = 8;
  intptr_t stack_with_regs = (intptr_t)rsp - 8*1 - 8*num_registers_to_save; // 8-byte PC + 8-byte saved registers
  me->guess = (void*) stack_with_regs; // store for debugging

  // round down to 4KB pages
  int64_t size = (stack_with_regs - ((intptr_t)me->base + 4096)) & 0xfffffffffffff000L;
  if( me != to &&           // don't protect if we're switching to ourselves
      me->base != NULL &&   // don't protect if it's the native host thread
      size > 0 ) {
    CHECK( 0 == mprotect( (void*)((intptr_t)me->base + 4096), size, PROT_READ ) ) << "mprotect failed; check errno";
  }
#endif

  me->running = 0;
  to->running = 1;

  val = swapstacks_inline(&(me->stack), &(to->stack), val);
  return val;
}

/// Called when a Thread completes its function.
/// Thread->next will contain retval.
/// This function does not return to the caller.
void thread_exit(Worker * me, void * retval);

/// Trampoline for spawning a new thread.
static void tramp(Worker * me, void * arg) {
  // Pass control back and forth a few times to get the info we need.
  Worker * master = (Worker *) arg;
  Worker * my_thr = (Worker *) coro_invoke(me, master, NULL);
  thread_func f = (thread_func) coro_invoke(me, master, NULL);
  void * f_arg = coro_invoke(me, master, NULL);
  // Next time we're invoked, it'll be for real.
  coro_invoke(me, master, NULL);

  StateTimer::setThreadState( StateTimer::SYSTEM );
  StateTimer::enterState_system();

  // create new Tau task, and top level timer for the task
#ifdef GRAPPA_TRACE
  int new_taskid;
  TAU_CREATE_TASK(new_taskid);
  my_thr->tau_taskid = new_taskid;
  thread_last_tau_taskid = new_taskid;
  GRAPPA_PROFILE_CREATE( mainprof, "start_thread", "()", TAU_DEFAULT );
  GRAPPA_PROFILE_THREAD_START( mainprof, my_thr );
#endif

  // call thread target function
  f(my_thr, f_arg);

  // stop top level Tau task timer
  GRAPPA_PROFILE_THREAD_STOP( mainprof, my_thr );

  // We shouldn't return, but if we do, kill the Thread.
  thread_exit(my_thr, NULL);
}

/// Spawn a new Worker belonging to the Scheduler.
/// Current Worker is parent. Does NOT enqueue into any scheduling queue.
Worker * worker_spawn(Worker * me, Scheduler * sched,
                     thread_func f, void * arg);

/// Tear down a coroutine
void destroy_coro(Worker * c);

/// Delete the thread.
void destroy_thread(Worker * thr);

/// Perform a context switch to another Thread
/// @param running the current Thread
/// @param next the Thread to switch to
/// @param val pass a value to next
inline void * thread_context_switch( Worker * running, Worker * next, void * val ) {
    // This timer ensures we are able to calculate exclusive time for the previous thing in this thread's callstack,
    // so that we don't count time in another thread
    GRAPPA_THREAD_FUNCTION_PROFILE( GRAPPA_SUSPEND_GROUP, running );
#ifdef VTRACE_FULL
  VT_TRACER("context switch");
#endif
    void * res = coro_invoke( running, next, val );
    StateTimer::enterState_thread();
    return res;
}

/// Return true if the thread is in the running state
inline int thread_is_running( Worker * thr ) {
    return thr->running;
}

/// Remove a Thread from the queue and return it
inline Worker * ThreadQueue::dequeue() {
    Worker * result = head;
    if (result != NULL) {
        head = result->next;
        result->next = NULL;
        len--;
    } else {
        tail = NULL;
    }
    return result;
}

/// Add a Thread to the queue
inline void ThreadQueue::enqueue( Worker * t) {
    if (head==NULL) {
        head = t;
    } else {
        tail->next = t;
    }
    tail = t;
    t->next = NULL;
    len++;
}

typedef Worker Thread; // FIXME: remove Thread references

#endif // WORKER_HPP
<commit_msg>put prefetch for ThreadQueue back in<commit_after>// Copyright 2010-2012 University of Washington. All Rights Reserved.
// LICENSE_PLACEHOLDER
// This software was created with Government support under DE
// AC05-76RL01830 awarded by the United States Department of
// Energy. The Government has certain rights in the software.

#ifndef WORKER_HPP
#define WORKER_HPP

#ifdef VTRACE
#include <vt_user.h>
#endif

#include "stack.h"
#include "StateTimer.hpp"
#include "PerformanceTools.hpp"
#include <sys/mman.h> // mprotect

#ifdef ENABLE_VALGRIND
#include <valgrind/valgrind.h>
#endif

#ifdef CORO_PROTECT_UNUSED_STACK
// for assertions: might as well not include if not using mprotect
#include <glog/logging.h>
#endif

class Scheduler;
typedef uint32_t threadid_t;
#ifdef GRAPPA_TRACE
// keeps track of last id assigned
extern int thread_last_tau_taskid;
#endif

/// Size in bytes of the stack allocated for every Thread
const size_t STACK_SIZE = 1L<<19;

class Worker;

/// A queue of threads
class ThreadQueue {
    private:
        Worker * head;
        Worker * tail;
        int len;

        std::ostream& dump( std::ostream& o ) const {
            return o << "[length:" << len
                     << "; head:" << (void*)head
                     << "; tail:" << (void*)tail << "]";
        }

    public:
        ThreadQueue ( )
            : head ( NULL )
            , tail ( NULL )
            , len ( 0 ) { }

        void enqueue(Worker * t);
        Worker * dequeue();
        void prefetch();
        int length() {
            return len;
        }

        bool empty() {
            return (head==NULL);
        }

        friend std::ostream& operator<< ( std::ostream& o, const ThreadQueue& tq );
};

/// Worker/coroutine
class Worker {
  //TODO
  //enum State { RUNNING, SUSPENDED };

  public: // TODO: privatize some members
    Worker() {}

  /* used on a context switch */
  // current stack pointer.  Since stack grows down in x86,
  // stack >= base (hopefully!)
  void * stack;

  /* used on synchronization objects (and maybe switch?) */
  // next thread field for wait queues
  Worker * next;

  /* use for just assertions? */
  // status flag
  //TODO State runstate;
  bool running;
  bool suspended;
  bool idle;

  /* used at startup and shutdown */
  Scheduler * sched;
  bool done;

  /* used less often */
  // start of the stack
  void * base;
  // size of the stack
  size_t ssize;
  threadid_t id;

  /* debugging state */
#ifdef CORO_PROTECT_UNUSED_STACK
  void *guess;
#endif

#ifdef ENABLE_VALGRIND
  // valgrind
  unsigned valgrind_stack_id;
#endif

  // pointers for tracking all coroutines
  Worker * tracking_prev;
  Worker * tracking_next;

#ifdef GRAPPA_TRACE
  int state;
  int tau_taskid;
#endif

  public:
  inline intptr_t stack_remaining() {
    register long rsp asm("rsp");
    int64_t remain = static_cast<int64_t>(rsp) - reinterpret_cast<int64_t>(this->base) - 4096;
    DCHECK_LT(remain, static_cast<int64_t>(STACK_SIZE)) << "rsp = " << reinterpret_cast<void*>(rsp) << ", base = " << base << ", STACK_SIZE = " << STACK_SIZE;
    DCHECK_GE(remain, 0) << "rsp = " << reinterpret_cast<void*>(rsp) << ", base = " << base << ", STACK_SIZE = " << STACK_SIZE;

    return remain;
  }

  /// prefetch the Thread execution state
  inline void prefetch() {
    __builtin_prefetch( stack, 0, 3 );                           // try to keep stack in cache
    //USUSED?if( data_prefetch ) __builtin_prefetch( data_prefetch, 0, 0 );   // for low-locality data
  }

};

typedef void (*thread_func)(Worker *, void *arg);

/// Turns the current (system) thread into a Grappa Thread.
/// This is simply necessary so that a Scheduler can
/// cause a context switch back to the system thread when
/// the Scheduler is done
Worker * convert_to_master( Worker * me = NULL );

/// spawn a new coroutine, creating a stack and everything, but
/// doesn't run until scheduled
void coro_spawn(Worker * me, Worker * c, coro_func f, size_t ssize);

/// pass control to <to> (giving it <val>, either as an argument for a
/// new coro or the return value of its last invoke.)
static inline void * coro_invoke(Worker * me, Worker * to, void * val) {
#ifdef CORO_PROTECT_UNUSED_STACK
  if( to->base != NULL ) CHECK( 0 == mprotect( (void*)((intptr_t)to->base + 4096), to->ssize, PROT_READ | PROT_WRITE ) ) << "mprotect failed; check errno";

  // compute expected stack pointer

  // get current stack pointer
  void * rsp = NULL;
  asm volatile("mov %%rsp, %0" : "=r"(rsp) );

  // adjust by register count
  // TODO: couple this with save/restore strategies in stack.S
  int num_registers_to_save = 8;
  intptr_t stack_with_regs = (intptr_t)rsp - 8*1 - 8*num_registers_to_save; // 8-byte PC + 8-byte saved registers
  me->guess = (void*) stack_with_regs; // store for debugging

  // round down to 4KB pages
  int64_t size = (stack_with_regs - ((intptr_t)me->base + 4096)) & 0xfffffffffffff000L;
  if( me != to &&           // don't protect if we're switching to ourselves
      me->base != NULL &&   // don't protect if it's the native host thread
      size > 0 ) {
    CHECK( 0 == mprotect( (void*)((intptr_t)me->base + 4096), size, PROT_READ ) ) << "mprotect failed; check errno";
  }
#endif

  me->running = 0;
  to->running = 1;

  val = swapstacks_inline(&(me->stack), &(to->stack), val);
  return val;
}

/// Called when a Thread completes its function.
/// Thread->next will contain retval.
/// This function does not return to the caller.
void thread_exit(Worker * me, void * retval);

/// Trampoline for spawning a new thread.
static void tramp(Worker * me, void * arg) {
  // Pass control back and forth a few times to get the info we need.
  Worker * master = (Worker *) arg;
  Worker * my_thr = (Worker *) coro_invoke(me, master, NULL);
  thread_func f = (thread_func) coro_invoke(me, master, NULL);
  void * f_arg = coro_invoke(me, master, NULL);
  // Next time we're invoked, it'll be for real.
  coro_invoke(me, master, NULL);

  StateTimer::setThreadState( StateTimer::SYSTEM );
  StateTimer::enterState_system();

  // create new Tau task, and top level timer for the task
#ifdef GRAPPA_TRACE
  int new_taskid;
  TAU_CREATE_TASK(new_taskid);
  my_thr->tau_taskid = new_taskid;
  thread_last_tau_taskid = new_taskid;
  GRAPPA_PROFILE_CREATE( mainprof, "start_thread", "()", TAU_DEFAULT );
  GRAPPA_PROFILE_THREAD_START( mainprof, my_thr );
#endif

  // call thread target function
  f(my_thr, f_arg);

  // stop top level Tau task timer
  GRAPPA_PROFILE_THREAD_STOP( mainprof, my_thr );

  // We shouldn't return, but if we do, kill the Thread.
  thread_exit(my_thr, NULL);
}

/// Spawn a new Worker belonging to the Scheduler.
/// Current Worker is parent. Does NOT enqueue into any scheduling queue.
Worker * worker_spawn(Worker * me, Scheduler * sched,
                     thread_func f, void * arg);

/// Tear down a coroutine
void destroy_coro(Worker * c);

/// Delete the thread.
void destroy_thread(Worker * thr);

/// Perform a context switch to another Thread
/// @param running the current Thread
/// @param next the Thread to switch to
/// @param val pass a value to next
inline void * thread_context_switch( Worker * running, Worker * next, void * val ) {
    // This timer ensures we are able to calculate exclusive time for the previous thing in this thread's callstack,
    // so that we don't count time in another thread
    GRAPPA_THREAD_FUNCTION_PROFILE( GRAPPA_SUSPEND_GROUP, running );
#ifdef VTRACE_FULL
  VT_TRACER("context switch");
#endif
    void * res = coro_invoke( running, next, val );
    StateTimer::enterState_thread();
    return res;
}

/// Return true if the thread is in the running state
inline int thread_is_running( Worker * thr ) {
    return thr->running;
}

/// Remove a Thread from the queue and return it
inline Worker * ThreadQueue::dequeue() {
    Worker * result = head;
    if (result != NULL) {
        head = result->next;
        result->next = NULL;
        len--;
    } else {
        tail = NULL;
    }
    return result;
}

/// Add a Thread to the queue
inline void ThreadQueue::enqueue( Worker * t) {
    if (head==NULL) {
        head = t;
    } else {
        tail->next = t;
    }
    tail = t;
    t->next = NULL;
    len++;
}

/// Prefetch some Thread close to the front of the queue
inline void ThreadQueue::prefetch() {
    Worker * result = head;
    // try to prefetch 4 away
    if( result ) {
      if( result->next ) result = result->next;
      if( result->next ) result = result->next;
      if( result->next ) result = result->next;
      if( result->next ) result = result->next;
      result->prefetch();
    }
}

typedef Worker Thread; // FIXME: remove Thread references

#endif // WORKER_HPP
<|endoftext|>


<commit_before>#pragma once

#include <systemd/sd-bus.h>

#include <sdbusplus/utility/memory.hpp>

#include <string>

namespace sdbusplus
{

namespace message
{

namespace details
{

/** Simple wrapper class for std::string to allow conversion to and from an
 *  alternative typename. */
struct string_wrapper
{
    std::string str;

    string_wrapper() = default;
    string_wrapper(const string_wrapper&) = default;
    string_wrapper& operator=(const string_wrapper&) = default;
    string_wrapper(string_wrapper&&) = default;
    string_wrapper& operator=(string_wrapper&&) = default;
    ~string_wrapper() = default;

    string_wrapper(const std::string& str) : str(str)
    {}
    string_wrapper(std::string&& str) : str(std::move(str))
    {}

    operator const std::string&() const volatile&
    {
        return const_cast<const string_wrapper*>(this)->str;
    }
    operator std::string&&() &&
    {
        return std::move(str);
    }

    bool operator==(const string_wrapper& r) const
    {
        return str == r.str;
    }
    bool operator!=(const string_wrapper& r) const
    {
        return str != r.str;
    }
    bool operator<(const string_wrapper& r) const
    {
        return str < r.str;
    }
    bool operator==(const std::string& r) const
    {
        return str == r;
    }
    bool operator!=(const std::string& r) const
    {
        return str != r;
    }
    bool operator<(const std::string& r) const
    {
        return str < r;
    }

    friend bool operator==(const std::string& l, const string_wrapper& r)
    {
        return l == r.str;
    }
    friend bool operator!=(const std::string& l, const string_wrapper& r)
    {
        return l != r.str;
    }
    friend bool operator<(const std::string& l, const string_wrapper& r)
    {
        return l < r.str;
    }
};

/** Simple wrapper class for std::string to allow conversion to and from an
 *  alternative typename. */
struct string_path_wrapper
{
    std::string str;

    string_path_wrapper() = default;
    string_path_wrapper(const string_path_wrapper&) = default;
    string_path_wrapper& operator=(const string_path_wrapper&) = default;
    string_path_wrapper(string_path_wrapper&&) = default;
    string_path_wrapper& operator=(string_path_wrapper&&) = default;
    ~string_path_wrapper() = default;

    string_path_wrapper(const std::string& str) : str(str)
    {}
    string_path_wrapper(std::string&& str) : str(std::move(str))
    {}

    operator const std::string&() const volatile&
    {
        return const_cast<const string_path_wrapper*>(this)->str;
    }
    operator std::string&&() &&
    {
        return std::move(str);
    }

    bool operator==(const string_path_wrapper& r) const
    {
        return str == r.str;
    }
    bool operator!=(const string_path_wrapper& r) const
    {
        return str != r.str;
    }
    bool operator<(const string_path_wrapper& r) const
    {
        return str < r.str;
    }
    bool operator==(const std::string& r) const
    {
        return str == r;
    }
    bool operator!=(const std::string& r) const
    {
        return str != r;
    }
    bool operator<(const std::string& r) const
    {
        return str < r;
    }

    friend bool operator==(const std::string& l, const string_path_wrapper& r)
    {
        return l == r.str;
    }
    friend bool operator!=(const std::string& l, const string_path_wrapper& r)
    {
        return l != r.str;
    }
    friend bool operator<(const std::string& l, const string_path_wrapper& r)
    {
        return l < r.str;
    }

    std::string filename() const
    {
        std::string parent = parent_path();
        _cleanup_free_ char* out = nullptr;
        int r = sd_bus_path_decode(str.c_str(), parent.c_str(), &out);
        if (r <= 0)
        {
            return "";
        }
        std::string ret(out);
        return ret;
    }

    string_path_wrapper parent_path() const
    {
        auto index = str.rfind('/');
        if (index == std::string::npos)
        {
            return string_path_wrapper("/");
        }
        if (index <= 1)
        {
            return string_path_wrapper("/");
        }

        return str.substr(0, index);
    }

    string_path_wrapper operator/(const std::string& extId)
    {
        return this->operator/(extId.c_str());
    }

    string_path_wrapper operator/(const char* extId)
    {
        string_path_wrapper out;
        _cleanup_free_ char* encOut = nullptr;
        int ret = sd_bus_path_encode(str.c_str(), extId, &encOut);
        if (ret < 0)
        {
            return out;
        }
        out.str = encOut;
        return out;
    }

    string_path_wrapper& operator/=(const char* extId)
    {
        string_path_wrapper out = this->operator/(extId);
        this->str = std::move(out.str);
        return *this;
    }

    string_path_wrapper& operator/=(const std::string& extId)
    {
        string_path_wrapper out = this->operator/(extId);
        this->str = std::move(out.str);
        return *this;
    }
};

/** Typename for sdbus SIGNATURE types. */
struct signature_type
{};
/** Typename for sdbus UNIX_FD types. */
struct unix_fd_type
{
    int fd;

    unix_fd_type() = default;
    unix_fd_type(int f) : fd(f)
    {}

    operator int() const
    {
        return fd;
    }
};

} // namespace details

/** std::string wrapper for OBJECT_PATH. */
using object_path = details::string_path_wrapper;
/** std::string wrapper for SIGNATURE. */
using signature = details::string_wrapper;
using unix_fd = details::unix_fd_type;

} // namespace message
} // namespace sdbusplus

namespace std
{

/** Overload of std::hash for details::string_wrappers */
template <>
struct hash<sdbusplus::message::details::string_wrapper>
{
    using argument_type = sdbusplus::message::details::string_wrapper;
    using result_type = std::size_t;

    result_type operator()(argument_type const& s) const
    {
        return hash<std::string>()(s.str);
    }
};

/** Overload of std::hash for details::string_wrappers */
template <>
struct hash<sdbusplus::message::details::string_path_wrapper>
{
    using argument_type = sdbusplus::message::details::string_path_wrapper;
    using result_type = std::size_t;

    result_type operator()(argument_type const& s) const
    {
        return hash<std::string>()(s.str);
    }
};

} // namespace std
<commit_msg>Make object_path operator/ const<commit_after>#pragma once

#include <systemd/sd-bus.h>

#include <sdbusplus/utility/memory.hpp>

#include <string>

namespace sdbusplus
{

namespace message
{

namespace details
{

/** Simple wrapper class for std::string to allow conversion to and from an
 *  alternative typename. */
struct string_wrapper
{
    std::string str;

    string_wrapper() = default;
    string_wrapper(const string_wrapper&) = default;
    string_wrapper& operator=(const string_wrapper&) = default;
    string_wrapper(string_wrapper&&) = default;
    string_wrapper& operator=(string_wrapper&&) = default;
    ~string_wrapper() = default;

    string_wrapper(const std::string& str) : str(str)
    {}
    string_wrapper(std::string&& str) : str(std::move(str))
    {}

    operator const std::string&() const volatile&
    {
        return const_cast<const string_wrapper*>(this)->str;
    }
    operator std::string&&() &&
    {
        return std::move(str);
    }

    bool operator==(const string_wrapper& r) const
    {
        return str == r.str;
    }
    bool operator!=(const string_wrapper& r) const
    {
        return str != r.str;
    }
    bool operator<(const string_wrapper& r) const
    {
        return str < r.str;
    }
    bool operator==(const std::string& r) const
    {
        return str == r;
    }
    bool operator!=(const std::string& r) const
    {
        return str != r;
    }
    bool operator<(const std::string& r) const
    {
        return str < r;
    }

    friend bool operator==(const std::string& l, const string_wrapper& r)
    {
        return l == r.str;
    }
    friend bool operator!=(const std::string& l, const string_wrapper& r)
    {
        return l != r.str;
    }
    friend bool operator<(const std::string& l, const string_wrapper& r)
    {
        return l < r.str;
    }
};

/** Simple wrapper class for std::string to allow conversion to and from an
 *  alternative typename. */
struct string_path_wrapper
{
    std::string str;

    string_path_wrapper() = default;
    string_path_wrapper(const string_path_wrapper&) = default;
    string_path_wrapper& operator=(const string_path_wrapper&) = default;
    string_path_wrapper(string_path_wrapper&&) = default;
    string_path_wrapper& operator=(string_path_wrapper&&) = default;
    ~string_path_wrapper() = default;

    string_path_wrapper(const std::string& str) : str(str)
    {}
    string_path_wrapper(std::string&& str) : str(std::move(str))
    {}

    operator const std::string&() const volatile&
    {
        return const_cast<const string_path_wrapper*>(this)->str;
    }
    operator std::string&&() &&
    {
        return std::move(str);
    }

    bool operator==(const string_path_wrapper& r) const
    {
        return str == r.str;
    }
    bool operator!=(const string_path_wrapper& r) const
    {
        return str != r.str;
    }
    bool operator<(const string_path_wrapper& r) const
    {
        return str < r.str;
    }
    bool operator==(const std::string& r) const
    {
        return str == r;
    }
    bool operator!=(const std::string& r) const
    {
        return str != r;
    }
    bool operator<(const std::string& r) const
    {
        return str < r;
    }

    friend bool operator==(const std::string& l, const string_path_wrapper& r)
    {
        return l == r.str;
    }
    friend bool operator!=(const std::string& l, const string_path_wrapper& r)
    {
        return l != r.str;
    }
    friend bool operator<(const std::string& l, const string_path_wrapper& r)
    {
        return l < r.str;
    }

    std::string filename() const
    {
        std::string parent = parent_path();
        _cleanup_free_ char* out = nullptr;
        int r = sd_bus_path_decode(str.c_str(), parent.c_str(), &out);
        if (r <= 0)
        {
            return "";
        }
        std::string ret(out);
        return ret;
    }

    string_path_wrapper parent_path() const
    {
        auto index = str.rfind('/');
        if (index == std::string::npos)
        {
            return string_path_wrapper("/");
        }
        if (index <= 1)
        {
            return string_path_wrapper("/");
        }

        return str.substr(0, index);
    }

    string_path_wrapper operator/(const std::string& extId) const
    {
        return this->operator/(extId.c_str());
    }

    string_path_wrapper operator/(const char* extId) const
    {
        string_path_wrapper out;
        _cleanup_free_ char* encOut = nullptr;
        int ret = sd_bus_path_encode(str.c_str(), extId, &encOut);
        if (ret < 0)
        {
            return out;
        }
        out.str = encOut;
        return out;
    }

    string_path_wrapper& operator/=(const char* extId)
    {
        string_path_wrapper out = this->operator/(extId);
        this->str = std::move(out.str);
        return *this;
    }

    string_path_wrapper& operator/=(const std::string& extId)
    {
        string_path_wrapper out = this->operator/(extId);
        this->str = std::move(out.str);
        return *this;
    }
};

/** Typename for sdbus SIGNATURE types. */
struct signature_type
{};
/** Typename for sdbus UNIX_FD types. */
struct unix_fd_type
{
    int fd;

    unix_fd_type() = default;
    unix_fd_type(int f) : fd(f)
    {}

    operator int() const
    {
        return fd;
    }
};

} // namespace details

/** std::string wrapper for OBJECT_PATH. */
using object_path = details::string_path_wrapper;
/** std::string wrapper for SIGNATURE. */
using signature = details::string_wrapper;
using unix_fd = details::unix_fd_type;

} // namespace message
} // namespace sdbusplus

namespace std
{

/** Overload of std::hash for details::string_wrappers */
template <>
struct hash<sdbusplus::message::details::string_wrapper>
{
    using argument_type = sdbusplus::message::details::string_wrapper;
    using result_type = std::size_t;

    result_type operator()(argument_type const& s) const
    {
        return hash<std::string>()(s.str);
    }
};

/** Overload of std::hash for details::string_wrappers */
template <>
struct hash<sdbusplus::message::details::string_path_wrapper>
{
    using argument_type = sdbusplus::message::details::string_path_wrapper;
    using result_type = std::size_t;

    result_type operator()(argument_type const& s) const
    {
        return hash<std::string>()(s.str);
    }
};

} // namespace std
<|endoftext|>