src/occ_405/gpu/gpu.c
// Function Specification
//
// Name:  gpu_set_pwr_limit_sm
//
// Description: Called from gpu_task_sm to set GPU power limit for G_current_gpu_id
//              This function should only return that complete is TRUE when the set
//              is complete (or determined failed) and ready for a different GPU
//
// Pre-Req:  Caller must have G_current_gpu_id set for GPU to read
//
// End Function Specification
Function Specification
Name:  gpu_set_pwr_limit_sm

Called from gpu_task_sm to set GPU power limit for G_current_gpu_id
This function should only return that complete is TRUE when the set
is complete (or determined failed) and ready for a different GPU

Pre-Req:  Caller must have G_current_gpu_id set for GPU to read

End Function Specification
Function Specification Name : gpu_set_pwr_limit_sm Called from gpu_task_sm to set GPU power limit for G_current_gpu_id This function should only return that complete is TRUE when the set is complete ( or determined failed ) and ready for a different GPU Pre - Req : Caller must have G_current_gpu_id set for GPU to read End Function Specification
bool gpu_set_pwr_limit_sm()
{
    bool l_complete = FALSE;   
    static bool L_scheduled = FALSE;  
    static uint8_t L_state_failure_count = 0;
    static uint8_t L_set_pwr_limit_failure_count[MAX_NUM_GPU_PER_DOMAIN] = {0};
    static gpuSetPwrLimitState_e L_set_pwr_limit_state = GPU_STATE_SET_PWR_LIMIT_NEW;
    static bool L_error_logged[MAX_NUM_GPU_PER_DOMAIN] = {FALSE};
    static uint32_t L_attempts = 0;
    static uint32_t L_last_pcap[MAX_NUM_GPU_PER_DOMAIN] = {0};
    static uint32_t L_num_ticks = 0;
    static bool L_retry_necessary = FALSE;
    L_num_ticks++;
    if (async_request_is_idle(&G_gpu_op_request.request))
    {
        if( (L_set_pwr_limit_state != GPU_STATE_SET_PWR_LIMIT_NEW) &&
            (!L_scheduled || (GPE_RC_SUCCESS != G_gpu_op_req_args.error.rc)) )
        {
            if( (L_set_pwr_limit_state == GPU_STATE_SET_PWR_LIMIT_4_FINISH) &&
                (GPE_RC_NOT_COMPLETE == G_gpu_op_req_args.error.rc) &&
                (L_attempts <= GPU_TIMEOUT) )
            {
                L_set_pwr_limit_state = GPU_STATE_SET_PWR_LIMIT_4_START;
                L_state_failure_count = 0;
                L_attempts++;
            }
            else if( (L_set_pwr_limit_state == GPU_STATE_SET_PWR_LIMIT_3_3) &&
                     (GPE_RC_GPU_BUSY == G_gpu_op_req_args.error.rc) )
            {
                 L_set_pwr_limit_state = GPU_STATE_SET_PWR_LIMIT_3_FINISH;
                 L_retry_necessary = TRUE;
            }
            else if(G_gpu_op_req_args.error.rc == GPE_RC_GPU_DRIVER_CHANGE)
            {
                handle_driver_change();
                L_state_failure_count = 0;
                L_set_pwr_limit_failure_count[G_current_gpu_id] = 0; 
                L_set_pwr_limit_state = GPU_STATE_SET_PWR_LIMIT_NEW;
                L_attempts = 0;
                L_retry_necessary = FALSE;
                return TRUE;  
            }
            else if( (L_state_failure_count > MAX_GPU_READ_ATTEMPT) ||
                     (L_attempts > GPU_TIMEOUT) )
            {
                if(L_attempts > GPU_TIMEOUT)
                {
                    g_amec->gpu[G_current_gpu_id].pcap.set_failed = true;
                    L_set_pwr_limit_failure_count[G_current_gpu_id] = 0;
                    INTR_TRAC_ERR("gpu_set_pwr_limit: Timedout setting power limit %d for GPU%d [attempts:%d][state_fail:%d]",
                                  G_gpu_op_req_args.data[0], G_current_gpu_id, L_attempts, L_state_failure_count);
                }
                if(g_amec->gpu[G_current_gpu_id].status.notReset)
                {
                    if(++L_set_pwr_limit_failure_count[G_current_gpu_id] > GPU_SET_PWR_LIMIT_ERROR_COUNT)
                    {
                        INTR_TRAC_ERR("gpu_set_pwr_limit: Failed to set power limit %d for GPU%d RC: 0x%02X",
                                       G_gpu_op_req_args.data[0],
                                       G_current_gpu_id,
                                       G_gpu_op_req_args.gpu_rc);
                        g_amec->gpu[G_current_gpu_id].pcap.set_failed = true;
                        L_set_pwr_limit_failure_count[G_current_gpu_id] = 0;
                        if(!L_error_logged[G_current_gpu_id])
                        {
                            L_error_logged[G_current_gpu_id] = TRUE;
                            errlHndl_t l_err = createErrl(GPU_MID_GPU_SET_PWR_LIMIT,
                                                          GPU_FAILURE,
                                                          ERC_GPU_SET_PWR_LIMIT_FAILURE,
                                                          ERRL_SEV_PREDICTIVE,
                                                          NULL,
                                                          DEFAULT_TRACE_SIZE,
                                                          G_current_gpu_id,
                                                          G_gpu_op_req_args.gpu_rc);
                            if(G_sysConfigData.gpu_sensor_ids[G_current_gpu_id])
                            {
                                addCalloutToErrl(l_err,
                                                 ERRL_CALLOUT_TYPE_GPU_ID,
                                                 G_sysConfigData.gpu_sensor_ids[G_current_gpu_id],
                                                 ERRL_CALLOUT_PRIORITY_MED);
                            }
                            commitErrl(&l_err);
                        } 
                    } 
                } 
                L_set_pwr_limit_state = GPU_STATE_SET_PWR_LIMIT_NEW;
                L_state_failure_count = 0;
                L_attempts = 0;
                return TRUE;  
            } 
            else
            {
                L_state_failure_count++;
            }
        }
        else 
        {
            L_state_failure_count = 0;
            if( (GPU_STATE_SET_PWR_LIMIT_4_FINISH == L_set_pwr_limit_state ) &&
                (L_retry_necessary) )
            {
                L_set_pwr_limit_state = GPU_STATE_SET_PWR_LIMIT_NEW;
                return TRUE;
            }
            else
            {
                L_set_pwr_limit_state++;
            }
        }
        L_scheduled = FALSE;  
        switch (L_set_pwr_limit_state)
        {
            case GPU_STATE_SET_PWR_LIMIT_1_START:
                if(!L_retry_necessary) L_num_ticks = 1;
                L_retry_necessary = FALSE;
                L_scheduled = schedule_gpu_req(GPU_REQ_SET_PWR_LIMIT_1_START, G_new_gpu_req_args);
                break;
            case GPU_STATE_SET_PWR_LIMIT_1_2:
                L_scheduled = schedule_gpu_req(GPU_REQ_SET_PWR_LIMIT_1_2, G_new_gpu_req_args);
                break;
            case GPU_STATE_SET_PWR_LIMIT_1_3:
                L_scheduled = schedule_gpu_req(GPU_REQ_SET_PWR_LIMIT_1_3, G_new_gpu_req_args);
                break;
            case GPU_STATE_SET_PWR_LIMIT_1_FINISH:
                L_scheduled = schedule_gpu_req(GPU_REQ_SET_PWR_LIMIT_1_FINISH, G_new_gpu_req_args);
                break;
            case GPU_STATE_SET_PWR_LIMIT_2_START:
                GPU_DBG("gpu_set_pwr_limit_sm: setting power limit to %dmW on GPU%d",
                         g_amec->gpu[G_current_gpu_id].pcap.gpu_desired_pcap_mw, G_current_gpu_id);
                G_new_gpu_req_args.data[1] = g_amec->gpu[G_current_gpu_id].pcap.gpu_desired_pcap_mw;
                L_scheduled = schedule_gpu_req(GPU_REQ_SET_PWR_LIMIT_2_START, G_new_gpu_req_args);
                break;
            case GPU_STATE_SET_PWR_LIMIT_2_2:
                L_scheduled = schedule_gpu_req(GPU_REQ_SET_PWR_LIMIT_2_2, G_new_gpu_req_args);
                break;
            case GPU_STATE_SET_PWR_LIMIT_2_3:
                L_scheduled = schedule_gpu_req(GPU_REQ_SET_PWR_LIMIT_2_3, G_new_gpu_req_args);
                break;
            case GPU_STATE_SET_PWR_LIMIT_2_FINISH:
                L_scheduled = schedule_gpu_req(GPU_REQ_SET_PWR_LIMIT_2_FINISH, G_new_gpu_req_args);
                break;
            case GPU_STATE_SET_PWR_LIMIT_3_START:
                L_scheduled = schedule_gpu_req(GPU_REQ_SET_PWR_LIMIT_3_START, G_new_gpu_req_args);
                break;
            case GPU_STATE_SET_PWR_LIMIT_3_2:
                L_scheduled = schedule_gpu_req(GPU_REQ_SET_PWR_LIMIT_3_2, G_new_gpu_req_args);
                break;
            case GPU_STATE_SET_PWR_LIMIT_3_3:
                L_scheduled = schedule_gpu_req(GPU_REQ_SET_PWR_LIMIT_3_3, G_new_gpu_req_args);
                break;
            case GPU_STATE_SET_PWR_LIMIT_3_FINISH:
                L_attempts = 0;
                L_scheduled = schedule_gpu_req(GPU_REQ_SET_PWR_LIMIT_3_FINISH, G_new_gpu_req_args);
                break;
            case GPU_STATE_SET_PWR_LIMIT_4_START:
                L_scheduled = schedule_gpu_req(GPU_REQ_SET_PWR_LIMIT_4_START, G_new_gpu_req_args);
                break;
            case GPU_STATE_SET_PWR_LIMIT_4_2:
                L_scheduled = schedule_gpu_req(GPU_REQ_SET_PWR_LIMIT_4_2, G_new_gpu_req_args);
                break;
            case GPU_STATE_SET_PWR_LIMIT_4_FINISH:
                L_scheduled = schedule_gpu_req(GPU_REQ_SET_PWR_LIMIT_4_FINISH, G_new_gpu_req_args);
                break;
            case GPU_STATE_SET_PWR_LIMIT_COMPLETE:
                update_gpu_tick_sensor(&G_gpu_tick_times.setpcap[G_current_gpu_id], L_num_ticks);
                GPU_DBG("gpu_set_pwr_limit_sm: took %d ticks to finish setting pcap for GPU%d", L_attempts, G_current_gpu_id);
                g_amec->gpu[G_current_gpu_id].pcap.gpu_requested_pcap_mw = (uint32_t) G_gpu_op_req_args.data[0];
                if(g_amec->gpu[G_current_gpu_id].pcap.gpu_requested_pcap_mw != L_last_pcap[G_current_gpu_id])
                {
                    L_last_pcap[G_current_gpu_id] = g_amec->gpu[G_current_gpu_id].pcap.gpu_requested_pcap_mw;
                    TRAC_IMP("gpu_set_pwr_limit_sm: successfully set power limit to %dmW on GPU%d",
                              g_amec->gpu[G_current_gpu_id].pcap.gpu_desired_pcap_mw, G_current_gpu_id);
                }
                L_set_pwr_limit_failure_count[G_current_gpu_id] = 0;
                L_set_pwr_limit_state = GPU_STATE_SET_PWR_LIMIT_NEW;
                L_attempts = 0;
                l_complete = TRUE;
                break;
            default:
                INTR_TRAC_ERR("gpu_set_pwr_limit: INVALID STATE: 0x%02X", L_set_pwr_limit_state);
                L_set_pwr_limit_state = GPU_STATE_SET_PWR_LIMIT_NEW;
                l_complete = TRUE;
                break;
        } 
        if(L_scheduled)
        {
            GPU_DBG("gpu_set_pwr_limit: Scheduled set power cap state 0x%02X at tick %d",
                    L_set_pwr_limit_state, GPU_TICK);
        }
        else if(!l_complete)  
        {
            INTR_TRAC_ERR("gpu_set_pwr_limit: failed to schedule state 0x%02X", L_set_pwr_limit_state);
        }
    } 
    else
    {
        INTR_TRAC_ERR("gpu_set_pwr_limit: NOT idle for state 0x%02X", L_set_pwr_limit_state);
    }
    return l_complete;
}
bool gpu_set_pwr_limit_sm ( ) { bool l_complete = FALSE ; static bool L_scheduled = FALSE ; static uint8_t L_state_failure_count = 0 ; static uint8_t L_set_pwr_limit_failure_count [ MAX_NUM_GPU_PER_DOMAIN ] = { 0 } ; static gpuSetPwrLimitState_e L_set_pwr_limit_state = GPU_STATE_SET_PWR_LIMIT_NEW ; static bool L_error_logged [ MAX_NUM_GPU_PER_DOMAIN ] = { FALSE } ; static uint32_t L_attempts = 0 ; static uint32_t L_last_pcap [ MAX_NUM_GPU_PER_DOMAIN ] = { 0 } ; static uint32_t L_num_ticks = 0 ; static bool L_retry_necessary = FALSE ; L_num_ticks ++ ; if ( async_request_is_idle ( & G_gpu_op_request . request ) ) { if ( ( L_set_pwr_limit_state != GPU_STATE_SET_PWR_LIMIT_NEW ) && ( ! L_scheduled || ( GPE_RC_SUCCESS != G_gpu_op_req_args . error . rc ) ) ) { if ( ( L_set_pwr_limit_state == GPU_STATE_SET_PWR_LIMIT_4_FINISH ) && ( GPE_RC_NOT_COMPLETE == G_gpu_op_req_args . error . rc ) && ( L_attempts <= GPU_TIMEOUT ) ) { L_set_pwr_limit_state = GPU_STATE_SET_PWR_LIMIT_4_START ; L_state_failure_count = 0 ; L_attempts ++ ; } else if ( ( L_set_pwr_limit_state == GPU_STATE_SET_PWR_LIMIT_3_3 ) && ( GPE_RC_GPU_BUSY == G_gpu_op_req_args . error . rc ) ) { L_set_pwr_limit_state = GPU_STATE_SET_PWR_LIMIT_3_FINISH ; L_retry_necessary = TRUE ; } else if ( G_gpu_op_req_args . error . rc == GPE_RC_GPU_DRIVER_CHANGE ) { handle_driver_change ( ) ; L_state_failure_count = 0 ; L_set_pwr_limit_failure_count [ G_current_gpu_id ] = 0 ; L_set_pwr_limit_state = GPU_STATE_SET_PWR_LIMIT_NEW ; L_attempts = 0 ; L_retry_necessary = FALSE ; return TRUE ; } else if ( ( L_state_failure_count > MAX_GPU_READ_ATTEMPT ) || ( L_attempts > GPU_TIMEOUT ) ) { if ( L_attempts > GPU_TIMEOUT ) { g_amec -> gpu [ G_current_gpu_id ] . pcap . set_failed = true ; L_set_pwr_limit_failure_count [ G_current_gpu_id ] = 0 ; INTR_TRAC_ERR ( " " , G_gpu_op_req_args . data [ 0 ] , G_current_gpu_id , L_attempts , L_state_failure_count ) ; } if ( g_amec -> gpu [ G_current_gpu_id ] . status . notReset ) { if ( ++ L_set_pwr_limit_failure_count [ G_current_gpu_id ] > GPU_SET_PWR_LIMIT_ERROR_COUNT ) { INTR_TRAC_ERR ( " " , G_gpu_op_req_args . data [ 0 ] , G_current_gpu_id , G_gpu_op_req_args . gpu_rc ) ; g_amec -> gpu [ G_current_gpu_id ] . pcap . set_failed = true ; L_set_pwr_limit_failure_count [ G_current_gpu_id ] = 0 ; if ( ! L_error_logged [ G_current_gpu_id ] ) { L_error_logged [ G_current_gpu_id ] = TRUE ; errlHndl_t l_err = createErrl ( GPU_MID_GPU_SET_PWR_LIMIT , GPU_FAILURE , ERC_GPU_SET_PWR_LIMIT_FAILURE , ERRL_SEV_PREDICTIVE , NULL , DEFAULT_TRACE_SIZE , G_current_gpu_id , G_gpu_op_req_args . gpu_rc ) ; if ( G_sysConfigData . gpu_sensor_ids [ G_current_gpu_id ] ) { addCalloutToErrl ( l_err , ERRL_CALLOUT_TYPE_GPU_ID , G_sysConfigData . gpu_sensor_ids [ G_current_gpu_id ] , ERRL_CALLOUT_PRIORITY_MED ) ; } commitErrl ( & l_err ) ; } } } L_set_pwr_limit_state = GPU_STATE_SET_PWR_LIMIT_NEW ; L_state_failure_count = 0 ; L_attempts = 0 ; return TRUE ; } else { L_state_failure_count ++ ; } } else { L_state_failure_count = 0 ; if ( ( GPU_STATE_SET_PWR_LIMIT_4_FINISH == L_set_pwr_limit_state ) && ( L_retry_necessary ) ) { L_set_pwr_limit_state = GPU_STATE_SET_PWR_LIMIT_NEW ; return TRUE ; } else { L_set_pwr_limit_state ++ ; } } L_scheduled = FALSE ; switch ( L_set_pwr_limit_state ) { case GPU_STATE_SET_PWR_LIMIT_1_START : if ( ! L_retry_necessary ) L_num_ticks = 1 ; L_retry_necessary = FALSE ; L_scheduled = schedule_gpu_req ( GPU_REQ_SET_PWR_LIMIT_1_START , G_new_gpu_req_args ) ; break ; case GPU_STATE_SET_PWR_LIMIT_1_2 : L_scheduled = schedule_gpu_req ( GPU_REQ_SET_PWR_LIMIT_1_2 , G_new_gpu_req_args ) ; break ; case GPU_STATE_SET_PWR_LIMIT_1_3 : L_scheduled = schedule_gpu_req ( GPU_REQ_SET_PWR_LIMIT_1_3 , G_new_gpu_req_args ) ; break ; case GPU_STATE_SET_PWR_LIMIT_1_FINISH : L_scheduled = schedule_gpu_req ( GPU_REQ_SET_PWR_LIMIT_1_FINISH , G_new_gpu_req_args ) ; break ; case GPU_STATE_SET_PWR_LIMIT_2_START : GPU_DBG ( " " , g_amec -> gpu [ G_current_gpu_id ] . pcap . gpu_desired_pcap_mw , G_current_gpu_id ) ; G_new_gpu_req_args . data [ 1 ] = g_amec -> gpu [ G_current_gpu_id ] . pcap . gpu_desired_pcap_mw ; L_scheduled = schedule_gpu_req ( GPU_REQ_SET_PWR_LIMIT_2_START , G_new_gpu_req_args ) ; break ; case GPU_STATE_SET_PWR_LIMIT_2_2 : L_scheduled = schedule_gpu_req ( GPU_REQ_SET_PWR_LIMIT_2_2 , G_new_gpu_req_args ) ; break ; case GPU_STATE_SET_PWR_LIMIT_2_3 : L_scheduled = schedule_gpu_req ( GPU_REQ_SET_PWR_LIMIT_2_3 , G_new_gpu_req_args ) ; break ; case GPU_STATE_SET_PWR_LIMIT_2_FINISH : L_scheduled = schedule_gpu_req ( GPU_REQ_SET_PWR_LIMIT_2_FINISH , G_new_gpu_req_args ) ; break ; case GPU_STATE_SET_PWR_LIMIT_3_START : L_scheduled = schedule_gpu_req ( GPU_REQ_SET_PWR_LIMIT_3_START , G_new_gpu_req_args ) ; break ; case GPU_STATE_SET_PWR_LIMIT_3_2 : L_scheduled = schedule_gpu_req ( GPU_REQ_SET_PWR_LIMIT_3_2 , G_new_gpu_req_args ) ; break ; case GPU_STATE_SET_PWR_LIMIT_3_3 : L_scheduled = schedule_gpu_req ( GPU_REQ_SET_PWR_LIMIT_3_3 , G_new_gpu_req_args ) ; break ; case GPU_STATE_SET_PWR_LIMIT_3_FINISH : L_attempts = 0 ; L_scheduled = schedule_gpu_req ( GPU_REQ_SET_PWR_LIMIT_3_FINISH , G_new_gpu_req_args ) ; break ; case GPU_STATE_SET_PWR_LIMIT_4_START : L_scheduled = schedule_gpu_req ( GPU_REQ_SET_PWR_LIMIT_4_START , G_new_gpu_req_args ) ; break ; case GPU_STATE_SET_PWR_LIMIT_4_2 : L_scheduled = schedule_gpu_req ( GPU_REQ_SET_PWR_LIMIT_4_2 , G_new_gpu_req_args ) ; break ; case GPU_STATE_SET_PWR_LIMIT_4_FINISH : L_scheduled = schedule_gpu_req ( GPU_REQ_SET_PWR_LIMIT_4_FINISH , G_new_gpu_req_args ) ; break ; case GPU_STATE_SET_PWR_LIMIT_COMPLETE : update_gpu_tick_sensor ( & G_gpu_tick_times . setpcap [ G_current_gpu_id ] , L_num_ticks ) ; GPU_DBG ( " " , L_attempts , G_current_gpu_id ) ; g_amec -> gpu [ G_current_gpu_id ] . pcap . gpu_requested_pcap_mw = ( uint32_t ) G_gpu_op_req_args . data [ 0 ] ; if ( g_amec -> gpu [ G_current_gpu_id ] . pcap . gpu_requested_pcap_mw != L_last_pcap [ G_current_gpu_id ] ) { L_last_pcap [ G_current_gpu_id ] = g_amec -> gpu [ G_current_gpu_id ] . pcap . gpu_requested_pcap_mw ; TRAC_IMP ( " " , g_amec -> gpu [ G_current_gpu_id ] . pcap . gpu_desired_pcap_mw , G_current_gpu_id ) ; } L_set_pwr_limit_failure_count [ G_current_gpu_id ] = 0 ; L_set_pwr_limit_state = GPU_STATE_SET_PWR_LIMIT_NEW ; L_attempts = 0 ; l_complete = TRUE ; break ; default : INTR_TRAC_ERR ( " " , L_set_pwr_limit_state ) ; L_set_pwr_limit_state = GPU_STATE_SET_PWR_LIMIT_NEW ; l_complete = TRUE ; break ; } if ( L_scheduled ) { GPU_DBG ( " " , L_set_pwr_limit_state , GPU_TICK ) ; } else if ( ! l_complete ) { INTR_TRAC_ERR ( " " , L_set_pwr_limit_state ) ; } } else { INTR_TRAC_ERR ( " " , L_set_pwr_limit_state ) ; } return l_complete ; }
Function Specification
Name:  gpu_set_pwr_limit_sm
Function Specification Name : gpu_set_pwr_limit_sm
// only return TRUE when complete or failed // indicates if a GPU GPE request was scheduled // If not starting a new set limit then need to check status of current state before moving on // stay in current state if the schedule failed or the state isn't finished/failed // Repeat step 4 until it succeeds. More details on this in GPE code. // Check if failure was due to driver change // Request can't be processed by GPU at this time so we are done with this GPU // setup to start new request // clear failure count since there's a driver change // Done with this GPU, let GPU SM move to next // If reached retry count give up on this read // give up trying to set power limit for this GPU // It will be retried if detected that GPU is put in reset and then taken out or driver change // if GPU is not in reset then INC error count and check if reached threshold // give up trying to set power limit for this GPU // It will be retried if detected that GPU is put in reset and then taken out or driver change // log error that power limit could not be set // Log error /* @
                             * @errortype
                             * @moduleid    GPU_MID_GPU_SET_PWR_LIMIT
                             * @reasoncode  GPU_FAILURE
                             * @userdata1   GPU ID
                             * @userdata2   GPU RC
                             * @userdata4   ERC_GPU_SET_PWR_LIMIT_FAILURE
                             * @devdesc     Failure to set GPU power limit
                             *
                             */ // Callout the GPU if have sensor ID for it // Commit Error // if error not logged // if reached error count // if notReset // Done with this GPU, let GPU SM move to next // if reached retry count // INC failure count and retry current state // success on last state go to next state and process it // Let SM move to next // default nothing scheduled // Step 1 // Step 2 // send the desired GPU power cap to the GPE to send to GPU // Step 3 // Step 4 // Update the requested power limit since it was successfully sent // NOTE: want this value to be sent back from the GPE to know what was set in case AMEC // has caluclated a new desired pcap while this one was already in process of being set // Done with this GPU ready to move to new one // switch L_set_pwr_limit_state // if not complete there must have been a failure on the schedule // if async_request_is_idle