src/occ_405/gpu/gpu.c
// Function Specification
//
// Name:  gpu_read_memory_temp_sm
//
// Description: Called from gpu_task_sm to read GPU memory temperature of G_current_gpu_id
//              This function should only return that complete is TRUE when the temperature
//              read is complete (or determined failed) and ready to start reading a different GPU
//
// Pre-Req:  Caller must have G_current_gpu_id set for GPU to read
//
// End Function Specification
Function Specification
Name:  gpu_read_memory_temp_sm

Called from gpu_task_sm to read GPU memory temperature of G_current_gpu_id
This function should only return that complete is TRUE when the temperature
read is complete (or determined failed) and ready to start reading a different GPU

Pre-Req:  Caller must have G_current_gpu_id set for GPU to read

End Function Specification
Function Specification Name : gpu_read_memory_temp_sm Called from gpu_task_sm to read GPU memory temperature of G_current_gpu_id This function should only return that complete is TRUE when the temperature read is complete ( or determined failed ) and ready to start reading a different GPU Pre - Req : Caller must have G_current_gpu_id set for GPU to read End Function Specification
bool gpu_read_memory_temp_sm()
{
    bool l_complete = FALSE;   
    uint16_t l_temp = 0;
    static bool L_scheduled = FALSE;  
    static uint8_t L_read_failure_count = 0;
    static gpuReadMemTempState_e L_read_temp_state = GPU_STATE_READ_MEM_TEMP_NEW;  
    static uint32_t L_num_ticks = 0;
    L_num_ticks++;
    if (async_request_is_idle(&G_gpu_op_request.request))
    {
       if( (L_read_temp_state != GPU_STATE_READ_MEM_TEMP_NEW) &&
           (!L_scheduled || (GPE_RC_SUCCESS != G_gpu_op_req_args.error.rc)) )
       {
          if(G_gpu_op_req_args.error.rc == GPE_RC_GPU_DRIVER_CHANGE)
          {
             handle_driver_change();
             L_read_failure_count = 0;
             g_amec->gpu[G_current_gpu_id].status.memErrorCount = 0;
             L_read_temp_state = GPU_STATE_READ_MEM_TEMP_NEW;
             return TRUE;  
          }
          else if( (L_read_failure_count > MAX_GPU_READ_ATTEMPT) ||
                   (G_gpu_op_req_args.error.rc == GPE_RC_GPU_CMD_NOT_SUPPORTED) )
          {
             if( (g_amec->gpu[G_current_gpu_id].status.notReset) ||
                 (G_gpu_op_req_args.error.rc == GPE_RC_GPU_CMD_NOT_SUPPORTED) )
             {
                g_amec->gpu[G_current_gpu_id].status.memErrorCount++;
                uint8_t max_read_timeout = G_data_cnfg->thrm_thresh.data[DATA_FRU_GPU_MEM].max_read_timeout;
                if((max_read_timeout) && (max_read_timeout != 0xFF) &&
                   (g_amec->gpu[G_current_gpu_id].status.memErrorCount >= max_read_timeout) )
                {
                    g_amec->gpu[G_current_gpu_id].status.memTempSupported = FALSE;
                    g_amec->gpu[G_current_gpu_id].status.memTempFailure = true;
                    g_amec->gpu[G_current_gpu_id].status.memTempNotAvailable = true;
                    INTR_TRAC_ERR("gpu_read_memory_temp: disabling memory temp for GPU%d due to %d consecutive errors",
                                    G_current_gpu_id, g_amec->gpu[G_current_gpu_id].status.memErrorCount);
                    if(g_amec->gpu[G_current_gpu_id].status.commErrorLogged == false)
                    {
                       INTR_TRAC_ERR("notReset: %d rc: 0x%0X", g_amec->gpu[G_current_gpu_id].status.notReset,
                                     G_gpu_op_req_args.error.rc);
                       errlHndl_t l_err = createErrl(GPU_MID_GPU_READ_MEM_TEMP,
                                                     GPU_FAILURE,
                                                     ERC_GPU_READ_MEM_TEMP_TIMEOUT,
                                                     ERRL_SEV_PREDICTIVE,
                                                     NULL,
                                                     DEFAULT_TRACE_SIZE,
                                                     G_gpu_op_req_args.gpu_rc,
                                                     g_amec->gpu[G_current_gpu_id].status.memErrorCount);
                       if(G_sysConfigData.gpu_sensor_ids[G_current_gpu_id])
                       {
                          addCalloutToErrl(l_err,
                                           ERRL_CALLOUT_TYPE_GPU_ID,
                                           G_sysConfigData.gpu_sensor_ids[G_current_gpu_id],
                                           ERRL_CALLOUT_PRIORITY_MED);
                       }
                       commitErrl(&l_err);
                       g_amec->gpu[G_current_gpu_id].status.commErrorLogged = true;
                    } 
                } 
                else if(G_gpu_op_req_args.error.rc == GPE_RC_GPU_CMD_NOT_SUPPORTED)
                {
                    g_amec->gpu[G_current_gpu_id].status.checkMemTempSupport = true;
                    g_amec->gpu[G_current_gpu_id].status.memTempSupported = false;
                }
             } 
             L_read_failure_count = 0;
             L_read_temp_state = GPU_STATE_READ_MEM_TEMP_NEW;
             return TRUE;  
          }  
          else
          {
             L_read_failure_count++;
          }
       }
       else 
       {
          L_read_failure_count = 0;
          L_read_temp_state++;
       }
       L_scheduled = FALSE;  
       switch (L_read_temp_state)
       {
           case GPU_STATE_READ_MEM_TEMP_START:
               L_num_ticks = 1;
               L_scheduled = schedule_gpu_req(GPU_REQ_READ_MEM_TEMP_START, G_new_gpu_req_args);
               break;
           case GPU_STATE_READ_MEM_TEMP_2:
               L_scheduled = schedule_gpu_req(GPU_REQ_READ_MEM_TEMP_2, G_new_gpu_req_args);
               break;
           case GPU_STATE_READ_MEM_TEMP_3:
               L_scheduled = schedule_gpu_req(GPU_REQ_READ_MEM_TEMP_3, G_new_gpu_req_args);
               break;
            case GPU_STATE_READ_MEM_TEMP_READ:
                L_scheduled = schedule_gpu_req(GPU_REQ_READ_MEM_TEMP_FINISH, G_new_gpu_req_args);
                break;
           case GPU_STATE_READ_MEM_TEMP_COMPLETE:
               update_gpu_tick_sensor(&G_gpu_tick_times.memtemp[G_current_gpu_id], L_num_ticks);
               l_temp = G_gpu_op_req_args.data[0];
               sensor_update(AMECSENSOR_PTR(TEMPGPU0MEM + G_current_gpu_id), l_temp);
               g_amec->gpu[G_current_gpu_id].status.memTempFailure = false;
               g_amec->gpu[G_current_gpu_id].status.memTempNotAvailable = false;
               g_amec->gpu[G_current_gpu_id].status.memErrorCount = 0;
               if((G_data_cnfg->thrm_thresh.data[DATA_FRU_GPU_MEM].error) &&
                  (l_temp > G_data_cnfg->thrm_thresh.data[DATA_FRU_GPU_MEM].error) &&
                  (!g_amec->gpu[G_current_gpu_id].status.memOvertempError) )
               {
                   g_amec->gpu[G_current_gpu_id].status.memOvertempError = TRUE;
                   INTR_TRAC_ERR("gpu_read_memory_temp: GPU%d memory OT! temp[%d]",
                                  G_current_gpu_id, l_temp);
                   errlHndl_t l_err = createErrl(GPU_MID_GPU_READ_MEM_TEMP,
                                                 GPU_MEMORY_ERROR_TEMP,
                                                 OCC_NO_EXTENDED_RC,
                                                 ERRL_SEV_PREDICTIVE,
                                                 NULL,
                                                 DEFAULT_TRACE_SIZE,
                                                 G_current_gpu_id,
                                                 l_temp);
                   addCalloutToErrl(l_err,
                                    ERRL_CALLOUT_TYPE_COMPONENT_ID,
                                    ERRL_COMPONENT_ID_OVER_TEMPERATURE,
                                    ERRL_CALLOUT_PRIORITY_HIGH);
                   if(G_sysConfigData.gpu_sensor_ids[G_current_gpu_id])
                   {
                      addCalloutToErrl(l_err,
                                       ERRL_CALLOUT_TYPE_GPU_ID,
                                       G_sysConfigData.gpu_sensor_ids[G_current_gpu_id],
                                       ERRL_CALLOUT_PRIORITY_MED);
                   }
                   commitErrl(&l_err);
               } 
               L_read_temp_state = GPU_STATE_READ_MEM_TEMP_NEW;
               l_complete = TRUE;
               break;
           default:
               INTR_TRAC_ERR("gpu_read_memory_temp_sm: INVALID STATE: 0x%02X", L_read_temp_state);
               L_read_temp_state = GPU_STATE_READ_MEM_TEMP_NEW;
               l_complete = TRUE;
               break;
       } 
       if(L_scheduled)
       {
          GPU_DBG("gpu_read_memory_temp_sm: Scheduled read temp state 0x%02X at tick %d",
                   L_read_temp_state, GPU_TICK);
       }
       else if(!l_complete)  
       {
          INTR_TRAC_ERR("gpu_read_memory_temp_sm: failed to schedule state 0x%02X", L_read_temp_state);
       }
    } 
    else
    {
       INTR_TRAC_ERR("gpu_read_memory_temp_sm: NOT idle for state 0x%02X", L_read_temp_state);
    }
    return l_complete;
}
bool gpu_read_memory_temp_sm ( ) { bool l_complete = FALSE ; uint16_t l_temp = 0 ; static bool L_scheduled = FALSE ; static uint8_t L_read_failure_count = 0 ; static gpuReadMemTempState_e L_read_temp_state = GPU_STATE_READ_MEM_TEMP_NEW ; static uint32_t L_num_ticks = 0 ; L_num_ticks ++ ; if ( async_request_is_idle ( & G_gpu_op_request . request ) ) { if ( ( L_read_temp_state != GPU_STATE_READ_MEM_TEMP_NEW ) && ( ! L_scheduled || ( GPE_RC_SUCCESS != G_gpu_op_req_args . error . rc ) ) ) { if ( G_gpu_op_req_args . error . rc == GPE_RC_GPU_DRIVER_CHANGE ) { handle_driver_change ( ) ; L_read_failure_count = 0 ; g_amec -> gpu [ G_current_gpu_id ] . status . memErrorCount = 0 ; L_read_temp_state = GPU_STATE_READ_MEM_TEMP_NEW ; return TRUE ; } else if ( ( L_read_failure_count > MAX_GPU_READ_ATTEMPT ) || ( G_gpu_op_req_args . error . rc == GPE_RC_GPU_CMD_NOT_SUPPORTED ) ) { if ( ( g_amec -> gpu [ G_current_gpu_id ] . status . notReset ) || ( G_gpu_op_req_args . error . rc == GPE_RC_GPU_CMD_NOT_SUPPORTED ) ) { g_amec -> gpu [ G_current_gpu_id ] . status . memErrorCount ++ ; uint8_t max_read_timeout = G_data_cnfg -> thrm_thresh . data [ DATA_FRU_GPU_MEM ] . max_read_timeout ; if ( ( max_read_timeout ) && ( max_read_timeout != 0xFF ) && ( g_amec -> gpu [ G_current_gpu_id ] . status . memErrorCount >= max_read_timeout ) ) { g_amec -> gpu [ G_current_gpu_id ] . status . memTempSupported = FALSE ; g_amec -> gpu [ G_current_gpu_id ] . status . memTempFailure = true ; g_amec -> gpu [ G_current_gpu_id ] . status . memTempNotAvailable = true ; INTR_TRAC_ERR ( " " , G_current_gpu_id , g_amec -> gpu [ G_current_gpu_id ] . status . memErrorCount ) ; if ( g_amec -> gpu [ G_current_gpu_id ] . status . commErrorLogged == false ) { INTR_TRAC_ERR ( " " , g_amec -> gpu [ G_current_gpu_id ] . status . notReset , G_gpu_op_req_args . error . rc ) ; errlHndl_t l_err = createErrl ( GPU_MID_GPU_READ_MEM_TEMP , GPU_FAILURE , ERC_GPU_READ_MEM_TEMP_TIMEOUT , ERRL_SEV_PREDICTIVE , NULL , DEFAULT_TRACE_SIZE , G_gpu_op_req_args . gpu_rc , g_amec -> gpu [ G_current_gpu_id ] . status . memErrorCount ) ; if ( G_sysConfigData . gpu_sensor_ids [ G_current_gpu_id ] ) { addCalloutToErrl ( l_err , ERRL_CALLOUT_TYPE_GPU_ID , G_sysConfigData . gpu_sensor_ids [ G_current_gpu_id ] , ERRL_CALLOUT_PRIORITY_MED ) ; } commitErrl ( & l_err ) ; g_amec -> gpu [ G_current_gpu_id ] . status . commErrorLogged = true ; } } else if ( G_gpu_op_req_args . error . rc == GPE_RC_GPU_CMD_NOT_SUPPORTED ) { g_amec -> gpu [ G_current_gpu_id ] . status . checkMemTempSupport = true ; g_amec -> gpu [ G_current_gpu_id ] . status . memTempSupported = false ; } } L_read_failure_count = 0 ; L_read_temp_state = GPU_STATE_READ_MEM_TEMP_NEW ; return TRUE ; } else { L_read_failure_count ++ ; } } else { L_read_failure_count = 0 ; L_read_temp_state ++ ; } L_scheduled = FALSE ; switch ( L_read_temp_state ) { case GPU_STATE_READ_MEM_TEMP_START : L_num_ticks = 1 ; L_scheduled = schedule_gpu_req ( GPU_REQ_READ_MEM_TEMP_START , G_new_gpu_req_args ) ; break ; case GPU_STATE_READ_MEM_TEMP_2 : L_scheduled = schedule_gpu_req ( GPU_REQ_READ_MEM_TEMP_2 , G_new_gpu_req_args ) ; break ; case GPU_STATE_READ_MEM_TEMP_3 : L_scheduled = schedule_gpu_req ( GPU_REQ_READ_MEM_TEMP_3 , G_new_gpu_req_args ) ; break ; case GPU_STATE_READ_MEM_TEMP_READ : L_scheduled = schedule_gpu_req ( GPU_REQ_READ_MEM_TEMP_FINISH , G_new_gpu_req_args ) ; break ; case GPU_STATE_READ_MEM_TEMP_COMPLETE : update_gpu_tick_sensor ( & G_gpu_tick_times . memtemp [ G_current_gpu_id ] , L_num_ticks ) ; l_temp = G_gpu_op_req_args . data [ 0 ] ; sensor_update ( AMECSENSOR_PTR ( TEMPGPU0MEM + G_current_gpu_id ) , l_temp ) ; g_amec -> gpu [ G_current_gpu_id ] . status . memTempFailure = false ; g_amec -> gpu [ G_current_gpu_id ] . status . memTempNotAvailable = false ; g_amec -> gpu [ G_current_gpu_id ] . status . memErrorCount = 0 ; if ( ( G_data_cnfg -> thrm_thresh . data [ DATA_FRU_GPU_MEM ] . error ) && ( l_temp > G_data_cnfg -> thrm_thresh . data [ DATA_FRU_GPU_MEM ] . error ) && ( ! g_amec -> gpu [ G_current_gpu_id ] . status . memOvertempError ) ) { g_amec -> gpu [ G_current_gpu_id ] . status . memOvertempError = TRUE ; INTR_TRAC_ERR ( " " , G_current_gpu_id , l_temp ) ; errlHndl_t l_err = createErrl ( GPU_MID_GPU_READ_MEM_TEMP , GPU_MEMORY_ERROR_TEMP , OCC_NO_EXTENDED_RC , ERRL_SEV_PREDICTIVE , NULL , DEFAULT_TRACE_SIZE , G_current_gpu_id , l_temp ) ; addCalloutToErrl ( l_err , ERRL_CALLOUT_TYPE_COMPONENT_ID , ERRL_COMPONENT_ID_OVER_TEMPERATURE , ERRL_CALLOUT_PRIORITY_HIGH ) ; if ( G_sysConfigData . gpu_sensor_ids [ G_current_gpu_id ] ) { addCalloutToErrl ( l_err , ERRL_CALLOUT_TYPE_GPU_ID , G_sysConfigData . gpu_sensor_ids [ G_current_gpu_id ] , ERRL_CALLOUT_PRIORITY_MED ) ; } commitErrl ( & l_err ) ; } L_read_temp_state = GPU_STATE_READ_MEM_TEMP_NEW ; l_complete = TRUE ; break ; default : INTR_TRAC_ERR ( " " , L_read_temp_state ) ; L_read_temp_state = GPU_STATE_READ_MEM_TEMP_NEW ; l_complete = TRUE ; break ; } if ( L_scheduled ) { GPU_DBG ( " " , L_read_temp_state , GPU_TICK ) ; } else if ( ! l_complete ) { INTR_TRAC_ERR ( " " , L_read_temp_state ) ; } } else { INTR_TRAC_ERR ( " " , L_read_temp_state ) ; } return l_complete ; }
Function Specification
Name:  gpu_read_memory_temp_sm
Function Specification Name : gpu_read_memory_temp_sm
// only return TRUE when the read is complete or failed // indicates if a GPU GPE request was scheduled // 1st state for reading temp // If not starting a new read then need to check status of current state before moving on // stay in current state if the schedule failed or the state isn't finished/failed // Check if failure was due to driver change // Request can't be processed by GPU at this time so we are done with this GPU // setup to start new request // Done with this GPU, let GPU SM move to next // If reached retry count or GPU indicated cmd not supported then give up on this read // if GPU is not in reset or the GPU responded with command not supported then // INC memory error count and check if reached timeout for new mem temp // Disable memory temp reading for this GPU and log error // so BMC knows there is an error for fan control set failure // Log error /* @
                        * @errortype
                        * @moduleid    GPU_MID_GPU_READ_MEM_TEMP
                        * @reasoncode  GPU_FAILURE
                        * @userdata1   GPU ID
                        * @userdata2   GPU RC
                        * @userdata4   ERC_GPU_READ_MEM_TEMP_TIMEOUT
                        * @devdesc     Timeout reading new GPU memory temperature
                        *
                        */ // Callout the GPU if have sensor ID for it // Commit Error // if !commErrorLogged // if timeout error // GPU indicated command not supported, re-check mem temp capability // if we try to read mem temp again that means mem temp was reported capable // and if this continues to fail eventually an error will be logged above at timeout // if notReset or command not supported // setup to start new request // Done with this GPU, let GPU SM move to next // else if failure count exceeded or command not supported // INC failure count and retry current state // success on last state go to next state and process it // default nothing scheduled // Update sensor // Clear past errors // check if there is an overtemp that hasn't been reported // Log an OT error /* @
                    * @errortype
                    * @moduleid    GPU_MID_GPU_READ_MEM_TEMP
                    * @reasoncode  GPU_MEMORY_ERROR_TEMP
                    * @userdata1   GPU ID
                    * @userdata2   GPU memory temperature
                    * @userdata4   OCC_NO_EXTENDED_RC
                    * @devdesc     GPU memory has reached error temperature
                    *
                    */ // Callout the over temperature procedure // Callout the GPU if have sensor ID for it // Commit Error // if OT error // Done with this GPU ready to move to new one // switch L_read_temp_state // if not complete there must have been a failure on the schedule // if async_request_is_idle