src/occ_405/gpu/gpu.c
// Called after a failure reading core temp for a specified GPU.  The error will
// be counted and if threshold is reached, an error will be created with
// the GPU as a callout if the GPU is not in reset
Called after a failure reading core temp for a specified GPU.  The error will
be counted and if threshold is reached, an error will be created with
the GPU as a callout if the GPU is not in reset
Called after a failure reading core temp for a specified GPU . The error will be counted and if threshold is reached an error will be created with the GPU as a callout if the GPU is not in reset
void mark_gpu_failed(const gpu_sm_args_t *i_arg)
{
    uint32_t gpu_id = i_arg->gpu_id;
    do
    {
        if((false == g_amec->gpu[gpu_id].status.disabled) &&
           (true == g_amec->gpu[gpu_id].status.readOnce))
        {
            GPU_DBG("mark_gpu_failed: GPU%d failed in op/rc/count=0x%06X "
                    "(ffdc 0x%08X%08X)",
                    gpu_id, (i_arg->operation << 16) | (i_arg->error.rc << 8) | g_amec->gpu[gpu_id].status.errorCount,
                    WORD_HIGH(i_arg->error.ffdc), WORD_LOW(i_arg->error.ffdc));
        }
        g_amec->gpu[gpu_id].status.retryCount++;
        if(g_amec->gpu[gpu_id].status.notReset)
        {
            if( ++g_amec->gpu[gpu_id].status.errorCount > GPU_INIT_ERROR_COUNT)
            {
               g_amec->gpu[gpu_id].status.memTempFailure = true;
               g_amec->gpu[gpu_id].status.memTempNotAvailable = true;
               g_amec->gpu[gpu_id].status.coreTempFailure = true;
               g_amec->gpu[gpu_id].status.coreTempNotAvailable = true;
               g_amec->gpu[gpu_id].status.disabled = true;
               INTR_TRAC_ERR("mark_gpu_failed: disabling GPU%d due to %d consecutive errors (op=%d)",
                             gpu_id, g_amec->gpu[gpu_id].status.errorCount, i_arg->operation);
               if(g_amec->gpu[gpu_id].status.commErrorLogged == false)
               {
                  errlHndl_t l_err = NULL;
                  l_err = createErrl(GPU_MID_MARK_GPU_FAILED,
                                     GPU_FAILURE,
                                     ERC_GPU_COMPLETE_FAILURE,
                                     ERRL_SEV_PREDICTIVE,
                                     NULL,
                                     DEFAULT_TRACE_SIZE,
                                     i_arg->error.rc,
                                     0);
                  addUsrDtlsToErrl(l_err,
                                   (uint8_t*)&i_arg->error.ffdc,
                                   sizeof(i_arg->error.ffdc),
                                   ERRL_STRUCT_VERSION_1,
                                   ERRL_USR_DTL_BINARY_DATA);
                  if(G_sysConfigData.gpu_sensor_ids[gpu_id])
                  {
                     addCalloutToErrl(l_err,
                                      ERRL_CALLOUT_TYPE_GPU_ID,
                                      G_sysConfigData.gpu_sensor_ids[gpu_id],
                                      ERRL_CALLOUT_PRIORITY_MED);
                  }
                  commitErrl(&l_err);
                  g_amec->gpu[gpu_id].status.commErrorLogged = true;
               } 
            } 
        } 
    } while(0);
    if( g_amec->gpu[gpu_id].status.retryCount > GPU_ERRORS_BEFORE_I2C_RESET)
    {
       g_amec->gpu[gpu_id].status.retryCount = 0;
       G_gpu_i2c_reset_required = true;
       G_gpu_reset_cause = gpu_id<<24 | (i_arg->error.rc & 0xFFFF);
    }
}
void mark_gpu_failed ( const gpu_sm_args_t * i_arg ) { uint32_t gpu_id = i_arg -> gpu_id ; do { if ( ( false == g_amec -> gpu [ gpu_id ] . status . disabled ) && ( true == g_amec -> gpu [ gpu_id ] . status . readOnce ) ) { GPU_DBG ( " " " " , gpu_id , ( i_arg -> operation << 16 ) | ( i_arg -> error . rc << 8 ) | g_amec -> gpu [ gpu_id ] . status . errorCount , WORD_HIGH ( i_arg -> error . ffdc ) , WORD_LOW ( i_arg -> error . ffdc ) ) ; } g_amec -> gpu [ gpu_id ] . status . retryCount ++ ; if ( g_amec -> gpu [ gpu_id ] . status . notReset ) { if ( ++ g_amec -> gpu [ gpu_id ] . status . errorCount > GPU_INIT_ERROR_COUNT ) { g_amec -> gpu [ gpu_id ] . status . memTempFailure = true ; g_amec -> gpu [ gpu_id ] . status . memTempNotAvailable = true ; g_amec -> gpu [ gpu_id ] . status . coreTempFailure = true ; g_amec -> gpu [ gpu_id ] . status . coreTempNotAvailable = true ; g_amec -> gpu [ gpu_id ] . status . disabled = true ; INTR_TRAC_ERR ( " " , gpu_id , g_amec -> gpu [ gpu_id ] . status . errorCount , i_arg -> operation ) ; if ( g_amec -> gpu [ gpu_id ] . status . commErrorLogged == false ) { errlHndl_t l_err = NULL ; l_err = createErrl ( GPU_MID_MARK_GPU_FAILED , GPU_FAILURE , ERC_GPU_COMPLETE_FAILURE , ERRL_SEV_PREDICTIVE , NULL , DEFAULT_TRACE_SIZE , i_arg -> error . rc , 0 ) ; addUsrDtlsToErrl ( l_err , ( uint8_t * ) & i_arg -> error . ffdc , sizeof ( i_arg -> error . ffdc ) , ERRL_STRUCT_VERSION_1 , ERRL_USR_DTL_BINARY_DATA ) ; if ( G_sysConfigData . gpu_sensor_ids [ gpu_id ] ) { addCalloutToErrl ( l_err , ERRL_CALLOUT_TYPE_GPU_ID , G_sysConfigData . gpu_sensor_ids [ gpu_id ] , ERRL_CALLOUT_PRIORITY_MED ) ; } commitErrl ( & l_err ) ; g_amec -> gpu [ gpu_id ] . status . commErrorLogged = true ; } } } } while ( 0 ) ; if ( g_amec -> gpu [ gpu_id ] . status . retryCount > GPU_ERRORS_BEFORE_I2C_RESET ) { g_amec -> gpu [ gpu_id ] . status . retryCount = 0 ; G_gpu_i2c_reset_required = true ; G_gpu_reset_cause = gpu_id << 24 | ( i_arg -> error . rc & 0xFFFF ) ; } }
Called after a failure reading core temp for a specified GPU.
Called after a failure reading core temp for a specified GPU .
// Always inc retry count for I2C reset regardless of if GPU is in reset or not // Only inc error count if it is known that GPU is NOT in reset // NOTE: Default is IN reset so this will only be true when OPAL/OS supports telling the OCC reset status // if OS never tells the OCC reset status the OCC will never disable or log a comm error // INC count and check if reached error threshold // set that GPU temperature readings failed // Disable this GPU.  GPU will get re-enabled if detected that GPU is put in reset and then taken out /*
                   * @errortype
                   * @moduleid    GPU_MID_MARK_GPU_FAILED
                   * @reasoncode  GPU_FAILURE
                   * @userdata1   GPE returned rc code
                   * @userdata4   ERC_GPU_COMPLETE_FAILURE
                   * @devdesc     GPU failure
                   */ // Callout the GPU if have sensor ID for it // if !commErrorLogged // if errorCount > threshold // if notReset // Do an I2C reset if reached retry count // don't want to do I2C reset every time since could be that this GPU really is in reset and // while resetting I2C we are unable to read other GPUs that may not be in reset