av1/encoder/mcomp.c
// Calculates the variance of prediction residue.
Calculates the variance of prediction residue.
Calculates the variance of prediction residue .
static int upsampled_pref_error(MACROBLOCKD *xd, const AV1_COMMON *cm,
                                const MV *this_mv, const uint8_t *src,
                                int src_stride, const uint8_t *ref,
                                int ref_stride,
                                const SUBPEL_SEARCH_VAR_PARAMS *var_params,
                                unsigned int *sse) {
  const aom_variance_fn_ptr_t *vfp = var_params->vfp;
  const SUBPEL_SEARCH_TYPE subpel_search_type = var_params->subpel_search_type;
  const uint8_t *second_pred = var_params->second_pred;
  const uint8_t *mask = var_params->mask;
  const int mask_stride = var_params->mask_stride;
  const int invert_mask = var_params->invert_mask;
  const int w = var_params->w;
  const int h = var_params->h;
  const int mi_row = xd->mi_row;
  const int mi_col = xd->mi_col;
  const int subpel_x_q3 = sp(this_mv->col);
  const int subpel_y_q3 = sp(this_mv->row);
  ref = pre(ref, ref_stride, this_mv->row, this_mv->col);
  unsigned int besterr;
#if CONFIG_AV1_HIGHBITDEPTH
  if (is_cur_buf_hbd(xd)) {
    DECLARE_ALIGNED(16, uint16_t, pred16[MAX_SB_SQUARE]);
    uint8_t *pred8 = CONVERT_TO_BYTEPTR(pred16);
    if (second_pred != NULL) {
      if (mask) {
        aom_highbd_comp_mask_upsampled_pred(
            xd, cm, mi_row, mi_col, this_mv, pred8, second_pred, w, h,
            subpel_x_q3, subpel_y_q3, ref, ref_stride, mask, mask_stride,
            invert_mask, xd->bd, subpel_search_type);
      } else {
        aom_highbd_comp_avg_upsampled_pred(
            xd, cm, mi_row, mi_col, this_mv, pred8, second_pred, w, h,
            subpel_x_q3, subpel_y_q3, ref, ref_stride, xd->bd,
            subpel_search_type);
      }
    } else {
      aom_highbd_upsampled_pred(xd, cm, mi_row, mi_col, this_mv, pred8, w, h,
                                subpel_x_q3, subpel_y_q3, ref, ref_stride,
                                xd->bd, subpel_search_type);
    }
    besterr = vfp->vf(pred8, w, src, src_stride, sse);
  } else {
    DECLARE_ALIGNED(16, uint8_t, pred[MAX_SB_SQUARE]);
    if (second_pred != NULL) {
      if (mask) {
        aom_comp_mask_upsampled_pred(
            xd, cm, mi_row, mi_col, this_mv, pred, second_pred, w, h,
            subpel_x_q3, subpel_y_q3, ref, ref_stride, mask, mask_stride,
            invert_mask, subpel_search_type);
      } else {
        aom_comp_avg_upsampled_pred(xd, cm, mi_row, mi_col, this_mv, pred,
                                    second_pred, w, h, subpel_x_q3, subpel_y_q3,
                                    ref, ref_stride, subpel_search_type);
      }
    } else {
      aom_upsampled_pred(xd, cm, mi_row, mi_col, this_mv, pred, w, h,
                         subpel_x_q3, subpel_y_q3, ref, ref_stride,
                         subpel_search_type);
    }
    besterr = vfp->vf(pred, w, src, src_stride, sse);
  }
#else
  DECLARE_ALIGNED(16, uint8_t, pred[MAX_SB_SQUARE]);
  if (second_pred != NULL) {
    if (mask) {
      aom_comp_mask_upsampled_pred(xd, cm, mi_row, mi_col, this_mv, pred,
                                   second_pred, w, h, subpel_x_q3, subpel_y_q3,
                                   ref, ref_stride, mask, mask_stride,
                                   invert_mask, subpel_search_type);
    } else {
      aom_comp_avg_upsampled_pred(xd, cm, mi_row, mi_col, this_mv, pred,
                                  second_pred, w, h, subpel_x_q3, subpel_y_q3,
                                  ref, ref_stride, subpel_search_type);
    }
  } else {
    aom_upsampled_pred(xd, cm, mi_row, mi_col, this_mv, pred, w, h, subpel_x_q3,
                       subpel_y_q3, ref, ref_stride, subpel_search_type);
  }
  besterr = vfp->vf(pred, w, src, src_stride, sse);
#endif
  return besterr;
}
static int upsampled_pref_error ( MACROBLOCKD * xd , const AV1_COMMON * cm , const MV * this_mv , const uint8_t * src , int src_stride , const uint8_t * ref , int ref_stride , const SUBPEL_SEARCH_VAR_PARAMS * var_params , unsigned int * sse ) { const aom_variance_fn_ptr_t * vfp = var_params -> vfp ; const SUBPEL_SEARCH_TYPE subpel_search_type = var_params -> subpel_search_type ; const uint8_t * second_pred = var_params -> second_pred ; const uint8_t * mask = var_params -> mask ; const int mask_stride = var_params -> mask_stride ; const int invert_mask = var_params -> invert_mask ; const int w = var_params -> w ; const int h = var_params -> h ; const int mi_row = xd -> mi_row ; const int mi_col = xd -> mi_col ; const int subpel_x_q3 = sp ( this_mv -> col ) ; const int subpel_y_q3 = sp ( this_mv -> row ) ; ref = pre ( ref , ref_stride , this_mv -> row , this_mv -> col ) ; unsigned int besterr ; #if CONFIG_AV1_HIGHBITDEPTH if ( is_cur_buf_hbd ( xd ) ) { DECLARE_ALIGNED ( 16 , uint16_t , pred16 [ MAX_SB_SQUARE ] ) ; uint8_t * pred8 = CONVERT_TO_BYTEPTR ( pred16 ) ; if ( second_pred != NULL ) { if ( mask ) { aom_highbd_comp_mask_upsampled_pred ( xd , cm , mi_row , mi_col , this_mv , pred8 , second_pred , w , h , subpel_x_q3 , subpel_y_q3 , ref , ref_stride , mask , mask_stride , invert_mask , xd -> bd , subpel_search_type ) ; } else { aom_highbd_comp_avg_upsampled_pred ( xd , cm , mi_row , mi_col , this_mv , pred8 , second_pred , w , h , subpel_x_q3 , subpel_y_q3 , ref , ref_stride , xd -> bd , subpel_search_type ) ; } } else { aom_highbd_upsampled_pred ( xd , cm , mi_row , mi_col , this_mv , pred8 , w , h , subpel_x_q3 , subpel_y_q3 , ref , ref_stride , xd -> bd , subpel_search_type ) ; } besterr = vfp -> vf ( pred8 , w , src , src_stride , sse ) ; } else { DECLARE_ALIGNED ( 16 , uint8_t , pred [ MAX_SB_SQUARE ] ) ; if ( second_pred != NULL ) { if ( mask ) { aom_comp_mask_upsampled_pred ( xd , cm , mi_row , mi_col , this_mv , pred , second_pred , w , h , subpel_x_q3 , subpel_y_q3 , ref , ref_stride , mask , mask_stride , invert_mask , subpel_search_type ) ; } else { aom_comp_avg_upsampled_pred ( xd , cm , mi_row , mi_col , this_mv , pred , second_pred , w , h , subpel_x_q3 , subpel_y_q3 , ref , ref_stride , subpel_search_type ) ; } } else { aom_upsampled_pred ( xd , cm , mi_row , mi_col , this_mv , pred , w , h , subpel_x_q3 , subpel_y_q3 , ref , ref_stride , subpel_search_type ) ; } besterr = vfp -> vf ( pred , w , src , src_stride , sse ) ; } #else DECLARE_ALIGNED ( 16 , uint8_t , pred [ MAX_SB_SQUARE ] ) ; if ( second_pred != NULL ) { if ( mask ) { aom_comp_mask_upsampled_pred ( xd , cm , mi_row , mi_col , this_mv , pred , second_pred , w , h , subpel_x_q3 , subpel_y_q3 , ref , ref_stride , mask , mask_stride , invert_mask , subpel_search_type ) ; } else { aom_comp_avg_upsampled_pred ( xd , cm , mi_row , mi_col , this_mv , pred , second_pred , w , h , subpel_x_q3 , subpel_y_q3 , ref , ref_stride , subpel_search_type ) ; } } else { aom_upsampled_pred ( xd , cm , mi_row , mi_col , this_mv , pred , w , h , subpel_x_q3 , subpel_y_q3 , ref , ref_stride , subpel_search_type ) ; } besterr = vfp -> vf ( pred , w , src , src_stride , sse ) ; #endif return besterr ; }
Calculates the variance of prediction residue.
Calculates the variance of prediction residue .