original_docstring: 
      Compute Consensus Scores (CS) from Shah et al. (2019): https://arxiv.org/abs/1902.05660
      :param result_accs: A list of VQA accuracies for all questions
      :param qgroupings: A dictionary or list of tuples of {original_index: [perturbed_index1, perturbed_index2, ...], }
      :param k_vals: k from the CS definition
      
docstring: Compute Consensus Scores (CS) from Shah et al.
docstring_tokens: Compute Consensus Scores ( CS ) from Shah et al .
code: def evaluate_consistency(result_accs, qgroupings, k_vals=(1, 2, 3, 4)):
      """
      Compute Consensus Scores (CS) from Shah et al. (2019): https://arxiv.org/abs/1902.05660
      :param result_accs: A list of VQA accuracies for all questions
      :param qgroupings: A dictionary or list of tuples of {original_index: [perturbed_index1, perturbed_index2, ...], }
      :param k_vals: k from the CS definition
      """
      collective_acc = sum(result_accs) / len(result_accs)
  
      allk_total_cons = {}
      allk_results_cons = {}
      num_groupings = len(qgroupings)
      num_og_questions = 0
      num_pert_questions = 0
  
      og_acc = 0.
      pert_acc = 0.
  
      if isinstance(qgroupings, dict):
          groupings_list = list(qgroupings.items())
      else:
          groupings_list = qgroupings
  
      for i, csk in enumerate(k_vals):
          total_cons = 0.
          results_cons = []
  
          for og_idx, pert_indices in groupings_list:
              group_og_acc = [result_accs[og_idx]]
              group_pert_acc = [result_accs[gqidx] for gqidx in pert_indices]
              group_accs = group_og_acc + group_pert_acc
              group_cons = calc_consistency(group_accs, csk)
              total_cons += group_cons
              results_cons.append(group_cons)
  
              if i == 0:
                  og_acc += sum(group_og_acc)
                  num_og_questions += len(group_og_acc)
  
                  pert_acc += sum(group_pert_acc)
                  num_pert_questions += len(group_pert_acc)
  
          allk_total_cons[csk] = total_cons / num_groupings
          allk_results_cons[csk] = results_cons
  
      og_acc /= num_og_questions
      pert_acc /= num_pert_questions
  
      mean_pert_acc = (og_acc + pert_acc) / 2.
  
      return allk_total_cons, allk_results_cons, og_acc, pert_acc, collective_acc, mean_pert_acc
code_tokens: def evaluate_consistency ( result_accs , qgroupings , k_vals = ( 1 , 2 , 3 , 4 ) ) : collective_acc = sum ( result_accs ) / len ( result_accs ) allk_total_cons = { } allk_results_cons = { } num_groupings = len ( qgroupings ) num_og_questions = 0 num_pert_questions = 0 og_acc = 0. pert_acc = 0. if isinstance ( qgroupings , dict ) : groupings_list = list ( qgroupings . items ( ) ) else : groupings_list = qgroupings for i , csk in enumerate ( k_vals ) : total_cons = 0. results_cons = [ ] for og_idx , pert_indices in groupings_list : group_og_acc = [ result_accs [ og_idx ] ] group_pert_acc = [ result_accs [ gqidx ] for gqidx in pert_indices ] group_accs = group_og_acc + group_pert_acc group_cons = calc_consistency ( group_accs , csk ) total_cons += group_cons results_cons . append ( group_cons ) if i == 0 : og_acc += sum ( group_og_acc ) num_og_questions += len ( group_og_acc ) pert_acc += sum ( group_pert_acc ) num_pert_questions += len ( group_pert_acc ) allk_total_cons [ csk ] = total_cons / num_groupings allk_results_cons [ csk ] = results_cons og_acc /= num_og_questions pert_acc /= num_pert_questions mean_pert_acc = ( og_acc + pert_acc ) / 2. return allk_total_cons , allk_results_cons , og_acc , pert_acc , collective_acc , mean_pert_acc
short_docstring: Compute Consensus Scores (CS) from Shah et al.
short_docstring_tokens: Compute Consensus Scores ( CS ) from Shah et al .
comment: """
      Compute Consensus Scores (CS) from Shah et al. (2019): https://arxiv.org/abs/1902.05660
      :param result_accs: A list of VQA accuracies for all questions
      :param qgroupings: A dictionary or list of tuples of {original_index: [perturbed_index1, perturbed_index2, ...], }
      :param k_vals: k from the CS definition
      """