This function creates a dataframe (a matrix with one row per dataset record and one column for fields of interest)
           from the benchmark run and the dataset data.
           For rows with multiple mutations, there may be multiple values for some fields e.g. wildtype residue exposure.
           We take the approach of marking these records as None (to be read as: N/A).
           Another approach is to take averages of continuous and binary values.
           This function also determines scalar_adjustments used to scale the predictions to try to improve the fraction
           correct score and the MAE.

def create_dataframe(self, pdb_data = {}, verbose = True):
        '''This function creates a dataframe (a matrix with one row per dataset record and one column for fields of interest)
           from the benchmark run and the dataset data.
           For rows with multiple mutations, there may be multiple values for some fields e.g. wildtype residue exposure.
           We take the approach of marking these records as None (to be read as: N/A).
           Another approach is to take averages of continuous and binary values.
           This function also determines scalar_adjustments used to scale the predictions to try to improve the fraction
           correct score and the MAE.
        '''

        if self.use_existing_benchmark_data and self.store_data_on_disk and os.path.exists(self.analysis_pandas_input_filepath):
            self.read_dataframe(self.analysis_pandas_input_filepath)
            return

        analysis_data = self.analysis_data
        dataset_cases = self.dataset_cases

        # Create XY data
        if self.store_data_on_disk:
            self.log('Creating the analysis input file %s and human-readable CSV and JSON versions %s and %s.' % (self.analysis_pandas_input_filepath, self.analysis_csv_input_filepath, self.analysis_json_input_filepath))
        if len(analysis_data) > len(dataset_cases):
            raise colortext.Exception('ERROR: There seems to be an error - there are more predictions than cases in the dataset. Exiting.')
        elif len(analysis_data) < len(dataset_cases):
            self.log('\nWARNING: %d cases missing for analysis; there are %d predictions in the output directory but %d cases in the dataset. The analysis below does not cover the complete dataset.\n' % (len(dataset_cases) - len(analysis_data), len(analysis_data), len(dataset_cases)), colortext.error)

        #  ddg_analysis_type can be set to 'DDG', 'DDG_Top[x]' (e.g. 'DDG_Top3'), eyc.
        # 'DDG' uses the value reported by the application. For the Rosetta application ddg_monomer by Kellogg et al., this is the value output at the end of a run (which is not the recommended value - the publication uses take_lowest := 3).
        # 'DDG_Top3' (generated by default) uses the metric from Kellogg et al. based on the three lowest scoring mutant structures and the three lowest scoring wildtype structures
        if self.use_single_reported_value or self.ddg_analysis_type == 'DDG':
            assert( self.ddg_analysis_type == 'DDG' )
            self.ddg_analysis_type_description = '\nThe predicted DDG value per case is the single DDG value reported by the application.'
        elif self.ddg_analysis_type[4:].startswith('Top') and int(self.ddg_analysis_type[7:]) == 3:
            take_lowest = int(self.ddg_analysis_type[7:])
            self.ddg_analysis_type_description = '\nThe predicted DDG value per case is computed using the {0} lowest-scoring mutant structures and the {0} lowest-scoring wildtype structures as in the paper by Kellogg et al.'.format(take_lowest)
        elif self.ddg_analysis_type[4:].startswith('Top'):
            take_lowest = int(self.ddg_analysis_type[7:])
            self.ddg_analysis_type_description = '\nThe predicted DDG value per case is computed using the {0} lowest-scoring mutant structures and the {0} lowest-scoring wildtype structures.'.format(take_lowest)
        elif self.ddg_analysis_type[4:].startswith('Random'):
            ddg_analysis_type = self.ddg_analysis_type[4:]
            if len( ddg_analysis_type ) > len('Random'):
                self.ddg_analysis_type_description = '\nThe predicted DDG value per case is computed by pairing {0} random mutant structures with {0} random wildtype structures.'.format( int(ddg_analysis_type[len('Random'):]) )
            else:
                self.ddg_analysis_type_description = '\nThe predicted DDG value per case is computed by pairing random mutant structures with random wildtype structures.'
        elif self.ddg_analysis_type[4:] == 'AvgAllPairs':
            self.ddg_analysis_type_description = '\nThe predicted DDG value per case is computed by constructing all pairs of all mutant structures with all wildtype structures.'
        elif self.ddg_analysis_type[4:] == 'MatchPairs':
            self.ddg_analysis_type_description = '\nThe predicted DDG value per case is computed by matching each wildtype structure with its corresponding (round number) mutant structure.'
        elif self.ddg_analysis_type[4:].startswith( 'CplxBoltzWT' ):
            assert( len(self.ddg_analysis_type[4:]) > len( 'CplxBoltzWT' ) )
            self.ddg_analysis_type_description = '\nThe predicted DDG value per case is computed by boltzmann weighting matching DDG scores on the wildtype complex score (temperature %.2f).' % float(self.ddg_analysis_type[4+len('CplxBoltzWT'):])
        elif self.ddg_analysis_type[4:].startswith( 'CplxBoltzMut' ):
            assert( len(self.ddg_analysis_type[4:]) > len( 'CplxBoltzMut' ) )
            self.ddg_analysis_type_description = '\nThe predicted DDG value per case is computed by boltzmann weighting matching DDG scores on the mutant complex score (temperature %.2f).' % float(self.ddg_analysis_type[4+len('CplxBoltzMut'):])
        elif self.ddg_analysis_type[4:].startswith( 'CplxBoltzBoth' ):
            assert( len(self.ddg_analysis_type[4:]) > len( 'CplxBoltzBoth' ) )
            self.ddg_analysis_type_description = '\nThe predicted DDG value per case is computed by boltzmann weighting matching DDG scores on both the mutant complex score and the wildtype complex score, added together (temperature %.2f).' % float(self.ddg_analysis_type[4+len('CplxBoltzBoth'):])
        else:
            raise Exception("Couldn't parse ddg_analysis_type: " + str(ddg_analysis_type))
        self.log(self.ddg_analysis_type_description)

        # Initialize the data structures
        #csv_file = []

        # Set the PDB input path
        if not pdb_data:
            try:
                pdb_data_ = json.loads(read_file('../../input/json/pdbs.json'))
                for k, v in pdb_data_.iteritems():
                    pdb_data[k.upper()] = v
            except Exception, e:
                self.log('input/json/pdbs.json could not be found - PDB-specific analysis cannot be performed.', colortext.error)
        else:
            # Normalize to upper case to avoid matching problems later
            new_pdb_data = {}
            for k, v in pdb_data.iteritems():
                assert(k.upper() not in new_pdb_data)
                new_pdb_data[k.upper()] = v
            pdb_data = new_pdb_data

        # Determine columns specific to the prediction data to be added
        additional_prediction_data_columns = set()
        for adv in analysis_data.values():
            additional_prediction_data_columns = additional_prediction_data_columns.union(set(adv.keys()))
        assert(len(additional_prediction_data_columns.intersection(set(self.csv_headers))) == 0)
        assert(self.ddg_analysis_type in additional_prediction_data_columns)
        additional_prediction_data_columns.remove(self.ddg_analysis_type)
        additional_prediction_data_columns = sorted(additional_prediction_data_columns)

        # Initialize the dataframe
        self.reset_csv_headers() # this is necessary for the DBBenchmarkRun class which is missing the Experimental, AbsoluteError, and StabilityClassification columns since it adds new columns per analysis set.
        res = pandas.DataFrame(columns=(self.csv_headers + additional_prediction_data_columns))
        dataframe_columns = self.csv_headers + additional_prediction_data_columns
        additional_prediction_data_columns = tuple(additional_prediction_data_columns)

        # Create the dataframe
        dataframe_table = {}
        indices = []
        for record_id, predicted_data in sorted(analysis_data.iteritems()):
            dataframe_record = self.get_dataframe_row(dataset_cases, predicted_data, pdb_data, record_id, additional_prediction_data_columns)
            if dataframe_record:
                indices.append(dataframe_record['DatasetID'])
                for h in dataframe_columns:
                    dataframe_table[h] = dataframe_table.get(h, [])
                    dataframe_table[h].append(dataframe_record[h])
                assert(sorted(dataframe_columns) == sorted(dataframe_record.keys()))
        dataframe = pandas.DataFrame(dataframe_table, index = indices)
        return self.set_dataframe(dataframe, verbose = verbose)