class Analyzer:
    """Class to perform analysis on all metrics, predictions, etc."""

    def __init__(self, logger=None):
        self.logger = logger if logger else logging.getLogger(__name__)

    @staticmethod
    def check_frame_names(data_container, dataframe_names):
        """
        Check that all specified dataframes are available.

        This method checks to make sure all specified DataFrames
        are in the given data container object.

        Parameters
        ----------
        data_container : container.DataContainer
            A DataContainer object
        dataframe_names : list of str
            The names of the DataFrames expected in the
            DataContainer object.

        Raises
        ------
        KeyError
            If a given dataframe_name is not in the DataContainer object.
        """
        for dataframe_name in dataframe_names:
            if dataframe_name not in data_container:
                raise KeyError('The DataFrame `{}` does not exist in the '
                               'DataContainer object.'.format(dataframe_name))

    @staticmethod
    def check_param_names(configuration_obj, parameter_names):
        """
        Check that all specified parameters are available.

        This method checks to make sure all specified parameters
        are in the given configuration object.

        Parameters
        ----------
        configuration_obj : configuration_parser.Configuration
            A configuration object
        parameter_names : list of str
            The names of the parameters (keys) expected in the
            Configuration object.

        Raises
        ------
        KeyError
            If a given parameter_name is not in the Configuration object.
        """
        for parameter_name in parameter_names:
            if parameter_name not in configuration_obj:
                raise KeyError('The parameter `{}` does not exist in the '
                               'Configuration object.'.format(parameter_name))

    @staticmethod
    def analyze_excluded_responses(df,
                                   features,
                                   header,
                                   exclude_zero_scores=True,
                                   exclude_listwise=False):
        """
        Compute statistics for responses excluded from analyses.

        This method computes various statistics for the responses that
        were excluded from analyses, either in the training set or in
        the test set.

        Parameters
        ----------
        df : pandas DataFrame
            Data frame containing the excluded responses
        features : list of str
            List of column names containing the features
            to which we want to restrict the analyses.
        header : str
            String to be used as the table header for the
            output data frame.
        exclude_zero_scores : bool, optional
            Whether or not the zero-score responses
            should be counted in the exclusion statistics.
            Defaults to ``True``.
        exclude_listwise : bool, optional
            Whether or not the candidates were excluded
            based on minimal number of responses.
            Defaults to ``False``.

        Returns
        -------
        df_full_crosstab : pandas DataFrame
            Two-dimensional data frame containing the
            exclusion statistics.
        """
        # create an empty output data frame
        df_full_crosstab = pd.DataFrame({'all features numeric': [0, 0, 0],
                                         'non-numeric feature values': [0, 0, 0]},
                                        index=['numeric non-zero human score',
                                               'zero human score',
                                               'non-numeric human score'])

        if not df.empty:
            # re-code human scores into numeric, missing or zero
            df['score_category'] = 'numeric non-zero human score'
            df.loc[df['sc1'].isnull(), 'score_category'] = 'non-numeric human score'
            df.loc[df['sc1'].astype(float) == 0, 'score_category'] = 'zero human score'

            # recode feature values: a response with at least one
            # missing feature is assigned 'non-numeric feature values'
            df_features_only = df[features + ['spkitemid']]
            null_feature_rows = df_features_only.isnull().any(axis=1)
            df_null_features = df_features_only[null_feature_rows]
            df['feat_category'] = 'all features numeric'
            df.loc[df['spkitemid'].isin(df_null_features['spkitemid']),
                   'feat_category'] = 'non-numeric feature values'

            # crosstabulate
            df_crosstab = pd.crosstab(df['score_category'],
                                      df['feat_category'])
            df_full_crosstab.update(df_crosstab)
            # convert back to integers as these are all counts
            df_full_crosstab = df_full_crosstab.astype(int)
            df_full_crosstab.insert(0, header, df_full_crosstab.index)

        if not exclude_listwise:
            # if we are not excluding listwise, rename the first cell so
            # that it is not set to zero
            assert(df_full_crosstab.loc['numeric non-zero human score',
                                        'all features numeric'] == 0)
            df_full_crosstab.loc['numeric non-zero human score',
                                 'all features numeric'] = '-'

            # if we are not excluding the zeros, rename the corresponding cells
            # so that they are not set to zero. We do not do this for listwise exclusion
            if not exclude_zero_scores:
                assert(df_full_crosstab.loc['zero human score',
                                            'all features numeric'] == 0)
                df_full_crosstab.loc['zero human score',
                                     'all features numeric'] = '-'

        return df_full_crosstab

    @staticmethod
    def analyze_used_responses(df_train, df_test, subgroups, candidate_column):
        """
        Compute statistics for responses used in analyses.

        This method computes various statistics on the responses that
        were used in analyses, either in the training set or in the
        test set.

        Parameters
        ----------
        df_train : pandas DataFrame
            Data frame containing the response information
            for the training set.
        df_test : pandas DataFrame
            Data frame containing the response information
            for the test set.
        subgroups : list of str
            List of column names that contain grouping
            information.
        candidate_column : str
            Column name that contains candidate
            identification information.

        Returns
        -------
        df_analysis : pandas DataFrame
            Data frame containing information about the used
            responses.
        """
        # create a basic data frame for responses only
        train_responses = set(df_train['spkitemid'])
        test_responses = set(df_test['spkitemid'])

        rows = [{'partition': 'Training', 'responses': len(train_responses)},
                {'partition': 'Evaluation', 'responses': len(test_responses)},
                {'partition': 'Overlapping', 'responses': len(train_responses & test_responses)},
                {'partition': 'Total', 'responses': len(train_responses | test_responses)}]

        df_analysis = pd.DataFrame.from_dict(rows)

        columns = ['partition', 'responses'] + subgroups

        if candidate_column:
            train_candidates = set(df_train['candidate'])
            test_candidates = set(df_test['candidate'])
            df_analysis['candidates'] = [len(train_candidates), len(test_candidates),
                                         len(train_candidates & test_candidates),
                                         len(train_candidates | test_candidates)]

            columns = ['partition', 'responses', 'candidates'] + subgroups

        for group in subgroups:
            train_group = set(df_train[group])
            test_group = set(df_test[group])
            df_analysis[group] = [len(train_group), len(test_group),
                                  len(train_group & test_group),
                                  len(train_group | test_group)]

        df_analysis = df_analysis[columns]
        return df_analysis

    @staticmethod
    def analyze_used_predictions(df_test, subgroups, candidate_column):
        """
        Compute various statistics for predictions used in analyses.

        Parameters
        ----------
        df_test : pandas DataFrame
            Data frame containing the test set predictions.
        subgroups : list of str
            List of column names that contain grouping
            information.
        candidate_column : str
            Column name that contains candidate
            identification information.

        Returns
        -------
        df_analysis : pandas DataFrame
            Data frame containing information about the used
            predictions.
        """
        rows = [{'partition': 'Evaluation', 'responses': df_test['spkitemid'].size}]

        df_analysis = pd.DataFrame.from_dict(rows)
        df_columns = ['partition', 'responses'] + subgroups

        if candidate_column:
            df_analysis['candidates'] = [df_test['candidate'].unique().size]
            df_columns = ['partition', 'responses', 'candidates'] + subgroups

        for group in subgroups:
            df_analysis[group] = [df_test[group].unique().size]

        df_analysis = df_analysis[df_columns]
        return df_analysis

    @staticmethod
    def compute_basic_descriptives(df, selected_features):
        """
        Compute basic descriptive statistics for columns in the given data frame.

        Parameters
        ----------
        df : pandas DataFrame
            Input data frame containing the feature values.
        selected_features : list of str
            List of feature names for which to compute
            the descriptives.

        Returns
        -------
        df_desc : pandas DataFrame
            DataFrame containing the descriptives for
            each of the features.
        """
        # select only feature columns
        df_desc = df[selected_features]

        # get the H1 scores
        scores = df['sc1']

        # compute correlations and p-values separately for efficiency
        cor_series = df_desc.apply(lambda s: pearsonr(s, scores))
        cors = cor_series.apply(lambda t: t[0])
        pvalues = cor_series.apply(lambda t: t[1])

        # create a data frame with all the descriptives
        df_output = pd.DataFrame({'mean': df_desc.mean(),
                                  'min': df_desc.min(),
                                  'max': df_desc.max(),
                                  'std. dev.': df_desc.std(),
                                  'skewness': df_desc.skew(),
                                  'kurtosis': df_desc.apply(lambda s: kurtosis(s, fisher=False)),
                                  'Correlation': cors,
                                  'p': pvalues,
                                  'N': len(df_desc)})

        # reorder the columns to make it look better
        df_output = df_output[['mean', 'std. dev.', 'min', 'max', 'skewness',
                               'kurtosis', 'Correlation', 'p', 'N']]

        return df_output

    @staticmethod
    def compute_percentiles(df,
                            selected_features,
                            percentiles=None):
        """
        Compute percentiles and outliers for columns in the given data frame.

        Parameters
        ----------
        df : pandas DataFrame
            Input data frame containing the feature values.
        selected_features : list of str
            List of feature names for which to compute the
            percentile descriptives.
        percentiles : list of ints, optional
            The percentiles to calculate. If ``None``, use the percentiles
            {1, 5, 25, 50, 75, 95, 99}.
            Defaults to ``None``.

        Returns
        -------
        df_output : pandas DataFrame
            Data frame containing the percentile information
            for each of the features.
        """
        # select only feature columns
        df_desc = df[selected_features]

        # compute the various percentile levels
        if percentiles is None:
            percentiles = [1, 5, 25, 50, 75, 95, 99]

        df_output = df_desc.apply(lambda series: pd.Series(np.percentile(series,
                                                                         percentiles,
                                                                         method='lower')))
        df_output = df_output.transpose()

        # change the column names to be more readable
        df_output.columns = ['{}%'.format(p) for p in percentiles]

        # add the inter-quartile range column
        df_output['IQR'] = df_output['75%'] - df_output['25%']

        # compute the various outlier statistics
        mild_upper = df_output['75%'] + 1.5 * df_output['IQR']
        mild_bottom = df_output['25%'] - 1.5 * df_output['IQR']

        extreme_upper = df_output['75%'] + 3 * df_output['IQR']
        extreme_bottom = df_output['25%'] - 3 * df_output['IQR']

        # compute the mild and extreme outliers
        num_mild_outliers = {}
        num_extreme_outliers = {}
        for c in df_desc.columns:
            is_extreme = (df_desc[c] <= extreme_bottom[c]) | (df_desc[c] >= extreme_upper[c])

            is_mild = ((df_desc[c] > extreme_bottom[c]) & (df_desc[c] <= mild_bottom[c]))
            is_mild = is_mild | ((df_desc[c] >= mild_upper[c]) & (df_desc[c] < extreme_upper[c]))
            num_mild_outliers[c] = len(df_desc[is_mild])
            num_extreme_outliers[c] = len(df_desc[is_extreme])

        # add those to the output data frame
        df_output['Mild outliers'] = pd.Series(num_mild_outliers)
        df_output['Extreme outliers'] = pd.Series(num_extreme_outliers)

        return df_output

    @staticmethod
    def compute_outliers(df, selected_features):
        """
        Compute number and percentage of outliers for given columns.

        This method computes the number and percentage of outliers
        that lie outside the range mean +/- 4 SD for each of the
        given columns in the given data frame.

        Parameters
        ----------
        df : pandas DataFrame
            Input data frame containing the feature values.
        selected_features : list of str
            List of feature names for which to compute
            outlier information.

        Returns
        -------
        df_output : pandas DataFrame
            Data frame containing outlier information
            for each of the features.
        """
        # select only feature columns
        df_desc = df[selected_features]

        # compute the means and standard deviations
        means = df_desc.mean()
        stds = df_desc.std()

        # compute the number of upper and lower outliers
        lower_outliers = {}
        upper_outliers = {}
        for c in df_desc.columns:
            lower_outliers[c] = len(df_desc[df_desc[c] < means[c] - 4 * stds[c]])
            upper_outliers[c] = len(df_desc[df_desc[c] > means[c] + 4 * stds[c]])

        # generate the output data frame
        lower_s = pd.Series(lower_outliers)
        upper_s = pd.Series(upper_outliers)
        both_s = lower_s + upper_s
        df_output = pd.DataFrame({'lower': lower_s,
                                  'upper': upper_s,
                                  'both': both_s,
                                  'lowerperc': round(lower_s / len(df_desc) * 100, 2),
                                  'upperperc': round(upper_s / len(df_desc) * 100, 2),
                                  'bothperc': round(both_s / len(df_desc) * 100, 2)})

        return df_output

    @staticmethod
    def compute_pca(df, selected_features):
        """
        Compute PCA decomposition of the given features.

        This method computes the PCA decomposition of features in the
        data frame, restricted to the given columns. The number of components
        is set to be min(n_features, n_samples).

        Parameters
        ----------
        df : pandas DataFrame
            Input data frame containing feature values.
        selected_features : list of str
            List of feature names to be used in the
            PCA decomposition.

        Returns
        -------
        df_components : pandas DataFrame
            Data frame containing the PCA components.
        df_variance : pandas DataFrame
            Data frame containing the variance information.
        """
        # restrict to the given features
        df_pca = df[selected_features]

        # fit the PCA
        n_components = min(len(selected_features), len(df_pca))
        pca = PCA(n_components=n_components)
        pca.fit(df_pca)

        df_components = pd.DataFrame(pca.components_)
        n_components = len(df_components)
        df_components.columns = selected_features
        df_components.index = ['PC{}'.format(i) for i in range(1, n_components + 1)]
        df_components = df_components.transpose()

        # compute the variance data frame
        df_variance = {'Eigenvalues': pca.explained_variance_,
                       'Percentage of variance': pca.explained_variance_ratio_,
                       'Cumulative percentage of '
                       'variance': np.cumsum(pca.explained_variance_ratio_)
                       }

        df_variance = pd.DataFrame(df_variance)

        # reorder the columns
        df_variance = df_variance[['Eigenvalues', 'Percentage of variance',
                                   'Cumulative percentage of variance']]

        # set the row names and take the transpose
        df_variance.index = ['PC{}'.format(i) for i in range(1, n_components + 1)]
        df_variance = df_variance.transpose()

        return df_components, df_variance

    @staticmethod
    def correlation_helper(df,
                           target_variable,
                           grouping_variable,
                           include_length=False):
        """
        Compute marginal and partial correlations for all columns.

        This helper method computes marginal and partial correlations of
        all the columns in the given data frame against the target variable
        separately for each level in the the grouping variable.
        If ``include_length`` is ``True``, it additionally computes partial
        correlations of each column in the data frame against the target
        variable after controlling for the "length" column.

        Parameters
        ----------
        df : pandas DataFrame
            Input data frame containing numeric feature values, the numeric
            `target variable` and the `grouping variable`.
        target_variable: str
            The name of the column used as a reference for computing correlations.
        grouping_variable: str
            The name of the column defining groups in the data
        include_length: bool, optional
            If True compute additional partial correlations of each column
            in the data frame against `target variable` only partialling out
            "length" column.

        Returns
        -------
        df_target_cors : pandas DataFrame
            Data frame containing Pearson's correlation coefficients for
            marginal correlations between features and `target_variable`.
        df_target_partcors : pandas DataFrame
            Data frame containing Pearson's correlation coefficients for
            partial correlations between each feature and `target_variable`
            after controlling for all other features. If ``include_length`` is
            set to ``True``, the "length" column will not be included in the
            partial correlation computation.
        df_target_partcors_no_length: pandas DataFrame
            If ``include_length`` is set to ``True``: Data frame containing
            Pearson's correlation coefficients for partial correlations
            between each feature and ``target_variable`` after controlling
            for "length". Otherwise, it will be an empty data frame.
        """
        # group by the group columns
        grouped = df.groupby(grouping_variable)

        df_target_cors = pd.DataFrame()
        df_target_pcorr = pd.DataFrame()
        df_target_pcorr_no_length = pd.DataFrame()

        for group, df_group in grouped:
            df_group = df_group.drop(grouping_variable, axis=1)

            # first check if we have at least 2 cases and return np.nan otherwise
            if len(df_group) == 1:
                df_target_cors[group] = pd.Series(data=np.nan,
                                                  index=df_group.columns)
                df_target_pcorr[group] = pd.Series(data=np.nan,
                                                   index=df_group.columns)
                df_target_pcorr_no_length[group] = pd.Series(data=np.nan,
                                                             index=df_group.columns)
            else:
                # if we are asked to include length, that means 'length' is
                # in the data frame which means that we want to exclude that
                # before computing the regular marginal and partial correlations
                if not include_length:
                    df_target_cors[group] = df_group.apply(lambda s:
                                                           pearsonr(s,
                                                                    df_group[target_variable])[0])
                    df_target_pcorr[group] = partial_correlations(df_group)[target_variable]
                else:
                    df_group_no_length = df_group.drop('length', axis=1)

                    partial_pearsonr = partial(pearsonr, y=df_group_no_length[target_variable])
                    df_target_cors[group] = df_group_no_length.apply(lambda s:
                                                                     partial_pearsonr(s)[0])

                    df_target_pcorr[group] = partial_correlations(df_group_no_length)[target_variable]
                    pcor_dict = {}
                    columns = [c for c in df_group.columns if c not in ['sc1', 'length']]
                    for c in columns:
                        pcor_dict[c] = partial_correlations(df_group[[c,
                                                                      'sc1',
                                                                      'length']])['sc1'][c]
                    df_target_pcorr_no_length[group] = pd.Series(pcor_dict)

        # remove the row containing the correlation of the target variable
        # with itself and take the transpose
        df_target_cors = df_target_cors.drop(target_variable).transpose()
        df_target_pcorr = df_target_pcorr.drop(target_variable).transpose()
        df_target_pcorr_no_length = df_target_pcorr_no_length.transpose()

        return (df_target_cors,
                df_target_pcorr,
                df_target_pcorr_no_length)

    @staticmethod
    def metrics_helper(human_scores,
                       system_scores,
                       population_human_score_sd=None,
                       population_system_score_sd=None,
                       population_human_score_mn=None,
                       population_system_score_mn=None,
                       smd_method='unpooled',
                       use_diff_std_means=False):
        """
        Compute basic association metrics between system and human scores.

        Parameters
        ----------
        human_scores : pandas Series
            Series containing numeric human (reference) scores.
        system_scores: pandas Series
            Series containing numeric scores predicted by the model.
        population_human_score_sd : float, optional
            Reference standard deviation for human scores.
            This must be specified when the function is used to compute
            association metrics for a subset of responses, for example,
            responses from a particular demographic subgroup. If ``smd_method``
            is set to "williamson" or "johnson", this should be the standard
            deviation for the whole population (in most cases, the standard
            deviation for the whole test set). If ``use_diff_std_means`` is
            ``True``, this must be the standard deviation for the whole
            population and ``population_human_score_mn`` must also be specified.
            Otherwise, it is ignored.
            Defaults to ``None``.
        population_system_score_sd : float, optional
            Reference standard deviation for system scores.
            This must be specified when the function is used to compute
            association metrics for a subset of responses, for example,
            responses from a particular demographic subgroup. If ``smd_method``
            is set to "williamson", this should be the standard deviation for
            the whole population (in most cases, the standard deviation for the
            whole test set). If ``use_diff_std_means`` is ``True``, this must
            be the standard deviation for the whole population and
            ``population_system_score_mn`` must also be specified. Otherwise,
            it is ignored.
            Defaults to ``None``.
        population_human_score_mn : float, optional
            Reference mean for human scores. This must be specified when the
            function is used to compute association metrics for a subset of
            responses, for example, responses from a particular demographic
            subgroup. If ``use_diff_std_means`` is ``True``, this must be the
            mean for the whole population (in most cases, the full test set)
            and  ``population_human_score_sd`` must also be specified.
            Otherwise, it is ignored.
            Defaults to ``None``.
        population_system_score_mn : float, optional
            Reference mean for system scores. This must be specified when the
            function is used to compute association metrics for a subset of
            responses, for example, responses from a particular demographic
            subgroup. If ``use_diff_std_means`` is ``True``, this must be the
            mean for the whole population (in most cases, the full test set)
            and ``population_system_score_sd`` must also be specified. Otherwise,
            it is ignored.
            Defaults to ``None``.
        smd_method : {"williamson", "johnson", "pooled", "unpooled"}, optional
            The SMD method to use, only used if ``use_diff_std_means`` is
            ``False``. All methods have the same numerator
            mean(`y_pred`) - mean(`y_true_observed`) and the following
            denominators :

            - "williamson": pooled population standard deviation of
              `y_true_observed` and `y_pred` computed using
              ``population_human_score_sd`` and ``population_system_score_sd``.
            - "johnson": ``population_human_score_sd``.
            - "pooled": pooled standard deviation of `y_true_observed` and
              `y_pred` for this group.
            - "unpooled": standard deviation of `y_true_observed` for this
              group.

            Defaults to "unpooled".

        use_diff_std_means : bool, optional
            Whether to use the difference of standardized means, rather than
            the standardized mean difference. This is most useful with subgroup
            analysis.
            Defaults to ``False``.

        Returns
        -------
        metrics: pandas Series
            Series containing different evaluation metrics comparing human
            and system scores. The following metrics are included:

            - `kappa`:  unweighted Cohen's kappa
            - `wtkappa`:  quadratic weighted kappa
            - `exact_agr`: exact agreement
            - `adj_agr`: adjacent agreement with tolerance set to 1
            - One of the following :

              * `SMD`: standardized mean difference, if ``use_diff_std_means``
                is ``False``.
              * `DSM`: difference of standardized means, if ``use_diff_std_means``
                is ``True``.

            - `corr`: Pearson's r
            - `R2`: r squared
            - `RMSE`: root mean square error
            - `sys_min`: min system score
            - `sys_max`: max system score
            - `sys_mean`: mean system score (ddof=1)
            - `sys_sd`: standard deviation of system scores (ddof=1)
            - `h_min`: min human score
            - `h_max`: max human score
            - `h_mean`: mean human score (ddof=1)
            - `h_sd`: standard deviation of human scores (ddof=1)
            - `N`: total number of responses
        """
        # compute the kappas
        unweighted_kappa = kappa(human_scores, system_scores)
        weighted_kappa = quadratic_weighted_kappa(human_scores,
                                                  system_scores)

        # compute the agreement statistics
        human_system_agreement = agreement(human_scores, system_scores)
        human_system_adjacent_agreement = agreement(human_scores,
                                                    system_scores,
                                                    tolerance=1)

        # compute the Pearson correlation after removing
        # any cases where either of the scores are NaNs.
        df = pd.DataFrame({'human': human_scores,
                           'system': system_scores}).dropna(how='any')

        if (len(df) == 1 or
                len(df['human'].unique()) == 1 or
                len(df['system'].unique()) == 1):
            # set correlations to 1 if we have a single instance or zero variance
            correlations = np.nan
        else:
            correlations = pearsonr(df['human'], df['system'])[0]

        # compute the min/max/mean/std. dev. for the system and human scores
        min_system_score = np.min(system_scores)
        min_human_score = np.min(human_scores)

        max_system_score = np.max(system_scores)
        max_human_score = np.max(human_scores)

        mean_system_score = np.mean(system_scores)
        mean_human_score = np.mean(human_scores)

        system_score_sd = np.std(system_scores, ddof=1)
        human_score_sd = np.std(human_scores, ddof=1)

        if use_diff_std_means:

            # calculate the difference of standardized means
            smd_name = 'DSM'
            smd = difference_of_standardized_means(human_scores,
                                                   system_scores,
                                                   population_human_score_mn,
                                                   population_system_score_mn,
                                                   population_human_score_sd,
                                                   population_system_score_sd)

        else:

            # calculate the standardized mean difference
            smd_name = 'SMD'
            smd = standardized_mean_difference(human_scores,
                                               system_scores,
                                               population_human_score_sd,
                                               population_system_score_sd,
                                               method=smd_method)

        # compute r2
        if len(df) == 1:
            r2 = np.nan
        else:
            r2 = r2_score(human_scores, system_scores)

        # compute MSE
        mse = mean_squared_error(human_scores, system_scores)
        rmse = np.sqrt(mse)

        # return everything as a series
        metrics = pd.Series({'kappa': unweighted_kappa,
                             'wtkappa': weighted_kappa,
                             'exact_agr': human_system_agreement,
                             'adj_agr': human_system_adjacent_agreement,
                             smd_name: smd,
                             'corr': correlations,
                             'R2': r2,
                             'RMSE': rmse,
                             'sys_min': min_system_score,
                             'sys_max': max_system_score,
                             'sys_mean': mean_system_score,
                             'sys_sd': system_score_sd,
                             'h_min': min_human_score,
                             'h_max': max_human_score,
                             'h_mean': mean_human_score,
                             'h_sd': human_score_sd,
                             'N': len(system_scores)})

        return metrics

    @staticmethod
    def compute_disattenuated_correlations(human_system_corr,
                                           human_human_corr):
        """
        Compute disattenuated correlations between human and system scores.

        These are computed as the Pearson's correlation between the human score
        and the system score divided by the square root of correlation between
        two human raters.

        Parameters
        ----------
        human_system_corr : pandas Series
            Series containing of pearson's correlation coefficients human-system
            correlations.
        human_human_corr : pandas Series
            Series containing of pearson's correlation coefficients for human-human
            correlations. This can contain a single value or have the index
            matching that of human-system correlations.

        Returns
        -------
        df_correlations: pandas DataFrame
            Data frame containing the human-system correlations, human-human
            correlations, and disattenuated correlations.
        """
        # if we only have a single value for human correlation and the index
        # is not in human-system values, we use the same HH value in all cases
        if (len(human_human_corr) == 1 and
                not human_human_corr.index[0] in human_system_corr.index):
            human_human_corr = pd.Series(human_human_corr.values.repeat(len(human_system_corr)),
                                         index=human_system_corr.index)

        # we now concatenate the two series on index
        df_correlations = pd.concat([human_system_corr, human_human_corr],
                                    axis=1,
                                    sort=True,
                                    keys=['corr_HM', 'corr_HH'])

        # if any of the HH correlations are negative, we will ignore these
        # and treat them as Nones
        with np.errstate(invalid='ignore'):
            df_correlations['sqrt_HH'] = np.sqrt(df_correlations['corr_HH'])

        df_correlations['corr_disattenuated'] = (df_correlations['corr_HM'] /
                                                 df_correlations['sqrt_HH'])

        return df_correlations

    def compute_correlations_by_group(self,
                                      df,
                                      selected_features,
                                      target_variable,
                                      grouping_variable,
                                      include_length=False):
        """
        Compute marginal and partial correlations against target variable.

        This method computes various marginal and partial correlations of the
        given columns in the given data frame against the target variable for
        all data and for each level of the grouping variable.

        Parameters
        ----------
        df : pandas DataFrame
            Input data frame.
        selected_features : list of str
            List of feature names for which to compute
            the correlations.
        target_variable : str
            Feature name indicating the target variable i.e., the
            dependent variable
        grouping_variable : str
            Feature name that contain the grouping information
        include_length : bool, optional
            Whether or not to include the length when
            computing the partial correlations.
            Defaults to ``False``.

        Returns
        -------
        df_output : pandas DataFrame
            Data frame containing the correlations.
        """
        df_desc = df.copy()

        columns = selected_features + [target_variable, grouping_variable]
        if include_length:
            columns.append('length')
        df_desc = df_desc[columns]

        # create a duplicate data frame to compute correlations
        # over the whole data, i.e., across all grouping variables
        df_desc_all = df_desc.copy()
        df_desc_all[grouping_variable] = 'All data'

        # combine the two data frames
        df_desc_combined = pd.concat([df_desc, df_desc_all], sort=True)
        df_desc_combined.reset_index(drop=True, inplace=True)

        # compute the various (marginal and partial) correlations with score
        ret = self.correlation_helper(df_desc_combined,
                                      target_variable,
                                      grouping_variable,
                                      include_length=include_length)

        return ret

    def filter_metrics(self,
                       df_metrics,
                       use_scaled_predictions=False,
                       chosen_metric_dict=None):
        """
        Filter data frame to retain only the given metrics.

        This method filters the data frame ``df_metrics`` -- containing
        all of the metric values by all score types (raw, raw_trim etc.)
        -- to retain only the metrics as defined in the given dictionary
        ``chosen_metric_dict``. This dictionary maps score types ("raw",
        "scale", "raw_trim" etc.) to metric names. The available metric
        names are:

        - "corr"
        - "kappa"
        - "wtkappa"
        - "exact_agr"
        - "adj_agr"
        - "SMD" or "DSM", depending on what is in ``df_metrics``.
        - "RMSE"
        - "R2"
        - "sys_min"
        - "sys_max"
        - "sys_mean"
        - "sys_sd"
        - "h_min"
        - "h_max"
        - "h_mean"
        - "h_sd"
        - "N"

        Parameters
        ----------
        df_metrics : pd.DataFrame
            The DataFrame to filter.
        use_scaled_predictions : bool, optional
            Whether to use scaled predictions.
            Defaults to ``False``.
        chosen_metric_dict : dict, optional
            The dictionary to map score types to metrics that should be
            computer for them.
            Defaults to ``None``.

        Note
        ----
        The last five metrics will be the `same` for all score types.
        If ``chosen_metric_dict`` is not specified then, the following default
        dictionary, containing the recommended metrics, is used::

            {"X_trim": ["N", "h_mean", "h_sd", "sys_mean", "sys_sd", "wtkappa",
                          "corr", "RMSE", "R2", "SMD"],
             "X_trim_round": ["sys_mean", "sys_sd", "kappa",
                                "exact_agr", "adj_agr", "SMD"]}

        where X = "raw" or "scale" depending on whether
        ``use_scaled_predictions`` is ``False`` or ``True``, respectively.
        """
        # do we want the raw or the scaled metrics
        score_prefix = 'scale' if use_scaled_predictions else 'raw'

        # what metrics are we choosing to include?
        if chosen_metric_dict:
            chosen_metrics = chosen_metric_dict
        else:
            smd_name = 'DSM' if 'DSM' in df_metrics else 'SMD'
            chosen_metrics = {'{}_trim'.format(score_prefix): ['N',
                                                               'h_mean',
                                                               'h_sd',
                                                               'sys_mean',
                                                               'sys_sd',
                                                               'wtkappa',
                                                               'corr',
                                                               smd_name,
                                                               'RMSE',
                                                               'R2'],
                              '{}_trim_round'.format(score_prefix): ['sys_mean',
                                                                     'sys_sd',
                                                                     'kappa',
                                                                     'exact_agr',
                                                                     'adj_agr',
                                                                     smd_name]}

        # extract the metrics we need from the given metrics frame
        metricdict = {}
        for score_type in chosen_metrics:
            for metric in chosen_metrics[score_type]:
                colname = (metric if metric in ['h_mean', 'h_sd', 'N']
                           else '{}.{}'.format(metric, score_type))
                values = df_metrics[metric][score_type]
                metricdict[colname] = values

        df_filtered_metrics = pd.DataFrame([metricdict])
        return df_filtered_metrics

    def compute_metrics(self,
                        df,
                        compute_shortened=False,
                        use_scaled_predictions=False,
                        include_second_score=False,
                        population_sd_dict=None,
                        population_mn_dict=None,
                        smd_method='unpooled',
                        use_diff_std_means=False):
        """
        Compute association metrics for scores in the given data frame.

        This function compute association metrics for all score types.
        If ``include_second_score`` is ``True``, then it is assumed that
        a column called `sc2` containing a second human score is available
        and it should be used to compute the human-human evaluation stats
        and the performance degradation statistics.

        If ``compute_shortened`` is ``True``, then this function also
        computes a shortened version of the full human-system metrics data
        frame. See ``filter_metrics()`` for the description of the default
        columns included in the shortened data frame.

        Parameters
        ----------
        df : pandas DataFrame
            Input data frame
        compute_shortened : bool, optional
            Also compute a shortened version of the full
            metrics data frame.
            Defaults to ``False``.
        use_scaled_predictions : bool, optional
            Use evaluations based on scaled predictions in
            the shortened version of the metrics data frame.
            Defaults to ``False``.
        include_second_score : bool, optional
            Second human score available.
            Defaults to ``False``.
        population_sd_dict : dict, optional
            Dictionary containing population standard deviation for each column containing
            human or system scores. This is used to compute SMD for subgroups.
            Defaults to ``None``.
        population_mn_dict : dict, optional
            Dictionary containing population mean for each column containing
            human or system scores. This is used to compute SMD for subgroups.
            Defaults to ``None``.
        smd_method : {"williamson", "johnson", pooled", "unpooled"}, optional
            The SMD method to use, only used if ``use_diff_std_means`` is
            ``False``. All methods have the same numerator
            mean(`y_pred`) - mean(`y_true_observed`) and the following
            denominators:

            - "williamson": pooled population standard deviation of human and
              system scores computed based on values in ``population_sd_dict``.
            - "johnson": population standard deviation of human scores computed
              based on values in ``population_sd_dict``.
            - "pooled": pooled standard deviation of `y_true_observed` and
              `y_pred` for this group.
            - "unpooled": standard deviation of `y_true_observed` for this
              group.

            Defaults to "unpooled".

        use_diff_std_means : bool, optional
            Whether to use the difference of standardized means, rather than the standardized mean
            difference. This is most useful with subgroup analysis.
            Defaults to ``False``.

        Returns
        -------
        df_human_system_eval : pandas DataFrame
            Data frame containing the full set of evaluation
            metrics.
        df_human_system_eval_filtered : pandas DataFrame
            Data frame containing the human-human statistics
            but is empty if ``include_second_score`` is ``False``.
        df_human_human_eval : pandas DataFrame
            A shortened version of the first data frame but
            is empty if ``compute_shortened`` is ``False``.
        """
        # shorter variable name is easier to work with
        use_scaled = use_scaled_predictions

        # are we using DSM or SMD?
        smd_name = 'DSM' if use_diff_std_means else 'SMD'

        # get the population standard deviations for SMD if none were supplied
        if not population_sd_dict:
            population_sd_dict = {col: None for col in df.columns}

        # get the population standard deviations for SMD if none were supplied
        if not population_mn_dict:
            population_mn_dict = {col: None for col in df.columns}

        # if the second human score column is available, the values are
        # probably not available for all of the responses in the test
        # set and so we want to exclude 'sc2' from human-system metrics
        # computation. In addition, we also want to compute the human-human
        # metrics only on the data that is double scored.
        df_human_human = pd.DataFrame()
        if include_second_score:

            df_single = df.drop('sc2', axis=1)

            df_human_system = df_single.apply(lambda s:
                                              self.metrics_helper(df_single['sc1'],
                                                                  s,
                                                                  population_sd_dict['sc1'],
                                                                  population_sd_dict[s.name],
                                                                  population_mn_dict['sc1'],
                                                                  population_mn_dict[s.name],
                                                                  smd_method,
                                                                  use_diff_std_means))
            df_double = df[df['sc2'].notnull()][['sc1', 'sc2']]
            df_human_human = df_double.apply(lambda s:
                                             self.metrics_helper(df_double['sc1'],
                                                                 s,
                                                                 population_sd_dict['sc1'],
                                                                 population_sd_dict[s.name],
                                                                 population_mn_dict['sc1'],
                                                                 population_mn_dict[s.name],
                                                                 'pooled',
                                                                 use_diff_std_means))
            # drop the sc1 column from the human-human agreement frame
            df_human_human = df_human_human.drop('sc1', axis=1)

            # sort the rows in the correct order
            df_human_human = df_human_human.reindex(['N', 'h_mean', 'h_sd',
                                                          'h_min', 'h_max',
                                                          'sys_mean', 'sys_sd',
                                                          'sys_min', 'sys_max',
                                                          'corr', 'wtkappa', 'R2',
                                                          'kappa', 'exact_agr',
                                                          'adj_agr', smd_name, 'RMSE'])
            # rename `h_*` -> `h1_*` and `sys_*` -> `h2_*`
            df_human_human.rename(lambda c: c.replace('h_', 'h1_').replace('sys_', 'h2_'),
                                  inplace=True)
            # drop RMSE and R2 because they are not meaningful for human raters
            df_human_human.drop(['R2', 'RMSE'], inplace=True)
            df_human_human = df_human_human.transpose()
            # convert N to integer if it's not empty else set to 0
            try:
                df_human_human['N'] = df_human_human['N'].astype(int)
            except ValueError:
                df_human_human['N'] = 0
            df_human_human.index = ['']
        else:
            df_human_system = df.apply(lambda s: self.metrics_helper(df['sc1'],
                                                                     s,
                                                                     population_sd_dict['sc1'],
                                                                     population_sd_dict[s.name],
                                                                     population_mn_dict['sc1'],
                                                                     population_mn_dict[s.name],
                                                                     smd_method,
                                                                     use_diff_std_means))

        # drop 'sc1' column from the human-system frame and transpose
        df_human_system = df_human_system.drop('sc1', axis=1)
        df_human_system = df_human_system.transpose()

        # sort the columns and rows in the correct order
        df_human_system = df_human_system[['N',
                                           'h_mean', 'h_sd',
                                           'h_min', 'h_max',
                                           'sys_mean', 'sys_sd',
                                           'sys_min', 'sys_max',
                                           'corr',
                                           'wtkappa', 'R2', 'kappa',
                                           'exact_agr', 'adj_agr',
                                           smd_name, 'RMSE']]

        # make N column an integer if it's not NaN else set it to 0
        df_human_system['N'] = df_human_system['N'].astype(int)
        all_rows_order = ['raw', 'raw_trim', 'raw_trim_round',
                          'scale', 'scale_trim', 'scale_trim_round']
        existing_rows_index = [row for row in all_rows_order if row in df_human_system.index]
        df_human_system = df_human_system.reindex(existing_rows_index)

        # extract some default metrics for a shorter version of this data frame
        # if we were asked to do so
        if compute_shortened:
            df_human_system_filtered = self.filter_metrics(df_human_system,
                                                           use_scaled_predictions=use_scaled)
        else:
            df_human_system_filtered = pd.DataFrame()

        # return all data frames
        return (df_human_system,
                df_human_system_filtered,
                df_human_human)

    def compute_metrics_by_group(self,
                                 df_test,
                                 grouping_variable,
                                 use_scaled_predictions=False,
                                 include_second_score=False):
        """
        Compute a subset of evaluation metrics by subgroups.

        This method computes a subset of evalution metrics for the scores
        in the given data frame by group specified in ``grouping_variable``.
        See ``filter_metrics()`` above for a description of the subset
        that is selected.

        Parameters
        ----------
        df_test : pandas DataFrame
            Input data frame.
        grouping_variable : str
            Feature name indicating the column that
            contains grouping information.
        use_scaled_predictions : bool, optional
            Include scaled predictions when computing
            the evaluation metrics.
            Defaults to ``False``.
        include_second_score : bool, optional
            Include human-human association statistics.
            Defaults to ``False``.

        Returns
        -------
        df_human_system_by_group : pandas DataFrame
            Data frame containing the correlation
            human-system association statistics.
        df_human_human_by_group : pandas DataFrame
            Data frame that either contains the human-human
            statistics or is an empty data frame, depending
            on whether ``include_second_score`` is `True``.
        """
        # get the population standard deviation that we will need to compute SMD for all columns
        # other than id and subgroup
        population_sd_dict = {col: df_test[col].std(ddof=1)
                              for col in df_test.columns if col not in ['spkitemid',
                                                                        grouping_variable]}

        population_mn_dict = {col: df_test[col].mean()
                              for col in df_test.columns if col not in ['spkitemid',
                                                                        grouping_variable]}

        # check if any of the standard deviations is zero and
        # tell user to expect to see many warnings.
        zero_sd_scores = [score for (score, sd) in population_sd_dict.items() if
                          np.isclose(sd, 0, atol=1e-07)]
        if len(zero_sd_scores) > 0:
            warnings.warn("The standard deviation for {} scores "
                          "is zero (all values are the same). You "
                          "will see multiple warnings about DSM computation "
                          "since this metric is computed separately for "
                          "each subgroup.".format(', '.join(zero_sd_scores)))

        # create a duplicate data frame to compute evaluations
        # over the whole data, i.e., across groups
        df_preds_all = df_test.copy()
        df_preds_all[grouping_variable] = 'All data'

        # combine the two data frames
        df_preds_combined = pd.concat([df_test, df_preds_all], sort=True)
        df_preds_combined.reset_index(drop=True, inplace=True)

        # group by the grouping_variable columns
        grouped = df_preds_combined.groupby(grouping_variable)

        df_human_system_by_group = pd.DataFrame()
        df_human_human_by_group = pd.DataFrame()

        for group, df_group in grouped:
            df_group = df_group.drop(grouping_variable, axis=1)

            (df_human_system_metrics,
             df_human_system_metrics_short,
             df_human_human_metrics
             ) = self.compute_metrics(df_group,
                                      compute_shortened=True,
                                      use_scaled_predictions=use_scaled_predictions,
                                      include_second_score=include_second_score,
                                      population_sd_dict=population_sd_dict,
                                      population_mn_dict=population_mn_dict,
                                      use_diff_std_means=True)

            # we need to convert the shortened data frame to a series here
            df_human_system_by_group[group] = df_human_system_metrics_short.iloc[0]

            # update the by group human-human metrics frame if
            # we have the second score column available
            if include_second_score:
                df_human_human_metrics.index = [group]
                df_human_human_by_group = pd.concat([df_human_human_by_group, df_human_human_metrics])

        # transpose the by group human-system metrics frame
        df_human_system_by_group = df_human_system_by_group.transpose()

        return (df_human_system_by_group, df_human_human_by_group)

    def compute_degradation_and_disattenuated_correlations(self,
                                                           df,
                                                           use_all_responses=True):
        """
        Compute the degradation in performance when using system score.

        This method computes the degradation in performance when using the
        system to predict the score instead of a second human and also the
        disattenuated correlations between human and system scores.
        These are computed as the Pearson's correlation between the human score
        and the system score divided by the square root of correlation between
        two human raters.

        For this, we can compute the system performance either only on the
        double scored data or on the full dataset. Both options have their
        pros and cons. The default is to use the full dataset. This function
        also assumes that the `sc2` column exists in the given data frame,
        in addition to `sc1` and the various types of predictions.

        Parameters
        ----------
        df : pandas DataFrame
            Input data frame.
        use_all_responses : bool, optional
            Use the full data set instead of only using the double-scored subset.
            Defaults to ``True``.

        Returns
        -------
        df_degradation : pandas DataFrame
            Data frame containing the degradation statistics.
        df_correlations : pandas DataFrame
            Data frame containing the human-system correlations, human-human
            correlations and disattenuated correlation.
        """
        if use_all_responses:
            df_responses = df
        else:
            # use only double scored data
            df_responses = df[df['sc2'].notnull()]

        # compute the human-system and human-human metrics
        (df_human_system_eval,
         _,
         df_human_human_eval) = self.compute_metrics(df_responses,
                                                     include_second_score=True)

        # compute disattenuated correlations
        df_correlations = self.compute_disattenuated_correlations(df_human_system_eval['corr'],
                                                                  df_human_human_eval['corr'])

        # Compute degradation. we only care about the degradation in these metrics
        degradation_metrics = ['corr', 'kappa', 'wtkappa',
                               'exact_agr', 'adj_agr', 'SMD']
        df_human_system_eval = df_human_system_eval[degradation_metrics]
        df_human_human_eval = df_human_human_eval[degradation_metrics]
        df_degradation = df_human_system_eval.apply(lambda row:
                                                    row - df_human_human_eval.loc[''], axis=1)

        return (df_degradation, df_correlations)

    def run_training_analyses(self,
                              data_container,
                              configuration):
        """
        Run all analyses on the training data.

        Parameters
        ----------
        data_container : container.DataContainer
            The DataContainer object. This container must include the following
            DataFrames: {"train_features", "train_metadata",
            "train_preprocessed_features", "train_length", "train_features"}.

        configuration : configuration_parser.Configuration
            The Configuration object. This configuration object must include the
            following parameters (keys): {"length_column", "subgroups",
            "selected_features"}.

        Returns
        -------
        data_container : container.DataContainer
            A new DataContainer object with the following DataFrames:

            - feature_descriptives
            - feature_descriptivesExtra
            - feature_outliers
            - cors_orig
            - cors_processed
            - margcor_score_all_data
            - pcor_score_all_data
            - pcor_score_no_length_all_data
            - margcor_length_all_data
            - pcor_length_all_data
            - pca
            - pcavar
            - margcor_length_by_*
            - pcor_length_by_*
            - margcor_score_by_*
            - pcor_score_by_*
            - pcor_score_no_length_by_*

        configuration : configuration_parser.Configuration
            A new Configuration object.
        """
        frame_names = ['train_features', 'train_metadata',
                       'train_preprocessed_features', 'train_length',
                       'train_features']

        param_names = ['length_column', 'subgroups', 'selected_features']

        self.check_frame_names(data_container, frame_names)

        self.check_param_names(configuration, param_names)

        # only use the features selected by the model but keep their order the same
        # as in the original file as ordering may affect the sign in pca
        df_train = data_container.train_features.copy()
        df_train_length = data_container.train_length.copy()
        df_train_metadata = data_container.train_metadata.copy()
        df_train_preprocessed_features = data_container.train_preprocessed_features.copy()

        subgroups = configuration['subgroups']
        selected_features = configuration['selected_features']

        df_train_preprocessed = pd.merge(df_train_preprocessed_features,
                                         df_train_metadata, on='spkitemid')

        assert (len(df_train_preprocessed.index) ==
                len(df_train_preprocessed_features.index) ==
                len(df_train_metadata.index))

        # get descriptives, percentiles and outliers for the original feature values
        df_descriptives = self.compute_basic_descriptives(df_train, selected_features)
        df_percentiles = self.compute_percentiles(df_train, selected_features)
        df_outliers = self.compute_outliers(df_train, selected_features)

        # set a general boolean flag indicating if we should include length
        include_length = not df_train_length.empty

        # include length if available
        if include_length:
            columns = selected_features + ['sc1', 'length']
            df_train_with_length = df_train.merge(df_train_length, on='spkitemid')
            df_train_preprocess_length = df_train_preprocessed.merge(df_train_length,
                                                                     on='spkitemid')
        else:
            columns = selected_features + ['sc1']
            df_train_with_length = df_train
            df_train_preprocess_length = df_train_preprocessed

        # get pairwise correlations against the original training features
        # as well as the pre-processed training features
        df_pairwise_cors_orig = df_train_with_length[columns].corr(method='pearson')
        df_pairwise_cors_preprocess = df_train_preprocess_length[columns].corr(method='pearson')

        # get marginal and partial correlations against sc1 for all data
        # for partial correlations, we partial out all other features
        df_train_with_group_for_all = df_train_preprocess_length.copy()
        df_train_with_group_for_all = df_train_with_group_for_all[columns]
        df_train_with_group_for_all['all_data'] = 'All data'

        (df_margcor_sc1,
         df_pcor_sc1,
         df_pcor_sc1_no_length) = self.correlation_helper(df_train_with_group_for_all,
                                                          'sc1',
                                                          'all_data',
                                                          include_length=include_length)

        # get marginal and partial correlations against length for all data
        # if the length column is available
        df_margcor_length = pd.DataFrame()
        df_pcor_length = pd.DataFrame()

        if include_length:

            df_train_with_group_for_all = df_train_preprocess_length.copy()
            columns = selected_features + ['length']

            df_train_with_group_for_all = df_train_with_group_for_all[columns]
            df_train_with_group_for_all['all_data'] = 'All data'

            (df_margcor_length,
             df_pcor_length,
             _) = self.correlation_helper(df_train_with_group_for_all,
                                          'length',
                                          'all_data')

        # get marginal and partial correlations against sc1 by group (preprocessed features)
        # also include partial correlations with length if length is available
        score_corr_by_group_dict = {}
        include_length = 'length' in df_train_preprocess_length
        for grouping_variable in subgroups:

            corr_by_group = self.compute_correlations_by_group(df_train_preprocess_length,
                                                               selected_features,
                                                               'sc1',
                                                               grouping_variable,
                                                               include_length=include_length)

            score_corr_by_group_dict[grouping_variable] = corr_by_group

        # get marginal and partial correlations against sc1 by group (preprocessed features)
        length_corr_by_group_dict = {}
        if include_length:
            for grouping_variable in subgroups:

                corr_by_group = self.compute_correlations_by_group(df_train_preprocess_length,
                                                                   selected_features,
                                                                   'length',
                                                                   grouping_variable)

                length_corr_by_group_dict[grouping_variable] = corr_by_group

        # get PCA information
        df_pca_components, df_pca_variance = self.compute_pca(df_train_preprocessed,
                                                              selected_features)

        # Datasets to add
        datasets = [{'name': 'feature_descriptives', 'frame': df_descriptives},
                    {'name': 'feature_descriptivesExtra', 'frame': df_percentiles},
                    {'name': 'feature_outliers', 'frame': df_outliers},
                    {'name': 'cors_orig', 'frame': df_pairwise_cors_orig},
                    {'name': 'cors_processed', 'frame': df_pairwise_cors_preprocess},
                    {'name': 'margcor_score_all_data', 'frame': df_margcor_sc1},
                    {'name': 'pcor_score_all_data', 'frame': df_pcor_sc1},
                    {'name': 'pcor_score_no_length_all_data', 'frame': df_pcor_sc1_no_length},
                    {'name': 'margcor_length_all_data', 'frame': df_margcor_length},
                    {'name': 'pcor_length_all_data', 'frame': df_pcor_length},
                    {'name': 'pca', 'frame': df_pca_components},
                    {'name': 'pcavar', 'frame': df_pca_variance}]

        # Add length correlation by group datasets
        for group in length_corr_by_group_dict:

            (length_marg_cors,
             length_part_cors,
             _) = length_corr_by_group_dict.get(group,
                                                (pd.DataFrame(),
                                                 pd.DataFrame(),
                                                 pd.DataFrame()))

            datasets.extend([{'name': 'margcor_length_by_{}'.format(group),
                              'frame': length_marg_cors},
                             {'name': 'pcor_length_by_{}'.format(group),
                              'frame': length_part_cors}])

        # Add score correlations by group datasets
        for group in score_corr_by_group_dict:

            (sc1_marg_cors,
             sc1_part_cors,
             sc1_part_cors_no_length) = score_corr_by_group_dict[group]

            datasets.extend([{'name': 'margcor_score_by_{}'.format(group),
                              'frame': sc1_marg_cors},
                             {'name': 'pcor_score_by_{}'.format(group),
                              'frame': sc1_part_cors},
                             {'name': 'pcor_score_no_length_by_{}'.format(group),
                              'frame': sc1_part_cors_no_length}])

        return configuration, DataContainer(datasets=datasets)

    def run_prediction_analyses(self,
                                data_container,
                                configuration):
        """
        Run all analyses on the system scores (predictions).

        Parameters
        ----------
        data_container : container.DataContainer
            The DataContainer object. This container must include the following
            DataFrames: {"train_features", "train_metadata",
            "train_preprocessed_features", "train_length",
            "train_features"}.

        configuration : configuration_parser.Configuration
            The Configuration object.  This configuration object must include the
            following parameters (keys):  {"subgroups", "second_human_score_column",
            "use_scaled_predictions"}.

        Returns
        -------
        data_container : container.DataContainer
            A new DataContainer object with the following DataFrames:

            - eval
            - eval_short
            - consistency
            - degradation
            - disattenudated_correlations
            - confMatrix
            - score_dist
            - eval_by_*
            - consistency_by_*
            - disattenduated_correlations_by_*
            - true_score_eval

        configuration : configuration_parser.Configuration
            A new Configuration object.
        """
        frame_names = ['pred_test', 'test_metadata', 'test_human_scores']

        param_names = ['subgroups',
                       'second_human_score_column',
                       'use_scaled_predictions']

        self.check_frame_names(data_container, frame_names)

        self.check_param_names(configuration, param_names)

        df_test = data_container.pred_test.copy()
        df_test_metadata = data_container.test_metadata.copy()
        df_test_human_scores = data_container.test_human_scores.copy()

        subgroups = configuration['subgroups']
        use_scaled_predictions = configuration['use_scaled_predictions']

        df_preds = pd.merge(df_test, df_test_metadata, on='spkitemid')

        assert len(df_preds.index) == len(df_test.index) == len(df_test_metadata.index)

        # set a general boolean flag indicating if
        # we should include the second human score
        include_second_score = not df_test_human_scores.empty

        # extract the columns that contain predictions
        prediction_columns = [column for column in df_test if column != 'spkitemid']

        # if a second score is available, use it
        if include_second_score:
            prediction_columns.append('sc2')
            df_preds_second_score = df_preds.merge(df_test_human_scores[['spkitemid', 'sc2']],
                                                   on='spkitemid')
        else:
            df_preds_second_score = df_preds

        # compute the evaluation metrics over the whole data set
        (df_human_system,
         df_human_system_short,
         df_human_human) = self.compute_metrics(df_preds_second_score[prediction_columns],
                                                compute_shortened=True,
                                                use_scaled_predictions=use_scaled_predictions,
                                                include_second_score=include_second_score)

        # compute the evaluation metrics by group
        eval_by_group_dict = {}
        for group in subgroups:
            group_columns = prediction_columns + [group]
            metrics = self.compute_metrics_by_group(df_preds_second_score[group_columns],
                                                    group,
                                                    use_scaled_predictions=use_scaled_predictions,
                                                    include_second_score=include_second_score)
            eval_by_group_dict[group] = metrics

        # compute the degradation statistics and disattenuated correlations
        # if we have the second human score available
        df_degradation = pd.DataFrame()
        df_correlations = pd.DataFrame()
        if include_second_score:
            (df_degradation,
             df_correlations) = self.compute_degradation_and_disattenuated_correlations(df_preds_second_score[prediction_columns])

        # the following two evaluations require rounded human scores
        # we create a column for this
        df_preds['sc1_round'] = np.round(df_preds['sc1'])

        # compute the confusion matrix as a data frame
        score_type = 'scale' if use_scaled_predictions else 'raw'
        human_scores = df_preds['sc1_round'].astype('int64')
        system_scores = df_preds['{}_trim_round'.format(score_type)].astype('int64')
        conf_matrix = confusion_matrix(human_scores, system_scores)
        labels = sorted(pd.concat([human_scores, system_scores]).unique())
        df_confmatrix = pd.DataFrame(conf_matrix, index=labels, columns=labels).transpose()

        # compute the score distributions of the rounded human and system scores
        df_score_dist = df_preds[['sc1_round',
                                  '{}_trim_round'
                                  ''.format(score_type)]].apply(lambda s:
                                                                (s.value_counts() /
                                                                 len(df_test) * 100))

        # Replace any NaNs, which we might get because our model never
        # predicts a particular score label, with zeros.
        df_score_dist.fillna(0, inplace=True)

        df_score_dist.columns = ['human', 'sys_{}'.format(score_type)]
        df_score_dist['difference'] = (df_score_dist['sys_{}'.format(score_type)] -
                                       df_score_dist['human'])
        df_score_dist['score'] = df_score_dist.index

        df_score_dist = df_score_dist[['score', 'human',
                                       'sys_{}'.format(score_type),
                                       'difference']]
        df_score_dist.sort_values(by='score', inplace=True)

        datasets = [{'name': 'eval', 'frame': df_human_system},
                    {'name': 'eval_short', 'frame': df_human_system_short},
                    {'name': 'consistency', 'frame': df_human_human},
                    {'name': 'degradation', 'frame': df_degradation},
                    {'name': 'disattenuated_correlations', 'frame': df_correlations},
                    {'name': 'confMatrix', 'frame': df_confmatrix},
                    {'name': 'score_dist', 'frame': df_score_dist}]

        # compute true-score analyses if we have second score
        # or have been given rater error variance
        rater_error_variance = configuration.get_rater_error_variance()

        if include_second_score or rater_error_variance is not None:
            system_score_columns = [col for col in prediction_columns
                                    if col not in ['sc1', 'sc2']]

            human_score_columns = [col for col in prediction_columns
                                   if col in ['sc1', 'sc2']]

            df_prmse = get_true_score_evaluations(df_preds_second_score,
                                                  system_score_columns,
                                                  human_score_columns,
                                                  rater_error_variance)

            datasets.extend([{'name': 'true_score_eval', 'frame': df_prmse}])

        for group in eval_by_group_dict:
            eval_by_group, consistency_by_group = eval_by_group_dict[group]

            # compute disattenuated correlations if we have the second human score
            if include_second_score:
                dis_corr_by_group = self.compute_disattenuated_correlations(eval_by_group['corr.{}_trim'.format(score_type)],
                                                                            consistency_by_group['corr'])
            else:
                dis_corr_by_group = pd.DataFrame()

            datasets.extend([{'name': 'eval_by_{}'.format(group),
                              'frame': eval_by_group},
                             {'name': 'consistency_by_{}'.format(group),
                              'frame': consistency_by_group},
                             {'name': 'disattenuated_correlations_by_{}'.format(group),
                              'frame': dis_corr_by_group}])

        return configuration, DataContainer(datasets=datasets)

    def run_data_composition_analyses_for_rsmtool(self,
                                                  data_container,
                                                  configuration):
        """
        Run all data composition analyses for RSMTool.

        Parameters
        ----------
        data_container : container.DataContainer
            The DataContainer object. This container must include the following
            DataFrames: {"test_metadata", "train_metadata","train_excluded",
            "test_excluded", "train_features"}.
        configuration : configuration_parser.Configuration
            The Configuration object.  This configuration object must include the
            following parameters (keys): {"subgroups", "candidate_column",
            "exclude_zero_scores", "exclude_listwise"}.

        Returns
        -------
        data_container : container.DataContainer
            A new DataContainer object with the following DataFrames:

            - test_excluded_composition
            - train_excluded_composition
            - data_composition
            - data_composition_by_*

        configuration : configuration_parser.Configuration
            A new Configuration object.
        """
        frame_names = ['train_metadata', 'test_metadata',
                       'train_excluded', 'test_excluded',
                       'train_features']

        param_names = ['candidate_column', 'subgroups',
                       'exclude_zero_scores', 'exclude_listwise']

        self.check_frame_names(data_container, frame_names)

        self.check_param_names(configuration, param_names)

        features = [column for column in data_container.train_features.columns
                    if column not in ['spkitemid', 'sc1']]

        exclude_scores = configuration['exclude_zero_scores']
        exclude_listwise = configuration['exclude_listwise']

        subgroups = configuration['subgroups']
        candidate_column = configuration['candidate_column']

        df_train_excluded = self.analyze_excluded_responses(data_container['train_excluded'],
                                                            features,
                                                            'Score/Features',
                                                            exclude_zero_scores=exclude_scores,
                                                            exclude_listwise=exclude_listwise)

        df_test_excluded = self.analyze_excluded_responses(data_container['test_excluded'],
                                                           features,
                                                           'Score/Features',
                                                           exclude_zero_scores=exclude_scores,
                                                           exclude_listwise=exclude_listwise)

        df_data_composition = self.analyze_used_responses(data_container['train_metadata'],
                                                          data_container['test_metadata'],
                                                          subgroups, candidate_column)

        # do the analysis by subgroups
        # first create a joint data frame with both sets
        df_train_metadata_with_set = data_container['train_metadata'].copy()
        df_test_metadata_with_set = data_container['test_metadata'].copy()

        df_train_metadata_with_set['set'] = 'Training set'
        df_test_metadata_with_set['set'] = 'Evaluation set'

        df_both_metadata = pd.merge(df_train_metadata_with_set,
                                    df_test_metadata_with_set,
                                    how='outer')

        # create contingency table for each subgroup
        data_composition_by_group_dict = {}
        for grouping_variable in subgroups:
            df_crosstab_group = pd.crosstab(df_both_metadata[grouping_variable],
                                            df_both_metadata['set'])
            df_crosstab_group = df_crosstab_group[['Training set',
                                                   'Evaluation set']]
            df_crosstab_group.insert(0, grouping_variable, df_crosstab_group.index)
            data_composition_by_group_dict[grouping_variable] = df_crosstab_group

        datasets = [{'name': 'test_excluded_composition', 'frame': df_test_excluded},
                    {'name': 'train_excluded_composition', 'frame': df_train_excluded},
                    {'name': 'data_composition', 'frame': df_data_composition}]

        for group in data_composition_by_group_dict:

            datasets.append({'name': 'data_composition_by_{}'.format(group),
                             'frame': data_composition_by_group_dict[group]})

        return configuration, DataContainer(datasets=datasets)

    def run_data_composition_analyses_for_rsmeval(self,
                                                  data_container,
                                                  configuration):
        """
        Run all data composition analyses for RSMEval.

        Parameters
        ----------
        data_container : container.DataContainer
            The DataContainer object. This container must include the following
            DataFrames: {"test_metadata", "test_excluded"}.

        configuration : configuration_parser.Configuration
            The Configuration object.  This configuration object must include the
            following parameters (keys): {"subgroups", "candidate_column",
            "exclude_zero_scores", "exclude_listwise"}.

        Returns
        -------
        data_container : container.DataContainer
            A new DataContainer object with the following DataFrames:

            - test_excluded_composition
            - data_composition
            - data_composition_by_*

        configuration : configuration_parser.Configuration
            A new Configuration object.
        """
        frame_names = ['test_metadata', 'test_excluded']

        param_names = ['candidate_column', 'subgroups',
                       'exclude_zero_scores', 'exclude_listwise']

        self.check_frame_names(data_container, frame_names)

        self.check_param_names(configuration, param_names)

        exclude_scores = configuration['exclude_zero_scores']
        exclude_listwise = configuration['exclude_listwise']

        subgroups = configuration['subgroups']
        candidate_column = configuration['candidate_column']

        # analyze excluded responses
        df_test_excluded = self.analyze_excluded_responses(data_container['test_excluded'],
                                                           ['raw'],
                                                           'Human/System',
                                                           exclude_zero_scores=exclude_scores,
                                                           exclude_listwise=exclude_listwise)

        # rename the columns and index in the analysis data frame
        df_test_excluded.rename(columns={'all features numeric':
                                         'numeric system score',
                                         'non-numeric feature values':
                                         'non-numeric system score'},

                                inplace=True)
        df_data_composition = self.analyze_used_predictions(data_container['test_metadata'],
                                                            subgroups,
                                                            candidate_column)

        # create contingency table for each group
        data_composition_by_group_dict = {}
        for grouping_variable in subgroups:
            series_crosstab_group = pd.pivot_table(data_container['test_metadata'],
                                                   values='spkitemid',
                                                   index=[grouping_variable],
                                                   aggfunc=len)

            df_crosstab_group = pd.DataFrame(series_crosstab_group)
            df_crosstab_group.insert(0, grouping_variable, df_crosstab_group.index)
            df_crosstab_group.rename(columns={'spkitemid': 'N responses'},
                                     inplace=True)
            data_composition_by_group_dict[grouping_variable] = df_crosstab_group

        datasets = [{'name': 'test_excluded_composition', 'frame': df_test_excluded},
                    {'name': 'data_composition', 'frame': df_data_composition}]

        for group in data_composition_by_group_dict:

            datasets.append({'name': 'data_composition_by_{}'.format(group),
                             'frame': data_composition_by_group_dict[group]})

        return configuration, DataContainer(datasets=datasets) def configure(context, config_file_or_obj_or_dict):
    """
    Create a Configuration object.

    Get the configuration for ``context`` from the input
    ``config_file_or_obj_or_dict``.

    Parameters
    ----------
    context : str
        The context that is being configured. Must be one of
        "rsmtool", "rsmeval", "rsmcompare", "rsmsummarize",
        "rsmpredict", or "rsmxval".
    config_file_or_obj_or_dict : str or pathlib.Path or dict or Configuration
        Path to the experiment configuration file either a a string
        or as a ``pathlib.Path`` object. Users can also pass a
        ``Configuration`` object that is in memory or a Python dictionary
        with keys corresponding to fields in the configuration file. Given a
        configuration file, any relative paths in the configuration file
        will be interpreted relative to the location of the file. Given a
        ``Configuration`` object, relative paths will be interpreted
        relative to the ``configdir`` attribute, that _must_ be set. Given
        a dictionary, the reference path is set to the current directory.

    Returns
    -------
    configuration : Configuration
        The Configuration object for the tool.

    Raises
    ------
    AttributeError
        If the ``configdir`` attribute for the Configuration input is not set.
    ValueError
        If ``config_file_or_obj_or_dict`` contains anything except a string,
        a path, a dictionary, or a ``Configuration`` object.
    """
    # check what sort of input we got
    # if we got a string we consider this to be path to config file
    if isinstance(config_file_or_obj_or_dict, (str, Path)):

        # Instantiate configuration parser object
        parser = ConfigurationParser(config_file_or_obj_or_dict)
        configuration = parser.parse(context=context)

    elif isinstance(config_file_or_obj_or_dict, dict):

        # directly instantiate the Configuration from the dictionary
        configuration = Configuration(config_file_or_obj_or_dict,
                                      context=context)

    elif isinstance(config_file_or_obj_or_dict, Configuration):

        # raise an error if we are passed a Configuration object
        # without a configdir attribute. This can only
        # happen if the object was constructed using an earlier version
        # of RSMTool and stored
        try:
            assert config_file_or_obj_or_dict.configdir is not None
        except AssertionError:
            raise AttributeError("Configuration object must have configdir attribute.")
        else:
            configuration = config_file_or_obj_or_dict

    else:
        raise ValueError("The input to run_experiment must be "
                         "a path to the file (str), a dictionary, "
                         "or a configuration object. You passed "
                         "{}.".format(type(config_file_or_obj_or_dict)))

    return configuration class Modeler:
    """
    Class to train model and generate predictions with built-in or SKLL models.
    """

    def __init__(self, logger=None):
        """Instantiate empty instance with no learner and given logger, if any."""
        self.learner = None
        self.logger = logger if logger else logging.getLogger(__name__)

    @classmethod
    def load_from_file(cls, model_path):
        """
        Load a modeler object from a file on disk.

        Parameters
        ----------
        model_path : str
            The path to the modeler file.

        Returns
        -------
        model : Modeler
            A Modeler instance.

        Raises
        ------
        ValueError
            If ``model_path`` does not end with ".model".
        """
        if not model_path.lower().endswith('.model'):
            raise ValueError('The file `{}` does not end with the '
                             'proper extension. Please make sure that '
                             'it is a `.model` file.'.format(model_path))

        # Create SKLL learner from file
        learner = Learner.from_file(model_path)
        return cls.load_from_learner(learner)

    @classmethod
    def load_from_learner(cls, learner):
        """
        Create a new Modeler instance with a pre-populated learner.

        Parameters
        ----------
        learner : skll.learner.Learner
            A SKLL Learner object.

        Returns
        -------
        modeler : Modeler
            A Modeler object.

        Raises
        ------
        TypeError
            If ``learner`` is not a SKLL Learner instance.
        """
        if not isinstance(learner, Learner):
            raise TypeError('The `learner` argument must be a '
                            '` SKLL Learner` instance, not `{}`.'
                            ''.format(type(learner)))

        # Create Modeler instance
        modeler = Modeler()
        modeler.learner = learner
        return modeler

    @staticmethod
    def model_fit_to_dataframe(fit):
        """
        Extract fit metrics from a ``statsmodels`` fit object into a data frame.

        Parameters
        ----------
        fit : statsmodels.RegressionResults
            Model fit object obtained from a linear model trained using
            ``statsmodels.OLS``.

        Returns
        -------
        df_fit : pandas DataFrame
            The output data frame with the main model fit metrics.
        """
        df_fit = pd.DataFrame({"N responses": [int(fit.nobs)]})
        df_fit['N features'] = int(fit.df_model)
        df_fit['R2'] = fit.rsquared
        df_fit['R2_adjusted'] = fit.rsquared_adj
        return df_fit

    @staticmethod
    def ols_coefficients_to_dataframe(coefs):
        """
        Convert series containing OLS coefficients to a data frame.

        Parameters
        ----------
        coefs : pandas Series
            Series with feature names in the index and the coefficient
            values as the data, obtained from a linear model trained
            using ``statsmodels.OLS``.

        Returns
        -------
        df_coef : pandas DataFrame
            Data frame with two columns: the feature name and
            the coefficient value.

        Note
        ----
        The first row in the output data frame is always for the intercept
        and the rest are sorted by feature name.
        """
        # first create a sorted data frame for all the non-intercept features
        non_intercept_columns = [c for c in coefs.index if c != 'const']
        df_non_intercept = pd.DataFrame(coefs.filter(non_intercept_columns),
                                        columns=['coefficient'])
        df_non_intercept.index.name = 'feature'
        df_non_intercept = df_non_intercept.sort_index()
        df_non_intercept.reset_index(inplace=True)

        # now create a data frame that just has the intercept
        df_intercept = pd.DataFrame([{'feature': 'Intercept',
                                      'coefficient': coefs['const']}])

        # append the non-intercept frame to the intercept one
        df_coef = pd.concat([df_intercept, df_non_intercept], sort=True, ignore_index=True)

        # we always want to have the feature column first
        df_coef = df_coef[['feature', 'coefficient']]

        return df_coef

    @staticmethod
    def skll_learner_params_to_dataframe(learner):
        """
        Extract parameters from the given SKLL learner into a data frame.

        Parameters
        ----------
        learner : skll.learner.Learner
            A SKLL learner object.

        Returns
        -------
        df_coef : pandas DataFrame
            The data frame containing the model parameters
            from the given SKLL learner object.

        Note
        ----
        1. We use the ``coef_`` attribute of the scikit-learn model underlying
           the SKLL learner instead of the latter's ``model_params`` attribute.
           This is because ``model_params`` ignores zero coefficients, which we
           do not want.
        2. The first row in the output data frame is always for the intercept
           and the rest are sorted by feature name.
        """
        # get the intercept, coefficients, and feature names
        intercept = learner.model.intercept_
        coefficients = learner.model.coef_
        feature_names = learner.feat_vectorizer.get_feature_names()

        # first create a sorted data frame for all the non-intercept features
        df_non_intercept = pd.DataFrame({'feature': feature_names,
                                         'coefficient': coefficients})
        df_non_intercept = df_non_intercept.sort_values(by=['feature'])

        # now create a data frame that just has the intercept
        df_intercept = pd.DataFrame([{'feature': 'Intercept',
                                      'coefficient': intercept}])

        # append the non-intercept frame to the intercept one
        df_coef = pd.concat([df_intercept, df_non_intercept], sort=True, ignore_index=True)

        # we always want to have the feature column first
        df_coef = df_coef[['feature', 'coefficient']]

        return df_coef

    def create_fake_skll_learner(self, df_coefficients):
        """
        Create a fake SKLL linear regression learner from given coefficients.

        Parameters
        ----------
        df_coefficients : pandas DataFrame
            The data frame containing the linear coefficients
            we want to create the fake SKLL model with.

        Returns
        -------
        learner: skll.learner.Learner
            SKLL Learner object representing a ``LinearRegression`` model
            with the specified coefficients.
        """
        # initialize a random number generator
        randgen = RandomState(1234567890)

        # iterate over the coefficients
        coefdict = {}
        for feature, coefficient in df_coefficients.itertuples(index=False):
            if feature == 'Intercept':
                intercept = coefficient
            else:
                # exclude NA coefficients
                if coefficient == np.nan:
                    self.logger.warning("No coefficient was estimated for "
                                        "{}. This is likely due to exact "
                                        "collinearity in the model. This "
                                        "feature will not be used for model "
                                        "building".format(feature))
                else:
                    coefdict[feature] = coefficient

        learner = Learner('LinearRegression')
        num_features = len(coefdict)  # excluding the intercept
        fake_feature_values = randgen.rand(num_features)
        fake_features = [dict(zip(coefdict, fake_feature_values))]
        fake_fs = FeatureSet('fake', ids=['1'], labels=[1.0], features=fake_features)
        learner.train(fake_fs, grid_search=False)

        # now create its parameters from the coefficients from the built-in model
        learner.model.coef_ = learner.feat_vectorizer.transform(coefdict).toarray()[0]
        learner.model.intercept_ = intercept
        return learner

    def train_linear_regression(self, df_train, feature_columns):
        """
        Train a "LinearRegression" model.

        This model is a simple linear regression model.

        Parameters
        ----------
        df_train : pandas DataFrame
            Data frame containing the features on which
            to train the model.
        feature_columns : list
            A list of feature columns to use in training the model.

        Returns
        -------
        learner : skll.learner.Learner
            The SKLL learner object.
        fit : statsmodels.RegressionResults
            A ``statsmodels`` regression results object.
        df_coef : pandas DataFrame
            Data frame containing the model coefficients.
        used_features : list of str
            A list of features used in the final model.
        """
        # get the feature columns
        X = df_train[feature_columns]

        # add the intercept
        X = sm.add_constant(X)

        # fit the model
        fit = sm.OLS(df_train['sc1'], X).fit()
        df_coef = self.ols_coefficients_to_dataframe(fit.params)
        learner = self.create_fake_skll_learner(df_coef)

        # we used all the features
        used_features = feature_columns

        return learner, fit, df_coef, used_features

    def train_equal_weights_lr(self, df_train, feature_columns):
        """
        Train an "EqualWeightsLR" model.

        This model assigns the same weight to all features.

        Parameters
        ----------
        df_train : pandas DataFrame
            Data frame containing the features on which
            to train the model.
        feature_columns : list of str
            A list of feature columns to use in training the model.

        Returns
        -------
        learner : skll.learner.Learner
            The SKLL learner object.
        fit : statsmodels.RegressionResults
            A ``statsmodels`` regression results object.
        df_coef : pandas DataFrame
            Data frame containing the model coefficients.
        used_features : list of str
            A list of features used in the final model.
        """
        # we first compute a single feature that is simply the sum of all features
        df_train_eqwt = df_train.copy()
        df_train_eqwt['sumfeature'] = df_train_eqwt[feature_columns].apply(np.sum,
                                                                           axis=1)

        # train a plain Linear Regression model
        X = df_train_eqwt['sumfeature']
        X = sm.add_constant(X)
        fit = sm.OLS(df_train_eqwt['sc1'], X).fit()

        # get the coefficient for the summed feature and the intercept
        coef = fit.params['sumfeature']
        const = fit.params['const']

        # now we need to assign this coefficient to all of the original
        # features and create a fake SKLL learner with these weights
        original_features = [c for c in df_train_eqwt.columns if c not in ['sc1',
                                                                           'sumfeature',
                                                                           'spkitemid']]
        coefs = pd.Series(dict([(origf, coef)
                                for origf in original_features] + [('const', const)]))
        df_coef = self.ols_coefficients_to_dataframe(coefs)

        # create fake SKLL learner with these coefficients
        learner = self.create_fake_skll_learner(df_coef)

        # we used all the features
        used_features = feature_columns

        return learner, fit, df_coef, used_features

    def train_rebalanced_lr(self, df_train, feature_columns):
        """
        Train a "RebalancedLR" model.

        This model balances empirical weights by changing betas (adapted
        from `here <https://stats.stackexchange.com/q/30876>`_).

        Parameters
        ----------
        df_train : pandas DataFrame
            Data frame containing the features on which
            to train the model.
        feature_columns : list of str
            A list of feature columns to use in training the model.

        Returns
        -------
        learner : skll.learner.Learner
            The SKLL learner object.
        fit : statsmodels.RegressionResults
            A ``statsmodels`` regression results object.
        df_coef : pandas DataFrame
             Data frame containing the model coefficients.
        used_features : list of str
            A list of features used in the final model.
        """
        # train a plain Linear Regression model
        X = df_train[feature_columns]
        X = sm.add_constant(X)
        fit = sm.OLS(df_train['sc1'], X).fit()

        # convert the model parameters into a data frame
        df_params = self.ols_coefficients_to_dataframe(fit.params)
        df_params = df_params.set_index('feature')

        # compute the betas for the non-intercept coefficients
        df_weights = df_params.loc[feature_columns]
        df_betas = df_weights.copy()

        df_train_std = df_train[feature_columns].std()
        df_betas['coefficient'] = (df_weights['coefficient'].multiply(df_train_std,
                                                                      axis='index') /
                                   df_train['sc1'].std())

        # replace each negative beta with delta and adjust
        # all the positive betas to account for this
        RT = 0.05
        df_positive_betas = df_betas[df_betas['coefficient'] > 0]
        df_negative_betas = df_betas[df_betas['coefficient'] < 0]
        delta = np.sum(df_positive_betas['coefficient']) * RT / len(df_negative_betas)
        df_betas['coefficient'] = df_betas.apply(lambda row: row['coefficient'] * (1 - RT)
                                                 if row['coefficient'] > 0 else delta, axis=1)

        # rescale the adjusted betas to get the new coefficients
        df_coef = df_betas['coefficient'] * df_train['sc1'].std()
        df_coef = df_coef.divide(df_train[feature_columns].std(), axis='index')

        # add the intercept back to the new coefficients
        df_coef['Intercept'] = df_params.loc['Intercept'].coefficient
        df_coef = df_coef.sort_index().reset_index()
        df_coef.columns = ['feature', 'coefficient']

        # create fake SKLL learner with these coefficients
        learner = self.create_fake_skll_learner(df_coef)

        # we used all the features
        used_features = feature_columns

        return learner, fit, df_coef, used_features

    def train_lasso_fixed_lambda_then_lr(self, df_train, feature_columns):
        """
        Train a "LassoFixedLambdaThenLR" model.

        First do feature selection using lasso regression with
        a fixed lambda and then use only those features to train
        a second linear regression

        Parameters
        ----------
        df_train : pandas DataFrame
            Data frame containing the features on which
            to train the model.
        feature_columns : list of str
            A list of feature columns to use in training the model.

        Returns
        -------
        learner : skll.learner.Learner
            The SKLL learner object
        fit : statsmodels.RegressionResults
            A ``statsmodels`` regression results object.
        df_coef : pandas DataFrame
            The model coefficients in a data_frame
        used_features : list of str
            A list of features used in the final model.
        """
        # train a Lasso Regression model with this featureset with a preset lambda
        p_lambda = sqrt(len(df_train) * log10(len(feature_columns)))

        # create a SKLL FeatureSet instance from the given data frame
        fs_train = FeatureSet.from_data_frame(df_train[feature_columns + ['sc1']],
                                              'train',
                                              labels_column='sc1')

        # note that 'alpha' in sklearn is different from this lambda
        # so we need to normalize looking at the sklearn objective equation
        p_alpha = p_lambda / len(df_train)
        l_lasso = Learner('Lasso', model_kwargs={'alpha': p_alpha, 'positive': True})
        l_lasso.train(fs_train, grid_search=False)

        # get the feature names that have the non-zero coefficients
        non_zero_features = list(l_lasso.model_params[0].keys())

        # now train a new vanilla linear regression with just the non-zero features
        X = df_train[non_zero_features]
        X = sm.add_constant(X)
        fit = sm.OLS(df_train['sc1'], X).fit()

        # get the coefficients data frame
        df_coef = self.ols_coefficients_to_dataframe(fit.params)

        # create fake SKLL learner with these coefficients
        learner = self.create_fake_skll_learner(df_coef)

        # we used only the non-zero features
        used_features = non_zero_features

        return learner, fit, df_coef, used_features

    def train_positive_lasso_cv_then_lr(self, df_train, feature_columns):
        """
        Train a "PositiveLassoCVThenLR" model.

        First do feature selection using lasso regression optimized
        for log likelihood using cross validation and then use only
        those features to train a second linear regression.

        Parameters
        ----------
        df_train : pandas DataFrame
            Data frame containing the features on which
            to train the model.
        feature_columns : list of str
            A list of feature columns to use in training the model.

        Returns
        -------
        learner : skll.learner.Learner
            The SKLL learner object.
        fit : statsmodels.RegressionResults
            A statsmodels regression results object.
        df_coef : pandas DataFrame
            Data frame containing the model coefficients.
        used_features : list of str
            A list of features used in the final model.
        """
        # train a LassoCV outside of SKLL since it's not exposed there
        X = df_train[feature_columns].values
        y = df_train['sc1'].values
        clf = LassoCV(cv=10, positive=True, random_state=1234567890)
        model = clf.fit(X, y)

        # get the non-zero features from this model
        non_zero_features = []
        for feature, coefficient in zip(feature_columns, model.coef_):
            if coefficient != 0:
                non_zero_features.append(feature)

        # now train a new linear regression with just these non-zero features
        X = df_train[non_zero_features]
        X = sm.add_constant(X)
        fit = sm.OLS(df_train['sc1'], X).fit()

        # convert the model parameters into a data frame
        df_coef = self.ols_coefficients_to_dataframe(fit.params)

        # create fake SKLL learner with these coefficients
        learner = self.create_fake_skll_learner(df_coef)

        # we used only the non-zero features
        used_features = non_zero_features

        return learner, fit, df_coef, used_features

    def train_non_negative_lr(self, df_train, feature_columns):
        """
        Train an "NNLR" model.

        To do this, we first do feature selection using non-negative
        least squares (NNLS) and then use only its non-zero features to
        train another linear regression (LR) model. We do the regular LR
        at the end since we want an LR object so that we have access to R^2
        and other useful statistics. There should be no difference between
        the non-zero coefficients from NNLS and the coefficients that end
        up coming out of the subsequent LR.

        Parameters
        ----------
        df_train : pandas DataFrame
            Data frame containing the features on which
            to train the model.
        feature_columns : list of str
            A list of feature columns to use in training the model.

        Returns
        -------
        learner : skll.learner.Learner
            The SKLL learner object.
        fit : statsmodels.RegressionResults
            A statsmodels regression results object.
        df_coef : pandas DataFrame
             Data frame containing the model coefficients.
        used_features : list of str
            A list of features used in the final model.
        """
        # add an intercept to the features manually
        X = df_train[feature_columns].values
        intercepts = np.ones((len(df_train), 1))
        X_plus_intercept = np.concatenate([intercepts, X], axis=1)
        y = df_train['sc1'].values

        # fit an NNLS model on this data
        coefs, rnorm = nnls(X_plus_intercept, y)

        # check whether the intercept is set to 0 and if so then we need
        # to flip the sign and refit the model to ensure that it is always
        # kept in the model
        if coefs[0] == 0:
            intercepts = -1 * np.ones((len(df_train), 1))
            X_plus_intercept = np.concatenate([intercepts, X], axis=1)
            coefs, rnorm = nnls(X_plus_intercept, y)

        # separate the intercept and feature coefficients
        # intercept = coefs[0]
        coefficients = coefs[1:].tolist()

        # get the non-zero features from this model
        non_zero_features = []
        for feature, coefficient in zip(feature_columns, coefficients):
            if coefficient != 0:
                non_zero_features.append(feature)

        # now train a new linear regression with just these non-zero features
        X = df_train[non_zero_features]
        X = sm.add_constant(X)
        fit = sm.OLS(df_train['sc1'], X).fit()

        # convert this model's parameters to a data frame
        df_coef = self.ols_coefficients_to_dataframe(fit.params)

        # create fake SKLL learner with these coefficients
        learner = self.create_fake_skll_learner(df_coef)

        # we used only the non-zero features
        used_features = non_zero_features

        return learner, fit, df_coef, used_features

    def train_non_negative_lr_iterative(self, df_train, feature_columns):
        """
        Train an "NNLR_iterative" model.

        For applications where there is a concern that standard NNLS may not
        converge, an alternate method of training NNLR by iteratively fitting
        OLS models, checking the coefficients, and dropping negative
        coefficients. First, fit an OLS model. Then, identify any variables
        whose coefficients are negative. Drop these variables from the model.
        Finally, refit the model. If any coefficients are still negative, set
        these to zero.

        Parameters
        ----------
        df_train : pandas DataFrame
            Data frame containing the features on which
            to train the model.
        feature_columns : list of str
            A list of feature columns to use in training the model.

        Returns
        -------
        learner : skll.learner.Learner
            The SKLL learner object.
        fit : statsmodels.RegressionResults
            A statsmodels regression results object.
        df_coef : pandas DataFrame
            Data frame containing the model coefficients.
        used_features : list of str
            A list of features used in the final model.
        """
        X = df_train[feature_columns]
        X = sm.add_constant(X)

        y = df_train['sc1']

        fit = sm.OLS(y, X).fit()

        positive_features = []
        for name, value in fit.params.items():
            if value >= 0 and name != 'const':
                positive_features.append(name)

        X = df_train[positive_features]
        X = sm.add_constant(X)

        fit = sm.OLS(y, X).fit()

        # if any parameters are still negative, set them to zero
        params = fit.params.copy()
        params = params.drop('const')
        if not (params >= 0).all():
            fit.params[(fit.params < 0) & (fit.params.index != 'const')] = 0

        # convert this model's parameters to a data frame
        df_coef = self.ols_coefficients_to_dataframe(fit.params)

        # create fake SKLL learner with these coefficients
        learner = self.create_fake_skll_learner(df_coef)

        # we used only the non-zero features
        used_features = positive_features

        return learner, fit, df_coef, used_features

    def train_lasso_fixed_lambda_then_non_negative_lr(self, df_train, feature_columns):
        """
        Train an "LassoFixedLambdaThenNNLR" model.

        First do feature selection using lasso regression and positive
        only weights. Then fit an NNLR (see above) on those features.

        Parameters
        ----------
        df_train : pandas DataFrame
            Data frame containing the features on which
            to train the model.
        feature_columns : list of str
            A list of feature columns to use in training the model.

        Returns
        -------
        learner : skll.learner.Learner
            The SKLL learner object.
        fit : statsmodels.RegressionResults
            A statsmodels regression results object.
        df_coef : pandas DataFrame
            Data frame containing the model coefficients.
        used_features : list of str
            A list of features used in the final model.
        """
        # train a Lasso Regression model with a preset lambda
        p_lambda = sqrt(len(df_train) * log10(len(feature_columns)))

        # create a SKLL FeatureSet instance from the given data frame
        fs_train = FeatureSet.from_data_frame(df_train[feature_columns + ['sc1']],
                                              'train',
                                              labels_column='sc1')

        # note that 'alpha' in sklearn is different from this lambda
        # so we need to normalize looking at the sklearn objective equation
        p_alpha = p_lambda / len(df_train)
        l_lasso = Learner('Lasso', model_kwargs={'alpha': p_alpha, 'positive': True})
        l_lasso.train(fs_train, grid_search=False)

        # get the feature names that have the non-zero coefficients
        non_zero_features = list(l_lasso.model_params[0].keys())

        # now train an NNLS regression using these non-zero features
        # first add an intercept to the features manually
        X = df_train[feature_columns].values
        intercepts = np.ones((len(df_train), 1))
        X_plus_intercept = np.concatenate([intercepts, X], axis=1)
        y = df_train['sc1'].values

        # fit an NNLS model on this data
        coefs, rnorm = nnls(X_plus_intercept, y)

        # check whether the intercept is set to 0 and if so then we need
        # to flip the sign and refit the model to ensure that it is always
        # kept in the model
        if coefs[0] == 0:
            intercepts = -1 * np.ones((len(df_train), 1))
            X_plus_intercept = np.concatenate([intercepts, X], axis=1)
            coefs, rnorm = nnls(X_plus_intercept, y)

        # separate the intercept and feature coefficients
        # even though we do not use intercept in the code
        # we define it here for readability
        # intercept = coefs[0]
        coefficients = coefs[1:].tolist()

        # get the non-zero features from this model
        non_zero_features = []
        for feature, coefficient in zip(feature_columns, coefficients):
            if coefficient != 0:
                non_zero_features.append(feature)

        # now train a new linear regression with just these non-zero features
        X = df_train[non_zero_features]
        X = sm.add_constant(X)
        fit = sm.OLS(df_train['sc1'], X).fit()

        # convert this model's parameters into a data frame
        df_coef = self.ols_coefficients_to_dataframe(fit.params)

        # create fake SKLL learner with these coefficients
        learner = self.create_fake_skll_learner(df_coef)

        # we used only the positive features
        used_features = non_zero_features

        return learner, fit, df_coef, used_features

    def train_lasso_fixed_lambda(self, df_train, feature_columns):
        """
        Train a "LassoFixedLambda" model.

        This is a Lasso model with a fixed lambda.

        Parameters
        ----------
        df_train : pandas DataFrame
            Data frame containing the features on which
            to train the model.
        feature_columns : list of str
            A list of feature columns to use in training the model.

        Returns
        -------
        learner : skll.learner.Learner
            The SKLL learner object
        fit : ``None``
            This is always ``None`` since there is no OLS
            model fitted in this case.
        df_coef : pandas DataFrame
            Data frame containing the model coefficients.
        used_features : list of str
            A list of features used in the final model.
        """
        # train a Lasso Regression model with a preset lambda
        p_lambda = sqrt(len(df_train) * log10(len(feature_columns)))

        # create a SKLL FeatureSet instance from the given data frame
        fs_train = FeatureSet.from_data_frame(df_train[feature_columns + ['sc1']],
                                              'train',
                                              labels_column='sc1')

        # note that 'alpha' in sklearn is different from this lambda
        # so we need to normalize looking at the sklearn objective equation
        alpha = p_lambda / len(df_train)
        learner = Learner('Lasso', model_kwargs={'alpha': alpha, 'positive': True})
        learner.train(fs_train, grid_search=False)

        # convert this model's parameters to a data frame
        df_coef = self.skll_learner_params_to_dataframe(learner)

        # there's no OLS fit object in this case
        fit = None

        # we used all the features
        used_features = feature_columns

        return learner, fit, df_coef, used_features

    def train_positive_lasso_cv(self, df_train, feature_columns):
        """
        Train a "PositiveLassoCV" model.

        Do feature selection using lasso regression optimized for
        log likelihood using cross validation. All coefficients
        are constrained to have positive values.

        Parameters
        ----------
        df_train : pandas DataFrame
            Data frame containing the features on which
            to train the model.
        feature_columns : list of str
            A list of feature columns to use in training the model.

        Returns
        -------
        learner : skll.learner.Learner
            The SKLL learner object
        fit : ``None``
            This is always ``None`` since there is no OLS
            model fitted in this case.
        df_coef : pandas DataFrame
            Data frame containing the model coefficients.
        used_features : list of str
            A list of features used in the final model.
        """
        # train a LassoCV outside of SKLL since it's not exposed there
        X = df_train[feature_columns].values
        y = df_train['sc1'].values
        clf = LassoCV(cv=10, positive=True, random_state=1234567890)
        model = clf.fit(X, y)

        # save the non-zero model coefficients and intercept to a data frame
        non_zero_features, non_zero_feature_values = [], []
        for feature, coefficient in zip(feature_columns, model.coef_):
            if coefficient != 0:
                non_zero_features.append(feature)
                non_zero_feature_values.append(coefficient)

        # initialize the coefficient data frame with just the intercept
        df_coef = pd.DataFrame([('Intercept', model.intercept_)])
        df_coef = pd.concat([df_coef, pd.DataFrame(zip(non_zero_features, non_zero_feature_values))],
                            ignore_index=True)
        df_coef.columns = ['feature', 'coefficient']

        # create a fake SKLL learner with these non-zero weights
        learner = self.create_fake_skll_learner(df_coef)

        # there's no OLS fit object in this case
        fit = None

        # we used only the non-zero features
        used_features = non_zero_features

        return learner, fit, df_coef, used_features

    def train_score_weighted_lr(self, df_train, feature_columns):
        """
        Train a "ScoreWeightedLR" model.

        This is a linear regression model weighted by score.

        Parameters
        ----------
        df_train : pandas DataFrame
            Data frame containing the features on which
            to train the model.
        feature_columns : list of str
            A list of feature columns to use in training the model.

        Returns
        -------
        learner : skll.learner.Learner
            The SKLL learner object
        fit : statsmodels.RegressionResults
            A statsmodels regression results object.
        df_coef : pandas DataFrame
            Data frame containing the model coefficients.
        used_features : list of str
            A list of features used in the final model.
        """
        # train weighted least squares regression
        # get the feature columns

        X = df_train[feature_columns]

        # add the intercept
        X = sm.add_constant(X)

        # define the weights as inverse proportion of total
        # number of data points for each score
        score_level_dict = df_train['sc1'].value_counts()
        expected_proportion = 1 / len(score_level_dict)
        score_weights_dict = {sc1: expected_proportion / count
                              for sc1, count in score_level_dict.items()}
        weights = [score_weights_dict[sc1] for sc1 in df_train['sc1']]

        # fit the model
        fit = sm.WLS(df_train['sc1'], X, weights=weights).fit()
        df_coef = self.ols_coefficients_to_dataframe(fit.params)
        learner = self.create_fake_skll_learner(df_coef)

        # we used all the features
        used_features = feature_columns

        return learner, fit, df_coef, used_features

    def train_builtin_model(self,
                            model_name,
                            df_train,
                            experiment_id,
                            filedir,
                            figdir,
                            file_format='csv'):
        """
        Train one of the :ref:`built-in linear regression models <builtin_models>`.

        Parameters
        ----------
        model_name : str
            Name of the built-in model to train.
        df_train : pandas DataFrame
            Data frame containing the features on which to train
            the model. The data frame must contain the ID column
            named "spkitemid" and the numeric label column named
            "sc1".
        experiment_id : str
            The experiment ID.
        filedir : str
            Path to the `output` experiment output directory.
        figdir : str
            Path to the `figure` experiment output directory.
        file_format : str, optional
            The format in which to save files. One of {"csv", "tsv", "xlsx"}.
            Defaults to "csv".

        Returns
        -------
        learner : skll.learner.Learner
            SKLL ``LinearRegression`` `Learner <https://skll.readthedocs.io/en/latest/api/learner.html#skll.learner.Learner>`_
            object containing the coefficients learned by training
            the built-in model.
        """
        # get the columns that actually contain the feature values
        feature_columns = [c for c in df_train.columns if c not in ['spkitemid', 'sc1']]

        # LinearRegression
        if model_name == 'LinearRegression':
            result = self.train_linear_regression(df_train, feature_columns)

        # EqualWeightsLR
        elif model_name == 'EqualWeightsLR':
            result = self.train_equal_weights_lr(df_train, feature_columns)

        # RebalancedLR
        elif model_name == 'RebalancedLR':
            result = self.train_rebalanced_lr(df_train, feature_columns)

        # LassoFixedLambdaThenLR
        elif model_name == 'LassoFixedLambdaThenLR':
            result = self.train_lasso_fixed_lambda_then_lr(df_train, feature_columns)

        # PositiveLassoCVThenLR
        elif model_name == 'PositiveLassoCVThenLR':
            result = self.train_positive_lasso_cv_then_lr(df_train, feature_columns)

        # NNLR
        elif model_name == 'NNLR':
            result = self.train_non_negative_lr(df_train, feature_columns)

        # NNLRIterative
        elif model_name == 'NNLRIterative':
            result = self.train_non_negative_lr_iterative(df_train, feature_columns)

        # LassoFixedLambdaThenNNLR
        elif model_name == 'LassoFixedLambdaThenNNLR':
            result = self.train_lasso_fixed_lambda_then_non_negative_lr(df_train, feature_columns)

        # LassoFixedLambda
        elif model_name == 'LassoFixedLambda':
            result = self.train_lasso_fixed_lambda(df_train, feature_columns)

        # PositiveLassoCV
        elif model_name == 'PositiveLassoCV':
            result = self.train_positive_lasso_cv(df_train, feature_columns)

        # ScoreWeightedLR
        elif model_name == 'ScoreWeightedLR':
            result = self.train_score_weighted_lr(df_train, feature_columns)

        writer = DataWriter(experiment_id)
        frames = []

        # unpack all results
        learner, fit, df_coef, used_features = result

        # add raw coefficients to frame list
        frames.append({'name': 'coefficients', 'frame': df_coef})

        # compute the standardized and relative coefficients (betas) for the
        # non-intercept features and save to a file
        df_betas = df_coef.set_index('feature').loc[used_features]
        df_betas = df_betas.multiply(df_train[used_features].std(),
                                     axis='index') / df_train['sc1'].std()
        df_betas.columns = ['standardized']
        df_betas['relative'] = df_betas / sum(abs(df_betas['standardized']))
        df_betas.reset_index(inplace=True)

        # add betas to frame list
        frames.append({'name': 'betas', 'frame': df_betas})

        # save the OLS fit object and its summary to files
        if fit:
            ols_file = join(filedir, '{}.ols'.format(experiment_id))
            summary_file = join(filedir, '{}_ols_summary.txt'.format(experiment_id))
            with open(ols_file, 'wb') as olsf, open(summary_file, 'w') as summf:
                pickle.dump(fit, olsf)
                summf.write(str(fit.summary()))

            # create a data frame with main model fit metrics and save to the file
            df_model_fit = self.model_fit_to_dataframe(fit)

            # add model_fit to frame list
            frames.append({'name': 'model_fit', 'frame': df_model_fit})

        # save the SKLL model to a file
        model_file = join(filedir, '{}.model'.format(experiment_id))
        learner.save(model_file)

        container = DataContainer(frames)
        writer.write_experiment_output(filedir, container, file_format=file_format)

        self.learner = learner

        return learner

    def train_skll_model(self,
                         model_name,
                         df_train,
                         experiment_id,
                         filedir,
                         figdir,
                         file_format='csv',
                         custom_fixed_parameters=None,
                         custom_objective=None,
                         predict_expected_scores=False):
        """
        Train a SKLL classification or regression model.

        Parameters
        ----------
        model_name : str
            Name of the SKLL model to train.
        df_train : pandas DataFrame
            Data frame containing the features on which
            to train the model.
        experiment_id : str
            The experiment ID.
        filedir : str
            Path to the "output" experiment output directory.
        figdir : str
            Path to the "figure" experiment output directory.
        file_format : str, optional
            The format in which to save files. For SKLL models,
            this argument does not actually change the format of
            the output files at this time, as no betas are computed.
            One of {"csv", "tsv", "xlsx"}.
            Defaults to "csv".
        custom_fixed_parameters : dict, optional
            A dictionary containing any fixed parameters for the SKLL
            model.
            Defaults to ``None``.
        custom_objective : str, optional
            Name of custom user-specified objective. If not specified
            or ``None``, "neg_mean_squared_error" is used as the objective.
            Defaults to ``None``.
        predict_expected_scores : bool, optional
            Whether we want the trained classifiers to predict expected scores.
            Defaults to ``False``.

        Returns
        -------
        learner_and_objective : tuple
            A 2-tuple containing a SKLL Learner object of the appropriate type
            and the chosen tuning objective.
        """
        # Instantiate the given SKLL learner and set its probability value
        # and fixed parameters appropriately
        model_kwargs = custom_fixed_parameters if custom_fixed_parameters is not None else {}
        learner = Learner(model_name,
                          model_kwargs=model_kwargs,
                          probability=predict_expected_scores)

        # get the features, IDs, and labels from the given data frame
        feature_columns = [c for c in df_train.columns if c not in ['spkitemid', 'sc1']]
        features = df_train[feature_columns].to_dict(orient='records')
        ids = df_train['spkitemid'].tolist()
        labels = df_train['sc1'].tolist()

        # create a FeatureSet and train the model
        fs = FeatureSet('train', ids=ids, labels=labels, features=features)

        # If we are training a SKLL regressor, then we want to use either the
        # user-specified objective or `neg_mean_squared_error`. If it's SKLL
        # classifier, then the choice is between the user-specified objective
        # and `f1_score_micro`.
        if learner.model_type._estimator_type == 'regressor':
            objective = 'neg_mean_squared_error' if not custom_objective else custom_objective
        else:
            objective = 'f1_score_micro' if not custom_objective else custom_objective

        learner.train(fs, grid_search=True, grid_objective=objective, grid_jobs=1)

        # TODO: compute betas for linear SKLL models?

        # save the SKLL model to disk with the given model name prefix
        model_file = join(filedir, '{}.model'.format(experiment_id))
        learner.save(model_file)

        self.learner = learner

        # return the SKLL learner object and the chosen objective
        return learner, objective

    def train(self,
              configuration,
              data_container,
              filedir,
              figdir,
              file_format='csv'):
        """
        Train the given model on the given data and save the results.

        The main driver function to train the given model on the given
        data and save the results in the given directories using the
        given experiment ID as the prefix.

        parameters
        ----------
        configuration : configuration_parser.Configuration
            A configuration object containing "experiment_id" and "model_name"
            parameters.
        data_container : container.DataContainer
            A data container object containing "train_preprocessed_features"
            data set.
        filedir : str
            Path to the "output" experiment output directory.
        figdir : str
            Path to the "figure" experiment output directory.
        file_format : str, optional
            The format in which to save files. One of {"csv", "tsv", "xlsx"}.
            Defaults to "csv".

        Returns
        -------
        name : SKLL Learner object
        """
        Analyzer.check_param_names(configuration, ['model_name', 'experiment_id'])
        Analyzer.check_frame_names(data_container, ['train_preprocessed_features'])

        model_name = configuration['model_name']
        experiment_id = configuration['experiment_id']

        df_train = data_container['train_preprocessed_features']

        args = [model_name, df_train, experiment_id, filedir, figdir]
        kwargs = {'file_format': file_format}

        # add user-specified SKLL objective to the arguments if we are
        # training a SKLL model
        if is_skll_model(model_name):
            kwargs.update({'custom_fixed_parameters': configuration['skll_fixed_parameters'],
                           'custom_objective': configuration['skll_objective'],
                           'predict_expected_scores': configuration['predict_expected_scores']})
            model, chosen_objective = self.train_skll_model(*args, **kwargs)
            configuration['skll_objective'] = chosen_objective
        else:
            model = self.train_builtin_model(*args, **kwargs)

        return model

    def predict(self, df, min_score, max_score, predict_expected=False):
        """
        Get raw predictions from given SKLL model on data in given data frame.

        Parameters
        ----------
        df : pandas DataFrame
            Data frame containing features on which to make the predictions.
            The data must contain pre-processed feature values, an ID column
            named "spkitemid", and a label column named "sc1".
        min_score : int
            Minimum score level to be used if computing expected scores.
        max_score : int
            Maximum score level to be used if computing expected scores.
        predict_expected : bool, optional
            Predict expected scores for classifiers that return probability
            distributions over score. This will be ignored with a warning
            if the specified model does not support probability distributions.
            Note also that this assumes that the score range consists of
            contiguous integers - starting at ``min_score`` and ending at
            ``max_score``. Defaults to ``False``.

        Returns
        -------
        df_predictions : pandas DataFrame
            Data frame containing the raw predictions, the IDs, and the
            human scores.

        Raises
        ------
        ValueError
            If the model cannot predict probability distributions and
            ``predict_expected`` is set to ``True``.
        ValueError
            If  the score range specified by ``min_score`` and ``max_score``
            does not match what the model predicts in its probability
            distribution.
        """
        model = self.learner

        feature_columns = [c for c in df.columns if c not in ['spkitemid', 'sc1']]
        features = df[feature_columns].to_dict(orient='records')
        ids = df['spkitemid'].tolist()

        # if we have the labels, save them in the featureset
        labels = None
        if 'sc1' in df:
            labels = df['sc1'].tolist()

        fs = FeatureSet('data', ids=ids, labels=labels, features=features)
        # if we are predicting expected scores, then call a different function
        predictions = compute_expected_scores_from_model(model,
                                                         fs,
                                                         min_score,
                                                         max_score) if predict_expected else model.predict(fs)

        df_predictions = pd.DataFrame()
        df_predictions['spkitemid'] = ids
        df_predictions['raw'] = predictions

        # save the labels in the dataframe if they existed in the first place
        if labels:
            df_predictions['sc1'] = labels

        return df_predictions

    def predict_train_and_test(self,
                               df_train,
                               df_test,
                               configuration):
        """
        Generate raw, scaled, and trimmed predictions on given data.

        Parameters
        ----------
        df_train : pandas DataFrame
            Data frame containing the pre-processed training
            set features.
        df_test : pandas DataFrame
            Data frame containing the pre-processed test
            set features.
        configuration : configuration_parser.Configuration
            A configuration object containing "trim_max"
            and "trim_min" keys.

        Returns
        -------
        List of data frames containing predictions and other
        information.
        """
        Analyzer.check_param_names(configuration, ['trim_max',
                                                   'trim_min',
                                                   'trim_tolerance'])

        (trim_min,
         trim_max,
         trim_tolerance) = configuration.get_trim_min_max_tolerance()

        predict_expected_scores = configuration['predict_expected_scores']

        df_train_predictions = self.predict(df_train,
                                            int(trim_min),
                                            int(trim_max),
                                            predict_expected=predict_expected_scores)
        df_test_predictions = self.predict(df_test,
                                           int(trim_min),
                                           int(trim_max),
                                           predict_expected=predict_expected_scores)

        # get the mean and SD of the training set predictions
        train_predictions_mean = df_train_predictions['raw'].mean()
        train_predictions_sd = df_train_predictions['raw'].std()

        # get the mean and SD of the human labels
        human_labels_mean = df_train['sc1'].mean()
        human_labels_sd = df_train['sc1'].std()

        self.logger.info('Processing train set predictions.')
        feature_preprocessor = FeaturePreprocessor()
        df_train_predictions = feature_preprocessor.process_predictions(df_train_predictions,
                                                                        train_predictions_mean,
                                                                        train_predictions_sd,
                                                                        human_labels_mean,
                                                                        human_labels_sd,
                                                                        trim_min,
                                                                        trim_max,
                                                                        trim_tolerance)

        self.logger.info('Processing test set predictions.')
        df_test_predictions = feature_preprocessor.process_predictions(df_test_predictions,
                                                                       train_predictions_mean,
                                                                       train_predictions_sd,
                                                                       human_labels_mean,
                                                                       human_labels_sd,
                                                                       trim_min,
                                                                       trim_max,
                                                                       trim_tolerance)

        df_postproc_params = pd.DataFrame([{'trim_min': trim_min,
                                            'trim_max': trim_max,
                                            'trim_tolerance': trim_tolerance,
                                            'h1_mean': human_labels_mean,
                                            'h1_sd': human_labels_sd,
                                            'train_predictions_mean': train_predictions_mean,
                                            'train_predictions_sd': train_predictions_sd}])

        datasets = [{'name': 'pred_train', 'frame': df_train_predictions},
                    {'name': 'pred_test', 'frame': df_test_predictions},
                    {'name': 'postprocessing_params', 'frame': df_postproc_params}]

        # configuration options that are entirely for internal use
        internal_options_dict = {'train_predictions_mean': train_predictions_mean,
                                 'train_predictions_sd': train_predictions_sd,
                                 'human_labels_mean': human_labels_mean,
                                 'human_labels_sd': human_labels_sd}

        new_configuration = configuration.copy()
        for key, value in internal_options_dict.items():
            new_configuration[key] = value

        return new_configuration, DataContainer(datasets=datasets)

    def get_feature_names(self):
        """
        Get the feature names, if available.

        Returns
        -------
        feature_names : list or ``None``
            A list of feature names, or None if no learner was trained.
        """
        if self.learner is not None:
            return self.learner.feat_vectorizer.get_feature_names_out().tolist()
        return None

    def get_intercept(self):
        """
        Get the intercept of the model, if available.

        Returns
        -------
        intercept : float or ``None``
           The intercept of the model.
        """
        if self.learner is not None:
            return self.learner.model.intercept_
        return None

    def get_coefficients(self):
        """
        Get the coefficients of the model, if available.

        Returns
        -------
        coefficients : np.array or ``None``
           The coefficients of the model, if available.
        """
        if self.learner is not None:
            return self.learner.model.coef_
        return None

    def scale_coefficients(self, configuration):
        """
        Scale coefficients using human scores & training set predictions.

        This procedure approximates what is done in operational
        setting but does not apply trimming to predictions.

        Parameters
        ----------
        configuration : configuration_parser.Configuration
            A configuration object containing the "train_predictions_mean",
            "train_predictions_sd", and "human_labels_sd" parameters.

        Returns
        -------
        data_container : container.DataContainer
            A container object containing the "coefficients_scaled"
            dataset. The frame for this dataset contains the scaled
            coefficients and the feature names, along with the
            intercept.

        Raises
        ------
        RuntimeError
            If the model is non-linear and no coefficients
            are available.
        """
        Analyzer.check_param_names(configuration, ['train_predictions_mean',
                                                   'train_predictions_sd',
                                                   'human_labels_sd'])

        train_predictions_mean = configuration['train_predictions_mean']
        train_predictions_sd = configuration['train_predictions_sd']
        h1_sd = configuration['human_labels_sd']

        feature_names = self.get_feature_names()

        # try to get the model coefficients, if available
        try:
            coefficients = self.get_coefficients()
        except AttributeError:
            raise RuntimeError("no coefficients available for this model.")

        intercept = self.get_intercept()

        # scale the coefficients and the intercept
        scaled_coefficients = coefficients * h1_sd / train_predictions_sd

        # adjust the intercept to set the mean predicted score
        # to the mean of the training variable
        new_intercept = intercept * (h1_sd / train_predictions_sd)
        new_intercept += train_predictions_mean * (1 - h1_sd / train_predictions_sd)

        intercept_and_feature_names = ['Intercept'] + feature_names
        intercept_and_feature_values = [new_intercept] + list(scaled_coefficients)

        # create a data frame with new values
        df_scaled_coefficients = pd.DataFrame({'feature': intercept_and_feature_names,
                                               'coefficient': intercept_and_feature_values},
                                              columns=['feature', 'coefficient'])

        scaled_dataset = [{'name': 'coefficients_scaled',
                           'frame': df_scaled_coefficients}]

        return DataContainer(datasets=scaled_dataset) class FeaturePreprocessor:
    """Class to preprocess features in training and testing sets."""

    def __init__(self, logger=None):
        self.logger = logger if logger else logging.getLogger(__name__)

    def check_model_name(self, model_name):
        """
        Check that the given model name is valid and determine its type.

        Parameters
        ----------
        model_name : str
            Name of the model.

        Returns
        -------
        model_type: str
            One of "BUILTIN" or "SKLL".

        Raises
        ------
        ValueError
            If the model is not supported.
        """
        if is_built_in_model(model_name):
            model_type = 'BUILTIN'
        elif is_skll_model(model_name):
            model_type = 'SKLL'
        else:
            raise ValueError("The specified model {} "
                             "was not found. Please "
                             "check the spelling.".format(model_name))

        return model_type

    def trim(self,
             values,
             trim_min,
             trim_max,
             tolerance=0.4998):
        """
        Trim values in given numpy array.

        The trimming uses ``trim_min`` - ``tolerance`` as the floor and
        ``trim_max`` + ``tolerance`` as the ceiling.

        Parameters
        ----------
        values : list or np.array
            The values to trim.
        trim_min : float
            The lowest score on the score point, used for
            trimming the raw regression predictions.
        trim_max : float
            The highest score on the score point, used for
            trimming the raw regression predictions.
        tolerance : float, optional
            The tolerance that will be used to compute the
            trim interval.
            Defaults to ``0.4998``.

        Returns
        -------
        trimmed_values : np.array
            Trimmed values.
        """
        if isinstance(values, list):
            values = np.array(values)

        new_max = trim_max + tolerance
        new_min = trim_min - tolerance
        trimmed_values = values.copy()
        trimmed_values[trimmed_values > new_max] = new_max
        trimmed_values[trimmed_values < new_min] = new_min
        return trimmed_values

    @staticmethod
    def remove_outliers(values,
                        mean=None,
                        sd=None,
                        sd_multiplier=4):  # noqa: D301
        """
        Remove outliers from given array of values by clamping them.

        Clamp any given values that are
        ± ``sd_multiplier`` (:math:`m`) standard deviations (:math:`\\sigma`)
        away from the mean (:math:`\\mu`). Use given ``mean`` and ``sd``
        instead of computing :math:`\\sigma` and :math:`\\mu`, if specified.
        The values are clamped to the interval:

        .. math::
            [\\mu - m * \\sigma, \\mu + m * \\sigma]

        Parameters
        ----------
        values : np.array
            The values from which to remove outliers.
        mean : int or float, optional
            Use the given mean value when computing outliers
            instead of the mean from the data.
            Defaults to ``None``.
        sd : None, optional
            Use the given std. dev. value when computing
            outliers instead of the std. dev. from the
            data.
            Defaults to ``None``.
        sd_multiplier : int, optional
            Use the given multipler for the std. dev. when
            computing the outliers. Defaults to 4.
            Defaults to 4.

        Returns
        -------
        new_values : np.array
            Numpy array with the outliers clamped.
        """
        # convert data to a numpy float array before doing any clamping
        new_values = np.array(values, dtype=np.float64)

        if not mean:
            mean = new_values.mean()
        if not sd:
            sd = new_values.std()

        floor = mean - sd_multiplier * sd
        ceiling = mean + sd_multiplier * sd

        new_values[new_values > ceiling] = ceiling
        new_values[new_values < floor] = floor
        return new_values

    def remove_outliers_using_truncations(self,
                                          values,
                                          feature_name,
                                          truncations):
        """
        Remove outliers using pre-specified truncation groups.

        This is different from ``remove_outliers()`` which calculates the
        outliers based on the training set.

        Parameters
        ----------
        values : np.array
            The values from which to remove outliers.
        feature_name : str
            Name of the feature whose outliers are
            being clamped.
        truncations : pandas DataFrame
            A data frame with truncation values. The
            features should be set as the index.

        Returns
        -------
        new_values : numpy array
            Numpy array with the outliers clamped.
        """
        # convert data to a numpy float array before doing any clamping
        new_values = np.array(values, dtype=np.float64)

        minimum = truncations.loc[feature_name, 'min']
        maximum = truncations.loc[feature_name, 'max']

        new_values[new_values > maximum] = maximum
        new_values[new_values < minimum] = minimum
        return new_values

    def select_candidates(self,
                          df,
                          N,
                          candidate_col='candidate'):
        """
        Select candidates which have responses to N or more items.

        Parameters
        ----------
        df : pandas DataFrame
            The data frame from which to select candidates with N or more items.
        N: int
            Minimal number of items per candidate
        candidate_col : str, optional
            Name of the column which contains candidate ids.
            Defaults to "candidate".

        Returns
        -------
        df_included: pandas DataFrame
            Data frame with responses from candidates with responses to N
            or more items.
        df_excluded: pandas DataFrame
            Data frame with responses from candidates with responses to
            less than N items.
        """
        items_per_candidate = df[candidate_col].value_counts()

        selected_candidates = items_per_candidate[items_per_candidate >= N]
        selected_candidates = selected_candidates.index

        df_included = df[df[candidate_col].isin(selected_candidates)].copy()
        df_excluded = df[~df[candidate_col].isin(selected_candidates)].copy()

        # reset indices
        df_included.reset_index(drop=True, inplace=True)
        df_excluded.reset_index(drop=True, inplace=True)

        return (df_included,
                df_excluded)

    def check_subgroups(self, df, subgroups):
        """
        Validate subgroup names in the given data.

        Check that all subgroups, if specified, correspond to columns in the
        provided data frame, and replace all NaNs in subgroups values with
        'No info' for later convenience.

        Raises an exception if any specified subgroup columns are missing.

        Parameters
        ----------
        df : pandas DataFrame
            Input data frame with subgroups to check.
        subgroups : list of str
            List of column names that contain grouping information.

        Returns
        -------
        df : pandas DataFrame
             Modified input data frame with NaNs replaced.

        Raises
        ------
        KeyError
            If the data does not contain columns for all specified subgroups.
        """
        missing_sub_cols = set(subgroups).difference(df.columns)
        if missing_sub_cols:
            raise KeyError("The data does not contain columns "
                           "for all subgroups specified in the "
                           "configuration file. Please check for "
                           "capitalization and other spelling "
                           "errors and make sure the subgroup "
                           "names do not contain hyphens. "
                           "The data does not have columns "
                           "for the following "
                           "subgroups: {}".format(', '.join(missing_sub_cols)))

        # replace any empty values in subgroups values by "No info"
        empty_value = re.compile(r"^\s*$")
        df[subgroups] = df[subgroups].replace(to_replace=empty_value,
                                              value='No info')
        return df

    def rename_default_columns(self,
                               df,
                               requested_feature_names,
                               id_column,
                               first_human_score_column,
                               second_human_score_column,
                               length_column,
                               system_score_column,
                               candidate_column):
        """
        Standardize column names and rename columns with reserved column names.

        RSMTool reserves some column names for internal use, e.g., "sc1",
        "spkitemid" etc. If the given data already contains columns with
        these names, then they must be renamed to prevent conflict. This
        method renames such columns to "##NAME##", e.g., an existing column
        named "sc1" will be renamed to "##sc1##".

        Parameters
        ----------
        df : pandas DataFrame
            The data frame containing the columns to rename.
        requested_feature_names : list of str
            List of feature column names that we want
            to include in the scoring model.
        id_column : str
            Column name containing the response IDs.
        first_human_score_column : str or ``None``.
            Column name containing the H1 scores.
            Should be ``None`` if no H1 scores are available.
        second_human_score_column : str or ``None``
            Column name containing the H2 scores.
            Should be ``None`` if no H2 scores are available.
        length_column : str or ``None``
            Column name containing response lengths.
            Should be ``None`` if lengths are not available.
        system_score_column : str
            Column name containing the score predicted
            by the system. This is only used for rsmeval.
        candidate_column : str or ``None``
            Column name containing identifying information
            at the candidate level. Should be None if such
            information is not available.

        Returns
        -------
        df : pandas DataFrame
            Modified input data frame with all the approximate
            re-namings.
        """
        df = df.copy()

        columns = [id_column,
                   first_human_score_column,
                   second_human_score_column,
                   length_column,
                   system_score_column,
                   candidate_column]

        defaults = ['spkitemid', 'sc1', 'sc2', 'length', 'raw', 'candidate']

        # create a dictionary of name mapping for used columns
        name_mapping = dict(filter(lambda t: t[0] is not None, zip(columns,
                                                                   defaults)))

        # find the columns where the names match the default names
        correct_defaults = [column for (column, default)
                            in name_mapping.items()
                            if column == default]

        # find the columns with default names reserved for other columns
        # which are not used as features in the model
        columns_with_incorrect_default_names = [column for column in df.columns
                                                if (column in defaults and
                                                    column not in correct_defaults and
                                                    column not in requested_feature_names)]
        # rename these columns
        if columns_with_incorrect_default_names:
            new_column_names = ['##{}##'.format(column) for column
                                in columns_with_incorrect_default_names]
            df.rename(columns=dict(zip(columns_with_incorrect_default_names,
                                       new_column_names)),
                      inplace=True)

        # find the columns where the names do not match the default
        columns_with_custom_names = [column for column in name_mapping
                                     if column not in correct_defaults]

        # rename the custom-named columns to default values
        for column in columns_with_custom_names:

            # if the column has already been renamed because it used a
            # default name, then use the updated name
            if column in columns_with_incorrect_default_names:
                df.rename(columns={'##{}##'.format(column): name_mapping[column]},
                          inplace=True)
            else:
                df.rename(columns={column: name_mapping[column]},
                          inplace=True)

        return df

    def filter_on_column(self,
                         df,
                         column,
                         id_column,
                         exclude_zeros=False,
                         exclude_zero_sd=False):  # noqa: D301
        """
        Filter out rows containing non-numeric values.

        Filter out the rows in the given data frame that contain non-numeric
        (or zero, if specified) values in the specified column. Additionally,
        it may exclude any columns if they have a standard deviation
        (:math:`\\sigma`) of 0.

        Parameters
        ----------
        df : pandas DataFrame
            The data frame containing the data to be filtered.
        column : str
            Name of the column from which to filter out values.
        id_column : str
            Name of the column containing the unique response IDs.
        exclude_zeros : bool, optional
            Whether to exclude responses containing zeros
            in the specified column.
            Defaults to ``False``.
        exclude_zero_sd : bool, optional
            Whether to perform the additional filtering step of removing
            columns that have :math:`\\sigma = 0`.
            Defaults to ``False``.

        Returns
        -------
        df_filtered : pandas DataFrame
            Data frame containing the responses that were *not* filtered out.
        df_excluded : pandas DataFrame
            Data frame containing the non-numeric or zero responses that
            were filtered out.

        Note
        ----
        The columns with :math:`\\sigma=0` are removed from both output
        data frames, assuming ``exclude_zero_scores`` is ``True``.
        """
        # create a copy of the original data frame
        df_filter = df.copy()

        # we start out assuming that we will not drop this column
        drop_column = False

        # return a copy of the original data frame if
        # the given column does not exist at all
        if column not in df.columns:
            return df_filter

        # Force convert the label column to numeric and
        # convert whatever can't be converted to a NaN
        df_filter[column] = pd.to_numeric(df_filter[column],
                                          errors='coerce').astype(float)

        # Save the values that have been converted to NaNs
        # as a separate data frame. We want to keep them as NaNs
        # to do more analyses later. We also filter out inf values.
        # Since these can only be generated during transformations,
        # we include them with NaNs for consistency.
        bad_rows = df_filter[column].isnull() | np.isinf(df_filter[column])
        df_bad_rows = df_filter[bad_rows]

        # if the column contained only non-numeric values, we need to drop it
        if len(df_bad_rows) == len(df_filter):
            self.logger.info(f"Feature {column} was excluded from the model "
                             f"because it only contains non-numeric values.")
            drop_column = True

        # now drop the above bad rows containing NaNs from our data frame
        df_filter = df_filter[~bad_rows]

        # exclude zeros if specified
        if exclude_zeros:
            zero_rows = df_filter[column] == 0
            df_zero_rows = df_filter[zero_rows]
            df_filter = df_filter[~zero_rows]
        else:
            df_zero_rows = pd.DataFrame()

        # combine all the filtered rows into a single data frame
        df_exclude = pd.concat([df_bad_rows, df_zero_rows], sort=True)

        # reset the index so that the indexing works correctly
        # for the next feature with missing values
        df_filter.reset_index(drop=True, inplace=True)
        df_exclude.reset_index(drop=True, inplace=True)

        # Check if the the standard deviation equals zero:
        # for training set sd == 0 will break normalization.
        # We set the tolerance level to the 6th digit
        # to account for the possibility that the exact value
        # computed by `std()` is not 0
        if exclude_zero_sd is True:
            feature_sd = df_filter[column].std()
            if np.isclose(feature_sd, 0, atol=1e-07):
                self.logger.info(f"Feature {column} was excluded from the model "
                                 f"because its standard deviation in the "
                                 f"training set is equal to 0.")
                drop_column = True

        # if `drop_column` is true, then we need to drop the column
        if drop_column:
            df_filter = df_filter.drop(column, axis=1)
            df_exclude = df_exclude.drop(column, axis=1)

        # return the filtered rows and the new data frame
        return (df_filter, df_exclude)

    def process_predictions(self,
                            df_test_predictions,
                            train_predictions_mean,
                            train_predictions_sd,
                            human_labels_mean,
                            human_labels_sd,
                            trim_min,
                            trim_max,
                            trim_tolerance=0.4998):
        """
        Process predictions to create scaled, trimmed and rounded predictions.

        Parameters
        ----------
        df_test_predictions : pandas DataFrame
            Data frame containing the test set predictions.
        train_predictions_mean : float
            The mean of the predictions on the training set.
        train_predictions_sd : float
            The std. dev. of the predictions on the training
            set.
        human_labels_mean : float
            The mean of the human scores used to train the
            model.
        human_labels_sd : float
            The std. dev. of the human scores used to train
            the model.
        trim_min : float
            The lowest score on the score point, used for
            trimming the raw regression predictions.
        trim_max : float
            The highest score on the score point, used for
            trimming the raw regression predictions.
        trim_tolerance: float
            Tolerance to be added to trim_max and substracted from
            trim_min. Defaults to 0.4998.

        Returns
        -------
        df_pred_processed : pandas DataFrame
            Data frame containing the various trimmed
            and rounded predictions.
        """
        # rescale the test set predictions by boosting
        # them to match the human mean and SD
        scaled_test_predictions = (df_test_predictions['raw'] -
                                   train_predictions_mean) / train_predictions_sd
        scaled_test_predictions = scaled_test_predictions * human_labels_sd + human_labels_mean

        df_pred_process = df_test_predictions.copy()
        df_pred_process['scale'] = scaled_test_predictions

        # trim and round the predictions before running the analyses
        df_pred_process['raw_trim'] = self.trim(df_pred_process['raw'],
                                                trim_min,
                                                trim_max,
                                                trim_tolerance)

        df_pred_process['raw_trim_round'] = np.rint(df_pred_process['raw_trim'])
        df_pred_process['raw_trim_round'] = df_pred_process['raw_trim_round'].astype('int64')

        df_pred_process['scale_trim'] = self.trim(df_pred_process['scale'],
                                                  trim_min,
                                                  trim_max,
                                                  trim_tolerance)

        df_pred_process['scale_trim_round'] = np.rint(df_pred_process['scale_trim'])
        df_pred_process['scale_trim_round'] = df_pred_process['scale_trim_round'].astype('int64')

        return df_pred_process

    def filter_on_flag_columns(self,
                               df,
                               flag_column_dict):
        """
        Filter based on specific flag columns.

        Check that all flag_columns are present in the given
        data frame, convert these columns to strings and filter
        out the values which do not match the condition in
        ``flag_column_dict``.

        Parameters
        ----------
        df : pandas DataFrame
            The DataFrame to filter on.
        flag_column_dict : dict
            Dictionary containing the flag column information.

        Returns
        -------
        df_responses_with_requested_flags : pandas DataFrame
            Data frame containing the responses remaining
            after filtering using the specified flag
            columns.
        df_responses_with_excluded_flags : pandas DataFrame
            Data frame containing the responses filtered
            out using the specified flag columns.

        Raises
        ------
        KeyError
            If the columns listed in the dictionary are
            not actually present in the data frame.
        ValueError
            If no responses remain after filtering based
            on the flag column information.
        """
        df = df.copy()

        flag_columns = list(flag_column_dict.keys())

        if not flag_columns:
            return df.copy(), pd.DataFrame(columns=df.columns)
        else:
            # check that all columns are present
            missing_flag_columns = set(flag_columns).difference(df.columns)
            if missing_flag_columns:
                raise KeyError("The data does not contain columns "
                               "for all flag columns specified in the "
                               "configuration file. Please check for "
                               "capitalization and other spelling "
                               "errors and make sure the flag column "
                               "names do not contain hyphens. "
                               "The data does not have the following columns: "
                               "{}".format(', '.join(missing_flag_columns)))

            # since flag column may be a mix of strings and numeric values
            # we convert all strings and integers to floats such that, for
            # example, “1”, 1, and “1.0" all map to 1.0. To do this, we will
            # first convert all the strings to numbers and then convert
            # all the integers to floats.

            flag_column_dict_to_float = {key: list(map(convert_to_float, value))
                                         for (key, value)
                                         in flag_column_dict.items()}

            # and now convert the the values in the feature column
            # in the data frame
            df_new = df[flag_columns].copy()
            df_new = df_new.applymap(convert_to_float)

            # identify responses with values which satisfy the condition
            full_mask = df_new.isin(flag_column_dict_to_float)
            flag_mask = full_mask[list(flag_column_dict_to_float.keys())].all(1)

            # return the columns from the original frame that was passed in
            # so that all data types remain the same and are not changed
            df_responses_with_requested_flags = df[flag_mask].copy()
            df_responses_with_excluded_flags = df[~flag_mask].copy()

            # make sure that the remaining data frame is not empty
            if len(df_responses_with_requested_flags) == 0:
                raise ValueError("No responses remaining after filtering "
                                 "on flag columns. No further analysis can "
                                 "be run.")
            # reset the index
            df_responses_with_requested_flags.reset_index(drop=True,
                                                          inplace=True)

            df_responses_with_excluded_flags.reset_index(drop=True,
                                                         inplace=True)

            return (df_responses_with_requested_flags,
                    df_responses_with_excluded_flags)

    def generate_feature_names(self,
                               df,
                               reserved_column_names,
                               feature_subset_specs,
                               feature_subset):
        """
        Generate feature names from column names in data frame.

        This method also selects the specified subset of features.

        Parameters
        ----------
        df : pandas DataFrame
            The data frame from which to generate feature names.
        reserved_column_names : list of str
            Names of reserved columns.
        feature_subset_specs : pandas DataFrame
            Feature subset specifications.
        feature_subset : str
            Feature subset column.

        Returns
        -------
        feature_names : list of str
            A list of features names.
        """
        df = df.copy()

        # Exclude the reserved names
        possible_feature_names = [cname for cname in df.columns
                                  if cname not in reserved_column_names]

        # Select the features by subset.
        # In the future, we may add option to select
        # by other methods, if needed.
        if feature_subset is not None:
            feature_names = FeatureSubsetProcessor().select_by_subset(possible_feature_names,
                                                                      feature_subset_specs,
                                                                      feature_subset)
        else:
            feature_names = possible_feature_names
        return feature_names

    def preprocess_feature(self,
                           values,
                           feature_name,
                           feature_transform,
                           feature_mean,
                           feature_sd,
                           exclude_zero_sd=False,
                           raise_error=True,
                           truncations=None):
        """
        Remove outliers and transform the values in given numpy array.

        Use the given outlier and transformation parameters.

        Parameters
        ----------
        values : np.array
            The feature values to preprocess.
        feature_name : str
            Name of the feature being pre-processed.
        feature_transform : str
            Name of the transformation function to apply.
        feature_mean : float
            Mean value to use for outlier detection instead
            of the mean of the given feature values.
        feature_sd : float
            Std. dev. value to use for outlier detection instead
            of the std. dev. of the given feature values.
        exclude_zero_sd : bool, optional
            Exclude the feature if it has zero standard deviation.
            Defaults to ``False``.
        raise_error : bool, optional
            Raise an error if any of the transformations lead to "inf" values
            or may change the ranking of feature values.
            Defaults to ``True``.
        truncations : pandas DataFrame, optional
            A set of pre-defined truncation values.
            Defaults to ``None``.

        Returns
        -------
        transformed_feature : np.array
            Numpy array containing the transformed and clamped
            feature values.

        Raises
        ------
        ValueError
            If the preprocessed feature values have zero standard deviation
            and ``exclude_zero_sd`` is set to ``True``.
        """
        if truncations is not None:
            # clamp outlier values using the truncations set
            features_no_outliers = self.remove_outliers_using_truncations(values,
                                                                          feature_name,
                                                                          truncations)

        else:
            # clamp any outlier values that are 4 standard deviations
            # away from the mean
            features_no_outliers = self.remove_outliers(values,
                                                        mean=feature_mean,
                                                        sd=feature_sd)

        # apply the requested transformation to the feature
        transformed_feature = FeatureTransformer().transform_feature(features_no_outliers,
                                                                     feature_name,
                                                                     feature_transform,
                                                                     raise_error=raise_error)

        # check the standard deviation of the transformed feature
        # we set ddof to 1 so that np.std gave the same result as pandas .std
        # we also set the tolerance limit to account for cases where std
        # is computed as a very low decimal rather than 0
        # We only do this for the training set.
        if exclude_zero_sd:
            feature_sd = np.std(transformed_feature, ddof=1)
            if np.isclose(feature_sd, 0, atol=1e-07):
                raise ValueError("The standard deviation for "
                                 "feature {} is 0 after pre-processing. "
                                 "Please exclude this feature and re-run "
                                 "the experiment.".format(feature_name))

        return transformed_feature

    def preprocess_features(self,
                            df_train,
                            df_test,
                            df_feature_specs,
                            standardize_features=True,
                            use_truncations=False):
        """
        Preprocess features in given data using corresponding specifications.

        Preprocess the feature values in the training and testing data
        frames whose specifications are contained in ``df_feature_specs``.
        Also returns a third data frame containing the feature specifications
        and other information.

        Parameters
        ----------
        df_train : pandas DataFrame
            Data frame containing the raw feature values
            for the training set.
        df_test : pandas DataFrame
            Data frame containing the raw feature values
            for the test set.
        df_feature_specs : pandas DataFrame
            Data frame containing the various specifications
            from the feature file.
        standardize_features : bool, optional
            Whether to standardize the features.
            Defaults to ``True``.
        use_truncations : bool, optional
            Whether we should use the truncation set
            for removing outliers.
            Defaults to ``False``.

        Returns
        -------
        df_train_preprocessed : pandas DataFrame
            Data frame with preprocessed training data.
        df_test_preprocessed : pandas DataFrame
            Data frame with preprocessed test data.
        df_feature_info : pandas DataFrame
            Data frame with feature information.
        """
        # keep the original data frames and make copies
        # that only include features used in the model
        df_train_preprocessed = df_train.copy()
        df_test_preprocessed = df_test.copy()

        # we also need to create a data frame that includes
        # all relevant information about each feature
        df_feature_info = pd.DataFrame()

        # make feature the index of df_feature_specs
        df_feature_specs.index = df_feature_specs['feature']

        # if we are should be using truncations, then we create the truncations
        # set from the feature specifications
        if use_truncations:
            truncations = df_feature_specs[['feature', 'min', 'max']].set_index('feature')
        else:
            truncations = None

        # now iterate over each feature
        for feature_name in df_feature_specs['feature']:

            feature_transformation = df_feature_specs.at[feature_name, 'transform']
            feature_sign = df_feature_specs.at[feature_name, 'sign']

            train_feature_mean = df_train[feature_name].mean()
            train_feature_sd = df_train[feature_name].std()

            training_feature_values = df_train[feature_name].values
            df_train_preprocessed[feature_name] = self.preprocess_feature(training_feature_values,
                                                                          feature_name,
                                                                          feature_transformation,
                                                                          train_feature_mean,
                                                                          train_feature_sd,
                                                                          exclude_zero_sd=True,
                                                                          truncations=truncations)

            testing_feature_values = df_test[feature_name].values
            df_test_preprocessed[feature_name] = self.preprocess_feature(testing_feature_values,
                                                                         feature_name,
                                                                         feature_transformation,
                                                                         train_feature_mean,
                                                                         train_feature_sd,
                                                                         truncations=truncations)

            # Standardize the features using the mean and sd computed on the
            # training set. These are computed separately because we need to
            # get the mean of transformed feature before standardization.
            train_transformed_mean = df_train_preprocessed[feature_name].mean()
            train_transformed_sd = df_train_preprocessed[feature_name].std()

            if standardize_features:

                df_train_without_mean = (df_train_preprocessed[feature_name] -
                                         train_transformed_mean)
                df_train_preprocessed[feature_name] = df_train_without_mean / train_transformed_sd

                df_test_without_mean = (df_test_preprocessed[feature_name] -
                                        train_transformed_mean)
                df_test_preprocessed[feature_name] = df_test_without_mean / train_transformed_sd

            # Multiply both train and test feature by sign.
            df_train_preprocessed[feature_name] = (df_train_preprocessed[feature_name] *
                                                   feature_sign)
            df_test_preprocessed[feature_name] = (df_test_preprocessed[feature_name] *
                                                  feature_sign)

            # update the feature preprocessing metadata frame
            df_feature = pd.DataFrame([{"feature": feature_name,
                                        "transform": feature_transformation,
                                        "sign": feature_sign,
                                        "train_mean": train_feature_mean,
                                        "train_sd": train_feature_sd,
                                        "train_transformed_mean": train_transformed_mean,
                                        "train_transformed_sd": train_transformed_sd}])

            df_feature_info = pd.concat([df_feature_info, df_feature])

        # reset the index for the feature metadata frame
        # since we built it up row by row
        df_feature_info = df_feature_info.reset_index().drop('index', axis=1)

        # return the three data frames
        return (df_train_preprocessed,
                df_test_preprocessed,
                df_feature_info)

    def filter_data(self,
                    df,
                    label_column,
                    id_column,
                    length_column,
                    second_human_score_column,
                    candidate_column,
                    requested_feature_names,
                    reserved_column_names,
                    given_trim_min,
                    given_trim_max,
                    flag_column_dict,
                    subgroups,
                    exclude_zero_scores=True,
                    exclude_zero_sd=False,
                    feature_subset_specs=None,
                    feature_subset=None,
                    min_candidate_items=None,
                    use_fake_labels=False):
        """
        Filter rows with zero/non-numeric values for ``label_column``.

        Check whether any features that are specifically requested in
        ``requested_feature_names`` are missing from the data. If no
        feature names are requested, the feature list is generated based
        on column names and subset information, if available. The function
        then excludes non-numeric values for any feature. It will also exclude
        zero scores if ``exclude_zero_scores`` is ``True``. If the user
        requested to exclude candidates with less than ``min_candidate_items``,
        such candidates are also excluded.

        It also generates fake labels between 1 and 10 if
        ``use_fake_parameters`` is ``True``. Finally, it renames the ID
        and label columns and splits the data into: (a) data frame with
        feature values and scores (b) data frame with information about
        subgroup and candidate (metadata) and (c) the data frame with all other
        columns.

        Parameters
        ----------
        df : pandas DataFrame
            The data frame to filter.
        label_column : str
            The label column in the data.
        id_column : str
            The ID column in the data.
        length_column : str
            The length column in the data.
        second_human_score_column : str
            The second human score column in the data.
        candidate_column : str
            The candidate column in the data.
        requested_feature_names : list
            A list of requested feature names.
        reserved_column_names : list
            A list of reserved column names.
        given_trim_min : float
            The minimum trim value.
        given_trim_max : float
            The maximum trim value.
        flag_column_dict : dict
            A dictionary of flag columns.
        subgroups : list, optional
            A list of subgroups, if any.
        exclude_zero_scores : bool
            Whether to exclude zero scores.
            Defaults to ``True``.
        exclude_zero_sd : bool, optional
            Whether to exclude zero standard deviation.
            Defaults to ``False``.
        feature_subset_specs : pandas DataFrame, optional
            The data frame containing the feature subset specifications.
            Defaults to ``None``.
        feature_subset : str, optional
            The feature subset group (e.g. 'A').
            Defaults to ``None``.
        min_candidate_items : int, optional
            The minimum number of items needed to include candidate.
            Defaults to ``None``.
        use_fake_labels : bool, optional
            Whether to use fake labels.
            Defaults to ``False``.

        Returns
        -------
        df_filtered_features : pandas DataFrame
            Data frame with filtered features.
        df_filtered_metadata : pandas DataFrame
            Data frame with filtered metadata.
        df_filtered_other_columns : pandas DataFrame
            Data frame with other columns filtered.
        df_excluded : pandas DataFrame
            Data frame with excluded records.
        df_filtered_length : pandas DataFrame
            Data frame with length column(s) filtered.
        df_filtered_human_scores : pandas DataFrame
            Data frame with human scores filtered.
        df_responses_with_excluded_flags : pandas DataFrame
            Data frame containing responses with excluded flags.
        trim_min : float
            The maximum trim value.
        trim_max : float
            The minimum trim value.
        feature_names  : list
            A list of feature names.
        """
        # make sure that the columns specified in the
        # config file actually exist
        columns_to_check = [id_column, label_column]

        if length_column:
            columns_to_check.append(length_column)

        if second_human_score_column:
            columns_to_check.append(second_human_score_column)

        if candidate_column:
            columns_to_check.append(candidate_column)

        missing_columns = set(columns_to_check).difference(df.columns)
        if missing_columns:
            raise KeyError("Columns {} from the config file "
                           "do not exist in the data.".format(missing_columns))

        # it is possible for the `id_column` and `candidate_column` to be
        # set to the same column name in the CSV file, e.g., if there is
        # only one response per candidate. If this happens, we neeed to
        # create a duplicate column for candidates or id for the downstream
        # processing to work as usual.
        if id_column == candidate_column:
            # if the name for both columns is `candidate`, we need to
            # create a separate id_column name
            if id_column == 'candidate':
                df['spkitemid'] = df['candidate'].copy()
                id_column = 'spkitemid'
            # else we create a separate `candidate` column
            else:
                df['candidate'] = df[id_column].copy()
                candidate_column = 'candidate'

        df = self.rename_default_columns(df,
                                         requested_feature_names,
                                         id_column,
                                         label_column,
                                         second_human_score_column,
                                         length_column,
                                         None,
                                         candidate_column)

        # check that the id_column contains unique values
        if df['spkitemid'].size != df['spkitemid'].unique().size:
            raise ValueError("The data contains duplicate response IDs in "
                             "'{}'. Please make sure all response IDs are "
                             "unique and re-run the tool.".format(id_column))

        # Generate feature names if no specific features were requested by the user
        if len(requested_feature_names) == 0:
            feature_names = self.generate_feature_names(df,
                                                        reserved_column_names,
                                                        feature_subset_specs=feature_subset_specs,
                                                        feature_subset=feature_subset)
        else:
            feature_names = requested_feature_names

        # make sure that feature names do not contain reserved column names
        illegal_feature_names = set(feature_names).intersection(reserved_column_names)
        if illegal_feature_names:
            raise ValueError("The following reserved "
                             "column names cannot be "
                             "used as feature names: '{}'. "
                             "Please rename these columns "
                             "and re-run the "
                             "experiment.".format(', '.join(illegal_feature_names)))

        # check to make sure that the subgroup columns are all present
        df = self.check_subgroups(df, subgroups)

        # filter out the responses based on flag columns
        (df_responses_with_requested_flags,
         df_responses_with_excluded_flags) = self.filter_on_flag_columns(df, flag_column_dict)

        # filter out the rows that have non-numeric or zero labels
        # unless we are going to generate fake labels in the first place
        if not use_fake_labels:
            (df_filtered,
             df_excluded) = self.filter_on_column(df_responses_with_requested_flags,
                                                  'sc1',
                                                  'spkitemid',
                                                  exclude_zeros=exclude_zero_scores)

            # make sure that the remaining data frame is not empty
            if len(df_filtered) == 0:
                raise ValueError("No responses remaining after filtering out "
                                 "non-numeric human scores. No further analysis "
                                 "can be run. ")

            trim_min = given_trim_min if given_trim_min else df_filtered['sc1'].min()
            trim_max = given_trim_max if given_trim_max else df_filtered['sc1'].max()
        else:
            df_filtered = df_responses_with_requested_flags.copy()
            trim_min = given_trim_min if given_trim_min else 1
            trim_max = given_trim_max if given_trim_max else 10
            self.logger.info("Generating labels randomly "
                             "from [{}, {}]".format(trim_min, trim_max))
            randgen = RandomState(seed=1234567890)
            df_filtered[label_column] = randgen.random_integers(trim_min,
                                                                trim_max,
                                                                size=len(df_filtered))

        # make sure there are no missing features in the data
        missing_features = set(feature_names).difference(df_filtered.columns)
        if not missing_features:
            # make sure all features selected for model building are numeric
            # and also replace any non-numeric feature values in already
            # excluded data with NaNs for consistency
            for feat in feature_names:
                df_excluded[feat] = pd.to_numeric(df_excluded[feat],
                                                  errors='coerce').astype(float)
                newdf, newdf_excluded = self.filter_on_column(df_filtered,
                                                              feat,
                                                              'spkitemid',
                                                              exclude_zeros=False,
                                                              exclude_zero_sd=exclude_zero_sd)
                del df_filtered
                df_filtered = newdf
                with np.errstate(divide='ignore'):
                    df_excluded = pd.concat([df_excluded, newdf_excluded], sort=True)

            # make sure that the remaining data frame is not empty
            if len(df_filtered) == 0:
                raise ValueError("No responses remaining after filtering "
                                 "out non-numeric feature values. No further "
                                 "analysis can be run.")

            # Raise warning if we excluded features that were
            # specified in the .json file because sd == 0.
            omitted_features = set(requested_feature_names).difference(df_filtered.columns)
            if omitted_features:
                raise ValueError("The following requested features "
                                 "were excluded because their standard "
                                 "deviation on the training set was 0: {}.\n"
                                 "Please edit the feature file to exclude "
                                 "these features and re-run the "
                                 "tool".format(', '.join(omitted_features)))
            # Update the feature names
            feature_names = [feature for feature in feature_names
                             if feature in df_filtered]
        else:
            raise KeyError("DataFrame does not contain "
                           "columns for all features specified in "
                           "the feature file. Please check for "
                           "capitalization and other spelling "
                           "errors and make sure the feature "
                           "names do not contain hyphens. "
                           "The data does not have columns "
                           "for the following features: "
                           "{}".format(', '.join(missing_features)))

        # if ``length_column`` exists, make sure it's converted to numeric;
        # values that cannot be coerced to numeric will be set to ``np.nan``
        if length_column:
            df_filtered['length'] = pd.to_numeric(df_filtered['length'], errors='coerce')

        # check the values for length column. We do this after filtering
        # to make sure we have removed responses that have not been
        # processed correctly. Else rename length column to
        # ##ORIGINAL_NAME##.
        if (length_column and
            (len(df_filtered[df_filtered['length'].isnull()]) != 0 or
                df_filtered['length'].std() <= 0)):
            self.logger.warning("The {} column either has missing values or a standard "
                                "deviation <= 0. No length-based analysis will be "
                                "provided. The column will be renamed as ##{}## and "
                                "saved in *train_other_columns.csv.".format(length_column,
                                                                            length_column))
            df_filtered.rename(columns={'length': '##{}##'.format(length_column)},
                               inplace=True)

        # if requested, exclude the candidates with less than X responses
        # left after filtering
        if min_candidate_items:
            (df_filtered_candidates,
             df_excluded_candidates) = self.select_candidates(df_filtered, min_candidate_items)
            # check that there are still responses left for analysis
            if len(df_filtered_candidates) == 0:
                raise ValueError("After filtering non-numeric scores and "
                                 "non-numeric feature values there were "
                                 "no candidates with {} or more responses "
                                 "left for analysis".format(min_candidate_items))

            # redefine df_filtered
            df_filtered = df_filtered_candidates.copy()

            # update df_excluded
            df_excluded = pd.concat([df_excluded, df_excluded_candidates], sort=True)

        # create separate data frames for features and sc1, all other
        # information, and responses excluded during filtering
        not_other_columns = set()
        feature_columns = ['spkitemid', 'sc1'] + feature_names
        df_filtered_features = df_filtered[feature_columns]
        not_other_columns.update(feature_columns)

        metadata_columns = ['spkitemid'] + subgroups
        if candidate_column:
            metadata_columns.append('candidate')
        df_filtered_metadata = df_filtered[metadata_columns]
        not_other_columns.update(metadata_columns)

        df_filtered_length = pd.DataFrame()
        length_columns = ['spkitemid', 'length']
        if length_column and 'length' in df_filtered:
            df_filtered_length = df_filtered[length_columns]
            not_other_columns.update(length_columns)

        df_filtered_human_scores = pd.DataFrame()
        human_score_columns = ['spkitemid', 'sc1', 'sc2']
        if second_human_score_column and 'sc2' in df_filtered:
            df_filtered_human_scores = df_filtered[human_score_columns].copy()
            not_other_columns.update(['sc2'])

            # filter out any non-numeric value rows
            # as well as zeros, if we were asked to
            df_filtered_human_scores['sc2'] = pd.to_numeric(df_filtered_human_scores['sc2'],
                                                            errors='coerce').astype(float)
            if exclude_zero_scores:
                df_filtered_human_scores['sc2'] = df_filtered_human_scores['sc2'].replace(0,
                                                                                          np.nan)

        # we need to make sure that `spkitemid` is the first column
        df_excluded = df_excluded[['spkitemid'] + [column for column in df_excluded
                                                   if column != 'spkitemid']]

        # now extract all other columns and add 'spkitemid'
        other_columns = ['spkitemid'] + [column for column in df_filtered
                                         if column not in not_other_columns]
        df_filtered_other_columns = df_filtered[other_columns]

        return (df_filtered_features,
                df_filtered_metadata,
                df_filtered_other_columns,
                df_excluded,
                df_filtered_length,
                df_filtered_human_scores,
                df_responses_with_excluded_flags,
                trim_min,
                trim_max,
                feature_names)

    def process_data_rsmtool(self, config_obj, data_container_obj):
        """
        Set up rsmtool experiment by loading & preprocessing train/test data.

        This function takes a configuration object and a container object
        as input and returns the same types of objects as output after
        the loading, normalizing, and preprocessing.

        Parameters
        ----------
        config_obj : configuration_parser.Configuration
            A configuration object.
        data_container_obj : container.DataContainer
            A data container object.

        Returns
        -------
        config_obj : configuration_parser.Configuration
            A new configuration object.
        data_container : container.DataContainer
            A new data container object.

        Raises
        ------
        ValueError
            If columns specified in the configuration do not exist in the data.
        ValueError
            If the test label column and second human score columns have the
            same name.
        ValueError
            If the length column is requested as a feature.
        ValueError
            If the second human score column is requested as a feature.
        ValueError
            If "use_truncations" was specified in the configuration,
            but no feature CSV file was found.
        """
        train = data_container_obj.train
        test = data_container_obj.test
        feature_specs = data_container_obj.get_frame('feature_specs')
        feature_subset = data_container_obj.get_frame('feature_subset_specs')

        feature_specs_processor = FeatureSpecsProcessor()
        feature_subset_processor = FeatureSubsetProcessor()

        configdir = config_obj.configdir

        (test_file_location,
         train_file_location) = DataReader.locate_files([config_obj['test_file'],
                                                         config_obj['train_file']],
                                                        configdir)

        feature_subset_file = config_obj['feature_subset_file']

        if feature_subset_file is not None:
            feature_subset_file = DataReader.locate_files(feature_subset_file, configdir)

        # get the column name for the labels for the training and testing data
        train_label_column = config_obj['train_label_column']
        test_label_column = config_obj['test_label_column']

        # get the column name that will hold the ID for
        # both the training and the test data
        id_column = config_obj['id_column']

        # get the specified trim min, trim max and trim tolerance values
        (spec_trim_min,
         spec_trim_max,
         spec_trim_tolerance) = config_obj.get_trim_min_max_tolerance()

        # get the name of the optional column that
        # contains response length.
        length_column = config_obj['length_column']

        # get the name of the optional column that
        # contains the second human score
        second_human_score_column = config_obj['second_human_score_column']

        # get the name of the optional column that
        # contains the candidate ID
        candidate_column = config_obj['candidate_column']

        # if the test label column is the same as the
        # second human score column, raise an error
        if test_label_column == second_human_score_column:
            raise ValueError("'test_label_column' and "
                             "'second_human_score_column' cannot have the "
                             "same value.")

        # check if we are excluding candidates based on number of responses
        exclude_listwise = config_obj.check_exclude_listwise()
        min_items = config_obj['min_items_per_candidate']

        # get the name of the model that we want to train and
        # check that it's valid
        model_name = config_obj['model']
        model_type = self.check_model_name(model_name)

        # are we excluding zero scores?
        exclude_zero_scores = config_obj['exclude_zero_scores']

        # should we standardize the features
        standardize_features = config_obj['standardize_features']

        # if we are excluding zero scores but trim_min
        # is set to 0, then we need to warn the user
        if exclude_zero_scores and spec_trim_min == 0:
            self.logger.warning("'exclude_zero_scores' is set to True but "
                                "'trim_min' is set to 0. This may cause "
                                "unexpected behavior.")

        # are we filtering on any other columns?
        # is `flag_column` applied to training partition only
        # or both partitions?
        if 'flag_column_test' in config_obj:
            flag_partition = 'train'
        else:
            flag_partition = 'both'

        flag_column_dict = config_obj.check_flag_column(partition=flag_partition)
        flag_column_test_dict = config_obj.check_flag_column('flag_column_test',
                                                             partition='test')

        if (flag_column_dict and not flag_column_test_dict):
            flag_column_test_dict = flag_column_dict

        # are we generating fake labels?
        use_fake_train_labels = train_label_column == 'fake'
        use_fake_test_labels = test_label_column == 'fake'

        # are we using truncations from the feature specs?
        use_truncations = config_obj['use_truncation_thresholds']

        # get the subgroups if any
        subgroups = config_obj.get('subgroups')

        # are there specific general report sections we want to include?
        general_report_sections = config_obj['general_sections']

        # what about the special or custom sections?
        special_report_sections = config_obj['special_sections']

        custom_report_section_paths = config_obj['custom_sections']

        if custom_report_section_paths and configdir is not None:
            self.logger.info('Locating custom report sections')
            custom_report_sections = Reporter.locate_custom_sections(custom_report_section_paths,
                                                                     configdir)
        else:
            custom_report_sections = []

        section_order = config_obj['section_order']

        chosen_notebook_files = Reporter().get_ordered_notebook_files(general_report_sections,
                                                                      special_report_sections,
                                                                      custom_report_sections,
                                                                      section_order,
                                                                      subgroups,
                                                                      model_type=model_type,
                                                                      context='rsmtool')

        # Location of feature file
        feature_field = config_obj['features']

        feature_subset_field = config_obj['feature_subset']

        # if the user requested feature_subset file and feature subset,
        # read the file and check its format
        if feature_subset is not None and feature_subset_field:
            feature_subset_processor.check_feature_subset_file(feature_subset)

        # Do we need to automatically find the best transformations/change sign?
        select_transformations = config_obj['select_transformations']
        feature_sign = config_obj['sign']
        requested_features = []
        generate_feature_specs_automatically = True

        # if the feature field is a list, then simply
        # assign it to `requested_features`
        if isinstance(feature_field, list):
            requested_features = feature_field

        elif feature_field is not None:
            generate_feature_specs_automatically = False
            feature_specs = feature_specs_processor.validate_feature_specs(feature_specs,
                                                                           use_truncations)
            requested_features = feature_specs['feature'].tolist()

        # if we get to this point and both ``generate_feature_specs_automatically``
        # and ``use_truncations`` are True, then we need to raise an error
        if use_truncations and generate_feature_specs_automatically:
            raise ValueError('You have specified the ``use_truncations`` configuration '
                             'option, but a feature file could not be found.')

        # check to make sure that `length_column` or `second_human_score_column`
        # are not also included in the requested features, if they are specified
        if (length_column and
                length_column in requested_features):
            raise ValueError("The value of 'length_column' ('{}') cannot be "
                             "used as a model feature.".format(length_column))

        if (second_human_score_column and
                second_human_score_column in requested_features):
            raise ValueError("The value of 'second_human_score_column' ('{}') cannot be "
                             "used as a model feature.".format(second_human_score_column))

        # Specify column names that cannot be used as features
        reserved_column_names = list(set(['spkitemid', 'spkitemlab',
                                          'itemType', 'r1', 'r2', 'score',
                                          'sc', 'sc1', 'adj',
                                          train_label_column,
                                          test_label_column,
                                          id_column] + subgroups + list(flag_column_dict.keys())))

        # if `second_human_score_column` is specified, then
        # we need to add the original name as well as `sc2` to the list of reserved column
        # names. And same for 'length' and 'candidate', if `length_column`
        # and `candidate_column` are specified. We add both names to
        # simplify things downstream since neither the original name nor
        # the standardized name should be used as feature names
        if second_human_score_column:
            reserved_column_names.append(second_human_score_column)
            reserved_column_names.append('sc2')
        if length_column:
            reserved_column_names.append(length_column)
            reserved_column_names.append('length')
        if candidate_column:
            reserved_column_names.append(candidate_column)
            reserved_column_names.append('candidate')

        # remove duplicates (if any) from the list of reserved column names
        reserved_column_names = list(set(reserved_column_names))

        # Make sure that the training data as specified in the
        # config file actually exists on disk and if it does,
        # load it and filter out the bad rows and features with
        # zero standard deviation. Also double check that the requested
        # features exist in the data or obtain the feature names if
        # no feature file was given.
        (df_train_features,
         df_train_metadata,
         df_train_other_columns,
         df_train_excluded,
         df_train_length,
         _,
         df_train_flagged_responses,
         used_trim_min,
         used_trim_max,
         feature_names) = self.filter_data(train,
                                           train_label_column,
                                           id_column,
                                           length_column,
                                           None,
                                           candidate_column,
                                           requested_features,
                                           reserved_column_names,
                                           spec_trim_min,
                                           spec_trim_max,
                                           flag_column_dict,
                                           subgroups,
                                           exclude_zero_scores=exclude_zero_scores,
                                           exclude_zero_sd=True,
                                           feature_subset_specs=feature_subset,
                                           feature_subset=feature_subset_field,
                                           min_candidate_items=min_items,
                                           use_fake_labels=use_fake_train_labels)

        # Generate feature specifications now that we know what features to use
        if generate_feature_specs_automatically:
            if select_transformations:

                feature_specs = feature_specs_processor.generate_specs(df_train_features,
                                                                       feature_names,
                                                                       'sc1',
                                                                       feature_subset=feature_subset,
                                                                       feature_sign=feature_sign)
            else:
                feature_specs = feature_specs_processor.generate_default_specs(feature_names)

        # Do the same for the test data except we can ignore the trim min
        # and max since we already have that from the training data and
        # we have the feature_names when no feature file was specified.
        # We also allow features with 0 standard deviation in the test file.
        if (test_file_location == train_file_location and
                train_label_column == test_label_column):
            self.logger.warning("The same data file and label "
                                "column are used for both training "
                                "and evaluating the model. No second "
                                "score analysis will be performed, even "
                                "if requested.")

            df_test_features = df_train_features.copy()
            df_test_metadata = df_train_metadata.copy()
            df_test_excluded = df_train_excluded.copy()
            df_test_other_columns = df_train_other_columns.copy()
            df_test_flagged_responses = df_train_flagged_responses.copy()
            df_test_human_scores = pd.DataFrame()
        else:

            (df_test_features,
             df_test_metadata,
             df_test_other_columns,
             df_test_excluded,
             _,
             df_test_human_scores,
             df_test_flagged_responses,
             _, _, _) = self.filter_data(test,
                                         test_label_column,
                                         id_column,
                                         None,
                                         second_human_score_column,
                                         candidate_column,
                                         feature_names,
                                         reserved_column_names,
                                         used_trim_min,
                                         used_trim_max,
                                         flag_column_test_dict,
                                         subgroups,
                                         exclude_zero_scores=exclude_zero_scores,
                                         exclude_zero_sd=False,
                                         min_candidate_items=min_items,
                                         use_fake_labels=use_fake_test_labels)

        self.logger.info('Pre-processing training and test set features')
        (df_train_preprocessed_features,
         df_test_preprocessed_features,
         df_feature_info) = self.preprocess_features(df_train_features,
                                                     df_test_features,
                                                     feature_specs,
                                                     standardize_features,
                                                     use_truncations)

        # configuration options that either override previous values or are
        # entirely for internal use
        new_config_obj = config_obj.copy()
        internal_options_dict = {'chosen_notebook_files': chosen_notebook_files,
                                 'exclude_listwise': exclude_listwise,
                                 'feature_subset_file': feature_subset_file,
                                 'model_name': model_name,
                                 'model_type': model_type,
                                 'test_file_location': test_file_location,
                                 'train_file_location': train_file_location,
                                 'trim_min': used_trim_min,
                                 'trim_max': used_trim_max}

        for key, value in internal_options_dict.items():
            new_config_obj[key] = value

        new_container = [{'name': 'train_features',
                          'frame': df_train_features},
                         {'name': 'test_features',
                          'frame': df_test_features},
                         {'name': 'train_preprocessed_features',
                          'frame': df_train_preprocessed_features},
                         {'name': 'test_preprocessed_features',
                          'frame': df_test_preprocessed_features},
                         {'name': 'train_metadata', 'frame': df_train_metadata},
                         {'name': 'test_metadata', 'frame': df_test_metadata},
                         {'name': 'train_other_columns', 'frame': df_train_other_columns},
                         {'name': 'test_other_columns', 'frame': df_test_other_columns},
                         {'name': 'train_excluded', 'frame': df_train_excluded},
                         {'name': 'test_excluded', 'frame': df_test_excluded},
                         {'name': 'train_length', 'frame': df_train_length},
                         {'name': 'test_human_scores', 'frame': df_test_human_scores},
                         {'name': 'train_flagged', 'frame': df_train_flagged_responses},
                         {'name': 'test_flagged', 'frame': df_test_flagged_responses},
                         {'name': 'feature_specs', 'frame': feature_specs},
                         {'name': 'feature_info', 'frame': df_feature_info}]

        new_container = DataContainer(new_container)

        return new_config_obj, new_container

    def process_data_rsmeval(self, config_obj, data_container_obj):
        """
        Set up rsmeval experiment by loading & preprocessing evaluation data.

        This function takes a configuration object and a container object
        as input and returns the same types of objects as output after
        the loading, normalizing, and preprocessing.

        Parameters
        ----------
        config_obj : configuration_parser.Configuration
            A configuration object.
        data_container_obj : container.DataContainer
            A data container object.

        Returns
        -------
        config_obj : configuration_parser.Configuration
            A new configuration object.
        data_container : container.DataContainer
            A new data container object.

        Raises
        ------
        KeyError
            If columns specified in the configuration do not exist in the
            predictions file.
        ValueError
            If the columns containing the human scores and the system
            scores in the predictions file have the same name.
        ValueError
            If the columns containing the first set of human scores
            and the second set of human scores in the predictions file
            have the same name.
        ValueError
            If the predictions file contains the same response ID more than
            once.
        ValueError
            No responses were left after filtering out zero or non-numeric
            values for the various columns.
        """
        # get the directory where the config file lives
        configpath = config_obj.configdir
        pred_file_location = DataReader.locate_files(config_obj['predictions_file'],
                                                     configpath)

        # get the column name for the labels for the training and testing data
        human_score_column = config_obj['human_score_column']
        system_score_column = config_obj['system_score_column']

        # if the human score column is the same as the
        # system score column, raise an error
        if human_score_column == system_score_column:
            raise ValueError("'human_score_column' and "
                             "'system_score_column' "
                             "cannot have the same value.")

        # get the name of the optional column that
        # contains the second human score
        second_human_score_column = config_obj['second_human_score_column']

        # if the human score column is the same as the
        # second human score column, raise an error
        if human_score_column == second_human_score_column:
            raise ValueError("'human_score_column' and "
                             "'second_human_score_column' "
                             "cannot have the same value.")

        # get the column name that will hold the ID for
        # both the training and the test data
        id_column = config_obj['id_column']

        # get the specified trim min and max, if any
        # and make sure they are numeric
        (spec_trim_min,
         spec_trim_max,
         spec_trim_tolerance) = config_obj.get_trim_min_max_tolerance()

        # get the subgroups if any
        subgroups = config_obj.get('subgroups')

        # get the candidate column if any and convert it to string
        candidate_column = config_obj['candidate_column']

        # check if we are excluding candidates based on number of responses
        exclude_listwise = config_obj.check_exclude_listwise()
        min_items_per_candidate = config_obj['min_items_per_candidate']

        general_report_sections = config_obj['general_sections']

        # get any special sections that the user might have specified
        special_report_sections = config_obj['special_sections']

        # get any custom sections and locate them to make sure
        # that they exist, otherwise raise an exception
        custom_report_section_paths = config_obj['custom_sections']
        if custom_report_section_paths:
            self.logger.info('Locating custom report sections')
            custom_report_sections = Reporter.locate_custom_sections(custom_report_section_paths,
                                                                     configpath)
        else:
            custom_report_sections = []

        section_order = config_obj['section_order']

        # check all sections values and order and get the
        # ordered list of notebook files
        chosen_notebook_files = Reporter().get_ordered_notebook_files(general_report_sections,
                                                                      special_report_sections,
                                                                      custom_report_sections,
                                                                      section_order,
                                                                      subgroups,
                                                                      model_type=None,
                                                                      context='rsmeval')

        # are we excluding zero scores?
        exclude_zero_scores = config_obj['exclude_zero_scores']

        # if we are excluding zero scores but trim_min
        # is set to 0, then we need to warn the user
        if exclude_zero_scores and spec_trim_min == 0:
            self.logger.warning("'exclude_zero_scores' is set to True but "
                                " 'trim_min' is set to 0. This may cause "
                                " unexpected behavior.")

        # are we filtering on any other columns?
        flag_column_dict = config_obj.check_flag_column(partition='test')

        # do we have the training set predictions and human scores CSV file
        scale_with = config_obj.get('scale_with')

        # use scaled predictions for the analyses unless
        # we were told not to
        use_scaled_predictions = (scale_with is not None)

        # log an appropriate message
        if scale_with is None:
            message = ('Assuming given system predictions '
                       'are unscaled and will be used as such.')
        elif scale_with == 'asis':
            message = ('Assuming given system predictions '
                       'are already scaled and will be used as such.')
        else:
            message = ('Assuming given system predictions '
                       'are unscaled and will be scaled before use.')

        self.logger.info(message)

        df_pred = data_container_obj.predictions

        # make sure that the columns specified in the config file actually exist
        columns_to_check = [id_column, human_score_column, system_score_column]

        if second_human_score_column:
            columns_to_check.append(second_human_score_column)

        if candidate_column:
            columns_to_check.append(candidate_column)

        missing_columns = set(columns_to_check).difference(df_pred.columns)
        if missing_columns:
            raise KeyError('Columns {} from the config file do not exist '
                           'in the predictions file.'.format(missing_columns))

        df_pred = self.rename_default_columns(df_pred,
                                              [],
                                              id_column,
                                              human_score_column,
                                              second_human_score_column,
                                              None,
                                              system_score_column,
                                              candidate_column)

        # check that the id_column contains unique values
        if df_pred['spkitemid'].size != df_pred['spkitemid'].unique().size:
            raise ValueError("The data contains duplicate response IDs "
                             "in '{}'. Please make sure all response IDs "
                             "are unique and re-run the tool.".format(id_column))

        df_pred = self.check_subgroups(df_pred, subgroups)

        # filter out the responses based on flag columns
        (df_responses_with_requested_flags,
         df_responses_with_excluded_flags) = self.filter_on_flag_columns(df_pred,
                                                                         flag_column_dict)

        # filter out rows that have non-numeric or zero human scores
        df_filtered, df_excluded = self.filter_on_column(df_responses_with_requested_flags,
                                                         'sc1',
                                                         'spkitemid',
                                                         exclude_zeros=exclude_zero_scores)

        # make sure that the remaining data frame is not empty
        if len(df_filtered) == 0:
            raise ValueError("No responses remaining after filtering out "
                             "non-numeric human scores. No further analysis "
                             "can be run. ")

        # Change all non-numeric machine scores in excluded
        # data to NaNs for consistency with rsmtool.
        # NOTE: This will *not* work if *all* of the values
        # in column are non-numeric. This is a known bug in
        # pandas: https://github.com/pydata/pandas/issues/9589
        # Therefore, we need add an additional check after this.
        df_excluded['raw'] = pd.to_numeric(df_excluded['raw'], errors='coerce').astype(float)

        # filter out the non-numeric machine scores from the rest of the data
        newdf, newdf_excluded = self.filter_on_column(df_filtered,
                                                      'raw',
                                                      'spkitemid',
                                                      exclude_zeros=False)

        del df_filtered
        df_filtered_pred = newdf

        # make sure that the remaining data frame is not empty
        if len(df_filtered_pred) == 0:
            raise ValueError("No responses remaining after filtering out "
                             "non-numeric machine scores. No further analysis "
                             "can be run. ")

        with np.errstate(divide='ignore'):
            df_excluded = pd.concat([df_excluded, newdf_excluded], sort=True)

        # if requested, exclude the candidates with less than X responses
        # left after filtering
        if exclude_listwise:
            (df_filtered_candidates,
             df_excluded_candidates) = self.select_candidates(df_filtered_pred,
                                                              min_items_per_candidate)

            # check that there are still responses left for analysis
            if len(df_filtered_candidates) == 0:
                raise ValueError("After filtering non-numeric human and system scores "
                                 "there were "
                                 "no candidates with {} or more responses "
                                 "left for analysis".format(str(min_items_per_candidate)))

            # redefine df_filtered_pred
            df_filtered_pred = df_filtered_candidates.copy()

            # update df_excluded
            df_excluded = pd.concat([df_excluded, df_excluded_candidates], sort=True)
            df_excluded = df_excluded[['spkitemid'] + [column for column in df_excluded
                                                       if column != 'spkitemid']]

        # set default values for scaling
        scale_pred_mean = 0
        scale_pred_sd = 1
        scale_human_mean = 0
        scale_human_sd = 1

        if data_container_obj.get_frame('scale') is not None:
            if ('sc1' not in data_container_obj.scale.columns and
                    'prediction' not in data_container_obj.scale.columns):
                raise KeyError('The CSV file specified for scaling ',
                               'must have the "prediction" and the "sc1" '
                               'columns.')
            else:
                scale_pred_mean, scale_pred_sd = (data_container_obj.scale['prediction'].mean(),
                                                  data_container_obj.scale['prediction'].std())
                scale_human_mean, scale_human_sd = (data_container_obj.scale['sc1'].mean(),
                                                    data_container_obj.scale['sc1'].std())

        self.logger.info('Processing predictions')
        df_pred_processed = self.process_predictions(df_filtered_pred,
                                                     scale_pred_mean,
                                                     scale_pred_sd,
                                                     scale_human_mean,
                                                     scale_human_sd,
                                                     spec_trim_min,
                                                     spec_trim_max,
                                                     spec_trim_tolerance)
        if not scale_with:
            expected_score_types = ['raw', 'raw_trim', 'raw_trim_round']
        elif scale_with == 'asis':
            expected_score_types = ['scale', 'scale_trim', 'scale_trim_round']
        else:
            expected_score_types = ['raw', 'raw_trim', 'raw_trim_round',
                                    'scale', 'scale_trim', 'scale_trim_round']

        # extract separated data frames that we will write out
        # as separate files
        not_other_columns = set()

        prediction_columns = ['spkitemid', 'sc1'] + expected_score_types
        df_predictions_only = df_pred_processed[prediction_columns]
        not_other_columns.update(prediction_columns)

        metadata_columns = ['spkitemid'] + subgroups
        if candidate_column:
            metadata_columns.append('candidate')
        df_test_metadata = df_filtered_pred[metadata_columns]
        not_other_columns.update(metadata_columns)

        df_test_human_scores = pd.DataFrame()
        human_score_columns = ['spkitemid', 'sc1', 'sc2']
        if second_human_score_column and 'sc2' in df_filtered_pred:
            df_test_human_scores = df_filtered_pred[human_score_columns].copy()
            not_other_columns.update(['sc2'])
            # filter out any non-numeric values nows
            # as well as zeros, if we were asked to
            df_test_human_scores['sc2'] = pd.to_numeric(df_test_human_scores['sc2'],
                                                        errors='coerce').astype(float)
            if exclude_zero_scores:
                df_test_human_scores['sc2'] = df_test_human_scores['sc2'].replace(0, np.nan)

        # remove 'spkitemid' from `not_other_columns`
        # because we want that in the other columns
        # data frame
        not_other_columns.remove('spkitemid')

        # extract all of the other columns in the predictions file
        other_columns = [column for column in df_filtered_pred.columns
                         if column not in not_other_columns]

        df_pred_other_columns = df_filtered_pred[other_columns]

        # add internal configuration options that we need
        new_config_obj = config_obj.copy()
        internal_options_dict = {'pred_file_location': pred_file_location,
                                 'exclude_listwise': exclude_listwise,
                                 'use_scaled_predictions': use_scaled_predictions,
                                 'chosen_notebook_files': chosen_notebook_files}

        for key, value in internal_options_dict.items():
            new_config_obj[key] = value

        # we need to make sure that `spkitemid` is the first column
        df_excluded = df_excluded[['spkitemid'] + [column for column in df_excluded
                                                   if column != 'spkitemid']]

        frames = [df_predictions_only,
                  df_test_metadata,
                  df_pred_other_columns,
                  df_test_human_scores,
                  df_excluded,
                  df_responses_with_excluded_flags]

        names = ['pred_test',
                 'test_metadata',
                 'test_other_columns',
                 'test_human_scores',
                 'test_excluded',
                 'test_responses_with_excluded_flags']

        new_container = [{'name': name, 'frame': frame}
                         for frame, name in zip(frames, names)]

        new_container = DataContainer(new_container)

        return new_config_obj, new_container

    def process_data_rsmpredict(self, config_obj, data_container_obj):
        """
        Process data for rsmpredict experiments.

        This function takes a configuration object and a container object
        as input and returns the same types of objects as output after
        the loading, normalizing, and preprocessing.

        Parameters
        ----------
        config_obj : configuration_parser.Configuration
            A configuration object.
        data_container_obj : container.DataContainer
            A data container object.

        Returns
        -------
        config_obj : configuration_parser.Configuration
            A new configuration object.
        data_congtainer : container.DataContainer
            A new data container object.

        Raises
        ------
        KeyError
            If columns specified in the configuration do not exist in the data.
        ValueError
            If data contains duplicate response IDs.
        """
        df_input = data_container_obj.input_features
        df_feature_info = data_container_obj.feature_info
        df_postproc_params = data_container_obj.postprocessing_params

        # get the column name that will hold the ID
        id_column = config_obj['id_column']

        # get the column name for human score (if any)
        human_score_column = config_obj['human_score_column']

        # get the column name for second human score (if any)
        second_human_score_column = config_obj['second_human_score_column']

        # get the column name for subgroups (if any)
        subgroups = config_obj['subgroups']

        # get the model
        model = config_obj['model']

        # should features be standardized?
        standardize_features = config_obj.get('standardize_features', True)

        # should we predict expected scores
        predict_expected_scores = config_obj['predict_expected_scores']

        # get the column names for flag columns (if any)
        flag_column_dict = config_obj.check_flag_column(partition='test')

        # get the name for the candidate_column (if any)
        candidate_column = config_obj['candidate_column']

        # make sure that the columns specified in the config file actually exist
        columns_to_check = [id_column] + subgroups + list(flag_column_dict.keys())

        # add subgroups and the flag columns to the list of columns
        # that will be added to the final file
        columns_to_copy = subgroups + list(flag_column_dict.keys())

        # human_score_column will be set to sc1 by default
        # we only raise an error if it's set to something else.
        # However, since we cannot distinguish whether the column was set
        # to sc1 by default or specified as such in the config file
        # we append it to output anyway as long as
        # it is in the input file
        if human_score_column != 'sc1' or 'sc1' in df_input.columns:
            columns_to_check.append(human_score_column)
            columns_to_copy.append('sc1')

        if candidate_column:
            columns_to_check.append(candidate_column)
            columns_to_copy.append('candidate')

        if second_human_score_column:
            columns_to_check.append(second_human_score_column)
            columns_to_copy.append('sc2')

        missing_columns = set(columns_to_check).difference(df_input.columns)
        if missing_columns:
            raise KeyError("Columns {} from the config file "
                           "do not exist in the data.".format(missing_columns))

        # rename all columns
        df_input = self.rename_default_columns(df_input,
                                               [],
                                               id_column,
                                               human_score_column,
                                               second_human_score_column,
                                               None,
                                               None,
                                               candidate_column=candidate_column)

        # check that the id_column contains unique values
        if df_input['spkitemid'].size != df_input['spkitemid'].unique().size:
            raise ValueError("The data contains repeated response IDs in {}. "
                             "Please make sure all response IDs are unique and "
                             "re-run the tool.".format(id_column))

        (df_features_preprocessed,
         df_excluded) = self.preprocess_new_data(df_input,
                                                 df_feature_info,
                                                 standardize_features)

        trim_min = df_postproc_params['trim_min'].values[0]
        trim_max = df_postproc_params['trim_max'].values[0]
        h1_mean = df_postproc_params['h1_mean'].values[0]
        h1_sd = df_postproc_params['h1_sd'].values[0]

        # if we are using a newly trained model, use trim_tolerance from the
        # df_postproc_params. If not, set it to the default value and show
        # warning
        if 'trim_tolerance' in df_postproc_params:
            trim_tolerance = df_postproc_params['trim_tolerance'].values[0]
        else:
            trim_tolerance = 0.4998
            self.logger.warning("The tolerance for trimming scores will be assumed to be 0.4998, "
                                "the default value in previous versions of RSMTool. "
                                "We recommend re-training the model to ensure future "
                                "compatibility.")

        # now generate the predictions for the features using this model
        logged_str = 'Generating predictions'
        logged_str += ' (expected scores).' if predict_expected_scores else '.'
        self.logger.info(logged_str)

        # compute minimum and maximum score for expected predictions
        min_score = int(np.rint(trim_min - trim_tolerance))
        max_score = int(np.rint(trim_max + trim_tolerance))

        df_predictions = model.predict(df_features_preprocessed,
                                       min_score,
                                       max_score,
                                       predict_expected=predict_expected_scores)

        train_predictions_mean = df_postproc_params['train_predictions_mean'].values[0]
        train_predictions_sd = df_postproc_params['train_predictions_sd'].values[0]

        df_predictions = self.process_predictions(df_predictions,
                                                  train_predictions_mean,
                                                  train_predictions_sd,
                                                  h1_mean,
                                                  h1_sd,
                                                  trim_min, trim_max,
                                                  trim_tolerance)

        # add back the columns that we were requested to copy if any
        if len(columns_to_copy) > 0:
            df_predictions_with_metadata = pd.merge(df_predictions,
                                                    df_input[['spkitemid'] + columns_to_copy])
            assert(len(df_predictions) == len(df_predictions_with_metadata))
        else:
            df_predictions_with_metadata = df_predictions.copy()

        # we need to make sure that `spkitemid` is the first column
        df_excluded = df_excluded[['spkitemid'] + [column for column in df_excluded
                                                   if column != 'spkitemid']]

        datasets = [{'name': 'features_processed', 'frame': df_features_preprocessed},
                    {'name': 'excluded', 'frame': df_excluded},
                    {'name': 'predictions_with_metadata', 'frame': df_predictions_with_metadata},
                    {'name': 'predictions', 'frame': df_predictions}]

        return config_obj, DataContainer(datasets)

    def process_data(self, config_obj, data_container_obj, context='rsmtool'):
        """
        Process and setup the data for an experiment in the given context.

        Parameters
        ----------
        config_obj : configuration_parser.Configuration
            A configuration object.
        data_container_obj : container.DataContainer
            A data container object.
        context : str
            The tool context: one of {"rsmtool", "rsmeval", "rsmpredict"}.
            Defaults to "rsmtool".

        Returns
        -------
        config_obj : configuration_parser.Configuration
            A new configuration object.
        data_container : container.DataContainer
            A new data container object.

        Raises
        ------
        ValueError
            If the context is not one of {"rsmtool", "rsmeval", "rsmpredict"}.
        """
        if context == 'rsmtool':
            return self.process_data_rsmtool(config_obj, data_container_obj)
        elif context == 'rsmeval':
            return self.process_data_rsmeval(config_obj, data_container_obj)
        elif context == 'rsmpredict':
            return self.process_data_rsmpredict(config_obj, data_container_obj)
        else:
            raise ValueError("The 'context' argument must be in the set: "
                             "{'rsmtool', 'rsmeval', 'rsmpredict'}. "
                             "You specified `{}`.".format(context))

    def preprocess_new_data(self,
                            df_input,
                            df_feature_info,
                            standardize_features=True):
        """
        Preprocess feature values using the parameters in ``df_feature_info``.

        For more details on what these preprocessing parameters are,
        see :ref:`documentation <preprocessing_parameters>`.

        Parameters
        ----------
        df_input : pandas DataFrame
            Data frame with raw feature values that will be used to generate
            the scores. Each feature is stored in a separate column. Each row
            corresponds to one response. There should also be a column named
            "spkitemid" containing a unique ID for each response.
        df_feature_info : pandas DataFrame
            Data frame with preprocessing parameters in the following columns:

            - "feature" : the name of the feature; should match the feature names
              in ``df_input``.
            - "sign" : 1 or -1.  Indicates whether the feature value needs to
              be multiplied by -1.
            - "transform" : :ref:`transformation <select_transformations_rsmtool>`
              that needs to be applied to this feature.
            - "train_mean", "train_sd" : mean and standard deviation for outlier
              truncation.
            - "train_transformed_mean", "train_transformed_sd" : mean and standard
              deviation for computing z-scores.

        standardize_features : bool, optional
            Whether the features should be standardized prior to prediction.
            Defaults to ``True``.

        Returns
        -------
        df_features_preprocessed : pandas DataFrame
            Data frame with processed feature values.
        df_excluded: pandas DataFrame
            Data frame with responses excluded from further analysis
            due to non-numeric feature values in the original file
            or after applying transformations. This data frame always
            contains the original feature values.

        Raises
        ------
        KeyError
            if some of the features specified in ``df_feature_info`` are
            not present in ``df_input``.
        ValueError
            If all responses have at least one non-numeric feature value
            and, therefore,  no score can be generated for any of the
            responses.
        """
        # get the list of required features
        required_features = df_feature_info.index.tolist()

        # ensure that all the features that are needed by the model
        # are present in the input file
        input_feature_columns = [c for c in df_input if c != 'spkitemid']
        missing_features = set(required_features).difference(input_feature_columns)
        if missing_features:
            raise KeyError('The input feature file is missing the '
                           'following features: {}'.format(missing_features))

        extra_features = set(input_feature_columns).difference(required_features + ['spkitemid'])
        if extra_features:
            self.logger.warning("The following extraneous features "
                                "will be ignored: {}".format(extra_features))

        # keep the required features plus the id
        features_to_keep = ['spkitemid'] + required_features

        # check if actually have the human scores for this data and add
        # sc1 to preprocessed features for consistency with other tools
        has_human_scores = 'sc1' in df_input
        if has_human_scores:
            features_to_keep.append('sc1')

        df_features = df_input[features_to_keep]

        # preprocess the feature values
        self.logger.info('Pre-processing input features')

        # first we need to filter out NaNs and any other
        # weird features, the same way we did for rsmtool.
        df_filtered = df_features.copy()
        df_excluded = pd.DataFrame(columns=df_filtered.columns)

        for feature_name in required_features:
            newdf, newdf_excluded = self.filter_on_column(df_filtered,
                                                          feature_name,
                                                          'spkitemid',
                                                          exclude_zeros=False,
                                                          exclude_zero_sd=False)
            del df_filtered
            df_filtered = newdf
            with np.errstate(divide='ignore'):
                df_excluded = pd.concat([df_excluded, newdf_excluded], sort=True)

        # make sure that the remaining data frame is not empty
        if len(df_filtered) == 0:
            raise ValueError("There are no responses left after "
                             "filtering out non-numeric feature values. No analysis "
                             "will be run")

        df_features = df_filtered.copy()
        df_features_preprocess = df_features.copy()
        for feature_name in required_features:

            feature_values = df_features_preprocess[feature_name].values

            feature_transformation = df_feature_info.loc[feature_name]['transform']
            feature_sign = df_feature_info.loc[feature_name]['sign']

            train_feature_mean = df_feature_info.loc[feature_name]['train_mean']
            train_feature_sd = df_feature_info.loc[feature_name]['train_sd']

            train_transformed_mean = df_feature_info.loc[feature_name]['train_transformed_mean']
            train_transformed_sd = df_feature_info.loc[feature_name]['train_transformed_sd']

            # transform the feature values and remove outliers
            df_features_preprocess[feature_name] = self.preprocess_feature(feature_values,
                                                                           feature_name,
                                                                           feature_transformation,
                                                                           train_feature_mean,
                                                                           train_feature_sd,
                                                                           exclude_zero_sd=False,
                                                                           raise_error=False)

            # filter the feature values once again to remove possible NaN and inf values that
            # might have emerged when applying transformations.
            # We do not need to do that if no transformation was applied.
            if feature_transformation not in ['raw', 'org']:
                # check that there are indeed inf or Nan values
                if np.isnan(df_features_preprocess[feature_name]).any() or \
                   np.isinf(df_features_preprocess[feature_name]).any():
                    (newdf,
                     newdf_excluded) = self.filter_on_column(df_features_preprocess,
                                                             feature_name,
                                                             'spkitemid',
                                                             exclude_zeros=False,
                                                             exclude_zero_sd=False)
                    del df_features_preprocess
                    df_features_preprocess = newdf

                    # add the response(s) with missing values to the excluded responses
                    # but make sure we are adding the original values, not the
                    # preprocessed ones
                    missing_values = df_features['spkitemid'].isin(newdf_excluded['spkitemid'])

                    df_excluded_original = df_features[missing_values].copy()
                    df_excluded = pd.merge(df_excluded, df_excluded_original, how='outer')

            # print(standardized_features)
            if standardize_features:

                # now standardize the feature values
                df_feature_minus_mean = (df_features_preprocess[feature_name] -
                                         train_transformed_mean)
                df_features_preprocess[feature_name] = (df_feature_minus_mean /
                                                        train_transformed_sd)

            # Multiply features by sign.
            df_features_preprocess[feature_name] = (df_features_preprocess[feature_name] *
                                                    feature_sign)

        # we need to make sure that `spkitemid` is the first column
        df_excluded = df_excluded[['spkitemid'] + [column for column in df_excluded
                                                   if column != 'spkitemid']]

        return (df_features_preprocess, df_excluded) class DataReader:
    """Class to generate DataContainer objects."""

    def __init__(self,
                 filepaths,
                 framenames,
                 file_converters=None):
        """
        Initialize a DataReader object.

        Parameters
        ----------
        filepaths : list of str
            A list of paths to files that are to be read in.
        framenames : list of str
            A list of names for the data sets to be included in the container.
        file_converters : dict of dicts, optional
            A dictionary of file converter dicts. The keys are the data set
            names and the values are the converter dictionaries to be applied
            to the corresponding data set.
            Defaults to ``None``.

        Raises
        ------
        AssertionError
            If ``len(filepaths)`` does not equal ``len(framenames)``.
        ValueError
            If ``file_converters`` is not a dictionary or if any of its
            values is not a dictionary.
        NameError
            If a key in ``file_converters`` does not exist in ``framenames``.
        ValueError
            If any of the specified file paths is ``None``.
        """
        # Default datasets list
        self.datasets = []

        # Make sure filepaths length matches frame names length
        assert len(filepaths) == len(framenames)

        # Make sure that there are no Nones in the filepaths
        if None in filepaths:
            frames_with_no_path = [framenames[i] for i in range(len(framenames))
                                   if filepaths[i] is None]

            raise ValueError("No path specified for "
                             "{}".format(' ,'.join(frames_with_no_path)))

        # Assign names and paths lists
        self.dataset_names = framenames
        self.dataset_paths = filepaths

        # If `file_converters` exists, then
        # check to make sure it is the correct length
        # and add all elements to `file_converters` list
        if file_converters is not None:
            if not isinstance(file_converters, dict):
                raise ValueError("The 'file_converters' argument must be a `dict`, "
                                 "not `{}`.".format(type(file_converters)))

            for file_converter_name in file_converters:
                # Make sure file_converter name is in `dataset_names`
                if file_converter_name not in self.dataset_names:
                    raise NameError('The file converter name ``{}`` '
                                    'does not exist in the '
                                    'dataset names that you '
                                    'passed.'.format(file_converter_name))

                # Make sure file converter is a `dict`
                file_converter = file_converters[file_converter_name]
                if not isinstance(file_converter, dict):
                    raise ValueError('Value for {} must be``dict`` ',
                                     'not {}'.format(file_converter_name,
                                                     type(file_converter)))

        # Default file_converters dict
        self.file_converters = {} if file_converters is None else file_converters

    @staticmethod
    def read_from_file(filename, converters=None, **kwargs):
        """
        Read a CSV/TSV/XLSX/JSONLINES/SAS7BDAT file and return a data frame.

        Parameters
        ----------
        filename : str
            Name of file to read.
        converters : dict, optional
            A dictionary specifying how the types of the columns
            in the file should be converted. Specified in the same
            format as for `pd.read_csv() <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html>`_.
            Defaults to ``None``.

        Returns
        -------
        df : pandas DataFrame
            Data frame containing the data in the given file.

        Raises
        ------
        ValueError
            If the file has an unsuppored extension.
        pandas.errors.ParserError
            If the file is badly formatted or corrupt.

        Note
        ----
        Any additional keyword arguments are passed to the underlying
        pandas IO reader function.
        """
        file_extension = splitext(filename)[1].lower()

        if file_extension in ['.csv', '.tsv']:
            sep = '\t' if file_extension == '.tsv' else ','
            do_read = partial(pd.read_csv, sep=sep, converters=converters)
        elif file_extension == '.xlsx':
            do_read = partial(pd.read_excel, converters=converters)
        elif file_extension == '.sas7bdat':
            if 'encoding' not in kwargs:
                encoding = 'latin-1'
            else:
                encoding = kwargs.pop('encoding')
            do_read = partial(pd.read_sas, encoding=encoding)
        elif file_extension in ['.jsonlines']:
            do_read = partial(read_jsonlines, converters=converters)
        else:
            raise ValueError("RSMTool only supports files in .csv, "
                             ".tsv, .xlsx, and .sas7bdat formats. "
                             "Input files should have one of these "
                             "extensions. The file you "
                             "passed is: {}.".format(filename))

        # ignore warnings about mixed data types for large files
        with warnings.catch_warnings():
            warnings.filterwarnings('ignore', category=DtypeWarning)
            try:
                df = do_read(filename, **kwargs)
            except pd.errors.ParserError:
                raise pd.errors.ParserError('Cannot read {}. Please check that it is '
                                            'not corrupt or in an incompatible format. '
                                            '(Try running dos2unix?)'.format(filename))
        return df

    @staticmethod
    def locate_files(filepaths, configdir):
        """
        Locate an experiment file, or a list of experiment files.

        If the given path doesn't exist, then maybe the path is relative
        to the path of the config file. If neither exists, then return
        ``None``.

        Parameters
        ----------
        filepaths : str or list
            Name(s) of the experiment file we want to locate.
        configdir : str
            Path to the reference configuration directory
            (usually the directory of the config file)

        Returns
        --------
        retval :  str or list
            Absolute path to the experiment file or ``None``
            if the file could not be located. If ``filepaths``
            was a string, this method will return a string.
            Otherwise, it will return a list.

        Raises
        ------
        ValueError
            If ``filepaths`` is not a string or a list.
        """
        # the feature config file can be in the 'feature' directory
        # at the same level as the main config file
        if not (isinstance(filepaths, str) or
                isinstance(filepaths, list)):
            raise ValueError("The 'filepaths' argument must be a "
                             "string or a list, not {}.".format(type(filepaths)))

        if isinstance(filepaths, str):
            filepaths = [filepaths]
            return_string = True
        else:
            return_string = False

        located_paths = []
        for filepath in filepaths:

            retval = None
            alternate_path = abspath(join(configdir, filepath))

            # if the given path exists as is, convert
            # that to an absolute path and return
            if exists(filepath):
                retval = abspath(filepath)

            # otherwise check if it exists relative
            # to the reference directory
            elif exists(alternate_path):
                retval = alternate_path

            located_paths.append(retval)

        if return_string:
            return located_paths[0]

        return located_paths

    def read(self, kwargs_dict=None):
        """
        Read all files contained in ``self.dataset_paths``.

        Parameters
        ----------
        kwargs_dict : dict of dicts, optional
            Any additional keyword arguments to pass to a particular DataFrame.
            These arguments will be passed to the pandas IO reader function.
            Defaults to ``None``.

        Returns
        -------
        datacontainer : container.DataContainer
            A data container object.

        Raises
        ------
        FileNotFoundError
            If any of the files in ``self.dataset_paths`` does not exist.
        """
        for idx, set_path in enumerate(self.dataset_paths):

            name = self.dataset_names[idx]
            converter = self.file_converters.get(name, None)

            if not exists(set_path):
                raise FileNotFoundError('The file {} does not exist'.format(set_path))

            if kwargs_dict is not None:
                kwargs = kwargs_dict.get(name, {})
            else:
                kwargs = {}

            dataframe = self.read_from_file(set_path, converter, **kwargs)

            # Add to list of datasets
            self.datasets.append({'name': name.strip(),
                                  'path': set_path,
                                  'frame': dataframe})

        return DataContainer(self.datasets) class Reporter:
    """Class to generate Jupyter notebook reports and convert them to HTML."""

    def __init__(self, logger=None):
        self.logger = logger if logger else logging.getLogger(__name__)

    @staticmethod
    def locate_custom_sections(custom_report_section_paths, configdir):
        """
        Locate custom report section files.

        Get the absolute paths for custom report sections and check that
        the files exist. If a file does not exist, raise an exception.

        Parameters
        ----------
        custom_report_section_paths : list of str
            List of paths to IPython notebook
            files representing the custom sections.
        configdir : str
            Path to the experiment configuration directory.

        Returns
        -------
        custom_report_sections :  list of str
            List of absolute paths to the custom section
            notebooks.

        Raises
        ------
        FileNotFoundError
            If any of the files cannot be found.
        """
        custom_report_sections = []
        for cs_path in custom_report_section_paths:
            cs_location = DataReader.locate_files(cs_path, configdir)
            if not cs_location:
                raise FileNotFoundError("Error: custom section not found at "
                                        "{}.".format(cs_path))
            else:
                custom_report_sections.append(cs_location)
        return custom_report_sections

    @staticmethod
    def merge_notebooks(notebook_files, output_file):
        """
        Merge the given Jupyter notebooks into a single Jupyter notebook.

        Parameters
        ----------
        notebook_files : list of str
            List of paths to the input Jupyter notebook files.
        output_file : str
            Path to output Jupyter notebook file

        Note
        ----
        Adapted from: https://stackoverflow.com/q/20454668.
        """
        # Merging ipython notebooks basically means that we keep the
        # metadata from the "first" notebook and then add in the cells
        # from all the other notebooks.
        first_notebook = notebook_files[0]
        merged_notebook = json.loads(open(first_notebook, 'r', encoding='utf-8').read())
        for notebook in notebook_files[1:]:
            section_cells = json.loads(open(notebook, 'r', encoding='utf-8').read())["cells"]
            merged_notebook['cells'].extend(section_cells)

        # output the merged cells into a report
        with open(output_file, 'w') as outf:
            json.dump(merged_notebook, outf, indent=1)

    @staticmethod
    def check_section_names(specified_sections,
                            section_type,
                            context='rsmtool'):
        """
        Validate the specified section names.

        This function checks whether the specified section names are valid
        and raises an exception if they are not.

        Parameters
        ----------
        specified_sections : list of str
            List of report section names.
        section_type : str
            One of "general" or "special".
        context : str, optional
            Context in which we are validating the section
            names. One of {"rsmtool", "rsmeval", "rsmcompare"}.
            Defaults to "rsmtool".

        Raises
        ------
        ValueError
            If any of the section names of the given type
            are not valid in the context of the given tool.
        """
        master_section_list = master_section_dict[section_type][context]
        invalid_section_names = set(specified_sections).difference(master_section_list)
        if invalid_section_names:
            raise ValueError("The following {} report section "
                             "names are invalid or not supported for {}: {}\n"
                             "The following sections are currently "
                             "available: {}".format(section_type,
                                                    context,
                                                    invalid_section_names,
                                                    master_section_list))

    @staticmethod
    def check_section_order(chosen_sections, section_order):
        """
        Check the order of the specified sections.

        Parameters
        ----------
        chosen_sections : list of str
            List of chosen section names.
        section_order : list of str
            An ordered list of the chosen section names.

        Raises
        ------
        ValueError
            If any sections specified in the order are missing
            from the list of chosen sections or vice versa.
        """
        if sorted(chosen_sections) != sorted(section_order):

            # check for discrepancies and create a helpful error message
            missing_sections = set(chosen_sections).difference(set(section_order))
            if missing_sections:
                error_message_missing = ("'section_order' must list all "
                                         "sections selected for your experiment: "
                                         "Please edit section order to include the "
                                         "following missing sections: "
                                         "{}".format(', '.join(missing_sections)))

            extra_sections = set(section_order).difference(set(chosen_sections))
            if extra_sections:
                error_message_extra = ("'section order' can only include "
                                       "sections availabe for this experiment. "
                                       "The following sections are either unavailable "
                                       "or were not selected for this experiment "
                                       "{}".format(', '.join(extra_sections)))

            # raise an appropriate error message or a combination of messages
            if missing_sections and not extra_sections:
                raise ValueError(error_message_missing)
            elif extra_sections and not missing_sections:
                raise ValueError(error_message_extra)
            else:
                raise ValueError("{}\n{}".format(error_message_missing,
                                                 error_message_extra))

    @staticmethod
    def convert_ipynb_to_html(notebook_file, html_file):
        """
        Convert given Jupyter notebook (``.ipynb``) to HTML file.

        Parameters
        ----------
        notebook_file : str
            Path to input Jupyter notebook file.
        html_file : str
            Path to output HTML file.

        Note
        ----
        This function is also exposed as the
        :ref:`render_notebook <render_notebook>` command-line utility.
        """
        # `nbconvert` uses `asyncio` which uses an entirely default
        # implemention of the event loop on Windows for Cpython 3.8
        # which breaks the report generation unless we include the
        # following workaround
        if (sys.version_info[0] == 3 and
                sys.version_info[1] >= 8 and
                sys.platform.startswith('win')):
            asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())

        # set a high timeout for datasets with a large number of features
        report_config = Config({'ExecutePreprocessor': {'enabled': True,
                                                        'timeout': 3600},
                                'HTMLExporter': {"template_name": "classic",
                                                 "template_file": join(template_path,
                                                                       'report.tpl')}})

        exportHtml = HTMLExporter(config=report_config)
        output, _ = exportHtml.from_filename(notebook_file)
        open(html_file, mode='w', encoding='utf-8').write(output)

    def determine_chosen_sections(self,
                                  general_sections,
                                  special_sections,
                                  custom_sections,
                                  subgroups,
                                  context='rsmtool'):
        """
        Compile a combined list of section names to be included in the report.

        Parameters
        ----------
        general_sections : list of str
            List of specified general section names.
        special_sections : str
            List of specified special section names, if any.
        custom_sections : list of str
            List of specified custom sections, if any.
        subgroups : list of str
            List of column names that contain grouping
            information.
        context : str, optional
            Context of the tool in which we are validating.
            One of  {"rsmtool", "rsmeval", "rsmcompare"}
            Defaults to "rsmtool".

        Returns
        -------
        chosen_sections : list of str
            Final list of chosen sections that are to
            be included in the HTML report.

        Raises
        ------
        ValueError
            If a subgroup report section is requested but no
            subgroups were specified in the configuration file.
        """
        # 1. Include all general sections unless we are asked to include
        # a specific (and valid) subset.
        general_section_list = master_section_dict['general'][context]
        chosen_general_sections = general_section_list
        all_general_sections = True

        if general_sections != ['all']:
            self.check_section_names(general_sections, 'general', context)
            chosen_general_sections = [s for s in general_sections
                                       if s in general_section_list]
            all_general_sections = False

        # 2. Exclude the subgroup sections if we do not have subgroup information.

        if len(subgroups) == 0:
            subgroup_sections = [section for section in chosen_general_sections
                                 if section.endswith('by_group') or section == 'fairness_analyses']
            # if we were given a list of general sections, raise an error if
            # that list included subgroup sections but no subgroups were specified

            if not all_general_sections and len(subgroup_sections) != 0:
                raise ValueError("You requested sections for subgroup analysis "
                                 "but did not specify any subgroups. "
                                 "Please amend the config files to define "
                                 "the subgroups or delete the following "
                                 "sections from the list of sections: {}"
                                 .format(', '.join(subgroup_sections)))

            # if we are using the default list, we simply remove the
            # subgroup sections
            chosen_general_sections = [section for section in chosen_general_sections
                                       if section not in subgroup_sections]

        # 3. Include the specified (and valid) subset of the special sections
        chosen_special_sections = []
        if special_sections:
            special_section_list = master_section_dict['special'][context]
            self.check_section_names(special_sections, 'special', context=context)
            chosen_special_sections = [s for s in special_sections
                                       if s in special_section_list]

        # 4. For the custom sections use the basename and strip off the `.ipynb` extension
        chosen_custom_sections = []
        if custom_sections:
            chosen_custom_sections = [splitext(basename(cs))[0] for cs in custom_sections]

        # return the final list of chosen sections
        chosen_sections = (chosen_general_sections +
                           chosen_special_sections +
                           chosen_custom_sections)

        return chosen_sections

    def get_section_file_map(self,
                             special_sections,
                             custom_sections,
                             model_type=None,
                             context='rsmtool'):
        """
        Map section names to IPython notebook filenames.

        Parameters
        ----------
        special_sections : list of str
            List of special sections.
        custom_sections : list of str
            List of custom sections.
        model_type : str, optional
            Type of the model. One of {"BUILTIN", "SKLL", ``None``}.
            We allow ``None`` here so that rsmeval can use the same
            function.
            Defaults to ``None``.
        context : str, optional
            Context of the tool in which we are validating.
            One of {"rsmtool", "rsmeval", "rsmcompare"}.
            Defaults to "rsmtool".

        Returns
        -------
        section_file_map : dict
            Dictionary mapping each section name to the
            corresponding IPython notebook filename.
        """
        # create the original section file map for general sections
        selected_notebook_path = notebook_path_dict['general'][context]
        general_sections = master_section_dict['general'][context]

        section_file_map = {s: join(selected_notebook_path, '{}.ipynb'.format(s))
                            for s in general_sections + ['header', 'footer']}

        # update the file map to point the 'model section to either the built-in
        # or the SKLL model notebook depending on the model type that
        # was passed in
        if context == 'rsmtool' and model_type:
            section_file_map['model'] = join(selected_notebook_path,
                                             '{}_model.ipynb'.format(model_type.lower()))

        # update the file map to include the special sections
        if special_sections:
            selected_special_notebook_path = notebook_path_dict['special'][context]
            section_file_map.update({ss: join(selected_special_notebook_path,
                                              "{}.ipynb".format(ss))
                                     for ss in special_sections})

        # update the file map to include the custom sections with
        # the file names (without the `.ipynb` extension) as the
        # names (keys) and full paths as values
        if custom_sections:
            section_file_map.update({splitext(basename(cs))[0]: cs
                                     for cs in custom_sections})

        return section_file_map

    def get_ordered_notebook_files(self,
                                   general_sections,
                                   special_sections=[],
                                   custom_sections=[],
                                   section_order=None,
                                   subgroups=[],
                                   model_type=None,
                                   context='rsmtool'):
        """
        Check all section names and the order of the sections.

        Combine all section names with the appropriate file mapping,
        and generate an ordered list of notebook files that are
        needed to generate the final report.

        Parameters
        ----------
        general_sections : str
            List of specified general sections.
        special_sections : list, optional
            List of specified special sections, if any.
            Defaults to ``[]``.
        custom_sections : list, optional
            List of specified custom sections, if any.
            Defaults to ``[]``.
        section_order : list, optional
            Ordered list in which the user wants the specified
            sections.
            Defaults to ``None``.
        subgroups : list, optional
            List of column names that contain grouping
            information.
            Defaults to ``[]``.
        model_type : None, optional
            Type of the model. Possible values are
            {"BUILTIN", "SKLL", ``None``.}. We allow ``None``
            here so that rsmeval can use the same function.
            Defaults to ``None``.
        context : str, optional
            Context of the tool in which we are validating.
            One of {"rsmtool", "rsmeval", "rsmcompare"}.
            Defaults to "rsmtool".

        Returns
        -------
        chosen_notebook_files : list of str
            List of the IPython notebook files that have
            to be rendered into the HTML report.
        """
        chosen_sections = self.determine_chosen_sections(general_sections,
                                                         special_sections,
                                                         custom_sections,
                                                         subgroups,
                                                         context=context)

        # check to make sure that if a custom section ordering is
        # specified by the user, that it actually contains
        # *all* of the sections that have been chosen for the
        # final report.
        if section_order:
            self.check_section_order(chosen_sections, section_order)

        # determine which order to use by default
        if context == 'rsmtool':
            ordered_section_list = ordered_section_list_rsmtool
        elif context == 'rsmeval':
            ordered_section_list = ordered_section_list_rsmeval
        elif context == 'rsmcompare':
            ordered_section_list = ordered_section_list_rsmcompare
        elif context == 'rsmsummarize':
            ordered_section_list = ordered_section_list_rsmsummarize

        # add all custom sections to the end of the default ordered list
        ordered_section_list.extend([splitext(basename(cs))[0] for cs in custom_sections])

        # get the section file map
        section_file_map = self.get_section_file_map(special_sections,
                                                     custom_sections,
                                                     model_type,
                                                     context=context)

        # order the section list either according to the default
        # order in `ordered_section_list` or according to the custom
        # order that has been passed in via `section_order`
        order_to_use = section_order if section_order else ordered_section_list
        chosen_sections = [s for s in order_to_use if s in chosen_sections]

        # add the header and the footer to the chosen sections
        chosen_sections = ['header'] + chosen_sections + ['footer']
        chosen_notebook_files = [section_file_map[cs] for cs in chosen_sections]

        return chosen_notebook_files

    def create_report(self,
                      config,
                      csvdir,
                      figdir,
                      context='rsmtool'):
        """
        Generate HTML report for an rsmtool/rsmeval experiment.

        Parameters
        ----------
        config : configuration_parser.Configuration
            A configuration object
        csvdir : str
            The CSV output directory.
        figdir : str
            The figure output directory
        context : str
            Context of the tool in which we are validating.
            One of {"rsmtool", "rsmeval"}.
            Defaults to "rsmtool".

        Raises
        ------
        KeyError
            If "test_file_location" or "pred_file_location" fields
            are not specified in the configuration.
        """
        test_file_location = config.get('test_file_location')
        if test_file_location is None:
            test_file_location = config.get('pred_file_location')

        # raise error if location is still None
        if test_file_location is None:
            raise KeyError('Could not find `test_file_location` or `pred_file_location` '
                           'in Configuration object. Please make sure you have included '
                           'one of these parameters in the configuration object.')

        # if the features subset file is None, just use empty string
        feature_subset_file = ('' if config.get('feature_subset_file') is None
                               else config['feature_subset_file'])

        # if the min items is None, just set it to zero
        min_items = (0 if config['min_items_per_candidate'] is None
                     else config['min_items_per_candidate'])

        # determine minimum and maximum scores for trimming
        (used_trim_min,
         used_trim_max,
         trim_tolerance) = config.get_trim_min_max_tolerance()
        min_score = used_trim_min - trim_tolerance
        max_score = used_trim_max + trim_tolerance

        environ_config = {'EXPERIMENT_ID': config['experiment_id'],
                          'DESCRIPTION': config['description'],
                          'MODEL_TYPE': config.get('model_type', ''),
                          'MODEL_NAME': config.get('model_name', ''),
                          'TRAIN_FILE_LOCATION': config.get('train_file_location', ''),
                          'TEST_FILE_LOCATION': test_file_location,
                          'SUBGROUPS': config.get('subgroups', []),
                          'GROUPS_FOR_DESCRIPTIVES': config.get('subgroups', []),
                          'GROUPS_FOR_EVALUATIONS': config.get('subgroups', []),
                          'MIN_N_PER_GROUP': config.get('min_n_per_group', {}),
                          'LENGTH_COLUMN': config.get('length_column', None),
                          'H2_COLUMN': config['second_human_score_column'],
                          'MIN_ITEMS': min_items,
                          'FEATURE_SUBSET_FILE': feature_subset_file,
                          'EXCLUDE_ZEROS': config.get('exclude_zero_scores', True),
                          'SCALED': config.get('use_scaled_predictions', False),
                          'MIN_SCORE': min_score,
                          'MAX_SCORE': max_score,
                          'STANDARDIZE_FEATURES': config.get('standardize_features', True),
                          'FILE_FORMAT': config.get('file_format', 'csv'),
                          'USE_THUMBNAILS': config.get('use_thumbnails', False),
                          'SKLL_FIXED_PARAMETERS': config.get('skll_fixed_parameters', {}),
                          'SKLL_OBJECTIVE': config.get('skll_objective', ''),
                          'PREDICT_EXPECTED_SCORES': config.get('predict_expected_scores', False),
                          'RATER_ERROR_VARIANCE': config.get('rater_error_variance', None),
                          'CONTEXT': context,
                          'JAVASCRIPT_PATH': javascript_path,
                          'OUTPUT_DIR': csvdir,
                          'FIGURE_DIR': figdir
                          }

        # get the report directory which is at the same level
        # as the output and the figure directory
        reportdir = abspath(join(csvdir, '..', 'report'))
        report_name = '{}_report'.format(config['experiment_id'])
        merged_notebook_file = join(reportdir, '{}.ipynb'.format(report_name))
        environ_config_file = join(reportdir, '.environ.json')

        # set the report directory as an environment variable
        os.environ['RSM_REPORT_DIR'] = reportdir

        # write out hidden environment JSON file
        with open(environ_config_file, 'w') as out_environ_config:
            json.dump(environ_config, out_environ_config)

        # merge all the given sections
        self.logger.info('Merging sections')
        self.merge_notebooks(config['chosen_notebook_files'], merged_notebook_file)

        # run the merged notebook and save the output as
        # an HTML file in the report directory
        self.logger.info('Exporting HTML')
        self.convert_ipynb_to_html(merged_notebook_file,
                                   join(reportdir, '{}.html'.format(report_name)))

    def create_comparison_report(self,
                                 config,
                                 csvdir_old,
                                 figdir_old,
                                 csvdir_new,
                                 figdir_new,
                                 output_dir):
        """
        Generate an HTML report for comparing two rsmtool experiments.

        Parameters
        ----------
        config : configuration_parser.Configuration
            A configuration object
        csvdir_old : str
            The old experiment CSV output directory.
        figdir_old : str
            The old figure output directory
        csvdir_new : str
            The new experiment CSV output directory.
        figdir_new : str
            The old figure output directory
        output_dir : str
            The output dir for the new report.
        """
        # whether to use scaled predictions for both new and old; default to false
        use_scaled_predictions_old = config.get('use_scaled_predictions_old', False)
        use_scaled_predictions_new = config.get('use_scaled_predictions_new', False)

        environ_config = {'EXPERIMENT_ID_OLD': config['experiment_id_old'],
                          'EXPERIMENT_ID_NEW': config['experiment_id_new'],
                          'DESCRIPTION_OLD': config['description_old'],
                          'DESCRIPTION_NEW': config['description_new'],
                          'GROUPS_FOR_DESCRIPTIVES': config.get('subgroups', []),
                          'GROUPS_FOR_EVALUATIONS': config.get('subgroups', []),
                          'SCALED_OLD': use_scaled_predictions_old,
                          'SCALED_NEW': use_scaled_predictions_new,
                          'USE_THUMBNAILS': config.get('use_thumbnails', False),
                          'JAVASCRIPT_PATH': javascript_path,
                          'OUTPUT_DIR_NEW': csvdir_new,
                          'FIGURE_DIR_NEW': figdir_new,
                          'OUTPUT_DIR_OLD': csvdir_old,
                          'FIGURE_DIR_OLD': figdir_old,
                          }

        # create the output directory
        os.makedirs(output_dir, exist_ok=True)
        report_name = '{}_report'.format(config['comparison_id'])
        merged_notebook_file = join(output_dir, '{}.ipynb'.format(report_name))
        environ_config_file = join(output_dir, '.environ.json')

        # set the report directory as an environment variable
        os.environ['RSM_REPORT_DIR'] = abspath(output_dir)

        # write out hidden environment JSON file
        with open(environ_config_file, 'w') as out_environ_config:
            json.dump(environ_config, out_environ_config)

        # merge all the given sections
        self.logger.info('Merging sections')
        self.merge_notebooks(config['chosen_notebook_files'], merged_notebook_file)

        # run the merged notebook and save the output as
        # an HTML file in the report directory
        self.logger.info('Exporting HTML')
        self.convert_ipynb_to_html(merged_notebook_file,
                                   join(output_dir, '{}.html'.format(report_name)))

    def create_summary_report(self,
                              config,
                              all_experiments,
                              csvdir):
        """
        Generate an HTML report for summarizing the given rsmtool experiments.

        Parameters
        ----------
        config : configuration_parser.Configuration
            A configuration object
        all_experiments : list of str
            A list of experiment configuration files to summarize.
        csvdir : str
            The experiment CSV output directory.
        """
        environ_config = {'SUMMARY_ID': config['summary_id'],
                          'DESCRIPTION': config['description'],
                          'GROUPS_FOR_DESCRIPTIVES': config.get('subgroups', []),
                          'GROUPS_FOR_EVALUATIONS': config.get('subgroups', []),
                          'USE_THUMBNAILS': config.get('use_thumbnails', False),
                          'FILE_FORMAT': config.get('file_format', 'csv'),
                          'JSONS': all_experiments,
                          'JAVASCRIPT_PATH': javascript_path,
                          'OUTPUT_DIR': csvdir
                          }

        report_name = '{}_report'.format(config['summary_id'])
        reportdir = abspath(join(csvdir, '..', 'report'))
        merged_notebook_file = join(reportdir, '{}.ipynb'.format(report_name))
        environ_config_file = join(reportdir, '.environ.json')

        # set the report directory as an environment variable
        os.environ['RSM_REPORT_DIR'] = reportdir

        # write out hidden environment JSON file
        with open(environ_config_file, 'w') as out_environ_config:
            json.dump(environ_config, out_environ_config)

        # merge all the given sections
        self.logger.info('Merging sections')
        self.merge_notebooks(config['chosen_notebook_files'], merged_notebook_file)

        # run the merged notebook and save the output as
        # an HTML file in the report directory
        self.logger.info('Exporting HTML')
        self.convert_ipynb_to_html(merged_notebook_file,
                                   join(reportdir, '{}.html'.format(report_name))) class ConfigurationGenerator:
    """
    Class to encapsulate automated batch-mode and interactive generation.

    Attributes
    ----------
    context : str
        Name of the command-line tool for which we are generating the
        configuration file.
    as_string : bool, optional
        If ``True``, return a formatted and indented string representation
        of the configuration, rather than a dictionary. Note that this only
        affects the batch-mode generation. Interactive generation always
        returns a string.
        Defaults to ``False``.
    suppress_warnings : bool, optional
        If ``True``, do not generate any warnings for batch-mode generation.
        Defaults to ``False``.
    use_subgroups : bool, optional
        If ``True``, include subgroup-related sections in the list of general sections
        in the configuration file.
        Defaults to ``False``.
    """

    def __init__(self,
                 context,
                 as_string=False,
                 suppress_warnings=False,
                 use_subgroups=False):  # noqa
        self.context = context
        self.use_subgroups = use_subgroups
        self.suppress_warnings = suppress_warnings
        self.as_string = as_string
        self.logger = logging.getLogger(__name__)

        # we need to save the first required and first optional field we will
        # insert since we will use them as sign posts to insert comments later
        self._required_fields = CHECK_FIELDS[self.context]['required']
        self._optional_fields = sorted(CHECK_FIELDS[self.context]['optional'])
        self._first_required_field = self._required_fields[0]
        self._first_optional_field = self._optional_fields[0]

    def _convert_to_string(self,
                           config_object,
                           insert_url_comment=True,
                           insert_required_comment=True,
                           insert_optional_comment=True):
        configuration = str(config_object)

        # insert the URL comment first, right above the first required field
        if insert_url_comment:
            base_url = 'https://rsmtool.readthedocs.io/en/stable'
            doc_slug = CONFIGURATION_DOCUMENTATION_SLUGS[self.context]
            doc_url = f"{base_url}/{doc_slug}"
            configuration = re.sub(fr'([ ]+)("{self._first_required_field}": [^,]+,\n)',
                                   fr'\1// Reference: {doc_url}\n\1\2',
                                   configuration)

        # insert first comment right above the first required field
        if insert_required_comment:
            configuration = re.sub(fr'([ ]+)("{self._first_required_field}": [^,]+,\n)',
                                   fr'\1// REQUIRED: replace "ENTER_VALUE_HERE" with the appropriate value!\n\1\2',
                                   configuration)

        # insert second comment right above the first optional field
        if insert_optional_comment:
            configuration = re.sub(fr'([ ]+)("{self._first_optional_field}": [^,]+,\n)',
                                   r'\1// OPTIONAL: replace default values below based on your data.\n\1\2',
                                   configuration)

        return configuration

    def _get_all_general_section_names(self):

        default_general_sections_value = DEFAULTS.get('general_sections', '')
        default_special_sections_value = DEFAULTS.get('special_sections', '')
        default_custom_sections_value = DEFAULTS.get('custom_sections', '')

        # if we are told ot use subgroups then just make up a dummy subgroup
        # value so that the subgroup-based sections will be included in the
        # section list. This value is not actually used in configuration file.
        subgroups_value = ['GROUP'] if self.use_subgroups else DEFAULTS.get('subgroups', '')
        return Reporter().determine_chosen_sections(default_general_sections_value,
                                                    default_special_sections_value,
                                                    default_custom_sections_value,
                                                    subgroups_value,
                                                    context=self.context)

    def interact(self):
        """
        Automatically generate an example configuration in interactive mode.

        Returns
        -------
        configuration : str
            The generated configuration as a formatted string.

        Note
        ----
        This method should *only* be used in terminals, and not in
        Jupyter notebooks.

        """
        # clear the screen first
        clear()

        # print the preamble and some instructions
        sys.stderr.write("\n")
        sys.stderr.write("Entering interactive mode:\n")
        sys.stderr.write(" - press ctrl-d to exit without generating a configuration\n")
        sys.stderr.write(" - press tab or start typing when choosing files/directories/models\n")
        sys.stderr.write(" - press enter to accept the default value for a field (underlined)\n")
        sys.stderr.write(" - press ctrl-c to cancel current entry for a field and enter again\n")
        sys.stderr.write(" - you may still need to edit the generated configuration\n")
        sys.stderr.write("\n")

        if not self.use_subgroups:
            sys.stderr.write("IMPORTANT: If you have subgroups and didn't specify the '-g' "
                             "option, exit now (ctrl-d) and re-run!\n")
            sys.stderr.write("\n")

        # instantiate a blank dictionary
        configdict = OrderedDict()

        # iterate over the required fields first, and then the (sorted) optional fields
        # keep track of which field type we are currently dealing with
        for field_type, field_name in chain(product(['required'], self._required_fields),
                                            product(['optional'], self._optional_fields)):

            # skip the subgroups field unless we were told to use subgroups
            if field_name == 'subgroups' and not self.use_subgroups:
                configdict['subgroups'] = DEFAULTS.get('subgroups')
                continue

            # if the field is not one that is meant to be filled interactively,
            # then just use its default value; for "general_sections", expand it
            # so that it is easy for the user to remove sections
            if field_name not in INTERACTIVE_MODE_METADATA:
                non_interactive_field_value = DEFAULTS.get(field_name, '')
                if field_name == 'general_sections':
                    non_interactive_field_value = self._get_all_general_section_names()
                configdict[field_name] = non_interactive_field_value
            else:
                # instantiate the interactive field first
                try:
                    interactive_field = InteractiveField(field_name,
                                                         field_type,
                                                         INTERACTIVE_MODE_METADATA[field_name])
                    configdict[field_name] = interactive_field.get_value()
                # if the user pressed Ctrl-D, then exit out of interactive mode
                # without generating anything and return an empty string
                except EOFError:
                    sys.stderr.write("\n")
                    sys.stderr.write("You exited interactive mode without a configuration.")
                    sys.stderr.write("\n")
                    return ''
                # otherwise get the field value and save it

        # create a Configuration instance from the dictionary we just generated
        sys.stderr.write("\n")
        config_object = Configuration(configdict,
                                      configdir=os.getcwd(),
                                      context=self.context)
        # convert the Configuration object to a string - we are using
        # a special wrapper method since we also want to insert comments
        return self._convert_to_string(config_object, insert_required_comment=False)

    def generate(self):
        """
        Automatically generate an example configuration in batch mode.

        Returns
        -------
        configuration : dict or str
            The generated configuration either as a dictionary or
            a formatted string, depending on the value of ``as_string``.
        """
        # instantiate a dictionary that remembers key insertion order
        configdict = OrderedDict()

        # insert the required fields first and give them a dummy value
        for required_field in self._required_fields:
            configdict[required_field] = 'ENTER_VALUE_HERE'

        # insert the optional fields in alphabetical order
        for optional_field in self._optional_fields:

            # to make it easy for users to add/remove sections, we should
            # populate the `general_sections` field with an explicit list
            # instead of the default value which is simply ``['all']``. To
            # do this, we can use the reporter class.
            if optional_field == 'general_sections':
                configdict['general_sections'] = self._get_all_general_section_names()
            else:
                configdict[optional_field] = DEFAULTS.get(optional_field, '')

        # create a Configuration object
        config_object = Configuration(configdict,
                                      configdir=os.getcwd(),
                                      context=self.context)

        # if we were asked for string output, then convert this dictionary to
        # a string that will also insert some useful comments
        if self.as_string:
            configuration = self._convert_to_string(config_object)
        # otherwise we just return the dictionary underlying the Configuration object
        else:
            configuration = config_object._config

        # print out a warning to make it clear that it cannot be used as is
        if not self.suppress_warnings:
            self.logger.warning("Automatically generated configuration files MUST "
                                "be edited to add values for required fields and "
                                "even for optional ones depending on your data.")

        # return either the Configuration object or the string
        return configuration def setup_rsmcmd_parser(name,
                        uses_output_directory=True,
                        allows_overwriting=False,
                        extra_run_options=[],
                        uses_subgroups=False):
    """
    Create argument parsers for RSM command-line utilities.

    Since the various RSM command-line utilities (``rsmtool``, ``rsmeval``,
    ``rsmcompare``, etc.) have very similar argument parsers, refactoring that
    shared code out into this helper function makes it easier to extend
    and modify the command-line interface to all utilities at the same time.
    In addition, it also improves the consistency among the tools.

    By default, this function adds the following options to the parser:
      - ``config_file`` : a positional argument for the tool's configuration file
      - ``-V``/``--version`` : an optional argument to print out the package version

    If ``uses_output_directory`` is ``True``, an ``output_dir`` positional
    argument will be added to the "run" subcommand parser.

    If ``allows_overwriting`` is ``True``, an ``-f``/``--force``
    optional argument will be added to the "run" subcommand parser.

    The ``extra_run_options`` list should contain a list of ``CmdOption``
    instances which are added to the "run" subcommand parser one by one.

    If ``uses_subgroups`` is ``True``, a ``--subgroups`` optional
    argument will be added to the "generate" subcommand parser.

    Parameters
    ----------
    name : str
        The name of the command-line tool for which we need the parser.
    uses_output_directory : bool, optional
        Add the ``output_dir`` positional argument to the "run" subcommand
        parser. This argument means that the respective tool uses an output
        directory to store its various outputs.
        Defaults to ``True``.
    allows_overwriting : bool, optional
        Add the ``-f``/``-force_write`` optional argument to the "run" subcommand
        parser. This argument allows the output for the respective
        tool to be overwritten even if it already exists (file) or contains
        output (directory).
        Defaults to ``False``.
    extra_run_options : list, optional
        Any additional options to be added to the "run" subcommand parser,
        each specified as a ``CmdOption`` instance.
        Defaults to ``[]``.
    uses_subgroups : bool, optional
        Add the ``--subgroups`` optional argument to the "generate" subcommand
        parser. This argument means that the tool for which we are automatically
        generating a configuration file includes additional information when
        subgroup information is available.
        Defaults to ``False``.

    Returns
    -------
    parser : arpgarse.ArgumentParser
        A fully instantiated argument parser for the respective tool.

    Raises
    ------
    RuntimeError
        If any of the ``CmdOption`` instances specified in
        ``extra_run_options`` do not contain the ``dest`` and
        ``help`` attributes.

    Note
    ----
    This function is only meant to be used by RSMTool developers.
    """
    # a special callable to test whether configuration files exist
    # or not; this is nested because it is only used within this function
    # and should never be used externally
    def existing_configuration_file(string):
        if Path(string).exists():
            return string
        else:
            msg = 'The configuration file %r does not exist.' % string
            raise argparse.ArgumentTypeError(msg)

    # initialize an argument parser
    parser = argparse.ArgumentParser(prog=f"{name}")

    # we always want to have a version flag for the main parser
    parser.add_argument('-V',
                        '--version',
                        action='version',
                        version=VERSION_STRING,
                        help=f"show the {name} version number and exit")

    # each RSM command-line utility has two subcommands
    # - generate : used to auto-generate configuration files
    # - run : used to run experiments

    # let's set up the sub-parsers corresponding to these subcommands
    subparsers = parser.add_subparsers(dest='subcommand', title='subcommands')
    parser_generate = subparsers.add_parser('generate',
                                            help=f"automatically generate an "
                                                 f"{name} configuration file")
    parser_run = subparsers.add_parser('run',
                                       help=f"run an {name} experiment")

    ###################################################
    # Setting up options for the "generate" subparser #
    ###################################################
    if uses_subgroups:
        # we need to display a special help message for ``rsmxval``
        # since its config does not actually contain a sections list
        if name == "rsmxval":
            parser_generate.add_argument('-g',
                                         '--subgroups',
                                         dest='subgroups',
                                         action='store_true',
                                         default=False,
                                         help=f"if specified, {name} will ensure that "
                                              f"subgroup sections are included in "
                                              f"the various reports")
        else:
            parser_generate.add_argument('-g',
                                         '--subgroups',
                                         dest='subgroups',
                                         action='store_true',
                                         default=False,
                                         help=f"if specified, the generated {name} "
                                              f"configuration file will include the "
                                              f"subgroup sections in the general "
                                              f"sections list")

    parser_generate.add_argument('-q',
                                 '--quiet',
                                 dest='quiet',
                                 action='store_true',
                                 default=False,
                                 help="if specified, the warning about not "
                                      "using the generated configuration "
                                      "as-is will be suppressed.")

    parser_generate.add_argument('-i',
                                 '--interactive',
                                 dest='interactive',
                                 action='store_true',
                                 default=False,
                                 help=f"if specified, generate the {name} "
                                      f"configuration file interactively")

    ##############################################
    # Setting up options for the "run" subparser #
    ##############################################

    # since this is an RSMTool command-line utility, we will
    # always need a configuration file
    parser_run.add_argument('config_file',
                            type=existing_configuration_file,
                            help=f"the {name} JSON configuration file to run")

    # if it uses an output directory, let's add that
    if uses_output_directory:
        parser_run.add_argument('output_dir',
                                nargs='?',
                                default=os.getcwd(),
                                help="the output directory where all the files "
                                     "for this run will be stored")

    # if it allows overwrting the output directory, let's add that
    if allows_overwriting:
        parser_run.add_argument('-f',
                                '--force',
                                dest='force_write',
                                action='store_true',
                                default=False,
                                help=f"if specified, {name} will overwrite the "
                                     f"contents of the output file or directory "
                                     f"even if it contains the output of a "
                                     f"previous run ")

    # add any extra options passed in for the rub subcommand;
    for parser_option in extra_run_options:

        # construct the arguments and keyword arguments needed for the
        # `add_argument()` call to the parser
        argparse_option_args = []
        argparse_option_kwargs = {}

        # first add the destination and the help string
        argparse_option_kwargs["dest"] = f"{parser_option.dest}"
        argparse_option_kwargs["help"] = f"{parser_option.help}"

        # now add any optional information
        if parser_option.shortname is not None:
            argparse_option_args.append(f"-{parser_option.shortname}")
        if parser_option.longname is not None:
            argparse_option_args.append(f"--{parser_option.longname}")
        if parser_option.action is not None:
            argparse_option_kwargs['action'] = f"{parser_option.action}"
        if parser_option.default is not None:
            argparse_option_kwargs["default"] = f"{parser_option.default}"
        if parser_option.required is not None:
            try:
                assert type(parser_option.required) == bool
            except AssertionError:
                raise TypeError(f"the 'required' field for CmdOption must be "
                                f"boolean, you specified '{parser_option.required}'")
            else:
                argparse_option_kwargs["required"] = parser_option.required
        if parser_option.nargs is not None:
            argparse_option_kwargs['nargs'] = f"{parser_option.nargs}"

        # add this argument to the parser
        parser_run.add_argument(*argparse_option_args, **argparse_option_kwargs)

    return parser VALID_PARSER_SUBCOMMANDS = ['generate', 'run'] class LogFormatter(logging.Formatter):
    """
    Custom logging formatter.

    Note
    ----
    This class is adapted from https://stackoverflow.com/q/1343227.
    """

    info_fmt = "%(msg)s"
    warn_fmt = "WARNING: %(msg)s"

    err_fmt = "ERROR: %(msg)s"
    dbg_fmt = "DEBUG: %(module)s: %(lineno)d: %(msg)s"

    def __init__(self, fmt="%(levelno)s: %(msg)s"):  # noqa: D107
        logging.Formatter.__init__(self, fmt)

    def format(self, record):
        """
        Format the given record.

        Parameters
        ----------
        record : logging.LogRecord
            The record to format.
        """
        # Save the original format configured by the user
        # when the logger formatter was instantiated
        format_orig = self._fmt

        # Replace the original format with one customized by logging level
        if record.levelno == logging.DEBUG:
            self._fmt = LogFormatter.dbg_fmt
            self._style = logging.PercentStyle(self._fmt)

        elif record.levelno == logging.WARNING:
            self._fmt = LogFormatter.warn_fmt
            self._style = logging.PercentStyle(self._fmt)

        elif record.levelno == logging.INFO:
            self._fmt = LogFormatter.info_fmt
            self._style = logging.PercentStyle(self._fmt)

        elif record.levelno == logging.ERROR:
            self._fmt = LogFormatter.err_fmt
            self._style = logging.PercentStyle(self._fmt)

        # Call the original formatter class to do the grunt work
        result = logging.Formatter.format(self, record)

        # Restore the original format configured by the user
        self._fmt = format_orig

        return result class DataWriter:
    """Class to write out DataContainer objects."""

    def __init__(self, experiment_id=None):  # noqa: D107
        self._id = experiment_id

    @staticmethod
    def write_frame_to_file(df, name_prefix, file_format="csv", index=False, **kwargs):
        """
        Write given data frame to disk with given name and file format.

        Parameters
        ----------
        df : pandas DataFrame
            Data frame to write to disk
        name_prefix : str
            The complete prefix for the file to be written to disk.
            This includes everything except the extension.
        file_format : str
            The file format (extension) for the file to be written to disk.
            One of {"csv", "xlsx", "tsv"}.
            Defaults to "csv".
        index : bool, optional
            Whether to include the index in the output file.
            Defaults to ``False``.

        Raises
        ------
        KeyError
            If ``file_format`` is not valid.
        """
        file_format = file_format.lower()

        if file_format == 'csv':
            name_prefix += '.csv'
            df.to_csv(name_prefix, index=index, **kwargs)

        elif file_format == 'tsv':
            name_prefix += '.tsv'
            df.to_csv(name_prefix, index=index, sep='\t', **kwargs)

        # Added jsonlines for experimental purposes, but leaving
        # this out of the documentation at this stage
        elif file_format == 'jsonlines':
            name_prefix += '.jsonlines'
            df.to_json(name_prefix, orient='records', lines=True, **kwargs)

        elif file_format == 'xlsx':
            name_prefix += '.xlsx'
            df.to_excel(name_prefix, index=index, **kwargs)

        else:
            raise KeyError("Please make sure that the `file_format` specified "
                           "is one of the following:\n{`csv`, `tsv`, `xlsx`}.\n"
                           f"You specified {file_format}.")

    def write_experiment_output(self,
                                csvdir,
                                container_or_dict,
                                dataframe_names=None,
                                new_names_dict=None,
                                include_experiment_id=True,
                                reset_index=False,
                                file_format='csv',
                                index=False,
                                **kwargs):
        """
        Write out each of the named frames to disk.

        This function writes out each of the given list of data frames as a
        ".csv", ".tsv", or ``.xlsx`` file in the given directory. Each data
        frame was generated as part of running an RSMTool experiment. All files
        are prefixed with the given experiment ID and suffixed with either the
        name of the data frame in the DataContainer (or dict) object, or a new
        name if ``new_names_dict`` is specified. Additionally, the indexes in
        the  data frames are reset if so specified.

        Parameters
        ----------
        csvdir : str
            Path to the output experiment sub-directory that will
            contain the CSV files corresponding to each of the data frames.
        container_or_dict : container.DataContainer or dict
            A DataContainer object or dict, where keys are data frame
            names and values are ``pd.DataFrame`` objects.
        dataframe_names : list of str, optional
            List of data frame names, one for each of the data frames.
            Defaults to ``None``.
        new_names_dict : dict, optional
            New dictionary with new names for the data frames, if desired.
            Defaults to ``None``.
        include_experiment_id : str, optional
            Whether to include the experiment ID in the file name.
            Defaults to ``True``.
        reset_index : bool, optional
            Whether to reset the index of each data frame
            before writing to disk.
            Defaults to ``False``.
        file_format : str, optional
            The file format in which to output the data.
            One of {"csv", "xlsx", "tsv"}.
            Defaults to "csv".
        index : bool, optional
            Whether to include the index in the output file.
            Defaults to ``False``.

        Raises
        ------
        KeyError
            If ``file_format`` is not valid, or a named data frame
            is not present in ``container_or_dict``.
        """
        container_or_dict = container_or_dict.copy()

        # If no `dataframe_names` specified, use all names
        if dataframe_names is None:
            dataframe_names = container_or_dict.keys()

        # Otherwise, check to make sure all specified names
        # are actually in the DataContainer
        else:
            for name in dataframe_names:
                if name not in container_or_dict:
                    raise KeyError("The name `{}` is not in the container "
                                   "or dictionary.".format(name))

        # Loop through DataFrames, and save
        # output in specified format
        for dataframe_name in dataframe_names:

            df = container_or_dict[dataframe_name]
            if df is None:
                raise KeyError('The DataFrame `{}` does not exist.'.format(dataframe_name))

            # If the DataFrame is empty, skip it
            if df.empty:
                continue

            # If there is a desire to rename the DataFrame,
            # get the new name
            if new_names_dict is not None:
                if dataframe_name in new_names_dict:
                    dataframe_name = new_names_dict[dataframe_name]

            # Reset the index, if desired
            if reset_index:
                df.index.name = ''
                df.reset_index(inplace=True)

            # If include_experiment_id is True, and the experiment_id exists
            # include it in the file name; otherwise, do not include it.
            if include_experiment_id and self._id is not None:
                outfile = join(csvdir, '{}_{}'.format(self._id, dataframe_name))
            else:
                outfile = join(csvdir, dataframe_name)

            # write out the frame to disk in the given file
            self.write_frame_to_file(df,
                                     outfile,
                                     file_format=file_format,
                                     index=index,
                                     **kwargs)

    def write_feature_csv(self,
                          featuredir,
                          data_container,
                          selected_features,
                          include_experiment_id=True,
                          file_format='csv'):
        """
        Write out the selected features to disk.

        Parameters
        ----------
        featuredir : str
            Path to the experiment output directory where the
            feature JSON file will be saved.
        data_container : container.DataContainer
            A data container object.
        selected_features : list of str
            List of features that were selected for model building.
        include_experiment_id : bool, optional
            Whether to include the experiment ID in the file name.
            Defaults to ``True``.
        file_format : str, optional
            The file format in which to output the data.
            One of {"csv", "tsv", "xlsx"}.
            Defaults to "csv".
        """
        df_feature_specs = data_container['feature_specs']

        # Select specific features used in training
        df_selected = df_feature_specs[df_feature_specs['feature'].isin(selected_features)]

        # Replace existing `feature_specs` with selected features specs
        data_container.add_dataset({'frame': df_selected,
                                    'name': 'feature_specs'}, update=True)

        makedirs(featuredir, exist_ok=True)
        self.write_experiment_output(featuredir,
                                     data_container,
                                     ['feature_specs'],
                                     {'feature_specs': 'selected'},
                                     include_experiment_id=include_experiment_id,
                                     file_format=file_format)
import logging
import sys
from os import listdir, makedirs
from os.path import abspath, exists, join
from .analyzer import Analyzer
from .configuration_parser import configure
from .modeler import Modeler
from .preprocessor import FeaturePreprocessor
from .reader import DataReader
from .reporter import Reporter
from .utils.commandline import ConfigurationGenerator, setup_rsmcmd_parser
from .utils.constants import VALID_PARSER_SUBCOMMANDS
from .utils.logging import LogFormatter
from .writer import DataWriter


def run_experiment(config_file_or_obj_or_dict,
                   output_dir,
                   overwrite_output=False,
                   logger=None):
    """
    Run an rsmtool experiment using the given configuration.

    Run rsmtool experiment using the given configuration file, object, or
    dictionary. All outputs are generated under ``output_dir``. If
    ``overwrite_output`` is ``True``, any existing output in ``output_dir``
    is overwritten.

    Parameters
    ----------
    config_file_or_obj_or_dict : str or pathlib.Path or dict or Configuration
        Path to the experiment configuration file either a a string
        or as a ``pathlib.Path`` object. Users can also pass a
        ``Configuration`` object that is in memory or a Python dictionary
        with keys corresponding to fields in the configuration file. Given a
        configuration file, any relative paths in the configuration file
        will be interpreted relative to the location of the file. Given a
        ``Configuration`` object, relative paths will be interpreted
        relative to the ``configdir`` attribute, that _must_ be set. Given
        a dictionary, the reference path is set to the current directory.
    output_dir : str
        Path to the experiment output directory.
    overwrite_output : bool, optional
        If ``True``, overwrite any existing output under ``output_dir``.
        Defaults to ``False``.
    logger : logging object, optional
        A logging object. If ``None`` is passed, get logger from ``__name__``.
        Defaults to ``None``.

    Raises
    ------
    FileNotFoundError
        If any of the files contained in ``config_file_or_obj_or_dict`` cannot
        be located.
    IOError
        If ``output_dir`` already contains the output of a previous experiment
        and ``overwrite_output`` is ``False``.
    ValueError
        If the current configuration specifies a non-linear model but
        ``output_dir`` already contains the output of a previous
        experiment that used a linear model with the same experiment ID.
    """
    logger = logger if logger else logging.getLogger(__name__)

    # create the 'output' and the 'figure' sub-directories
    # where all the experiment output such as the CSV files
    # and the box plots will be saved

    # Get absolute paths to output directories
    csvdir = abspath(join(output_dir, 'output'))
    figdir = abspath(join(output_dir, 'figure'))
    reportdir = abspath(join(output_dir, 'report'))
    featuredir = abspath(join(output_dir, 'feature'))

    # Make directories, if necessary
    makedirs(csvdir, exist_ok=True)
    makedirs(figdir, exist_ok=True)
    makedirs(reportdir, exist_ok=True)

    # Raise an error if the specified output directory
    # already contains a non-empty `output` directory, unless
    # `overwrite_output` was specified, in which case we assume
    # that the user knows what she is doing and simply
    # output a warning saying that the report might
    # not be correct.
    non_empty_csvdir = exists(csvdir) and listdir(csvdir)
    if non_empty_csvdir:
        if not overwrite_output:
            raise IOError("'{}' already contains a non-empty 'output' "
                          "directory.".format(output_dir))
        else:
            logger.warning("{} already contains a non-empty 'output' directory. "
                           "The generated report might contain "
                           "unexpected information from a previous "
                           "experiment.".format(output_dir))

    configuration = configure('rsmtool', config_file_or_obj_or_dict)

    logger.info('Saving configuration file.')
    configuration.save(output_dir)

    # Get output format
    file_format = configuration.get('file_format', 'csv')

    # Get DataWriter object
    writer = DataWriter(configuration['experiment_id'])

    # Get the paths and names for the DataReader

    (file_names,
     file_paths_org) = configuration.get_names_and_paths(['train_file', 'test_file',
                                                          'features',
                                                          'feature_subset_file'],
                                                         ['train', 'test',
                                                          'feature_specs',
                                                          'feature_subset_specs'])

    file_paths = DataReader.locate_files(file_paths_org, configuration.configdir)

    # if there are any missing files after trying to locate
    # all expected files, raise an error
    if None in file_paths:
        missing_file_paths = [file_paths_org[idx] for idx, path in enumerate(file_paths)
                              if path is None]
        raise FileNotFoundError('The following files were not found: '
                                '{}'.format(repr(missing_file_paths)))

    # Use the default converter for both train and test
    converters = {'train': configuration.get_default_converter(),
                  'test': configuration.get_default_converter()}

    logger.info('Reading in all data from files.')

    # Initialize the reader

    reader = DataReader(file_paths, file_names, converters)