class Configuration:
    """
    Configuration class.

    Encapsulates all of the configuration parameters and methods to 
    access these parameters.
    """

    def __init__(self,
                 configdict,
                 *,
                 configdir=None,
                 context='rsmtool',
                 logger=None):
        """
        Create an object of the `Configuration` class.

        This method can be used to directly instantiate a Configuration
        object.

        Parameters
        ----------
        configdict : dict
            A dictionary of configuration parameters.
            The dictionary must be a valid configuration dictionary
            with default values filled as necessary.
        configdir : str, optional, keyword-only
            The reference path used to resolve any relative paths
            in the configuration object. When ``None``, will be set during
            initialization to the current working directory.
            Defaults to ``None``,
        context : str
            The context of the tool. One of {"rsmtool", "rsmeval", "rsmcompare",
            "rsmpredict", "rsmsummarize"}.
            Defaults to "rsmtool".
        logger : logging object, optional
            A logging object. If ``None`` is passed, get logger from ``__name__``.
            Defaults to ``None``.
        """
        if not isinstance(configdict, dict):
            raise TypeError('The input must be a dictionary.')

        # create a logger instance
        self.logger = logger if logger else logging.getLogger(__name__)

        # process and validate the configuration dictionary
        configdict = ConfigurationParser.process_config(configdict)
        configdict = ConfigurationParser.validate_config(configdict, context=context)

        # set configdir to `cwd` if not given and let the user know
        if configdir is None:
            configdir = Path(getcwd())
            self.logger.info("Configuration directory will be set to {}".format(configdir))
        else:
            configdir = Path(configdir).resolve()

        self._config = configdict
        self._configdir = configdir
        self._context = context

    def __contains__(self, key):
        """
        Check if the configuration object contains a given key.

        Parameters
        ----------
        key : str
            Key to check in the Configuration object.

        Returns
        -------
        key_check : bool
            ``True`` if key is in configuration object, else ``False``.
        """
        return key in self._config

    def __getitem__(self, key):
        """
        Get configuration value for the given key.

        Parameters
        ----------
        key : str
            Key to check in the configuration object

        Returns
        -------
        value
            The value in the configuration object dictionary.
        """
        return self._config[key]

    def __setitem__(self, key, value):
        """
        Set configuration value for the given key.

        Parameters
        ----------
        key : str
            Key to set in the configuration object.
        value
            The value to be set for the key.
        """
        self._config[key] = value

    def __len__(self):
        """
        Return the size of the configuration dictionary.

        Returns
        -------
        size : int
            The number of elements in the configuration dictionary.
        """
        return len(self._config)

    def __str__(self):
        """
        Return string representation of the configuration dictionary.

        Returns
        -------
        config_string : str
            A string representation of the underlying configuration
            dictionary as encoded by ``json.dumps()``. It only
            includes the configuration options that can be set by
            the user.
        """
        expected_fields = (CHECK_FIELDS[self._context]['required'] +
                           CHECK_FIELDS[self._context]['optional'])

        output_config = {k: v for k, v in self._config.items() if k in expected_fields}
        return json.dumps(output_config, indent=4, separators=(',', ': '))

    def __iter__(self):
        """
        Iterate through configuration object keys.

        Yields
        ------
        key : str
            A key in the config dictionary
        """
        for key in self.keys():
            yield key

    @property
    def configdir(self):
        """
        Get the path to configuration directory.

        Get the path to the configuration reference directory that
        will be used to resolve any relative paths in
        the configuration.

        Returns
        -------
        configdir : str
            The path to the configuration reference directory.
        """
        return str(self._configdir)

    @configdir.setter
    def configdir(self, new_path):
        """
        Set a new configuration reference directory.

        Parameters
        ----------
        new_path : str
            Path to the new configuration reference
            directory used to resolve any relative paths
            in the configuration object.
        """
        if new_path is None:
            raise ValueError("The `configdir` attribute cannot be set to `None` ")

        # TODO: replace `Path(abspath(new_path))` with `Path(new_path).resolve()
        # once this Windows bug is fixed: https://bugs.python.org/issue38671
        self._configdir = Path(abspath(new_path))

    @property
    def context(self):
        """Get the context."""
        return self._context

    @context.setter
    def context(self, new_context):
        """
        Set a new context.

        Parameters
        ----------
        new_context : str
            A new context for the configuration object.
        """
        self._context = new_context

    def get(self, key, default=None):
        """
        Get value or default for the given key.

        Parameters
        ----------
        key : str
            Key to check in the Configuration object.
        default, optional
            The default value to return, if no key exists.
            Defaults to ``None``.

        Returns
        -------
        value
            The value in the configuration object dictionary.
        """
        return self._config.get(key, default)

    def to_dict(self):
        """
        Get a dictionary representation of the configuration object.

        Returns
        -------
        config : dict
            The configuration dictionary.
        """
        return self._config

    def keys(self):
        """
        Return keys as a list.

        Returns
        -------
        keys : list of str
            A list of keys in the configuration object.
        """
        return [k for k in self._config.keys()]

    def values(self):
        """
        Return configuration values as a list.

        Returns
        -------
        values : list
            A list of values in the configuration object.
        """
        return [v for v in self._config.values()]

    def items(self):
        """
        Return configuration items as a list of tuples.

        Returns
        -------
        items : list of tuples
            A list of (key, value) tuples in the
            configuration object.
        """
        return [(k, v) for k, v in self._config.items()]

    def pop(self, key, default=None):
        """
        Remove and return an element from the object having the given key.

        Parameters
        ----------
        key : str
            Key to pop in the configuration object.
        default, optional
            The default value to return, if no key exists.
            Defaults to ``None``.

        Returns
        -------
        value
            The value removed from the object.
        """
        return self._config.pop(key, default)

    def copy(self, deep=True):
        """
        Return a copy of the object.

        Parameters
        ----------
        deep : bool, optional
            Whether to perform a deep copy.
            Defaults to ``True``.

        Returns
        -------
        copy : Configuration
            A new configuration object.
        """
        if deep:
            return deepcopy(self)
        return copy(self)

    def save(self, output_dir=None):
        """
        Save the configuration file to the output directory specified.

        Parameters
        ----------
        output_dir : str
            The path to the output directory. If ``None``, the current
            directory is used.
            Defaults to ``None``.
        """
        # save a copy of the main config into the output directory
        if output_dir is None:
            output_dir = Path(getcwd())

        # Create output directory, if it does not exist
        output_dir = Path(output_dir).resolve() / 'output'
        output_dir.mkdir(parents=True, exist_ok=True)

        id_field = ID_FIELDS[self._context]
        experiment_id = self._config[id_field]
        context = self._context
        outjson = output_dir / f"{experiment_id}_{context}.json"

        with outjson.open(mode='w') as outfile:
            outfile.write(str(self))

    def check_exclude_listwise(self):
        """
        Check for candidate exclusion.

        Check if we are excluding candidates based on number of responses, and
        add this to the configuration file.

        Returns
        -------
        exclude_listwise : bool
            Whether to exclude list-wise.
        """
        exclude_listwise = False
        if self._config.get('min_items_per_candidate'):
            exclude_listwise = True
        return exclude_listwise

    def check_flag_column(self,
                          flag_column='flag_column',
                          partition='unknown'):
        """
        Make sure the column name in ``flag_column`` is correctly specified.

        Get flag columns and values for filtering, if any, and convert single
        values to lists. Raises an exception if the column name in ``flag_column``
        is not correctly specified in the configuration file.

        Parameters
        ----------
        flag_column : str
            The flag column name to check. Currently used names are
            "flag_column" or "flag_column_test".
            Defaults to "flag_column".

        partition: str
            The data partition which is filtered based on the flag column name.
            One of {"train", "test", "both", "unknown"}.
            Defaults to "both".

        Returns
        -------
        new_filtering_dict : dict
            Properly formatted dictionary for the column
            name in ``flag_column``.

        Raises
        ------
        ValueError
            If the specified value of the column name in ``flag_column``
            is not a dictionary.
        ValueError
            If the value of ``partition`` is not in the expected list.
        ValueError
            If the value of ``partition`` does not match the ``flag_column``.
        """
        config = self._config

        new_filter_dict = {}

        flag_message = {"train": "training",
                        "test": "evaluating",
                        "both": "training and evaluating",
                        "unknown": "training and/or evaluating"}

        if partition not in flag_message:
            raise ValueError("Unknown value for partition: {} "
                             "This must be one of the following: {}."
                             "".format(partition, ','.join(flag_message.keys())))

        if flag_column == "flag_column_test":
            if partition in ["both", "train"]:
                raise ValueError("Conditions specified in 'flag_column_test' "
                                 "can only be applied to the evaluation partition.")

        if config.get(flag_column):

            original_filter_dict = config[flag_column]

            # first check that the value is a dictionary
            if not isinstance(original_filter_dict, dict):
                raise ValueError("`flag_column` must be a dictionary. "
                                 "Please refer to the documentation for "
                                 "further information.")

            for column in original_filter_dict:

                # if we were given a single value, convert it to list
                if not isinstance(original_filter_dict[column], list):
                    new_filter_dict[column] = [original_filter_dict[column]]
                    self.logger.warning("The filtering condition {}"
                                        " for column {} was converted "
                                        "to list. Only responses where "
                                        "{} == {} will be used for "
                                        "{} the "
                                        "model. You can ignore this "
                                        "warning if this is the correct "
                                        "interpretation of your "
                                        "configuration settings"
                                        ".".format(original_filter_dict[column],
                                                   column,
                                                   column,
                                                   original_filter_dict[column],
                                                   flag_message[partition])
                                        )
                else:
                    new_filter_dict[column] = original_filter_dict[column]

                    model_eval = ', '.join(map(str,
                                               original_filter_dict[column]))
                    self.logger.info("Only responses where "
                                     "{} equals one of the following values "
                                     "will be used for {} the model: "
                                     "{}.".format(column,
                                                  flag_message[partition],
                                                  model_eval))
        return new_filter_dict

    def get_trim_min_max_tolerance(self):
        """
        Get trim min, trim max, and tolerance values.

        Get the specified trim min and max, and trim_tolerance if any,
        and make sure they are numeric.

        Returns
        -------
        spec_trim_min : float
            Specified trim min value.
        spec_trim_max : float
            Specified trim max value.
        spec_trim_tolerance: float
            Specified trim tolerance value.
        """
        config = self._config

        spec_trim_min = config.get('trim_min', None)
        spec_trim_max = config.get('trim_max', None)
        spec_trim_tolerance = config.get('trim_tolerance', None)

        if spec_trim_min:
            spec_trim_min = float(spec_trim_min)
        if spec_trim_max:
            spec_trim_max = float(spec_trim_max)
        if spec_trim_tolerance:
            spec_trim_tolerance = float(spec_trim_tolerance)
        return (spec_trim_min, spec_trim_max, spec_trim_tolerance)

    def get_rater_error_variance(self):
        """
        Get specified rater error variance, if any, and make sure it's numeric.

        Returns
        -------
        rater_error_variance : float
            Specified rater error variance.
        """
        config = self._config

        rater_error_variance = config.get('rater_error_variance', None)

        if rater_error_variance:
            rater_error_variance = float(rater_error_variance)

        return rater_error_variance

    def get_default_converter(self):
        """
        Get default converter dictionary for data reader.

        Returns
        -------
        default_converter : dict
            The default converter for a train or test file.
        """
        string_columns = [self._config['id_column']]

        candidate = self._config.get('candidate_column')
        if candidate is not None:
            string_columns.append(candidate)

        subgroups = self._config.get('subgroups')
        if subgroups:
            string_columns.extend(subgroups)

        return dict([(column, str) for column in string_columns if column])

    def get_names_and_paths(self, keys, names):
        """
        Get a list of values for the given keys.

        Remove any values that are ``None``.

        Parameters
        -------
        keys : list
            A list of keys whose values to retrieve.
        names : list
            The default value to use if key cannot be found.
            Defaults to ``None``.

        Returns
        -------
        values : list
            The list of values.

        Raises
        ------
        ValueError
            If there are any duplicate keys or names.
        """
        assert len(keys) == len(names)

        # Make sure keys are not duplicated
        if not len(set(keys)) == len(keys):
            raise ValueError('The ``keys`` must be unique. However, the '
                             'following duplicate keys were found: {}.'
                             ''.format(', '.join([key for key, val in Counter(keys).items()
                                                  if val > 1])))
        # Make sure names are not duplicated
        if not len(set(names)) == len(names):
            raise ValueError('The``names`` must be unique. However, the '
                             'following duplicate names were found: {}.'
                             ''.format(', '.join([name for name, val in Counter(names).items()
                                                  if val > 1])))
        existing_names = []
        existing_paths = []
        for idx, key in enumerate(keys):

            path = self._config.get(key)

            # if the `features` field is a list,
            # do not include it in the container
            if key == 'features':
                if isinstance(path, list):
                    continue

            if path is not None:
                existing_paths.append(path)
                existing_names.append(names[idx])

        return existing_names, existing_paths def configure(context, config_file_or_obj_or_dict):
    """
    Create a Configuration object.

    Get the configuration for ``context`` from the input
    ``config_file_or_obj_or_dict``.

    Parameters
    ----------
    context : str
        The context that is being configured. Must be one of
        "rsmtool", "rsmeval", "rsmcompare", "rsmsummarize",
        "rsmpredict", or "rsmxval".
    config_file_or_obj_or_dict : str or pathlib.Path or dict or Configuration
        Path to the experiment configuration file either a a string
        or as a ``pathlib.Path`` object. Users can also pass a
        ``Configuration`` object that is in memory or a Python dictionary
        with keys corresponding to fields in the configuration file. Given a
        configuration file, any relative paths in the configuration file
        will be interpreted relative to the location of the file. Given a
        ``Configuration`` object, relative paths will be interpreted
        relative to the ``configdir`` attribute, that _must_ be set. Given
        a dictionary, the reference path is set to the current directory.

    Returns
    -------
    configuration : Configuration
        The Configuration object for the tool.

    Raises
    ------
    AttributeError
        If the ``configdir`` attribute for the Configuration input is not set.
    ValueError
        If ``config_file_or_obj_or_dict`` contains anything except a string,
        a path, a dictionary, or a ``Configuration`` object.
    """
    # check what sort of input we got
    # if we got a string we consider this to be path to config file
    if isinstance(config_file_or_obj_or_dict, (str, Path)):

        # Instantiate configuration parser object
        parser = ConfigurationParser(config_file_or_obj_or_dict)
        configuration = parser.parse(context=context)

    elif isinstance(config_file_or_obj_or_dict, dict):

        # directly instantiate the Configuration from the dictionary
        configuration = Configuration(config_file_or_obj_or_dict,
                                      context=context)

    elif isinstance(config_file_or_obj_or_dict, Configuration):

        # raise an error if we are passed a Configuration object
        # without a configdir attribute. This can only
        # happen if the object was constructed using an earlier version
        # of RSMTool and stored
        try:
            assert config_file_or_obj_or_dict.configdir is not None
        except AssertionError:
            raise AttributeError("Configuration object must have configdir attribute.")
        else:
            configuration = config_file_or_obj_or_dict

    else:
        raise ValueError("The input to run_experiment must be "
                         "a path to the file (str), a dictionary, "
                         "or a configuration object. You passed "
                         "{}.".format(type(config_file_or_obj_or_dict)))

    return configuration def run_evaluation(config_file_or_obj_or_dict,
                   output_dir,
                   overwrite_output=False,
                   logger=None):
    """
    Run an rsmeval experiment using the given configuration.

    All outputs are generated under ``output_dir``. If ``overwrite_output``
    is ``True``, any existing output in ``output_dir`` is overwritten.

    Parameters
    ----------
    config_file_or_obj_or_dict : str or pathlib.Path or dict or Configuration
        Path to the experiment configuration file either a a string
        or as a ``pathlib.Path`` object. Users can also pass a
        ``Configuration`` object that is in memory or a Python dictionary
        with keys corresponding to fields in the configuration file. Given a
        configuration file, any relative paths in the configuration file
        will be interpreted relative to the location of the file. Given a
        ``Configuration`` object, relative paths will be interpreted
        relative to the ``configdir`` attribute, that _must_ be set. Given
        a dictionary, the reference path is set to the current directory.
    output_dir : str
        Path to the experiment output directory.
    overwrite_output : bool, optional
        If ``True``, overwrite any existing output under ``output_dir``.
        Defaults to ``False``.
    logger : logging object, optional
        A logging object. If ``None`` is passed, get logger from ``__name__``.
        Defaults to ``None``.

    Raises
    ------
    FileNotFoundError
        If any of the files contained in ``config_file_or_obj_or_dict`` cannot
        be located.
    IOError
        If ``output_dir`` already contains the output of a previous experiment
        and ``overwrite_output`` is ``False``.
    """
    logger = logger if logger else logging.getLogger(__name__)

    # create the 'output' and the 'figure' sub-directories
    # where all the experiment output such as the CSV files
    # and the box plots will be saved
    csvdir = abspath(join(output_dir, 'output'))
    figdir = abspath(join(output_dir, 'figure'))
    reportdir = abspath(join(output_dir, 'report'))
    os.makedirs(csvdir, exist_ok=True)
    os.makedirs(figdir, exist_ok=True)
    os.makedirs(reportdir, exist_ok=True)

    # Raise an error if the specified output directory
    # already contains a non-empty `output` directory, unless
    # ``overwrite_output`` was specified, in which case we assume
    # that the user knows what she is doing and simply
    # output a warning saying that the report might
    # not be correct.
    non_empty_csvdir = exists(csvdir) and listdir(csvdir)
    if non_empty_csvdir:
        if not overwrite_output:
            raise IOError("'{}' already contains a non-empty 'output' "
                          "directory.".format(output_dir))
        else:
            logger.warning("{} already contains a non-empty 'output' directory. "
                           "The generated report might contain "
                           "unexpected information from a previous "
                           "experiment.".format(output_dir))

    configuration = configure('rsmeval', config_file_or_obj_or_dict)

    logger.info('Saving configuration file.')
    configuration.save(output_dir)

    # Get output format
    file_format = configuration.get('file_format', 'csv')

    # Get DataWriter object
    writer = DataWriter(configuration['experiment_id'])

    # Make sure prediction file can be located
    if not DataReader.locate_files(configuration['predictions_file'],
                                   configuration.configdir):
        raise FileNotFoundError('Error: Predictions file {} '
                                'not found.\n'.format(configuration['predictions_file']))

    scale_with = configuration.get('scale_with')

    # scale_with can be one of the following:
    # (a) None       : the predictions are assumed to be 'raw' and should be used as is
    #                  when computing the metrics; the names for the final columns are
    #                  'raw', 'raw_trim' and 'raw_trim_round'.
    # (b) 'asis'     : the predictions are assumed to be pre-scaled and should be used as is
    #                  when computing the metrics; the names for the final columns are
    #                  'scale', 'scale_trim' and 'scale_trim_round'.
    # (c) a CSV file : the predictions are assumed to be 'raw' and should be scaled
    #                  before computing the metrics; the names for the final columns are
    #                  'scale', 'scale_trim' and 'scale_trim_round'.

    # Check whether we want to do scaling
    do_scaling = (scale_with is not None and
                  scale_with != 'asis')

    # The paths to files and names for data container properties
    paths = ['predictions_file']
    names = ['predictions']

    # If we want to do scaling, get the scale file
    if do_scaling:

        # Make sure scale file can be located
        scale_file_location = DataReader.locate_files(scale_with,
                                                      configuration.configdir)
        if not scale_file_location:
            raise FileNotFoundError('Could not find scaling file {}.'
                                    ''.format(scale_file_location))

        paths.append('scale_with')
        names.append('scale')

    # Get the paths, names, and converters for the DataReader
    (file_names,
     file_paths) = configuration.get_names_and_paths(paths, names)

    file_paths = DataReader.locate_files(file_paths, configuration.configdir)

    converters = {'predictions': configuration.get_default_converter()}

    logger.info('Reading predictions: {}.'.format(configuration['predictions_file']))

    # Initialize the reader
    reader = DataReader(file_paths, file_names, converters)
    data_container = reader.read()

    logger.info('Preprocessing predictions.')

    # Initialize the processor
    processor = FeaturePreprocessor(logger=logger)

    (processed_config,
     processed_container) = processor.process_data(configuration,
                                                   data_container,
                                                   context='rsmeval')

    logger.info('Saving pre-processed predictions and metadata to disk.')
    writer.write_experiment_output(csvdir,
                                   processed_container,
                                   new_names_dict={'pred_test':
                                                   'pred_processed',
                                                   'test_excluded':
                                                   'test_excluded_responses'},
                                   file_format=file_format)

    # Initialize the analyzer
    analyzer = Analyzer(logger=logger)

    # do the data composition stats
    (analyzed_config,
     analyzed_container) = analyzer.run_data_composition_analyses_for_rsmeval(processed_container,
                                                                              processed_config)
    # Write out files
    writer.write_experiment_output(csvdir,
                                   analyzed_container,
                                   file_format=file_format)

    for_pred_data_container = analyzed_container + processed_container

    # run the analyses on the predictions of the model`
    logger.info('Running analyses on predictions.')
    (pred_analysis_config,
     pred_analysis_data_container) = analyzer.run_prediction_analyses(for_pred_data_container,
                                                                      analyzed_config)

    writer.write_experiment_output(csvdir,
                                   pred_analysis_data_container,
                                   reset_index=True,
                                   file_format=file_format)

    # Initialize reporter
    reporter = Reporter(logger=logger)

    # generate the report
    logger.info('Starting report generation.')
    reporter.create_report(pred_analysis_config,
                           csvdir,
                           figdir,
                           context='rsmeval') def run_experiment(config_file_or_obj_or_dict,
                   output_dir,
                   overwrite_output=False,
                   logger=None):
    """
    Run an rsmtool experiment using the given configuration.

    Run rsmtool experiment using the given configuration file, object, or
    dictionary. All outputs are generated under ``output_dir``. If
    ``overwrite_output`` is ``True``, any existing output in ``output_dir``
    is overwritten.

    Parameters
    ----------
    config_file_or_obj_or_dict : str or pathlib.Path or dict or Configuration
        Path to the experiment configuration file either a a string
        or as a ``pathlib.Path`` object. Users can also pass a
        ``Configuration`` object that is in memory or a Python dictionary
        with keys corresponding to fields in the configuration file. Given a
        configuration file, any relative paths in the configuration file
        will be interpreted relative to the location of the file. Given a
        ``Configuration`` object, relative paths will be interpreted
        relative to the ``configdir`` attribute, that _must_ be set. Given
        a dictionary, the reference path is set to the current directory.
    output_dir : str
        Path to the experiment output directory.
    overwrite_output : bool, optional
        If ``True``, overwrite any existing output under ``output_dir``.
        Defaults to ``False``.
    logger : logging object, optional
        A logging object. If ``None`` is passed, get logger from ``__name__``.
        Defaults to ``None``.

    Raises
    ------
    FileNotFoundError
        If any of the files contained in ``config_file_or_obj_or_dict`` cannot
        be located.
    IOError
        If ``output_dir`` already contains the output of a previous experiment
        and ``overwrite_output`` is ``False``.
    ValueError
        If the current configuration specifies a non-linear model but
        ``output_dir`` already contains the output of a previous
        experiment that used a linear model with the same experiment ID.
    """
    logger = logger if logger else logging.getLogger(__name__)

    # create the 'output' and the 'figure' sub-directories
    # where all the experiment output such as the CSV files
    # and the box plots will be saved

    # Get absolute paths to output directories
    csvdir = abspath(join(output_dir, 'output'))
    figdir = abspath(join(output_dir, 'figure'))
    reportdir = abspath(join(output_dir, 'report'))
    featuredir = abspath(join(output_dir, 'feature'))

    # Make directories, if necessary
    makedirs(csvdir, exist_ok=True)
    makedirs(figdir, exist_ok=True)
    makedirs(reportdir, exist_ok=True)

    # Raise an error if the specified output directory
    # already contains a non-empty `output` directory, unless
    # `overwrite_output` was specified, in which case we assume
    # that the user knows what she is doing and simply
    # output a warning saying that the report might
    # not be correct.
    non_empty_csvdir = exists(csvdir) and listdir(csvdir)
    if non_empty_csvdir:
        if not overwrite_output:
            raise IOError("'{}' already contains a non-empty 'output' "
                          "directory.".format(output_dir))
        else:
            logger.warning("{} already contains a non-empty 'output' directory. "
                           "The generated report might contain "
                           "unexpected information from a previous "
                           "experiment.".format(output_dir))

    configuration = configure('rsmtool', config_file_or_obj_or_dict)

    logger.info('Saving configuration file.')
    configuration.save(output_dir)

    # Get output format
    file_format = configuration.get('file_format', 'csv')

    # Get DataWriter object
    writer = DataWriter(configuration['experiment_id'])

    # Get the paths and names for the DataReader

    (file_names,
     file_paths_org) = configuration.get_names_and_paths(['train_file', 'test_file',
                                                          'features',
                                                          'feature_subset_file'],
                                                         ['train', 'test',
                                                          'feature_specs',
                                                          'feature_subset_specs'])

    file_paths = DataReader.locate_files(file_paths_org, configuration.configdir)

    # if there are any missing files after trying to locate
    # all expected files, raise an error
    if None in file_paths:
        missing_file_paths = [file_paths_org[idx] for idx, path in enumerate(file_paths)
                              if path is None]
        raise FileNotFoundError('The following files were not found: '
                                '{}'.format(repr(missing_file_paths)))

    # Use the default converter for both train and test
    converters = {'train': configuration.get_default_converter(),
                  'test': configuration.get_default_converter()}

    logger.info('Reading in all data from files.')

    # Initialize the reader
    reader = DataReader(file_paths, file_names, converters)
    data_container = reader.read()

    logger.info('Preprocessing all features.')

    # Initialize the processor
    processor = FeaturePreprocessor(logger=logger)

    (processed_config,
     processed_container) = processor.process_data(configuration,
                                                   data_container)

    # Rename certain frames with more descriptive names
    # for writing out experiment files
    rename_dict = {'train_excluded': 'train_excluded_responses',
                   'test_excluded': 'test_excluded_responses',
                   'train_length': 'train_response_lengths',
                   'train_flagged': 'train_responses_with_excluded_flags',
                   'test_flagged': 'test_responses_with_excluded_flags'}

    logger.info('Saving training and test set data to disk.')

    # Write out files
    writer.write_experiment_output(csvdir,
                                   processed_container,
                                   ['train_features',
                                    'test_features',
                                    'train_metadata',
                                    'test_metadata',
                                    'train_other_columns',
                                    'test_other_columns',
                                    'train_preprocessed_features',
                                    'test_preprocessed_features',
                                    'train_excluded',
                                    'test_excluded',
                                    'train_length',
                                    'test_human_scores',
                                    'train_flagged',
                                    'test_flagged'],
                                   rename_dict,
                                   file_format=file_format)

    # Initialize the analyzer
    analyzer = Analyzer(logger=logger)

    (_,
     analyzed_container) = analyzer.run_data_composition_analyses_for_rsmtool(processed_container,
                                                                              processed_config)

    # Write out files
    writer.write_experiment_output(csvdir,
                                   analyzed_container,
                                   file_format=file_format)

    logger.info('Training {} model.'.format(processed_config['model_name']))

    # Initialize modeler
    modeler = Modeler(logger=logger)

    modeler.train(processed_config,
                  processed_container,
                  csvdir,
                  figdir,
                  file_format)

    # Identify the features used by the model
    selected_features = modeler.get_feature_names()

    # Add selected features to processed configuration
    processed_config['selected_features'] = selected_features

    # Write out files
    writer.write_feature_csv(featuredir,
                             processed_container,
                             selected_features,
                             file_format=file_format)

    features_data_container = processed_container.copy()

    # Get selected feature info, and write out to file
    df_feature_info = features_data_container.feature_info.copy()
    df_selected_feature_info = df_feature_info[df_feature_info['feature'].isin(selected_features)]
    selected_feature_dataset_dict = {'name': 'selected_feature_info',
                                     'frame': df_selected_feature_info}

    features_data_container.add_dataset(selected_feature_dataset_dict,
                                        update=True)

    writer.write_experiment_output(csvdir,
                                   features_data_container,
                                   dataframe_names=['selected_feature_info'],
                                   new_names_dict={'selected_feature_info': 'feature'},
                                   file_format=file_format)

    logger.info('Running analyses on training set.')

    (_,
     train_analyzed_container) = analyzer.run_training_analyses(processed_container,
                                                                processed_config)

    # Write out files
    writer.write_experiment_output(csvdir,
                                   train_analyzed_container,
                                   reset_index=True,
                                   file_format=file_format)

    # Use only selected features for predictions
    columns_for_prediction = ['spkitemid', 'sc1'] + selected_features
    train_for_prediction = processed_container.train_preprocessed_features[columns_for_prediction]
    test_for_prediction = processed_container.test_preprocessed_features[columns_for_prediction]

    logged_str = 'Generating training and test set predictions'
    logged_str += ' (expected scores).' if configuration['predict_expected_scores'] else '.'
    logger.info(logged_str)
    (pred_config,
     pred_data_container) = modeler.predict_train_and_test(train_for_prediction,
                                                           test_for_prediction,
                                                           processed_config)

    # Write out files
    writer.write_experiment_output(csvdir,
                                   pred_data_container,
                                   new_names_dict={'pred_test': 'pred_processed'},
                                   file_format=file_format)

    original_coef_file = join(csvdir, '{}_coefficients.{}'.format(pred_config['experiment_id'],
                                                                  file_format))

    # If coefficients file exists, then try to generate the scaled
    # coefficients and save them to a file
    if exists(original_coef_file):
        logger.info('Scaling the coefficients and saving them to disk')
        try:

            # scale coefficients, and return DataContainer w/ scaled coefficients
            scaled_data_container = modeler.scale_coefficients(pred_config)

        # raise an error if the coefficient file exists but the
        # coefficients are not available for the current model
        # which can happen if the user is re-running the same experiment
        # with the same ID but with a non-linear model whereas the previous
        # run of the same ID was with a linear model and the user has not
        # cleared the directory
        except RuntimeError:
            raise ValueError("It appears you previously ran an experiment with the "
                             "same ID using a linear model and saved its output to "
                             "the same directory. That output is interfering with "
                             "the current experiment. Either clear the contents "
                             "of the output directory or re-run the current "
                             "experiment using a different experiment ID.")
        else:

            # Write out scaled coefficients to disk
            writer.write_experiment_output(csvdir,
                                           scaled_data_container,
                                           file_format=file_format)

    # Add processed data_container frames to pred_data_container
    new_pred_data_container = pred_data_container + processed_container

    logger.info('Running prediction analyses.')
    (pred_analysis_config,
     pred_analysis_data_container) = analyzer.run_prediction_analyses(new_pred_data_container,
                                                                      pred_config)

    # Write out files
    writer.write_experiment_output(csvdir,
                                   pred_analysis_data_container,
                                   reset_index=True,
                                   file_format=file_format)
    # Initialize reporter
    reporter = Reporter(logger=logger)

    # generate the report
    logger.info('Starting report generation.')
    reporter.create_report(pred_analysis_config,
                           csvdir,
                           figdir) def run_summary(config_file_or_obj_or_dict,
                output_dir,
                overwrite_output=False,
                logger=None):
    """
    Run rsmsummarize experiment using the given configuration.

    Summarize several rsmtool experiments using the given configuration
    file, object, or dictionary. All outputs are generated under ``output_dir``.
    If ``overwrite_output`` is ``True``, any existing output in ``output_dir``
    is overwritten.

    Parameters
    ----------
    config_file_or_obj_or_dict : str or pathlib.Path or dict or Configuration
        Path to the experiment configuration file either a a string
        or as a ``pathlib.Path`` object. Users can also pass a
        ``Configuration`` object that is in memory or a Python dictionary
        with keys corresponding to fields in the configuration file. Given a
        configuration file, any relative paths in the configuration file
        will be interpreted relative to the location of the file. Given a
        ``Configuration`` object, relative paths will be interpreted
        relative to the ``configdir`` attribute, that _must_ be set. Given
        a dictionary, the reference path is set to the current directory.
    output_dir : str
        Path to the experiment output directory.
    overwrite_output : bool, optional
        If ``True``, overwrite any existing output under ``output_dir``.
        Defaults to ``False``.
    logger : logging object, optional
        A logging object. If ``None`` is passed, get logger from ``__name__``.
        Defaults to ``None``.

    Raises
    ------
    IOError
        If ``output_dir`` already contains the output of a previous experiment
        and ``overwrite_output`` is ``False``.
    """
    logger = logger if logger else logging.getLogger(__name__)

    # create the 'output' and the 'figure' sub-directories
    # where all the experiment output such as the CSV files
    # and the box plots will be saved
    csvdir = abspath(join(output_dir, 'output'))
    figdir = abspath(join(output_dir, 'figure'))
    reportdir = abspath(join(output_dir, 'report'))

    os.makedirs(csvdir, exist_ok=True)
    os.makedirs(figdir, exist_ok=True)
    os.makedirs(reportdir, exist_ok=True)

    # Raise an error if the specified output directory
    # already contains a non-empty `output` directory, unless
    # `overwrite_output` was specified, in which case we assume
    # that the user knows what she is doing and simply
    # output a warning saying that the report might
    # not be correct.
    non_empty_csvdir = exists(csvdir) and listdir(csvdir)
    if non_empty_csvdir:
        if not overwrite_output:
            raise IOError("'{}' already contains a non-empty 'output' "
                          "directory.".format(output_dir))
        else:
            logger.warning("{} already contains a non-empty 'output' directory. "
                           "The generated report might contain "
                           "unexpected information from a previous "
                           "experiment.".format(output_dir))

    configuration = configure('rsmsummarize', config_file_or_obj_or_dict)

    logger.info('Saving configuration file.')
    configuration.save(output_dir)

    # get the list of the experiment dirs
    experiment_dirs = configuration['experiment_dirs']

    # Get experiment names if any
    experiment_names = configuration.get('experiment_names')
    experiment_names = experiment_names if experiment_names else [None] * len(experiment_dirs)
    dirs_with_names = zip(experiment_dirs, experiment_names)

    # check the experiment dirs and assemble the list of csvdir and jsons
    all_experiments = []
    for (experiment_dir, experiment_name) in dirs_with_names:
        experiments = check_experiment_dir(experiment_dir,
                                           experiment_name,
                                           configuration.configdir)
        all_experiments.extend(experiments)

    # get the subgroups if any
    # Note: at the moment no comparison are reported for subgroups.
    # this option is added to the code to make it easier to add
    # subgroup comparisons in future versions
    subgroups = configuration.get('subgroups')

    general_report_sections = configuration['general_sections']

    # get any special sections that the user might have specified
    special_report_sections = configuration['special_sections']

    # get any custom sections and locate them to make sure
    # that they exist, otherwise raise an exception
    custom_report_section_paths = configuration['custom_sections']
    if custom_report_section_paths:
        logger.info('Locating custom report sections')
        custom_report_sections = Reporter.locate_custom_sections(custom_report_section_paths,
                                                                 configuration.configdir)
    else:
        custom_report_sections = []

    section_order = configuration['section_order']

    # Initialize reporter
    reporter = Reporter(logger=logger)

    # check all sections values and order and get the
    # ordered list of notebook files
    chosen_notebook_files = reporter.get_ordered_notebook_files(general_report_sections,
                                                                special_report_sections,
                                                                custom_report_sections,
                                                                section_order,
                                                                subgroups,
                                                                model_type=None,
                                                                context='rsmsummarize')

    # add chosen notebook files to configuration
    configuration['chosen_notebook_files'] = chosen_notebook_files

    # now generate the comparison report
    logger.info('Starting report generation')
    reporter.create_summary_report(configuration,
                                   all_experiments,
                                   csvdir) class ConfigurationGenerator:
    """
    Class to encapsulate automated batch-mode and interactive generation.

    Attributes
    ----------
    context : str
        Name of the command-line tool for which we are generating the
        configuration file.
    as_string : bool, optional
        If ``True``, return a formatted and indented string representation
        of the configuration, rather than a dictionary. Note that this only
        affects the batch-mode generation. Interactive generation always
        returns a string.
        Defaults to ``False``.
    suppress_warnings : bool, optional
        If ``True``, do not generate any warnings for batch-mode generation.
        Defaults to ``False``.
    use_subgroups : bool, optional
        If ``True``, include subgroup-related sections in the list of general sections
        in the configuration file.
        Defaults to ``False``.
    """

    def __init__(self,
                 context,
                 as_string=False,
                 suppress_warnings=False,
                 use_subgroups=False):  # noqa
        self.context = context
        self.use_subgroups = use_subgroups
        self.suppress_warnings = suppress_warnings
        self.as_string = as_string
        self.logger = logging.getLogger(__name__)

        # we need to save the first required and first optional field we will
        # insert since we will use them as sign posts to insert comments later
        self._required_fields = CHECK_FIELDS[self.context]['required']
        self._optional_fields = sorted(CHECK_FIELDS[self.context]['optional'])
        self._first_required_field = self._required_fields[0]
        self._first_optional_field = self._optional_fields[0]

    def _convert_to_string(self,
                           config_object,
                           insert_url_comment=True,
                           insert_required_comment=True,
                           insert_optional_comment=True):
        configuration = str(config_object)

        # insert the URL comment first, right above the first required field
        if insert_url_comment:
            base_url = 'https://rsmtool.readthedocs.io/en/stable'
            doc_slug = CONFIGURATION_DOCUMENTATION_SLUGS[self.context]
            doc_url = f"{base_url}/{doc_slug}"
            configuration = re.sub(fr'([ ]+)("{self._first_required_field}": [^,]+,\n)',
                                   fr'\1// Reference: {doc_url}\n\1\2',
                                   configuration)

        # insert first comment right above the first required field
        if insert_required_comment:
            configuration = re.sub(fr'([ ]+)("{self._first_required_field}": [^,]+,\n)',
                                   fr'\1// REQUIRED: replace "ENTER_VALUE_HERE" with the appropriate value!\n\1\2',
                                   configuration)

        # insert second comment right above the first optional field
        if insert_optional_comment:
            configuration = re.sub(fr'([ ]+)("{self._first_optional_field}": [^,]+,\n)',
                                   r'\1// OPTIONAL: replace default values below based on your data.\n\1\2',
                                   configuration)

        return configuration

    def _get_all_general_section_names(self):

        default_general_sections_value = DEFAULTS.get('general_sections', '')
        default_special_sections_value = DEFAULTS.get('special_sections', '')
        default_custom_sections_value = DEFAULTS.get('custom_sections', '')

        # if we are told ot use subgroups then just make up a dummy subgroup
        # value so that the subgroup-based sections will be included in the
        # section list. This value is not actually used in configuration file.
        subgroups_value = ['GROUP'] if self.use_subgroups else DEFAULTS.get('subgroups', '')
        return Reporter().determine_chosen_sections(default_general_sections_value,
                                                    default_special_sections_value,
                                                    default_custom_sections_value,
                                                    subgroups_value,
                                                    context=self.context)

    def interact(self):
        """
        Automatically generate an example configuration in interactive mode.

        Returns
        -------
        configuration : str
            The generated configuration as a formatted string.

        Note
        ----
        This method should *only* be used in terminals, and not in
        Jupyter notebooks.

        """
        # clear the screen first
        clear()

        # print the preamble and some instructions
        sys.stderr.write("\n")
        sys.stderr.write("Entering interactive mode:\n")
        sys.stderr.write(" - press ctrl-d to exit without generating a configuration\n")
        sys.stderr.write(" - press tab or start typing when choosing files/directories/models\n")
        sys.stderr.write(" - press enter to accept the default value for a field (underlined)\n")
        sys.stderr.write(" - press ctrl-c to cancel current entry for a field and enter again\n")
        sys.stderr.write(" - you may still need to edit the generated configuration\n")
        sys.stderr.write("\n")

        if not self.use_subgroups:
            sys.stderr.write("IMPORTANT: If you have subgroups and didn't specify the '-g' "
                             "option, exit now (ctrl-d) and re-run!\n")
            sys.stderr.write("\n")

        # instantiate a blank dictionary
        configdict = OrderedDict()

        # iterate over the required fields first, and then the (sorted) optional fields
        # keep track of which field type we are currently dealing with
        for field_type, field_name in chain(product(['required'], self._required_fields),
                                            product(['optional'], self._optional_fields)):

            # skip the subgroups field unless we were told to use subgroups
            if field_name == 'subgroups' and not self.use_subgroups:
                configdict['subgroups'] = DEFAULTS.get('subgroups')
                continue

            # if the field is not one that is meant to be filled interactively,
            # then just use its default value; for "general_sections", expand it
            # so that it is easy for the user to remove sections
            if field_name not in INTERACTIVE_MODE_METADATA:
                non_interactive_field_value = DEFAULTS.get(field_name, '')
                if field_name == 'general_sections':
                    non_interactive_field_value = self._get_all_general_section_names()
                configdict[field_name] = non_interactive_field_value
            else:
                # instantiate the interactive field first
                try:
                    interactive_field = InteractiveField(field_name,
                                                         field_type,
                                                         INTERACTIVE_MODE_METADATA[field_name])
                    configdict[field_name] = interactive_field.get_value()
                # if the user pressed Ctrl-D, then exit out of interactive mode
                # without generating anything and return an empty string
                except EOFError:
                    sys.stderr.write("\n")
                    sys.stderr.write("You exited interactive mode without a configuration.")
                    sys.stderr.write("\n")
                    return ''
                # otherwise get the field value and save it

        # create a Configuration instance from the dictionary we just generated
        sys.stderr.write("\n")
        config_object = Configuration(configdict,
                                      configdir=os.getcwd(),
                                      context=self.context)
        # convert the Configuration object to a string - we are using
        # a special wrapper method since we also want to insert comments
        return self._convert_to_string(config_object, insert_required_comment=False)

    def generate(self):
        """
        Automatically generate an example configuration in batch mode.

        Returns
        -------
        configuration : dict or str
            The generated configuration either as a dictionary or
            a formatted string, depending on the value of ``as_string``.
        """
        # instantiate a dictionary that remembers key insertion order
        configdict = OrderedDict()

        # insert the required fields first and give them a dummy value
        for required_field in self._required_fields:
            configdict[required_field] = 'ENTER_VALUE_HERE'

        # insert the optional fields in alphabetical order
        for optional_field in self._optional_fields:

            # to make it easy for users to add/remove sections, we should
            # populate the `general_sections` field with an explicit list
            # instead of the default value which is simply ``['all']``. To
            # do this, we can use the reporter class.
            if optional_field == 'general_sections':
                configdict['general_sections'] = self._get_all_general_section_names()
            else:
                configdict[optional_field] = DEFAULTS.get(optional_field, '')

        # create a Configuration object
        config_object = Configuration(configdict,
                                      configdir=os.getcwd(),
                                      context=self.context)

        # if we were asked for string output, then convert this dictionary to
        # a string that will also insert some useful comments
        if self.as_string:
            configuration = self._convert_to_string(config_object)
        # otherwise we just return the dictionary underlying the Configuration object
        else:
            configuration = config_object._config

        # print out a warning to make it clear that it cannot be used as is
        if not self.suppress_warnings:
            self.logger.warning("Automatically generated configuration files MUST "
                                "be edited to add values for required fields and "
                                "even for optional ones depending on your data.")

        # return either the Configuration object or the string
        return configuration def setup_rsmcmd_parser(name,
                        uses_output_directory=True,
                        allows_overwriting=False,
                        extra_run_options=[],
                        uses_subgroups=False):
    """
    Create argument parsers for RSM command-line utilities.

    Since the various RSM command-line utilities (``rsmtool``, ``rsmeval``,
    ``rsmcompare``, etc.) have very similar argument parsers, refactoring that
    shared code out into this helper function makes it easier to extend
    and modify the command-line interface to all utilities at the same time.
    In addition, it also improves the consistency among the tools.

    By default, this function adds the following options to the parser:
      - ``config_file`` : a positional argument for the tool's configuration file
      - ``-V``/``--version`` : an optional argument to print out the package version

    If ``uses_output_directory`` is ``True``, an ``output_dir`` positional
    argument will be added to the "run" subcommand parser.

    If ``allows_overwriting`` is ``True``, an ``-f``/``--force``
    optional argument will be added to the "run" subcommand parser.

    The ``extra_run_options`` list should contain a list of ``CmdOption``
    instances which are added to the "run" subcommand parser one by one.

    If ``uses_subgroups`` is ``True``, a ``--subgroups`` optional
    argument will be added to the "generate" subcommand parser.

    Parameters
    ----------
    name : str
        The name of the command-line tool for which we need the parser.
    uses_output_directory : bool, optional
        Add the ``output_dir`` positional argument to the "run" subcommand
        parser. This argument means that the respective tool uses an output
        directory to store its various outputs.
        Defaults to ``True``.
    allows_overwriting : bool, optional
        Add the ``-f``/``-force_write`` optional argument to the "run" subcommand
        parser. This argument allows the output for the respective
        tool to be overwritten even if it already exists (file) or contains
        output (directory).
        Defaults to ``False``.
    extra_run_options : list, optional
        Any additional options to be added to the "run" subcommand parser,
        each specified as a ``CmdOption`` instance.
        Defaults to ``[]``.
    uses_subgroups : bool, optional
        Add the ``--subgroups`` optional argument to the "generate" subcommand
        parser. This argument means that the tool for which we are automatically
        generating a configuration file includes additional information when
        subgroup information is available.
        Defaults to ``False``.

    Returns
    -------
    parser : arpgarse.ArgumentParser
        A fully instantiated argument parser for the respective tool.

    Raises
    ------
    RuntimeError
        If any of the ``CmdOption`` instances specified in
        ``extra_run_options`` do not contain the ``dest`` and
        ``help`` attributes.

    Note
    ----
    This function is only meant to be used by RSMTool developers.
    """
    # a special callable to test whether configuration files exist
    # or not; this is nested because it is only used within this function
    # and should never be used externally
    def existing_configuration_file(string):
        if Path(string).exists():
            return string
        else:
            msg = 'The configuration file %r does not exist.' % string
            raise argparse.ArgumentTypeError(msg)

    # initialize an argument parser
    parser = argparse.ArgumentParser(prog=f"{name}")

    # we always want to have a version flag for the main parser
    parser.add_argument('-V',
                        '--version',
                        action='version',
                        version=VERSION_STRING,
                        help=f"show the {name} version number and exit")

    # each RSM command-line utility has two subcommands
    # - generate : used to auto-generate configuration files
    # - run : used to run experiments

    # let's set up the sub-parsers corresponding to these subcommands
    subparsers = parser.add_subparsers(dest='subcommand', title='subcommands')
    parser_generate = subparsers.add_parser('generate',
                                            help=f"automatically generate an "
                                                 f"{name} configuration file")
    parser_run = subparsers.add_parser('run',
                                       help=f"run an {name} experiment")

    ###################################################
    # Setting up options for the "generate" subparser #
    ###################################################
    if uses_subgroups:
        # we need to display a special help message for ``rsmxval``
        # since its config does not actually contain a sections list
        if name == "rsmxval":
            parser_generate.add_argument('-g',
                                         '--subgroups',
                                         dest='subgroups',
                                         action='store_true',
                                         default=False,
                                         help=f"if specified, {name} will ensure that "
                                              f"subgroup sections are included in "
                                              f"the various reports")
        else:
            parser_generate.add_argument('-g',
                                         '--subgroups',
                                         dest='subgroups',
                                         action='store_true',
                                         default=False,
                                         help=f"if specified, the generated {name} "
                                              f"configuration file will include the "
                                              f"subgroup sections in the general "
                                              f"sections list")

    parser_generate.add_argument('-q',
                                 '--quiet',
                                 dest='quiet',
                                 action='store_true',
                                 default=False,
                                 help="if specified, the warning about not "
                                      "using the generated configuration "
                                      "as-is will be suppressed.")

    parser_generate.add_argument('-i',
                                 '--interactive',
                                 dest='interactive',
                                 action='store_true',
                                 default=False,
                                 help=f"if specified, generate the {name} "
                                      f"configuration file interactively")

    ##############################################
    # Setting up options for the "run" subparser #
    ##############################################

    # since this is an RSMTool command-line utility, we will
    # always need a configuration file
    parser_run.add_argument('config_file',
                            type=existing_configuration_file,
                            help=f"the {name} JSON configuration file to run")

    # if it uses an output directory, let's add that
    if uses_output_directory:
        parser_run.add_argument('output_dir',
                                nargs='?',
                                default=os.getcwd(),
                                help="the output directory where all the files "
                                     "for this run will be stored")

    # if it allows overwrting the output directory, let's add that
    if allows_overwriting:
        parser_run.add_argument('-f',
                                '--force',
                                dest='force_write',
                                action='store_true',
                                default=False,
                                help=f"if specified, {name} will overwrite the "
                                     f"contents of the output file or directory "
                                     f"even if it contains the output of a "
                                     f"previous run ")

    # add any extra options passed in for the rub subcommand;
    for parser_option in extra_run_options:

        # construct the arguments and keyword arguments needed for the
        # `add_argument()` call to the parser
        argparse_option_args = []
        argparse_option_kwargs = {}

        # first add the destination and the help string
        argparse_option_kwargs["dest"] = f"{parser_option.dest}"
        argparse_option_kwargs["help"] = f"{parser_option.help}"

        # now add any optional information
        if parser_option.shortname is not None:
            argparse_option_args.append(f"-{parser_option.shortname}")
        if parser_option.longname is not None:
            argparse_option_args.append(f"--{parser_option.longname}")
        if parser_option.action is not None:
            argparse_option_kwargs['action'] = f"{parser_option.action}"
        if parser_option.default is not None:
            argparse_option_kwargs["default"] = f"{parser_option.default}"
        if parser_option.required is not None:
            try:
                assert type(parser_option.required) == bool
            except AssertionError:
                raise TypeError(f"the 'required' field for CmdOption must be "
                                f"boolean, you specified '{parser_option.required}'")
            else:
                argparse_option_kwargs["required"] = parser_option.required
        if parser_option.nargs is not None:
            argparse_option_kwargs['nargs'] = f"{parser_option.nargs}"

        # add this argument to the parser
        parser_run.add_argument(*argparse_option_args, **argparse_option_kwargs)

    return parser VALID_PARSER_SUBCOMMANDS = ['generate', 'run'] def combine_fold_prediction_files(foldsdir, file_format="csv"):
    """
    Combine predictions from all folds into a single data frame.

    Parameters
    ----------
    foldsdir : str
        The directory which stores the output for each fold experiment.
    file_format : str
        The file format (extension) for the file to be written to disk.
        One of {"csv", "xlsx", "tsv"}.
        Defaults to "csv".

    Returns
    -------
    df_all_predictions : pandas DataFrame
        Data frame containing the combined predictions for all folds.
    """
    # initialize empty list to hold each prediction data frame
    prediction_dfs = []

    # iterate over each fold in the given directory & read and save predictions
    for prediction_file in Path(foldsdir).glob(f"**/*pred_processed*.{file_format}"):
        df_fold_predictions = DataReader.read_from_file(prediction_file, converters={'spkitemid': str})
        prediction_dfs.append(df_fold_predictions)

    # concatenate all of the predictions into a single frame using
    # "spkitemid" column which must exist since this is the output of RSMTool
    df_all_predictions = concat(prediction_dfs, keys="spkitemid").reset_index(drop=True)

    return df_all_predictions def create_xval_files(configuration, output_dir, logger=None):
    """
    Create all files needed for the cross-validation experiment.

    Parameters
    ----------
    configuration : rsmtool.configuration_parser.Configuration
        A Configuration object holding the user-specified configuration
        for "rsmxval".
    output_dir : str
        Path to the output directory specified for "rsmxval".
    logger : None, optional
        Logger object used to log messages from this function.

    Returns
    -------
    df_train : pandas DataFrame
        DataFrame containing the raw training data file.
    folds : int
        The number of folds for which the files were created.

    Raises
    ------
    FileNotFoundError
        If the training data file specified in the configuration is not found.
    """
    # instantiate a logger if one is not given
    logger = logger if logger else logging.getLogger(__name__)

    # get the paths to the "<output_dir>/folds" directory as well as to
    # the "<output_dir>/final-model" directory since those are the two
    # directories under which we will write files
    foldsdir = Path(output_dir) / "folds"
    modeldir = Path(output_dir) / "final-model"

    # get the file format specified by the user
    given_file_format = configuration["file_format"]

    # locate the training data file, the folds file, and feature file(s)
    located_filepaths = {}
    located_filepaths["train"] = DataReader.locate_files(configuration.get("train_file"),
                                                         configuration.configdir)
    if not located_filepaths["train"]:
        raise FileNotFoundError('The training data file was not found: '
                                '{}'.format(repr(configuration.get("train_file"))))

    # "features" could be a list of features so check for that
    additional_filenames = ["folds_file", "feature_subset_file"]
    if isinstance(configuration.get("features"), str):
        additional_filenames.append("features")

    for additional_filename in additional_filenames:
        if additional_file := configuration.get(additional_filename):
            located_filepaths[additional_filename] = DataReader.locate_files(additional_file,
                                                                             configuration.configdir)

    # read the training file into a dataframe
    df_train = DataReader.read_from_file(located_filepaths["train"])

    # we need to sub-sample the full training data file to create a dummy
    # test file that we need to use when running RSMTool on the full
    # training dataset to get the model/feature descriptives; we use 10%
    # of the training data to create this dummy test set
    df_sampled_test = df_train.sample(frac=0.1,
                                      replace=False,
                                      random_state=1234567890,
                                      axis=0)
    DataWriter.write_frame_to_file(df_sampled_test,
                                   str(modeldir / "dummy_test"),
                                   file_format=given_file_format,
                                   index=False)

    # read the folds file if it exists, otherwise use specified number of folds
    folds_file = located_filepaths.get("folds_file")
    if folds_file and Path(folds_file).exists():
        cv_folds = load_cv_folds(located_filepaths["folds_file"])
        folds = len(set(cv_folds.values()))
        logger.info(f"Using {folds} folds specified in {located_filepaths['folds_file']}")
        logo = LeaveOneGroupOut()
        id_column = configuration.get("id_column")
        try:
            train_ids = df_train[id_column]
        except KeyError:
            logger.error("Column f{id_column} not found! Check the value of "
                         "'id_column' in the configuration file.")
        else:
            fold_groups = [cv_folds[train_id] for train_id in train_ids.values]
            fold_generator = logo.split(range(len(df_train)), y=None, groups=fold_groups)
    else:
        # if we are in this code path but the configuration did
        # specify "folds_file", then we must not have found the file
        # so raise a warning to that effect
        if configuration.get("folds_file"):
            logging.warning("Specified folds file not found. Ignoring.")
        folds = configuration.get("folds")
        logger.info(f"Generating {folds} folds after shuffling")
        kfold = KFold(n_splits=folds, random_state=1234567890, shuffle=True)
        fold_generator = kfold.split(range(len(df_train)))

    # iterate over each of the folds and generate an rsmtool configuration file
    # which is then saved to disk in a directory specific to each fold
    for fold_num, (fold_train_indices,
                   fold_test_indices) in enumerate(fold_generator, start=1):

        # get the train and test files for this fold and
        # write them out to disk
        df_fold_train = df_train.loc[fold_train_indices]
        df_fold_test = df_train.loc[fold_test_indices]
        this_fold_dir = Path(foldsdir) / f"{fold_num:02}"
        this_fold_dir.mkdir()
        fold_train_file_prefix = str(this_fold_dir / "train")
        DataWriter.write_frame_to_file(df_fold_train,
                                       fold_train_file_prefix,
                                       file_format=given_file_format,
                                       index=False)
        fold_test_file_prefix = str(this_fold_dir / "test")
        DataWriter.write_frame_to_file(df_fold_test,
                                       fold_test_file_prefix,
                                       file_format=given_file_format,
                                       index=False)

        # update the value of "train_file" and add "test_file"
        # & "test_label_column" in the configuration file
        fold_configuration = configuration.copy(deep=True)
        fold_configuration["train_file"] = f"{fold_train_file_prefix}.{given_file_format}"
        fold_configuration["test_file"] = f"{fold_test_file_prefix}.{given_file_format}"
        fold_configuration["test_label_column"] = configuration["train_label_column"]

        # use the same file format as the user specified
        fold_configuration["file_format"] = given_file_format

        # update "experiment_id" and "description" in configuration file
        fold_configuration["experiment_id"] = f"{configuration['experiment_id']}_fold{fold_num:02}"
        fold_configuration["description"] = f"{configuration['description']} (Fold {fold_num:02})"

        # if "feature_subset" was specified in the original configuration,
        # then we need it in the fold configuration as well
        if subset := configuration.get("feature_subset"):
            fold_configuration["feature_subset"] = subset

        # update "configdir" and "context" attributes in configuration file
        fold_configuration.configdir = str(this_fold_dir)
        fold_configuration.context = "rsmtool"
        with open(this_fold_dir / "rsmtool.json", "w") as fold_config_file:
            fold_config_file.write(str(fold_configuration))

        # copy "features" or "features_subset_file" to the
        # same directory as where the configuration was saved;
        # note that if features is a list instead of a file,
        # it will get ignored automatically
        for filename in ["features", "feature_subset_file"]:
            if filepath := located_filepaths.get(filename):
                copyfile(filepath, this_fold_dir / Path(filepath).name)

    return df_train, folds def process_fold(fold_num, foldsdir):
    """
    Run RSMTool on the specified numbered fold.

    Parameters
    ----------
    fold_num : int
        The number of the fold to run RSMTool on.
    foldsdir : str
        The directory which stores the output for each fold experiment.
    """
    # create a directory name specific to the given fold
    fold_dir = Path(foldsdir) / f"{fold_num:02}"

    # get a file logger since we cannot show the log for RSMTool on screen
    log_file_path = fold_dir / "rsmtool.log"
    logger = get_file_logger(f"fold{fold_num:02}", log_file_path)

    # run RSMTool on the given fold and have it log its output to the file
    config_file = fold_dir / "rsmtool.json"
    run_experiment(config_file, fold_dir, False, logger=logger) class LogFormatter(logging.Formatter):
    """
    Custom logging formatter.

    Note
    ----
    This class is adapted from https://stackoverflow.com/q/1343227.
    """

    info_fmt = "%(msg)s"
    warn_fmt = "WARNING: %(msg)s"

    err_fmt = "ERROR: %(msg)s"
    dbg_fmt = "DEBUG: %(module)s: %(lineno)d: %(msg)s"

    def __init__(self, fmt="%(levelno)s: %(msg)s"):  # noqa: D107
        logging.Formatter.__init__(self, fmt)

    def format(self, record):
        """
        Format the given record.

        Parameters
        ----------
        record : logging.LogRecord
            The record to format.
        """
        # Save the original format configured by the user
        # when the logger formatter was instantiated
        format_orig = self._fmt

        # Replace the original format with one customized by logging level
        if record.levelno == logging.DEBUG:
            self._fmt = LogFormatter.dbg_fmt
            self._style = logging.PercentStyle(self._fmt)

        elif record.levelno == logging.WARNING:
            self._fmt = LogFormatter.warn_fmt
            self._style = logging.PercentStyle(self._fmt)

        elif record.levelno == logging.INFO:
            self._fmt = LogFormatter.info_fmt
            self._style = logging.PercentStyle(self._fmt)

        elif record.levelno == logging.ERROR:
            self._fmt = LogFormatter.err_fmt
            self._style = logging.PercentStyle(self._fmt)

        # Call the original formatter class to do the grunt work
        result = logging.Formatter.format(self, record)

        # Restore the original format configured by the user
        self._fmt = format_orig

        return result @contextlib.contextmanager
def tqdm_joblib(tqdm_object):
    """
    Patch joblib to report into tqdm progress bar given as argument.

    This function creates a joblib-compatible context manager with
    a progress bar attached.

    Adapted from: https://stackoverflow.com/a/58936697

    Parameters
    ----------
    tqdm_object : tqdm progress bar
        The given tqdm progress bar into which joblib should report
    """
    class TqdmBatchCompletionCallback(joblib.parallel.BatchCompletionCallBack):
        def __call__(self, *args, **kwargs):
            tqdm_object.update(n=self.batch_size)
            return super().__call__(*args, **kwargs)

    old_batch_callback = joblib.parallel.BatchCompletionCallBack
    joblib.parallel.BatchCompletionCallBack = TqdmBatchCompletionCallback
    try:
        yield tqdm_object
    finally:
        joblib.parallel.BatchCompletionCallBack = old_batch_callback
        tqdm_object.close() def get_file_logger(logger_name, log_file_path):
    """
    Create and return a file-based logger.

    Instantiate a logger with the given name and attach a
    file handler that uses the given log file. The logging
    level is set to INFO.

    Parameters
    ----------
    logger_name : str
        Name to use for the logger.
    log_file_path : str
        File path to which the logger should output messages.

    Returns
    -------
    logger
        A logging.Logger object attached to a file handler.
    """
    logger = logging.getLogger(logger_name)
    logger.propagate = False
    formatter = logging.Formatter('%(message)s')
    file_handler = logging.FileHandler(log_file_path, mode="w")
    file_handler.setFormatter(formatter)
    file_handler.setLevel(logging.INFO)
    logger.addHandler(file_handler)
    logger.setLevel(logging.INFO)
    return logger class DataWriter:
    """Class to write out DataContainer objects."""

    def __init__(self, experiment_id=None):  # noqa: D107
        self._id = experiment_id

    @staticmethod
    def write_frame_to_file(df, name_prefix, file_format="csv", index=False, **kwargs):
        """
        Write given data frame to disk with given name and file format.

        Parameters
        ----------
        df : pandas DataFrame
            Data frame to write to disk
        name_prefix : str
            The complete prefix for the file to be written to disk.
            This includes everything except the extension.
        file_format : str
            The file format (extension) for the file to be written to disk.
            One of {"csv", "xlsx", "tsv"}.
            Defaults to "csv".
        index : bool, optional
            Whether to include the index in the output file.
            Defaults to ``False``.

        Raises
        ------
        KeyError
            If ``file_format`` is not valid.
        """
        file_format = file_format.lower()

        if file_format == 'csv':
            name_prefix += '.csv'
            df.to_csv(name_prefix, index=index, **kwargs)

        elif file_format == 'tsv':
            name_prefix += '.tsv'
            df.to_csv(name_prefix, index=index, sep='\t', **kwargs)

        # Added jsonlines for experimental purposes, but leaving
        # this out of the documentation at this stage
        elif file_format == 'jsonlines':
            name_prefix += '.jsonlines'
            df.to_json(name_prefix, orient='records', lines=True, **kwargs)

        elif file_format == 'xlsx':
            name_prefix += '.xlsx'
            df.to_excel(name_prefix, index=index, **kwargs)

        else:
            raise KeyError("Please make sure that the `file_format` specified "
                           "is one of the following:\n{`csv`, `tsv`, `xlsx`}.\n"
                           f"You specified {file_format}.")

    def write_experiment_output(self,
                                csvdir,
                                container_or_dict,
                                dataframe_names=None,
                                new_names_dict=None,
                                include_experiment_id=True,
                                reset_index=False,
                                file_format='csv',
                                index=False,
                                **kwargs):
        """
        Write out each of the named frames to disk.

        This function writes out each of the given list of data frames as a
        ".csv", ".tsv", or ``.xlsx`` file in the given directory. Each data
        frame was generated as part of running an RSMTool experiment. All files
        are prefixed with the given experiment ID and suffixed with either the
        name of the data frame in the DataContainer (or dict) object, or a new
        name if ``new_names_dict`` is specified. Additionally, the indexes in
        the  data frames are reset if so specified.

        Parameters
        ----------
        csvdir : str
            Path to the output experiment sub-directory that will
            contain the CSV files corresponding to each of the data frames.
        container_or_dict : container.DataContainer or dict
            A DataContainer object or dict, where keys are data frame
            names and values are ``pd.DataFrame`` objects.
        dataframe_names : list of str, optional
            List of data frame names, one for each of the data frames.
            Defaults to ``None``.
        new_names_dict : dict, optional
            New dictionary with new names for the data frames, if desired.
            Defaults to ``None``.
        include_experiment_id : str, optional
            Whether to include the experiment ID in the file name.
            Defaults to ``True``.
        reset_index : bool, optional
            Whether to reset the index of each data frame
            before writing to disk.
            Defaults to ``False``.
        file_format : str, optional
            The file format in which to output the data.
            One of {"csv", "xlsx", "tsv"}.
            Defaults to "csv".
        index : bool, optional
            Whether to include the index in the output file.
            Defaults to ``False``.

        Raises
        ------
        KeyError
            If ``file_format`` is not valid, or a named data frame
            is not present in ``container_or_dict``.
        """
        container_or_dict = container_or_dict.copy()

        # If no `dataframe_names` specified, use all names
        if dataframe_names is None:
            dataframe_names = container_or_dict.keys()

        # Otherwise, check to make sure all specified names
        # are actually in the DataContainer
        else:
            for name in dataframe_names:
                if name not in container_or_dict:
                    raise KeyError("The name `{}` is not in the container "
                                   "or dictionary.".format(name))

        # Loop through DataFrames, and save
        # output in specified format
        for dataframe_name in dataframe_names:

            df = container_or_dict[dataframe_name]
            if df is None:
                raise KeyError('The DataFrame `{}` does not exist.'.format(dataframe_name))

            # If the DataFrame is empty, skip it
            if df.empty:
                continue

            # If there is a desire to rename the DataFrame,
            # get the new name
            if new_names_dict is not None:
                if dataframe_name in new_names_dict:
                    dataframe_name = new_names_dict[dataframe_name]

            # Reset the index, if desired
            if reset_index:
                df.index.name = ''
                df.reset_index(inplace=True)

            # If include_experiment_id is True, and the experiment_id exists
            # include it in the file name; otherwise, do not include it.
            if include_experiment_id and self._id is not None:
                outfile = join(csvdir, '{}_{}'.format(self._id, dataframe_name))
            else:
                outfile = join(csvdir, dataframe_name)

            # write out the frame to disk in the given file
            self.write_frame_to_file(df,
                                     outfile,
                                     file_format=file_format,
                                     index=index,
                                     **kwargs)

    def write_feature_csv(self,
                          featuredir,
                          data_container,
                          selected_features,
                          include_experiment_id=True,
                          file_format='csv'):
        """
        Write out the selected features to disk.

        Parameters
        ----------
        featuredir : str
            Path to the experiment output directory where the
            feature JSON file will be saved.
        data_container : container.DataContainer
            A data container object.
        selected_features : list of str
            List of features that were selected for model building.
        include_experiment_id : bool, optional
            Whether to include the experiment ID in the file name.
            Defaults to ``True``.
        file_format : str, optional
            The file format in which to output the data.
            One of {"csv", "tsv", "xlsx"}.
            Defaults to "csv".
        """
        df_feature_specs = data_container['feature_specs']

        # Select specific features used in training
        df_selected = df_feature_specs[df_feature_specs['feature'].isin(selected_features)]

        # Replace existing `feature_specs` with selected features specs
        data_container.add_dataset({'frame': df_selected,
                                    'name': 'feature_specs'}, update=True)

        makedirs(featuredir, exist_ok=True)
        self.write_experiment_output(featuredir,
                                     data_container,
                                     ['feature_specs'],
                                     {'feature_specs': 'selected'},
                                     include_experiment_id=include_experiment_id,
                                     file_format=file_format)
import logging
import sys
from os import listdir, makedirs
from os.path import abspath, exists, join
from pathlib import Path
from joblib.parallel import Parallel, delayed
from tqdm import tqdm
from .configuration_parser import Configuration, configure
from .rsmeval import run_evaluation
from .rsmtool import run_experiment
from .rsmsummarize import run_summary
from .utils.commandline import ConfigurationGenerator, setup_rsmcmd_parser
from .utils.constants import VALID_PARSER_SUBCOMMANDS
from .utils.cross_validation import combine_fold_prediction_files, create_xval_files, process_fold
from .utils.logging import LogFormatter, tqdm_joblib, get_file_logger
from .writer import DataWriter
        "description": f"{configuration['description']} (Fold Summary)",
        "file_format": given_file_format,
        "use_thumbnails": f"{configuration['use_thumbnails']}"
    }
    fold_summary_configuration = Configuration(fold_summary_configdict,
                                               configdir=summarydir,
                                               context="rsmsummarize")

    # run rsmsummarize on all of the fold directories
    summary_logger = get_file_logger("fold-summary",
                                     Path(summarydir) / "rsmsummarize.log")
    run_summary(fold_summary_configuration, summarydir, False, logger=summary_logger)

    # combine all of the fold prediction files for evaluation
    df_predictions = combine_fold_prediction_files(foldsdir, given_file_format)

    # if there were subgroups and a second human score column specified, then
    # we need to add those to the combined predictions file as well
    id_column = configuration["id_column"]
    columns_to_use = [id_column]
    if subgroups := configuration["subgroups"]:
        columns_to_use.extend(subgroups)
    if second_human_score_column := configuration["second_human_score_column"]:
        columns_to_use.append(second_human_score_column)
    if len(columns_to_use) > 1:
        df_to_add = df_train[columns_to_use]
        df_predictions = df_predictions.merge(df_to_add,
                                              left_on="spkitemid",
                                              right_on=id_column)
        # drop any extra ID column if we have added one
        if id_column != "spkitemid":
            df_predictions.drop(id_column, axis=1, inplace=True)

    # write out the predictions file to disk
    predictions_file_prefix = str(Path(evaldir) / "xval_predictions")
    DataWriter.write_frame_to_file(df_predictions,
                                   predictions_file_prefix,
                                   file_format=given_file_format,
                                   index=False)

    # create a new rsmeval configuration dictionary;
    # note that we do not want to set "id_column" and "human_score_column"
    # here since those columns already have default names given that the
    # predictions are generated from RSMTool
    evaluation_configdict = {
        "experiment_id": f"{configuration['experiment_id']}_evaluation",
        "description": f"{configuration['description']} (Evaluating Fold Predictions)",
        "predictions_file": f"{predictions_file_prefix}.{given_file_format}",
        "system_score_column": "scale" if configuration["use_scaled_predictions"] else "raw",
        "scale_with": "asis" if configuration["use_scaled_predictions"] else "raw",
        "trim_min": int(f"{configuration['trim_min']}"),
        "trim_max": int(f"{configuration['trim_max']}"),
        "file_format": given_file_format
    }

    # copy over the relevant configuration fields from the main configuration
    for field_name in ["second_human_score_column",
                       "candidate_column",
                       "flag_column",
                       "exclude_zero_scores",
                       "min_items_per_candidate",
                       "rater_error_variance",
                       "subgroups",
                       "trim_tolerance",
                       "use_thumbnails"]:
        evaluation_configdict[field_name] = configuration[field_name]

    # create an rsmeval configuration object with this dictionary
    evaluation_configuration = Configuration(evaluation_configdict,
                                             configdir=evaldir,
                                             context="rsmeval")

    # run rsmeval on the combined predictions file
    logger.info("Evaluating combined fold predictions")
    eval_logger = get_file_logger("evaluation", Path(evaldir) / "rsmeval.log")
    run_evaluation(evaluation_configuration, evaldir, False, logger=eval_logger)

    # run rsmtool on the full dataset and generate only model/feature
    # descriptives report; we will use the dummy test set that we
    # created by calling `create_xval_files()` for this purpose
    logger.info("Training model on full data")
    model_logger = get_file_logger("final_model", Path(modeldir) / "rsmtool.log")
    final_rsmtool_configdict = configuration.to_dict().copy()
    final_rsmtool_configdict["experiment_id"] = f"{configuration['experiment_id']}_model"
    final_rsmtool_configdict["test_file"] = str(Path(modeldir) / f"dummy_test.{given_file_format}")
    final_rsmtool_configdict["test_label_column"] = final_rsmtool_configdict["train_label_column"]

    # remove by-group sections if we don't have the info
    sections_to_use = []
    for section in FINAL_MODEL_SECTION_LIST:
        if section.endswith("_by_group") and not subgroups:
            continue
        sections_to_use.append(section)
    final_rsmtool_configdict["general_sections"] = sections_to_use
    final_rsmtool_configuration = Configuration(final_rsmtool_configdict,
                                                configdir=configuration.configdir,
                                                context="rsmtool")
    run_experiment(final_rsmtool_configuration, modeldir, False, logger=model_logger)


def main():  # noqa: D103

    # set up the basic logging configuration
    formatter = LogFormatter()

    # we need two handlers, one that prints to stdout
    # for the "run" command and one that prints to stderr
    # from the "generate" command; the latter is important
    # because do not want the warning to show up in the
    # generated configuration file
    stdout_handler = logging.StreamHandler(sys.stdout)
    stdout_handler.setFormatter(formatter)

    stderr_handler = logging.StreamHandler(sys.stderr)
    stderr_handler.setFormatter(formatter)

    logging.root.setLevel(logging.INFO)
    logger = logging.getLogger(__name__)

    # set up an argument parser via our helper function

    parser = setup_rsmcmd_parser('rsmxval',