class Configuration:
    """
    Configuration class.

    Encapsulates all of the configuration parameters and methods to 
    access these parameters.
    """

    def __init__(self,
                 configdict,
                 *,
                 configdir=None,
                 context='rsmtool',
                 logger=None):
        """
        Create an object of the `Configuration` class.

        This method can be used to directly instantiate a Configuration
        object.

        Parameters
        ----------
        configdict : dict
            A dictionary of configuration parameters.
            The dictionary must be a valid configuration dictionary
            with default values filled as necessary.
        configdir : str, optional, keyword-only
            The reference path used to resolve any relative paths
            in the configuration object. When ``None``, will be set during
            initialization to the current working directory.
            Defaults to ``None``,
        context : str
            The context of the tool. One of {"rsmtool", "rsmeval", "rsmcompare",
            "rsmpredict", "rsmsummarize"}.
            Defaults to "rsmtool".
        logger : logging object, optional
            A logging object. If ``None`` is passed, get logger from ``__name__``.
            Defaults to ``None``.
        """
        if not isinstance(configdict, dict):
            raise TypeError('The input must be a dictionary.')

        # create a logger instance
        self.logger = logger if logger else logging.getLogger(__name__)

        # process and validate the configuration dictionary
        configdict = ConfigurationParser.process_config(configdict)
        configdict = ConfigurationParser.validate_config(configdict, context=context)

        # set configdir to `cwd` if not given and let the user know
        if configdir is None:
            configdir = Path(getcwd())
            self.logger.info("Configuration directory will be set to {}".format(configdir))
        else:
            configdir = Path(configdir).resolve()

        self._config = configdict
        self._configdir = configdir
        self._context = context

    def __contains__(self, key):
        """
        Check if the configuration object contains a given key.

        Parameters
        ----------
        key : str
            Key to check in the Configuration object.

        Returns
        -------
        key_check : bool
            ``True`` if key is in configuration object, else ``False``.
        """
        return key in self._config

    def __getitem__(self, key):
        """
        Get configuration value for the given key.

        Parameters
        ----------
        key : str
            Key to check in the configuration object

        Returns
        -------
        value
            The value in the configuration object dictionary.
        """
        return self._config[key]

    def __setitem__(self, key, value):
        """
        Set configuration value for the given key.

        Parameters
        ----------
        key : str
            Key to set in the configuration object.
        value
            The value to be set for the key.
        """
        self._config[key] = value

    def __len__(self):
        """
        Return the size of the configuration dictionary.

        Returns
        -------
        size : int
            The number of elements in the configuration dictionary.
        """
        return len(self._config)

    def __str__(self):
        """
        Return string representation of the configuration dictionary.

        Returns
        -------
        config_string : str
            A string representation of the underlying configuration
            dictionary as encoded by ``json.dumps()``. It only
            includes the configuration options that can be set by
            the user.
        """
        expected_fields = (CHECK_FIELDS[self._context]['required'] +
                           CHECK_FIELDS[self._context]['optional'])

        output_config = {k: v for k, v in self._config.items() if k in expected_fields}
        return json.dumps(output_config, indent=4, separators=(',', ': '))

    def __iter__(self):
        """
        Iterate through configuration object keys.

        Yields
        ------
        key : str
            A key in the config dictionary
        """
        for key in self.keys():
            yield key

    @property
    def configdir(self):
        """
        Get the path to configuration directory.

        Get the path to the configuration reference directory that
        will be used to resolve any relative paths in
        the configuration.

        Returns
        -------
        configdir : str
            The path to the configuration reference directory.
        """
        return str(self._configdir)

    @configdir.setter
    def configdir(self, new_path):
        """
        Set a new configuration reference directory.

        Parameters
        ----------
        new_path : str
            Path to the new configuration reference
            directory used to resolve any relative paths
            in the configuration object.
        """
        if new_path is None:
            raise ValueError("The `configdir` attribute cannot be set to `None` ")

        # TODO: replace `Path(abspath(new_path))` with `Path(new_path).resolve()
        # once this Windows bug is fixed: https://bugs.python.org/issue38671
        self._configdir = Path(abspath(new_path))

    @property
    def context(self):
        """Get the context."""
        return self._context

    @context.setter
    def context(self, new_context):
        """
        Set a new context.

        Parameters
        ----------
        new_context : str
            A new context for the configuration object.
        """
        self._context = new_context

    def get(self, key, default=None):
        """
        Get value or default for the given key.

        Parameters
        ----------
        key : str
            Key to check in the Configuration object.
        default, optional
            The default value to return, if no key exists.
            Defaults to ``None``.

        Returns
        -------
        value
            The value in the configuration object dictionary.
        """
        return self._config.get(key, default)

    def to_dict(self):
        """
        Get a dictionary representation of the configuration object.

        Returns
        -------
        config : dict
            The configuration dictionary.
        """
        return self._config

    def keys(self):
        """
        Return keys as a list.

        Returns
        -------
        keys : list of str
            A list of keys in the configuration object.
        """
        return [k for k in self._config.keys()]

    def values(self):
        """
        Return configuration values as a list.

        Returns
        -------
        values : list
            A list of values in the configuration object.
        """
        return [v for v in self._config.values()]

    def items(self):
        """
        Return configuration items as a list of tuples.

        Returns
        -------
        items : list of tuples
            A list of (key, value) tuples in the
            configuration object.
        """
        return [(k, v) for k, v in self._config.items()]

    def pop(self, key, default=None):
        """
        Remove and return an element from the object having the given key.

        Parameters
        ----------
        key : str
            Key to pop in the configuration object.
        default, optional
            The default value to return, if no key exists.
            Defaults to ``None``.

        Returns
        -------
        value
            The value removed from the object.
        """
        return self._config.pop(key, default)

    def copy(self, deep=True):
        """
        Return a copy of the object.

        Parameters
        ----------
        deep : bool, optional
            Whether to perform a deep copy.
            Defaults to ``True``.

        Returns
        -------
        copy : Configuration
            A new configuration object.
        """
        if deep:
            return deepcopy(self)
        return copy(self)

    def save(self, output_dir=None):
        """
        Save the configuration file to the output directory specified.

        Parameters
        ----------
        output_dir : str
            The path to the output directory. If ``None``, the current
            directory is used.
            Defaults to ``None``.
        """
        # save a copy of the main config into the output directory
        if output_dir is None:
            output_dir = Path(getcwd())

        # Create output directory, if it does not exist
        output_dir = Path(output_dir).resolve() / 'output'
        output_dir.mkdir(parents=True, exist_ok=True)

        id_field = ID_FIELDS[self._context]
        experiment_id = self._config[id_field]
        context = self._context
        outjson = output_dir / f"{experiment_id}_{context}.json"

        with outjson.open(mode='w') as outfile:
            outfile.write(str(self))

    def check_exclude_listwise(self):
        """
        Check for candidate exclusion.

        Check if we are excluding candidates based on number of responses, and
        add this to the configuration file.

        Returns
        -------
        exclude_listwise : bool
            Whether to exclude list-wise.
        """
        exclude_listwise = False
        if self._config.get('min_items_per_candidate'):
            exclude_listwise = True
        return exclude_listwise

    def check_flag_column(self,
                          flag_column='flag_column',
                          partition='unknown'):
        """
        Make sure the column name in ``flag_column`` is correctly specified.

        Get flag columns and values for filtering, if any, and convert single
        values to lists. Raises an exception if the column name in ``flag_column``
        is not correctly specified in the configuration file.

        Parameters
        ----------
        flag_column : str
            The flag column name to check. Currently used names are
            "flag_column" or "flag_column_test".
            Defaults to "flag_column".

        partition: str
            The data partition which is filtered based on the flag column name.
            One of {"train", "test", "both", "unknown"}.
            Defaults to "both".

        Returns
        -------
        new_filtering_dict : dict
            Properly formatted dictionary for the column
            name in ``flag_column``.

        Raises
        ------
        ValueError
            If the specified value of the column name in ``flag_column``
            is not a dictionary.
        ValueError
            If the value of ``partition`` is not in the expected list.
        ValueError
            If the value of ``partition`` does not match the ``flag_column``.
        """
        config = self._config

        new_filter_dict = {}

        flag_message = {"train": "training",
                        "test": "evaluating",
                        "both": "training and evaluating",
                        "unknown": "training and/or evaluating"}

        if partition not in flag_message:
            raise ValueError("Unknown value for partition: {} "
                             "This must be one of the following: {}."
                             "".format(partition, ','.join(flag_message.keys())))

        if flag_column == "flag_column_test":
            if partition in ["both", "train"]:
                raise ValueError("Conditions specified in 'flag_column_test' "
                                 "can only be applied to the evaluation partition.")

        if config.get(flag_column):

            original_filter_dict = config[flag_column]

            # first check that the value is a dictionary
            if not isinstance(original_filter_dict, dict):
                raise ValueError("`flag_column` must be a dictionary. "
                                 "Please refer to the documentation for "
                                 "further information.")

            for column in original_filter_dict:

                # if we were given a single value, convert it to list
                if not isinstance(original_filter_dict[column], list):
                    new_filter_dict[column] = [original_filter_dict[column]]
                    self.logger.warning("The filtering condition {}"
                                        " for column {} was converted "
                                        "to list. Only responses where "
                                        "{} == {} will be used for "
                                        "{} the "
                                        "model. You can ignore this "
                                        "warning if this is the correct "
                                        "interpretation of your "
                                        "configuration settings"
                                        ".".format(original_filter_dict[column],
                                                   column,
                                                   column,
                                                   original_filter_dict[column],
                                                   flag_message[partition])
                                        )
                else:
                    new_filter_dict[column] = original_filter_dict[column]

                    model_eval = ', '.join(map(str,
                                               original_filter_dict[column]))
                    self.logger.info("Only responses where "
                                     "{} equals one of the following values "
                                     "will be used for {} the model: "
                                     "{}.".format(column,
                                                  flag_message[partition],
                                                  model_eval))
        return new_filter_dict

    def get_trim_min_max_tolerance(self):
        """
        Get trim min, trim max, and tolerance values.

        Get the specified trim min and max, and trim_tolerance if any,
        and make sure they are numeric.

        Returns
        -------
        spec_trim_min : float
            Specified trim min value.
        spec_trim_max : float
            Specified trim max value.
        spec_trim_tolerance: float
            Specified trim tolerance value.
        """
        config = self._config

        spec_trim_min = config.get('trim_min', None)
        spec_trim_max = config.get('trim_max', None)
        spec_trim_tolerance = config.get('trim_tolerance', None)

        if spec_trim_min:
            spec_trim_min = float(spec_trim_min)
        if spec_trim_max:
            spec_trim_max = float(spec_trim_max)
        if spec_trim_tolerance:
            spec_trim_tolerance = float(spec_trim_tolerance)
        return (spec_trim_min, spec_trim_max, spec_trim_tolerance)

    def get_rater_error_variance(self):
        """
        Get specified rater error variance, if any, and make sure it's numeric.

        Returns
        -------
        rater_error_variance : float
            Specified rater error variance.
        """
        config = self._config

        rater_error_variance = config.get('rater_error_variance', None)

        if rater_error_variance:
            rater_error_variance = float(rater_error_variance)

        return rater_error_variance

    def get_default_converter(self):
        """
        Get default converter dictionary for data reader.

        Returns
        -------
        default_converter : dict
            The default converter for a train or test file.
        """
        string_columns = [self._config['id_column']]

        candidate = self._config.get('candidate_column')
        if candidate is not None:
            string_columns.append(candidate)

        subgroups = self._config.get('subgroups')
        if subgroups:
            string_columns.extend(subgroups)

        return dict([(column, str) for column in string_columns if column])

    def get_names_and_paths(self, keys, names):
        """
        Get a list of values for the given keys.

        Remove any values that are ``None``.

        Parameters
        -------
        keys : list
            A list of keys whose values to retrieve.
        names : list
            The default value to use if key cannot be found.
            Defaults to ``None``.

        Returns
        -------
        values : list
            The list of values.

        Raises
        ------
        ValueError
            If there are any duplicate keys or names.
        """
        assert len(keys) == len(names)

        # Make sure keys are not duplicated
        if not len(set(keys)) == len(keys):
            raise ValueError('The ``keys`` must be unique. However, the '
                             'following duplicate keys were found: {}.'
                             ''.format(', '.join([key for key, val in Counter(keys).items()
                                                  if val > 1])))
        # Make sure names are not duplicated
        if not len(set(names)) == len(names):
            raise ValueError('The``names`` must be unique. However, the '
                             'following duplicate names were found: {}.'
                             ''.format(', '.join([name for name, val in Counter(names).items()
                                                  if val > 1])))
        existing_names = []
        existing_paths = []
        for idx, key in enumerate(keys):

            path = self._config.get(key)

            # if the `features` field is a list,
            # do not include it in the container
            if key == 'features':
                if isinstance(path, list):
                    continue

            if path is not None:
                existing_paths.append(path)
                existing_names.append(names[idx])

        return existing_names, existing_paths class ConfigurationParser:
    """``ConfigurationParser`` class to create ``Configuration`` objects."""

    # initialize class logger attribute to None
    logger = None

    def __init__(self, pathlike, logger=None):
        """
        Instantiate a ``ConfigurationParser`` for a given config file path.

        Parameters
        ----------
        pathlike : str or pathlib.Path
            A string containing the path to the configuration file
            that is to be parsed. A ``pathlib.Path`` instance is also
            acceptable.
        logger : logging.Logger object, optional
            Custom logger object to use, if not ``None``. Otherwise
            a new logger is created.
            Defaults to ``None``.

        Raises
        ------
        FileNotFoundError
            If the given path does not exist.
        OSError
            If the given path is a directory, not a file.
        ValueError
            If the file at the given path does not have
            a valid extension (".json").
        """
        # if we passed in a string, convert it to a Path
        if isinstance(pathlike, str):
            pathlike = Path(pathlike)

        # raise an exception if the file does not exist
        if not pathlike.exists():
            raise FileNotFoundError(f"The configuration file {pathlike} "
                                    f"was not found.")

        # raise an exception if the user specified a directory
        if not pathlike.is_file():
            raise OSError(f"The given path {pathlike} should be a "
                          f"file, not a directory.")

        # make sure we have a file that ends in ".json"
        extension = pathlike.suffix.lower()
        if extension != '.json':
            raise ValueError(f"The configuration file must be in `.json` "
                             f"format. You specified: {extension}.")

        # set the various attributes to None
        self._filename = pathlike.name
        self._configdir = pathlike.resolve().parent

        # set the `logger` class attribute to the given logger, if provided
        # or else create a new logger
        self.__class__.logger = logger if logger else logging.getLogger(__name__)

    @staticmethod
    def _fix_json(json_string):
        """
        Fix the configuration JSON.

        Take a bit of JSON that might have bad quotes
        or capitalized booleans and fixes that stuff.

        Parameters
        ----------
        json_string : str
            A string to be reformatted for JSON parsing.

        Return
        ------
        json_string : str
            The updated string.
        """
        json_string = json_string.replace('True', 'true')
        json_string = json_string.replace('False', 'false')
        json_string = json_string.replace("'", '"')
        return json_string

    def _parse_json_file(self, filepath):
        """
        Parse the configuration JSON.

        A private method to parse JSON configuration files and return
        a Python dictionary.

        Parameters
        ----------
        filepath : pathlib.Path
            A ``pathlib.Path`` object containing the JSON configuration
            filepath.

        Returns
        -------
        configdict : dict
            A Python dictionary containing the parameters from the
            JSON configuration file.

        Raises
        ------
        ValueError
            If the JSON file could not be parsed.
        """
        try:
            configdict = parse_json_with_comments(filepath)
        except ValueError:
            raise ValueError('The main configuration file `{}` exists but '
                             'is formatted incorrectly. Please check that '
                             'each line ends with a comma, there is no comma '
                             'at the end of the last line, and that all quotes '
                             'match.'.format(filepath))

        return configdict

    def parse(self, context='rsmtool'):
        """
        Parse configuration file.

        Parse the configuration file for which this parser was
        instantiated.

        Parameters
        ----------
        context : str, optional
            Context of the tool in which we are validating. One of:
            {"rsmtool", "rsmeval", "rsmpredict", "rsmcompare",
            "rsmsummarize", "rsmxval"}.
            Defaults to "rsmtool".

        Returns
        -------
        configuration : Configuration
            A configuration object containing the parameters in the
            file that we instantiated the parser for.
        """
        filepath = self._configdir / self._filename
        configdict = self._parse_json_file(filepath)

        # create a new Configuration object which will automatically
        # process and validate the configuration
        # dictionary being passed in
        return Configuration(configdict,
                             configdir=self._configdir,
                             context=context)

    @classmethod
    def validate_config(cls, config, context='rsmtool'):
        """
        Validate configuration file.

        Ensure that all required fields are specified, add default values
        values for all unspecified fields, and ensure that all specified
        fields are valid.

        Parameters
        ----------
        context : str, optional
            Context of the tool in which we are validating.
            One of {"rsmtool", "rsmeval", "rsmpredict",
            "rsmcompare", "rsmsummarize"}.
            Defaults to "rsmtool".
        inplace : bool
            Maintain the state of the config object produced by
            this method.
            Defaults to ``True``.

        Returns
        -------
        config_obj : Configuration
            A configuration object

        Raises
        ------
        ValueError
            If config does not exist, and no config passed.
        """
        # make a copy of the given parameter dictionary
        new_config = deepcopy(config)

        # 1. Check to make sure all required fields are specified
        required_fields = CHECK_FIELDS[context]['required']

        for field in required_fields:
            if field not in new_config:
                raise ValueError("The config file must "
                                 "specify '{}'".format(field))

        # 2. Add default values for unspecified optional fields
        # for given RSMTool context
        defaults = DEFAULTS

        for field in defaults:
            if field not in new_config:
                new_config[field] = defaults[field]

        # 3. Check to make sure no unrecognized fields are specified
        for field in new_config:
            if field not in defaults and field not in required_fields:
                raise ValueError("Unrecognized field '{}'"
                                 " in json file".format(field))

        # 4. Check to make sure that the ID fields that will be
        # used as part of filenames are formatted correctly
        # i.e., they do not contain any spaces and are < 200 characters
        id_field = ID_FIELDS[context]
        id_field_values = {id_field: new_config[id_field]}

        for id_field, id_field_value in id_field_values.items():
            if len(id_field_value) > 200:
                raise ValueError("{} is too long (must be "
                                 "<=200 characters)".format(id_field))

            if re.search(r'\s', id_field_value):
                raise ValueError("{} cannot contain any "
                                 "spaces".format(id_field))

        # 5. Check that the feature file and feature subset/subset file are not
        # specified together
        msg = ("You cannot specify BOTH \"features\" and \"{}\". "
               "Please refer to the \"Selecting Feature Columns\" "
               "section in the documentation for more details.")
        if new_config['features'] and new_config['feature_subset_file']:
            msg = msg.format("feature_subset_file")
            raise ValueError(msg)
        if new_config['features'] and new_config['feature_subset']:
            msg = msg.format("feature_subset")
            raise ValueError(msg)

        # 6. Check for fields that require feature_subset_file and try
        # to use the default feature file
        if (new_config['feature_subset'] and
                not new_config['feature_subset_file']):

            # Check if we have the default subset file from rsmextra
            if HAS_RSMEXTRA:
                default_basename = Path(default_feature_subset_file).name
                new_config['feature_subset_file'] = default_feature_subset_file
                cls.logger.warning("You requested feature subsets but did not "
                                   "specify any feature file. "
                                   "The tool will use the default "
                                   "feature file {} available via "
                                   "rsmextra".format(default_basename))
            else:
                raise ValueError("If you want to use feature subsets, you "
                                 "must specify a feature subset file")

        if new_config['sign'] and not new_config['feature_subset_file']:

            # Check if we have the default subset file from rsmextra
            if HAS_RSMEXTRA:
                default_basename = Path(default_feature_subset_file).name
                new_config['feature_subset_file'] = default_feature_subset_file
                cls.logger.warning("You specified the expected sign of "
                                   "correlation but did not specify a feature "
                                   "subset file. The tool will use "
                                   "the default feature subset file {} "
                                   "available via "
                                   "rsmextra".format(default_basename))
            else:
                raise ValueError("If you want to specify the expected sign of "
                                 " correlation for each feature, you must "
                                 "specify a feature subset file")

        # Use the default sign if we are using the default feature file
        # and sign has not been specified in the config file
        if HAS_RSMEXTRA:
            default_feature = default_feature_subset_file
            if (new_config['feature_subset_file'] == default_feature and
                    not new_config['sign']):
                new_config['sign'] = default_feature_sign

        # 7. Check for fields that must be specified together
        if (new_config['min_items_per_candidate'] and
                not new_config['candidate_column']):
            raise ValueError("If you want to filter out candidates with "
                             "responses to less than X items, you need "
                             "to specify the name of the column which "
                             "contains candidate IDs.")

        # 8. Check that if "skll_objective" is specified, it's
        # one of the metrics that SKLL allows for AND that it is
        # specified for a SKLL model and _not_ a built-in
        # linear regression model
        if new_config['skll_objective']:
            if not is_skll_model(new_config['model']):
                warnings.warn("You specified a custom SKLL objective but also chose a "
                              "non-SKLL model. The objective will be ignored.")
            else:
                if new_config['skll_objective'] not in SCORERS:
                    raise ValueError("Invalid SKLL objective. Please refer to the SKLL "
                                     "documentation and choose a valid tuning objective.")

        # 9. Check that if "skll_fixed_parameters" is specified,
        # it's specified for SKLL model and _not_ a built-in linear
        # regression model; we cannot check whether the parameters
        # are valid at parse time but SKLL will raise an error
        # at run time for any invalid parameters
        if new_config['skll_fixed_parameters']:
            if not is_skll_model(new_config['model']):
                warnings.warn("You specified custom SKLL fixed parameters but "
                              "also chose a non-SKLL model. The parameters will "
                              "be ignored.")

        # 10. Check that if we are running rsmtool to ask for
        # expected scores then the SKLL model type must actually
        # support probabilistic classification. If it's not a SKLL
        # model at all, we just treat it as a LinearRegression model
        # which is basically what they all are in the end.
        if context == 'rsmtool' and new_config['predict_expected_scores']:
            model_name = new_config['model']
            dummy_learner = Learner(model_name) if is_skll_model(model_name) else Learner('LinearRegression')
            if not hasattr(dummy_learner.model_type, 'predict_proba'):
                raise ValueError("{} does not support expected scores "
                                 "since it is not a probablistic classifier.".format(model_name))
            del dummy_learner

        # 11. Check the fields that requires rsmextra
        if not HAS_RSMEXTRA:
            if new_config['special_sections']:
                raise ValueError("Special sections are only available to ETS"
                                 " users by installing the rsmextra package.")

        # 12. Raise a warning if we are specifiying a feature file but also
        # telling the system to automatically select transformations
        if new_config['features'] and new_config['select_transformations']:
            # Show a warning unless a user passed a list of features.
            if not isinstance(new_config['features'], list):
                warnings.warn("You specified a feature file but also set "
                              "`select_transformations` to True. Any "
                              "transformations or signs specified in "
                              "the feature file will be overwritten by "
                              "the automatically selected transformations "
                              "and signs.")

        # 13. If we have `experiment_names`, check that the length of the list
        # matches the list of experiment_dirs.
        if context == 'rsmsummarize' and new_config['experiment_names']:
            if len(new_config['experiment_names']) != len(new_config['experiment_dirs']):
                raise ValueError("The number of specified experiment names should be the same"
                                 " as the number of specified experiment directories.")

        # 14. Check that if the user specified min_n_per_group, they also
        # specified subgroups. If they supplied a dictionary, make
        # sure the keys match
        if new_config['min_n_per_group']:
            # make sure we have subgroups
            if 'subgroups' not in new_config:
                raise ValueError("You must specify a list of subgroups in "
                                 "in the `subgroups` field if "
                                 "you want to use the `min_n_per_group` field")
            # if we got dictionary, make sure the keys match
            elif isinstance(new_config['min_n_per_group'], dict):
                if sorted(new_config['min_n_per_group'].keys()) != sorted(new_config['subgroups']):
                    raise ValueError("The keys in `min_n_per_group` must "
                                     "match the subgroups in `subgroups` field")
            # else convert to dictionary
            else:
                new_config['min_n_per_group'] = {group: new_config['min_n_per_group']
                                                 for group in new_config['subgroups']}

        # 15. Clean up config dict to keep only context-specific fields
        context_relevant_fields = (CHECK_FIELDS[context]['optional'] +
                                   CHECK_FIELDS[context]['required'])

        new_config = {k: v for k, v in new_config.items()
                      if k in context_relevant_fields}

        return new_config

    @classmethod
    def process_config(cls, config):
        """
        Process configuration file.

        Converts fields which are read in as string to the
        appropriate format. Fields which can take multiple
        string values are converted to lists if they have
        not been already formatted as such.

        Parameters
        ----------
        inplace : bool
            Maintain the state of the config object produced by
            this method.
            Defaults to ``True``.

        Returns
        -------
        config_obj : Configuration
            A configuration object

        Raises
        ------
        NameError
            If ``config`` does not exist, or ``config`` could not be read.
        ValueError
            If boolean configuration fields contain a value other then
            "true" or "false" (in JSON).
        """
        # Get the parameter dictionary
        new_config = deepcopy(config)

        # convert multiple values into lists
        for field in LIST_FIELDS:
            if field in new_config and new_config[field] is not None:
                if not isinstance(new_config[field], list):
                    new_config[field] = new_config[field].split(',')
                    new_config[field] = [prefix.strip() for prefix
                                         in new_config[field]]

        # make sure all boolean values are boolean
        for field in BOOLEAN_FIELDS:
            error_message = ('Field {} can only be set to '
                             'True or False.'.format(field))
            if field in new_config and new_config[field] is not None:
                if not isinstance(new_config[field], bool):
                    # we first convert the value to string to avoid
                    # attribute errors in case the user supplied an integer.
                    given_value = str(new_config[field]).strip()
                    m = re.match(r'^(true|false)$', given_value, re.I)
                    if not m:
                        raise ValueError(error_message)
                    else:
                        bool_value = json.loads(m.group().lower())
                        new_config[field] = bool_value

        return new_config class DataReader:
    """Class to generate DataContainer objects."""

    def __init__(self,
                 filepaths,
                 framenames,
                 file_converters=None):
        """
        Initialize a DataReader object.

        Parameters
        ----------
        filepaths : list of str
            A list of paths to files that are to be read in.
        framenames : list of str
            A list of names for the data sets to be included in the container.
        file_converters : dict of dicts, optional
            A dictionary of file converter dicts. The keys are the data set
            names and the values are the converter dictionaries to be applied
            to the corresponding data set.
            Defaults to ``None``.

        Raises
        ------
        AssertionError
            If ``len(filepaths)`` does not equal ``len(framenames)``.
        ValueError
            If ``file_converters`` is not a dictionary or if any of its
            values is not a dictionary.
        NameError
            If a key in ``file_converters`` does not exist in ``framenames``.
        ValueError
            If any of the specified file paths is ``None``.
        """
        # Default datasets list
        self.datasets = []

        # Make sure filepaths length matches frame names length
        assert len(filepaths) == len(framenames)

        # Make sure that there are no Nones in the filepaths
        if None in filepaths:
            frames_with_no_path = [framenames[i] for i in range(len(framenames))
                                   if filepaths[i] is None]

            raise ValueError("No path specified for "
                             "{}".format(' ,'.join(frames_with_no_path)))

        # Assign names and paths lists
        self.dataset_names = framenames
        self.dataset_paths = filepaths

        # If `file_converters` exists, then
        # check to make sure it is the correct length
        # and add all elements to `file_converters` list
        if file_converters is not None:
            if not isinstance(file_converters, dict):
                raise ValueError("The 'file_converters' argument must be a `dict`, "
                                 "not `{}`.".format(type(file_converters)))

            for file_converter_name in file_converters:
                # Make sure file_converter name is in `dataset_names`
                if file_converter_name not in self.dataset_names:
                    raise NameError('The file converter name ``{}`` '
                                    'does not exist in the '
                                    'dataset names that you '
                                    'passed.'.format(file_converter_name))

                # Make sure file converter is a `dict`
                file_converter = file_converters[file_converter_name]
                if not isinstance(file_converter, dict):
                    raise ValueError('Value for {} must be``dict`` ',
                                     'not {}'.format(file_converter_name,
                                                     type(file_converter)))

        # Default file_converters dict
        self.file_converters = {} if file_converters is None else file_converters

    @staticmethod
    def read_from_file(filename, converters=None, **kwargs):
        """
        Read a CSV/TSV/XLSX/JSONLINES/SAS7BDAT file and return a data frame.

        Parameters
        ----------
        filename : str
            Name of file to read.
        converters : dict, optional
            A dictionary specifying how the types of the columns
            in the file should be converted. Specified in the same
            format as for `pd.read_csv() <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html>`_.
            Defaults to ``None``.

        Returns
        -------
        df : pandas DataFrame
            Data frame containing the data in the given file.

        Raises
        ------
        ValueError
            If the file has an unsuppored extension.
        pandas.errors.ParserError
            If the file is badly formatted or corrupt.

        Note
        ----
        Any additional keyword arguments are passed to the underlying
        pandas IO reader function.
        """
        file_extension = splitext(filename)[1].lower()

        if file_extension in ['.csv', '.tsv']:
            sep = '\t' if file_extension == '.tsv' else ','
            do_read = partial(pd.read_csv, sep=sep, converters=converters)
        elif file_extension == '.xlsx':
            do_read = partial(pd.read_excel, converters=converters)
        elif file_extension == '.sas7bdat':
            if 'encoding' not in kwargs:
                encoding = 'latin-1'
            else:
                encoding = kwargs.pop('encoding')
            do_read = partial(pd.read_sas, encoding=encoding)
        elif file_extension in ['.jsonlines']:
            do_read = partial(read_jsonlines, converters=converters)
        else:
            raise ValueError("RSMTool only supports files in .csv, "
                             ".tsv, .xlsx, and .sas7bdat formats. "
                             "Input files should have one of these "
                             "extensions. The file you "
                             "passed is: {}.".format(filename))

        # ignore warnings about mixed data types for large files
        with warnings.catch_warnings():
            warnings.filterwarnings('ignore', category=DtypeWarning)
            try:
                df = do_read(filename, **kwargs)
            except pd.errors.ParserError:
                raise pd.errors.ParserError('Cannot read {}. Please check that it is '
                                            'not corrupt or in an incompatible format. '
                                            '(Try running dos2unix?)'.format(filename))
        return df

    @staticmethod
    def locate_files(filepaths, configdir):
        """
        Locate an experiment file, or a list of experiment files.

        If the given path doesn't exist, then maybe the path is relative
        to the path of the config file. If neither exists, then return
        ``None``.

        Parameters
        ----------
        filepaths : str or list
            Name(s) of the experiment file we want to locate.
        configdir : str
            Path to the reference configuration directory
            (usually the directory of the config file)

        Returns
        --------
        retval :  str or list
            Absolute path to the experiment file or ``None``
            if the file could not be located. If ``filepaths``
            was a string, this method will return a string.
            Otherwise, it will return a list.

        Raises
        ------
        ValueError
            If ``filepaths`` is not a string or a list.
        """
        # the feature config file can be in the 'feature' directory
        # at the same level as the main config file
        if not (isinstance(filepaths, str) or
                isinstance(filepaths, list)):
            raise ValueError("The 'filepaths' argument must be a "
                             "string or a list, not {}.".format(type(filepaths)))

        if isinstance(filepaths, str):
            filepaths = [filepaths]
            return_string = True
        else:
            return_string = False

        located_paths = []
        for filepath in filepaths:

            retval = None
            alternate_path = abspath(join(configdir, filepath))

            # if the given path exists as is, convert
            # that to an absolute path and return
            if exists(filepath):
                retval = abspath(filepath)

            # otherwise check if it exists relative
            # to the reference directory
            elif exists(alternate_path):
                retval = alternate_path

            located_paths.append(retval)

        if return_string:
            return located_paths[0]

        return located_paths

    def read(self, kwargs_dict=None):
        """
        Read all files contained in ``self.dataset_paths``.

        Parameters
        ----------
        kwargs_dict : dict of dicts, optional
            Any additional keyword arguments to pass to a particular DataFrame.
            These arguments will be passed to the pandas IO reader function.
            Defaults to ``None``.

        Returns
        -------
        datacontainer : container.DataContainer
            A data container object.

        Raises
        ------
        FileNotFoundError
            If any of the files in ``self.dataset_paths`` does not exist.
        """
        for idx, set_path in enumerate(self.dataset_paths):

            name = self.dataset_names[idx]
            converter = self.file_converters.get(name, None)

            if not exists(set_path):
                raise FileNotFoundError('The file {} does not exist'.format(set_path))

            if kwargs_dict is not None:
                kwargs = kwargs_dict.get(name, {})
            else:
                kwargs = {}

            dataframe = self.read_from_file(set_path, converter, **kwargs)

            # Add to list of datasets
            self.datasets.append({'name': name.strip(),
                                  'path': set_path,
                                  'frame': dataframe})

        return DataContainer(self.datasets) def setup_rsmcmd_parser(name,
                        uses_output_directory=True,
                        allows_overwriting=False,
                        extra_run_options=[],
                        uses_subgroups=False):
    def existing_configuration_file(string):
    def __init__(self, field_name, field_type, field_metadata):
    def _make_boolean_validator(self, allow_empty=False):
    def _make_choice_validator(self, choices):
    def _make_directory_completer(self):
    def _make_directory_validator(self):
    def _make_file_completer(self):
        def valid_file(filename):
    def _make_file_validator(self, allow_empty=False):
        def is_valid(path):
        def is_empty(path):
    def _make_file_format_validator(self):
    def _make_id_validator(self):
    def _make_integer_validator(self, allow_empty=False):
    def _get_user_input(self):
    def _finalize(self, user_input):
    def get_value(self):
    def __init__(self,
                 context,
                 as_string=False,
                 suppress_warnings=False,
                 use_subgroups=False):  # noqa
    def _convert_to_string(self,
                           config_object,
                           insert_url_comment=True,
                           insert_required_comment=True,
                           insert_optional_comment=True):
    def _get_all_general_section_names(self):
    def interact(self):
    def generate(self):
class InteractiveField:
class ConfigurationGenerator: CHECK_FIELDS = {'rsmtool': {'required': ['experiment_id',
                                         'model',
                                         'train_file',
                                         'test_file'],
                            'optional': ['description',
                                         'features',
                                         'feature_subset_file',
                                         'feature_subset',
                                         'file_format',
                                         'sign',
                                         'id_column',
                                         'use_thumbnails',
                                         'train_label_column',
                                         'test_label_column',
                                         'length_column',
                                         'second_human_score_column',
                                         'flag_column',
                                         'flag_column_test',
                                         'exclude_zero_scores',
                                         'rater_error_variance',
                                         'trim_min',
                                         'trim_max',
                                         'trim_tolerance',
                                         'predict_expected_scores',
                                         'select_transformations',
                                         'use_scaled_predictions',
                                         'use_truncation_thresholds',
                                         'subgroups',
                                         'min_n_per_group',
                                         'general_sections',
                                         'custom_sections',
                                         'special_sections',
                                         'skll_fixed_parameters',
                                         'skll_objective',
                                         'section_order',
                                         'candidate_column',
                                         'standardize_features',
                                         'min_items_per_candidate']},
                'rsmxval': {'required': ['experiment_id',
                                         'model',
                                         'train_file'],
                            'optional': ['description',
                                         'folds',
                                         'folds_file',
                                         'features',
                                         'feature_subset_file',
                                         'feature_subset',
                                         'file_format',
                                         'sign',
                                         'id_column',
                                         'use_thumbnails',
                                         'train_label_column',
                                         'length_column',
                                         'second_human_score_column',
                                         'flag_column',
                                         'flag_column_test',
                                         'exclude_zero_scores',
                                         'rater_error_variance',
                                         'trim_min',
                                         'trim_max',
                                         'trim_tolerance',
                                         'predict_expected_scores',
                                         'select_transformations',
                                         'use_scaled_predictions',
                                         'use_truncation_thresholds',
                                         'subgroups',
                                         'min_n_per_group',
                                         'skll_fixed_parameters',
                                         'skll_objective',
                                         'candidate_column',
                                         'standardize_features',
                                         'min_items_per_candidate']},
                'rsmeval': {'required': ['experiment_id',
                                         'predictions_file',
                                         'system_score_column',
                                         'trim_min',
                                         'trim_max'],
                            'optional': ['description',
                                         'id_column',
                                         'human_score_column',
                                         'second_human_score_column',
                                         'file_format',
                                         'flag_column',
                                         'exclude_zero_scores',
                                         'use_thumbnails',
                                         'scale_with',
                                         'trim_tolerance',
                                         'rater_error_variance',
                                         'subgroups',
                                         'min_n_per_group',
                                         'general_sections',
                                         'custom_sections',
                                         'special_sections',
                                         'section_order',
                                         'candidate_column',
                                         'min_items_per_candidate']},
                'rsmpredict': {'required': ['experiment_id',
                                            'experiment_dir',
                                            'input_features_file'],
                               'optional': ['id_column',
                                            'candidate_column',
                                            'file_format',
                                            'predict_expected_scores',
                                            'human_score_column',
                                            'second_human_score_column',
                                            'standardize_features',
                                            'subgroups',
                                            'flag_column']},
                'rsmcompare': {'required': ['comparison_id',
                                            'experiment_id_old',
                                            'experiment_dir_old',
                                            'experiment_id_new',
                                            'experiment_dir_new',
                                            'description_old',
                                            'description_new'],
                               'optional': ['use_scaled_predictions_old',
                                            'use_scaled_predictions_new',
                                            'subgroups',
                                            'use_thumbnails',
                                            'general_sections',
                                            'custom_sections',
                                            'special_sections',
                                            'section_order']},
                'rsmsummarize': {'required': ['summary_id',
                                              'experiment_dirs'],
                                 'optional': ['description',
                                              'experiment_names',
                                              'file_format',
                                              'general_sections',
                                              'custom_sections',
                                              'use_thumbnails',
                                              'special_sections',
                                              'subgroups',
                                              'section_order']}} DEFAULTS = {'id_column': 'spkitemid',
            'description': '',
            'description_old': '',
            'description_new': '',
            'train_label_column': 'sc1',
            'test_label_column': 'sc1',
            'human_score_column': 'sc1',
            'exclude_zero_scores': True,
            'use_scaled_predictions': False,
            'use_scaled_predictions_old': False,
            'use_scaled_predictions_new': False,
            'select_transformations': False,
            'standardize_features': True,
            'use_thumbnails': False,
            'use_truncation_thresholds': False,
            'scale_with': None,
            'predict_expected_scores': False,
            'sign': None,
            'features': None,
            'length_column': None,
            'second_human_score_column': None,
            'file_format': 'csv',
            'form_level_scores': None,
            'candidate_column': None,
            'general_sections': ['all'],
            'special_sections': None,
            'custom_sections': None,
            'feature_subset_file': None,
            'feature_subset': None,
            'rater_error_variance': None,
            'trim_min': None,
            'trim_max': None,
            'trim_tolerance': 0.4998,
            'subgroups': [],
            'min_n_per_group': None,
            'skll_fixed_parameters': {},
            'skll_objective': None,
            'section_order': None,
            'flag_column': None,
            'flag_column_test': None,
            'min_items_per_candidate': None,
            'experiment_names': None,
            'folds_file': None,
            'folds': 5} INTERACTIVE_MODE_METADATA = {'experiment_id': {'label': 'Experiment ID',
                                               'type': 'id'},
                             'comparison_id': {'label': 'Comparison ID',
                                               'type': 'id'},
                             'summary_id': {'label': 'Summary ID',
                                            'type': 'id'},
                             'model': {'label': 'Model to use',
                                       'type': 'choice',
                                       'choices': sorted(set(BUILTIN_MODELS + VALID_SKLL_MODELS))},
                             'train_file': {'label': 'Path to training data file',
                                            'type': 'file'},
                             'test_file': {'label': 'Path to evaluation data file',
                                           'type': 'file'},
                             'folds': {'label': 'Number of cross-validation folds to use (<u>5</u>)',
                                       'type': 'integer'},
                             'folds_file': {'label': 'Optional file with custom folds (overrides # of folds)',
                                            'type': 'file'},
                             'predictions_file': {'label': 'Path to file containing predictions',
                                                  'type': 'file'},
                             'system_score_column': {'label': 'Name of column containing predictions'},
                             'trim_min': {'label': 'The lowest possible human score',
                                          'type': 'integer'},
                             'trim_max': {'label': 'The highest possible human score',
                                          'type': 'integer'},
                             'experiment_dir': {'label': 'Path to the directory containing RSMTool experiment',
                                                'type': 'dir'},
                             'input_features_file': {'label': 'Path to input file containing features',
                                                     'type': 'file'},
                             'experiment_id_old': {'label': 'ID for old RSMTool experiment'},
                             'experiment_dir_old': {'label': 'Path to old RSMTool experiment',
                                                    'type': 'dir'},
                             'description_old': {'label': 'Description of old RSMTool experiment'},
                             'experiment_id_new': {'label': 'ID for new RSMTool experiment'},
                             'experiment_dir_new': {'label': 'Path to new RSMTool experiment',
                                                    'type': 'dir'},
                             'description_new': {'label': 'Description of new RSMTool experiment'},
                             'experiment_dirs': {'label': 'Paths to directories containing RSMTool experiments', 'type': 'dir', 'count': 'multiple'},
                             'description': {'label': 'Description of experiment'},
                             'file_format': {'label': 'Format for intermediate files (<u>csv</u>/tsv/xlsx)',
                                             'type': 'format'},
                             'id_column': {'label': 'Name of column that contains response IDs (<u>spkitemid</u>)'},
                             'use_thumbnails': {'label': 'Use clickable thumbnails in report instead '
                                                         'of full-sized images? (true/<u>false</u>)',
                                                'type': 'boolean'},
                             'train_label_column': {'label': 'Name of column in training data that '
                                                             'contains human scores (<u>sc1</u>)'},
                             'test_label_column': {'label': 'Name of column in evaluation data that '
                                                            'contains human scores (<u>sc1</u>)'},
                             'length_column': {'label': 'Name of column in training/evaluation data '
                                                        'that contains response lengths, if any'},
                             'human_score_column': {'label': 'Name of column in evaluation data '
                                                             'that contains human scores (<u>sc1</u>)'},
                             'second_human_score_column': {'label': 'Name of column in evaluation '
                                                                    'data that contains scores from '
                                                                    'a second human, if any'},
                             'exclude_zero_scores': {'label': 'Keep responses with human scores of 0 in '
                                                              'training/evaluation data '
                                                              '(true/<u>false</u>)',
                                                     'type': 'boolean'},
                             'use_scaled_predictions': {'label': 'Use scaled predictions instead of '
                                                                 'raw in report analyses '
                                                                 '(true/<u>false</u>)',
                                                        'type': 'boolean'},
                             'standardize_features': {'label': 'Standardize all features '
                                                               '(<u>true</u>/false)',
                                                      'type': 'boolean'},
                             'subgroups': {'label': 'List of column names containing subgroup variables',
                                           'count': 'multiple'}
                             } def convert_to_float(value):
    """
    Convert value to float, if possible.

    Parameters
    ----------
    value
        Name of the experiment file we want to locate.

    Returns
    -------
    value
        Value converted to float, if possible
    """
    return int_to_float(string_to_number(value)) def int_to_float(value):
    """
    Convert integer to float, if possible.

    Parameters
    ----------
    value
        Name of the value we want to convert.

    Returns
    -------
    value
        Value converted to float, if possible
    """
    return float(value) if type(value) == int else value def combine_fold_prediction_files(foldsdir, file_format="csv"):
    """
    Combine predictions from all folds into a single data frame.

    Parameters
    ----------
    foldsdir : str
        The directory which stores the output for each fold experiment.
    file_format : str
        The file format (extension) for the file to be written to disk.
        One of {"csv", "xlsx", "tsv"}.
        Defaults to "csv".

    Returns
    -------
    df_all_predictions : pandas DataFrame
        Data frame containing the combined predictions for all folds.
    """
    # initialize empty list to hold each prediction data frame
    prediction_dfs = []

    # iterate over each fold in the given directory & read and save predictions
    for prediction_file in Path(foldsdir).glob(f"**/*pred_processed*.{file_format}"):
        df_fold_predictions = DataReader.read_from_file(prediction_file, converters={'spkitemid': str})
        prediction_dfs.append(df_fold_predictions)

    # concatenate all of the predictions into a single frame using
    # "spkitemid" column which must exist since this is the output of RSMTool
    df_all_predictions = concat(prediction_dfs, keys="spkitemid").reset_index(drop=True)

    return df_all_predictions def create_xval_files(configuration, output_dir, logger=None):
    """
    Create all files needed for the cross-validation experiment.

    Parameters
    ----------
    configuration : rsmtool.configuration_parser.Configuration
        A Configuration object holding the user-specified configuration
        for "rsmxval".
    output_dir : str
        Path to the output directory specified for "rsmxval".
    logger : None, optional
        Logger object used to log messages from this function.

    Returns
    -------
    df_train : pandas DataFrame
        DataFrame containing the raw training data file.
    folds : int
        The number of folds for which the files were created.

    Raises
    ------
    FileNotFoundError
        If the training data file specified in the configuration is not found.
    """
    # instantiate a logger if one is not given
    logger = logger if logger else logging.getLogger(__name__)

    # get the paths to the "<output_dir>/folds" directory as well as to
    # the "<output_dir>/final-model" directory since those are the two
    # directories under which we will write files
    foldsdir = Path(output_dir) / "folds"
    modeldir = Path(output_dir) / "final-model"

    # get the file format specified by the user
    given_file_format = configuration["file_format"]

    # locate the training data file, the folds file, and feature file(s)
    located_filepaths = {}
    located_filepaths["train"] = DataReader.locate_files(configuration.get("train_file"),
                                                         configuration.configdir)
    if not located_filepaths["train"]:
        raise FileNotFoundError('The training data file was not found: '
                                '{}'.format(repr(configuration.get("train_file"))))

    # "features" could be a list of features so check for that
    additional_filenames = ["folds_file", "feature_subset_file"]
    if isinstance(configuration.get("features"), str):
        additional_filenames.append("features")

    for additional_filename in additional_filenames:
        if additional_file := configuration.get(additional_filename):
            located_filepaths[additional_filename] = DataReader.locate_files(additional_file,
                                                                             configuration.configdir)

    # read the training file into a dataframe
    df_train = DataReader.read_from_file(located_filepaths["train"])

    # we need to sub-sample the full training data file to create a dummy
    # test file that we need to use when running RSMTool on the full
    # training dataset to get the model/feature descriptives; we use 10%
    # of the training data to create this dummy test set
    df_sampled_test = df_train.sample(frac=0.1,
                                      replace=False,
                                      random_state=1234567890,
                                      axis=0)
    DataWriter.write_frame_to_file(df_sampled_test,
                                   str(modeldir / "dummy_test"),
                                   file_format=given_file_format,
                                   index=False)

    # read the folds file if it exists, otherwise use specified number of folds
    folds_file = located_filepaths.get("folds_file")
    if folds_file and Path(folds_file).exists():
        cv_folds = load_cv_folds(located_filepaths["folds_file"])
        folds = len(set(cv_folds.values()))
        logger.info(f"Using {folds} folds specified in {located_filepaths['folds_file']}")
        logo = LeaveOneGroupOut()
        id_column = configuration.get("id_column")
        try:
            train_ids = df_train[id_column]
        except KeyError:
            logger.error("Column f{id_column} not found! Check the value of "
                         "'id_column' in the configuration file.")
        else:
            fold_groups = [cv_folds[train_id] for train_id in train_ids.values]
            fold_generator = logo.split(range(len(df_train)), y=None, groups=fold_groups)
    else:
        # if we are in this code path but the configuration did
        # specify "folds_file", then we must not have found the file
        # so raise a warning to that effect
        if configuration.get("folds_file"):
            logging.warning("Specified folds file not found. Ignoring.")
        folds = configuration.get("folds")
        logger.info(f"Generating {folds} folds after shuffling")
        kfold = KFold(n_splits=folds, random_state=1234567890, shuffle=True)
        fold_generator = kfold.split(range(len(df_train)))

    # iterate over each of the folds and generate an rsmtool configuration file
    # which is then saved to disk in a directory specific to each fold
    for fold_num, (fold_train_indices,
                   fold_test_indices) in enumerate(fold_generator, start=1):

        # get the train and test files for this fold and
        # write them out to disk
        df_fold_train = df_train.loc[fold_train_indices]
        df_fold_test = df_train.loc[fold_test_indices]
        this_fold_dir = Path(foldsdir) / f"{fold_num:02}"
        this_fold_dir.mkdir()
        fold_train_file_prefix = str(this_fold_dir / "train")
        DataWriter.write_frame_to_file(df_fold_train,
                                       fold_train_file_prefix,
                                       file_format=given_file_format,
                                       index=False)
        fold_test_file_prefix = str(this_fold_dir / "test")
        DataWriter.write_frame_to_file(df_fold_test,
                                       fold_test_file_prefix,
                                       file_format=given_file_format,
                                       index=False)

        # update the value of "train_file" and add "test_file"
        # & "test_label_column" in the configuration file
        fold_configuration = configuration.copy(deep=True)
        fold_configuration["train_file"] = f"{fold_train_file_prefix}.{given_file_format}"
        fold_configuration["test_file"] = f"{fold_test_file_prefix}.{given_file_format}"
        fold_configuration["test_label_column"] = configuration["train_label_column"]

        # use the same file format as the user specified
        fold_configuration["file_format"] = given_file_format

        # update "experiment_id" and "description" in configuration file
        fold_configuration["experiment_id"] = f"{configuration['experiment_id']}_fold{fold_num:02}"
        fold_configuration["description"] = f"{configuration['description']} (Fold {fold_num:02})"

        # if "feature_subset" was specified in the original configuration,
        # then we need it in the fold configuration as well
        if subset := configuration.get("feature_subset"):
            fold_configuration["feature_subset"] = subset

        # update "configdir" and "context" attributes in configuration file
        fold_configuration.configdir = str(this_fold_dir)
        fold_configuration.context = "rsmtool"
        with open(this_fold_dir / "rsmtool.json", "w") as fold_config_file:
            fold_config_file.write(str(fold_configuration))

        # copy "features" or "features_subset_file" to the
        # same directory as where the configuration was saved;
        # note that if features is a list instead of a file,
        # it will get ignored automatically
        for filename in ["features", "feature_subset_file"]:
            if filepath := located_filepaths.get(filename):
                copyfile(filepath, this_fold_dir / Path(filepath).name)

    return df_train, folds def get_output_directory_extension(directory, experiment_id):
    """
    Check output directory to determine what file extensions exist.

    If more than one extension (in the possible list of
    extensions) exists, then raise a ``ValueError``. Otherwise,
    return the one file extension. If no extensions can be found, then
    "csv" will be returned by default.

    Possible extensions include: "csv", "tsv", and "xlsx". Files in the
    directory with none of these extensions are ignored.

    Parameters
    ----------
    directory : str
        The path to the directory where output is located.
    experiment_id : str
        The ID of the experiment.

    Returns
    -------
    extension : str
        The extension that output files in this directory
        end with. One of {"csv", "tsv", "xlsx"}.

    Raises
    ------
    ValueError
        If any files in the directory have extensions
        other than "csv", "tsv", or "xlsx".
    """
    extension = 'csv'
    extensions_identified = {ext for ext in POSSIBLE_EXTENSIONS
                             if has_files_with_extension(directory, ext)}

    if len(extensions_identified) > 1:
        raise ValueError('Some of the files in the experiment output directory (`{}`) '
                         'for `{}` have different extensions. All files in this directory '
                         'must have the same extension. The following extensions were '
                         'identified : {}'.format(directory,
                                                  experiment_id,
                                                  ', '.join(extensions_identified)))

    elif len(extensions_identified) == 1:
        extension = list(extensions_identified)[0]

    return extension def has_files_with_extension(directory, ext):
    """
    Check if the directory has any files with the given extension.

    Parameters
    ----------
    directory : str
        The path to the directory where output is located.
    ext : str
        The the given extension.

    Returns
    -------
    ans : bool
        ``True`` if directory contains files with given extension,
        else ``False``.
    """
    files_with_extension = glob(join(directory, '*.{}'.format(ext)))
    return len(files_with_extension) > 0 def parse_json_with_comments(pathlike):
    """
    Parse a JSON file after removing any comments.

    Comments can use either ``//`` for single-line
    comments or or ``/* ... */`` for multi-line comments.
    The input filepath can be a string or ``pathlib.Path``.

    Parameters
    ----------
    filename : str or os.PathLike
        Path to the input JSON file either as a string
        or as a ``pathlib.Path`` object.

    Returns
    -------
    obj : dict
        JSON object representing the input file.

    Note
    ----
    This code was adapted from:
    https://web.archive.org/web/20150520154859/http://www.lifl.fr/~riquetd/parse-a-json-file-with-comments.html
    """
    # Regular expression to identify comments
    comment_re = re.compile(r'(^)?[^\S\n]*(?:/\*(.*?)\*/[^\S\n]*|(?<!https:)(?<!http:)//[^\n]*)($)?',
                            re.DOTALL | re.MULTILINE)

    # if we passed in a string, convert it to a Path
    if isinstance(pathlike, str):
        pathlike = Path(pathlike)

    with open(pathlike, 'r') as file_buff:
        content = ''.join(file_buff.readlines())

        # Looking for comments
        match = comment_re.search(content)
        while match:

            # single line comment
            content = content[:match.start()] + content[match.end():]
            match = comment_re.search(content)

        # Return JSON object
        config = json.loads(content)
        return config def get_file_logger(logger_name, log_file_path):
    """
    Create and return a file-based logger.

    Instantiate a logger with the given name and attach a
    file handler that uses the given log file. The logging
    level is set to INFO.

    Parameters
    ----------
    logger_name : str
        Name to use for the logger.
    log_file_path : str
        File path to which the logger should output messages.

    Returns
    -------
    logger
        A logging.Logger object attached to a file handler.
    """
    logger = logging.getLogger(logger_name)
    logger.propagate = False
    formatter = logging.Formatter('%(message)s')
    file_handler = logging.FileHandler(log_file_path, mode="w")
    file_handler.setFormatter(formatter)
    file_handler.setLevel(logging.INFO)
    logger.addHandler(file_handler)
    logger.setLevel(logging.INFO)
    return logger def compute_expected_scores_from_model(model, featureset, min_score, max_score):
    """
    Compute expected scores using probability distributions over labels.

    This function only works with SKLL models.

    Parameters
    ----------
    model : skll.learner.Learner
        The SKLL learner object to use for computing the expected scores.
    featureset : skll.data.FeatureSet
        The SKLL featureset object for which predictions are to be made.
    min_score : int
        Minimum score level to be used for computing expected scores.
    max_score : int
        Maximum score level to be used for computing expected scores.

    Returns
    -------
    expected_scores: np.array
        A numpy array containing the expected scores.

    Raises
    ------
    ValueError
        If the given model cannot predict probability distributions.
    ValueError
        If the score range specified by ``min_score`` and ``max_score``
        does not match what the model predicts in its probability
        distribution.
    """
    if hasattr(model.model, "predict_proba"):
        # Tell the model we want probabiltiies as output. This is likely already set
        # to True but it might not be, e.g., when using rsmpredict.
        model.probability = True
        probability_distributions = model.predict(featureset, class_labels=False)
        # check to make sure that the number of labels in the probability
        # distributions matches the number of score points we have
        num_score_points_specified = max_score - min_score + 1
        num_score_points_in_learner = probability_distributions.shape[1]
        if num_score_points_specified != num_score_points_in_learner:
            raise ValueError('The specified number of score points ({}) '
                             'does not match that from the the learner '
                             '({}).'.format(num_score_points_specified,
                                            num_score_points_in_learner))
        expected_scores = probability_distributions.dot(range(min_score, max_score + 1))
    else:
        if model.model_type.__name__ == 'SVC':
            raise ValueError("Expected scores cannot be computed since the SVC model was "
                             "not originally trained to predict probabilities.")
        else:
            raise ValueError("Expected scores cannot be computed since {} is not a "
                             "probabilistic classifier.".format(model.model_type.__name__))

    return expected_scores def difference_of_standardized_means(y_true_observed,
                                     y_pred,
                                     population_y_true_observed_mn=None,
                                     population_y_pred_mn=None,
                                     population_y_true_observed_sd=None,
                                     population_y_pred_sd=None,
                                     ddof=1):
    """
    Calculate the difference between standardized means.

    First, standardize both observed and predicted scores to z-scores using
    mean and standard deviation for the whole population. Then
    calculate differences between standardized means for each subgroup.

    Parameters
    ----------
    y_true_observed : array-like
        The observed scores for the group or subgroup.
    y_pred : array-like
        The predicted score for the group or subgroup.
        The predicted scores.
    population_y_true_observed_mn : float, optional
        The population true score mean.
        When the DSM is being calculated for a subgroup,
        this should be the mean for the whole
        population.
        Defaults to ``None``.
    population_y_pred_mn : float, optional
        The predicted score mean.
        When the DSM is being calculated for a subgroup,
        this should be the mean for the whole
        population.
        Defaults to ``None``.
    population_y_true_observed_sd : float, optional
        The population true score standard deviation.
        When the DSM is being calculated for a subgroup,
        this should be the standard deviation for the whole
        population.
        Defaults to ``None``.
    population_y_pred_sd : float, optional
        The predicted score standard deviation.
        When the DSM is being calculated for a subgroup,
        this should be the standard deviation for the whole
        population.
        Defaults to None.
    ddof : int, optional
        The delta degrees of freedom. The divisor used in
        calculations is N - ddof, where N represents the
        number of elements.
        Defaults to 1.

    Returns
    -------
    difference_of_std_means : array-like
        The difference of standardized means

    Raises
    ------
    ValueError
        If only one of ``population_y_true_observed_mn`` and
        ``population_y_true_observed_sd`` is not ``None``.
    ValueError
        If only one of ``population_y_pred_mn`` and ``population_y_pred_sd``
        is not ``None``.
    """
    assert len(y_true_observed) == len(y_pred)

    # all of this is just to make sure users aren't passing the population
    # standard deviation and not population mean for either true or predicted
    y_true_observed_population_params = [population_y_true_observed_mn,
                                         population_y_true_observed_sd]
    y_pred_population_params = [population_y_pred_mn,
                                population_y_pred_sd]

    if len([param for param in y_true_observed_population_params
            if param is None]) == 1:
        raise ValueError('You must pass both `population_y_true_observed_mn` and '
                         '`population_y_true_observed_sd` or neither.')

    if len([param for param in y_pred_population_params
            if param is None]) == 1:
        raise ValueError('You must pass both `population_y_pred_mn` and '
                         '`population_y_pred_sd` or neither.')

    warning_msg = ('You did not pass population mean and std. for `{}`; '
                   'thus, the calculated z-scores will be zero.')

    # if the population means and standard deviations were not provided, calculate from the data
    # We only check for mean since the function requires
    # both of these to be set or both to be None
    if population_y_true_observed_mn is None:

        warnings.warn(warning_msg.format('y_true_observed'))
        (population_y_true_observed_sd,
         population_y_true_observed_mn) = (np.std(y_true_observed, ddof=ddof),
                                           np.mean(y_true_observed))

    if population_y_pred_mn is None:

        warnings.warn(warning_msg.format('y_pred'))
        (population_y_pred_sd,
         population_y_pred_mn) = (np.std(y_pred, ddof=ddof),
                                  np.mean(y_pred))

    # if any of the standard deviations equal zero
    # raise a warning and return None.
    # We use np.isclose since sometimes sd for float
    # values is a value very close to 0.
    # We use the same tolerance as used for identifying
    # features with zero standard deviation
    if (np.isclose(population_y_pred_sd, 0, atol=1e-07) or
            np.isclose(population_y_true_observed_sd, 0, atol=1e-07)):
        warnings.warn("Population standard deviations for the computation of "
                      "DSM is zero. No value will be computed.")
        return None

    # calculate the z-scores for observed and predicted
    y_true_observed_subgroup_z = ((y_true_observed - population_y_true_observed_mn) /
                                  population_y_true_observed_sd)
    y_pred_subgroup_z = ((y_pred - population_y_pred_mn) /
                         population_y_pred_sd)

    # calculate the DSM, given the z-scores for observed and predicted
    difference_of_std_means = np.mean(y_pred_subgroup_z - y_true_observed_subgroup_z)

    return difference_of_std_means def partial_correlations(df):
    """
    Implement the R ``pcor`` function from ``ppcor`` package in Python.

    This computes partial correlations of each pair of variables
    in the given data frame ``df``, excluding all other variables.

    Parameters
    ----------
    df : pd.DataFrame
        Data frame containing the feature values.

    Returns
    -------
    df_pcor : pd.DataFrame
        Data frame containing the partial correlations of of each
        pair of variables in the given data frame ``df``,
        excluding all other variables.
    """
    numrows, numcols = df.shape
    df_cov = df.cov()
    columns = df_cov.columns

    # return a matrix of nans if the number of columns is
    # greater than the number of rows. When the ncol == nrows
    # we get the degenerate matrix with 1 only. It is not meaningful
    # to compute partial correlations when ncol > nrows.

    # create empty array for when we cannot compute the
    # matrix inversion
    empty_array = np.empty((len(columns), len(columns)))
    empty_array[:] = np.nan
    if numcols > numrows:
        icvx = empty_array
    else:
        # if the determinant is less than the lowest representable
        # 32 bit integer, then we use the pseudo-inverse;
        # otherwise, use the inverse; if a linear algebra error
        # occurs, then we just set the matrix to empty
        try:
            assert np.linalg.det(df_cov) > np.finfo(np.float32).eps
            icvx = np.linalg.inv(df_cov)
        except AssertionError:
            icvx = np.linalg.pinv(df_cov)
            warnings.warn('When computing partial correlations '
                          'the inverse of the variance-covariance matrix '
                          'was calculated using the Moore-Penrose generalized '
                          'matrix inversion, due to its determinant being at '
                          'or very close to zero.')
        except np.linalg.LinAlgError:
            icvx = empty_array

    pcor = -1 * covariance_to_correlation(icvx)
    np.fill_diagonal(pcor, 1.0)
    df_pcor = pd.DataFrame(pcor, columns=columns, index=columns)
    return df_pcor def quadratic_weighted_kappa(y_true_observed, y_pred, ddof=0):  # noqa: D301
    """
    Calculate quadratic-weighted kappa for both discrete and continuous values.

    The formula to compute quadratic-weighted kappa for continuous values
    was developed at ETS by Shelby Haberman.
    See `Haberman (2019) <https://onlinelibrary.wiley.com/doi/abs/10.1002/ets2.12258>`_
    for the full derivation. The discrete case is simply treated as a
    special case of the continuous one.

    The formula is as follows:

    :math:`QWK=\\displaystyle\\frac{2*Cov(M,H)}{Var(H)+Var(M)+(\\bar{M}-\\bar{H})^2}`, where

        - :math:`Cov` - covariance with normalization by :math:`N` (the total number of observations given)
        - :math:`H` - the human score
        - :math:`M` - the system score
        - :math:`\\bar{H}` - mean of :math:`H`
        - :math:`\\bar{M}` - mean of :math:`M`
        - :math:`Var(X)` - variance of X

    Parameters
    ----------
    y_true_observed : array-like
        The observed scores.
    y_pred : array-like
        The predicted scores.
    ddof : int, optional
        Means Delta Degrees of Freedom. The divisor used in
        calculations is N - ddof, where N represents the
        number of elements. When ddof is set to zero, the results
        for discrete case match those from the standard implementations.
        Defaults to 0.

    Returns
    -------
    kappa : float
        The quadratic weighted kappa

    Raises
    ------
    AssertionError
        If the number of elements in ``y_true_observed`` is not equal
        to the number of elements in ``y_pred``.
    """
    assert len(y_true_observed) == len(y_pred)
    y_true_observed_var, y_true_observed_avg = (np.var(y_true_observed, ddof=ddof),
                                                np.mean(y_true_observed))
    y_pred_var, y_pred_avg = (np.var(y_pred, ddof=ddof),
                              np.mean(y_pred))

    numerator = 2 * np.cov(y_true_observed, y_pred, ddof=ddof)[0][1]
    denominator = y_true_observed_var + y_pred_var + (y_true_observed_avg - y_pred_avg)**2
    kappa = numerator / denominator
    return kappa def standardized_mean_difference(y_true_observed,
                                 y_pred,
                                 population_y_true_observed_sd=None,
                                 population_y_pred_sd=None,
                                 method='unpooled',
                                 ddof=1):
    """
    Compute the standardized mean difference between system and human scores.

    The numerator is calculated as mean(y_pred) - mean(y_true_observed)
    for all of the available methods.

    Parameters
    ----------
    y_true_observed : array-like
        The observed scores for the group or subgroup.
    y_pred : array-like
        The predicted score for the group or subgroup.
    population_y_true_observed_sd : float, optional
        The population true score standard deviation.
        When the SMD is being calculated for a subgroup,
        this should be the standard deviation for the whole
        population.
        Defaults to ``None``.
    population_y_pred_sd : float, optional
        The predicted score standard deviation.
        When the SMD is being calculated for a subgroup,
        this should be the standard deviation for the whole
        population.
        Defaults to ``None``.
    method : str, optional
        The SMD method to use. Possible options are:

        - "williamson": Denominator is the pooled population standard deviation
          of ``y_true_observed`` and ``y_pred`` computed using
          ``population_y_true_observed_sd`` and ``population_y_pred_sd``.
        - "johnson": Denominator is ``population_y_true_observed_sd``.
        - "pooled": Denominator is the pooled standard deviation of
          ``y_true_observed`` and ``y_pred`` for this group.
        - "unpooled": Denominator is the standard deviation of
          ``y_true_observed`` for this group.

        Defaults to "unpooled".

    ddof : int, optional
        The delta degrees of freedom. The divisor used in
        calculations is N - ddof, where N represents the
        number of elements.
        Defaults to 1.

    Returns
    -------
    smd : float
        The SMD for the given group or subgroup.

    Raises
    ------
    ValueError
        If method is "williamson" and either ``population_y_true_observed_sd``
        or ``population_y_pred_sd`` is ``None``.
    ValueError
        If method is "johnson" and ``population_y_true_observed_sd`` is ``None``.
    ValueError
        If method is not one of {"unpooled", "pooled", "williamson", "johnson"}.

    Note
    ----
    - The "williamson" implementation was recommended by Williamson,
      et al. (2012).
    - The metric is only applicable when both sets of scores are on
      the same scale.
    """
    numerator = np.mean(y_pred) - np.mean(y_true_observed)

    method = method.lower()
    if method == 'unpooled':
        denominator = np.std(y_true_observed, ddof=ddof)
    elif method == 'pooled':
        denominator = np.sqrt((np.std(y_true_observed, ddof=ddof)**2 +
                               np.std(y_pred, ddof=ddof)**2) / 2)
    elif method == 'johnson':
        if population_y_true_observed_sd is None:
            raise ValueError("If `method='johnson'`, then `population_y_true_observed_sd` "
                             "must be provided.")
        denominator = population_y_true_observed_sd
    elif method == 'williamson':
        if population_y_true_observed_sd is None or population_y_pred_sd is None:
            raise ValueError("If `method='williamson'`, both `population_y_true_observed_sd` "
                             "and `population_y_pred_sd` must be provided.")
        denominator = np.sqrt((population_y_true_observed_sd**2 +
                               population_y_pred_sd**2) / 2)
    else:
        possible_methods = {"'unpooled'", "'pooled'", "'johnson'", "'williamson'"}
        raise ValueError("The available methods are {{{}}}; you selected {}."
                         "".format(', '.join(possible_methods), method))

    # if the denominator is zero, then return NaN as the SMD
    smd = np.nan if denominator == 0 else numerator / denominator
    return smd def bold_highlighter(num, low=0, high=1, prec=3, absolute=False):
    """
    Instantiate a ``custom_highlighter()`` with "bold" as default class.

    Parameters:
    -----------
    num : float
        The floating point number to format.
    low : float
        The number will be displayed as an HTML span it is below this value.
        Defaults to 0.
    high : float
        The number will be displayed as an HTML span it is above this value.
        Defaults to 1.
    prec : int
        The number of decimal places to display for x.
        Defaults to 3.
    absolute : bool
        If ``True``, use the absolute value of x for comparison.
        Defaults to ``False``.

    Returns:
    --------
    ans : str
        The formatted highlighter with bold class as default.
    """
    ans = custom_highlighter(num, low, high, prec, absolute, 'bold')
    return ans def color_highlighter(num, low=0, high=1, prec=3, absolute=False):
    """
    Instantiate a ``custom_highlighter()`` with "color" as the default class.

    Parameters:
    -----------
    num : float
        The floating point number to format.
    low : float
        The number will be displayed as an HTML span it is below this value.
        Defaults to 0.
    high : float
        The number will be displayed as an HTML span it is above this value.
        Defaults to 1.
    prec : int
        The number of decimal places to display for x.
        Defaults to 3.
    absolute : bool
        If ``True``, use the absolute value of x for comparison.
        Defaults to ``False``.

    Returns:
    --------
    ans : str
        The formatted highlighter with color class as default.
    """
    ans = custom_highlighter(num, low, high, prec, absolute, 'color')
    return ans def compute_subgroup_plot_params(group_names, num_plots):
    """
    Compute subgroup plot and figure parameters.

    The parameters are computed based on the number of subgroups and the
    number of plots to be generated.

    Parameters
    ----------
    group_names : list of str
        A list of subgroup names for plots.
    num_plots : int
        The number of plots to compute.

    Returns
    -------
    figure_width : int
        The width of the figure.
    figure_height : int
        The height of the figure.
    num_rows : int
        The number of rows for the plots.
    num_columns : int
        The number of columns for the plots.
    wrapped_group_names : list of str
        A list of group names for plots.
    """
    wrapped_group_names = ['\n'.join(wrap(str(gn), 20)) for gn in group_names]
    plot_height = 4 if wrapped_group_names == group_names else 6
    num_groups = len(group_names)
    if num_groups <= 6:
        num_columns = 2
        num_rows = ceil(num_plots / num_columns)
        figure_width = num_columns * num_groups
        figure_height = plot_height * num_rows
    else:
        num_columns = 1
        num_rows = num_plots
        figure_width = 10
        figure_height = plot_height * num_plots

    return (figure_width, figure_height, num_rows, num_columns, wrapped_group_names) def custom_highlighter(num,
                       low=0,
                       high=1,
                       prec=3,
                       absolute=False,
                       span_class='bold'):
    """
    Convert float to an HTML <span> element with given class.

    The conversion only happens if the given float is below ``low``
    or above ``high``. If it does not meet those constraints, then
    a plain string is returned with specified number of decimal places.

    Parameters:
    -----------
    num : float
        The floating point number to format.
    low : float
        The number will be displayed as an HTML span it is below this value.
        Defaults to 0.
    high : float
        The number will be displayed as an HTML span it is above this value.
        Defaults to 1.
    prec : int
        The number of decimal places to display for x. Defaults to 3.
    absolute : bool
        If ``True``, use the absolute value of x for comparison.
        Defaults to ``False``.
    span_class : str
        One of "bold" or "color". These are the two classes
        available for the HTML span tag.

    Returns:
    --------
    ans : str
        The plain or HTML string representing the given number.
    """
    abs_num = abs(num) if absolute else num
    val = float_format_func(num, prec=prec)
    ans = ('<span class="highlight_{}">{}</span>'.format(span_class, val)
           if abs_num < low or abs_num > high else val)
    return ans def float_format_func(num, prec=3):
    """
    Format given float to the specified precision as a string.

    Parameters:
    ----------
    num : float
        The floating point number to format.
    prec: int, optional
        The number of decimal places to use when displaying the number.
        Defaults to 3.

    Returns:
    -------
    ans : str
        The formatted string representing the given number.
    """
    formatter_string = Template('{:.${prec}f}').substitute(prec=prec)
    ans = formatter_string.format(num)
    return ans def get_files_as_html(output_dir, experiment_id, file_format, replace_dict={}):
    """
    Generate an HTML list for each output file in given directory.

    Optionally pass a replacement dictionary to use more descriptive
    titles for the file names.

    Parameters
    ----------
    output_dir : str
        The output directory.
    experiment_id : str
        The experiment ID.
    file_format : str
        The format of the output files.
    replace_dict : dict, optional
        A dictionary which makes file names to descriptions.
        Defaults to ``{}``.

    Returns
    ------
    html_string : str
        HTML string with file descriptions and links.
    """
    output_dir = Path(output_dir)
    parent_dir = output_dir.parent
    files = output_dir.glob('*.{}'.format(file_format))
    html_string = ''
    for file in sorted(files):
        relative_file = ".." / file.relative_to(parent_dir)
        relative_name = relative_file.stem.replace('{}_'.format(experiment_id), '')

        # check if relative name is in the replacement dictionary and,
        # if it is, use the more descriptive name in the replacement
        # dictionary. Otherwise, normalize the file name and use that
        # as the description instead.
        if relative_name in replace_dict:
            descriptive_name = replace_dict[relative_name]
        else:
            descriptive_name_components = relative_name.split('_')
            descriptive_name = ' '.join(descriptive_name_components).title()

        html_string += HTML_STRING.format(descriptive_name,
                                          relative_file,
                                          file_format)

    return """<ul><html>""" + html_string + """</ul></html>""" def get_thumbnail_as_html(path_to_image, image_id, path_to_thumbnail=None):
    """
    Generate HTML for a clickable thumbnail of given image.

    Given the path to an image file, generate the HTML for
    a clickable thumbnail version of the image. When clicked,
    this HTML will open the full-sized version of the image in
    a new window.

    Parameters
    ----------
    path_to_image : str
        The absolute or relative path to the image.
        If an absolute path is provided, it will be
        converted to a relative path.
    image_id : int
        The id of the <img> tag in the HTML. This must
        be unique for each <img> tag.
    path_to_thumbnail : str, optional
        If you would like to use a different thumbnail
        image, specify the path to this thumbnail.
        Defaults to ``None``.

    Returns
    -------
    image : str
        The HTML string generated for the image.

    Raises
    ------
    FileNotFoundError
        If the image file cannot be located.
    """
    error_message = 'The file `{}` could not be located.'
    if not exists(path_to_image):
        raise FileNotFoundError(error_message.format(path_to_image))

    # check if the path is relative or absolute
    if isabs(path_to_image):
        rel_image_path = relpath(path_to_image)
    else:
        rel_image_path = path_to_image

    # if `path_to_thumbnail` is None, use `path_to_image`;
    # otherwise, get the relative path to the thumbnail
    if path_to_thumbnail is None:
        rel_thumbnail_path = rel_image_path
    else:
        if not exists(path_to_thumbnail):
            raise FileNotFoundError(error_message.format(path_to_thumbnail))
        rel_thumbnail_path = relpath(path_to_thumbnail)

    # specify the thumbnail style
    style = """
    <style>
    img {
        border: 1px solid #ddd;
        border-radius: 4px;
        padding: 5px;
        width: 150px;
        cursor: pointer;
    }
    </style>
    """

    # on click, open larger image in new window
    script = """
    <script>
    function getPicture(picpath) {{
        window.open(picpath, 'Image', resizable=1);
    }};
    </script>""".format(image_id)

    # generate image tags
    image = ("""<img id='{}' src='{}' onclick='getPicture("{}")' """
             """title="Click to enlarge">"""
             """</img>""").format(image_id,
                                  rel_image_path,
                                  rel_thumbnail_path)

    # create the image HTML
    image += style
    image += script
    return image def int_or_float_format_func(num, prec=3):
    """
    Identify whether the number is float or integer.

    When displaying integers, use no decimal. For a float, round to the
    specified number of decimal places and convert to a string.

    Parameters:
    -----------
    num : float or int
        The number to format and display.
    prec : int, optional
        The number of decimal places to display if x is a float.
        Defaults to 3.

    Returns:
    -------
    ans : str
        The formatted string representing the given number.
    """
    if float.is_integer(num):
        ans = '{}'.format(int(num))
    else:
        ans = float_format_func(num, prec=prec)
    return ans class DataWriter:
    """Class to write out DataContainer objects."""

    def __init__(self, experiment_id=None):  # noqa: D107
        self._id = experiment_id

    @staticmethod
    def write_frame_to_file(df, name_prefix, file_format="csv", index=False, **kwargs):
        """
        Write given data frame to disk with given name and file format.

        Parameters
        ----------
        df : pandas DataFrame
            Data frame to write to disk
        name_prefix : str
            The complete prefix for the file to be written to disk.
            This includes everything except the extension.
        file_format : str
            The file format (extension) for the file to be written to disk.
            One of {"csv", "xlsx", "tsv"}.
            Defaults to "csv".
        index : bool, optional
            Whether to include the index in the output file.
            Defaults to ``False``.

        Raises
        ------
        KeyError
            If ``file_format`` is not valid.
        """
        file_format = file_format.lower()

        if file_format == 'csv':
            name_prefix += '.csv'
            df.to_csv(name_prefix, index=index, **kwargs)

        elif file_format == 'tsv':
            name_prefix += '.tsv'
            df.to_csv(name_prefix, index=index, sep='\t', **kwargs)

        # Added jsonlines for experimental purposes, but leaving
        # this out of the documentation at this stage
        elif file_format == 'jsonlines':
            name_prefix += '.jsonlines'
            df.to_json(name_prefix, orient='records', lines=True, **kwargs)

        elif file_format == 'xlsx':
            name_prefix += '.xlsx'
            df.to_excel(name_prefix, index=index, **kwargs)

        else:
            raise KeyError("Please make sure that the `file_format` specified "
                           "is one of the following:\n{`csv`, `tsv`, `xlsx`}.\n"
                           f"You specified {file_format}.")

    def write_experiment_output(self,
                                csvdir,
                                container_or_dict,
                                dataframe_names=None,
                                new_names_dict=None,
                                include_experiment_id=True,
                                reset_index=False,
                                file_format='csv',
                                index=False,
                                **kwargs):
        """
        Write out each of the named frames to disk.

        This function writes out each of the given list of data frames as a
        ".csv", ".tsv", or ``.xlsx`` file in the given directory. Each data
        frame was generated as part of running an RSMTool experiment. All files
        are prefixed with the given experiment ID and suffixed with either the
        name of the data frame in the DataContainer (or dict) object, or a new
        name if ``new_names_dict`` is specified. Additionally, the indexes in
        the  data frames are reset if so specified.

        Parameters
        ----------
        csvdir : str
            Path to the output experiment sub-directory that will
            contain the CSV files corresponding to each of the data frames.
        container_or_dict : container.DataContainer or dict
            A DataContainer object or dict, where keys are data frame
            names and values are ``pd.DataFrame`` objects.
        dataframe_names : list of str, optional
            List of data frame names, one for each of the data frames.
            Defaults to ``None``.
        new_names_dict : dict, optional
            New dictionary with new names for the data frames, if desired.
            Defaults to ``None``.
        include_experiment_id : str, optional
            Whether to include the experiment ID in the file name.
            Defaults to ``True``.
        reset_index : bool, optional
            Whether to reset the index of each data frame
            before writing to disk.
            Defaults to ``False``.
        file_format : str, optional
            The file format in which to output the data.
            One of {"csv", "xlsx", "tsv"}.
            Defaults to "csv".
        index : bool, optional
            Whether to include the index in the output file.
            Defaults to ``False``.

        Raises
        ------
        KeyError
            If ``file_format`` is not valid, or a named data frame
            is not present in ``container_or_dict``.
        """
        container_or_dict = container_or_dict.copy()

        # If no `dataframe_names` specified, use all names
        if dataframe_names is None:
            dataframe_names = container_or_dict.keys()

        # Otherwise, check to make sure all specified names
        # are actually in the DataContainer
        else:
            for name in dataframe_names:
                if name not in container_or_dict:
                    raise KeyError("The name `{}` is not in the container "
                                   "or dictionary.".format(name))

        # Loop through DataFrames, and save
        # output in specified format
        for dataframe_name in dataframe_names:

            df = container_or_dict[dataframe_name]
            if df is None:
                raise KeyError('The DataFrame `{}` does not exist.'.format(dataframe_name))

            # If the DataFrame is empty, skip it
            if df.empty:
                continue

            # If there is a desire to rename the DataFrame,
            # get the new name
            if new_names_dict is not None:
                if dataframe_name in new_names_dict:
                    dataframe_name = new_names_dict[dataframe_name]

            # Reset the index, if desired
            if reset_index:
                df.index.name = ''
                df.reset_index(inplace=True)

            # If include_experiment_id is True, and the experiment_id exists
            # include it in the file name; otherwise, do not include it.
            if include_experiment_id and self._id is not None:
                outfile = join(csvdir, '{}_{}'.format(self._id, dataframe_name))
            else:
                outfile = join(csvdir, dataframe_name)

            # write out the frame to disk in the given file
            self.write_frame_to_file(df,
                                     outfile,
                                     file_format=file_format,
                                     index=index,
                                     **kwargs)

    def write_feature_csv(self,
                          featuredir,
                          data_container,
                          selected_features,
                          include_experiment_id=True,
                          file_format='csv'):
        """
        Write out the selected features to disk.

        Parameters
        ----------
        featuredir : str
            Path to the experiment output directory where the
            feature JSON file will be saved.
        data_container : container.DataContainer
            A data container object.
        selected_features : list of str
            List of features that were selected for model building.
        include_experiment_id : bool, optional
            Whether to include the experiment ID in the file name.
            Defaults to ``True``.
        file_format : str, optional
            The file format in which to output the data.
            One of {"csv", "tsv", "xlsx"}.
            Defaults to "csv".
        """
        df_feature_specs = data_container['feature_specs']

        # Select specific features used in training
        df_selected = df_feature_specs[df_feature_specs['feature'].isin(selected_features)]

        # Replace existing `feature_specs` with selected features specs
        data_container.add_dataset({'frame': df_selected,
                                    'name': 'feature_specs'}, update=True)

        makedirs(featuredir, exist_ok=True)
        self.write_experiment_output(featuredir,
                                     data_container,
                                     ['feature_specs'],
                                     {'feature_specs': 'selected'},
                                     include_experiment_id=include_experiment_id,
                                     file_format=file_format)
import argparse
import filecmp
import warnings
import numpy as np
import pandas as pd
from io import StringIO
from itertools import count, product
from os import environ, getcwd, listdir, makedirs, unlink
from os.path import abspath, join, relpath
from pathlib import Path
from shutil import rmtree
from tempfile import NamedTemporaryFile, TemporaryDirectory, mkdtemp
from unittest.mock import patch
from uuid import uuid4
from nose.tools import assert_dict_equal, assert_equal, eq_, ok_, raises
from numpy.testing import assert_almost_equal, assert_array_equal
from pandas.testing import assert_frame_equal
from prompt_toolkit.formatted_text import HTML
from prompt_toolkit.shortcuts import CompleteStyle
from rsmtool.configuration_parser import Configuration, ConfigurationParser
from rsmtool.reader import DataReader
from rsmtool.utils.commandline import CmdOption, ConfigurationGenerator, InteractiveField, setup_rsmcmd_parser
from rsmtool.utils.constants import CHECK_FIELDS, DEFAULTS, INTERACTIVE_MODE_METADATA
from rsmtool.utils.conversion import convert_to_float, int_to_float
from rsmtool.utils.cross_validation import combine_fold_prediction_files, create_xval_files
from rsmtool.utils.files import get_output_directory_extension, has_files_with_extension, parse_json_with_comments
from rsmtool.utils.logging import get_file_logger
from rsmtool.utils.metrics import (compute_expected_scores_from_model,
                                   difference_of_standardized_means,
                                   partial_correlations,
                                   quadratic_weighted_kappa,
                                   standardized_mean_difference)
from rsmtool.utils.notebook import (bold_highlighter,
                                    color_highlighter,
                                    compute_subgroup_plot_params,
                                    custom_highlighter,
                                    float_format_func,
                                    get_files_as_html,
                                    get_thumbnail_as_html,
                                    int_or_float_format_func)
from sklearn.datasets import make_classification
from sklearn.exceptions import ConvergenceWarning
from sklearn.metrics import cohen_kappa_score
from skll.data import FeatureSet
from skll.learner import Learner
from skll.metrics import kappa
from rsmtool.writer import DataWriter
    from rsmtool.test_utils import rsmtool_test_dir

        cls.mocked_rsmcompare_interactive_values = ["testcompare",       # comparison_id
                                                    "rsmtool1",          # experiment_id_old
                                                    "/a/b/c",            # experiment_dir_old
                                                    "rsmtool2",          # experiment_id_new
                                                    '/d/e',              # experiment_dir_new
                                                    "rsmtool expt 1",    # description_old
                                                    "rsmtool expt 2",    # description_new
                                                    [],                  # subgroups
                                                    True                 # use_thumbnails
                                                    ]

        cls.mocked_rsmpredict_interactive_values = ["testpred",          # experiment_id
                                                    "/a/b",              # experiment_dir_new
                                                    "features.csv",      # input_features_file
                                                    "csv",               # file_format
                                                    "score",             # human_score_column
                                                    "spkitemid",         # id_column
                                                    None,                # second_human_score_column
                                                    True,                # standardize_features
                                                    ]

        cls.mocked_rsmsummarize_interactive_values = ["testsumm",          # summary_id
                                                      ["/a/b",
                                                       "/d",
                                                       "/e/f/g"],          # experiment_dirs
                                                      'summary test',      # description
                                                      "tsv",               # file_format
                                                      True,                # use_thumbnails
                                                      ]

        cls.mocked_rsmxval_interactive_values = ["testxval",          # experiment_id
                                                 "LinearSVC",         # model
                                                 "train.csv",         # train_file
                                                 'xval test',         # description
                                                 True,                # exclude_zero_scores
                                                 "xlsx",              # file_format
                                                 3,                   # folds
                                                 None,                # folds file
                                                 "ID",                # id_column
                                                 "length",            # length_column
                                                 "score2",            # second_human_score_column
                                                 True,                # standardize_features
                                                 ["L1", "QUESTION"],  # subgroups
                                                 "score",             # train_label_column
                                                 1,                   # trim_min
                                                 5,                   # trim_max,
                                                 True,                # use_scaled_predictions
                                                 False                # use_thumbnails
                                                 ]

        cls.mocked_rsmxval_interactive_values_folds_file = ["testxval",          # experiment_id
                                                            "LinearSVC",         # model
                                                            "train.csv",         # train_file
                                                            'xval test',         # description
                                                            True,                # exclude_zero_scores
                                                            "xlsx",              # file_format
                                                            5,                   # default folds
                                                            "folds.csv",         # folds file
                                                            "ID",                # id_column
                                                            "length",            # length_column
                                                            "score2",            # second_human_score_column
                                                            True,                # standardize_features
                                                            ["L1"],              # subgroups
                                                            "score",             # train_label_column
                                                            1,                   # trim_min
                                                            5,                   # trim_max,
                                                            True,                # use_scaled_predictions
                                                            False                # use_thumbnails
                                                            ]

    def check_tool_interact(self, context, subgroups=False, with_folds_file=False):
        """
        A helper method that runs `ConfigurationGenerator.interact()`
        and compares its output to expected output.

        Parameters
        ----------
        context : str
            Name of the tool being tested.
            One of {"rsmtool", "rsmeval", "rsmcompare", "rsmpredict", "rsmsummarize"}.
        subgroups : bool, optional
            Whether to include subgroup information in the generated configuration.
        with_folds_file : bool, optional
            Whether to use "folds_file" for rsmxval.
        """
        # if we are using subgroups, then define a suffix for the expected file
        groups_suffix = '_groups' if subgroups else ''
        folds_file_suffix = '_folds_file' if with_folds_file else ''

        # get the appropriate list of mocked values for this tool but make
        # a copy since we may need to modify it below
        mocked_values = getattr(self, f"mocked_{context}_interactive_values{folds_file_suffix}")[:]

        # if we are not using subgroups, delete the subgroup entry
        # from the list of mocked values
        if not subgroups:
            if context in ['rsmtool', 'rsmeval']:
                del mocked_values[11]
            elif context == 'rsmcompare':
                del mocked_values[7]
            elif context == 'rsmxval':
                del mocked_values[12]

        # point to the right file holding the expected configuration
        expected_file = f"interactive_{context}_config{groups_suffix}{folds_file_suffix}.json"
        expected_path = join(rsmtool_test_dir, 'data', 'output', expected_file)

        # we need to patch stderr and `prompt_toolkit.shortcuts.clear()`` so
        # that calling 'interact()' doesn't actually print out anything
        # to stderr and doesn't clear the screen
        sys_stderr_patcher = patch('sys.stderr', new_callable=StringIO)
        clear_patcher = patch('rsmtool.utils.commandline.clear')
        _ = clear_patcher.start()
        _ = sys_stderr_patcher.start()

        # mock the `InteractiveField.get_value()` method to return the
        # pre-determined mocked values in order and check that the
        # configuration generated by `interact()` is what we expect it to be
        with patch.object(InteractiveField, 'get_value', side_effect=mocked_values):

            generator = ConfigurationGenerator(context, use_subgroups=subgroups)