import gzip
import os
from pkg_resources import iter_entry_points
from glob2.fnmatch import fnmatch

from .asset_attributes import AssetAttributes
from .assets import build_asset
from .cache import SimpleCache
from .compat import bytes
from .exceptions import FileNotFound
from .manifest import Manifest
from .processors import (
    DirectivesProcessor,
    HexdigestPathsProcessor,
    SemicolonsProcessor
)
from .utils import get_condition_func, unique

DEFAULT_PUBLIC_ASSETS = (
    lambda path: not any(path.endswith(ext) for ext in ('.css', '.js')),
    r'^css/style\.css$',
    r'^js/script\.js$',
)

class Finders(list):
    """The registry for file finders. This is just a list of finder objects.
    Each finder object must be an instance of any
    :class:`~gears.finders.BaseFinder` subclass. Finders from this registry are
    used by :class:`~gears.environment.Environment` object in the order they
    were added.
    """

    def register(self, finder):
        """Append passed ``finder`` to the list of finders."""
        if finder not in self:
            self.append(finder)

    def unregister(self, finder):
        """Remove passed ``finder`` from the list of finders. If ``finder``
        does not found in the registry, nothing happens.
        """
        if finder in self:
            self.remove(finder)

    def list(self, path):
        for finder in self:
            for item in finder.list(path):
                yield item

class MIMETypes(dict):
    """The registry for MIME types. It acts like a dict with extensions as
    keys and MIME types as values. Every registered extension can have only one
    MIME type.
    """

    def register_defaults(self):
        """Register MIME types for ``.js`` and ``.css`` extensions."""
        self.register('.css', 'text/css')
        self.register('.js', 'application/javascript')

    def register(self, extension, mimetype):
        """Register passed ``mimetype`` MIME type with ``extension`` extension.
        """
        self[extension] = mimetype

    def unregister(self, extension):
        """Remove registered MIME type for passed ``extension`` extension. If
        MIME type for this extension does not found in the registry, nothing
        happens.
        """
        if extension in self:
            del self[extension]

class Compilers(dict):
    """The registry for compilers. It acts like a dict with extensions as keys
    and compilers as values. Every registered extension can have only one
    compiler.
    """

    def register(self, extension, compiler):
        """Register passed `compiler` with passed `extension`."""
        self[extension] = compiler

    def unregister(self, extension):
        """Remove registered compiler for passed `extension`. If compiler for
        this extension does not found in the registry, nothing happens.
        """
        if extension in self:
            del self[extension]

class Processors(dict):
    """Base class for processors registries."""

    def register(self, mimetype, processor):
        """Register passed `processor` for passed `mimetype`."""
        if mimetype not in self or processor not in self[mimetype]:
            self.setdefault(mimetype, []).append(processor)

    def unregister(self, mimetype, processor):
        """Remove passed `processor` for passed `mimetype`. If processor for
        this MIME type does not found in the registry, nothing happens.
        """
        if mimetype in self and processor in self[mimetype]:
            self[mimetype].remove(processor)

    def get(self, mimetype):
        """Return a list of processors, registered for passed `mimetype`. If
        no processors are registered for this MIME type, empty list is
        returned.
        """
        return super(Processors, self).get(mimetype, [])

class Preprocessors(Processors):
    """The registry for asset preprocessors. It acts like a dictionary with
    MIME types as keys and lists of processors as values. Every registered MIME
    type can have many preprocessors. Preprocessors for the MIME type are used
    in the order they were added.
    """

    def register_defaults(self):
        """Register :class:`~gears.processors.DirectivesProcessor` as
        a preprocessor for `text/css` and `application/javascript` MIME types.
        """
        self.register('text/css', DirectivesProcessor.as_handler())
        self.register('application/javascript', DirectivesProcessor.as_handler())

class Postprocessors(Processors):
    """The registry for asset postprocessors. It acts like a dictionary with
    MIME types as keys and lists of processors as values. Every registered MIME
    type can have many postprocessors. Postprocessors for the MIME type are
    used in the order they were added.
    """

    def register_defaults(self):
        self.register('application/javascript', SemicolonsProcessor.as_handler())
        self.register('text/css', HexdigestPathsProcessor.as_handler())

class Compressors(dict):
    """The registry for asset compressors. It acts like a dictionary with
    MIME types as keys and compressors as values. Every registered MIME type
    can have only one compressor.
    """

    def register(self, mimetype, compressor):
        """Register passed `compressor` for passed `mimetype`."""
        self[mimetype] = compressor

    def unregister(self, mimetype):
        """Remove registered compressors for passed `mimetype`. If compressor
        for this MIME type does not found in the registry, nothing happens.
        """
        if mimetype in self:
            del self[mimetype]

class Suffixes(list):
    """The registry for asset suffixes. It acts like a list of dictionaries.
    Every dictionary has three keys: ``extensions``, ``result_mimetype`` and
    ``mimetype``:

    - ``suffix`` is a suffix as a list of extensions (e.g. ``['.js', '.coffee']``);
    - ``result_mimetype`` is a MIME type of a compiled asset with this suffix;
    - ``mimetype`` is a MIME type, for which this suffix is registered.
    """

    def register(self, extension, root=False, to=None, mimetype=None):
        if root:
            self.append({
                'suffix': [extension],
                'full': [extension],
                'result_mimetype': mimetype,
                'mimetype': mimetype,
            })
            return
        new = []
        for item in self:
            if to is not None and item['mimetype'] != to:
                continue
            suffix = list(item['suffix'])
            suffix.append(extension)
            full = list(item['full'])
            full.append(extension)
            new.append({
                'suffix': suffix,
                'full': full,
                'result_mimetype': item['result_mimetype'],
                'mimetype': mimetype,
            })
            if to is not None:
                new.append({
                    'suffix': [extension],
                    'full': full,
                    'result_mimetype': item['result_mimetype'],
                    'mimetype': mimetype,
                })
        self.extend(new)

    def unregister(self, extension):
        for item in list(self):
            if extension in item['full']:
                self.remove(item)

    def find(self, mimetype=None):
        suffixes = []
        for item in self:
            if mimetype is None or item['result_mimetype'] == mimetype:
                suffixes.append(''.join(item['suffix']))
        return suffixes

class Environment(object):
    """This is the central object, that links all Gears parts. It is passed the
    absolute path to the directory where public assets will be saved.
    Environment contains registries for file finders, compilers, compressors,
    processors and supported MIME types.

    :param root: the absolute path to the directory where handled public assets
                 will be saved by :meth:`save` method.
    :param public_assets: a list of public assets paths.
    :param cache: a cache object. It is used by assets and dependencies to
                  store compilation results.
    :param fingerprinting: if set to `True`, fingerprinted versions of assets
                           won't be created.
    """

    def __init__(self, root, public_assets=DEFAULT_PUBLIC_ASSETS,
                 manifest_path=None, cache=None, gzip=False,
                 fingerprinting=True):
        self.root = root
        self.public_assets = [get_condition_func(c) for c in public_assets]

        if manifest_path is not None:
            self.manifest_path = manifest_path
        else:
            self.manifest_path = os.path.join(self.root, '.manifest.json')
        self.manifest = Manifest(self.manifest_path)

        if cache is not None:
            self.cache = cache
        else:
            self.cache = SimpleCache()

        self.gzip = gzip
        self.fingerprinting = fingerprinting

        #: The registry for file finders. See
        #: :class:`~gears.environment.Finders` for more information.
        self.finders = Finders()

        #: The registry for asset compilers. See
        #: :class:`~gears.environment.Compilers` for more information.
        self.compilers = Compilers()

        #: The registry for supported MIME types. See
        #: :class:`~gears.environment.MIMETypes` for more information.
        self.mimetypes = MIMETypes()

        #: The registry for asset compressors. See
        #: :class:`~gears.environment.Compressors` for more information.
        self.compressors = Compressors()

        #: The registry for asset preprocessors. See
        #: :class:`~gears.environment.Preprocessors` for more information.
        self.preprocessors = Preprocessors()

        #: The registry for asset postprocessors. See
        #: :class:`~gears.environment.Postprocessors` for more information.
        self.postprocessors = Postprocessors()

    @property
    def suffixes(self):
        """The registry for supported suffixes of assets. It is built from
        MIME types and compilers registries, and is cached at the first call.
        See :class:`~gears.environment.Suffixes` for more information.
        """
        if not hasattr(self, '_suffixes'):
            suffixes = Suffixes()
            for extension, mimetype in self.mimetypes.items():
                suffixes.register(extension, root=True, mimetype=mimetype)
            for extension, compiler in self.compilers.items():
                suffixes.register(extension, to=compiler.result_mimetype)
            self._suffixes = suffixes
        return self._suffixes

    @property
    def paths(self):
        """The list of search paths. It is built from registered finders, which
        has ``paths`` property. Can be useful for compilers to resolve internal
        dependencies.
        """
        if not hasattr(self, '_paths'):
            paths = []
            for finder in self.finders:
                if hasattr(finder, 'paths'):
                    paths.extend(finder.paths)
            self._paths = paths
        return self._paths

    def register_defaults(self):
        """Register default compilers, preprocessors and MIME types."""
        self.mimetypes.register_defaults()
        self.preprocessors.register_defaults()
        self.postprocessors.register_defaults()

    def register_entry_points(self, exclude=()):
        """Allow Gears plugins to inject themselves to the environment. For
        example, if your plugin's package contains such ``entry_points``
        definition in ``setup.py``, ``gears_plugin.register`` function will be
        called with current environment during ``register_entry_points`` call::

            entry_points = {
                'gears': [
                    'register = gears_plugin:register',
                ],
            }

        Here is an example of such function::

            def register(environment):
                assets_dir = os.path.join(os.path.dirname(__file__), 'assets')
                assets_dir = os.path.absolute_path(assets_dir)
                environment.register(FileSystemFinder([assets_dir]))

        If you want to disable this behavior for some plugins, list their
        packages using ``exclude`` argument::

            environment.register_entry_points(exclude=['plugin'])
        """
        for entry_point in iter_entry_points('gears', 'register'):
            if entry_point.module_name not in exclude:
                register = entry_point.load()
                register(self)

    def find(self, item, logical=False):
        """Find files using :attr:`finders` registry. The ``item`` parameter
        can be an instance of :class:`~gears.asset_attributes.AssetAttributes`
        class, a path to the asset or a logical path to the asset. If ``item``
        is a logical path, `logical` parameter must be set to ``True``.

        Returns a tuple with :class:`~gears.asset_attributes.AssetAttributes`
        instance for found file path as first item, and absolute path to this
        file as second item.

        If nothing is found, :class:`gears.exceptions.FileNotFound` exception
        is rased.
        """
        if isinstance(item, AssetAttributes):
            for path in item.search_paths:
                try:
                    return self.find(path, logical)
                except FileNotFound:
                    continue
            raise FileNotFound(item.path)
        if logical:
            asset_attributes = AssetAttributes(self, item)
            suffixes = self.suffixes.find(asset_attributes.mimetype)
            if not suffixes:
                return self.find(item)
            path = asset_attributes.path_without_suffix
            for suffix in suffixes:
                try:
                    return self.find(path + suffix)
                except FileNotFound:
                    continue
        else:
            for finder in self.finders:
                try:
                    absolute_path = finder.find(item)
                except FileNotFound:
                    continue
                return AssetAttributes(self, item), absolute_path
        raise FileNotFound(item)

    def list(self, path, mimetype=None):
        """Yield two-tuples for all files found in the directory given by
        ``path`` parameter. Result can be filtered by the second parameter,
        ``mimetype``, that must be a MIME type of assets compiled source code.
        Each tuple has :class:`~gears.asset_attributes.AssetAttributes`
        instance for found file path as first item, and absolute path to this
        file as second item.

        Usage example::

            # Yield all files from 'js/templates' directory.
            environment.list('js/templates/*')

            # Yield only files that are in 'js/templates' directory and have
            # 'application/javascript' MIME type of compiled source code.
            environment.list('js/templates/*', mimetype='application/javascript')
        """
        basename_pattern = os.path.basename(path)

        if path.endswith('**'):
            paths = [path]
        else:
            paths = AssetAttributes(self, path).search_paths
        paths = map(lambda p: p if p.endswith('*') else p + '*', paths)

        results = unique(self._list_paths(paths), lambda x: x[0])
        for logical_path, absolute_path in results:
            asset_attributes = AssetAttributes(self, logical_path)
            if mimetype is not None and asset_attributes.mimetype != mimetype:
                continue

            basename = os.path.basename(asset_attributes.path_without_suffix)
            if not fnmatch(basename, basename_pattern) and basename != 'index':
                continue

            yield asset_attributes, absolute_path

    def _list_paths(self, paths):
        for path in paths:
            for result in self.finders.list(path):
                yield result

    def save(self):
        """Save handled public assets to :attr:`root` directory."""
        for asset_attributes, absolute_path in self.list('**'):
            logical_path = os.path.normpath(asset_attributes.logical_path)
            check_asset = build_asset(self, logical_path, check=True)
            if check_asset.is_public:
                asset = build_asset(self, logical_path)
                source = bytes(asset)
                self.save_file(logical_path, source, asset.gzippable)
                if self.fingerprinting:
                    self.save_file(asset.hexdigest_path, source, asset.gzippable)
                    self.manifest.files[logical_path] = asset.hexdigest_path
        self.manifest.dump()

    def save_file(self, path, source, gzippable=False):
        filename = os.path.join(self.root, path)
        path = os.path.dirname(filename)
        if not os.path.exists(path):
            os.makedirs(path)
        elif not os.path.isdir(path):
            raise OSError("%s exists and is not a directory." % path)
        with open(filename, 'wb') as f:
            f.write(source)
        if self.gzip and gzippable:
            with gzip.open('{}.gz'.format(filename), 'wb') as f:
                f.write(source)

    def is_public(self, logical_path):
        return any(condition(logical_path) for condition in self.public_assets)


import json

from idaapi import PluginForm
from PyQt5 import QtCore
from PyQt5 import QtGui
from PyQt5 import QtWidgets

import ida_settings

class IdaSettingsEditor(PluginForm):
    def OnCreate(self, form):
        # 6.8 and below
        #self.parent = self.FormToPySideWidget(form)
        # 6.9 and above
        self.parent = self.FormToPyQtWidget(form)
        self.PopulateForm()

    def PopulateForm(self):
        """
        +-----------------------------------------------------------------------+
        | +--- splitter ------------------------------------------------------+ |
        | | +-- list widget--------------+  +- IdaSettingsView -------------+ | |
        | | |                            |  |                               | | |
        | | |  - plugin name             |  |                               | | |
        | | |  - plugin name             |  |                               | | |
        | | |  - plugin name             |  |                               | | |
        | | |                            |  |                               | | |
        | | |                            |  |                               | | |
        | | |                            |  |                               | | |
        | | |                            |  |                               | | |
        | | |                            |  |                               | | |
        | | |                            |  |                               | | |
        | | |                            |  |                               | | |
        | | |                            |  |                               | | |
        | | |                            |  |                               | | |
        | | |                            |  |                               | | |
        | | |                            |  |                               | | |
        | | |                            |  |                               | | |
        | | |                            |  |                               | | |
        | | +----------------------------+  +-------------------------------+ | |
        | +-------------------------------------------------------------------+ |
        +-----------------------------------------------------------------------+
        """
        hbox = QtWidgets.QHBoxLayout(self.parent)

        self._splitter = QtWidgets.QSplitter(QtCore.Qt.Horizontal)
        self._plugin_list = QtWidgets.QListWidget()

        plugin_names = set([])
        for scope, fn in (("idb", ida_settings.IDASettings.get_idb_plugin_names),
                ("directory", ida_settings.IDASettings.get_directory_plugin_names),
                ("user", ida_settings.IDASettings.get_user_plugin_names),
                ("system", ida_settings.IDASettings.get_system_plugin_names)):
            for plugin_name in fn():
                plugin_names.add(plugin_name)
        for plugin_name in plugin_names:
            self._plugin_list.addItem(plugin_name)
        self._splitter.addWidget(self._plugin_list)

        hbox.addWidget(self._splitter)
        self.parent.setLayout(hbox)

        self._plugin_list.currentItemChanged.connect(self._handle_plugin_changed)

    def _clear_settings_widgets(self):
        for i in range(1, self._splitter.count()):
            w = self._splitter.widget(i)
            if w is not None:
                w.deleteLater()

    def _set_settings_widget(self, settings):
        self._clear_settings_widgets()
        w = IdaSettingsView(settings, parent=self.parent)
        self._splitter.insertWidget(1, w)

    def _handle_plugin_changed(self, current, previous):
        plugin_name = str(current.text())
        settings = ida_settings.IDASettings(plugin_name)
        self._set_settings_widget(settings)

class IdaSettingsView(QtWidgets.QWidget):
    def __init__(self, settings, parent=None):
        """
        +-----------------------------------------------------------------------+
        | +--- hbox ----------------------------------------------------------+ |
        | | +-- list widget--------------+  +- vbox ------------------------+ | |
        | | |                            |  | +- QTextEdit ---------------+ | | |
        | | |  - key                     |  | |                           | | | |
        | | |  - key                     |  | |  value                    | | | |
        | | |  - key                     |  | |                           | | | |
        | | |                            |  | |                           | | | |
        | | |                            |  | |                           | | | |
        | | |                            |  | |                           | | | |
        | | |                            |  | |                           | | | |
        | | |                            |  | |                           | | | |
        | | |                            |  | +---------------------------+ | | |
        | | |                            |  |                               | | |
        | | |                            |  | +- QPushButton -------------+ | | |
        | | |                            |  | |                           | | | |
        | | |                            |  | |  save                     | | | |
        | | |                            |  | |                           | | | |
        | | |                            |  | +---------------------------+ | | |
        | | +----------------------------+  +-------------------------------+ | |
        | +-------------------------------------------------------------------+ |
        +-----------------------------------------------------------------------+
        """
        super(IdaSettingsView, self).__init__(parent=parent)
        self._settings = settings
        self._current_key = None
        self._current_scope = None

        hbox = QtWidgets.QHBoxLayout(self)
        self._key_list = QtWidgets.QListWidget()
        for scope, keys in (("idb", self._settings.idb.iterkeys()),
                            ("directory", self._settings.directory.iterkeys()),
                            ("user", self._settings.user.iterkeys()),
                            ("system", self._settings.system.iterkeys())):
            for key in keys:
                self._key_list.addItem("({scope:s}) {key:s}".format(scope=scope, key=key))

        hbox.addWidget(self._key_list)

        vbox = QtWidgets.QVBoxLayout(self)
        self._value_view = QtWidgets.QTextEdit(self)
        self._save_button = QtWidgets.QPushButton("save")
        vbox.addWidget(self._value_view)
        vbox.addWidget(self._save_button)

        hbox.addLayout(vbox)

        self._key_list.currentItemChanged.connect(self._handle_key_changed)
        self._save_button.clicked.connect(self._handle_save_value)

        self.setLayout(hbox)

    def _handle_key_changed(self, current, previous):
        if current is None:
            return

        self._value_view.clear()
        scope, _, key = str(current.text()).partition(" ")
        scope = scope.lstrip("(").rstrip(")")
        self._current_scope = scope
        self._current_key = key

        v = getattr(self._settings, scope)[key]
        self._value_view.setText(json.dumps(v))

    def _handle_save_value(self):
        v = str(self._value_view.toPlainText())
        s = getattr(self._settings, self._current_scope)
        s[self._current_key] = json.loads(v)

def main():
    v = IdaSettingsEditor()
    v.Show("Plugin Settings Editor")

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-

from functools import wraps
from subprocess import Popen, PIPE

class AssetHandlerError(Exception):
    pass

class BaseAssetHandler(object):
    """Base class for all asset handlers (processors, compilers and
    compressors). A subclass has to implement :meth:`__call__` which is called
    with asset as argument.
    """

    supports_check_mode = False

    def __call__(self, asset):
        """Subclasses have to override this method to implement the actual
        handler function code. This method is called with asset as argument.
        Depending on the type of the handler, this method must change asset
        state (as it does in :class:`~gears.processors.Directivesprocessor`)
        or return some value (in case of asset compressors).
        """
        raise NotImplementedError

    @classmethod
    def as_handler(cls, **initkwargs):
        """Converts the class into an actual handler function that can be used
        when registering different types of processors in
        :class:`~gears.environment.Environment` class instance.

        The arguments passed to :meth:`as_handler` are forwarded to the
        constructor of the class.
        """
        @wraps(cls, updated=())
        def handler(asset, *args, **kwargs):
            return handler.handler_class(**initkwargs)(asset, *args, **kwargs)
        handler.handler_class = cls
        handler.supports_check_mode = cls.supports_check_mode
        return handler

class ExecMixin(object):
    """Provides the ability to process asset through external command."""

    #: The name of the executable to run. It must be a command name, if it is
    #: available in the PATH environment variable, or a path to the executable.
    executable = None

    #: The list of executable parameters.
    params = []

    def run(self, input):
        """Runs :attr:`executable` with ``input`` as stdin.
        :class:`AssetHandlerError` exception is raised, if execution is failed,
        otherwise stdout is returned.
        """
        p = self.get_process()
        output, errors = p.communicate(input=input.encode('utf-8'))
        if p.returncode != 0:
            raise AssetHandlerError(errors)
        return output.decode('utf-8')

    def get_process(self):
        """Returns :class:`subprocess.Popen` instance with args from
        :meth:`get_args` result and piped stdin, stdout and stderr.
        """
        return Popen(self.get_args(), stdin=PIPE, stdout=PIPE, stderr=PIPE)

    def get_args(self):
        """Returns the list of :class:`subprocess.Popen` arguments."""
        return [self.executable] + self.params


"""
ida_settings provides a mechanism for settings and fetching
configration values for IDAPython scripts and plugins.
Configurations are namespaced by plugin name,
and scoped to the global system, current user,
working directory, or current IDB file. Configurations
can be exported and imported using an .ini-style intermediate
representation.

Example fetching a configuration value:

    settings = IDASettings("MSDN-doc")
    if settings["verbosity"] == "high":
        ...

Example setting a global configuration value:

    settings = IDASettings("MSDN-doc")
    settings.system["verbosity"] = "high"

Example setting a working directory configuration value:

    settings = IDASettings("MSDN-doc")
    settings.directory["verbosity"] = "high"

Use the properties "system", "user", "directory" and "idb"
to scope configuration accesses and mutations to the global
system, current user, working directory, or current IDB file.

Plugins that write settings should pick the appropriate
scope for their settings. Plugins that read settings should
fetch them from the default scope. This allows for precedence
of scopes, such as the current IDB over system-wide configuration.
For example:

    settings = IDASettings("MSDN-doc")
    # when writing, use a scope:
    settings.user["verbosity"] = "high"

    # when reading, use the default scope:
    settings["verbosity"] --> "high"

Generally, treat a settings instance like a dictionary. For example:

    settings = IDASettings("MSDN-doc")
    "verbosity" in settings.user --> False
    settings.user["verbosity"] = "high"
    settings.user["verbosity"] --> "high"
    settings.user.keys()       --> ["verbosity"]
    settings.user.values()     --> ["high"]
    settings.user.items()      --> [("verbosity", "high')]

The value of a particular settings entry must be a JSON-encodable
value. For example, these are fine:

    settings = IDASettings("MSDN-doc")
    settings.user["verbosity"] = "high"
    settings.user["count"] = 1
    settings.user["percentage"] = 0.75
    settings.user["filenames"] = ["a.txt", "b.txt"]
    settings.user["aliases"] = {"bp": "breakpoint", "g": "go"}

and this is not:

    settings.user["object"] = hashlib.md5()      # this is not JSON-encodable

To export the current effective settings, use the `export_settings`
function. For example:

    settings = IDASettings("MSDN-doc")
    export_settings(settings, "/home/user/desktop/current.ini")

To import existing settings into a settings instance, such as
the open IDB, use the `import_settings` function. For example:

    settings = IDASettings("MSDN-doc")
    import_settings(settings.idb, "/home/user/desktop/current.ini")

Enumerate the plugin names for the various levels using the
IDASettings class properties:

    IDASettings.get_system_plugin_names()     --> ["plugin-1", "plugin-2"]
    IDASettings.get_user_plugin_names()       --> ["plugin-3", "plugin-4"]
    IDASettings.get_directory_plugin_names()  --> ["plugin-5", "plugin-6"]
    IDASettings.get_idb_plugin_names()        --> ["plugin-7", "plugin-8"]

This module is a single file that you can include in IDAPython
plugin module or scripts.

It depends on ida-netnode, which you can download here:
https://github.com/williballenthin/ida-netnode

This project is licensed under the Apache 2.0 license.

Author: Willi Ballenthin <william.ballenthin@fireeye.com>
"""
import os
import re
import sys
import abc
import json
import logging
import datetime
import unittest
import contextlib

try:
    import idc
    import idaapi
    import netnode
except ImportError:
    pass

# we'll use a function here to avoid polluting our global variable namespace.
def import_qtcore():
    """
    This nasty piece of code is here to force the loading of IDA's
     Qt bindings.
    Without it, Python attempts to load PySide from the site-packages
     directory, and failing, as it does not play nicely with IDA.

    via: github.com/tmr232/Cute
    """
    has_ida = False
    try:
        # if we're running under IDA,
        # then we'll use IDA's Qt bindings
        import idaapi
        has_ida = True
    except ImportError:
        # not running under IDA,
        # so use default Qt installation
        has_ida = False

    if has_ida:
        old_path = sys.path[:]
        try:
            ida_python_path = os.path.dirname(idaapi.__file__)
            sys.path.insert(0, ida_python_path)
            if idaapi.IDA_SDK_VERSION >= 690:
                from PyQt5 import QtCore
                return QtCore
            else:
                from PySide import QtCore
                return QtCore
        finally:
            sys.path = old_path
    else:
        try:
            from PyQt5 import QtCore
            return QtCore
        except ImportError:
            pass

        try:
            from PySide import QtCore
            return QtCore
        except ImportError:
            pass

        raise ImportError("No module named PySide or PyQt")

QtCore = import_qtcore()

CONFIG_FILE_NANE = ".ida-settings.ini"
IDA_SETTINGS_ORGANIZATION = "IDAPython"
IDA_SETTINGS_APPLICATION = "IDA-Settings"

g_logger = logging.getLogger("ida-settings")

# enforce methods required by settings providers
class IDASettingsInterface:
    __metaclass__ = abc.ABCMeta

    @abc.abstractmethod
    def get_value(self, key):
        """
        Fetch the settings value with the given key, or raise KeyError.

        type key: basestring
        rtype value: Union[basestring, int, float, List, Dict]
        """
        raise NotImplemented

    @abc.abstractmethod
    def set_value(self, key, value):
        """
        Set the settings value with the given key.

        type key: basestring
        type value: Union[basestring, int, float, List, Dict]
        """
        raise NotImplemented

    @abc.abstractmethod
    def del_value(self, key):
        """
        Remove the settings value with the given key.
        Does not raise an error if the key does not already exist.

        type key: basestring
        """
        raise NotImplemented

    @abc.abstractmethod
    def get_keys(self):
        """
        Fetch an iterable of the settings keys, which are strings.

        rtype: Iterable[basestring]
        """
        raise NotImplemented

    @abc.abstractmethod
    def clear(self):
        """
        Delete all settings for this settings instance.

        rtype: None
        """
        raise NotImplemented

def validate(s):
    # the slash character is used by QSettings to denote a subgroup
    # we want to have a single nested structure of settings
    if "/" in s:
        return False
    if "\\" in s:
        # QSettings automatically translates '\' to '/'
        return False
    return True

# provide base constructor args required by settings providers
class IDASettingsBase(IDASettingsInterface):
    def __init__(self, plugin_name):
        super(IDASettingsBase, self).__init__()
        if not validate(plugin_name):
            raise RuntimeError("invalid plugin name")
        self._plugin_name = plugin_name

# allow IDASettings to look like dicts
class DictMixin:
    def __getitem__(self, key):
        if not isinstance(key, basestring):
            raise TypeError("key must be a string")
        return self.get_value(key)

    def __setitem__(self, key, value):
        if not isinstance(key, basestring):
            raise TypeError("key must be a string")
        return self.set_value(key, value)

    def __delitem__(self, key):
        if not isinstance(key, basestring):
            raise TypeError("key must be a string")
        return self.del_value(key)

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def __contains__(self, key):
        try:
            if self[key] is not None:
                return True
            return False
        except KeyError:
            return False

    def iterkeys(self):
        return self.get_keys()

    def keys(self):
        return [k for k in self.iterkeys()]

    def itervalues(self):
        for k in self.iterkeys():
            yield self[k]

    def values(self):
        return [v for v in self.itervalues()]

    def iteritems(self):
        for k in self.iterkeys():
            yield k, self[k]

    def items(self):
        return [(k, v) for k, v in self.iteritems()]

MARKER_KEY = "__meta/permission_check"

def has_qsettings_write_permission(settings):
    value = datetime.datetime.now().isoformat("T")
    settings.setValue(MARKER_KEY, value)
    settings.sync()
    # there's a race here, if another thread/process also
    # performs the same check at the same time
    if settings.status() != QtCore.QSettings.NoError:
        return False
    if settings.value(MARKER_KEY) != value:
        return False
    settings.remove(MARKER_KEY)
    settings.sync()
    return True

class PermissionError(IOError):
    def __init__(self):
        super(PermissionError, self).__init__("Unable to write to QSettings")

class QSettingsIDASettings(IDASettingsInterface):
    """
    An IDASettings implementation that uses an existing QSettings
     instance to persist the keys and values.
    """

    def __init__(self, qsettings):
        super(QSettingsIDASettings, self).__init__()
        self._settings = qsettings
        self._has_perms = None

    def _check_perms(self):
        if self._has_perms is None:
            self._has_perms = has_qsettings_write_permission(self._settings)
        if not self._has_perms:
            raise PermissionError()

    def get_value(self, key):
        v = self._settings.value(key)
        if v is None:
            raise KeyError("key not found")
        return json.loads(v)

    def set_value(self, key, value):
        self._check_perms()
        self._settings.setValue(key, json.dumps(value))

    def del_value(self, key):
        self._check_perms()
        return self._settings.remove(key)

    def get_keys(self):
        for k in self._settings.allKeys():
            yield k

    def clear(self):
        self._check_perms()
        # Qt: the empty string removes all entries in the current group
        self._settings.remove("")

class SystemIDASettings(IDASettingsBase, DictMixin):
    """
    An IDASettings implementation that persists keys and values in the
     system scope using a QSettings instance.
    """

    def __init__(self, plugin_name, *args, **kwargs):
        super(SystemIDASettings, self).__init__(plugin_name, *args, **kwargs)
        s = QtCore.QSettings(QtCore.QSettings.SystemScope,
                             IDA_SETTINGS_ORGANIZATION,
                             IDA_SETTINGS_APPLICATION)
        s.beginGroup(self._plugin_name)
        self._qsettings = QSettingsIDASettings(s)

    def get_value(self, key):
        return self._qsettings.get_value(key)

    def set_value(self, key, value):
        return self._qsettings.set_value(key, value)

    def del_value(self, key):
        return self._qsettings.del_value(key)

    def get_keys(self):
        return self._qsettings.get_keys()

    def clear(self):
        return self._qsettings.clear()

class UserIDASettings(IDASettingsBase, DictMixin):
    """
    An IDASettings implementation that persists keys and values in the
     user scope using a QSettings instance.
    """

    def __init__(self, plugin_name, *args, **kwargs):
        super(UserIDASettings, self).__init__(plugin_name, *args, **kwargs)
        s = QtCore.QSettings(QtCore.QSettings.UserScope,
                             IDA_SETTINGS_ORGANIZATION,
                             IDA_SETTINGS_APPLICATION)
        s.beginGroup(self._plugin_name)
        self._qsettings = QSettingsIDASettings(s)

    def get_value(self, key):
        return self._qsettings.get_value(key)

    def set_value(self, key, value):
        return self._qsettings.set_value(key, value)

    def del_value(self, key):
        return self._qsettings.del_value(key)

    def get_keys(self):
        return self._qsettings.get_keys()

    def clear(self):
        return self._qsettings.clear()

def get_directory_config_path(directory=None):
    if directory is None:
        directory = os.path.dirname(idc.GetIdbPath())
    config_path = os.path.join(directory, CONFIG_FILE_NANE)
    return config_path

class DirectoryIDASettings(IDASettingsBase, DictMixin):
    """
    An IDASettings implementation that persists keys and values in the
     directory scope using a QSettings instance.
    """

    def __init__(self, plugin_name, *args, **kwargs):
        config_directory = kwargs.pop("directory")
        super(DirectoryIDASettings, self).__init__(plugin_name, *args, **kwargs)
        config_path = get_directory_config_path(config_directory)
        s = QtCore.QSettings(config_path, QtCore.QSettings.IniFormat)
        s.beginGroup(self._plugin_name)
        self._qsettings = QSettingsIDASettings(s)

    def get_value(self, key):
        return self._qsettings.get_value(key)

    def set_value(self, key, value):
        return self._qsettings.set_value(key, value)

    def del_value(self, key):
        return self._qsettings.del_value(key)

    def get_keys(self):
        return self._qsettings.get_keys()

    def clear(self):
        return self._qsettings.clear()

def get_meta_netnode():
    """
    Get the netnode used to store settings metadata in the current IDB.
    Note that this implicitly uses the open IDB via the idc iterface.
    """
    node_name = "$ {org:s}.{application:s}".format(
        org=IDA_SETTINGS_ORGANIZATION,
        application=IDA_SETTINGS_APPLICATION)
    return netnode.Netnode(node_name)

PLUGIN_NAMES_KEY = "plugin_names"

def get_netnode_plugin_names():
    """
    Get a iterable of the plugin names registered in the current IDB.
    Note that this implicitly uses the open IDB via the idc iterface.
    """
    try:
        return json.loads(get_meta_netnode()[PLUGIN_NAMES_KEY])
    except KeyError:
        # TODO: there may be other exception types to catch here
        return []

def add_netnode_plugin_name(plugin_name):
    """
    Add the given plugin name to the list of plugin names registered in
      the current IDB.
    Note that this implicitly uses the open IDB via the idc iterface.
    """
    current_names = set(get_netnode_plugin_names())
    if plugin_name in current_names:
        return

    current_names.add(plugin_name)

    get_meta_netnode()[PLUGIN_NAMES_KEY] = json.dumps(list(current_names))

def del_netnode_plugin_name(plugin_name):
    """
    Remove the given plugin name to the list of plugin names registered in
      the current IDB.
    Note that this implicitly uses the open IDB via the idc iterface.
    """
    current_names = set(get_netnode_plugin_names())
    if plugin_name not in current_names:
        return

    try:
        current_names.remove(plugin_name)
    except KeyError:
        return

    get_meta_netnode()[PLUGIN_NAMES_KEY] = json.dumps(list(current_names))

class IDBIDASettings(IDASettingsBase, DictMixin):
    """
    An IDASettings implementation that persists keys and values in the
     current IDB database.
    """

    @property
    def _netnode(self):
        node_name = "$ {org:s}.{application:s}.{plugin_name:s}".format(
            org=IDA_SETTINGS_ORGANIZATION,
            application=IDA_SETTINGS_APPLICATION,
            plugin_name=self._plugin_name)
        return netnode.Netnode(node_name)

    def get_value(self, key):
        if not isinstance(key, basestring):
            raise TypeError("key must be a string")

        try:
            v = self._netnode[key]
        except TypeError:
            raise KeyError("key not found")
        if v is None:
            raise KeyError("key not found")

        return json.loads(v)

    def set_value(self, key, value):
        if not isinstance(key, basestring):
            raise TypeError("key must be a string")

        self._netnode[key] = json.dumps(value)
        add_netnode_plugin_name(self._plugin_name)

    def del_value(self, key):
        if not isinstance(key, basestring):
            raise TypeError("key must be a string")

        try:
            del self._netnode[key]
        except KeyError:
            pass

    def get_keys(self):
        return self._netnode.iterkeys()

    def clear(self):
        for k in self.get_keys():
            self.del_value(k)
        self._netnode.kill()
        del_netnode_plugin_name(self._plugin_name)

def ensure_ida_loaded():
    try:
        import idc
        import idaapi
    except ImportError:
        raise EnvironmentError("Must be running in IDA to access IDB or directory settings")

class IDASettings(object):
    def __init__(self, plugin_name, directory=None):
        super(IDASettings, self).__init__()
        if not validate(plugin_name):
            raise RuntimeError("invalid plugin name")
        self._plugin_name = plugin_name
        self._config_directory = directory

    @property
    def idb(self):
        """
        Fetch the IDASettings instance for the curren plugin with IDB scope.

        rtype: IDASettingsInterface
        """
        ensure_ida_loaded()
        return IDBIDASettings(self._plugin_name)

    @property
    def directory(self):
        """
        Fetch the IDASettings instance for the curren plugin with directory scope.

        rtype: IDASettingsInterface
        """
        if self._config_directory is None:
            ensure_ida_loaded()
        return DirectoryIDASettings(self._plugin_name, directory=self._config_directory)

    @property
    def user(self):
        """
        Fetch the IDASettings instance for the curren plugin with user scope.

        rtype: IDASettingsInterface
        """
        return UserIDASettings(self._plugin_name)

    @property
    def system(self):
        """
        Fetch the IDASettings instance for the curren plugin with system scope.

        rtype: IDASettingsInterface
        """
        return SystemIDASettings(self._plugin_name)

    def get_value(self, key):
        """
        Fetch the settings value with the highest precedence for the given
         key, or raise KeyError.
        Precedence:
          - IDB scope
          - directory scope
          - user scope
          - system scope

        type key: basestring
        rtype value: Union[basestring, int, float, List, Dict]
        """
        try:
            return self.idb.get_value(key)
        except (KeyError, EnvironmentError):
            pass
        try:
            return self.directory.get_value(key)
        except (KeyError, EnvironmentError):
            pass
        try:
            return self.user.get_value(key)
        except KeyError:
            pass
        try:
            return self.system.get_value(key)
        except KeyError:
            pass

        raise KeyError("key not found")

    def iterkeys(self):
        """
        Enumerate the keys found at any scope for the current plugin.

        rtype: Generator[str]
        """
        visited_keys = set()
        try:
            for key in self.idb.iterkeys():
                if key not in visited_keys:
                    yield key
                    visited_keys.add(key)
        except (PermissionError, EnvironmentError):
            pass

        try:
            for key in self.directory.iterkeys():
                if key not in visited_keys:
                    yield key
                    visited_keys.add(key)
        except (PermissionError, EnvironmentError):
            pass

        try:
            for key in self.user.iterkeys():
                if key not in visited_keys:
                    yield key
                    visited_keys.add(key)
        except (PermissionError, EnvironmentError):
            pass

        try:
            for key in self.system.iterkeys():
                if key not in visited_keys:
                    yield key
                    visited_keys.add(key)
        except (PermissionError, EnvironmentError):
            pass

    def keys(self):
        """
        Enumerate the keys found at any scope for the current plugin.

        rtype: Generator[str]
        """

        return list(self.iterkeys())

    def itervalues(self):
        """
        Enumerate the values found at any scope for the current plugin.

        rtype: Generator[jsonable]
        """

        for key in self.iterkeys():
            yield self[key]

    def values(self):
        """
        Enumerate the values found at any scope for the current plugin.

        rtype: Sequence[jsonable]
        """

        return list(self.itervalues())

    def iteritems(self):
        """
        Enumerate the (key, value) pairs found at any scope for the current plugin.

        rtype: Sequence[Tuple[str, jsonable]]
        """
        for key in self.iterkeys():
            yield (key, self[key])

    def items(self):
        """
        Enumerate the (key, value) pairs found at any scope for the current plugin.

        rtype: Sequence[Tuple[str, jsonable]]
        """
        return list(self.iteritems())

    def __getitem__(self, key):
        return self.get_value(key)

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def __contains__(self, key):
        try:
            if self[key] is not None:
                return True
            return False
        except KeyError:
            return False

    @staticmethod
    def get_system_plugin_names():
        """
        Get the names of all plugins at the system scope.
        As this is a static method, you can call the directly on IDASettings:

            import ida_settings
            print( ida_settings.IDASettings.get_system_plugin_names() )

        rtype: Sequence[str]
        """
        return QtCore.QSettings(QtCore.QSettings.SystemScope,
                                IDA_SETTINGS_ORGANIZATION,
                                IDA_SETTINGS_APPLICATION).childGroups()[:]

    @staticmethod
    def get_user_plugin_names():
        """
        Get the names of all plugins at the user scope.
        As this is a static method, you can call the directly on IDASettings:

            import ida_settings
            print( ida_settings.IDASettings.get_user_plugin_names() )

        rtype: Sequence[str]
        """
        return QtCore.QSettings(QtCore.QSettings.UserScope,
                                IDA_SETTINGS_ORGANIZATION,
                                IDA_SETTINGS_APPLICATION).childGroups()[:]

    @staticmethod
    def get_directory_plugin_names(config_directory=None):
        """
        Get the names of all plugins at the directory scope.
        Provide a config directory path to use this method outside of IDA.
        As this is a static method, you can call the directly on IDASettings:

            import ida_settings
            print( ida_settings.IDASettings.get_directory_plugin_names("/tmp/ida/1/") )

        type config_directory: str
        rtype: Sequence[str]
        """
        ensure_ida_loaded()
        return QtCore.QSettings(get_directory_config_path(directory=config_directory),
                                QtCore.QSettings.IniFormat).childGroups()[:]

    @staticmethod
    def get_idb_plugin_names():
        """
        Get the names of all plugins at the IDB scope.
        Cannot be used outside of IDA.
        As this is a static method, you can call the directly on IDASettings:

            import ida_settings
            print( ida_settings.IDASettings.get_idb_plugin_names() )

        rtype: Sequence[str]
        """
        ensure_ida_loaded()
        return get_netnode_plugin_names()

def import_settings(settings, config_path):
    """
    Import settings from the given file system path to given settings instance.

    type settings: IDASettingsInterface
    type config_path: str
    """
    other = QtCore.QSettings(config_path, QtCore.QSettings.IniFormat)
    for k in other.allKeys():
        settings[k] = other.value(k)

def export_settings(settings, config_path):
    """
    Export the given settings instance to the given file system path.

    type settings: IDASettingsInterface
    type config_path: str
    """
    other = QtCore.QSettings(config_path, QtCore.QSettings.IniFormat)
    for k, v in settings.iteritems():
        other.setValue(k, v)

#######################################################################################
#
# Test Cases
#  run this file as an IDAPython script to invoke the tests.
#
#######################################################################################

PLUGIN_1 = "plugin1"
PLUGIN_2 = "plugin2"
KEY_1 = "key_1"
KEY_2 = "key_2"
VALUE_1 = bytes("hello")
VALUE_2 = bytes("goodbye")
VALUE_INT = 69
VALUE_FLOAT = 69.69
VALUE_LIST = ["a", "b", "c"]
VALUE_DICT = {"a": 1, "b": "2", "c": 3.0}

class TestSync(unittest.TestCase):
    """
    Demonstrate that creating new instances of the settings objects shows the same data.
    """

    def test_system(self):
        try:
            # this may fail if the user is not running as admin
            IDASettings(PLUGIN_1).system.set_value(KEY_1, VALUE_1)
            self.assertEqual(IDASettings(PLUGIN_1).system.get_value(KEY_1), VALUE_1)
        except PermissionError:
            g_logger.warn("swallowing PermissionError during testing")

    def test_user(self):
        try:
            IDASettings(PLUGIN_1).user.set_value(KEY_1, VALUE_1)
            self.assertEqual(IDASettings(PLUGIN_1).user.get_value(KEY_1), VALUE_1)
        except PermissionError:
            g_logger.warn("swallowing PermissionError during testing")

    def test_directory(self):
        try:
            IDASettings(PLUGIN_1).directory.set_value(KEY_1, VALUE_1)
            self.assertEqual(IDASettings(PLUGIN_1).directory.get_value(KEY_1), VALUE_1)
        except PermissionError:
            g_logger.warn("swallowing PermissionError during testing")

    def test_idb(self):
        try:
            IDASettings(PLUGIN_1).idb.set_value(KEY_1, VALUE_1)
            self.assertEqual(IDASettings(PLUGIN_1).idb.get_value(KEY_1), VALUE_1)
        except PermissionError:
            g_logger.warn("swallowing PermissionError during testing")

@contextlib.contextmanager
def clearing(settings):
    settings.clear()
    try:
        yield
    finally:
        settings.clear()

class TestSettingsMixin(object):
    """
    A mixin that adds standard tests test cases with:
      - self.settings, an IDASettingsInterface implementor
    """

    def test_set(self):
        try:
            with clearing(self.settings):
                # simple set
                self.settings.set_value(KEY_1, VALUE_1)
                self.assertEqual(self.settings.get_value(KEY_1), VALUE_1)
                # overwrite
                self.settings.set_value(KEY_1, VALUE_2)
                self.assertEqual(self.settings.get_value(KEY_1), VALUE_2)
        except PermissionError:
            g_logger.warn("swallowing PermissionError during testing")

    def test_del(self):
        try:
            with clearing(self.settings):
                self.settings.set_value(KEY_1, VALUE_1)
                self.settings.del_value(KEY_1)
                with self.assertRaises(KeyError):
                    self.settings.get_value(KEY_1)
        except PermissionError:
            g_logger.warn("swallowing PermissionError during testing")

    def test_keys(self):
        try:
            with clearing(self.settings):
                self.settings.del_value(KEY_1)
                self.settings.del_value(KEY_2)
                self.settings.set_value(KEY_1, VALUE_1)
                self.assertEqual(list(self.settings.get_keys()), [KEY_1])
                self.settings.set_value(KEY_2, VALUE_2)
                self.assertEqual(list(self.settings.get_keys()), [KEY_1, KEY_2])
        except PermissionError:
            g_logger.warn("swallowing PermissionError during testing")

    def test_dict(self):
        try:
            with clearing(self.settings):
                self.assertFalse(KEY_1 in self.settings)
                self.settings[KEY_1] = VALUE_1
                self.assertEquals(self.settings[KEY_1], VALUE_1)
                self.settings[KEY_2] = VALUE_2
                self.assertEquals(self.settings[KEY_2], VALUE_2)
                self.assertEquals(self.settings.keys(), [KEY_1, KEY_2])
                self.assertEquals(self.settings.values(), [VALUE_1, VALUE_2])
                del self.settings[KEY_1]
                self.assertEquals(self.settings.keys(), [KEY_2])
        except PermissionError:
            g_logger.warn("swallowing PermissionError during testing")

    def test_types(self):
        try:
            with clearing(self.settings):
                for v in [VALUE_INT, VALUE_FLOAT, VALUE_LIST, VALUE_DICT]:
                    self.settings.set_value(KEY_1, v)
                    self.assertEquals(self.settings[KEY_1], v)
        except PermissionError:
            g_logger.warn("swallowing PermissionError during testing")

    def test_large_values(self):
        large_value_1 = "".join(VALUE_1 * 1000)
        large_value_2 = "".join(VALUE_2 * 1000)
        try:
            with clearing(self.settings):
                # simple set
                self.settings.set_value(KEY_1, large_value_1)
                self.assertEqual(self.settings.get_value(KEY_1), large_value_1)
                # overwrite
                self.settings.set_value(KEY_1, large_value_2)
                self.assertEqual(self.settings.get_value(KEY_1), large_value_2)
        except PermissionError:
            g_logger.warn("swallowing PermissionError during testing")

class TestSystemSettings(unittest.TestCase, TestSettingsMixin):
    def setUp(self):
        self.settings = IDASettings(PLUGIN_1).system

class TestUserSettings(unittest.TestCase, TestSettingsMixin):
    def setUp(self):
        self.settings = IDASettings(PLUGIN_1).user

class TestDirectorySettings(unittest.TestCase, TestSettingsMixin):
    def setUp(self):
        self.settings = IDASettings(PLUGIN_1).directory

class TestIdbSettings(unittest.TestCase, TestSettingsMixin):
    def setUp(self):
        self.settings = IDASettings(PLUGIN_1).idb

class TestUserAndSystemSettings(unittest.TestCase):
    def setUp(self):
        self.system = IDASettings(PLUGIN_1).system
        self.user = IDASettings(PLUGIN_1).user

    def test_system_fallback(self):
        """
        QSettings instances with scope "user" automatically fall back to
         scope "system" if the key doesn't exist.
        """
        try:
            with clearing(self.system):
                with clearing(self.user):
                    self.system.set_value(KEY_1, VALUE_1)
                    self.assertEqual(self.user.get_value(KEY_1), VALUE_1)
        except PermissionError:
            g_logger.warn("swallowing PermissionError during testing")

class TestSettingsPrecendence(unittest.TestCase):
    def setUp(self):
        self.system = IDASettings(PLUGIN_1).system
        self.user = IDASettings(PLUGIN_1).user
        self.directory = IDASettings(PLUGIN_1).directory
        self.idb = IDASettings(PLUGIN_1).idb
        self.mux = IDASettings(PLUGIN_1)

    def test_user_gt_system(self):
        try:
            with clearing(self.system):
                with clearing(self.user):
                    self.system.set_value(KEY_1, VALUE_1)
                    self.user.set_value(KEY_1, VALUE_2)
                    self.assertEqual(self.mux.get_value(KEY_1), VALUE_2)
        except PermissionError:
            g_logger.warn("swallowing PermissionError during testing")

    def test_directory_gt_user(self):
        try:
            with clearing(self.user):
                with clearing(self.directory):
                    self.user.set_value(KEY_1, VALUE_1)
                    self.directory.set_value(KEY_1, VALUE_2)
                    self.assertEqual(self.mux.get_value(KEY_1), VALUE_2)
        except PermissionError:
            g_logger.warn("swallowing PermissionError during testing")

    def test_idb_gt_directory(self):
        try:
            with clearing(self.directory):
                with clearing(self.idb):
                    self.directory.set_value(KEY_1, VALUE_1)
                    self.idb.set_value(KEY_1, VALUE_2)
                    self.assertEqual(self.mux.get_value(KEY_1), VALUE_2)
        except PermissionError:
            g_logger.warn("swallowing PermissionError during testing")

class TestPluginNamesAccessors(unittest.TestCase):
    def test_system_plugin_names(self):
        try:
            self.assertEqual(set(IDASettings.get_system_plugin_names()),
                             set([]))

            s1 = IDASettings(PLUGIN_1).system
            with clearing(s1):
                s1[KEY_1] = VALUE_1
                self.assertEqual(set(IDASettings.get_system_plugin_names()),
                                 set([PLUGIN_1]))

                s2 = IDASettings(PLUGIN_2).system
                with clearing(s2):
                    s2[KEY_1] = VALUE_1
                    self.assertEqual(set(IDASettings.get_system_plugin_names()),
                                     set([PLUGIN_1, PLUGIN_2]))

            self.assertEqual(set(IDASettings.get_system_plugin_names()), set([]))
        except PermissionError:
            g_logger.warn("swallowing PermissionError during testing")

    def test_user_plugin_names(self):
        try:
            self.assertEqual(set(IDASettings.get_user_plugin_names()),
                             set([]))

            s1 = IDASettings(PLUGIN_1).user
            with clearing(s1):
                s1[KEY_1] = VALUE_1
                self.assertEqual(set(IDASettings.get_user_plugin_names()),
                                 set([PLUGIN_1]))

                s2 = IDASettings(PLUGIN_2).user
                with clearing(s2):
                    s2[KEY_1] = VALUE_1
                    self.assertEqual(set(IDASettings.get_user_plugin_names()),
                                     set([PLUGIN_1, PLUGIN_2]))

            self.assertEqual(set(IDASettings.get_user_plugin_names()), set([]))
        except PermissionError:
            g_logger.warn("swallowing PermissionError during testing")

    def test_directory_plugin_names(self):
        try:
            self.assertEqual(set(IDASettings.get_directory_plugin_names()),
                             set([]))

            s1 = IDASettings(PLUGIN_1).directory
            with clearing(s1):
                s1[KEY_1] = VALUE_1
                self.assertEqual(set(IDASettings.get_directory_plugin_names()),
                                 set([PLUGIN_1]))

                s2 = IDASettings(PLUGIN_2).directory
                with clearing(s2):
                    s2[KEY_1] = VALUE_1
                    self.assertEqual(set(IDASettings.get_directory_plugin_names()),
                                     set([PLUGIN_1, PLUGIN_2]))

            self.assertEqual(set(IDASettings.get_directory_plugin_names()),
                             set([]))
        except PermissionError:
            g_logger.warn("swallowing PermissionError during testing")

    def test_idb_plugin_names(self):
        try:
            self.assertEqual(set(IDASettings.get_idb_plugin_names()),
                             set([]))

            s1 = IDASettings(PLUGIN_1).idb
            with clearing(s1):
                s1[KEY_1] = VALUE_1
                self.assertEqual(set(IDASettings.get_idb_plugin_names()),
                                 set([PLUGIN_1]))

                s2 = IDASettings(PLUGIN_2).idb
                with clearing(s2):
                    s2[KEY_1] = VALUE_1
                    self.assertEqual(set(IDASettings.get_idb_plugin_names()),
                                     set([PLUGIN_1, PLUGIN_2]))

            self.assertEqual(set(IDASettings.get_idb_plugin_names()),
                             set([]))
        except PermissionError:
            g_logger.warn("swallowing PermissionError during testing")

def main():
    try:
        unittest.main()
    except SystemExit:
        pass

if __name__ == "__main__":
    main()


from __future__ import unicode_literals

from django.core.exceptions import PermissionDenied
from django.http import Http404
from rest_framework import status, exceptions
from rest_framework.response import Response

def simple_error_handler(exc, *args):
    """
    Returns the response that should be used for any given exception.

    By default we handle the REST framework `APIException`, and also
    Django's builtin `Http404` and `PermissionDenied` exceptions.

    Any unhandled exceptions may return `None`, which will cause a 500 error
    to be raised.
    """
    if isinstance(exc, exceptions.APIException):
        headers = {}
        if getattr(exc, 'auth_header', None):
            headers['WWW-Authenticate'] = exc.auth_header
        if getattr(exc, 'wait', None):
            headers['X-Throttle-Wait-Seconds'] = '%d' % exc.wait

        return Response({'error': exc.detail},
                        status=exc.status_code,
                        headers=headers)

    elif isinstance(exc, Http404):
        return Response({'error': 'Not found'},
                        status=status.HTTP_404_NOT_FOUND)

    elif isinstance(exc, PermissionDenied):
        return Response({'error': 'Permission denied'},
                        status=status.HTTP_403_FORBIDDEN)

    # Note: Unhandled exceptions will raise a 500 error.
    return None


# -*- coding: utf-8 -*-

import boto
from numbers import Number
from boto.dynamodb.exceptions import DynamoDBKeyNotFoundError

class Table(object):
    def __init__(self, table=None, eager=False):
        self.table = table
        self.is_eager = eager

    def __repr__(self):
        return '<table \'{0}\'>'.format(self.name)

    @property
    def name(self):
        return self.table.__dict__['_dict']['TableName']

    def item(self, item):
        return Item(item, self)

    def delete(self):
        return self.table.delete()

    def scale(self, read=None, write=None):
        read = read or self.table.read_units
        write = write or self.table.read_units
        return self.table.update_throughput(read_units=read, write_units=write)

    def __getitem__(self, key):
        try:
            if isinstance(key, (basestring, Number)):
                key = [key]
            i = self.table.get_item(*key)
            i = self.item(i)
        except DynamoDBKeyNotFoundError:
            return self.__magic_get(key)
        return i

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def __setitem__(self, key, values):
        if isinstance(key, (basestring, Number)):
            key = [key]
        i = self.table.new_item(*key, attrs=values)
        i = self.item(i)
        i.put()
        return i

    def __delitem__(self, key):
        return self[key].delete()

    def __magic_get(self, key):
        if self.is_eager:
            self[key] = {}
            return self.item(self[key])

    def __contains__(self, key):
        return not self.get(key) is None

    def new(self, name):
        table = self.table.layer2.create_table(
            name=name,
            schema=self.table._schema,
            read_units=self.table.read_units,
            write_units=self.table.write_units
        )
        return Table(table=table, eager=self.is_eager)

class Item(object):
    def __init__(self, item, table):
        self.item = item
        self.table = table

    @property
    def is_eager(self):
        return self.table.is_eager

    def __getattr__(self, key):
        if not key in ['item']:
            try:
                return getattr(object, key)
            except AttributeError:
                return getattr(self.item, key)

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def __repr__(self):
        return repr(self.item)

    def __getitem__(self, key):
        return self.item[key]

    def __setitem__(self, key, value):
        self.item[key] = value
        if self.is_eager:
            self.item.save()

    def __contains__(self, key):
        return key in self.item

def table(name, auth=None, eager=True):
    """Returns a given table for the given user."""
    auth = auth or []
    dynamodb = boto.connect_dynamodb(*auth)

    table = dynamodb.get_table(name)
    return Table(table=table, eager=eager)

def tables(auth=None, eager=True):
    """Returns a list of tables for the given user."""
    auth = auth or []
    dynamodb = boto.connect_dynamodb(*auth)

    return [table(t, auth, eager=eager) for t in dynamodb.list_tables()]


# -*- coding: utf-8 -*-
#
# Copyright (C) 2015-2017 Bitergia
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA.
#
# Authors:
#     Valerio Cosentino <valcos@bitergia.com>
#

import json
import logging

import requests

from grimoirelab_toolkit.datetime import (datetime_utcnow,
                                          datetime_to_utc,
                                          str_to_datetime)
from grimoirelab_toolkit.uris import urijoin

from ...backend import (Backend,
                        BackendCommand,
                        BackendCommandArgumentParser)
from ...client import HttpClient
from ...utils import DEFAULT_DATETIME

CRATES_URL = "https://crates.io/"
CRATES_API_URL = 'https://crates.io/api/v1/'

CATEGORY_CRATES = 'crates'
CATEGORY_SUMMARY = 'summary'

SLEEP_TIME = 60

logger = logging.getLogger(__name__)

class Crates(Backend):
    """Crates.io backend for Perceval.

    This class allows the fetch the packages stored in Crates.io

    :param sleep_time: sleep time in case of connection lost
    :param tag: label used to mark the data
    :param archive: archive to store/retrieve items
    """
    version = '0.3.2'

    CATEGORIES = [CATEGORY_CRATES, CATEGORY_SUMMARY]

    def __init__(self, sleep_time=SLEEP_TIME, tag=None, archive=None):
        origin = CRATES_URL

        super().__init__(origin, tag=tag, archive=archive)
        self.sleep_time = sleep_time

        self.client = None

    def fetch(self, category=CATEGORY_CRATES, from_date=DEFAULT_DATETIME):
        """Fetch package data.

        The method retrieves packages and summary from Crates.io.

        :param category: the category of items to fetch
        :param from_date: obtain packages updated since this date

        :returns: a summary and crate items
        """
        if not from_date:
            from_date = DEFAULT_DATETIME

        from_date = datetime_to_utc(from_date)

        kwargs = {"from_date": from_date}
        items = super().fetch(category, **kwargs)

        return items

    def fetch_items(self, category, **kwargs):
        """Fetch packages and summary from Crates.io

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        from_date = kwargs['from_date']

        if category == CATEGORY_CRATES:
            return self.__fetch_crates(from_date)
        else:
            return self.__fetch_summary()

    @classmethod
    def has_archiving(cls):
        """Returns whether it supports archiving items on the fetch process.

        :returns: this backend supports items archive
        """
        return True

    @classmethod
    def has_resuming(cls):
        """Returns whether it supports to resume the fetch process.

        :returns: this backend supports items resuming
        """
        return False

    @staticmethod
    def metadata_id(item):
        """Extracts the identifier from an item depending on its type."""

        if Crates.metadata_category(item) == CATEGORY_CRATES:
            return str(item['id'])
        else:
            ts = item['fetched_on']
            ts = str_to_datetime(ts)
            return str(ts.timestamp())

    @staticmethod
    def metadata_updated_on(item):
        """Extracts the update time from an item.

        Depending on the item, the timestamp is extracted from the
        'updated_at' or 'fetched_on' fields.
        This date is converted to UNIX timestamp format.

        :param item: item generated by the backend

        :returns: a UNIX timestamp
        """
        if Crates.metadata_category(item) == CATEGORY_CRATES:
            ts = item['updated_at']
        else:
            ts = item['fetched_on']

        ts = str_to_datetime(ts)

        return ts.timestamp()

    @staticmethod
    def metadata_category(item):
        """Extracts the category from an item.

        This backend generates two types of item: 'summary' and 'crate'.
        """
        if 'num_downloads' in item:
            return CATEGORY_SUMMARY
        else:
            return CATEGORY_CRATES

    def _init_client(self, from_archive=False):
        """Init client"""

        return CratesClient(self.sleep_time, self.archive, from_archive)

    def __fetch_summary(self):
        """Fetch summary"""

        raw_summary = self.client.summary()
        summary = json.loads(raw_summary)
        summary['fetched_on'] = str(datetime_utcnow())

        yield summary

    def __fetch_crates(self, from_date):
        """Fetch crates"""

        from_date = datetime_to_utc(from_date)

        crates_groups = self.client.crates()

        for raw_crates in crates_groups:
            crates = json.loads(raw_crates)

            for crate_container in crates['crates']:

                if str_to_datetime(crate_container['updated_at']) < from_date:
                    continue

                crate_id = crate_container['id']

                crate = self.__fetch_crate_data(crate_id)
                crate['owner_team_data'] = self.__fetch_crate_owner_team(crate_id)
                crate['owner_user_data'] = self.__fetch_crate_owner_user(crate_id)
                crate['version_downloads_data'] = self.__fetch_crate_version_downloads(crate_id)
                crate['versions_data'] = self.__fetch_crate_versions(crate_id)

                yield crate

    def __fetch_crate_owner_team(self, crate_id):
        """Get crate team owner"""

        raw_owner_team = self.client.crate_attribute(crate_id, 'owner_team')

        owner_team = json.loads(raw_owner_team)

        return owner_team

    def __fetch_crate_owner_user(self, crate_id):
        """Get crate user owners"""

        raw_owner_user = self.client.crate_attribute(crate_id, 'owner_user')

        owner_user = json.loads(raw_owner_user)

        return owner_user

    def __fetch_crate_versions(self, crate_id):
        """Get crate versions data"""

        raw_versions = self.client.crate_attribute(crate_id, "versions")

        version_downloads = json.loads(raw_versions)

        return version_downloads

    def __fetch_crate_version_downloads(self, crate_id):
        """Get crate version downloads"""

        raw_version_downloads = self.client.crate_attribute(crate_id, "downloads")

        version_downloads = json.loads(raw_version_downloads)

        return version_downloads

    def __fetch_crate_data(self, crate_id):
        """Get crate data"""

        raw_crate = self.client.crate(crate_id)

        crate = json.loads(raw_crate)
        return crate['crate']

class CratesClient(HttpClient):
    """Crates API client.

    Client for fetching information from the Crates API.

    :param sleep_time: time to sleep in case
        of connection problems
    :param archive: an archive to store/read fetched data
    :param from_archive: it tells whether to write/read the archive
    """

    MAX_RETRIES = 5

    def __init__(self, sleep_time=SLEEP_TIME, archive=None, from_archive=False):
        super().__init__(CRATES_API_URL, sleep_time=sleep_time, max_retries=CratesClient.MAX_RETRIES,
                         extra_headers={'Content-type': 'application/json'},
                         archive=archive, from_archive=from_archive)

    def summary(self):
        """Get Crates.io summary"""

        path = urijoin(CRATES_API_URL, CATEGORY_SUMMARY)
        raw_content = self.fetch(path)

        return raw_content

    def crates(self, from_page=1):
        """Get crates in alphabetical order"""

        path = urijoin(CRATES_API_URL, CATEGORY_CRATES)
        raw_crates = self.__fetch_items(path, from_page)

        return raw_crates

    def crate(self, crate_id):
        """Get a crate by its ID"""

        path = urijoin(CRATES_API_URL, CATEGORY_CRATES, crate_id)
        raw_crate = self.fetch(path)

        return raw_crate

    def crate_attribute(self, crate_id, attribute):
        """Get crate attribute"""

        path = urijoin(CRATES_API_URL, CATEGORY_CRATES, crate_id, attribute)
        raw_attribute_data = self.fetch(path)

        return raw_attribute_data

    def __fetch_items(self, path, page=1):
        """Return the items from Crates.io API using pagination"""

        fetch_data = True
        parsed_crates = 0
        total_crates = 0

        while fetch_data:
            logger.debug("Fetching page: %i", page)

            try:
                payload = {'sort': 'alphabetical', 'page': page}
                raw_content = self.fetch(path, payload=payload)
                content = json.loads(raw_content)

                parsed_crates += len(content['crates'])

                if not total_crates:
                    total_crates = content['meta']['total']

            except requests.exceptions.HTTPError as e:
                logger.error("HTTP exception raised - %s", e.response.text)
                raise e

            yield raw_content
            page += 1

            if parsed_crates >= total_crates:
                fetch_data = False

    def fetch(self, url, payload=None):
        """Return the textual content associated to the Response object"""

        response = super().fetch(url, payload=payload)

        return response.text

class CratesCommand(BackendCommand):
    """Class to run Crates.io backend from the command line."""

    BACKEND = Crates

    @classmethod
    def setup_cmd_parser(cls):
        """Returns the Crates argument parser."""

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              from_date=True,
                                              archive=True,
                                              token_auth=True)

        # Optional arguments
        group = parser.parser.add_argument_group('Crates.io arguments')
        group.add_argument('--sleep-time', dest='sleep_time',
                           default=SLEEP_TIME, type=int,
                           help="Sleep time in case of connection lost")

        return parser


# -*- coding: utf-8 -*-
#
# Copyright (C) 2016-2017 Bitergia
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA.
#
# Authors:
#     Alvaro del Castillo <acs@bitergia.com>
#

import json
import logging

import requests

from grimoirelab_toolkit.datetime import str_to_datetime
from grimoirelab_toolkit.uris import urijoin

from ...backend import (Backend,
                        BackendCommand,
                        BackendCommandArgumentParser)
from ...client import HttpClient
from ...errors import ParseError

logger = logging.getLogger(__name__)

KITSUNE_URL = "https://support.mozilla.org"
DEFAULT_OFFSET = 0

CATEGORY_QUESTION = "question"

class Kitsune(Backend):
    """Kitsune backend for Perceval.

    This class retrieves the questions and answers from a
    Kitsune URL. To initialize this class a URL may be provided.
    If not, https://support.mozilla.org will be used. The origin
    of the data will be set to this URL.

    Questions and answers are returned from older to newer.

    :param url: Kitsune URL
    :param tag: label used to mark the data
    :param archive: archive to store/retrieve items
    """
    version = '0.6.3'

    CATEGORIES = [CATEGORY_QUESTION]

    def __init__(self, url=None, tag=None, archive=None):
        if not url:
            url = KITSUNE_URL
        origin = url

        super().__init__(origin, tag=tag, archive=archive)
        self.url = url

        self.client = None

    def fetch(self, category=CATEGORY_QUESTION, offset=DEFAULT_OFFSET):
        """Fetch questions from the Kitsune url.

        :param category: the category of items to fetch
        :offset: obtain questions after offset
        :returns: a generator of questions
        """
        if not offset:
            offset = DEFAULT_OFFSET

        kwargs = {"offset": offset}
        items = super().fetch(category, **kwargs)

        return items

    def fetch_items(self, category, **kwargs):
        """Fetch questions from the Kitsune url

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        offset = kwargs['offset']

        logger.info("Looking for questions at url '%s' using offset %s",
                    self.url, str(offset))

        nquestions = 0  # number of questions processed
        tquestions = 0  # number of questions from API data
        equestions = 0  # number of questions dropped by errors

        # Always get complete pages so the first item is always
        # the first one in the page
        page = int(offset / KitsuneClient.ITEMS_PER_PAGE)
        page_offset = page * KitsuneClient.ITEMS_PER_PAGE
        # drop questions from page before the offset
        drop_questions = offset - page_offset
        current_offset = offset

        questions_page = self.client.get_questions(offset)

        while True:
            try:
                raw_questions = next(questions_page)
            except StopIteration:
                break
            except requests.exceptions.HTTPError as e:
                # Continue with the next page if it is a 500 error
                if e.response.status_code == 500:
                    logger.exception(e)
                    logger.error("Problem getting Kitsune questions. "
                                 "Loosing %i questions. Going to the next page.",
                                 KitsuneClient.ITEMS_PER_PAGE)
                    equestions += KitsuneClient.ITEMS_PER_PAGE
                    current_offset += KitsuneClient.ITEMS_PER_PAGE
                    questions_page = self.client.get_questions(current_offset)
                    continue
                else:
                    # If it is another error just propagate the exception
                    raise e

            try:
                questions_data = json.loads(raw_questions)
                tquestions = questions_data['count']
                questions = questions_data['results']
            except (ValueError, KeyError) as ex:
                logger.error(ex)
                cause = ("Bad JSON format for mozilla_questions: %s" % (raw_questions))
                raise ParseError(cause=cause)

            for question in questions:
                if drop_questions > 0:
                    # Remove extra questions due to page base retrieval
                    drop_questions -= 1
                    continue
                question['offset'] = current_offset
                current_offset += 1
                question['answers_data'] = []
                for raw_answers in self.client.get_question_answers(question['id']):
                    answers = json.loads(raw_answers)['results']
                    question['answers_data'] += answers
                yield question
                nquestions += 1

            logger.debug("Questions: %i/%i", nquestions + offset, tquestions)

        logger.info("Total number of questions: %i (%i total)", nquestions, tquestions)
        logger.info("Questions with errors dropped: %i", equestions)

    def metadata(self, item, filter_classified=False):
        """Kitsune metadata.

        This method takes items overrides `metadata` method to add extra
        information related to Kitsune (offset of the question).

        :param item: an item fetched by a backend
        :param filter_classified: sets if classified fields were filtered
        """
        item = super().metadata(item, filter_classified=filter_classified)
        item['offset'] = item['data'].pop('offset')

        return item

    @classmethod
    def has_archiving(cls):
        """Returns whether it supports archiving items on the fetch process.

        :returns: this backend supports items archive
        """
        return True

    @classmethod
    def has_resuming(cls):
        """Returns whether it supports to resume the fetch process.

        :returns: this backend supports items resuming
        """
        return True

    @staticmethod
    def metadata_id(item):
        """Extracts the identifier from a Kitsune item."""

        return str(item['id'])

    @staticmethod
    def metadata_updated_on(item):
        """Extracts the update time from a Kitsune item.

        The timestamp is extracted from 'updated' field.
        This date is a UNIX timestamp but needs to be converted to
        a float value.

        :param item: item generated by the backend

        :returns: a UNIX timestamp
        """
        return float(str_to_datetime(item['updated']).timestamp())

    @staticmethod
    def metadata_category(item):
        """Extracts the category from a Kitsune item.

        This backend only generates one type of item which is
        'question'.
        """
        return CATEGORY_QUESTION

    def _init_client(self, from_archive=False):
        """Init client"""

        return KitsuneClient(self.url, self.archive, from_archive)

class KitsuneClient(HttpClient):
    """Kitsune API client.

    This class implements a simple client to retrieve questions and answers from
    a Kitsune site.

    :param url: URL of Kitsune (sample https://support.mozilla.org)
    :param archive: an archive to store/read fetched data
    :param from_archive: it tells whether to write/read the archive

    :raises HTTPError: when an error occurs doing the request
    """
    FIRST_PAGE = 1  # Initial page in Kitsune
    ITEMS_PER_PAGE = 20  # Items per page in Kitsune API

    def __init__(self, url, archive=None, from_archive=False):
        super().__init__(urijoin(url, '/api/2/'), archive=archive, from_archive=from_archive)

    def get_questions(self, offset=None):
        """Retrieve questions from older to newer updated starting offset"""

        page = KitsuneClient.FIRST_PAGE

        if offset:
            page += int(offset / KitsuneClient.ITEMS_PER_PAGE)

        while True:
            api_questions_url = urijoin(self.base_url, '/question') + '/'

            params = {
                "page": page,
                "ordering": "updated"
            }

            questions = self.fetch(api_questions_url, params)
            yield questions

            questions_json = json.loads(questions)
            next_uri = questions_json['next']
            if not next_uri:
                break
            page += 1

    def get_question_answers(self, question_id):
        """Retrieve all answers for a question from older to newer (updated)"""

        page = KitsuneClient.FIRST_PAGE

        while True:
            api_answers_url = urijoin(self.base_url, '/answer') + '/'
            params = {
                "page": page,
                "question": question_id,
                "ordering": "updated"
            }

            answers_raw = self.fetch(api_answers_url, params)
            yield answers_raw

            answers = json.loads(answers_raw)
            if not answers['next']:
                break
            page += 1

    def fetch(self, url, params):
        """Return the textual content associated to the Response object"""

        logger.debug("Kitsune client calls API: %s params: %s",
                     url, str(params))

        response = super().fetch(url, payload=params)

        return response.text

class KitsuneCommand(BackendCommand):
    """Class to run Kitsune backend from the command line."""

    BACKEND = Kitsune

    @classmethod
    def setup_cmd_parser(cls):
        """Returns the Kitsune argument parser."""

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              offset=True,
                                              archive=True)

        # Required arguments
        parser.parser.add_argument('url', nargs='?',
                                   default="https://support.mozilla.org",
                                   help="Kitsune URL (default: https://support.mozilla.org)")

        return parser