# -*- coding: utf-8 -*-
#
# Copyright (C) 2015-2019 Bitergia
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA.
#
# Authors:
#     Santiago Dueñas <sduenas@bitergia.com>
#

import argparse
import hashlib
import importlib
import json
import logging
import os
import pkgutil
import sys

from grimoirelab_toolkit.introspect import find_signature_parameters
from grimoirelab_toolkit.datetime import (datetime_utcnow,
                                          str_to_datetime)
from .archive import Archive, ArchiveManager
from .errors import ArchiveError, BackendError
from ._version import __version__

logger = logging.getLogger(__name__)

ARCHIVES_DEFAULT_PATH = '~/.perceval/archives/'

class Backend:
    """Abstract class for backends.

    Base class to fetch data from a repository. This repository
    will be named as 'origin'. During the initialization, an `Archive`
    object can be provided for archiving raw data from the repositories.

    Derivated classes have to implement `fetch_items`, `has_archiving` and
    `has_resuming` methods. Otherwise, `NotImplementedError`
    exception will be raised. Metadata decorator can be used together with
    fetch methods but requires the implementation of `metadata_id`,
    `metadata_updated_on` and `metadata_category` static methods.

    The fetched items can be tagged using the `tag` parameter. It will
    be useful to trace data. When it is set to `None` or to an empty
    string, the tag will be the same that the `origin` attribute.

    To track which version of the backend was used during the fetching
    process, this class provides a `version` attribute that each backend
    may override.

    Each backend should implement a class attribute named `CLASSIFIED_FIELDS`.
    It will allow to filter from items those fields that may be considered
    sensible or confidential. This attribute is a list of lists.
    As items returned are dicts that may contain nested dicts, each entry
    is a list which stores the "path" or nested dicts keys to the field to
    remove. For example, `['my', 'classified', 'field']` will remove `field`
    from `item['data']['my']['classified']` dict.

    Classified data filtering and archiving are not compatible to prevent
    data leaks or security issues.

    :param origin: identifier of the repository
    :param tag: tag items using this label
    :param archive: archive to store/retrieve data

    :raises ValueError: raised when `archive` is not an instance of
        `Archive` class
    """
    version = '0.8.0'

    CATEGORIES = []
    CLASSIFIED_FIELDS = []

    def __init__(self, origin, tag=None, archive=None):
        self._origin = origin
        self.tag = tag if tag else origin
        self.archive = archive or None

    @property
    def origin(self):
        return self._origin

    @property
    def archive(self):
        return self._archive

    @archive.setter
    def archive(self, obj):
        if obj and not isinstance(obj, Archive):
            msg = "obj is not an instance of Archive. %s object given" \
                % (str(type(obj)))
            raise ValueError(msg)

        self._archive = obj

    @property
    def categories(self):
        return self.CATEGORIES

    @property
    def classified_fields(self):
        cfs = ['.'.join(cf) for cf in self.CLASSIFIED_FIELDS]
        return cfs

    def fetch_items(self, category, **kwargs):
        raise NotImplementedError

    def fetch(self, category, filter_classified=False, **kwargs):
        """Fetch items from the repository.

        The method retrieves items from a repository.

        To removed classified fields from the resulting items, set
        the parameter `filter_classified`. Take into account this
        parameter is incompatible with archiving items. Raw client
        data are archived before any other process. Therefore,
        classified data  are stored within the archive. To prevent
        from possible data leaks or security issues when users do
        not need these fields, archiving and filtering are not
        compatible.

        :param category: the category of the items fetched
        :param filter_classified: remove classified fields from the resulting items
        :param kwargs: a list of other parameters (e.g., from_date, offset, etc.
        specific for each backend)

        :returns: a generator of items

        :raises BackendError: either when the category is not valid or
            'filter_classified' and 'archive' are active at the same time.
        """
        if category not in self.categories:
            cause = "%s category not valid for %s" % (category, self.__class__.__name__)
            raise BackendError(cause=cause)

        if filter_classified and self.archive:
            cause = "classified fields filtering is not compatible with archiving items"
            raise BackendError(cause=cause)

        if self.archive:
            self.archive.init_metadata(self.origin, self.__class__.__name__, self.version, category,
                                       kwargs)

        self.client = self._init_client()

        for item in self.fetch_items(category, **kwargs):
            if filter_classified:
                item = self.filter_classified_data(item)

            yield self.metadata(item, filter_classified=filter_classified)

    def fetch_from_archive(self):
        """Fetch the questions from an archive.

        It returns the items stored within an archive. If this method is called but
        no archive was provided, the method will raise a `ArchiveError` exception.

        :returns: a generator of items

        :raises ArchiveError: raised when an error occurs accessing an archive
        """
        if not self.archive:
            raise ArchiveError(cause="archive instance was not provided")

        self.client = self._init_client(from_archive=True)

        for item in self.fetch_items(self.archive.category, **self.archive.backend_params):
            yield self.metadata(item)

    def filter_classified_data(self, item):
        """Remove classified or confidential data from an item.

        It removes those fields that contain data considered as classified.
        Classified fields are defined in `CLASSIFIED_FIELDS` class attribute.

        :param item: fields will be removed from this item

        :returns: the same item but with confidential data filtered
        """
        item_uuid = uuid(self.origin, self.metadata_id(item))

        logger.debug("Filtering classified data for item %s", item_uuid)

        for cf in self.CLASSIFIED_FIELDS:
            try:
                _remove_key_from_nested_dict(item, cf)
            except KeyError:
                logger.debug("Classified field '%s' not found for item %s; field ignored",
                             '.'.join(cf), item_uuid)

        logger.debug("Classified data filtered for item %s", item_uuid)

        return item

    def metadata(self, item, filter_classified=False):
        """Add metadata to an item.

        It adds metadata to a given item such as how and
        when it was fetched. The contents from the original item will
        be stored under the 'data' keyword.

        :param item: an item fetched by a backend
        :param filter_classified: sets if classified fields were filtered
        """
        item = {
            'backend_name': self.__class__.__name__,
            'backend_version': self.version,
            'perceval_version': __version__,
            'timestamp': datetime_utcnow().timestamp(),
            'origin': self.origin,
            'uuid': uuid(self.origin, self.metadata_id(item)),
            'updated_on': self.metadata_updated_on(item),
            'classified_fields_filtered': self.classified_fields if filter_classified else None,
            'category': self.metadata_category(item),
            'tag': self.tag,
            'data': item,
        }

        return item

    @classmethod
    def has_archiving(cls):
        raise NotImplementedError

    @classmethod
    def has_resuming(cls):
        raise NotImplementedError

    @staticmethod
    def metadata_id(item):
        raise NotImplementedError

    @staticmethod
    def metadata_updated_on(item):
        raise NotImplementedError

    @staticmethod
    def metadata_category(item):
        raise NotImplementedError

    def _init_client(self, from_archive=False):
        raise NotImplementedError

def _remove_key_from_nested_dict(nested_dict, path_to_field):
    if len(path_to_field) == 0:
        return

    key = path_to_field[0]

    if len(path_to_field) == 1:
        nested_dict.pop(key)
    else:
        _remove_key_from_nested_dict(nested_dict[key], path_to_field[1:])

class BackendCommandArgumentParser:
    """Manage and parse backend command arguments.

    This class defines and parses a set of arguments common to
    backends commands. Some parameters like archive or the different
    types of authentication can be set during the initialization
    of the instance.

    :param categories: set category argument
    :param from_date: set from_date argument
    :param to_date: set to_date argument
    :param offset: set offset argument
    :param basic_auth: set basic authentication arguments
    :param token_auth: set token/key authentication arguments
    :param archive: set archiving arguments
    :param aliases: define aliases for parsed arguments

    :raises AttributeArror: when both `from_date` and `offset` are set
        to `True`
    """

    def __init__(self, categories, from_date=False, to_date=False, offset=False,
                 basic_auth=False, token_auth=False, archive=False,
                 aliases=None):
        self._from_date = from_date
        self._to_date = to_date
        self._archive = archive
        self._categories = categories

        self.aliases = aliases or {}
        self.parser = argparse.ArgumentParser()

        group = self.parser.add_argument_group('general arguments')
        group.add_argument('--category', dest='category',
                           help="type of the items to fetch (%s)" % ','.join(self._categories))
        group.add_argument('--tag', dest='tag',
                           help="tag the items generated during the fetching process")
        group.add_argument('--filter-classified', dest='filter_classified',
                           action='store_true',
                           help="filter classified fields, if any, from fetched items")

        if (from_date or to_date) and offset:
            raise AttributeError("date and offset parameters are incompatible")

        if from_date:
            group.add_argument('--from-date', dest='from_date',
                               default='1970-01-01',
                               help="fetch items updated since this \
                                     date (in any ISO 8601 format, e.g., 'YYYY-MM-DD HH:mm:SS +|-HH:MM')")
        if to_date:
            group.add_argument('--to-date', dest='to_date',
                               help="fetch items updated before this \
                                    date (in any ISO 8601 format, e.g., 'YYYY-MM-DD HH:mm:SS +|-HH:MM')")
        if offset:
            group.add_argument('--offset', dest='offset',
                               type=int, default=0,
                               help="offset to start fetching items")

        if basic_auth or token_auth:
            self._set_auth_arguments(basic_auth=basic_auth,
                                     token_auth=token_auth)

        if archive:
            self._set_archive_arguments()

        self._set_output_arguments()

    def parse(self, *args):
        """Parse a list of arguments.

        Parse argument strings needed to run a backend command. The result
        will be a `argparse.Namespace` object populated with the values
        obtained after the validation of the parameters.

        :param args: argument strings

        :result: an object with the parsed values
        """
        parsed_args = self.parser.parse_args(args)

        # Category was not set, remove it
        if parsed_args.category is None:
            delattr(parsed_args, 'category')

        if self._from_date:
            parsed_args.from_date = str_to_datetime(parsed_args.from_date)
        if self._to_date and parsed_args.to_date:
            parsed_args.to_date = str_to_datetime(parsed_args.to_date)
        if self._archive and parsed_args.archived_since:
            parsed_args.archived_since = str_to_datetime(parsed_args.archived_since)

        if self._archive and parsed_args.fetch_archive and parsed_args.no_archive:
            raise AttributeError("fetch-archive and no-archive arguments are not compatible")
        if self._archive and parsed_args.fetch_archive and not parsed_args.category:
            raise AttributeError("fetch-archive needs a category to work with")

        # Set aliases
        for alias, arg in self.aliases.items():
            if (alias not in parsed_args) and (arg in parsed_args):
                value = getattr(parsed_args, arg, None)
                setattr(parsed_args, alias, value)

        return parsed_args

    def _set_auth_arguments(self, basic_auth=True, token_auth=False):
        """Activate authentication arguments parsing"""

        group = self.parser.add_argument_group('authentication arguments')

        if basic_auth:
            group.add_argument('-u', '--backend-user', dest='user',
                               help="backend user")
            group.add_argument('-p', '--backend-password', dest='password',
                               help="backend password")
        if token_auth:
            group.add_argument('-t', '--api-token', dest='api_token',
                               help="backend authentication token / API key")

    def _set_archive_arguments(self):
        """Activate archive arguments parsing"""

        group = self.parser.add_argument_group('archive arguments')
        group.add_argument('--archive-path', dest='archive_path', default=None,
                           help="directory path to the archives")
        group.add_argument('--no-archive', dest='no_archive', action='store_true',
                           help="do not archive data")
        group.add_argument('--fetch-archive', dest='fetch_archive', action='store_true',
                           help="fetch data from the archives")
        group.add_argument('--archived-since', dest='archived_since', default='1970-01-01',
                           help="retrieve items archived since the given date")

    def _set_output_arguments(self):
        """Activate output arguments parsing"""

        group = self.parser.add_argument_group('output arguments')
        group.add_argument('-o', '--output', type=argparse.FileType('w'),
                           dest='outfile', default=sys.stdout,
                           help="output file")
        group.add_argument('--json-line', dest='json_line', action='store_true',
                           help="produce a JSON line for each output item")

class BackendCommand:
    """Abstract class to run backends from the command line.

    When the class is initialized, it parses the given arguments using
    the defined argument parser on `setump_cmd_parser` method. Those
    arguments will be stored in the attribute `parsed_args`.

    The arguments will be used to inizialize and run the `Backend` object
    assigned to this command. The backend used to run the command is stored
    under `BACKEND` class attributed. Any class derived from this and must
    set its own `Backend` class.

    Moreover, the method `setup_cmd_parser` must be implemented to exectute
    the backend.
    """
    BACKEND = None

    def __init__(self, *args):
        parser = self.setup_cmd_parser()
        self.parsed_args = parser.parse(*args)

        self.archive_manager = None

        self._pre_init()
        self._initialize_archive()
        self._post_init()

        self.outfile = self.parsed_args.outfile
        self.json_line = self.parsed_args.json_line

    def run(self):
        """Fetch and write items.

        This method runs the backend to fetch the items from the given
        origin. Items are converted to JSON objects and written to the
        defined output.

        If `fetch-archive` parameter was given as an argument during
        the inizialization of the instance, the items will be retrieved
        using the archive manager.
        """
        backend_args = vars(self.parsed_args)
        category = backend_args.pop('category', None)
        filter_classified = backend_args.pop('filter_classified', False)
        archived_since = backend_args.pop('archived_since', None)

        if self.archive_manager and self.parsed_args.fetch_archive:
            items = fetch_from_archive(self.BACKEND, backend_args,
                                       self.archive_manager,
                                       category,
                                       archived_since)
        else:
            items = fetch(self.BACKEND, backend_args, category,
                          filter_classified=filter_classified,
                          manager=self.archive_manager)

        try:
            for item in items:
                if self.json_line:
                    obj = json.dumps(item, separators=(',', ':'), sort_keys=True)
                else:
                    obj = json.dumps(item, indent=4, sort_keys=True)
                self.outfile.write(obj)
                self.outfile.write('\n')
        except IOError as e:
            raise RuntimeError(str(e))
        except Exception as e:
            raise RuntimeError(str(e))

    def _pre_init(self):
        """Override to execute before backend is initialized."""
        pass

    def _post_init(self):
        """Override to execute after backend is initialized."""
        pass

    def _initialize_archive(self):
        """Initialize archive based on the parsed parameters"""

        if 'archive_path' not in self.parsed_args:
            manager = None
        elif self.parsed_args.no_archive:
            manager = None
        else:
            if not self.parsed_args.archive_path:
                archive_path = os.path.expanduser(ARCHIVES_DEFAULT_PATH)
            else:
                archive_path = self.parsed_args.archive_path

            manager = ArchiveManager(archive_path)

        self.archive_manager = manager

    @classmethod
    def setup_cmd_parser(cls):
        raise NotImplementedError

def uuid(*args):
    """Generate a UUID based on the given parameters.

    The UUID will be the SHA1 of the concatenation of the values
    from the list. The separator bewteedn these values is ':'.
    Each value must be a non-empty string, otherwise, the function
    will raise an exception.

    :param *args: list of arguments used to generate the UUID

    :returns: a universal unique identifier

    :raises ValueError: when anyone of the values is not a string,
        is empty or `None`.
    """
    def check_value(v):
        if not isinstance(v, str):
            raise ValueError("%s value is not a string instance" % str(v))
        elif not v:
            raise ValueError("value cannot be None or empty")
        else:
            return v

    s = ':'.join(map(check_value, args))

    sha1 = hashlib.sha1(s.encode('utf-8', errors='surrogateescape'))
    uuid_sha1 = sha1.hexdigest()

    return uuid_sha1

def fetch(backend_class, backend_args, category, filter_classified=False,
          manager=None):
    """Fetch items using the given backend.

    Generator to get items using the given backend class. When
    an archive manager is given, this function will store
    the fetched items in an `Archive`. If an exception is raised,
    this archive will be removed to avoid corrupted archives.

    The parameters needed to initialize the `backend` class and
    get the items are given using `backend_args` dict parameter.

    :param backend_class: backend class to fetch items
    :param backend_args: dict of arguments needed to fetch the items
    :param category: category of the items to retrieve.
       If None, it will use the default backend category
    :param filter_classified: remove classified fields from the resulting items
    :param manager: archive manager needed to store the items

    :returns: a generator of items
    """
    init_args = find_signature_parameters(backend_class.__init__,
                                          backend_args)
    archive = manager.create_archive() if manager else None
    init_args['archive'] = archive

    backend = backend_class(**init_args)

    if category:
        backend_args['category'] = category
    if filter_classified:
        backend_args['filter_classified'] = filter_classified

    fetch_args = find_signature_parameters(backend.fetch,
                                           backend_args)
    items = backend.fetch(**fetch_args)

    try:
        for item in items:
            yield item
    except Exception as e:
        if manager:
            archive_path = archive.archive_path
            manager.remove_archive(archive_path)
        raise e

def fetch_from_archive(backend_class, backend_args, manager,
                       category, archived_after):
    """Fetch items from an archive manager.

    Generator to get the items of a category (previously fetched
    by the given backend class) from an archive manager. Only those
    items archived after the given date will be returned.

    The parameters needed to initialize `backend` and get the
    items are given using `backend_args` dict parameter.

    :param backend_class: backend class to retrive items
    :param backend_args: dict of arguments needed to retrieve the items
    :param manager: archive manager where the items will be retrieved
    :param category: category of the items to retrieve
    :param archived_after: return items archived after this date

    :returns: a generator of archived items
    """
    init_args = find_signature_parameters(backend_class.__init__,
                                          backend_args)
    backend = backend_class(**init_args)

    filepaths = manager.search(backend.origin,
                               backend.__class__.__name__,
                               category,
                               archived_after)

    for filepath in filepaths:
        backend.archive = Archive(filepath)
        items = backend.fetch_from_archive()

        try:
            for item in items:
                yield item
        except ArchiveError as e:
            logger.warning("Ignoring %s archive due to: %s", filepath, str(e))

def find_backends(top_package):
    """Find available backends.

    Look for the Perceval backends and commands under `top_package`
    and its sub-packages. When `top_package` defines a namespace,
    backends under that same namespace will be found too.

    :param top_package: package storing backends

    :returns: a tuple with two dicts: one with `Backend` classes and one
        with `BackendCommand` classes
    """
    candidates = pkgutil.walk_packages(top_package.__path__,
                                       prefix=top_package.__name__ + '.')

    modules = [name for _, name, is_pkg in candidates if not is_pkg]

    return _import_backends(modules)

def _import_backends(modules):
    for module in modules:
        importlib.import_module(module)

    bkls = _find_classes(Backend, modules)
    ckls = _find_classes(BackendCommand, modules)

    backends = {name: kls for name, kls in bkls}
    commands = {name: klass for name, klass in ckls}

    return backends, commands

def _find_classes(parent, modules):
    parents = parent.__subclasses__()

    while parents:
        kls = parents.pop()

        m = kls.__module__

        if m not in modules:
            continue

        name = m.split('.')[-1]
        parents.extend(kls.__subclasses__())

        yield name, kls


# -*- coding: utf-8 -*-
#
# Copyright (C) 2015-2019 Bitergia
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA.
#
# Authors:
#     Santiago Dueñas <sduenas@bitergia.com>
#     Germán Poo-Caamaño <gpoo@gnome.org>
#
# Note: some ot this code was taken from the MailingListStats project
#

import logging
import mailbox
import os
import tempfile

import gzip
import bz2
import zipfile

from grimoirelab_toolkit.datetime import (InvalidDateError,
                                          datetime_to_utc,
                                          str_to_datetime)

from ...backend import (Backend,
                        BackendCommand,
                        BackendCommandArgumentParser)
from ...utils import (DEFAULT_DATETIME,
                      check_compressed_file_type,
                      message_to_dict)

CATEGORY_MESSAGE = "message"

logger = logging.getLogger(__name__)

class MBox(Backend):
    """MBox backend.

    This class allows the fetch the email messages stored one or several
    mbox files. Initialize this class passing the directory path where
    the mbox files are stored. The origin of the data will be set to to
    the value of `uri`.

    :param uri: URI of the mboxes; typically, the URL of their
        mailing list
    :param dirpath: directory path where the mboxes are stored
    :param tag: label used to mark the data
    :param archive: archive to store/retrieve items
    """
    version = '0.11.0'

    CATEGORIES = [CATEGORY_MESSAGE]

    DATE_FIELD = 'Date'
    MESSAGE_ID_FIELD = 'Message-ID'

    def __init__(self, uri, dirpath, tag=None, archive=None):
        origin = uri

        super().__init__(origin, tag=tag, archive=archive)
        self.uri = uri
        self.dirpath = dirpath

    def fetch(self, category=CATEGORY_MESSAGE, from_date=DEFAULT_DATETIME):
        """Fetch the messages from a set of mbox files.

        The method retrieves, from mbox files, the messages stored in
        these containers.

        :param category: the category of items to fetch
        :param from_date: obtain messages since this date

        :returns: a generator of messages
        """
        if not from_date:
            from_date = DEFAULT_DATETIME

        kwargs = {'from_date': from_date}
        items = super().fetch(category, **kwargs)

        return items

    def fetch_items(self, category, **kwargs):
        """Fetch the messages

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        from_date = kwargs['from_date']

        logger.info("Looking for messages from '%s' on '%s' since %s",
                    self.uri, self.dirpath, str(from_date))

        mailing_list = MailingList(self.uri, self.dirpath)

        messages = self._fetch_and_parse_messages(mailing_list, from_date)

        for message in messages:
            yield message

        logger.info("Fetch process completed")

    @classmethod
    def has_archiving(cls):
        """Returns whether it supports archiving items on the fetch process.

        :returns: this backend does not support items archive
        """
        return False

    @classmethod
    def has_resuming(cls):
        """Returns whether it supports to resume the fetch process.

        :returns: this backend supports items resuming
        """
        return True

    @staticmethod
    def metadata_id(item):
        """Extracts the identifier from a MBox item."""

        return item[MBox.MESSAGE_ID_FIELD]

    @staticmethod
    def metadata_updated_on(item):
        """Extracts the update time from a MBox item.

        The timestamp used is extracted from 'Date' field in its
        several forms. This date is converted to UNIX timestamp
        format.

        :param item: item generated by the backend

        :returns: a UNIX timestamp
        """
        ts = item[MBox.DATE_FIELD]
        ts = str_to_datetime(ts)

        return ts.timestamp()

    @staticmethod
    def metadata_category(item):
        """Extracts the category from a MBox item.

        This backend only generates one type of item which is
        'message'.
        """
        return CATEGORY_MESSAGE

    @staticmethod
    def parse_mbox(filepath):
        """Parse a mbox file.

        This method parses a mbox file and returns an iterator of dictionaries.
        Each one of this contains an email message.

        :param filepath: path of the mbox to parse

        :returns : generator of messages; each message is stored in a
            dictionary of type `requests.structures.CaseInsensitiveDict`
        """
        mbox = _MBox(filepath, create=False)

        for msg in mbox:
            message = message_to_dict(msg)
            yield message

    def _init_client(self, from_archive=False):
        pass

    def _fetch_and_parse_messages(self, mailing_list, from_date):
        """Fetch and parse the messages from a mailing list"""

        from_date = datetime_to_utc(from_date)

        nmsgs, imsgs, tmsgs = (0, 0, 0)

        for mbox in mailing_list.mboxes:
            tmp_path = None

            try:
                tmp_path = self._copy_mbox(mbox)

                for message in self.parse_mbox(tmp_path):
                    tmsgs += 1

                    if not self._validate_message(message):
                        imsgs += 1
                        continue

                    # Ignore those messages sent before the given date
                    dt = str_to_datetime(message[MBox.DATE_FIELD])

                    if dt < from_date:
                        logger.debug("Message %s sent before %s; skipped",
                                     message['unixfrom'], str(from_date))
                        tmsgs -= 1
                        continue

                    # Convert 'CaseInsensitiveDict' to dict
                    message = self._casedict_to_dict(message)

                    nmsgs += 1
                    logger.debug("Message %s parsed", message['unixfrom'])

                    yield message
            except (OSError, EOFError) as e:
                logger.warning("Ignoring %s mbox due to: %s", mbox.filepath, str(e))
            except Exception as e:
                if tmp_path and os.path.exists(tmp_path):
                    os.remove(tmp_path)
                raise e
            finally:
                if tmp_path and os.path.exists(tmp_path):
                    os.remove(tmp_path)

        logger.info("Done. %s/%s messages fetched; %s ignored",
                    nmsgs, tmsgs, imsgs)

    def _copy_mbox(self, mbox):
        """Copy the contents of a mbox to a temporary file"""

        tmp_path = tempfile.mktemp(prefix='perceval_')

        with mbox.container as f_in:
            with open(tmp_path, mode='wb') as f_out:
                for l in f_in:
                    f_out.write(l)
        return tmp_path

    def _validate_message(self, message):
        """Check if the given message has the mandatory fields"""

        # This check is "case insensitive" because we're
        # using 'CaseInsensitiveDict' from requests.structures
        # module to store the contents of a message.
        if self.MESSAGE_ID_FIELD not in message:
            logger.warning("Field 'Message-ID' not found in message %s; ignoring",
                           message['unixfrom'])
            return False

        if not message[self.MESSAGE_ID_FIELD]:
            logger.warning("Field 'Message-ID' is empty in message %s; ignoring",
                           message['unixfrom'])
            return False

        if self.DATE_FIELD not in message:
            logger.warning("Field 'Date' not found in message %s; ignoring",
                           message['unixfrom'])
            return False

        if not message[self.DATE_FIELD]:
            logger.warning("Field 'Date' is empty in message %s; ignoring",
                           message['unixfrom'])
            return False

        try:
            str_to_datetime(message[self.DATE_FIELD])
        except InvalidDateError:
            logger.warning("Invalid date %s in message %s; ignoring",
                           message[self.DATE_FIELD], message['unixfrom'])
            return False

        return True

    def _casedict_to_dict(self, message):
        """Convert a message in CaseInsensitiveDict to dict.

        This method also converts well known problematic headers,
        such as Message-ID and Date to a common name.
        """
        message_id = message.pop(self.MESSAGE_ID_FIELD)
        date = message.pop(self.DATE_FIELD)

        msg = {k: v for k, v in message.items()}
        msg[self.MESSAGE_ID_FIELD] = message_id
        msg[self.DATE_FIELD] = date

        return msg

class _MBox(mailbox.mbox):
    """Wrapper of `mailbox.mbox` to catch unhandled errors"""

    def get_message(self, key):
        """Return a Message representation or raise a KeyError."""

        start, stop = self._lookup(key)
        self._file.seek(start)
        from_line = self._file.readline().replace(mailbox.linesep, b'')
        string = self._file.read(stop - self._file.tell())
        msg = self._message_factory(string.replace(mailbox.linesep, b'\n'))

        try:
            msg.set_from(from_line[5:].decode('ascii'))
            return msg
        except UnicodeDecodeError:
            pass

        try:
            msg.set_from(from_line[5:].decode('utf-8'))
        except UnicodeDecodeError:
            msg.set_from(from_line[5:].decode('iso-8859-1'))

        return msg

class MBoxCommand(BackendCommand):
    """Class to run MBox backend from the command line."""

    BACKEND = MBox

    @classmethod
    def setup_cmd_parser(cls):
        """Returns the MBox argument parser."""

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              from_date=True)

        # Required arguments
        parser.parser.add_argument('uri',
                                   help="URI of the mboxes, usually the URL to their mailing list")
        parser.parser.add_argument('dirpath',
                                   help="Path to the mbox directory")

        return parser

class MBoxArchive(object):
    """Class to access a mbox archive.

    MBOX archives can be stored into plain or compressed files
    (gzip, bz2 or zip).

    :param filepath: path to the mbox file
    """
    def __init__(self, filepath):
        self._filepath = filepath
        self._compressed = check_compressed_file_type(filepath)

    @property
    def filepath(self):
        return self._filepath

    @property
    def container(self):
        if not self.is_compressed():
            return open(self.filepath, mode='rb')

        if self.compressed_type == 'bz2':
            return bz2.open(self.filepath, mode='rb')
        elif self.compressed_type == 'gz':
            return gzip.open(self.filepath, mode='rb')
        elif self.compressed_type == "zip":
            _zip = zipfile.ZipFile(self.filepath)
            if len(_zip.infolist()) > 1:
                logger.error("Zip %s contains more than one file, only the first uncompressed", self.filepath)
            return _zip.open(_zip.infolist()[0].filename)

    @property
    def compressed_type(self):
        return self._compressed

    def is_compressed(self):
        return self._compressed is not None

class MailingList(object):
    """Manage mailing lists archives.

    This class gives access to the local mboxes archives that a
    mailing list manages.

    :param uri: URI of the mailing lists, usually its URL address
    :param dirpath: path to the mboxes archives
    """
    def __init__(self, uri, dirpath):
        self.uri = uri
        self.dirpath = dirpath

    @property
    def mboxes(self):
        """Get the mboxes managed by this mailing list.

        Returns the archives sorted by name.

        :returns: a list of `.MBoxArchive` objects
        """
        archives = []

        if os.path.isfile(self.dirpath):
            try:
                archives.append(MBoxArchive(self.dirpath))
            except OSError as e:
                logger.warning("Ignoring %s mbox due to: %s", self.dirpath, str(e))
        else:
            for root, _, files in os.walk(self.dirpath):
                for filename in sorted(files):
                    try:
                        location = os.path.join(root, filename)
                        archives.append(MBoxArchive(location))
                    except OSError as e:
                        logger.warning("Ignoring %s mbox due to: %s", filename, str(e))
        return archives


# -*- coding: utf-8 -*-
#
# Copyright (C) 2015-2019 Bitergia
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA.
#
# Authors:
#     Santiago Dueñas <sduenas@bitergia.com>
#

import collections
import io
import logging
import os
import re
import subprocess
import threading

import dulwich.client
import dulwich.repo

from grimoirelab_toolkit.datetime import datetime_to_utc, str_to_datetime

from ...backend import (Backend,
                        BackendCommand,
                        BackendCommandArgumentParser)
from ...errors import RepositoryError, ParseError
from ...utils import DEFAULT_DATETIME, DEFAULT_LAST_DATETIME

CATEGORY_COMMIT = 'commit'

logger = logging.getLogger(__name__)

class Git(Backend):
    """Git backend.

    This class allows the fetch the commits from a Git repository
    (local or remote) or from a log file. To initialize this class,
    you have to provide the URI repository and a value for `gitpath`.
    This `uri` will be set as the origin of the data.

    When `gitpath` is a directory or does not exist, it will be
    considered as the place where the repository is/will be cloned;
    when `gitpath` is a file it will be considered as a Git log file.

    :param uri: URI of the Git repository
    :param gitpath: path to the repository or to the log file
    :param tag: label used to mark the data
    :param archive: archive to store/retrieve items

    :raises RepositoryError: raised when there was an error cloning or
        updating the repository.
    """
    version = '0.11.1'

    CATEGORIES = [CATEGORY_COMMIT]

    def __init__(self, uri, gitpath, tag=None, archive=None):
        origin = uri

        super().__init__(origin, tag=tag, archive=archive)
        self.uri = uri
        self.gitpath = gitpath

    def fetch(self, category=CATEGORY_COMMIT, from_date=DEFAULT_DATETIME, to_date=DEFAULT_LAST_DATETIME,
              branches=None, latest_items=False, no_update=False):
        """Fetch commits.

        The method retrieves from a Git repository or a log file
        a list of commits. Commits are returned in the same order
        they were obtained.

        When `from_date` parameter is given it returns items commited
        since the given date.

        The list of `branches` is a list of strings, with the names of
        the branches to fetch. If the list of branches is empty, no
        commit is fetched. If the list of branches is None, all commits
        for all branches will be fetched.

        The parameter `latest_items` returns only those commits which
        are new since the last time this method was called.

        The parameter `no_update` returns all commits without performing
        an update of the repository before.

        Take into account that `from_date` and `branches` are ignored
        when the commits are fetched from a Git log file or when
        `latest_items` flag is set.

        The class raises a `RepositoryError` exception when an error
        occurs accessing the repository.

        :param category: the category of items to fetch
        :param from_date: obtain commits newer than a specific date
            (inclusive)
        :param to_date: obtain commits older than a specific date
        :param branches: names of branches to fetch from (default: None)
        :param latest_items: sync with the repository to fetch only the
            newest commits
        :param no_update: if enabled, don't update the repo with the latest changes

        :returns: a generator of commits
        """
        if not from_date:
            from_date = DEFAULT_DATETIME
        if not to_date:
            to_date = DEFAULT_LAST_DATETIME

        kwargs = {
            'from_date': from_date,
            'to_date': to_date,
            'branches': branches,
            'latest_items': latest_items,
            'no_update': no_update
        }
        items = super().fetch(category, **kwargs)

        return items

    def fetch_items(self, category, **kwargs):
        """Fetch the commits

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        from_date = kwargs['from_date']
        to_date = kwargs['to_date']
        branches = kwargs['branches']
        latest_items = kwargs['latest_items']
        no_update = kwargs['no_update']

        ncommits = 0

        try:
            if os.path.isfile(self.gitpath):
                commits = self.__fetch_from_log()
            else:
                commits = self.__fetch_from_repo(from_date, to_date, branches,
                                                 latest_items, no_update)

            for commit in commits:
                yield commit
                ncommits += 1
        except EmptyRepositoryError:
            pass

        logger.info("Fetch process completed: %s commits fetched",
                    ncommits)

    @classmethod
    def has_archiving(cls):
        """Returns whether it supports archiving items on the fetch process.

        :returns: this backend does not support items archive
        """
        return False

    @classmethod
    def has_resuming(cls):
        """Returns whether it supports to resume the fetch process.

        :returns: this backend supports items resuming
        """
        return True

    @staticmethod
    def metadata_id(item):
        """Extracts the identifier from a Git item."""

        return item['commit']

    @staticmethod
    def metadata_updated_on(item):
        """Extracts the update time from a Git item.

        The timestamp used is extracted from 'CommitDate' field.
        This date is converted to UNIX timestamp format taking into
        account the timezone of the date.

        :param item: item generated by the backend

        :returns: a UNIX timestamp
        """
        ts = item['CommitDate']
        ts = str_to_datetime(ts)

        return ts.timestamp()

    @staticmethod
    def metadata_category(item):
        """Extracts the category from a Git item.

        This backend only generates one type of item which is
        'commit'.
        """
        return CATEGORY_COMMIT

    @staticmethod
    def parse_git_log_from_file(filepath):
        """Parse a Git log file.

        The method parses the Git log file and returns an iterator of
        dictionaries. Each one of this, contains a commit.

        :param filepath: path to the log file

        :returns: a generator of parsed commits

        :raises ParseError: raised when the format of the Git log file
            is invalid
        :raises OSError: raised when an error occurs reading the
            given file
        """
        with open(filepath, 'r', errors='surrogateescape',
                  newline=os.linesep) as f:
            parser = GitParser(f)

            for commit in parser.parse():
                yield commit

    @staticmethod
    def parse_git_log_from_iter(iterator):
        """Parse a Git log obtained from an iterator.

        The method parses the Git log fetched from an iterator, where
        each item is a line of the log. It returns and iterator of
        dictionaries. Each dictionary contains a commit.

        :param iterator: iterator of Git log lines

        :raises ParseError: raised when the format of the Git log
            is invalid
        """
        parser = GitParser(iterator)

        for commit in parser.parse():
            yield commit

    def _init_client(self, from_archive=False):
        pass

    def __fetch_from_log(self):
        logger.info("Fetching commits: '%s' git repository from log file %s",
                    self.uri, self.gitpath)
        return self.parse_git_log_from_file(self.gitpath)

    def __fetch_from_repo(self, from_date, to_date, branches, latest_items=False, no_update=False):
        # When no latest items are set or the repository has not
        # been cloned use the default mode
        default_mode = not latest_items or not os.path.exists(self.gitpath)

        repo = self.__create_git_repository()

        if default_mode:
            commits = self.__fetch_commits_from_repo(repo, from_date, to_date, branches, no_update)
        else:
            commits = self.__fetch_newest_commits_from_repo(repo)

        return commits

    def __fetch_commits_from_repo(self, repo, from_date, to_date, branches, no_update):
        if branches is None:
            branches_text = "all"
        elif len(branches) == 0:
            branches_text = "no"
        else:
            branches_text = ", ".join(branches)

        logger.info("Fetching commits: '%s' git repository from %s to %s; %s branches",
                    self.uri, str(from_date), str(to_date), branches_text)

        # Ignore default datetime to avoid problems with git
        # or convert to UTC
        if to_date == DEFAULT_LAST_DATETIME:
            to_date = None
        else:
            to_date = datetime_to_utc(to_date)

        if from_date == DEFAULT_DATETIME:
            from_date = None
        else:
            from_date = datetime_to_utc(from_date)

        if not no_update:
            repo.update()

        gitlog = repo.log(from_date, to_date, branches)
        return self.parse_git_log_from_iter(gitlog)

    def __fetch_newest_commits_from_repo(self, repo):
        logger.info("Fetching latest commits: '%s' git repository",
                    self.uri)

        hashes = repo.sync()
        if not hashes:
            return []

        gitshow = repo.show(hashes)
        return self.parse_git_log_from_iter(gitshow)

    def __create_git_repository(self):
        if not os.path.exists(self.gitpath):
            repo = GitRepository.clone(self.uri, self.gitpath)
        elif os.path.isdir(self.gitpath):
            repo = GitRepository(self.uri, self.gitpath)
        return repo

class GitCommand(BackendCommand):
    """Class to run Git backend from the command line."""

    BACKEND = Git

    def _pre_init(self):
        """Initialize repositories directory path"""

        if self.parsed_args.git_log:
            git_path = self.parsed_args.git_log
        elif not self.parsed_args.git_path:
            base_path = os.path.expanduser('~/.perceval/repositories/')
            processed_uri = self.parsed_args.uri.lstrip('/')
            git_path = os.path.join(base_path, processed_uri) + '-git'
        else:
            git_path = self.parsed_args.git_path

        setattr(self.parsed_args, 'gitpath', git_path)

    @classmethod
    def setup_cmd_parser(cls):
        """Returns the Git argument parser."""

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              from_date=True,
                                              to_date=True)

        # Optional arguments
        group = parser.parser.add_argument_group('Git arguments')
        group.add_argument('--branches', dest='branches',
                           nargs='+', type=str, default=None,
                           help="Fetch commits only from these branches")

        # Mutual exclusive parameters
        exgroup = group.add_mutually_exclusive_group()
        exgroup.add_argument('--git-path', dest='git_path',
                             help="Path where the Git repository will be cloned")
        exgroup.add_argument('--git-log', dest='git_log',
                             help="Path to the Git log file")

        exgroup_fetch = group.add_mutually_exclusive_group()
        exgroup_fetch.add_argument('--latest-items', dest='latest_items',
                                   action='store_true',
                                   help="Fetch latest commits added to the repository")
        exgroup_fetch.add_argument('--no-update', dest='no_update',
                                   action='store_true',
                                   help="Fetch all commits without updating the repository")

        # Required arguments
        parser.parser.add_argument('uri',
                                   help="URI of the Git log repository")

        return parser

class GitParser:
    """Git log parser.

    This class parses a plain Git log stream, converting plain commits
    into dict items.

    Not every Git log output is valid to be parsed. The Git log stream
    must have a specific structure. It must contain raw commits data and
    stats about modified files. The next excerpt shows an example of a
    valid log:

        commit aaa7a9209f096aaaadccaaa7089aaaa3f758a703
        Author:     John Smith <jsmith@example.com>
        AuthorDate: Tue Aug 14 14:30:13 2012 -0300
        Commit:     John Smith <jsmith@example.com>
        CommitDate: Tue Aug 14 14:30:13 2012 -0300

            Commit for testing

        :000000 100644 0000000... aaaaaaa... A	aaa/otherthing
        :000000 100644 0000000... aaaaaaa... A	aaa/something
        :000000 100644 0000000... aaaaaaa... A	bbb/bthing
        0	0	aaa/otherthing
        0	0	aaa/something
        0	0	bbb/bthing

    Each commit starts with the 'commit' tag that is followed by the
    SHA-1 of the commit, its parents (two or more parents in the case
    of a merge) and a list of refs, if any.

        commit 456a68ee1407a77f3e804a30dff245bb6c6b872f
               ce8e0b86a1e9877f42fe9453ede418519115f367
               51a3b654f252210572297f47597b31527c475fb8
               (HEAD -> refs/heads/master)

    The commit line is followed by one or more headers. Each header
    has a key and a value:

        Author:     John Smith <jsmith@example.com>
        AuthorDate: Tue Aug 14 14:30:13 2012 -0300
        Commit:     John Smith <jsmith@example.com>
        CommitDate: Tue Aug 14 14:30:13 2012 -0300

    Then, an empty line divides the headers from the commit message.

        First line of the commit

        Commit message splitted into one or several lines.
        Each line of the message stars with 4 spaces.

    Commit messages can contain a list of 'trailers'. These trailers
    have the same format of headers but their meaning is project
    dependent. This is an example of a commit message with trailers:

        Commit message with trailers

        This is the body of the message where trailers are included.
        Trailers are part of the body so each line of the message
        stars with 4 spaces.

        Signed-off-by: John Doe <jdoe@example.com>
        Signed-off-by: Jane Rae <jrae@example.com>

    After a new empty line, actions and stats over files can be found.
    A action line starts with one or more ':' chars and contain data
    about the old and new permissions of a file, its old and new indexes,
    the action code and the filepath to the file. In the case of a copied,
    renamed or moved file, the new filepath to that file is included.

        :100644 100644 e69de29... e69de29... R100	aaa/otherthing	aaa/otherthing.renamed

    Stats lines include the number of lines added and removed, and the
    name of the file. The new name is also included for moved or renamed
    files.

        10	0	aaa/{otherthing => otherthing.renamed}

    The commit ends with an empty line.

    Take into account that one empty line is valid at the beginning
    of the log. This allows to parse empty logs without raising
    exceptions.

    This example was generated using the next command:

        git log --raw --numstat --pretty=fuller --decorate=full \
                --parents -M -C -c --remotes=origin --all

    :param stream: a file object which stores the log
    """
    COMMIT_PATTERN = r"""^commit[ \t](?P<commit>[a-f0-9]{40})
                     (?:[ \t](?P<parents>[a-f0-9][a-f0-9 \t]+))?
                     (?:[ \t]\((?P<refs>.+)\))?$
                     """

    HEADER_TRAILER_PATTERN = r"^(?P<name>[a-zA-z0-9\-]+)\:[ \t]+(?P<value>.+)$"

    MESSAGE_LINE_PATTERN = r"^[\s]{4}(?P<msg>.*)$"

    ACTION_PATTERN = r"""^(?P<sc>\:+)
                      (?P<modes>(?:\d{6}[ \t])+)
                      (?P<indexes>(?:[a-f0-9]+\.{,3}[ \t])+)
                      (?P<action>[^\t]+)\t+
                      (?P<file>[^\t]+)
                      (?:\t+(?P<newfile>.+))?$"""

    STATS_PATTERN = r"^(?P<added>\d+|-)[ \t]+(?P<removed>\d+|-)[ \t]+(?P<file>.+)$"

    EMPTY_LINE_PATTERN = r"^$"

    # Compiled patterns
    GIT_COMMIT_REGEXP = re.compile(COMMIT_PATTERN, re.VERBOSE)
    GIT_HEADER_TRAILER_REGEXP = re.compile(HEADER_TRAILER_PATTERN, re.VERBOSE)
    GIT_MESSAGE_REGEXP = re.compile(MESSAGE_LINE_PATTERN, re.VERBOSE)
    GIT_ACTION_REGEXP = re.compile(ACTION_PATTERN, re.VERBOSE)
    GIT_STATS_REGEXP = re.compile(STATS_PATTERN, re.VERBOSE)
    GIT_NEXT_STATE_REGEXP = re.compile(EMPTY_LINE_PATTERN, re.VERBOSE)

    # Git parser status
    (INIT,
     COMMIT,
     HEADER,
     MESSAGE,
     FILE) = range(5)

    # Git trailers
    TRAILERS = ['Signed-off-by']

    def __init__(self, stream):
        self.stream = stream
        self.nline = 0
        self.state = self.INIT

        # Aux vars to store the commit that is being parsed
        self.commit = None
        self.commit_files = {}

        self.handlers = {
            self.INIT: self._handle_init,
            self.COMMIT: self._handle_commit,
            self.HEADER: self._handle_header,
            self.MESSAGE: self._handle_message,
            self.FILE: self._handle_file
        }

    def parse(self):
        """Parse the Git log stream."""

        for line in self.stream:
            line = line.rstrip('\n')
            parsed = False
            self.nline += 1

            while not parsed:
                parsed = self.handlers[self.state](line)

                if self.state == self.COMMIT and self.commit:
                    commit = self._build_commit()
                    logger.debug("Commit %s parsed", commit['commit'])
                    yield commit

        # Return the last commit, if any
        if self.commit:
            commit = self._build_commit()
            logger.debug("Commit %s parsed", commit['commit'])
            yield commit

    def _build_commit(self):
        def remove_none_values(d):
            return {k: v for k, v in d.items() if v is not None}

        commit = self.commit
        commit = remove_none_values(commit)
        commit['files'] = [remove_none_values(item)
                           for _, item in sorted(self.commit_files.items())]

        self.commit = None
        self.commit_files = {}

        return commit

    def _handle_init(self, line):
        m = self.GIT_NEXT_STATE_REGEXP.match(line)

        # In both cases, the parser advances to the next state.
        # It only has to check whether the line has to be parsed
        # again or not
        self.state = self.COMMIT
        parsed = m is not None

        return parsed

    def _handle_commit(self, line):
        m = self.GIT_COMMIT_REGEXP.match(line)
        if not m:
            msg = "commit expected on line %s" % (str(self.nline))
            raise ParseError(cause=msg)

        parents = self.__parse_data_list(m.group('parents'))
        refs = self.__parse_data_list(m.group('refs'), sep=',')

        # Initialize a new commit
        self.commit = {}
        self.commit['commit'] = m.group('commit')
        self.commit['parents'] = parents
        self.commit['refs'] = refs

        self.state = self.HEADER

        return True

    def _handle_header(self, line):
        m = self.GIT_NEXT_STATE_REGEXP.match(line)
        if m:
            self.state = self.MESSAGE
            return True

        m = self.GIT_HEADER_TRAILER_REGEXP.match(line)
        if not m:
            msg = "invalid header format on line %s" % (str(self.nline))
            raise ParseError(cause=msg)

        header = m.group('name')
        value = m.group('value')
        self.commit[header] = value

        return True

    def _handle_message(self, line):
        m = self.GIT_NEXT_STATE_REGEXP.match(line)
        if m:
            self.state = self.FILE
            return True

        m = self.GIT_MESSAGE_REGEXP.match(line)
        if not m:
            logger.debug("Invalid message format on line %s. Skipping.",
                         str(self.nline))
            self.state = self.FILE
            return False

        msg_line = m.group('msg')

        # Concatenate message lines
        if 'message' not in self.commit:
            self.commit['message'] = ''
        else:
            self.commit['message'] += '\n'
        self.commit['message'] += msg_line

        # Check trailers
        self._handle_trailer(msg_line)

        return True

    def _handle_file(self, line):
        m = self.GIT_NEXT_STATE_REGEXP.match(line)
        if m:
            self.state = self.COMMIT
            return True

        m = self.GIT_ACTION_REGEXP.match(line)
        if m:
            data = m.groupdict()
            self._handle_action_data(data)
            return True

        m = self.GIT_STATS_REGEXP.match(line)
        if m:
            data = m.groupdict()
            self._handle_stats_data(data)
            return True

        # No match case
        logger.debug("Invalid action format on line %s. Skipping.",
                     str(self.nline))
        self.state = self.COMMIT
        return False

    def _handle_trailer(self, line):
        m = self.GIT_HEADER_TRAILER_REGEXP.match(line)
        if not m:
            return

        trailer = m.group('name')
        value = m.group('value')

        if trailer not in self.TRAILERS:
            logger.debug("Trailer %s found on line %s but is not a core trailer. Skipping.",
                         trailer, str(self.nline))
            return

        self.commit.setdefault(trailer, []).append(value)

    def _handle_action_data(self, data):
        modes = self.__parse_data_list(data['modes'])
        indexes = self.__parse_data_list(data['indexes'])
        filename = data['file']

        if filename not in self.commit_files:
            self.commit_files[filename] = {}

        self.commit_files[filename]['modes'] = modes
        self.commit_files[filename]['indexes'] = indexes
        self.commit_files[filename]['action'] = data['action']
        self.commit_files[filename]['file'] = filename
        self.commit_files[filename]['newfile'] = data['newfile']

    def _handle_stats_data(self, data):
        filename = self.__get_old_filepath(data['file'])

        if filename not in self.commit_files:
            self.commit_files[filename] = {'file': filename}

        self.commit_files[filename]['added'] = data['added']
        self.commit_files[filename]['removed'] = data['removed']

    def __parse_data_list(self, data, sep=' '):
        if data:
            lst = data.strip().split(sep)
            return [e.strip() for e in lst]
        else:
            return []

    def __get_old_filepath(self, f):
        """Get the old filepath of a moved/renamed file.

        Moved or renamed files can be found in the log with any of the
        next patterns:
          'old_name => new_name'
          '{old_prefix => new_prefix}/name'
          'name/{old_suffix => new_suffix}'

        This method returns the filepath before the file was moved or
        renamed.
        """
        i = f.find('{')
        j = f.find('}')

        if i > -1 and j > -1:
            prefix = f[0:i]
            inner = f[i + 1:f.find(' => ', i)]
            suffix = f[j + 1:]
            return prefix + inner + suffix
        elif ' => ' in f:
            return f.split(' => ')[0]
        else:
            return f

class EmptyRepositoryError(RepositoryError):
    """Exception raised when a repository is empty"""

    message = "%(repository)s is empty"

GitRef = collections.namedtuple('GitRef', ['hash', 'refname'])

class _GraphWalker:
    """Commit walker needed by fetch_pack"""

    def __init__(self, local_refs):
        self.heads = [
            ref.hash.encode('utf-8') for ref in local_refs
            if ref.refname.startswith('refs/heads/')
        ]

    def ack(self, sha):
        pass

    def next(self):
        if self.heads:
            ret = self.heads.pop()
            return ret
        return None

    __next__ = next

class GitRepository:
    """Manage a Git repository.

    This class provides access to a Git repository running some
    common commands such as `clone`, `pull` or `log`.
    To create an instance from a remote repository, use `clone()`
    class method.

    :param uri: URI of the repository
    :param dirpath: local directory where the repository is stored
    """
    GIT_PRETTY_OUTPUT_OPTS = [
        '--raw',  # show data in raw format
        '--numstat',  # show added/deleted lines per file
        '--pretty=fuller',  # pretty output
        '--decorate=full',  # show full refs
        '--parents',  # show parents information
        '-M',  # detect and report renames
        '-C',  # detect and report copies
        '-c',  # show merge info
    ]

    def __init__(self, uri, dirpath):
        gitdir = os.path.join(dirpath, 'HEAD')

        if not os.path.exists(dirpath):
            cause = "directory '%s' for Git repository '%s' does not exist" % (dirpath, uri)
            raise RepositoryError(cause=cause)
        elif not os.path.exists(gitdir):
            warning = "Working directories for Git repositories no longer supported." \
                "Please remove it or clone it using --mirror option."
            logger.warning(warning)
            cause = "directory '%s' is not a Git mirror of repository '%s'" % (dirpath, uri)
            raise RepositoryError(cause=cause)

        self.uri = uri
        self.dirpath = dirpath
        self.gitenv = {
            'LANG': 'C',
            'PAGER': '',
            'HOME': os.getenv('HOME', '')
        }

    @classmethod
    def clone(cls, uri, dirpath):
        """Clone a Git repository.

        Make a bare copy of the repository stored in `uri` into `dirpath`.
        The repository would be either local or remote.

        :param uri: URI of the repository
        :param dirtpath: directory where the repository will be cloned

        :returns: a `GitRepository` class having cloned the repository

        :raises RepositoryError: when an error occurs cloning the given
            repository
        """
        cmd = ['git', 'clone', '--bare', uri, dirpath]
        env = {
            'LANG': 'C',
            'HOME': os.getenv('HOME', '')
        }

        cls._exec(cmd, env=env)

        logger.debug("Git %s repository cloned into %s",
                     uri, dirpath)

        return cls(uri, dirpath)

    def count_objects(self):
        """Count the objects of a repository.

        The method returns the total number of objects (packed and unpacked)
        available on the repository.

        :raises RepositoryError: when an error occurs counting the objects
            of a repository
        """
        cmd_count = ['git', 'count-objects', '-v']

        outs = self._exec(cmd_count, cwd=self.dirpath, env=self.gitenv)
        outs = outs.decode('utf-8', errors='surrogateescape').rstrip()

        try:
            cobjs = {k: v for k, v in (x.split(': ') for x in outs.split('\n'))}
            nobjs = int(cobjs['count']) + int(cobjs['in-pack'])
        except KeyError as e:
            error = "unable to parse 'count-objects' output; reason: '%s' entry not found" \
                % e.args[0]
            raise RepositoryError(cause=error)
        except ValueError as e:
            error = "unable to parse 'count-objects' output; reason: %s" % str(e)
            raise RepositoryError(cause=error)

        logger.debug("Git %s repository has %s objects",
                     self.uri, str(nobjs))

        return nobjs

    def is_detached(self):
        """Check if the repo is in a detached state.

        The repository is in a detached state when HEAD is not a symbolic
        reference.

        :returns: whether the repository is detached or not

        :raises RepositoryError: when an error occurs checking the state
            of the repository
        """
        cmd_sym = ['git', 'symbolic-ref', 'HEAD']

        try:
            self._exec(cmd_sym, cwd=self.dirpath, env=self.gitenv)
        except RepositoryError as e:
            if e.msg.find("ref HEAD is not a symbolic ref") == -1:
                raise e
            return True
        else:
            return False

    def is_empty(self):
        """Determines whether the repository is empty or not.

        Returns `True` when the repository is empty. Under the hood,
        it checks the number of objects on the repository. When
        this number is 0, the repositoy is empty.

        :raises RepositoryError: when an error occurs accessing the
            repository
        """
        return self.count_objects() == 0

    def update(self):
        """Update repository from its remote.

        Calling this method, the repository will be synchronized with
        the remote repository using 'fetch' command for 'heads' refs.
        Any commit stored in the local copy will be removed; refs
        will be overwritten.

        :raises RepositoryError: when an error occurs updating the
            repository
        """
        cmd_update = ['git', 'fetch', 'origin', '+refs/heads/*:refs/heads/*', '--prune']
        self._exec(cmd_update, cwd=self.dirpath, env=self.gitenv)

        logger.debug("Git %s repository updated into %s",
                     self.uri, self.dirpath)

    def sync(self):
        """Keep the repository in sync.

        This method will synchronize the repository with its 'origin',
        fetching newest objects and updating references. It uses low
        level commands which allow to keep track of which things
        have changed in the repository.

        The method also returns a list of hashes related to the new
        commits fetched during the process.

        :returns: list of new commits

        :raises RepositoryError: when an error occurs synchronizing
            the repository
        """
        pack_name, refs = self._fetch_pack()

        if pack_name:
            commits = self._read_commits_from_pack(pack_name)
        else:
            commits = []
            logger.debug("Git repository %s (%s) does not have any new object",
                         self.uri, self.dirpath)

        self._update_references(refs)

        logger.debug("Git repository %s (%s) is synced",
                     self.uri, self.dirpath)

        return commits

    def rev_list(self, branches=None):
        """Read the list commits from the repository

        The list of branches is a list of strings, with the names of the
        branches to fetch. If the list of branches is empty, no commit
        is fetched. If the list of branches is None, all commits
        for all branches will be fetched.

        The method returns the Git rev-list of the repository using the
        following options:

            git rev-list --topo-order

        :param branches: names of branches to fetch from (default: None)

        :raises EmptyRepositoryError: when the repository is empty and
            the action cannot be performed
        :raises RepositoryError: when an error occurs executing the command
        """
        if self.is_empty():
            logger.warning("Git %s repository is empty; unable to get the rev-list",
                           self.uri)
            raise EmptyRepositoryError(repository=self.uri)

        cmd_rev_list = ['git', 'rev-list', '--topo-order']

        if branches is None:
            cmd_rev_list.extend(['--branches', '--tags', '--remotes=origin'])
        elif len(branches) == 0:
            cmd_rev_list.extend(['--branches', '--tags', '--max-count=0'])
        else:
            branches = ['refs/heads/' + branch for branch in branches]
            cmd_rev_list.extend(branches)

        for line in self._exec_nb(cmd_rev_list, cwd=self.dirpath, env=self.gitenv):
            yield line.rstrip('\n')

        logger.debug("Git rev-list fetched from %s repository (%s)",
                     self.uri, self.dirpath)

    def log(self, from_date=None, to_date=None, branches=None, encoding='utf-8'):
        """Read the commit log from the repository.

        The method returns the Git log of the repository using the
        following options:

            git log --raw --numstat --pretty=fuller --decorate=full
                --all --reverse --topo-order --parents -M -C -c
                --remotes=origin

        When `from_date` is given, it gets the commits equal or older
        than that date. This date is given in a datetime object.

        The list of branches is a list of strings, with the names of the
        branches to fetch. If the list of branches is empty, no commit
        is fetched. If the list of branches is None, all commits
        for all branches will be fetched.

        :param from_date: fetch commits newer than a specific
            date (inclusive)
        :param branches: names of branches to fetch from (default: None)
        :param encoding: encode the log using this format

        :returns: a generator where each item is a line from the log

        :raises EmptyRepositoryError: when the repository is empty and
            the action cannot be performed
        :raises RepositoryError: when an error occurs fetching the log
        """
        if self.is_empty():
            logger.warning("Git %s repository is empty; unable to get the log",
                           self.uri)
            raise EmptyRepositoryError(repository=self.uri)

        cmd_log = ['git', 'log', '--reverse', '--topo-order']
        cmd_log.extend(self.GIT_PRETTY_OUTPUT_OPTS)

        if from_date:
            dt = from_date.strftime("%Y-%m-%d %H:%M:%S %z")
            cmd_log.append('--since=' + dt)

        if to_date:
            dt = to_date.strftime("%Y-%m-%d %H:%M:%S %z")
            cmd_log.append('--until=' + dt)

        if branches is None:
            cmd_log.extend(['--branches', '--tags', '--remotes=origin'])
        elif len(branches) == 0:
            cmd_log.append('--max-count=0')
        else:
            branches = ['refs/heads/' + branch for branch in branches]
            cmd_log.extend(branches)

        for line in self._exec_nb(cmd_log, cwd=self.dirpath, env=self.gitenv):
            yield line

        logger.debug("Git log fetched from %s repository (%s)",
                     self.uri, self.dirpath)

    def show(self, commits=None, encoding='utf-8'):
        """Show the data of a set of commits.

        The method returns the output of Git show command for a
        set of commits using the following options:

            git show --raw --numstat --pretty=fuller --decorate=full
                --parents -M -C -c [<commit>...<commit>]

        When the list of commits is empty, the command will return
        data about the last commit, like the default behaviour of
        `git show`.

        :param commits: list of commits to show data
        :param encoding: encode the output using this format

        :returns: a generator where each item is a line from the show output

        :raises EmptyRepositoryError: when the repository is empty and
            the action cannot be performed
        :raises RepositoryError: when an error occurs fetching the show output
        """
        if self.is_empty():
            logger.warning("Git %s repository is empty; unable to run show",
                           self.uri)
            raise EmptyRepositoryError(repository=self.uri)

        if commits is None:
            commits = []

        cmd_show = ['git', 'show']
        cmd_show.extend(self.GIT_PRETTY_OUTPUT_OPTS)
        cmd_show.extend(commits)

        for line in self._exec_nb(cmd_show, cwd=self.dirpath, env=self.gitenv):
            yield line

        logger.debug("Git show fetched from %s repository (%s)",
                     self.uri, self.dirpath)

    def _fetch_pack(self):
        """Fetch changes and store them in a pack."""

        def prepare_refs(refs):
            return [ref.hash.encode('utf-8') for ref in refs
                    if not ref.refname.endswith('^{}')]

        def determine_wants(refs):
            remote_refs = prepare_refs(self._discover_refs(remote=True))
            local_refs = prepare_refs(self._discover_refs())
            wants = [ref for ref in remote_refs if ref not in local_refs]
            return wants

        client, repo_path = dulwich.client.get_transport_and_path(self.uri)
        repo = dulwich.repo.Repo(self.dirpath)
        fd = io.BytesIO()

        local_refs = self._discover_refs()
        graph_walker = _GraphWalker(local_refs)

        result = client.fetch_pack(repo_path,
                                   determine_wants,
                                   graph_walker,
                                   fd.write)
        refs = [GitRef(ref_hash.decode('utf-8'), ref_name.decode('utf-8'))
                for ref_name, ref_hash in result.refs.items()]

        if len(fd.getvalue()) > 0:
            fd.seek(0)
            pack = repo.object_store.add_thin_pack(fd.read, None)
            pack_name = pack.name().decode('utf-8')
        else:
            pack_name = None

        return (pack_name, refs)

    def _read_commits_from_pack(self, packet_name):
        """Read the commits of a pack."""

        filepath = 'objects/pack/pack-' + packet_name

        cmd_verify_pack = ['git', 'verify-pack', '-v', filepath]

        outs = self._exec(cmd_verify_pack, cwd=self.dirpath, env=self.gitenv)
        outs = outs.decode('utf-8', errors='surrogateescape').rstrip()

        lines = [line.split(' ') for line in outs.split('\n')]

        # Commits usually come in the pack ordered from newest to oldest
        commits = [parts[0] for parts in lines if parts[1] == 'commit']
        commits.reverse()

        return commits

    def _update_references(self, refs):
        """Update references removing old ones."""

        new_refs = [ref.refname for ref in refs]

        # Delete old references
        for old_ref in self._discover_refs():
            if not old_ref.refname.startswith('refs/heads/'):
                continue
            if old_ref.refname in new_refs:
                continue
            self._update_ref(old_ref, delete=True)

        # Update new references
        for new_ref in refs:
            refname = new_ref.refname

            if refname.endswith('^{}'):
                logger.debug("Annotated tag %s ignored for updating in sync process",
                             refname)
                continue
            elif not refname.startswith('refs/heads/') and not refname.startswith('refs/tags/'):
                logger.debug("Reference %s not needed; ignored for updating in sync process",
                             refname)
                continue
            else:
                self._update_ref(new_ref)

        # Prune repository to remove old branches
        cmd = ['git', 'remote', 'prune', 'origin']
        self._exec(cmd, cwd=self.dirpath, env=self.gitenv)

    def _discover_refs(self, remote=False):
        """Get the current list of local or remote refs."""

        if remote:
            cmd_refs = ['git', 'ls-remote', '-h', '-t', '--exit-code', 'origin']
            sep = '\t'
            ignored_error_codes = [2]
        else:
            # Check first whether the local repo is empty;
            # Running 'show-ref' in empty repos gives an error
            if self.is_empty():
                raise EmptyRepositoryError(repository=self.uri)

            cmd_refs = ['git', 'show-ref', '--heads', '--tags']
            sep = ' '
            ignored_error_codes = [1]

        # Error codes returned when no matching refs (i.e, no heads
        # or tags) are found in a repository will be ignored. Otherwise,
        # the full process would fail for those situations.
        outs = self._exec(cmd_refs, cwd=self.dirpath,
                          env=self.gitenv,
                          ignored_error_codes=ignored_error_codes)
        outs = outs.decode('utf-8', errors='surrogateescape').rstrip()
        outs = outs.split('\n') if outs else []

        refs = []

        for line in outs:
            data = line.split(sep)
            ref = GitRef(data[0], data[1])
            refs.append(ref)

        return refs

    def _update_ref(self, ref, delete=False):
        """Update a reference."""

        cmd = ['git', 'update-ref']

        if delete:
            cmd.extend(['-d', ref.refname])
            action = 'deleted'
        else:
            cmd.extend([ref.refname, ref.hash])
            action = 'updated to %s' % ref.hash

        try:
            self._exec(cmd, cwd=self.dirpath, env=self.gitenv)
        except RepositoryError as e:
            logger.warning("Git %s ref could not be %s during sync process in %s (%s); skipped",
                           ref.refname, action, self.uri, self.dirpath)
        else:
            logger.debug("Git %s ref %s in %s (%s)",
                         ref.refname, action, self.uri, self.dirpath)

    def _exec_nb(self, cmd, cwd=None, env=None, encoding='utf-8'):
        """Run a command with a non blocking call.

        Execute `cmd` command with a non blocking call. The command will
        be run in the directory set by `cwd`. Enviroment variables can be
        set using the `env` dictionary. The output data is returned
        as encoded bytes in an iterator. Each item will be a line of the
        output.

        :returns: an iterator with the output of the command as encoded bytes

        :raises RepositoryError: when an error occurs running the command
        """
        self.failed_message = None

        logger.debug("Running command %s (cwd: %s, env: %s)",
                     ' '.join(cmd), cwd, str(env))

        try:
            self.proc = subprocess.Popen(cmd,
                                         stdout=subprocess.PIPE,
                                         stderr=subprocess.PIPE,
                                         cwd=cwd,
                                         env=env)
            err_thread = threading.Thread(target=self._read_stderr,
                                          kwargs={'encoding': encoding},
                                          daemon=True)
            err_thread.start()
            for line in self.proc.stdout:
                yield line.decode(encoding, errors='surrogateescape')
            err_thread.join()

            self.proc.communicate()
            self.proc.stdout.close()
            self.proc.stderr.close()
        except OSError as e:
            err_thread.join()
            raise RepositoryError(cause=str(e))

        if self.proc.returncode != 0:
            cause = "git command - %s (return code: %d)" % \
                (self.failed_message, self.proc.returncode)
            raise RepositoryError(cause=cause)

    def _read_stderr(self, encoding='utf-8'):
        """Reads self.proc.stderr.

        Usually, this should be read in a thread, to prevent blocking
        the read from stdout of the stderr buffer is filled, and this
        function is not called becuase the program is busy in the
        stderr reading loop.

        Reads self.proc.stderr (self.proc is the subprocess running
        the git command), and reads / writes self.failed_message
        (the message sent to stderr when git fails, usually one line).
        """
        for line in self.proc.stderr:
            err_line = line.decode(encoding, errors='surrogateescape')

            if self.proc.returncode != 0:
                # If the subprocess didn't finish successfully, we expect
                # the last line in stderr to provide the cause
                if self.failed_message is not None:
                    # We had a message, there is a newer line, print it
                    logger.debug("Git log stderr: " + self.failed_message)
                self.failed_message = err_line
            else:
                # The subprocess is successfully up to now, print the line
                logger.debug("Git log stderr: " + err_line)

    @staticmethod
    def _exec(cmd, cwd=None, env=None, ignored_error_codes=None,
              encoding='utf-8'):
        """Run a command.

        Execute `cmd` command in the directory set by `cwd`. Environment
        variables can be set using the `env` dictionary. The output
        data is returned as encoded bytes.

        Commands which their returning status codes are non-zero will
        be treated as failed. Error codes considered as valid can be
        ignored giving them in the `ignored_error_codes` list.

        :returns: the output of the command as encoded bytes

        :raises RepositoryError: when an error occurs running the command
        """
        if ignored_error_codes is None:
            ignored_error_codes = []

        logger.debug("Running command %s (cwd: %s, env: %s)",
                     ' '.join(cmd), cwd, str(env))

        try:
            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE,
                                    cwd=cwd, env=env)
            (outs, errs) = proc.communicate()
        except OSError as e:
            raise RepositoryError(cause=str(e))

        if proc.returncode != 0 and proc.returncode not in ignored_error_codes:
            err = errs.decode(encoding, errors='surrogateescape')
            cause = "git command - %s" % err
            raise RepositoryError(cause=cause)
        else:
            logger.debug(errs.decode(encoding, errors='surrogateescape'))

        return outs


# -*- coding: utf-8 -*-
#
# Copyright (C) 2015-2019 Bitergia
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA.
#
# Authors:
#     Valerio Cosentino <valcos@bitergia.com>
#

import json
import logging

from grimoirelab_toolkit.datetime import datetime_utcnow, str_to_datetime

from ...backend import (Backend,
                        BackendCommand,
                        BackendCommandArgumentParser)
from ...client import HttpClient, RateLimitHandler
from ...errors import BackendError

CATEGORY_TWEET = "tweet"
MAX_SEARCH_QUERY = 500

TWITTER_URL = 'https://twitter.com/'
TWITTER_API_URL = 'https://api.twitter.com/1.1/search/tweets.json'
MAX_ITEMS = 100

# Range before sleeping until rate limit reset
MIN_RATE_LIMIT = 1

# Time to avoid too many request exception
SLEEP_TIME = 30

TWEET_TYPE_MIXED = "mixed"
TWEET_TYPE_RECENT = "recent"
TWEET_TYPE_POPULAR = "popular"

RATE_LIMIT_HEADER = "x-rate-limit-remaining"
RATE_LIMIT_RESET_HEADER = "x-rate-limit-reset"

logger = logging.getLogger(__name__)

class Twitter(Backend):
    """Twitter backend.

    This class allows to fetch samples of tweets containing specific
    keywords. Initialize this class passing API key needed
    for authentication with the parameter `api_key`.

    :param query: query to fetch tweets
    :param api_token: token or key needed to use the API
    :param max_items: maximum number of issues requested on the same query
    :param sleep_for_rate: sleep until rate limit is reset
    :param min_rate_to_sleep: minimun rate needed to sleep until
         it will be reset
    :param sleep_time: time (in seconds) to sleep in case
        of connection problems
    :param tag: label used to mark the data
    :param archive: archive to store/retrieve items
    """
    version = '0.2.2'

    CATEGORIES = [CATEGORY_TWEET]

    def __init__(self, query, api_token, max_items=MAX_ITEMS,
                 sleep_for_rate=False, min_rate_to_sleep=MIN_RATE_LIMIT,
                 sleep_time=SLEEP_TIME,
                 tag=None, archive=None):
        origin = TWITTER_URL

        if len(query) >= MAX_SEARCH_QUERY:
            msg = "Search query length exceeded %s, max is %s" % (len(query), MAX_SEARCH_QUERY)
            raise BackendError(cause=msg)

        super().__init__(origin, tag=tag, archive=archive)
        self.query = query
        self.api_token = api_token
        self.max_items = max_items
        self.sleep_for_rate = sleep_for_rate
        self.min_rate_to_sleep = min_rate_to_sleep
        self.sleep_time = sleep_time

        self.client = None

    def fetch(self, category=CATEGORY_TWEET, since_id=None, max_id=None,
              geocode=None, lang=None,
              include_entities=True, tweets_type=TWEET_TYPE_MIXED):
        """Fetch the tweets from the server.

        This method fetches tweets from the TwitterSearch API published in the last seven days.

        :param category: the category of items to fetch
        :param since_id: if not null, it returns results with an ID greater than the specified ID
        :param max_id: when it is set or if not None, it returns results with an ID less than the specified ID
        :param geocode: if enabled, returns tweets by users located at latitude,longitude,"mi"|"km"
        :param lang: if enabled, restricts tweets to the given language, given by an ISO 639-1 code
        :param include_entities: if disabled, it excludes entities node
        :param tweets_type: type of tweets returned. Default is “mixed”, others are "recent" and "popular"

        :returns: a generator of tweets
        """
        kwargs = {"since_id": since_id,
                  "max_id": max_id,
                  "geocode": geocode,
                  "lang": lang,
                  "include_entities": include_entities,
                  "result_type": tweets_type}
        items = super().fetch(category, **kwargs)

        return items

    def fetch_items(self, category, **kwargs):
        """Fetch the tweets

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        since_id = kwargs['since_id']
        max_id = kwargs['max_id']
        geocode = kwargs['geocode']
        lang = kwargs['lang']
        entities = kwargs['include_entities']
        tweets_type = kwargs['result_type']

        logger.info("Fetching tweets %s from %s to %s",
                    self.query, str(since_id),
                    str(max_id) if max_id else '--')

        tweets_ids = []
        min_date = None
        max_date = None
        group_tweets = self.client.tweets(self.query, since_id=since_id, max_id=max_id, geocode=geocode,
                                          lang=lang, include_entities=entities, result_type=tweets_type)

        for tweets in group_tweets:
            for i in range(len(tweets)):
                tweet = tweets[i]
                tweets_ids.append(tweet['id'])

                if tweets[-1] == tweet:
                    min_date = str_to_datetime(tweets[-1]['created_at'])

                if tweets[0] == tweet and not max_date:
                    max_date = str_to_datetime(tweets[0]['created_at'])

                yield tweet

        logger.info("Fetch process completed: %s (unique %s) tweets fetched, from %s to %s",
                    len(tweets_ids), len(list(set(tweets_ids))), min_date, max_date)

    @classmethod
    def has_archiving(cls):
        """Returns whether it supports archiving items on the fetch process.

        :returns: this backend supports items archive
        """
        return True

    @classmethod
    def has_resuming(cls):
        """Returns whether it supports to resume the fetch process.

        :returns: this backend supports items resuming
        """
        return False

    @staticmethod
    def metadata_id(item):
        """Extracts the identifier from a Twitter item."""

        return str(item['id_str'])

    @staticmethod
    def metadata_updated_on(item):
        """Extracts and coverts the update time from a Twitter item.

        The timestamp is extracted from 'created_at' field and converted
        to a UNIX timestamp.

        :param item: item generated by the backend

        :returns: a UNIX timestamp
        """
        ts = item['created_at']
        ts = str_to_datetime(ts)

        return ts.timestamp()

    @staticmethod
    def metadata_category(item):
        """Extracts the category from a Twitter item.

        This backend only generates one type of item which is
        'tweet'.
        """
        return CATEGORY_TWEET

    def _init_client(self, from_archive=False):
        """Init client"""

        return TwitterClient(self.api_token, self.max_items,
                             self.sleep_for_rate, self.min_rate_to_sleep, self.sleep_time,
                             self.archive, from_archive)

class TwitterClient(HttpClient, RateLimitHandler):
    """Twitter API client.

    Client for fetching information from the Twitter server
    using its REST API v1.1.

    :param api_key: key needed to use the API
    :param max_items: maximum number of items per request
    :param sleep_for_rate: sleep until rate limit is reset
    :param min_rate_to_sleep: minimun rate needed to sleep until
         it will be reset
    :param sleep_time: time (in seconds) to sleep in case
        of connection problems
    :param archive: an archive to store/read fetched data
    :param from_archive: it tells whether to write/read the archive
    """
    def __init__(self, api_key, max_items=MAX_ITEMS,
                 sleep_for_rate=False, min_rate_to_sleep=MIN_RATE_LIMIT, sleep_time=SLEEP_TIME,
                 archive=None, from_archive=False):
        self.api_key = api_key
        self.max_items = max_items

        super().__init__(TWITTER_API_URL, sleep_time=sleep_time, extra_status_forcelist=[429],
                         archive=archive, from_archive=from_archive)
        super().setup_rate_limit_handler(sleep_for_rate=sleep_for_rate, min_rate_to_sleep=min_rate_to_sleep,
                                         rate_limit_header=RATE_LIMIT_HEADER,
                                         rate_limit_reset_header=RATE_LIMIT_RESET_HEADER)

    def calculate_time_to_reset(self):
        """Number of seconds to wait. They are contained in the rate limit reset header"""

        time_to_reset = self.rate_limit_reset_ts - (datetime_utcnow().replace(microsecond=0).timestamp() + 1)
        time_to_reset = 0 if time_to_reset < 0 else time_to_reset

        return time_to_reset

    @staticmethod
    def sanitize_for_archive(url, headers, payload):
        """Sanitize payload of a HTTP request by removing the token information
        before storing/retrieving archived items

        :param: url: HTTP url request
        :param: headers: HTTP headers request
        :param: payload: HTTP payload request

        :returns url, headers and the sanitized payload
        """
        if 'Authorization' in headers:
            headers.pop('Authorization')

        return url, headers, payload

    def tweets(self, query, since_id=None, max_id=None, geocode=None, lang=None,
               include_entities=True, result_type=TWEET_TYPE_MIXED):
        """Fetch tweets for a given query between since_id and max_id.

        :param query: query to fetch tweets
        :param since_id: if not null, it returns results with an ID greater than the specified ID
        :param max_id: if not null, it returns results with an ID less than the specified ID
        :param geocode: if enabled, returns tweets by users located at latitude,longitude,"mi"|"km"
        :param lang: if enabled, restricts tweets to the given language, given by an ISO 639-1 code
        :param include_entities: if disabled, it excludes entities node
        :param result_type: type of tweets returned. Default is “mixed”, others are "recent" and "popular"

        :returns: a generator of tweets
        """
        resource = self.base_url
        params = {'q': query,
                  'count': self.max_items}

        if since_id:
            params['since_id'] = since_id

        if max_id:
            params['max_id'] = max_id

        if geocode:
            params['geocode'] = geocode

        if lang:
            params['lang'] = lang

        params['include_entities'] = include_entities
        params['result_type'] = result_type

        while True:
            raw_tweets = self._fetch(resource, params=params)
            tweets = json.loads(raw_tweets)

            if not tweets['statuses']:
                break

            params['max_id'] = tweets['statuses'][-1]['id'] - 1
            yield tweets['statuses']

    def _fetch(self, url, params):
        """Fetch a resource.

        Method to fetch and to iterate over the contents of a
        type of resource. The method returns a generator of
        pages for that resource and parameters.

        :param url: the endpoint of the API
        :param params: parameters to filter

        :returns: the text of the response
        """
        if not self.from_archive:
            self.sleep_for_rate_limit()

        headers = {'Authorization': 'Bearer ' + self.api_key}
        r = self.fetch(url, payload=params, headers=headers)

        if not self.from_archive:
            self.update_rate_limit(r)

        return r.text

class TwitterCommand(BackendCommand):
    """Class to run Twitter backend from the command line."""

    BACKEND = Twitter

    @classmethod
    def setup_cmd_parser(cls):
        """Returns the Twitter argument parser."""

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              token_auth=True,
                                              archive=True)

        # Backend token is required
        action = parser.parser._option_string_actions['--api-token']
        action.required = True

        # Meetup options
        group = parser.parser.add_argument_group('Twitter arguments')
        group.add_argument('--max-items', dest='max_items',
                           type=int, default=MAX_ITEMS,
                           help="Maximum number of items requested on the same query")
        group.add_argument('--no-entities', dest='include_entities',
                           action='store_false',
                           help=" Exclude entities node")
        group.add_argument('--geo-code', dest='geocode',
                           help="Select tweets by users located at latitude,longitude,radius")
        group.add_argument('--lang', dest='lang',
                           help="Select tweets to the given language in ISO 639-1 code")
        group.add_argument('--tweets-type', dest='tweets_type', default=TWEET_TYPE_MIXED,
                           help="Type of tweets returned. Default is 'mixed', others are 'recent' and 'popular'")
        group.add_argument('--sleep-for-rate', dest='sleep_for_rate',
                           action='store_true',
                           help="sleep for getting more rate")
        group.add_argument('--min-rate-to-sleep', dest='min_rate_to_sleep',
                           default=MIN_RATE_LIMIT, type=int,
                           help="sleep until reset when the rate limit reaches this value")
        group.add_argument('--sleep-time', dest='sleep_time',
                           default=SLEEP_TIME, type=int,
                           help="minimun sleeping time to avoid too many request exception")

        # Required arguments
        parser.parser.add_argument('query',
                                   help="Search query including operators, max 500 chars")

        return parser