# -*- coding: utf-8 -*-
#
# Copyright (C) 2015-2019 Bitergia
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA.
#
# Authors:
#     Valerio Cosentino <valcos@bitergia.com>
#

import bs4
import logging
import re

from grimoirelab_toolkit.datetime import datetime_utcnow

from ...backend import (Backend,
                        BackendCommand,
                        BackendCommandArgumentParser,
                        uuid)
from ...client import HttpClient
from ...errors import BackendError

# Default sleep time and retries to deal with connection/server problems
DEFAULT_SLEEP_TIME = 1
MAX_RETRIES = 5

CATEGORY_HITS = "google_hits"
GOOGLE_SEARCH_URL = 'https://www.google.com/search'

logger = logging.getLogger(__name__)

class GoogleHits(Backend):
    """GoogleHits backend for Perceval.

    This class retrieves the number of hits for a given list of
    keywords via the Google API. To initialize this class a list
    of keywords is needed.

    :param keywords: a list of keywords
    :param tag: label used to mark the data
    :param archive: archive to store/retrieve items
    :param max_retries: number of max retries to a data source
        before raising a RetryError exception
    :param sleep_time: time (in seconds) to sleep in case
        of connection problems
    """
    version = '0.2.2'

    CATEGORIES = [CATEGORY_HITS]

    def __init__(self, keywords, tag=None, archive=None,
                 max_retries=MAX_RETRIES, sleep_time=DEFAULT_SLEEP_TIME):

        if len(keywords) == 1 and keywords[0].strip() == "":
            cause = "No keywords provided"
            raise BackendError(cause=cause)

        self.keywords = keywords
        super().__init__(GOOGLE_SEARCH_URL, tag=tag, archive=archive)

        self.max_retries = max_retries
        self.sleep_time = sleep_time

        self.client = None

    def fetch(self, category=CATEGORY_HITS):
        """Fetch data from Google API.

        The method retrieves a list of hits for some
        given keywords using the Google API.

        :param category: the category of items to fetch

        :returns: a generator of data
        """
        kwargs = {}
        items = super().fetch(category, **kwargs)

        return items

    def fetch_items(self, category, **kwargs):
        """Fetch Google hit items

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        logger.info("Fetching data for '%s'", self.keywords)

        hits_raw = self.client.hits(self.keywords)
        hits = self.__parse_hits(hits_raw)

        yield hits

        logger.info("Fetch process completed")

    @classmethod
    def has_archiving(cls):
        """Returns whether it supports archiving items on the fetch process.

        :returns: this backend supports items archive
        """
        return True

    @classmethod
    def has_resuming(cls):
        """Returns whether it supports to resume the fetch process.

        :returns: this backend supports items resuming
        """
        return True

    @staticmethod
    def metadata_id(item):
        """Extracts the identifier from a GoogleHit item."""

        return item['id']

    @staticmethod
    def metadata_updated_on(item):
        """Extracts the update time from a GoogleHit item.

        The timestamp is based on the current time when the hit was extracted.
        This field is not part of the data provided by Google API. It is added
        by this backend.

        :param item: item generated by the backend

        :returns: a UNIX timestamp
        """
        return item['fetched_on']

    @staticmethod
    def metadata_category(item):
        """Extracts the category from a GoogleHits item.

        This backend only generates one type of item which is
        'google_hits'.
        """
        return CATEGORY_HITS

    def _init_client(self, from_archive=False):
        """Init client"""

        return GoogleHitsClient(self.sleep_time, self.max_retries,
                                archive=self.archive, from_archive=from_archive)

    def __parse_hits(self, hit_raw):
        """Parse the hits returned by the Google Search API"""

        # Create the soup and get the desired div
        bs_result = bs4.BeautifulSoup(hit_raw, 'html.parser')
        hit_string = bs_result.find("div", id="resultStats").text

        # Remove commas or dots
        hit_string = hit_string.replace(',', u'')
        hit_string = hit_string.replace('.', u'')

        fetched_on = datetime_utcnow().timestamp()
        id_args = self.keywords[:]
        id_args.append(str(fetched_on))

        hits_json = {
            'fetched_on': fetched_on,
            'id': uuid(*id_args),
            'keywords': self.keywords,
            'type': 'googleSearchHits'
        }

        if not hit_string:
            logger.warning("No hits for %s", self.keywords)
            hits_json['hits'] = 0

            return hits_json

        str_hits = re.search(r'\d+', hit_string).group(0)
        hits = int(str_hits)
        hits_json['hits'] = hits

        return hits_json

class GoogleHitsClient(HttpClient):
    """GoogleHits API client.

    Client for fetching hits data from Google API.

    :param sleep_time: time (in seconds) to sleep in case
        of connection problems
    :param max_retries: number of max retries to a data source
        before raising a RetryError exception
    :param archive: an archive to store/read fetched data
    :param from_archive: it tells whether to write/read the archive
    """
    EXTRA_STATUS_FORCELIST = [429]

    def __init__(self, sleep_time=DEFAULT_SLEEP_TIME, max_retries=MAX_RETRIES,
                 archive=None, from_archive=False):
        super().__init__(GOOGLE_SEARCH_URL, extra_status_forcelist=self.EXTRA_STATUS_FORCELIST,
                         sleep_time=sleep_time, max_retries=max_retries,
                         archive=archive, from_archive=from_archive)

    def hits(self, keywords):
        """Fetch information about a list of keywords."""

        if len(keywords) == 1:
            query_str = keywords[0]
        else:
            query_str = ' '.join([k for k in keywords])

        logger.info("Fetching hits for '%s'", query_str)
        params = {'q': query_str}

        # Make the request
        req = self.fetch(GOOGLE_SEARCH_URL, payload=params)

        return req.text

class GoogleHitsCommand(BackendCommand):
    """Class to run GoogleHits backend from the command line."""

    BACKEND = GoogleHits

    @classmethod
    def setup_cmd_parser(cls):
        """Returns the GoogleHits argument parser."""

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              archive=True)

        group = parser.parser.add_argument_group('GoogleHits arguments')
        # Generic client options
        group.add_argument('--max-retries', dest='max_retries',
                           default=MAX_RETRIES, type=int,
                           help="number of API call retries")
        group.add_argument('--sleep-time', dest='sleep_time',
                           default=DEFAULT_SLEEP_TIME, type=int,
                           help="sleeping time between API call retries")

        # Required arguments
        parser.parser.add_argument('keywords', nargs='+',
                                   help="Keywords to search as Google hits")

        return parser


# -*- coding: utf-8 -*-
#
# Copyright (C) 2015-2019 Bitergia
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA.
#
# Authors:
#     Alvaro del Castillo San Felix <acs@bitergia.com>
#     Santiago Dueñas <sduenas@bitergia.com>
#     Alberto Martín <alberto.martin@bitergia.com>
#

import json
import logging

import requests
from grimoirelab_toolkit.datetime import (datetime_to_utc,
                                          datetime_utcnow,
                                          str_to_datetime)
from grimoirelab_toolkit.uris import urijoin

from ...backend import (Backend,
                        BackendCommand,
                        BackendCommandArgumentParser)
from ...client import HttpClient, RateLimitHandler
from ...utils import DEFAULT_DATETIME, DEFAULT_LAST_DATETIME

CATEGORY_ISSUE = "issue"
CATEGORY_PULL_REQUEST = "pull_request"
CATEGORY_REPO = 'repository'

GITHUB_URL = "https://github.com/"
GITHUB_API_URL = "https://api.github.com"

# Range before sleeping until rate limit reset
MIN_RATE_LIMIT = 10
MAX_RATE_LIMIT = 500
# Use this factor of the current token's remaining API points before switching to the next token
TOKEN_USAGE_BEFORE_SWITCH = 0.1

PER_PAGE = 100

# Default sleep time and retries to deal with connection/server problems
DEFAULT_SLEEP_TIME = 1
MAX_RETRIES = 5

TARGET_ISSUE_FIELDS = ['user', 'assignee', 'assignees', 'comments', 'reactions']
TARGET_PULL_FIELDS = ['user', 'review_comments', 'requested_reviewers', "merged_by", "commits"]

logger = logging.getLogger(__name__)

class GitHub(Backend):
    """GitHub backend for Perceval.

    This class allows the fetch the issues stored in GitHub
    repository.

    :param owner: GitHub owner
    :param repository: GitHub repository from the owner
    :param api_token: list of GitHub auth tokens to access the API
    :param base_url: GitHub URL in enterprise edition case;
        when no value is set the backend will be fetch the data
        from the GitHub public site.
    :param tag: label used to mark the data
    :param archive: archive to store/retrieve items
    :param sleep_for_rate: sleep until rate limit is reset
    :param min_rate_to_sleep: minimun rate needed to sleep until
         it will be reset
    :param max_retries: number of max retries to a data source
        before raising a RetryError exception
    :param sleep_time: time to sleep in case
        of connection problems
    """
    version = '0.20.1'

    CATEGORIES = [CATEGORY_ISSUE, CATEGORY_PULL_REQUEST, CATEGORY_REPO]

    def __init__(self, owner=None, repository=None,
                 api_token=None, base_url=None,
                 tag=None, archive=None,
                 sleep_for_rate=False, min_rate_to_sleep=MIN_RATE_LIMIT,
                 max_retries=MAX_RETRIES, sleep_time=DEFAULT_SLEEP_TIME):
        if api_token is None:
            api_token = []
        origin = base_url if base_url else GITHUB_URL
        origin = urijoin(origin, owner, repository)

        super().__init__(origin, tag=tag, archive=archive)

        self.owner = owner
        self.repository = repository
        self.api_token = api_token
        self.base_url = base_url

        self.sleep_for_rate = sleep_for_rate
        self.min_rate_to_sleep = min_rate_to_sleep
        self.max_retries = max_retries
        self.sleep_time = sleep_time

        self.client = None
        self._users = {}  # internal users cache

    def fetch(self, category=CATEGORY_ISSUE, from_date=DEFAULT_DATETIME, to_date=DEFAULT_LAST_DATETIME):
        """Fetch the issues/pull requests from the repository.

        The method retrieves, from a GitHub repository, the issues/pull requests
        updated since the given date.

        :param category: the category of items to fetch
        :param from_date: obtain issues/pull requests updated since this date
        :param to_date: obtain issues/pull requests until a specific date (included)

        :returns: a generator of issues
        """
        if not from_date:
            from_date = DEFAULT_DATETIME
        if not to_date:
            to_date = DEFAULT_LAST_DATETIME

        from_date = datetime_to_utc(from_date)
        to_date = datetime_to_utc(to_date)

        kwargs = {
            'from_date': from_date,
            'to_date': to_date
        }
        items = super().fetch(category, **kwargs)

        return items

    def fetch_items(self, category, **kwargs):
        """Fetch the items (issues or pull_requests)

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        from_date = kwargs['from_date']
        to_date = kwargs['to_date']

        if category == CATEGORY_ISSUE:
            items = self.__fetch_issues(from_date, to_date)
        elif category == CATEGORY_PULL_REQUEST:
            items = self.__fetch_pull_requests(from_date, to_date)
        else:
            items = self.__fetch_repo_info()

        return items

    @classmethod
    def has_archiving(cls):
        """Returns whether it supports archiving items on the fetch process.

        :returns: this backend supports items archive
        """
        return True

    @classmethod
    def has_resuming(cls):
        """Returns whether it supports to resume the fetch process.

        :returns: this backend supports items resuming
        """
        return True

    @staticmethod
    def metadata_id(item):
        """Extracts the identifier from a GitHub item."""

        if "forks_count" in item:
            return str(item['fetched_on'])
        else:
            return str(item['id'])

    @staticmethod
    def metadata_updated_on(item):
        """Extracts the update time from a GitHub item.

        The timestamp used is extracted from 'updated_at' field.
        This date is converted to UNIX timestamp format. As GitHub
        dates are in UTC the conversion is straightforward.

        :param item: item generated by the backend

        :returns: a UNIX timestamp
        """
        if "forks_count" in item:
            return item['fetched_on']
        else:
            ts = item['updated_at']
            ts = str_to_datetime(ts)

            return ts.timestamp()

    @staticmethod
    def metadata_category(item):
        """Extracts the category from a GitHub item.

        This backend generates two types of item which are
        'issue' and 'pull_request'.
        """

        if "base" in item:
            category = CATEGORY_PULL_REQUEST
        elif "forks_count" in item:
            category = CATEGORY_REPO
        else:
            category = CATEGORY_ISSUE

        return category

    def _init_client(self, from_archive=False):
        """Init client"""

        return GitHubClient(self.owner, self.repository, self.api_token, self.base_url,
                            self.sleep_for_rate, self.min_rate_to_sleep,
                            self.sleep_time, self.max_retries,
                            self.archive, from_archive)

    def __fetch_issues(self, from_date, to_date):
        """Fetch the issues"""

        issues_groups = self.client.issues(from_date=from_date)

        for raw_issues in issues_groups:
            issues = json.loads(raw_issues)
            for issue in issues:

                if str_to_datetime(issue['updated_at']) > to_date:
                    return

                self.__init_extra_issue_fields(issue)
                for field in TARGET_ISSUE_FIELDS:

                    if not issue[field]:
                        continue

                    if field == 'user':
                        issue[field + '_data'] = self.__get_user(issue[field]['login'])
                    elif field == 'assignee':
                        issue[field + '_data'] = self.__get_issue_assignee(issue[field])
                    elif field == 'assignees':
                        issue[field + '_data'] = self.__get_issue_assignees(issue[field])
                    elif field == 'comments':
                        issue[field + '_data'] = self.__get_issue_comments(issue['number'])
                    elif field == 'reactions':
                        issue[field + '_data'] = \
                            self.__get_issue_reactions(issue['number'], issue['reactions']['total_count'])

                yield issue

    def __fetch_pull_requests(self, from_date, to_date):
        """Fetch the pull requests"""

        raw_pulls = self.client.pulls(from_date=from_date)
        for raw_pull in raw_pulls:
            pull = json.loads(raw_pull)

            if str_to_datetime(pull['updated_at']) > to_date:
                return

            self.__init_extra_pull_fields(pull)
            for field in TARGET_PULL_FIELDS:

                if not pull[field]:
                    continue

                if field == 'user':
                    pull[field + '_data'] = self.__get_user(pull[field]['login'])
                elif field == 'merged_by':
                    pull[field + '_data'] = self.__get_user(pull[field]['login'])
                elif field == 'review_comments':
                    pull[field + '_data'] = self.__get_pull_review_comments(pull['number'])
                elif field == 'requested_reviewers':
                    pull[field + '_data'] = self.__get_pull_requested_reviewers(pull['number'])
                elif field == 'commits':
                    pull[field + '_data'] = self.__get_pull_commits(pull['number'])

            yield pull

    def __fetch_repo_info(self):
        """Get repo info about stars, watchers and forks"""

        raw_repo = self.client.repo()
        repo = json.loads(raw_repo)

        fetched_on = datetime_utcnow()
        repo['fetched_on'] = fetched_on.timestamp()

        yield repo

    def __get_issue_reactions(self, issue_number, total_count):
        """Get issue reactions"""

        reactions = []

        if total_count == 0:
            return reactions

        group_reactions = self.client.issue_reactions(issue_number)

        for raw_reactions in group_reactions:

            for reaction in json.loads(raw_reactions):
                reaction['user_data'] = self.__get_user(reaction['user']['login'])
                reactions.append(reaction)

        return reactions

    def __get_issue_comments(self, issue_number):
        """Get issue comments"""

        comments = []
        group_comments = self.client.issue_comments(issue_number)

        for raw_comments in group_comments:

            for comment in json.loads(raw_comments):
                comment_id = comment.get('id')
                comment['user_data'] = self.__get_user(comment['user']['login'])
                comment['reactions_data'] = \
                    self.__get_issue_comment_reactions(comment_id, comment['reactions']['total_count'])
                comments.append(comment)

        return comments

    def __get_issue_comment_reactions(self, comment_id, total_count):
        """Get reactions on issue comments"""

        reactions = []

        if total_count == 0:
            return reactions

        group_reactions = self.client.issue_comment_reactions(comment_id)

        for raw_reactions in group_reactions:

            for reaction in json.loads(raw_reactions):
                reaction['user_data'] = self.__get_user(reaction['user']['login'])
                reactions.append(reaction)

        return reactions

    def __get_issue_assignee(self, raw_assignee):
        """Get issue assignee"""

        assignee = self.__get_user(raw_assignee['login'])

        return assignee

    def __get_issue_assignees(self, raw_assignees):
        """Get issue assignees"""

        assignees = []
        for ra in raw_assignees:
            assignees.append(self.__get_user(ra['login']))

        return assignees

    def __get_pull_requested_reviewers(self, pr_number):
        """Get pull request requested reviewers"""

        requested_reviewers = []
        group_requested_reviewers = self.client.pull_requested_reviewers(pr_number)

        for raw_requested_reviewers in group_requested_reviewers:
            group_requested_reviewers = json.loads(raw_requested_reviewers)

            for requested_reviewer in group_requested_reviewers['users']:
                user_data = self.__get_user(requested_reviewer['login'])
                requested_reviewers.append(user_data)

        return requested_reviewers

    def __get_pull_commits(self, pr_number):
        """Get pull request commit hashes"""

        hashes = []
        group_pull_commits = self.client.pull_commits(pr_number)

        for raw_pull_commits in group_pull_commits:

            for commit in json.loads(raw_pull_commits):
                commit_hash = commit['sha']
                hashes.append(commit_hash)

        return hashes

    def __get_pull_review_comments(self, pr_number):
        """Get pull request review comments"""

        comments = []
        group_comments = self.client.pull_review_comments(pr_number)

        for raw_comments in group_comments:

            for comment in json.loads(raw_comments):
                comment_id = comment.get('id')

                user = comment.get('user', None)
                if not user:
                    logger.warning("Missing user info for %s", comment['url'])
                    comment['user_data'] = None
                else:
                    comment['user_data'] = self.__get_user(user['login'])

                comment['reactions_data'] = \
                    self.__get_pull_review_comment_reactions(comment_id, comment['reactions']['total_count'])
                comments.append(comment)

        return comments

    def __get_pull_review_comment_reactions(self, comment_id, total_count):
        """Get pull review comment reactions"""

        reactions = []

        if total_count == 0:
            return reactions

        group_reactions = self.client.pull_review_comment_reactions(comment_id)

        for raw_reactions in group_reactions:

            for reaction in json.loads(raw_reactions):
                reaction['user_data'] = self.__get_user(reaction['user']['login'])
                reactions.append(reaction)

        return reactions

    def __get_user(self, login):
        """Get user and org data for the login"""

        user = {}

        if not login:
            return user

        user_raw = self.client.user(login)
        user = json.loads(user_raw)
        user_orgs_raw = \
            self.client.user_orgs(login)
        user['organizations'] = json.loads(user_orgs_raw)

        return user

    def __init_extra_issue_fields(self, issue):
        """Add fields to an issue"""

        issue['user_data'] = {}
        issue['assignee_data'] = {}
        issue['assignees_data'] = []
        issue['comments_data'] = []
        issue['reactions_data'] = []

    def __init_extra_pull_fields(self, pull):
        """Add fields to a pull request"""

        pull['user_data'] = {}
        pull['review_comments_data'] = {}
        pull['requested_reviewers_data'] = []
        pull['merged_by_data'] = []
        pull['commits_data'] = []

class GitHubClient(HttpClient, RateLimitHandler):
    """Client for retieving information from GitHub API

    :param owner: GitHub owner
    :param repository: GitHub repository from the owner
    :param tokens: list of GitHub auth tokens to access the API
    :param base_url: GitHub URL in enterprise edition case;
        when no value is set the backend will be fetch the data
        from the GitHub public site.
    :param sleep_for_rate: sleep until rate limit is reset
    :param min_rate_to_sleep: minimun rate needed to sleep until
         it will be reset
    :param sleep_time: time to sleep in case
        of connection problems
    :param max_retries: number of max retries to a data source
        before raising a RetryError exception
    :param archive: collect issues already retrieved from an archive
    :param from_archive: it tells whether to write/read the archive
    """
    EXTRA_STATUS_FORCELIST = [403, 500, 502, 503]

    _users = {}       # users cache
    _users_orgs = {}  # users orgs cache

    def __init__(self, owner, repository, tokens,
                 base_url=None, sleep_for_rate=False, min_rate_to_sleep=MIN_RATE_LIMIT,
                 sleep_time=DEFAULT_SLEEP_TIME, max_retries=MAX_RETRIES,
                 archive=None, from_archive=False):
        self.owner = owner
        self.repository = repository
        self.tokens = tokens
        self.n_tokens = len(self.tokens)
        self.current_token = None
        self.last_rate_limit_checked = None

        if base_url:
            base_url = urijoin(base_url, 'api', 'v3')
        else:
            base_url = GITHUB_API_URL

        super().__init__(base_url, sleep_time=sleep_time, max_retries=max_retries,
                         extra_headers=self._set_extra_headers(),
                         extra_status_forcelist=self.EXTRA_STATUS_FORCELIST,
                         archive=archive, from_archive=from_archive)
        super().setup_rate_limit_handler(sleep_for_rate=sleep_for_rate, min_rate_to_sleep=min_rate_to_sleep)

        # Choose best API token (with maximum API points remaining)
        if not self.from_archive:
            self._choose_best_api_token()

    def calculate_time_to_reset(self):
        """Calculate the seconds to reset the token requests, by obtaining the different
        between the current date and the next date when the token is fully regenerated.
        """

        time_to_reset = self.rate_limit_reset_ts - (datetime_utcnow().replace(microsecond=0).timestamp() + 1)
        time_to_reset = 0 if time_to_reset < 0 else time_to_reset

        return time_to_reset

    def issue_reactions(self, issue_number):
        """Get reactions of an issue"""

        payload = {
            'per_page': PER_PAGE,
            'direction': 'asc',
            'sort': 'updated'
        }

        path = urijoin("issues", str(issue_number), "reactions")
        return self.fetch_items(path, payload)

    def issue_comment_reactions(self, comment_id):
        """Get reactions of an issue comment"""

        payload = {
            'per_page': PER_PAGE,
            'direction': 'asc',
            'sort': 'updated'
        }

        path = urijoin("issues", "comments", str(comment_id), "reactions")
        return self.fetch_items(path, payload)

    def issue_comments(self, issue_number):
        """Get the issue comments from pagination"""

        payload = {
            'per_page': PER_PAGE,
            'direction': 'asc',
            'sort': 'updated'
        }

        path = urijoin("issues", str(issue_number), "comments")
        return self.fetch_items(path, payload)

    def issues(self, from_date=None):
        """Fetch the issues from the repository.

        The method retrieves, from a GitHub repository, the issues
        updated since the given date.

        :param from_date: obtain issues updated since this date

        :returns: a generator of issues
        """
        payload = {
            'state': 'all',
            'per_page': PER_PAGE,
            'direction': 'asc',
            'sort': 'updated'}

        if from_date:
            payload['since'] = from_date.isoformat()

        path = urijoin("issues")
        return self.fetch_items(path, payload)

    def pulls(self, from_date=None):
        """Fetch the pull requests from the repository.

        The method retrieves, from a GitHub repository, the pull requests
        updated since the given date.

        :param from_date: obtain pull requests updated since this date

        :returns: a generator of pull requests
        """
        issues_groups = self.issues(from_date=from_date)

        for raw_issues in issues_groups:
            issues = json.loads(raw_issues)
            for issue in issues:

                if "pull_request" not in issue:
                    continue

                pull_number = issue["number"]
                path = urijoin(self.base_url, 'repos', self.owner, self.repository, "pulls", pull_number)

                r = self.fetch(path)
                pull = r.text

                yield pull

    def repo(self):
        """Get repository data"""

        path = urijoin(self.base_url, 'repos', self.owner, self.repository)

        r = self.fetch(path)
        repo = r.text

        return repo

    def pull_requested_reviewers(self, pr_number):
        """Get pull requested reviewers"""

        requested_reviewers_url = urijoin("pulls", str(pr_number), "requested_reviewers")
        return self.fetch_items(requested_reviewers_url, {})

    def pull_commits(self, pr_number):
        """Get pull request commits"""

        payload = {
            'per_page': PER_PAGE,
        }

        commit_url = urijoin("pulls", str(pr_number), "commits")
        return self.fetch_items(commit_url, payload)

    def pull_review_comments(self, pr_number):
        """Get pull request review comments"""

        payload = {
            'per_page': PER_PAGE,
            'direction': 'asc',
            'sort': 'updated'
        }

        comments_url = urijoin("pulls", str(pr_number), "comments")
        return self.fetch_items(comments_url, payload)

    def pull_review_comment_reactions(self, comment_id):
        """Get reactions of a review comment"""

        payload = {
            'per_page': PER_PAGE,
            'direction': 'asc',
            'sort': 'updated'
        }

        path = urijoin("pulls", "comments", str(comment_id), "reactions")
        return self.fetch_items(path, payload)

    def user(self, login):
        """Get the user information and update the user cache"""
        user = None

        if login in self._users:
            return self._users[login]

        url_user = urijoin(self.base_url, 'users', login)

        logging.info("Getting info for %s" % (url_user))

        r = self.fetch(url_user)
        user = r.text
        self._users[login] = user

        return user

    def user_orgs(self, login):
        """Get the user public organizations"""
        if login in self._users_orgs:
            return self._users_orgs[login]

        url = urijoin(self.base_url, 'users', login, 'orgs')
        try:
            r = self.fetch(url)
            orgs = r.text
        except requests.exceptions.HTTPError as error:
            # 404 not found is wrongly received sometimes
            if error.response.status_code == 404:
                logger.error("Can't get github login orgs: %s", error)
                orgs = '[]'
            else:
                raise error

        self._users_orgs[login] = orgs

        return orgs

    def fetch(self, url, payload=None, headers=None, method=HttpClient.GET, stream=False, verify=True):
        """Fetch the data from a given URL.

        :param url: link to the resource
        :param payload: payload of the request
        :param headers: headers of the request
        :param method: type of request call (GET or POST)
        :param stream: defer downloading the response body until the response content is available

        :returns a response object
        """
        if not self.from_archive:
            self.sleep_for_rate_limit()

        response = super().fetch(url, payload, headers, method, stream, verify)

        if not self.from_archive:
            if self._need_check_tokens():
                self._choose_best_api_token()
            else:
                self.update_rate_limit(response)

        return response

    def fetch_items(self, path, payload):
        """Return the items from github API using links pagination"""

        page = 0  # current page
        last_page = None  # last page
        url_next = urijoin(self.base_url, 'repos', self.owner, self.repository, path)

        logger.debug("Get GitHub paginated items from " + url_next)

        response = self.fetch(url_next, payload=payload)

        items = response.text
        page += 1

        if 'last' in response.links:
            last_url = response.links['last']['url']
            last_page = last_url.split('&page=')[1].split('&')[0]
            last_page = int(last_page)
            logger.debug("Page: %i/%i" % (page, last_page))

        while items:
            yield items

            items = None

            if 'next' in response.links:
                url_next = response.links['next']['url']
                response = self.fetch(url_next, payload=payload)
                page += 1

                items = response.text
                logger.debug("Page: %i/%i" % (page, last_page))

    def _get_token_rate_limit(self, token):
        """Return token's remaining API points"""

        rate_url = urijoin(self.base_url, "rate_limit")
        self.session.headers.update({'Authorization': 'token ' + token})
        remaining = 0
        try:
            headers = super().fetch(rate_url).headers
            if self.rate_limit_header in headers:
                remaining = int(headers[self.rate_limit_header])
        except requests.exceptions.HTTPError as error:
            logger.warning("Rate limit not initialized: %s", error)
        return remaining

    def _get_tokens_rate_limits(self):
        """Return array of all tokens remaining API points"""

        remainings = [0] * self.n_tokens
        # Turn off archiving when checking rates, because that would cause
        # archive key conflict (the same URLs giving different responses)
        arch = self.archive
        self.archive = None
        for idx, token in enumerate(self.tokens):
            # Pass flag to skip disabling archiving because this function doies it
            remainings[idx] = self._get_token_rate_limit(token)
        # Restore archiving to whatever state it was
        self.archive = arch
        logger.debug("Remaining API points: {}".format(remainings))
        return remainings

    def _choose_best_api_token(self):
        """Check all API tokens defined and choose one with most remaining API points"""

        # Return if no tokens given
        if self.n_tokens == 0:
            return

        # If multiple tokens given, choose best
        token_idx = 0
        if self.n_tokens > 1:
            remainings = self._get_tokens_rate_limits()
            token_idx = remainings.index(max(remainings))
            logger.debug("Remaining API points: {}, choosen index: {}".format(remainings, token_idx))

        # If we have any tokens - use best of them
        self.current_token = self.tokens[token_idx]
        self.session.headers.update({'Authorization': 'token ' + self.current_token})
        # Update rate limit data for the current token
        self._update_current_rate_limit()

    def _need_check_tokens(self):
        """Check if we need to switch GitHub API tokens"""

        if self.n_tokens <= 1 or self.rate_limit is None:
            return False
        elif self.last_rate_limit_checked is None:
            self.last_rate_limit_checked = self.rate_limit
            return True

        # If approaching minimum rate limit for sleep
        approaching_limit = float(self.min_rate_to_sleep) * (1.0 + TOKEN_USAGE_BEFORE_SWITCH) + 1
        if self.rate_limit <= approaching_limit:
            self.last_rate_limit_checked = self.rate_limit
            return True

        # Only switch token when used predefined factor of the current token's remaining API points
        ratio = float(self.rate_limit) / float(self.last_rate_limit_checked)
        if ratio < 1.0 - TOKEN_USAGE_BEFORE_SWITCH:
            self.last_rate_limit_checked = self.rate_limit
            return True
        elif ratio > 1.0:
            self.last_rate_limit_checked = self.rate_limit
            return False
        else:
            return False

    def _update_current_rate_limit(self):
        """Update rate limits data for the current token"""

        url = urijoin(self.base_url, "rate_limit")
        try:
            # Turn off archiving when checking rates, because that would cause
            # archive key conflict (the same URLs giving different responses)
            arch = self.archive
            self.archive = None
            response = super().fetch(url)
            self.archive = arch
            self.update_rate_limit(response)
            self.last_rate_limit_checked = self.rate_limit
        except requests.exceptions.HTTPError as error:
            if error.response.status_code == 404:
                logger.warning("Rate limit not initialized: %s", error)
            else:
                raise error

    def _set_extra_headers(self):
        """Set extra headers for session"""

        headers = {}
        headers.update({'Accept': 'application/vnd.github.squirrel-girl-preview'})
        return headers

class GitHubCommand(BackendCommand):
    """Class to run GitHub backend from the command line."""

    BACKEND = GitHub

    @classmethod
    def setup_cmd_parser(cls):
        """Returns the GitHub argument parser."""

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              from_date=True,
                                              to_date=True,
                                              token_auth=False,
                                              archive=True)
        # GitHub options
        group = parser.parser.add_argument_group('GitHub arguments')
        group.add_argument('--enterprise-url', dest='base_url',
                           help="Base URL for GitHub Enterprise instance")
        group.add_argument('--sleep-for-rate', dest='sleep_for_rate',
                           action='store_true',
                           help="sleep for getting more rate")
        group.add_argument('--min-rate-to-sleep', dest='min_rate_to_sleep',
                           default=MIN_RATE_LIMIT, type=int,
                           help="sleep until reset when the rate limit reaches this value")
        # GitHub token(s)
        group.add_argument('-t', '--api-token', dest='api_token',
                           nargs='+',
                           default=[],
                           help="list of GitHub API tokens")

        # Generic client options
        group.add_argument('--max-retries', dest='max_retries',
                           default=MAX_RETRIES, type=int,
                           help="number of API call retries")
        group.add_argument('--sleep-time', dest='sleep_time',
                           default=DEFAULT_SLEEP_TIME, type=int,
                           help="sleeping time between API call retries")

        # Positional arguments
        parser.parser.add_argument('owner',
                                   help="GitHub owner")
        parser.parser.add_argument('repository',
                                   help="GitHub repository")

        return parser


# -*- coding: utf-8 -*-
#
# Copyright (C) 2015-2019 Bitergia
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA.
#
# Authors:
#     Valerio Cosentino <valcos@bitergia.com>
#     Santiago Dueñas <sduenas@bitergia.com>
#

import hashlib
import json
import logging
import os
import pickle
import sqlite3
import uuid

from grimoirelab_toolkit.datetime import (datetime_utcnow,
                                          datetime_to_utc,
                                          str_to_datetime)

from .errors import ArchiveError, ArchiveManagerError

logger = logging.getLogger(__name__)

class Archive:
    """Basic class for archiving raw items fetched by Perceval.

    This class allows to archive raw items - usually HTML pages or
    JSON documents - for a further recovery. These raw items will
    be fetched, stored and retrieved back by a backend.

    Each stored item will have a hash code used as unique identifier.
    Hash codes are generated using URIs and other parameters needed
    to fetch raw items.

    When an instance of `Archive` is initialized it will expect
    to access an existing archive file. To create a new and empty
    archive used `create` class method instead. Metadata must be
    initialized calling to `init_metadata` method after creating
    a new archive.

    :param archive_path: path where this archive is stored

    :raises ArchiveError: when the archive does not exist or is invalid
    """

    ARCHIVE_TABLE = "archive"
    METADATA_TABLE = "metadata"

    # Table structure
    ARCHIVE_CREATE_STMT = "CREATE TABLE " + ARCHIVE_TABLE + " ( " \
                          "id INTEGER PRIMARY KEY AUTOINCREMENT, " \
                          "hashcode VARCHAR(256) UNIQUE NOT NULL, " \
                          "uri TEXT, " \
                          "payload BLOB, " \
                          "headers BLOB, " \
                          "data BLOB)"

    METADATA_CREATE_STMT = "CREATE TABLE " + METADATA_TABLE + " ( " \
                           "origin TEXT, " \
                           "backend_name TEXT, " \
                           "backend_version TEXT, " \
                           "category TEXT, " \
                           "backend_params BLOB, " \
                           "created_on TEXT)"

    def __init__(self, archive_path):
        if not os.path.exists(archive_path):
            raise ArchiveError(cause="archive %s does not exist" % (archive_path))

        self.archive_path = archive_path
        self.origin = None
        self.backend_name = None
        self.backend_version = None
        self.category = None
        self.backend_params = None
        self.created_on = None

        self._db = sqlite3.connect(self.archive_path)

        self._verify_archive()
        self._load_metadata()

    def __del__(self):
        conn = getattr(self, '_db', None)
        if conn:
            conn.close()

    def init_metadata(self, origin, backend_name, backend_version,
                      category, backend_params):
        """Init metadata information.

        Metatada is composed by basic information needed to identify
        where archived data came from and how it can be retrieved
        and built into Perceval items.

        :param: origin: identifier of the repository
        :param: backend_name: name of the backend
        :param: backend_version: version of the backend
        :param: category: category of the items fetched
        :param: backend_params: dict representation of the fetch parameters

        raises ArchiveError: when an error occurs initializing the metadata
        """
        created_on = datetime_to_utc(datetime_utcnow())
        created_on_dumped = created_on.isoformat()
        backend_params_dumped = pickle.dumps(backend_params, 0)

        metadata = (origin, backend_name, backend_version, category,
                    backend_params_dumped, created_on_dumped,)

        try:
            cursor = self._db.cursor()
            insert_stmt = "INSERT INTO " + self.METADATA_TABLE + " "\
                          "(origin, backend_name, backend_version, " \
                          "category, backend_params, created_on) " \
                          "VALUES (?, ?, ?, ?, ?, ?)"
            cursor.execute(insert_stmt, metadata)

            self._db.commit()
            cursor.close()
        except sqlite3.DatabaseError as e:
            msg = "metadata initialization error; cause: %s" % str(e)
            raise ArchiveError(cause=msg)

        self.origin = origin
        self.backend_name = backend_name
        self.backend_version = backend_version
        self.category = category
        self.backend_params = backend_params
        self.created_on = created_on

        logger.debug("Metadata of archive %s initialized to %s",
                     self.archive_path, metadata)

    def store(self, uri, payload, headers, data):
        """Store a raw item in this archive.

        The method will store `data` content in this archive. The unique
        identifier for that item will be generated using the rest of the
        parameters.

        :param uri: request URI
        :param payload: request payload
        :param headers: request headers
        :param data: data to store in this archive

        :raises ArchiveError: when an error occurs storing the given data
        """
        hashcode = self.make_hashcode(uri, payload, headers)
        payload_dump = pickle.dumps(payload, 0)
        headers_dump = pickle.dumps(headers, 0)
        data_dump = pickle.dumps(data, 0)

        logger.debug("Archiving %s with %s %s %s in %s",
                     hashcode, uri, payload, headers, self.archive_path)

        try:
            cursor = self._db.cursor()
            insert_stmt = "INSERT INTO " + self.ARCHIVE_TABLE + " (" \
                          "id, hashcode, uri, payload, headers, data) " \
                          "VALUES(?,?,?,?,?,?)"
            cursor.execute(insert_stmt, (None, hashcode, uri,
                                         payload_dump, headers_dump, data_dump))
            self._db.commit()
            cursor.close()
        except sqlite3.IntegrityError as e:
            msg = "data storage error; cause: duplicated entry %s" % hashcode
            raise ArchiveError(cause=msg)
        except sqlite3.DatabaseError as e:
            msg = "data storage error; cause: %s" % str(e)
            raise ArchiveError(cause=msg)

        logger.debug("%s data archived in %s", hashcode, self.archive_path)

    def retrieve(self, uri, payload, headers):
        """Retrieve a raw item from the archive.

        The method will return the `data` content corresponding to the
        hascode derived from the given parameters.

        :param uri: request URI
        :param payload: request payload
        :param headers: request headers

        :returns: the archived data

        :raises ArchiveError: when an error occurs retrieving data
        """
        hashcode = self.make_hashcode(uri, payload, headers)

        logger.debug("Retrieving entry %s with %s %s %s in %s",
                     hashcode, uri, payload, headers, self.archive_path)

        self._db.row_factory = sqlite3.Row

        try:
            cursor = self._db.cursor()
            select_stmt = "SELECT data " \
                          "FROM " + self.ARCHIVE_TABLE + " " \
                          "WHERE hashcode = ?"
            cursor.execute(select_stmt, (hashcode,))
            row = cursor.fetchone()
            cursor.close()
        except sqlite3.DatabaseError as e:
            msg = "data retrieval error; cause: %s" % str(e)
            raise ArchiveError(cause=msg)

        if row:
            found = pickle.loads(row['data'])
        else:
            msg = "entry %s not found in archive %s" % (hashcode, self.archive_path)
            raise ArchiveError(cause=msg)

        return found

    @classmethod
    def create(cls, archive_path):
        """Create a brand new archive.

         Call this method to create a new and empty archive. It will initialize
         the storage file in the path defined by `archive_path`.

        :param archive_path: absolute path where the archive file will be created

        :raises ArchiveError: when the archive file already exists
        """
        if os.path.exists(archive_path):
            msg = "archive %s already exists; remove it before creating a new one"
            raise ArchiveError(cause=msg % (archive_path))

        conn = sqlite3.connect(archive_path)

        cursor = conn.cursor()
        cursor.execute(cls.METADATA_CREATE_STMT)
        cursor.execute(cls.ARCHIVE_CREATE_STMT)
        conn.commit()

        cursor.close()
        conn.close()

        logger.debug("Creating archive %s", archive_path)
        archive = cls(archive_path)
        logger.debug("Achive %s was created", archive_path)

        return archive

    @staticmethod
    def make_hashcode(uri, payload, headers):
        """Generate a SHA1 based on the given arguments.

        Hashcodes created by this method will used as unique identifiers
        for the raw items or resources stored by this archive.

        :param uri: URI to the resource
        :param payload: payload of the request needed to fetch the resource
        :param headers: headers of the request needed to fetch the resource

        :returns: a SHA1 hash code
        """
        def dict_to_json_str(data):
            return json.dumps(data, sort_keys=True)

        content = ':'.join([uri, dict_to_json_str(payload), dict_to_json_str(headers)])
        hashcode = hashlib.sha1(content.encode('utf-8'))
        return hashcode.hexdigest()

    def _verify_archive(self):
        """Check whether the archive is valid or not.

        This method will check if tables were created and if they
        contain valid data.
        """
        nentries = self._count_table_rows(self.ARCHIVE_TABLE)
        nmetadata = self._count_table_rows(self.METADATA_TABLE)

        if nmetadata > 1:
            msg = "archive %s metadata corrupted; multiple metadata entries" % (self.archive_path)
            raise ArchiveError(cause=msg)
        if nmetadata == 0 and nentries > 0:
            msg = "archive %s metadata is empty but %s entries were achived" % (self.archive_path)
            raise ArchiveError(cause=msg)

        logger.debug("Integrity of archive %s OK; entries: %s rows, metadata: %s rows",
                     self.archive_path, nentries, nmetadata)

    def _load_metadata(self):
        """Load metadata from the archive file"""

        logger.debug("Loading metadata infomation of archive %s", self.archive_path)

        cursor = self._db.cursor()
        select_stmt = "SELECT origin, backend_name, backend_version, " \
                      "category, backend_params, created_on " \
                      "FROM " + self.METADATA_TABLE + " " \
                      "LIMIT 1"
        cursor.execute(select_stmt)
        row = cursor.fetchone()
        cursor.close()

        if row:
            self.origin = row[0]
            self.backend_name = row[1]
            self.backend_version = row[2]
            self.category = row[3]
            self.backend_params = pickle.loads(row[4])
            self.created_on = str_to_datetime(row[5])
        else:
            logger.debug("Metadata of archive %s was empty", self.archive_path)

        logger.debug("Metadata of archive %s loaded", self.archive_path)

    def _count_table_rows(self, table_name):
        """Fetch the number of rows in a table"""

        cursor = self._db.cursor()
        select_stmt = "SELECT COUNT(*) FROM " + table_name

        try:
            cursor.execute(select_stmt)
            row = cursor.fetchone()
        except sqlite3.DatabaseError as e:
            msg = "invalid archive file; cause: %s" % str(e)
            raise ArchiveError(cause=msg)
        finally:
            cursor.close()

        return row[0]

class ArchiveManager:
    """Manager for handling archives in Perceval.

    This class manages the creation, deletion and access of `Archive`
    objects. Archives are stored under `dirpath` directory, using
    a random SHA1 for each file. The first byte of the hashcode will
    be the name of the subdirectory; the remaining bytes, the archive
    name.

    :param: dirpath: path where the archives are stored
    """

    STORAGE_EXT = '.sqlite3'

    def __init__(self, dirpath):
        self.dirpath = dirpath

        if not os.path.exists(self.dirpath):
            os.makedirs(self.dirpath)

    def create_archive(self):
        """Create a new archive.

        The method creates in the filesystem a brand new archive with
        a random SHA1 as its name. The first byte of the hashcode will
        be the name of the subdirectory; the remaining bytes, the
        archive name.

        :returns: a new `Archive` object

        :raises ArchiveManagerError: when an error occurs creating the
            new archive
        """
        hashcode = uuid.uuid4().hex
        archive_dir = os.path.join(self.dirpath, hashcode[0:2])
        archive_name = hashcode[2:] + self.STORAGE_EXT
        archive_path = os.path.join(archive_dir, archive_name)

        if not os.path.exists(archive_dir):
            os.makedirs(archive_dir)

        try:
            archive = Archive.create(archive_path)
        except ArchiveError as e:
            raise ArchiveManagerError(cause=str(e))

        return archive

    def remove_archive(self, archive_path):
        """Remove an archive.

        This method deletes from the filesystem the archive stored
        in `archive_path`.

        :param archive_path: path to the archive

        :raises ArchiveManangerError: when an error occurs removing the
            archive
        """
        try:
            Archive(archive_path)
        except ArchiveError as e:
            raise ArchiveManagerError(cause=str(e))

        os.remove(archive_path)

    def search(self, origin, backend_name, category, archived_after):
        """Search archives.

        Get the archives which store data based on the given parameters.
        These parameters define which the origin was (`origin`), how data
        was fetched (`backend_name`) and data type ('category').
        Only those archives created on or after `archived_after` will be
        returned.

        The method returns a list with the file paths to those archives.
        The list is sorted by the date of creation of each archive.

        :param origin: data origin
        :param backend_name: backed used to fetch data
        :param category: type of the items fetched by the backend
        :param archived_after: get archives created on or after this date

        :returns: a list with archive names which match the search criteria
        """
        archives = self._search_archives(origin, backend_name,
                                         category, archived_after)
        archives = [(fp, date) for fp, date in archives]
        archives = [fp for fp, _ in sorted(archives, key=lambda x: x[1])]

        return archives

    def _search_archives(self, origin, backend_name, category, archived_after):
        """Search archives using filters."""

        for archive_path in self._search_files():
            try:
                archive = Archive(archive_path)
            except ArchiveError:
                continue

            match = archive.origin == origin and \
                archive.backend_name == backend_name and \
                archive.category == category and \
                archive.created_on >= archived_after

            if not match:
                continue

            yield archive_path, archive.created_on

    def _search_files(self):
        """Retrieve the file paths stored under the base path."""

        for root, _, files in os.walk(self.dirpath):
            for filename in files:
                location = os.path.join(root, filename)
                yield location


# -*- coding: utf-8 -*-
#
# Copyright (C) 2015-2019 Bitergia
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA.
#
# Authors:
#     Santiago Dueñas <sduenas@bitergia.com>
#     Germán Poo-Caamaño <gpoo@gnome.org>
#

import datetime
import email
import logging
import mailbox
import re
import sys

import xml.etree.ElementTree

import dateutil.rrule
import dateutil.tz

import requests

from .errors import ParseError

logger = logging.getLogger(__name__)

DEFAULT_DATETIME = datetime.datetime(1970, 1, 1, 0, 0, 0,
                                     tzinfo=dateutil.tz.tzutc())
DEFAULT_LAST_DATETIME = datetime.datetime(2100, 1, 1, 0, 0, 0,
                                          tzinfo=dateutil.tz.tzutc())

def check_compressed_file_type(filepath):
    """Check if filename is a compressed file supported by the tool.

    This function uses magic numbers (first four bytes) to determine
    the type of the file. Supported types are 'gz' and 'bz2'. When
    the filetype is not supported, the function returns `None`.

    :param filepath: path to the file

    :returns: 'gz' or 'bz2'; `None` if the type is not supported
    """
    def compressed_file_type(content):
        magic_dict = {
            b'\x1f\x8b\x08': 'gz',
            b'\x42\x5a\x68': 'bz2',
            b'PK\x03\x04': 'zip'
        }

        for magic, filetype in magic_dict.items():
            if content.startswith(magic):
                return filetype

        return None

    with open(filepath, mode='rb') as f:
        magic_number = f.read(4)
    return compressed_file_type(magic_number)

def months_range(from_date, to_date):
    """Generate a months range.

    Generator of months starting on `from_date` util `to_date`. Each
    returned item is a tuple of two datatime objects like in (month, month+1).
    Thus, the result will follow the sequence:
        ((fd, fd+1), (fd+1, fd+2), ..., (td-2, td-1), (td-1, td))

    :param from_date: generate dates starting on this month
    :param to_date: generate dates until this month

    :result: a generator of months range
    """
    start = datetime.datetime(from_date.year, from_date.month, 1)
    end = datetime.datetime(to_date.year, to_date.month, 1)

    month_gen = dateutil.rrule.rrule(freq=dateutil.rrule.MONTHLY,
                                     dtstart=start, until=end)
    months = [d for d in month_gen]

    pos = 0
    for x in range(1, len(months)):
        yield months[pos], months[x]
        pos = x

def message_to_dict(msg):
    """Convert an email message into a dictionary.

    This function transforms an `email.message.Message` object
    into a dictionary. Headers are stored as key:value pairs
    while the body of the message is stored inside `body` key.
    Body may have two other keys inside, 'plain', for plain body
    messages and 'html', for HTML encoded messages.

    The returned dictionary has the type `requests.structures.CaseInsensitiveDict`
    due to same headers with different case formats can appear in
    the same message.

    :param msg: email message of type `email.message.Message`

    :returns : dictionary of type `requests.structures.CaseInsensitiveDict`

    :raises ParseError: when an error occurs transforming the message
        to a dictionary
    """
    def parse_headers(msg):
        headers = {}

        for header, value in msg.items():
            hv = []

            for text, charset in email.header.decode_header(value):
                if type(text) == bytes:
                    charset = charset if charset else 'utf-8'
                    try:
                        text = text.decode(charset, errors='surrogateescape')
                    except (UnicodeError, LookupError):
                        # Try again with a 7bit encoding
                        text = text.decode('ascii', errors='surrogateescape')
                hv.append(text)

            v = ' '.join(hv)
            headers[header] = v if v else None

        return headers

    def parse_payload(msg):
        body = {}

        if not msg.is_multipart():
            payload = decode_payload(msg)
            subtype = msg.get_content_subtype()
            body[subtype] = [payload]
        else:
            # Include all the attached texts if it is multipart
            # Ignores binary parts by default
            for part in email.iterators.typed_subpart_iterator(msg):
                payload = decode_payload(part)
                subtype = part.get_content_subtype()
                body.setdefault(subtype, []).append(payload)

        return {k: '\n'.join(v) for k, v in body.items()}

    def decode_payload(msg_or_part):
        charset = msg_or_part.get_content_charset('utf-8')
        payload = msg_or_part.get_payload(decode=True)

        try:
            payload = payload.decode(charset, errors='surrogateescape')
        except (UnicodeError, LookupError):
            # Try again with a 7bit encoding
            payload = payload.decode('ascii', errors='surrogateescape')
        return payload

    # The function starts here
    message = requests.structures.CaseInsensitiveDict()

    if isinstance(msg, mailbox.mboxMessage):
        message['unixfrom'] = msg.get_from()
    else:
        message['unixfrom'] = None

    try:
        for k, v in parse_headers(msg).items():
            message[k] = v
        message['body'] = parse_payload(msg)
    except UnicodeError as e:
        raise ParseError(cause=str(e))

    return message

def remove_invalid_xml_chars(raw_xml):
    """Remove control and invalid characters from an xml stream.

    Looks for invalid characters and subtitutes them with whitespaces.
    This solution is based on these two posts: Olemis Lang's reponse
    on StackOverflow (http://stackoverflow.com/questions/1707890) and
    lawlesst's on GitHub Gist (https://gist.github.com/lawlesst/4110923),
    that is based on the previous answer.

    :param xml: XML stream

    :returns: a purged XML stream
    """
    illegal_unichrs = [(0x00, 0x08), (0x0B, 0x1F),
                       (0x7F, 0x84), (0x86, 0x9F)]

    illegal_ranges = ['%s-%s' % (chr(low), chr(high))
                      for (low, high) in illegal_unichrs
                      if low < sys.maxunicode]

    illegal_xml_re = re.compile('[%s]' % ''.join(illegal_ranges))

    purged_xml = ''

    for c in raw_xml:
        if illegal_xml_re.search(c) is not None:
            c = ' '
        purged_xml += c

    return purged_xml

def xml_to_dict(raw_xml):
    """Convert a XML stream into a dictionary.

    This function transforms a xml stream into a dictionary. The
    attributes are stored as single elements while child nodes are
    stored into lists. The text node is stored using the special
    key '__text__'.

    This code is based on Winston Ewert's solution to this problem.
    See http://codereview.stackexchange.com/questions/10400/convert-elementtree-to-dict
    for more info. The code was licensed as cc by-sa 3.0.

    :param raw_xml: XML stream

    :returns: a dict with the XML data

    :raises ParseError: raised when an error occurs parsing the given
        XML stream
    """
    def node_to_dict(node):
        d = {}
        d.update(node.items())

        text = getattr(node, 'text', None)

        if text is not None:
            d['__text__'] = text

        childs = {}
        for child in node:
            childs.setdefault(child.tag, []).append(node_to_dict(child))

        d.update(childs.items())

        return d

    purged_xml = remove_invalid_xml_chars(raw_xml)

    try:
        tree = xml.etree.ElementTree.fromstring(purged_xml)
    except xml.etree.ElementTree.ParseError as e:
        cause = "XML stream %s" % (str(e))
        raise ParseError(cause=cause)

    d = node_to_dict(tree)

    return d


# -*- coding: utf-8 -*-
#
# Copyright (C) 2015-2019 Bitergia
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA.
#
# Authors:
#     Santiago Dueñas <sduenas@bitergia.com>
#

import json
import logging

import requests

from grimoirelab_toolkit.datetime import datetime_to_utc, str_to_datetime
from grimoirelab_toolkit.uris import urijoin

from ...backend import (Backend,
                        BackendCommand,
                        BackendCommandArgumentParser)
from ...client import HttpClient
from ...utils import DEFAULT_DATETIME

CATEGORY_ISSUE = "issue"

MAX_ISSUES = 100  # Maximum number of issues per query
USER_FIELDS = ['assigned_to', 'author']

logger = logging.getLogger(__name__)

class Redmine(Backend):
    """Redmine backend.

    This class allows to fetch the issues stored on a Redmine
    server. Initialize this class passing the URL of this server.
    Some servers require authentication to get access to some
    data, if this is the case, pass the API token to `api_token`
    parameter.

    :param url: URL of the server
    :param api_token: token needed to use the API
    :param max_issues:  maximum number of issues requested on the same query
    :param tag: label used to mark the data
    :param archive: archive to store/retrieve items
    """
    version = '0.9.4'

    CATEGORIES = [CATEGORY_ISSUE]

    def __init__(self, url, api_token=None, max_issues=MAX_ISSUES,
                 tag=None, archive=None):
        origin = url

        super().__init__(origin, tag=tag, archive=archive)
        self.url = url
        self.api_token = api_token
        self.max_issues = max_issues
        self.client = None

        self._users = {}

    def fetch(self, category=CATEGORY_ISSUE, from_date=DEFAULT_DATETIME):
        """Fetch the issues from the server.

        This method fetches the issues stored on the server that were
        updated since the given date. Data about attachments, journals
        and watchers (among others) are included within each issue.

        :param category: the category of items to fetch
        :param from_date: obtain issues updated since this date

        :returns: a generator of issues
        """
        if not from_date:
            from_date = DEFAULT_DATETIME

        from_date = datetime_to_utc(from_date)
        kwargs = {'from_date': from_date}
        items = super().fetch(category, **kwargs)

        return items

    def fetch_items(self, category, **kwargs):
        """Fetch the issues

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        from_date = kwargs['from_date']

        logger.info("Fetching issues of '%s' from %s",
                    self.url, str(from_date))

        nissues = 0

        for issue_id in self.__fetch_issues_ids(from_date):
            issue = self.__fetch_and_parse_issue(issue_id)

            for key in USER_FIELDS:
                if key not in issue:
                    continue

                user = self.__get_or_fetch_user(issue[key]['id'])
                issue[key + '_data'] = user

            for journal in issue['journals']:
                if 'user' not in journal:
                    continue

                user = self.__get_or_fetch_user(journal['user']['id'])
                journal['user_data'] = user

            yield issue
            nissues += 1

        logger.info("Fetch process completed: %s issues fetched", nissues)

    @classmethod
    def has_archiving(cls):
        """Returns whether it supports archiving items on the fetch process.

        :returns: this backend supports items archive
        """
        return True

    @classmethod
    def has_resuming(cls):
        """Returns whether it supports to resume the fetch process.

        :returns: this backend supports items resuming
        """
        return True

    @staticmethod
    def metadata_id(item):
        """Extracts the identifier from a Redmine item."""

        return str(item['id'])

    @staticmethod
    def metadata_updated_on(item):
        """Extracts and coverts the update time from a Redmine item.

        The timestamp is extracted from 'updated_on' field and converted
        to a UNIX timestamp.

        :param item: item generated by the backend

        :returns: a UNIX timestamp
        """
        ts = item['updated_on']
        ts = str_to_datetime(ts)

        return ts.timestamp()

    @staticmethod
    def metadata_category(item):
        """Extracts the category from a Redmine item.

        This backend only generates one type of item which is
        'issue'.
        """
        return CATEGORY_ISSUE

    @staticmethod
    def parse_issues(raw_json):
        """Parse a Redmine issues JSON stream.

        The method parses a JSON stream and returns a list iterator.
        Each item is a dictionary that contains the issue parsed data.

        :param raw_json: JSON string to parse

        :returns: a generator of parsed issues
        """
        results = json.loads(raw_json)

        issues = results['issues']
        for issue in issues:
            yield issue

    @staticmethod
    def parse_issue_data(raw_json):
        """Parse a Redmine issue JSON stream.

        The method parses a JSON stream and returns a dictionary
        with the parsed data for the given issue.

        :param raw_json: JSON string to parse

        :returns: a dictionary with the parsed issue data
        """
        result = json.loads(raw_json)
        return result['issue']

    @staticmethod
    def parse_user_data(raw_json):
        """Parse a Redmine user JSON stream.

        The method parses a JSON stream and returns a dictionary
        with the parsed data for the given user.

        :param raw_json: JSON string to parse

        :returns: a dictionary with the parsed user data
        """
        result = json.loads(raw_json)
        return result['user']

    def _init_client(self, from_archive=False):
        """Init client"""

        return RedmineClient(self.url, self.api_token, self.archive, from_archive)

    def __fetch_issues_ids(self, from_date):
        offset = 0
        issues = self.__fetch_and_parse_issues_page(from_date, offset,
                                                    self.max_issues)

        while issues:
            issue = issues.pop(0)
            issue_id = issue['id']
            yield issue_id

            if not issues:
                offset += self.max_issues
                issues = self.__fetch_and_parse_issues_page(from_date, offset,
                                                            self.max_issues)

    def __get_or_fetch_user(self, user_id):
        if user_id in self._users:
            return self._users[user_id]

        logger.debug("User %s not found on client cache; fetching it", user_id)

        try:
            user = self.__fetch_and_parse_user(user_id)
        except requests.exceptions.HTTPError as e:
            if e.response.status_code == 404:
                logger.warning("User %s not found on the server; skipping it",
                               user_id)
                user = {}
            else:
                raise e

        self._users[user_id] = user

        return user

    def __fetch_and_parse_issues_page(self, from_date, offset, max_issues):
        logger.debug("Fetching and parsing issues page from %s (offset: %s)",
                     str(from_date), str(offset))
        raw_json = self.client.issues(from_date=from_date, offset=offset,
                                      max_issues=max_issues)
        issues = self.parse_issues(raw_json)
        return [issue for issue in issues]

    def __fetch_and_parse_issue(self, issue_id):
        logger.debug("Fetching and parsing issue #%s", issue_id)
        raw_issue = self.client.issue(issue_id)
        return self.parse_issue_data(raw_issue)

    def __fetch_and_parse_user(self, user_id):
        logger.debug("Fetching and parsing user #%s", user_id)
        raw_user = self.client.user(user_id)
        return self.parse_user_data(raw_user)

class RedmineCommand(BackendCommand):
    """Class to run Redmine backend from the command line."""

    BACKEND = Redmine

    @classmethod
    def setup_cmd_parser(cls):
        """Returns the Redmine argument parser."""

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              from_date=True,
                                              token_auth=True,
                                              archive=True)

        # Redmine options
        group = parser.parser.add_argument_group('Redmine arguments')
        group.add_argument('--max-issues', dest='max_issues',
                           type=int, default=MAX_ISSUES,
                           help="Maximum number of issues requested on the same query")

        # Required arguments
        parser.parser.add_argument('url',
                                   help="URL of the Redmine server")

        return parser

class RedmineClient(HttpClient):
    """Redmine API client.

    This class implements a client that retrieves issues from
    a Redmine server. Remine servers provides a REST API that
    returns its results in JSON format.

    :param base_url: URL of the Phabricator server
    :param api_token: token to get access to restricted data
        stored in the server
    :param archive: an archive to store/read fetched data
    :param from_archive: it tells whether to write/read the archive
    """
    URL = '%(base)s/%(resource)s'

    RISSUES = 'issues'
    RUSERS = 'users'

    PINCLUDE = 'include'
    PKEY = 'key'
    PLIMIT = 'limit'
    POFFSET = 'offset'
    PSORT = 'sort'
    PSTATUS_ID = 'status_id'
    PUPDATED_ON = 'updated_on'

    CJSON = '.json'
    CATTACHMENTS = 'attachments'
    CCHANGESETS = 'changesets'
    CCHILDREN = 'children'
    CJOURNALS = 'journals'
    CRELATIONS = 'relations'
    CWATCHERS = 'watchers'

    def __init__(self, base_url, api_token=None, archive=None, from_archive=False):
        super().__init__(base_url.rstrip('/'), archive=archive, from_archive=from_archive)
        self.api_token = api_token

    def issues(self, from_date=DEFAULT_DATETIME,
               offset=None, max_issues=MAX_ISSUES):
        """Get the information of a list of issues.

        :param from_date: retrieve issues that where updated from that date;
            dates are converted to UTC
        :param offset: starting position for the search
        :param max_issues: maximum number of issues to reteurn per query
        """
        resource = self.RISSUES + self.CJSON

        ts = datetime_to_utc(from_date)
        ts = ts.strftime("%Y-%m-%dT%H:%M:%SZ")

        # By default, Redmine returns open issues only.
        # Parameter 'status_id' is set to get all the statuses.
        params = {
            self.PSTATUS_ID: '*',
            self.PSORT: self.PUPDATED_ON,
            self.PUPDATED_ON: '>=' + ts,
            self.PLIMIT: max_issues
        }

        if offset is not None:
            params[self.POFFSET] = offset

        response = self._call(resource, params)

        return response

    def issue(self, issue_id):
        """Get the information of the given issue.

        :param issue_id: issue identifier
        """
        resource = urijoin(self.RISSUES, str(issue_id) + self.CJSON)

        params = {
            self.PINCLUDE: ','.join([self.CATTACHMENTS, self.CCHANGESETS,
                                     self.CCHILDREN, self.CJOURNALS,
                                     self.CRELATIONS, self.CWATCHERS])
        }

        response = self._call(resource, params)

        return response

    def user(self, user_id):
        """Get the information of the given user.

        :param user_id: user identifier
        """
        resource = urijoin(self.RUSERS, str(user_id) + self.CJSON)

        params = {}

        response = self._call(resource, params)

        return response

    @staticmethod
    def sanitize_for_archive(url, headers, payload):
        """Sanitize payload of a HTTP request by removing the token information
        before storing/retrieving archived items

        :param: url: HTTP url request
        :param: headers: HTTP headers request
        :param: payload: HTTP payload request

        :returns url, headers and the sanitized payload
        """
        if RedmineClient.PKEY in payload:
            payload.pop(RedmineClient.PKEY)

        return url, headers, payload

    def _call(self, resource, params):
        """Call to get a resource.

        :param method: resource to get
        :param params: dict with the HTTP parameters needed to get
            the given resource
        """
        url = self.URL % {'base': self.base_url, 'resource': resource}

        if self.api_token:
            params[self.PKEY] = self.api_token

        logger.debug("Redmine client requests: %s params: %s",
                     resource, str(params))

        r = self.fetch(url, payload=params, verify=False)

        return r.text


# -*- coding: utf-8 -*-
#
# Copyright (C) 2015-2019 Bitergia
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA.
#
# Authors:
#     Santiago Dueñas <sduenas@bitergia.com>
#

import datetime
import logging
import os

import dateutil.parser
import dateutil.relativedelta
import dateutil.tz

from grimoirelab_toolkit.datetime import datetime_to_utc, datetime_utcnow
from grimoirelab_toolkit.uris import urijoin

from .mbox import MBox, MailingList, CATEGORY_MESSAGE
from ...backend import (BackendCommand,
                        BackendCommandArgumentParser)
from ...client import HttpClient
from ...utils import (DEFAULT_DATETIME,
                      months_range)

logger = logging.getLogger(__name__)

class HyperKitty(MBox):
    """HyperKitty backend.

    This class allows the fetch the email messages stored on a HyperKitty
    archiver. Initialize this class passing the URL where the mailing list
    archiver is and the directory path where the mbox files will be fetched
    and stored. The origin of the data will be set to the value of `url`.

    :param url: URL to the HyperKitty mailing list archiver
    :param dirpath: directory path where the mboxes are stored
    :param tag: label used to mark the data
    :param archive: archive to store/retrieve items
    """
    version = '0.4.2'

    CATEGORIES = [CATEGORY_MESSAGE]

    def __init__(self, url, dirpath, tag=None, archive=None):
        super().__init__(url, dirpath, tag=tag, archive=archive)
        self.url = url

    def fetch(self, category=CATEGORY_MESSAGE, from_date=DEFAULT_DATETIME):
        """Fetch the messages from the HyperKitty mailing list archiver.

        The method fetches the mbox files from a remote HyperKitty
        mailing list archiver and retrieves the messages stored on them.

        Take into account that HyperKitty does not provide yet any kind
        of info to know which is the first message on the mailing list.
        For this reason, using a value in `from_date` previous to the
        date where the first message was sent will make to download
        empty mbox files.

        :param category: the category of items to fetch
        :param from_date: obtain messages since this date

        :returns: a generator of messages
        """
        items = super().fetch(category, from_date)

        return items

    def fetch_items(self, category, **kwargs):
        """Fetch the messages

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        from_date = kwargs['from_date']

        logger.info("Looking for messages from '%s' since %s",
                    self.url, str(from_date))

        mailing_list = HyperKittyList(self.url, self.dirpath)
        mailing_list.fetch(from_date=from_date)

        messages = self._fetch_and_parse_messages(mailing_list, from_date)

        for message in messages:
            yield message

        logger.info("Fetch process completed")

    @classmethod
    def has_archiving(cls):
        """Returns whether it supports archiving items on the fetch process.

        :returns: this backend does not support items archive
        """
        return False

    @classmethod
    def has_resuming(cls):
        """Returns whether it supports to resume the fetch process.

        :returns: this backend supports items resuming
        """
        return True

class HyperKittyList(MailingList):
    """Manage mailing list archives stored by HyperKitty archiver.

    This class gives access to remote and local mboxes archives
    from a mailing list stored by HyperKitty. This class also allows
    to keep them in sync.

    Notice that this class only works with HyperKitty version 1.0.4
    or greater. Previous versions do not export messages in MBox
    format.

    :param url: URL to the HyperKitty archiver for this list
    :param dirpath: path to the local mboxes archives
    """
    def __init__(self, url, dirpath):
        super().__init__(url, dirpath)
        self.client = HttpClient(url)

    def fetch(self, from_date=DEFAULT_DATETIME):
        """Fetch the mbox files from the remote archiver.

        This method stores the archives in the path given during the
        initialization of this object.

        HyperKitty archives are accessed month by month and stored following
        the schema year-month. Archives are fetched from the given month
        till the current month.

        :param from_date: fetch archives that store messages
            equal or after the given date; only year and month values
            are compared

        :returns: a list of tuples, storing the links and paths of the
            fetched archives
        """
        logger.info("Downloading mboxes from '%s' to since %s",
                    self.client.base_url, str(from_date))
        logger.debug("Storing mboxes in '%s'", self.dirpath)

        self.client.fetch(self.client.base_url)

        from_date = datetime_to_utc(from_date)
        to_end = datetime_utcnow()
        to_end += dateutil.relativedelta.relativedelta(months=1)

        months = months_range(from_date, to_end)

        fetched = []

        if not os.path.exists(self.dirpath):
            os.makedirs(self.dirpath)

        tmbox = 0

        for dts in months:
            tmbox += 1
            start, end = dts[0], dts[1]
            filename = start.strftime("%Y-%m.mbox.gz")
            filepath = os.path.join(self.dirpath, filename)

            url = urijoin(self.client.base_url, 'export', filename)

            params = {
                'start': start.strftime("%Y-%m-%d"),
                'end': end.strftime("%Y-%m-%d")
            }

            success = self._download_archive(url, params, filepath)

            if success:
                fetched.append((url, filepath))

        logger.info("%s/%s MBoxes downloaded", len(fetched), tmbox)

        return fetched

    @property
    def mboxes(self):
        """Get the mboxes managed by this mailing list.

        Returns the archives sorted by date in ascending order.

        :returns: a list of `.MBoxArchive` objects
        """
        archives = []

        for mbox in super().mboxes:
            dt = self._parse_date_from_filepath(mbox.filepath)
            archives.append((dt, mbox))

        archives.sort(key=lambda x: x[0])

        return [a[1] for a in archives]

    def _parse_date_from_filepath(self, filepath):
        default_dt = datetime.datetime(2100, 1, 1,
                                       tzinfo=dateutil.tz.tzutc())

        try:
            name = os.path.basename(filepath)
            dt = dateutil.parser.parse(name, default=default_dt,
                                       fuzzy=True)
        except (AttributeError, TypeError, ValueError) as e:
            dt = default_dt
            logger.debug("Date of file %s not detected due to %s",
                         filepath, str(e))
            logger.debug("Date set to default: %s", str(dt))

        return dt

    def _download_archive(self, url, params, filepath):
        r = self.client.fetch(url, payload=params, stream=True)

        try:
            with open(filepath, 'wb') as fd:
                fd.write(r.raw.read())
        except OSError as e:
            logger.warning("Ignoring %s archive due to: %s", url, str(e))
            return False

        logger.debug("%s archive downloaded and stored in %s", url, filepath)

        return True

class HyperKittyCommand(BackendCommand):
    """Class to run HyperKitty backend from the command line."""

    BACKEND = HyperKitty

    def _pre_init(self):
        """Initialize mailing lists directory path"""

        if not self.parsed_args.mboxes_path:
            base_path = os.path.expanduser('~/.perceval/mailinglists/')
            dirpath = os.path.join(base_path, self.parsed_args.url)
        else:
            dirpath = self.parsed_args.mboxes_path

        setattr(self.parsed_args, 'dirpath', dirpath)

    @classmethod
    def setup_cmd_parser(cls):
        """Returns the HyperKitty argument parser."""

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              from_date=True)

        # Optional arguments
        group = parser.parser.add_argument_group('HyperKitty arguments')
        group.add_argument('--mboxes-path', dest='mboxes_path',
                           help="Path where mbox files will be stored")

        # Required arguments
        parser.parser.add_argument('url',
                                   help="URL of the mailing list archiver")

        return parser


# -*- coding: utf-8 -*-
#
# Copyright (C) 2015-2019 Bitergia
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA.
#
# Authors:
#     Santiago Dueñas <sduenas@bitergia.com>
#

import json
import logging

from grimoirelab_toolkit.datetime import datetime_utcnow
from grimoirelab_toolkit.uris import urijoin

from ...backend import (Backend,
                        BackendCommand,
                        BackendCommandArgumentParser)
from ...client import HttpClient

CATEGORY_DOCKERHUB_DATA = "dockerhub-data"

DOCKERHUB_URL = "https://hub.docker.com/"
DOCKERHUB_API_URL = urijoin(DOCKERHUB_URL, 'v2')

DOCKER_OWNER = 'library'
DOCKER_SHORTCUT_OWNER = '_'

logger = logging.getLogger(__name__)

class DockerHub(Backend):
    """DockerHub backend for Perceval.

    This class retrieves data from a repository stored
    in the Docker Hub site. To initialize this class owner
    and repositories where data will be fetched must be provided.
    The origin of the data will be built with both parameters.

    Shortcut `_` owner for official Docker repositories will
    be replaced by its long name: `library`.

    :param owner: DockerHub owner
    :param repository: DockerHub repository owned by `owner`
    :param tag: label used to mark the data
    :param archive: archive to store/retrieve items
    """
    version = '0.4.2'

    CATEGORIES = [CATEGORY_DOCKERHUB_DATA]

    def __init__(self, owner, repository, tag=None, archive=None):
        if owner == DOCKER_SHORTCUT_OWNER:
            owner = DOCKER_OWNER

        origin = urijoin(DOCKERHUB_URL, owner, repository)

        super().__init__(origin, tag=tag, archive=archive)
        self.owner = owner
        self.repository = repository
        self.client = None

    def fetch(self, category=CATEGORY_DOCKERHUB_DATA):
        """Fetch data from a Docker Hub repository.

        The method retrieves, from a repository stored in Docker Hub,
        its data which includes number of pulls, stars, description,
        among other data.

        :param category: the category of items to fetch

        :returns: a generator of data
        """
        kwargs = {}
        items = super().fetch(category, **kwargs)

        return items

    def fetch_items(self, category, **kwargs):
        """Fetch the Dockher Hub items

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        logger.info("Fetching data from '%s' repository of '%s' owner",
                    self.repository, self.owner)

        raw_data = self.client.repository(self.owner, self.repository)
        fetched_on = datetime_utcnow().timestamp()

        data = self.parse_json(raw_data)
        data['fetched_on'] = fetched_on
        yield data

        logger.info("Fetch process completed")

    @classmethod
    def has_archiving(cls):
        """Returns whether it supports archiving items on the fetch process.

        :returns: this backend supports items archive
        """
        return True

    @classmethod
    def has_resuming(cls):
        """Returns whether it supports to resume the fetch process.

        :returns: this backend supports items resuming
        """
        return True

    @staticmethod
    def metadata_id(item):
        """Extracts the identifier from a Docker Hub item."""

        return str(item['fetched_on'])

    @staticmethod
    def metadata_updated_on(item):
        """Extracts and coverts the update time from a Docker Hub item.

        The timestamp is extracted from 'fetched_on' field. This field
        is not part of the data provided by Docker Hub. It is added
        by this backend.

        :param item: item generated by the backend

        :returns: a UNIX timestamp
        """
        return item['fetched_on']

    @staticmethod
    def metadata_category(item):
        """Extracts the category from a Docker Hub item.

        This backend only generates one type of item which is
        'dockerhub-data'.
        """
        return CATEGORY_DOCKERHUB_DATA

    @staticmethod
    def parse_json(raw_json):
        """Parse a Docker Hub JSON stream.

        The method parses a JSON stream and returns a
        dict with the parsed data.

        :param raw_json: JSON string to parse

        :returns: a dict with the parsed data
        """
        result = json.loads(raw_json)
        return result

    def _init_client(self, from_archive=False):
        """Init client"""

        return DockerHubClient(archive=self.archive, from_archive=from_archive)

class DockerHubClient(HttpClient):
    """DockerHub API client.

    Client for fetching information from the DockerHub server
    using its REST API v2.

    :param archive: an archive to store/read fetched data
    :param from_archive: it tells whether to write/read the archive
    """
    RREPOSITORY = 'repositories'

    def __init__(self, archive=None, from_archive=False):
        super().__init__(DOCKERHUB_API_URL, archive=archive, from_archive=from_archive)

    def repository(self, owner, repository):
        """Fetch information about a repository."""

        url = urijoin(self.base_url, self.RREPOSITORY, owner, repository)

        logger.debug("DockerHub client requests: %s", url)

        response = self.fetch(url)

        return response.text

class DockerHubCommand(BackendCommand):
    """Class to run DockerHub backend from the command line."""

    BACKEND = DockerHub

    @classmethod
    def setup_cmd_parser(cls):
        """Returns the DockerHub argument parser."""

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              archive=True)

        # Required arguments
        parser.parser.add_argument('owner',
                                   help="Docker Hub owner")
        parser.parser.add_argument('repository',
                                   help="Docker Hub repository")

        return parser