def next_retrieve_group_item(self, last_item=None, entry=None):
        """Return the item to start from in next reviews group."""

        next_item = None

        gerrit_version = self.version

        if gerrit_version[0] == 2 and gerrit_version[1] > 9:
            if last_item is None:
                next_item = 0
            else:
                next_item = last_item
        elif gerrit_version[0] == 2 and gerrit_version[1] == 9:
            # https://groups.google.com/forum/#!topic/repo-discuss/yQgRR5hlS3E
            cause = "Gerrit 2.9.0 does not support pagination"
            raise BackendError(cause=cause)
        else:
            if entry is not None:
                next_item = entry['sortKey']

        return next_item

Return the item to start from in next reviews group .


def __execute(self, cmd):
        """Execute gerrit command"""

        if self.from_archive:
            response = self.__execute_from_archive(cmd)
        else:
            response = self.__execute_from_remote(cmd)

        return response

def __execute ( self , cmd ) : if self . from_archive : response = self . __execute_from_archive ( cmd ) else : response = self . __execute_from_remote ( cmd ) return response


def __execute_from_archive(self, cmd):
        """Execute gerrit command against the archive"""

        cmd = self.sanitize_for_archive(cmd)
        response = self.archive.retrieve(cmd, None, None)

        if isinstance(response, RuntimeError):
            raise response

        return response

def __execute_from_archive ( self , cmd ) : cmd = self . sanitize_for_archive ( cmd ) response = self . archive . retrieve ( cmd , None , None ) if isinstance ( response , RuntimeError ) : raise response return response


def __execute_from_remote(self, cmd):
        """Execute gerrit command with retry if it fails"""

        result = None  # data result from the cmd execution
        retries = 0

        while retries < self.MAX_RETRIES:
            try:
                result = subprocess.check_output(cmd, shell=True)
                break
            except subprocess.CalledProcessError as ex:
                logger.error("gerrit cmd %s failed: %s", cmd, ex)
                time.sleep(self.RETRY_WAIT * retries)
                retries += 1

        if result is None:
            result = RuntimeError(cmd + " failed " + str(self.MAX_RETRIES) + " times. Giving up!")

        if self.archive:
            cmd = self.sanitize_for_archive(cmd)
            self.archive.store(cmd, None, None, result)

        if isinstance(result, RuntimeError):
            raise result

        return result

def
__execute_from_remote
(
self
,
cmd
)
:
result
=
# data result from the cmd execution
retries
=
0
while
retries
<
self
.
MAX_RETRIES
:
try
:
result
=
subprocess
.
check_output
(
cmd
,
shell
=
True
)
break
except
subprocess
.
CalledProcessError
as
ex
:
logger
.
error
(
"gerrit cmd %s failed: %s"
,
cmd
,
ex
)
time
.
sleep
(
self
.
RETRY_WAIT
*
retries
)
retries
+=
1
if
result
is
:
result
=
RuntimeError
(
cmd
+
" failed "
+
str
(
self
.
MAX_RETRIES
)
+
" times. Giving up!"
)
if
self
.
archive
:
cmd
=
self
.
sanitize_for_archive
(
cmd
)
self
.
archive
.
store
(
cmd
,
,
,
result
)
if
isinstance
(
result
,
RuntimeError
)
:
raise
result
return
result


def setup_cmd_parser(cls):
        """Returns the Gerrit argument parser."""

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              from_date=True,
                                              archive=True)

        # Gerrit options
        group = parser.parser.add_argument_group('Gerrit arguments')
        group.add_argument('--user', dest='user',
                           help="Gerrit ssh user")
        group.add_argument('--max-reviews', dest='max_reviews',
                           type=int, default=MAX_REVIEWS,
                           help="Max number of reviews per ssh query.")
        group.add_argument('--blacklist-reviews', dest='blacklist_reviews',
                           nargs='*',
                           help="Wrong reviews that must not be retrieved.")
        group.add_argument('--disable-host-key-check', dest='disable_host_key_check', action='store_true',
                           help="Don't check remote host identity")
        group.add_argument('--ssh-port', dest='port',
                           default=PORT, type=int,
                           help="Set SSH port of the Gerrit server")

        # Required arguments
        parser.parser.add_argument('hostname',
                                   help="Hostname of the Gerrit server")

        return parser


def __fetch_issue_data(self, issue_id):
        """Get data associated to an issue"""

        raw_issue = self.client.issue(issue_id)
        issue = json.loads(raw_issue)

        return issue

def __fetch_issue_data ( self , issue_id ) : raw_issue = self . client . issue ( issue_id ) issue = json . loads ( raw_issue ) return issue


def __fetch_issue_attachments(self, issue_id):
        """Get attachments of an issue"""

        for attachments_raw in self.client.issue_collection(issue_id, "attachments"):
            attachments = json.loads(attachments_raw)

            for attachment in attachments['entries']:
                yield attachment

def __fetch_issue_attachments ( self , issue_id ) : for attachments_raw in self . client . issue_collection ( issue_id , "attachments" ) : attachments = json . loads ( attachments_raw ) for attachment in attachments [ 'entries' ] : yield attachment


def __fetch_issue_messages(self, issue_id):
        """Get messages of an issue"""

        for messages_raw in self.client.issue_collection(issue_id, "messages"):
            messages = json.loads(messages_raw)

            for msg in messages['entries']:
                msg['owner_data'] = self.__fetch_user_data('{OWNER}', msg['owner_link'])
                yield msg

def __fetch_issue_messages ( self , issue_id ) : for messages_raw in self . client . issue_collection ( issue_id , "messages" ) : messages = json . loads ( messages_raw ) for msg in messages [ 'entries' ] : msg [ 'owner_data' ] = self . __fetch_user_data ( '{OWNER}' , msg [ 'owner_link' ] ) yield msg


def __fetch_issue_activities(self, issue_id):
        """Get activities on an issue"""

        for activities_raw in self.client.issue_collection(issue_id, "activity"):
            activities = json.loads(activities_raw)

            for act in activities['entries']:
                act['person_data'] = self.__fetch_user_data('{PERSON}', act['person_link'])
                yield act

def __fetch_issue_activities ( self , issue_id ) : for activities_raw in self . client . issue_collection ( issue_id , "activity" ) : activities = json . loads ( activities_raw ) for act in activities [ 'entries' ] : act [ 'person_data' ] = self . __fetch_user_data ( '{PERSON}' , act [ 'person_link' ] ) yield act


def __fetch_user_data(self, tag_type, user_link):
        """Get data associated to an user"""

        user_name = self.client.user_name(user_link)

        user = {}

        if not user_name:
            return user

        user_raw = self.client.user(user_name)
        user = json.loads(user_raw)

        return user

def __fetch_user_data ( self , tag_type , user_link ) : user_name = self . client . user_name ( user_link ) user = { } if not user_name : return user user_raw = self . client . user ( user_name ) user = json . loads ( user_raw ) return user


def user(self, user_name):
        """Get the user data by URL"""

        user = None

        if user_name in self._users:
            return self._users[user_name]

        url_user = self.__get_url("~" + user_name)

        logger.info("Getting info for %s" % (url_user))

        try:
            raw_user = self.__send_request(url_user)
            user = raw_user
        except requests.exceptions.HTTPError as e:
            if e.response.status_code in [404, 410]:
                logger.warning("Data is not available - %s", url_user)
                user = '{}'
            else:
                raise e

        self._users[user_name] = user

        return user

def
user
(
self
,
user_name
)
:
user
=
if
user_name
in
self
.
_users
:
return
self
.
_users
[
user_name
]
url_user
=
self
.
__get_url
(
"~"
+
user_name
)
logger
.
info
(
"Getting info for %s"
%
(
url_user
)
)
try
:
raw_user
=
self
.
__send_request
(
url_user
)
user
=
raw_user
except
requests
.
exceptions
.
HTTPError
as
e
:
if
e
.
response
.
status_code
in
[
404
,
410
]
:
logger
.
warning
(
"Data is not available - %s"
,
url_user
)
user
=
'{}'
else
:
raise
e
self
.
_users
[
user_name
]
=
user
return
user


def issue(self, issue_id):
        """Get the issue data by its ID"""

        path = urijoin("bugs", str(issue_id))
        url_issue = self.__get_url(path)
        raw_text = self.__send_request(url_issue)

        return raw_text

def issue ( self , issue_id ) : path = urijoin ( "bugs" , str ( issue_id ) ) url_issue = self . __get_url ( path ) raw_text = self . __send_request ( url_issue ) return raw_text


def issue_collection(self, issue_id, collection_name):
        """Get a collection list of a given issue"""

        path = urijoin("bugs", str(issue_id), collection_name)
        url_collection = self.__get_url(path)
        payload = {'ws.size': self.items_per_page, 'ws.start': 0, 'order_by': 'date_last_updated'}

        raw_items = self.__fetch_items(path=url_collection, payload=payload)

        return raw_items

def issue_collection ( self , issue_id , collection_name ) : path = urijoin ( "bugs" , str ( issue_id ) , collection_name ) url_collection = self . __get_url ( path ) payload = { 'ws.size' : self . items_per_page , 'ws.start' : 0 , 'order_by' : 'date_last_updated' } raw_items = self . __fetch_items ( path = url_collection , payload = payload ) return raw_items


def __get_url_project(self):
        """Build URL project"""

        if self.package:
            url = self.__get_url_distribution_package()
        else:
            url = self.__get_url_distribution()

        return url

def __get_url_project ( self ) : if self . package : url = self . __get_url_distribution_package ( ) else : url = self . __get_url_distribution ( ) return url


def __fetch_items(self, path, payload):
        """Return the items from Launchpad API using pagination"""

        page = 0  # current page
        url_next = path
        fetch_data = True

        while fetch_data:
            logger.debug("Fetching page: %i", page)

            try:
                raw_content = self.__send_request(url_next, payload)
                content = json.loads(raw_content)
            except requests.exceptions.HTTPError as e:
                if e.response.status_code in [410]:
                    logger.warning("Data is not available - %s", url_next)
                    raw_content = '{"total_size": 0, "start": 0, "entries": []}'
                    content = json.loads(raw_content)
                else:
                    raise e

            if 'next_collection_link' in content:
                url_next = content['next_collection_link']
                payload = None
            else:
                fetch_data = False

            yield raw_content
            page += 1


def subscriptions(self, per_page=PER_PAGE):
        """Fetch the groupsio paginated subscriptions for a given token

        :param per_page: number of subscriptions per page

        :returns: an iterator of subscriptions
        """
        url = urijoin(GROUPSIO_API_URL, self.GET_SUBSCRIPTIONS)
        logger.debug("Get groupsio paginated subscriptions from " + url)

        keep_fetching = True
        payload = {
            "limit": per_page
        }

        while keep_fetching:
            r = self.__fetch(url, payload)
            response_raw = r.json()
            subscriptions = response_raw['data']
            yield subscriptions

            total_subscriptions = response_raw['total_count']
            logger.debug("Subscriptions: %i/%i" % (response_raw['end_item'], total_subscriptions))

            payload['page_token'] = response_raw['next_page_token']
            keep_fetching = response_raw['has_more']


def __find_group_id(self):
        """Find the id of a group given its name by iterating on the list of subscriptions"""

        group_subscriptions = self.subscriptions(self.auth)

        for subscriptions in group_subscriptions:
            for sub in subscriptions:
                if sub['group_name'] == self.group_name:
                    return sub['group_id']

        msg = "Group id not found for group name %s" % self.group_name
        raise BackendError(cause=msg)

def
__find_group_id
(
self
)
:
group_subscriptions
=
self
.
subscriptions
(
self
.
auth
)
for
subscriptions
in
group_subscriptions
:
for
sub
in
subscriptions
:
if
sub
[
'group_name'
]
==
self
.
group_name
:
return
sub
[
'group_id'
]
msg
=
"Group id not found for group name %s"
%
self
.
group_name
raise
BackendError
(
cause
=
msg
)


def __fetch(self, url, payload):
        """Fetch requests from groupsio API"""

        r = requests.get(url, params=payload, auth=self.auth, verify=self.verify)
        try:
            r.raise_for_status()
        except requests.exceptions.HTTPError as e:
            raise e

        return r

def __fetch ( self , url , payload ) : r = requests . get ( url , params = payload , auth = self . auth , verify = self . verify ) try : r . raise_for_status ( ) except requests . exceptions . HTTPError as e : raise e return r


def setup_cmd_parser(cls):
        """Returns the Groupsio argument parser."""

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              from_date=True,
                                              token_auth=True)

        # Backend token is required
        action = parser.parser._option_string_actions['--api-token']
        action.required = True

        # Optional arguments
        group = parser.parser.add_argument_group('Groupsio arguments')
        group.add_argument('--mboxes-path', dest='mboxes_path',
                           help="Path where mbox files will be stored")
        group.add_argument('--no-verify', dest='verify',
                           action='store_false',
                           help="Value 'True' enable SSL verification")

        # Required arguments
        parser.parser.add_argument('group_name',
                                   help="Name of the group on Groups.io")

        return parser

def
setup_cmd_parser
(
cls
)
:
parser
=
BackendCommandArgumentParser
(
cls
.
BACKEND
.
CATEGORIES
,
from_date
=
True
,
token_auth
=
True
)
# Backend token is required
action
=
parser
.
parser
.
_option_string_actions
[
'--api-token'
]
action
.
required
=
True
# Optional arguments
group
=
parser
.
parser
.
add_argument_group
(
'Groupsio arguments'
)
group
.
add_argument
(
'--mboxes-path'
,
dest
=
'mboxes_path'
,
help
=
"Path where mbox files will be stored"
)
group
.
add_argument
(
'--no-verify'
,
dest
=
'verify'
,
action
=
'store_false'
,
help
=
"Value 'True' enable SSL verification"
)
# Required arguments
parser
.
parser
.
add_argument
(
'group_name'
,
help
=
"Name of the group on Groups.io"
)
return
parser


def uuid(*args):
    """Generate a UUID based on the given parameters.

    The UUID will be the SHA1 of the concatenation of the values
    from the list. The separator bewteedn these values is ':'.
    Each value must be a non-empty string, otherwise, the function
    will raise an exception.

    :param *args: list of arguments used to generate the UUID

    :returns: a universal unique identifier

    :raises ValueError: when anyone of the values is not a string,
        is empty or `None`.
    """
    def check_value(v):
        if not isinstance(v, str):
            raise ValueError("%s value is not a string instance" % str(v))
        elif not v:
            raise ValueError("value cannot be None or empty")
        else:
            return v

    s = ':'.join(map(check_value, args))

    sha1 = hashlib.sha1(s.encode('utf-8', errors='surrogateescape'))
    uuid_sha1 = sha1.hexdigest()

    return uuid_sha1

def
uuid
(
*
args
)
:
def
check_value
(
v
)
:
if
not
isinstance
(
v
,
str
)
:
raise
ValueError
(
"%s value is not a string instance"
%
str
(
v
)
)
elif
not
v
:
raise
ValueError
(
"value cannot be None or empty"
)
else
:
return
v
s
=
':'
.
join
(
map
(
check_value
,
args
)
)
sha1
=
hashlib
.
sha1
(
s
.
encode
(
'utf-8'
,
errors
=
'surrogateescape'
)
)
uuid_sha1
=
sha1
.
hexdigest
(
)
return
uuid_sha1

Generate a UUID based on the given parameters .


def fetch(backend_class, backend_args, category, filter_classified=False,
          manager=None):
    """Fetch items using the given backend.

    Generator to get items using the given backend class. When
    an archive manager is given, this function will store
    the fetched items in an `Archive`. If an exception is raised,
    this archive will be removed to avoid corrupted archives.

    The parameters needed to initialize the `backend` class and
    get the items are given using `backend_args` dict parameter.

    :param backend_class: backend class to fetch items
    :param backend_args: dict of arguments needed to fetch the items
    :param category: category of the items to retrieve.
       If None, it will use the default backend category
    :param filter_classified: remove classified fields from the resulting items
    :param manager: archive manager needed to store the items

    :returns: a generator of items
    """
    init_args = find_signature_parameters(backend_class.__init__,
                                          backend_args)
    archive = manager.create_archive() if manager else None
    init_args['archive'] = archive

    backend = backend_class(**init_args)

    if category:
        backend_args['category'] = category
    if filter_classified:
        backend_args['filter_classified'] = filter_classified

    fetch_args = find_signature_parameters(backend.fetch,
                                           backend_args)
    items = backend.fetch(**fetch_args)

    try:
        for item in items:
            yield item
    except Exception as e:
        if manager:
            archive_path = archive.archive_path
            manager.remove_archive(archive_path)
        raise e

def fetch ( backend_class , backend_args , category , filter_classified = False , manager = None ) : init_args = find_signature_parameters ( backend_class . __init__ , backend_args ) archive = manager . create_archive ( ) if manager else None init_args [ 'archive' ] = archive backend = backend_class ( * * init_args ) if category : backend_args [ 'category' ] = category if filter_classified : backend_args [ 'filter_classified' ] = filter_classified fetch_args = find_signature_parameters ( backend . fetch , backend_args ) items = backend . fetch ( * * fetch_args ) try : for item in items : yield item except Exception as e : if manager : archive_path = archive . archive_path manager . remove_archive ( archive_path ) raise e


def fetch_from_archive(backend_class, backend_args, manager,
                       category, archived_after):
    """Fetch items from an archive manager.

    Generator to get the items of a category (previously fetched
    by the given backend class) from an archive manager. Only those
    items archived after the given date will be returned.

    The parameters needed to initialize `backend` and get the
    items are given using `backend_args` dict parameter.

    :param backend_class: backend class to retrive items
    :param backend_args: dict of arguments needed to retrieve the items
    :param manager: archive manager where the items will be retrieved
    :param category: category of the items to retrieve
    :param archived_after: return items archived after this date

    :returns: a generator of archived items
    """
    init_args = find_signature_parameters(backend_class.__init__,
                                          backend_args)
    backend = backend_class(**init_args)

    filepaths = manager.search(backend.origin,
                               backend.__class__.__name__,
                               category,
                               archived_after)

    for filepath in filepaths:
        backend.archive = Archive(filepath)
        items = backend.fetch_from_archive()

        try:
            for item in items:
                yield item
        except ArchiveError as e:
            logger.warning("Ignoring %s archive due to: %s", filepath, str(e))

def
fetch_from_archive
(
backend_class
,
backend_args
,
manager
,
category
,
archived_after
)
:
init_args
=
find_signature_parameters
(
backend_class
.
__init__
,
backend_args
)
backend
=
backend_class
(
*
*
init_args
)
filepaths
=
manager
.
search
(
backend
.
origin
,
backend
.
__class__
.
__name__
,
category
,
archived_after
)
for
filepath
in
filepaths
:
backend
.
archive
=
Archive
(
filepath
)
items
=
backend
.
fetch_from_archive
(
)
try
:
for
item
in
items
:
yield
item
except
ArchiveError
as
e
:
logger
.
warning
(
"Ignoring %s archive due to: %s"
,
filepath
,
str
(
e
)
)


def find_backends(top_package):
    """Find available backends.

    Look for the Perceval backends and commands under `top_package`
    and its sub-packages. When `top_package` defines a namespace,
    backends under that same namespace will be found too.

    :param top_package: package storing backends

    :returns: a tuple with two dicts: one with `Backend` classes and one
        with `BackendCommand` classes
    """
    candidates = pkgutil.walk_packages(top_package.__path__,
                                       prefix=top_package.__name__ + '.')

    modules = [name for _, name, is_pkg in candidates if not is_pkg]

    return _import_backends(modules)

def find_backends ( top_package ) : candidates = pkgutil . walk_packages ( top_package . __path__ , prefix = top_package . __name__ + '.' ) modules = [ name for _ , name , is_pkg in candidates if not is_pkg ] return _import_backends ( modules )


def fetch(self, category, filter_classified=False, **kwargs):
        """Fetch items from the repository.

        The method retrieves items from a repository.

        To removed classified fields from the resulting items, set
        the parameter `filter_classified`. Take into account this
        parameter is incompatible with archiving items. Raw client
        data are archived before any other process. Therefore,
        classified data  are stored within the archive. To prevent
        from possible data leaks or security issues when users do
        not need these fields, archiving and filtering are not
        compatible.

        :param category: the category of the items fetched
        :param filter_classified: remove classified fields from the resulting items
        :param kwargs: a list of other parameters (e.g., from_date, offset, etc.
        specific for each backend)

        :returns: a generator of items

        :raises BackendError: either when the category is not valid or
            'filter_classified' and 'archive' are active at the same time.
        """
        if category not in self.categories:
            cause = "%s category not valid for %s" % (category, self.__class__.__name__)
            raise BackendError(cause=cause)

        if filter_classified and self.archive:
            cause = "classified fields filtering is not compatible with archiving items"
            raise BackendError(cause=cause)

        if self.archive:
            self.archive.init_metadata(self.origin, self.__class__.__name__, self.version, category,
                                       kwargs)

        self.client = self._init_client()

        for item in self.fetch_items(category, **kwargs):
            if filter_classified:
                item = self.filter_classified_data(item)

            yield self.metadata(item, filter_classified=filter_classified)


def fetch_from_archive(self):
        """Fetch the questions from an archive.

        It returns the items stored within an archive. If this method is called but
        no archive was provided, the method will raise a `ArchiveError` exception.

        :returns: a generator of items

        :raises ArchiveError: raised when an error occurs accessing an archive
        """
        if not self.archive:
            raise ArchiveError(cause="archive instance was not provided")

        self.client = self._init_client(from_archive=True)

        for item in self.fetch_items(self.archive.category, **self.archive.backend_params):
            yield self.metadata(item)

def
fetch_from_archive
(
self
)
:
if
not
self
.
archive
:
raise
ArchiveError
(
cause
=
"archive instance was not provided"
)
self
.
client
=
self
.
_init_client
(
from_archive
=
True
)
for
item
in
self
.
fetch_items
(
self
.
archive
.
category
,
*
*
self
.
archive
.
backend_params
)
:
yield
self
.
metadata
(
item
)


def filter_classified_data(self, item):
        """Remove classified or confidential data from an item.

        It removes those fields that contain data considered as classified.
        Classified fields are defined in `CLASSIFIED_FIELDS` class attribute.

        :param item: fields will be removed from this item

        :returns: the same item but with confidential data filtered
        """
        item_uuid = uuid(self.origin, self.metadata_id(item))

        logger.debug("Filtering classified data for item %s", item_uuid)

        for cf in self.CLASSIFIED_FIELDS:
            try:
                _remove_key_from_nested_dict(item, cf)
            except KeyError:
                logger.debug("Classified field '%s' not found for item %s; field ignored",
                             '.'.join(cf), item_uuid)

        logger.debug("Classified data filtered for item %s", item_uuid)

        return item

Remove classified or confidential data from an item .


def parse(self, *args):
        """Parse a list of arguments.

        Parse argument strings needed to run a backend command. The result
        will be a `argparse.Namespace` object populated with the values
        obtained after the validation of the parameters.

        :param args: argument strings

        :result: an object with the parsed values
        """
        parsed_args = self.parser.parse_args(args)

        # Category was not set, remove it
        if parsed_args.category is None:
            delattr(parsed_args, 'category')

        if self._from_date:
            parsed_args.from_date = str_to_datetime(parsed_args.from_date)
        if self._to_date and parsed_args.to_date:
            parsed_args.to_date = str_to_datetime(parsed_args.to_date)
        if self._archive and parsed_args.archived_since:
            parsed_args.archived_since = str_to_datetime(parsed_args.archived_since)

        if self._archive and parsed_args.fetch_archive and parsed_args.no_archive:
            raise AttributeError("fetch-archive and no-archive arguments are not compatible")
        if self._archive and parsed_args.fetch_archive and not parsed_args.category:
            raise AttributeError("fetch-archive needs a category to work with")

        # Set aliases
        for alias, arg in self.aliases.items():
            if (alias not in parsed_args) and (arg in parsed_args):
                value = getattr(parsed_args, arg, None)
                setattr(parsed_args, alias, value)

        return parsed_args


def _set_auth_arguments(self, basic_auth=True, token_auth=False):
        """Activate authentication arguments parsing"""

        group = self.parser.add_argument_group('authentication arguments')

        if basic_auth:
            group.add_argument('-u', '--backend-user', dest='user',
                               help="backend user")
            group.add_argument('-p', '--backend-password', dest='password',
                               help="backend password")
        if token_auth:
            group.add_argument('-t', '--api-token', dest='api_token',
                               help="backend authentication token / API key")


def _set_archive_arguments(self):
        """Activate archive arguments parsing"""

        group = self.parser.add_argument_group('archive arguments')
        group.add_argument('--archive-path', dest='archive_path', default=None,
                           help="directory path to the archives")
        group.add_argument('--no-archive', dest='no_archive', action='store_true',
                           help="do not archive data")
        group.add_argument('--fetch-archive', dest='fetch_archive', action='store_true',
                           help="fetch data from the archives")
        group.add_argument('--archived-since', dest='archived_since', default='1970-01-01',
                           help="retrieve items archived since the given date")


def _set_output_arguments(self):
        """Activate output arguments parsing"""

        group = self.parser.add_argument_group('output arguments')
        group.add_argument('-o', '--output', type=argparse.FileType('w'),
                           dest='outfile', default=sys.stdout,
                           help="output file")
        group.add_argument('--json-line', dest='json_line', action='store_true',
                           help="produce a JSON line for each output item")


def run(self):
        """Fetch and write items.

        This method runs the backend to fetch the items from the given
        origin. Items are converted to JSON objects and written to the
        defined output.

        If `fetch-archive` parameter was given as an argument during
        the inizialization of the instance, the items will be retrieved
        using the archive manager.
        """
        backend_args = vars(self.parsed_args)
        category = backend_args.pop('category', None)
        filter_classified = backend_args.pop('filter_classified', False)
        archived_since = backend_args.pop('archived_since', None)

        if self.archive_manager and self.parsed_args.fetch_archive:
            items = fetch_from_archive(self.BACKEND, backend_args,
                                       self.archive_manager,
                                       category,
                                       archived_since)
        else:
            items = fetch(self.BACKEND, backend_args, category,
                          filter_classified=filter_classified,
                          manager=self.archive_manager)

        try:
            for item in items:
                if self.json_line:
                    obj = json.dumps(item, separators=(',', ':'), sort_keys=True)
                else:
                    obj = json.dumps(item, indent=4, sort_keys=True)
                self.outfile.write(obj)
                self.outfile.write('\n')
        except IOError as e:
            raise RuntimeError(str(e))
        except Exception as e:
            raise RuntimeError(str(e))

def run ( self ) : backend_args = vars ( self . parsed_args ) category = backend_args . pop ( 'category' , None ) filter_classified = backend_args . pop ( 'filter_classified' , False ) archived_since = backend_args . pop ( 'archived_since' , None ) if self . archive_manager and self . parsed_args . fetch_archive : items = fetch_from_archive ( self . BACKEND , backend_args , self . archive_manager , category , archived_since ) else : items = fetch ( self . BACKEND , backend_args , category , filter_classified = filter_classified , manager = self . archive_manager ) try : for item in items : if self . json_line : obj = json . dumps ( item , separators = ( ',' , ':' ) , sort_keys = True ) else : obj = json . dumps ( item , indent = 4 , sort_keys = True ) self . outfile . write ( obj ) self . outfile . write ( '\n' ) except IOError as e : raise RuntimeError ( str ( e ) ) except Exception as e : raise RuntimeError ( str ( e ) )


def _initialize_archive(self):
        """Initialize archive based on the parsed parameters"""

        if 'archive_path' not in self.parsed_args:
            manager = None
        elif self.parsed_args.no_archive:
            manager = None
        else:
            if not self.parsed_args.archive_path:
                archive_path = os.path.expanduser(ARCHIVES_DEFAULT_PATH)
            else:
                archive_path = self.parsed_args.archive_path

            manager = ArchiveManager(archive_path)

        self.archive_manager = manager

def _initialize_archive ( self ) : if 'archive_path' not in self . parsed_args : manager = None elif self . parsed_args . no_archive : manager = None else : if not self . parsed_args . archive_path : archive_path = os . path . expanduser ( ARCHIVES_DEFAULT_PATH ) else : archive_path = self . parsed_args . archive_path manager = ArchiveManager ( archive_path ) self . archive_manager = manager


def metadata_updated_on(item):
        """Extracts the update time from a MBox item.

        The timestamp used is extracted from 'Date' field in its
        several forms. This date is converted to UNIX timestamp
        format.

        :param item: item generated by the backend

        :returns: a UNIX timestamp
        """
        ts = item[MBox.DATE_FIELD]
        ts = str_to_datetime(ts)

        return ts.timestamp()

def metadata_updated_on ( item ) : ts = item [ MBox . DATE_FIELD ] ts = str_to_datetime ( ts ) return ts . timestamp ( )

Extracts the update time from a MBox item .


def parse_mbox(filepath):
        """Parse a mbox file.

        This method parses a mbox file and returns an iterator of dictionaries.
        Each one of this contains an email message.

        :param filepath: path of the mbox to parse

        :returns : generator of messages; each message is stored in a
            dictionary of type `requests.structures.CaseInsensitiveDict`
        """
        mbox = _MBox(filepath, create=False)

        for msg in mbox:
            message = message_to_dict(msg)
            yield message

def parse_mbox ( filepath ) : mbox = _MBox ( filepath , create = False ) for msg in mbox : message = message_to_dict ( msg ) yield message


def _fetch_and_parse_messages(self, mailing_list, from_date):
        """Fetch and parse the messages from a mailing list"""

        from_date = datetime_to_utc(from_date)

        nmsgs, imsgs, tmsgs = (0, 0, 0)

        for mbox in mailing_list.mboxes:
            tmp_path = None

            try:
                tmp_path = self._copy_mbox(mbox)

                for message in self.parse_mbox(tmp_path):
                    tmsgs += 1

                    if not self._validate_message(message):
                        imsgs += 1
                        continue

                    # Ignore those messages sent before the given date
                    dt = str_to_datetime(message[MBox.DATE_FIELD])

                    if dt < from_date:
                        logger.debug("Message %s sent before %s; skipped",
                                     message['unixfrom'], str(from_date))
                        tmsgs -= 1
                        continue

                    # Convert 'CaseInsensitiveDict' to dict
                    message = self._casedict_to_dict(message)

                    nmsgs += 1
                    logger.debug("Message %s parsed", message['unixfrom'])

                    yield message
            except (OSError, EOFError) as e:
                logger.warning("Ignoring %s mbox due to: %s", mbox.filepath, str(e))
            except Exception as e:
                if tmp_path and os.path.exists(tmp_path):
                    os.remove(tmp_path)
                raise e
            finally:
                if tmp_path and os.path.exists(tmp_path):
                    os.remove(tmp_path)

        logger.info("Done. %s/%s messages fetched; %s ignored",
                    nmsgs, tmsgs, imsgs)


def _copy_mbox(self, mbox):
        """Copy the contents of a mbox to a temporary file"""

        tmp_path = tempfile.mktemp(prefix='perceval_')

        with mbox.container as f_in:
            with open(tmp_path, mode='wb') as f_out:
                for l in f_in:
                    f_out.write(l)
        return tmp_path

def _copy_mbox ( self , mbox ) : tmp_path = tempfile . mktemp ( prefix = 'perceval_' ) with mbox . container as f_in : with open ( tmp_path , mode = 'wb' ) as f_out : for l in f_in : f_out . write ( l ) return tmp_path


def _validate_message(self, message):
        """Check if the given message has the mandatory fields"""

        # This check is "case insensitive" because we're
        # using 'CaseInsensitiveDict' from requests.structures
        # module to store the contents of a message.
        if self.MESSAGE_ID_FIELD not in message:
            logger.warning("Field 'Message-ID' not found in message %s; ignoring",
                           message['unixfrom'])
            return False

        if not message[self.MESSAGE_ID_FIELD]:
            logger.warning("Field 'Message-ID' is empty in message %s; ignoring",
                           message['unixfrom'])
            return False

        if self.DATE_FIELD not in message:
            logger.warning("Field 'Date' not found in message %s; ignoring",
                           message['unixfrom'])
            return False

        if not message[self.DATE_FIELD]:
            logger.warning("Field 'Date' is empty in message %s; ignoring",
                           message['unixfrom'])
            return False

        try:
            str_to_datetime(message[self.DATE_FIELD])
        except InvalidDateError:
            logger.warning("Invalid date %s in message %s; ignoring",
                           message[self.DATE_FIELD], message['unixfrom'])
            return False

        return True


def _casedict_to_dict(self, message):
        """Convert a message in CaseInsensitiveDict to dict.

        This method also converts well known problematic headers,
        such as Message-ID and Date to a common name.
        """
        message_id = message.pop(self.MESSAGE_ID_FIELD)
        date = message.pop(self.DATE_FIELD)

        msg = {k: v for k, v in message.items()}
        msg[self.MESSAGE_ID_FIELD] = message_id
        msg[self.DATE_FIELD] = date

        return msg

def _casedict_to_dict ( self , message ) : message_id = message . pop ( self . MESSAGE_ID_FIELD ) date = message . pop ( self . DATE_FIELD ) msg = { k : v for k , v in message . items ( ) } msg [ self . MESSAGE_ID_FIELD ] = message_id msg [ self . DATE_FIELD ] = date return msg

Convert a message in CaseInsensitiveDict to dict .


def get_message(self, key):
        """Return a Message representation or raise a KeyError."""

        start, stop = self._lookup(key)
        self._file.seek(start)
        from_line = self._file.readline().replace(mailbox.linesep, b'')
        string = self._file.read(stop - self._file.tell())
        msg = self._message_factory(string.replace(mailbox.linesep, b'\n'))

        try:
            msg.set_from(from_line[5:].decode('ascii'))
            return msg
        except UnicodeDecodeError:
            pass

        try:
            msg.set_from(from_line[5:].decode('utf-8'))
        except UnicodeDecodeError:
            msg.set_from(from_line[5:].decode('iso-8859-1'))

        return msg

def get_message ( self , key ) : start , stop = self . _lookup ( key ) self . _file . seek ( start ) from_line = self . _file . readline ( ) . replace ( mailbox . linesep , b'' ) string = self . _file . read ( stop - self . _file . tell ( ) ) msg = self . _message_factory ( string . replace ( mailbox . linesep , b'\n' ) ) try : msg . set_from ( from_line [ 5 : ] . decode ( 'ascii' ) ) return msg except UnicodeDecodeError : pass try : msg . set_from ( from_line [ 5 : ] . decode ( 'utf-8' ) ) except UnicodeDecodeError : msg . set_from ( from_line [ 5 : ] . decode ( 'iso-8859-1' ) ) return msg

Return a Message representation or raise a KeyError .


def fetch(self, category=CATEGORY_COMMIT, from_date=DEFAULT_DATETIME, to_date=DEFAULT_LAST_DATETIME,
              branches=None, latest_items=False, no_update=False):
        """Fetch commits.

        The method retrieves from a Git repository or a log file
        a list of commits. Commits are returned in the same order
        they were obtained.

        When `from_date` parameter is given it returns items commited
        since the given date.

        The list of `branches` is a list of strings, with the names of
        the branches to fetch. If the list of branches is empty, no
        commit is fetched. If the list of branches is None, all commits
        for all branches will be fetched.

        The parameter `latest_items` returns only those commits which
        are new since the last time this method was called.

        The parameter `no_update` returns all commits without performing
        an update of the repository before.

        Take into account that `from_date` and `branches` are ignored
        when the commits are fetched from a Git log file or when
        `latest_items` flag is set.

        The class raises a `RepositoryError` exception when an error
        occurs accessing the repository.

        :param category: the category of items to fetch
        :param from_date: obtain commits newer than a specific date
            (inclusive)
        :param to_date: obtain commits older than a specific date
        :param branches: names of branches to fetch from (default: None)
        :param latest_items: sync with the repository to fetch only the
            newest commits
        :param no_update: if enabled, don't update the repo with the latest changes

        :returns: a generator of commits
        """
        if not from_date:
            from_date = DEFAULT_DATETIME
        if not to_date:
            to_date = DEFAULT_LAST_DATETIME

        kwargs = {
            'from_date': from_date,
            'to_date': to_date,
            'branches': branches,
            'latest_items': latest_items,
            'no_update': no_update
        }
        items = super().fetch(category, **kwargs)

        return items

def fetch ( self , category = CATEGORY_COMMIT , from_date = DEFAULT_DATETIME , to_date = DEFAULT_LAST_DATETIME , branches = None , latest_items = False , no_update = False ) : if not from_date : from_date = DEFAULT_DATETIME if not to_date : to_date = DEFAULT_LAST_DATETIME kwargs = { 'from_date' : from_date , 'to_date' : to_date , 'branches' : branches , 'latest_items' : latest_items , 'no_update' : no_update } items = super ( ) . fetch ( category , * * kwargs ) return items


def fetch_items(self, category, **kwargs):
        """Fetch the commits

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        from_date = kwargs['from_date']
        to_date = kwargs['to_date']
        branches = kwargs['branches']
        latest_items = kwargs['latest_items']
        no_update = kwargs['no_update']

        ncommits = 0

        try:
            if os.path.isfile(self.gitpath):
                commits = self.__fetch_from_log()
            else:
                commits = self.__fetch_from_repo(from_date, to_date, branches,
                                                 latest_items, no_update)

            for commit in commits:
                yield commit
                ncommits += 1
        except EmptyRepositoryError:
            pass

        logger.info("Fetch process completed: %s commits fetched",
                    ncommits)


def parse_git_log_from_file(filepath):
        """Parse a Git log file.

        The method parses the Git log file and returns an iterator of
        dictionaries. Each one of this, contains a commit.

        :param filepath: path to the log file

        :returns: a generator of parsed commits

        :raises ParseError: raised when the format of the Git log file
            is invalid
        :raises OSError: raised when an error occurs reading the
            given file
        """
        with open(filepath, 'r', errors='surrogateescape',
                  newline=os.linesep) as f:
            parser = GitParser(f)

            for commit in parser.parse():
                yield commit

def parse_git_log_from_file ( filepath ) : with open ( filepath , 'r' , errors = 'surrogateescape' , newline = os . linesep ) as f : parser = GitParser ( f ) for commit in parser . parse ( ) : yield commit


def _pre_init(self):
        """Initialize repositories directory path"""

        if self.parsed_args.git_log:
            git_path = self.parsed_args.git_log
        elif not self.parsed_args.git_path:
            base_path = os.path.expanduser('~/.perceval/repositories/')
            processed_uri = self.parsed_args.uri.lstrip('/')
            git_path = os.path.join(base_path, processed_uri) + '-git'
        else:
            git_path = self.parsed_args.git_path

        setattr(self.parsed_args, 'gitpath', git_path)

def _pre_init ( self ) : if self . parsed_args . git_log : git_path = self . parsed_args . git_log elif not self . parsed_args . git_path : base_path = os . path . expanduser ( '~/.perceval/repositories/' ) processed_uri = self . parsed_args . uri . lstrip ( '/' ) git_path = os . path . join ( base_path , processed_uri ) + '-git' else : git_path = self . parsed_args . git_path setattr ( self . parsed_args , 'gitpath' , git_path )


def setup_cmd_parser(cls):
        """Returns the Git argument parser."""

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              from_date=True,
                                              to_date=True)

        # Optional arguments
        group = parser.parser.add_argument_group('Git arguments')
        group.add_argument('--branches', dest='branches',
                           nargs='+', type=str, default=None,
                           help="Fetch commits only from these branches")

        # Mutual exclusive parameters
        exgroup = group.add_mutually_exclusive_group()
        exgroup.add_argument('--git-path', dest='git_path',
                             help="Path where the Git repository will be cloned")
        exgroup.add_argument('--git-log', dest='git_log',
                             help="Path to the Git log file")

        exgroup_fetch = group.add_mutually_exclusive_group()
        exgroup_fetch.add_argument('--latest-items', dest='latest_items',
                                   action='store_true',
                                   help="Fetch latest commits added to the repository")
        exgroup_fetch.add_argument('--no-update', dest='no_update',
                                   action='store_true',
                                   help="Fetch all commits without updating the repository")

        # Required arguments
        parser.parser.add_argument('uri',
                                   help="URI of the Git log repository")

        return parser


def parse(self):
        """Parse the Git log stream."""

        for line in self.stream:
            line = line.rstrip('\n')
            parsed = False
            self.nline += 1

            while not parsed:
                parsed = self.handlers[self.state](line)

                if self.state == self.COMMIT and self.commit:
                    commit = self._build_commit()
                    logger.debug("Commit %s parsed", commit['commit'])
                    yield commit

        # Return the last commit, if any
        if self.commit:
            commit = self._build_commit()
            logger.debug("Commit %s parsed", commit['commit'])
            yield commit

def
parse
(
self
)
:
for
line
in
self
.
stream
:
line
=
line
.
rstrip
(
'\n'
)
parsed
=
False
self
.
nline
+=
1
while
not
parsed
:
parsed
=
self
.
handlers
[
self
.
state
]
(
line
)
if
self
.
state
==
self
.
COMMIT
and
self
.
commit
:
commit
=
self
.
_build_commit
(
)
logger
.
debug
(
"Commit %s parsed"
,
commit
[
'commit'
]
)
yield
commit
# Return the last commit, if any
if
self
.
commit
:
commit
=
self
.
_build_commit
(
)
logger
.
debug
(
"Commit %s parsed"
,
commit
[
'commit'
]
)
yield
commit


def clone(cls, uri, dirpath):
        """Clone a Git repository.

        Make a bare copy of the repository stored in `uri` into `dirpath`.
        The repository would be either local or remote.

        :param uri: URI of the repository
        :param dirtpath: directory where the repository will be cloned

        :returns: a `GitRepository` class having cloned the repository

        :raises RepositoryError: when an error occurs cloning the given
            repository
        """
        cmd = ['git', 'clone', '--bare', uri, dirpath]
        env = {
            'LANG': 'C',
            'HOME': os.getenv('HOME', '')
        }

        cls._exec(cmd, env=env)

        logger.debug("Git %s repository cloned into %s",
                     uri, dirpath)

        return cls(uri, dirpath)

def
clone
(
cls
,
uri
,
dirpath
)
:
cmd
=
[
'git'
,
'clone'
,
'--bare'
,
uri
,
dirpath
]
env
=
{
'LANG'
:
'C'
,
'HOME'
:
os
.
getenv
(
'HOME'
,
''
)
}
cls
.
_exec
(
cmd
,
env
=
env
)
logger
.
debug
(
"Git %s repository cloned into %s"
,
uri
,
dirpath
)
return
cls
(
uri
,
dirpath
)


def count_objects(self):
        """Count the objects of a repository.

        The method returns the total number of objects (packed and unpacked)
        available on the repository.

        :raises RepositoryError: when an error occurs counting the objects
            of a repository
        """
        cmd_count = ['git', 'count-objects', '-v']

        outs = self._exec(cmd_count, cwd=self.dirpath, env=self.gitenv)
        outs = outs.decode('utf-8', errors='surrogateescape').rstrip()

        try:
            cobjs = {k: v for k, v in (x.split(': ') for x in outs.split('\n'))}
            nobjs = int(cobjs['count']) + int(cobjs['in-pack'])
        except KeyError as e:
            error = "unable to parse 'count-objects' output; reason: '%s' entry not found" \
                % e.args[0]
            raise RepositoryError(cause=error)
        except ValueError as e:
            error = "unable to parse 'count-objects' output; reason: %s" % str(e)
            raise RepositoryError(cause=error)

        logger.debug("Git %s repository has %s objects",
                     self.uri, str(nobjs))

        return nobjs


def is_detached(self):
        """Check if the repo is in a detached state.

        The repository is in a detached state when HEAD is not a symbolic
        reference.

        :returns: whether the repository is detached or not

        :raises RepositoryError: when an error occurs checking the state
            of the repository
        """
        cmd_sym = ['git', 'symbolic-ref', 'HEAD']

        try:
            self._exec(cmd_sym, cwd=self.dirpath, env=self.gitenv)
        except RepositoryError as e:
            if e.msg.find("ref HEAD is not a symbolic ref") == -1:
                raise e
            return True
        else:
            return False

def
is_detached
(
self
)
:
cmd_sym
=
[
'git'
,
'symbolic-ref'
,
'HEAD'
]
try
:
self
.
_exec
(
cmd_sym
,
cwd
=
self
.
dirpath
,
env
=
self
.
gitenv
)
except
RepositoryError
as
e
:
if
e
.
msg
.
find
(
"ref HEAD is not a symbolic ref"
)
==
1
:
raise
e
return
True
else
:
return
False

Check if the repo is in a detached state .


def update(self):
        """Update repository from its remote.

        Calling this method, the repository will be synchronized with
        the remote repository using 'fetch' command for 'heads' refs.
        Any commit stored in the local copy will be removed; refs
        will be overwritten.

        :raises RepositoryError: when an error occurs updating the
            repository
        """
        cmd_update = ['git', 'fetch', 'origin', '+refs/heads/*:refs/heads/*', '--prune']
        self._exec(cmd_update, cwd=self.dirpath, env=self.gitenv)

        logger.debug("Git %s repository updated into %s",
                     self.uri, self.dirpath)

def
update
(
self
)
:
cmd_update
=
[
'git'
,
'fetch'
,
'origin'
,
'+refs/heads/*:refs/heads/*'
,
'--prune'
]
self
.
_exec
(
cmd_update
,
cwd
=
self
.
dirpath
,
env
=
self
.
gitenv
)
logger
.
debug
(
"Git %s repository updated into %s"
,
self
.
uri
,
self
.
dirpath
)


def sync(self):
        """Keep the repository in sync.

        This method will synchronize the repository with its 'origin',
        fetching newest objects and updating references. It uses low
        level commands which allow to keep track of which things
        have changed in the repository.

        The method also returns a list of hashes related to the new
        commits fetched during the process.

        :returns: list of new commits

        :raises RepositoryError: when an error occurs synchronizing
            the repository
        """
        pack_name, refs = self._fetch_pack()

        if pack_name:
            commits = self._read_commits_from_pack(pack_name)
        else:
            commits = []
            logger.debug("Git repository %s (%s) does not have any new object",
                         self.uri, self.dirpath)

        self._update_references(refs)

        logger.debug("Git repository %s (%s) is synced",
                     self.uri, self.dirpath)

        return commits


def rev_list(self, branches=None):
        """Read the list commits from the repository

        The list of branches is a list of strings, with the names of the
        branches to fetch. If the list of branches is empty, no commit
        is fetched. If the list of branches is None, all commits
        for all branches will be fetched.

        The method returns the Git rev-list of the repository using the
        following options:

            git rev-list --topo-order

        :param branches: names of branches to fetch from (default: None)

        :raises EmptyRepositoryError: when the repository is empty and
            the action cannot be performed
        :raises RepositoryError: when an error occurs executing the command
        """
        if self.is_empty():
            logger.warning("Git %s repository is empty; unable to get the rev-list",
                           self.uri)
            raise EmptyRepositoryError(repository=self.uri)

        cmd_rev_list = ['git', 'rev-list', '--topo-order']

        if branches is None:
            cmd_rev_list.extend(['--branches', '--tags', '--remotes=origin'])
        elif len(branches) == 0:
            cmd_rev_list.extend(['--branches', '--tags', '--max-count=0'])
        else:
            branches = ['refs/heads/' + branch for branch in branches]
            cmd_rev_list.extend(branches)

        for line in self._exec_nb(cmd_rev_list, cwd=self.dirpath, env=self.gitenv):
            yield line.rstrip('\n')

        logger.debug("Git rev-list fetched from %s repository (%s)",
                     self.uri, self.dirpath)


def log(self, from_date=None, to_date=None, branches=None, encoding='utf-8'):
        """Read the commit log from the repository.

        The method returns the Git log of the repository using the
        following options:

            git log --raw --numstat --pretty=fuller --decorate=full
                --all --reverse --topo-order --parents -M -C -c
                --remotes=origin

        When `from_date` is given, it gets the commits equal or older
        than that date. This date is given in a datetime object.

        The list of branches is a list of strings, with the names of the
        branches to fetch. If the list of branches is empty, no commit
        is fetched. If the list of branches is None, all commits
        for all branches will be fetched.

        :param from_date: fetch commits newer than a specific
            date (inclusive)
        :param branches: names of branches to fetch from (default: None)
        :param encoding: encode the log using this format

        :returns: a generator where each item is a line from the log

        :raises EmptyRepositoryError: when the repository is empty and
            the action cannot be performed
        :raises RepositoryError: when an error occurs fetching the log
        """
        if self.is_empty():
            logger.warning("Git %s repository is empty; unable to get the log",
                           self.uri)
            raise EmptyRepositoryError(repository=self.uri)

        cmd_log = ['git', 'log', '--reverse', '--topo-order']
        cmd_log.extend(self.GIT_PRETTY_OUTPUT_OPTS)

        if from_date:
            dt = from_date.strftime("%Y-%m-%d %H:%M:%S %z")
            cmd_log.append('--since=' + dt)

        if to_date:
            dt = to_date.strftime("%Y-%m-%d %H:%M:%S %z")
            cmd_log.append('--until=' + dt)

        if branches is None:
            cmd_log.extend(['--branches', '--tags', '--remotes=origin'])
        elif len(branches) == 0:
            cmd_log.append('--max-count=0')
        else:
            branches = ['refs/heads/' + branch for branch in branches]
            cmd_log.extend(branches)

        for line in self._exec_nb(cmd_log, cwd=self.dirpath, env=self.gitenv):
            yield line

        logger.debug("Git log fetched from %s repository (%s)",
                     self.uri, self.dirpath)

Read the commit log from the repository .


def show(self, commits=None, encoding='utf-8'):
        """Show the data of a set of commits.

        The method returns the output of Git show command for a
        set of commits using the following options:

            git show --raw --numstat --pretty=fuller --decorate=full
                --parents -M -C -c [<commit>...<commit>]

        When the list of commits is empty, the command will return
        data about the last commit, like the default behaviour of
        `git show`.

        :param commits: list of commits to show data
        :param encoding: encode the output using this format

        :returns: a generator where each item is a line from the show output

        :raises EmptyRepositoryError: when the repository is empty and
            the action cannot be performed
        :raises RepositoryError: when an error occurs fetching the show output
        """
        if self.is_empty():
            logger.warning("Git %s repository is empty; unable to run show",
                           self.uri)
            raise EmptyRepositoryError(repository=self.uri)

        if commits is None:
            commits = []

        cmd_show = ['git', 'show']
        cmd_show.extend(self.GIT_PRETTY_OUTPUT_OPTS)
        cmd_show.extend(commits)

        for line in self._exec_nb(cmd_show, cwd=self.dirpath, env=self.gitenv):
            yield line

        logger.debug("Git show fetched from %s repository (%s)",
                     self.uri, self.dirpath)


def _fetch_pack(self):
        """Fetch changes and store them in a pack."""

        def prepare_refs(refs):
            return [ref.hash.encode('utf-8') for ref in refs
                    if not ref.refname.endswith('^{}')]

        def determine_wants(refs):
            remote_refs = prepare_refs(self._discover_refs(remote=True))
            local_refs = prepare_refs(self._discover_refs())
            wants = [ref for ref in remote_refs if ref not in local_refs]
            return wants

        client, repo_path = dulwich.client.get_transport_and_path(self.uri)
        repo = dulwich.repo.Repo(self.dirpath)
        fd = io.BytesIO()

        local_refs = self._discover_refs()
        graph_walker = _GraphWalker(local_refs)

        result = client.fetch_pack(repo_path,
                                   determine_wants,
                                   graph_walker,
                                   fd.write)
        refs = [GitRef(ref_hash.decode('utf-8'), ref_name.decode('utf-8'))
                for ref_name, ref_hash in result.refs.items()]

        if len(fd.getvalue()) > 0:
            fd.seek(0)
            pack = repo.object_store.add_thin_pack(fd.read, None)
            pack_name = pack.name().decode('utf-8')
        else:
            pack_name = None

        return (pack_name, refs)

def _fetch_pack ( self ) : def prepare_refs ( refs ) : return [ ref . hash . encode ( 'utf-8' ) for ref in refs if not ref . refname . endswith ( '^{}' ) ] def determine_wants ( refs ) : remote_refs = prepare_refs ( self . _discover_refs ( remote = True ) ) local_refs = prepare_refs ( self . _discover_refs ( ) ) wants = [ ref for ref in remote_refs if ref not in local_refs ] return wants client , repo_path = dulwich . client . get_transport_and_path ( self . uri ) repo = dulwich . repo . Repo ( self . dirpath ) fd = io . BytesIO ( ) local_refs = self . _discover_refs ( ) graph_walker = _GraphWalker ( local_refs ) result = client . fetch_pack ( repo_path , determine_wants , graph_walker , fd . write ) refs = [ GitRef ( ref_hash . decode ( 'utf-8' ) , ref_name . decode ( 'utf-8' ) ) for ref_name , ref_hash in result . refs . items ( ) ] if len ( fd . getvalue ( ) ) > 0 : fd . seek ( 0 ) pack = repo . object_store . add_thin_pack ( fd . read , None ) pack_name = pack . name ( ) . decode ( 'utf-8' ) else : pack_name = None return ( pack_name , refs )

Fetch changes and store them in a pack .


def _read_commits_from_pack(self, packet_name):
        """Read the commits of a pack."""

        filepath = 'objects/pack/pack-' + packet_name

        cmd_verify_pack = ['git', 'verify-pack', '-v', filepath]

        outs = self._exec(cmd_verify_pack, cwd=self.dirpath, env=self.gitenv)
        outs = outs.decode('utf-8', errors='surrogateescape').rstrip()

        lines = [line.split(' ') for line in outs.split('\n')]

        # Commits usually come in the pack ordered from newest to oldest
        commits = [parts[0] for parts in lines if parts[1] == 'commit']
        commits.reverse()

        return commits


def _update_references(self, refs):
        """Update references removing old ones."""

        new_refs = [ref.refname for ref in refs]

        # Delete old references
        for old_ref in self._discover_refs():
            if not old_ref.refname.startswith('refs/heads/'):
                continue
            if old_ref.refname in new_refs:
                continue
            self._update_ref(old_ref, delete=True)

        # Update new references
        for new_ref in refs:
            refname = new_ref.refname

            if refname.endswith('^{}'):
                logger.debug("Annotated tag %s ignored for updating in sync process",
                             refname)
                continue
            elif not refname.startswith('refs/heads/') and not refname.startswith('refs/tags/'):
                logger.debug("Reference %s not needed; ignored for updating in sync process",
                             refname)
                continue
            else:
                self._update_ref(new_ref)

        # Prune repository to remove old branches
        cmd = ['git', 'remote', 'prune', 'origin']
        self._exec(cmd, cwd=self.dirpath, env=self.gitenv)


def _discover_refs(self, remote=False):
        """Get the current list of local or remote refs."""

        if remote:
            cmd_refs = ['git', 'ls-remote', '-h', '-t', '--exit-code', 'origin']
            sep = '\t'
            ignored_error_codes = [2]
        else:
            # Check first whether the local repo is empty;
            # Running 'show-ref' in empty repos gives an error
            if self.is_empty():
                raise EmptyRepositoryError(repository=self.uri)

            cmd_refs = ['git', 'show-ref', '--heads', '--tags']
            sep = ' '
            ignored_error_codes = [1]

        # Error codes returned when no matching refs (i.e, no heads
        # or tags) are found in a repository will be ignored. Otherwise,
        # the full process would fail for those situations.
        outs = self._exec(cmd_refs, cwd=self.dirpath,
                          env=self.gitenv,
                          ignored_error_codes=ignored_error_codes)
        outs = outs.decode('utf-8', errors='surrogateescape').rstrip()
        outs = outs.split('\n') if outs else []

        refs = []

        for line in outs:
            data = line.split(sep)
            ref = GitRef(data[0], data[1])
            refs.append(ref)

        return refs

Get the current list of local or remote refs .


def _update_ref(self, ref, delete=False):
        """Update a reference."""

        cmd = ['git', 'update-ref']

        if delete:
            cmd.extend(['-d', ref.refname])
            action = 'deleted'
        else:
            cmd.extend([ref.refname, ref.hash])
            action = 'updated to %s' % ref.hash

        try:
            self._exec(cmd, cwd=self.dirpath, env=self.gitenv)
        except RepositoryError as e:
            logger.warning("Git %s ref could not be %s during sync process in %s (%s); skipped",
                           ref.refname, action, self.uri, self.dirpath)
        else:
            logger.debug("Git %s ref %s in %s (%s)",
                         ref.refname, action, self.uri, self.dirpath)


def _exec_nb(self, cmd, cwd=None, env=None, encoding='utf-8'):
        """Run a command with a non blocking call.

        Execute `cmd` command with a non blocking call. The command will
        be run in the directory set by `cwd`. Enviroment variables can be
        set using the `env` dictionary. The output data is returned
        as encoded bytes in an iterator. Each item will be a line of the
        output.

        :returns: an iterator with the output of the command as encoded bytes

        :raises RepositoryError: when an error occurs running the command
        """
        self.failed_message = None

        logger.debug("Running command %s (cwd: %s, env: %s)",
                     ' '.join(cmd), cwd, str(env))

        try:
            self.proc = subprocess.Popen(cmd,
                                         stdout=subprocess.PIPE,
                                         stderr=subprocess.PIPE,
                                         cwd=cwd,
                                         env=env)
            err_thread = threading.Thread(target=self._read_stderr,
                                          kwargs={'encoding': encoding},
                                          daemon=True)
            err_thread.start()
            for line in self.proc.stdout:
                yield line.decode(encoding, errors='surrogateescape')
            err_thread.join()

            self.proc.communicate()
            self.proc.stdout.close()
            self.proc.stderr.close()
        except OSError as e:
            err_thread.join()
            raise RepositoryError(cause=str(e))

        if self.proc.returncode != 0:
            cause = "git command - %s (return code: %d)" % \
                (self.failed_message, self.proc.returncode)
            raise RepositoryError(cause=cause)

def
_exec_nb
(
self
,
cmd
,
cwd
=
,
env
=
,
encoding
=
'utf-8'
)
:
self
.
failed_message
=
logger
.
debug
(
"Running command %s (cwd: %s, env: %s)"
,
' '
.
join
(
cmd
)
,
cwd
,
str
(
env
)
)
try
:
self
.
proc
=
subprocess
.
Popen
(
cmd
,
stdout
=
subprocess
.
PIPE
,
stderr
=
subprocess
.
PIPE
,
cwd
=
cwd
,
env
=
env
)
err_thread
=
threading
.
Thread
(
target
=
self
.
_read_stderr
,
kwargs
=
{
'encoding'
:
encoding
}
,
daemon
=
True
)
err_thread
.
start
(
)
for
line
in
self
.
proc
.
stdout
:
yield
line
.
decode
(
encoding
,
errors
=
'surrogateescape'
)
err_thread
.
join
(
)
self
.
proc
.
communicate
(
)
self
.
proc
.
stdout
.
close
(
)
self
.
proc
.
stderr
.
close
(
)
except
OSError
as
e
:
err_thread
.
join
(
)
raise
RepositoryError
(
cause
=
str
(
e
)
)
if
self
.
proc
.
returncode
!=
0
:
cause
=
"git command - %s (return code: %d)"
%
(
self
.
failed_message
,
self
.
proc
.
returncode
)
raise
RepositoryError
(
cause
=
cause
)

Run a command with a non blocking call .


def _read_stderr(self, encoding='utf-8'):
        """Reads self.proc.stderr.

        Usually, this should be read in a thread, to prevent blocking
        the read from stdout of the stderr buffer is filled, and this
        function is not called becuase the program is busy in the
        stderr reading loop.

        Reads self.proc.stderr (self.proc is the subprocess running
        the git command), and reads / writes self.failed_message
        (the message sent to stderr when git fails, usually one line).
        """
        for line in self.proc.stderr:
            err_line = line.decode(encoding, errors='surrogateescape')

            if self.proc.returncode != 0:
                # If the subprocess didn't finish successfully, we expect
                # the last line in stderr to provide the cause
                if self.failed_message is not None:
                    # We had a message, there is a newer line, print it
                    logger.debug("Git log stderr: " + self.failed_message)
                self.failed_message = err_line
            else:
                # The subprocess is successfully up to now, print the line
                logger.debug("Git log stderr: " + err_line)


def _exec(cmd, cwd=None, env=None, ignored_error_codes=None,
              encoding='utf-8'):
        """Run a command.

        Execute `cmd` command in the directory set by `cwd`. Environment
        variables can be set using the `env` dictionary. The output
        data is returned as encoded bytes.

        Commands which their returning status codes are non-zero will
        be treated as failed. Error codes considered as valid can be
        ignored giving them in the `ignored_error_codes` list.

        :returns: the output of the command as encoded bytes

        :raises RepositoryError: when an error occurs running the command
        """
        if ignored_error_codes is None:
            ignored_error_codes = []

        logger.debug("Running command %s (cwd: %s, env: %s)",
                     ' '.join(cmd), cwd, str(env))

        try:
            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE,
                                    cwd=cwd, env=env)
            (outs, errs) = proc.communicate()
        except OSError as e:
            raise RepositoryError(cause=str(e))

        if proc.returncode != 0 and proc.returncode not in ignored_error_codes:
            err = errs.decode(encoding, errors='surrogateescape')
            cause = "git command - %s" % err
            raise RepositoryError(cause=cause)
        else:
            logger.debug(errs.decode(encoding, errors='surrogateescape'))

        return outs

def
_exec
(
cmd
,
cwd
=
,
env
=
,
ignored_error_codes
=
,
encoding
=
'utf-8'
)
:
if
ignored_error_codes
is
:
ignored_error_codes
=
[
]
logger
.
debug
(
"Running command %s (cwd: %s, env: %s)"
,
' '
.
join
(
cmd
)
,
cwd
,
str
(
env
)
)
try
:
proc
=
subprocess
.
Popen
(
cmd
,
stdout
=
subprocess
.
PIPE
,
stderr
=
subprocess
.
PIPE
,
cwd
=
cwd
,
env
=
env
)
(
outs
,
errs
)
=
proc
.
communicate
(
)
except
OSError
as
e
:
raise
RepositoryError
(
cause
=
str
(
e
)
)
if
proc
.
returncode
!=
0
and
proc
.
returncode
not
in
ignored_error_codes
:
err
=
errs
.
decode
(
encoding
,
errors
=
'surrogateescape'
)
cause
=
"git command - %s"
%
err
raise
RepositoryError
(
cause
=
cause
)
else
:
logger
.
debug
(
errs
.
decode
(
encoding
,
errors
=
'surrogateescape'
)
)
return
outs


def fetch(self, category=CATEGORY_TWEET, since_id=None, max_id=None,
              geocode=None, lang=None,
              include_entities=True, tweets_type=TWEET_TYPE_MIXED):
        """Fetch the tweets from the server.

        This method fetches tweets from the TwitterSearch API published in the last seven days.

        :param category: the category of items to fetch
        :param since_id: if not null, it returns results with an ID greater than the specified ID
        :param max_id: when it is set or if not None, it returns results with an ID less than the specified ID
        :param geocode: if enabled, returns tweets by users located at latitude,longitude,"mi"|"km"
        :param lang: if enabled, restricts tweets to the given language, given by an ISO 639-1 code
        :param include_entities: if disabled, it excludes entities node
        :param tweets_type: type of tweets returned. Default is “mixed”, others are "recent" and "popular"

        :returns: a generator of tweets
        """
        kwargs = {"since_id": since_id,
                  "max_id": max_id,
                  "geocode": geocode,
                  "lang": lang,
                  "include_entities": include_entities,
                  "result_type": tweets_type}
        items = super().fetch(category, **kwargs)

        return items

def fetch ( self , category = CATEGORY_TWEET , since_id = None , max_id = None , geocode = None , lang = None , include_entities = True , tweets_type = TWEET_TYPE_MIXED ) : kwargs = { "since_id" : since_id , "max_id" : max_id , "geocode" : geocode , "lang" : lang , "include_entities" : include_entities , "result_type" : tweets_type } items = super ( ) . fetch ( category , * * kwargs ) return items


def fetch_items(self, category, **kwargs):
        """Fetch the tweets

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        since_id = kwargs['since_id']
        max_id = kwargs['max_id']
        geocode = kwargs['geocode']
        lang = kwargs['lang']
        entities = kwargs['include_entities']
        tweets_type = kwargs['result_type']

        logger.info("Fetching tweets %s from %s to %s",
                    self.query, str(since_id),
                    str(max_id) if max_id else '--')

        tweets_ids = []
        min_date = None
        max_date = None
        group_tweets = self.client.tweets(self.query, since_id=since_id, max_id=max_id, geocode=geocode,
                                          lang=lang, include_entities=entities, result_type=tweets_type)

        for tweets in group_tweets:
            for i in range(len(tweets)):
                tweet = tweets[i]
                tweets_ids.append(tweet['id'])

                if tweets[-1] == tweet:
                    min_date = str_to_datetime(tweets[-1]['created_at'])

                if tweets[0] == tweet and not max_date:
                    max_date = str_to_datetime(tweets[0]['created_at'])

                yield tweet

        logger.info("Fetch process completed: %s (unique %s) tweets fetched, from %s to %s",
                    len(tweets_ids), len(list(set(tweets_ids))), min_date, max_date)


def tweets(self, query, since_id=None, max_id=None, geocode=None, lang=None,
               include_entities=True, result_type=TWEET_TYPE_MIXED):
        """Fetch tweets for a given query between since_id and max_id.

        :param query: query to fetch tweets
        :param since_id: if not null, it returns results with an ID greater than the specified ID
        :param max_id: if not null, it returns results with an ID less than the specified ID
        :param geocode: if enabled, returns tweets by users located at latitude,longitude,"mi"|"km"
        :param lang: if enabled, restricts tweets to the given language, given by an ISO 639-1 code
        :param include_entities: if disabled, it excludes entities node
        :param result_type: type of tweets returned. Default is “mixed”, others are "recent" and "popular"

        :returns: a generator of tweets
        """
        resource = self.base_url
        params = {'q': query,
                  'count': self.max_items}

        if since_id:
            params['since_id'] = since_id

        if max_id:
            params['max_id'] = max_id

        if geocode:
            params['geocode'] = geocode

        if lang:
            params['lang'] = lang

        params['include_entities'] = include_entities
        params['result_type'] = result_type

        while True:
            raw_tweets = self._fetch(resource, params=params)
            tweets = json.loads(raw_tweets)

            if not tweets['statuses']:
                break

            params['max_id'] = tweets['statuses'][-1]['id'] - 1
            yield tweets['statuses']

def tweets ( self , query , since_id = None , max_id = None , geocode = None , lang = None , include_entities = True , result_type = TWEET_TYPE_MIXED ) : resource = self . base_url params = { 'q' : query , 'count' : self . max_items } if since_id : params [ 'since_id' ] = since_id if max_id : params [ 'max_id' ] = max_id if geocode : params [ 'geocode' ] = geocode if lang : params [ 'lang' ] = lang params [ 'include_entities' ] = include_entities params [ 'result_type' ] = result_type while True : raw_tweets = self . _fetch ( resource , params = params ) tweets = json . loads ( raw_tweets ) if not tweets [ 'statuses' ] : break params [ 'max_id' ] = tweets [ 'statuses' ] [ - 1 ] [ 'id' ] - 1 yield tweets [ 'statuses' ]

Fetch tweets for a given query between since_id and max_id .


def setup_cmd_parser(cls):
        """Returns the Twitter argument parser."""

        parser = BackendCommandArgumentParser(cls.BACKEND.CATEGORIES,
                                              token_auth=True,
                                              archive=True)

        # Backend token is required
        action = parser.parser._option_string_actions['--api-token']
        action.required = True

        # Meetup options
        group = parser.parser.add_argument_group('Twitter arguments')
        group.add_argument('--max-items', dest='max_items',
                           type=int, default=MAX_ITEMS,
                           help="Maximum number of items requested on the same query")
        group.add_argument('--no-entities', dest='include_entities',
                           action='store_false',
                           help=" Exclude entities node")
        group.add_argument('--geo-code', dest='geocode',
                           help="Select tweets by users located at latitude,longitude,radius")
        group.add_argument('--lang', dest='lang',
                           help="Select tweets to the given language in ISO 639-1 code")
        group.add_argument('--tweets-type', dest='tweets_type', default=TWEET_TYPE_MIXED,
                           help="Type of tweets returned. Default is 'mixed', others are 'recent' and 'popular'")
        group.add_argument('--sleep-for-rate', dest='sleep_for_rate',
                           action='store_true',
                           help="sleep for getting more rate")
        group.add_argument('--min-rate-to-sleep', dest='min_rate_to_sleep',
                           default=MIN_RATE_LIMIT, type=int,
                           help="sleep until reset when the rate limit reaches this value")
        group.add_argument('--sleep-time', dest='sleep_time',
                           default=SLEEP_TIME, type=int,
                           help="minimun sleeping time to avoid too many request exception")

        # Required arguments
        parser.parser.add_argument('query',
                                   help="Search query including operators, max 500 chars")

        return parser


def fetch(self, category=CATEGORY_HITS):
        """Fetch data from Google API.

        The method retrieves a list of hits for some
        given keywords using the Google API.

        :param category: the category of items to fetch

        :returns: a generator of data
        """
        kwargs = {}
        items = super().fetch(category, **kwargs)

        return items

def fetch ( self , category = CATEGORY_HITS ) : kwargs = { } items = super ( ) . fetch ( category , * * kwargs ) return items


def fetch_items(self, category, **kwargs):
        """Fetch Google hit items

        :param category: the category of items to fetch
        :param kwargs: backend arguments

        :returns: a generator of items
        """
        logger.info("Fetching data for '%s'", self.keywords)

        hits_raw = self.client.hits(self.keywords)
        hits = self.__parse_hits(hits_raw)

        yield hits

        logger.info("Fetch process completed")

def
fetch_items
(
self
,
category
,
*
*
kwargs
)
:
logger
.
info
(
"Fetching data for '%s'"
,
self
.
keywords
)
hits_raw
=
self
.
client
.
hits
(
self
.
keywords
)
hits
=
self
.
__parse_hits
(
hits_raw
)
yield
hits
logger
.
info
(
"Fetch process completed"
)


def __parse_hits(self, hit_raw):
        """Parse the hits returned by the Google Search API"""

        # Create the soup and get the desired div
        bs_result = bs4.BeautifulSoup(hit_raw, 'html.parser')
        hit_string = bs_result.find("div", id="resultStats").text

        # Remove commas or dots
        hit_string = hit_string.replace(',', u'')
        hit_string = hit_string.replace('.', u'')

        fetched_on = datetime_utcnow().timestamp()
        id_args = self.keywords[:]
        id_args.append(str(fetched_on))

        hits_json = {
            'fetched_on': fetched_on,
            'id': uuid(*id_args),
            'keywords': self.keywords,
            'type': 'googleSearchHits'
        }

        if not hit_string:
            logger.warning("No hits for %s", self.keywords)
            hits_json['hits'] = 0

            return hits_json

        str_hits = re.search(r'\d+', hit_string).group(0)
        hits = int(str_hits)
        hits_json['hits'] = hits

        return hits_json


def hits(self, keywords):
        """Fetch information about a list of keywords."""

        if len(keywords) == 1:
            query_str = keywords[0]
        else:
            query_str = ' '.join([k for k in keywords])

        logger.info("Fetching hits for '%s'", query_str)
        params = {'q': query_str}

        # Make the request
        req = self.fetch(GOOGLE_SEARCH_URL, payload=params)

        return req.text

def
hits
(
self
,
keywords
)
:
if
len
(
keywords
)
==
1
:
query_str
=
keywords
[
0
]
else
:
query_str
=
' '
.
join
(
[
k
for
k
in
keywords
]
)
logger
.
info
(
"Fetching hits for '%s'"
,
query_str
)
params
=
{
'q'
:
query_str
}
# Make the request
req
=
self
.
fetch
(
GOOGLE_SEARCH_URL
,
payload
=
params
)
return
req
.
text

Fetch information about a list of keywords .


def metadata_updated_on(item):
        """Extracts the update time from a GitHub item.

        The timestamp used is extracted from 'updated_at' field.
        This date is converted to UNIX timestamp format. As GitHub
        dates are in UTC the conversion is straightforward.

        :param item: item generated by the backend

        :returns: a UNIX timestamp
        """
        if "forks_count" in item:
            return item['fetched_on']
        else:
            ts = item['updated_at']
            ts = str_to_datetime(ts)

            return ts.timestamp()

def metadata_updated_on ( item ) : if "forks_count" in item : return item [ 'fetched_on' ] else : ts = item [ 'updated_at' ] ts = str_to_datetime ( ts ) return ts . timestamp ( )

Extracts the update time from a GitHub item .


def metadata_category(item):
        """Extracts the category from a GitHub item.

        This backend generates two types of item which are
        'issue' and 'pull_request'.
        """

        if "base" in item:
            category = CATEGORY_PULL_REQUEST
        elif "forks_count" in item:
            category = CATEGORY_REPO
        else:
            category = CATEGORY_ISSUE

        return category

def metadata_category ( item ) : if "base" in item : category = CATEGORY_PULL_REQUEST elif "forks_count" in item : category = CATEGORY_REPO else : category = CATEGORY_ISSUE return category

Extracts the category from a GitHub item .


def __fetch_pull_requests(self, from_date, to_date):
        """Fetch the pull requests"""

        raw_pulls = self.client.pulls(from_date=from_date)
        for raw_pull in raw_pulls:
            pull = json.loads(raw_pull)

            if str_to_datetime(pull['updated_at']) > to_date:
                return

            self.__init_extra_pull_fields(pull)
            for field in TARGET_PULL_FIELDS:

                if not pull[field]:
                    continue

                if field == 'user':
                    pull[field + '_data'] = self.__get_user(pull[field]['login'])
                elif field == 'merged_by':
                    pull[field + '_data'] = self.__get_user(pull[field]['login'])
                elif field == 'review_comments':
                    pull[field + '_data'] = self.__get_pull_review_comments(pull['number'])
                elif field == 'requested_reviewers':
                    pull[field + '_data'] = self.__get_pull_requested_reviewers(pull['number'])
                elif field == 'commits':
                    pull[field + '_data'] = self.__get_pull_commits(pull['number'])

            yield pull

def __fetch_pull_requests ( self , from_date , to_date ) : raw_pulls = self . client . pulls ( from_date = from_date ) for raw_pull in raw_pulls : pull = json . loads ( raw_pull ) if str_to_datetime ( pull [ 'updated_at' ] ) > to_date : return self . __init_extra_pull_fields ( pull ) for field in TARGET_PULL_FIELDS : if not pull [ field ] : continue if field == 'user' : pull [ field + '_data' ] = self . __get_user ( pull [ field ] [ 'login' ] ) elif field == 'merged_by' : pull [ field + '_data' ] = self . __get_user ( pull [ field ] [ 'login' ] ) elif field == 'review_comments' : pull [ field + '_data' ] = self . __get_pull_review_comments ( pull [ 'number' ] ) elif field == 'requested_reviewers' : pull [ field + '_data' ] = self . __get_pull_requested_reviewers ( pull [ 'number' ] ) elif field == 'commits' : pull [ field + '_data' ] = self . __get_pull_commits ( pull [ 'number' ] ) yield pull


def __fetch_repo_info(self):
        """Get repo info about stars, watchers and forks"""

        raw_repo = self.client.repo()
        repo = json.loads(raw_repo)

        fetched_on = datetime_utcnow()
        repo['fetched_on'] = fetched_on.timestamp()

        yield repo

def __fetch_repo_info ( self ) : raw_repo = self . client . repo ( ) repo = json . loads ( raw_repo ) fetched_on = datetime_utcnow ( ) repo [ 'fetched_on' ] = fetched_on . timestamp ( ) yield repo

Get repo info about stars watchers and forks


def __get_issue_reactions(self, issue_number, total_count):
        """Get issue reactions"""

        reactions = []

        if total_count == 0:
            return reactions

        group_reactions = self.client.issue_reactions(issue_number)

        for raw_reactions in group_reactions:

            for reaction in json.loads(raw_reactions):
                reaction['user_data'] = self.__get_user(reaction['user']['login'])
                reactions.append(reaction)

        return reactions

def __get_issue_reactions ( self , issue_number , total_count ) : reactions = [ ] if total_count == 0 : return reactions group_reactions = self . client . issue_reactions ( issue_number ) for raw_reactions in group_reactions : for reaction in json . loads ( raw_reactions ) : reaction [ 'user_data' ] = self . __get_user ( reaction [ 'user' ] [ 'login' ] ) reactions . append ( reaction ) return reactions


def __get_issue_comment_reactions(self, comment_id, total_count):
        """Get reactions on issue comments"""

        reactions = []

        if total_count == 0:
            return reactions

        group_reactions = self.client.issue_comment_reactions(comment_id)

        for raw_reactions in group_reactions:

            for reaction in json.loads(raw_reactions):
                reaction['user_data'] = self.__get_user(reaction['user']['login'])
                reactions.append(reaction)

        return reactions

def __get_issue_comment_reactions ( self , comment_id , total_count ) : reactions = [ ] if total_count == 0 : return reactions group_reactions = self . client . issue_comment_reactions ( comment_id ) for raw_reactions in group_reactions : for reaction in json . loads ( raw_reactions ) : reaction [ 'user_data' ] = self . __get_user ( reaction [ 'user' ] [ 'login' ] ) reactions . append ( reaction ) return reactions


def __get_issue_assignees(self, raw_assignees):
        """Get issue assignees"""

        assignees = []
        for ra in raw_assignees:
            assignees.append(self.__get_user(ra['login']))

        return assignees

def __get_issue_assignees ( self , raw_assignees ) : assignees = [ ] for ra in raw_assignees : assignees . append ( self . __get_user ( ra [ 'login' ] ) ) return assignees


def __get_pull_requested_reviewers(self, pr_number):
        """Get pull request requested reviewers"""

        requested_reviewers = []
        group_requested_reviewers = self.client.pull_requested_reviewers(pr_number)

        for raw_requested_reviewers in group_requested_reviewers:
            group_requested_reviewers = json.loads(raw_requested_reviewers)

            for requested_reviewer in group_requested_reviewers['users']:
                user_data = self.__get_user(requested_reviewer['login'])
                requested_reviewers.append(user_data)

        return requested_reviewers

def __get_pull_requested_reviewers ( self , pr_number ) : requested_reviewers = [ ] group_requested_reviewers = self . client . pull_requested_reviewers ( pr_number ) for raw_requested_reviewers in group_requested_reviewers : group_requested_reviewers = json . loads ( raw_requested_reviewers ) for requested_reviewer in group_requested_reviewers [ 'users' ] : user_data = self . __get_user ( requested_reviewer [ 'login' ] ) requested_reviewers . append ( user_data ) return requested_reviewers


def __get_pull_commits(self, pr_number):
        """Get pull request commit hashes"""

        hashes = []
        group_pull_commits = self.client.pull_commits(pr_number)

        for raw_pull_commits in group_pull_commits:

            for commit in json.loads(raw_pull_commits):
                commit_hash = commit['sha']
                hashes.append(commit_hash)

        return hashes

def __get_pull_commits ( self , pr_number ) : hashes = [ ] group_pull_commits = self . client . pull_commits ( pr_number ) for raw_pull_commits in group_pull_commits : for commit in json . loads ( raw_pull_commits ) : commit_hash = commit [ 'sha' ] hashes . append ( commit_hash ) return hashes


def __get_pull_review_comment_reactions(self, comment_id, total_count):
        """Get pull review comment reactions"""

        reactions = []

        if total_count == 0:
            return reactions

        group_reactions = self.client.pull_review_comment_reactions(comment_id)

        for raw_reactions in group_reactions:

            for reaction in json.loads(raw_reactions):
                reaction['user_data'] = self.__get_user(reaction['user']['login'])
                reactions.append(reaction)

        return reactions

def __get_pull_review_comment_reactions ( self , comment_id , total_count ) : reactions = [ ] if total_count == 0 : return reactions group_reactions = self . client . pull_review_comment_reactions ( comment_id ) for raw_reactions in group_reactions : for reaction in json . loads ( raw_reactions ) : reaction [ 'user_data' ] = self . __get_user ( reaction [ 'user' ] [ 'login' ] ) reactions . append ( reaction ) return reactions


def __get_user(self, login):
        """Get user and org data for the login"""

        user = {}

        if not login:
            return user

        user_raw = self.client.user(login)
        user = json.loads(user_raw)
        user_orgs_raw = \
            self.client.user_orgs(login)
        user['organizations'] = json.loads(user_orgs_raw)

        return user

def __get_user ( self , login ) : user = { } if not login : return user user_raw = self . client . user ( login ) user = json . loads ( user_raw ) user_orgs_raw = self . client . user_orgs ( login ) user [ 'organizations' ] = json . loads ( user_orgs_raw ) return user


def issue_reactions(self, issue_number):
        """Get reactions of an issue"""

        payload = {
            'per_page': PER_PAGE,
            'direction': 'asc',
            'sort': 'updated'
        }

        path = urijoin("issues", str(issue_number), "reactions")
        return self.fetch_items(path, payload)

def issue_reactions ( self , issue_number ) : payload = { 'per_page' : PER_PAGE , 'direction' : 'asc' , 'sort' : 'updated' } path = urijoin ( "issues" , str ( issue_number ) , "reactions" ) return self . fetch_items ( path , payload )


def issues(self, from_date=None):
        """Fetch the issues from the repository.

        The method retrieves, from a GitHub repository, the issues
        updated since the given date.

        :param from_date: obtain issues updated since this date

        :returns: a generator of issues
        """
        payload = {
            'state': 'all',
            'per_page': PER_PAGE,
            'direction': 'asc',
            'sort': 'updated'}

        if from_date:
            payload['since'] = from_date.isoformat()

        path = urijoin("issues")
        return self.fetch_items(path, payload)

def issues ( self , from_date = None ) : payload = { 'state' : 'all' , 'per_page' : PER_PAGE , 'direction' : 'asc' , 'sort' : 'updated' } if from_date : payload [ 'since' ] = from_date . isoformat ( ) path = urijoin ( "issues" ) return self . fetch_items ( path , payload )


def pulls(self, from_date=None):
        """Fetch the pull requests from the repository.

        The method retrieves, from a GitHub repository, the pull requests
        updated since the given date.

        :param from_date: obtain pull requests updated since this date

        :returns: a generator of pull requests
        """
        issues_groups = self.issues(from_date=from_date)

        for raw_issues in issues_groups:
            issues = json.loads(raw_issues)
            for issue in issues:

                if "pull_request" not in issue:
                    continue

                pull_number = issue["number"]
                path = urijoin(self.base_url, 'repos', self.owner, self.repository, "pulls", pull_number)

                r = self.fetch(path)
                pull = r.text

                yield pull

def pulls ( self , from_date = None ) : issues_groups = self . issues ( from_date = from_date ) for raw_issues in issues_groups : issues = json . loads ( raw_issues ) for issue in issues : if "pull_request" not in issue : continue pull_number = issue [ "number" ] path = urijoin ( self . base_url , 'repos' , self . owner , self . repository , "pulls" , pull_number ) r = self . fetch ( path ) pull = r . text yield pull

Fetch the pull requests from the repository .


def repo(self):
        """Get repository data"""

        path = urijoin(self.base_url, 'repos', self.owner, self.repository)

        r = self.fetch(path)
        repo = r.text

        return repo

def repo ( self ) : path = urijoin ( self . base_url , 'repos' , self . owner , self . repository ) r = self . fetch ( path ) repo = r . text return repo


def pull_requested_reviewers(self, pr_number):
        """Get pull requested reviewers"""

        requested_reviewers_url = urijoin("pulls", str(pr_number), "requested_reviewers")
        return self.fetch_items(requested_reviewers_url, {})

def pull_requested_reviewers ( self , pr_number ) : requested_reviewers_url = urijoin ( "pulls" , str ( pr_number ) , "requested_reviewers" ) return self . fetch_items ( requested_reviewers_url , { } )


def pull_commits(self, pr_number):
        """Get pull request commits"""

        payload = {
            'per_page': PER_PAGE,
        }

        commit_url = urijoin("pulls", str(pr_number), "commits")
        return self.fetch_items(commit_url, payload)

def pull_commits ( self , pr_number ) : payload = { 'per_page' : PER_PAGE , } commit_url = urijoin ( "pulls" , str ( pr_number ) , "commits" ) return self . fetch_items ( commit_url , payload )


def pull_review_comment_reactions(self, comment_id):
        """Get reactions of a review comment"""

        payload = {
            'per_page': PER_PAGE,
            'direction': 'asc',
            'sort': 'updated'
        }

        path = urijoin("pulls", "comments", str(comment_id), "reactions")
        return self.fetch_items(path, payload)

def pull_review_comment_reactions ( self , comment_id ) : payload = { 'per_page' : PER_PAGE , 'direction' : 'asc' , 'sort' : 'updated' } path = urijoin ( "pulls" , "comments" , str ( comment_id ) , "reactions" ) return self . fetch_items ( path , payload )


def user(self, login):
        """Get the user information and update the user cache"""
        user = None

        if login in self._users:
            return self._users[login]

        url_user = urijoin(self.base_url, 'users', login)

        logging.info("Getting info for %s" % (url_user))

        r = self.fetch(url_user)
        user = r.text
        self._users[login] = user

        return user

def
user
(
self
,
login
)
:
user
=
if
login
in
self
.
_users
:
return
self
.
_users
[
login
]
url_user
=
urijoin
(
self
.
base_url
,
'users'
,
login
)
logging
.
info
(
"Getting info for %s"
%
(
url_user
)
)
r
=
self
.
fetch
(
url_user
)
user
=
r
.
text
self
.
_users
[
login
]
=
user
return
user


def user_orgs(self, login):
        """Get the user public organizations"""
        if login in self._users_orgs:
            return self._users_orgs[login]

        url = urijoin(self.base_url, 'users', login, 'orgs')
        try:
            r = self.fetch(url)
            orgs = r.text
        except requests.exceptions.HTTPError as error:
            # 404 not found is wrongly received sometimes
            if error.response.status_code == 404:
                logger.error("Can't get github login orgs: %s", error)
                orgs = '[]'
            else:
                raise error

        self._users_orgs[login] = orgs

        return orgs


def _get_token_rate_limit(self, token):
        """Return token's remaining API points"""

        rate_url = urijoin(self.base_url, "rate_limit")
        self.session.headers.update({'Authorization': 'token ' + token})
        remaining = 0
        try:
            headers = super().fetch(rate_url).headers
            if self.rate_limit_header in headers:
                remaining = int(headers[self.rate_limit_header])
        except requests.exceptions.HTTPError as error:
            logger.warning("Rate limit not initialized: %s", error)
        return remaining

def
_get_token_rate_limit
(
self
,
token
)
:
rate_url
=
urijoin
(
self
.
base_url
,
"rate_limit"
)
self
.
session
.
headers
.
update
(
{
'Authorization'
:
'token '
+
token
}
)
remaining
=
0
try
:
headers
=
super
(
)
.
fetch
(
rate_url
)
.
headers
if
self
.
rate_limit_header
in
headers
:
remaining
=
int
(
headers
[
self
.
rate_limit_header
]
)
except
requests
.
exceptions
.
HTTPError
as
error
:
logger
.
warning
(
"Rate limit not initialized: %s"
,
error
)
return
remaining