"""Queries for and processes xmatters instance data

.. _Google Python Style Guide:
   http://google.github.io/styleguide/pyguide.html

"""

import json
import sys
import pprint
from io import TextIOBase
import urllib.parse

import requests
from requests.auth import HTTPBasicAuth

import config
import common_logger

_logger = None
_users = None
_sites_cache = {}
# Holds admin info: company_admins, roles, timezones, country, language
_admin_objects = None

def _update_admin(a_type: str, a_value: str):
    """Updates the admin objects set
        
    Puts the value into the appropriate set.
        
    Args:
        a_type (str): The name of a set in the admin dict
        a_value (str): The value to add to the dict
    """
    _admin_objects[a_type].add(a_value)

def _log_xm_error(url, response):
    """Captures and logs errors
        
        Logs the error caused by attempting to call url.
        
        Args:
        url (str): The location being requested that caused the error
        response (object): JSON object that holds the error response
        """
    body = response.json()
    if response.status_code == 404:
        _logger.warn(config.ERR_INITIAL_REQUEST_FAILED_MSG,
                     response.status_code, url)
    else:
        _logger.error(config.ERR_INITIAL_REQUEST_FAILED_MSG,
                      response.status_code, url)
        _logger.error('Response - code: %s, reason: %s, message: %s',
                    str(body['code']) if 'code' in body else "none",
                    str(body['reason']) if 'reason' in body else "none",
                    str(body['message']) if 'message' in body else "none")

def _create_out_file(filename: str) -> TextIOBase:
    """Creates and opens results file

    Args:
        filename (str): Name of file to hold output

    Returns:
        file: outFile
    """
    outFile = open(filename, 'w')
    return outFile

def _lookup_site_name(site_id: str):
    """Retrieves a Site name by ID

    Attempts to find the Site by it's ID in the Site cache,
    if not found, then retrieve the Site object from xMatters.

    Args:
        site_id (str): The ID of the site to find

    Return:
        None
    """
    if site_id in _sites_cache:
        return _sites_cache[site_id]
    
    # Site was not in the Cache, so get it from xMatters

    # Initialize conditions
    url = config.xmod_url + '/api/xm/1/sites/' + site_id
    _logger.debug('Retrieving Site, url=%s', url)

    # Get the site records
    response = requests.get(url, auth=config.basic_auth)
    if response.status_code not in [200, 404]:
        _log_xm_error(url, response)
        _sites_cache[site_id] = None
        return None

    # Process the responses
    site = response.json()
    _sites_cache[site['id']] = site['name']
    return site['name']

def _process_sites():
    """Capture and save the instances Site objects

    Retrieves the Site object records from xMatters and saves them in
    JSON payload format to the output file.

    Args:
        None

    Return:
        None
    """
    _logger.info('Begin Gathering Sites.')
    sites_file = _create_out_file(config.sites_filename)
    sites_file.write('[\n')

    # Initialize conditions
    site_objects = []
    total_sites = 0
    cnt = 0
    url = config.xmod_url + '/api/xm/1/sites?offset=0&limit=' + str(config.page_size)
    _logger.debug('Gathering Sites via url=%s', url)

    while True:

        # Get the site records
        response = requests.get(url, auth=config.basic_auth)
        if response.status_code not in [200]:
            _log_xm_error(url, response)
            break

        # Process the responses
        bodys = response.json()
        total_sites = bodys['total']
        if bodys['count'] > 0:
            _logger.debug("%d Count of %d Total Sites found via url=%s", bodys['count'], bodys['total'], url)
            for body in bodys['data']:
                cnt += 1
                _logger.info(f'Capturing Site "{body["name"]}"')
                json.dump(body, sites_file)
                sites_file.write(',\n') if cnt < total_sites else sites_file.write('\n')
                _sites_cache[body['id']] = body['name']
                # Update admin sets
                if 'language' in body: _update_admin('languages', body['language'])
                if 'timezone' in body: _update_admin('timezones', body['timezone'])
                if 'country' in body: _update_admin('countries', body['country'])
            site_objects += bodys['data']

        # See if there are any more to get
        if 'links' in bodys and 'next' in bodys['links']:
            url = config.xmod_url + bodys['links']['next']
        else:
            break
            
    _logger.info("Collected %d of a possible %d Sites.", len(site_objects), total_sites)

    sites_file.write(']')
    sites_file.close()

def _get_user_devices(user_id: str, target_name: str):
    """Return a User's Devices

    Retrieves the Device records from xMatters for the specified User and
    returns them as a list.

    Args:
        user_id (str): The User's UUID
        target_name (str): The User's targetName field

    Return:
        device_list (list): List of dictionaries of the User's devices.
    """
    # Initialize conditions
    device_list = []
    total_devices = 0
    url = config.xmod_url + '/api/xm/1/people/' + user_id + '/devices/?embed=timeframes&offset=0&limit=' + str(config.page_size)
    _logger.debug('Gathering Devices for user "%s", url=%s', target_name, url)

    while True:

        # Get the site records
        response = requests.get(url, auth=config.basic_auth)
        if response.status_code not in [200, 404]:
            _log_xm_error(url, response)
            break

        # Process the responses
        bodys = response.json()
        total_devices = bodys['total']
        if bodys['count'] > 0:
            _logger.debug('%d Count of %d Total Devices found for User "%s" via url=%s', bodys['count'], bodys['total'], target_name, url)
            device_list += bodys['data']
            # Use Timeframe to update the timezones admin set
            for body in bodys['data']:
                if 'timeframes' in body:
                    for timeframe in body['timeframes']:
                        if 'timezone' in timeframe: _update_admin('timezones', timeframe['timezone'])
                _update_admin('devices', body['deviceType'] + '|' + body['name'])
                if 'provider' in body: _update_admin('usps', body['provider']['id'])

        # See if there are any more to get
        if 'links' in bodys and 'next' in bodys['links']:
            url = config.xmod_url + bodys['links']['next']
        else:
            break

    _logger.debug('Collected %d of a possible %d Devices for User "%s".', len(device_list), total_devices, target_name)

    return device_list

def _get_user(user_id: str, target_name: str):
    """Attempst to retrieve User by id.
        
        If the User exists, retrieve and return the object.
        If not, return None
        
        Args:
        user_id (str): UUID of User to retrieve
        target_name (str): targetName field value for the specified user.
        """
    _logger.info(f'Capturing User: "{target_name}".')
    
    # Set our resource URLs
    url = config.xmod_url + '/api/xm/1/people/' + urllib.parse.quote(user_id) + '?embed=roles,supervisors'
    _logger.debug('Attempting to retrieve User "%s" via url: %s', target_name, url)
    
    # Make the request
    try:
        response = requests.get(url, auth=config.basic_auth)
    except requests.exceptions.RequestException as e:
        _logger.error(config.ERR_REQUEST_EXCEPTION_CODE, url, repr(e))
        return None
    
    # If the initial response fails, log and return null
    if response.status_code != 200:
        _log_xm_error(url, response)
        return None
    
    # Process the response
    user_obj = response.json()
    # _logger.debug('Found User "%s" - json body: %s', user_obj['firstName'] + ' ' + user_obj['lastName'], pprint.pformat(user_obj))
    _logger.debug('Found User "%s" - json body.id: %s', user_obj['firstName'] + ' ' + user_obj['lastName'], user_obj['id'])

    # Update the Admin object
    if 'language' in user_obj: _update_admin('languages', user_obj['language'])
    if 'timezone' in user_obj: _update_admin('timezones', user_obj['timezone'])
    if 'roles' in user_obj and user_obj['roles']['total'] > 0:
        for role in user_obj['roles']['data']:
            _update_admin('roles', role['name'])
            if role['name'] == config.company_admin_role:
                _update_admin('admins', user_obj['targetName'])

    return user_obj

def _process_users(include_devices: bool):
    """Capture and save the instances User objects

    Retrieves the User object records from xMatters and saves them in
    JSON payload format to the output file.

    Args:
        include_devices (bool): If True, get the User's devices too

    Return:
        None
    """
    _logger.info('Begin gathering Users.')
    users_file = _create_out_file(config.users_filename)
    users_file.write('[\n')

    # Initialize conditions
    user_objects = []
    total_users = 0
    cnt = 0
    url = config.xmod_url + '/api/xm/1/people?offset=0&limit=' + str(config.page_size)
    _logger.debug('Gathering Users via url=%s', url)

    while True:

        # Get the user records
        response = requests.get(url, auth=config.basic_auth)
        if response.status_code not in [200, 404]:
            _log_xm_error(url, response)
            break

        # Process the responses
        bodys = response.json()
        total_users = bodys['total']
        if bodys['count'] > 0:
            _logger.debug("%d Count of %d Total Users found via url=%s", bodys['count'], bodys['total'], url)
            for body in bodys['data']:

                # Get the full user object, including Roles and Supervisors
                user_obj = {}
                a_user = _get_user(body['id'], body['targetName'])
                if a_user is not None:
                    user_obj['user'] = a_user

                    # Get the devices, if requested
                    if include_devices:
                        user_obj['devices'] = _get_user_devices(a_user['id'], a_user['targetName'])

                    # Save the User
                    cnt += 1
                    json.dump(user_obj, users_file)
                    users_file.write(',\n') if cnt < total_users else users_file.write('\n')

            user_objects += bodys['data']

        # See if there are any more to get
        if 'links' in bodys and 'next' in bodys['links']:
            url = config.xmod_url + bodys['links']['next']
        else:
            break
            
    _logger.info("Collected %d of a possible %d Users.", len(user_objects), total_users)

    users_file.write(']')
    users_file.close()

def _get_group(group_id: str, target_name: str):
    """Attempst to retrieve Group by id.
        
        If the Group exists, retrieve and return the object.
        If not, return None
        
        Args:
        group_id (str): UUID of Group to retrieve
        target_name (str): targetName field value for the specified Group.
        """
    _logger.info(f"Retrieving Group: {target_name}")
    
    # Set our resource URLs
    url = config.xmod_url + '/api/xm/1/groups/' + urllib.parse.quote(group_id) + '?embed=supervisors'
    _logger.debug(f'Attempting to retrieve Group "{target_name}" via url: {url}')
    
    # Make the request
    try:
        response = requests.get(url, auth=config.basic_auth)
    except requests.exceptions.RequestException as e:
        _logger.error(config.ERR_REQUEST_EXCEPTION_CODE, url, repr(e))
        return None
    
    # If the initial response fails, log and return null
    if response.status_code != 200:
        _log_xm_error(url, response)
        return None
    
    # Process the response
    group_obj = response.json()
    # If present, translate the Site from an ID to a name
    if 'site' in group_obj:
        site_name = _lookup_site_name(group_obj['site']['id'])
        del group_obj['site']
        group_obj['site'] = site_name
    # _logger.debug(f'Found Group "{group_obj["targetName"]}" - json body: {pprint.pformat(group_obj)}')
    _logger.debug(f'Found Group "{group_obj["targetName"]}" - json body.id: {group_obj["id"]}')
    return group_obj

def _get_group_shifts(group_id: str, target_name: str):
    """Return a Group's Shifts

    Retrieves the Shift records from xMatters for the specified Group and
    returns them as a list.

    Args:
        group_id (str): The Group's UUID
        target_name (str): The Group's targetName field

    Return:
        shift_list (list): List of dictionaries of the Group's Shifts.
    """
    # Initialize conditions
    shift_list = []
    total_shifts = 0
    url = config.xmod_url + '/api/xm/1/groups/' + group_id + '/shifts/?embed=members,rotation&offset=0&limit=' + str(config.page_size)
    _logger.debug(f'Gathering Shifts for Group "{target_name}", url={url}')

    while True:

        # Get the site records
        response = requests.get(url, auth=config.basic_auth)
        if response.status_code not in [200, 404]:
            _log_xm_error(url, response)
            break

        # Process the responses
        bodys = response.json()
        total_shifts = bodys['total']
        if bodys['count'] > 0:
            _logger.debug('%d Count of %d Total Shifts found for Group "%s" via url=%s', bodys['count'], bodys['total'], target_name, url)
            shift_list += bodys['data']

        # See if there are any more to get
        if 'links' in bodys and 'next' in bodys['links']:
            url = config.xmod_url + bodys['links']['next']
        else:
            break

    _logger.debug(f'Collected {len(shift_list)} of a possible {total_shifts} Shifts for Group "{target_name}".')

    return shift_list

def _process_groups():
    """Capture and save the instances Group objects

    Retrieves the User's Group object records from xMatters and saves them in
    JSON payload format to the output file.

    Args:
        None

    Return:
        None
    """
    _logger.info('Begin capturing Groups.')
    groups_file = _create_out_file(config.groups_filename)
    groups_file.write('[\n')

    # Initialize conditions
    group_objects = []
    total_groups = 0
    cnt = 0
    url = config.xmod_url + '/api/xm/1/groups?offset=0&limit=' + str(config.page_size)
    _logger.debug('Gathering Groups via url=%s', url)

    while True:

        # Get the group records
        response = requests.get(url, auth=config.basic_auth)
        if response.status_code not in [200, 404]:
            _log_xm_error(url, response)
            break

        # Process the responses
        bodys = response.json()
        total_groups = bodys['total']
        if bodys['count'] > 0:
            _logger.debug("%d Count of %d Total Groups found via url=%s", bodys['count'], bodys['total'], url)
            for body in bodys['data']:

                # Get the full Group object, including Roles and Supervisors
                group_obj = {}
                a_group = _get_group(body['id'], body['targetName'])
                if a_group is not None:
                    group_obj['group'] = a_group

                    # Get the shifts,
                    group_obj['shifts'] = _get_group_shifts(a_group['id'], a_group['targetName'])

                    # Save the Group
                    cnt += 1
                    json.dump(group_obj, groups_file)
                    groups_file.write(',\n') if cnt < total_groups else groups_file.write('\n')

            group_objects += bodys['data']

        # See if there are any more to get
        if 'links' in bodys and 'next' in bodys['links']:
            url = config.xmod_url + bodys['links']['next']
        else:
            break
            
    _logger.info(f"Collected {len(group_objects)} of a possible {total_groups} Groups.")

    groups_file.write(']')
    groups_file.close()

def _save_admin_data():
    """Saves the collected admin sets

    Preservs the collected admin sets to the file system.

    Args:
        None

    Return:
        None
    """
    admin_dict = {
        'admins': list(_admin_objects['admins']),
        'roles': list(_admin_objects['roles']),
        'timezones': list(_admin_objects['timezones']),
        'countries': list(_admin_objects['countries']),
        'languages': list(_admin_objects['languages']),
        'devices': list(_admin_objects['devices']),
        'usps': list(_admin_objects['usps'])
    }
    admin_file = _create_out_file(config.admin_filename)
    json.dump(admin_dict, admin_file, indent=2)
    admin_file.close()

def process(objects_to_process: list):
    """Capture objects for this instance.

    If requeste contains 'sites', then capture and save Sites.
    If requeste contains 'users', then capture and save Users.
    If requeste contains 'devices', then capture and save Devices.
    If requeste contains 'groups', then capture and save Groups.

    Args:
        objects_to_process (list): The list of object types to capture
    """
    global _logger, _admin_objects # pylint: disable=global-statement

    ### Get the current logger
    _logger = common_logger.get_logger()

    # Initialize the Admin object
    _admin_objects = {
        'admins': set(),
        'roles': set(),
        'timezones': set(),
        'countries': set(),
        'languages': set(),
        'devices': set(),
        'usps': set()
    }

    # Capture and save the Site objects
    if 'sites' in objects_to_process:
        _process_sites()

    # Capture and save the User objects, and possibly devices
    if 'users' in objects_to_process:
        _process_users('devices' in objects_to_process)
    elif 'devices' in objects_to_process:
        _process_users(True)

    # Capture and save the Device objects
    if 'groups' in objects_to_process:
        _process_groups()

    # Preserve the collected admin data
    _save_admin_data()

def main():
    """In case we need to execute the module directly"""
    pass

if __name__ == '__main__':
    main()

"""Queries for and processes xmatters instance data

.. _Google Python Style Guide:
   http://google.github.io/styleguide/pyguide.html

"""

import json
import sys
import pprint
from io import TextIOBase
import urllib.parse

import requests
from requests.auth import HTTPBasicAuth

import config
import common_logger

_logger = None
_users = None
_sites_cache = {}
# Holds admin info: company_admins, roles, timezones, country, language
_admin_objects = None

def _update_admin(a_type: str, a_value: str):
    """Updates the admin objects set
        
    Puts the value into the appropriate set.
        
    Args:
        a_type (str): The name of a set in the admin dict
        a_value (str): The value to add to the dict
    """
    _admin_objects[a_type].add(a_value)

def _log_xm_error(url, response):
    """Captures and logs errors
        
        Logs the error caused by attempting to call url.
        
        Args:
        url (str): The location being requested that caused the error
        response (object): JSON object that holds the error response
        """
    body = response.json()
    if response.status_code == 404:
        _logger.warn(config.ERR_INITIAL_REQUEST_FAILED_MSG,
                     response.status_code, url)
    else:
        _logger.error(config.ERR_INITIAL_REQUEST_FAILED_MSG,
                      response.status_code, url)
        _logger.error('Response - code: %s, reason: %s, message: %s',
                    str(body['code']) if 'code' in body else "none",
                    str(body['reason']) if 'reason' in body else "none",
                    str(body['message']) if 'message' in body else "none")

def _create_out_file(filename: str) -> TextIOBase:
    """Creates and opens results file

    Args:
        filename (str): Name of file to hold output

    Returns:
        file: outFile
    """
    outFile = open(filename, 'w')
    return outFile

def _lookup_site_name(site_id: str):
    """Retrieves a Site name by ID

    Attempts to find the Site by it's ID in the Site cache,
    if not found, then retrieve the Site object from xMatters.

    Args:
        site_id (str): The ID of the site to find

    Return:
        None
    """
    if site_id in _sites_cache:
        return _sites_cache[site_id]
    
    # Site was not in the Cache, so get it from xMatters

    # Initialize conditions
    url = config.xmod_url + '/api/xm/1/sites/' + site_id
    _logger.debug('Retrieving Site, url=%s', url)

    # Get the site records
    response = requests.get(url, auth=config.basic_auth)
    if response.status_code not in [200, 404]:
        _log_xm_error(url, response)
        _sites_cache[site_id] = None
        return None

    # Process the responses
    site = response.json()
    _sites_cache[site['id']] = site['name']
    return site['name']

def _process_sites():
    """Capture and save the instances Site objects

    Retrieves the Site object records from xMatters and saves them in
    JSON payload format to the output file.

    Args:
        None

    Return:
        None
    """
    _logger.info('Begin Gathering Sites.')
    sites_file = _create_out_file(config.sites_filename)
    sites_file.write('[\n')

    # Initialize conditions
    site_objects = []
    total_sites = 0
    cnt = 0
    url = config.xmod_url + '/api/xm/1/sites?offset=0&limit=' + str(config.page_size)
    _logger.debug('Gathering Sites via url=%s', url)

    while True:

        # Get the site records
        response = requests.get(url, auth=config.basic_auth)
        if response.status_code not in [200]:
            _log_xm_error(url, response)
            break

        # Process the responses
        bodys = response.json()
        total_sites = bodys['total']
        if bodys['count'] > 0:
            _logger.debug("%d Count of %d Total Sites found via url=%s", bodys['count'], bodys['total'], url)
            for body in bodys['data']:
                cnt += 1
                _logger.info(f'Capturing Site "{body["name"]}"')
                json.dump(body, sites_file)
                sites_file.write(',\n') if cnt < total_sites else sites_file.write('\n')
                _sites_cache[body['id']] = body['name']
                # Update admin sets
                if 'language' in body: _update_admin('languages', body['language'])
                if 'timezone' in body: _update_admin('timezones', body['timezone'])
                if 'country' in body: _update_admin('countries', body['country'])
            site_objects += bodys['data']

        # See if there are any more to get
        if 'links' in bodys and 'next' in bodys['links']:
            url = config.xmod_url + bodys['links']['next']
        else:
            break
            
    _logger.info("Collected %d of a possible %d Sites.", len(site_objects), total_sites)

    sites_file.write(']')
    sites_file.close()

def _get_user_devices(user_id: str, target_name: str):
    """Return a User's Devices

    Retrieves the Device records from xMatters for the specified User and
    returns them as a list.

    Args:
        user_id (str): The User's UUID
        target_name (str): The User's targetName field

    Return:
        device_list (list): List of dictionaries of the User's devices.
    """
    # Initialize conditions
    device_list = []
    total_devices = 0
    url = config.xmod_url + '/api/xm/1/people/' + user_id + '/devices/?embed=timeframes&offset=0&limit=' + str(config.page_size)
    _logger.debug('Gathering Devices for user "%s", url=%s', target_name, url)

    while True:

        # Get the site records
        response = requests.get(url, auth=config.basic_auth)
        if response.status_code not in [200, 404]:
            _log_xm_error(url, response)
            break

        # Process the responses
        bodys = response.json()
        total_devices = bodys['total']
        if bodys['count'] > 0:
            _logger.debug('%d Count of %d Total Devices found for User "%s" via url=%s', bodys['count'], bodys['total'], target_name, url)
            device_list += bodys['data']
            # Use Timeframe to update the timezones admin set
            for body in bodys['data']:
                if 'timeframes' in body:
                    for timeframe in body['timeframes']:
                        if 'timezone' in timeframe: _update_admin('timezones', timeframe['timezone'])
                _update_admin('devices', body['deviceType'] + '|' + body['name'])
                if 'provider' in body: _update_admin('usps', body['provider']['id'])

        # See if there are any more to get
        if 'links' in bodys and 'next' in bodys['links']:
            url = config.xmod_url + bodys['links']['next']
        else:
            break

    _logger.debug('Collected %d of a possible %d Devices for User "%s".', len(device_list), total_devices, target_name)

    return device_list

def _get_user(user_id: str, target_name: str):
    """Attempst to retrieve User by id.
        
        If the User exists, retrieve and return the object.
        If not, return None
        
        Args:
        user_id (str): UUID of User to retrieve
        target_name (str): targetName field value for the specified user.
        """
    _logger.info(f'Capturing User: "{target_name}".')
    
    # Set our resource URLs
    url = config.xmod_url + '/api/xm/1/people/' + urllib.parse.quote(user_id) + '?embed=roles,supervisors'
    _logger.debug('Attempting to retrieve User "%s" via url: %s', target_name, url)
    
    # Make the request
    try:
        response = requests.get(url, auth=config.basic_auth)
    except requests.exceptions.RequestException as e:
        _logger.error(config.ERR_REQUEST_EXCEPTION_CODE, url, repr(e))
        return None
    
    # If the initial response fails, log and return null
    if response.status_code != 200:
        _log_xm_error(url, response)
        return None
    
    # Process the response
    user_obj = response.json()
    # _logger.debug('Found User "%s" - json body: %s', user_obj['firstName'] + ' ' + user_obj['lastName'], pprint.pformat(user_obj))
    _logger.debug('Found User "%s" - json body.id: %s', user_obj['firstName'] + ' ' + user_obj['lastName'], user_obj['id'])

    # Update the Admin object
    if 'language' in user_obj: _update_admin('languages', user_obj['language'])
    if 'timezone' in user_obj: _update_admin('timezones', user_obj['timezone'])
    if 'roles' in user_obj and user_obj['roles']['total'] > 0:
        for role in user_obj['roles']['data']:
            _update_admin('roles', role['name'])
            if role['name'] == config.company_admin_role:
                _update_admin('admins', user_obj['targetName'])

    return user_obj

def _process_users(include_devices: bool):
    """Capture and save the instances User objects

    Retrieves the User object records from xMatters and saves them in
    JSON payload format to the output file.

    Args:
        include_devices (bool): If True, get the User's devices too

    Return:
        None
    """
    _logger.info('Begin gathering Users.')
    users_file = _create_out_file(config.users_filename)
    users_file.write('[\n')

    # Initialize conditions
    user_objects = []
    total_users = 0
    cnt = 0
    url = config.xmod_url + '/api/xm/1/people?offset=0&limit=' + str(config.page_size)
    _logger.debug('Gathering Users via url=%s', url)

    while True:

        # Get the user records
        response = requests.get(url, auth=config.basic_auth)
        if response.status_code not in [200, 404]:
            _log_xm_error(url, response)
            break

        # Process the responses
        bodys = response.json()
        total_users = bodys['total']
        if bodys['count'] > 0:
            _logger.debug("%d Count of %d Total Users found via url=%s", bodys['count'], bodys['total'], url)
            for body in bodys['data']:

                # Get the full user object, including Roles and Supervisors
                user_obj = {}
                a_user = _get_user(body['id'], body['targetName'])
                if a_user is not None:
                    user_obj['user'] = a_user

                    # Get the devices, if requested
                    if include_devices:
                        user_obj['devices'] = _get_user_devices(a_user['id'], a_user['targetName'])

                    # Save the User
                    cnt += 1
                    json.dump(user_obj, users_file)
                    users_file.write(',\n') if cnt < total_users else users_file.write('\n')

            user_objects += bodys['data']

        # See if there are any more to get
        if 'links' in bodys and 'next' in bodys['links']:
            url = config.xmod_url + bodys['links']['next']
        else:
            break
            
    _logger.info("Collected %d of a possible %d Users.", len(user_objects), total_users)

    users_file.write(']')
    users_file.close()

def _get_group(group_id: str, target_name: str):
    """Attempst to retrieve Group by id.
        
        If the Group exists, retrieve and return the object.
        If not, return None
        
        Args:
        group_id (str): UUID of Group to retrieve
        target_name (str): targetName field value for the specified Group.
        """
    _logger.info(f"Retrieving Group: {target_name}")
    
    # Set our resource URLs
    url = config.xmod_url + '/api/xm/1/groups/' + urllib.parse.quote(group_id) + '?embed=supervisors'
    _logger.debug(f'Attempting to retrieve Group "{target_name}" via url: {url}')
    
    # Make the request
    try:
        response = requests.get(url, auth=config.basic_auth)
    except requests.exceptions.RequestException as e:
        _logger.error(config.ERR_REQUEST_EXCEPTION_CODE, url, repr(e))
        return None
    
    # If the initial response fails, log and return null
    if response.status_code != 200:
        _log_xm_error(url, response)
        return None
    
    # Process the response
    group_obj = response.json()
    # If present, translate the Site from an ID to a name
    if 'site' in group_obj:
        site_name = _lookup_site_name(group_obj['site']['id'])
        del group_obj['site']
        group_obj['site'] = site_name
    # _logger.debug(f'Found Group "{group_obj["targetName"]}" - json body: {pprint.pformat(group_obj)}')
    _logger.debug(f'Found Group "{group_obj["targetName"]}" - json body.id: {group_obj["id"]}')
    return group_obj

def _get_group_shifts(group_id: str, target_name: str):
    """Return a Group's Shifts

    Retrieves the Shift records from xMatters for the specified Group and
    returns them as a list.

    Args:
        group_id (str): The Group's UUID
        target_name (str): The Group's targetName field

    Return:
        shift_list (list): List of dictionaries of the Group's Shifts.
    """
    # Initialize conditions
    shift_list = []
    total_shifts = 0
    url = config.xmod_url + '/api/xm/1/groups/' + group_id + '/shifts/?embed=members,rotation&offset=0&limit=' + str(config.page_size)
    _logger.debug(f'Gathering Shifts for Group "{target_name}", url={url}')

    while True:

        # Get the site records
        response = requests.get(url, auth=config.basic_auth)
        if response.status_code not in [200, 404]:
            _log_xm_error(url, response)
            break

        # Process the responses
        bodys = response.json()
        total_shifts = bodys['total']
        if bodys['count'] > 0:
            _logger.debug('%d Count of %d Total Shifts found for Group "%s" via url=%s', bodys['count'], bodys['total'], target_name, url)
            shift_list += bodys['data']

        # See if there are any more to get
        if 'links' in bodys and 'next' in bodys['links']:
            url = config.xmod_url + bodys['links']['next']
        else:
            break

    _logger.debug(f'Collected {len(shift_list)} of a possible {total_shifts} Shifts for Group "{target_name}".')

    return shift_list

def _process_groups():
    """Capture and save the instances Group objects

    Retrieves the User's Group object records from xMatters and saves them in
    JSON payload format to the output file.

    Args:
        None

    Return:
        None
    """
    _logger.info('Begin capturing Groups.')
    groups_file = _create_out_file(config.groups_filename)
    groups_file.write('[\n')

    # Initialize conditions
    group_objects = []
    total_groups = 0
    cnt = 0
    url = config.xmod_url + '/api/xm/1/groups?offset=0&limit=' + str(config.page_size)
    _logger.debug('Gathering Groups via url=%s', url)

    while True:

        # Get the group records
        response = requests.get(url, auth=config.basic_auth)
        if response.status_code not in [200, 404]:
            _log_xm_error(url, response)
            break

        # Process the responses
        bodys = response.json()
        total_groups = bodys['total']
        if bodys['count'] > 0:
            _logger.debug("%d Count of %d Total Groups found via url=%s", bodys['count'], bodys['total'], url)
            for body in bodys['data']:

                # Get the full Group object, including Roles and Supervisors
                group_obj = {}
                a_group = _get_group(body['id'], body['targetName'])
                if a_group is not None:
                    group_obj['group'] = a_group

                    # Get the shifts,
                    group_obj['shifts'] = _get_group_shifts(a_group['id'], a_group['targetName'])

                    # Save the Group
                    cnt += 1
                    json.dump(group_obj, groups_file)
                    groups_file.write(',\n') if cnt < total_groups else groups_file.write('\n')

            group_objects += bodys['data']

        # See if there are any more to get
        if 'links' in bodys and 'next' in bodys['links']:
            url = config.xmod_url + bodys['links']['next']
        else:
            break
            
    _logger.info(f"Collected {len(group_objects)} of a possible {total_groups} Groups.")

    groups_file.write(']')
    groups_file.close()

def _save_admin_data():
    """Saves the collected admin sets

    Preservs the collected admin sets to the file system.

    Args:
        None

    Return:
        None
    """
    admin_dict = {
        'admins': list(_admin_objects['admins']),
        'roles': list(_admin_objects['roles']),
        'timezones': list(_admin_objects['timezones']),
        'countries': list(_admin_objects['countries']),
        'languages': list(_admin_objects['languages']),
        'devices': list(_admin_objects['devices']),
        'usps': list(_admin_objects['usps'])
    }
    admin_file = _create_out_file(config.admin_filename)
    json.dump(admin_dict, admin_file, indent=2)
    admin_file.close()

def process(objects_to_process: list):
    """Capture objects for this instance.

    If requeste contains 'sites', then capture and save Sites.
    If requeste contains 'users', then capture and save Users.
    If requeste contains 'devices', then capture and save Devices.
    If requeste contains 'groups', then capture and save Groups.

    Args:
        objects_to_process (list): The list of object types to capture
    """
    global _logger, _admin_objects # pylint: disable=global-statement

    ### Get the current logger
    _logger = common_logger.get_logger()

    # Initialize the Admin object
    _admin_objects = {
        'admins': set(),
        'roles': set(),
        'timezones': set(),
        'countries': set(),
        'languages': set(),
        'devices': set(),
        'usps': set()
    }

    # Capture and save the Site objects
    if 'sites' in objects_to_process:
        _process_sites()

    # Capture and save the User objects, and possibly devices
    if 'users' in objects_to_process:
        _process_users('devices' in objects_to_process)
    elif 'devices' in objects_to_process:
        _process_users(True)

    # Capture and save the Device objects
    if 'groups' in objects_to_process:
        _process_groups()

    # Preserve the collected admin data
    _save_admin_data()

def main():
    """In case we need to execute the module directly"""
    pass

if __name__ == '__main__':
    main()

Queries for and processes xmatters instance data  .. _Google Python Style Guide:    http://google.github.io/styleguide/pyguide.html # Holds admin info: company_admins, roles, timezones, country, language Updates the admin objects set              Puts the value into the appropriate set.              Args:         a_type (str): The name of a set in the admin dict         a_value (str): The value to add to the dict Captures and logs errors                  Logs the error caused by attempting to call url.                  Args:         url (str): The location being requested that caused the error         response (object): JSON object that holds the error response Creates and opens results file      Args:         filename (str): Name of file to hold output      Returns:         file: outFile Retrieves a Site name by ID      Attempts to find the Site by it's ID in the Site cache,     if not found, then retrieve the Site object from xMatters.      Args:         site_id (str): The ID of the site to find      Return:         None # Site was not in the Cache, so get it from xMatters # Initialize conditions # Get the site records # Process the responses Capture and save the instances Site objects      Retrieves the Site object records from xMatters and saves them in     JSON payload format to the output file.      Args:         None      Return:         None # Initialize conditions # Get the site records # Process the responses # Update admin sets # See if there are any more to get Return a User's Devices      Retrieves the Device records from xMatters for the specified User and     returns them as a list.      Args:         user_id (str): The User's UUID         target_name (str): The User's targetName field      Return:         device_list (list): List of dictionaries of the User's devices. # Initialize conditions # Get the site records # Process the responses # Use Timeframe to update the timezones admin set # See if there are any more to get Attempst to retrieve User by id.                  If the User exists, retrieve and return the object.         If not, return None                  Args:         user_id (str): UUID of User to retrieve         target_name (str): targetName field value for the specified user. # Set our resource URLs # Make the request # If the initial response fails, log and return null # Process the response # _logger.debug('Found User "%s" - json body: %s', user_obj['firstName'] + ' ' + user_obj['lastName'], pprint.pformat(user_obj)) # Update the Admin object Capture and save the instances User objects      Retrieves the User object records from xMatters and saves them in     JSON payload format to the output file.      Args:         include_devices (bool): If True, get the User's devices too      Return:         None # Initialize conditions # Get the user records # Process the responses # Get the full user object, including Roles and Supervisors # Get the devices, if requested # Save the User # See if there are any more to get Attempst to retrieve Group by id.                  If the Group exists, retrieve and return the object.         If not, return None                  Args:         group_id (str): UUID of Group to retrieve         target_name (str): targetName field value for the specified Group. # Set our resource URLs # Make the request # If the initial response fails, log and return null # Process the response # If present, translate the Site from an ID to a name # _logger.debug(f'Found Group "{group_obj["targetName"]}" - json body: {pprint.pformat(group_obj)}') Return a Group's Shifts      Retrieves the Shift records from xMatters for the specified Group and     returns them as a list.      Args:         group_id (str): The Group's UUID         target_name (str): The Group's targetName field      Return:         shift_list (list): List of dictionaries of the Group's Shifts. # Initialize conditions # Get the site records # Process the responses # See if there are any more to get Capture and save the instances Group objects      Retrieves the User's Group object records from xMatters and saves them in     JSON payload format to the output file.      Args:         None      Return:         None # Initialize conditions # Get the group records # Process the responses # Get the full Group object, including Roles and Supervisors # Get the shifts, # Save the Group # See if there are any more to get Saves the collected admin sets      Preservs the collected admin sets to the file system.      Args:         None      Return:         None Capture objects for this instance.      If requeste contains 'sites', then capture and save Sites.     If requeste contains 'users', then capture and save Users.     If requeste contains 'devices', then capture and save Devices.     If requeste contains 'groups', then capture and save Groups.      Args:         objects_to_process (list): The list of object types to capture # pylint: disable=global-statement ### Get the current logger # Initialize the Admin object # Capture and save the Site objects # Capture and save the User objects, and possibly devices # Capture and save the Device objects # Preserve the collected admin data In case we need to execute the module directly