""" Meta-analysis tools """

import logging
import numpy as np
from scipy.stats import norm
from neurosynth.base import imageutils
from neurosynth.analysis import stats
from os.path import join, exists
from os import makedirs

logger = logging.getLogger('neurosynth.meta')

def analyze_features(dataset, features=None, image_type='association-test_z',
                     threshold=0.001, q=0.01, output_dir=None, prefix=None):
    """ Generate meta-analysis images for a set of features.
    Args:
        dataset: A Dataset instance containing feature and activation data.
        features: A list of named features to generate meta-analysis maps for.
            If None, analyzes all features in the current dataset.
        image_type: The type of image to return. Specify one of the extensions
            generated by the MetaAnalysis procedure--e.g., association-test_z,
            uniformity-test_z, etc.  By default, will use
            association-test_z (i.e., z-scores reflecting the association
            between presence of activation and presence of feature).
        threshold: The threshold for determining whether or not a Mappable has
            a feature. By default, this is 0.001, which is only sensible in the
            case of term-based features (so be sure to specify it for other
            kinds).
        q: The FDR rate to use for multiple comparisons correction (default =
            0.05).
        output_dir: Directory to save all meta-analysis images to. If none,
            returns all the data as a matrix.
        prefix: All output images will be prepended with this string (if None,
            defaults to the name of the feature).
    Returns:
        If output_dir is None, an n_voxels x n_features 2D numpy array.
    """
    if features is None:
        features = dataset.get_feature_names()
    if output_dir is None:
        result = np.zeros((dataset.masker.n_vox_in_mask, len(features)))

    for i, f in enumerate(features):
        ids = dataset.get_studies(features=f, frequency_threshold=threshold)
        ma = MetaAnalysis(dataset, ids, q=q)
        if output_dir is None:
            result[:, i] = ma.images[image_type]
        else:
            pfx = f if prefix is None else prefix + '_' + f
            ma.save_results(output_dir=output_dir, prefix=pfx)

    if output_dir is None:
        return result

class MetaAnalysis(object):

    """ Meta-analysis of a Dataset. Currently contrasts two subsets of
    studies within a Dataset and saves a bunch of statistical images.
    Only one list of study IDs (ids) needs to be passed; the Universe will
    be bisected into studies that are and are not included in the
    list, and the contrast is then performed across these two groups.
    If a second optional second study list is provided (ids2), the Dataset
    is first constrained to the union of ids1 and ids2, and the standard
    contrast is then performed."""

    def __init__(self, dataset, ids, ids2=None, q=0.01, prior=0.5,
                 min_studies=1):
        """ Initialize a new MetaAnalysis instance and run an analysis.
        Args:
            dataset: A Dataset instance.
            ids: A list of Mappable IDs to include in the meta-analysis.
            ids2: Optional second list of Mappable IDs. If passed, the set of
                studies will be restricted to the union of ids and ids2 before
                performing the meta-analysis. This is useful for meta-analytic
                contrasts, as the resulting images will in effect identify
                regions that are reported/activated more frequently in one
                list than in the other.
            q: The FDR threshold to use when correcting for multiple
                comparisons. Set to .01 by default.
            prior: The prior to use when calculating conditional probabilities.
                This is the prior probability of a feature being used in a
                study (i.e., p(F)). For example, if set to 0.25, the analysis
                will assume that 1/4 of studies load on the target feature, as
                opposed to the empirically estimated p(F), which is len(ids) /
                total number of studies in the dataset. If prior is not passed,
                defaults to 0.5, reflecting an effort to put all terms on level
                footing and avoid undue influence of base rates (because some
                terms are much more common than others). Note that modifying
                the prior will only affect the effect size/probability maps,
                and not the statistical inference (z-score) maps.
            min_studies: Integer or float indicating which voxels to mask out
                from results due to lack of stability. If an integer is passed,
                all voxels that activate in fewer than this number of studies
                will be ignored (i.e., a value of 0 will be assigned in all
                output images). If a float in the range of 0 - 1 is passed,
                this will be interpreted as a proportion to use as the cut-off
                (e.g., passing 0.03 will exclude all voxels active in fewer
                than 3% of the entire dataset). Defaults to 1, meaning all
                voxels that activate at least one study will be kept.
        """

        self.dataset = dataset
        mt = dataset.image_table
        self.selected_ids = list(set(mt.ids) & set(ids))
        self.selected_id_indices = np.in1d(mt.ids, ids)

        # If ids2 is provided, we only use mappables explicitly in either ids or ids2.
        # Otherwise, all mappables not in the ids list are used as the control
        # condition.
        unselected_id_indices = ~self.selected_id_indices if ids2 is None \
            else np.in1d(mt.ids, ids2)
        mappable_id_indices = self.selected_id_indices | unselected_id_indices

        # Calculate different count variables
        logger.debug("Calculating counts...")
        n_selected = len(self.selected_ids)
        n_unselected = np.sum(unselected_id_indices)
        n_mappables = n_selected + n_unselected

        n_selected_active_voxels = mt.data.dot(self.selected_id_indices)
        n_unselected_active_voxels = mt.data.dot(unselected_id_indices)

        # Nomenclature for variables below: p = probability, F = feature present, g = given,
        # U = unselected, A = activation. So, e.g., pAgF = p(A|F) = probability of activation
        # in a voxel if we know that the feature is present in a study.
        pF = (n_selected * 1.0) / n_mappables
        mappable_data = mt.data if ids2 is None else mt.data[:, mappable_id_indices]
        pA = np.array((mappable_data.sum(axis=1) * 1.0) / n_mappables).squeeze()

        # Conditional probabilities
        logger.debug("Calculating conditional probabilities...")
        pAgF = n_selected_active_voxels * 1.0 / n_selected
        pAgU = n_unselected_active_voxels * 1.0 / n_unselected
        pFgA = pAgF * pF / pA

        # Recompute conditionals with uniform prior
        logger.debug("Recomputing with uniform priors...")
        pA_prior = prior * pAgF + (1 - prior) * pAgU
        pFgA_prior = pAgF * prior / pA_prior

        def p_to_z(p, sign):
            p = p / 2  # convert to two-tailed
            # prevent underflow
            p[p < 1e-240] = 1e-240
            # Convert to z and assign tail
            z = np.abs(norm.ppf(p)) * sign
            # Set very large z's to max precision
            z[np.isinf(z)] = norm.ppf(1e-240) * -1
            return z

        # One-way chi-square test for consistency of activation
        p_vals = stats.one_way(
            np.squeeze(n_selected_active_voxels), n_selected)
        p_vals[p_vals < 1e-240] = 1e-240
        z_sign = np.sign(
            n_selected_active_voxels - np.mean(
                n_selected_active_voxels)).ravel()
        pAgF_z = p_to_z(p_vals, z_sign)
        fdr_thresh = stats.fdr(p_vals, q)
        pAgF_z_FDR = imageutils.threshold_img(
            pAgF_z, fdr_thresh, p_vals, mask_out='above')

        # Two-way chi-square for specificity of activation
        cells = np.squeeze(
            np.array([[n_selected_active_voxels, n_unselected_active_voxels],
                      [n_selected - n_selected_active_voxels, n_unselected -
                       n_unselected_active_voxels]]).T)
        p_vals = stats.two_way(cells)
        p_vals[p_vals < 1e-240] = 1e-240
        z_sign = np.sign(pAgF - pAgU).ravel()
        pFgA_z = p_to_z(p_vals, z_sign)
        fdr_thresh = stats.fdr(p_vals, q)
        pFgA_z_FDR = imageutils.threshold_img(
            pFgA_z, fdr_thresh, p_vals, mask_out='above')

        # Retain any images we may want to save or access later
        self.images = {
            'pA': pA,
            'pAgF': pAgF,
            'pFgA': pFgA,
            ('pA_given_pF=%0.2f' % prior): pA_prior,
            ('pFgA_given_pF=%0.2f' % prior): pFgA_prior,
            'uniformity-test_z': pAgF_z,
            'association-test_z': pFgA_z,
            ('uniformity-test_z_FDR_%s' % q): pAgF_z_FDR,
            ('association-test_z_FDR_%s' % q): pFgA_z_FDR
        }

        # Mask out all voxels below num_studies threshold
        if min_studies > 0:
            if isinstance(min_studies, int):
                min_studies = float(
                    min_studies) / n_mappables  # Recalculate as proportion
            vox_to_exclude = np.where(pA < min_studies)[0]  # Create mask
            # Mask each image
            for k in self.images:
                self.images[k][vox_to_exclude] = 0

    def save_results(self, output_dir='.', prefix='', prefix_sep='_',
                     image_list=None):
        """ Write out any images generated by the meta-analysis.
        Args:
            output_dir (str): folder to write images to
            prefix (str): all image files will be prepended with this string
            prefix_sep (str): glue between the prefix and rest of filename
            image_list (list): optional list of images to save--e.g.,
                ['pFgA_z', 'pAgF']. If image_list is None (default), will save
                all images.
        """

        if prefix == '':
            prefix_sep = ''

        if not exists(output_dir):
            makedirs(output_dir)

        logger.debug("Saving results...")
        if image_list is None:
            image_list = self.images.keys()
        for suffix, img in self.images.items():
            if suffix in image_list:
                filename = prefix + prefix_sep + suffix + '.nii.gz'
                outpath = join(output_dir, filename)
                imageutils.save_img(img, outpath, self.dataset.masker)


""" Various transformations between coordinate frames, atlas spaces, etc. """

import numpy as np
from numpy import linalg
import logging

logger = logging.getLogger('neurosynth.transformations')

def transform(foci, mat):
    """ Convert coordinates from one space to another using provided
    transformation matrix. """
    t = linalg.pinv(mat)
    foci = np.hstack((foci, np.ones((foci.shape[0], 1))))
    return np.dot(foci, t)[:, 0:3]

def xyz_to_mat(foci, xyz_dims=None, mat_dims=None):
    """ Convert an N x 3 array of XYZ coordinates to matrix indices. """
    foci = np.hstack((foci, np.ones((foci.shape[0], 1))))
    mat = np.array([[-0.5, 0, 0, 45], [0, 0.5, 0, 63], [0, 0, 0.5, 36]]).T
    result = np.dot(foci, mat)[:, ::-1]  # multiply and reverse column order
    return np.round_(result).astype(int)  # need to round indices to ints

def mat_to_xyz(foci, mat_dims=None, xyz_dims=None):
    """ Convert an N x 3 array of matrix indices to XYZ coordinates. """
    foci = np.hstack((foci, np.ones((foci.shape[0], 1))))
    mat = np.array([[-2, 0, 0, 90], [0, 2, 0, -126], [0, 0, 2, -72]]).T
    result = np.dot(foci, mat)[:, ::-1]  # multiply and reverse column order
    return np.round_(result).astype(int)  # need to round indices to ints

def t88_to_mni():
    """ Convert Talairach to MNI coordinates using the Lancaster transform.
    Adapted from BrainMap scripts; see http://brainmap.org/icbm2tal/
    Details are described in Lancaster et al. (2007)
    (http://brainmap.org/new/pubs/LancasterHBM07.pdf). """
    return np.array([[0.9254, 0.0024, -0.0118, -1.0207],
                     [-0.0048, 0.9316, -0.0871, -1.7667],
                     [0.0152, 0.0883, 0.8924, 4.0926],
                     [0.0, 0.0, 0.0, 1.0]]).T

class Transformer(object):

    """ The Transformer class supports transformations between different
    image spaces. """

    def __init__(self, transforms=None, target='MNI'):
        """ Initialize a transformer instance. """
        self.target = target
        self.transformations = transforms

    def add(self, name, mat):
        """ Add a named linear transformation. """
        self.transformations[name] = mat

    def apply(self, name, foci):
        """ Apply a named transformation to a set of foci.

        If the named transformation doesn't exist, return foci untransformed.
        """
        if name in self.transformations:
            return transform(foci, self.transformations[name])
        else:
            logger.info(
                "No transformation named '%s' found; coordinates left "
                "untransformed." % name)
            return foci


import numpy as np
import nibabel as nb
from six import string_types

class Masker(object):

    """ Handles vectorization/masking/unmasking of images. """

    def __init__(self, volume, layers=None):
        """ Initialize a new Masker.
        Args:
            volume: A volume indicating the global space within which all
                subsequent layers must reside. Any voxel in the mask with a
                non-zero valid is considered valid for analyses. Can be either
                an image filename or a NiBabel image.
            layers: Optional masking layers to add; see docstring for add().
        """
        if isinstance(volume, string_types):
            volume = nb.load(volume)
        self.volume = volume
        data = self.volume.get_data()
        self.dims = data.shape
        self.vox_dims = self.get_header().get_zooms()
        self.full = np.float64(data.ravel())
        self.global_mask = np.where(self.full)

        self.reset()
        if layers is not None:
            self.add(layers)

    def reset(self):
        """ Reset/remove all layers, keeping only the initial volume. """
        self.layers = {}
        self.stack = []
        self.set_mask()
        self.n_vox_in_vol = len(np.where(self.current_mask)[0])

    def add(self, layers, above=None, below=None):
        """ Add one or more layers to the stack of masking layers.
        Args:
            layers: A string, NiBabel image, list, or dict. If anything other
                than a dict is passed, assigns sequential layer names based on
                the current position in stack; if a dict, uses key as the name
                and value as the mask image.
        """

        def add_named_layer(name, image):
            image = self.get_image(image, output='vector')
            if above is not None:
                image[image < above] = 0.
            if below is not None:
                image[image > below] = 0.
            self.layers[name] = image
            self.stack.append(name)

        if isinstance(layers, dict):
            for (name, image) in layers.items():
                add_named_layer(name, image)

        else:
            if not isinstance(layers, list):
                layers = [layers]
            for image in layers:
                name = 'layer_%d' % len(self.stack)
                add_named_layer(name, image)

        self.set_mask()

    def remove(self, layers):
        """ Remove one or more layers from the stack of masking layers.
        Args:
            layers: An int, string or list of strings and/or ints. Ints are
                interpreted as indices in the stack to remove; strings are
                interpreted as names of layers to remove. Negative ints will
                also work--i.e., remove(-1) will drop the last layer added.
        """
        if not isinstance(layers, list):
            layers = [layers]
        for l in layers:
            if isinstance(l, string_types):
                if l not in self.layers:
                    raise ValueError("There's no image/layer named '%s' in "
                                     "the masking stack!" % l)
                self.stack.remove(l)
            else:
                l = self.stack.pop(l)
            del self.layers[l]

        self.set_mask()

    def get_image(self, image, output='vector'):
        """ A flexible method for transforming between different
        representations of image data.
        Args:
            image: The input image. Can be a string (filename of image),
                NiBabel image, N-dimensional array (must have same shape as
                self.volume), or vectorized image data (must have same length
                as current conjunction mask).
            output: The format of the returned image representation. Must be
                one of:
                    'vector': A 1D vectorized array
                    'array': An N-dimensional array, with
                        shape = self.volume.shape
                    'image': A NiBabel image
        Returns: An object containing image data; see output options above.
        """
        if isinstance(image, string_types):
            image = nb.load(image)

        if type(image).__module__.startswith('nibabel'):
            if output == 'image':
                return image
            image = image.get_data()

        if not type(image).__module__.startswith('numpy'):
            raise ValueError("Input image must be a string, a NiBabel image, "
                             "or a numpy array.")

        if image.shape[:3] == self.volume.shape:
            if output == 'image':
                return nb.nifti1.Nifti1Image(image, None, self.get_header())
            elif output == 'array':
                return image
            else:
                image = image.ravel()

        if output == 'vector':
            return image.ravel()

        image = np.reshape(image, self.volume.shape)

        if output == 'array':
            return image

        return nb.nifti1.Nifti1Image(image, None, self.get_header())

    def mask(self, image, nan_to_num=True, layers=None, in_global_mask=False):
        """ Vectorize an image and mask out all invalid voxels.

        Args:
            images: The image to vectorize and mask. Input can be any object
                handled by get_image().
            layers: Which mask layers to use (specified as int, string, or
                list of ints and strings). When None, applies the conjunction
                of all layers.
            nan_to_num: boolean indicating whether to convert NaNs to 0.
            in_global_mask: Whether to return the resulting masked vector in
                the globally masked space (i.e., n_voxels =
                len(self.global_mask)). If False (default), returns in the full
                image space (i.e., n_voxels = len(self.volume)).
        Returns:
          A 1D NumPy array of in-mask voxels.
        """
        self.set_mask(layers)
        image = self.get_image(image, output='vector')

        if in_global_mask:
            masked_data = image[self.global_mask]
            masked_data[~self.get_mask(in_global_mask=True)] = 0
        else:
            masked_data = image[self.current_mask]

        if nan_to_num:
            masked_data = np.nan_to_num(masked_data)

        return masked_data

    def unmask(self, data, layers=None, output='array'):
        """ Reconstruct a masked vector into the original 3D volume.
        Args:
            data: The 1D vector to reconstruct. (Can also be a 2D vector where
                the second dimension is time, but then output will always
                be set to 'array'--i.e., a 4D image will be returned.)
            layers: Which mask layers to use (specified as int, string, or list
                of ints and strings). When None, applies the conjunction of all
                layers. Note that the layers specified here must exactly match
                the layers used in the mask() operation, otherwise the shape of
                the mask will be incorrect and bad things will happen.
            output: What kind of object to return. See options in get_image().
                By default, returns an N-dimensional array of reshaped data.
        """
        self.set_mask(layers)
        if data.ndim == 2:
            n_volumes = data.shape[1]
            # Assume 1st dimension is voxels, 2nd is time
            # but we generate x,y,z,t volume
            image = np.zeros(self.full.shape + (n_volumes,))
            image[self.current_mask, :] = data
            image = np.reshape(image, self.volume.shape + (n_volumes,))
        else:
            # img = self.full.copy()
            image = np.zeros(self.full.shape)
            image[self.current_mask] = data
        return self.get_image(image, output)

    def get_mask(self, layers=None, output='vector', in_global_mask=True):
        """ Set the current mask by taking the conjunction of all specified
        layers.

        Args:
            layers: Which layers to include. See documentation for add() for
                format.
            include_global_mask: Whether or not to automatically include the
                global mask (i.e., self.volume) in the conjunction.
        """
        if in_global_mask:
            output = 'vector'

        if layers is None:
            layers = self.layers.keys()
        elif not isinstance(layers, list):
            layers = [layers]

        layers = map(lambda x: x if isinstance(x, string_types)
                     else self.stack[x], layers)
        layers = [self.layers[l] for l in layers if l in self.layers]

        # Always include the original volume
        layers.append(self.full)
        layers = np.vstack(layers).T.astype(bool)
        mask = layers.all(axis=1)
        mask = self.get_image(mask, output)
        return mask[self.global_mask] if in_global_mask else mask

    def set_mask(self, layers=None):
        self.current_mask = self.get_mask(layers, in_global_mask=False)
        self.n_vox_in_mask = len(np.where(self.current_mask)[0])

    def get_header(self):
        """ A wrapper for the NiBabel method. """
        return self.volume.get_header()


""" Miscellaneous image-related utility functions. """

import logging
import nibabel as nb
from nibabel import nifti1
import numpy as np
from six import string_types

logger = logging.getLogger('neurosynth.imageutils')

def get_sphere(coords, r=4, vox_dims=(2, 2, 2), dims=(91, 109, 91)):
    """ # Return all points within r mm of coordinates. Generates a cube
    and then discards all points outside sphere. Only returns values that
    fall within the dimensions of the image."""
    r = float(r)
    xx, yy, zz = [slice(-r / vox_dims[i], r / vox_dims[
                        i] + 0.01, 1) for i in range(len(coords))]
    cube = np.vstack([row.ravel() for row in np.mgrid[xx, yy, zz]])
    sphere = cube[:, np.sum(np.dot(np.diag(
        vox_dims), cube) ** 2, 0) ** .5 <= r]
    sphere = np.round(sphere.T + coords)
    return sphere[(np.min(sphere, 1) >= 0) &
                  (np.max(np.subtract(sphere, dims), 1) <= -1), :].astype(int)

def map_peaks_to_image(peaks, r=4, vox_dims=(2, 2, 2), dims=(91, 109, 91),
                       header=None):
    """ Take a set of discrete foci (i.e., 2-D array of xyz coordinates)
    and generate a corresponding image, convolving each focus with a
    hard sphere of radius r."""
    data = np.zeros(dims)
    for p in peaks:
        valid = get_sphere(p, r, vox_dims, dims)
        valid = valid[:, ::-1]
        data[tuple(valid.T)] = 1
    return nifti1.Nifti1Image(data, None, header=header)

def load_imgs(filenames, masker, nan_to_num=True):
    """ Load multiple images from file into an ndarray.

    Args:
      filenames: A single filename or list of filenames pointing to valid
        images.
      masker: A Masker instance.
      nan_to_num: Optional boolean indicating whether to convert NaNs to zero.

    Returns:
      An m x n 2D numpy array, where m = number of voxels in mask and
      n = number of images passed.
    """
    if isinstance(filenames, string_types):
        filenames = [filenames]
    data = np.zeros((masker.n_vox_in_mask, len(filenames)))
    for i, f in enumerate(filenames):
        data[:, i] = masker.mask(f, nan_to_num)
    return data

def save_img(data, filename, masker, header=None):
    """ Save a vectorized image to file. """
    if not header:
        header = masker.get_header()
    header.set_data_dtype(data.dtype)  # Avoids loss of precision
    # Update min/max -- this should happen on save, but doesn't seem to
    header['cal_max'] = data.max()
    header['cal_min'] = data.min()
    img = nifti1.Nifti1Image(masker.unmask(data), None, header)
    img.to_filename(filename)

def threshold_img(data, threshold, mask=None, mask_out='below'):
    """ Threshold data, setting all values in the array above/below threshold
    to zero.
    Args:
        data (ndarray): The image data to threshold.
        threshold (float): Numeric threshold to apply to image.
        mask (ndarray): Optional 1D-array with the same length as the data. If
            passed, the threshold is first applied to the mask, and the
            resulting indices are used to threshold the data. This is primarily
            useful when, e.g., applying a statistical threshold to a z-value
            image based on a p-value threshold.
        mask_out (str): Thresholding direction. Can be 'below' the threshold
            (default) or 'above' the threshold. Note: use 'above' when masking
            based on p values.
    """
    if mask is not None:
        mask = threshold_img(mask, threshold, mask_out=mask_out)
        return data * mask.astype(bool)
    if mask_out.startswith('b'):
        data[data < threshold] = 0
    elif mask_out.startswith('a'):
        data[data > threshold] = 0
    return data

def create_grid(image, scale=4, apply_mask=True, save_file=None):
    """ Creates an image containing labeled cells in a 3D grid.
    Args:
        image: String or nibabel image. The image used to define the grid
            dimensions. Also used to define the mask to apply to the grid.
            Only voxels with non-zero values in the mask will be retained; all
            other voxels will be zeroed out in the returned image.
        scale: The scaling factor which controls the grid size. Value reflects
            diameter of cube in voxels.
        apply_mask: Boolean indicating whether or not to zero out voxels not in
            image.
        save_file: Optional string giving the path to save image to. Image
            written out is a standard Nifti image. If save_file is None, no
            image is written.
    Returns:
        A nibabel image with the same dimensions as the input image. All voxels
        in each cell in the 3D grid are assigned the same non-zero label.
    """
    if isinstance(image, string_types):
        image = nb.load(image)

    # create a list of cluster centers
    centers = []
    x_length, y_length, z_length = image.shape
    for x in range(0, x_length, scale):
        for y in range(0, y_length, scale):
            for z in range(0, z_length, scale):
                centers.append((x, y, z))

    # create a box around each center with the diameter equal to the scaling
    # factor
    grid = np.zeros(image.shape)
    for (i, (x, y, z)) in enumerate(centers):
        for mov_x in range((-scale + 1) // 2, (scale + 1) // 2):
            for mov_y in range((-scale + 1) // 2, (scale + 1) // 2):
                for mov_z in range((-scale + 1) // 2, (scale + 1) // 2):
                    try:  # Ignore voxels outside bounds of image
                        grid[x + mov_x, y + mov_y, z + mov_z] = i + 1
                    except:
                        pass

    if apply_mask:
        mask = image
        if isinstance(mask, string_types):
            mask = nb.load(mask)
        if type(mask).__module__ != np.__name__:
            mask = mask.get_data()
        grid[~mask.astype(bool)] = 0.0

    grid = nb.Nifti1Image(grid, image.get_affine(), image.get_header())

    if save_file is not None:
        nb.save(grid, save_file)

    return grid


# emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
# ex: set sts=4 ts=4 sw=4 et:
"""NeuroSynth -- large-scale synthesis of functional neuroimaging data.

"""
__all__ = ["analysis", "base", "set_logging_level", "__version__"]

# Easier access to key modules/classes
from .base.dataset import Dataset
from .base.mask import Masker
from .analysis.cluster import Clusterable
from .analysis.meta import MetaAnalysis
from .analysis.decode import Decoder
from .base import dataset, imageutils, lexparser, mask, transformations
from .analysis import classify, cluster, decode, meta, network, reduce

import logging
import sys
import os

from .version import __version__

logger = logging.getLogger("neurosynth")

def set_logging_level(level=None):
    """Set neurosynth's logging level

    Args
      level : str
        Name of the logging level (warning, error, info, etc) known
        to logging module.  If no level provided, it would get that one
        from environment variable NEUROSYNTH_LOGLEVEL
    """
    if level is None:
        level = os.environ.get('NEUROSYNTH_LOGLEVEL', 'warn')
    if level is not None:
        logger.setLevel(getattr(logging, level.upper()))
    return logger.getEffectiveLevel()

def _setup_logger(logger):
    # Basic logging setup
    console = logging.StreamHandler(sys.stdout)
    console.setFormatter(logging.Formatter("%(levelname)-6s %(module)-7s %(message)s"))
    logger.addHandler(console)
    set_logging_level()

# Setup neurosynth's logger
_setup_logger(logger)


# -*- coding: utf-8 -*-
"""Python bindings to libpostal expand_address."""

from __future__ import unicode_literals
from postal import _expand
from postal.utils.encoding import safe_decode

def expand_address(address, languages=None, **kw):
    """
    Expand the given address into one or more normalized strings.

    Required
    --------
    @param address: the address as either Unicode or a UTF-8 encoded string

    Options
    -------
    @param languages: a tuple or list of ISO language code strings (e.g. "en", "fr", "de", etc.)
                      to use in expansion. If None is passed, use language classifier
                      to detect language automatically.
    @param address_components: an integer (bit-set) of address component expansions
                               to use e.g. ADDRESS_NAME | ADDRESS_STREET would use
                               only expansions which apply to venue names or streets.
    @param latin_ascii: use the Latin to ASCII transliterator, which normalizes e.g. æ => ae
    @param transliterate: use any available transliterators for non-Latin scripts, e.g.
                          for the Greek phrase διαφορετικούς becomes diaphoretikoús̱
    @param strip_accents: strip accented characters e.g. é => e, ç => c. This loses some
                          information in various languags, but in general we want
    @param decompose: perform Unicode normalization (NFD form)
    @param lowercase: UTF-8 lowercase the string
    @param trim_string: trim spaces on either side of the string
    @param replace_word_hyphens: add version of the string replacing hyphens with space
    @param delete_word_hyphens: add version of the string with hyphens deleted
    @param replace_numeric_hyphens: add version of the string with numeric hyphens replaced
                                    e.g. 12345-6789 => 12345 6789
    @param delete_numeric_hyphens: add version of the string with numeric hyphens removed
                                   e.g. 12345-6789 => 123456789
    @param split_alpha_from_numeric: split tokens like CR17 into CR 17, helps with expansion
                                     of certain types of highway abbreviations
    @param delete_final_periods: remove final periods on abbreviations e.g. St. => St
    @param delete_acronym_periods: remove periods in acronyms e.g. U.S.A. => USA
    @param drop_english_possessives: normalize possessives e.g. Mark's => Marks
    @param delete_apostrophes: delete other types of hyphens e.g. O'Malley => OMalley
    @param expand_numex: converts numeric expressions e.g. Twenty sixth => 26th,
                         using either the supplied languages or the result of
                         automated language classification.
    @param roman_numerals: normalize Roman numerals e.g. IX => 9. Since these can be
                           ambiguous (especially I and V), turning this on simply
                           adds another version of the string if any potential
                           Roman numerals are found.
    """
    address = safe_decode(address, 'utf-8')
    return _expand.expand_address(address, languages=languages, **kw)

# Constants for address components
ADDRESS_NONE = _expand.ADDRESS_NONE
ADDRESS_ANY = _expand.ADDRESS_ANY
ADDRESS_NAME = _expand.ADDRESS_NAME
ADDRESS_HOUSE_NUMBER = _expand.ADDRESS_HOUSE_NUMBER
ADDRESS_STREET = _expand.ADDRESS_STREET
ADDRESS_UNIT = _expand.ADDRESS_UNIT
ADDRESS_LEVEL = _expand.ADDRESS_LEVEL
ADDRESS_STAIRCASE = _expand.ADDRESS_STAIRCASE
ADDRESS_ENTRANCE = _expand.ADDRESS_ENTRANCE
ADDRESS_CATEGORY = _expand.ADDRESS_CATEGORY
ADDRESS_NEAR = _expand.ADDRESS_NEAR
ADDRESS_TOPONYM = _expand.ADDRESS_TOPONYM
ADDRESS_POSTAL_CODE = _expand.ADDRESS_POSTAL_CODE
ADDRESS_PO_BOX = _expand.ADDRESS_PO_BOX
ADDRESS_ALL = _expand.ADDRESS_ALL


# -*- coding: utf-8 -*-

from postal import _normalize
from postal.token_types import token_types

from postal.utils.encoding import safe_decode

# String options
NORMALIZE_STRING_LATIN_ASCII = _normalize.NORMALIZE_STRING_LATIN_ASCII
NORMALIZE_STRING_TRANSLITERATE = _normalize.NORMALIZE_STRING_TRANSLITERATE
NORMALIZE_STRING_STRIP_ACCENTS = _normalize.NORMALIZE_STRING_STRIP_ACCENTS
NORMALIZE_STRING_DECOMPOSE = _normalize.NORMALIZE_STRING_DECOMPOSE
NORMALIZE_STRING_LOWERCASE = _normalize.NORMALIZE_STRING_LOWERCASE
NORMALIZE_STRING_TRIM = _normalize.NORMALIZE_STRING_TRIM
NORMALIZE_STRING_REPLACE_HYPHENS = _normalize.NORMALIZE_STRING_REPLACE_HYPHENS
NORMALIZE_STRING_SIMPLE_LATIN_ASCII = _normalize.NORMALIZE_STRING_SIMPLE_LATIN_ASCII
NORMALIZE_STRING_REPLACE_NUMEX = _normalize.NORMALIZE_STRING_REPLACE_NUMEX

DEFAULT_STRING_OPTIONS = _normalize.NORMALIZE_DEFAULT_STRING_OPTIONS
STRING_OPTIONS_NUMEX = _normalize.NORMALIZE_DEFAULT_STRING_OPTIONS | NORMALIZE_STRING_REPLACE_NUMEX

# Token options
NORMALIZE_TOKEN_REPLACE_HYPHENS = _normalize.NORMALIZE_TOKEN_REPLACE_HYPHENS
NORMALIZE_TOKEN_DELETE_HYPHENS = _normalize.NORMALIZE_TOKEN_DELETE_HYPHENS
NORMALIZE_TOKEN_DELETE_FINAL_PERIOD = _normalize.NORMALIZE_TOKEN_DELETE_FINAL_PERIOD
NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS = _normalize.NORMALIZE_TOKEN_DELETE_ACRONYM_PERIODS
NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES = _normalize.NORMALIZE_TOKEN_DROP_ENGLISH_POSSESSIVES
NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE = _normalize.NORMALIZE_TOKEN_DELETE_OTHER_APOSTROPHE
NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC = _normalize.NORMALIZE_TOKEN_SPLIT_ALPHA_FROM_NUMERIC
NORMALIZE_TOKEN_REPLACE_DIGITS = _normalize.NORMALIZE_TOKEN_REPLACE_DIGITS

DEFAULT_TOKEN_OPTIONS = _normalize.NORMALIZE_DEFAULT_TOKEN_OPTIONS

TOKEN_OPTIONS_DROP_PERIODS = _normalize.NORMALIZE_TOKEN_OPTIONS_DROP_PERIODS
DEFAULT_TOKEN_OPTIONS_NUMERIC = _normalize.NORMALIZE_DEFAULT_TOKEN_OPTIONS_NUMERIC

def remove_parens(tokens):
    new_tokens = []
    open_parens = 0
    for t, c in tokens:
        if c == token_types.PUNCT_OPEN:
            open_parens += 1
        elif c == token_types.PUNCT_CLOSE:
            if open_parens > 0:
                open_parens -= 1
        elif open_parens <= 0:
            new_tokens.append((t, c))
    return new_tokens

def normalize_string(s, string_options=DEFAULT_STRING_OPTIONS, languages=None):
    s = safe_decode(s)
    return _normalize.normalize_string(s, string_options, languages=languages)

def normalized_tokens(s, string_options=DEFAULT_STRING_OPTIONS,
                      token_options=DEFAULT_TOKEN_OPTIONS,
                      strip_parentheticals=True, whitespace=False,
                      languages=None):
    '''
    Normalizes a string, tokenizes, and normalizes each token
    with string and token-level options.

    This version only uses libpostal's deterministic normalizations
    i.e. methods with a single output. The string tree version will
    return multiple normalized strings, each with tokens.

    Usage:
        normalized_tokens(u'St.-Barthélemy')
    '''
    s = safe_decode(s)
    normalized_tokens = _normalize.normalized_tokens(s, string_options, token_options, whitespace, languages=languages)

    if strip_parentheticals:
        normalized_tokens = remove_parens(normalized_tokens)

    return [(s, token_types.from_id(token_type)) for s, token_type in normalized_tokens]


"""Python bindings to libpostal parse_address."""
from postal import _parser
from postal.utils.encoding import safe_decode

def parse_address(address, language=None, country=None):
    """
    Parse address into components.

    @param address: the address as either Unicode or a UTF-8 encoded string
    @param language (optional): language code
    @param country (optional): country code
    """
    address = safe_decode(address, 'utf-8')
    return _parser.parse_address(address, language=language, country=country)


"""Python bindings to libpostal near_dupe_hashes."""

from postal import _near_dupe

def near_dupe_hashes(labels, values, languages=None, **kw):
    """
    Hash the given address into normalized strings that can be used to group similar
    addresses together for more detailed pairwise comparison. This can be thought of
    as the blocking function in record linkage or locally-sensitive hashing in the
    document near-duplicate detection.

    Required
    --------
    @param labels: array of component labels as either Unicode or UTF-8 encoded strings
                   e.g. ["house_number", "road", "postcode"]
    @param values: array of component values as either Unicode or UTF-8 encoded strings
                   e.g. ["123", "Broadway", "11216"]. Note len(values) must be equal to
                   len(labels).

    Options
    -------
    @param languages: a tuple or list of ISO language code strings (e.g. "en", "fr", "de", etc.)
                      to use in expansion. If None is passed, use language classifier
                      to detect language automatically.
    @param with_name: use name in the hashes
    @param with_address: use house_number & street in the hashes
    @param with_unit: use secondary unit as part of the hashes
    @param with_city_or_equivalent: use the city, city_district, suburb, or island name as one of
                                    the geo qualifiers
    @param with_small_containing_boundaries: use small containing boundaries (currently state_district)
                                             as one of the geo qualifiers
    @param with_postal_code: use postal code as one of the geo qualifiers
    @param with_latlon: use geohash + neighbors as one of the geo qualifiers
    @param latitude: latitude (Y coordinate)
    @param longitude: longitude (X coordinate)
    @param geohash_precision: geohash tile size (default = 6)
    @param name_and_address_keys: include keys with name + address + geo
    @param name_only_keys: include keys with name + geo
    @param address_only_keys: include keys with address + geo
    """
    return _near_dupe.near_dupe_hashes(labels, values, languages=languages, **kw)


#!/usr/bin/env python

from __future__ import print_function
import glob
import re
import argparse

fixtures_directory = 'tests/fixtures/'

# restclient api header configuration
zero_api_regex = r'(\[Token )0{40}(\])'
real_api_regex = r'(\[Token ).{40}(\])'
zero_token_string = '[Token ' + 40 * '0' + ']'

def has_api_key(file_name):
    """
    Detect whether the file contains an api key in the Token object that is not 40*'0'.
    See issue #86.
    :param file: path-to-file to check
    :return: boolean
    """
    f = open(file_name, 'r')
    text = f.read()
    if re.search(real_api_regex, text) is not None and  \
            re.search(zero_api_regex, text) is None:
        return True
    return False

def remove_api_key(file_name):
    """
    Change the api key in the Token object to 40*'0'.  See issue #86.
    :param file: path-to-file to change
    """
    with open(file_name, 'r') as fp:
        text = fp.read()
    text = re.sub(real_api_regex, zero_token_string, text)
    with open(file_name, 'w') as fp:
        fp.write(text)
    return

def main(path):
    if path[-1] != '/':
        raise ValueError('Final character in path must be /.')
    n_files_changed = 0
    for filename in glob.glob(path+'*.yaml'):
        if has_api_key(filename):
            remove_api_key(filename)
            n_files_changed += 1
    print("Changed {} files.".format(n_files_changed))

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("path", help="path to test fixtures",
                        nargs='?', default=fixtures_directory)
    args = parser.parse_args()
    main(args.path)


# -*- coding: utf-8 -*-

import os
import sys
import pkg_resources
import csv
import json
from collections import namedtuple
from zipfile import ZipFile
from tiingo.restclient import RestClient
import requests
import re

try:
    import pandas as pd
    pandas_is_installed = True
except ImportError:
    pandas_is_installed = False

VERSION = pkg_resources.get_distribution("tiingo").version

# These methods enable python 2 + 3 compatibility.
def get_zipfile_from_response(response):
    if sys.version_info < (3, 0):  # python 2
        from StringIO import StringIO as Buffer
    else:  # python 3
        from io import BytesIO as Buffer
    buffered = Buffer(response.content)
    return ZipFile(buffered)

def get_buffer_from_zipfile(zipfile, filename):
    if sys.version_info < (3, 0):  # python 2
        from StringIO import StringIO
        return StringIO(zipfile.read(filename))
    else:  # python 3
        # Source:
        # https://stackoverflow.com/questions/5627954/py3k-how-do-you-read-a-file-inside-a-zip-file-as-text-not-bytes
        from io import (TextIOWrapper, BytesIO)
        return TextIOWrapper(BytesIO(zipfile.read(filename)))

def dict_to_object(item, object_name):
    """Converts a python dict to a namedtuple, saving memory."""
    fields = item.keys()
    values = item.values()
    return json.loads(json.dumps(item),
                      object_hook=lambda d:
                      namedtuple(object_name, fields)(*values))

class InstallPandasException(Exception):
    pass

class APIColumnNameError(Exception):
    pass

class InvalidFrequencyError(Exception):
    pass

class TiingoClient(RestClient):
    """Class for managing interactions with the Tiingo REST API

        Supply API Key via Environment Variable TIINGO_API_KEY
        or via the Config Object
    """

    def __init__(self, *args, **kwargs):
        super(TiingoClient, self).__init__(*args, **kwargs)
        self._base_url = "https://api.tiingo.com"

        try:
            api_key = self._config['api_key']
        except KeyError:
            api_key = os.environ.get('TIINGO_API_KEY')
        self._api_key = api_key

        if not(api_key):
            raise RuntimeError("Tiingo API Key not provided. Please provide"
                               " via environment variable or config argument.")

        self._headers = {
            'Authorization': "Token {}".format(api_key),
            'Content-Type': 'application/json',
            'User-Agent': 'tiingo-python-client {}'.format(VERSION)
        }

        self._frequency_pattern = re.compile('^[0-9]+(min|hour)$', re.IGNORECASE)

    def __repr__(self):
        return '<TiingoClient(url="{}")>'.format(self._base_url)

    def _is_eod_frequency(self,frequency):
        return frequency.lower() in ['daily', 'weekly', 'monthly', 'annually']

    # TICKER PRICE ENDPOINTS
    # https://api.tiingo.com/docs/tiingo/daily
    def list_tickers(self, assetType):
        """Return a list of dicts of metadata tickers for all supported tickers
            of the specified asset type, as well as metadata about each ticker.
            This includes supported date range, the exchange the ticker is traded
            on, and the currency the stock is traded on.
           Tickers for unrelated products are omitted.
           https://apimedia.tiingo.com/docs/tiingo/daily/supported_tickers.zip
           """
        listing_file_url = "https://apimedia.tiingo.com/docs/tiingo/daily/supported_tickers.zip"
        response = requests.get(listing_file_url)
        zipdata = get_zipfile_from_response(response)
        raw_csv = get_buffer_from_zipfile(zipdata, 'supported_tickers.csv')
        reader = csv.DictReader(raw_csv)

        return [row for row in reader
                if row.get('assetType') == assetType]

    def list_stock_tickers(self):
        return self.list_tickers('Stock')

    def list_etf_tickers(self):
        return self.list_tickers('ETF')

    def list_fund_tickers(self):
        return self.list_tickers('Mutual Fund')

    def get_ticker_metadata(self, ticker, fmt='json'):
        """Return metadata for 1 ticker
           Use TiingoClient.list_tickers() to get available options

            Args:
                ticker (str) : Unique identifier for stock
        """
        url = "tiingo/daily/{}".format(ticker)
        response = self._request('GET', url)
        data = response.json()
        if fmt == 'json':
            return data
        elif fmt == 'object':
            return dict_to_object(data, "Ticker")

    def _invalid_frequency(self, frequency):
        """
        Check to see that frequency was specified correctly
        :param frequency (string): frequency string
        :return (boolean):
        """
        is_valid = self._is_eod_frequency(frequency) or re.match(self._frequency_pattern, frequency)
        return not is_valid

    def _get_url(self, ticker, frequency):
        """
        Return url based on frequency.  Daily, weekly, or yearly use Tiingo
        EOD api; anything less than daily uses the iex intraday api.
        :param ticker (string): ticker to be embedded in the url
        :param frequency (string): valid frequency per Tiingo api
        :return (string): url
        """
        if self._invalid_frequency(frequency):
            etext = ("Error: {} is an invalid frequency.  Check Tiingo API documentation "
                     "for valid EOD or intraday frequency format.")
            raise InvalidFrequencyError(etext.format(frequency))
        else:
            if self._is_eod_frequency(frequency):
                return "tiingo/daily/{}/prices".format(ticker)
            else:
                return "iex/{}/prices".format(ticker)

    def get_ticker_price(self, ticker,
                         startDate=None, endDate=None,
                         fmt='json', frequency='daily'):
        """By default, return latest EOD Composite Price for a stock ticker.
           On average, each feed contains 3 data sources.

            Supported tickers + Available Day Ranges are here:
            https://apimedia.tiingo.com/docs/tiingo/daily/supported_tickers.zip

            Args:
                ticker (string): Unique identifier for stock ticker
                startDate (string): Start of ticker range in YYYY-MM-DD format
                endDate (string): End of ticker range in YYYY-MM-DD format
                fmt (string): 'csv' or 'json'
                frequency (string): Resample frequency
        """
        url = self._get_url(ticker, frequency)
        params = {
            'format': fmt if fmt != "object" else 'json',  # conversion local
            'resampleFreq': frequency
        }

        if startDate:
            params['startDate'] = startDate
        if endDate:
            params['endDate'] = endDate

        # TODO: evaluate whether to stream CSV to cache on disk, or
        # load as array in memory, or just pass plain text
        response = self._request('GET', url, params=params)
        if fmt == "json":
            return response.json()
        elif fmt == "object":
            data = response.json()
            return [dict_to_object(item, "TickerPrice") for item in data]
        else:
            return response.content.decode("utf-8")

    def get_dataframe(self, tickers,
                      startDate=None, endDate=None, metric_name=None, frequency='daily'):

        """ Return a pandas.DataFrame of historical prices for one or more ticker symbols.

            By default, return latest EOD Composite Price for a list of stock tickers.
            On average, each feed contains 3 data sources.

            Supported tickers + Available Day Ranges are here:
            https://apimedia.tiingo.com/docs/tiingo/daily/supported_tickers.zip
            or from the TiingoClient.list_tickers() method.

            Args:
                tickers (string/list): One or more unique identifiers for a stock ticker.
                startDate (string): Start of ticker range in YYYY-MM-DD format.
                endDate (string): End of ticker range in YYYY-MM-DD format.
                metric_name (string): Optional parameter specifying metric to be returned for each
                    ticker.  In the event of a single ticker, this is optional and if not specified
                    all of the available data will be returned.  In the event of a list of tickers,
                    this parameter is required.
                frequency (string): Resample frequency (defaults to daily).
        """

        valid_columns = ['open', 'high', 'low', 'close', 'volume', 'adjOpen', 'adjHigh', 'adjLow',
                         'adjClose', 'adjVolume', 'divCash', 'splitFactor']

        if metric_name is not None and metric_name not in valid_columns:
            raise APIColumnNameError('Valid data items are: ' + str(valid_columns))

        params = {
            'format': 'json',
            'resampleFreq': frequency
        }
        if startDate:
            params['startDate'] = startDate
        if endDate:
            params['endDate'] = endDate

        if pandas_is_installed:
            if type(tickers) is str:
                stock = tickers
                url = self._get_url(stock, frequency)
                response = self._request('GET', url, params=params)
                df = pd.DataFrame(response.json())
                if metric_name is not None:
                    prices = df[metric_name]
                    prices.index = df['date']
                else:
                    prices = df
                    prices.index = df['date']
                    del (prices['date'])
            else:
                prices = pd.DataFrame()
                for stock in tickers:
                    url = self._get_url(stock, frequency)
                    response = self._request('GET', url, params=params)
                    df = pd.DataFrame(response.json())
                    df.index = df['date']
                    df.rename(index=str, columns={metric_name: stock}, inplace=True)
                    prices = pd.concat([prices, df[stock]], axis=1)
            prices.index = pd.to_datetime(prices.index)
            return prices
        else:
            error_message = ("Pandas is not installed, but .get_ticker_price() was "
                             "called with fmt=pandas.  In order to install tiingo with "
                             "pandas, reinstall with pandas as an optional dependency. \n"
                             "Install tiingo with pandas dependency: \'pip install tiingo[pandas]\'\n"
                             "Alternatively, just install pandas: pip install pandas.")
            raise InstallPandasException(error_message)

    # NEWS FEEDS
    # tiingo/news
    def get_news(self, tickers=[], tags=[], sources=[], startDate=None,
                 endDate=None, limit=100, offset=0, sortBy="publishedDate",
                 fmt='json'):
        """Return list of news articles matching given search terms
            https://api.tiingo.com/docs/tiingo/news

            # Dates are in YYYY-MM-DD Format.

            Args:
                tickers [string] : List of unique Stock Tickers to search
                tags [string] : List of topics tagged by Tiingo Algorithms
                sources [string]: List of base urls to include as news sources
                startDate, endDate [date]: Boundaries of news search window
                limit (int): Max results returned. Default 100, max 1000
                offset (int): Search results offset, used for paginating
                sortBy (string): "publishedDate" OR (#TODO: UPDATE THIS)
        """
        url = "tiingo/news"
        params = {
            'limit': limit,
            'offset': offset,
            'sortBy': sortBy,
            'tickers': tickers,
            'sources': sources,
            'tags': tags,
            'startDate': startDate,
            'endDate': endDate
        }
        response = self._request('GET', url, params=params)
        data = response.json()
        if fmt == 'json':
            return data
        elif fmt == 'object':
            return [dict_to_object(item, "NewsArticle") for item in data]

    def get_bulk_news(self, file_id=None, fmt='json'):
        """Only available to institutional clients.
            If ID is NOT provided, return array of available file_ids.
            If ID is provided, provides URL which you can use to download your
            file, as well as some metadata about that file.
        """
        if file_id:
            url = "tiingo/news/bulk_download/{}".format(file_id)
        else:
            url = "tiingo/news/bulk_download"

        response = self._request('GET', url)
        data = response.json()
        if fmt == 'json':
            return data
        elif fmt == 'object':
            return dict_to_object(data, "BulkNews")


# -*- coding: utf-8 -*-

import logging
import requests
from requests.exceptions import HTTPError

# TODO: Possibly print HTTP json response if available?
class RestClientError(Exception):
    "Wrapper around HTTP Errors"
    pass

class RestClient(object):

    def __init__(self, config={}):
        """Base class for interacting with RESTful APIs
           Child class MUST have a ._base_url property!

            Args:
                config (dict): Arbitrary options that child classes can access
        """
        self._config = config

        # The child class should override these properties or else the
        # restclient won't work. Reevalute whether to do this as an abstract
        # base class so it doesn't get used by itself.
        self._headers = {}
        self._base_url = ""

        if config.get('session'):
            self._session = requests.Session()
        else:
            self._session = requests

    def __repr__(self):
        return '<RestClient(url="{}")>'.format(self._base_url)

    def _request(self, method, url, **kwargs):
        """Make HTTP request and return response object

            Args:
                method (str): GET, POST, PUT, DELETE
                url (str): path appended to the base_url to create request
                **kwargs: passed directly to a requests.request object
        """
        resp = self._session.request(method,
                                     '{}/{}'.format(self._base_url, url),
                                     headers=self._headers,
                                     **kwargs)

        try:
            resp.raise_for_status()
        except HTTPError as e:
            logging.error(resp.content)
            raise RestClientError(e)

        return resp


import asyncio
import json
import random
import string
from base64 import b64encode
from urllib.parse import quote

import aiohttp

from .errors import HTTPException, Forbidden, NotFound, SpotifyException

__all__ = ('HTTPClient', 'HTTPUserClient', 'Route')

_GET_BEARER_ERR = '%s was `None` when getting a bearer token.'

class Route:
    """Used for constructing URLs for API endpoints.

    Parameters
    ----------
    method : str
        The HTTP/REST method used.
    path : str
        A path to be formatted.
    kwargs : Any
        The arguments to used to format the path.

    Attributes
    ----------
    path : str
        The path template.
    method : str
        The HTTP method used.
    url : str
        The formatted path.
    """
    BASE = 'https://api.spotify.com/v1'

    def __init__(self, method, path, **kwargs):
        self.path = path
        self.method = method
        self.url = (self.BASE + self.path)

        if kwargs:
            parameters = {key: (quote(v) if isinstance(v, str) else v) for key, v in kwargs.items()}
            self.url = self.url.format(**parameters)

class HTTPClient:
    """Represents an HTTP client sending HTTP requests to the Spotify API.

    Parameters
    ----------
    client_id : str
        The client id provided by spotify for the app.
    client_secret : str
        The client secret for the app.
    loop : Optional[event loop]
        The event loop the client should run on, if no loop is specified `asyncio.get_event_loop()` is called and used instead.

    Attributes
    ----------
    loop : AbstractEventLoop
        The loop the client is running with.
    client_id : str
        The client id of the app.
    client_secret : str
        The client secret.
    """
    RETRY_AMOUNT = 10

    def __init__(self, client_id, client_secret, loop=None):
        self.loop = loop or asyncio.get_event_loop()
        self._session = aiohttp.ClientSession(loop=self.loop)

        self.client_id = client_id
        self.client_secret = client_secret

        self.bearer_info = None

    async def get_bearer_info(self):
        """Get the application bearer token from client_id and client_secret."""
        if self.client_id is None:
            raise SpotifyException(_GET_BEARER_ERR % 'client_id')

        elif self.client_secret is None:
            raise SpotifyException(_GET_BEARER_ERR % 'client_secret')

        token = b64encode(':'.join((self.client_id, self.client_secret)).encode())

        kwargs = {
            'url': 'https://accounts.spotify.com/api/token',
            'data': {'grant_type': 'client_credentials'},
            'headers': {'Authorization': 'Basic ' + token.decode()}
        }

        async with self._session.post(**kwargs) as resp:
            return json.loads(await resp.text(encoding='utf-8'))

    async def request(self, route, **kwargs):
        """Make a request to the spotify API with the current bearer credentials.

        Parameters
        ----------
        route : Union[tuple[str, str], Route]
            A tuple of the method and url or a :class:`Route` object.
        kwargs : Any
            keyword arguments to pass into :class:`aiohttp.ClientSession.request`
        """
        if isinstance(route, tuple):
            method, url = route
        else:
            method = route.method
            url = route.url

        if self.bearer_info is None:
            self.bearer_info = bearer_info = await self.get_bearer_info()
            access_token = bearer_info['access_token']
        else:
            access_token = self.bearer_info['access_token']

        headers = {
            'Authorization': 'Bearer ' + access_token,
            'Content-Type': kwargs.get('content_type', 'application/json'),
            **kwargs.pop('headers', {})
        }

        for _ in range(self.RETRY_AMOUNT):
            r = await self._session.request(method, url, headers=headers, **kwargs)
            try:
                status = r.status

                try:
                    data = json.loads(await r.text(encoding='utf-8'))
                except json.decoder.JSONDecodeError:
                    data = {}

                if 300 > status >= 200:
                    return data

                if status == 401:
                    self.bearer_info = bearer_info = await self.get_bearer_info()
                    headers['Authorization'] = 'Bearer ' + bearer_info['access_token']
                    continue

                if status == 429:
                    # we're being rate limited.
                    amount = r.headers.get('Retry-After')
                    await asyncio.sleep(int(amount), loop=self.loop)
                    continue

                if status in (502, 503):
                    # unconditional retry
                    continue

                if status == 403:
                    raise Forbidden(r, data)
                elif status == 404:
                    raise NotFound(r, data)
            finally:
                await r.release()
        else:
            raise HTTPException(r, data)

    async def close(self):
        """Close the underlying HTTP session."""
        await self._session.close()

    def album(self, spotify_id, market='US'):
        """Get a spotify album by its ID.

        Parameters
        ----------
        spotify_id : str
            The spotify_id to search by.
        market : Optional[str]
            An ISO 3166-1 alpha-2 country code.

        Returns
        -------
        album : Dict
            The album object.
        """
        route = Route('GET', '/albums/{spotify_id}', spotify_id=spotify_id)
        payload = {}

        if market:
            payload['market'] = market

        return self.request(route, params=payload)

    def album_tracks(self, spotify_id, limit=20, offset=0, market='US'):
        """Get an albums tracks by an ID.

        Parameters
        ----------
        spotify_id : str
            The spotify_id to search by.
        limit : Optional[int]
            The maximum number of items to return. Default: 20. Minimum: 1. Maximum: 50.
        offset : Optiona[int]
            The offset of which Spotify should start yielding from.
        market : Optional[str]
            An ISO 3166-1 alpha-2 country code.
        """
        route = Route('GET', '/albums/{spotify_id}/tracks', spotify_id=spotify_id)
        payload = {'limit': limit, 'offset': offset}

        if market:
            payload['market'] = market

        return self.request(route, params=payload)

    def albums(self, spotify_ids, market='US'):
        """Get a spotify album by its ID.

        Parameters
        ----------
        spotify_ids : List[str]
            The spotify_ids to search by.
        market : Optional[str]
            An ISO 3166-1 alpha-2 country code.
        """
        route = Route('GET', '/albums/')
        payload = {'ids': spotify_ids}

        if market:
            payload['market'] = market

        return self.request(route, params=payload)

    def artist(self, spotify_id):
        """Get a spotify artist by their ID.

        Parameters
        ----------
        spotify_id : str
            The spotify_id to search by.
        """
        route = Route('GET', '/artists/{spotify_id}', spotify_id=spotify_id)
        return self.request(route)

    def artist_albums(self, spotify_id, include_groups=None, limit=20, offset=0, market='US'):
        """Get an artists tracks by their ID.

        Parameters
        ----------
        spotify_id : str
            The spotify_id to search by.
        include_groups : INCLUDE_GROUPS_TP
            INCLUDE_GROUPS
        limit : Optional[int]
            The maximum number of items to return. Default: 20. Minimum: 1. Maximum: 50.
        offset : Optiona[int]
            The offset of which Spotify should start yielding from.
        market : Optional[str]
            An ISO 3166-1 alpha-2 country code.
        """
        route = Route('GET', '/artists/{spotify_id}/albums', spotify_id=spotify_id)
        payload = {'limit': limit, 'offset': offset}

        if include_groups:
            payload['include_groups'] = include_groups

        if market:
            payload['market'] = market

        return self.request(route, params=payload)

    def artist_top_tracks(self, spotify_id, country):
        """Get an artists top tracks per country with their ID.

        Parameters
        ----------
        spotify_id : str
            The spotify_id to search by.
        country : COUNTRY_TP
            COUNTRY
        """
        route = Route('GET', '/artists/{spotify_id}/top-tracks', spotify_id=spotify_id)
        payload = {'country': country}
        return self.request(route, params=payload)

    def artist_related_artists(self, spotify_id):
        """Get related artists for an artist by their ID.

        Parameters
        ----------
        spotify_id : str
            The spotify_id to search by.
        """
        route = Route('GET', '/artists/{spotify_id}/related-artists', spotify_id=spotify_id)
        return self.request(route)

    def artists(self, spotify_ids):
        """Get a spotify artists by their IDs.

        Parameters
        ----------
        spotify_id : List[str]
            The spotify_ids to search with.
        """
        route = Route('GET', '/artists')
        payload = {'ids': spotify_ids}
        return self.request(route, params=payload)

    def category(self, category_id, country=None, locale=None):
        """Get a single category used to tag items in Spotify.

        Parameters
        ----------
        category_id : str
            The Spotify category ID for the category.
        country : COUNTRY_TP
            COUNTRY
        locale : LOCALE_TP
            LOCALE
        """
        route = Route('GET', '/browse/categories/{category_id}', category_id=category_id)
        payload = {}

        if country:
            payload['country'] = country

        if locale:
            payload['locale'] = locale

        return self.request(route, params=payload)

    def category_playlists(self, category_id, limit=20, offset=0, country=None):
        """Get a list of Spotify playlists tagged with a particular category.

        Parameters
        ----------
        category_id : str
            The Spotify category ID for the category.
        limit : Optional[int]
            The maximum number of items to return. Default: 20. Minimum: 1. Maximum: 50.
        offset : Optional[int]
            The index of the first item to return. Default: 0
        country : COUNTRY_TP
            COUNTRY
        """
        route = Route('GET', '/browse/categories/{category_id}/playlists', category_id=category_id)
        payload = {'limit': limit, 'offset': offset}

        if country:
            payload['country'] = country

        return self.request(route, params=payload)

    def categories(self, limit=20, offset=0, country=None, locale=None):
        """Get a list of categories used to tag items in Spotify.

        Parameters
        ----------
        limit : Optional[int]
            The maximum number of items to return. Default: 20. Minimum: 1. Maximum: 50.
        offset : Optional[int]
            The index of the first item to return. Default: 0
        country : COUNTRY_TP
            COUNTRY
        locale : LOCALE_TP
            LOCALE
        """
        route = Route('GET', '/browse/categories')
        payload = {'limit': limit, 'offset': offset}

        if country:
            payload['country'] = country

        if locale:
            payload['locale'] = locale

        return self.request(route, params=payload)

    def featured_playlists(self, locale=None, country=None, timestamp=None, limit=20, offset=0):
        """Get a list of Spotify featured playlists.

        Parameters
        ----------
        locale : LOCALE_TP
            LOCALE
        country : COUNTRY_TP
            COUNTRY
        timestamp : TIMESTAMP_TP
            TIMESTAMP
        limit : Optional[int]
            The maximum number of items to return. Default: 20. Minimum: 1. Maximum: 50.
        offset : Optional[int]
            The index of the first item to return. Default: 0
        """
        route = Route('GET', '/browse/featured-playlists')
        payload = {'limit': limit, 'offset': offset}

        if country:
            payload['country'] = country

        if locale:
            payload['locale'] = locale

        if timestamp:
            payload['timestamp'] = timestamp

        return self.request(route, params=payload)

    def new_releases(self, *, country=None, limit=20, offset=0):
        """Get a list of new album releases featured in Spotify.

        Parameters
        ----------
        limit : Optional[int]
            The maximum number of items to return. Default: 20. Minimum: 1. Maximum: 50.
        offset : Optional[int]
            The index of the first item to return. Default: 0
        country : COUNTRY_TP
            COUNTRY
        """
        route = Route('GET', '/browse/new-releases')
        payload = {'limit': limit, 'offset': offset}

        if country:
            payload['country'] = country

        return self.request(route, params=payload)

    def recommendations(self, seed_artists, seed_genres, seed_tracks, *, limit=20, market=None, **filters):
        """Get Recommendations Based on Seeds.

        Parameters
        ----------
        seed_artists : str
            A comma separated list of Spotify IDs for seed artists. Up to 5 seed values may be provided.
        seed_genres : str
            A comma separated list of any genres in the set of available genre seeds. Up to 5 seed values may be provided.
        seed_tracks : str
            A comma separated list of Spotify IDs for a seed track. Up to 5 seed values may be provided.
        limit : Optional[int]
            The maximum number of items to return. Default: 20. Minimum: 1. Maximum: 50.
        market : Optional[str]
            An ISO 3166-1 alpha-2 country code.
        max_* : Optional[Keyword arguments]
            For each tunable track attribute, a hard ceiling on the selected track attribute’s value can be provided.
        min_* : Optional[Keyword arguments]
            For each tunable track attribute, a hard floor on the selected track attribute’s value can be provided.
        target_* : Optional[Keyword arguments]
            For each of the tunable track attributes (below) a target value may be provided.
        """
        route = Route('GET', '/recommendations')
        payload = {'seed_artists': seed_artists, 'seed_genres': seed_genres, 'seed_tracks': seed_tracks, 'limit': limit}

        if market:
            payload['market'] = market

        if filters:
            payload.update(filters)

        return self.request(route, param=payload)

    def following_artists_or_users(self, ids, *, type='artist'):
        """Check to see if the current user is following one or more artists or other Spotify users.

        Parameters
        ----------
        ids : List[str]
            A comma-separated list of the artist or the user Spotify IDs to check.
            A maximum of 50 IDs can be sent in one request.
        type : Optional[str]
            The ID type: either "artist" or "user".
            Default: "artist"
        """
        route = Route('GET', '/me/following/contains')
        payload = {'ids': ids, 'type': type}

        return self.request(route, params=payload)

    def following_playlists(self, owner_id, playlist_id, *, ids):
        route = Route('GET', '/users/{owner_id}/playlists/{playlist_id}/followers/contains', owner_id=owner_id, playlist_id=playlist_id)
        payload = {'ids': ids}

        return self.request(route, params=payload)

    def follow_artist_or_user(self, ids, *, type='artist'):
        """Add the current user as a follower of one or more artists or other Spotify users.

        Parameters
        ----------
        ids : List[str]
            A comma-separated list of the artist or the user Spotify IDs to check.
            A maximum of 50 IDs can be sent in one request.
        type : Optional[str]
            The ID type: either "artist" or "user".
            Default: "artist"
        """
        route = Route('PUT', '/me/following')
        payload = {'ids': ids, 'type': type}

        return self.request(route, params=payload)

    def follow_playlist(self, owner_id, playlist_id, *, public=False):
        route = Route('PUT', '/users/{owner_id}/playlists/{playlist_id}/followers', owner_id=owner_id, playlist_id=playlist_id)

        content = json.dumps({'public': public})

        return self.request(route, content=content)

    def followed_artists(self, limit=20, after=None):
        route = Route('GET', '/me/following')
        payload = {'limit': limit, 'type': 'artist'}

        if after:
            payload['after'] = after

        return self.request(route, params=payload)

    def unfollow_artists_or_users(self, ids, *, type='artist'):
        route = Route('DELETE', '/me/following')
        payload = {'ids': ids, 'type': type}

        return self.request(route, params=payload)

    def unfollow_playlist(self, owner_id, playlist_id):
        route = Route('DELETE', '/users/{owner_id}/playlists/{playlist_id}/followers', owner_id=owner_id, playlist_id=playlist_id)

        return self.request(route)

    def is_saved_album(self, ids):
        route = Route('GET', '/me/albums/contains')
        payload = {'ids': ids}

        return self.request(route, params=payload)

    def is_saved_track(self, ids):
        route = Route('GET', '/me/tracks/contains')
        payload = {'ids': ids}

        return self.request(route, params=payload)

    def saved_albums(self, *, limit=20, offset=0, market=None):
        route = Route('GET', '/me/albums')
        payload = {'limit': limit, 'offset': offset}

        if market:
            payload['market'] = market

        return self.request(route, params=payload)

    def saved_tracks(self, *, limit=20, offset=0, market=None):
        route = Route('GET', '/me/tracks')
        payload = {'limit': limit, 'offset': offset}

        if market:
            payload['market'] = market

        return self.request(route, params=payload)

    def delete_saved_albums(self, ids):
        route = Route('DELETE', '/me/albums')
        payload = {'ids': ids}

        return self.request(route, params=payload)

    def delete_saved_tracks(self, ids):
        route = Route('DELETE', '/me/tracks')
        payload = {'ids': ids}

        return self.request(route, params=payload)

    def save_tracks(self, ids):
        route = Route('PUT', '/me/tracks')
        payload = {'ids': ids}

        return self.request(route, params=payload)

    def save_albums(self, ids):
        route = Route('PUT', '/me/albums')
        payload = {'ids': ids}

        return self.request(route, params=payload)

    def top_artists_or_tracks(self, type, *, limit=20, offset=0, time_range=None):
        route = Route('GET', '/me/top/{type}', type=type)
        payload = {'limit': limit, 'offset': offset}

        if time_range:
            payload['time_range'] = time_range

        return self.request(route)

    def available_devices(self):
        route = Route('GET', '/me/player/devices')
        return self.request(route)

    def current_player(self, *, market=None):
        route = Route('GET', '/me/player')
        payload = {}

        if market:
            payload['market'] = market

        return self.request(route, params=payload)

    def recently_played(self, *, limit=20, before=None, after=None):
        route = Route('GET', '/me/player/recently-played')
        payload = {'limit': limit}

        if before:
            payload['before'] = before
        elif after:
            payload['after'] = after

        return self.request(route, params=payload)

    def currently_playing(self, *, market=None):
        route = Route('GET', '/me/player/currently-playing')
        payload = {}

        if market:
            payload['market'] = market

        return self.request(route, params=payload)

    def pause_playback(self, *, device_id=None):
        route = Route('PUT', '/me/player/pause')
        payload = {}

        if device_id:
            payload['device_id'] = device_id

        return self.request(route, params=payload)

    def seek_playback(self, position_ms, *, device_id=None):
        route = Route('PUT', '/me/player/seek')
        payload = {'position_ms': position_ms}

        if device_id:
            payload['device_id'] = device_id

        return self.request(route, params=payload)

    def repeat_playback(self, state, *, device_id=None):
        route = Route('PUT', '/me/player/repeat')
        payload = {'state': state}

        if device_id:
            payload['device_id'] = device_id

        return self.request(route, params=payload)

    def set_playback_volume(self, volume, *, device_id=None):
        route = Route('PUT', '/me/player/volume')
        payload = {'volume_percent': volume}

        if device_id:
            payload['device_id'] = device_id

        return self.request(route, params=payload)

    def skip_next(self, *, device_id=None):
        route = Route('POST', '/me/player/next')
        payload = {}

        if device_id:
            payload['device_id'] = device_id

        return self.request(route, params=payload)

    def skip_previous(self, *, device_id=None):
        route = Route('POST', '/me/player/previous')
        payload = {}

        if device_id:
            payload['device_id'] = device_id

        return self.request(route, params=payload)

    def play_playback(self, context_uri, *, offset=None, device_id=None):
        route = Route('PUT', '/me/player/play')
        payload = {}

        if isinstance(context_uri, str):
            payload['context_uri'] = {'context_uri': context_uri}
        elif context_uri is not None:
            payload['uris'] = {'uris': list(*context_uri)}

        if offset:
            payload['offset'] = offset

        if device_id:
            payload['device_id'] = device_id

        return self.request(route, data=json.dumps(payload))

    def shuffle_playback(self, state, *, device_id=None):
        route = Route('PUT', '/me/player/seek')
        payload = {'state': state}

        if device_id:
            payload['device_id'] = device_id

        return self.request(route, params=payload)

    def transfer_player(self, device_id, *, play=None):
        route = Route('PUT', '/me/player')
        payload = {'device_ids': [device_id]}

        if play:
            payload['play'] = play

        return self.request(route, data=json.dumps(payload))

    def add_playlist_tracks(self, user_id, playlist_id, tracks, *, position=None):
        route = Route('POST', '/users/{user_id}/playlists/{playlist_id}/tracks', user_id=user_id, playlist_id=playlist_id)
        payload = {'uris': tracks}

        if position:
            payload['position'] = position

        return self.request(route, params=payload)

    def change_playlist_details(self, user_id, playlist_id, *, data):
        route = Route('PUT', '/users/{user_id}/playlists/{playlist_id}/tracks', user_id=user_id, playlist_id=playlist_id)
        return self.request(route, data=json.dumps(data))

    def create_playlist(self, user_id, *, data):
        route = Route('POST', '/users/{user_id}/playlists', user_id=user_id)
        return self.request(route, data=json.dumps(data))

    def current_playlists(self, *, limit=20, offset=0):
        route = Route('GET', '/me/playlists')
        return self.request(route, params={'limit': limit, 'offset': offset})

    def get_playlists(self, user_id, *, limit=20, offset=0):
        route = Route('GET', '/users/{user_id}/playlists', user_id=user_id)
        return self.request(route, params={'limit': limit, 'offset': offset})

    def get_playlist_cover_image(self, user_id, playlist_id):
        route = Route('GET', '/users/{user_id}/playlists/{playlist_id}/images', user_id=user_id, playlist_id=playlist_id)
        return self.request(route)

    def get_playlist(self, user_id, playlist_id, *, fields=None, market=None):
        route = Route('GET', '/users/{user_id}/playlists/{playlist_id}', user_id=user_id, playlist_id=playlist_id)
        payload = {}

        if fields:
            payload['fields'] = fields

        if market:
            payload['market'] = market

        return self.request(route, params=payload)

    def get_playlist_tracks(self, user_id, playlist_id, *, fields=None, market=None, limit=20, offset=0):
        route = Route('GET', '/users/{user_id}/playlists/{playlist_id}/tracks', user_id=user_id, playlist_id=playlist_id)
        payload = {'limit': limit, 'offset': offset}

        if fields:
            payload['fields'] = fields

        if market:
            payload['market'] = market

        return self.request(route, params=payload)

    def remove_playlist_tracks(self, user_id, playlist_id, tracks, *, position=None, snapshot_id=None):
        route = Route('DELETE ', '/users/{user_id}/playlists/{playlist_id}/tracks', user_id=user_id, playlist_id=playlist_id)
        payload = {'uris': tracks}

        if position:
            payload['position'] = position

        if snapshot_id:
            payload['snapshot_id'] = snapshot_id

        return self.request(route, params=payload)

    def reorder_playlists_tracks(self, user_id, playlist_id, range_start, range_length, insert_before, *, snapshot_id=None):
        route = Route('PUT', '/users/{user_id}/playlists/{playlist_id}/tracks', user_id=user_id, playlist_id=playlist_id)
        payload = {'range_start': range_start, 'range_length': range_length, 'insert_before': insert_before}

        if snapshot_id:
            payload['snapshot_id'] = snapshot_id

        return self.request(route, data=payload)

    def replace_playlist_tracks(self, user_id, playlist_id, tracks):
        route = Route('PUT', '/users/{user_id}/playlists/{playlist_id}/tracks', user_id=user_id, playlist_id=playlist_id)
        payload = {'uris': tracks}

        return self.request(route, params=payload)

    def upload_playlist_cover_image(self, user_id, playlist_id, file):
        route = Route('PUT', '/users/{user_id}/playlists/{playlist_id}/images', user_id=user_id, playlist_id=playlist_id)
        return self.request(route, data=b64encode(file.read()), content_type='image/jpeg')

    def track_audio_analysis(self, track_id):
        route = Route('GET', '/audio-analysis/{id}', id=track_id)
        return self.request(route)

    def track_audio_features(self, track_id):
        route = Route('GET', '/audio-features/{id}', id=track_id)
        return self.request(route)

    def audio_features(self, track_ids):
        route = Route('GET', '/audio-features')
        return self.request(route, params={'ids': track_ids})

    def track(self, track_id):
        route = Route('GET', '/tracks/{id}', id=track_id)
        return self.request(route)

    def tracks(self, track_ids):
        route = Route('GET', '/tracks')
        return self.request(route, params={'ids': track_ids})

    def current_user(self):
        route = Route('GET', '/me')
        return self.request(route)

    def user(self, user_id):
        route = Route('GET', '/users/{user_id}', user_id=user_id)
        return self.request(route)

    def search(self, q, queary_type='track,playlist,artist,album', market='US', limit=20, offset=0):
        route = Route('GET', '/search')
        payload = {'q': q, 'type': queary_type, 'limit': limit, 'offset': offset}

        if market:
            payload['market'] = market

        return self.request(route, params=payload)

class HTTPUserClient(HTTPClient):
    '''HTTPClient for access to user endpoints.'''
    def __init__(self, token, loop=None):
        self.loop = loop or asyncio.get_event_loop()
        self._session = aiohttp.ClientSession(loop=self.loop)

        self.bearer_info = {'access_token': token}
        self.token = token

    async def get_bearer_info(self):
        return {'access_token': self.token}


from typing import Optional, List

from . import SpotifyBase, URIBase, Image

class Artist(URIBase):
    """A Spotify Artist.

    Attributes
    ----------
    id : str
        The Spotify ID of the artist.
    uri : str
        The URI of the artist.
    url : str
        The open.spotify URL.
    href : str
        A link to the Web API endpoint providing full details of the artist.
    name : str
        The name of the artist.
    genres : List[str]
        A list of the genres the artist is associated with.
        For example: "Prog Rock" , "Post-Grunge". (If not yet classified, the array is empty.)
    followers : Optional[int]
        The total number of followers.
    popularity : int
        The popularity of the artist.
        The value will be between 0 and 100, with 100 being the most popular.
        The artist’s popularity is calculated from the popularity of all the artist’s tracks.
    images : List[Image]
        Images of the artist in various sizes, widest first.
    """
    def __init__(self, client, data):
        self.__client = client

        # Simplified object attributes
        self.id = data.pop('id')
        self.uri = data.pop('uri')
        self.url = data.pop('external_urls').get('spotify', None)
        self.href = data.pop('href')
        self.name = data.pop('name')

        # Full object attributes
        self.genres = data.pop('genres', None)
        self.followers = data.pop('followers', {}).get('total', None)
        self.popularity = data.pop('popularity', None)
        self.images = list(Image(**image) for image in data.pop('images', []))

    def __repr__(self):
        return '<spotify.Artist: "%s">' % self.name

    async def get_albums(self, *, limit: Optional[int] = 20, offset: Optional[int] = 0, include_groups=None, market: Optional[str] = None) -> List[Album]:
        """Get the albums of a Spotify artist.

        Parameters
        ----------
        limit : Optional[int]
            The maximum number of items to return. Default: 20. Minimum: 1. Maximum: 50.
        offset : Optiona[int]
            The offset of which Spotify should start yielding from.
        include_groups : INCLUDE_GROUPS_TP
            INCLUDE_GROUPS
        market : Optional[str]
            An ISO 3166-1 alpha-2 country code.

        Returns
        -------
        albums : List[Album]
            The albums of the artist.
        """
        from .album import Album

        data = await self.__client.http.artist_albums(self.id, limit=limit, offset=offset, include_groups=include_groups, market=market)
        return list(Album(self.__client, item) for item in data['items'])

    async def get_all_albums(self, *, market='US') -> List[Album]:
        """loads all of the artists albums, depending on how many the artist has this may be a long operation.

        Parameters
        ----------
        market : Optional[str]
            An ISO 3166-1 alpha-2 country code.

        Returns
        -------
        albums : List[Album]
            The albums of the artist.
        """
        from .album import Album

        albums = []
        offset = 0
        total = await self.total_albums(market=market)

        while len(albums) < total:
            data = await self.__client.http.artist_albums(self.id, limit=50, offset=offset, market=market)

            offset += 50
            albums += list(Album(self.__client, item) for item in data['items'])

        return albums

    async def total_albums(self, *, market: str = None) -> int:
        """get the total amout of tracks in the album.

        Parameters
        ----------
        market : Optional[str]
            An ISO 3166-1 alpha-2 country code.

        Returns
        -------
        total : int
            The total amount of albums.
        """
        data = await self.__client.http.artist_albums(self.id, limit=1, offset=0, market=market)
        return data['total']

    async def top_tracks(self, country: str = 'US') -> List[Track]:
        """Get Spotify catalog information about an artist’s top tracks by country.

        Parameters
        ----------
        country : str
            The country to search for, it defaults to 'US'.

        Returns
        -------
        tracks : List[Track]
            The artists top tracks.
        """
        from .track import Track

        top = await self.__client.http.artist_top_tracks(self.id, country=country)
        return list(Track(self.__client, item) for item in top['tracks'])

    async def related_artists(self) -> List[Artist]:
        """Get Spotify catalog information about artists similar to a given artist.

        Similarity is based on analysis of the Spotify community’s listening history.

        Returns
        -------
        artists : List[Artits]
            The artists deemed similar.
        """
        related = await self.__client.http.artist_related_artists(self.id)
        return list(Artist(self.__client, item) for item in related['artists'])