from __future__ import absolute_import

import logging
import os

from pip._vendor import pkg_resources
from pip._vendor import requests

from pip.download import (url_to_path, unpack_url)
from pip.exceptions import (InstallationError, BestVersionAlreadyInstalled,
                            DistributionNotFound, PreviousBuildDirError)
from pip.locations import (PIP_DELETE_MARKER_FILENAME, build_prefix)
from pip.req.req_install import InstallRequirement
from pip.utils import (display_path, rmtree, dist_in_usersite,
                       _make_build_dir, normalize_path)
from pip.utils.logging import indent_log
from pip.vcs import vcs
from pip.wheel import wheel_ext

logger = logging.getLogger(__name__)

class Requirements(object):

    def __init__(self):
        self._keys = []
        self._dict = {}

    def keys(self):
        return self._keys

    def values(self):
        return [self._dict[key] for key in self._keys]

    def __contains__(self, item):
        return item in self._keys

    def __setitem__(self, key, value):
        if key not in self._keys:
            self._keys.append(key)
        self._dict[key] = value

    def __getitem__(self, key):
        return self._dict[key]

    def __repr__(self):
        values = ['%s: %s' % (repr(k), repr(self[k])) for k in self.keys()]
        return 'Requirements({%s})' % ', '.join(values)

class RequirementSet(object):

    def __init__(self, build_dir, src_dir, download_dir, upgrade=False,
                 ignore_installed=False, as_egg=False, target_dir=None,
                 ignore_dependencies=False, force_reinstall=False,
                 use_user_site=False, session=None, pycompile=True,
                 isolated=False, wheel_download_dir=None):
        if session is None:
            raise TypeError(
                "RequirementSet() missing 1 required keyword argument: "
                "'session'"
            )

        self.build_dir = build_dir
        self.src_dir = src_dir
        self.download_dir = download_dir
        self.upgrade = upgrade
        self.ignore_installed = ignore_installed
        self.force_reinstall = force_reinstall
        self.requirements = Requirements()
        # Mapping of alias: real_name
        self.requirement_aliases = {}
        self.unnamed_requirements = []
        self.ignore_dependencies = ignore_dependencies
        self.successfully_downloaded = []
        self.successfully_installed = []
        self.reqs_to_cleanup = []
        self.as_egg = as_egg
        self.use_user_site = use_user_site
        self.target_dir = target_dir  # set from --target option
        self.session = session
        self.pycompile = pycompile
        self.isolated = isolated
        if wheel_download_dir:
            wheel_download_dir = normalize_path(wheel_download_dir)
        self.wheel_download_dir = wheel_download_dir

    def __str__(self):
        reqs = [req for req in self.requirements.values()
                if not req.comes_from]
        reqs.sort(key=lambda req: req.name.lower())
        return ' '.join([str(req.req) for req in reqs])

    def add_requirement(self, install_req):
        if not install_req.match_markers():
            logger.debug("Ignore %s: markers %r don't match",
                         install_req.name, install_req.markers)
            return

        name = install_req.name
        install_req.as_egg = self.as_egg
        install_req.use_user_site = self.use_user_site
        install_req.target_dir = self.target_dir
        install_req.pycompile = self.pycompile
        if not name:
            # url or path requirement w/o an egg fragment
            self.unnamed_requirements.append(install_req)
        else:
            if self.has_requirement(name):
                raise InstallationError(
                    'Double requirement given: %s (already in %s, name=%r)'
                    % (install_req, self.get_requirement(name), name))
            self.requirements[name] = install_req
            # FIXME: what about other normalizations?  E.g., _ vs. -?
            if name.lower() != name:
                self.requirement_aliases[name.lower()] = name

    def has_requirement(self, project_name):
        for name in project_name, project_name.lower():
            if name in self.requirements or name in self.requirement_aliases:
                return True
        return False

    @property
    def has_requirements(self):
        return list(self.requirements.values()) or self.unnamed_requirements

    @property
    def is_download(self):
        if self.download_dir:
            self.download_dir = os.path.expanduser(self.download_dir)
            if os.path.exists(self.download_dir):
                return True
            else:
                logger.critical('Could not find download directory')
                raise InstallationError(
                    "Could not find or access download directory '%s'"
                    % display_path(self.download_dir))
        return False

    def get_requirement(self, project_name):
        for name in project_name, project_name.lower():
            if name in self.requirements:
                return self.requirements[name]
            if name in self.requirement_aliases:
                return self.requirements[self.requirement_aliases[name]]
        raise KeyError("No project with the name %r" % project_name)

    def uninstall(self, auto_confirm=False):
        for req in self.requirements.values():
            req.uninstall(auto_confirm=auto_confirm)
            req.commit_uninstall()

    def locate_files(self):
        # FIXME: duplicates code from prepare_files; relevant code should
        #        probably be factored out into a separate method
        unnamed = list(self.unnamed_requirements)
        reqs = list(self.requirements.values())
        while reqs or unnamed:
            if unnamed:
                req_to_install = unnamed.pop(0)
            else:
                req_to_install = reqs.pop(0)
            install_needed = True
            if not self.ignore_installed and not req_to_install.editable:
                req_to_install.check_if_exists()
                if req_to_install.satisfied_by:
                    if self.upgrade:
                        # don't uninstall conflict if user install and
                        # conflict is not user install
                        if not (self.use_user_site
                                and not dist_in_usersite(
                                    req_to_install.satisfied_by
                                )):
                            req_to_install.conflicts_with = \
                                req_to_install.satisfied_by
                        req_to_install.satisfied_by = None
                    else:
                        install_needed = False
                if req_to_install.satisfied_by:
                    logger.info(
                        'Requirement already satisfied (use --upgrade to '
                        'upgrade): %s',
                        req_to_install,
                    )

            if req_to_install.editable:
                if req_to_install.source_dir is None:
                    req_to_install.source_dir = req_to_install.build_location(
                        self.src_dir
                    )
            elif install_needed:
                req_to_install.source_dir = req_to_install.build_location(
                    self.build_dir,
                )

            if (req_to_install.source_dir is not None
                    and not os.path.isdir(req_to_install.source_dir)):
                raise InstallationError(
                    'Could not install requirement %s because source folder %s'
                    ' does not exist (perhaps --no-download was used without '
                    'first running an equivalent install with --no-install?)' %
                    (req_to_install, req_to_install.source_dir)
                )

    def prepare_files(self, finder):
        """
        Prepare process. Create temp directories, download and/or unpack files.
        """
        from pip.index import Link

        unnamed = list(self.unnamed_requirements)
        reqs = list(self.requirements.values())
        while reqs or unnamed:
            if unnamed:
                req_to_install = unnamed.pop(0)
            else:
                req_to_install = reqs.pop(0)
            install = True
            best_installed = False
            not_found = None

            # ############################################# #
            # # Search for archive to fulfill requirement # #
            # ############################################# #

            if not self.ignore_installed and not req_to_install.editable:
                req_to_install.check_if_exists()
                if req_to_install.satisfied_by:
                    if self.upgrade:
                        if not self.force_reinstall and not req_to_install.url:
                            try:
                                url = finder.find_requirement(
                                    req_to_install, self.upgrade)
                            except BestVersionAlreadyInstalled:
                                best_installed = True
                                install = False
                            except DistributionNotFound as exc:
                                not_found = exc
                            else:
                                # Avoid the need to call find_requirement again
                                req_to_install.url = url.url

                        if not best_installed:
                            # don't uninstall conflict if user install and
                            # conflict is not user install
                            if not (self.use_user_site
                                    and not dist_in_usersite(
                                        req_to_install.satisfied_by
                                    )):
                                req_to_install.conflicts_with = \
                                    req_to_install.satisfied_by
                            req_to_install.satisfied_by = None
                    else:
                        install = False
                if req_to_install.satisfied_by:
                    if best_installed:
                        logger.info(
                            'Requirement already up-to-date: %s',
                            req_to_install,
                        )
                    else:
                        logger.info(
                            'Requirement already satisfied (use --upgrade to '
                            'upgrade): %s',
                            req_to_install,
                        )
            if req_to_install.editable:
                logger.info('Obtaining %s', req_to_install)
            elif install:
                if (req_to_install.url
                        and req_to_install.url.lower().startswith('file:')):
                    path = url_to_path(req_to_install.url)
                    logger.info('Processing %s', display_path(path))
                else:
                    logger.info('Collecting %s', req_to_install)

            with indent_log():
                # ################################ #
                # # vcs update or unpack archive # #
                # ################################ #

                is_wheel = False
                if req_to_install.editable:
                    if req_to_install.source_dir is None:
                        location = req_to_install.build_location(self.src_dir)
                        req_to_install.source_dir = location
                    else:
                        location = req_to_install.source_dir
                    if not os.path.exists(self.build_dir):
                        _make_build_dir(self.build_dir)
                    req_to_install.update_editable(not self.is_download)
                    if self.is_download:
                        req_to_install.run_egg_info()
                        req_to_install.archive(self.download_dir)
                    else:
                        req_to_install.run_egg_info()
                elif install:
                    # @@ if filesystem packages are not marked
                    # editable in a req, a non deterministic error
                    # occurs when the script attempts to unpack the
                    # build directory

                    # NB: This call can result in the creation of a temporary
                    # build directory
                    location = req_to_install.build_location(
                        self.build_dir,
                    )
                    unpack = True
                    url = None

                    # If a checkout exists, it's unwise to keep going.  version
                    # inconsistencies are logged later, but do not fail the
                    # installation.
                    if os.path.exists(os.path.join(location, 'setup.py')):
                        raise PreviousBuildDirError(
                            "pip can't proceed with requirements '%s' due to a"
                            " pre-existing build directory (%s). This is "
                            "likely due to a previous installation that failed"
                            ". pip is being responsible and not assuming it "
                            "can delete this. Please delete it and try again."
                            % (req_to_install, location)
                        )
                    else:
                        # FIXME: this won't upgrade when there's an existing
                        # package unpacked in `location`
                        if req_to_install.url is None:
                            if not_found:
                                raise not_found
                            url = finder.find_requirement(
                                req_to_install,
                                upgrade=self.upgrade,
                            )
                        else:
                            # FIXME: should req_to_install.url already be a
                            # link?
                            url = Link(req_to_install.url)
                            assert url
                        if url:
                            try:

                                if (
                                    url.filename.endswith(wheel_ext)
                                    and self.wheel_download_dir
                                ):
                                    # when doing 'pip wheel`
                                    download_dir = self.wheel_download_dir
                                    do_download = True
                                else:
                                    download_dir = self.download_dir
                                    do_download = self.is_download
                                unpack_url(
                                    url, location, download_dir,
                                    do_download, session=self.session,
                                )
                            except requests.HTTPError as exc:
                                logger.critical(
                                    'Could not install requirement %s because '
                                    'of error %s',
                                    req_to_install,
                                    exc,
                                )
                                raise InstallationError(
                                    'Could not install requirement %s because '
                                    'of HTTP error %s for URL %s' %
                                    (req_to_install, exc, url)
                                )
                        else:
                            unpack = False
                    if unpack:
                        is_wheel = url and url.filename.endswith(wheel_ext)
                        if self.is_download:
                            req_to_install.source_dir = location
                            if not is_wheel:
                                # FIXME:https://github.com/pypa/pip/issues/1112
                                req_to_install.run_egg_info()
                            if url and url.scheme in vcs.all_schemes:
                                req_to_install.archive(self.download_dir)
                        elif is_wheel:
                            req_to_install.source_dir = location
                            req_to_install.url = url.url
                        else:
                            req_to_install.source_dir = location
                            req_to_install.run_egg_info()
                            req_to_install.assert_source_matches_version()
                        # req_to_install.req is only avail after unpack for URL
                        # pkgs repeat check_if_exists to uninstall-on-upgrade
                        # (#14)
                        if not self.ignore_installed:
                            req_to_install.check_if_exists()
                        if req_to_install.satisfied_by:
                            if self.upgrade or self.ignore_installed:
                                # don't uninstall conflict if user install and
                                # conflict is not user install
                                if not (self.use_user_site
                                        and not dist_in_usersite(
                                            req_to_install.satisfied_by)):
                                    req_to_install.conflicts_with = \
                                        req_to_install.satisfied_by
                                req_to_install.satisfied_by = None
                            else:
                                logger.info(
                                    'Requirement already satisfied (use '
                                    '--upgrade to upgrade): %s',
                                    req_to_install,
                                )
                                install = False

                # ###################### #
                # # parse dependencies # #
                # ###################### #
                if (req_to_install.extras):
                    logger.debug(
                        "Installing extra requirements: %r",
                        ','.join(req_to_install.extras),
                    )

                if is_wheel:
                    dist = list(
                        pkg_resources.find_distributions(location)
                    )[0]
                else:  # sdists
                    if req_to_install.satisfied_by:
                        dist = req_to_install.satisfied_by
                    else:
                        dist = req_to_install.get_dist()
                    # FIXME: shouldn't be globally added:
                    if dist.has_metadata('dependency_links.txt'):
                        finder.add_dependency_links(
                            dist.get_metadata_lines('dependency_links.txt')
                        )

                if not self.ignore_dependencies:
                    for subreq in dist.requires(
                            req_to_install.extras):
                        if self.has_requirement(
                                subreq.project_name):
                            # FIXME: check for conflict
                            continue
                        subreq = InstallRequirement(
                            str(subreq),
                            req_to_install,
                            isolated=self.isolated,
                        )
                        reqs.append(subreq)
                        self.add_requirement(subreq)

                if not self.has_requirement(req_to_install.name):
                    # 'unnamed' requirements will get added here
                    self.add_requirement(req_to_install)

                # cleanup tmp src
                if (self.is_download or
                        req_to_install._temp_build_dir is not None):
                    self.reqs_to_cleanup.append(req_to_install)

                if install:
                    self.successfully_downloaded.append(req_to_install)

    def cleanup_files(self):
        """Clean up files, remove builds."""
        logger.debug('Cleaning up...')
        with indent_log():
            for req in self.reqs_to_cleanup:
                req.remove_temporary_source()

            if self._pip_has_created_build_dir():
                logger.debug('Removing temporary dir %s...', self.build_dir)
                rmtree(self.build_dir)

    def _pip_has_created_build_dir(self):
        return (
            self.build_dir == build_prefix
            and os.path.exists(
                os.path.join(self.build_dir, PIP_DELETE_MARKER_FILENAME)
            )
        )

    def install(self, install_options, global_options=(), *args, **kwargs):
        """
        Install everything in this set (after having downloaded and unpacked
        the packages)
        """
        to_install = [r for r in self.requirements.values()[::-1]
                      if not r.satisfied_by]

        # DISTRIBUTE TO SETUPTOOLS UPGRADE HACK (1 of 3 parts)
        # move the distribute-0.7.X wrapper to the end because it does not
        # install a setuptools package. by moving it to the end, we ensure it's
        # setuptools dependency is handled first, which will provide the
        # setuptools package
        # TODO: take this out later
        distribute_req = pkg_resources.Requirement.parse("distribute>=0.7")
        for req in to_install:
            if (req.name == 'distribute'
                    and req.installed_version is not None
                    and req.installed_version in distribute_req):
                to_install.remove(req)
                to_install.append(req)

        if to_install:
            logger.info(
                'Installing collected packages: %s',
                ', '.join([req.name for req in to_install]),
            )

        with indent_log():
            for requirement in to_install:

                # DISTRIBUTE TO SETUPTOOLS UPGRADE HACK (1 of 3 parts)
                # when upgrading from distribute-0.6.X to the new merged
                # setuptools in py2, we need to force setuptools to uninstall
                # distribute. In py3, which is always using distribute, this
                # conversion is already happening in distribute's
                # pkg_resources. It's ok *not* to check if setuptools>=0.7
                # because if someone were actually trying to ugrade from
                # distribute to setuptools 0.6.X, then all this could do is
                # actually help, although that upgade path was certainly never
                # "supported"
                # TODO: remove this later
                if requirement.name == 'setuptools':
                    try:
                        # only uninstall distribute<0.7. For >=0.7, setuptools
                        # will also be present, and that's what we need to
                        # uninstall
                        distribute_requirement = \
                            pkg_resources.Requirement.parse("distribute<0.7")
                        existing_distribute = \
                            pkg_resources.get_distribution("distribute")
                        if existing_distribute in distribute_requirement:
                            requirement.conflicts_with = existing_distribute
                    except pkg_resources.DistributionNotFound:
                        # distribute wasn't installed, so nothing to do
                        pass

                if requirement.conflicts_with:
                    logger.info(
                        'Found existing installation: %s',
                        requirement.conflicts_with,
                    )
                    with indent_log():
                        requirement.uninstall(auto_confirm=True)
                try:
                    requirement.install(
                        install_options,
                        global_options,
                        *args,
                        **kwargs
                    )
                except:
                    # if install did not succeed, rollback previous uninstall
                    if (requirement.conflicts_with
                            and not requirement.install_succeeded):
                        requirement.rollback_uninstall()
                    raise
                else:
                    if (requirement.conflicts_with
                            and requirement.install_succeeded):
                        requirement.commit_uninstall()
                requirement.remove_temporary_source()

        self.successfully_installed = to_install


'''The press module is used to create Record Collections.

'''

import shutil
import sys
import os
import pandas as pd
import turntable.utils
import traceback

import turntable

class RecordPress(object):

    '''This class auto-seralizes any attributes assigned to an instance and clears them from memmory
    when an attribute is called via the dot operator, it is read from disk

    Parameters
    ----------
    pickle : Boolean [True]
        if False, the instance will behave as a normal class
    pickle_path : string ['./tmp']
        the path underwhich the files will be stored

    Methods
    -------
    clean_disk()
        deletes all files stored by the instance
    clean_memmory()
        sets the in memory attribute values to None reducing the memory footprint

    Examples
    --------
    This class can be encapsulated to be used elsewhere

    >>> class NewClass(RecordPress):
    >>>
    >>>  def __init__(self, pickle = True, pickle_path = './tmp'):
    >>>     self.pickle = pickle
    >>>     self.class_path = turntable.utils.path_to_filename(pickle_path+'/'+self.__class__.__name__)[0]
    >>>     self.pickles = []
    >>>
    >>> newClass = NewClass()
    >>> newClass.x = 10
    >>> y = newClass.x
    >>> newClass.clean_disk()

    '''

    def __init__(self, pickle=True, pickle_path='./tmp'):
        '''
        Parameters
        ----------
        pickle : pickle the class properties
        pickle_path : where to pickle

        '''

        self.pickle = pickle  # required for over setattr override function
        self.class_path = turntable.utils.path2filename(
            pickle_path + '/' + self.__class__.__name__)[0]
        self.pickles = []

    def __getattribute__(*args):
        # ignore these standard calls to class properties
        ignore_args_list = ['__dict__', '__members__', '__methods__',
                            '__class__', 'trait_names', '_getAttributeNames']
        if any([attr in args[1]
                for attr in ['pickle', 'class_path', 'pickles'] + ignore_args_list]):
            # if the name of the args matches the standard or required names,
            # recurse normally
            return object.__getattribute__(*args)
        else:
            self = args[0]  # 0th index is the class instance
            name = args[1]  # 1st index is the attribute lable requested
            # if this is a attribute from the pickles list, grab it from disk
            if self.pickle & (name in self.pickles):
                # print "Reading disk: ", self.class_path + name + '.pkl'
                return turntable.utils.from_pickle(
                    filename=self.class_path + name + '.pkl', clean_disk=False)
            else:  # otherwise return the attribute from the class instance
                # print "Reading memory: ", name
                return object.__getattribute__(*args)

    def __setattr__(self, name, value):
        # first set the attribute to the instance of the class
        object.__setattr__(self, name, value)

        # test for the pickle command
        if self.pickle:
            # if the name of the assignend attribute is not a required name, go
            # ahead and pickle it
            if any([attr in name for attr in ['pickle', 'class_path', 'pickles']]) == False:
                return_value = turntable.utils.to_pickle(
                    obj=value,
                    filename=self.class_path+name+'.pkl',
                    clean_memory=True)
                # return_value = None #for testing memmory
                self.__dict__[name] = return_value
                # use the below line to not default memory clearing
                # turntable.utils.to_pickle(obj = value, filename = self.class_path + name + '.pkl', clean_memory = False)
                # add the name to the list of pickled attributes, so we know which attributes have been pickled
                self.pickles.append(name)
                self.pickles = list(set(self.pickles))
        #         print "writing to disk: ", self.class_path + name + '.pkl'
        #     else:
        #         pass
        #         print "writing to memory: ", name
        # else:
        #     pass
        #     print "writing to memory: ", name

    def clean_memory(self):
        '''sets all attribute values from the attribute list pickles to None
        -this is the default behaviure so this method is redundent'''
        for key in self.pickles:
            self.__dict__[key] = None

    def clean_disk(self):
        '''clean removes all files and folders under the class_path directory'''
        shutil.rmtree(self.class_path)

class RecordSetter:

    '''RecordSetter provides a simple interface for initalizing arguments passed in kwargs
    and a run method for running a class method by name

    Parameters
    ----------
    kwargs : name : value

    Examples
    --------
    RecordSetter is a general python class that assigns **kwargs as instances of it self.

    >>> obj = RecordSetter(name = 'me')
    >>> print obj.name
    me

    '''

    def __init__(self, **kwargs):
        self.set_attributes(kwargs)

    def set_attributes(self, kwargs):
        '''
        Initalizes the given argument structure as properties of the class
        to be used by name in specific method execution.

        Parameters
        ----------
        kwargs : dictionary
            Dictionary of extra attributes,
            where keys are attributes names and values attributes values.

        '''

        for key, value in kwargs.items():
            setattr(self, key, value)
            # print key, value

    def run_method(self, method):
        '''
        Calls a specied method by name using run_method

        '''

        method_to_call = getattr(self, method)
        result = method_to_call()

class SeriesLoader(object):

    '''SeriesLoader assigns given properties to a special pandas.Series property: self.series.

    Parameters
    ----------
    series : all atributes of the class get added to an internal pandas series

    Examples
    --------
    This class can be encapsulated to be used elsewhere

    >>> series_loader = SeriesLoader()
    >>> series_loader.one = 'one'
    >>> print series_loader.series

    '''

    def __init__(self):
        pass

    def __setattr__(self, name, value):
        # first set the attribute to the instance of the class
        object.__setattr__(self, name, value)
        # if the name of the assigned attribute is not a required name, go
        # ahead and pickle it
        blacklisted_keys = ['series']
        if any([attr in name for attr in blacklisted_keys]) == False:
            if 'series' in self.__dict__.keys():
                self.series[name] = value
            else:
                self.series = pd.Series(dict(
                    [(key, value) for key, value in self.__dict__.items() if key not in blacklisted_keys]))

class Record(RecordSetter, SeriesLoader):

    '''Record is a container object with the special property "series". Any
    property added to Record will also be added to the pandas.Series

    Properties
    ----------
    series : pandas.Series
        container for parameters set to the instance

    Methods
    -------
    set_attributes : assigns items of a dictionary to the class and to the series parameter
    runMethod : runs a method by a string call

    Examples
    --------
    lets see how we can add a propertie to the record object

    >>> record = Record(first_item = 'one')
    >>> record.second_item = 'two'
    >>> print record.series
    '''

    def __init__(self, **kwargs):
        self.set_attributes(kwargs)

def load_record(index_series_tuple, kwargs):
    '''
    Generates an instance of Record() from a tuple of the form (index, pandas.Series)
    with associated parameters kwargs

    Paremeters
    ----------
    index_series_tuple : tuple
        tuple consisting of (index, pandas.Series)
    kwargs : dict
        aditional arguments

    Returns
    -------
    Record : object

    '''

    index_record = index_series_tuple[0]
    series = index_series_tuple[1]
    record = Record()
    record.series = series
    record.index_record = index_record
    record.set_attributes(kwargs)
    return record

def build_collection(df, **kwargs):
    '''
    Generates a list of Record objects given a DataFrame.
    Each Record instance has a series attribute which is a pandas.Series of the same attributes
    in the DataFrame.
    Optional data can be passed in through kwargs which will be included by the name of each object.

    parameters
    ----------
    df : pandas.DataFrame
    kwargs : alternate arguments to be saved by name to the series of each object

    Returns
    -------
    collection : list
        list of Record objects where each Record represents one row from a dataframe

    Examples
    --------
    This is how we generate a Record Collection from a DataFrame.

    >>> import pandas as pd
    >>> import turntable
    >>>
    >>> df = pd.DataFrame({'Artist':"""Michael Jackson, Pink Floyd, Whitney Houston, Meat Loaf,
        Eagles, Fleetwood Mac, Bee Gees, AC/DC""".split(', '),
    >>> 'Album' :"""Thriller, The Dark Side of the Moon, The Bodyguard, Bat Out of Hell,
        Their Greatest Hits (1971-1975), Rumours, Saturday Night Fever, Back in Black""".split(', ')})
    >>> collection = turntable.press.build_collection(df, my_favorite_record = 'nevermind')
    >>> record = collection[0]
    >>> print record.series

    '''

    print 'Generating the Record Collection...\n'

    df['index_original'] = df.index
    df.reset_index(drop=True, inplace=True)

    if pd.__version__ >= '0.15.0':
        d = df.T.to_dict(orient='series')
    else:
        d = df.T.to_dict(outtype='series')

    collection = [load_record(item, kwargs) for item in d.items()]
    return collection

def collection_to_df(collection):
    '''
    Converts a collection back into a pandas DataFrame

    parameters
    ----------
    collection : list
        list of Record objects where each Record represents one row from a dataframe

    Returns
    -------
    df : pandas.DataFrame
        DataFrame of length=len(collection) where each row represents one Record

    '''

    return pd.concat([record.series for record in collection], axis=1).T

def spin_frame(df, method):
    '''
    Runs the full turntable process on a pandas DataFrame

    parameters
    ----------
    df : pandas.DataFrame
        each row represents a record
    method : def method(record)
        function used to process each row

    Returns
    -------
    df : pandas.DataFrame
        DataFrame processed by method

    Example
    -------
    >>> import pandas as pd
    >>> import turntable
    >>>
    >>> df = pd.DataFrame({'Artist':"""Michael Jackson, Pink Floyd, Whitney Houston, Meat Loaf, Eagles, Fleetwood Mac, Bee Gees, AC/DC""".split(', '), 'Album':"""Thriller, The Dark Side of the Moon, The Bodyguard, Bat Out of Hell, Their Greatest Hits (1971–1975), Rumours, Saturday Night Fever, Back in Black""".split(', ')})
    >>>
    >>> def method(record):
    >>>    record.cost = 40
    >>>    return record
    >>>
    >>> turntable.press.spin_frame(df, method)

    '''
    collection = build_collection(df)
    collection = turntable.spin.batch(collection, method)
    return collection_to_df(collection)


"""
A simple logger object that consolidates messages incoming from ipcluster processes.

Authors:

* MinRK

"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import logging
import sys

import zmq
from zmq.eventloop import ioloop, zmqstream

from IPython.config.configurable import LoggingConfigurable
from IPython.utils.traitlets import Int, Unicode, Instance, List

#-----------------------------------------------------------------------------
# Classes
#-----------------------------------------------------------------------------

class LogWatcher(LoggingConfigurable):
    """A simple class that receives messages on a SUB socket, as published
    by subclasses of `zmq.log.handlers.PUBHandler`, and logs them itself.

    This can subscribe to multiple topics, but defaults to all topics.
    """

    # configurables
    topics = List([''], config=True,
        help="The ZMQ topics to subscribe to. Default is to subscribe to all messages")
    url = Unicode('tcp://127.0.0.1:20202', config=True,
        help="ZMQ url on which to listen for log messages")

    # internals
    stream = Instance('zmq.eventloop.zmqstream.ZMQStream')

    context = Instance(zmq.Context)
    def _context_default(self):
        return zmq.Context.instance()

    loop = Instance(zmq.eventloop.ioloop.IOLoop)
    def _loop_default(self):
        return ioloop.IOLoop.instance()

    def __init__(self, **kwargs):
        super(LogWatcher, self).__init__(**kwargs)
        s = self.context.socket(zmq.SUB)
        s.bind(self.url)
        self.stream = zmqstream.ZMQStream(s, self.loop)
        self.subscribe()
        self.on_trait_change(self.subscribe, 'topics')

    def start(self):
        self.stream.on_recv(self.log_message)

    def stop(self):
        self.stream.stop_on_recv()

    def subscribe(self):
        """Update our SUB socket's subscriptions."""
        self.stream.setsockopt(zmq.UNSUBSCRIBE, '')
        if '' in self.topics:
            self.log.debug("Subscribing to: everything")
            self.stream.setsockopt(zmq.SUBSCRIBE, '')
        else:
            for topic in self.topics:
                self.log.debug("Subscribing to: %r"%(topic))
                self.stream.setsockopt(zmq.SUBSCRIBE, topic)

    def _extract_level(self, topic_str):
        """Turn 'engine.0.INFO.extra' into (logging.INFO, 'engine.0.extra')"""
        topics = topic_str.split('.')
        for idx,t in enumerate(topics):
            level = getattr(logging, t, None)
            if level is not None:
                break

        if level is None:
            level = logging.INFO
        else:
            topics.pop(idx)

        return level, '.'.join(topics)

    def log_message(self, raw):
        """receive and parse a message, then log it."""
        if len(raw) != 2 or '.' not in raw[0]:
            self.log.error("Invalid log message: %s"%raw)
            return
        else:
            topic, msg = raw
            # don't newline, since log messages always newline:
            topic,level_name = topic.rsplit('.',1)
            level,topic = self._extract_level(topic)
            if msg[-1] == '\n':
                msg = msg[:-1]
            self.log.log(level, "[%s] %s" % (topic, msg))


"""Example showing how to merge multiple remote data streams.
"""
# Slightly modified version of:
# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/511509
from __future__ import print_function

import heapq
from IPython.parallel.error import RemoteError

def mergesort(list_of_lists, key=None):
    """ Perform an N-way merge operation on sorted lists.

    @param list_of_lists: (really iterable of iterable) of sorted elements
    (either by naturally or by C{key})
    @param key: specify sort key function (like C{sort()}, C{sorted()})

    Yields tuples of the form C{(item, iterator)}, where the iterator is the
    built-in list iterator or something you pass in, if you pre-generate the
    iterators.

    This is a stable merge; complexity O(N lg N)

    Examples::

    >>> print list(mergesort([[1,2,3,4],
    ...                      [2,3.25,3.75,4.5,6,7],
    ...                      [2.625,3.625,6.625,9]]))
    [1, 2, 2, 2.625, 3, 3.25, 3.625, 3.75, 4, 4.5, 6, 6.625, 7, 9]

    # note stability
    >>> print list(mergesort([[1,2,3,4],
    ...                      [2,3.25,3.75,4.5,6,7],
    ...                      [2.625,3.625,6.625,9]],
    ...                      key=int))
    [1, 2, 2, 2.625, 3, 3.25, 3.75, 3.625, 4, 4.5, 6, 6.625, 7, 9]

    >>> print list(mergesort([[4, 3, 2, 1],
    ...                      [7, 6, 4.5, 3.75, 3.25, 2],
    ...                      [9, 6.625, 3.625, 2.625]],
    ...                      key=lambda x: -x))
    [9, 7, 6.625, 6, 4.5, 4, 3.75, 3.625, 3.25, 3, 2.625, 2, 2, 1]
    """

    heap = []
    for i, itr in enumerate(iter(pl) for pl in list_of_lists):
        try:
            item = itr.next()
            if key:
                toadd = (key(item), i, item, itr)
            else:
                toadd = (item, i, itr)
            heap.append(toadd)
        except StopIteration:
            pass
    heapq.heapify(heap)

    if key:
        while heap:
            _, idx, item, itr = heap[0]
            yield item
            try:
                item = itr.next()
                heapq.heapreplace(heap, (key(item), idx, item, itr) )
            except StopIteration:
                heapq.heappop(heap)

    else:
        while heap:
            item, idx, itr = heap[0]
            yield item
            try:
                heapq.heapreplace(heap, (itr.next(), idx, itr))
            except StopIteration:
                heapq.heappop(heap)

def remote_iterator(view,name):
    """Return an iterator on an object living on a remote engine.
    """
    view.execute('it%s=iter(%s)'%(name,name), block=True)
    while True:
        try:
            result = view.apply_sync(lambda x: x.next(), Reference('it'+name))
        # This causes the StopIteration exception to be raised.
        except RemoteError as e:
            if e.ename == 'StopIteration':
                raise StopIteration
            else:
                raise e
        else:
            yield result

# Main, interactive testing
if __name__ == '__main__':

    from IPython.parallel import Client, Reference
    rc = Client()
    view = rc[:]
    print('Engine IDs:', rc.ids)

    # Make a set of 'sorted datasets'
    a0 = range(5,20)
    a1 = range(10)
    a2 = range(15,25)

    # Now, imagine these had been created in the remote engines by some long
    # computation.  In this simple example, we just send them over into the
    # remote engines.  They will all be called 'a' in each engine.
    rc[0]['a'] = a0
    rc[1]['a'] = a1
    rc[2]['a'] = a2

    # And we now make a local object which represents the remote iterator
    aa0 = remote_iterator(rc[0],'a')
    aa1 = remote_iterator(rc[1],'a')
    aa2 = remote_iterator(rc[2],'a')

    # Let's merge them, both locally and remotely:
    print('Merge the local datasets:')
    print(list(mergesort([a0,a1,a2])))

    print('Locally merge the remote sets:')
    print(list(mergesort([aa0,aa1,aa2])))


"""Code for converting notebooks to and from the v2 format.

Authors:

* Brian Granger
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from .nbbase import (
    new_code_cell, new_text_cell, new_worksheet, new_notebook, new_output
)

#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------

def convert_to_this_nbformat(nb, orig_version=1):
    """Convert a notebook to the v2 format.

    Parameters
    ----------
    nb : NotebookNode
        The Python representation of the notebook to convert.
    orig_version : int
        The original version of the notebook to convert.
    """
    if orig_version == 1:
        newnb = new_notebook()
        ws = new_worksheet()
        for cell in nb.cells:
            if cell.cell_type == u'code':
                newcell = new_code_cell(input=cell.get('code'),prompt_number=cell.get('prompt_number'))
            elif cell.cell_type == u'text':
                newcell = new_text_cell(u'markdown',source=cell.get('text'))
            ws.cells.append(newcell)
        newnb.worksheets.append(ws)
        return newnb
    else:
        raise ValueError('Cannot convert a notebook from v%s to v2' % orig_version)