# Copyright 2015 Rackspace
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the
#    License. You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing,
#    software distributed under the License is distributed on an "AS
#    IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
#    express or implied. See the License for the specific language
#    governing permissions and limitations under the License.

import abc
import collections
import os
import sys

import six
import yaml

from timid import entry
from timid import utils

NAMESPACE_ACTION = 'timid.actions'
NAMESPACE_MODIFIER = 'timid.modifiers'

# Possible result states
SKIPPED = 0
SUCCESS = 1
FAILURE = 2
ERROR = 3
states = ['SKIPPED', 'SUCCESS', 'FAILURE', 'ERROR']

class ConfigError(Exception):
    """
    Report a configuration error.
    """

    def __init__(self, msg, step_addr=None):
        """
        Initialize a ``ConfigError`` exception.

        :param msg: A message describing the configuration error.
        :param step_addr: The address of the step in the test
                          configuration.
        """

        # Add the step index to the message
        if step_addr is not None:
            msg += ' (%s)' % step_addr

        # Initialize the exception
        super(ConfigError, self).__init__(msg)

        # Save the step index
        self.step_addr = step_addr

class StepAddress(object):
    """
    The "address" of a step.
    """

    def __init__(self, fname, idx, key=None):
        """
        Initialize a ``StepAddress`` instance.

        :param fname: The file name of the YAML file.
        :param idx: The index of the step within the file, or within
                    the designated key of the file.
        :param key: A key within the file.  If the file consists of a
                    single list, this should be ``None`` (the
                    default).
        """

        self.fname = fname
        self.idx = idx
        self.key = key

        # Cache for the string representation
        self._str = None

    def __str__(self):
        """
        Retrieve a string representation for the address.

        :returns: The string representation.
        """

        # Build the string if necessary...
        if self._str is None:
            if self.key is None:
                self._str = '%s step %d' % (self.fname, self.idx + 1)
            else:
                self._str = '%s[%s] step %d' % (self.fname, self.key,
                                                self.idx + 1)

        return self._str

@six.add_metaclass(abc.ABCMeta)
class StepPart(object):
    """
    A superclass for actions and modifiers that contains common
    pieces, such as config validation.
    """

    def __init__(self, ctxt, name, config, step_addr):
        """
        Initialize the action or modifier.  This should process and store
        the configuration provided.

        :param ctxt: The context object.
        :param name: The name.
        :param config: The configuration.  This may be a scalar value
                       (e.g., "run: command"), a list, or a
                       dictionary.  If the configuration provided is
                       invalid, a ``ConfigError`` should be raised.
        :param step_addr: The address of the step in the test
                          configuration.  Should be passed to the
                          ``ConfigError``.
        """

        # Validate the configuration
        self.validate_conf(name, config, step_addr)

        # Store the name and base configuration
        self.name = name
        self.config = config
        self.step_addr = step_addr

    def validate_conf(self, name, config, step_addr):
        """
        Use JSONSchema validation to validate the configuration.
        Validation errors will be reported by raising ConfigError.

        :param name: The name of the action or modifier.
        :param config: The actual configuration.
        :param step_addr: The address of the step in the test
                          configuration.
        """

        utils.schema_validate(config, self.schema, ConfigError, name,
                              step_addr=step_addr)

    @abc.abstractproperty
    def schema(self):
        """
        A JSONSchema.  This is used in validating the configuration.
        """

        pass  # pragma: no cover

class Action(StepPart):
    """
    A step *action*.  Actions are responsible for the actual operation
    performed by the test step.  Each step must have exactly one
    action.
    """

    # Specify as True to designate a "step" action, an action which
    # should be executed during step parsing, and which returns a list
    # of steps
    step_action = False

    @abc.abstractmethod
    def __call__(self, ctxt):
        """
        Invoke the action.  This should perform the actual step.  Note
        that it is possible, given some modifiers, that this method
        may be called any number of times (including 0 times) with
        slightly modified contexts.

        :param ctxt: The context object.

        :returns: A ``StepResult`` object, or if the ``step_action``
                  class attribute is ``True``, a list of zero or more
                  ``Step`` objects.
        """

        pass  # pragma: no cover

class Modifier(StepPart):
    """
    A step *modifier*.  Modifiers modify a step in some fashion, such
    as through repetition or applying a condition.
    """

    # Value for "restriction" to indicate a modifier compatible only
    # with "normal" actions, that is, actions that are not step
    # actions.
    NORMAL = 0x01

    # Value for "restriction" to indicate a modifier compatible only
    # with step actions.
    STEP = 0x02

    # Value for "restriction" to indicate a modifier compatible with
    # either normal or step actions.
    UNRESTRICTED = NORMAL | STEP

    # Specify one of the above restriction values to control which
    # actions the modifier is compatible with.
    restriction = NORMAL

    def action_conf(self, ctxt, action_class, action_name, config, step_addr):
        """
        A modifier hook function.  This is called in priority order prior
        to initializing the ``Action`` for the step.  This allows a
        modifier to alter the configuration to be fed to the action.

        :param ctxt: The context object.
        :param action_class: The ``Action`` subclass the modifier is
                             modifying.  This method should not
                             attempt to initialize the action.
        :param action_name: The name of the action.
        :param config: The configuration for the action.  This may be
                       a scalar value (e.g., "run: command"), a list,
                       or a dictionary.  If the configuration provided
                       is invalid for the action, a ``ConfigError``
                       should be raised.
        :param step_addr: The address of the step in the test
                          configuration.  Should be passed to the
                          ``ConfigError``.

        :returns: The configuration for the action, optionally
                  modified.  If the configuration is not modified,
                  ``config`` must be returned unchanged.  The default
                  implementation of this method does so.
        """

        return config

    def pre_call(self, ctxt, pre_mod, post_mod, action):
        """
        A modifier hook function.  This is called in priority order prior
        to invoking the ``Action`` for the step.  This allows a
        modifier to alter the context, or to take over subsequent
        action invocation.

        :param ctxt: The context object.
        :param pre_mod: A list of the modifiers preceding this
                        modifier in the list of modifiers that is
                        applicable to the action.  This list is in
                        priority order.
        :param post_mod: A list of the modifiers following this
                         modifier in the list of modifiers that is
                         applicable to the action.  This list is in
                         priority order.
        :param action: The action that will be performed.

        :returns: A ``None`` return value indicates that the modifier
                  is taking no action.  A non-``None`` return value
                  should consist of a ``StepResult`` object; this will
                  suspend further ``pre_call()`` processing and
                  proceed to the ``post_call()`` processing.  The
                  default implementation of this method returns
                  ``None``.
        """

        return None

    def post_call(self, ctxt, result, action, post_mod, pre_mod):
        """
        A modifier hook function.  This is called in reverse-priority
        order after invoking the ``Action`` for the step.  This allows
        a modifier to inspect or alter the result of the step.

        :param ctxt: The context object.
        :param result: The result of the action.  This will be a
                       ``StepResult`` object.
        :param action: The action that was performed.
        :param post_mod: A list of modifiers following this modifier
                         in the list of modifiers that is applicable
                         to the action.  This list is in priority
                         order.
        :param pre_mod: A list of modifiers preceding this modifier in
                        the list of modifiers that is applicable to
                        the action.  This list is in priority order.

        :returns: The result for the action, optionally modified.  If
                  the result is not modified, ``result`` must be
                  returned unchanged.  The default implementation of
                  this method does so.
        """

        return result

    @abc.abstractproperty
    def priority(self):
        """
        An integer value.  This provides a primitive means of controlling
        the order of application of modifiers.  The higher the number,
        the later the modifier will be applied.  The action has a
        priority equivalent to positive infinity.

        This must be a class attribute, not a property.
        """

        pass  # pragma: no cover

class StepItem(object):
    """
    A helper class to represent a step item, either an action or a
    modifier.  This is only used during step parsing.
    """

    def __init__(self, cls, name, conf):
        """
        Initialize a ``StepItem`` instance.

        :param cls: The class implementing the action or modifier.
        :param name: The name of the action or modifier.
        :param conf: The configuration for the action or modifier.
        """

        self.cls = cls
        self.name = name
        self.conf = conf

    def init(self, ctxt, step_addr):
        """
        Initialize the item.  This calls the class constructor with the
        appropriate arguments and returns the initialized object.

        :param ctxt: The context object.
        :param step_addr: The address of the step in the test
                          configuration.
        """

        return self.cls(ctxt, self.name, self.conf, step_addr)

class Step(object):
    """
    Represents a test step.
    """

    schemas = {
        'name': {'type': 'string'},
        'description': {'type': 'string'},
    }

    @classmethod
    def parse_file(cls, ctxt, fname, key=None, step_addr=None):
        """
        Parse a YAML file containing test steps.

        :param ctxt: The context object.
        :param fname: The name of the file to parse.
        :param key: An optional dictionary key.  If specified, the
                    file must be a YAML dictionary, and the referenced
                    value will be interpreted as a list of steps.  If
                    not provided, the file must be a YAML list, which
                    will be interpreted as the list of steps.
        :param step_addr: The address of the step in the test
                          configuration.  This may be used in the case
                          of includes, for instance.

        :returns: A list of ``Step`` objects.
        """

        # Load the YAML file
        try:
            with open(fname) as f:
                step_data = yaml.load(f)
        except Exception as exc:
            raise ConfigError(
                'Failed to read file "%s": %s' % (fname, exc),
                step_addr,
            )

        # Do we have a key?
        if key is not None:
            if (not isinstance(step_data, collections.Mapping) or
                    key not in step_data):
                raise ConfigError(
                    'Bad step configuration file "%s": expecting dictionary '
                    'with key "%s"' % (fname, key),
                    step_addr,
                )

            # Extract just the step data
            step_data = step_data[key]

        # Validate that it's a sequence
        if not isinstance(step_data, collections.Sequence):
            addr = ('%s[%s]' % (fname, key)) if key is not None else fname
            raise ConfigError(
                'Bad step configuration sequence at %s: expecting list, '
                'not "%s"' % (addr, step_data.__class__.__name__),
                step_addr,
            )

        # OK, assemble the step list and return it
        steps = []
        for idx, step_conf in enumerate(step_data):
            steps.extend(cls.parse_step(
                ctxt, StepAddress(fname, idx, key), step_conf))

        return steps

    @classmethod
    def parse_step(cls, ctxt, step_addr, step_conf):
        """
        Parse a step dictionary.

        :param ctxt: The context object.
        :param step_addr: The address of the step in the test
                          configuration.
        :param step_conf: The description of the step.  This may be a
                          scalar string or a dictionary.

        :returns: A list of steps.
        """

        # Make sure the step makes sense
        if isinstance(step_conf, six.string_types):
            # Convert string to a dict for uniformity of processing
            step_conf = {step_conf: None}
        elif not isinstance(step_conf, collections.Mapping):
            raise ConfigError(
                'Unable to parse step configuration: expecting string or '
                'dictionary, not "%s"' % step_conf.__class__.__name__,
                step_addr,
            )

        # Parse the configuration into the action and modifier classes
        # and the configuration to apply to each
        action_item = None
        mod_items = {}
        kwargs = {}  # extra args for Step.__init__()
        for key, key_conf in step_conf.items():
            # Handle special keys first
            if key in cls.schemas:
                # Validate the key
                utils.schema_validate(key_conf, cls.schemas[key], ConfigError,
                                      key, step_addr=step_addr)

                # Save the value
                kwargs[key] = key_conf

            # Is it an action?
            elif key in entry.points[NAMESPACE_ACTION]:
                if action_item is not None:
                    raise ConfigError(
                        'Bad step configuration: action "%s" specified, '
                        'but action "%s" already processed' %
                        (key, action_item.name),
                        step_addr,
                    )

                action_item = StepItem(
                    entry.points[NAMESPACE_ACTION][key], key, key_conf)

            # OK, is it a modifier?
            elif key in entry.points[NAMESPACE_MODIFIER]:
                mod_class = entry.points[NAMESPACE_MODIFIER][key]

                # Store it in priority order
                mod_items.setdefault(mod_class.priority, [])
                mod_items[mod_class.priority].append(StepItem(
                    mod_class, key, key_conf))

            # Couldn't resolve it
            else:
                raise ConfigError(
                    'Bad step configuration: unable to resolve action '
                    '"%s"' % key,
                    step_addr,
                )

        # Make sure we have an action
        if action_item is None:
            raise ConfigError(
                'Bad step configuration: no action specified',
                step_addr,
            )

        # What is the action type?
        action_type = (Modifier.STEP if action_item.cls.step_action
                       else Modifier.NORMAL)

        # OK, build our modifiers list and preprocess the action
        # configuration
        modifiers = []
        for mod_item in utils.iter_prio_dict(mod_items):
            # Verify that the modifier is compatible with the
            # action
            if mod_item.cls.restriction & action_type == 0:
                raise ConfigError(
                    'Bad step configuration: modifier "%s" is '
                    'incompatible with the action "%s"' %
                    (mod_item.name, action_item.name),
                    step_addr,
                )

            # Initialize the modifier
            modifier = mod_item.init(ctxt, step_addr)

            # Add it to the list of modifiers
            modifiers.append(modifier)

            # Apply the modifier's configuration processing
            action_item.conf = modifier.action_conf(
                ctxt, action_item.cls, action_item.name, action_item.conf,
                step_addr)

        # Now we can initialize the action
        action = action_item.init(ctxt, step_addr)

        # Create the step
        step = cls(step_addr, action, modifiers, **kwargs)

        # If the final_action is a StepAction, invoke it now and
        # return the list of steps.  We do this after creating the
        # Step object so that we can take advantage of its handling of
        # modifiers.
        if action_item.cls.step_action:
            return step(ctxt)

        # Not a step action, return the step as a list of one element
        return [step]

    def __init__(self, step_addr, action, modifiers=None, name=None,
                 description=None):
        """
        Initialize a ``Step`` instance.

        :param step_addr: The address of the step in the test
                          configuration.
        :param action: An ``Action`` instance.
        :param modifiers: A list of ``Modifier`` instances, in the
                          order in which processing should be
                          performed.  Optional.
        :param name: A name for the step.  Optional.
        :param description: A description of the step.  Optional.
        """

        self.step_addr = step_addr
        self.action = action
        self.modifiers = modifiers or []
        self.name = name or action.__class__.__name__
        self.description = description

    def __call__(self, ctxt):
        """
        Invoke the step.

        :param ctxt: The context object.

        :returns: A ``StepResult`` object, or if the action is a step
                  action, a list of zero or more ``Step`` objects.
        """

        # Begin by walking the modifiers
        i = -1
        for i in six.moves.range(len(self.modifiers)):
            result = self.modifiers[i].pre_call(
                ctxt, self.modifiers[:i], self.modifiers[i + 1:], self.action)

            # Did a modifier return a result?
            if result is not None:
                break
        else:
            # All modifiers have weighed in without returning a
            # result, so let's call the action
            try:
                result = self.action(ctxt)
            except Exception:
                # Wrap the exception in a StepResult instance
                result = StepResult(exc_info=sys.exc_info())
            else:
                # Convert a None into an error StepResult
                if result is None:
                    result = StepResult(state=ERROR)

        # Now walk the modifiers in reverse order for result
        # processing
        for j in six.moves.range(i, -1, -1):
            result = self.modifiers[j].post_call(
                ctxt, result, self.action, self.modifiers[j + 1:],
                self.modifiers[:j])

        return result

class StepResult(object):
    """
    Represent the result(s) of an action.
    """

    def __init__(self, state=None, msg=None, ignore=None,
                 returncode=None, exc_info=None, results=None):
        """
        Initialize a ``StepResult`` instance.

        :param state: One of the possible result states.  If not
                      provided, its value will be inferred from the
                      other arguments.
        :param msg: A message giving more detail regarding the result.
        :param ignore: Whether to ignore errors.  A ``True`` will
                       ignore an error condition.  Defaults to
                       ``False``.
        :param returncode: The return code of an external process.  If
                           provided, ``state`` is inferred to be
                           SUCCESS or FAILURE based on whether the
                           value is zero or non-zero, respectively.
        :param exc_info: Exception information.  If provided,
                         ``state`` is inferred to be ERROR.
        :param results: Used when the ``StepResult`` is encapsulating
                        a list of other ``StepResult`` instances.
        """

        # Save the result message
        self.msg = msg

        # If an exception was recorded, store it and default the state
        # appropriately
        self.exc_info = exc_info
        if exc_info is not None and state is None:
            state = ERROR

        # If the return code was provided, default the state
        # appropriately
        self.returncode = returncode
        if returncode is not None and state is None:
            state = SUCCESS if returncode == 0 else FAILURE

        # If a list of instances was provided, default the state
        # appropriately
        self.results = results or []
        if results is not None:
            if state is None:
                state = max(r.state for r in results)
            if ignore is None:
                ignore = any(r.ignore for r in results)

        # Save the error state
        self.state = state

        # Track whether we're ignoring the error
        self._ignore = ignore

    def __bool__(self):
        """
        Cast the ``StepResult`` instance to boolean.  It tests as
        equivalent to ``True`` if the state is "skipped" or "success",
        and ``False`` if the state is "error" or "failure".

        :returns: A boolean value indicating if the result succeeded
                  or failed.
        """

        return self._ignore or self.state in (SKIPPED, SUCCESS)
    __nonzero__ = __bool__

    @property
    def ignore(self):
        """
        Determine whether an error result should be ignored.
        """

        return self._ignore or False

    @ignore.setter
    def ignore(self, value):
        """
        Update the ignore state if it hasn't been set yet.
        """

        if self._ignore is None:
            self._ignore = value

class SensitiveDictAction(Action):
    """
    An abstract action for updating a ``SensitiveDict`` instance.  The
    base usage is::

        - name:
            set:
              var: value
            unset:
            - other_var
            sensitive:
            - sens_var
            files:
            - fname1.yaml
            - fname2.yaml

    This action would set the variable named "var" to the value
    "value", unset the variable "other_var", mark the variable
    "sens_var" as a sensitive variable, and read the files
    "fname1.yaml" and "fname2.yaml" from the directory containing the
    file specifying the action.

    Note that if a variable is present under both the "set" and
    "unset" keys, the "set" will take precedence.  Also note that
    variable file reading is performed before any other operations.
    """

    # Schema for validating the configuration
    schema = {
        'type': 'object',
        'properties': {
            'set': {'type': 'object'},
            'unset': {
                'type': 'array',
                'items': {'type': 'string'},
            },
            'sensitive': {
                'type': 'array',
                'items': {'type': 'string'},
            },
            'files': {
                'type': 'array',
                'items': {'type': 'string'},
            },
        },
        'additionalProperties': False,
    }

    def __init__(self, ctxt, name, config, step_addr):
        """
        Initialize a ``SensitiveDictAction`` instance.

        :param ctxt: The context object.
        :param name: The name of the action.
        :param config: The configuration for the action.  This may be
                       a scalar value (e.g., "run: command"), a list,
                       or a dictionary.  If the configuration provided
                       is invalid for the action, a ``ConfigError``
                       should be raised.
        :param step_addr: The address of the step in the test
                          configuration.  Should be passed to the
                          ``ConfigError``.
        """

        # Perform superclass initialization
        super(SensitiveDictAction, self).__init__(
            ctxt, name, config, step_addr)

        # Set up the 'set' portion
        self.set_vars = {}
        for key, value in config.get('set', {}).items():
            self.set_vars[key] = ctxt.template(value)

        # Now do the unset and sensitive sets
        self.unset_vars = set(config.get('unset', []))
        self.sensitive_vars = set(config.get('sensitive', []))
        self.files = [ctxt.template(fn) for fn in config.get('files', [])]

        # Save the directory of the file the action is in for relative
        # interpretation
        self.dirname = os.path.dirname(step_addr.fname) or os.curdir

    def __call__(self, ctxt):
        """
        Invoke the action.  This updates the appropriate ``SensitiveDict``
        instance from the context as specified in the configuration.

        :param ctxt: The context object.

        :returns: A ``StepResult`` object.
        """

        # First, select the correct context attribute
        sensitive_dict = getattr(ctxt, self.context_attr)

        # Next, read in the files in order
        for ftmpl in self.files:
            fpath = utils.canonicalize_path(self.dirname, ftmpl(ctxt))
            try:
                with open(fpath) as f:
                    var_data = yaml.load(f)
            except Exception as exc:
                # Ignore missing variable files
                continue

            # Make sure the contents are a dictionary
            if not isinstance(var_data, collections.Mapping):
                continue

            # Update the dictionary
            sensitive_dict.update(var_data)

        # Now, apply the sensitive and unset changes
        for var in self.sensitive_vars:
            sensitive_dict.declare_sensitive(var)
        for var in self.unset_vars:
            sensitive_dict.pop(var, None)

        # Finally, apply the sets
        for var, tmpl in self.set_vars.items():
            sensitive_dict[var] = tmpl(ctxt)

        # We're done!
        return StepResult(state=SUCCESS)

    @abc.abstractproperty
    def context_attr(self):
        """
        The name of an attribute of the context containing the
        ``SensitiveDict`` instance that should be acted upon by this
        action.
        """

        pass  # pragma: no cover

class IncludeAction(Action):
    """
    An action that includes steps from another file.  The base usage
    is::

        - include: filename.yaml

    With this syntax, all steps in the "filename.yaml" (which must be
    a YAML list) will be included at the point of the "include"
    action.  Note that the filename is resolved relative to the
    directory of the file containing the "include" action.

    More advanced syntax allows selection of subsets of the steps from
    the referenced file::

        - include:
            path: filename.yaml
            key: special
            start: 2
            stop: 7

    The only required element is the "path" element.  The "key"
    element causes the file to be treated as a YAML mapping, and the
    list contained in the value for the specified key will be
    included.  The "start" and "stop" elements allow selection of a
    subset, and are interpreted as in a Python range syntax; that is,
    for the example above, steps 2, 3, 4, 5, and 6 will be included,
    but not steps 0, 1, 7, etc.
    """

    # This is a special "step" action, an action that returns a list
    # of steps
    step_action = True

    # Schema for validating the configuration
    schema = {
        'type': 'object',
        'properties': {
            'path': {'type': 'string'},
            'key': {'type': 'string'},
            'start': {'type': 'integer'},
            'stop': {'type': 'integer'},
        },
        'additionalProperties': False,
        'required': ['path'],
    }

    def __init__(self, ctxt, name, config, step_addr):
        """
        Initialize an ``IncludeAction`` instance.

        :param ctxt: The context object.
        :param name: The name of the action.
        :param config: The configuration for the action.  This may be
                       a scalar value (e.g., "run: command"), a list,
                       or a dictionary.  If the configuration provided
                       is invalid for the action, a ``ConfigError``
                       should be raised.
        :param step_addr: The address of the step in the test
                          configuration.  Should be passed to the
                          ``ConfigError``.
        """

        # Convert bare strings intelligently
        if isinstance(config, six.string_types):
            config = {'path': config}

        # Perform superclass initialization
        super(IncludeAction, self).__init__(ctxt, name, config, step_addr)

        # Extract the appropriate values
        self.path = ctxt.template(config['path'])
        self.key = ctxt.template(config.get('key'))
        self.start = config.get('start')
        self.stop = config.get('stop')

        # Save the directory of the file the include is in for
        # relative interpretation
        self.dirname = os.path.dirname(step_addr.fname) or os.curdir

    def __call__(self, ctxt):
        """
        Invoke the action.  This reads and interprets the referenced steps
        file.

        :param ctxt: The context object.

        :returns: A list of zero or more ``Step`` objects.
        """

        # Interpret the path
        path = utils.canonicalize_path(self.dirname, self.path(ctxt))

        # Import the desired steps
        steps = Step.parse_file(
            ctxt, path, self.key(ctxt), self.step_addr)

        # Narrow the steps, if desired
        if self.start is not None or self.stop is not None:
            return steps[self.start:self.stop]

        return steps


# encoding: utf-8
"""
An application for IPython.

All top-level applications should use the classes in this module for
handling configuration and creating componenets.

The job of an :class:`Application` is to create the master configuration
object and then create the configurable objects, passing the config to them.

Authors:

* Brian Granger
* Fernando Perez
* Min RK

"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import atexit
import glob
import logging
import os
import shutil
import sys

from IPython.config.application import Application, catch_config_error
from IPython.config.configurable import Configurable
from IPython.config.loader import Config, ConfigFileNotFound
from IPython.core import release, crashhandler
from IPython.core.profiledir import ProfileDir, ProfileDirError
from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
from IPython.utils.traitlets import List, Unicode, Type, Bool, Dict
from IPython.utils import py3compat

#-----------------------------------------------------------------------------
# Classes and functions
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Base Application Class
#-----------------------------------------------------------------------------

# aliases and flags

base_aliases = {
    'profile' : 'BaseIPythonApplication.profile',
    'ipython-dir' : 'BaseIPythonApplication.ipython_dir',
    'log-level' : 'Application.log_level',
}

base_flags = dict(
    debug = ({'Application' : {'log_level' : logging.DEBUG}},
            "set log level to logging.DEBUG (maximize logging output)"),
    quiet = ({'Application' : {'log_level' : logging.CRITICAL}},
            "set log level to logging.CRITICAL (minimize logging output)"),
    init = ({'BaseIPythonApplication' : {
                    'copy_config_files' : True,
                    'auto_create' : True}
            }, """Initialize profile with default config files.  This is equivalent
            to running `ipython profile create <profile>` prior to startup.
            """)
)

class BaseIPythonApplication(Application):

    name = Unicode(u'ipython')
    description = Unicode(u'IPython: an enhanced interactive Python shell.')
    version = Unicode(release.version)

    aliases = Dict(base_aliases)
    flags = Dict(base_flags)
    classes = List([ProfileDir])

    # Track whether the config_file has changed,
    # because some logic happens only if we aren't using the default.
    config_file_specified = Bool(False)

    config_file_name = Unicode(u'ipython_config.py')
    def _config_file_name_default(self):
        return self.name.replace('-','_') + u'_config.py'
    def _config_file_name_changed(self, name, old, new):
        if new != old:
            self.config_file_specified = True

    # The directory that contains IPython's builtin profiles.
    builtin_profile_dir = Unicode(
        os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
    )

    config_file_paths = List(Unicode)
    def _config_file_paths_default(self):
        return [os.getcwdu()]

    profile = Unicode(u'default', config=True,
        help="""The IPython profile to use."""
    )

    def _profile_changed(self, name, old, new):
        self.builtin_profile_dir = os.path.join(
                get_ipython_package_dir(), u'config', u'profile', new
        )

    ipython_dir = Unicode(get_ipython_dir(), config=True,
        help="""
        The name of the IPython directory. This directory is used for logging
        configuration (through profiles), history storage, etc. The default
        is usually $HOME/.ipython. This options can also be specified through
        the environment variable IPYTHONDIR.
        """
    )

    overwrite = Bool(False, config=True,
        help="""Whether to overwrite existing config files when copying""")
    auto_create = Bool(False, config=True,
        help="""Whether to create profile dir if it doesn't exist""")

    config_files = List(Unicode)
    def _config_files_default(self):
        return [u'ipython_config.py']

    copy_config_files = Bool(False, config=True,
        help="""Whether to install the default config files into the profile dir.
        If a new profile is being created, and IPython contains config files for that
        profile, then they will be staged into the new directory.  Otherwise,
        default config files will be automatically generated.
        """)

    verbose_crash = Bool(False, config=True,
        help="""Create a massive crash report when IPython encounters what may be an
        internal error.  The default is to append a short message to the
        usual traceback""")

    # The class to use as the crash handler.
    crash_handler_class = Type(crashhandler.CrashHandler)

    def __init__(self, **kwargs):
        super(BaseIPythonApplication, self).__init__(**kwargs)
        # ensure even default IPYTHONDIR exists
        if not os.path.exists(self.ipython_dir):
            self._ipython_dir_changed('ipython_dir', self.ipython_dir, self.ipython_dir)

    #-------------------------------------------------------------------------
    # Various stages of Application creation
    #-------------------------------------------------------------------------

    def init_crash_handler(self):
        """Create a crash handler, typically setting sys.excepthook to it."""
        self.crash_handler = self.crash_handler_class(self)
        sys.excepthook = self.excepthook
        def unset_crashhandler():
            sys.excepthook = sys.__excepthook__
        atexit.register(unset_crashhandler)

    def excepthook(self, etype, evalue, tb):
        """this is sys.excepthook after init_crashhandler

        set self.verbose_crash=True to use our full crashhandler, instead of
        a regular traceback with a short message (crash_handler_lite)
        """

        if self.verbose_crash:
            return self.crash_handler(etype, evalue, tb)
        else:
            return crashhandler.crash_handler_lite(etype, evalue, tb)

    def _ipython_dir_changed(self, name, old, new):
        if old in sys.path:
            sys.path.remove(old)
        sys.path.append(os.path.abspath(new))
        if not os.path.isdir(new):
            os.makedirs(new, mode=0777)
        readme = os.path.join(new, 'README')
        if not os.path.exists(readme):
            path = os.path.join(get_ipython_package_dir(), u'config', u'profile')
            shutil.copy(os.path.join(path, 'README'), readme)
        self.log.debug("IPYTHONDIR set to: %s" % new)

    def load_config_file(self, suppress_errors=True):
        """Load the config file.

        By default, errors in loading config are handled, and a warning
        printed on screen. For testing, the suppress_errors option is set
        to False, so errors will make tests fail.
        """
        self.log.debug("Searching path %s for config files", self.config_file_paths)
        base_config = 'ipython_config.py'
        self.log.debug("Attempting to load config file: %s" %
                       base_config)
        try:
            Application.load_config_file(
                self,
                base_config,
                path=self.config_file_paths
            )
        except ConfigFileNotFound:
            # ignore errors loading parent
            self.log.debug("Config file %s not found", base_config)
            pass
        if self.config_file_name == base_config:
            # don't load secondary config
            return
        self.log.debug("Attempting to load config file: %s" %
                       self.config_file_name)
        try:
            Application.load_config_file(
                self,
                self.config_file_name,
                path=self.config_file_paths
            )
        except ConfigFileNotFound:
            # Only warn if the default config file was NOT being used.
            if self.config_file_specified:
                msg = self.log.warn
            else:
                msg = self.log.debug
            msg("Config file not found, skipping: %s", self.config_file_name)
        except:
            # For testing purposes.
            if not suppress_errors:
                raise
            self.log.warn("Error loading config file: %s" %
                          self.config_file_name, exc_info=True)

    def init_profile_dir(self):
        """initialize the profile dir"""
        try:
            # location explicitly specified:
            location = self.config.ProfileDir.location
        except AttributeError:
            # location not specified, find by profile name
            try:
                p = ProfileDir.find_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
            except ProfileDirError:
                # not found, maybe create it (always create default profile)
                if self.auto_create or self.profile=='default':
                    try:
                        p = ProfileDir.create_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
                    except ProfileDirError:
                        self.log.fatal("Could not create profile: %r"%self.profile)
                        self.exit(1)
                    else:
                        self.log.info("Created profile dir: %r"%p.location)
                else:
                    self.log.fatal("Profile %r not found."%self.profile)
                    self.exit(1)
            else:
                self.log.info("Using existing profile dir: %r"%p.location)
        else:
            # location is fully specified
            try:
                p = ProfileDir.find_profile_dir(location, self.config)
            except ProfileDirError:
                # not found, maybe create it
                if self.auto_create:
                    try:
                        p = ProfileDir.create_profile_dir(location, self.config)
                    except ProfileDirError:
                        self.log.fatal("Could not create profile directory: %r"%location)
                        self.exit(1)
                    else:
                        self.log.info("Creating new profile dir: %r"%location)
                else:
                    self.log.fatal("Profile directory %r not found."%location)
                    self.exit(1)
            else:
                self.log.info("Using existing profile dir: %r"%location)

        self.profile_dir = p
        self.config_file_paths.append(p.location)

    def init_config_files(self):
        """[optionally] copy default config files into profile dir."""
        # copy config files
        path = self.builtin_profile_dir
        if self.copy_config_files:
            src = self.profile

            cfg = self.config_file_name
            if path and os.path.exists(os.path.join(path, cfg)):
                self.log.warn("Staging %r from %s into %r [overwrite=%s]"%(
                        cfg, src, self.profile_dir.location, self.overwrite)
                )
                self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite)
            else:
                self.stage_default_config_file()
        else:
            # Still stage *bundled* config files, but not generated ones
            # This is necessary for `ipython profile=sympy` to load the profile
            # on the first go
            files = glob.glob(os.path.join(path, '*.py'))
            for fullpath in files:
                cfg = os.path.basename(fullpath)
                if self.profile_dir.copy_config_file(cfg, path=path, overwrite=False):
                    # file was copied
                    self.log.warn("Staging bundled %s from %s into %r"%(
                            cfg, self.profile, self.profile_dir.location)
                    )

    def stage_default_config_file(self):
        """auto generate default config file, and stage it into the profile."""
        s = self.generate_config_file()
        fname = os.path.join(self.profile_dir.location, self.config_file_name)
        if self.overwrite or not os.path.exists(fname):
            self.log.warn("Generating default config file: %r"%(fname))
            with open(fname, 'w') as f:
                f.write(s)

    @catch_config_error
    def initialize(self, argv=None):
        # don't hook up crash handler before parsing command-line
        self.parse_command_line(argv)
        self.init_crash_handler()
        if self.subapp is not None:
            # stop here if subapp is taking over
            return
        cl_config = self.config
        self.init_profile_dir()
        self.init_config_files()
        self.load_config_file()
        # enforce cl-opts override configfile opts:
        self.update_config(cl_config)


"""Coverage data for Coverage."""

import os

from coverage.backward import iitems, pickle, sorted    # pylint: disable=W0622
from coverage.files import PathAliases
from coverage.misc import file_be_gone

class CoverageData(object):
    """Manages collected coverage data, including file storage.

    The data file format is a pickled dict, with these keys:

        * collector: a string identifying the collecting software

        * lines: a dict mapping filenames to sorted lists of line numbers
          executed:
            { 'file1': [17,23,45],  'file2': [1,2,3], ... }

        * arcs: a dict mapping filenames to sorted lists of line number pairs:
            { 'file1': [(17,23), (17,25), (25,26)], ... }

    """

    def __init__(self, basename=None, collector=None, debug=None):
        """Create a CoverageData.

        `basename` is the name of the file to use for storing data.

        `collector` is a string describing the coverage measurement software.

        `debug` is a `DebugControl` object for writing debug messages.

        """
        self.collector = collector or 'unknown'
        self.debug = debug

        self.use_file = True

        # Construct the filename that will be used for data file storage, if we
        # ever do any file storage.
        self.filename = basename or ".coverage"
        self.filename = os.path.abspath(self.filename)

        # A map from canonical Python source file name to a dictionary in
        # which there's an entry for each line number that has been
        # executed:
        #
        #   {
        #       'filename1.py': { 12: None, 47: None, ... },
        #       ...
        #       }
        #
        self.lines = {}

        # A map from canonical Python source file name to a dictionary with an
        # entry for each pair of line numbers forming an arc:
        #
        #   {
        #       'filename1.py': { (12,14): None, (47,48): None, ... },
        #       ...
        #       }
        #
        self.arcs = {}

    def usefile(self, use_file=True):
        """Set whether or not to use a disk file for data."""
        self.use_file = use_file

    def read(self):
        """Read coverage data from the coverage data file (if it exists)."""
        if self.use_file:
            self.lines, self.arcs = self._read_file(self.filename)
        else:
            self.lines, self.arcs = {}, {}

    def write(self, suffix=None):
        """Write the collected coverage data to a file.

        `suffix` is a suffix to append to the base file name. This can be used
        for multiple or parallel execution, so that many coverage data files
        can exist simultaneously.  A dot will be used to join the base name and
        the suffix.

        """
        if self.use_file:
            filename = self.filename
            if suffix:
                filename += "." + suffix
            self.write_file(filename)

    def erase(self):
        """Erase the data, both in this object, and from its file storage."""
        if self.use_file:
            if self.filename:
                file_be_gone(self.filename)
        self.lines = {}
        self.arcs = {}

    def line_data(self):
        """Return the map from filenames to lists of line numbers executed."""
        return dict(
            [(f, sorted(lmap.keys())) for f, lmap in iitems(self.lines)]
            )

    def arc_data(self):
        """Return the map from filenames to lists of line number pairs."""
        return dict(
            [(f, sorted(amap.keys())) for f, amap in iitems(self.arcs)]
            )

    def write_file(self, filename):
        """Write the coverage data to `filename`."""

        # Create the file data.
        data = {}

        data['lines'] = self.line_data()
        arcs = self.arc_data()
        if arcs:
            data['arcs'] = arcs

        if self.collector:
            data['collector'] = self.collector

        if self.debug and self.debug.should('dataio'):
            self.debug.write("Writing data to %r" % (filename,))

        # Write the pickle to the file.
        fdata = open(filename, 'wb')
        try:
            pickle.dump(data, fdata, 2)
        finally:
            fdata.close()

    def read_file(self, filename):
        """Read the coverage data from `filename`."""
        self.lines, self.arcs = self._read_file(filename)

    def raw_data(self, filename):
        """Return the raw pickled data from `filename`."""
        if self.debug and self.debug.should('dataio'):
            self.debug.write("Reading data from %r" % (filename,))
        fdata = open(filename, 'rb')
        try:
            data = pickle.load(fdata)
        finally:
            fdata.close()
        return data

    def _read_file(self, filename):
        """Return the stored coverage data from the given file.

        Returns two values, suitable for assigning to `self.lines` and
        `self.arcs`.

        """
        lines = {}
        arcs = {}
        try:
            data = self.raw_data(filename)
            if isinstance(data, dict):
                # Unpack the 'lines' item.
                lines = dict([
                    (f, dict.fromkeys(linenos, None))
                        for f, linenos in iitems(data.get('lines', {}))
                    ])
                # Unpack the 'arcs' item.
                arcs = dict([
                    (f, dict.fromkeys(arcpairs, None))
                        for f, arcpairs in iitems(data.get('arcs', {}))
                    ])
        except Exception:
            pass
        return lines, arcs

    def combine_parallel_data(self, aliases=None):
        """Combine a number of data files together.

        Treat `self.filename` as a file prefix, and combine the data from all
        of the data files starting with that prefix plus a dot.

        If `aliases` is provided, it's a `PathAliases` object that is used to
        re-map paths to match the local machine's.

        """
        aliases = aliases or PathAliases()
        data_dir, local = os.path.split(self.filename)
        localdot = local + '.'
        for f in os.listdir(data_dir or '.'):
            if f.startswith(localdot):
                full_path = os.path.join(data_dir, f)
                new_lines, new_arcs = self._read_file(full_path)
                for filename, file_data in iitems(new_lines):
                    filename = aliases.map(filename)
                    self.lines.setdefault(filename, {}).update(file_data)
                for filename, file_data in iitems(new_arcs):
                    filename = aliases.map(filename)
                    self.arcs.setdefault(filename, {}).update(file_data)
                if f != local:
                    os.remove(full_path)

    def add_line_data(self, line_data):
        """Add executed line data.

        `line_data` is { filename: { lineno: None, ... }, ...}

        """
        for filename, linenos in iitems(line_data):
            self.lines.setdefault(filename, {}).update(linenos)

    def add_arc_data(self, arc_data):
        """Add measured arc data.

        `arc_data` is { filename: { (l1,l2): None, ... }, ...}

        """
        for filename, arcs in iitems(arc_data):
            self.arcs.setdefault(filename, {}).update(arcs)

    def touch_file(self, filename):
        """Ensure that `filename` appears in the data, empty if needed."""
        self.lines.setdefault(filename, {})

    def measured_files(self):
        """A list of all files that had been measured."""
        return list(self.lines.keys())

    def executed_lines(self, filename):
        """A map containing all the line numbers executed in `filename`.

        If `filename` hasn't been collected at all (because it wasn't executed)
        then return an empty map.

        """
        return self.lines.get(filename) or {}

    def executed_arcs(self, filename):
        """A map containing all the arcs executed in `filename`."""
        return self.arcs.get(filename) or {}

    def add_to_hash(self, filename, hasher):
        """Contribute `filename`'s data to the Md5Hash `hasher`."""
        hasher.update(self.executed_lines(filename))
        hasher.update(self.executed_arcs(filename))

    def summary(self, fullpath=False):
        """Return a dict summarizing the coverage data.

        Keys are based on the filenames, and values are the number of executed
        lines.  If `fullpath` is true, then the keys are the full pathnames of
        the files, otherwise they are the basenames of the files.

        """
        summ = {}
        if fullpath:
            filename_fn = lambda f: f
        else:
            filename_fn = os.path.basename
        for filename, lines in iitems(self.lines):
            summ[filename_fn(filename)] = len(lines)
        return summ

    def has_arcs(self):
        """Does this data have arcs?"""
        return bool(self.arcs)

if __name__ == '__main__':
    # Ad-hoc: show the raw data in a data file.
    import pprint, sys
    covdata = CoverageData()
    if sys.argv[1:]:
        fname = sys.argv[1]
    else:
        fname = covdata.filename
    pprint.pprint(covdata.raw_data(fname))


# -*- coding: utf-8 -*-
"""Subclass of InteractiveShell for terminal based frontends."""

#-----------------------------------------------------------------------------
#  Copyright (C) 2001 Janko Hauser <jhauser@zscout.de>
#  Copyright (C) 2001-2007 Fernando Perez. <fperez@colorado.edu>
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import bdb
import os
import re
import sys
import textwrap

# We need to use nested to support python 2.6, once we move to >=2.7, we can
# use the with keyword's new builtin support for nested managers
try:
    from contextlib import nested
except:
    from IPython.utils.nested_context import nested

from IPython.core.error import TryNext, UsageError
from IPython.core.usage import interactive_usage, default_banner
from IPython.core.inputsplitter import IPythonInputSplitter
from IPython.core.interactiveshell import InteractiveShell, InteractiveShellABC
from IPython.core.magic import Magics, magics_class, line_magic
from IPython.testing.skipdoctest import skip_doctest
from IPython.utils.encoding import get_stream_enc
from IPython.utils import py3compat
from IPython.utils.terminal import toggle_set_term_title, set_term_title
from IPython.utils.process import abbrev_cwd
from IPython.utils.warn import warn, error
from IPython.utils.text import num_ini_spaces, SList, strip_email_quotes
from IPython.utils.traitlets import Integer, CBool, Unicode

#-----------------------------------------------------------------------------
# Utilities
#-----------------------------------------------------------------------------

def get_default_editor():
    try:
        ed = os.environ['EDITOR']
    except KeyError:
        if os.name == 'posix':
            ed = 'vi'  # the only one guaranteed to be there!
        else:
            ed = 'notepad' # same in Windows!
    return ed

def get_pasted_lines(sentinel, l_input=py3compat.input):
    """ Yield pasted lines until the user enters the given sentinel value.
    """
    print "Pasting code; enter '%s' alone on the line to stop or use Ctrl-D." \
          % sentinel
    while True:
        try:
            l = l_input(':')
            if l == sentinel:
                return
            else:
                yield l
        except EOFError:
            print '<EOF>'
            return

#------------------------------------------------------------------------
# Terminal-specific magics
#------------------------------------------------------------------------

@magics_class
class TerminalMagics(Magics):
    def __init__(self, shell):
        super(TerminalMagics, self).__init__(shell)
        self.input_splitter = IPythonInputSplitter(input_mode='line')

    def cleanup_input(self, block):
        """Apply all possible IPython cleanups to an input block.

        This means:

        - remove any global leading whitespace (dedent)
        - remove any email quotes ('>') if they are present in *all* lines
        - apply all static inputsplitter transforms and break into sub-blocks
        - apply prefilter() to each sub-block that is a single line.

        Parameters
        ----------
        block : str
          A possibly multiline input string of code.

        Returns
        -------
        transformed block : str
          The input, with all transformations above applied.
        """
        # We have to effectively implement client-side the loop that is done by
        # the terminal frontend, and furthermore do it on a block that can
        # possibly contain multiple statments pasted in one go.

        # First, run the input through the block splitting code.  We should
        # eventually make this a self-contained method in the inputsplitter.
        isp = self.input_splitter
        isp.reset()
        b = textwrap.dedent(block)

        # Remove email quotes first.  These must be consistently applied to
        # *all* lines to be removed
        b = strip_email_quotes(b)

        # Split the input into independent sub-blocks so we can later do
        # prefiltering (which must be done *only* to single-line inputs)
        blocks = []
        last_block = []
        for line in b.splitlines():
            isp.push(line)
            last_block.append(line)
            if not isp.push_accepts_more():
                blocks.append(isp.source_reset())
                last_block = []
        if last_block:
            blocks.append('\n'.join(last_block))

        # Now, apply prefiltering to any one-line block to match the behavior
        # of the interactive terminal
        final_blocks = []
        for block in blocks:
            lines = block.splitlines()
            if len(lines) == 1:
                final_blocks.append(self.shell.prefilter(lines[0]))
            else:
                final_blocks.append(block)

        # We now have the final version of the input code as a list of blocks,
        # with all inputsplitter transformations applied and single-line blocks
        # run through prefilter.  For further processing, turn into a single
        # string as the rest of our apis use string inputs.
        return '\n'.join(final_blocks)

    def store_or_execute(self, block, name):
        """ Execute a block, or store it in a variable, per the user's request.
        """
        b = self.cleanup_input(block)
        if name:
            # If storing it for further editing
            self.shell.user_ns[name] = SList(b.splitlines())
            print "Block assigned to '%s'" % name
        else:
            self.shell.user_ns['pasted_block'] = b
            self.shell.run_cell(b)

    def rerun_pasted(self, name='pasted_block'):
        """ Rerun a previously pasted command.
        """
        b = self.shell.user_ns.get(name)

        # Sanity checks
        if b is None:
            raise UsageError('No previous pasted block available')
        if not isinstance(b, basestring):
            raise UsageError(
                "Variable 'pasted_block' is not a string, can't execute")

        print "Re-executing '%s...' (%d chars)"% (b.split('\n',1)[0], len(b))
        self.shell.run_cell(b)

    @line_magic
    def autoindent(self, parameter_s = ''):
        """Toggle autoindent on/off (if available)."""

        self.shell.set_autoindent()
        print "Automatic indentation is:",['OFF','ON'][self.shell.autoindent]

    @skip_doctest
    @line_magic
    def cpaste(self, parameter_s=''):
        """Paste & execute a pre-formatted code block from clipboard.

        You must terminate the block with '--' (two minus-signs) or Ctrl-D
        alone on the line. You can also provide your own sentinel with '%paste
        -s %%' ('%%' is the new sentinel for this operation)

        The block is dedented prior to execution to enable execution of method
        definitions. '>' and '+' characters at the beginning of a line are
        ignored, to allow pasting directly from e-mails, diff files and
        doctests (the '...' continuation prompt is also stripped).  The
        executed block is also assigned to variable named 'pasted_block' for
        later editing with '%edit pasted_block'.

        You can also pass a variable name as an argument, e.g. '%cpaste foo'.
        This assigns the pasted block to variable 'foo' as string, without
        dedenting or executing it (preceding >>> and + is still stripped)

        '%cpaste -r' re-executes the block previously entered by cpaste.

        Do not be alarmed by garbled output on Windows (it's a readline bug).
        Just press enter and type -- (and press enter again) and the block
        will be what was just pasted.

        IPython statements (magics, shell escapes) are not supported (yet).

        See also
        --------
        paste: automatically pull code from clipboard.

        Examples
        --------
        ::

          In [8]: %cpaste
          Pasting code; enter '--' alone on the line to stop.
          :>>> a = ["world!", "Hello"]
          :>>> print " ".join(sorted(a))
          :--
          Hello world!
        """
        opts, name = self.parse_options(parameter_s, 'rs:', mode='string')
        if 'r' in opts:
            self.rerun_pasted()
            return

        sentinel = opts.get('s', '--')
        block = '\n'.join(get_pasted_lines(sentinel))
        self.store_or_execute(block, name)

    @line_magic
    def paste(self, parameter_s=''):
        """Paste & execute a pre-formatted code block from clipboard.

        The text is pulled directly from the clipboard without user
        intervention and printed back on the screen before execution (unless
        the -q flag is given to force quiet mode).

        The block is dedented prior to execution to enable execution of method
        definitions. '>' and '+' characters at the beginning of a line are
        ignored, to allow pasting directly from e-mails, diff files and
        doctests (the '...' continuation prompt is also stripped).  The
        executed block is also assigned to variable named 'pasted_block' for
        later editing with '%edit pasted_block'.

        You can also pass a variable name as an argument, e.g. '%paste foo'.
        This assigns the pasted block to variable 'foo' as string, without
        executing it (preceding >>> and + is still stripped).

        Options
        -------

          -r: re-executes the block previously entered by cpaste.

          -q: quiet mode: do not echo the pasted text back to the terminal.

        IPython statements (magics, shell escapes) are not supported (yet).

        See also
        --------
        cpaste: manually paste code into terminal until you mark its end.
        """
        opts, name = self.parse_options(parameter_s, 'rq', mode='string')
        if 'r' in opts:
            self.rerun_pasted()
            return
        try:
            block = self.shell.hooks.clipboard_get()
        except TryNext as clipboard_exc:
            message = getattr(clipboard_exc, 'args')
            if message:
                error(message[0])
            else:
                error('Could not get text from the clipboard.')
            return

        # By default, echo back to terminal unless quiet mode is requested
        if 'q' not in opts:
            write = self.shell.write
            write(self.shell.pycolorize(block))
            if not block.endswith('\n'):
                write('\n')
            write("## -- End pasted text --\n")

        self.store_or_execute(block, name)

    # Class-level: add a '%cls' magic only on Windows
    if sys.platform == 'win32':
        @line_magic
        def cls(self, s):
            """Clear screen.
            """
            os.system("cls")

#-----------------------------------------------------------------------------
# Main class
#-----------------------------------------------------------------------------

class TerminalInteractiveShell(InteractiveShell):

    autoedit_syntax = CBool(False, config=True,
        help="auto editing of files with syntax errors.")
    banner = Unicode('')
    banner1 = Unicode(default_banner, config=True,
        help="""The part of the banner to be printed before the profile"""
    )
    banner2 = Unicode('', config=True,
        help="""The part of the banner to be printed after the profile"""
    )
    confirm_exit = CBool(True, config=True,
        help="""
        Set to confirm when you try to exit IPython with an EOF (Control-D
        in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
        you can force a direct exit without any confirmation.""",
    )
    # This display_banner only controls whether or not self.show_banner()
    # is called when mainloop/interact are called.  The default is False
    # because for the terminal based application, the banner behavior
    # is controlled by Global.display_banner, which IPythonApp looks at
    # to determine if *it* should call show_banner() by hand or not.
    display_banner = CBool(False) # This isn't configurable!
    embedded = CBool(False)
    embedded_active = CBool(False)
    editor = Unicode(get_default_editor(), config=True,
        help="Set the editor used by IPython (default to $EDITOR/vi/notepad)."
    )
    pager = Unicode('less', config=True,
        help="The shell program to be used for paging.")

    screen_length = Integer(0, config=True,
        help=
        """Number of lines of your screen, used to control printing of very
        long strings.  Strings longer than this number of lines will be sent
        through a pager instead of directly printed.  The default value for
        this is 0, which means IPython will auto-detect your screen size every
        time it needs to print certain potentially long strings (this doesn't
        change the behavior of the 'print' keyword, it's only triggered
        internally). If for some reason this isn't working well (it needs
        curses support), specify it yourself. Otherwise don't change the
        default.""",
    )
    term_title = CBool(False, config=True,
        help="Enable auto setting the terminal title."
    )

    # In the terminal, GUI control is done via PyOS_InputHook
    from IPython.lib.inputhook import enable_gui
    enable_gui = staticmethod(enable_gui)

    def __init__(self, config=None, ipython_dir=None, profile_dir=None,
                 user_ns=None, user_module=None, custom_exceptions=((),None),
                 usage=None, banner1=None, banner2=None, display_banner=None):

        super(TerminalInteractiveShell, self).__init__(
            config=config, profile_dir=profile_dir, user_ns=user_ns,
            user_module=user_module, custom_exceptions=custom_exceptions
        )
        # use os.system instead of utils.process.system by default,
        # because piped system doesn't make sense in the Terminal:
        self.system = self.system_raw

        self.init_term_title()
        self.init_usage(usage)
        self.init_banner(banner1, banner2, display_banner)

    #-------------------------------------------------------------------------
    # Things related to the terminal
    #-------------------------------------------------------------------------

    @property
    def usable_screen_length(self):
        if self.screen_length == 0:
            return 0
        else:
            num_lines_bot = self.separate_in.count('\n')+1
            return self.screen_length - num_lines_bot

    def init_term_title(self):
        # Enable or disable the terminal title.
        if self.term_title:
            toggle_set_term_title(True)
            set_term_title('IPython: ' + abbrev_cwd())
        else:
            toggle_set_term_title(False)

    #-------------------------------------------------------------------------
    # Things related to aliases
    #-------------------------------------------------------------------------

    def init_alias(self):
        # The parent class defines aliases that can be safely used with any
        # frontend.
        super(TerminalInteractiveShell, self).init_alias()

        # Now define aliases that only make sense on the terminal, because they
        # need direct access to the console in a way that we can't emulate in
        # GUI or web frontend
        if os.name == 'posix':
            aliases = [('clear', 'clear'), ('more', 'more'), ('less', 'less'),
                       ('man', 'man')]
        elif os.name == 'nt':
            aliases = [('cls', 'cls')]

        for name, cmd in aliases:
            self.alias_manager.define_alias(name, cmd)

    #-------------------------------------------------------------------------
    # Things related to the banner and usage
    #-------------------------------------------------------------------------

    def _banner1_changed(self):
        self.compute_banner()

    def _banner2_changed(self):
        self.compute_banner()

    def _term_title_changed(self, name, new_value):
        self.init_term_title()

    def init_banner(self, banner1, banner2, display_banner):
        if banner1 is not None:
            self.banner1 = banner1
        if banner2 is not None:
            self.banner2 = banner2
        if display_banner is not None:
            self.display_banner = display_banner
        self.compute_banner()

    def show_banner(self, banner=None):
        if banner is None:
            banner = self.banner
        self.write(banner)

    def compute_banner(self):
        self.banner = self.banner1
        if self.profile and self.profile != 'default':
            self.banner += '\nIPython profile: %s\n' % self.profile
        if self.banner2:
            self.banner += '\n' + self.banner2

    def init_usage(self, usage=None):
        if usage is None:
            self.usage = interactive_usage
        else:
            self.usage = usage

    #-------------------------------------------------------------------------
    # Mainloop and code execution logic
    #-------------------------------------------------------------------------

    def mainloop(self, display_banner=None):
        """Start the mainloop.

        If an optional banner argument is given, it will override the
        internally created default banner.
        """

        with nested(self.builtin_trap, self.display_trap):

            while 1:
                try:
                    self.interact(display_banner=display_banner)
                    #self.interact_with_readline()
                    # XXX for testing of a readline-decoupled repl loop, call
                    # interact_with_readline above
                    break
                except KeyboardInterrupt:
                    # this should not be necessary, but KeyboardInterrupt
                    # handling seems rather unpredictable...
                    self.write("\nKeyboardInterrupt in interact()\n")

    def _replace_rlhist_multiline(self, source_raw, hlen_before_cell):
        """Store multiple lines as a single entry in history"""

        # do nothing without readline or disabled multiline
        if not self.has_readline or not self.multiline_history:
            return hlen_before_cell

        # windows rl has no remove_history_item
        if not hasattr(self.readline, "remove_history_item"):
            return hlen_before_cell

        # skip empty cells
        if not source_raw.rstrip():
            return hlen_before_cell

        # nothing changed do nothing, e.g. when rl removes consecutive dups
        hlen = self.readline.get_current_history_length()
        if hlen == hlen_before_cell:
            return hlen_before_cell

        for i in range(hlen - hlen_before_cell):
            self.readline.remove_history_item(hlen - i - 1)
        stdin_encoding = get_stream_enc(sys.stdin, 'utf-8')
        self.readline.add_history(py3compat.unicode_to_str(source_raw.rstrip(),
                                    stdin_encoding))
        return self.readline.get_current_history_length()

    def interact(self, display_banner=None):
        """Closely emulate the interactive Python console."""

        # batch run -> do not interact
        if self.exit_now:
            return

        if display_banner is None:
            display_banner = self.display_banner

        if isinstance(display_banner, basestring):
            self.show_banner(display_banner)
        elif display_banner:
            self.show_banner()

        more = False

        if self.has_readline:
            self.readline_startup_hook(self.pre_readline)
            hlen_b4_cell = self.readline.get_current_history_length()
        else:
            hlen_b4_cell = 0
        # exit_now is set by a call to %Exit or %Quit, through the
        # ask_exit callback.

        while not self.exit_now:
            self.hooks.pre_prompt_hook()
            if more:
                try:
                    prompt = self.prompt_manager.render('in2')
                except:
                    self.showtraceback()
                if self.autoindent:
                    self.rl_do_indent = True

            else:
                try:
                    prompt = self.separate_in + self.prompt_manager.render('in')
                except:
                    self.showtraceback()
            try:
                line = self.raw_input(prompt)
                if self.exit_now:
                    # quick exit on sys.std[in|out] close
                    break
                if self.autoindent:
                    self.rl_do_indent = False

            except KeyboardInterrupt:
                #double-guard against keyboardinterrupts during kbdint handling
                try:
                    self.write('\nKeyboardInterrupt\n')
                    source_raw = self.input_splitter.source_raw_reset()[1]
                    hlen_b4_cell = \
                        self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
                    more = False
                except KeyboardInterrupt:
                    pass
            except EOFError:
                if self.autoindent:
                    self.rl_do_indent = False
                    if self.has_readline:
                        self.readline_startup_hook(None)
                self.write('\n')
                self.exit()
            except bdb.BdbQuit:
                warn('The Python debugger has exited with a BdbQuit exception.\n'
                     'Because of how pdb handles the stack, it is impossible\n'
                     'for IPython to properly format this particular exception.\n'
                     'IPython will resume normal operation.')
            except:
                # exceptions here are VERY RARE, but they can be triggered
                # asynchronously by signal handlers, for example.
                self.showtraceback()
            else:
                self.input_splitter.push(line)
                more = self.input_splitter.push_accepts_more()
                if (self.SyntaxTB.last_syntax_error and
                    self.autoedit_syntax):
                    self.edit_syntax_error()
                if not more:
                    source_raw = self.input_splitter.source_raw_reset()[1]
                    self.run_cell(source_raw, store_history=True)
                    hlen_b4_cell = \
                        self._replace_rlhist_multiline(source_raw, hlen_b4_cell)

        # Turn off the exit flag, so the mainloop can be restarted if desired
        self.exit_now = False

    def raw_input(self, prompt=''):
        """Write a prompt and read a line.

        The returned line does not include the trailing newline.
        When the user enters the EOF key sequence, EOFError is raised.

        Optional inputs:

          - prompt(''): a string to be printed to prompt the user.

          - continue_prompt(False): whether this line is the first one or a
          continuation in a sequence of inputs.
        """
        # Code run by the user may have modified the readline completer state.
        # We must ensure that our completer is back in place.

        if self.has_readline:
            self.set_readline_completer()

        # raw_input expects str, but we pass it unicode sometimes
        prompt = py3compat.cast_bytes_py2(prompt)

        try:
            line = py3compat.str_to_unicode(self.raw_input_original(prompt))
        except ValueError:
            warn("\n********\nYou or a %run:ed script called sys.stdin.close()"
                 " or sys.stdout.close()!\nExiting IPython!\n")
            self.ask_exit()
            return ""

        # Try to be reasonably smart about not re-indenting pasted input more
        # than necessary.  We do this by trimming out the auto-indent initial
        # spaces, if the user's actual input started itself with whitespace.
        if self.autoindent:
            if num_ini_spaces(line) > self.indent_current_nsp:
                line = line[self.indent_current_nsp:]
                self.indent_current_nsp = 0

        return line

    #-------------------------------------------------------------------------
    # Methods to support auto-editing of SyntaxErrors.
    #-------------------------------------------------------------------------

    def edit_syntax_error(self):
        """The bottom half of the syntax error handler called in the main loop.

        Loop until syntax error is fixed or user cancels.
        """

        while self.SyntaxTB.last_syntax_error:
            # copy and clear last_syntax_error
            err = self.SyntaxTB.clear_err_state()
            if not self._should_recompile(err):
                return
            try:
                # may set last_syntax_error again if a SyntaxError is raised
                self.safe_execfile(err.filename,self.user_ns)
            except:
                self.showtraceback()
            else:
                try:
                    f = open(err.filename)
                    try:
                        # This should be inside a display_trap block and I
                        # think it is.
                        sys.displayhook(f.read())
                    finally:
                        f.close()
                except:
                    self.showtraceback()

    def _should_recompile(self,e):
        """Utility routine for edit_syntax_error"""

        if e.filename in ('<ipython console>','<input>','<string>',
                          '<console>','<BackgroundJob compilation>',
                          None):

            return False
        try:
            if (self.autoedit_syntax and
                not self.ask_yes_no('Return to editor to correct syntax error? '
                              '[Y/n] ','y')):
                return False
        except EOFError:
            return False

        def int0(x):
            try:
                return int(x)
            except TypeError:
                return 0
        # always pass integer line and offset values to editor hook
        try:
            self.hooks.fix_error_editor(e.filename,
                int0(e.lineno),int0(e.offset),e.msg)
        except TryNext:
            warn('Could not open editor')
            return False
        return True

    #-------------------------------------------------------------------------
    # Things related to exiting
    #-------------------------------------------------------------------------

    def ask_exit(self):
        """ Ask the shell to exit. Can be overiden and used as a callback. """
        self.exit_now = True

    def exit(self):
        """Handle interactive exit.

        This method calls the ask_exit callback."""
        if self.confirm_exit:
            if self.ask_yes_no('Do you really want to exit ([y]/n)?','y'):
                self.ask_exit()
        else:
            self.ask_exit()

    #-------------------------------------------------------------------------
    # Things related to magics
    #-------------------------------------------------------------------------

    def init_magics(self):
        super(TerminalInteractiveShell, self).init_magics()
        self.register_magics(TerminalMagics)

    def showindentationerror(self):
        super(TerminalInteractiveShell, self).showindentationerror()
        print("If you want to paste code into IPython, try the "
              "%paste and %cpaste magic functions.")

InteractiveShellABC.register(TerminalInteractiveShell)


"""Handles all VCS (version control) support"""

import os
import shutil

from pip.backwardcompat import urlparse, urllib
from pip.log import logger
from pip.util import (display_path, backup_dir, find_command,
                      ask, rmtree, ask_path_exists)

__all__ = ['vcs', 'get_src_requirement']

class VcsSupport(object):
    _registry = {}
    schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp', 'svn']

    def __init__(self):
        # Register more schemes with urlparse for various version control systems
        urlparse.uses_netloc.extend(self.schemes)
        # Python >= 2.7.4, 3.3 doesn't have uses_fragment
        if getattr(urlparse, 'uses_fragment', None):
            urlparse.uses_fragment.extend(self.schemes)
        super(VcsSupport, self).__init__()

    def __iter__(self):
        return self._registry.__iter__()

    @property
    def backends(self):
        return list(self._registry.values())

    @property
    def dirnames(self):
        return [backend.dirname for backend in self.backends]

    @property
    def all_schemes(self):
        schemes = []
        for backend in self.backends:
            schemes.extend(backend.schemes)
        return schemes

    def register(self, cls):
        if not hasattr(cls, 'name'):
            logger.warn('Cannot register VCS %s' % cls.__name__)
            return
        if cls.name not in self._registry:
            self._registry[cls.name] = cls

    def unregister(self, cls=None, name=None):
        if name in self._registry:
            del self._registry[name]
        elif cls in self._registry.values():
            del self._registry[cls.name]
        else:
            logger.warn('Cannot unregister because no class or name given')

    def get_backend_name(self, location):
        """
        Return the name of the version control backend if found at given
        location, e.g. vcs.get_backend_name('/path/to/vcs/checkout')
        """
        for vc_type in self._registry.values():
            path = os.path.join(location, vc_type.dirname)
            if os.path.exists(path):
                return vc_type.name
        return None

    def get_backend(self, name):
        name = name.lower()
        if name in self._registry:
            return self._registry[name]

    def get_backend_from_location(self, location):
        vc_type = self.get_backend_name(location)
        if vc_type:
            return self.get_backend(vc_type)
        return None

vcs = VcsSupport()

class VersionControl(object):
    name = ''
    dirname = ''

    def __init__(self, url=None, *args, **kwargs):
        self.url = url
        self._cmd = None
        super(VersionControl, self).__init__(*args, **kwargs)

    def _filter(self, line):
        return (logger.INFO, line)

    def _is_local_repository(self, repo):
        """
           posix absolute paths start with os.path.sep,
           win32 ones ones start with drive (like c:\\folder)
        """
        drive, tail = os.path.splitdrive(repo)
        return repo.startswith(os.path.sep) or drive

    @property
    def cmd(self):
        if self._cmd is not None:
            return self._cmd
        command = find_command(self.name)
        logger.info('Found command %r at %r' % (self.name, command))
        self._cmd = command
        return command

    def get_url_rev(self):
        """
        Returns the correct repository URL and revision by parsing the given
        repository URL
        """
        error_message= (
           "Sorry, '%s' is a malformed VCS url. "
           "Ihe format is <vcs>+<protocol>://<url>, "
           "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp")
        assert '+' in self.url, error_message % self.url
        url = self.url.split('+', 1)[1]
        scheme, netloc, path, query, frag = urlparse.urlsplit(url)
        rev = None
        if '@' in path:
            path, rev = path.rsplit('@', 1)
        url = urlparse.urlunsplit((scheme, netloc, path, query, ''))
        return url, rev

    def get_info(self, location):
        """
        Returns (url, revision), where both are strings
        """
        assert not location.rstrip('/').endswith(self.dirname), 'Bad directory: %s' % location
        return self.get_url(location), self.get_revision(location)

    def normalize_url(self, url):
        """
        Normalize a URL for comparison by unquoting it and removing any trailing slash.
        """
        return urllib.unquote(url).rstrip('/')

    def compare_urls(self, url1, url2):
        """
        Compare two repo URLs for identity, ignoring incidental differences.
        """
        return (self.normalize_url(url1) == self.normalize_url(url2))

    def parse_vcs_bundle_file(self, content):
        """
        Takes the contents of the bundled text file that explains how to revert
        the stripped off version control data of the given package and returns
        the URL and revision of it.
        """
        raise NotImplementedError

    def obtain(self, dest):
        """
        Called when installing or updating an editable package, takes the
        source path of the checkout.
        """
        raise NotImplementedError

    def switch(self, dest, url, rev_options):
        """
        Switch the repo at ``dest`` to point to ``URL``.
        """
        raise NotImplemented

    def update(self, dest, rev_options):
        """
        Update an already-existing repo to the given ``rev_options``.
        """
        raise NotImplementedError

    def check_destination(self, dest, url, rev_options, rev_display):
        """
        Prepare a location to receive a checkout/clone.

        Return True if the location is ready for (and requires) a
        checkout/clone, False otherwise.
        """
        checkout = True
        prompt = False
        if os.path.exists(dest):
            checkout = False
            if os.path.exists(os.path.join(dest, self.dirname)):
                existing_url = self.get_url(dest)
                if self.compare_urls(existing_url, url):
                    logger.info('%s in %s exists, and has correct URL (%s)' %
                                (self.repo_name.title(), display_path(dest),
                                 url))
                    logger.notify('Updating %s %s%s' %
                                  (display_path(dest), self.repo_name,
                                   rev_display))
                    self.update(dest, rev_options)
                else:
                    logger.warn('%s %s in %s exists with URL %s' %
                                (self.name, self.repo_name,
                                 display_path(dest), existing_url))
                    prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ',
                              ('s', 'i', 'w', 'b'))
            else:
                logger.warn('Directory %s already exists, '
                            'and is not a %s %s.' %
                            (dest, self.name, self.repo_name))
                prompt = ('(i)gnore, (w)ipe, (b)ackup ', ('i', 'w', 'b'))
        if prompt:
            logger.warn('The plan is to install the %s repository %s' %
                        (self.name, url))
            response = ask_path_exists('What to do?  %s' % prompt[0],
                                       prompt[1])

            if response == 's':
                logger.notify('Switching %s %s to %s%s' %
                              (self.repo_name, display_path(dest), url,
                               rev_display))
                self.switch(dest, url, rev_options)
            elif response == 'i':
                # do nothing
                pass
            elif response == 'w':
                logger.warn('Deleting %s' % display_path(dest))
                rmtree(dest)
                checkout = True
            elif response == 'b':
                dest_dir = backup_dir(dest)
                logger.warn('Backing up %s to %s'
                            % (display_path(dest), dest_dir))
                shutil.move(dest, dest_dir)
                checkout = True
        return checkout

    def unpack(self, location):
        if os.path.exists(location):
            rmtree(location)
        self.obtain(location)

    def get_src_requirement(self, dist, location, find_tags=False):
        raise NotImplementedError

def get_src_requirement(dist, location, find_tags):
    version_control = vcs.get_backend_from_location(location)
    if version_control:
        return version_control().get_src_requirement(dist, location, find_tags)
    logger.warn('cannot determine version of editable source in %s (is not SVN checkout, Git clone, Mercurial clone or Bazaar branch)' % location)
    return dist.as_requirement()