# -*- coding: utf-8 -*-
"""
Command Session

TODO:
    - contextmanager to allow caller to set stream preference
    - override of environment and cwd
    - ParamDict should unpack itself (CommandSession should nto have to)
"""
__all__ = ["CommandError", "CommandSession", "CommandSessionMixin", "ParamDict"]

import os
import sys
import six
import subprocess

from boltons.dictutils import OrderedMultiDict

ParamDict = OrderedMultiDict

class CommandError(subprocess.CalledProcessError):
    def __init__(self, session):
        super(CommandError, self).__init__(
            session.last_returncode,
            session.last_command,
            output=session.last_output
        )
    def __str__(self):
        return "Command {} returned {} with {}".format(
            self.cmd,
            self.returncode,
            self.output
        )

class CommandSession(object):
    """Wrapper around multiple invocations of subprocess.Popen, tracking
    and saving execution results and output.
    Provides methods having similar interface as `subprocess.call()`,
    `subprocess.check_call()`, and subprocess.check_output()`.

    Params:
        stream (bool): should output be streamed to stdout, in additin to being
        captured?
        env (dict): environment to use, in place of current process environment
        cwd (str): current working directory to replace `os.cwd()`
        force_shell (bool): whether or not to force `shell=True` param
            to `Popen()`; if the shell (/bin/sh) is required (for instance,
            if you intend to use a shell builtin), you can force this;
            normally, `shell` param is only used for commands given as
            strings (subprocess prefers commands given as lists, which is
            the default for this module).
    """
    def __init__(self, stream=False, env=None, cwd=None, force_shell=False):
        self.log = []
        self._stream = sys.stdout if stream else None
        self._env = env
        self._cwd = cwd
        self._shell = force_shell

    @property
    def last_returncode(self):
        """Get the return code of the last command exevuted."""
        try:
            return self.log[-1][1]
        except IndexError:
            raise RuntimeError('Nothing executed')

    @property
    def last_command(self):
        """Get the output of the last command exevuted."""
        if not len(self.log):
            raise RuntimeError('Nothing executed')

        return self.log[-1][0]

    @property
    def last_output(self):
        """Get the output of the last command exevuted."""
        if not len(self.log):
            raise RuntimeError('Nothing executed')

        return self.log[-1][2]

    @property
    def last_error(self):
        """Get the output of the last command exevuted."""
        if not len(self.log):
            raise RuntimeError('Nothing executed')

        try:
            errs = [l for l in self.log if l[1] != 0]

            return errs[-1][2]
        except IndexError:
            # odd case where there were no errors
            #TODO
            return 'no last error'

    @property
    def output(self):
        """Get the output of the entire session."""

        return '\n'.join(['\n'.join(l[2]) for l in self.log])

    def _stream_write(self, line):
        if self._stream:
            self._stream.write(line)
            self._stream.write(os.linesep)

    def _exec(self, cmd):
        shell = self._shell
        if isinstance(cmd, six.string_types):
            shell=True

        popen_kwargs = {
            'shell': shell,
            'stdout': subprocess.PIPE,
            'stderr': subprocess.STDOUT
        }
        if self._env:
            popen_kwargs['env'] = self._env

        if self._cwd:
            popen_kwargs['cwd'] = self._cwd

        p = subprocess.Popen(
            cmd, **popen_kwargs
        )
        output = []
        for line in iter(p.stdout.readline, six.binary_type('', 'utf-8')):
            line = line.decode('utf-8').strip()
            self._stream_write(line)
            output.append(line)

        p.wait()

        log = [
            ' '.join(cmd) if not shell else cmd,
            p.returncode,
            output
        ]
        self.log.append(log)

        return p.returncode, '\n'.join(output)

    def check_output(self, cmd):
        """Wrapper for subprocess.check_output."""
        ret, output = self._exec(cmd)
        if not ret == 0:
            raise CommandError(self)

        return output

    def check_call(self, cmd):
        """Fake the interface of subprocess.call()."""
        ret, _ = self._exec(cmd)
        if not ret == 0:
            raise CommandError(self)

        return ret

    def call(self, cmd):
        """Fake the interface of subprocess.call()."""
        ret, _ = self._exec(cmd)
        return ret

    @staticmethod
    def unpack_args(*args, **kwargs):
        gnu = kwargs.pop('gnu', False)
        assert isinstance(gnu, bool)
        def _transform(argname):
            """Transform a python identifier into a
            shell-appropriate argument name
            """
            if len(argname) == 1:
                return '-{}'.format(argname)

            return '--{}'.format(argname.replace('_', '-'))

        ret = []
        for k, v in kwargs.items():
            if isinstance(v, list):
                for item in v:
                    if gnu:
                        ret.append('{}={}'.format(
                            _transform(k),
                            str(item)
                        ))
                    else:
                        ret.extend([
                            _transform(k),
                            str(item)
                        ])
            else:
                if gnu:
                    ret.append('{}={}'.format(_transform(k), str(v)))
                else:
                    ret.extend([
                        _transform(k),
                        str(v)
                    ])

        if len(args):
            for item in args:
                ret.append(_transform(item))

        return ret

    @staticmethod
    def unpack_pargs(positional_args, param_kwargs, gnu=False):
        """Unpack multidict and positional args into a
        list appropriate for subprocess.
        :param param_kwargs:
            ``ParamDict`` storing '--param' style data.
        :param positional_args: flags
        :param gnu:
            if True, long-name args are unpacked as:
                --parameter=argument
            otherwise, they are unpacked as:
                --parameter argument
        :returns: list appropriate for sending to subprocess
        """

        def _transform(argname):
            """Transform a python identifier into a
            shell-appropriate argument name
            """
            if len(argname) == 1:
                return '-{}'.format(argname)

            return '--{}'.format(argname.replace('_', '-'))

        args = []
        for item in param_kwargs.keys():
            for value in param_kwargs.getlist(item):
                if gnu:
                    args.append('{}={}'.format(
                        _transform(item),
                        value
                    ))
                else:
                    args.extend([
                        _transform(item),
                        value
                    ])
        if positional_args:
            for item in positional_args:
                args.append(_transform(item))

        return args

class CommandSessionMixin(object):
    def __init__(self, session=None):
        self.session = session or CommandSession()


"""Results of coverage measurement."""

import os

from coverage.backward import iitems, set, sorted       # pylint: disable=W0622
from coverage.misc import format_lines, join_regex, NoSource
from coverage.parser import CodeParser

class Analysis(object):
    """The results of analyzing a code unit."""

    def __init__(self, cov, code_unit):
        self.coverage = cov
        self.code_unit = code_unit

        self.filename = self.code_unit.filename
        actual_filename, source = self.find_source(self.filename)

        self.parser = CodeParser(
            text=source, filename=actual_filename,
            exclude=self.coverage._exclude_regex('exclude')
            )
        self.statements, self.excluded = self.parser.parse_source()

        # Identify missing statements.
        executed = self.coverage.data.executed_lines(self.filename)
        exec1 = self.parser.first_lines(executed)
        self.missing = self.statements - exec1

        if self.coverage.data.has_arcs():
            self.no_branch = self.parser.lines_matching(
                join_regex(self.coverage.config.partial_list),
                join_regex(self.coverage.config.partial_always_list)
                )
            n_branches = self.total_branches()
            mba = self.missing_branch_arcs()
            n_partial_branches = sum(
                [len(v) for k,v in iitems(mba) if k not in self.missing]
                )
            n_missing_branches = sum([len(v) for k,v in iitems(mba)])
        else:
            n_branches = n_partial_branches = n_missing_branches = 0
            self.no_branch = set()

        self.numbers = Numbers(
            n_files=1,
            n_statements=len(self.statements),
            n_excluded=len(self.excluded),
            n_missing=len(self.missing),
            n_branches=n_branches,
            n_partial_branches=n_partial_branches,
            n_missing_branches=n_missing_branches,
            )

    def find_source(self, filename):
        """Find the source for `filename`.

        Returns two values: the actual filename, and the source.

        The source returned depends on which of these cases holds:

            * The filename seems to be a non-source file: returns None

            * The filename is a source file, and actually exists: returns None.

            * The filename is a source file, and is in a zip file or egg:
              returns the source.

            * The filename is a source file, but couldn't be found: raises
              `NoSource`.

        """
        source = None

        base, ext = os.path.splitext(filename)
        TRY_EXTS = {
            '.py':  ['.py', '.pyw'],
            '.pyw': ['.pyw'],
        }
        try_exts = TRY_EXTS.get(ext)
        if not try_exts:
            return filename, None

        for try_ext in try_exts:
            try_filename = base + try_ext
            if os.path.exists(try_filename):
                return try_filename, None
            source = self.coverage.file_locator.get_zip_data(try_filename)
            if source:
                return try_filename, source
        raise NoSource("No source for code: '%s'" % filename)

    def missing_formatted(self):
        """The missing line numbers, formatted nicely.

        Returns a string like "1-2, 5-11, 13-14".

        """
        return format_lines(self.statements, self.missing)

    def has_arcs(self):
        """Were arcs measured in this result?"""
        return self.coverage.data.has_arcs()

    def arc_possibilities(self):
        """Returns a sorted list of the arcs in the code."""
        arcs = self.parser.arcs()
        return arcs

    def arcs_executed(self):
        """Returns a sorted list of the arcs actually executed in the code."""
        executed = self.coverage.data.executed_arcs(self.filename)
        m2fl = self.parser.first_line
        executed = [(m2fl(l1), m2fl(l2)) for (l1,l2) in executed]
        return sorted(executed)

    def arcs_missing(self):
        """Returns a sorted list of the arcs in the code not executed."""
        possible = self.arc_possibilities()
        executed = self.arcs_executed()
        missing = [
            p for p in possible
                if p not in executed
                    and p[0] not in self.no_branch
            ]
        return sorted(missing)

    def arcs_unpredicted(self):
        """Returns a sorted list of the executed arcs missing from the code."""
        possible = self.arc_possibilities()
        executed = self.arcs_executed()
        # Exclude arcs here which connect a line to itself.  They can occur
        # in executed data in some cases.  This is where they can cause
        # trouble, and here is where it's the least burden to remove them.
        unpredicted = [
            e for e in executed
                if e not in possible
                    and e[0] != e[1]
            ]
        return sorted(unpredicted)

    def branch_lines(self):
        """Returns a list of line numbers that have more than one exit."""
        exit_counts = self.parser.exit_counts()
        return [l1 for l1,count in iitems(exit_counts) if count > 1]

    def total_branches(self):
        """How many total branches are there?"""
        exit_counts = self.parser.exit_counts()
        return sum([count for count in exit_counts.values() if count > 1])

    def missing_branch_arcs(self):
        """Return arcs that weren't executed from branch lines.

        Returns {l1:[l2a,l2b,...], ...}

        """
        missing = self.arcs_missing()
        branch_lines = set(self.branch_lines())
        mba = {}
        for l1, l2 in missing:
            if l1 in branch_lines:
                if l1 not in mba:
                    mba[l1] = []
                mba[l1].append(l2)
        return mba

    def branch_stats(self):
        """Get stats about branches.

        Returns a dict mapping line numbers to a tuple:
        (total_exits, taken_exits).
        """

        exit_counts = self.parser.exit_counts()
        missing_arcs = self.missing_branch_arcs()
        stats = {}
        for lnum in self.branch_lines():
            exits = exit_counts[lnum]
            try:
                missing = len(missing_arcs[lnum])
            except KeyError:
                missing = 0
            stats[lnum] = (exits, exits - missing)
        return stats

class Numbers(object):
    """The numerical results of measuring coverage.

    This holds the basic statistics from `Analysis`, and is used to roll
    up statistics across files.

    """
    # A global to determine the precision on coverage percentages, the number
    # of decimal places.
    _precision = 0
    _near0 = 1.0              # These will change when _precision is changed.
    _near100 = 99.0

    def __init__(self, n_files=0, n_statements=0, n_excluded=0, n_missing=0,
                    n_branches=0, n_partial_branches=0, n_missing_branches=0
                    ):
        self.n_files = n_files
        self.n_statements = n_statements
        self.n_excluded = n_excluded
        self.n_missing = n_missing
        self.n_branches = n_branches
        self.n_partial_branches = n_partial_branches
        self.n_missing_branches = n_missing_branches

    def set_precision(cls, precision):
        """Set the number of decimal places used to report percentages."""
        assert 0 <= precision < 10
        cls._precision = precision
        cls._near0 = 1.0 / 10**precision
        cls._near100 = 100.0 - cls._near0
    set_precision = classmethod(set_precision)

    def _get_n_executed(self):
        """Returns the number of executed statements."""
        return self.n_statements - self.n_missing
    n_executed = property(_get_n_executed)

    def _get_n_executed_branches(self):
        """Returns the number of executed branches."""
        return self.n_branches - self.n_missing_branches
    n_executed_branches = property(_get_n_executed_branches)

    def _get_pc_covered(self):
        """Returns a single percentage value for coverage."""
        if self.n_statements > 0:
            pc_cov = (100.0 * (self.n_executed + self.n_executed_branches) /
                        (self.n_statements + self.n_branches))
        else:
            pc_cov = 100.0
        return pc_cov
    pc_covered = property(_get_pc_covered)

    def _get_pc_covered_str(self):
        """Returns the percent covered, as a string, without a percent sign.

        Note that "0" is only returned when the value is truly zero, and "100"
        is only returned when the value is truly 100.  Rounding can never
        result in either "0" or "100".

        """
        pc = self.pc_covered
        if 0 < pc < self._near0:
            pc = self._near0
        elif self._near100 < pc < 100:
            pc = self._near100
        else:
            pc = round(pc, self._precision)
        return "%.*f" % (self._precision, pc)
    pc_covered_str = property(_get_pc_covered_str)

    def pc_str_width(cls):
        """How many characters wide can pc_covered_str be?"""
        width = 3   # "100"
        if cls._precision > 0:
            width += 1 + cls._precision
        return width
    pc_str_width = classmethod(pc_str_width)

    def __add__(self, other):
        nums = Numbers()
        nums.n_files = self.n_files + other.n_files
        nums.n_statements = self.n_statements + other.n_statements
        nums.n_excluded = self.n_excluded + other.n_excluded
        nums.n_missing = self.n_missing + other.n_missing
        nums.n_branches = self.n_branches + other.n_branches
        nums.n_partial_branches = (
            self.n_partial_branches + other.n_partial_branches
            )
        nums.n_missing_branches = (
            self.n_missing_branches + other.n_missing_branches
            )
        return nums

    def __radd__(self, other):
        # Implementing 0+Numbers allows us to sum() a list of Numbers.
        if other == 0:
            return self
        return NotImplemented


import re
from django import template
from django.template import Node, TemplateSyntaxError
from django.conf import settings
from django.utils.safestring import mark_safe
from ..utils.query import get_text_tokenizer

register = template.Library()

def highlight_text(needles, haystack, cls_name='highlighted', words=False, case=False):
    """ Applies cls_name to all needles found in haystack. """

    if not needles:
        return haystack
    if not haystack:
        return ''

    if words:
        pattern = r"(%s)" % "|".join(['\\b{}\\b'.format(re.escape(n)) for n in needles])
    else:
        pattern = r"(%s)" % "|".join([re.escape(n) for n in needles])

    if case:
        regex = re.compile(pattern)
    else:
        regex = re.compile(pattern, re.I)

    i, out = 0, ""
    for m in regex.finditer(haystack):
        out += "".join([haystack[i:m.start()], '' % cls_name,
            haystack[m.start():m.end()], ""])
        i = m.end()
    return mark_safe(out + haystack[i:])

@register.filter
def highlight(string, keywords, cls_name='highlighted'):
    """ Given an list of words, this function highlights the matched text in the given string. """

    if not keywords:
        return string
    if not string:
        return ''
    include, exclude = get_text_tokenizer(keywords)
    highlighted = highlight_text(include, string, cls_name)
    return highlighted

@register.filter
def highlight_words(string, keywords, cls_name='highlighted'):
    """ Given an list of words, this function highlights the matched words in the given string. """

    if not keywords:
        return string
    if not string:
        return ''
    include, exclude = get_text_tokenizer(keywords)
    highlighted = highlight_text(include, string, cls_name, words=True)
    return highlighted


import os, sys, __builtin__, tempfile, operator, pkg_resources
_os = sys.modules[os.name]
try:
    _file = file
except NameError:
    _file = None
_open = open
from distutils.errors import DistutilsError
__all__ = [
    "AbstractSandbox", "DirectorySandbox", "SandboxViolation", "run_setup",
]
def run_setup(setup_script, args):
    """Run a distutils setup script, sandboxed in its directory"""
    old_dir = os.getcwd()
    save_argv = sys.argv[:]
    save_path = sys.path[:]
    setup_dir = os.path.abspath(os.path.dirname(setup_script))
    temp_dir = os.path.join(setup_dir,'temp')
    if not os.path.isdir(temp_dir): os.makedirs(temp_dir)
    save_tmp = tempfile.tempdir
    save_modules = sys.modules.copy()
    pr_state = pkg_resources.__getstate__()
    try:
        tempfile.tempdir = temp_dir
        os.chdir(setup_dir)
        try:
            sys.argv[:] = [setup_script]+list(args)
            sys.path.insert(0, setup_dir)
            DirectorySandbox(setup_dir).run(
                lambda: execfile(
                    "setup.py",
                    {'__file__':setup_script, '__name__':'__main__'}
                )
            )
        except SystemExit, v:
            if v.args and v.args[0]:
                raise
            # Normal exit, just return
    finally:
        pkg_resources.__setstate__(pr_state)
        sys.modules.update(save_modules)
        # remove any modules imported within the sandbox
        del_modules = [
            mod_name for mod_name in sys.modules
            if mod_name not in save_modules
            # exclude any encodings modules. See #285
            and not mod_name.startswith('encodings.')
        ]
        map(sys.modules.__delitem__, del_modules)
        os.chdir(old_dir)
        sys.path[:] = save_path
        sys.argv[:] = save_argv
        tempfile.tempdir = save_tmp

class AbstractSandbox:
    """Wrap 'os' module and 'open()' builtin for virtualizing setup scripts"""

    _active = False

    def __init__(self):
        self._attrs = [
            name for name in dir(_os)
                if not name.startswith('_') and hasattr(self,name)
        ]

    def _copy(self, source):
        for name in self._attrs:
            setattr(os, name, getattr(source,name))

    def run(self, func):
        """Run 'func' under os sandboxing"""
        try:
            self._copy(self)
            if _file:
                __builtin__.file = self._file
            __builtin__.open = self._open
            self._active = True
            return func()
        finally:
            self._active = False
            if _file:
                __builtin__.file = _file
            __builtin__.open = _open
            self._copy(_os)

    def _mk_dual_path_wrapper(name):
        original = getattr(_os,name)
        def wrap(self,src,dst,*args,**kw):
            if self._active:
                src,dst = self._remap_pair(name,src,dst,*args,**kw)
            return original(src,dst,*args,**kw)
        return wrap

    for name in ["rename", "link", "symlink"]:
        if hasattr(_os,name): locals()[name] = _mk_dual_path_wrapper(name)

    def _mk_single_path_wrapper(name, original=None):
        original = original or getattr(_os,name)
        def wrap(self,path,*args,**kw):
            if self._active:
                path = self._remap_input(name,path,*args,**kw)
            return original(path,*args,**kw)
        return wrap

    if _file:
        _file = _mk_single_path_wrapper('file', _file)
    _open = _mk_single_path_wrapper('open', _open)
    for name in [
        "stat", "listdir", "chdir", "open", "chmod", "chown", "mkdir",
        "remove", "unlink", "rmdir", "utime", "lchown", "chroot", "lstat",
        "startfile", "mkfifo", "mknod", "pathconf", "access"
    ]:
        if hasattr(_os,name): locals()[name] = _mk_single_path_wrapper(name)

    def _mk_single_with_return(name):
        original = getattr(_os,name)
        def wrap(self,path,*args,**kw):
            if self._active:
                path = self._remap_input(name,path,*args,**kw)
                return self._remap_output(name, original(path,*args,**kw))
            return original(path,*args,**kw)
        return wrap

    for name in ['readlink', 'tempnam']:
        if hasattr(_os,name): locals()[name] = _mk_single_with_return(name)

    def _mk_query(name):
        original = getattr(_os,name)
        def wrap(self,*args,**kw):
            retval = original(*args,**kw)
            if self._active:
                return self._remap_output(name, retval)
            return retval
        return wrap

    for name in ['getcwd', 'tmpnam']:
        if hasattr(_os,name): locals()[name] = _mk_query(name)

    def _validate_path(self,path):
        """Called to remap or validate any path, whether input or output"""
        return path

    def _remap_input(self,operation,path,*args,**kw):
        """Called for path inputs"""
        return self._validate_path(path)

    def _remap_output(self,operation,path):
        """Called for path outputs"""
        return self._validate_path(path)

    def _remap_pair(self,operation,src,dst,*args,**kw):
        """Called for path pairs like rename, link, and symlink operations"""
        return (
            self._remap_input(operation+'-from',src,*args,**kw),
            self._remap_input(operation+'-to',dst,*args,**kw)
        )

if hasattr(os, 'devnull'):
    _EXCEPTIONS = [os.devnull,]
else:
    _EXCEPTIONS = []

try:
    from win32com.client.gencache import GetGeneratePath
    _EXCEPTIONS.append(GetGeneratePath())
    del GetGeneratePath
except ImportError:
    # it appears pywin32 is not installed, so no need to exclude.
    pass

class DirectorySandbox(AbstractSandbox):
    """Restrict operations to a single subdirectory - pseudo-chroot"""

    write_ops = dict.fromkeys([
        "open", "chmod", "chown", "mkdir", "remove", "unlink", "rmdir",
        "utime", "lchown", "chroot", "mkfifo", "mknod", "tempnam",
    ])

    def __init__(self, sandbox, exceptions=_EXCEPTIONS):
        self._sandbox = os.path.normcase(os.path.realpath(sandbox))
        self._prefix = os.path.join(self._sandbox,'')
        self._exceptions = [os.path.normcase(os.path.realpath(path)) for path in exceptions]
        AbstractSandbox.__init__(self)

    def _violation(self, operation, *args, **kw):
        raise SandboxViolation(operation, args, kw)

    if _file:
        def _file(self, path, mode='r', *args, **kw):
            if mode not in ('r', 'rt', 'rb', 'rU', 'U') and not self._ok(path):
                self._violation("file", path, mode, *args, **kw)
            return _file(path,mode,*args,**kw)

    def _open(self, path, mode='r', *args, **kw):
        if mode not in ('r', 'rt', 'rb', 'rU', 'U') and not self._ok(path):
            self._violation("open", path, mode, *args, **kw)
        return _open(path,mode,*args,**kw)

    def tmpnam(self):
        self._violation("tmpnam")

    def _ok(self,path):
        active = self._active
        try:
            self._active = False
            realpath = os.path.normcase(os.path.realpath(path))
            if (self._exempted(realpath) or realpath == self._sandbox
                or realpath.startswith(self._prefix)):
                return True
        finally:
            self._active = active

    def _exempted(self, filepath):
        exception_matches = map(filepath.startswith, self._exceptions)
        return True in exception_matches

    def _remap_input(self,operation,path,*args,**kw):
        """Called for path inputs"""
        if operation in self.write_ops and not self._ok(path):
            self._violation(operation, os.path.realpath(path), *args, **kw)
        return path

    def _remap_pair(self,operation,src,dst,*args,**kw):
        """Called for path pairs like rename, link, and symlink operations"""
        if not self._ok(src) or not self._ok(dst):
            self._violation(operation, src, dst, *args, **kw)
        return (src,dst)

    def open(self, file, flags, mode=0777):
        """Called for low-level os.open()"""
        if flags & WRITE_FLAGS and not self._ok(file):
            self._violation("os.open", file, flags, mode)
        return _os.open(file,flags,mode)

WRITE_FLAGS = reduce(
    operator.or_,
    [getattr(_os, a, 0) for a in
        "O_WRONLY O_RDWR O_APPEND O_CREAT O_TRUNC O_TEMPORARY".split()]
)

class SandboxViolation(DistutilsError):
    """A setup script attempted to modify the filesystem outside the sandbox"""

    def __str__(self):
        return """SandboxViolation: %s%r %s

The package setup script has attempted to modify files on your system
that are not within the EasyInstall build area, and has been aborted.

This package cannot be safely installed by EasyInstall, and may not
support alternate installation locations even if you run its setup
script by hand.  Please inform the package's author and the EasyInstall
maintainers to find out if a fix or workaround is available.""" % self.args

#


# encoding: utf-8
"""
Utilities for working with strings and text.
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import __main__

import os
import re
import shutil
import sys
import textwrap
from string import Formatter

from IPython.external.path import path
from IPython.testing.skipdoctest import skip_doctest_py3, skip_doctest
from IPython.utils import py3compat
from IPython.utils.io import nlprint
from IPython.utils.data import flatten

#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------

def unquote_ends(istr):
    """Remove a single pair of quotes from the endpoints of a string."""

    if not istr:
        return istr
    if (istr[0]=="'" and istr[-1]=="'") or \
       (istr[0]=='"' and istr[-1]=='"'):
        return istr[1:-1]
    else:
        return istr

class LSString(str):
    """String derivative with a special access attributes.

    These are normal strings, but with the special attributes:

        .l (or .list) : value as list (split on newlines).
        .n (or .nlstr): original value (the string itself).
        .s (or .spstr): value as whitespace-separated string.
        .p (or .paths): list of path objects

    Any values which require transformations are computed only once and
    cached.

    Such strings are very useful to efficiently interact with the shell, which
    typically only understands whitespace-separated options for commands."""

    def get_list(self):
        try:
            return self.__list
        except AttributeError:
            self.__list = self.split('\n')
            return self.__list

    l = list = property(get_list)

    def get_spstr(self):
        try:
            return self.__spstr
        except AttributeError:
            self.__spstr = self.replace('\n',' ')
            return self.__spstr

    s = spstr = property(get_spstr)

    def get_nlstr(self):
        return self

    n = nlstr = property(get_nlstr)

    def get_paths(self):
        try:
            return self.__paths
        except AttributeError:
            self.__paths = [path(p) for p in self.split('\n') if os.path.exists(p)]
            return self.__paths

    p = paths = property(get_paths)

# FIXME: We need to reimplement type specific displayhook and then add this
# back as a custom printer. This should also be moved outside utils into the
# core.

# def print_lsstring(arg):
#     """ Prettier (non-repr-like) and more informative printer for LSString """
#     print "LSString (.p, .n, .l, .s available). Value:"
#     print arg
#
#
# print_lsstring = result_display.when_type(LSString)(print_lsstring)

class SList(list):
    """List derivative with a special access attributes.

    These are normal lists, but with the special attributes:

        .l (or .list) : value as list (the list itself).
        .n (or .nlstr): value as a string, joined on newlines.
        .s (or .spstr): value as a string, joined on spaces.
        .p (or .paths): list of path objects

    Any values which require transformations are computed only once and
    cached."""

    def get_list(self):
        return self

    l = list = property(get_list)

    def get_spstr(self):
        try:
            return self.__spstr
        except AttributeError:
            self.__spstr = ' '.join(self)
            return self.__spstr

    s = spstr = property(get_spstr)

    def get_nlstr(self):
        try:
            return self.__nlstr
        except AttributeError:
            self.__nlstr = '\n'.join(self)
            return self.__nlstr

    n = nlstr = property(get_nlstr)

    def get_paths(self):
        try:
            return self.__paths
        except AttributeError:
            self.__paths = [path(p) for p in self if os.path.exists(p)]
            return self.__paths

    p = paths = property(get_paths)

    def grep(self, pattern, prune = False, field = None):
        """ Return all strings matching 'pattern' (a regex or callable)

        This is case-insensitive. If prune is true, return all items
        NOT matching the pattern.

        If field is specified, the match must occur in the specified
        whitespace-separated field.

        Examples::

            a.grep( lambda x: x.startswith('C') )
            a.grep('Cha.*log', prune=1)
            a.grep('chm', field=-1)
        """

        def match_target(s):
            if field is None:
                return s
            parts = s.split()
            try:
                tgt = parts[field]
                return tgt
            except IndexError:
                return ""

        if isinstance(pattern, basestring):
            pred = lambda x : re.search(pattern, x, re.IGNORECASE)
        else:
            pred = pattern
        if not prune:
            return SList([el for el in self if pred(match_target(el))])
        else:
            return SList([el for el in self if not pred(match_target(el))])

    def fields(self, *fields):
        """ Collect whitespace-separated fields from string list

        Allows quick awk-like usage of string lists.

        Example data (in var a, created by 'a = !ls -l')::
            -rwxrwxrwx  1 ville None      18 Dec 14  2006 ChangeLog
            drwxrwxrwx+ 6 ville None       0 Oct 24 18:05 IPython

        a.fields(0) is ['-rwxrwxrwx', 'drwxrwxrwx+']
        a.fields(1,0) is ['1 -rwxrwxrwx', '6 drwxrwxrwx+']
        (note the joining by space).
        a.fields(-1) is ['ChangeLog', 'IPython']

        IndexErrors are ignored.

        Without args, fields() just split()'s the strings.
        """
        if len(fields) == 0:
            return [el.split() for el in self]

        res = SList()
        for el in [f.split() for f in self]:
            lineparts = []

            for fd in fields:
                try:
                    lineparts.append(el[fd])
                except IndexError:
                    pass
            if lineparts:
                res.append(" ".join(lineparts))

        return res

    def sort(self,field= None,  nums = False):
        """ sort by specified fields (see fields())

        Example::
            a.sort(1, nums = True)

        Sorts a by second field, in numerical order (so that 21 > 3)

        """

        #decorate, sort, undecorate
        if field is not None:
            dsu = [[SList([line]).fields(field),  line] for line in self]
        else:
            dsu = [[line,  line] for line in self]
        if nums:
            for i in range(len(dsu)):
                numstr = "".join([ch for ch in dsu[i][0] if ch.isdigit()])
                try:
                    n = int(numstr)
                except ValueError:
                    n = 0;
                dsu[i][0] = n

        dsu.sort()
        return SList([t[1] for t in dsu])

# FIXME: We need to reimplement type specific displayhook and then add this
# back as a custom printer. This should also be moved outside utils into the
# core.

# def print_slist(arg):
#     """ Prettier (non-repr-like) and more informative printer for SList """
#     print "SList (.p, .n, .l, .s, .grep(), .fields(), sort() available):"
#     if hasattr(arg,  'hideonce') and arg.hideonce:
#         arg.hideonce = False
#         return
#
#     nlprint(arg)
#
# print_slist = result_display.when_type(SList)(print_slist)

def esc_quotes(strng):
    """Return the input string with single and double quotes escaped out"""

    return strng.replace('"','\\"').replace("'","\\'")

def qw(words,flat=0,sep=None,maxsplit=-1):
    """Similar to Perl's qw() operator, but with some more options.

    qw(words,flat=0,sep=' ',maxsplit=-1) -> words.split(sep,maxsplit)

    words can also be a list itself, and with flat=1, the output will be
    recursively flattened.

    Examples:

    >>> qw('1 2')
    ['1', '2']

    >>> qw(['a b','1 2',['m n','p q']])
    [['a', 'b'], ['1', '2'], [['m', 'n'], ['p', 'q']]]

    >>> qw(['a b','1 2',['m n','p q']],flat=1)
    ['a', 'b', '1', '2', 'm', 'n', 'p', 'q']
    """

    if isinstance(words, basestring):
        return [word.strip() for word in words.split(sep,maxsplit)
                if word and not word.isspace() ]
    if flat:
        return flatten(map(qw,words,[1]*len(words)))
    return map(qw,words)

def qwflat(words,sep=None,maxsplit=-1):
    """Calls qw(words) in flat mode. It's just a convenient shorthand."""
    return qw(words,1,sep,maxsplit)

def qw_lol(indata):
    """qw_lol('a b') -> [['a','b']],
    otherwise it's just a call to qw().

    We need this to make sure the modules_some keys *always* end up as a
    list of lists."""

    if isinstance(indata, basestring):
        return [qw(indata)]
    else:
        return qw(indata)

def grep(pat,list,case=1):
    """Simple minded grep-like function.
    grep(pat,list) returns occurrences of pat in list, None on failure.

    It only does simple string matching, with no support for regexps. Use the
    option case=0 for case-insensitive matching."""

    # This is pretty crude. At least it should implement copying only references
    # to the original data in case it's big. Now it copies the data for output.
    out=[]
    if case:
        for term in list:
            if term.find(pat)>-1: out.append(term)
    else:
        lpat=pat.lower()
        for term in list:
            if term.lower().find(lpat)>-1: out.append(term)

    if len(out): return out
    else: return None

def dgrep(pat,*opts):
    """Return grep() on dir()+dir(__builtins__).

    A very common use of grep() when working interactively."""

    return grep(pat,dir(__main__)+dir(__main__.__builtins__),*opts)

def idgrep(pat):
    """Case-insensitive dgrep()"""

    return dgrep(pat,0)

def igrep(pat,list):
    """Synonym for case-insensitive grep."""

    return grep(pat,list,case=0)

def indent(instr,nspaces=4, ntabs=0, flatten=False):
    """Indent a string a given number of spaces or tabstops.

    indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.

    Parameters
    ----------

    instr : basestring
        The string to be indented.
    nspaces : int (default: 4)
        The number of spaces to be indented.
    ntabs : int (default: 0)
        The number of tabs to be indented.
    flatten : bool (default: False)
        Whether to scrub existing indentation.  If True, all lines will be
        aligned to the same indentation.  If False, existing indentation will
        be strictly increased.

    Returns
    -------

    str|unicode : string indented by ntabs and nspaces.

    """
    if instr is None:
        return
    ind = '\t'*ntabs+' '*nspaces
    if flatten:
        pat = re.compile(r'^\s*', re.MULTILINE)
    else:
        pat = re.compile(r'^', re.MULTILINE)
    outstr = re.sub(pat, ind, instr)
    if outstr.endswith(os.linesep+ind):
        return outstr[:-len(ind)]
    else:
        return outstr

def native_line_ends(filename,backup=1):
    """Convert (in-place) a file to line-ends native to the current OS.

    If the optional backup argument is given as false, no backup of the
    original file is left.  """

    backup_suffixes = {'posix':'~','dos':'.bak','nt':'.bak','mac':'.bak'}

    bak_filename = filename + backup_suffixes[os.name]

    original = open(filename).read()
    shutil.copy2(filename,bak_filename)
    try:
        new = open(filename,'wb')
        new.write(os.linesep.join(original.splitlines()))
        new.write(os.linesep) # ALWAYS put an eol at the end of the file
        new.close()
    except:
        os.rename(bak_filename,filename)
    if not backup:
        try:
            os.remove(bak_filename)
        except:
            pass

def list_strings(arg):
    """Always return a list of strings, given a string or list of strings
    as input.

    :Examples:

        In [7]: list_strings('A single string')
        Out[7]: ['A single string']

        In [8]: list_strings(['A single string in a list'])
        Out[8]: ['A single string in a list']

        In [9]: list_strings(['A','list','of','strings'])
        Out[9]: ['A', 'list', 'of', 'strings']
    """

    if isinstance(arg,basestring): return [arg]
    else: return arg

def marquee(txt='',width=78,mark='*'):
    """Return the input string centered in a 'marquee'.

    :Examples:

        In [16]: marquee('A test',40)
        Out[16]: '**************** A test ****************'

        In [17]: marquee('A test',40,'-')
        Out[17]: '---------------- A test ----------------'

        In [18]: marquee('A test',40,' ')
        Out[18]: '                 A test                 '

    """
    if not txt:
        return (mark*width)[:width]
    nmark = (width-len(txt)-2)//len(mark)//2
    if nmark < 0: nmark =0
    marks = mark*nmark
    return '%s %s %s' % (marks,txt,marks)

ini_spaces_re = re.compile(r'^(\s+)')

def num_ini_spaces(strng):
    """Return the number of initial spaces in a string"""

    ini_spaces = ini_spaces_re.match(strng)
    if ini_spaces:
        return ini_spaces.end()
    else:
        return 0

def format_screen(strng):
    """Format a string for screen printing.

    This removes some latex-type format codes."""
    # Paragraph continue
    par_re = re.compile(r'\\$',re.MULTILINE)
    strng = par_re.sub('',strng)
    return strng

def dedent(text):
    """Equivalent of textwrap.dedent that ignores unindented first line.

    This means it will still dedent strings like:
    '''foo
    is a bar
    '''

    For use in wrap_paragraphs.
    """

    if text.startswith('\n'):
        # text starts with blank line, don't ignore the first line
        return textwrap.dedent(text)

    # split first line
    splits = text.split('\n',1)
    if len(splits) == 1:
        # only one line
        return textwrap.dedent(text)

    first, rest = splits
    # dedent everything but the first line
    rest = textwrap.dedent(rest)
    return '\n'.join([first, rest])

def wrap_paragraphs(text, ncols=80):
    """Wrap multiple paragraphs to fit a specified width.

    This is equivalent to textwrap.wrap, but with support for multiple
    paragraphs, as separated by empty lines.

    Returns
    -------

    list of complete paragraphs, wrapped to fill `ncols` columns.
    """
    paragraph_re = re.compile(r'\n(\s*\n)+', re.MULTILINE)
    text = dedent(text).strip()
    paragraphs = paragraph_re.split(text)[::2] # every other entry is space
    out_ps = []
    indent_re = re.compile(r'\n\s+', re.MULTILINE)
    for p in paragraphs:
        # presume indentation that survives dedent is meaningful formatting,
        # so don't fill unless text is flush.
        if indent_re.search(p) is None:
            # wrap paragraph
            p = textwrap.fill(p, ncols)
        out_ps.append(p)
    return out_ps

def long_substr(data):
    """Return the longest common substring in a list of strings.

    Credit: http://stackoverflow.com/questions/2892931/longest-common-substring-from-more-than-two-strings-python
    """
    substr = ''
    if len(data) > 1 and len(data[0]) > 0:
        for i in range(len(data[0])):
            for j in range(len(data[0])-i+1):
                if j > len(substr) and all(data[0][i:i+j] in x for x in data):
                    substr = data[0][i:i+j]
    elif len(data) == 1:
        substr = data[0]
    return substr

def strip_email_quotes(text):
    """Strip leading email quotation characters ('>').

    Removes any combination of leading '>' interspersed with whitespace that
    appears *identically* in all lines of the input text.

    Parameters
    ----------
    text : str

    Examples
    --------

    Simple uses::

        In [2]: strip_email_quotes('> > text')
        Out[2]: 'text'

        In [3]: strip_email_quotes('> > text\\n> > more')
        Out[3]: 'text\\nmore'

    Note how only the common prefix that appears in all lines is stripped::

        In [4]: strip_email_quotes('> > text\\n> > more\\n> more...')
        Out[4]: '> text\\n> more\\nmore...'

    So if any line has no quote marks ('>') , then none are stripped from any
    of them ::

        In [5]: strip_email_quotes('> > text\\n> > more\\nlast different')
        Out[5]: '> > text\\n> > more\\nlast different'
    """
    lines = text.splitlines()
    matches = set()
    for line in lines:
        prefix = re.match(r'^(\s*>[ >]*)', line)
        if prefix:
            matches.add(prefix.group(1))
        else:
            break
    else:
        prefix = long_substr(list(matches))
        if prefix:
            strip = len(prefix)
            text = '\n'.join([ ln[strip:] for ln in lines])
    return text

class EvalFormatter(Formatter):
    """A String Formatter that allows evaluation of simple expressions.

    Note that this version interprets a : as specifying a format string (as per
    standard string formatting), so if slicing is required, you must explicitly
    create a slice.

    This is to be used in templating cases, such as the parallel batch
    script templates, where simple arithmetic on arguments is useful.

    Examples
    --------

    In  [1]: f = EvalFormatter()
    In  [2]: f.format('{n//4}', n=8)
    Out [2]: '2'

    In  [3]: f.format("{greeting[slice(2,4)]}", greeting="Hello")
    Out [3]: 'll'
    """
    def get_field(self, name, args, kwargs):
        v = eval(name, kwargs)
        return v, name

@skip_doctest_py3
class FullEvalFormatter(Formatter):
    """A String Formatter that allows evaluation of simple expressions.

    Any time a format key is not found in the kwargs,
    it will be tried as an expression in the kwargs namespace.

    Note that this version allows slicing using [1:2], so you cannot specify
    a format string. Use :class:`EvalFormatter` to permit format strings.

    Examples
    --------

    In [1]: f = FullEvalFormatter()
    In [2]: f.format('{n//4}', n=8)
    Out[2]: u'2'

    In [3]: f.format('{list(range(5))[2:4]}')
    Out[3]: u'[2, 3]'

    In [4]: f.format('{3*2}')
    Out[4]: u'6'
    """
    # copied from Formatter._vformat with minor changes to allow eval
    # and replace the format_spec code with slicing
    def _vformat(self, format_string, args, kwargs, used_args, recursion_depth):
        if recursion_depth < 0:
            raise ValueError('Max string recursion exceeded')
        result = []
        for literal_text, field_name, format_spec, conversion in \
                self.parse(format_string):

            # output the literal text
            if literal_text:
                result.append(literal_text)

            # if there's a field, output it
            if field_name is not None:
                # this is some markup, find the object and do
                # the formatting

                if format_spec:
                    # override format spec, to allow slicing:
                    field_name = ':'.join([field_name, format_spec])

                # eval the contents of the field for the object
                # to be formatted
                obj = eval(field_name, kwargs)

                # do any conversion on the resulting object
                obj = self.convert_field(obj, conversion)

                # format the object and append to the result
                result.append(self.format_field(obj, ''))

        return u''.join(py3compat.cast_unicode(s) for s in result)

@skip_doctest_py3
class DollarFormatter(FullEvalFormatter):
    """Formatter allowing Itpl style $foo replacement, for names and attribute
    access only. Standard {foo} replacement also works, and allows full
    evaluation of its arguments.

    Examples
    --------
    In [1]: f = DollarFormatter()
    In [2]: f.format('{n//4}', n=8)
    Out[2]: u'2'

    In [3]: f.format('23 * 76 is $result', result=23*76)
    Out[3]: u'23 * 76 is 1748'

    In [4]: f.format('$a or {b}', a=1, b=2)
    Out[4]: u'1 or 2'
    """
    _dollar_pattern = re.compile("(.*?)\$(\$?[\w\.]+)")
    def parse(self, fmt_string):
        for literal_txt, field_name, format_spec, conversion \
                    in Formatter.parse(self, fmt_string):

            # Find $foo patterns in the literal text.
            continue_from = 0
            txt = ""
            for m in self._dollar_pattern.finditer(literal_txt):
                new_txt, new_field = m.group(1,2)
                # $$foo --> $foo
                if new_field.startswith("$"):
                    txt += new_txt + new_field
                else:
                    yield (txt + new_txt, new_field, "", None)
                    txt = ""
                continue_from = m.end()

            # Re-yield the {foo} style pattern
            yield (txt + literal_txt[continue_from:], field_name, format_spec, conversion)

#-----------------------------------------------------------------------------
# Utils to columnize a list of string
#-----------------------------------------------------------------------------

def _chunks(l, n):
    """Yield successive n-sized chunks from l."""
    for i in xrange(0, len(l), n):
        yield l[i:i+n]

def _find_optimal(rlist , separator_size=2 , displaywidth=80):
    """Calculate optimal info to columnize a list of string"""
    for nrow in range(1, len(rlist)+1) :
        chk = map(max,_chunks(rlist, nrow))
        sumlength = sum(chk)
        ncols = len(chk)
        if sumlength+separator_size*(ncols-1) <= displaywidth :
            break;
    return {'columns_numbers' : ncols,
            'optimal_separator_width':(displaywidth - sumlength)/(ncols-1) if (ncols -1) else 0,
            'rows_numbers' : nrow,
            'columns_width' : chk
           }

def _get_or_default(mylist, i, default=None):
    """return list item number, or default if don't exist"""
    if i >= len(mylist):
        return default
    else :
        return mylist[i]

@skip_doctest
def compute_item_matrix(items, empty=None, *args, **kwargs) :
    """Returns a nested list, and info to columnize items

    Parameters :
    ------------

    items :
        list of strings to columize
    empty : (default None)
        default value to fill list if needed
    separator_size : int (default=2)
        How much caracters will be used as a separation between each columns.
    displaywidth : int (default=80)
        The width of the area onto wich the columns should enter

    Returns :
    ---------

    Returns a tuple of (strings_matrix, dict_info)

    strings_matrix :

        nested list of string, the outer most list contains as many list as
        rows, the innermost lists have each as many element as colums. If the
        total number of elements in `items` does not equal the product of
        rows*columns, the last element of some lists are filled with `None`.

    dict_info :
        some info to make columnize easier:

        columns_numbers : number of columns
        rows_numbers    : number of rows
        columns_width   : list of with of each columns
        optimal_separator_width : best separator width between columns

    Exemple :
    ---------

    In [1]: l = ['aaa','b','cc','d','eeeee','f','g','h','i','j','k','l']
       ...: compute_item_matrix(l,displaywidth=12)
    Out[1]:
        ([['aaa', 'f', 'k'],
        ['b', 'g', 'l'],
        ['cc', 'h', None],
        ['d', 'i', None],
        ['eeeee', 'j', None]],
        {'columns_numbers': 3,
        'columns_width': [5, 1, 1],
        'optimal_separator_width': 2,
        'rows_numbers': 5})

    """
    info = _find_optimal(map(len, items), *args, **kwargs)
    nrow, ncol = info['rows_numbers'], info['columns_numbers']
    return ([[ _get_or_default(items, c*nrow+i, default=empty) for c in range(ncol) ] for i in range(nrow) ], info)

def columnize(items, separator='  ', displaywidth=80):
    """ Transform a list of strings into a single string with columns.

    Parameters
    ----------
    items : sequence of strings
        The strings to process.

    separator : str, optional [default is two spaces]
        The string that separates columns.

    displaywidth : int, optional [default is 80]
        Width of the display in number of characters.

    Returns
    -------
    The formatted string.
    """
    if not items :
        return '\n'
    matrix, info = compute_item_matrix(items, separator_size=len(separator), displaywidth=displaywidth)
    fmatrix = [filter(None, x) for x in matrix]
    sjoin = lambda x : separator.join([ y.ljust(w, ' ') for y, w in zip(x, info['columns_width'])])
    return '\n'.join(map(sjoin, fmatrix))+'\n'


"""
Tools to open .py files as Unicode, using the encoding specified within the file,
as per PEP 263.

Much of the code is taken from the tokenize module in Python 3.2.
"""
from __future__ import absolute_import

import io
from io import TextIOWrapper
import re
import urllib

cookie_re = re.compile(ur"coding[:=]\s*([-\w.]+)", re.UNICODE)
cookie_comment_re = re.compile(ur"^\s*#.*coding[:=]\s*([-\w.]+)", re.UNICODE)

try:
    # Available in Python 3
    from tokenize import detect_encoding
except ImportError:
    from codecs import lookup, BOM_UTF8

    # Copied from Python 3.2 tokenize
    def _get_normal_name(orig_enc):
        """Imitates get_normal_name in tokenizer.c."""
        # Only care about the first 12 characters.
        enc = orig_enc[:12].lower().replace("_", "-")
        if enc == "utf-8" or enc.startswith("utf-8-"):
            return "utf-8"
        if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \
           enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):
            return "iso-8859-1"
        return orig_enc

    # Copied from Python 3.2 tokenize
    def detect_encoding(readline):
        """
        The detect_encoding() function is used to detect the encoding that should
        be used to decode a Python source file.  It requires one argment, readline,
        in the same way as the tokenize() generator.

        It will call readline a maximum of twice, and return the encoding used
        (as a string) and a list of any lines (left as bytes) it has read in.

        It detects the encoding from the presence of a utf-8 bom or an encoding
        cookie as specified in pep-0263.  If both a bom and a cookie are present,
        but disagree, a SyntaxError will be raised.  If the encoding cookie is an
        invalid charset, raise a SyntaxError.  Note that if a utf-8 bom is found,
        'utf-8-sig' is returned.

        If no encoding is specified, then the default of 'utf-8' will be returned.
        """
        bom_found = False
        encoding = None
        default = 'utf-8'
        def read_or_stop():
            try:
                return readline()
            except StopIteration:
                return b''

        def find_cookie(line):
            try:
                line_string = line.decode('ascii')
            except UnicodeDecodeError:
                return None

            matches = cookie_re.findall(line_string)
            if not matches:
                return None
            encoding = _get_normal_name(matches[0])
            try:
                codec = lookup(encoding)
            except LookupError:
                # This behaviour mimics the Python interpreter
                raise SyntaxError("unknown encoding: " + encoding)

            if bom_found:
                if codec.name != 'utf-8':
                    # This behaviour mimics the Python interpreter
                    raise SyntaxError('encoding problem: utf-8')
                encoding += '-sig'
            return encoding

        first = read_or_stop()
        if first.startswith(BOM_UTF8):
            bom_found = True
            first = first[3:]
            default = 'utf-8-sig'
        if not first:
            return default, []

        encoding = find_cookie(first)
        if encoding:
            return encoding, [first]

        second = read_or_stop()
        if not second:
            return default, [first]

        encoding = find_cookie(second)
        if encoding:
            return encoding, [first, second]

        return default, [first, second]

try:
    # Available in Python 3.2 and above.
    from tokenize import open
except ImportError:
    # Copied from Python 3.2 tokenize
    def open(filename):
        """Open a file in read only mode using the encoding detected by
        detect_encoding().
        """
        buffer = io.open(filename, 'rb')   # Tweaked to use io.open for Python 2
        encoding, lines = detect_encoding(buffer.readline)
        buffer.seek(0)
        text = TextIOWrapper(buffer, encoding, line_buffering=True)
        text.mode = 'r'
        return text

def strip_encoding_cookie(filelike):
    """Generator to pull lines from a text-mode file, skipping the encoding
    cookie if it is found in the first two lines.
    """
    it = iter(filelike)
    try:
        first = next(it)
        if not cookie_comment_re.match(first):
            yield first
        second = next(it)
        if not cookie_comment_re.match(second):
            yield second
    except StopIteration:
        return

    for line in it:
        yield line

def read_py_file(filename, skip_encoding_cookie=True):
    """Read a Python file, using the encoding declared inside the file.

    Parameters
    ----------
    filename : str
      The path to the file to read.
    skip_encoding_cookie : bool
      If True (the default), and the encoding declaration is found in the first
      two lines, that line will be excluded from the output - compiling a
      unicode string with an encoding declaration is a SyntaxError in Python 2.

    Returns
    -------
    A unicode string containing the contents of the file.
    """
    with open(filename) as f:   # the open function defined in this module.
        if skip_encoding_cookie:
            return "".join(strip_encoding_cookie(f))
        else:
            return f.read()

def read_py_url(url, errors='replace', skip_encoding_cookie=True):
    """Read a Python file from a URL, using the encoding declared inside the file.

    Parameters
    ----------
    url : str
      The URL from which to fetch the file.
    errors : str
      How to handle decoding errors in the file. Options are the same as for
      bytes.decode(), but here 'replace' is the default.
    skip_encoding_cookie : bool
      If True (the default), and the encoding declaration is found in the first
      two lines, that line will be excluded from the output - compiling a
      unicode string with an encoding declaration is a SyntaxError in Python 2.

    Returns
    -------
    A unicode string containing the contents of the file.
    """
    response = urllib.urlopen(url)
    buffer = io.BytesIO(response.read())
    encoding, lines = detect_encoding(buffer.readline)
    buffer.seek(0)
    text = TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True)
    text.mode = 'r'
    if skip_encoding_cookie:
        return "".join(strip_encoding_cookie(text))
    else:
        return text.read()


""" A minimal application base mixin for all ZMQ based IPython frontends.

This is not a complete console app, as subprocess will not be able to receive
input, there is no real readline support, among other limitations. This is a
refactoring of what used to be the IPython/frontend/qt/console/qtconsoleapp.py

Authors:

* Evan Patterson
* Min RK
* Erik Tollerud
* Fernando Perez
* Bussonnier Matthias
* Thomas Kluyver
* Paul Ivanov

"""

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

# stdlib imports
import atexit
import json
import os
import signal
import sys
import uuid

# Local imports
from IPython.config.application import boolean_flag
from IPython.config.configurable import Configurable
from IPython.core.profiledir import ProfileDir
from IPython.lib.kernel import tunnel_to_kernel, find_connection_file, swallow_argv
from IPython.zmq.blockingkernelmanager import BlockingKernelManager
from IPython.utils.path import filefind
from IPython.utils.py3compat import str_to_bytes
from IPython.utils.traitlets import (
    Dict, List, Unicode, CUnicode, Int, CBool, Any
)
from IPython.zmq.ipkernel import (
    flags as ipkernel_flags,
    aliases as ipkernel_aliases,
    IPKernelApp
)
from IPython.zmq.session import Session, default_secure
from IPython.zmq.zmqshell import ZMQInteractiveShell

#-----------------------------------------------------------------------------
# Network Constants
#-----------------------------------------------------------------------------

from IPython.utils.localinterfaces import LOCALHOST, LOCAL_IPS

#-----------------------------------------------------------------------------
# Globals
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Aliases and Flags
#-----------------------------------------------------------------------------

flags = dict(ipkernel_flags)

# the flags that are specific to the frontend
# these must be scrubbed before being passed to the kernel,
# or it will raise an error on unrecognized flags
app_flags = {
    'existing' : ({'IPythonConsoleApp' : {'existing' : 'kernel*.json'}},
            "Connect to an existing kernel. If no argument specified, guess most recent"),
}
app_flags.update(boolean_flag(
    'confirm-exit', 'IPythonConsoleApp.confirm_exit',
    """Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
       to force a direct exit without any confirmation.
    """,
    """Don't prompt the user when exiting. This will terminate the kernel
       if it is owned by the frontend, and leave it alive if it is external.
    """
))
flags.update(app_flags)

aliases = dict(ipkernel_aliases)

# also scrub aliases from the frontend
app_aliases = dict(
    hb = 'IPythonConsoleApp.hb_port',
    shell = 'IPythonConsoleApp.shell_port',
    iopub = 'IPythonConsoleApp.iopub_port',
    stdin = 'IPythonConsoleApp.stdin_port',
    ip = 'IPythonConsoleApp.ip',
    existing = 'IPythonConsoleApp.existing',
    f = 'IPythonConsoleApp.connection_file',

    ssh = 'IPythonConsoleApp.sshserver',
)
aliases.update(app_aliases)

#-----------------------------------------------------------------------------
# Classes
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# IPythonConsole
#-----------------------------------------------------------------------------

classes = [IPKernelApp, ZMQInteractiveShell, ProfileDir, Session]

try:
    from IPython.zmq.pylab.backend_inline import InlineBackend
except ImportError:
    pass
else:
    classes.append(InlineBackend)

class IPythonConsoleApp(Configurable):
    name = 'ipython-console-mixin'
    default_config_file_name='ipython_config.py'

    description = """
        The IPython Mixin Console.

        This class contains the common portions of console client (QtConsole,
        ZMQ-based terminal console, etc).  It is not a full console, in that
        launched terminal subprocesses will not be able to accept input.

        The Console using this mixing supports various extra features beyond
        the single-process Terminal IPython shell, such as connecting to
        existing kernel, via:

            ipython <appname> --existing

        as well as tunnel via SSH

    """

    classes = classes
    flags = Dict(flags)
    aliases = Dict(aliases)
    kernel_manager_class = BlockingKernelManager

    kernel_argv = List(Unicode)
    # frontend flags&aliases to be stripped when building kernel_argv
    frontend_flags = Any(app_flags)
    frontend_aliases = Any(app_aliases)

    # create requested profiles by default, if they don't exist:
    auto_create = CBool(True)
    # connection info:
    ip = Unicode(LOCALHOST, config=True,
        help="""Set the kernel\'s IP address [default localhost].
        If the IP address is something other than localhost, then
        Consoles on other machines will be able to connect
        to the Kernel, so be careful!"""
    )

    sshserver = Unicode('', config=True,
        help="""The SSH server to use to connect to the kernel.""")
    sshkey = Unicode('', config=True,
        help="""Path to the ssh key to use for logging in to the ssh server.""")

    hb_port = Int(0, config=True,
        help="set the heartbeat port [default: random]")
    shell_port = Int(0, config=True,
        help="set the shell (ROUTER) port [default: random]")
    iopub_port = Int(0, config=True,
        help="set the iopub (PUB) port [default: random]")
    stdin_port = Int(0, config=True,
        help="set the stdin (DEALER) port [default: random]")
    connection_file = Unicode('', config=True,
        help="""JSON file in which to store connection info [default: kernel-<pid>.json]

        This file will contain the IP, ports, and authentication key needed to connect
        clients to this kernel. By default, this file will be created in the security-dir
        of the current profile, but can be specified by absolute path.
        """)
    def _connection_file_default(self):
        return 'kernel-%i.json' % os.getpid()

    existing = CUnicode('', config=True,
        help="""Connect to an already running kernel""")

    confirm_exit = CBool(True, config=True,
        help="""
        Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
        to force a direct exit without any confirmation.""",
    )

    def build_kernel_argv(self, argv=None):
        """build argv to be passed to kernel subprocess"""
        if argv is None:
            argv = sys.argv[1:]
        self.kernel_argv = swallow_argv(argv, self.frontend_aliases, self.frontend_flags)
        # kernel should inherit default config file from frontend
        self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name)

    def init_connection_file(self):
        """find the connection file, and load the info if found.

        The current working directory and the current profile's security
        directory will be searched for the file if it is not given by
        absolute path.

        When attempting to connect to an existing kernel and the `--existing`
        argument does not match an existing file, it will be interpreted as a
        fileglob, and the matching file in the current profile's security dir
        with the latest access time will be used.

        After this method is called, self.connection_file contains the *full path*
        to the connection file, never just its name.
        """
        if self.existing:
            try:
                cf = find_connection_file(self.existing)
            except Exception:
                self.log.critical("Could not find existing kernel connection file %s", self.existing)
                self.exit(1)
            self.log.info("Connecting to existing kernel: %s" % cf)
            self.connection_file = cf
        else:
            # not existing, check if we are going to write the file
            # and ensure that self.connection_file is a full path, not just the shortname
            try:
                cf = find_connection_file(self.connection_file)
            except Exception:
                # file might not exist
                if self.connection_file == os.path.basename(self.connection_file):
                    # just shortname, put it in security dir
                    cf = os.path.join(self.profile_dir.security_dir, self.connection_file)
                else:
                    cf = self.connection_file
                self.connection_file = cf

        # should load_connection_file only be used for existing?
        # as it is now, this allows reusing ports if an existing
        # file is requested
        try:
            self.load_connection_file()
        except Exception:
            self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True)
            self.exit(1)

    def load_connection_file(self):
        """load ip/port/hmac config from JSON connection file"""
        # this is identical to KernelApp.load_connection_file
        # perhaps it can be centralized somewhere?
        try:
            fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir])
        except IOError:
            self.log.debug("Connection File not found: %s", self.connection_file)
            return
        self.log.debug(u"Loading connection file %s", fname)
        with open(fname) as f:
            s = f.read()
        cfg = json.loads(s)
        if self.ip == LOCALHOST and 'ip' in cfg:
            # not overridden by config or cl_args
            self.ip = cfg['ip']
        for channel in ('hb', 'shell', 'iopub', 'stdin'):
            name = channel + '_port'
            if getattr(self, name) == 0 and name in cfg:
                # not overridden by config or cl_args
                setattr(self, name, cfg[name])
        if 'key' in cfg:
            self.config.Session.key = str_to_bytes(cfg['key'])

    def init_ssh(self):
        """set up ssh tunnels, if needed."""
        if not self.sshserver and not self.sshkey:
            return

        if self.sshkey and not self.sshserver:
            # specifying just the key implies that we are connecting directly
            self.sshserver = self.ip
            self.ip = LOCALHOST

        # build connection dict for tunnels:
        info = dict(ip=self.ip,
                    shell_port=self.shell_port,
                    iopub_port=self.iopub_port,
                    stdin_port=self.stdin_port,
                    hb_port=self.hb_port
        )

        self.log.info("Forwarding connections to %s via %s"%(self.ip, self.sshserver))

        # tunnels return a new set of ports, which will be on localhost:
        self.ip = LOCALHOST
        try:
            newports = tunnel_to_kernel(info, self.sshserver, self.sshkey)
        except:
            # even catch KeyboardInterrupt
            self.log.error("Could not setup tunnels", exc_info=True)
            self.exit(1)

        self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = newports

        cf = self.connection_file
        base,ext = os.path.splitext(cf)
        base = os.path.basename(base)
        self.connection_file = os.path.basename(base)+'-ssh'+ext
        self.log.critical("To connect another client via this tunnel, use:")
        self.log.critical("--existing %s" % self.connection_file)

    def _new_connection_file(self):
        cf = ''
        while not cf:
            # we don't need a 128b id to distinguish kernels, use more readable
            # 48b node segment (12 hex chars).  Users running more than 32k simultaneous
            # kernels can subclass.
            ident = str(uuid.uuid4()).split('-')[-1]
            cf = os.path.join(self.profile_dir.security_dir, 'kernel-%s.json' % ident)
            # only keep if it's actually new.  Protect against unlikely collision
            # in 48b random search space
            cf = cf if not os.path.exists(cf) else ''
        return cf

    def init_kernel_manager(self):
        # Don't let Qt or ZMQ swallow KeyboardInterupts.
        signal.signal(signal.SIGINT, signal.SIG_DFL)

        # Create a KernelManager and start a kernel.
        self.kernel_manager = self.kernel_manager_class(
                                ip=self.ip,
                                shell_port=self.shell_port,
                                iopub_port=self.iopub_port,
                                stdin_port=self.stdin_port,
                                hb_port=self.hb_port,
                                connection_file=self.connection_file,
                                config=self.config,
        )
        # start the kernel
        if not self.existing:
            self.kernel_manager.start_kernel(extra_arguments=self.kernel_argv)
        elif self.sshserver:
            # ssh, write new connection file
            self.kernel_manager.write_connection_file()
        atexit.register(self.kernel_manager.cleanup_connection_file)
        self.kernel_manager.start_channels()

    def initialize(self, argv=None):
        """
        Classes which mix this class in should call:
               IPythonConsoleApp.initialize(self,argv)
        """
        self.init_connection_file()
        default_secure(self.config)
        self.init_ssh()
        self.init_kernel_manager()


#!/usr/bin/python3
import os
import subprocess
import iot_message.abstract.message_interface as message_interface
import json

__author__ = 'Bartosz Kościów'

class Message(message_interface.Message):
    """Class Message"""
    protocol = "iot:1"

    def __init__(self, node, chip_id=None):
        self._node = node
        if chip_id is None:
            self._chip_id = self._get_id()
        else:
            self._chip_id = chip_id

    @property
    def node(self):
        """
        Return node name
        :return string
        """
        return self._node

    @property
    def chip_id(self):
        """
        Return chip id
        :return string
        """
        return self._chip_id

    def _get_id(self):
        """:return string"""
        if 'nt' in os.name:
            return subprocess.getoutput('wmic csproduct get uuid')
        else:
            return subprocess.getoutput('cat /var/lib/dbus/machine-id')

    def prepare_message(self, data=None):
        """
        Return message as dict
        :return dict
        """
        message = {
            'protocol': self.protocol,
            'node': self._node,
            'chip_id': self._chip_id,
            'event': '',
            'parameters': {},
            'response': '',
            'targets': [
                'ALL'
            ]
        }
        if type(data) is dict:
            for k, v in data.items():
                if k in message:
                    message[k] = v

        return message

    def decode_message(self, message):
        """
        Decode json string to dict. Validate against node name(targets) and protocol version
        :return dict | None
        """
        try:
            message = json.loads(message)
            if not self._validate_message(message):
                message = None
        except ValueError:
            message = None

        return message

    def _validate_message(self, message):
        """:return boolean"""
        if 'protocol' not in message or 'targets' not in message or \
                type(message['targets']) is not list:
            return False

        if message['protocol'] != self.protocol:
            return False

        if self.node not in message['targets'] and 'ALL' not in message['targets']:
            return False

        return True


# -*- coding: utf-8 -*-
"""
    pretty
    ~~

    Python advanced pretty printer.  This pretty printer is intended to
    replace the old `pprint` python module which does not allow developers
    to provide their own pretty print callbacks.

    This module is based on ruby's `prettyprint.rb` library by `Tanaka Akira`.

    Example Usage
    =============

    To directly print the representation of an object use `pprint`::

        from pretty import pprint
        pprint(complex_object)

    To get a string of the output use `pretty`::

        from pretty import pretty
        string = pretty(complex_object)

    Extending
    =========

    The pretty library allows developers to add pretty printing rules for their
    own objects.  This process is straightforward.  All you have to do is to
    add a `_repr_pretty_` method to your object and call the methods on the
    pretty printer passed::

        class MyObject(object):

            def _repr_pretty_(self, p, cycle):
                ...

    Depending on the python version you want to support you have two
    possibilities.  The following list shows the python 2.5 version and the
    compatibility one.

    Here the example implementation of a `_repr_pretty_` method for a list
    subclass for python 2.5 and higher (python 2.5 requires the with statement
    __future__ import)::

        class MyList(list):

            def _repr_pretty_(self, p, cycle):
                if cycle:
                    p.text('MyList(...)')
                else:
                    with p.group(8, 'MyList([', '])'):
                        for idx, item in enumerate(self):
                            if idx:
                                p.text(',')
                                p.breakable()
                            p.pretty(item)

    The `cycle` parameter is `True` if pretty detected a cycle.  You *have* to
    react to that or the result is an infinite loop.  `p.text()` just adds
    non breaking text to the output, `p.breakable()` either adds a whitespace
    or breaks here.  If you pass it an argument it's used instead of the
    default space.  `p.pretty` prettyprints another object using the pretty print
    method.

    The first parameter to the `group` function specifies the extra indentation
    of the next line.  In this example the next item will either be not
    breaked (if the items are short enough) or aligned with the right edge of
    the opening bracked of `MyList`.

    If you want to support python 2.4 and lower you can use this code::

        class MyList(list):

            def _repr_pretty_(self, p, cycle):
                if cycle:
                    p.text('MyList(...)')
                else:
                    p.begin_group(8, 'MyList([')
                    for idx, item in enumerate(self):
                        if idx:
                            p.text(',')
                            p.breakable()
                        p.pretty(item)
                    p.end_group(8, '])')

    If you just want to indent something you can use the group function
    without open / close parameters.  Under python 2.5 you can also use this
    code::

        with p.indent(2):
            ...

    Or under python2.4 you might want to modify ``p.indentation`` by hand but
    this is rather ugly.

    :copyright: 2007 by Armin Ronacher.
                Portions (c) 2009 by Robert Kern.
    :license: BSD License.
"""
from __future__ import with_statement
from contextlib import contextmanager
import sys
import types
import re
import datetime
from StringIO import StringIO
from collections import deque

__all__ = ['pretty', 'pprint', 'PrettyPrinter', 'RepresentationPrinter',
    'for_type', 'for_type_by_name']

_re_pattern_type = type(re.compile(''))

def pretty(obj, verbose=False, max_width=79, newline='\n'):
    """
    Pretty print the object's representation.
    """
    stream = StringIO()
    printer = RepresentationPrinter(stream, verbose, max_width, newline)
    printer.pretty(obj)
    printer.flush()
    return stream.getvalue()

def pprint(obj, verbose=False, max_width=79, newline='\n'):
    """
    Like `pretty` but print to stdout.
    """
    printer = RepresentationPrinter(sys.stdout, verbose, max_width, newline)
    printer.pretty(obj)
    printer.flush()
    sys.stdout.write(newline)
    sys.stdout.flush()

class _PrettyPrinterBase(object):

    @contextmanager
    def indent(self, indent):
        """with statement support for indenting/dedenting."""
        self.indentation += indent
        try:
            yield
        finally:
            self.indentation -= indent

    @contextmanager
    def group(self, indent=0, open='', close=''):
        """like begin_group / end_group but for the with statement."""
        self.begin_group(indent, open)
        try:
            yield
        finally:
            self.end_group(indent, close)

class PrettyPrinter(_PrettyPrinterBase):
    """
    Baseclass for the `RepresentationPrinter` prettyprinter that is used to
    generate pretty reprs of objects.  Contrary to the `RepresentationPrinter`
    this printer knows nothing about the default pprinters or the `_repr_pretty_`
    callback method.
    """

    def __init__(self, output, max_width=79, newline='\n'):
        self.output = output
        self.max_width = max_width
        self.newline = newline
        self.output_width = 0
        self.buffer_width = 0
        self.buffer = deque()

        root_group = Group(0)
        self.group_stack = [root_group]
        self.group_queue = GroupQueue(root_group)
        self.indentation = 0

    def _break_outer_groups(self):
        while self.max_width < self.output_width + self.buffer_width:
            group = self.group_queue.deq()
            if not group:
                return
            while group.breakables:
                x = self.buffer.popleft()
                self.output_width = x.output(self.output, self.output_width)
                self.buffer_width -= x.width
            while self.buffer and isinstance(self.buffer[0], Text):
                x = self.buffer.popleft()
                self.output_width = x.output(self.output, self.output_width)
                self.buffer_width -= x.width

    def text(self, obj):
        """Add literal text to the output."""
        width = len(obj)
        if self.buffer:
            text = self.buffer[-1]
            if not isinstance(text, Text):
                text = Text()
                self.buffer.append(text)
            text.add(obj, width)
            self.buffer_width += width
            self._break_outer_groups()
        else:
            self.output.write(obj)
            self.output_width += width

    def breakable(self, sep=' '):
        """
        Add a breakable separator to the output.  This does not mean that it
        will automatically break here.  If no breaking on this position takes
        place the `sep` is inserted which default to one space.
        """
        width = len(sep)
        group = self.group_stack[-1]
        if group.want_break:
            self.flush()
            self.output.write(self.newline)
            self.output.write(' ' * self.indentation)
            self.output_width = self.indentation
            self.buffer_width = 0
        else:
            self.buffer.append(Breakable(sep, width, self))
            self.buffer_width += width
            self._break_outer_groups()

    def begin_group(self, indent=0, open=''):
        """
        Begin a group.  If you want support for python < 2.5 which doesn't has
        the with statement this is the preferred way:

            p.begin_group(1, '{')
            ...
            p.end_group(1, '}')

        The python 2.5 expression would be this:

            with p.group(1, '{', '}'):
                ...

        The first parameter specifies the indentation for the next line (usually
        the width of the opening text), the second the opening text.  All
        parameters are optional.
        """
        if open:
            self.text(open)
        group = Group(self.group_stack[-1].depth + 1)
        self.group_stack.append(group)
        self.group_queue.enq(group)
        self.indentation += indent

    def end_group(self, dedent=0, close=''):
        """End a group. See `begin_group` for more details."""
        self.indentation -= dedent
        group = self.group_stack.pop()
        if not group.breakables:
            self.group_queue.remove(group)
        if close:
            self.text(close)

    def flush(self):
        """Flush data that is left in the buffer."""
        for data in self.buffer:
            self.output_width += data.output(self.output, self.output_width)
        self.buffer.clear()
        self.buffer_width = 0

def _get_mro(obj_class):
    """ Get a reasonable method resolution order of a class and its superclasses
    for both old-style and new-style classes.
    """
    if not hasattr(obj_class, '__mro__'):
        # Old-style class. Mix in object to make a fake new-style class.
        try:
            obj_class = type(obj_class.__name__, (obj_class, object), {})
        except TypeError:
            # Old-style extension type that does not descend from object.
            # FIXME: try to construct a more thorough MRO.
            mro = [obj_class]
        else:
            mro = obj_class.__mro__[1:-1]
    else:
        mro = obj_class.__mro__
    return mro

class RepresentationPrinter(PrettyPrinter):
    """
    Special pretty printer that has a `pretty` method that calls the pretty
    printer for a python object.

    This class stores processing data on `self` so you must *never* use
    this class in a threaded environment.  Always lock it or reinstanciate
    it.

    Instances also have a verbose flag callbacks can access to control their
    output.  For example the default instance repr prints all attributes and
    methods that are not prefixed by an underscore if the printer is in
    verbose mode.
    """

    def __init__(self, output, verbose=False, max_width=79, newline='\n',
        singleton_pprinters=None, type_pprinters=None, deferred_pprinters=None):

        PrettyPrinter.__init__(self, output, max_width, newline)
        self.verbose = verbose
        self.stack = []
        if singleton_pprinters is None:
            singleton_pprinters = _singleton_pprinters.copy()
        self.singleton_pprinters = singleton_pprinters
        if type_pprinters is None:
            type_pprinters = _type_pprinters.copy()
        self.type_pprinters = type_pprinters
        if deferred_pprinters is None:
            deferred_pprinters = _deferred_type_pprinters.copy()
        self.deferred_pprinters = deferred_pprinters

    def pretty(self, obj):
        """Pretty print the given object."""
        obj_id = id(obj)
        cycle = obj_id in self.stack
        self.stack.append(obj_id)
        self.begin_group()
        try:
            obj_class = getattr(obj, '__class__', None) or type(obj)
            # First try to find registered singleton printers for the type.
            try:
                printer = self.singleton_pprinters[obj_id]
            except (TypeError, KeyError):
                pass
            else:
                return printer(obj, self, cycle)
            # Next walk the mro and check for either:
            #   1) a registered printer
            #   2) a _repr_pretty_ method
            for cls in _get_mro(obj_class):
                if cls in self.type_pprinters:
                    # printer registered in self.type_pprinters
                    return self.type_pprinters[cls](obj, self, cycle)
                else:
                    # deferred printer
                    printer = self._in_deferred_types(cls)
                    if printer is not None:
                        return printer(obj, self, cycle)
                    else:
                        # Finally look for special method names.
                        # Some objects automatically create any requested
                        # attribute. Try to ignore most of them by checking for
                        # callability.
                        if '_repr_pretty_' in obj_class.__dict__:
                            meth = obj_class._repr_pretty_
                            if callable(meth):
                                return meth(obj, self, cycle)
            return _default_pprint(obj, self, cycle)
        finally:
            self.end_group()
            self.stack.pop()

    def _in_deferred_types(self, cls):
        """
        Check if the given class is specified in the deferred type registry.

        Returns the printer from the registry if it exists, and None if the
        class is not in the registry. Successful matches will be moved to the
        regular type registry for future use.
        """
        mod = getattr(cls, '__module__', None)
        name = getattr(cls, '__name__', None)
        key = (mod, name)
        printer = None
        if key in self.deferred_pprinters:
            # Move the printer over to the regular registry.
            printer = self.deferred_pprinters.pop(key)
            self.type_pprinters[cls] = printer
        return printer

class Printable(object):

    def output(self, stream, output_width):
        return output_width

class Text(Printable):

    def __init__(self):
        self.objs = []
        self.width = 0

    def output(self, stream, output_width):
        for obj in self.objs:
            stream.write(obj)
        return output_width + self.width

    def add(self, obj, width):
        self.objs.append(obj)
        self.width += width

class Breakable(Printable):

    def __init__(self, seq, width, pretty):
        self.obj = seq
        self.width = width
        self.pretty = pretty
        self.indentation = pretty.indentation
        self.group = pretty.group_stack[-1]
        self.group.breakables.append(self)

    def output(self, stream, output_width):
        self.group.breakables.popleft()
        if self.group.want_break:
            stream.write(self.pretty.newline)
            stream.write(' ' * self.indentation)
            return self.indentation
        if not self.group.breakables:
            self.pretty.group_queue.remove(self.group)
        stream.write(self.obj)
        return output_width + self.width

class Group(Printable):

    def __init__(self, depth):
        self.depth = depth
        self.breakables = deque()
        self.want_break = False

class GroupQueue(object):

    def __init__(self, *groups):
        self.queue = []
        for group in groups:
            self.enq(group)

    def enq(self, group):
        depth = group.depth
        while depth > len(self.queue) - 1:
            self.queue.append([])
        self.queue[depth].append(group)

    def deq(self):
        for stack in self.queue:
            for idx, group in enumerate(reversed(stack)):
                if group.breakables:
                    del stack[idx]
                    group.want_break = True
                    return group
            for group in stack:
                group.want_break = True
            del stack[:]

    def remove(self, group):
        try:
            self.queue[group.depth].remove(group)
        except ValueError:
            pass

try:
    _baseclass_reprs = (object.__repr__, types.InstanceType.__repr__)
except AttributeError:  # Python 3
    _baseclass_reprs = (object.__repr__,)

def _default_pprint(obj, p, cycle):
    """
    The default print function.  Used if an object does not provide one and
    it's none of the builtin objects.
    """
    klass = getattr(obj, '__class__', None) or type(obj)
    if getattr(klass, '__repr__', None) not in _baseclass_reprs:
        # A user-provided repr.
        p.text(repr(obj))
        return
    p.begin_group(1, '<')
    p.pretty(klass)
    p.text(' at 0x%x' % id(obj))
    if cycle:
        p.text(' ...')
    elif p.verbose:
        first = True
        for key in dir(obj):
            if not key.startswith('_'):
                try:
                    value = getattr(obj, key)
                except AttributeError:
                    continue
                if isinstance(value, types.MethodType):
                    continue
                if not first:
                    p.text(',')
                p.breakable()
                p.text(key)
                p.text('=')
                step = len(key) + 1
                p.indentation += step
                p.pretty(value)
                p.indentation -= step
                first = False
    p.end_group(1, '>')

def _seq_pprinter_factory(start, end, basetype):
    """
    Factory that returns a pprint function useful for sequences.  Used by
    the default pprint for tuples, dicts, lists, sets and frozensets.
    """
    def inner(obj, p, cycle):
        typ = type(obj)
        if basetype is not None and typ is not basetype and typ.__repr__ != basetype.__repr__:
            # If the subclass provides its own repr, use it instead.
            return p.text(typ.__repr__(obj))

        if cycle:
            return p.text(start + '...' + end)
        step = len(start)
        p.begin_group(step, start)
        for idx, x in enumerate(obj):
            if idx:
                p.text(',')
                p.breakable()
            p.pretty(x)
        if len(obj) == 1 and type(obj) is tuple:
            # Special case for 1-item tuples.
            p.text(',')
        p.end_group(step, end)
    return inner

def _dict_pprinter_factory(start, end, basetype=None):
    """
    Factory that returns a pprint function used by the default pprint of
    dicts and dict proxies.
    """
    def inner(obj, p, cycle):
        typ = type(obj)
        if basetype is not None and typ is not basetype and typ.__repr__ != basetype.__repr__:
            # If the subclass provides its own repr, use it instead.
            return p.text(typ.__repr__(obj))

        if cycle:
            return p.text('{...}')
        p.begin_group(1, start)
        keys = obj.keys()
        try:
            keys.sort()
        except Exception, e:
            # Sometimes the keys don't sort.
            pass
        for idx, key in enumerate(keys):
            if idx:
                p.text(',')
                p.breakable()
            p.pretty(key)
            p.text(': ')
            p.pretty(obj[key])
        p.end_group(1, end)
    return inner

def _super_pprint(obj, p, cycle):
    """The pprint for the super type."""
    p.begin_group(8, '<super: ')
    p.pretty(obj.__self_class__)
    p.text(',')
    p.breakable()
    p.pretty(obj.__self__)
    p.end_group(8, '>')

def _re_pattern_pprint(obj, p, cycle):
    """The pprint function for regular expression patterns."""
    p.text('re.compile(')
    pattern = repr(obj.pattern)
    if pattern[:1] in 'uU':
        pattern = pattern[1:]
        prefix = 'ur'
    else:
        prefix = 'r'
    pattern = prefix + pattern.replace('\\\\', '\\')
    p.text(pattern)
    if obj.flags:
        p.text(',')
        p.breakable()
        done_one = False
        for flag in ('TEMPLATE', 'IGNORECASE', 'LOCALE', 'MULTILINE', 'DOTALL',
            'UNICODE', 'VERBOSE', 'DEBUG'):
            if obj.flags & getattr(re, flag):
                if done_one:
                    p.text('|')
                p.text('re.' + flag)
                done_one = True
    p.text(')')

def _type_pprint(obj, p, cycle):
    """The pprint for classes and types."""
    if obj.__module__ in ('__builtin__', 'exceptions'):
        name = obj.__name__
    else:
        name = obj.__module__ + '.' + obj.__name__
    p.text(name)

def _repr_pprint(obj, p, cycle):
    """A pprint that just redirects to the normal repr function."""
    p.text(repr(obj))

def _function_pprint(obj, p, cycle):
    """Base pprint for all functions and builtin functions."""
    if obj.__module__ in ('__builtin__', 'exceptions') or not obj.__module__:
        name = obj.__name__
    else:
        name = obj.__module__ + '.' + obj.__name__
    p.text('<function %s>' % name)

def _exception_pprint(obj, p, cycle):
    """Base pprint for all exceptions."""
    if obj.__class__.__module__ in ('exceptions', 'builtins'):
        name = obj.__class__.__name__
    else:
        name = '%s.%s' % (
            obj.__class__.__module__,
            obj.__class__.__name__
        )
    step = len(name) + 1
    p.begin_group(step, name + '(')
    for idx, arg in enumerate(getattr(obj, 'args', ())):
        if idx:
            p.text(',')
            p.breakable()
        p.pretty(arg)
    p.end_group(step, ')')

#: the exception base
try:
    _exception_base = BaseException
except NameError:
    _exception_base = Exception

#: printers for builtin types
_type_pprinters = {
    int:                        _repr_pprint,
    long:                       _repr_pprint,
    float:                      _repr_pprint,
    str:                        _repr_pprint,
    unicode:                    _repr_pprint,
    tuple:                      _seq_pprinter_factory('(', ')', tuple),
    list:                       _seq_pprinter_factory('[', ']', list),
    dict:                       _dict_pprinter_factory('{', '}', dict),

    set:                        _seq_pprinter_factory('set([', '])', set),
    frozenset:                  _seq_pprinter_factory('frozenset([', '])', frozenset),
    super:                      _super_pprint,
    _re_pattern_type:           _re_pattern_pprint,
    type:                       _type_pprint,
    types.FunctionType:         _function_pprint,
    types.BuiltinFunctionType:  _function_pprint,
    types.SliceType:            _repr_pprint,
    types.MethodType:           _repr_pprint,

    datetime.datetime:          _repr_pprint,
    datetime.timedelta:         _repr_pprint,
    _exception_base:            _exception_pprint
}

try:
    _type_pprinters[types.DictProxyType] = _dict_pprinter_factory('<dictproxy {', '}>')
    _type_pprinters[types.ClassType] = _type_pprint
except AttributeError: # Python 3
    pass

try:
    _type_pprinters[xrange] = _repr_pprint
except NameError:
    _type_pprinters[range] = _repr_pprint

#: printers for types specified by name
_deferred_type_pprinters = {
}

def for_type(typ, func):
    """
    Add a pretty printer for a given type.
    """
    oldfunc = _type_pprinters.get(typ, None)
    if func is not None:
        # To support easy restoration of old pprinters, we need to ignore Nones.
        _type_pprinters[typ] = func
    return oldfunc

def for_type_by_name(type_module, type_name, func):
    """
    Add a pretty printer for a type specified by the module and name of a type
    rather than the type object itself.
    """
    key = (type_module, type_name)
    oldfunc = _deferred_type_pprinters.get(key, None)
    if func is not None:
        # To support easy restoration of old pprinters, we need to ignore Nones.
        _deferred_type_pprinters[key] = func
    return oldfunc

#: printers for the default singletons
_singleton_pprinters = dict.fromkeys(map(id, [None, True, False, Ellipsis,
                                      NotImplemented]), _repr_pprint)

if __name__ == '__main__':
    from random import randrange
    class Foo(object):
        def __init__(self):
            self.foo = 1
            self.bar = re.compile(r'\s+')
            self.blub = dict.fromkeys(range(30), randrange(1, 40))
            self.hehe = 23424.234234
            self.list = ["blub", "blah", self]

        def get_foo(self):
            print "foo"

    pprint(Foo(), verbose=True)