#!/usr/bin/env python

""" PickleShare - a small 'shelve' like datastore with concurrency support

Like shelve, a PickleShareDB object acts like a normal dictionary. Unlike
shelve, many processes can access the database simultaneously. Changing a
value in database is immediately visible to other processes accessing the
same database.

Concurrency is possible because the values are stored in separate files. Hence
the "database" is a directory where *all* files are governed by PickleShare.

Example usage::

    from pickleshare import *
    db = PickleShareDB('~/testpickleshare')
    db.clear()
    print "Should be empty:",db.items()
    db['hello'] = 15
    db['aku ankka'] = [1,2,313]
    db['paths/are/ok/key'] = [1,(5,46)]
    print db.keys()
    del db['aku ankka']

This module is certainly not ZODB, but can be used for low-load
(non-mission-critical) situations where tiny code size trumps the
advanced features of a "real" object database.

Installation guide: easy_install pickleshare

Author: Ville Vainio <vivainio@gmail.com>
License: MIT open source license.

"""

from IPython.external.path import path as Path
import os,stat,time
import collections
import cPickle as pickle
import glob

def gethashfile(key):
    return ("%02x" % abs(hash(key) % 256))[-2:]

_sentinel = object()

class PickleShareDB(collections.MutableMapping):
    """ The main 'connection' object for PickleShare database """
    def __init__(self,root):
        """ Return a db object that will manage the specied directory"""
        self.root = Path(root).expanduser().abspath()
        if not self.root.isdir():
            self.root.makedirs()
        # cache has { 'key' : (obj, orig_mod_time) }
        self.cache = {}

    def __getitem__(self,key):
        """ db['key'] reading """
        fil = self.root / key
        try:
            mtime = (fil.stat()[stat.ST_MTIME])
        except OSError:
            raise KeyError(key)

        if fil in self.cache and mtime == self.cache[fil][1]:
            return self.cache[fil][0]
        try:
            # The cached item has expired, need to read
            obj = pickle.loads(fil.open("rb").read())
        except:
            raise KeyError(key)

        self.cache[fil] = (obj,mtime)
        return obj

    def __setitem__(self,key,value):
        """ db['key'] = 5 """
        fil = self.root / key
        parent = fil.parent
        if parent and not parent.isdir():
            parent.makedirs()
        # We specify protocol 2, so that we can mostly go between Python 2
        # and Python 3. We can upgrade to protocol 3 when Python 2 is obsolete.
        pickled = pickle.dump(value,fil.open('wb'), protocol=2)
        try:
            self.cache[fil] = (value,fil.mtime)
        except OSError,e:
            if e.errno != 2:
                raise

    def hset(self, hashroot, key, value):
        """ hashed set """
        hroot = self.root / hashroot
        if not hroot.isdir():
            hroot.makedirs()
        hfile = hroot / gethashfile(key)
        d = self.get(hfile, {})
        d.update( {key : value})
        self[hfile] = d

    def hget(self, hashroot, key, default = _sentinel, fast_only = True):
        """ hashed get """
        hroot = self.root / hashroot
        hfile = hroot / gethashfile(key)

        d = self.get(hfile, _sentinel )
        #print "got dict",d,"from",hfile
        if d is _sentinel:
            if fast_only:
                if default is _sentinel:
                    raise KeyError(key)

                return default

            # slow mode ok, works even after hcompress()
            d = self.hdict(hashroot)

        return d.get(key, default)

    def hdict(self, hashroot):
        """ Get all data contained in hashed category 'hashroot' as dict """
        hfiles = self.keys(hashroot + "/*")
        hfiles.sort()
        last = len(hfiles) and hfiles[-1] or ''
        if last.endswith('xx'):
            # print "using xx"
            hfiles = [last] + hfiles[:-1]

        all = {}

        for f in hfiles:
            # print "using",f
            try:
                all.update(self[f])
            except KeyError:
                print "Corrupt",f,"deleted - hset is not threadsafe!"
                del self[f]

            self.uncache(f)

        return all

    def hcompress(self, hashroot):
        """ Compress category 'hashroot', so hset is fast again

        hget will fail if fast_only is True for compressed items (that were
        hset before hcompress).

        """
        hfiles = self.keys(hashroot + "/*")
        all = {}
        for f in hfiles:
            # print "using",f
            all.update(self[f])
            self.uncache(f)

        self[hashroot + '/xx'] = all
        for f in hfiles:
            p = self.root / f
            if p.basename() == 'xx':
                continue
            p.remove()

    def __delitem__(self,key):
        """ del db["key"] """
        fil = self.root / key
        self.cache.pop(fil,None)
        try:
            fil.remove()
        except OSError:
            # notfound and permission denied are ok - we
            # lost, the other process wins the conflict
            pass

    def _normalized(self, p):
        """ Make a key suitable for user's eyes """
        return str(self.root.relpathto(p)).replace('\\','/')

    def keys(self, globpat = None):
        """ All keys in DB, or all keys matching a glob"""

        if globpat is None:
            files = self.root.walkfiles()
        else:
            files = [Path(p) for p in glob.glob(self.root/globpat)]
        return [self._normalized(p) for p in files if p.isfile()]

    def __iter__(self):
        return iter(self.keys())

    def __len__(self):
        return len(self.keys())

    def uncache(self,*items):
        """ Removes all, or specified items from cache

        Use this after reading a large amount of large objects
        to free up memory, when you won't be needing the objects
        for a while.

        """
        if not items:
            self.cache = {}
        for it in items:
            self.cache.pop(it,None)

    def waitget(self,key, maxwaittime = 60 ):
        """ Wait (poll) for a key to get a value

        Will wait for `maxwaittime` seconds before raising a KeyError.
        The call exits normally if the `key` field in db gets a value
        within the timeout period.

        Use this for synchronizing different processes or for ensuring
        that an unfortunately timed "db['key'] = newvalue" operation
        in another process (which causes all 'get' operation to cause a
        KeyError for the duration of pickling) won't screw up your program
        logic.
        """

        wtimes = [0.2] * 3 + [0.5] * 2 + [1]
        tries = 0
        waited = 0
        while 1:
            try:
                val = self[key]
                return val
            except KeyError:
                pass

            if waited > maxwaittime:
                raise KeyError(key)

            time.sleep(wtimes[tries])
            waited+=wtimes[tries]
            if tries < len(wtimes) -1:
                tries+=1

    def getlink(self,folder):
        """ Get a convenient link for accessing items  """
        return PickleShareLink(self, folder)

    def __repr__(self):
        return "PickleShareDB('%s')" % self.root

class PickleShareLink:
    """ A shortdand for accessing nested PickleShare data conveniently.

    Created through PickleShareDB.getlink(), example::

        lnk = db.getlink('myobjects/test')
        lnk.foo = 2
        lnk.bar = lnk.foo + 5

    """
    def __init__(self, db, keydir ):
        self.__dict__.update(locals())

    def __getattr__(self,key):
        return self.__dict__['db'][self.__dict__['keydir']+'/' + key]
    def __setattr__(self,key,val):
        self.db[self.keydir+'/' + key] = val
    def __repr__(self):
        db = self.__dict__['db']
        keys = db.keys( self.__dict__['keydir'] +"/*")
        return "<PickleShareLink '%s': %s>" % (
            self.__dict__['keydir'],
            ";".join([Path(k).basename() for k in keys]))

def test():
    db = PickleShareDB('~/testpickleshare')
    db.clear()
    print "Should be empty:",db.items()
    db['hello'] = 15
    db['aku ankka'] = [1,2,313]
    db['paths/nest/ok/keyname'] = [1,(5,46)]
    db.hset('hash', 'aku', 12)
    db.hset('hash', 'ankka', 313)
    print "12 =",db.hget('hash','aku')
    print "313 =",db.hget('hash','ankka')
    print "all hashed",db.hdict('hash')
    print db.keys()
    print db.keys('paths/nest/ok/k*')
    print dict(db) # snapsot of whole db
    db.uncache() # frees memory, causes re-reads later

    # shorthand for accessing deeply nested files
    lnk = db.getlink('myobjects/test')
    lnk.foo = 2
    lnk.bar = lnk.foo + 5
    print lnk.bar # 7

def stress():
    db = PickleShareDB('~/fsdbtest')
    import time,sys
    for i in range(1000):
        for j in range(1000):
            if i % 15 == 0 and i < 200:
                if str(j) in db:
                    del db[str(j)]
                continue

            if j%33 == 0:
                time.sleep(0.02)

            db[str(j)] = db.get(str(j), []) + [(i,j,"proc %d" % os.getpid())]
            db.hset('hash',j, db.hget('hash',j,15) + 1 )

        print i,
        sys.stdout.flush()
        if i % 10 == 0:
            db.uncache()

def main():
    import textwrap
    usage = textwrap.dedent("""\
    pickleshare - manage PickleShare databases

    Usage:

        pickleshare dump /path/to/db > dump.txt
        pickleshare load /path/to/db < dump.txt
        pickleshare test /path/to/db
    """)
    DB = PickleShareDB
    import sys
    if len(sys.argv) < 2:
        print usage
        return

    cmd = sys.argv[1]
    args = sys.argv[2:]
    if cmd == 'dump':
        if not args: args= ['.']
        db = DB(args[0])
        import pprint
        pprint.pprint(db.items())
    elif cmd == 'load':
        cont = sys.stdin.read()
        db = DB(args[0])
        data = eval(cont)
        db.clear()
        for k,v in db.items():
            db[k] = v
    elif cmd == 'testwait':
        db = DB(args[0])
        db.clear()
        print db.waitget('250')
    elif cmd == 'test':
        test()
        stress()

if __name__== "__main__":
    main()


"""a simple completer for the qtconsole"""
#-----------------------------------------------------------------------------
# Copyright (c) 2012, IPython Development Team.$
#
# Distributed under the terms of the Modified BSD License.$
#
# The full license is in the file COPYING.txt, distributed with this software.
#-------------------------------------------------------------------

# System library imports
from IPython.external.qt import QtCore, QtGui
import IPython.utils.text as text

class CompletionPlain(QtGui.QWidget):
    """ A widget for tab completion,  navigable by arrow keys """

    #--------------------------------------------------------------------------
    # 'QObject' interface
    #--------------------------------------------------------------------------

    def __init__(self, console_widget):
        """ Create a completion widget that is attached to the specified Qt
            text edit widget.
        """
        assert isinstance(console_widget._control, (QtGui.QTextEdit, QtGui.QPlainTextEdit))
        super(CompletionPlain, self).__init__()

        self._text_edit = console_widget._control
        self._console_widget = console_widget

        self._text_edit.installEventFilter(self)

    def eventFilter(self, obj, event):
        """ Reimplemented to handle keyboard input and to auto-hide when the
            text edit loses focus.
        """
        if obj == self._text_edit:
            etype = event.type()

            if etype in( QtCore.QEvent.KeyPress, QtCore.QEvent.FocusOut ):
                self.cancel_completion()

        return super(CompletionPlain, self).eventFilter(obj, event)

    #--------------------------------------------------------------------------
    # 'CompletionPlain' interface
    #--------------------------------------------------------------------------
    def cancel_completion(self):
        """Cancel the completion, reseting internal variable, clearing buffer """
        self._console_widget._clear_temporary_buffer()

    def show_items(self, cursor, items):
        """ Shows the completion widget with 'items' at the position specified
            by 'cursor'.
        """
        if not items :
            return
        self.cancel_completion()
        strng = text.columnize(items)
        self._console_widget._fill_temporary_buffer(cursor, strng, html=False)


"""
This plugin captures logging statements issued during test execution. When an
error or failure occurs, the captured log messages are attached to the running
test in the test.capturedLogging attribute, and displayed with the error failure
output. It is enabled by default but can be turned off with the option
``--nologcapture``.

You can filter captured logging statements with the ``--logging-filter`` option.
If set, it specifies which logger(s) will be captured; loggers that do not match
will be passed. Example: specifying ``--logging-filter=sqlalchemy,myapp``
will ensure that only statements logged via sqlalchemy.engine, myapp
or myapp.foo.bar logger will be logged.

You can remove other installed logging handlers with the
``--logging-clear-handlers`` option.
"""

import logging
from logging.handlers import BufferingHandler
import threading

from nose.plugins.base import Plugin
from nose.util import anyp, ln, safe_str

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

log = logging.getLogger(__name__)

class FilterSet(object):
    def __init__(self, filter_components):
        self.inclusive, self.exclusive = self._partition(filter_components)

    # @staticmethod
    def _partition(components):
        inclusive, exclusive = [], []
        for component in components:
            if component.startswith('-'):
                exclusive.append(component[1:])
            else:
                inclusive.append(component)
        return inclusive, exclusive
    _partition = staticmethod(_partition)

    def allow(self, record):
        """returns whether this record should be printed"""
        if not self:
            # nothing to filter
            return True
        return self._allow(record) and not self._deny(record)

    # @staticmethod
    def _any_match(matchers, record):
        """return the bool of whether `record` starts with
        any item in `matchers`"""
        def record_matches_key(key):
            return record == key or record.startswith(key + '.')
        return anyp(bool, map(record_matches_key, matchers))
    _any_match = staticmethod(_any_match)

    def _allow(self, record):
        if not self.inclusive:
            return True
        return self._any_match(self.inclusive, record)

    def _deny(self, record):
        if not self.exclusive:
            return False
        return self._any_match(self.exclusive, record)

class MyMemoryHandler(BufferingHandler):
    def __init__(self, capacity, logformat, logdatefmt, filters):
        BufferingHandler.__init__(self, capacity)
        fmt = logging.Formatter(logformat, logdatefmt)
        self.setFormatter(fmt)
        self.filterset = FilterSet(filters)
    def flush(self):
        pass # do nothing
    def truncate(self):
        self.buffer = []
    def filter(self, record):
        return self.filterset.allow(record.name)
    def __getstate__(self):
        state = self.__dict__.copy()
        del state['lock']
        return state
    def __setstate__(self, state):
        self.__dict__.update(state)
        self.lock = threading.RLock()

class LogCapture(Plugin):
    """
    Log capture plugin. Enabled by default. Disable with --nologcapture.
    This plugin captures logging statements issued during test execution,
    appending any output captured to the error or failure output,
    should the test fail or raise an error.
    """
    enabled = True
    env_opt = 'NOSE_NOLOGCAPTURE'
    name = 'logcapture'
    score = 500
    logformat = '%(name)s: %(levelname)s: %(message)s'
    logdatefmt = None
    clear = False
    filters = ['-nose']

    def options(self, parser, env):
        """Register commandline options.
        """
        parser.add_option(
            "--nologcapture", action="store_false",
            default=not env.get(self.env_opt), dest="logcapture",
            help="Disable logging capture plugin. "
                 "Logging configurtion will be left intact."
                 " [NOSE_NOLOGCAPTURE]")
        parser.add_option(
            "--logging-format", action="store", dest="logcapture_format",
            default=env.get('NOSE_LOGFORMAT') or self.logformat,
            metavar="FORMAT",
            help="Specify custom format to print statements. "
                 "Uses the same format as used by standard logging handlers."
                 " [NOSE_LOGFORMAT]")
        parser.add_option(
            "--logging-datefmt", action="store", dest="logcapture_datefmt",
            default=env.get('NOSE_LOGDATEFMT') or self.logdatefmt,
            metavar="FORMAT",
            help="Specify custom date/time format to print statements. "
                 "Uses the same format as used by standard logging handlers."
                 " [NOSE_LOGDATEFMT]")
        parser.add_option(
            "--logging-filter", action="store", dest="logcapture_filters",
            default=env.get('NOSE_LOGFILTER'),
            metavar="FILTER",
            help="Specify which statements to filter in/out. "
                 "By default, everything is captured. If the output is too"
                 " verbose,\nuse this option to filter out needless output.\n"
                 "Example: filter=foo will capture statements issued ONLY to\n"
                 " foo or foo.what.ever.sub but not foobar or other logger.\n"
                 "Specify multiple loggers with comma: filter=foo,bar,baz.\n"
                 "If any logger name is prefixed with a minus, eg filter=-foo,\n"
                 "it will be excluded rather than included. Default: "
                 "exclude logging messages from nose itself (-nose)."
                 " [NOSE_LOGFILTER]\n")
        parser.add_option(
            "--logging-clear-handlers", action="store_true",
            default=False, dest="logcapture_clear",
            help="Clear all other logging handlers")
        parser.add_option(
            "--logging-level", action="store",
            default='NOTSET', dest="logcapture_level",
            help="Set the log level to capture")

    def configure(self, options, conf):
        """Configure plugin.
        """
        self.conf = conf
        # Disable if explicitly disabled, or if logging is
        # configured via logging config file
        if not options.logcapture or conf.loggingConfig:
            self.enabled = False
        self.logformat = options.logcapture_format
        self.logdatefmt = options.logcapture_datefmt
        self.clear = options.logcapture_clear
        self.loglevel = options.logcapture_level
        if options.logcapture_filters:
            self.filters = options.logcapture_filters.split(',')

    def setupLoghandler(self):
        # setup our handler with root logger
        root_logger = logging.getLogger()
        if self.clear:
            if hasattr(root_logger, "handlers"):
                for handler in root_logger.handlers:
                    root_logger.removeHandler(handler)
            for logger in logging.Logger.manager.loggerDict.values():
                if hasattr(logger, "handlers"):
                    for handler in logger.handlers:
                        logger.removeHandler(handler)
        # make sure there isn't one already
        # you can't simply use "if self.handler not in root_logger.handlers"
        # since at least in unit tests this doesn't work --
        # LogCapture() is instantiated for each test case while root_logger
        # is module global
        # so we always add new MyMemoryHandler instance
        for handler in root_logger.handlers[:]:
            if isinstance(handler, MyMemoryHandler):
                root_logger.handlers.remove(handler)
        root_logger.addHandler(self.handler)
        # to make sure everything gets captured
        loglevel = getattr(self, "loglevel", "NOTSET")
        root_logger.setLevel(getattr(logging, loglevel))

    def begin(self):
        """Set up logging handler before test run begins.
        """
        self.start()

    def start(self):
        self.handler = MyMemoryHandler(1000, self.logformat, self.logdatefmt,
                                       self.filters)
        self.setupLoghandler()

    def end(self):
        pass

    def beforeTest(self, test):
        """Clear buffers and handlers before test.
        """
        self.setupLoghandler()

    def afterTest(self, test):
        """Clear buffers after test.
        """
        self.handler.truncate()

    def formatFailure(self, test, err):
        """Add captured log messages to failure output.
        """
        return self.formatError(test, err)

    def formatError(self, test, err):
        """Add captured log messages to error output.
        """
        # logic flow copied from Capture.formatError
        test.capturedLogging = records = self.formatLogRecords()
        if not records:
            return err
        ec, ev, tb = err
        return (ec, self.addCaptureToErr(ev, records), tb)

    def formatLogRecords(self):
        format = self.handler.format
        return [safe_str(format(r)) for r in self.handler.buffer]

    def addCaptureToErr(self, ev, records):
        return '\n'.join([safe_str(ev), ln('>> begin captured logging <<')] + \
                          records + \
                          [ln('>> end captured logging <<')])


# encoding: utf-8
"""
An embedded IPython shell.

Authors:

* Brian Granger
* Fernando Perez

Notes
-----
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from __future__ import with_statement

import sys
import warnings

# We need to use nested to support python 2.6, once we move to >=2.7, we can
# use the with keyword's new builtin support for nested managers
try:
    from contextlib import nested
except:
    from IPython.utils.nested_context import nested

from IPython.core import ultratb
from IPython.core.magic import Magics, magics_class, line_magic
from IPython.frontend.terminal.interactiveshell import TerminalInteractiveShell
from IPython.frontend.terminal.ipapp import load_default_config

from IPython.utils.traitlets import Bool, CBool, Unicode
from IPython.utils.io import ask_yes_no

#-----------------------------------------------------------------------------
# Classes and functions
#-----------------------------------------------------------------------------

# This is an additional magic that is exposed in embedded shells.
@magics_class
class EmbeddedMagics(Magics):

    @line_magic
    def kill_embedded(self, parameter_s=''):
        """%kill_embedded : deactivate for good the current embedded IPython.

        This function (after asking for confirmation) sets an internal flag so
        that an embedded IPython will never activate again.  This is useful to
        permanently disable a shell that is being called inside a loop: once
        you've figured out what you needed from it, you may then kill it and
        the program will then continue to run without the interactive shell
        interfering again.
        """

        kill = ask_yes_no("Are you sure you want to kill this embedded instance "
                         "(y/n)? [y/N] ",'n')
        if kill:
            self.shell.embedded_active = False
            print ("This embedded IPython will not reactivate anymore "
                   "once you exit.")

class InteractiveShellEmbed(TerminalInteractiveShell):

    dummy_mode = Bool(False)
    exit_msg = Unicode('')
    embedded = CBool(True)
    embedded_active = CBool(True)
    # Like the base class display_banner is not configurable, but here it
    # is True by default.
    display_banner = CBool(True)

    def __init__(self, config=None, ipython_dir=None, user_ns=None,
                 user_module=None, custom_exceptions=((),None),
                 usage=None, banner1=None, banner2=None,
                 display_banner=None, exit_msg=u'', user_global_ns=None):

        if user_global_ns is not None:
            warnings.warn("user_global_ns has been replaced by user_module. The\
                           parameter will be ignored.", DeprecationWarning)

        super(InteractiveShellEmbed,self).__init__(
            config=config, ipython_dir=ipython_dir, user_ns=user_ns,
            user_module=user_module, custom_exceptions=custom_exceptions,
            usage=usage, banner1=banner1, banner2=banner2,
            display_banner=display_banner
        )

        self.exit_msg = exit_msg

        # don't use the ipython crash handler so that user exceptions aren't
        # trapped
        sys.excepthook = ultratb.FormattedTB(color_scheme=self.colors,
                                             mode=self.xmode,
                                             call_pdb=self.pdb)

    def init_sys_modules(self):
        pass

    def init_magics(self):
        super(InteractiveShellEmbed, self).init_magics()
        self.register_magics(EmbeddedMagics)

    def __call__(self, header='', local_ns=None, module=None, dummy=None,
                 stack_depth=1, global_ns=None):
        """Activate the interactive interpreter.

        __call__(self,header='',local_ns=None,module=None,dummy=None) -> Start
        the interpreter shell with the given local and global namespaces, and
        optionally print a header string at startup.

        The shell can be globally activated/deactivated using the
        dummy_mode attribute. This allows you to turn off a shell used
        for debugging globally.

        However, *each* time you call the shell you can override the current
        state of dummy_mode with the optional keyword parameter 'dummy'. For
        example, if you set dummy mode on with IPShell.dummy_mode = True, you
        can still have a specific call work by making it as IPShell(dummy=False).
        """

        # If the user has turned it off, go away
        if not self.embedded_active:
            return

        # Normal exits from interactive mode set this flag, so the shell can't
        # re-enter (it checks this variable at the start of interactive mode).
        self.exit_now = False

        # Allow the dummy parameter to override the global __dummy_mode
        if dummy or (dummy != 0 and self.dummy_mode):
            return

        if self.has_readline:
            self.set_readline_completer()

        # self.banner is auto computed
        if header:
            self.old_banner2 = self.banner2
            self.banner2 = self.banner2 + '\n' + header + '\n'
        else:
            self.old_banner2 = ''

        # Call the embedding code with a stack depth of 1 so it can skip over
        # our call and get the original caller's namespaces.
        self.mainloop(local_ns, module, stack_depth=stack_depth, global_ns=global_ns)

        self.banner2 = self.old_banner2

        if self.exit_msg is not None:
            print self.exit_msg

    def mainloop(self, local_ns=None, module=None, stack_depth=0,
                 display_banner=None, global_ns=None):
        """Embeds IPython into a running python program.

        Input:

          - header: An optional header message can be specified.

          - local_ns, module: working local namespace (a dict) and module (a
          module or similar object). If given as None, they are automatically
          taken from the scope where the shell was called, so that
          program variables become visible.

          - stack_depth: specifies how many levels in the stack to go to
          looking for namespaces (when local_ns or module is None).  This
          allows an intermediate caller to make sure that this function gets
          the namespace from the intended level in the stack.  By default (0)
          it will get its locals and globals from the immediate caller.

        Warning: it's possible to use this in a program which is being run by
        IPython itself (via %run), but some funny things will happen (a few
        globals get overwritten). In the future this will be cleaned up, as
        there is no fundamental reason why it can't work perfectly."""

        if (global_ns is not None) and (module is None):
            class DummyMod(object):
                """A dummy module object for embedded IPython."""
                pass
            warnings.warn("global_ns is deprecated, use module instead.", DeprecationWarning)
            module = DummyMod()
            module.__dict__ = global_ns

        # Get locals and globals from caller
        if (local_ns is None or module is None) and self.default_user_namespaces:
            call_frame = sys._getframe(stack_depth).f_back

            if local_ns is None:
                local_ns = call_frame.f_locals
            if module is None:
                global_ns = call_frame.f_globals
                module = sys.modules[global_ns['__name__']]

        # Save original namespace and module so we can restore them after
        # embedding; otherwise the shell doesn't shut down correctly.
        orig_user_module = self.user_module
        orig_user_ns = self.user_ns

        # Update namespaces and fire up interpreter

        # The global one is easy, we can just throw it in
        if module is not None:
            self.user_module = module

        # But the user/local one is tricky: ipython needs it to store internal
        # data, but we also need the locals. We'll throw our hidden variables
        # like _ih and get_ipython() into the local namespace, but delete them
        # later.
        if local_ns is not None:
            self.user_ns = local_ns
            self.init_user_ns()

        # Patch for global embedding to make sure that things don't overwrite
        # user globals accidentally. Thanks to Richard <rxe@renre-europe.com>
        # FIXME. Test this a bit more carefully (the if.. is new)
        # N.B. This can't now ever be called. Not sure what it was for.
        # And now, since it wasn't called in the previous version, I'm
        # commenting out these lines so they can't be called with my new changes
        # --TK, 2011-12-10
        #if local_ns is None and module is None:
        #    self.user_global_ns.update(__main__.__dict__)

        # make sure the tab-completer has the correct frame information, so it
        # actually completes using the frame's locals/globals
        self.set_completer_frame()

        with nested(self.builtin_trap, self.display_trap):
            self.interact(display_banner=display_banner)

        # now, purge out the local namespace of IPython's hidden variables.
        if local_ns is not None:
            for name in self.user_ns_hidden:
                local_ns.pop(name, None)

        # Restore original namespace so shell can shut down when we exit.
        self.user_module = orig_user_module
        self.user_ns = orig_user_ns

_embedded_shell = None

def embed(**kwargs):
    """Call this to embed IPython at the current point in your program.

    The first invocation of this will create an :class:`InteractiveShellEmbed`
    instance and then call it.  Consecutive calls just call the already
    created instance.

    Here is a simple example::

        from IPython import embed
        a = 10
        b = 20
        embed('First time')
        c = 30
        d = 40
        embed

    Full customization can be done by passing a :class:`Struct` in as the
    config argument.
    """
    config = kwargs.get('config')
    header = kwargs.pop('header', u'')
    if config is None:
        config = load_default_config()
        config.InteractiveShellEmbed = config.TerminalInteractiveShell
        kwargs['config'] = config
    global _embedded_shell
    if _embedded_shell is None:
        _embedded_shell = InteractiveShellEmbed(**kwargs)
    _embedded_shell(header=header, stack_depth=2)


# encoding: utf-8
"""A fancy version of Python's builtin :func:`dir` function.
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------

def get_class_members(cls):
    ret = dir(cls)
    if hasattr(cls, '__bases__'):
        try:
            bases = cls.__bases__
        except AttributeError:
            # `obj` lied to hasattr (e.g. Pyro), ignore
            pass
        else:
            for base in bases:
                ret.extend(get_class_members(base))
    return ret

def dir2(obj):
    """dir2(obj) -> list of strings

    Extended version of the Python builtin dir(), which does a few extra
    checks, and supports common objects with unusual internals that confuse
    dir(), such as Traits and PyCrust.

    This version is guaranteed to return only a list of true strings, whereas
    dir() returns anything that objects inject into themselves, even if they
    are later not really valid for attribute access (many extension libraries
    have such bugs).
    """

    # Start building the attribute list via dir(), and then complete it
    # with a few extra special-purpose calls.

    words = set(dir(obj))

    if hasattr(obj, '__class__'):
        #words.add('__class__')
        words |= set(get_class_members(obj.__class__))

    # for objects with Enthought's traits, add trait_names() list
    # for PyCrust-style, add _getAttributeNames() magic method list
    for attr in ('trait_names', '_getAttributeNames'):
        if hasattr(obj, attr):
            try:
                func = getattr(obj, attr)
                if callable(func):
                    words |= set(func())
            except:
                # TypeError: obj is class not instance
                pass

    # filter out non-string attributes which may be stuffed by dir() calls
    # and poor coding in third-party modules

    words = [w for w in words if isinstance(w, basestring)]
    return sorted(words)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import ast
import os
import re
from itertools import izip_longest

import polib

from c3po.conf import settings
from c3po.converters.unicode import UnicodeWriter, UnicodeReader

METADATA_EMPTY = "{'comment': '', 'previous_msgctxt': None, " + \
                 "'encoding': 'utf-8', 'obsolete': 0, 'msgid_plural': '', " + \
                 "'msgstr_plural': {}, 'occurrences': [], 'msgctxt': None, " + \
                 "'flags': [], 'previous_msgid': None, " + \
                 "'previous_msgid_plural': None}"

def _get_all_po_filenames(locale_root, lang, po_files_path):
    """
    Get all po filenames from locale folder and return list of them.
    Assumes a directory structure:
    <locale_root>/<lang>/<po_files_path>/<filename>.
    """
    all_files = os.listdir(os.path.join(locale_root, lang, po_files_path))
    return filter(lambda s: s.endswith('.po'), all_files)

def _get_new_csv_writers(trans_title, meta_title,
                         trans_csv_path, meta_csv_path):
    """
    Prepare new csv writers, write title rows and return them.
    """
    trans_writer = UnicodeWriter(trans_csv_path)
    trans_writer.writerow(trans_title)

    meta_writer = UnicodeWriter(meta_csv_path)
    meta_writer.writerow(meta_title)

    return trans_writer, meta_writer

def _prepare_locale_dirs(languages, locale_root):
    """
    Prepare locale dirs for writing po files.
    Create new directories if they doesn't exist.
    """
    trans_languages = []
    for i, t in enumerate(languages):
        lang = t.split(':')[0]
        trans_languages.append(lang)
        lang_path = os.path.join(locale_root, lang)
        if not os.path.exists(lang_path):
            os.makedirs(lang_path)
    return trans_languages

def _prepare_polib_files(files_dict, filename, languages,
                         locale_root, po_files_path, header):
    """
    Prepare polib file object for writing/reading from them.
    Create directories and write header if needed. For each language,
    ensure there's a translation file named "filename" in the correct place.
    Assumes (and creates) a directory structure:
    <locale_root>/<lang>/<po_files_path>/<filename>.
    """
    files_dict[filename] = {}
    for lang in languages:
        file_path = os.path.join(locale_root, lang, po_files_path)
        if not os.path.exists(file_path):
            os.makedirs(file_path)

        if header is not None:
            _write_header(os.path.join(file_path, filename), lang, header)

        files_dict[filename][lang] = polib.pofile(
            os.path.join(file_path, filename), encoding="UTF-8")

def _write_entries(po_files, languages, msgid, msgstrs, metadata, comment):
    """
    Write msgstr for every language with all needed metadata and comment.
    Metadata are parser from string into dict, so read them only from gdocs.
    """
    start = re.compile(r'^[\s]+')
    end = re.compile(r'[\s]+$')
    for i, lang in enumerate(languages):
        meta = ast.literal_eval(metadata)
        entry = polib.POEntry(**meta)
        entry.tcomment = comment
        entry.msgid = msgid
        if msgstrs[i]:
            start_ws = start.search(msgid)
            end_ws = end.search(msgid)
            entry.msgstr = str(start_ws.group() if start_ws else '') + \
                unicode(msgstrs[i].strip()) + \
                str(end_ws.group() if end_ws else '')
        else:
            entry.msgstr = ''
        po_files[lang].append(entry)

def _write_header(po_path, lang, header):
    """
    Write header into po file for specific lang.
    Metadata are read from settings file.
    """
    po_file = open(po_path, 'w')
    po_file.write(header + '\n')
    po_file.write(
        'msgid ""' +
        '\nmsgstr ""' +
        '\n"MIME-Version: ' + settings.METADATA['MIME-Version'] + r'\n"'
        '\n"Content-Type: ' + settings.METADATA['Content-Type'] + r'\n"'
        '\n"Content-Transfer-Encoding: ' +
        settings.METADATA['Content-Transfer-Encoding'] + r'\n"'
        '\n"Language: ' + lang + r'\n"' + '\n')
    po_file.close()

def _write_new_messages(po_file_path, trans_writer, meta_writer,
                        msgids, msgstrs, languages):
    """
    Write new msgids which appeared in po files with empty msgstrs values
    and metadata. Look for all new msgids which are diffed with msgids list
    provided as an argument.
    """
    po_filename = os.path.basename(po_file_path)
    po_file = polib.pofile(po_file_path)

    new_trans = 0
    for entry in po_file:
        if entry.msgid not in msgids:
            new_trans += 1
            trans = [po_filename, entry.tcomment, entry.msgid, entry.msgstr]
            for lang in languages[1:]:
                trans.append(msgstrs[lang].get(entry.msgid, ''))

            meta = dict(entry.__dict__)
            meta.pop('msgid', None)
            meta.pop('msgstr', None)
            meta.pop('tcomment', None)

            trans_writer.writerow(trans)
            meta_writer.writerow([str(meta)])

    return new_trans

def _get_new_msgstrs(po_file_path, msgids):
    """
    Write new msgids which appeared in po files with empty msgstrs values
    and metadata. Look for all new msgids which are diffed with msgids list
    provided as an argument.
    """
    po_file = polib.pofile(po_file_path)

    msgstrs = {}

    for entry in po_file:
        if entry.msgid not in msgids:
            msgstrs[entry.msgid] = entry.msgstr

    return msgstrs

def po_to_csv_merge(languages, locale_root, po_files_path,
                    local_trans_csv, local_meta_csv,
                    gdocs_trans_csv, gdocs_meta_csv):
    """
    Converts po file to csv GDocs spreadsheet readable format.
    Merges them if some msgid aren't in the spreadsheet.
    :param languages: list of language codes
    :param locale_root: path to locale root folder containing directories
                        with languages
    :param po_files_path: path from lang directory to po file
    :param local_trans_csv: path where local csv with translations
                            will be created
    :param local_meta_csv: path where local csv with metadata will be created
    :param gdocs_trans_csv: path to gdoc csv with translations
    """
    msgids = []

    trans_reader = UnicodeReader(gdocs_trans_csv)
    meta_reader = UnicodeReader(gdocs_meta_csv)

    try:
        trans_title = trans_reader.next()
        meta_title = meta_reader.next()
    except StopIteration:
        trans_title = ['file', 'comment', 'msgid']
        trans_title += map(lambda s: s + ':msgstr', languages)
        meta_title = ['metadata']

    trans_writer, meta_writer = _get_new_csv_writers(
        trans_title, meta_title, local_trans_csv, local_meta_csv)

    for trans_row, meta_row in izip_longest(trans_reader, meta_reader):
        msgids.append(trans_row[2])
        trans_writer.writerow(trans_row)
        meta_writer.writerow(meta_row if meta_row else [METADATA_EMPTY])

    trans_reader.close()
    meta_reader.close()

    po_files = _get_all_po_filenames(locale_root, languages[0], po_files_path)

    new_trans = False
    for po_filename in po_files:
        new_msgstrs = {}
        for lang in languages[1:]:
            po_file_path = os.path.join(locale_root, lang,
                                        po_files_path, po_filename)
            if not os.path.exists(po_file_path):
                open(po_file_path, 'a').close()
            new_msgstrs[lang] = _get_new_msgstrs(po_file_path, msgids)

        if len(new_msgstrs[languages[1]].keys()) > 0:
            new_trans = True
            po_file_path = os.path.join(locale_root, languages[0],
                                        po_files_path, po_filename)
            _write_new_messages(po_file_path, trans_writer, meta_writer,
                                msgids, new_msgstrs, languages)

    trans_writer.close()
    meta_writer.close()

    return new_trans

def csv_to_po(trans_csv_path, meta_csv_path, locale_root,
              po_files_path, header=None):
    """
    Converts GDocs spreadsheet generated csv file into po file.
    :param trans_csv_path: path to temporary file with translations
    :param meta_csv_path: path to temporary file with meta information
    :param locale_root: path to locale root folder containing directories
                        with languages
    :param po_files_path: path from lang directory to po file
    """
    pattern = "^\w+.*po$"
    for root, dirs, files in os.walk(locale_root):
        for f in filter(lambda x: re.match(pattern, x), files):
            os.remove(os.path.join(root, f))

    # read title row and prepare descriptors for po files in each lang
    trans_reader = UnicodeReader(trans_csv_path)
    meta_reader = UnicodeReader(meta_csv_path)
    try:
        title_row = trans_reader.next()
    except StopIteration:
        # empty file
        return

    trans_languages = _prepare_locale_dirs(title_row[3:], locale_root)

    po_files = {}

    meta_reader.next()
    # go through every row in downloaded csv file
    for trans_row, meta_row in izip_longest(trans_reader, meta_reader):
        filename = trans_row[0].rstrip()
        metadata = meta_row[0].rstrip() if meta_row else METADATA_EMPTY
        comment = trans_row[1]
        msgid = trans_row[2]

        if filename not in po_files:
            _prepare_polib_files(po_files, filename, trans_languages,
                                 locale_root, po_files_path, header)

        _write_entries(po_files[filename], trans_languages, msgid,
                       trans_row[3:], metadata, comment)
    for filename in po_files:
        for lang in po_files[filename]:
            po_files[filename][lang].save()

    trans_reader.close()
    meta_reader.close()


# encoding: utf-8

""" notifo.py - python wrapper for notifo.com """

try:
  import json
except ImportError:
  try:
    import simplejson as json
  except ImportError:
    # For Google AppEngine
    from django.utils import simplejson as json

import urllib
import urllib2
from base64 import encodestring

class Notifo:
    """ Class for wrapping notifo.com """
    def __init__(self, user, secret):
        self.user = user
        self.secret = secret
        self.root_url = "https://api.notifo.com/v1/"

    def subscribe_user(self, user):
        """ method to subscribe a user to a service
        """
        url = self.root_url + "subscribe_user"
        values = {}
        values["username"] = user
        return self._query(url, values)

    def send_notification(self, to=None, msg=None, label=None,
                          title=None, uri=None):
        """ method to send a message to a user

            Parameters:
                to -> recipient
                msg -> message to send
                label -> application description
                title -> name of the notification event
                uri -> callback uri
        """
        url = self.root_url + "send_notification"
        values = {}
        if to is not None:
            values["to"] = to
        if msg is not None:
            values["msg"] = msg
        if label is not None:
            values["label"] = label
        if title is not None:
            values["title"] = title
        if uri is not None:
            values["uri"] = uri
        return self._query(url, values)

    def send_message(self, to=None, msg=None):
        """ method to send a message to a user

            Parameters:
                to -> recipient
                msg -> message to send
        """
        url = self.root_url + "send_message"
        values = {}
        if to is not None:
            values["to"] = to
        if msg is not None:
            values["msg"] = msg
        return self._query(url, values)

    def _query(self, url, data = None):
        """ query method to do HTTP POST/GET

            Parameters:
                url -> the url to POST/GET
                data -> header_data as a dict (only for POST)

            Returns:
                Parsed JSON data as dict
                or
                None on error
        """
        auth = encodestring('%s:%s' % (self.user, self.secret)).replace('\n', '')

        if data is not None: # we have POST data if there is data
            values = urllib.urlencode(data)
            request = urllib2.Request(url, values)
            request.add_header("Authorization", "Basic %s" % auth)
        else: # do a GET otherwise
            request = urllib2.Request(url)
            request.add_header("Authorization", "Basic %s" % auth)
        try:
            response = urllib2.urlopen(request)
        except IOError, e: # no connection
            return {"status" : "error",
                    "response_code" : e.code,
                    "response_message" : e.msg
                   }
        return json.loads(response.read())


#!/usr/bin/env python
# encoding: utf-8

""" executable to interact with notifo.com using the notifo.py library """

import notifo
from optparse import OptionParser

def init_parser():
    """ function to init option parser """
    usage = "usage: %prog -u user -s secret -n name [-l label] \
[-t title] [-c callback] [TEXT]"

    parser = OptionParser(usage, version="%prog " + notifo.__version__)
    parser.add_option("-u", "--user", action="store", dest="user",
                      help="your notifo username")
    parser.add_option("-s", "--secret", action="store", dest="secret",
                      help="your notifo API secret")
    parser.add_option("-n", "--name", action="store", dest="name",
                      help="recipient for the notification")
    parser.add_option("-l", "--label", action="store", dest="label",
                      help="label for the notification")
    parser.add_option("-t", "--title", action="store", dest="title",
                      help="title of the notification")
    parser.add_option("-c", "--callback", action="store", dest="callback",
                      help="callback URL to call")
    parser.add_option("-m", "--message", action="store_true", dest="message",
                      default=False, help="send message instead of notification")

    (options, args) = parser.parse_args()
    return (parser, options, args)

def main():
    """ main function """
    # get options and arguments
    (parser, options, args) = init_parser()

    # initialize result variable
    result = None

    # check for values which are always needed
    if not options.user:
        parser.error("No user given.")
    if not options.secret:
        parser.error("No API secret given.")
    if not options.name:
        parser.error("No recipient given.")

    # If there is no message, we probably want to subscribe a user
    if len(args) < 1:
        result = notifo.subscribe_user(options.user, options.secret, options.name)
    else:
        params = {}
        params["to"] = options.name
        m = ''
        for a in args:
            m = "%s %s" %(m, a)
        params["msg"] = m

        if options.message == True:
            result = notifo.send_message(options.user, options.secret, **params)
        else:
            if options.label:
                params["label"] = options.label
            if options.title:
                params["title"] = options.title
            if options.callback:
                params["uri"] = options.callback
            result = notifo.send_notification(options.user,options.secret,
                                              **params)

    if result is None:
        print "Something went wrong. Check parameters and try again."

if __name__ == '__main__':
    main()


"""Execute files of Python code."""

import imp, marshal, os, sys

from coverage.backward import exec_code_object, open_source
from coverage.misc import ExceptionDuringRun, NoCode, NoSource

try:
    # In Py 2.x, the builtins were in __builtin__
    BUILTINS = sys.modules['__builtin__']
except KeyError:
    # In Py 3.x, they're in builtins
    BUILTINS = sys.modules['builtins']

def rsplit1(s, sep):
    """The same as s.rsplit(sep, 1), but works in 2.3"""
    parts = s.split(sep)
    return sep.join(parts[:-1]), parts[-1]

def run_python_module(modulename, args):
    """Run a python module, as though with ``python -m name args...``.

    `modulename` is the name of the module, possibly a dot-separated name.
    `args` is the argument array to present as sys.argv, including the first
    element naming the module being executed.

    """
    openfile = None
    glo, loc = globals(), locals()
    try:
        try:
            # Search for the module - inside its parent package, if any - using
            # standard import mechanics.
            if '.' in modulename:
                packagename, name = rsplit1(modulename, '.')
                package = __import__(packagename, glo, loc, ['__path__'])
                searchpath = package.__path__
            else:
                packagename, name = None, modulename
                searchpath = None  # "top-level search" in imp.find_module()
            openfile, pathname, _ = imp.find_module(name, searchpath)

            # Complain if this is a magic non-file module.
            if openfile is None and pathname is None:
                raise NoSource(
                    "module does not live in a file: %r" % modulename
                    )

            # If `modulename` is actually a package, not a mere module, then we
            # pretend to be Python 2.7 and try running its __main__.py script.
            if openfile is None:
                packagename = modulename
                name = '__main__'
                package = __import__(packagename, glo, loc, ['__path__'])
                searchpath = package.__path__
                openfile, pathname, _ = imp.find_module(name, searchpath)
        except ImportError:
            _, err, _ = sys.exc_info()
            raise NoSource(str(err))
    finally:
        if openfile:
            openfile.close()

    # Finally, hand the file off to run_python_file for execution.
    pathname = os.path.abspath(pathname)
    args[0] = pathname
    run_python_file(pathname, args, package=packagename)

def run_python_file(filename, args, package=None):
    """Run a python file as if it were the main program on the command line.

    `filename` is the path to the file to execute, it need not be a .py file.
    `args` is the argument array to present as sys.argv, including the first
    element naming the file being executed.  `package` is the name of the
    enclosing package, if any.

    """
    # Create a module to serve as __main__
    old_main_mod = sys.modules['__main__']
    main_mod = imp.new_module('__main__')
    sys.modules['__main__'] = main_mod
    main_mod.__file__ = filename
    if package:
        main_mod.__package__ = package
    main_mod.__builtins__ = BUILTINS

    # Set sys.argv properly.
    old_argv = sys.argv
    sys.argv = args

    try:
        # Make a code object somehow.
        if filename.endswith(".pyc") or filename.endswith(".pyo"):
            code = make_code_from_pyc(filename)
        else:
            code = make_code_from_py(filename)

        # Execute the code object.
        try:
            exec_code_object(code, main_mod.__dict__)
        except SystemExit:
            # The user called sys.exit().  Just pass it along to the upper
            # layers, where it will be handled.
            raise
        except:
            # Something went wrong while executing the user code.
            # Get the exc_info, and pack them into an exception that we can
            # throw up to the outer loop.  We peel two layers off the traceback
            # so that the coverage.py code doesn't appear in the final printed
            # traceback.
            typ, err, tb = sys.exc_info()
            raise ExceptionDuringRun(typ, err, tb.tb_next.tb_next)
    finally:
        # Restore the old __main__
        sys.modules['__main__'] = old_main_mod

        # Restore the old argv and path
        sys.argv = old_argv

def make_code_from_py(filename):
    """Get source from `filename` and make a code object of it."""
    # Open the source file.
    try:
        source_file = open_source(filename)
    except IOError:
        raise NoSource("No file to run: %r" % filename)

    try:
        source = source_file.read()
    finally:
        source_file.close()

    # We have the source.  `compile` still needs the last line to be clean,
    # so make sure it is, then compile a code object from it.
    if not source or source[-1] != '\n':
        source += '\n'
    code = compile(source, filename, "exec")

    return code

def make_code_from_pyc(filename):
    """Get a code object from a .pyc file."""
    try:
        fpyc = open(filename, "rb")
    except IOError:
        raise NoCode("No file to run: %r" % filename)

    try:
        # First four bytes are a version-specific magic number.  It has to
        # match or we won't run the file.
        magic = fpyc.read(4)
        if magic != imp.get_magic():
            raise NoCode("Bad magic number in .pyc file")

        # Skip the junk in the header that we don't need.
        fpyc.read(4)            # Skip the moddate.
        if sys.version_info >= (3, 3):
            # 3.3 added another long to the header (size), skip it.
            fpyc.read(4)

        # The rest of the file is the code object we want.
        code = marshal.load(fpyc)
    finally:
        fpyc.close()

    return code


"""a navigable completer for the qtconsole"""
# coding : utf-8
#-----------------------------------------------------------------------------
# Copyright (c) 2012, IPython Development Team.$
#
# Distributed under the terms of the Modified BSD License.$
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

# System library imports
import IPython.utils.text as text

from IPython.external.qt import QtCore, QtGui

#--------------------------------------------------------------------------
# Return an HTML table with selected item in a special class
#--------------------------------------------------------------------------
def html_tableify(item_matrix, select=None, header=None , footer=None) :
    """ returnr a string for an html table"""
    if not item_matrix :
        return ''
    html_cols = []
    tds = lambda text : u''+text+u'  	'
    trs = lambda text : u'
'+text+u'
'
    tds_items = [map(tds, row) for row in item_matrix]
    if select :
        row, col = select
        tds_items[row][col] = u''\
                +item_matrix[row][col]\
                +u'  	'
    #select the right item
    html_cols = map(trs, (u''.join(row) for row in tds_items))
    head = ''
    foot = ''
    if header :
        head = (u'
'\
            +''.join((u''+header+u'	')*len(item_matrix[0]))\
            +'
')

    if footer :
        foot = (u'
'\
            +''.join((u''+footer+u'	')*len(item_matrix[0]))\
            +'
')
    html = (u'
'+head+(u''.join(html_cols))+foot+u'
')
    return html

class SlidingInterval(object):
    """a bound interval that follows a cursor

    internally used to scoll the completion view when the cursor
    try to go beyond the edges, and show '...' when rows are hidden
    """

    _min = 0
    _max = 1
    _current = 0
    def __init__(self, maximum=1, width=6, minimum=0, sticky_lenght=1):
        """Create a new bounded interval

        any value return by this will be bound between maximum and
        minimum. usual width will be 'width', and sticky_length
        set when the return  interval should expand to max and min
        """
        self._min = minimum
        self._max = maximum
        self._start = 0
        self._width = width
        self._stop = self._start+self._width+1
        self._sticky_lenght = sticky_lenght

    @property
    def current(self):
        """current cursor position"""
        return self._current

    @current.setter
    def current(self, value):
        """set current cursor position"""
        current = min(max(self._min, value), self._max)

        self._current = current

        if current > self._stop :
            self._stop = current
            self._start = current-self._width
        elif current < self._start :
            self._start = current
            self._stop = current + self._width

        if abs(self._start - self._min) <= self._sticky_lenght :
            self._start = self._min

        if abs(self._stop - self._max) <= self._sticky_lenght :
            self._stop = self._max

    @property
    def start(self):
        """begiiing of interval to show"""
        return self._start

    @property
    def stop(self):
        """end of interval to show"""
        return self._stop

    @property
    def width(self):
        return self._stop - self._start

    @property
    def nth(self):
        return self.current - self.start

class CompletionHtml(QtGui.QWidget):
    """ A widget for tab completion,  navigable by arrow keys """

    #--------------------------------------------------------------------------
    # 'QObject' interface
    #--------------------------------------------------------------------------

    _items = ()
    _index = (0, 0)
    _consecutive_tab = 0
    _size = (1, 1)
    _old_cursor = None
    _start_position = 0
    _slice_start = 0
    _slice_len = 4

    def __init__(self, console_widget):
        """ Create a completion widget that is attached to the specified Qt
            text edit widget.
        """
        assert isinstance(console_widget._control, (QtGui.QTextEdit, QtGui.QPlainTextEdit))
        super(CompletionHtml, self).__init__()

        self._text_edit = console_widget._control
        self._console_widget = console_widget
        self._text_edit.installEventFilter(self)
        self._sliding_interval = None
        self._justified_items = None

        # Ensure that the text edit keeps focus when widget is displayed.
        self.setFocusProxy(self._text_edit)

    def eventFilter(self, obj, event):
        """ Reimplemented to handle keyboard input and to auto-hide when the
            text edit loses focus.
        """
        if obj == self._text_edit:
            etype = event.type()
            if etype == QtCore.QEvent.KeyPress:
                key = event.key()
                if self._consecutive_tab == 0 and key in (QtCore.Qt.Key_Tab,):
                    return False
                elif self._consecutive_tab == 1 and key in (QtCore.Qt.Key_Tab,):
                    # ok , called twice, we grab focus, and show the cursor
                    self._consecutive_tab = self._consecutive_tab+1
                    self._update_list()
                    return True
                elif self._consecutive_tab == 2:
                    if key in (QtCore.Qt.Key_Return, QtCore.Qt.Key_Enter):
                        self._complete_current()
                        return True
                    if key in (QtCore.Qt.Key_Tab,):
                        self.select_right()
                        self._update_list()
                        return True
                    elif key in ( QtCore.Qt.Key_Down,):
                        self.select_down()
                        self._update_list()
                        return True
                    elif key in (QtCore.Qt.Key_Right,):
                        self.select_right()
                        self._update_list()
                        return True
                    elif key in ( QtCore.Qt.Key_Up,):
                        self.select_up()
                        self._update_list()
                        return True
                    elif key in ( QtCore.Qt.Key_Left,):
                        self.select_left()
                        self._update_list()
                        return True
                    elif key in ( QtCore.Qt.Key_Escape,):
                        self.cancel_completion()
                        return True
                    else :
                        self.cancel_completion()
                else:
                    self.cancel_completion()

            elif etype == QtCore.QEvent.FocusOut:
                self.cancel_completion()

        return super(CompletionHtml, self).eventFilter(obj, event)

    #--------------------------------------------------------------------------
    # 'CompletionHtml' interface
    #--------------------------------------------------------------------------
    def cancel_completion(self):
        """Cancel the completion

        should be called when the completer have to be dismissed

        This reset internal variable, clearing the temporary buffer
        of the console where the completion are shown.
        """
        self._consecutive_tab = 0
        self._slice_start = 0
        self._console_widget._clear_temporary_buffer()
        self._index = (0, 0)
        if(self._sliding_interval):
            self._sliding_interval = None

    #
    #  ...  2 4 4 4 4 4 4 4 4 4 4  4  4
    #   2   2 4 4 4 4 4 4 4 4 4 4  4  4
    #
    #2  2   x x x x x x x x x x x  5  5
    #6  6   x x x x x x x x x x x  5  5
    #6  6   x x x x x x x x x x ?  5  5
    #6  6   x x x x x x x x x x ?  1  1
    #
    #3  3   3 3 3 3 3 3 3 3 3 3 1  1  1 ...
    #3  3   3 3 3 3 3 3 3 3 3 3 1  1  1 ...
    def _select_index(self, row, col):
        """Change the selection index, and make sure it stays in the right range

        A little more complicated than just dooing modulo the number of row columns
        to be sure to cycle through all element.

        horizontaly, the element are maped like this :
        to r <-- a b c d e f --> to g
        to f <-- g h i j k l --> to m
        to l <-- m n o p q r --> to a

        and vertically
        a d g j m p
        b e h k n q
        c f i l o r
        """

        nr, nc = self._size
        nr = nr-1
        nc = nc-1

        # case 1
        if (row > nr and col >= nc) or (row >= nr and col > nc):
            self._select_index(0, 0)
        # case 2
        elif (row <= 0 and col < 0) or  (row < 0 and col <= 0):
            self._select_index(nr, nc)
        # case 3
        elif row > nr :
            self._select_index(0, col+1)
        # case 4
        elif row < 0 :
            self._select_index(nr, col-1)
        # case 5
        elif col > nc :
            self._select_index(row+1, 0)
        # case 6
        elif col < 0 :
            self._select_index(row-1, nc)
        elif 0 <= row and row <= nr and 0 <= col and col <= nc :
            self._index = (row, col)
        else :
            raise NotImplementedError("you'r trying to go where no completion\
                           have gone before : %d:%d (%d:%d)"%(row, col, nr, nc) )

    @property
    def _slice_end(self):
        end = self._slice_start+self._slice_len
        if end > len(self._items) :
            return None
        return end

    def select_up(self):
        """move cursor up"""
        r, c = self._index
        self._select_index(r-1, c)

    def select_down(self):
        """move cursor down"""
        r, c = self._index
        self._select_index(r+1, c)

    def select_left(self):
        """move cursor left"""
        r, c = self._index
        self._select_index(r, c-1)

    def select_right(self):
        """move cursor right"""
        r, c = self._index
        self._select_index(r, c+1)

    def show_items(self, cursor, items):
        """ Shows the completion widget with 'items' at the position specified
            by 'cursor'.
        """
        if not items :
            return
        self._start_position = cursor.position()
        self._consecutive_tab = 1
        items_m, ci = text.compute_item_matrix(items, empty=' ')
        self._sliding_interval = SlidingInterval(len(items_m)-1)

        self._items = items_m
        self._size = (ci['rows_numbers'], ci['columns_numbers'])
        self._old_cursor = cursor
        self._index = (0, 0)
        sjoin = lambda x : [ y.ljust(w, ' ') for y, w in zip(x, ci['columns_width'])]
        self._justified_items = map(sjoin, items_m)
        self._update_list(hilight=False)

    def _update_list(self, hilight=True):
        """ update the list of completion and hilight the currently selected completion """
        self._sliding_interval.current = self._index[0]
        head = None
        foot = None
        if self._sliding_interval.start > 0 :
            head = '...'

        if self._sliding_interval.stop < self._sliding_interval._max:
            foot = '...'
        items_m = self._justified_items[\
                       self._sliding_interval.start:\
                       self._sliding_interval.stop+1\
                                       ]

        self._console_widget._clear_temporary_buffer()
        if(hilight):
            sel = (self._sliding_interval.nth, self._index[1])
        else :
            sel = None

        strng = html_tableify(items_m, select=sel, header=head, footer=foot)
        self._console_widget._fill_temporary_buffer(self._old_cursor, strng, html=True)

    #--------------------------------------------------------------------------
    # Protected interface
    #--------------------------------------------------------------------------

    def _complete_current(self):
        """ Perform the completion with the currently selected item.
        """
        i = self._index
        item = self._items[i[0]][i[1]]
        item = item.strip()
        if item :
            self._current_text_cursor().insertText(item)
        self.cancel_completion()

    def _current_text_cursor(self):
        """ Returns a cursor with text between the start position and the
            current position selected.
        """
        cursor = self._text_edit.textCursor()
        if cursor.position() >= self._start_position:
            cursor.setPosition(self._start_position,
                               QtGui.QTextCursor.KeepAnchor)
        return cursor


"""Count the frequencies of words in a string"""

from __future__ import division
from __future__ import print_function

import cmath as math

def wordfreq(text, is_filename=False):
    """Return a dictionary of words and word counts in a string."""
    if is_filename:
        with open(text) as f:
            text = f.read()
    freqs = {}
    for word in text.split():
        lword = word.lower()
        freqs[lword] = freqs.get(lword, 0) + 1
    return freqs

def print_wordfreq(freqs, n=10):
    """Print the n most common words and counts in the freqs dict."""

    words, counts = freqs.keys(), freqs.values()
    items = zip(counts, words)
    items.sort(reverse=True)
    for (count, word) in items[:n]:
        print(word, count)

def wordfreq_to_weightsize(worddict, minsize=25, maxsize=50, minalpha=0.5, maxalpha=1.0):
    mincount = min(worddict.itervalues())
    maxcount = max(worddict.itervalues())
    weights = {}
    for k, v in worddict.iteritems():
        w = (v-mincount)/(maxcount-mincount)
        alpha = minalpha + (maxalpha-minalpha)*w
        size = minsize + (maxsize-minsize)*w
        weights[k] = (alpha, size)
    return weights

def tagcloud(worddict, n=10, minsize=25, maxsize=50, minalpha=0.5, maxalpha=1.0):
    from matplotlib import pyplot as plt
    import random

    worddict = wordfreq_to_weightsize(worddict, minsize, maxsize, minalpha, maxalpha)

    fig = plt.figure()
    ax = fig.add_subplot(111)
    ax.set_position([0.0,0.0,1.0,1.0])
    plt.xticks([])
    plt.yticks([])

    words = worddict.keys()
    alphas = [v[0] for v in worddict.values()]
    sizes = [v[1] for v in worddict.values()]
    items = zip(alphas, sizes, words)
    items.sort(reverse=True)
    for alpha, size, word in items[:n]:
        # xpos = random.normalvariate(0.5, 0.3)
        # ypos = random.normalvariate(0.5, 0.3)
        xpos = random.uniform(0.0,1.0)
        ypos = random.uniform(0.0,1.0)
        ax.text(xpos, ypos, word.lower(), alpha=alpha, fontsize=size)
    ax.autoscale_view()
    return ax


# encoding: utf-8
"""
Job and task components for writing .xml files that the Windows HPC Server
2008 can use to start jobs.

Authors:

* Brian Granger
* MinRK

"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import os
import re
import uuid

from xml.etree import ElementTree as ET

from IPython.config.configurable import Configurable
from IPython.utils.traitlets import (
    Unicode, Integer, List, Instance,
    Enum, Bool
)

#-----------------------------------------------------------------------------
# Job and Task classes
#-----------------------------------------------------------------------------

def as_str(value):
    if isinstance(value, str):
        return value
    elif isinstance(value, bool):
        if value:
            return 'true'
        else:
            return 'false'
    elif isinstance(value, (int, float)):
        return repr(value)
    else:
        return value

def indent(elem, level=0):
    i = "\n" + level*"  "
    if len(elem):
        if not elem.text or not elem.text.strip():
            elem.text = i + "  "
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
        for elem in elem:
            indent(elem, level+1)
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
    else:
        if level and (not elem.tail or not elem.tail.strip()):
            elem.tail = i

def find_username():
    domain = os.environ.get('USERDOMAIN')
    username = os.environ.get('USERNAME','')
    if domain is None:
        return username
    else:
        return '%s\\%s' % (domain, username)

class WinHPCJob(Configurable):

    job_id = Unicode('')
    job_name = Unicode('MyJob', config=True)
    min_cores = Integer(1, config=True)
    max_cores = Integer(1, config=True)
    min_sockets = Integer(1, config=True)
    max_sockets = Integer(1, config=True)
    min_nodes = Integer(1, config=True)
    max_nodes = Integer(1, config=True)
    unit_type = Unicode("Core", config=True)
    auto_calculate_min = Bool(True, config=True)
    auto_calculate_max = Bool(True, config=True)
    run_until_canceled = Bool(False, config=True)
    is_exclusive = Bool(False, config=True)
    username = Unicode(find_username(), config=True)
    job_type = Unicode('Batch', config=True)
    priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
        default_value='Highest', config=True)
    requested_nodes = Unicode('', config=True)
    project = Unicode('IPython', config=True)
    xmlns = Unicode('http://schemas.microsoft.com/HPCS2008/scheduler/')
    version = Unicode("2.000")
    tasks = List([])

    @property
    def owner(self):
        return self.username

    def _write_attr(self, root, attr, key):
        s = as_str(getattr(self, attr, ''))
        if s:
            root.set(key, s)

    def as_element(self):
        # We have to add _A_ type things to get the right order than
        # the MSFT XML parser expects.
        root = ET.Element('Job')
        self._write_attr(root, 'version', '_A_Version')
        self._write_attr(root, 'job_name', '_B_Name')
        self._write_attr(root, 'unit_type', '_C_UnitType')
        self._write_attr(root, 'min_cores', '_D_MinCores')
        self._write_attr(root, 'max_cores', '_E_MaxCores')
        self._write_attr(root, 'min_sockets', '_F_MinSockets')
        self._write_attr(root, 'max_sockets', '_G_MaxSockets')
        self._write_attr(root, 'min_nodes', '_H_MinNodes')
        self._write_attr(root, 'max_nodes', '_I_MaxNodes')
        self._write_attr(root, 'run_until_canceled', '_J_RunUntilCanceled')
        self._write_attr(root, 'is_exclusive', '_K_IsExclusive')
        self._write_attr(root, 'username', '_L_UserName')
        self._write_attr(root, 'job_type', '_M_JobType')
        self._write_attr(root, 'priority', '_N_Priority')
        self._write_attr(root, 'requested_nodes', '_O_RequestedNodes')
        self._write_attr(root, 'auto_calculate_max', '_P_AutoCalculateMax')
        self._write_attr(root, 'auto_calculate_min', '_Q_AutoCalculateMin')
        self._write_attr(root, 'project', '_R_Project')
        self._write_attr(root, 'owner', '_S_Owner')
        self._write_attr(root, 'xmlns', '_T_xmlns')
        dependencies = ET.SubElement(root, "Dependencies")
        etasks = ET.SubElement(root, "Tasks")
        for t in self.tasks:
            etasks.append(t.as_element())
        return root

    def tostring(self):
        """Return the string representation of the job description XML."""
        root = self.as_element()
        indent(root)
        txt = ET.tostring(root, encoding="utf-8")
        # Now remove the tokens used to order the attributes.
        txt = re.sub(r'_[A-Z]_','',txt)
        txt = '<?xml version="1.0" encoding="utf-8"?>\n' + txt
        return txt

    def write(self, filename):
        """Write the XML job description to a file."""
        txt = self.tostring()
        with open(filename, 'w') as f:
            f.write(txt)

    def add_task(self, task):
        """Add a task to the job.

        Parameters
        ----------
        task : :class:`WinHPCTask`
            The task object to add.
        """
        self.tasks.append(task)

class WinHPCTask(Configurable):

    task_id = Unicode('')
    task_name = Unicode('')
    version = Unicode("2.000")
    min_cores = Integer(1, config=True)
    max_cores = Integer(1, config=True)
    min_sockets = Integer(1, config=True)
    max_sockets = Integer(1, config=True)
    min_nodes = Integer(1, config=True)
    max_nodes = Integer(1, config=True)
    unit_type = Unicode("Core", config=True)
    command_line = Unicode('', config=True)
    work_directory = Unicode('', config=True)
    is_rerunnaable = Bool(True, config=True)
    std_out_file_path = Unicode('', config=True)
    std_err_file_path = Unicode('', config=True)
    is_parametric = Bool(False, config=True)
    environment_variables = Instance(dict, args=(), config=True)

    def _write_attr(self, root, attr, key):
        s = as_str(getattr(self, attr, ''))
        if s:
            root.set(key, s)

    def as_element(self):
        root = ET.Element('Task')
        self._write_attr(root, 'version', '_A_Version')
        self._write_attr(root, 'task_name', '_B_Name')
        self._write_attr(root, 'min_cores', '_C_MinCores')
        self._write_attr(root, 'max_cores', '_D_MaxCores')
        self._write_attr(root, 'min_sockets', '_E_MinSockets')
        self._write_attr(root, 'max_sockets', '_F_MaxSockets')
        self._write_attr(root, 'min_nodes', '_G_MinNodes')
        self._write_attr(root, 'max_nodes', '_H_MaxNodes')
        self._write_attr(root, 'command_line', '_I_CommandLine')
        self._write_attr(root, 'work_directory', '_J_WorkDirectory')
        self._write_attr(root, 'is_rerunnaable', '_K_IsRerunnable')
        self._write_attr(root, 'std_out_file_path', '_L_StdOutFilePath')
        self._write_attr(root, 'std_err_file_path', '_M_StdErrFilePath')
        self._write_attr(root, 'is_parametric', '_N_IsParametric')
        self._write_attr(root, 'unit_type', '_O_UnitType')
        root.append(self.get_env_vars())
        return root

    def get_env_vars(self):
        env_vars = ET.Element('EnvironmentVariables')
        for k, v in self.environment_variables.iteritems():
            variable = ET.SubElement(env_vars, "Variable")
            name = ET.SubElement(variable, "Name")
            name.text = k
            value = ET.SubElement(variable, "Value")
            value.text = v
        return env_vars

# By declaring these, we can configure the controller and engine separately!

class IPControllerJob(WinHPCJob):
    job_name = Unicode('IPController', config=False)
    is_exclusive = Bool(False, config=True)
    username = Unicode(find_username(), config=True)
    priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
        default_value='Highest', config=True)
    requested_nodes = Unicode('', config=True)
    project = Unicode('IPython', config=True)

class IPEngineSetJob(WinHPCJob):
    job_name = Unicode('IPEngineSet', config=False)
    is_exclusive = Bool(False, config=True)
    username = Unicode(find_username(), config=True)
    priority = Enum(('Lowest','BelowNormal','Normal','AboveNormal','Highest'),
        default_value='Highest', config=True)
    requested_nodes = Unicode('', config=True)
    project = Unicode('IPython', config=True)

class IPControllerTask(WinHPCTask):

    task_name = Unicode('IPController', config=True)
    controller_cmd = List(['ipcontroller.exe'], config=True)
    controller_args = List(['--log-to-file', '--log-level=40'], config=True)
    # I don't want these to be configurable
    std_out_file_path = Unicode('', config=False)
    std_err_file_path = Unicode('', config=False)
    min_cores = Integer(1, config=False)
    max_cores = Integer(1, config=False)
    min_sockets = Integer(1, config=False)
    max_sockets = Integer(1, config=False)
    min_nodes = Integer(1, config=False)
    max_nodes = Integer(1, config=False)
    unit_type = Unicode("Core", config=False)
    work_directory = Unicode('', config=False)

    def __init__(self, config=None):
        super(IPControllerTask, self).__init__(config=config)
        the_uuid = uuid.uuid1()
        self.std_out_file_path = os.path.join('log','ipcontroller-%s.out' % the_uuid)
        self.std_err_file_path = os.path.join('log','ipcontroller-%s.err' % the_uuid)

    @property
    def command_line(self):
        return ' '.join(self.controller_cmd + self.controller_args)

class IPEngineTask(WinHPCTask):

    task_name = Unicode('IPEngine', config=True)
    engine_cmd = List(['ipengine.exe'], config=True)
    engine_args = List(['--log-to-file', '--log-level=40'], config=True)
    # I don't want these to be configurable
    std_out_file_path = Unicode('', config=False)
    std_err_file_path = Unicode('', config=False)
    min_cores = Integer(1, config=False)
    max_cores = Integer(1, config=False)
    min_sockets = Integer(1, config=False)
    max_sockets = Integer(1, config=False)
    min_nodes = Integer(1, config=False)
    max_nodes = Integer(1, config=False)
    unit_type = Unicode("Core", config=False)
    work_directory = Unicode('', config=False)

    def __init__(self, config=None):
        super(IPEngineTask,self).__init__(config=config)
        the_uuid = uuid.uuid1()
        self.std_out_file_path = os.path.join('log','ipengine-%s.out' % the_uuid)
        self.std_err_file_path = os.path.join('log','ipengine-%s.err' % the_uuid)

    @property
    def command_line(self):
        return ' '.join(self.engine_cmd + self.engine_args)

# j = WinHPCJob(None)
# j.job_name = 'IPCluster'
# j.username = 'GNET\\bgranger'
# j.requested_nodes = 'GREEN'
#
# t = WinHPCTask(None)
# t.task_name = 'Controller'
# t.command_line = r"\\blue\domainusers$\bgranger\Python\Python25\Scripts\ipcontroller.exe --log-to-file -p default --log-level 10"
# t.work_directory = r"\\blue\domainusers$\bgranger\.ipython\cluster_default"
# t.std_out_file_path = 'controller-out.txt'
# t.std_err_file_path = 'controller-err.txt'
# t.environment_variables['PYTHONPATH'] = r"\\blue\domainusers$\bgranger\Python\Python25\Lib\site-packages"
# j.add_task(t)


"""
This module contains the main Timeline class.
"""
import requests
from cerberus import Validator as _Validator

from pypebbleapi import __version__, schemas

PEBBLE_API_ROOT = 'https://timeline-api.getpebble.com'
""" Default Pebble API root URL """

PEBBLE_CODES = {
    400: 'The pin object submitted was invalid.',
    403: 'The API key submitted was invalid.',
    410: 'The user token submitted was invalid or does not exist.',
    429: 'Server is sending updates too quickly.',
    503: 'Could not save pin due to a temporary server error.',
}
""" Mapping between error status codes and reason. """

def _raise_for_status(response):
    status = response.status_code
    if 400 <= response.status_code < 500:
        try:
            body = response.json()
            response.reason = body['errorMessage']
            return response.raise_for_status()
        except (KeyError, ValueError):
            pass

        try:
            response.reason = PEBBLE_CODES[status]
            return response.raise_for_status()
        except KeyError:
            pass

    response.raise_for_status()

def _request(method, url, api_key=None, user_token=None, topics_list=None,
        json=None, user_agent=None):
    headers = {}
    if api_key:
        headers['X-API-Key'] = api_key
    if user_token:
        headers['X-User-Token'] = user_token
    if topics_list:
        headers['X-PIN-Topics'] = ','.join(t for t in topics_list)
    if user_agent:
        headers['User-Agent'] = user_agent

    return requests.request(method, url, headers=headers, json=json)

def validate_pin(pin):
    """ Validate the given pin against the schema.

    :param dict pin: The pin to validate:
    :raises pypebbleapi.schemas.DocumentError: If the pin is not valid.
    """
    v = _Validator(schemas.pin)
    if v.validate(pin):
        return
    else:
        raise schemas.DocumentError(errors=v.errors)

class Timeline(object):
    """
    Timeline class used to send or delete pins and to access users' subscriptions.

    :param str api_key: Your application api key (either for production or sandbox)
    :param str api_root: Base URL to connect to. You should probably leave it to the default.
    """
    user_agent = '''pypebbleapi/{}'''.format(__version__)

    @property
    def api_key(self):
        return self._api_key

    @property
    def api_root(self):
        return self._api_root

    def __init__(self, api_key=None, api_root=PEBBLE_API_ROOT):
        self._api_key = api_key

        self._api_root = api_root

    def url_v1(self, partial_url=None):
        return '{}/v1{}'.format(self.api_root, partial_url)

    def send_shared_pin(self, topics, pin, skip_validation=False):
        """
        Send a shared pin for the given topics.

        :param list topics: The list of topics.
        :param dict pin: The pin.
        :param bool skip_validation: Whether to skip the validation.
        :raises pypebbleapi.schemas.DocumentError: If the validation process failed.
        :raises `requests.exceptions.HTTPError`: If an HTTP error occurred.
        """
        if not self.api_key:
            raise ValueError("You need to specify an api_key.")
        if not skip_validation:
            validate_pin(pin)

        response = _request('PUT',
            url=self.url_v1('/shared/pins/' + pin['id']),
            user_agent=self.user_agent,
            api_key=self.api_key,
            topics_list=topics,
            json=pin,
        )
        _raise_for_status(response)

    def delete_shared_pin(self, pin_id):
        """
        Delete a shared pin.

        :param str pin_id: The id of the pin to delete.
        :raises `requests.exceptions.HTTPError`: If an HTTP error occurred.
        """
        if not self.api_key:
            raise ValueError("You need to specify an api_key.")

        response = _request('DELETE',
            url=self.url_v1('/shared/pins/' + pin_id),
            user_agent=self.user_agent,
            api_key=self.api_key,
        )
        _raise_for_status(response)

    def send_user_pin(self, user_token, pin, skip_validation=False):
        """
        Send a user pin.

        :param str user_token: The token of the user.
        :param dict pin: The pin.
        :param bool skip_validation: Whether to skip the validation.
        :raises pypebbleapi.schemas.DocumentError: If the validation process failed.
        :raises `requests.exceptions.HTTPError`: If an HTTP error occurred.
        """
        if not skip_validation:
            validate_pin(pin)

        response = _request('PUT',
            url=self.url_v1('/user/pins/' + pin['id']),
            user_agent=self.user_agent,
            user_token=user_token,
            json=pin,
        )
        _raise_for_status(response)

    def delete_user_pin(self, user_token, pin_id):
        """
        Delete a user pin.

        :param str user_token: The token of the user.
        :param str pin_id: The id of the pin to delete.
        :raises `requests.exceptions.HTTPError`: If an HTTP error occurred.
        """

        response = _request('DELETE',
            url=self.url_v1('/user/pins/' + pin_id),
            user_agent=self.user_agent,
            user_token=user_token,
        )
        _raise_for_status(response)

    def subscribe(self, user_token, topic):
        """
        Subscribe a user to the given topic.

        :param str user_token: The token of the user.
        :param str topic: The topic.
        :raises `requests.exceptions.HTTPError`: If an HTTP error occurred.
        """
        response = _request('POST',
            url=self.url_v1('/user/subscriptions/' + topic),
            user_agent=self.user_agent,
            user_token=user_token,
        )
        _raise_for_status(response)

    def unsubscribe(self, user_token, topic):
        """
        Unsubscribe a user from the given topic.

        :param str user_token: The token of the user.
        :param str topic: The topic.
        :raises `requests.exceptions.HTTPError`: If an HTTP error occurred.
        """
        response = _request('DELETE',
            url=self.url_v1('/user/subscriptions/' + topic),
            user_agent=self.user_agent,
            user_token=user_token,
        )
        _raise_for_status(response)

    def list_subscriptions(self, user_token):
        """
        Get the list of the topics which a user is subscribed to.

        :param str user_token: The token of the user.
        :return: The list of the topics.
        :rtype: list
        :raises `requests.exceptions.HTTPError`: If an HTTP error occurred.
        """
        response = _request('GET',
            url=self.url_v1('/user/subscriptions'),
            user_agent=self.user_agent,
            user_token=user_token,
        )
        _raise_for_status(response)

        return response.json()['topics']


import inspect
import logging
from contextlib import contextmanager

log = logging.getLogger(__name__)

from functools import wraps

def monitored(total: int, name=None, message=None):
    """
    Decorate a function to automatically begin and end a task on the progressmonitor.
    The function must have a parameter called 'monitor'
    """
    def decorator(f):
        nonlocal name
        monitor_index = list(inspect.signature(f).parameters.keys()).index('monitor')
        if name is None:
            name=f.__name__
        @wraps(f)
        def wrapper(*args, **kargs):
            if len(args) > monitor_index:
                monitor = args[monitor_index]
            elif 'monitor' in kargs:
                monitor = kargs['monitor']
            else:
                monitor = kargs['monitor'] = NullMonitor()
            with monitor.task(total, name, message):
                f(*args, **kargs)
        return wrapper
    return decorator

class ProgressMonitor(object):
    """
    A Progress Monitor keeps track of pogress of tasks with optional subtasks.

    Usage:
    m = ProgressMonitor()
    m.begin(total_units, [task_name], [message])
    m.update(units[, message])
    [m.done()]

    When using sub monitors, an amount of work of this monitor (e.g. 50) is delegated to the submonitor,
    who has its own total_units (e.g. 10):

    sm = m.submonitor(50)
    sm.begin(10, "subtask name")
    sm.update(1)  # this will 'update' 5 units of m's work
    sm.done()     # this will 'update' the remainig units of m's work

    You can also use the context managers .task and .subtask and @monitored decorator:

    @monitored(100)
    def supertask(monitor):
        ...
        with monitor.submonitor(50) as sm:
            subtask(sm)

    @monitored(10, name="My first subtask")
    def subtask(monitor):
        monitor.update(5)  # this will update 5/10 * 50 = 25 of supertask's work
    """

    def __init__(self):
        self.total = None  # need to call begin before starting!
        self.name = None  # name is given with begin(.)
        self.worked = 0
        self.listeners = set()
        self.sub_monitors = {}  # monitor : units of (my) work

    def begin(self, total: int, name=None, message=None):
        """Call before starting work on a monitor, specifying name and amount of work"""
        self.total = total
        message = message or name or "Working..."
        self.name = name or "ProgressMonitor"
        self.update(0, message)

    @contextmanager
    def task(self, total: int, name=None, message=None):
        """Wrap code into a begin and end call on this monitor"""
        self.begin(total, name, message)
        try:
            yield self
        finally:
            self.done()

    @contextmanager
    def subtask(self, units: int):
        """Create a submonitor with the given units"""
        sm = self.submonitor(units)
        try:
            yield sm
        finally:
            if sm.total is None:
                # begin was never called, so the subtask cannot be closed
                self.update(units)
            else:
                sm.done()

    @property
    def progress(self)-> float:
        """What percentage (range 0-1) of work is done (including submonitors)"""
        if self.total is None:
            return 0
        my_progress = self.worked
        my_progress += sum(s.progress * weight
                           for (s, weight) in self.sub_monitors.items())
        return min(1, my_progress / self.total)

    def update(self, units: int=1, message: str=None):
        """Increment the monitor with N units worked and an optional message"""
        if self.total is None:
            raise Exception("Cannot call progressmonitor.update before calling begin")
        self.worked = min(self.total, self.worked+units)
        if message:
            self.message = message
        for listener in self.listeners:
            listener(self)

    def add_listener(self, func):
        """Add a function to listen to updates"""
        self.listeners.add(func)

    def remove_listener(self, func):
        """REmove a function from the listeners"""
        self.listeners.remove(func)

    def submonitor(self, units: int, *args, **kargs) -> 'ProgressMonitor':
        """
        Create a sub monitor that stands for N units of work in this monitor
        The sub task should call .begin (or use @monitored / with .task) before calling updates
        """
        submonitor = ProgressMonitor(*args, **kargs)
        self.sub_monitors[submonitor] = units
        submonitor.add_listener(self._submonitor_update)
        return submonitor

    def _submonitor_update(self, submonitor):
        self.update(0, submonitor.message)

    def done(self, message: str=None):
        """
        Signal that this task is done.
        This is completely optional and will just call .update with the remaining work.
        """
        if message is None:
            message = "{self.name} done".format(**locals()) if self.name else "Done"
        self.update(units=self.total - self.worked, message=message)

    @property
    def is_done(self):
        tot = self.worked
        for submonitor, weight in self.sub_monitors.items():
            if submonitor.is_done:
                tot += weight
        return tot >= self.total

class NullMonitor(ProgressMonitor):
    """Dummy monitor that will accept all calls and do nothing"""
    def submonitor(self, total, *args, **kargs):
        return NullMonitor()

    def update(self, *args, **kargs):
        pass


# encoding: utf-8
"""
A payload based version of page.

Authors:

* Brian Granger
* Fernando Perez
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

# Third-party
try:
    from docutils.core import publish_string
except ImportError:
    # html paging won't be available, but we don't raise any errors.  It's a
    # purely optional feature.
    pass

# Our own
from IPython.core.interactiveshell import InteractiveShell

#-----------------------------------------------------------------------------
# Classes and functions
#-----------------------------------------------------------------------------

def page(strng, start=0, screen_lines=0, pager_cmd=None,
         html=None, auto_html=False):
    """Print a string, piping through a pager.

    This version ignores the screen_lines and pager_cmd arguments and uses
    IPython's payload system instead.

    Parameters
    ----------
    strng : str
      Text to page.

    start : int
      Starting line at which to place the display.

    html : str, optional
      If given, an html string to send as well.

    auto_html : bool, optional
      If true, the input string is assumed to be valid reStructuredText and is
      converted to HTML with docutils.  Note that if docutils is not found,
      this option is silently ignored.

    Note
    ----

    Only one of the ``html`` and ``auto_html`` options can be given, not
    both.
    """

    # Some routines may auto-compute start offsets incorrectly and pass a
    # negative value.  Offset to 0 for robustness.
    start = max(0, start)
    shell = InteractiveShell.instance()

    if auto_html:
        try:
            # These defaults ensure user configuration variables for docutils
            # are not loaded, only our config is used here.
            defaults = {'file_insertion_enabled': 0,
                        'raw_enabled': 0,
                        '_disable_config': 1}
            html = publish_string(strng, writer_name='html',
                                  settings_overrides=defaults)
        except:
            pass

    payload = dict(
        source='IPython.zmq.page.page',
        text=strng,
        html=html,
        start_line_number=start
        )
    shell.payload_manager.write_payload(payload)

def install_payload_page():
    """Install this version of page as IPython.core.page.page."""
    from IPython.core import page as corepage
    corepage.page = page