"""Word completion for IPython.

This module is a fork of the rlcompleter module in the Python standard
library.  The original enhancements made to rlcompleter have been sent
upstream and were accepted as of Python 2.3, but we need a lot more
functionality specific to IPython, so this module will continue to live as an
IPython-specific utility.

Original rlcompleter documentation:

This requires the latest extension to the readline module (the
completes keywords, built-ins and globals in __main__; when completing
NAME.NAME..., it evaluates (!) the expression up to the last dot and
completes its attributes.

It's very cool to do "import string" type "string.", hit the
completion key (twice), and see the list of names defined by the
string module!

Tip: to use the tab key as the completion key, call

    readline.parse_and_bind("tab: complete")

Notes:

- Exceptions raised by the completer function are *ignored* (and
generally cause the completion to fail).  This is a feature -- since
readline sets the tty device in raw (or cbreak) mode, printing a
traceback wouldn't work well without some complicated hoopla to save,
reset and restore the tty state.

- The evaluation of the NAME.NAME... form may cause arbitrary
application defined code to be executed if an object with a
__getattr__ hook is found.  Since it is the responsibility of the
application (or the user) to enable this feature, I consider this an
acceptable risk.  More complicated expressions (e.g. function calls or
indexing operations) are *not* evaluated.

- GNU readline is also used by the built-in functions input() and
raw_input(), and thus these also benefit/suffer from the completer
features.  Clearly an interactive application can benefit by
specifying its own completer function and using raw_input() for all
its input.

- When the original stdin is not a tty device, GNU readline is never
used, and this module (and the readline module) are silently inactive.
"""

#*****************************************************************************
#
# Since this file is essentially a minimally modified copy of the rlcompleter
# module which is part of the standard Python distribution, I assume that the
# proper procedure is to maintain its copyright as belonging to the Python
# Software Foundation (in addition to my own, for all new code).
#
#       Copyright (C) 2008 IPython Development Team
#       Copyright (C) 2001 Fernando Perez. <fperez@colorado.edu>
#       Copyright (C) 2001 Python Software Foundation, www.python.org
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#
#*****************************************************************************

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import __builtin__
import __main__
import glob
import inspect
import itertools
import keyword
import os
import re
import shlex
import sys

from IPython.config.configurable import Configurable
from IPython.core.error import TryNext
from IPython.core.inputsplitter import ESC_MAGIC
from IPython.utils import generics
from IPython.utils import io
from IPython.utils.dir2 import dir2
from IPython.utils.process import arg_split
from IPython.utils.traitlets import CBool, Enum

#-----------------------------------------------------------------------------
# Globals
#-----------------------------------------------------------------------------

# Public API
__all__ = ['Completer','IPCompleter']

if sys.platform == 'win32':
    PROTECTABLES = ' '
else:
    PROTECTABLES = ' ()[]{}?=\\|;:\'#*"^&'

#-----------------------------------------------------------------------------
# Main functions and classes
#-----------------------------------------------------------------------------

def has_open_quotes(s):
    """Return whether a string has open quotes.

    This simply counts whether the number of quote characters of either type in
    the string is odd.

    Returns
    -------
    If there is an open quote, the quote character is returned.  Else, return
    False.
    """
    # We check " first, then ', so complex cases with nested quotes will get
    # the " to take precedence.
    if s.count('"') % 2:
        return '"'
    elif s.count("'") % 2:
        return "'"
    else:
        return False

def protect_filename(s):
    """Escape a string to protect certain characters."""

    return "".join([(ch in PROTECTABLES and '\\' + ch or ch)
                    for ch in s])

def expand_user(path):
    """Expand '~'-style usernames in strings.

    This is similar to :func:`os.path.expanduser`, but it computes and returns
    extra information that will be useful if the input was being used in
    computing completions, and you wish to return the completions with the
    original '~' instead of its expanded value.

    Parameters
    ----------
    path : str
      String to be expanded.  If no ~ is present, the output is the same as the
      input.

    Returns
    -------
    newpath : str
      Result of ~ expansion in the input path.
    tilde_expand : bool
      Whether any expansion was performed or not.
    tilde_val : str
      The value that ~ was replaced with.
    """
    # Default values
    tilde_expand = False
    tilde_val = ''
    newpath = path

    if path.startswith('~'):
        tilde_expand = True
        rest = len(path)-1
        newpath = os.path.expanduser(path)
        if rest:
            tilde_val = newpath[:-rest]
        else:
            tilde_val = newpath

    return newpath, tilde_expand, tilde_val

def compress_user(path, tilde_expand, tilde_val):
    """Does the opposite of expand_user, with its outputs.
    """
    if tilde_expand:
        return path.replace(tilde_val, '~')
    else:
        return path

class Bunch(object): pass

DELIMS = ' \t\n`!@#$^&*()=+[{]}\\|;:\'",<>?'
GREEDY_DELIMS = ' \r\n'

class CompletionSplitter(object):
    """An object to split an input line in a manner similar to readline.

    By having our own implementation, we can expose readline-like completion in
    a uniform manner to all frontends.  This object only needs to be given the
    line of text to be split and the cursor position on said line, and it
    returns the 'word' to be completed on at the cursor after splitting the
    entire line.

    What characters are used as splitting delimiters can be controlled by
    setting the `delims` attribute (this is a property that internally
    automatically builds the necessary regular expression)"""

    # Private interface

    # A string of delimiter characters.  The default value makes sense for
    # IPython's most typical usage patterns.
    _delims = DELIMS

    # The expression (a normal string) to be compiled into a regular expression
    # for actual splitting.  We store it as an attribute mostly for ease of
    # debugging, since this type of code can be so tricky to debug.
    _delim_expr = None

    # The regular expression that does the actual splitting
    _delim_re = None

    def __init__(self, delims=None):
        delims = CompletionSplitter._delims if delims is None else delims
        self.delims = delims

    @property
    def delims(self):
        """Return the string of delimiter characters."""
        return self._delims

    @delims.setter
    def delims(self, delims):
        """Set the delimiters for line splitting."""
        expr = '[' + ''.join('\\'+ c for c in delims) + ']'
        self._delim_re = re.compile(expr)
        self._delims = delims
        self._delim_expr = expr

    def split_line(self, line, cursor_pos=None):
        """Split a line of text with a cursor at the given position.
        """
        l = line if cursor_pos is None else line[:cursor_pos]
        return self._delim_re.split(l)[-1]

class Completer(Configurable):

    greedy = CBool(False, config=True,
        help="""Activate greedy completion

        This will enable completion on elements of lists, results of function calls, etc.,
        but can be unsafe because the code is actually evaluated on TAB.
        """
    )

    def __init__(self, namespace=None, global_namespace=None, config=None, **kwargs):
        """Create a new completer for the command line.

        Completer(namespace=ns,global_namespace=ns2) -> completer instance.

        If unspecified, the default namespace where completions are performed
        is __main__ (technically, __main__.__dict__). Namespaces should be
        given as dictionaries.

        An optional second namespace can be given.  This allows the completer
        to handle cases where both the local and global scopes need to be
        distinguished.

        Completer instances should be used as the completion mechanism of
        readline via the set_completer() call:

        readline.set_completer(Completer(my_namespace).complete)
        """

        # Don't bind to namespace quite yet, but flag whether the user wants a
        # specific namespace or to use __main__.__dict__. This will allow us
        # to bind to __main__.__dict__ at completion time, not now.
        if namespace is None:
            self.use_main_ns = 1
        else:
            self.use_main_ns = 0
            self.namespace = namespace

        # The global namespace, if given, can be bound directly
        if global_namespace is None:
            self.global_namespace = {}
        else:
            self.global_namespace = global_namespace

        super(Completer, self).__init__(config=config, **kwargs)

    def complete(self, text, state):
        """Return the next possible completion for 'text'.

        This is called successively with state == 0, 1, 2, ... until it
        returns None.  The completion should begin with 'text'.

        """
        if self.use_main_ns:
            self.namespace = __main__.__dict__

        if state == 0:
            if "." in text:
                self.matches = self.attr_matches(text)
            else:
                self.matches = self.global_matches(text)
        try:
            return self.matches[state]
        except IndexError:
            return None

    def global_matches(self, text):
        """Compute matches when text is a simple name.

        Return a list of all keywords, built-in functions and names currently
        defined in self.namespace or self.global_namespace that match.

        """
        #print 'Completer->global_matches, txt=%r' % text # dbg
        matches = []
        match_append = matches.append
        n = len(text)
        for lst in [keyword.kwlist,
                    __builtin__.__dict__.keys(),
                    self.namespace.keys(),
                    self.global_namespace.keys()]:
            for word in lst:
                if word[:n] == text and word != "__builtins__":
                    match_append(word)
        return matches

    def attr_matches(self, text):
        """Compute matches when text contains a dot.

        Assuming the text is of the form NAME.NAME....[NAME], and is
        evaluatable in self.namespace or self.global_namespace, it will be
        evaluated and its attributes (as revealed by dir()) are used as
        possible completions.  (For class instances, class members are are
        also considered.)

        WARNING: this can still invoke arbitrary C code, if an object
        with a __getattr__ hook is evaluated.

        """

        #io.rprint('Completer->attr_matches, txt=%r' % text) # dbg
        # Another option, seems to work great. Catches things like ''.<tab>
        m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text)

        if m:
            expr, attr = m.group(1, 3)
        elif self.greedy:
            m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer)
            if not m2:
                return []
            expr, attr = m2.group(1,2)
        else:
            return []

        try:
            obj = eval(expr, self.namespace)
        except:
            try:
                obj = eval(expr, self.global_namespace)
            except:
                return []

        if self.limit_to__all__ and hasattr(obj, '__all__'):
            words = get__all__entries(obj)
        else:
            words = dir2(obj)

        try:
            words = generics.complete_object(obj, words)
        except TryNext:
            pass
        except Exception:
            # Silence errors from completion function
            #raise # dbg
            pass
        # Build match list to return
        n = len(attr)
        res = ["%s.%s" % (expr, w) for w in words if w[:n] == attr ]
        return res

def get__all__entries(obj):
    """returns the strings in the __all__ attribute"""
    try:
        words = getattr(obj, '__all__')
    except:
        return []

    return [w for w in words if isinstance(w, basestring)]

class IPCompleter(Completer):
    """Extension of the completer class with IPython-specific features"""

    def _greedy_changed(self, name, old, new):
        """update the splitter and readline delims when greedy is changed"""
        if new:
            self.splitter.delims = GREEDY_DELIMS
        else:
            self.splitter.delims = DELIMS

        if self.readline:
            self.readline.set_completer_delims(self.splitter.delims)

    merge_completions = CBool(True, config=True,
        help="""Whether to merge completion results into a single list

        If False, only the completion results from the first non-empty
        completer will be returned.
        """
    )
    omit__names = Enum((0,1,2), default_value=2, config=True,
        help="""Instruct the completer to omit private method names

        Specifically, when completing on ``object.<tab>``.

        When 2 [default]: all names that start with '_' will be excluded.

        When 1: all 'magic' names (``__foo__``) will be excluded.

        When 0: nothing will be excluded.
        """
    )
    limit_to__all__ = CBool(default_value=False, config=True,
        help="""Instruct the completer to use __all__ for the completion

        Specifically, when completing on ``object.<tab>``.

        When True: only those names in obj.__all__ will be included.

        When False [default]: the __all__ attribute is ignored
        """
    )

    def __init__(self, shell=None, namespace=None, global_namespace=None,
                 alias_table=None, use_readline=True,
                 config=None, **kwargs):
        """IPCompleter() -> completer

        Return a completer object suitable for use by the readline library
        via readline.set_completer().

        Inputs:

        - shell: a pointer to the ipython shell itself.  This is needed
        because this completer knows about magic functions, and those can
        only be accessed via the ipython instance.

        - namespace: an optional dict where completions are performed.

        - global_namespace: secondary optional dict for completions, to
        handle cases (such as IPython embedded inside functions) where
        both Python scopes are visible.

        - If alias_table is supplied, it should be a dictionary of aliases
        to complete.

        use_readline : bool, optional
          If true, use the readline library.  This completer can still function
          without readline, though in that case callers must provide some extra
          information on each call about the current line."""

        self.magic_escape = ESC_MAGIC
        self.splitter = CompletionSplitter()

        # Readline configuration, only used by the rlcompleter method.
        if use_readline:
            # We store the right version of readline so that later code
            import IPython.utils.rlineimpl as readline
            self.readline = readline
        else:
            self.readline = None

        # _greedy_changed() depends on splitter and readline being defined:
        Completer.__init__(self, namespace=namespace, global_namespace=global_namespace,
                            config=config, **kwargs)

        # List where completion matches will be stored
        self.matches = []
        self.shell = shell
        if alias_table is None:
            alias_table = {}
        self.alias_table = alias_table
        # Regexp to split filenames with spaces in them
        self.space_name_re = re.compile(r'([^\\] )')
        # Hold a local ref. to glob.glob for speed
        self.glob = glob.glob

        # Determine if we are running on 'dumb' terminals, like (X)Emacs
        # buffers, to avoid completion problems.
        term = os.environ.get('TERM','xterm')
        self.dumb_terminal = term in ['dumb','emacs']

        # Special handling of backslashes needed in win32 platforms
        if sys.platform == "win32":
            self.clean_glob = self._clean_glob_win32
        else:
            self.clean_glob = self._clean_glob

        # All active matcher routines for completion
        self.matchers = [self.python_matches,
                         self.file_matches,
                         self.magic_matches,
                         self.alias_matches,
                         self.python_func_kw_matches,
                         ]

    def all_completions(self, text):
        """
        Wrapper around the complete method for the benefit of emacs
        and pydb.
        """
        return self.complete(text)[1]

    def _clean_glob(self,text):
        return self.glob("%s*" % text)

    def _clean_glob_win32(self,text):
        return [f.replace("\\","/")
                for f in self.glob("%s*" % text)]

    def file_matches(self, text):
        """Match filenames, expanding ~USER type strings.

        Most of the seemingly convoluted logic in this completer is an
        attempt to handle filenames with spaces in them.  And yet it's not
        quite perfect, because Python's readline doesn't expose all of the
        GNU readline details needed for this to be done correctly.

        For a filename with a space in it, the printed completions will be
        only the parts after what's already been typed (instead of the
        full completions, as is normally done).  I don't think with the
        current (as of Python 2.3) Python readline it's possible to do
        better."""

        #io.rprint('Completer->file_matches: <%r>' % text) # dbg

        # chars that require escaping with backslash - i.e. chars
        # that readline treats incorrectly as delimiters, but we
        # don't want to treat as delimiters in filename matching
        # when escaped with backslash
        if text.startswith('!'):
            text = text[1:]
            text_prefix = '!'
        else:
            text_prefix = ''

        text_until_cursor = self.text_until_cursor
        # track strings with open quotes
        open_quotes = has_open_quotes(text_until_cursor)

        if '(' in text_until_cursor or '[' in text_until_cursor:
            lsplit = text
        else:
            try:
                # arg_split ~ shlex.split, but with unicode bugs fixed by us
                lsplit = arg_split(text_until_cursor)[-1]
            except ValueError:
                # typically an unmatched ", or backslash without escaped char.
                if open_quotes:
                    lsplit = text_until_cursor.split(open_quotes)[-1]
                else:
                    return []
            except IndexError:
                # tab pressed on empty line
                lsplit = ""

        if not open_quotes and lsplit != protect_filename(lsplit):
            # if protectables are found, do matching on the whole escaped name
            has_protectables = True
            text0,text = text,lsplit
        else:
            has_protectables = False
            text = os.path.expanduser(text)

        if text == "":
            return [text_prefix + protect_filename(f) for f in self.glob("*")]

        # Compute the matches from the filesystem
        m0 = self.clean_glob(text.replace('\\',''))

        if has_protectables:
            # If we had protectables, we need to revert our changes to the
            # beginning of filename so that we don't double-write the part
            # of the filename we have so far
            len_lsplit = len(lsplit)
            matches = [text_prefix + text0 +
                       protect_filename(f[len_lsplit:]) for f in m0]
        else:
            if open_quotes:
                # if we have a string with an open quote, we don't need to
                # protect the names at all (and we _shouldn't_, as it
                # would cause bugs when the filesystem call is made).
                matches = m0
            else:
                matches = [text_prefix +
                           protect_filename(f) for f in m0]

        #io.rprint('mm', matches)  # dbg

        # Mark directories in input list by appending '/' to their names.
        matches = [x+'/' if os.path.isdir(x) else x for x in matches]
        return matches

    def magic_matches(self, text):
        """Match magics"""
        #print 'Completer->magic_matches:',text,'lb',self.text_until_cursor # dbg
        # Get all shell magics now rather than statically, so magics loaded at
        # runtime show up too.
        lsm = self.shell.magics_manager.lsmagic()
        line_magics = lsm['line']
        cell_magics = lsm['cell']
        pre = self.magic_escape
        pre2 = pre+pre

        # Completion logic:
        # - user gives %%: only do cell magics
        # - user gives %: do both line and cell magics
        # - no prefix: do both
        # In other words, line magics are skipped if the user gives %% explicitly
        bare_text = text.lstrip(pre)
        comp = [ pre2+m for m in cell_magics if m.startswith(bare_text)]
        if not text.startswith(pre2):
            comp += [ pre+m for m in line_magics if m.startswith(bare_text)]
        return comp

    def alias_matches(self, text):
        """Match internal system aliases"""
        #print 'Completer->alias_matches:',text,'lb',self.text_until_cursor # dbg

        # if we are not in the first 'item', alias matching
        # doesn't make sense - unless we are starting with 'sudo' command.
        main_text = self.text_until_cursor.lstrip()
        if ' ' in main_text and not main_text.startswith('sudo'):
            return []
        text = os.path.expanduser(text)
        aliases =  self.alias_table.keys()
        if text == '':
            return aliases
        else:
            return [a for a in aliases if a.startswith(text)]

    def python_matches(self,text):
        """Match attributes or global python names"""

        #io.rprint('Completer->python_matches, txt=%r' % text) # dbg
        if "." in text:
            try:
                matches = self.attr_matches(text)
                if text.endswith('.') and self.omit__names:
                    if self.omit__names == 1:
                        # true if txt is _not_ a __ name, false otherwise:
                        no__name = (lambda txt:
                                    re.match(r'.*\.__.*?__',txt) is None)
                    else:
                        # true if txt is _not_ a _ name, false otherwise:
                        no__name = (lambda txt:
                                    re.match(r'.*\._.*?',txt) is None)
                    matches = filter(no__name, matches)
            except NameError:
                # catches <undefined attributes>.<tab>
                matches = []
        else:
            matches = self.global_matches(text)

        return matches

    def _default_arguments(self, obj):
        """Return the list of default arguments of obj if it is callable,
        or empty list otherwise."""

        if not (inspect.isfunction(obj) or inspect.ismethod(obj)):
            # for classes, check for __init__,__new__
            if inspect.isclass(obj):
                obj = (getattr(obj,'__init__',None) or
                       getattr(obj,'__new__',None))
            # for all others, check if they are __call__able
            elif hasattr(obj, '__call__'):
                obj = obj.__call__
            # XXX: is there a way to handle the builtins ?
        try:
            args,_,_1,defaults = inspect.getargspec(obj)
            if defaults:
                return args[-len(defaults):]
        except TypeError: pass
        return []

    def python_func_kw_matches(self,text):
        """Match named parameters (kwargs) of the last open function"""

        if "." in text: # a parameter cannot be dotted
            return []
        try: regexp = self.__funcParamsRegex
        except AttributeError:
            regexp = self.__funcParamsRegex = re.compile(r'''
                '.*?(?<!\\)' |    # single quoted strings or
                ".*?(?<!\\)" |    # double quoted strings or
                \w+          |    # identifier
                \S                # other characters
                ''', re.VERBOSE | re.DOTALL)
        # 1. find the nearest identifier that comes before an unclosed
        # parenthesis before the cursor
        # e.g. for "foo (1+bar(x), pa<cursor>,a=1)", the candidate is "foo"
        tokens = regexp.findall(self.text_until_cursor)
        tokens.reverse()
        iterTokens = iter(tokens); openPar = 0
        for token in iterTokens:
            if token == ')':
                openPar -= 1
            elif token == '(':
                openPar += 1
                if openPar > 0:
                    # found the last unclosed parenthesis
                    break
        else:
            return []
        # 2. Concatenate dotted names ("foo.bar" for "foo.bar(x, pa" )
        ids = []
        isId = re.compile(r'\w+$').match
        while True:
            try:
                ids.append(iterTokens.next())
                if not isId(ids[-1]):
                    ids.pop(); break
                if not iterTokens.next() == '.':
                    break
            except StopIteration:
                break
        # lookup the candidate callable matches either using global_matches
        # or attr_matches for dotted names
        if len(ids) == 1:
            callableMatches = self.global_matches(ids[0])
        else:
            callableMatches = self.attr_matches('.'.join(ids[::-1]))
        argMatches = []
        for callableMatch in callableMatches:
            try:
                namedArgs = self._default_arguments(eval(callableMatch,
                                                         self.namespace))
            except:
                continue
            for namedArg in namedArgs:
                if namedArg.startswith(text):
                    argMatches.append("%s=" %namedArg)
        return argMatches

    def dispatch_custom_completer(self, text):
        #io.rprint("Custom! '%s' %s" % (text, self.custom_completers)) # dbg
        line = self.line_buffer
        if not line.strip():
            return None

        # Create a little structure to pass all the relevant information about
        # the current completion to any custom completer.
        event = Bunch()
        event.line = line
        event.symbol = text
        cmd = line.split(None,1)[0]
        event.command = cmd
        event.text_until_cursor = self.text_until_cursor

        #print "\ncustom:{%s]\n" % event # dbg

        # for foo etc, try also to find completer for %foo
        if not cmd.startswith(self.magic_escape):
            try_magic = self.custom_completers.s_matches(
                self.magic_escape + cmd)
        else:
            try_magic = []

        for c in itertools.chain(self.custom_completers.s_matches(cmd),
                 try_magic,
                 self.custom_completers.flat_matches(self.text_until_cursor)):
            #print "try",c # dbg
            try:
                res = c(event)
                if res:
                    # first, try case sensitive match
                    withcase = [r for r in res if r.startswith(text)]
                    if withcase:
                        return withcase
                    # if none, then case insensitive ones are ok too
                    text_low = text.lower()
                    return [r for r in res if r.lower().startswith(text_low)]
            except TryNext:
                pass

        return None

    def complete(self, text=None, line_buffer=None, cursor_pos=None):
        """Find completions for the given text and line context.

        This is called successively with state == 0, 1, 2, ... until it
        returns None.  The completion should begin with 'text'.

        Note that both the text and the line_buffer are optional, but at least
        one of them must be given.

        Parameters
        ----------
          text : string, optional
            Text to perform the completion on.  If not given, the line buffer
            is split using the instance's CompletionSplitter object.

          line_buffer : string, optional
            If not given, the completer attempts to obtain the current line
            buffer via readline.  This keyword allows clients which are
            requesting for text completions in non-readline contexts to inform
            the completer of the entire text.

          cursor_pos : int, optional
            Index of the cursor in the full line buffer.  Should be provided by
            remote frontends where kernel has no access to frontend state.

        Returns
        -------
        text : str
          Text that was actually used in the completion.

        matches : list
          A list of completion matches.
        """
        #io.rprint('\nCOMP1 %r %r %r' % (text, line_buffer, cursor_pos))  # dbg

        # if the cursor position isn't given, the only sane assumption we can
        # make is that it's at the end of the line (the common case)
        if cursor_pos is None:
            cursor_pos = len(line_buffer) if text is None else len(text)

        # if text is either None or an empty string, rely on the line buffer
        if not text:
            text = self.splitter.split_line(line_buffer, cursor_pos)

        # If no line buffer is given, assume the input text is all there was
        if line_buffer is None:
            line_buffer = text

        self.line_buffer = line_buffer
        self.text_until_cursor = self.line_buffer[:cursor_pos]
        #io.rprint('COMP2 %r %r %r' % (text, line_buffer, cursor_pos))  # dbg

        # Start with a clean slate of completions
        self.matches[:] = []
        custom_res = self.dispatch_custom_completer(text)
        if custom_res is not None:
            # did custom completers produce something?
            self.matches = custom_res
        else:
            # Extend the list of completions with the results of each
            # matcher, so we return results to the user from all
            # namespaces.
            if self.merge_completions:
                self.matches = []
                for matcher in self.matchers:
                    try:
                        self.matches.extend(matcher(text))
                    except:
                        # Show the ugly traceback if the matcher causes an
                        # exception, but do NOT crash the kernel!
                        sys.excepthook(*sys.exc_info())
            else:
                for matcher in self.matchers:
                    self.matches = matcher(text)
                    if self.matches:
                        break
        # FIXME: we should extend our api to return a dict with completions for
        # different types of objects.  The rlcomplete() method could then
        # simply collapse the dict into a list for readline, but we'd have
        # richer completion semantics in other evironments.
        self.matches = sorted(set(self.matches))
        #io.rprint('COMP TEXT, MATCHES: %r, %r' % (text, self.matches)) # dbg
        return text, self.matches

    def rlcomplete(self, text, state):
        """Return the state-th possible completion for 'text'.

        This is called successively with state == 0, 1, 2, ... until it
        returns None.  The completion should begin with 'text'.

        Parameters
        ----------
          text : string
            Text to perform the completion on.

          state : int
            Counter used by readline.
        """
        if state==0:

            self.line_buffer = line_buffer = self.readline.get_line_buffer()
            cursor_pos = self.readline.get_endidx()

            #io.rprint("\nRLCOMPLETE: %r %r %r" %
            #          (text, line_buffer, cursor_pos) ) # dbg

            # if there is only a tab on a line with only whitespace, instead of
            # the mostly useless 'do you want to see all million completions'
            # message, just do the right thing and give the user his tab!
            # Incidentally, this enables pasting of tabbed text from an editor
            # (as long as autoindent is off).

            # It should be noted that at least pyreadline still shows file
            # completions - is there a way around it?

            # don't apply this on 'dumb' terminals, such as emacs buffers, so
            # we don't interfere with their own tab-completion mechanism.
            if not (self.dumb_terminal or line_buffer.strip()):
                self.readline.insert_text('\t')
                sys.stdout.flush()
                return None

            # Note: debugging exceptions that may occur in completion is very
            # tricky, because readline unconditionally silences them.  So if
            # during development you suspect a bug in the completion code, turn
            # this flag on temporarily by uncommenting the second form (don't
            # flip the value in the first line, as the '# dbg' marker can be
            # automatically detected and is used elsewhere).
            DEBUG = False
            #DEBUG = True # dbg
            if DEBUG:
                try:
                    self.complete(text, line_buffer, cursor_pos)
                except:
                    import traceback; traceback.print_exc()
            else:
                # The normal production version is here

                # This method computes the self.matches array
                self.complete(text, line_buffer, cursor_pos)

        try:
            return self.matches[state]
        except IndexError:
            return None


"""A Task logger that presents our DB interface,
but exists entirely in memory and implemented with dicts.

Authors:

* Min RK

TaskRecords are dicts of the form:
{
    'msg_id' : str(uuid),
    'client_uuid' : str(uuid),
    'engine_uuid' : str(uuid) or None,
    'header' : dict(header),
    'content': dict(content),
    'buffers': list(buffers),
    'submitted': datetime,
    'started': datetime or None,
    'completed': datetime or None,
    'resubmitted': datetime or None,
    'result_header' : dict(header) or None,
    'result_content' : dict(content) or None,
    'result_buffers' : list(buffers) or None,
}
With this info, many of the special categories of tasks can be defined by query:

pending:  completed is None
client's outstanding: client_uuid = uuid && completed is None
MIA: arrived is None (and completed is None)
etc.

EngineRecords are dicts of the form:
{
    'eid' : int(id),
    'uuid': str(uuid)
}
This may be extended, but is currently.

We support a subset of mongodb operators:
    $lt,$gt,$lte,$gte,$ne,$in,$nin,$all,$mod,$exists
"""
#-----------------------------------------------------------------------------
#  Copyright (C) 2010-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

from copy import deepcopy as copy
from datetime import datetime

from IPython.config.configurable import LoggingConfigurable

from IPython.utils.traitlets import Dict, Unicode, Instance

filters = {
 '$lt' : lambda a,b: a < b,
 '$gt' : lambda a,b: b > a,
 '$eq' : lambda a,b: a == b,
 '$ne' : lambda a,b: a != b,
 '$lte': lambda a,b: a <= b,
 '$gte': lambda a,b: a >= b,
 '$in' : lambda a,b: a in b,
 '$nin': lambda a,b: a not in b,
 '$all': lambda a,b: all([ a in bb for bb in b ]),
 '$mod': lambda a,b: a%b[0] == b[1],
 '$exists' : lambda a,b: (b and a is not None) or (a is None and not b)
}

class CompositeFilter(object):
    """Composite filter for matching multiple properties."""

    def __init__(self, dikt):
        self.tests = []
        self.values = []
        for key, value in dikt.iteritems():
            self.tests.append(filters[key])
            self.values.append(value)

    def __call__(self, value):
        for test,check in zip(self.tests, self.values):
            if not test(value, check):
                return False
        return True

class BaseDB(LoggingConfigurable):
    """Empty Parent class so traitlets work on DB."""
    # base configurable traits:
    session = Unicode("")

class DictDB(BaseDB):
    """Basic in-memory dict-based object for saving Task Records.

    This is the first object to present the DB interface
    for logging tasks out of memory.

    The interface is based on MongoDB, so adding a MongoDB
    backend should be straightforward.
    """

    _records = Dict()

    def _match_one(self, rec, tests):
        """Check if a specific record matches tests."""
        for key,test in tests.iteritems():
            if not test(rec.get(key, None)):
                return False
        return True

    def _match(self, check):
        """Find all the matches for a check dict."""
        matches = []
        tests = {}
        for k,v in check.iteritems():
            if isinstance(v, dict):
                tests[k] = CompositeFilter(v)
            else:
                tests[k] = lambda o: o==v

        for rec in self._records.itervalues():
            if self._match_one(rec, tests):
                matches.append(copy(rec))
        return matches

    def _extract_subdict(self, rec, keys):
        """extract subdict of keys"""
        d = {}
        d['msg_id'] = rec['msg_id']
        for key in keys:
            d[key] = rec[key]
        return copy(d)

    def add_record(self, msg_id, rec):
        """Add a new Task Record, by msg_id."""
        if self._records.has_key(msg_id):
            raise KeyError("Already have msg_id %r"%(msg_id))
        self._records[msg_id] = rec

    def get_record(self, msg_id):
        """Get a specific Task Record, by msg_id."""
        if not msg_id in self._records:
            raise KeyError("No such msg_id %r"%(msg_id))
        return copy(self._records[msg_id])

    def update_record(self, msg_id, rec):
        """Update the data in an existing record."""
        self._records[msg_id].update(rec)

    def drop_matching_records(self, check):
        """Remove a record from the DB."""
        matches = self._match(check)
        for m in matches:
            del self._records[m['msg_id']]

    def drop_record(self, msg_id):
        """Remove a record from the DB."""
        del self._records[msg_id]

    def find_records(self, check, keys=None):
        """Find records matching a query dict, optionally extracting subset of keys.

        Returns dict keyed by msg_id of matching records.

        Parameters
        ----------

        check: dict
            mongodb-style query argument
        keys: list of strs [optional]
            if specified, the subset of keys to extract.  msg_id will *always* be
            included.
        """
        matches = self._match(check)
        if keys:
            return [ self._extract_subdict(rec, keys) for rec in matches ]
        else:
            return matches

    def get_history(self):
        """get all msg_ids, ordered by time submitted."""
        msg_ids = self._records.keys()
        return sorted(msg_ids, key=lambda m: self._records[m]['submitted'])

NODATA = KeyError("NoDB backend doesn't store any data. "
"Start the Controller with a DB backend to enable resubmission / result persistence."
)

class NoDB(DictDB):
    """A blackhole db backend that actually stores no information.

    Provides the full DB interface, but raises KeyErrors on any
    method that tries to access the records.  This can be used to
    minimize the memory footprint of the Hub when its record-keeping
    functionality is not required.
    """

    def add_record(self, msg_id, record):
        pass

    def get_record(self, msg_id):
        raise NODATA

    def update_record(self, msg_id, record):
        pass

    def drop_matching_records(self, check):
        pass

    def drop_record(self, msg_id):
        pass

    def find_records(self, check, keys=None):
        raise NODATA

    def get_history(self):
        raise NODATA


# -*- coding: utf-8 -*-
"""Displayhook for IPython.

This defines a callable class that IPython uses for `sys.displayhook`.

Authors:

* Fernando Perez
* Brian Granger
* Robert Kern
"""

#-----------------------------------------------------------------------------
#       Copyright (C) 2008-2011 The IPython Development Team
#       Copyright (C) 2001-2007 Fernando Perez <fperez@colorado.edu>
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import __builtin__

from IPython.config.configurable import Configurable
from IPython.utils import io
from IPython.utils.traitlets import Instance, List
from IPython.utils.warn import warn

#-----------------------------------------------------------------------------
# Main displayhook class
#-----------------------------------------------------------------------------

# TODO: Move the various attributes (cache_size, [others now moved]). Some
# of these are also attributes of InteractiveShell. They should be on ONE object
# only and the other objects should ask that one object for their values.

class DisplayHook(Configurable):
    """The custom IPython displayhook to replace sys.displayhook.

    This class does many things, but the basic idea is that it is a callable
    that gets called anytime user code returns a value.
    """

    shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')

    def __init__(self, shell=None, cache_size=1000, config=None):
        super(DisplayHook, self).__init__(shell=shell, config=config)

        cache_size_min = 3
        if cache_size <= 0:
            self.do_full_cache = 0
            cache_size = 0
        elif cache_size < cache_size_min:
            self.do_full_cache = 0
            cache_size = 0
            warn('caching was disabled (min value for cache size is %s).' %
                 cache_size_min,level=3)
        else:
            self.do_full_cache = 1

        self.cache_size = cache_size

        # we need a reference to the user-level namespace
        self.shell = shell

        self._,self.__,self.___ = '','',''

        # these are deliberately global:
        to_user_ns = {'_':self._,'__':self.__,'___':self.___}
        self.shell.user_ns.update(to_user_ns)

    @property
    def prompt_count(self):
        return self.shell.execution_count

    #-------------------------------------------------------------------------
    # Methods used in __call__. Override these methods to modify the behavior
    # of the displayhook.
    #-------------------------------------------------------------------------

    def check_for_underscore(self):
        """Check if the user has set the '_' variable by hand."""
        # If something injected a '_' variable in __builtin__, delete
        # ipython's automatic one so we don't clobber that.  gettext() in
        # particular uses _, so we need to stay away from it.
        if '_' in __builtin__.__dict__:
            try:
                del self.shell.user_ns['_']
            except KeyError:
                pass

    def quiet(self):
        """Should we silence the display hook because of ';'?"""
        # do not print output if input ends in ';'
        try:
            cell = self.shell.history_manager.input_hist_parsed[self.prompt_count]
            if cell.rstrip().endswith(';'):
                return True
        except IndexError:
            # some uses of ipshellembed may fail here
            pass
        return False

    def start_displayhook(self):
        """Start the displayhook, initializing resources."""
        pass

    def write_output_prompt(self):
        """Write the output prompt.

        The default implementation simply writes the prompt to
        ``io.stdout``.
        """
        # Use write, not print which adds an extra space.
        io.stdout.write(self.shell.separate_out)
        outprompt = self.shell.prompt_manager.render('out')
        if self.do_full_cache:
            io.stdout.write(outprompt)

    def compute_format_data(self, result):
        """Compute format data of the object to be displayed.

        The format data is a generalization of the :func:`repr` of an object.
        In the default implementation the format data is a :class:`dict` of
        key value pair where the keys are valid MIME types and the values
        are JSON'able data structure containing the raw data for that MIME
        type. It is up to frontends to determine pick a MIME to to use and
        display that data in an appropriate manner.

        This method only computes the format data for the object and should
        NOT actually print or write that to a stream.

        Parameters
        ----------
        result : object
            The Python object passed to the display hook, whose format will be
            computed.

        Returns
        -------
        format_data : dict
            A :class:`dict` whose keys are valid MIME types and values are
            JSON'able raw data for that MIME type. It is recommended that
            all return values of this should always include the "text/plain"
            MIME type representation of the object.
        """
        return self.shell.display_formatter.format(result)

    def write_format_data(self, format_dict):
        """Write the format data dict to the frontend.

        This default version of this method simply writes the plain text
        representation of the object to ``io.stdout``. Subclasses should
        override this method to send the entire `format_dict` to the
        frontends.

        Parameters
        ----------
        format_dict : dict
            The format dict for the object passed to `sys.displayhook`.
        """
        # We want to print because we want to always make sure we have a
        # newline, even if all the prompt separators are ''. This is the
        # standard IPython behavior.
        result_repr = format_dict['text/plain']
        if '\n' in result_repr:
            # So that multi-line strings line up with the left column of
            # the screen, instead of having the output prompt mess up
            # their first line.
            # We use the prompt template instead of the expanded prompt
            # because the expansion may add ANSI escapes that will interfere
            # with our ability to determine whether or not we should add
            # a newline.
            prompt_template = self.shell.prompt_manager.out_template
            if prompt_template and not prompt_template.endswith('\n'):
                # But avoid extraneous empty lines.
                result_repr = '\n' + result_repr

        print >>io.stdout, result_repr

    def update_user_ns(self, result):
        """Update user_ns with various things like _, __, _1, etc."""

        # Avoid recursive reference when displaying _oh/Out
        if result is not self.shell.user_ns['_oh']:
            if len(self.shell.user_ns['_oh']) >= self.cache_size and self.do_full_cache:
                warn('Output cache limit (currently '+
                      `self.cache_size`+' entries) hit.\n'
                     'Flushing cache and resetting history counter...\n'
                     'The only history variables available will be _,__,___ and _1\n'
                     'with the current result.')

                self.flush()
            # Don't overwrite '_' and friends if '_' is in __builtin__ (otherwise
            # we cause buggy behavior for things like gettext).

            if '_' not in __builtin__.__dict__:
                self.___ = self.__
                self.__ = self._
                self._ = result
                self.shell.push({'_':self._,
                                 '__':self.__,
                                '___':self.___}, interactive=False)

            # hackish access to top-level  namespace to create _1,_2... dynamically
            to_main = {}
            if self.do_full_cache:
                new_result = '_'+`self.prompt_count`
                to_main[new_result] = result
                self.shell.push(to_main, interactive=False)
                self.shell.user_ns['_oh'][self.prompt_count] = result

    def log_output(self, format_dict):
        """Log the output."""
        if self.shell.logger.log_output:
            self.shell.logger.log_write(format_dict['text/plain'], 'output')
        self.shell.history_manager.output_hist_reprs[self.prompt_count] = \
                                                    format_dict['text/plain']

    def finish_displayhook(self):
        """Finish up all displayhook activities."""
        io.stdout.write(self.shell.separate_out2)
        io.stdout.flush()

    def __call__(self, result=None):
        """Printing with history cache management.

        This is invoked everytime the interpreter needs to print, and is
        activated by setting the variable sys.displayhook to it.
        """
        self.check_for_underscore()
        if result is not None and not self.quiet():
            self.start_displayhook()
            self.write_output_prompt()
            format_dict = self.compute_format_data(result)
            self.write_format_data(format_dict)
            self.update_user_ns(result)
            self.log_output(format_dict)
            self.finish_displayhook()

    def flush(self):
        if not self.do_full_cache:
            raise ValueError,"You shouldn't have reached the cache flush "\
                  "if full caching is not enabled!"
        # delete auto-generated vars from global namespace

        for n in range(1,self.prompt_count + 1):
            key = '_'+`n`
            try:
                del self.shell.user_ns[key]
            except: pass
        # In some embedded circumstances, the user_ns doesn't have the
        # '_oh' key set up.
        oh = self.shell.user_ns.get('_oh', None)
        if oh is not None:
            oh.clear()

        # Release our own references to objects:
        self._, self.__, self.___ = '', '', ''

        if '_' not in __builtin__.__dict__:
            self.shell.user_ns.update({'_':None,'__':None, '___':None})
        import gc
        # TODO: Is this really needed?
        gc.collect()


# -*- coding: utf-8 -*-
"""
%store magic for lightweight persistence.

Stores variables, aliases and macros in IPython's database.

To automatically restore stored variables at startup, add this to your
:file:`ipython_config.py` file::

  c.StoreMagic.autorestore = True
"""
#-----------------------------------------------------------------------------
#  Copyright (c) 2012, The IPython Development Team.
#
#  Distributed under the terms of the Modified BSD License.
#
#  The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

# Stdlib
import inspect, os, sys, textwrap

# Our own
from IPython.core.error import UsageError
from IPython.core.fakemodule import FakeModule
from IPython.core.magic import Magics, magics_class, line_magic
from IPython.core.plugin import Plugin
from IPython.testing.skipdoctest import skip_doctest
from IPython.utils.traitlets import Bool, Instance

#-----------------------------------------------------------------------------
# Functions and classes
#-----------------------------------------------------------------------------

def restore_aliases(ip):
    staliases = ip.db.get('stored_aliases', {})
    for k,v in staliases.items():
        #print "restore alias",k,v # dbg
        #self.alias_table[k] = v
        ip.alias_manager.define_alias(k,v)

def refresh_variables(ip):
    db = ip.db
    for key in db.keys('autorestore/*'):
        # strip autorestore
        justkey = os.path.basename(key)
        try:
            obj = db[key]
        except KeyError:
            print "Unable to restore variable '%s', ignoring (use %%store -d to forget!)" % justkey
            print "The error was:", sys.exc_info()[0]
        else:
            #print "restored",justkey,"=",obj #dbg
            ip.user_ns[justkey] = obj

def restore_dhist(ip):
    ip.user_ns['_dh'] = ip.db.get('dhist',[])

def restore_data(ip):
    refresh_variables(ip)
    restore_aliases(ip)
    restore_dhist(ip)

@magics_class
class StoreMagics(Magics):
    """Lightweight persistence for python variables.

    Provides the %store magic."""

    @skip_doctest
    @line_magic
    def store(self, parameter_s=''):
        """Lightweight persistence for python variables.

        Example::

          In [1]: l = ['hello',10,'world']
          In [2]: %store l
          In [3]: exit

          (IPython session is closed and started again...)

          ville@badger:~$ ipython
          In [1]: l
          Out[1]: ['hello', 10, 'world']

        Usage:

        * ``%store``          - Show list of all variables and their current
                                values
        * ``%store spam``     - Store the *current* value of the variable spam
                                to disk
        * ``%store -d spam``  - Remove the variable and its value from storage
        * ``%store -z``       - Remove all variables from storage
        * ``%store -r``       - Refresh all variables from store (delete
                                current vals)
        * ``%store foo >a.txt``  - Store value of foo to new file a.txt
        * ``%store foo >>a.txt`` - Append value of foo to file a.txt

        It should be noted that if you change the value of a variable, you
        need to %store it again if you want to persist the new value.

        Note also that the variables will need to be pickleable; most basic
        python types can be safely %store'd.

        Also aliases can be %store'd across sessions.
        """

        opts,argsl = self.parse_options(parameter_s,'drz',mode='string')
        args = argsl.split(None,1)
        ip = self.shell
        db = ip.db
        # delete
        if opts.has_key('d'):
            try:
                todel = args[0]
            except IndexError:
                raise UsageError('You must provide the variable to forget')
            else:
                try:
                    del db['autorestore/' + todel]
                except:
                    raise UsageError("Can't delete variable '%s'" % todel)
        # reset
        elif opts.has_key('z'):
            for k in db.keys('autorestore/*'):
                del db[k]

        elif opts.has_key('r'):
            refresh_variables(ip)

        # run without arguments -> list variables & values
        elif not args:
            vars = db.keys('autorestore/*')
            vars.sort()
            if vars:
                size = max(map(len, vars))
            else:
                size = 0

            print 'Stored variables and their in-db values:'
            fmt = '%-'+str(size)+'s -> %s'
            get = db.get
            for var in vars:
                justkey = os.path.basename(var)
                # print 30 first characters from every var
                print fmt % (justkey, repr(get(var, '<unavailable>'))[:50])

        # default action - store the variable
        else:
            # %store foo >file.txt or >>file.txt
            if len(args) > 1 and args[1].startswith('>'):
                fnam = os.path.expanduser(args[1].lstrip('>').lstrip())
                if args[1].startswith('>>'):
                    fil = open(fnam, 'a')
                else:
                    fil = open(fnam, 'w')
                obj = ip.ev(args[0])
                print "Writing '%s' (%s) to file '%s'." % (args[0],
                  obj.__class__.__name__, fnam)

                if not isinstance (obj, basestring):
                    from pprint import pprint
                    pprint(obj, fil)
                else:
                    fil.write(obj)
                    if not obj.endswith('\n'):
                        fil.write('\n')

                fil.close()
                return

            # %store foo
            try:
                obj = ip.user_ns[args[0]]
            except KeyError:
                # it might be an alias
                # This needs to be refactored to use the new AliasManager stuff.
                if args[0] in ip.alias_manager:
                    name = args[0]
                    nargs, cmd = ip.alias_manager.alias_table[ name ]
                    staliases = db.get('stored_aliases',{})
                    staliases[ name ] = cmd
                    db['stored_aliases'] = staliases
                    print "Alias stored: %s (%s)" % (name, cmd)
                    return
                else:
                    raise UsageError("Unknown variable '%s'" % args[0])

            else:
                if isinstance(inspect.getmodule(obj), FakeModule):
                    print textwrap.dedent("""\
                    Warning:%s is %s
                    Proper storage of interactively declared classes (or instances
                    of those classes) is not possible! Only instances
                    of classes in real modules on file system can be %%store'd.
                    """ % (args[0], obj) )
                    return
                #pickled = pickle.dumps(obj)
                db[ 'autorestore/' + args[0] ] = obj
                print "Stored '%s' (%s)" % (args[0], obj.__class__.__name__)

class StoreMagic(Plugin):
    shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
    autorestore = Bool(False, config=True)

    def __init__(self, shell, config):
        super(StoreMagic, self).__init__(shell=shell, config=config)
        shell.register_magics(StoreMagics)

        if self.autorestore:
            restore_data(shell)

_loaded = False

def load_ipython_extension(ip):
    """Load the extension in IPython."""
    global _loaded
    if not _loaded:
        plugin = StoreMagic(shell=ip, config=ip.config)
        ip.plugin_manager.register_plugin('storemagic', plugin)
        _loaded = True


"""times module

This module provides some Date and Time classes for dealing with MySQL data.

Use Python datetime module to handle date and time columns."""

import math
from time import localtime
from datetime import date, datetime, time, timedelta
from _mysql import string_literal

Date = date
Time = time
TimeDelta = timedelta
Timestamp = datetime

DateTimeDeltaType = timedelta
DateTimeType = datetime

def DateFromTicks(ticks):
    """Convert UNIX ticks into a date instance."""
    return date(*localtime(ticks)[:3])

def TimeFromTicks(ticks):
    """Convert UNIX ticks into a time instance."""
    return time(*localtime(ticks)[3:6])

def TimestampFromTicks(ticks):
    """Convert UNIX ticks into a datetime instance."""
    return datetime(*localtime(ticks)[:6])

format_TIME = format_DATE = str

def format_TIMEDELTA(v):
    seconds = int(v.seconds) % 60
    minutes = int(v.seconds / 60) % 60
    hours = int(v.seconds / 3600) % 24
    return '%d %d:%d:%d' % (v.days, hours, minutes, seconds)

def format_TIMESTAMP(d):
    return d.isoformat(" ")

def DateTime_or_None(s):
    if ' ' in s:
        sep = ' '
    elif 'T' in s:
        sep = 'T'
    else:
        return Date_or_None(s)

    try:
        d, t = s.split(sep, 1)
        return datetime(*[ int(x) for x in d.split('-')+t.split(':') ])
    except (SystemExit, KeyboardInterrupt):
        raise
    except:
        return Date_or_None(s)

def TimeDelta_or_None(s):
    try:
        h, m, s = s.split(':')
        h, m, s = int(h), int(m), float(s)
        td = timedelta(hours=abs(h), minutes=m, seconds=int(s),
                       microseconds=int(math.modf(s)[0] * 1000000))
        if h < 0:
            return -td
        else:
            return td
    except ValueError:
        # unpacking or int/float conversion failed
        return None

def Time_or_None(s):
    try:
        h, m, s = s.split(':')
        h, m, s = int(h), int(m), float(s)
        return time(hour=h, minute=m, second=int(s),
                    microsecond=int(math.modf(s)[0] * 1000000))
    except ValueError:
        return None

def Date_or_None(s):
    try:
        return date(*[ int(x) for x in s.split('-',2)])
    except (SystemExit, KeyboardInterrupt):
        raise
    except:
        return None

def DateTime2literal(d, c):
    """Format a DateTime object as an ISO timestamp."""
    return string_literal(format_TIMESTAMP(d),c)

def DateTimeDelta2literal(d, c):
    """Format a DateTimeDelta object as a time."""
    return string_literal(format_TIMEDELTA(d),c)

def mysql_timestamp_converter(s):
    """Convert a MySQL TIMESTAMP to a Timestamp object."""
    # MySQL>4.1 returns TIMESTAMP in the same format as DATETIME
    if s[4] == '-': return DateTime_or_None(s)
    s = s + "0"*(14-len(s)) # padding
    parts = map(int, filter(None, (s[:4],s[4:6],s[6:8],
                                   s[8:10],s[10:12],s[12:14])))
    try:
        return Timestamp(*parts)
    except (SystemExit, KeyboardInterrupt):
        raise
    except:
        return None


from abc import ABCMeta
from ConfigParser import NoOptionError, NoSectionError
from functools import partial
import logging

from lxml import etree, objectify
from lxml.builder import ElementMaker
import requests

from .config import production_config, sandbox_config, CONFIG_PATH
from . import namespaces
from .utils import parser_from_schema

logging.basicConfig()
log = logging.getLogger(__name__)

# I wish I didn't have to hard-code this but there's no service to query, see:
# http://developer.ebay.com/DevZone/merchandising/docs/Concepts/SiteIDToGlobalID.html
SITE_ID_TO_GLOBAL_ID = {
    0: 'EBAY-US',
    2: 'EBAY-ENCA',
    3: 'EBAY-GB',
    15: 'EBAY-AU',
    16: 'EBAY-AT',
    23: 'EBAY-FRBE',
    71: 'EBAY-FR',
    77: 'EBAY-DE',
    100: 'EBAY-MOTOR',
    101: 'EBAY-IT',
    123: 'EBAY-NLBE',
    146: 'EBAY-NL',
    186: 'EBAY-ES',
    193: 'EBAY-CH',
    201: 'EBAY-HK',
    203: 'EBAY-IN',
    205: 'EBAY-IE',
    207: 'EBAY-MY',
    210: 'EBAY-FRCA',
    211: 'EBAY-PH',
    212: 'EBAY-PL',
    216: 'EBAY-SG',
}

NSMAP = {
    'soap': namespaces.SOAP_1_2,
    'soapenv': namespaces.SOAP_1_1,
    'ebl': namespaces.EBAY,
    'xsi': namespaces.XSI,
}

def dicttoxml(obj, factory):
    elements = []
    for key, value in obj.items():
        if hasattr(value, 'items'):
            elements.append(dicttoxml(value, factory))
        elif isinstance(value, (list, tuple)):
            elements.append([
                dicttoxml(sub_val, factory)
                for sub_val in value
            ])
        else:
            el = getattr(factory, key)(unicode(value))
            elements.append(el)
    return elements

class APIBase:
    """
    Abstract: make a concrete sub-class (some APIs are provided below)
    """
    __metaclass__ = ABCMeta

    SCHEMA_URL = None

    PRODUCTION_ENDPOINT = None
    SANDBOX_ENDPOINT = None

    SOAP_VERSION = None

    DEFAULT_NAMESPACE = namespaces.EBAY

    def __init__(self, schema_url=None, sandbox=False, **kwargs):
        # eBay API methods are all CamelCase so it should be safe to set
        # lowercase (or all-caps) attributes on self (see __getattr__)

        self.CONF_PREFIX = self._get_conf_prefix()
        self.sandbox = sandbox
        if sandbox:
            self.config = sandbox_config
        else:
            self.config = production_config

        # use passed in schema first, else try config file,
        # else use class default
        if schema_url is not None:
            self._schema = schema_url
        else:
            try:
                self._schema = self.config.get('soap',
                                               '%s_schema' % self.CONF_PREFIX)
            except (NoOptionError, NoSectionError):
                if self.SCHEMA_URL is None:
                    raise NotImplementedError(
                        'You must give a value for SCHEMA_URL on a sub-class,'
                        ' or define <api name>_schema in the conf file'
                    )
                self._schema = self.SCHEMA_URL

        # make a schema-aware parser (for deserializing responses into objects)
        self.parser, self.version = parser_from_schema(self._schema)

        # determine the service endpoint URI
        try:
            self._endpoint = self.config.get('soap',
                                             '%s_api' % self.CONF_PREFIX)
        except (NoOptionError, NoSectionError):
            if sandbox:
                if self.SANDBOX_ENDPOINT is None:
                    raise NotImplementedError(
                        'You must give a value for SANDBOX_ENDPOINT on a sub-'
                        'class, or define <api name>_api in the conf file'
                    )
                self._endpoint = self.SANDBOX_ENDPOINT
            else:
                if self.SANDBOX_ENDPOINT is None:
                    raise NotImplementedError(
                        'You must give a value for PRODUCTION_ENDPOINT on a '
                        'sub-class, or define <api name>_api in the conf file'
                    )
                self._endpoint = self.PRODUCTION_ENDPOINT

        if self.SOAP_VERSION is None:
            raise NotImplementedError(
                'You must give a value for SOAP_VERSION on a sub-class'
            )

        self.client = requests.Session()
        self.client.headers.update({
            'Content-Type': 'application/soap+xml; charset=utf-8',
            'SOAPAction': '',
        })

        log.info('CONFIG_PATH: %s', CONFIG_PATH)
        self.site_id = (
            kwargs.get('site_id') or self.config.get('site', 'site_id')
        )
        self.app_id = (
            kwargs.get('app_id') or self.config.get('keys', 'app_id')
        )
        self.dev_id = (
            kwargs.get('dev_id') or self.config.get('keys', 'dev_id')
        )
        self.cert_id = (
            kwargs.get('cert_id') or self.config.get('keys', 'cert_id')
        )
        self.token = (
            kwargs.get('token') or self.config.get('auth', 'token')
        )

        self._last_response = None

    def __getattr__(self, name):
        """
        make a SOAP request

        Note:
        eBay API methods are all CamelCase, or otherCamelCase
        """
        if name.lower() == name or name.startswith('_'):
            # avoid accidental API calls in ipython..!
            return super(APIBase, self).__getattr(name)
        return partial(self._execute, name=name)

    def _get_conf_prefix(self):
        # assume the class name ends in 'API'
        return self.__class__.__name__[:-3].lower()

    def _nsmap(self):
        nsmap = NSMAP.copy()
        nsmap[None] = self.DEFAULT_NAMESPACE
        return nsmap

    def get_soap_el_factory(self):
        if self.SOAP_VERSION == '1.1':
            return ElementMaker(
                namespace=namespaces.SOAP_1_1,
                nsmap=self._nsmap()
            )
        elif self.SOAP_VERSION == '1.2':
            return ElementMaker(
                namespace=namespaces.SOAP_1_2,
                nsmap=self._nsmap()
            )
        else:
            raise ValueError(
                "Invalid SOAP_VERSION: {}".format(self.SOAP_VERSION)
            )

    def get_msg_el_factory(self):
        return ElementMaker(nsmap=self._nsmap())

    def get_http_headers(self, name):
        return {}

    def _request(self, name, envelope):
        response = self.client.post(
            url=self._endpoint,
            data=etree.tostring(envelope),
            params=self.get_query_params(name),
            headers=self.get_http_headers(name),
        )
        response.raise_for_status()
        self._last_response = response
        return response

    def _execute(self, name, **kwargs):
        """
        this indirection gives us the same call signature as used by
        ebaysdk-python, i.e. `api.execute('GetUser', {})`

        (our __init__ signature and conf are different still at the moment)
        """
        return self.execute(name, kwargs)

    def execute(self, name, params):
        request_name = '{}Request'.format(name)  # can we rely on this?
        factory = self.get_msg_el_factory()
        body_el = getattr(factory, request_name)
        envelope = self.make_envelope([
            body_el(*dicttoxml(params, factory))
        ])

        log.debug(etree.tostring(envelope, pretty_print=True))

        response = self._request(name, envelope)
        return self.objectify_response(name, response)

    def objectify_response(self, name, response):
        soap_response = etree.fromstring(
            response.text.encode(response.encoding)
        )
        response_name = '{}Response'.format(name)  # can we rely on this?
        body_root = soap_response.xpath(
            "//*[local-name() = '%s']" % response_name
        )[0]

        # tostring->fromstring roundtrip is ugly but otherwise the objectified
        # tree is spoiled by namespaces (would have to use getattr everywhere)
        return objectify.fromstring(
            etree.tostring(body_root),
            parser=self.parser
        )

    def get_soap_header(self):
        return None

    def get_query_params(self, name):
        return {}

    def make_envelope(self, body_elements=None):
        """
        body_elements: <list> of etree.Elements or <None>
        """
        soap = self.get_soap_el_factory()
        body_elements = body_elements or []
        body = soap.Body(*body_elements)
        header = self.get_soap_header()
        if header is not None:
            elements = [header, body]
        else:
            elements = [body]
        return soap.Envelope(
            #{
            #    '{%s}encodingStyle' % namespaces.SOAP_1_2: (
            #        'http://www.w3.org/2001/12/soap-encoding')
            #},
            *elements
        )

class TradingAPI(APIBase):
    SCHEMA_URL = 'http://developer.ebay.com/webservices/latest/ebaySvc.xsd'

    PRODUCTION_ENDPOINT = 'https://api.ebay.com/wsapi'
    SANDBOX_ENDPOINT = 'https://api.sandbox.ebay.com/wsapi'

    SOAP_VERSION = '1.1'

    def __getattr__(self, name):
        method = super(TradingAPI, self).__getattr__(name=name)
        # the method call has to include the API version in the body
        # even though it's also provided on the querystring...
        return partial(method, Version=self.version)

    def get_soap_header(self):
        soap = self.get_soap_el_factory()
        payload = self.get_msg_el_factory()
        return soap.Header(
            payload.RequesterCredentials(
                payload.Credentials(
                    payload.AppID(self.app_id),
                    payload.DevId(self.dev_id),
                    payload.AuthCert(self.cert_id),
                ),
                payload.eBayAuthToken(self.token)
            )
        )

    def get_query_params(self, name):
        # for some reason ebay require some fields from the SOAP request to be
        # repeated as querystring args appended to the service url
        return {
            'callname': name,
            'siteid': self.site_id,
            'appid': self.app_id,
            'version': self.version,
            'routing': 'default',
        }

class PlatformNotificationsAPI(TradingAPI):
    """
    The calls to get and set platform notification preferences are actually
    part of the TradingAPI.

    The notifications received at your callback url can be decoded using the
    `objectify_response` method.
    """
    pass

class ShoppingAPI(APIBase):
    SCHEMA_URL = (
        'http://developer.ebay.com/webservices/latest/ShoppingService.xsd'
    )

    PRODUCTION_ENDPOINT = 'http://open.api.ebay.com/shopping'
    SANDBOX_ENDPOINT = 'http://open.api.sandbox.ebay.com/shopping'

    SOAP_VERSION = '1.1'

    def get_query_params(self, name):
        # for some reason ebay require some fields from the SOAP request to be
        # repeated as querystring args appended to the service url
        return {
            'callname': name,
            'siteid': self.site_id,
            'appid': self.app_id,
            'version': self.version,
            'responseencoding': 'SOAP',
            'requestencoding': 'SOAP',
        }

class FindingAPI(APIBase):
    SCHEMA_URL = (
        'http://developer.ebay.com/webservices/Finding/latest/'
        'FindingService.wsdl'
    )

    PRODUCTION_ENDPOINT = (
        'http://svcs.ebay.com/services/search/FindingService/v1'
    )
    SANDBOX_ENDPOINT = (
        'http://svcs.sandbox.ebay.com/services/search/FindingService/v1'
    )

    SOAP_VERSION = '1.2'

    DEFAULT_NAMESPACE = namespaces.EBAY_SEARCH

    def get_http_headers(self, name):
        site_id = int(self.site_id, 10)
        return {
            'X-EBAY-SOA-OPERATION-NAME': name,
            'X-EBAY-SOA-SERVICE-NAME': 'FindingService',  # this one is genius
            'X-EBAY-SOA-SERVICE-VERSION': self.version,
            'X-EBAY-SOA-GLOBAL-ID': SITE_ID_TO_GLOBAL_ID[site_id],
            'X-EBAY-SOA-SECURITY-APPNAME': self.app_id,
            'X-EBAY-SOA-REQUEST-DATA-FORMAT': 'SOAP',
            'X-EBAY-SOA-MESSAGE-PROTOCOL': 'SOAP12',
        }

class BusinessPoliciesAPI(APIBase):
    SCHEMA_URL = (
        'http://developer.ebay.com/webservices/business-policies/'
        'latest/SellerProfilesManagementService.wsdl'
    )

    PRODUCTION_ENDPOINT = (
        'https://svcs.ebay.com/services/selling/v1/'
        'SellerProfilesManagementService'
    )
    SANDBOX_ENDPOINT = (
        'http://svcs.sandbox.ebay.com/services/selling/v1/'
        'SellerProfilesManagementService'
    )

    SOAP_VERSION = '1.2'

    def get_http_headers(self, name):
        site_id = int(self.site_id, 10)
        return {
            'X-EBAY-SOA-OPERATION-NAME': name,
            'X-EBAY-SOA-SERVICE-VERSION': self.version,
            'X-EBAY-SOA-GLOBAL-ID': SITE_ID_TO_GLOBAL_ID[site_id],
            'X-EBAY-SOA-REQUEST-DATA-FORMAT': 'SOAP',
            'X-EBAY-SOA-MESSAGE-PROTOCOL': 'SOAP12',
            'X-EBAY-SOA-SECURITY-TOKEN': self.token,
        }