# -*- coding: utf-8 -*-
#   Copyright (C) 2008-2010, 2013-2016 Rocky Bernstein <rocky@gnu.org>
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""Debugger core routines.

This module contains the Debugger core routines for starting and
stopping trace event handling and breakpoint checking. See also
debugger for top-level Debugger class and module routine which
ultimately will call this. An event processor is responsible of
handling what to do when an event is triggered."""

# Common Python packages
import os, sys, threading

# External Egg packages
import tracer

# Our local modules
from trepan.lib import breakpoint, default, stack as Mstack
from trepan import misc as Mmisc
from trepan import clifns as Mclifns
from trepan.processor import trace as Mtrace, cmdproc as Mcmdproc

class TrepanCore:

    DEFAULT_INIT_OPTS = {
        'processor'   : None,

        # How many step events to skip before
        # entering event processor? Zero (0) means stop at the next one.
        # A negative number indicates no eventual stopping.
        'step_ignore' : 0,
        'ignore_filter': None,  # But see debugger.py
        }

    def __init__(self, debugger, opts=None):
        """ Create a debugger object. But depending on the value of
        key 'start' inside hash `opts', we may or may not initially
        start tracing events (i.e. enter the debugger).

        See also `start' and `stop'.
        """

        import trepan.bwprocessor as Mbwproc

        get_option       = lambda key: Mmisc.option_set(opts, key,
                                                        self.DEFAULT_INIT_OPTS)

        self.bpmgr           = breakpoint.BreakpointManager()
        self.current_bp      = None
        self.debugger        = debugger

        # Threading lock ensures that we don't have other traced threads
        # running when we enter the debugger. Later we may want to have
        # a switch to control.
        self.debugger_lock   = threading.Lock()

        self.filename_cache  = {}

        # Initially the event parameter of the event hook.
        # We can however modify it, such as for breakpoints
        self.event           = None

        # Is debugged program currently under execution?
        self.execution_status = 'Pre-execution'

        # main_dirname is the directory where the script resides.
        # Filenames in co_filename are often relative to this.
        self.main_dirname    = os.curdir

        # What event processor and processor options do we use?
        self.processor = get_option('processor')
        proc_opts      = get_option('proc_opts')
        if not self.processor:
            self.processor   = Mcmdproc.CommandProcessor(self, opts=proc_opts)
        elif self.processor == 'bullwinkle':
            self.processor   = Mbwproc.BWProcessor(self, opts=proc_opts)
            pass
        # What events are considered in stepping. Note: 'None' means *all*.
        self.step_events     = None
        # How many line events to skip before entering event processor?
        # If stop_level is None all breaks are counted otherwise just
        # those which less than or equal to stop_level.
        self.step_ignore     = get_option('step_ignore')

        # If stop_level is not None, then we are next'ing or
        # finish'ing and will ignore frames greater than stop_level.
        # We also will cache the last frame and thread number encountered
        # so we don't have to compute the current level all the time.
        self.last_frame      = None
        self.last_level      = 10000
        self.last_thread     = None
        self.stop_level      = None
        self.stop_on_finish  = False

        self.last_lineno     = None
        self.last_filename   = None
        self.different_line  = None

        # The reason we have stopped, e.g. 'breakpoint hit', 'next',
        # 'finish', 'step', or 'exception'.
        self.stop_reason     = ''

        self.trace_processor = Mtrace.PrintProcessor(self)

        # What routines (keyed by f_code) will we not trace into?
        self.ignore_filter = get_option('ignore_filter')

        self.search_path     = sys.path  # Source filename search path

        # When trace_hook_suspend is set True, we'll suspend
        # debugging.
        self.trace_hook_suspend = False

        self.until_condition = get_option('until_condition')

        return

    def add_ignore(self, *frames_or_fns):
        """Add `frame_or_fn' to the list of functions that are not to
        be debugged"""
        for frame_or_fn in frames_or_fns:
            rc = self.ignore_filter.add_include(frame_or_fn)
            pass
        return rc

    def canonic(self, filename):
        """ Turns `filename' into its canonic representation and returns this
        string. This allows a user to refer to a given file in one of several
        equivalent ways.

        Relative filenames need to be fully resolved, since the current working
        directory might change over the course of execution.

        If filename is enclosed in < ... >, then we assume it is
        one of the bogus internal Python names like <string> which is seen
        for example when executing "exec cmd".
        """

        if filename == "<" + filename[1:-1] + ">":
            return filename
        canonic = self.filename_cache.get(filename)
        if not canonic:
            lead_dir = filename.split(os.sep)[0]
            if lead_dir == os.curdir or lead_dir == os.pardir:
                # We may have invoked the program from a directory
                # other than where the program resides. filename is
                # relative to where the program resides. So make sure
                # to use that.
                canonic = os.path.abspath(os.path.join(self.main_dirname,
                                                       filename))
            else:
                canonic = os.path.abspath(filename)
                pass
            if not os.path.isfile(canonic):
                canonic = Mclifns.search_file(filename, self.search_path,
                                              self.main_dirname)
                # FIXME: is this is right for utter failure?
                if not canonic: canonic = filename
                pass
            canonic = os.path.realpath(os.path.normcase(canonic))
            self.filename_cache[filename] = canonic
        return canonic

    def canonic_filename(self, frame):
        """Picks out the file name from `frame' and returns its
         canonic() value, a string."""
        return self.canonic(frame.f_code.co_filename)

    def filename(self, filename=None):
        """Return filename or the basename of that depending on the
        basename setting"""
        if filename is None:
            if self.debugger.mainpyfile:
                filename = self.debugger.mainpyfile
            else:
                return None
        if self.debugger.settings['basename']:
            return(os.path.basename(filename))
        return filename

    def is_running(self):
        return 'Running' == self.execution_status

    def is_started(self):
        '''Return True if debugging is in progress.'''
        return (tracer.is_started() and
                not self.trace_hook_suspend
                and tracer.find_hook(self.trace_dispatch))

    def remove_ignore(self, frame_or_fn):
        """Remove `frame_or_fn' to the list of functions that are not to
        be debugged"""
        return self.ignore_filter.remove_include(frame_or_fn)

    def start(self, opts=None):
        """ We've already created a debugger object, but here we start
        debugging in earnest. We can also turn off debugging (but have
        the hooks suspended or not) using 'stop'.

        'opts' is a hash of every known value you might want to set when
        starting the debugger. See START_OPTS of module default.
        """

        # The below is our fancy equivalent of:
        #    sys.settrace(self._trace_dispatch)
        try:
            self.trace_hook_suspend = True
            get_option = lambda key: Mmisc.option_set(opts, key,
                                                      default.START_OPTS)

            add_hook_opts = get_option('add_hook_opts')

            # Has tracer been started?
            if not tracer.is_started() or get_option('force'):
                # FIXME: should filter out opts not for tracer

                tracer_start_opts = default.START_OPTS.copy()
                if opts:
                    tracer_start_opts.update(opts.get('tracer_start', {}))
                tracer_start_opts['trace_fn'] = self.trace_dispatch
                tracer_start_opts['add_hook_opts'] = add_hook_opts
                tracer.start(tracer_start_opts)
            elif not tracer.find_hook(self.trace_dispatch):
                tracer.add_hook(self.trace_dispatch, add_hook_opts)
                pass
            self.execution_status = 'Running'
        finally:
            self.trace_hook_suspend = False
        return

    def stop(self, options=None):
        # Our version of:
        #    sys.settrace(None)
        try:
            self.trace_hook_suspend = True
            get_option = lambda key: Mmisc.option_set(options, key,
                                                      default.STOP_OPTS)
            args = [self.trace_dispatch]
            remove = get_option('remove')
            if remove:
                args.append(remove)
                pass
            if tracer.is_started():
                try:
                    tracer.remove_hook(*args)
                except LookupError:
                    pass
                pass
        finally:
            self.trace_hook_suspend = False
        return

    def is_break_here(self, frame, arg):
        filename = self.canonic(frame.f_code.co_filename)
        if 'call' == self.event:
            find_name  = frame.f_code.co_name
            # Could check code object or decide not to
            # The below could be done as a list comprehension, but
            # I'm feeling in Fortran mood right now.
            for fn in self.bpmgr.fnlist:
                if fn.__name__ == find_name:
                    self.current_bp = bp = self.bpmgr.fnlist[fn][0]
                    if bp.temporary:
                        msg = 'temporary '
                        self.bpmgr.delete_breakpoint(bp)
                    else:
                        msg = ''
                        pass
                    self.stop_reason = ("at %scall breakpoint %d" %
                                        (msg, bp.number))
                    self.event = 'brkpt'
                    return True
                pass
            pass
        if (filename, frame.f_lineno) in list(self.bpmgr.bplist.keys()):
            (bp, clear_bp) = self.bpmgr.find_bp(filename, frame.f_lineno,
                                                frame)
            if bp:
                self.current_bp = bp
                if (clear_bp and bp.temporary):
                    msg = 'temporary '
                    self.bpmgr.delete_breakpoint(bp)
                else:
                    msg = ''
                    pass
                self.stop_reason = ("at %sline breakpoint %d" %
                                    (msg, bp.number))
                self.event = 'brkpt'
                return True
            else:
                return False
            pass
        return False

    def matches_condition(self, frame):
        # Conditional bp.
        # Ignore count applies only to those bpt hits where the
        # condition evaluates to true.
        try:
            val = eval(self.until_condition, frame.f_globals, frame.f_locals)
        except:
            # if eval fails, most conservative thing is to
            # stop on breakpoint regardless of ignore count.
            # Don't delete temporary, as another hint to user.
            return False
        return val

    def is_stop_here(self, frame, event, arg):
        """ Does the magic to determine if we stop here and run a
        command processor or not. If so, return True and set
        self.stop_reason; if not, return False.

        Determining factors can be whether a breakpoint was
        encountered, whether we are stepping, next'ing, finish'ing,
        and, if so, whether there is an ignore counter.
        """

        # Add an generic event filter here?
        # FIXME TODO: Check for
        #  - thread switching (under set option)

        # Check for "next" and "finish" stopping via stop_level

        # Do we want a different line and if so,
        # do we have one?
        lineno = frame.f_lineno
        filename = frame.f_code.co_filename
        if self.different_line and event == 'line':
            if self.last_lineno == lineno and self.last_filename == filename:
                return False
            pass
        self.last_lineno   = lineno
        self.last_filename = filename

        if self.stop_level is not None:
            if frame != self.last_frame:
                # Recompute stack_depth
                self.last_level = Mstack.count_frames(frame)
                self.last_frame = frame
                pass
            if self.last_level > self.stop_level:
                return False
            elif self.last_level == self.stop_level and \
                    self.stop_on_finish and event in ['return', 'c_return']:
                self.stop_level = None
                self.stop_reason = "in return for 'finish' command"
                return True
            pass

        # Check for stepping
        if self._is_step_next_stop(event):
            self.stop_reason = 'at a stepping statement'
            return True

        return False

    def _is_step_next_stop(self, event):
        if self.step_events and event not in self.step_events:
            return False
        if self.step_ignore == 0:
            return True
        elif self.step_ignore > 0:
            self.step_ignore -= 1
            pass
        return False

    def set_next(self, frame, step_ignore=0, step_events=None):
        "Sets to stop on the next event that happens in frame 'frame'."
        self.step_events      = None  # Consider all events
        self.stop_level       = Mstack.count_frames(frame)
        self.last_level       = self.stop_level
        self.last_frame       = frame
        self.stop_on_finish   = False
        self.step_ignore      = step_ignore
        return

    def trace_dispatch(self, frame, event, arg):
        '''A trace event occurred. Filter or pass the information to a
        specialized event processor. Note that there may be more filtering
        that goes on in the command processor (e.g. to force a
        different line). We could put that here, but since that seems
        processor-specific I think it best to distribute the checks.'''

        # For now we only allow one instance in a process
        # In Python 2.6 and beyond one can use "with threading.Lock():"
        try:
            self.debugger_lock.acquire()

            if self.trace_hook_suspend:
                return None

            self.event = event
            # FIXME: Understand what's going on here better.
            # When None gets returned, the frame's f_trace seems to get set
            # to None. Somehow this is changing other frames when get passed
            # to this routine which also have their f_trace set to None.
            # This will disallow a command like "jump" from working properly,
            # which will give a cryptic the message on setting f_lineno:
            #   f_lineno can only be set by a trace function
            if self.ignore_filter and self.ignore_filter.is_included(frame):
                return True

            if self.debugger.settings['trace']:
                print_event_set = self.debugger.settings['printset']
                if self.event in print_event_set:
                    self.trace_processor.event_processor(frame,
                                                         self.event, arg)
                    pass
                pass

            if self.until_condition:
                if not self.matches_condition(frame): return True
                pass

            trace_event_set = self.debugger.settings['events']
            if trace_event_set is None or self.event not in trace_event_set:
                return True

            # I think we *have* to run is_stop_here() before
            # is_break_here() because is_stop_here() sets various
            # stepping counts. But it might be more desirable from the
            # user's standpoint to test for breaks before steps. In
            # this case we will need to factor out the counting
            # updates.
            if ( self.is_stop_here(frame, event, arg) or
                 self.is_break_here(frame, arg) ):
                # Run the event processor
                return self.processor.event_processor(frame, self.event, arg)
            return True
        finally:
            try:
                self.debugger_lock.release()
            except:
                pass
            pass
        pass
    pass

# Demo it
if __name__=='__main__':
    class MockProcessor:
        pass
    opts = {'processor': MockProcessor()}
    dc = TrepanCore(None, opts=opts)
    dc.step_ignore = 1
    print('dc._is_step_next_stop():', dc._is_step_next_stop('line'))
    print('dc._is_step_next_stop():', dc._is_step_next_stop('line'))
    print('dc.step_ignore:', dc.step_ignore)
    print('dc.is_started:', dc.is_started())
    print(dc.canonic('<string>'))
    print(dc.canonic(__file__))
    pass


# -*- coding: utf-8 -*-
#   Copyright (C) 2009, 2014 Rocky Bernstein <rocky@gnu.org>
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
import sys, threading

# Our local modules
from trepan.processor.command import base_subcmd as Mbase_subcmd
from trepan.lib import stack as Mstack, thred as Mthread

# FIXME turn into yet another subcommand thingy.
class InfoThread(Mbase_subcmd.DebuggerSubcommand):
    """**info threads** [*thread-name*|*thread-number*] [**terse**|**verbose**]

List all currently-known thread name(s).

If no thread name is given, we list info for all threads. Unless a
terse listing, for each thread we give:

  - the class, thread name, and status as *Class(Thread-n, status)*

  - the top-most call-stack information for that thread.

Generally the top-most calls into the debugger and dispatcher are
omitted unless set dbg_trepan is *True*.

If 'verbose' appended to the end of the command, then the entire stack
trace is given for each frame.  If 'terse' is appended we just list
the thread name and thread id.

To get the full stack trace for a specific thread pass in the thread name.
"""
    min_abbrev = 2  # Min is "info th"
    max_args = 2
    need_stack = True
    short_help = "List thread info"

    def __init__(self, cmd):
        Mbase_subcmd.DebuggerSubcommand.__init__(self, cmd)
        self.name2id = {}
        return

    def stack_trace(self, f):
        """A mini stack trace routine for threads."""
        while f:
            if (not self.core.ignore_filter.is_included(f)
                or self.settings['dbg_trepan']):
                s = Mstack.format_stack_entry(self, (f, f.f_lineno))
                self.msg(" "*4 + s)
                pass
            f = f.f_back
            pass
        return

    def info_thread_terse(self, name2id, arg=None):
        if arg is not None:
            thread_name = arg
            if thread_name in list(name2id.keys()):
                self.info_thread_line(thread_name, name2id)
            else:
                self.errmsg("Don't know about thread name %s" % thread_name)
                return
            pass

        # Show all threads
        thread_name_list = list(name2id.keys())
        thread_name_list.sort()
        for thread_name in thread_name_list:
            self.info_thread_line(thread_name, name2id)
            pass
        # self.info_thread_missing()
        return

    def info_thread_line(self, thread_name, name2id):
        if thread_name == self.proc.frame_thread_name:
            prefix = '-> '
        elif thread_name == self.proc.thread_name:
            prefix = '=> '
        else:
            prefix = '   '
            pass

        self.msg("%s%s: %d" % (prefix, thread_name,
                               name2id[thread_name]))
        return

    def run(self, args):
        # FIXME: add thread locking here?

        self.thread_name = Mthread.current_thread_name()

        name2id = Mthread.map_thread_names()
        # invert threading._active
        for thread_id in list(threading._active.keys()):
            thread = threading._active[thread_id]
            name = thread.getName()
            if name not in list(self.name2id.keys()):
                self.name2id[name] = thread_id
                pass
            pass

        all_verbose = False
        if len(args) == 1:
            if args[0].startswith('verbose'):
                all_verbose = True
            elif args[0].startswith('terse'):
                self.info_thread_terse(name2id)
                return
            pass

        if len(args) > 0 and not all_verbose:
            thread_name = args[0]
            if thread_name == '.':
                thread_name = self.thread_name
            try:
                thread_id = int(thread_name)
                if thread_id not in list(threading._active.keys()):
                    self.errmsg("Don't know about thread number %s" %
                                thread_name)
                    self.info_thread_terse(name2id)
                    return
            except ValueError:
                if thread_name not in list(self.name2id.keys()):
                    self.errmsg("Don't know about thread %s" % thread_name)
                    self.info_thread_terse(name2id)
                    return
                thread_id = self.name2id[thread_name]
                pass

            frame = sys._current_frames()[thread_id]
            self.stack_trace(frame)
            return

        # Show info about *all* threads
        thread_key_list = list(self.name2id.keys())
        thread_key_list.sort()
        for thread_name in thread_key_list:
            thread_id = self.name2id[thread_name]
            frame = sys._current_frames()[thread_id]
            s = ''
            # Print location where thread was created and line number
            if thread_id in threading._active:
                thread = threading._active[thread_id]
                thread_name = thread.getName()
                if thread_name == self.proc.frame_thread_name:
                    prefix = '-> '
                    if not self.settings['dbg_trepan']:
                        frame = Mthread.find_debugged_frame(frame)
                        pass
                    pass
                elif thread_name == self.proc.thread_name:
                    prefix = '=> '
                else:
                    prefix='   '
                    pass
                s += "%s%s" % (prefix, str(thread))
                if all_verbose:
                    s += ": %d" % thread_id
                    pass
            else:
                s += "    thread id: %d" % thread_id
                pass
            s += "\n    "
            s += Mstack.format_stack_entry(self, (frame, frame.f_lineno),
                                           color=self.settings['highlight'])
            self.section('-' * 40)
            self.msg(s)
            frame = frame.f_back
            if all_verbose and frame:
                self.stack_trace(frame)
                pass
        return
    pass

if __name__ == '__main__':
    from trepan.processor.command import mock, info as Minfo
    d, cp = mock.dbg_setup()
    i = Minfo.InfoCommand(cp)
    sub = InfoThread(i)
    import inspect
    cp.curframe = inspect.currentframe()
    sub.run([])
    print('=' * 30)
    sub.run(['foo'])
    print('=' * 30)
    sub.run(['MainThread'])
    print('=' * 30)
    sub.run(['terse'])
    print('=' * 30)
    sub.run(['verbose'])
    print('=' * 30)
    sub.run(['MainThread', 'verbose'])
    # Try with threading.
    pass


# -*- coding: utf-8 -*-
#   Copyright (C) 2008-2009, 2012-2013, 2015 Rocky Bernstein <rocky@gnu.org>
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
import columnize, inspect, pyficache, sys, os

# Our local modules
from trepan.processor.command import base_subcmd as Mbase_subcmd
from trepan import misc as Mmisc
from trepan.lib import complete as Mcomplete
from trepan.lib.file import file_list

class InfoFiles(Mbase_subcmd.DebuggerSubcommand):
    """**info files** [*filename* [**all** | **brkpts** | **lines** | **sha1** | **size**]]

Show information about the current file. If no filename is given and
the program is running then the current file associated with the
current stack entry is used. Sub options which can be shown about a file are:

 * **brkpts** Line numbers where there are statement boundaries. These
 lines can be used in breakpoint commands.

 * **sha1**	A SHA1 hash of the source text. """ + \
"""This may be useful in comparing source code.

 * **size**	The number of lines in the file.

 * **all** All of the above information.
"""

    min_abbrev = 2
    need_stack = False
    short_help = 'Show information about an imported or loaded Python file'

    def complete(self, prefix):
        completions = sorted(['.'] + file_list())
        return Mcomplete.complete_token(completions, prefix)

    def run(self, args):
        """Get file information"""
        if len(args) == 0:
            if not self.proc.curframe:
                self.errmsg("No frame - no default file.")
                return False
            filename = self.proc.curframe.f_code.co_filename
        else:
            filename = args[0]
            pass

        m = filename + ' is'
        filename_cache = self.core.filename_cache
        if filename in filename_cache:
            m += " cached in debugger"
            if filename_cache[filename] != filename:
                m += ' as:'
                m = Mmisc.wrapped_lines(m, filename_cache[filename] + '.',
                                        self.settings['width'])
            else:
                m += '.'
                pass
            self.msg(m)
        else:
            matches = [file for file in file_list() if
                       file.endswith(filename)]
            if (len(matches) > 1):
                self.msg("Multiple files found ending filename string:")
                for match_file in matches:
                    self.msg("\t%s" % match_file)
                    pass
            elif len(matches) == 1:
                canonic_name = pyficache.unmap_file(matches[0])
                m += " matched debugger cache file:\n  "  + canonic_name
                self.msg(m)
            else:
                self.msg(m + ' not cached in debugger.')
            pass
        canonic_name = self.core.canonic(filename)
        self.msg(Mmisc.wrapped_lines('Canonic name:', canonic_name,
                                     self.settings['width']))
        for name in (canonic_name, filename):
            if name in sys.modules:
                for key in [k for k, v in list(sys.modules.items())
                            if name == v]:
                    self.msg("module: %s", key)
                    pass
                pass
            pass
        for arg in args[1:]:
            processed_arg = False
            if arg in ['all', 'size']:
                if pyficache.size(canonic_name):
                    self.msg("File has %d lines." %
                             pyficache.size(canonic_name))
                    pass
                processed_arg = True
                pass
            if arg in ['all', 'sha1']:
                self.msg("SHA1 is %s." % pyficache.sha1(canonic_name))
                processed_arg = True
                pass
            if arg in ['all', 'brkpts']:
                lines = pyficache.trace_line_numbers(canonic_name)
                if lines:
                    self.section("Possible breakpoint line numbers:")
                    fmt_lines = columnize.columnize(lines, ljust = False,
                                                    arrange_vertical = False,
                                                    lineprefix='  ')
                    self.msg(fmt_lines)
                    pass
                processed_arg = True
                pass
            if not processed_arg:
                self.errmsg("Don't understand sub-option %s." % arg)
                pass
            pass
        return
    pass

if __name__ == '__main__':
    from trepan.processor.command import mock, info as Minfo
    d, cp = mock.dbg_setup()
    i = Minfo.InfoCommand(cp)
    sub = InfoFiles(i)
    sub.run([])
    cp.curframe = inspect.currentframe()
    sub.run(['file.py', 'foo'])
    for width in (200, 80):
        sub.settings['width'] = width
        sub.run(['file.py', 'lines'])
        print(sub.run([]))
        pass
    sub.run(['file.py', 'all'])
    print(sub.complete(''))
    # sub.run(['file.py', 'lines', 'sha1'])
    pass


# -*- coding: utf-8 -*-
#  Copyright (C) 2009-2010, 2013, 2015 Rocky Bernstein
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os

# Our local modules
from trepan.processor.command import base_cmd
from trepan.processor import cmdfns as Mcmdfns

class NextCommand(base_cmd.DebuggerCommand):

    aliases       = ('next+', 'next-', 'n', 'n-', 'n+')
    category      = 'running'
    execution_set = ['Running']
    min_args      = 0
    max_args      = 1
    name          = os.path.basename(__file__).split('.')[0]
    need_stack    = True
    short_help    = 'Step program without entering called functions'

    def run(self, args):
        """**next**[**+**|**-**] [*count*]

Step one statement ignoring steps into function calls at this level.

With an integer argument, perform `next` that many times. However if
an exception occurs at this level, or we *return*, *yield* or the
thread changes, we stop regardless of count.

A suffix of `+` on the command or an alias to the command forces to
move to another line, while a suffix of `-` does the opposite and
disables the requiring a move to a new line. If no suffix is given,
the debugger setting 'different-line' determines this behavior.

See also:
---------

`step`, `skip`, `jump` (there's no `hop` yet), `continue`, and
`finish` for other ways to progress execution.
"""

        if len(args) <= 1:
            step_ignore = 0
        else:
            step_ignore = self.proc.get_int(args[1], default=1,
                                            cmdname='next')
            if step_ignore is None: return False
            # 0 means stop now or step 1, so we subtract 1.
            step_ignore -= 1
            pass
        self.core.different_line   = \
            Mcmdfns.want_different_line(args[0],
                                        self.debugger.settings['different'])
        self.core.set_next(self.proc.frame, step_ignore)
        self.proc.continue_running = True   # Break out of command read loop
        return True
    pass

if __name__ == '__main__':
    from mock import MockDebugger
    d = MockDebugger()
    cmd = NextCommand(d.core.processor)
    for c in (['n', '5'],
              ['next', '1+2'],
              ['n', 'foo']):
        d.core.step_ignore = 0
        cmd.continue_running = False
        result = cmd.run(c)
        print('Run result: %s' % result)
        print('step_ignore %d, continue_running: %s' % (d.core.step_ignore,
                                                        cmd.continue_running,))
        pass
    for c in (['n'], ['next+'], ['n-']):
        d.core.step_ignore = 0
        cmd.continue_running = False
        result = cmd.run(c)
        print(cmd.core.different_line)
        pass
    pass


# -*- coding: utf-8 -*-
#   Copyright (C) 2015, 2017 Rocky Bernstein <rocky@gnu.org>
"""Breakpoints as used in a debugger.

This code is a rewrite of the stock python bdb.Breakpoint"""

__all__ = ["BreakpointManager", "Breakpoint"]

import os.path

class BreakpointManager:
    """Manages the list of Breakpoints.

    Breakpoints are indexed by number in the `bpbynumber' list, and
    through a (file,line) tuple which is a key in the `bplist'
    dictionary. If the breakpoint is a function it is in `fnlist' as
    well.  Note there may be more than one breakpoint per line which
    may have different conditions associated with them.
    """
    def __init__(self):
        self.reset()
        return

    def bpnumbers(self):
        '''Returns a list of strings of breakpoint numbers'''
        return ["%d" % bp.number for bp in self.bpbynumber if
                bp is not None]

    def get_breakpoint(self, i):
        if isinstance(i, str):
            try:
                i = int(i)
            except ValueError:
                return(False, 'Breakpoint value %r is not a number.' % i,
                       None)
            pass
        if 1 == len(self.bpbynumber):
            return (False, 'No breakpoints set.', None)
        elif i >= len(self.bpbynumber) or i <= 0:
            return (False, 'Breakpoint number %d out of range 1..%d.' %
                    (i, len(self.bpbynumber)-1), None)
        bp = self.bpbynumber[i]
        if bp is None:
            return (False, 'Breakpoint %d previously deleted.' % i, None)
        return (True, None, bp)

    def add_breakpoint(self, filename, lineno, temporary=False, condition=None,
                       func=None):

        bpnum = len(self.bpbynumber)
        if filename: filename  = os.path.realpath(filename)
        brkpt = Breakpoint(bpnum, filename, lineno, temporary, condition,
                           func)
        # Build the internal lists of breakpoints
        self.bpbynumber.append(brkpt)
        if (filename, lineno) in self.bplist:
            self.bplist[filename, lineno].append(brkpt)
        else:
            self.bplist[filename, lineno] = [brkpt]
            pass
        if func:
            if func in self.fnlist:
                self.fnlist[func].append(brkpt)
            else:
                self.fnlist[func] = [brkpt]
                pass
        return brkpt

    def delete_all_breakpoints(self):
        bp_list = []
        for bp in self.bpbynumber:
            if bp:
                bp_list.append(str(bp.number))
                self.delete_breakpoint(bp)
                pass
        if not bp_list:
            return 'There are no breakpoints'
        else:
            return 'Deleted breakpoints %s' % ', '.join(bp_list)
        return

    def delete_breakpoint(self, bp):
        " remove breakpoint `bp'"
        bpnum = bp.number
        self.bpbynumber[bpnum] = None   # No longer in list
        index = (bp.filename, bp.line)
        if index not in self.bplist: return False
        self.bplist[index].remove(bp)
        if not self.bplist[index]:
            # No more breakpoints for this file:line combo
            del self.bplist[index]
        return True

    def delete_breakpoint_by_number(self, bpnum):
        "Remove a breakpoint given its breakpoint number."
        success, msg, bp = self.get_breakpoint(bpnum)
        if not success:
            return False, msg
        self.delete_breakpoint(bp)
        return (True, '')

    def en_disable_all_breakpoints(self,  do_enable=True):
        "Enable or disable all breakpoints."
        bp_list = [bp for bp in self.bpbynumber if bp]
        bp_nums = []
        if do_enable:
            endis = 'en'
        else:
            endis = 'dis'
            pass
        if not bp_list:
            return "No breakpoints to %sable" % endis
        for bp in bp_list:
            bp.enabled = do_enable
            bp_nums.append(str(bp.number))
            pass
        return ("Breakpoints %sabled: %s" % (endis, ", ".join(bp_nums)))

    def en_disable_breakpoint_by_number(self, bpnum, do_enable=True):
        "Enable or disable a breakpoint given its breakpoint number."
        success, msg, bp = self.get_breakpoint(bpnum)
        if not success:
            return success, msg
        if do_enable:
            endis = 'en'
        else:
            endis = 'dis'
            pass
        if bp.enabled == do_enable:
            return (False, ('Breakpoint (%r) previously %sabled' %
                            (str(bpnum), endis,)))
        bp.enabled = do_enable
        return (True, '')

    def delete_breakpoints_by_lineno(self, filename, lineno):
        """Removes all breakpoints at a give filename and line number.
        Returns a list of breakpoints numbers deleted.
        """
        if (filename, lineno) not in self.bplist:
            return []
        breakpoints = self.bplist[(filename, lineno)]
        bpnums = [bp.number for bp in breakpoints]
        for bp in list(breakpoints):
            self.delete_breakpoint(bp)
        return bpnums

    def find_bp(self, filename, lineno, frame):
        """Determine which breakpoint for this file:line is to be acted upon.

        Called only if we know there is a bpt at this
        location.  Returns breakpoint that was triggered and a flag
        that indicates if it is ok to delete a temporary breakpoint.

        """
        possibles = self.bplist[filename, lineno]
        for i in range(0, len(possibles)):
            b = possibles[i]
            if not b.enabled:
                continue
            if not checkfuncname(b, frame):
                continue
            # Count every hit when bp is enabled
            b.hits += 1
            if not b.condition:
                # If unconditional, and ignoring, go on to next, else
                # break
                if b.ignore > 0:
                    b.ignore = b.ignore -1
                    continue
                else:
                    # breakpoint and marker that's ok to delete if
                    # temporary
                    return (b, True)
            else:
                # Conditional bp.
                # Ignore count applies only to those bpt hits where the
                # condition evaluates to true.
                try:
                    val = eval(b.condition, frame.f_globals, frame.f_locals)
                    if val:
                        if b.ignore > 0:
                            b.ignore = b.ignore -1
                            # continue
                        else:
                            return (b, True)
                    # else:
                    #   continue
                except:
                    # if eval fails, most conservative thing is to
                    # stop on breakpoint regardless of ignore count.
                    # Don't delete temporary, as another hint to user.
                    return (b, False)
                pass
            pass
        return (None, None)

    def last(self):
        return len(self.bpbynumber)-1

    def reset(self):
        """ A list of breakpoints by breakpoint number.  Each entry is
        None or an instance of Breakpoint.  Index 0 is unused, except
        for marking an effective break .... see effective(). """
        self.bpbynumber = [None]

        # A list of breakpoints indexed by (file, lineno) tuple
        self.bplist = {}
        self.fnlist  = {}

        return

    pass  # BreakpointManager

class Breakpoint:

    """Breakpoint class implements temporary breakpoints, ignore
    counts, disabling and (re)-enabling breakpoints and breakpoint
    conditionals.
    """

    def __init__(self, number, filename, line, temporary=False,
                 condition=None, funcname=None):

        self.condition = condition
        self.enabled   = True

        self.filename  = filename
        if filename: self.filename  = os.path.realpath(filename)

        # Needed if funcname is not None.
        self.func_first_executable_line = None
        self.funcname  = funcname

        # Number of time breakpoint has been hit
        self.hits      = 0

        # Number of times to ignore breakpoint before stopping
        self.ignore    = 0

        self.line      = line
        self.number    = number

        # Delete breakpoint after hitting it.
        self.temporary = temporary
        return

    def __str__(self):
        if self.temporary:
            disp = 'del  '
        else:
            disp = 'keep '
        if self.enabled:
            disp = disp + 'yes  '
        else:
            disp = disp + 'no   '
        msg = '%-4dbreakpoint   %s at %s:%d' % (self.number, disp,
                                                self.filename, self.line)
        if self.condition:
            msg += '\n\tstop only if %s' % self.condition
        if self.ignore:
            msg += msg('\n\tignore next %d hits' % self.ignore)
        if self.hits:
            if (self.hits > 1): ss = 's'
            else: ss = ''
            msg +='\n\tbreakpoint already hit %d time%s' % self.hits, ss
        return msg

    def enable(self):
        self.enabled = True
        return self.enabled

    def disable(self):
        self.enabled = False
        return self.enabled

    def icon_char(self):
        """Return a one-character "icon" giving the state of the breakpoint
        't': temporary breakpoint
        'B': enabled breakpoint
        'b': disabled breakpoint
        """
        if self.temporary : return 't'
        elif self.enabled:  return 'B'
        else: return 'b'
        return

    pass  # end of Breakpoint class

def checkfuncname(b, frame):
    """Check whether we should break here because of `b.funcname`."""
    if not b.funcname:
        # Breakpoint was set via line number.
        if b.line != frame.f_lineno:
            # Breakpoint was set at a line with a def statement and the function
            # defined is called: don't break.
            return False
        return True

    # Breakpoint set via function name.

    if frame.f_code.co_name != b.funcname:
        # It's not a function call, but rather execution of def statement.
        return False

    # We are in the right frame.
    if not b.func_first_executable_line:
        # The function is entered for the 1st time.
        b.func_first_executable_line = frame.f_lineno

    if b.func_first_executable_line != frame.f_lineno:
        # But we are not at the first line number: don't break.
        return False
    return True

# Demo

if __name__=='__main__':
    bpmgr = BreakpointManager()
    print(bpmgr.last())
    bp = bpmgr.add_breakpoint('foo', 5)
    print(bp.icon_char())
    print(bpmgr.last())
    print(repr(bp))
    print(str(bp))
    bp.disable()
    print(str(bp))
    for i in 10, 1:
        status, msg = bpmgr.delete_breakpoint_by_number(i)
        print("Delete breakpoint %s: %s %s" % (i, status, msg,))
    import inspect
    frame = inspect.currentframe()
    print("Stop at bp: %s" % checkfuncname(bp, frame))

    def foo(bp, bpmgr):
        frame = inspect.currentframe()
        print("Stop at bp2: %s" % checkfuncname(bp, frame))
        # frame.f_lineno is constantly updated. So adjust for the
        # line difference between the add_breakpoint and the check.
        bp3 = bpmgr.add_breakpoint('foo', frame.f_lineno+1)
        print("Stop at bp3: %s" % checkfuncname(bp3, frame))
        return

    bp2 = bpmgr.add_breakpoint(None, None, False, None, 'foo')
    foo(bp2, bpmgr)
    bp3 = bpmgr.add_breakpoint('foo', 5, temporary=True)
    print(bp3.icon_char())
    print(bpmgr.bpnumbers())

    bp = bpmgr.add_breakpoint('bar', 3)
    filename = bp.filename
    for i in range(3):
        bp = bpmgr.add_breakpoint('bar', 6)
    print(bpmgr.delete_breakpoints_by_lineno(filename, 6))
    print(bpmgr.delete_breakpoints_by_lineno(filename, 6))
    print(bpmgr.bpnumbers())


# -*- coding: utf-8 -*-
#   Copyright (C) 2009, 2014-2015 Rocky Bernstein <rocky@gnu.org>
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""Debugger Script input interface. """

import io, types

from trepan.inout import base as Mbase

# Do we need this?
class ScriptInput(Mbase.DebuggerInputBase):
    """Debugger Script input - largely the same as DebuggerInput."""

    def __init__(self, inp, opts=None):

        self.input     = None
        self.line_edit = False  # Our name for GNU readline capability
        self.name      = None
        self.open(inp, opts)
        return

    def close(self):
        if self.input:
            self.input.close()
            pass
        return

    def open(self, inp, opts=None):
        """Use this to set what file to read from. """
        if isinstance(inp, io.TextIOWrapper):
            self.input = inp
        elif isinstance(inp, 'string'.__class__):  # FIXME
            self.name  = inp
            self.input = open(inp, 'r')
        else:
            raise IOError("Invalid input type (%s) for %s" %
                          (inp.__class__.__name__, inp))
        return

    def readline(self, prompt='', use_raw=None):
        """Read a line of input. Prompt and use_raw exist to be
        compatible with other input routines and are ignored.
        EOFError will be raised on EOF.
        """
        line = self.input.readline()
        if not line: raise EOFError
        return line.rstrip("\n")

# Demo
if __name__=='__main__':
    import os
    my_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                           'scriptin.py')
    inp = ScriptInput(my_file)
    line = inp.readline()
    print(line)
    inp.close()
    # Note opts below are aren't acted upon. They are there for
    # compatibility
    inp.open(my_file, opts={'use_raw': False})
    while True:
        try:
            inp.readline()
        except EOFError:
            break
        pass
    try:
        inp.readline()
    except EOFError:
        print('EOF handled correctly')
    pass


# -*- coding: utf-8 -*-
#   Copyright (C) 2009, 2013-2015, 2017 Rocky Bernstein <rocky@gnu.org>
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""Module for Server (i.e. program to communication-device) interaction"""
import atexit

# Our local modules
from trepan import interface as Minterface
from trepan.inout import tcpserver as Mtcpserver, fifoserver as Mfifoserver
from trepan.interfaces import comcodes as Mcomcodes

DEFAULT_INIT_CONNECTION_OPTS = {'IO': 'TCP',
                                'PORT': 1955}

class ServerInterface(Minterface.TrepanInterface):
    """Interface for debugging a program but having user control
    reside outside of the debugged process, possibly on another
    computer."""

    def __init__(self, inout=None, out=None, connection_opts={}):
        atexit.register(self.finalize)

        opts = DEFAULT_INIT_CONNECTION_OPTS.copy()
        opts.update(connection_opts)
        self.inout = None  # initialize in case assignment below fails
        if inout:
            self.inout = inout
        else:
            self.server_type = opts['IO']
            if 'FIFO' == self.server_type:
                self.inout = Mfifoserver.FIFOServer()
            else:
                self.inout = Mtcpserver.TCPServer(opts=opts)
                pass
            pass
        # For Compatability
        self.output = self.inout
        self.input  = self.inout
        self.interactive = True  # Or at least so we think initially
        self.histfile = None
        return

    def close(self):
        """ Closes both input and output """
        if self.inout:
            self.inout.close()
        return

    def confirm(self, prompt, default):
        """ Called when a dangerous action is about to be done to make sure
        it's okay. `prompt' is printed; user response is returned."""
        while True:
            try:
                self.write_confirm(prompt, default)
                reply = self.readline('').strip().lower()
            except EOFError:
                return default
            if reply in ('y', 'yes'):
                return True
            elif reply in ('n', 'no'):
                return False
            else:
                self.msg("Please answer y or n.")
                pass
            pass
        return default

    def errmsg(self, str, prefix="** "):
        """Common routine for reporting debugger error messages.
           """
        return self.msg("%s%s" %(prefix, str))

    def finalize(self, last_wishes=Mcomcodes.QUIT):
        # print exit annotation
        if self.is_connected():
            self.inout.writeline(last_wishes)
            pass
        self.close()
        return

    def is_connected(self):
        """ Return True if we are connected """
        return self.inout and 'connected' == self.inout.state

    def msg(self, msg):
        """ used to write to a debugger that is connected to this
        server; `str' written will have a newline added to it
        """
        self.inout.writeline(Mcomcodes.PRINT + msg)
        return

    def msg_nocr(self, msg):
        """ used to write to a debugger that is connected to this
        server; `str' written will not have a newline added to it
        """
        self.inout.write(Mcomcodes.PRINT +  msg)
        return

    def read_command(self, prompt):
        return self.readline(prompt)

    def read_data(self):
        return self.inout.read_data()

    def readline(self, prompt, add_to_history=True):
        if prompt:
            self.write_prompt(prompt)
            pass
        coded_line = self.inout.read_msg()
        self.read_ctrl = coded_line[0]
        return coded_line[1:]

    def state(self):
        """ Return connected """
        return self.inout.state

    def write_prompt(self, prompt):
        return self.inout.writeline(Mcomcodes.PROMPT + prompt)

    def write_confirm(self, prompt, default):
        if default:
            code = Mcomcodes.CONFIRM_TRUE
        else:
            code = Mcomcodes.CONFIRM_FALSE
            pass
        return self.inout.writeline(code + prompt)

    pass

# Demo
if __name__=='__main__':
    connection_opts={'IO': 'TCP', 'PORT': 1954}
    intf = ServerInterface(connection_opts=connection_opts)
    pass


"""
Simple SPARK-style scanner
Copyright (c) 2017-2018 Rocky Bernstein
"""

from __future__ import print_function
import re
from spark_parser.scanner import GenericScanner
from trepan.processor.parse.tok import Token

class ScannerError(Exception):
    def __init__(self, text, text_cursor):
        self.text = text
        self.text_cursor = text_cursor

    def __str__(self):
        return self.text + "\n" + self.text_cursor

class LocationScanner(GenericScanner):

    def error(self, s):
        """Show text and a caret under that. For example:
x = 2y + z
     ^
"""
        # print("Lexical error:")
        # print("%s" % s[:self.pos+10])  # + 10 for trailing context
        # print("%s^" % (" "*(self.pos-1)))
        # for t in self.rv: print(t)
        raise ScannerError( ("%s" % s),
                            ("%s^" % (" "*(self.pos-1))) )

    def tokenize(self, input):
        self.rv = []
        GenericScanner.tokenize(self, input)
        return self.rv

    def add_token(self, name, v):
        t = Token(kind=name, value=v, offset=self.pos)
        self.pos += len(str(v))
        self.rv.append(t)

    # The function names below begin with 't_'.
    # This indicates to GenericScanner that these routines
    # form the tokens. GenericScanner introspects on the
    # method names of this class and the docstrings to come
    # up with both the names of the tokens and the regular expressions
    # that make up those tokens

    # Recognize white space, but we don't create a token for it.
    # This has the effect of stripping white space between tokens
    def t_whitespace(self, s):
        r'\s+'
        self.add_token('SPACE', s)
        self.pos += len(s)
        pass

    def t_file_or_func(self, s):
        r'(?:[^*-+,\d\'"\t \n:][^\'"\t \n:,]*)|(?:^""".+""")|(?:\'\'\'.+\'\'\')'
        maybe_funcname = True
        if s == 'if':
            self.add_token('IF', s)
            return
        if s[0] in frozenset(('"', "'")):
            # Pick out text inside of triple-quoted string
            if ( (s.startswith("'''") and s.endswith("'''") ) or
                 (s.startswith('"""') and s.endswith('"""') ) ):
                base = s[3:-3]
            else:
                # Pick out text inside singly-quote string
                base = s[1:-1]
            maybe_funcname = False
        else:
            base = s
        if maybe_funcname and re.match('[a-zA-Z_][[a-zA-Z_.0-9\[\]]+\(\)', s):
            self.add_token('FUNCNAME', base)
        else:
            self.add_token('FILENAME', base)
        self.pos += len(s)

    def t_single_quote_file(self, s):
        r"'[^'].+'"
        # Pick out text inside of singe-quoted string
        base = s[1:-1]
        self.add_token('FILENAME', base)
        self.pos += len(s)

    def t_double_quote_file(self, s):
        r'"[^"]+"'
        # Pick out text inside of singe-quoted string
        base = s[1:-1]
        self.add_token('FILENAME', base)
        self.pos += len(s)

    def t_colon(self, s):
        r':'
        # Used to separate a filename from a line number
        self.add_token('COLON', s)
        self.pos += len(s)

    def t_comma(self, s):
        r','
        # Used in "list" to separate first from last
        self.add_token('COMMA', s)
        self.pos += len(s)

    def t_direction(self, s):
        r'^[+-]$'
        # Used in the "list" command
        self.add_token('DIRECTION', s)
        self.pos += len(s)

    # Recognize integers
    def t_number(self, s):
        r'\d+'
        pos = self.pos
        self.add_token('NUMBER', int(s))
        self.pos = pos + len(s)

    # Recognize list offsets (counts)
    def t_offset(self, s):
        r'[+]\d+'
        pos = self.pos
        self.add_token('OFFSET', s)
        self.pos = pos + len(s)

    # Recognize addresses (bytecode offsets)
    def t_address(self, s):
        r'[*]\d+'
        pos = self.pos
        self.add_token('ADDRESS', s)
        self.pos = pos + len(s)

if __name__ == "__main__":
    for line in (
            # '/tmp/foo.py:12',
            # "'''/tmp/foo.py:12'''",
            "'/tmp/foo.py:12'",
            "6",
            "*6",
            "8 *6",
            # "/tmp/foo.py line 12",
            # "\"\"\"/tmp/foo.py's line 12\"\"\"",
            # "12",
            # "../foo.py:5",
            # "gcd()",
            # "foo.py line 5 if x > 1",
            # "5 ,",
            # "5,",
            # "5,10",
            # ",10",
            ):
        try:
            tokens = LocationScanner().tokenize(line.strip())
            for t in tokens:
                print(t)
                pass
        except ScannerError as e:
            print("Lexical error at or around: ")
            print(e.text)
            print(e.text_cursor)

        pass
    pass


# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import subprocess
import tempfile
from .MetaMap import MetaMap
from .Concept import Corpus

class SubprocessBackend(MetaMap):
    def __init__(self, metamap_filename, version=None):
        """ Interface to MetaMap using subprocess. This creates a
            command line call to a specified metamap process.
        """
        MetaMap.__init__(self, metamap_filename, version)

    def extract_concepts(self, sentences=None, ids=None,
                         composite_phrase=4, filename=None,
                         file_format='sldi', allow_acronym_variants=False,
                         word_sense_disambiguation=False, allow_large_n=False,
                         strict_model=False, relaxed_model=False,
                         allow_overmatches=False, allow_concept_gaps=False,
                         term_processing=False, no_derivational_variants=False,
                         derivational_variants=False, ignore_word_order=False,
                         unique_acronym_variants=False,
                         prefer_multiple_concepts=False,
                         ignore_stop_phrases=False, compute_all_mappings=False,
                         mm_data_version=False, exclude_sources=[],
                         restrict_to_sources=[], restrict_to_sts=[], exclude_sts=[]):
        """ extract_concepts takes a list of sentences and ids(optional)
            then returns a list of Concept objects extracted via
            MetaMap.

            Supported Options:
                Composite Phrase -Q
                Word Sense Disambiguation -y
                use strict model -A
                use relaxed model -C
                allow large N -l
                allow overmatches -o
                allow concept gaps -g
                term processing -z
                No Derivational Variants -d
                All Derivational Variants -D
                Ignore Word Order -i
                Allow Acronym Variants -a
                Unique Acronym Variants -u
                Prefer Multiple Concepts -Y
                Ignore Stop Phrases -K
                Compute All Mappings -b
                MM Data Version -V
                Exclude Sources -e
                Restrict to Sources -R
                Restrict to Semantic Types -J
                Exclude Semantic Types -k

            For information about the available options visit
            http://metamap.nlm.nih.gov/.

            Note: If an error is encountered the process will be closed
                  and whatever was processed, if anything, will be
                  returned along with the error found.
        """
        if allow_acronym_variants and unique_acronym_variants:
            raise ValueError("You can't use both allow_acronym_variants and "
                             "unique_acronym_variants.")
        if (sentences is not None and filename is not None) or \
                (sentences is None and filename is None):
            raise ValueError("You must either pass a list of sentences "
                             "OR a filename.")
        if file_format not in ['sldi','sldiID']:
            raise ValueError("file_format must be either sldi or sldiID")

        input_file = None
        if sentences is not None:
            input_file = tempfile.NamedTemporaryFile(mode="wb", delete=False)
        else:
            input_file = open(filename, 'r')
        output_file = tempfile.NamedTemporaryFile(mode="r", delete=False)
        error = None
        try:
            if sentences is not None:
                if ids is not None:
                    for identifier, sentence in zip(ids, sentences):
                        input_file.write('{0!r}|{1!r}\n'.format(identifier, sentence).encode('utf8'))
                else:
                    for sentence in sentences:
                        input_file.write('{0!r}\n'.format(sentence).encode('utf8'))
                input_file.flush()

            command = [self.metamap_filename, '-N']
            command.append('-Q')
            command.append(str(composite_phrase))
            if mm_data_version is not False:
                if mm_data_version not in ['Base', 'USAbase', 'NLM']:
                    raise ValueError("mm_data_version must be Base, USAbase, or NLM.")
                command.append('-V')
                command.append(str(mm_data_version))
            if word_sense_disambiguation:
                command.append('-y')
            if strict_model:
                command.append('-A')
            if relaxed_model:
                command.append('-C')
            if allow_large_n:
                command.append('-l')
            if allow_overmatches:
                command.append('-o')
            if allow_concept_gaps:
                command.append('-g')
            if term_processing:
                command.append('-z')
            if no_derivational_variants:
                command.append('-d')
            if derivational_variants:
                command.append('-D')
            if ignore_word_order:
                command.append('-i')
            if allow_acronym_variants:
                command.append('-a')
            if unique_acronym_variants:
                command.append('-u')
            if prefer_multiple_concepts:
                command.append('-Y')
            if ignore_stop_phrases:
                command.append('-K')
            if compute_all_mappings:
                command.append('-b')
            if len(exclude_sources) > 0:
                command.append('-e')
                command.append(str(','.join(exclude_sources)))
            if len(restrict_to_sources) > 0:
                command.append('-R')
                command.append(str(','.join(restrict_to_sources)))
            if len(restrict_to_sts) > 0:
                command.append('-J')
                command.append(str(','.join(restrict_to_sts)))
            if len(exclude_sts) > 0:
                command.append('-k')
                command.append(str(','.join(exclude_sts)))
            if ids is not None or (file_format == 'sldiID' and
                    sentences is None):
                command.append('--sldiID')
            else:
                command.append('--sldi')
            command.append(input_file.name)
            command.append(output_file.name)

            metamap_process = subprocess.Popen(command, stdout=subprocess.PIPE)
            while metamap_process.poll() is None:
                stdout = str(metamap_process.stdout.readline())
                if 'ERROR' in stdout:
                    metamap_process.terminate()
                    error = stdout.rstrip()
            output = str(output_file.read())
        finally:
            if sentences is not None:
                os.remove(input_file.name)
            else:
                input_file.close()
            os.remove(output_file.name)

        concepts = Corpus.load(output.splitlines())
        return (concepts, error)


# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import subprocess
import tempfile
from .MetaMapLite import MetaMapLite
from .ConceptLite import CorpusLite

class SubprocessBackendLite(MetaMapLite):
    def __init__(self, metamap_filename):
        """ Interface to MetaMap using subprocess. This creates a
            command line call to a specified metamap process.
        """
        MetaMapLite.__init__(self, metamap_filename=metamap_filename)

    def extract_concepts(self, sentences=None, ids=None, filename=None,
                         restrict_to_sts=None, restrict_to_sources=None):
        """ extract_concepts takes a list of sentences and ids(optional)
            then returns a list of Concept objects extracted via
            MetaMapLite.

            Supported Options:
                Restrict to Semantic Types --restrict_to_sts
                Restrict to Sources --restrict_to_sources

            For information about the available options visit
            http://metamap.nlm.nih.gov/.

            Note: If an error is encountered the process will be closed
                  and whatever was processed, if anything, will be
                  returned along with the error found.
        """
        if (sentences is not None and filename is not None) or \
                (sentences is None and filename is None):
            raise ValueError("You must either pass a list of sentences "
                             "OR a filename.")

        input_file = None
        if sentences is not None:
            input_file = tempfile.NamedTemporaryFile(mode="wb", delete=False)
        else:
            input_file = open(filename, 'r')

        # Unlike MetaMap, MetaMapLite does not take an output filename as a parameter.
        # It creates a new output file at same location as "input_file" with the default file extension ".mmi".
        # output_file = tempfile.NamedTemporaryFile(mode="r", delete=False)
        output_file_name = None
        error = None
        try:
            if sentences is not None:
                if ids is not None:
                    for identifier, sentence in zip(ids, sentences):
                        input_file.write('{0!r}|{1!r}\n'.format(identifier, sentence).encode('utf8'))
                else:
                    for sentence in sentences:
                        input_file.write('{0!r}\n'.format(sentence).encode('utf8'))
                input_file.flush()

            command = ["bash", os.path.join(self.metamap_filename, "metamaplite.sh")]
            if restrict_to_sts:
                if isinstance(restrict_to_sts, str):
                    restrict_to_sts = [restrict_to_sts]
                if len(restrict_to_sts) > 0:
                    command.append('--restrict_to_sts')
                    command.append(str(','.join(restrict_to_sts)))

            if restrict_to_sources:
                if isinstance(restrict_to_sources, str):
                    restrict_to_sources = [restrict_to_sources]
                if len(restrict_to_sources) > 0:
                    command.append('--restrict_to_sources')
                    command.append(str(','.join(restrict_to_sources)))

            if ids is not None:
                command.append('--inputformat=sldiwi')

            command.append(input_file.name)
            # command.append(output_file.name)

            metamap_process = subprocess.Popen(command, stdout=subprocess.PIPE)
            while metamap_process.poll() is None:
                stdout = str(metamap_process.stdout.readline())
                if 'ERROR' in stdout:
                    metamap_process.terminate()
                    error = stdout.rstrip()

            # print("input file name: {0}".format(input_file.name))
            output_file_name, file_extension = os.path.splitext(input_file.name)
            output_file_name += "." + "mmi"
            # print("output_file_name: {0}".format(output_file_name))
            with open(output_file_name) as fd:
                output = fd.read()
            # output = str(output_file.read())
            # print("output: {0}".format(output))
        finally:
            if sentences is not None:
                os.remove(input_file.name)
            else:
                input_file.close()
            # os.remove(output_file.name)
            os.remove(output_file_name)

        concepts = CorpusLite.load(output.splitlines())
        return concepts, error


import multiprocessing
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager

from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base as sa_declarative_base
from sqlalchemy.orm import sessionmaker
from tornado.concurrent import Future, chain_future
from tornado.ioloop import IOLoop

__all__ = (
    'as_future',
    'declarative_base',
    'make_session_factory',
    'SessionMixin',
    'set_max_workers',
)

class MissingFactoryError(Exception):
    pass

class _AsyncExecution:
    """Tiny wrapper around ThreadPoolExecutor. This class is not meant to be
    instantiated externally, but internally we just use it as a wrapper around
    ThreadPoolExecutor so we can control the pool size and make the
    `as_future` function public.

    Parameters
    ----------
    max_workers : int
        Worker count for the ThreadPoolExecutor
    """

    def __init__(self, max_workers=None):
        self._max_workers = max_workers or multiprocessing.cpu_count()
        self._pool = None

    def set_max_workers(self, count):
        if self._pool:
            self._pool.shutdown(wait=True)

        self._max_workers = count
        self._pool = ThreadPoolExecutor(max_workers=self._max_workers)

    def as_future(self, query):
        """Wrap a `sqlalchemy.orm.query.Query` object into a
        `concurrent.futures.Future` so that it can be yielded.

        Parameters
        ----------
        query : sqlalchemy.orm.query.Query
            SQLAlchemy query object to execute

        Returns
        -------
            tornado.concurrent.Future
                A `Future` object wrapping the given query so that tornado can
                await/yield on it
        """
        # concurrent.futures.Future is not compatible with the "new style"
        # asyncio Future, and awaiting on such "old-style" futures does not
        # work.
        #
        # tornado includes a `run_in_executor` function to help with this
        # problem, but it's only included in version 5+. Hence, we copy a
        # little bit of code here to handle this incompatibility.

        if not self._pool:
            self._pool = ThreadPoolExecutor(max_workers=self._max_workers)

        old_future = self._pool.submit(query)
        new_future = Future()

        IOLoop.current().add_future(
            old_future, lambda f: chain_future(f, new_future)
        )

        return new_future

class SessionFactory:
    """SessionFactory is a wrapper around the functions that SQLAlchemy
    provides. The intention here is to let the user work at the session level
    instead of engines and connections.
    """

    def __init__(self, *args, **kwargs):
        self._engine = create_engine(*args, **kwargs)

        self._factory = sessionmaker()
        self._factory.configure(bind=self._engine)

    def make_session(self):
        return self._factory()

    @property
    def engine(self):
        return self._engine

class SessionMixin:
    _session = None

    @contextmanager
    def make_session(self):
        session = None

        try:
            session = self._make_session()

            yield session
        except Exception:
            if session:
                session.rollback()
            raise
        else:
            session.commit()
        finally:
            if session:
                session.close()

    def on_finish(self):
        next_on_finish = None

        try:
            next_on_finish = super(SessionMixin, self).on_finish
        except AttributeError:
            pass

        if self._session:
            self._session.commit()
            self._session.close()

        if next_on_finish:
            next_on_finish()

    @property
    def session(self):
        if not self._session:
            self._session = self._make_session()
        return self._session

    def _make_session(self):
        factory = self.application.settings.get('session_factory')

        if not factory:
            raise MissingFactoryError()

        return factory.make_session()

_async_exec = _AsyncExecution()

as_future = _async_exec.as_future

set_max_workers = _async_exec.set_max_workers

def make_session_factory(*args, **kwargs):
    return SessionFactory(*args, **kwargs)

class _declarative_base:
    def __init__(self):
        self._instance = None

    def __call__(self):
        if not self._instance:
            self._instance = sa_declarative_base()
        return self._instance

declarative_base = _declarative_base()


import contextlib
import logging
from datetime import timedelta

from django.conf import settings as django_settings
from django.contrib import messages
from django.contrib.admin.models import CHANGE, LogEntry
from django.contrib.auth import get_user_model, load_backend, login, logout
from django.contrib.auth.models import update_last_login
from django.contrib.auth.signals import user_logged_in
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ImproperlyConfigured
from django.core.signing import SignatureExpired, TimestampSigner

from . import settings as la_settings

signer = TimestampSigner()
logger = logging.getLogger(__name__)
username_field = la_settings.USERNAME_FIELD

@contextlib.contextmanager
def no_update_last_login():
    """
    Disconnect any signals to update_last_login() for the scope of the context
    manager, then restore.
    """
    kw = {"receiver": update_last_login}
    kw_id = {"receiver": update_last_login, "dispatch_uid": "update_last_login"}

    was_connected = user_logged_in.disconnect(**kw)
    was_connected_id = not was_connected and user_logged_in.disconnect(**kw_id)
    yield
    # Restore signal if needed
    if was_connected:
        user_logged_in.connect(**kw)
    elif was_connected_id:
        user_logged_in.connect(**kw_id)

def login_as(user, request, store_original_user=True):
    """
    Utility function for forcing a login as specific user -- be careful about
    calling this carelessly :)
    """

    # Save the original user pk before it is replaced in the login method
    original_user_pk = request.user.pk

    # Find a suitable backend.
    if not hasattr(user, "backend"):
        for backend in django_settings.AUTHENTICATION_BACKENDS:
            if not hasattr(load_backend(backend), "get_user"):
                continue

            if user == load_backend(backend).get_user(user.pk):
                user.backend = backend
                break
        else:
            raise ImproperlyConfigured("Could not found an appropriate authentication backend")

    # Add admin audit log entry
    if original_user_pk:
        change_message = "User {0} logged in as {1}.".format(request.user, user)
        LogEntry.objects.log_action(
            user_id=original_user_pk,
            content_type_id=ContentType.objects.get_for_model(user).pk,
            object_id=user.pk,
            object_repr=str(user),
            change_message=change_message,
            action_flag=CHANGE,
        )

    # Log the user in.
    if not hasattr(user, "backend"):
        return

    if la_settings.UPDATE_LAST_LOGIN:
        login(request, user)
    else:
        with no_update_last_login():
            login(request, user)

    # Set a flag on the session
    if store_original_user:
        messages.warning(
            request,
            la_settings.MESSAGE_LOGIN_SWITCH.format(username=user.__dict__[username_field]),
            extra_tags=la_settings.MESSAGE_EXTRA_TAGS,
        )
        request.session[la_settings.USER_SESSION_FLAG] = signer.sign(original_user_pk)

def restore_original_login(request):
    """
    Restore an original login session, checking the signed session
    """
    original_session = request.session.get(la_settings.USER_SESSION_FLAG)
    logout(request)

    if not original_session:
        return

    try:
        original_user_pk = signer.unsign(
            original_session, max_age=timedelta(days=la_settings.USER_SESSION_DAYS_TIMESTAMP).total_seconds()
        )
        user = get_user_model().objects.get(pk=original_user_pk)
        messages.info(
            request,
            la_settings.MESSAGE_LOGIN_REVERT.format(username=user.__dict__[username_field]),
            extra_tags=la_settings.MESSAGE_EXTRA_TAGS,
        )
        login_as(user, request, store_original_user=False)
        if la_settings.USER_SESSION_FLAG in request.session:
            del request.session[la_settings.USER_SESSION_FLAG]
    except SignatureExpired:
        pass

def is_impersonated_session(request):
    """
    Checks if the session in the request is impersonated or not
    """
    return la_settings.USER_SESSION_FLAG in request.session


from django.contrib import messages
from django.contrib.admin.utils import unquote
from django.core.exceptions import ImproperlyConfigured
from django.shortcuts import redirect
from django.utils import six
from django.utils.translation import ugettext_lazy as _
from django.views.decorators.csrf import csrf_protect
from django.views.decorators.http import require_POST

from . import settings as la_settings
from .utils import login_as, restore_original_login

try:
    from importlib import import_module
except ImportError:
    from django.utils.importlib import import_module  # type: ignore

try:
    from django.contrib.auth import get_user_model

    User = get_user_model()
except ImportError:
    from django.contrib.auth.models import User  # type: ignore

def _load_module(path):
    """Code to load create user module. Copied off django-browserid."""

    i = path.rfind(".")
    module, attr = path[:i], path[i + 1 :]

    try:
        mod = import_module(module)
    except ImportError:
        raise ImproperlyConfigured("Error importing CAN_LOGIN_AS function: {}".format(module))
    except ValueError:
        raise ImproperlyConfigured("Error importing CAN_LOGIN_AS" " function. Is CAN_LOGIN_AS a" " string?")

    try:
        can_login_as = getattr(mod, attr)
    except AttributeError:
        raise ImproperlyConfigured("Module {0} does not define a {1} " "function.".format(module, attr))
    return can_login_as

@csrf_protect
@require_POST
def user_login(request, user_id):
    user = User.objects.get(pk=unquote(user_id))

    if isinstance(la_settings.CAN_LOGIN_AS, six.string_types):
        can_login_as = _load_module(la_settings.CAN_LOGIN_AS)
    elif hasattr(la_settings.CAN_LOGIN_AS, "__call__"):
        can_login_as = la_settings.CAN_LOGIN_AS
    else:
        raise ImproperlyConfigured("The CAN_LOGIN_AS setting is neither a valid module nor callable.")

    if not can_login_as(request, user):
        messages.error(request, _("You do not have permission to do that."), extra_tags=la_settings.MESSAGE_EXTRA_TAGS)
        return redirect(request.META.get("HTTP_REFERER", "/"))

    login_as(user, request)

    return redirect(la_settings.LOGIN_REDIRECT)

def user_logout(request):
    """
    This can replace your default logout view. In you settings, do:

    from django.core.urlresolvers import reverse_lazy
    LOGOUT_URL = reverse_lazy('loginas-logout')
    """
    restore_original_login(request)

    return redirect(la_settings.LOGOUT_REDIRECT)


import argparse
import json
import sys
import os
from urllib.parse import urlparse
from tqdm import tqdm

from .v5_client import LuminosoClient
from .v5_constants import URL_BASE

DESCRIPTION = 'Download documents from a Luminoso project via the command line.'
DOCS_PER_BATCH = 1000

# If we aren't being asked for "expanded" results, we only output these fields.
CONCISE_FIELDS = ['title', 'text', 'metadata']

# Even in the "expanded" version, we don't need these fields.
UNNECESSARY_FIELDS = ['match_score', 'doc_id']

def _sanitize_filename(filename):
    """
    Get a filename that lacks the / character (so it doesn't express a path by
    accident) and also lacks spaces (just for tab-completion convenience).
    """
    return filename.replace('/', '_').replace(' ', '_')

def iterate_docs(client, expanded=False, progress=False):
    """
    Yield each document in a Luminoso project in turn. Requires a client whose
    URL points to a project.

    If expanded=True, it will include additional fields that Luminoso added in
    its analysis, such as 'terms' and 'vector'.

    Otherwise, it will contain only the fields necessary to reconstruct the
    document: 'title', 'text', and 'metadata'.

    Shows a progress bar if progress=True.
    """
    # Get total number of docs from the project record
    num_docs = client.get()['document_count']
    progress_bar = None
    try:
        if progress:
            progress_bar = tqdm(desc='Downloading documents', total=num_docs)

        for offset in range(0, num_docs, DOCS_PER_BATCH):
            response = client.get('docs', offset=offset, limit=DOCS_PER_BATCH)
            docs = response['result']
            for doc in docs:
                # Get the appropriate set of fields for each document
                if expanded:
                    for field in UNNECESSARY_FIELDS:
                        doc.pop(field, None)
                else:
                    doc = {field: doc[field] for field in CONCISE_FIELDS}

                if progress:
                    progress_bar.update()
                yield doc

    finally:
        if progress:
            progress_bar.close()

def download_docs(client, output_filename=None, expanded=False):
    """
    Given a LuminosoClient pointing to a project and a filename to write to,
    retrieve all its documents in batches, and write them to a JSON lines
    (.jsons) file with one document per line.
    """
    if output_filename is None:
        # Find a default filename to download to, based on the project name.
        projname = _sanitize_filename(client.get()['name'])
        output_filename = '{}.jsons'.format(projname)

        # If the file already exists, add .1, .2, ..., after the project name
        # to unobtrusively get a unique filename.
        counter = 0
        while os.access(output_filename, os.F_OK):
            counter += 1
            output_filename = '{}.{}.jsons'.format(projname, counter)

        print('Downloading project to {!r}'.format(output_filename))

    with open(output_filename, 'w', encoding='utf-8') as out:
        for doc in iterate_docs(client, expanded=expanded, progress=True):
            print(json.dumps(doc, ensure_ascii=False), file=out)

def _main(argv):
    """
    Handle arguments for the 'lumi-download' command.
    """
    parser = argparse.ArgumentParser(
        description=DESCRIPTION,
        formatter_class=argparse.RawDescriptionHelpFormatter
    )
    parser.add_argument(
        '-b',
        '--base-url',
        default=URL_BASE,
        help='API root url, default: %s' % URL_BASE,
    )
    parser.add_argument(
        '-e', '--expanded',
        help="Include Luminoso's analysis of each document, such as terms and"
             ' document vectors',
        action='store_true',
    )
    parser.add_argument('-t', '--token', help='API authentication token')
    parser.add_argument(
        '-s',
        '--save-token',
        action='store_true',
        help='save --token for --base-url to ~/.luminoso/tokens.json',
    )
    parser.add_argument(
        'project_id', help='The ID of the project in the Daylight API'
    )
    parser.add_argument(
        'output_file', nargs='?', default=None,
        help='The JSON lines (.jsons) file to write to'
    )
    args = parser.parse_args(argv)
    if args.save_token:
        if not args.token:
            raise ValueError("error: no token provided")
        LuminosoClient.save_token(args.token,
                                  domain=urlparse(args.base_url).netloc)

    client = LuminosoClient.connect(url=args.base_url, token=args.token)
    proj_client = client.client_for_path('projects/{}'.format(args.project_id))
    download_docs(proj_client, args.output_file, args.expanded)

def main():
    """
    The setuptools entry point.
    """
    _main(sys.argv[1:])


"""
This file helps to build a JSON stream from arbitrary kinds of input,
including messy Excel CSV files.

The output this produces -- either in a real file, or in a temporarily
file that it returns a reference to -- is a JSON stream (.jsons), a file
with one JSON object per line.

Its input can be:

- A CSV in the "excel" dialect, with a header row
  - Preferably, this file is UTF-8 encoded.
  - However, this can read files in many other encodings, including MacRoman,
    which Excel sometimes produces and which trips up chardet.
- A single JSON list of the documents
- Or a JSON stream, which will effectively be validated before uploading.

The dictionary keys in JSON, or the column labels in CSV, should be the
document properties defined in the documentation at
https://analytics.luminoso.com/api/v4.
"""
from __future__ import unicode_literals
import json
import io
import csv
import ftfy
import logging
import unicodedata
import sys
import tempfile
from datetime import datetime
from .compat import PY3, string_type
logger = logging.getLogger(__name__)

def transcode(input_filename, output_filename=None, date_format=None):
    """
    Convert a JSON or CSV file of input to a JSON stream (.jsons). This
    kind of file can be easily uploaded using `luminoso_api.upload`.
    """
    if output_filename is None:
        # transcode to standard output
        output = sys.stdout
    else:
        if output_filename.endswith('.json'):
            logger.warning("Changing .json to .jsons, because this program "
                           "outputs a JSON stream format that is not "
                           "technically JSON itself.")
            output_filename += 's'
        output = open(output_filename, 'w')

    for entry in open_json_or_csv_somehow(input_filename,
                                          date_format=date_format):
        output.write(json.dumps(entry, ensure_ascii=False).encode('utf-8'))
        output.write('\n')
    output.close()

def transcode_to_stream(input_filename, date_format=None):
    """
    Read a JSON or CSV file and convert it into a JSON stream, which will
    be saved in an anonymous temp file.
    """
    tmp = tempfile.TemporaryFile()
    for entry in open_json_or_csv_somehow(input_filename,
                                          date_format=date_format):
        tmp.write(json.dumps(entry, ensure_ascii=False).encode('utf-8'))
        tmp.write(b'\n')
    tmp.seek(0)
    return tmp

def open_json_or_csv_somehow(filename, date_format=None):
    """
    Deduce the format of a file, within reason.

    - If the filename ends with .csv or .txt, it's csv.
    - If the filename ends with .jsons, it's a JSON stream (conveniently the
      format we want to output).
    - If the filename ends with .json, it could be a legitimate JSON file, or
      it could be a JSON stream, following a nonstandard convention that many
      people including us are guilty of. In that case:
      - If the first line is a complete JSON document, and there is more in the
        file besides the first line, then it is a JSON stream.
      - Otherwise, it is probably really JSON.
    - If the filename does not end with .json, .jsons, or .csv, we have to guess
      whether it's still CSV or tab-separated values or something like that.
      If it's JSON, the first character would almost certainly have to be a
      bracket or a brace. If it isn't, assume it's CSV or similar.
    """
    fileformat = None
    if filename.endswith('.csv'):
        fileformat = 'csv'
    elif filename.endswith('.jsons'):
        fileformat = 'jsons'
    else:
        with open(filename) as opened:
            line = opened.readline()
            if line[0] not in '{[' and not filename.endswith('.json'):
                fileformat = 'csv'
            else:
                if (line.count('{') == line.count('}') and
                    line.count('[') == line.count(']')):
                    # This line contains a complete JSON document. This probably
                    # means it's in linewise JSON ('.jsons') format, unless the
                    # whole file is on one line.
                    char = ' '
                    while char.isspace():
                        char = opened.read()
                        if char == '':
                            fileformat = 'json'
                            break
                    if fileformat is None:
                        fileformat = 'jsons'
                else:
                    fileformat = 'json'

    if fileformat == 'json':
        stream = json.load(open(filename), encoding='utf-8')
    elif fileformat == 'csv':
        stream = open_csv_somehow(filename)
    else:
        stream = stream_json_lines(filename)

    return _normalize_data(stream, date_format=date_format)

def _normalize_data(stream, date_format=None):
    """
    This function is meant to normalize data for upload to the Luminoso
    Analytics system. Currently it only normalizes dates.

    If date_format is not specified, or if there's no date in a particular doc,
    the the doc is yielded unchanged.
    """
    for doc in stream:
        if 'date' in doc and date_format is not None:
            try:
                doc['date'] = _convert_date(doc['date'], date_format)
            except ValueError:
                # ValueErrors cover the cases when date_format does not match
                # the actual format of the date, both for epoch and non-epoch
                # times.
                logger.exception('%s does not match the date format %s;'
                                 % (doc['date'], date_format))
        yield doc

def _convert_date(date_string, date_format):
    """
    Convert a date in a given format to epoch time. Mostly a wrapper for
    datetime's strptime.
    """
    if date_format != 'epoch':
        return datetime.strptime(date_string, date_format).timestamp()
    else:
        return float(date_string)

def detect_file_encoding(filename):
    """
    Use ftfy to detect the encoding of a file, based on a sample of its
    first megabyte.

    ftfy's encoding detector is limited. The only encodings it can detect are
    UTF-8, CESU-8, UTF-16, Windows-1252, and occasionally MacRoman. But it
    does much better than chardet.
    """
    with open(filename, 'rb') as opened:
        sample = opened.read(2 ** 20)
        _, encoding = ftfy.guess_bytes(sample)
        return encoding

def stream_json_lines(file):
    """
    Load a JSON stream and return a generator, yielding one object at a time.
    """
    if isinstance(file, string_type):
        file = open(file, 'rb')
    for line in file:
        line = line.strip()
        if line:
            if isinstance(line, bytes):
                line = line.decode('utf-8')
            yield json.loads(line)

def open_csv_somehow(filename):
    """
    Given a filename that we're told is a CSV file, detect its encoding,
    parse its header, and return a generator yielding its rows as dictionaries.
    """
    if PY3:
        return open_csv_somehow_py3(filename)
    else:
        return open_csv_somehow_py2(filename)

def transcode_to_utf8(filename, encoding):
    """
    Convert a file in some other encoding into a temporary file that's in
    UTF-8.
    """
    tmp = tempfile.TemporaryFile()
    for line in io.open(filename, encoding=encoding):
        tmp.write(line.strip('\uFEFF').encode('utf-8'))

    tmp.seek(0)
    return tmp

def open_csv_somehow_py2(filename):
    """
    Open a CSV file using Python 2's CSV module, working around the deficiency
    where it can't handle the null bytes of UTF-16.
    """
    encoding = detect_file_encoding(filename)
    if encoding.startswith('UTF-16'):
        csvfile = transcode_to_utf8(filename, encoding)
        encoding = 'UTF-8'
    else:
        csvfile = open(filename, 'rU')
    line = csvfile.readline()
    csvfile.seek(0)

    if '\t' in line:
        # tab-separated
        reader = csv.reader(csvfile, delimiter='\t')
    else:
        reader = csv.reader(csvfile, dialect='excel')

    header = reader.next()
    header = [cell.decode(encoding).lower().strip() for cell in header]
    encode_fn = lambda x: x.decode(encoding, 'replace')
    return _read_csv(reader, header, encode_fn)

def open_csv_somehow_py3(filename):
    encoding = detect_file_encoding(filename)
    csvfile = open(filename, 'rU', encoding=encoding, newline='')
    line = csvfile.readline()
    csvfile.seek(0)

    if '\t' in line:
        # tab-separated
        reader = csv.reader(csvfile, delimiter='\t')
    else:
        reader = csv.reader(csvfile, dialect='excel')

    header = next(reader)
    header = [cell.lower().strip() for cell in header]
    encode_fn = lambda x: x
    return _read_csv(reader, header, encode_fn)

def _read_csv(reader, header, encode_fn):
    """
    Given a constructed CSV reader object, a header row that we've read, and
    a detected encoding, yield its rows as dictionaries.
    """
    for row in reader:
        if len(row) == 0:
            continue
        row = [encode_fn(cell) for cell in row]
        row_list = zip(header, row)
        row_dict = dict(row_list)
        if len(row_dict['text']) == 0:
            continue
        row_dict['text'] = unicodedata.normalize(
            'NFKC', row_dict['text'].strip()
        )
        if row_dict.get('title') == '':
            del row_dict['title']
        if 'date' in row_dict:
            # We handle dates further in open_json_or_csv_somehow
            if row_dict['date'] == '':
                del row_dict['date']
        if 'subset' in row_dict:
            subsets = [cell[1] for cell in row_list
                       if cell[1] != '' and cell[0] == 'subset']
            if subsets:
                row_dict['subsets'] = subsets
            if 'subset' in row_dict:
                del row_dict['subset']
        yield row_dict

def main():
    """
    Handle command line arguments to convert a file to a JSON stream as a
    script.
    """
    logging.basicConfig(level=logging.INFO)
    import argparse
    parser = argparse.ArgumentParser(
        description="Translate CSV or JSON input to a JSON stream, or verify "
                    "something that is already a JSON stream."
    )
    parser.add_argument('input',
        help='A CSV, JSON, or JSON stream file to read.')
    parser.add_argument('output', nargs='?', default=None,
        help="The filename to output to. Recommended extension is .jsons. "
             "If omitted, use standard output.")
    args = parser.parse_args()
    transcode(args.input, args.output)

if __name__ == '__main__':
    main()