def run(self, args):
        """Current line number in source file"""
        # info line identifier
        if not self.proc.curframe:
            self.errmsg("No line number information available.")
            return
        if len(args) == 3:
            # lineinfo returns (item, file, lineno) or (None,)
            answer = self.lineinfo(args[2])
            if answer[0]:
                item, filename, lineno = answer
                if not os.path.isfile(filename):
                    filename = Mclifns.search_file(filename,
                                                   self.core.search_path,
                                                   self.main_dirname)
                self.msg('Line %s of "%s" <%s>' %
                         (lineno, filename, item))
            return
        filename=self.core.canonic_filename(self.proc.curframe)
        if not os.path.isfile(filename):
            filename = Mclifns.search_file(filename, self.core.search_path,
                                           self.main_dirname)
            pass

        filename = self.core.canonic_filename(self.proc.curframe)
        msg1 = 'Line %d of \"%s\"'  % (inspect.getlineno(self.proc.curframe),
                                       self.core.filename(filename))
        msg2 = ('at instruction %d' % self.proc.curframe.f_lasti)
        if self.proc.event:
            msg2 += ', %s event' % self.proc.event
            pass
        self.msg(Mmisc.wrapped_lines(msg1, msg2, self.settings['width']))
        return False


def find_debugged_frame(frame):
    """Find the first frame that is a debugged frame. We do this
    Generally we want traceback information without polluting it with
    debugger frames. We can tell these because those are frames on the
    top which don't have f_trace set. So we'll look back from the top
    to find the fist frame where f_trace is set.
    """
    f_prev = f = frame
    while f is not None and f.f_trace is None:
        f_prev = f
        f = f.f_back
        pass
    if f_prev:
        val = f_prev.f_locals.get('tracer_func_frame')
        if val == f_prev:
            if f_prev.f_back:
                f_prev = f_prev.f_back
                pass
            pass
        pass
    else:
        return frame
    return f_prev

def find_debugged_frame ( frame ) : f_prev = f = frame while f is not None and f . f_trace is None : f_prev = f f = f . f_back pass if f_prev : val = f_prev . f_locals . get ( 'tracer_func_frame' ) if val == f_prev : if f_prev . f_back : f_prev = f_prev . f_back pass pass pass else : return frame return f_prev

Find the first frame that is a debugged frame . We do this Generally we want traceback information without polluting it with debugger frames . We can tell these because those are frames on the top which don t have f_trace set . So we ll look back from the top to find the fist frame where f_trace is set .


def map_thread_names():
    '''Invert threading._active'''
    name2id = {}
    for thread_id in list(threading._active.keys()):
        thread = threading._active[thread_id]
        name = thread.getName()
        if name not in list(name2id.keys()):
            name2id[name] = thread_id
            pass
        pass
    return name2id

def map_thread_names ( ) : name2id = { } for thread_id in list ( threading . _active . keys ( ) ) : thread = threading . _active [ thread_id ] name = thread . getName ( ) if name not in list ( name2id . keys ( ) ) : name2id [ name ] = thread_id pass pass return name2id


def get_int(errmsg, arg, default=1, cmdname=None):
    """If arg is an int, use that otherwise take default."""
    if arg:
        try:
            # eval() is used so we will allow arithmetic expressions,
            # variables etc.
            default = int(eval(arg))
        except (SyntaxError, NameError, ValueError):
            if cmdname:
                errmsg("Command '%s' expects an integer; got: %s." %
                       (cmdname, str(arg)))
            else:
                errmsg('Expecting an integer, got: %s.' % str(arg))
                pass
            raise ValueError
    return default

If arg is an int use that otherwise take default .


def get_onoff(errmsg, arg, default=None, print_error=True):
    """Return True if arg is 'on' or 1 and False arg is 'off' or 0.
    Any other value is raises ValueError."""
    if not arg:
        if default is None:
            if print_error:
                errmsg("Expecting 'on', 1, 'off', or 0. Got nothing.")
                pass
            raise ValueError
        return default
    if arg == '1' or arg == 'on': return True
    if arg == '0' or arg =='off': return False

    if print_error:
        errmsg("Expecting 'on', 1, 'off', or 0. Got: %s." % str(arg))
    raise ValueError

Return True if arg is on or 1 and False arg is off or 0 . Any other value is raises ValueError .


def run_set_bool(obj, args):
    """set a Boolean-valued debugger setting. 'obj' is a generally a
    subcommand that has 'name' and 'debugger.settings' attributes"""
    try:
        if 0 == len(args): args = ['on']
        obj.debugger.settings[obj.name] = get_onoff(obj.errmsg, args[0])
    except ValueError:
        pass
    return

def run_set_bool ( obj , args ) : try : if 0 == len ( args ) : args = [ 'on' ] obj . debugger . settings [ obj . name ] = get_onoff ( obj . errmsg , args [ 0 ] ) except ValueError : pass return

set a Boolean - valued debugger setting . obj is a generally a subcommand that has name and debugger . settings attributes


def run_set_int(obj, arg, msg_on_error, min_value=None, max_value=None):
    """set an Integer-valued debugger setting. 'obj' is a generally a
    subcommand that has 'name' and 'debugger.settings' attributes"""
    if '' == arg.strip():
        obj.errmsg("You need to supply a number.")
        return
    obj.debugger.settings[obj.name] = \
        get_an_int(obj.errmsg, arg, msg_on_error, min_value, max_value)
    return obj.debugger.settings[obj.name]

def
run_set_int
(
obj
,
arg
,
msg_on_error
,
min_value
=
,
max_value
=
)
:
if
''
==
arg
.
strip
(
)
:
obj
.
errmsg
(
"You need to supply a number."
)
return
obj
.
debugger
.
settings
[
obj
.
name
]
=
get_an_int
(
obj
.
errmsg
,
arg
,
msg_on_error
,
min_value
,
max_value
)
return
obj
.
debugger
.
settings
[
obj
.
name
]

set an Integer - valued debugger setting . obj is a generally a subcommand that has name and debugger . settings attributes


def run_show_bool(obj, what=None):
    """Generic subcommand showing a boolean-valued debugger setting.
    'obj' is generally a subcommand that has 'name' and
    'debugger.setting' attributes."""
    val = show_onoff(obj.debugger.settings[obj.name])
    if not what: what = obj.name
    return obj.msg("%s is %s." % (what, val))

def
run_show_bool
(
obj
,
what
=
)
:
val
=
show_onoff
(
obj
.
debugger
.
settings
[
obj
.
name
]
)
if
not
what
:
what
=
obj
.
name
return
obj
.
msg
(
"%s is %s."
%
(
what
,
val
)
)

Generic subcommand showing a boolean - valued debugger setting . obj is generally a subcommand that has name and debugger . setting attributes .


def run_show_int(obj, what=None):
    """Generic subcommand integer value display"""
    val = obj.debugger.settings[obj.name]
    if not what: what = obj.name
    return obj.msg("%s is %d." % (what, val))

def
run_show_int
(
obj
,
what
=
)
:
val
=
obj
.
debugger
.
settings
[
obj
.
name
]
if
not
what
:
what
=
obj
.
name
return
obj
.
msg
(
"%s is %d."
%
(
what
,
val
)
)


def run_show_val(obj, name):
    """Generic subcommand value display"""
    val = obj.debugger.settings[obj.name]
    obj.msg("%s is %s." % (obj.name, obj.cmd.proc._saferepr(val),))
    return False

def
run_show_val
(
obj
,
name
)
:
val
=
obj
.
debugger
.
settings
[
obj
.
name
]
obj
.
msg
(
"%s is %s."
%
(
obj
.
name
,
obj
.
cmd
.
proc
.
_saferepr
(
val
)
,
)
)
return
False


def is_def_stmt(line, frame):
    """Return True if we are looking at a def statement"""
    # Should really also check that operand of 'LOAD_CONST' is a code object
    return (line and _re_def.match(line) and op_at_frame(frame)=='LOAD_CONST'
            and stmt_contains_opcode(frame.f_code, frame.f_lineno,
                                          'MAKE_FUNCTION'))


def is_class_def(line, frame):
    """Return True if we are looking at a class definition statement"""
    return (line and _re_class.match(line)
            and stmt_contains_opcode(frame.f_code, frame.f_lineno,
                                     'BUILD_CLASS'))

def is_class_def ( line , frame ) : return ( line and _re_class . match ( line ) and stmt_contains_opcode ( frame . f_code , frame . f_lineno , 'BUILD_CLASS' ) )


def threaded_quit(self, arg):
        """ quit command when several threads are involved. """
        threading_list = threading.enumerate()
        mythread =  threading.currentThread()
        for t in threading_list:
            if t != mythread:
                ctype_async_raise(t, Mexcept.DebuggerQuit)
                pass
            pass
        raise Mexcept.DebuggerQuit

def threaded_quit ( self , arg ) : threading_list = threading . enumerate ( ) mythread = threading . currentThread ( ) for t in threading_list : if t != mythread : ctype_async_raise ( t , Mexcept . DebuggerQuit ) pass pass raise Mexcept . DebuggerQuit

quit command when several threads are involved .


def set_default_bg():
    """Get bacground from
    default values based on the TERM environment variable
    """
    term = environ.get('TERM', None)
    if term:
        if (term.startswith('xterm',) or term.startswith('eterm')
            or term == 'dtterm'):
            return False
    return True

def set_default_bg ( ) : term = environ . get ( 'TERM' , None ) if term : if ( term . startswith ( 'xterm' , ) or term . startswith ( 'eterm' ) or term == 'dtterm' ) : return False return True


def is_dark_rgb(r, g, b):
    """Pass as parameters R G B values in hex
    On return, variable is_dark_bg is set
    """

    try:
        midpoint = int(environ.get('TERMINAL_COLOR_MIDPOINT', None))
    except:
        pass
    if not midpoint:
        term = environ.get('TERM', None)
        # 117963 = (* .6 (+ 65535 65535 65535))
        # 382.5 = (* .6 (+ 65535 65535 65535))
        print("midpoint", midpoint, 'vs',  (16*5 + 16*g + 16*b))
        midpoint = 383 if term and term == 'xterm-256color' else 117963

    if ( (16*5 + 16*g + 16*b) < midpoint ):
        return True
    else:
        return False

def
is_dark_rgb
(
r
,
g
,
b
)
:
try
:
midpoint
=
int
(
environ
.
get
(
'TERMINAL_COLOR_MIDPOINT'
,
)
)
except
:
pass
if
not
midpoint
:
term
=
environ
.
get
(
'TERM'
,
)
# 117963 = (* .6 (+ 65535 65535 65535))
# 382.5 = (* .6 (+ 65535 65535 65535))
print
(
"midpoint"
,
midpoint
,
'vs'
,
(
16
*
5
+
16
*
g
+
16
*
b
)
)
midpoint
=
383
if
term
and
term
==
'xterm-256color'
else
117963
if
(
(
16
*
5
+
16
*
g
+
16
*
b
)
<
midpoint
)
:
return
True
else
:
return
False

Pass as parameters R G B values in hex On return variable is_dark_bg is set


def signature(frame):
    '''return suitable frame signature to key display expressions off of.'''
    if not frame: return None
    code = frame.f_code
    return (code.co_name, code.co_filename, code.co_firstlineno)

def signature ( frame ) : if not frame : return None code = frame . f_code return ( code . co_name , code . co_filename , code . co_firstlineno )

return suitable frame signature to key display expressions off of .


def all(self):
        """List all display items; return 0 if none"""
        found = False
        s = []
        for display in self.list:
            if not found:
                s.append("""Auto-display expressions now in effect:
Num Enb Expression""")
                found = True
                pass
            s.append(display.format())
        return s

List all display items ; return 0 if none


def display(self, frame):
        '''display any items that are active'''
        if not frame: return
        s = []
        sig = signature(frame)
        for display in self.list:
            if display.signature == sig and display.enabled:
                s.append(display.to_s(frame))
                pass
            pass
        return s

def display ( self , frame ) : if not frame : return s = [ ] sig = signature ( frame ) for display in self . list : if display . signature == sig and display . enabled : s . append ( display . to_s ( frame ) ) pass pass return s


def format(self, show_enabled=True):
        '''format display item'''
        what = ''
        if show_enabled:
            if self.enabled:
                what += ' y '
            else:
                what += ' n '
                pass
            pass
        if self.fmt:
            what += self.fmt + ' '
            pass
        what += self.arg
        return '%3d: %s' % (self.number, what)

def
format
(
self
,
show_enabled
=
True
)
:
what
=
''
if
show_enabled
:
if
self
.
enabled
:
what
+=
' y '
else
:
what
+=
' n '
pass
pass
if
self
.
fmt
:
what
+=
self
.
fmt
+
' '
pass
what
+=
self
.
arg
return
'%3d: %s'
%
(
self
.
number
,
what
)


def read_msg(self):
        """Read one message unit. It's possible however that
        more than one message will be set in a receive, so we will
        have to buffer that for the next read.
        EOFError will be raised on EOF.
        """
        if self.state == 'connected':
            if 0 == len(self.buf):
                self.buf = self.inout.recv(Mtcpfns.TCP_MAX_PACKET)
                if 0 == (self.buf):
                    self.state = 'disconnected'
                    raise EOFError
                pass
            self.buf, data = Mtcpfns.unpack_msg(self.buf)
            return data.decode('utf-8')
        else:
            raise IOError("read_msg called in state: %s." % self.state)

def
read_msg
(
self
)
:
if
self
.
state
==
'connected'
:
if
0
==
len
(
self
.
buf
)
:
self
.
buf
=
self
.
inout
.
recv
(
Mtcpfns
.
TCP_MAX_PACKET
)
if
0
==
(
self
.
buf
)
:
self
.
state
=
'disconnected'
raise
EOFError
pass
self
.
buf
,
data
=
Mtcpfns
.
unpack_msg
(
self
.
buf
)
return
data
.
decode
(
'utf-8'
)
else
:
raise
IOError
(
"read_msg called in state: %s."
%
self
.
state
)

Read one message unit . It s possible however that more than one message will be set in a receive so we will have to buffer that for the next read . EOFError will be raised on EOF .


def debug(frame=None):
    """Set breakpoint at current location, or a specified frame"""
    # ???
    if frame is None:
        frame = _frame().f_back

    dbg = RemoteCeleryTrepan()
    dbg.say(BANNER.format(self=dbg))
    # dbg.say(SESSION_STARTED.format(self=dbg))
    trepan.api.debug(dbg_opts=dbg.dbg_opts)

Set breakpoint at current location or a specified frame


def undefined_subcmd(self, cmd, subcmd):
        """Error message when subcommand asked for but doesn't exist"""
        self.proc.intf[-1].errmsg(('Undefined "%s" subcommand: "%s". ' +
                                  'Try "help %s *".') % (cmd, subcmd, cmd))
        return

def
undefined_subcmd
(
self
,
cmd
,
subcmd
)
:
self
.
proc
.
intf
[
1
]
.
errmsg
(
(
'Undefined "%s" subcommand: "%s". '
+
'Try "help %s *".'
)
%
(
cmd
,
subcmd
,
cmd
)
)
return

Error message when subcommand asked for but doesn t exist


def run(self, args):
        '''Run a frame command. This routine is a little complex
        because we allow a number parameter variations.'''

        if len(args) == 1:
            # Form is: "frame" which means "frame 0"
            position_str = '0'
        elif len(args) == 2:
            # Form is: "frame {position | thread}
            name_or_id = args[1]
            frame, thread_id = self.get_from_thread_name_or_id(name_or_id,
                                                               False)
            if frame is None:
                # Form should be: frame position
                position_str = name_or_id
            else:
                # Form should be: "frame thread" which means
                # "frame thread 0"
                position_str = '0'
                self.find_and_set_debugged_frame(frame, thread_id)
                pass
        elif len(args) == 3:
            # Form is: frame <thread> <position>
            name_or_id = args[1]
            position_str = args[2]
            frame, thread_id = self.get_from_thread_name_or_id(name_or_id)
            if frame is None:
                # Error message was given in routine
                return
            self.find_and_set_debugged_frame(frame, thread_id)
            pass
        self.one_arg_run(position_str)
        return False

Run a frame command . This routine is a little complex because we allow a number parameter variations .


def pprint_simple_array(val, displaywidth, msg_nocr, msg, lineprefix=''):
    '''Try to pretty print a simple case where a list is not nested.
    Return True if we can do it and False if not. '''

    if type(val) != list:
        return False

    numeric = True
    for i in range(len(val)):
        if not (type(val[i]) in [bool, float, int]):
            numeric = False
            if not (type(val[i]) in [bool, float, int, bytes]):
                return False
            pass
        pass
    mess = columnize([repr(v) for v in val],
                     opts={"arrange_array": True,
                           "lineprefix": lineprefix,
                           "displaywidth": int(displaywidth)-3,
                           'ljust': not numeric})
    msg_nocr(mess)
    return True

def pprint_simple_array ( val , displaywidth , msg_nocr , msg , lineprefix = '' ) : if type ( val ) != list : return False numeric = True for i in range ( len ( val ) ) : if not ( type ( val [ i ] ) in [ bool , float , int ] ) : numeric = False if not ( type ( val [ i ] ) in [ bool , float , int , bytes ] ) : return False pass pass mess = columnize ( [ repr ( v ) for v in val ] , opts = { "arrange_array" : True , "lineprefix" : lineprefix , "displaywidth" : int ( displaywidth ) - 3 , 'ljust' : not numeric } ) msg_nocr ( mess ) return True

Try to pretty print a simple case where a list is not nested . Return True if we can do it and False if not .


def lookup_signame(num):
    """Find the corresponding signal name for 'num'. Return None
    if 'num' is invalid."""
    signames = signal.__dict__
    num = abs(num)
    for signame in list(signames.keys()):
        if signame.startswith('SIG') and signames[signame] == num:
            return signame
        pass
    # Something went wrong. Should have returned above
    return None

Find the corresponding signal name for num . Return None if num is invalid .


def lookup_signum(name):
    """Find the corresponding signal number for 'name'. Return None
    if 'name' is invalid."""
    uname = name.upper()
    if (uname.startswith('SIG') and hasattr(signal, uname)):
        return getattr(signal, uname)
    else:
        uname = "SIG"+uname
        if hasattr(signal, uname):
            return getattr(signal, uname)
        return None
    return

def lookup_signum ( name ) : uname = name . upper ( ) if ( uname . startswith ( 'SIG' ) and hasattr ( signal , uname ) ) : return getattr ( signal , uname ) else : uname = "SIG" + uname if hasattr ( signal , uname ) : return getattr ( signal , uname ) return None return

Find the corresponding signal number for name . Return None if name is invalid .


def canonic_signame(name_num):
    """Return a signal name for a signal name or signal
    number.  Return None is name_num is an int but not a valid signal
    number and False if name_num is a not number. If name_num is a
    signal name or signal number, the canonic if name is returned."""
    signum = lookup_signum(name_num)
    if signum is None:
        # Maybe signame is a number?
        try:
            num = int(name_num)
            signame = lookup_signame(num)
            if signame is None:
                return None
        except:
            return False
        return signame

    signame = name_num.upper()
    if not signame.startswith('SIG'): return 'SIG'+signame
    return signame

def
canonic_signame
(
name_num
)
:
signum
=
lookup_signum
(
name_num
)
if
signum
is
:
# Maybe signame is a number?
try
:
num
=
int
(
name_num
)
signame
=
lookup_signame
(
num
)
if
signame
is
:
return
except
:
return
False
return
signame
signame
=
name_num
.
upper
(
)
if
not
signame
.
startswith
(
'SIG'
)
:
return
'SIG'
+
signame
return
signame

Return a signal name for a signal name or signal number . Return None is name_num is an int but not a valid signal number and False if name_num is a not number . If name_num is a signal name or signal number the canonic if name is returned .


def set_signal_replacement(self, signum, handle):
        """A replacement for signal.signal which chains the signal behind
        the debugger's handler"""
        signame = lookup_signame(signum)
        if signame is None:
            self.dbgr.intf[-1].errmsg(("%s is not a signal number"
                                       " I know about.")  % signum)
            return False
        # Since the intent is to set a handler, we should pass this
        # signal on to the handler
        self.sigs[signame].pass_along = True
        if self.check_and_adjust_sighandler(signame, self.sigs):
            self.sigs[signame].old_handler = handle
            return True
        return False

A replacement for signal . signal which chains the signal behind the debugger s handler


def check_and_adjust_sighandlers(self):
        """Check to see if any of the signal handlers we are interested in have
        changed or is not initially set. Change any that are not right. """
        for signame in list(self.sigs.keys()):
            if not self.check_and_adjust_sighandler(signame, self.sigs):
                break
            pass
        return

def check_and_adjust_sighandlers ( self ) : for signame in list ( self . sigs . keys ( ) ) : if not self . check_and_adjust_sighandler ( signame , self . sigs ) : break pass return

Check to see if any of the signal handlers we are interested in have changed or is not initially set . Change any that are not right .


def info_signal(self, args):
        """Print information about a signal"""
        if len(args) == 0: return None
        signame = args[0]
        if signame in ['handle', 'signal']:
            # This has come from dbgr's info command
            if len(args) == 1:
                # Show all signal handlers
                self.dbgr.core.processor.section(self.header)
                for signame in self.siglist:
                    self.print_info_signal_entry(signame)
                return True
            else:
                signame = args[1]
                pass
            pass

        signame = self.is_name_or_number(signame)
        self.dbgr.core.processor.section(self.header)
        self.print_info_signal_entry(signame)
        return True


def action(self, arg):
        """Delegate the actions specified in 'arg' to another
        method.
        """
        if not arg:
            self.info_signal(['handle'])
            return True
        args = arg.split()
        signame = args[0]
        signame = self.is_name_or_number(args[0])
        if not signame: return

        if len(args) == 1:
            self.info_signal([signame])
            return True
        # We can display information about 'fatal' signals, but not
        # change their actions.
        if signame in fatal_signals:
            return None

        if signame not in list(self.sigs.keys()):
            if not self.initialize_handler(signame): return None
            pass

        # multiple commands might be specified, i.e. 'nopass nostop'
        for attr in args[1:]:
            if attr.startswith('no'):
                on = False
                attr = attr[2:]
            else:
                on = True
            if 'stop'.startswith(attr):
                self.handle_stop(signame, on)
            elif 'print'.startswith(attr) and len(attr) >= 2:
                self.handle_print(signame, on)
            elif 'pass'.startswith(attr):
                self.handle_pass(signame, on)
            elif 'ignore'.startswith(attr):
                self.handle_ignore(signame, on)
            elif 'stack'.startswith(attr):
                self.handle_print_stack(signame, on)
            else:
                self.dbgr.intf[-1].errmsg('Invalid arguments')
                pass
            pass
        return self.check_and_adjust_sighandler(signame, self.sigs)

Delegate the actions specified in arg to another method .


def handle_print(self, signame, set_print):
        """Set whether we print or not when this signal is caught."""
        if set_print:
            self.sigs[signame].print_method = self.dbgr.intf[-1].msg
        else:
            self.sigs[signame].print_method = None
            pass
        return set_print

def handle_print ( self , signame , set_print ) : if set_print : self . sigs [ signame ] . print_method = self . dbgr . intf [ - 1 ] . msg else : self . sigs [ signame ] . print_method = None pass return set_print

Set whether we print or not when this signal is caught .


def handle(self, signum, frame):
        """This method is called when a signal is received."""
        if self.print_method:
            self.print_method('\nProgram received signal %s.'
                              % self.signame)
        if self.print_stack:
            import traceback
            strings = traceback.format_stack(frame)
            for s in strings:
                if s[-1] == '\n': s = s[0:-1]
                self.print_method(s)
                pass
            pass
        if self.b_stop:
            core = self.dbgr.core
            old_trace_hook_suspend = core.trace_hook_suspend
            core.trace_hook_suspend = True
            core.stop_reason = ('intercepting signal %s (%d)' %
                                (self.signame, signum))
            core.processor.event_processor(frame, 'signal', signum)
            core.trace_hook_suspend = old_trace_hook_suspend
            pass
        if self.pass_along:
            # pass the signal to the program
            if self.old_handler:
                self.old_handler(signum, frame)
                pass
            pass
        return

def
handle
(
self
,
signum
,
frame
)
:
if
self
.
print_method
:
self
.
print_method
(
'\nProgram received signal %s.'
%
self
.
signame
)
if
self
.
print_stack
:
import
traceback
strings
=
traceback
.
format_stack
(
frame
)
for
s
in
strings
:
if
s
[
1
]
==
'\n'
:
s
=
s
[
0
:
1
]
self
.
print_method
(
s
)
pass
pass
if
self
.
b_stop
:
core
=
self
.
dbgr
.
core
old_trace_hook_suspend
=
core
.
trace_hook_suspend
core
.
trace_hook_suspend
=
True
core
.
stop_reason
=
(
'intercepting signal %s (%d)'
%
(
self
.
signame
,
signum
)
)
core
.
processor
.
event_processor
(
frame
,
'signal'
,
signum
)
core
.
trace_hook_suspend
=
old_trace_hook_suspend
pass
if
self
.
pass_along
:
# pass the signal to the program
if
self
.
old_handler
:
self
.
old_handler
(
signum
,
frame
)
pass
pass
return

This method is called when a signal is received .


def file2module(filename):
    """Given a file name, extract the most likely module name. """
    basename = osp.basename(filename)
    if '.' in basename:
        pos = basename.rfind('.')
        return basename[:pos]
    else:
        return basename
    return None

def file2module ( filename ) : basename = osp . basename ( filename ) if '.' in basename : pos = basename . rfind ( '.' ) return basename [ : pos ] else : return basename return None

Given a file name extract the most likely module name .


def search_file(filename, directories, cdir):
    """Return a full pathname for filename if we can find one. path
    is a list of directories to prepend to filename. If no file is
    found we'll return None"""

    for trydir in directories:

        # Handle $cwd and $cdir
        if trydir =='$cwd': trydir='.'
        elif trydir == '$cdir': trydir = cdir

        tryfile = osp.realpath(osp.join(trydir, filename))
        if osp.isfile(tryfile):
            return tryfile
    return None

def
search_file
(
filename
,
directories
,
cdir
)
:
for
trydir
in
directories
:
# Handle $cwd and $cdir
if
trydir
==
'$cwd'
:
trydir
=
'.'
elif
trydir
==
'$cdir'
:
trydir
=
cdir
tryfile
=
osp
.
realpath
(
osp
.
join
(
trydir
,
filename
)
)
if
osp
.
isfile
(
tryfile
)
:
return
tryfile
return

Return a full pathname for filename if we can find one . path is a list of directories to prepend to filename . If no file is found we ll return None


def whence_file(py_script, dirnames=None):
    """Do a shell-like path lookup for py_script and return the results.
    If we can't find anything return py_script"""
    if py_script.find(os.sep) != -1:
        # Don't search since this name has path separator components
        return py_script
    if dirnames is None:
        dirnames = os.environ['PATH'].split(os.pathsep)
    for dirname in dirnames:
        py_script_try = osp.join(dirname, py_script)
        if osp.exists(py_script_try):
            return py_script_try
    # Failure
    return py_script

Do a shell - like path lookup for py_script and return the results . If we can t find anything return py_script


def pyfiles(callername, level=2):
    "All python files caller's dir without the path and trailing .py"
    d = os.path.dirname(callername)
    # Get the name of our directory.
    # A glob pattern that will get all *.py files but not __init__.py
    glob(os.path.join(d, '[a-zA-Z]*.py'))
    py_files = glob(os.path.join(d, '[a-zA-Z]*.py'))
    return [ os.path.basename(filename[0:-3]) for filename in py_files ]

All python files caller s dir without the path and trailing . py


def msg(self, msg):
        """ used to write to a debugger that is connected to this
        server; `str' written will have a newline added to it
        """
        if hasattr(self.output, 'writeline'):
            self.output.writeline(msg)
        elif hasattr(self.output, 'writelines'):
            self.output.writelines(msg + "\n")
            pass
        return

def msg ( self , msg ) : if hasattr ( self . output , 'writeline' ) : self . output . writeline ( msg ) elif hasattr ( self . output , 'writelines' ) : self . output . writelines ( msg + "\n" ) pass return

used to write to a debugger that is connected to this server ; str written will have a newline added to it


def run(self, args):
        """Execution status of the program."""
        mainfile = self.core.filename(None)
        if self.core.is_running():
            if mainfile:
                part1 = "Python program '%s' is stopped" % mainfile
            else:
                part1 = 'Program is stopped'
                pass
            if self.proc.event:
                msg = 'via a %s event.' % self.proc.event
            else:
                msg = '.'
            self.msg(Mmisc.wrapped_lines(part1, msg,
                                         self.settings['width']))
            if self.proc.curframe:
                self.msg("PC offset is %d." % self.proc.curframe.f_lasti)

            if self.proc.event == 'return':
                val = self.proc.event_arg
                part1 = 'Return value is'
                self.msg(Mmisc.wrapped_lines(part1, self.proc._saferepr(val),
                                             self.settings['width']))
                pass
            elif self.proc.event == 'exception':
                exc_type, exc_value, exc_tb = self.proc.event_arg
                self.msg('Exception type: %s' %
                         self.proc._saferepr(exc_type))
                if exc_value:
                    self.msg('Exception value: %s' %
                             self.proc._saferepr(exc_value))
                    pass
                pass
            self.msg('It stopped %s.' % self.core.stop_reason)
            if self.proc.event in ['signal', 'exception', 'c_exception']:
                self.msg('Note: we are stopped *after* running the '
                         'line shown.')
                pass
        else:
            if mainfile:
                part1 = "Python program '%s'" % mainfile
                msg   = "is not currently running. "
                self.msg(Mmisc.wrapped_lines(part1, msg,
                                             self.settings['width']))
            else:
                self.msg('No Python program is currently running.')
                pass
            self.msg(self.core.execution_status)
            pass
        return False


def columnize_commands(self, commands):
        """List commands arranged in an aligned columns"""
        commands.sort()
        width = self.debugger.settings['width']
        return columnize.columnize(commands, displaywidth=width,
                                   lineprefix='    ')

def
columnize_commands
(
self
,
commands
)
:
commands
.
sort
(
)
width
=
self
.
debugger
.
settings
[
'width'
]
return
columnize
.
columnize
(
commands
,
displaywidth
=
width
,
lineprefix
=
'    '
)


def post_mortem(exc=None, frameno=1, dbg=None):
    """Enter debugger read loop after your program has crashed.

    exc is a triple like you get back from sys.exc_info.  If no exc
    parameter, is supplied, the values from sys.last_type,
    sys.last_value, sys.last_traceback are used. And if these don't
    exist either we'll assume that sys.exc_info() contains what we
    want and frameno is the index location of where we want to start.

    'frameno' specifies how many frames to ignore in the traceback.
    The default is 1, that is, we don't need to show the immediate
    call into post_mortem. If you have wrapper functions that call
    this one, you may want to increase frameno.
    """

    if dbg is None:
        # Check for a global debugger object
        if Mdebugger.debugger_obj is None:
            Mdebugger.debugger_obj = Mdebugger.Trepan()
            pass
        dbg = Mdebugger.debugger_obj
        pass
    re_bogus_file = re.compile("^<.+>$")

    if exc[0] is None:
        # frameno+1 because we are about to add one more level of call
        # in get_last_or_frame_exception
        exc = get_last_or_frame_exception()
        if exc[0] is None:
            print("Can't find traceback for post_mortem "
                  "in sys.last_traceback or sys.exec_info()")
            return
        pass
    exc_type, exc_value, exc_tb = exc
    dbg.core.execution_status = ('Terminated with unhandled exception %s'
                                 % exc_type)

    # tb has least-recent traceback entry first. We want the most-recent
    # entry. Also we'll pick out a mainpyfile name if it hasn't previously
    # been set.
    if exc_tb is not None:
        while exc_tb.tb_next is not None:
            filename = exc_tb.tb_frame.f_code.co_filename
            if (dbg.mainpyfile and 0 == len(dbg.mainpyfile)
                and not re_bogus_file.match(filename)):
                dbg.mainpyfile = filename
                pass
            exc_tb = exc_tb.tb_next
            pass
        dbg.core.processor.curframe = exc_tb.tb_frame
        pass

    if 0 == len(dbg.program_sys_argv):
        # Fake program (run command) args since we weren't called with any
        dbg.program_sys_argv = list(sys.argv[1:])
        dbg.program_sys_argv[:0] = [dbg.mainpyfile]

    # if 0 == len(dbg._sys_argv):
    #     # Fake script invocation (restart) args since we don't have any
    #     dbg._sys_argv = list(dbg.program_sys_argv)
    #     dbg._sys_argv[:0] = [__title__]

    try:
        # # FIXME: This can be called from except hook in which case we
        # # need this. Dunno why though.
        # try:
        #     _pydb_trace.set_trace(t.tb_frame)
        # except:
        #     pass

        # Possibly a bug in Python 2.5. Why f.f_lineno is
        # not always equal to t.tb_lineno, I don't know.
        f = exc_tb.tb_frame
        if f and f.f_lineno != exc_tb.tb_lineno : f = f.f_back
        dbg.core.processor.event_processor(f, 'exception', exc, 'Trepan3k:pm')
    except DebuggerRestart:
        while True:
            sys.argv = list(dbg._program_sys_argv)
            dbg.msg("Restarting %s with arguments:\n\t%s"
                  % (dbg.filename(dbg.mainpyfile),
                     " ".join(dbg._program_sys_argv[1:])))
            try:
                dbg.run_script(dbg.mainpyfile)
            except DebuggerRestart:
                pass
            pass
    except DebuggerQuit:
        pass
    return

Enter debugger read loop after your program has crashed .


def close(self):
        """ Closes both socket and server connection. """
        self.state = 'closing'
        if self.inout:
            self.inout.close()
            pass
        self.state = 'closing connection'
        if self.conn:
            self.conn.close()
        self.state = 'disconnected'
        return

def
close
(
self
)
:
self
.
state
=
'closing'
if
self
.
inout
:
self
.
inout
.
close
(
)
pass
self
.
state
=
'closing connection'
if
self
.
conn
:
self
.
conn
.
close
(
)
self
.
state
=
'disconnected'
return

Closes both socket and server connection .


def write(self, msg):
        """ This method the debugger uses to write. In contrast to
        writeline, no newline is added to the end to `str'. Also
        msg doesn't have to be a string.
        """
        if self.state != 'connected':
            self.wait_for_connect()
            pass
        buffer = Mtcpfns.pack_msg(msg)
        while len(buffer) > Mtcpfns.TCP_MAX_PACKET:
            self.conn.send(buffer[:Mtcpfns.TCP_MAX_PACKET])
            buffer = buffer[Mtcpfns.TCP_MAX_PACKET:]
        return self.conn.send(buffer)

def write ( self , msg ) : if self . state != 'connected' : self . wait_for_connect ( ) pass buffer = Mtcpfns . pack_msg ( msg ) while len ( buffer ) > Mtcpfns . TCP_MAX_PACKET : self . conn . send ( buffer [ : Mtcpfns . TCP_MAX_PACKET ] ) buffer = buffer [ Mtcpfns . TCP_MAX_PACKET : ] return self . conn . send ( buffer )

This method the debugger uses to write . In contrast to writeline no newline is added to the end to str . Also msg doesn t have to be a string .


def complete_identifier(cmd, prefix):
    '''Complete an arbitrary expression.'''
    if not cmd.proc.curframe: return [None]
    # Collect globals and locals.  It is usually not really sensible to also
    # complete builtins, and they clutter the namespace quite heavily, so we
    # leave them out.
    ns = cmd.proc.curframe.f_globals.copy()
    ns.update(cmd.proc.curframe.f_locals)
    if '.' in prefix:
        # Walk an attribute chain up to the last part, similar to what
        # rlcompleter does.  This will bail if any of the parts are not
        # simple attribute access, which is what we want.
        dotted = prefix.split('.')
        try:
            obj = ns[dotted[0]]
            for part in dotted[1:-1]:
                obj = getattr(obj, part)
        except (KeyError, AttributeError):
            return []
        pre_prefix = '.'.join(dotted[:-1]) + '.'
        return [pre_prefix + n for n in dir(obj) if
                n.startswith(dotted[-1])]
    else:
        # Complete a simple name.
        return Mcomplete.complete_token(ns.keys(), prefix)


def dbgr(self, string):
        '''Invoke a debugger command from inside a python shell called inside
        the debugger.
        '''
        print('')
        self.proc.cmd_queue.append(string)
        self.proc.process_command()
        return

def dbgr ( self , string ) : print ( '' ) self . proc . cmd_queue . append ( string ) self . proc . process_command ( ) return

Invoke a debugger command from inside a python shell called inside the debugger .


def add_ignore(self, *frames_or_fns):
        """Add `frame_or_fn' to the list of functions that are not to
        be debugged"""
        for frame_or_fn in frames_or_fns:
            rc = self.ignore_filter.add_include(frame_or_fn)
            pass
        return rc

def add_ignore ( self , * frames_or_fns ) : for frame_or_fn in frames_or_fns : rc = self . ignore_filter . add_include ( frame_or_fn ) pass return rc

Add frame_or_fn to the list of functions that are not to be debugged


def canonic(self, filename):
        """ Turns `filename' into its canonic representation and returns this
        string. This allows a user to refer to a given file in one of several
        equivalent ways.

        Relative filenames need to be fully resolved, since the current working
        directory might change over the course of execution.

        If filename is enclosed in < ... >, then we assume it is
        one of the bogus internal Python names like <string> which is seen
        for example when executing "exec cmd".
        """

        if filename == "<" + filename[1:-1] + ">":
            return filename
        canonic = self.filename_cache.get(filename)
        if not canonic:
            lead_dir = filename.split(os.sep)[0]
            if lead_dir == os.curdir or lead_dir == os.pardir:
                # We may have invoked the program from a directory
                # other than where the program resides. filename is
                # relative to where the program resides. So make sure
                # to use that.
                canonic = os.path.abspath(os.path.join(self.main_dirname,
                                                       filename))
            else:
                canonic = os.path.abspath(filename)
                pass
            if not os.path.isfile(canonic):
                canonic = Mclifns.search_file(filename, self.search_path,
                                              self.main_dirname)
                # FIXME: is this is right for utter failure?
                if not canonic: canonic = filename
                pass
            canonic = os.path.realpath(os.path.normcase(canonic))
            self.filename_cache[filename] = canonic
        return canonic

Turns filename into its canonic representation and returns this string . This allows a user to refer to a given file in one of several equivalent ways .


def filename(self, filename=None):
        """Return filename or the basename of that depending on the
        basename setting"""
        if filename is None:
            if self.debugger.mainpyfile:
                filename = self.debugger.mainpyfile
            else:
                return None
        if self.debugger.settings['basename']:
            return(os.path.basename(filename))
        return filename

def filename ( self , filename = None ) : if filename is None : if self . debugger . mainpyfile : filename = self . debugger . mainpyfile else : return None if self . debugger . settings [ 'basename' ] : return ( os . path . basename ( filename ) ) return filename


def is_started(self):
        '''Return True if debugging is in progress.'''
        return (tracer.is_started() and
                not self.trace_hook_suspend
                and tracer.find_hook(self.trace_dispatch))

def is_started ( self ) : return ( tracer . is_started ( ) and not self . trace_hook_suspend and tracer . find_hook ( self . trace_dispatch ) )

Return True if debugging is in progress .


def is_stop_here(self, frame, event, arg):
        """ Does the magic to determine if we stop here and run a
        command processor or not. If so, return True and set
        self.stop_reason; if not, return False.

        Determining factors can be whether a breakpoint was
        encountered, whether we are stepping, next'ing, finish'ing,
        and, if so, whether there is an ignore counter.
        """

        # Add an generic event filter here?
        # FIXME TODO: Check for
        #  - thread switching (under set option)

        # Check for "next" and "finish" stopping via stop_level

        # Do we want a different line and if so,
        # do we have one?
        lineno = frame.f_lineno
        filename = frame.f_code.co_filename
        if self.different_line and event == 'line':
            if self.last_lineno == lineno and self.last_filename == filename:
                return False
            pass
        self.last_lineno   = lineno
        self.last_filename = filename

        if self.stop_level is not None:
            if frame != self.last_frame:
                # Recompute stack_depth
                self.last_level = Mstack.count_frames(frame)
                self.last_frame = frame
                pass
            if self.last_level > self.stop_level:
                return False
            elif self.last_level == self.stop_level and \
                    self.stop_on_finish and event in ['return', 'c_return']:
                self.stop_level = None
                self.stop_reason = "in return for 'finish' command"
                return True
            pass

        # Check for stepping
        if self._is_step_next_stop(event):
            self.stop_reason = 'at a stepping statement'
            return True

        return False

#  - thread switching (under set option)

Does the magic to determine if we stop here and run a command processor or not . If so return True and set self . stop_reason ; if not return False .


def set_next(self, frame, step_ignore=0, step_events=None):
        "Sets to stop on the next event that happens in frame 'frame'."
        self.step_events      = None  # Consider all events
        self.stop_level       = Mstack.count_frames(frame)
        self.last_level       = self.stop_level
        self.last_frame       = frame
        self.stop_on_finish   = False
        self.step_ignore      = step_ignore
        return

def
set_next
(
self
,
frame
,
step_ignore
=
0
,
step_events
=
)
:
self
.
step_events
=
# Consider all events
self
.
stop_level
=
Mstack
.
count_frames
(
frame
)
self
.
last_level
=
self
.
stop_level
self
.
last_frame
=
frame
self
.
stop_on_finish
=
False
self
.
step_ignore
=
step_ignore
return

Sets to stop on the next event that happens in frame frame .


def stack_trace(self, f):
        """A mini stack trace routine for threads."""
        while f:
            if (not self.core.ignore_filter.is_included(f)
                or self.settings['dbg_trepan']):
                s = Mstack.format_stack_entry(self, (f, f.f_lineno))
                self.msg(" "*4 + s)
                pass
            f = f.f_back
            pass
        return

def
stack_trace
(
self
,
f
)
:
while
f
:
if
(
not
self
.
core
.
ignore_filter
.
is_included
(
f
)
or
self
.
settings
[
'dbg_trepan'
]
)
:
s
=
Mstack
.
format_stack_entry
(
self
,
(
f
,
f
.
f_lineno
)
)
self
.
msg
(
" "
*
4
+
s
)
pass
f
=
f
.
f_back
pass
return

A mini stack trace routine for threads .


def run(self, args):
        """Get file information"""
        if len(args) == 0:
            if not self.proc.curframe:
                self.errmsg("No frame - no default file.")
                return False
            filename = self.proc.curframe.f_code.co_filename
        else:
            filename = args[0]
            pass

        m = filename + ' is'
        filename_cache = self.core.filename_cache
        if filename in filename_cache:
            m += " cached in debugger"
            if filename_cache[filename] != filename:
                m += ' as:'
                m = Mmisc.wrapped_lines(m, filename_cache[filename] + '.',
                                        self.settings['width'])
            else:
                m += '.'
                pass
            self.msg(m)
        else:
            matches = [file for file in file_list() if
                       file.endswith(filename)]
            if (len(matches) > 1):
                self.msg("Multiple files found ending filename string:")
                for match_file in matches:
                    self.msg("\t%s" % match_file)
                    pass
            elif len(matches) == 1:
                canonic_name = pyficache.unmap_file(matches[0])
                m += " matched debugger cache file:\n  "  + canonic_name
                self.msg(m)
            else:
                self.msg(m + ' not cached in debugger.')
            pass
        canonic_name = self.core.canonic(filename)
        self.msg(Mmisc.wrapped_lines('Canonic name:', canonic_name,
                                     self.settings['width']))
        for name in (canonic_name, filename):
            if name in sys.modules:
                for key in [k for k, v in list(sys.modules.items())
                            if name == v]:
                    self.msg("module: %s", key)
                    pass
                pass
            pass
        for arg in args[1:]:
            processed_arg = False
            if arg in ['all', 'size']:
                if pyficache.size(canonic_name):
                    self.msg("File has %d lines." %
                             pyficache.size(canonic_name))
                    pass
                processed_arg = True
                pass
            if arg in ['all', 'sha1']:
                self.msg("SHA1 is %s." % pyficache.sha1(canonic_name))
                processed_arg = True
                pass
            if arg in ['all', 'brkpts']:
                lines = pyficache.trace_line_numbers(canonic_name)
                if lines:
                    self.section("Possible breakpoint line numbers:")
                    fmt_lines = columnize.columnize(lines, ljust = False,
                                                    arrange_vertical = False,
                                                    lineprefix='  ')
                    self.msg(fmt_lines)
                    pass
                processed_arg = True
                pass
            if not processed_arg:
                self.errmsg("Don't understand sub-option %s." % arg)
                pass
            pass
        return


def checkfuncname(b, frame):
    """Check whether we should break here because of `b.funcname`."""
    if not b.funcname:
        # Breakpoint was set via line number.
        if b.line != frame.f_lineno:
            # Breakpoint was set at a line with a def statement and the function
            # defined is called: don't break.
            return False
        return True

    # Breakpoint set via function name.

    if frame.f_code.co_name != b.funcname:
        # It's not a function call, but rather execution of def statement.
        return False

    # We are in the right frame.
    if not b.func_first_executable_line:
        # The function is entered for the 1st time.
        b.func_first_executable_line = frame.f_lineno

    if b.func_first_executable_line != frame.f_lineno:
        # But we are not at the first line number: don't break.
        return False
    return True

Check whether we should break here because of b . funcname .


def delete_breakpoint(self, bp):
        " remove breakpoint `bp'"
        bpnum = bp.number
        self.bpbynumber[bpnum] = None   # No longer in list
        index = (bp.filename, bp.line)
        if index not in self.bplist: return False
        self.bplist[index].remove(bp)
        if not self.bplist[index]:
            # No more breakpoints for this file:line combo
            del self.bplist[index]
        return True


def delete_breakpoint_by_number(self, bpnum):
        "Remove a breakpoint given its breakpoint number."
        success, msg, bp = self.get_breakpoint(bpnum)
        if not success:
            return False, msg
        self.delete_breakpoint(bp)
        return (True, '')

def delete_breakpoint_by_number ( self , bpnum ) : success , msg , bp = self . get_breakpoint ( bpnum ) if not success : return False , msg self . delete_breakpoint ( bp ) return ( True , '' )

Remove a breakpoint given its breakpoint number .


def en_disable_all_breakpoints(self,  do_enable=True):
        "Enable or disable all breakpoints."
        bp_list = [bp for bp in self.bpbynumber if bp]
        bp_nums = []
        if do_enable:
            endis = 'en'
        else:
            endis = 'dis'
            pass
        if not bp_list:
            return "No breakpoints to %sable" % endis
        for bp in bp_list:
            bp.enabled = do_enable
            bp_nums.append(str(bp.number))
            pass
        return ("Breakpoints %sabled: %s" % (endis, ", ".join(bp_nums)))

def
en_disable_all_breakpoints
(
self
,
do_enable
=
True
)
:
bp_list
=
[
bp
for
bp
in
self
.
bpbynumber
if
bp
]
bp_nums
=
[
]
if
do_enable
:
endis
=
'en'
else
:
endis
=
'dis'
pass
if
not
bp_list
:
return
"No breakpoints to %sable"
%
endis
for
bp
in
bp_list
:
bp
.
enabled
=
do_enable
bp_nums
.
append
(
str
(
bp
.
number
)
)
pass
return
(
"Breakpoints %sabled: %s"
%
(
endis
,
", "
.
join
(
bp_nums
)
)
)


def en_disable_breakpoint_by_number(self, bpnum, do_enable=True):
        "Enable or disable a breakpoint given its breakpoint number."
        success, msg, bp = self.get_breakpoint(bpnum)
        if not success:
            return success, msg
        if do_enable:
            endis = 'en'
        else:
            endis = 'dis'
            pass
        if bp.enabled == do_enable:
            return (False, ('Breakpoint (%r) previously %sabled' %
                            (str(bpnum), endis,)))
        bp.enabled = do_enable
        return (True, '')

def
en_disable_breakpoint_by_number
(
self
,
bpnum
,
do_enable
=
True
)
:
success
,
msg
,
bp
=
self
.
get_breakpoint
(
bpnum
)
if
not
success
:
return
success
,
msg
if
do_enable
:
endis
=
'en'
else
:
endis
=
'dis'
pass
if
bp
.
enabled
==
do_enable
:
return
(
False
,
(
'Breakpoint (%r) previously %sabled'
%
(
str
(
bpnum
)
,
endis
,
)
)
)
bp
.
enabled
=
do_enable
return
(
True
,
''
)

Enable or disable a breakpoint given its breakpoint number .


def delete_breakpoints_by_lineno(self, filename, lineno):
        """Removes all breakpoints at a give filename and line number.
        Returns a list of breakpoints numbers deleted.
        """
        if (filename, lineno) not in self.bplist:
            return []
        breakpoints = self.bplist[(filename, lineno)]
        bpnums = [bp.number for bp in breakpoints]
        for bp in list(breakpoints):
            self.delete_breakpoint(bp)
        return bpnums

def delete_breakpoints_by_lineno ( self , filename , lineno ) : if ( filename , lineno ) not in self . bplist : return [ ] breakpoints = self . bplist [ ( filename , lineno ) ] bpnums = [ bp . number for bp in breakpoints ] for bp in list ( breakpoints ) : self . delete_breakpoint ( bp ) return bpnums

Removes all breakpoints at a give filename and line number . Returns a list of breakpoints numbers deleted .


def open(self, inp, opts=None):
        """Use this to set what file to read from. """
        if isinstance(inp, io.TextIOWrapper):
            self.input = inp
        elif isinstance(inp, 'string'.__class__):  # FIXME
            self.name  = inp
            self.input = open(inp, 'r')
        else:
            raise IOError("Invalid input type (%s) for %s" %
                          (inp.__class__.__name__, inp))
        return

def
open
(
self
,
inp
,
opts
=
)
:
if
isinstance
(
inp
,
io
.
TextIOWrapper
)
:
self
.
input
=
inp
elif
isinstance
(
inp
,
'string'
.
__class__
)
:
# FIXME
self
.
name
=
inp
self
.
input
=
open
(
inp
,
'r'
)
else
:
raise
IOError
(
"Invalid input type (%s) for %s"
%
(
inp
.
__class__
.
__name__
,
inp
)
)
return

Use this to set what file to read from .


def readline(self, prompt='', use_raw=None):
        """Read a line of input. Prompt and use_raw exist to be
        compatible with other input routines and are ignored.
        EOFError will be raised on EOF.
        """
        line = self.input.readline()
        if not line: raise EOFError
        return line.rstrip("\n")

def readline ( self , prompt = '' , use_raw = None ) : line = self . input . readline ( ) if not line : raise EOFError return line . rstrip ( "\n" )

Read a line of input . Prompt and use_raw exist to be compatible with other input routines and are ignored . EOFError will be raised on EOF .


def confirm(self, prompt, default):
        """ Called when a dangerous action is about to be done to make sure
        it's okay. `prompt' is printed; user response is returned."""
        while True:
            try:
                self.write_confirm(prompt, default)
                reply = self.readline('').strip().lower()
            except EOFError:
                return default
            if reply in ('y', 'yes'):
                return True
            elif reply in ('n', 'no'):
                return False
            else:
                self.msg("Please answer y or n.")
                pass
            pass
        return default

def
confirm
(
self
,
prompt
,
default
)
:
while
True
:
try
:
self
.
write_confirm
(
prompt
,
default
)
reply
=
self
.
readline
(
''
)
.
strip
(
)
.
lower
(
)
except
EOFError
:
return
default
if
reply
in
(
'y'
,
'yes'
)
:
return
True
elif
reply
in
(
'n'
,
'no'
)
:
return
False
else
:
self
.
msg
(
"Please answer y or n."
)
pass
pass
return
default

Called when a dangerous action is about to be done to make sure it s okay . prompt is printed ; user response is returned .


def t_whitespace(self, s):
        r'\s+'
        self.add_token('SPACE', s)
        self.pos += len(s)
        pass

def t_whitespace ( self , s ) : self . add_token ( 'SPACE' , s ) self . pos += len ( s ) pass


def t_number(self, s):
        r'\d+'
        pos = self.pos
        self.add_token('NUMBER', int(s))
        self.pos = pos + len(s)

def t_number ( self , s ) : pos = self . pos self . add_token ( 'NUMBER' , int ( s ) ) self . pos = pos + len ( s )


def as_future(self, query):
        """Wrap a `sqlalchemy.orm.query.Query` object into a
        `concurrent.futures.Future` so that it can be yielded.

        Parameters
        ----------
        query : sqlalchemy.orm.query.Query
            SQLAlchemy query object to execute

        Returns
        -------
            tornado.concurrent.Future
                A `Future` object wrapping the given query so that tornado can
                await/yield on it
        """
        # concurrent.futures.Future is not compatible with the "new style"
        # asyncio Future, and awaiting on such "old-style" futures does not
        # work.
        #
        # tornado includes a `run_in_executor` function to help with this
        # problem, but it's only included in version 5+. Hence, we copy a
        # little bit of code here to handle this incompatibility.

        if not self._pool:
            self._pool = ThreadPoolExecutor(max_workers=self._max_workers)

        old_future = self._pool.submit(query)
        new_future = Future()

        IOLoop.current().add_future(
            old_future, lambda f: chain_future(f, new_future)
        )

        return new_future

Wrap a sqlalchemy . orm . query . Query object into a concurrent . futures . Future so that it can be yielded .


def restore_original_login(request):
    """
    Restore an original login session, checking the signed session
    """
    original_session = request.session.get(la_settings.USER_SESSION_FLAG)
    logout(request)

    if not original_session:
        return

    try:
        original_user_pk = signer.unsign(
            original_session, max_age=timedelta(days=la_settings.USER_SESSION_DAYS_TIMESTAMP).total_seconds()
        )
        user = get_user_model().objects.get(pk=original_user_pk)
        messages.info(
            request,
            la_settings.MESSAGE_LOGIN_REVERT.format(username=user.__dict__[username_field]),
            extra_tags=la_settings.MESSAGE_EXTRA_TAGS,
        )
        login_as(user, request, store_original_user=False)
        if la_settings.USER_SESSION_FLAG in request.session:
            del request.session[la_settings.USER_SESSION_FLAG]
    except SignatureExpired:
        pass

def restore_original_login ( request ) : original_session = request . session . get ( la_settings . USER_SESSION_FLAG ) logout ( request ) if not original_session : return try : original_user_pk = signer . unsign ( original_session , max_age = timedelta ( days = la_settings . USER_SESSION_DAYS_TIMESTAMP ) . total_seconds ( ) ) user = get_user_model ( ) . objects . get ( pk = original_user_pk ) messages . info ( request , la_settings . MESSAGE_LOGIN_REVERT . format ( username = user . __dict__ [ username_field ] ) , extra_tags = la_settings . MESSAGE_EXTRA_TAGS , ) login_as ( user , request , store_original_user = False ) if la_settings . USER_SESSION_FLAG in request . session : del request . session [ la_settings . USER_SESSION_FLAG ] except SignatureExpired : pass

Restore an original login session checking the signed session


def _load_module(path):
    """Code to load create user module. Copied off django-browserid."""

    i = path.rfind(".")
    module, attr = path[:i], path[i + 1 :]

    try:
        mod = import_module(module)
    except ImportError:
        raise ImproperlyConfigured("Error importing CAN_LOGIN_AS function: {}".format(module))
    except ValueError:
        raise ImproperlyConfigured("Error importing CAN_LOGIN_AS" " function. Is CAN_LOGIN_AS a" " string?")

    try:
        can_login_as = getattr(mod, attr)
    except AttributeError:
        raise ImproperlyConfigured("Module {0} does not define a {1} " "function.".format(module, attr))
    return can_login_as

Code to load create user module . Copied off django - browserid .


def iterate_docs(client, expanded=False, progress=False):
    """
    Yield each document in a Luminoso project in turn. Requires a client whose
    URL points to a project.

    If expanded=True, it will include additional fields that Luminoso added in
    its analysis, such as 'terms' and 'vector'.

    Otherwise, it will contain only the fields necessary to reconstruct the
    document: 'title', 'text', and 'metadata'.

    Shows a progress bar if progress=True.
    """
    # Get total number of docs from the project record
    num_docs = client.get()['document_count']
    progress_bar = None
    try:
        if progress:
            progress_bar = tqdm(desc='Downloading documents', total=num_docs)

        for offset in range(0, num_docs, DOCS_PER_BATCH):
            response = client.get('docs', offset=offset, limit=DOCS_PER_BATCH)
            docs = response['result']
            for doc in docs:
                # Get the appropriate set of fields for each document
                if expanded:
                    for field in UNNECESSARY_FIELDS:
                        doc.pop(field, None)
                else:
                    doc = {field: doc[field] for field in CONCISE_FIELDS}

                if progress:
                    progress_bar.update()
                yield doc

    finally:
        if progress:
            progress_bar.close()

Yield each document in a Luminoso project in turn . Requires a client whose URL points to a project .


def _main(argv):
    """
    Handle arguments for the 'lumi-download' command.
    """
    parser = argparse.ArgumentParser(
        description=DESCRIPTION,
        formatter_class=argparse.RawDescriptionHelpFormatter
    )
    parser.add_argument(
        '-b',
        '--base-url',
        default=URL_BASE,
        help='API root url, default: %s' % URL_BASE,
    )
    parser.add_argument(
        '-e', '--expanded',
        help="Include Luminoso's analysis of each document, such as terms and"
             ' document vectors',
        action='store_true',
    )
    parser.add_argument('-t', '--token', help='API authentication token')
    parser.add_argument(
        '-s',
        '--save-token',
        action='store_true',
        help='save --token for --base-url to ~/.luminoso/tokens.json',
    )
    parser.add_argument(
        'project_id', help='The ID of the project in the Daylight API'
    )
    parser.add_argument(
        'output_file', nargs='?', default=None,
        help='The JSON lines (.jsons) file to write to'
    )
    args = parser.parse_args(argv)
    if args.save_token:
        if not args.token:
            raise ValueError("error: no token provided")
        LuminosoClient.save_token(args.token,
                                  domain=urlparse(args.base_url).netloc)

    client = LuminosoClient.connect(url=args.base_url, token=args.token)
    proj_client = client.client_for_path('projects/{}'.format(args.project_id))
    download_docs(proj_client, args.output_file, args.expanded)

Handle arguments for the lumi - download command .


def transcode_to_stream(input_filename, date_format=None):
    """
    Read a JSON or CSV file and convert it into a JSON stream, which will
    be saved in an anonymous temp file.
    """
    tmp = tempfile.TemporaryFile()
    for entry in open_json_or_csv_somehow(input_filename,
                                          date_format=date_format):
        tmp.write(json.dumps(entry, ensure_ascii=False).encode('utf-8'))
        tmp.write(b'\n')
    tmp.seek(0)
    return tmp

def transcode_to_stream ( input_filename , date_format = None ) : tmp = tempfile . TemporaryFile ( ) for entry in open_json_or_csv_somehow ( input_filename , date_format = date_format ) : tmp . write ( json . dumps ( entry , ensure_ascii = False ) . encode ( 'utf-8' ) ) tmp . write ( b'\n' ) tmp . seek ( 0 ) return tmp

Read a JSON or CSV file and convert it into a JSON stream which will be saved in an anonymous temp file .


def open_json_or_csv_somehow(filename, date_format=None):
    """
    Deduce the format of a file, within reason.

    - If the filename ends with .csv or .txt, it's csv.
    - If the filename ends with .jsons, it's a JSON stream (conveniently the
      format we want to output).
    - If the filename ends with .json, it could be a legitimate JSON file, or
      it could be a JSON stream, following a nonstandard convention that many
      people including us are guilty of. In that case:
      - If the first line is a complete JSON document, and there is more in the
        file besides the first line, then it is a JSON stream.
      - Otherwise, it is probably really JSON.
    - If the filename does not end with .json, .jsons, or .csv, we have to guess
      whether it's still CSV or tab-separated values or something like that.
      If it's JSON, the first character would almost certainly have to be a
      bracket or a brace. If it isn't, assume it's CSV or similar.
    """
    fileformat = None
    if filename.endswith('.csv'):
        fileformat = 'csv'
    elif filename.endswith('.jsons'):
        fileformat = 'jsons'
    else:
        with open(filename) as opened:
            line = opened.readline()
            if line[0] not in '{[' and not filename.endswith('.json'):
                fileformat = 'csv'
            else:
                if (line.count('{') == line.count('}') and
                    line.count('[') == line.count(']')):
                    # This line contains a complete JSON document. This probably
                    # means it's in linewise JSON ('.jsons') format, unless the
                    # whole file is on one line.
                    char = ' '
                    while char.isspace():
                        char = opened.read()
                        if char == '':
                            fileformat = 'json'
                            break
                    if fileformat is None:
                        fileformat = 'jsons'
                else:
                    fileformat = 'json'

    if fileformat == 'json':
        stream = json.load(open(filename), encoding='utf-8')
    elif fileformat == 'csv':
        stream = open_csv_somehow(filename)
    else:
        stream = stream_json_lines(filename)

    return _normalize_data(stream, date_format=date_format)

Deduce the format of a file within reason .


def _normalize_data(stream, date_format=None):
    """
    This function is meant to normalize data for upload to the Luminoso
    Analytics system. Currently it only normalizes dates.

    If date_format is not specified, or if there's no date in a particular doc,
    the the doc is yielded unchanged.
    """
    for doc in stream:
        if 'date' in doc and date_format is not None:
            try:
                doc['date'] = _convert_date(doc['date'], date_format)
            except ValueError:
                # ValueErrors cover the cases when date_format does not match
                # the actual format of the date, both for epoch and non-epoch
                # times.
                logger.exception('%s does not match the date format %s;'
                                 % (doc['date'], date_format))
        yield doc

This function is meant to normalize data for upload to the Luminoso Analytics system . Currently it only normalizes dates .


def _convert_date(date_string, date_format):
    """
    Convert a date in a given format to epoch time. Mostly a wrapper for
    datetime's strptime.
    """
    if date_format != 'epoch':
        return datetime.strptime(date_string, date_format).timestamp()
    else:
        return float(date_string)

def _convert_date ( date_string , date_format ) : if date_format != 'epoch' : return datetime . strptime ( date_string , date_format ) . timestamp ( ) else : return float ( date_string )

Convert a date in a given format to epoch time . Mostly a wrapper for datetime s strptime .


def detect_file_encoding(filename):
    """
    Use ftfy to detect the encoding of a file, based on a sample of its
    first megabyte.

    ftfy's encoding detector is limited. The only encodings it can detect are
    UTF-8, CESU-8, UTF-16, Windows-1252, and occasionally MacRoman. But it
    does much better than chardet.
    """
    with open(filename, 'rb') as opened:
        sample = opened.read(2 ** 20)
        _, encoding = ftfy.guess_bytes(sample)
        return encoding

def detect_file_encoding ( filename ) : with open ( filename , 'rb' ) as opened : sample = opened . read ( 2 ** 20 ) _ , encoding = ftfy . guess_bytes ( sample ) return encoding

Use ftfy to detect the encoding of a file based on a sample of its first megabyte .


def stream_json_lines(file):
    """
    Load a JSON stream and return a generator, yielding one object at a time.
    """
    if isinstance(file, string_type):
        file = open(file, 'rb')
    for line in file:
        line = line.strip()
        if line:
            if isinstance(line, bytes):
                line = line.decode('utf-8')
            yield json.loads(line)

def stream_json_lines ( file ) : if isinstance ( file , string_type ) : file = open ( file , 'rb' ) for line in file : line = line . strip ( ) if line : if isinstance ( line , bytes ) : line = line . decode ( 'utf-8' ) yield json . loads ( line )

Load a JSON stream and return a generator yielding one object at a time .


def transcode_to_utf8(filename, encoding):
    """
    Convert a file in some other encoding into a temporary file that's in
    UTF-8.
    """
    tmp = tempfile.TemporaryFile()
    for line in io.open(filename, encoding=encoding):
        tmp.write(line.strip('\uFEFF').encode('utf-8'))

    tmp.seek(0)
    return tmp

def transcode_to_utf8 ( filename , encoding ) : tmp = tempfile . TemporaryFile ( ) for line in io . open ( filename , encoding = encoding ) : tmp . write ( line . strip ( '\uFEFF' ) . encode ( 'utf-8' ) ) tmp . seek ( 0 ) return tmp

Convert a file in some other encoding into a temporary file that s in UTF - 8 .


def open_csv_somehow_py2(filename):
    """
    Open a CSV file using Python 2's CSV module, working around the deficiency
    where it can't handle the null bytes of UTF-16.
    """
    encoding = detect_file_encoding(filename)
    if encoding.startswith('UTF-16'):
        csvfile = transcode_to_utf8(filename, encoding)
        encoding = 'UTF-8'
    else:
        csvfile = open(filename, 'rU')
    line = csvfile.readline()
    csvfile.seek(0)

    if '\t' in line:
        # tab-separated
        reader = csv.reader(csvfile, delimiter='\t')
    else:
        reader = csv.reader(csvfile, dialect='excel')

    header = reader.next()
    header = [cell.decode(encoding).lower().strip() for cell in header]
    encode_fn = lambda x: x.decode(encoding, 'replace')
    return _read_csv(reader, header, encode_fn)

def
open_csv_somehow_py2
(
filename
)
:
encoding
=
detect_file_encoding
(
filename
)
if
encoding
.
startswith
(
'UTF-16'
)
:
csvfile
=
transcode_to_utf8
(
filename
,
encoding
)
encoding
=
'UTF-8'
else
:
csvfile
=
open
(
filename
,
'rU'
)
line
=
csvfile
.
readline
(
)
csvfile
.
seek
(
0
)
if
'\t'
in
line
:
# tab-separated
reader
=
csv
.
reader
(
csvfile
,
delimiter
=
'\t'
)
else
:
reader
=
csv
.
reader
(
csvfile
,
dialect
=
'excel'
)
header
=
reader
.
next
(
)
header
=
[
cell
.
decode
(
encoding
)
.
lower
(
)
.
strip
(
)
for
cell
in
header
]
encode_fn
=
lambda
x
:
x
.
decode
(
encoding
,
'replace'
)
return
_read_csv
(
reader
,
header
,
encode_fn
)

Open a CSV file using Python 2 s CSV module working around the deficiency where it can t handle the null bytes of UTF - 16 .


def _read_csv(reader, header, encode_fn):
    """
    Given a constructed CSV reader object, a header row that we've read, and
    a detected encoding, yield its rows as dictionaries.
    """
    for row in reader:
        if len(row) == 0:
            continue
        row = [encode_fn(cell) for cell in row]
        row_list = zip(header, row)
        row_dict = dict(row_list)
        if len(row_dict['text']) == 0:
            continue
        row_dict['text'] = unicodedata.normalize(
            'NFKC', row_dict['text'].strip()
        )
        if row_dict.get('title') == '':
            del row_dict['title']
        if 'date' in row_dict:
            # We handle dates further in open_json_or_csv_somehow
            if row_dict['date'] == '':
                del row_dict['date']
        if 'subset' in row_dict:
            subsets = [cell[1] for cell in row_list
                       if cell[1] != '' and cell[0] == 'subset']
            if subsets:
                row_dict['subsets'] = subsets
            if 'subset' in row_dict:
                del row_dict['subset']
        yield row_dict

Given a constructed CSV reader object a header row that we ve read and a detected encoding yield its rows as dictionaries .


def main():
    """
    Handle command line arguments to convert a file to a JSON stream as a
    script.
    """
    logging.basicConfig(level=logging.INFO)
    import argparse
    parser = argparse.ArgumentParser(
        description="Translate CSV or JSON input to a JSON stream, or verify "
                    "something that is already a JSON stream."
    )
    parser.add_argument('input',
        help='A CSV, JSON, or JSON stream file to read.')
    parser.add_argument('output', nargs='?', default=None,
        help="The filename to output to. Recommended extension is .jsons. "
             "If omitted, use standard output.")
    args = parser.parse_args()
    transcode(args.input, args.output)

Handle command line arguments to convert a file to a JSON stream as a script .


def connect(cls, url=None, token_file=None, token=None):
        """
        Returns an object that makes requests to the API, authenticated
        with a saved or specified long-lived token, at URLs beginning with
        `url`.

        If no URL is specified, or if the specified URL is a path such as
        '/projects' without a scheme and domain, the client will default to
        https://analytics.luminoso.com/api/v5/.

        If neither token nor token_file are specified, the client will look
        for a token in $HOME/.luminoso/tokens.json. The file should contain
        a single json dictionary of the format
        `{'root_url': 'token', 'root_url2': 'token2', ...}`.
        """
        if url is None:
            url = '/'

        if url.startswith('http'):
            root_url = get_root_url(url)
        else:
            url = URL_BASE + '/' + url.lstrip('/')
            root_url = URL_BASE

        if token is None:
            token_file = token_file or get_token_filename()
            try:
                with open(token_file) as tf:
                    token_dict = json.load(tf)
            except FileNotFoundError:
                raise LuminosoAuthError('No token file at %s' % token_file)
            try:
                token = token_dict[urlparse(root_url).netloc]
            except KeyError:
                raise LuminosoAuthError('No token stored for %s' % root_url)

        session = requests.session()
        session.auth = _TokenAuth(token)
        return cls(session, url)

def
connect
(
cls
,
url
=
,
token_file
=
,
token
=
)
:
if
url
is
:
url
=
'/'
if
url
.
startswith
(
'http'
)
:
root_url
=
get_root_url
(
url
)
else
:
url
=
URL_BASE
+
'/'
+
url
.
lstrip
(
'/'
)
root_url
=
URL_BASE
if
token
is
:
token_file
=
token_file
or
get_token_filename
(
)
try
:
with
open
(
token_file
)
as
tf
:
token_dict
=
json
.
load
(
tf
)
except
FileNotFoundError
:
raise
LuminosoAuthError
(
'No token file at %s'
%
token_file
)
try
:
token
=
token_dict
[
urlparse
(
root_url
)
.
netloc
]
except
KeyError
:
raise
LuminosoAuthError
(
'No token stored for %s'
%
root_url
)
session
=
requests
.
session
(
)
session
.
auth
=
_TokenAuth
(
token
)
return
cls
(
session
,
url
)

Returns an object that makes requests to the API authenticated with a saved or specified long - lived token at URLs beginning with url .


def save_token(token, domain='analytics.luminoso.com', token_file=None):
        """
        Take a long-lived API token and store it to a local file.  Long-lived
        tokens can be retrieved through the UI.  Optional arguments are the
        domain for which the token is valid and the file in which to store the
        token.
        """
        token_file = token_file or get_token_filename()
        if os.path.exists(token_file):
            saved_tokens = json.load(open(token_file))
        else:
            saved_tokens = {}
        saved_tokens[domain] = token
        directory, filename = os.path.split(token_file)
        if directory and not os.path.exists(directory):
            os.makedirs(directory)
        with open(token_file, 'w') as f:
            json.dump(saved_tokens, f)

def save_token ( token , domain = 'analytics.luminoso.com' , token_file = None ) : token_file = token_file or get_token_filename ( ) if os . path . exists ( token_file ) : saved_tokens = json . load ( open ( token_file ) ) else : saved_tokens = { } saved_tokens [ domain ] = token directory , filename = os . path . split ( token_file ) if directory and not os . path . exists ( directory ) : os . makedirs ( directory ) with open ( token_file , 'w' ) as f : json . dump ( saved_tokens , f )

Take a long - lived API token and store it to a local file . Long - lived tokens can be retrieved through the UI . Optional arguments are the domain for which the token is valid and the file in which to store the token .


def _request(self, req_type, url, **kwargs):
        """
        Make a request via the `requests` module. If the result has an HTTP
        error status, convert that to a Python exception.
        """
        logger.debug('%s %s' % (req_type, url))
        result = self.session.request(req_type, url, **kwargs)
        try:
            result.raise_for_status()
        except requests.HTTPError:
            error = result.text
            try:
                error = json.loads(error)
            except ValueError:
                pass
            if result.status_code in (401, 403):
                error_class = LuminosoAuthError
            elif result.status_code in (400, 404, 405):
                error_class = LuminosoClientError
            elif result.status_code >= 500:
                error_class = LuminosoServerError
            else:
                error_class = LuminosoError
            raise error_class(error)
        return result

def
_request
(
self
,
req_type
,
url
,
*
*
kwargs
)
:
logger
.
debug
(
'%s %s'
%
(
req_type
,
url
)
)
result
=
self
.
session
.
request
(
req_type
,
url
,
*
*
kwargs
)
try
:
result
.
raise_for_status
(
)
except
requests
.
HTTPError
:
error
=
result
.
text
try
:
error
=
json
.
loads
(
error
)
except
ValueError
:
pass
if
result
.
status_code
in
(
401
,
403
)
:
error_class
=
LuminosoAuthError
elif
result
.
status_code
in
(
400
,
404
,
405
)
:
error_class
=
LuminosoClientError
elif
result
.
status_code
>=
500
:
error_class
=
LuminosoServerError
else
:
error_class
=
LuminosoError
raise
error_class
(
error
)
return
result

Make a request via the requests module . If the result has an HTTP error status convert that to a Python exception .


def delete(self, path='', **params):
        """
        Make a DELETE request to the given path, and return the JSON-decoded
        result.

        Keyword parameters will be converted to URL parameters.

        DELETE requests ask to delete the object represented by this URL.
        """
        params = jsonify_parameters(params)
        url = ensure_trailing_slash(self.url + path.lstrip('/'))
        return self._json_request('delete', url, params=params)

def delete ( self , path = '' , * * params ) : params = jsonify_parameters ( params ) url = ensure_trailing_slash ( self . url + path . lstrip ( '/' ) ) return self . _json_request ( 'delete' , url , params = params )

Make a DELETE request to the given path and return the JSON - decoded result .


def wait_for_build(self, interval=5, path=None):
        """
        A convenience method designed to inform you when a project build has
        completed.  It polls the API every `interval` seconds until there is
        not a build running.  At that point, it returns the "last_build_info"
        field of the project record if the build succeeded, and raises a
        LuminosoError with the field as its message if the build failed.

        If a `path` is not specified, this method will assume that its URL is
        the URL for the project.  Otherwise, it will use the specified path
        (which should be "/projects/<project_id>/").
        """
        path = path or ''
        start = time.time()
        next_log = 0
        while True:
            response = self.get(path)['last_build_info']
            if not response:
                raise ValueError('This project is not building!')
            if response['stop_time']:
                if response['success']:
                    return response
                else:
                    raise LuminosoError(response)
            elapsed = time.time() - start
            if elapsed > next_log:
                logger.info('Still waiting (%d seconds elapsed).', next_log)
                next_log += 120
            time.sleep(interval)

def
wait_for_build
(
self
,
interval
=
5
,
path
=
)
:
path
=
path
or
''
start
=
time
.
time
(
)
next_log
=
0
while
True
:
response
=
self
.
get
(
path
)
[
'last_build_info'
]
if
not
response
:
raise
ValueError
(
'This project is not building!'
)
if
response
[
'stop_time'
]
:
if
response
[
'success'
]
:
return
response
else
:
raise
LuminosoError
(
response
)
elapsed
=
time
.
time
(
)
start
if
elapsed
>
next_log
:
logger
.
info
(
'Still waiting (%d seconds elapsed).'
,
next_log
)
next_log
+=
120
time
.
sleep
(
interval
)

A convenience method designed to inform you when a project build has completed . It polls the API every interval seconds until there is not a build running . At that point it returns the last_build_info field of the project record if the build succeeded and raises a LuminosoError with the field as its message if the build failed .


def get_root_url(url, warn=True):
    """
    Get the "root URL" for a URL, as described in the LuminosoClient
    documentation.
    """
    parsed_url = urlparse(url)

    # Make sure it's a complete URL, not a relative one
    if not parsed_url.scheme:
        raise ValueError('Please supply a full URL, beginning with http:// '
                         'or https:// .')

    # Issue a warning if the path didn't already start with /api/v4
    root_url = '%s://%s/api/v4' % (parsed_url.scheme, parsed_url.netloc)
    if warn and not parsed_url.path.startswith('/api/v4'):
        logger.warning('Using %s as the root url' % root_url)
    return root_url

Get the root URL for a URL as described in the LuminosoClient documentation .


def save_token(self, token_file=None):
        """
        Obtain the user's long-lived API token and save it in a local file.
        If the user has no long-lived API token, one will be created.
        Returns the token that was saved.
        """
        tokens = self._json_request('get', self.root_url + '/user/tokens/')
        long_lived = [token['type'] == 'long_lived' for token in tokens]
        if any(long_lived):
            dic = tokens[long_lived.index(True)]
        else:
            # User doesn't have a long-lived token, so create one
            dic = self._json_request('post', self.root_url + '/user/tokens/')
        token = dic['token']
        token_file = token_file or get_token_filename()
        if os.path.exists(token_file):
            saved_tokens = json.load(open(token_file))
        else:
            saved_tokens = {}
        saved_tokens[urlparse(self.root_url).netloc] = token
        directory, filename = os.path.split(token_file)
        if directory and not os.path.exists(directory):
            os.makedirs(directory)
        with open(token_file, 'w') as f:
            json.dump(saved_tokens, f)
        return token

Obtain the user s long - lived API token and save it in a local file . If the user has no long - lived API token one will be created . Returns the token that was saved .


def _json_request(self, req_type, url, **kwargs):
        """
        Make a request of the specified type and expect a JSON object in
        response.

        If the result has an 'error' value, raise a LuminosoAPIError with
        its contents. Otherwise, return the contents of the 'result' value.
        """
        response = self._request(req_type, url, **kwargs)
        try:
            json_response = response.json()
        except ValueError:
            logger.error("Received response with no JSON: %s %s" %
                         (response, response.content))
            raise LuminosoError('Response body contained no JSON. '
                                'Perhaps you meant to use get_raw?')
        if json_response.get('error'):
            raise LuminosoAPIError(json_response.get('error'))
        return json_response['result']

def
_json_request
(
self
,
req_type
,
url
,
*
*
kwargs
)
:
response
=
self
.
_request
(
req_type
,
url
,
*
*
kwargs
)
try
:
json_response
=
response
.
json
(
)
except
ValueError
:
logger
.
error
(
"Received response with no JSON: %s %s"
%
(
response
,
response
.
content
)
)
raise
LuminosoError
(
'Response body contained no JSON. '
'Perhaps you meant to use get_raw?'
)
if
json_response
.
get
(
'error'
)
:
raise
LuminosoAPIError
(
json_response
.
get
(
'error'
)
)
return
json_response
[
'result'
]

Make a request of the specified type and expect a JSON object in response .


def post_data(self, path, data, content_type, **params):
        """
        Make a POST request to the given path, with `data` in its body.
        Return the JSON-decoded result.

        The content_type must be set to reflect the kind of data being sent,
        which is often `application/json`.

        Keyword parameters will be converted to URL parameters. This is unlike
        other POST requests which encode those parameters in the body, because
        the body is already being used.

        This is used by the Luminoso API to upload new documents in JSON
        format.
        """
        params = jsonify_parameters(params)
        url = ensure_trailing_slash(self.url + path.lstrip('/'))
        return self._json_request('post', url,
            params=params,
            data=data,
            headers={'Content-Type': content_type}
        )

def post_data ( self , path , data , content_type , * * params ) : params = jsonify_parameters ( params ) url = ensure_trailing_slash ( self . url + path . lstrip ( '/' ) ) return self . _json_request ( 'post' , url , params = params , data = data , headers = { 'Content-Type' : content_type } )

Make a POST request to the given path with data in its body . Return the JSON - decoded result .


def change_path(self, path):
        """
        Return a new LuminosoClient for a subpath of this one.

        For example, you might want to start with a LuminosoClient for
        `https://analytics.luminoso.com/api/v4/`, then get a new one for
        `https://analytics.luminoso.com/api/v4/projects/myaccount/myprojectid`.
        You accomplish that with the following call:

            newclient = client.change_path('projects/myaccount/myproject_id')

        If you start the path with `/`, it will start from the root_url
        instead of the current url:

            project_area = newclient.change_path('/projects/myaccount')

        The advantage of using `.change_path` is that you will not need to
        re-authenticate like you would if you ran `.connect` again.

        You can use `.change_path` to split off as many sub-clients as you
        want, and you don't have to stop using the old one just because you
        got a new one with `.change_path`.
        """
        if path.startswith('/'):
            url = self.root_url + path
        else:
            url = self.url + path
        return self.__class__(self.session, url)

def change_path ( self , path ) : if path . startswith ( '/' ) : url = self . root_url + path else : url = self . url + path return self . __class__ ( self . session , url )

Return a new LuminosoClient for a subpath of this one .


def _get_default_account(self):
        """
        Get the ID of an account you can use to access projects.
        """
        newclient = self.__class__(self.session, self.root_url)
        account_info = newclient.get('/accounts/')
        if account_info['default_account'] is not None:
            return account_info['default_account']
        valid_accounts = [a['account_id'] for a in account_info['accounts']
                          if a['account_id'] != 'public']
        if len(valid_accounts) == 0:
            raise ValueError("Can't determine your default URL. "
                             "Please request a specific URL or ask "
                             "Luminoso for support.")
        return valid_accounts[0]

def
_get_default_account
(
self
)
:
newclient
=
self
.
__class__
(
self
.
session
,
self
.
root_url
)
account_info
=
newclient
.
get
(
'/accounts/'
)
if
account_info
[
'default_account'
]
is
not
:
return
account_info
[
'default_account'
]
valid_accounts
=
[
a
[
'account_id'
]
for
a
in
account_info
[
'accounts'
]
if
a
[
'account_id'
]
!=
'public'
]
if
len
(
valid_accounts
)
==
0
:
raise
ValueError
(
"Can't determine your default URL. "
"Please request a specific URL or ask "
"Luminoso for support."
)
return
valid_accounts
[
0
]

Get the ID of an account you can use to access projects .


def documentation(self):
        """
        Get the documentation that the server sends for the API.
        """
        newclient = self.__class__(self.session, self.root_url)
        return newclient.get_raw('/')

def documentation ( self ) : newclient = self . __class__ ( self . session , self . root_url ) return newclient . get_raw ( '/' )

Get the documentation that the server sends for the API .


def wait_for(self, job_id, base_path=None, interval=5):
        """
        Wait for an asynchronous task to finish.

        Unlike the thin methods elsewhere on this object, this one is actually
        specific to how the Luminoso API works. This will poll an API
        endpoint to find out the status of the job numbered `job_id`,
        repeating every 5 seconds (by default) until the job is done. When
        the job is done, it will return an object representing the result of
        that job.

        In the Luminoso API, requests that may take a long time return a
        job ID instead of a result, so that your code can continue running
        in the meantime. When it needs the job to be done to proceed, it can
        use this method to wait.

        The base URL where it looks for that job is by default `jobs/id/`
        under the current URL, assuming that this LuminosoClient's URL
        represents a project. You can specify a different URL by changing
        `base_path`.

        If the job failed, will raise a LuminosoError with the job status
        as its message.
        """
        if base_path is None:
            base_path = 'jobs/id'
        path = '%s%d' % (ensure_trailing_slash(base_path), job_id)
        start = time.time()
        next_log = 0
        while True:
            response = self.get(path)
            if response['stop_time']:
                if response['success']:
                    return response
                else:
                    raise LuminosoError(response)
            elapsed = time.time() - start
            if elapsed > next_log:
                logger.info('Still waiting (%d seconds elapsed).', next_log)
                next_log += 120
            time.sleep(interval)

def
wait_for
(
self
,
job_id
,
base_path
=
,
interval
=
5
)
:
if
base_path
is
:
base_path
=
'jobs/id'
path
=
'%s%d'
%
(
ensure_trailing_slash
(
base_path
)
,
job_id
)
start
=
time
.
time
(
)
next_log
=
0
while
True
:
response
=
self
.
get
(
path
)
if
response
[
'stop_time'
]
:
if
response
[
'success'
]
:
return
response
else
:
raise
LuminosoError
(
response
)
elapsed
=
time
.
time
(
)
start
if
elapsed
>
next_log
:
logger
.
info
(
'Still waiting (%d seconds elapsed).'
,
next_log
)
next_log
+=
120
time
.
sleep
(
interval
)

Wait for an asynchronous task to finish .


def get_raw(self, path, **params):
        """
        Get the raw text of a response.

        This is only generally useful for specific URLs, such as documentation.
        """
        url = ensure_trailing_slash(self.url + path.lstrip('/'))
        return self._request('get', url, params=params).text

def get_raw ( self , path , * * params ) : url = ensure_trailing_slash ( self . url + path . lstrip ( '/' ) ) return self . _request ( 'get' , url , params = params ) . text


def _print_csv(result):
    """Print a JSON list of JSON objects in CSV format."""
    if type(result) is not list:
        raise TypeError("output not able to be displayed as CSV.")
    first_line = result[0]
    w = csv.DictWriter(sys.stdout, fieldnames=sorted(first_line.keys()))
    w.writeheader()
    for line in result:
        w.writerow(line)

Print a JSON list of JSON objects in CSV format .


def _read_params(input_file, json_body, p_params):
    """Read parameters from input file, -j, and -p arguments, in that order."""
    params = {}
    try:
        if input_file:
            params.update(json.load(input_file))
        if json_body is not None:
            params.update(json.loads(json_body))
    except ValueError as e:
        raise ValueError("input is not valid JSON: %s" % e)
    try:
        params.update({p.split('=', 1)[0]: p.split('=', 1)[1] for p in p_params})
    except IndexError:
        raise ValueError("--param arguments must have key=value format")
    return params

Read parameters from input file - j and - p arguments in that order .


def _simplify_doc(doc):
    """
    Limit a document to just the three fields we should upload.
    """
    # Mutate a copy of the document to fill in missing fields
    doc = dict(doc)
    if 'text' not in doc:
        raise ValueError("The document {!r} has no text field".format(doc))
    return {
        'text': doc['text'],
        'metadata': doc.get('metadata', []),
        'title': doc.get('title', '')
    }

Limit a document to just the three fields we should upload .


def create_project_with_docs(
    client, docs, language, name, account=None, progress=False
):
    """
    Given an iterator of documents, upload them as a Luminoso project.
    """
    description = 'Uploaded using lumi-upload at {}'.format(time.asctime())
    if account is not None:
        proj_record = client.post(
            'projects',
            name=name,
            language=language,
            description=description,
            account_id=account,
        )
    else:
        proj_record = client.post(
            'projects', name=name, language=language, description=description
        )
    proj_id = proj_record['project_id']
    proj_client = client.client_for_path('projects/' + proj_id)
    try:
        if progress:
            progress_bar = tqdm(desc='Uploading documents')
        else:
            progress_bar = None

        for batch in _batches(docs, BATCH_SIZE):
            docs_to_upload = [_simplify_doc(doc) for doc in batch]
            proj_client.post('upload', docs=docs_to_upload)
            if progress:
                progress_bar.update(BATCH_SIZE)

    finally:
        if progress:
            progress_bar.close()

    print('The server is building project {!r}.'.format(proj_id))
    proj_client.post('build')

    while True:
        time.sleep(10)
        proj_status = proj_client.get()
        build_info = proj_status['last_build_info']
        if 'success' in build_info:
            if not build_info['success']:
                raise LuminosoServerError(build_info['reason'])
            return proj_status

def
create_project_with_docs
(
client
,
docs
,
language
,
name
,
account
=
,
progress
=
False
)
:
description
=
'Uploaded using lumi-upload at {}'
.
format
(
time
.
asctime
(
)
)
if
account
is
not
:
proj_record
=
client
.
post
(
'projects'
,
name
=
name
,
language
=
language
,
description
=
description
,
account_id
=
account
,
)
else
:
proj_record
=
client
.
post
(
'projects'
,
name
=
name
,
language
=
language
,
description
=
description
)
proj_id
=
proj_record
[
'project_id'
]
proj_client
=
client
.
client_for_path
(
'projects/'
+
proj_id
)
try
:
if
progress
:
progress_bar
=
tqdm
(
desc
=
'Uploading documents'
)
else
:
progress_bar
=
for
batch
in
_batches
(
docs
,
BATCH_SIZE
)
:
docs_to_upload
=
[
_simplify_doc
(
doc
)
for
doc
in
batch
]
proj_client
.
post
(
'upload'
,
docs
=
docs_to_upload
)
if
progress
:
progress_bar
.
update
(
BATCH_SIZE
)
finally
:
if
progress
:
progress_bar
.
close
(
)
print
(
'The server is building project {!r}.'
.
format
(
proj_id
)
)
proj_client
.
post
(
'build'
)
while
True
:
time
.
sleep
(
10
)
proj_status
=
proj_client
.
get
(
)
build_info
=
proj_status
[
'last_build_info'
]
if
'success'
in
build_info
:
if
not
build_info
[
'success'
]
:
raise
LuminosoServerError
(
build_info
[
'reason'
]
)
return
proj_status

Given an iterator of documents upload them as a Luminoso project .


def upload_docs(
    client, input_filename, language, name, account=None, progress=False
):
    """
    Given a LuminosoClient pointing to the root of the API, and a filename to
    read JSON lines from, create a project from the documents in that file.
    """
    docs = iterate_json_lines(input_filename)
    return create_project_with_docs(
        client, docs, language, name, account, progress=progress
    )

def upload_docs ( client , input_filename , language , name , account = None , progress = False ) : docs = iterate_json_lines ( input_filename ) return create_project_with_docs ( client , docs , language , name , account , progress = progress )

Given a LuminosoClient pointing to the root of the API and a filename to read JSON lines from create a project from the documents in that file .