def fail_unreachable(self, msg_id, why=error.ImpossibleDependency):
        """a task has become unreachable, send a reply with an ImpossibleDependency
        error."""
        if msg_id not in self.depending:
            self.log.error("msg %r already failed!", msg_id)
            return
        job = self.depending.pop(msg_id)
        for mid in job.dependents:
            if mid in self.graph:
                self.graph[mid].remove(msg_id)

        try:
            raise why()
        except:
            content = error.wrap_exception()

        self.all_done.add(msg_id)
        self.all_failed.add(msg_id)

        msg = self.session.send(self.client_stream, 'apply_reply', content,
                                                parent=job.header, ident=job.idents)
        self.session.send(self.mon_stream, msg, ident=[b'outtask']+job.idents)

        self.update_graph(msg_id, success=False)

def
fail_unreachable
(
self
,
msg_id
,
why
=
error
.
ImpossibleDependency
)
:
if
msg_id
not
in
self
.
depending
:
self
.
log
.
error
(
"msg %r already failed!"
,
msg_id
)
return
job
=
self
.
depending
.
pop
(
msg_id
)
for
mid
in
job
.
dependents
:
if
mid
in
self
.
graph
:
self
.
graph
[
mid
]
.
remove
(
msg_id
)
try
:
raise
why
(
)
except
:
content
=
error
.
wrap_exception
(
)
self
.
all_done
.
add
(
msg_id
)
self
.
all_failed
.
add
(
msg_id
)
msg
=
self
.
session
.
send
(
self
.
client_stream
,
'apply_reply'
,
content
,
parent
=
job
.
header
,
ident
=
job
.
idents
)
self
.
session
.
send
(
self
.
mon_stream
,
msg
,
ident
=
[
b'outtask'
]
+
job
.
idents
)
self
.
update_graph
(
msg_id
,
success
=
False
)

a task has become unreachable send a reply with an ImpossibleDependency error .


def maybe_run(self, job):
        """check location dependencies, and run if they are met."""
        msg_id = job.msg_id
        self.log.debug("Attempting to assign task %s", msg_id)
        if not self.targets:
            # no engines, definitely can't run
            return False

        if job.follow or job.targets or job.blacklist or self.hwm:
            # we need a can_run filter
            def can_run(idx):
                # check hwm
                if self.hwm and self.loads[idx] == self.hwm:
                    return False
                target = self.targets[idx]
                # check blacklist
                if target in job.blacklist:
                    return False
                # check targets
                if job.targets and target not in job.targets:
                    return False
                # check follow
                return job.follow.check(self.completed[target], self.failed[target])

            indices = filter(can_run, range(len(self.targets)))

            if not indices:
                # couldn't run
                if job.follow.all:
                    # check follow for impossibility
                    dests = set()
                    relevant = set()
                    if job.follow.success:
                        relevant = self.all_completed
                    if job.follow.failure:
                        relevant = relevant.union(self.all_failed)
                    for m in job.follow.intersection(relevant):
                        dests.add(self.destinations[m])
                    if len(dests) > 1:
                        self.depending[msg_id] = job
                        self.fail_unreachable(msg_id)
                        return False
                if job.targets:
                    # check blacklist+targets for impossibility
                    job.targets.difference_update(job.blacklist)
                    if not job.targets or not job.targets.intersection(self.targets):
                        self.depending[msg_id] = job
                        self.fail_unreachable(msg_id)
                        return False
                return False
        else:
            indices = None

        self.submit_task(job, indices)
        return True

check location dependencies and run if they are met .


def save_unmet(self, job):
        """Save a message for later submission when its dependencies are met."""
        msg_id = job.msg_id
        self.depending[msg_id] = job
        # track the ids in follow or after, but not those already finished
        for dep_id in job.after.union(job.follow).difference(self.all_done):
            if dep_id not in self.graph:
                self.graph[dep_id] = set()
            self.graph[dep_id].add(msg_id)

Save a message for later submission when its dependencies are met .


def submit_task(self, job, indices=None):
        """Submit a task to any of a subset of our targets."""
        if indices:
            loads = [self.loads[i] for i in indices]
        else:
            loads = self.loads
        idx = self.scheme(loads)
        if indices:
            idx = indices[idx]
        target = self.targets[idx]
        # print (target, map(str, msg[:3]))
        # send job to the engine
        self.engine_stream.send(target, flags=zmq.SNDMORE, copy=False)
        self.engine_stream.send_multipart(job.raw_msg, copy=False)
        # update load
        self.add_job(idx)
        self.pending[target][job.msg_id] = job
        # notify Hub
        content = dict(msg_id=job.msg_id, engine_id=target.decode('ascii'))
        self.session.send(self.mon_stream, 'task_destination', content=content,
                        ident=[b'tracktask',self.ident])

def
submit_task
(
self
,
job
,
indices
=
)
:
if
indices
:
loads
=
[
self
.
loads
[
i
]
for
i
in
indices
]
else
:
loads
=
self
.
loads
idx
=
self
.
scheme
(
loads
)
if
indices
:
idx
=
indices
[
idx
]
target
=
self
.
targets
[
idx
]
# print (target, map(str, msg[:3]))
# send job to the engine
self
.
engine_stream
.
send
(
target
,
flags
=
zmq
.
SNDMORE
,
copy
=
False
)
self
.
engine_stream
.
send_multipart
(
job
.
raw_msg
,
copy
=
False
)
# update load
self
.
add_job
(
idx
)
self
.
pending
[
target
]
[
job
.
msg_id
]
=
job
# notify Hub
content
=
dict
(
msg_id
=
job
.
msg_id
,
engine_id
=
target
.
decode
(
'ascii'
)
)
self
.
session
.
send
(
self
.
mon_stream
,
'task_destination'
,
content
=
content
,
ident
=
[
b'tracktask'
,
self
.
ident
]
)

Submit a task to any of a subset of our targets .


def dispatch_result(self, raw_msg):
        """dispatch method for result replies"""
        try:
            idents,msg = self.session.feed_identities(raw_msg, copy=False)
            msg = self.session.unserialize(msg, content=False, copy=False)
            engine = idents[0]
            try:
                idx = self.targets.index(engine)
            except ValueError:
                pass # skip load-update for dead engines
            else:
                self.finish_job(idx)
        except Exception:
            self.log.error("task::Invaid result: %r", raw_msg, exc_info=True)
            return

        header = msg['header']
        parent = msg['parent_header']
        if header.get('dependencies_met', True):
            success = (header['status'] == 'ok')
            msg_id = parent['msg_id']
            retries = self.retries[msg_id]
            if not success and retries > 0:
                # failed
                self.retries[msg_id] = retries - 1
                self.handle_unmet_dependency(idents, parent)
            else:
                del self.retries[msg_id]
                # relay to client and update graph
                self.handle_result(idents, parent, raw_msg, success)
                # send to Hub monitor
                self.mon_stream.send_multipart([b'outtask']+raw_msg, copy=False)
        else:
            self.handle_unmet_dependency(idents, parent)

def
dispatch_result
(
self
,
raw_msg
)
:
try
:
idents
,
msg
=
self
.
session
.
feed_identities
(
raw_msg
,
copy
=
False
)
msg
=
self
.
session
.
unserialize
(
msg
,
content
=
False
,
copy
=
False
)
engine
=
idents
[
0
]
try
:
idx
=
self
.
targets
.
index
(
engine
)
except
ValueError
:
pass
# skip load-update for dead engines
else
:
self
.
finish_job
(
idx
)
except
Exception
:
self
.
log
.
error
(
"task::Invaid result: %r"
,
raw_msg
,
exc_info
=
True
)
return
header
=
msg
[
'header'
]
parent
=
msg
[
'parent_header'
]
if
header
.
get
(
'dependencies_met'
,
True
)
:
success
=
(
header
[
'status'
]
==
'ok'
)
msg_id
=
parent
[
'msg_id'
]
retries
=
self
.
retries
[
msg_id
]
if
not
success
and
retries
>
0
:
# failed
self
.
retries
[
msg_id
]
=
retries
1
self
.
handle_unmet_dependency
(
idents
,
parent
)
else
:
del
self
.
retries
[
msg_id
]
# relay to client and update graph
self
.
handle_result
(
idents
,
parent
,
raw_msg
,
success
)
# send to Hub monitor
self
.
mon_stream
.
send_multipart
(
[
b'outtask'
]
+
raw_msg
,
copy
=
False
)
else
:
self
.
handle_unmet_dependency
(
idents
,
parent
)


def handle_result(self, idents, parent, raw_msg, success=True):
        """handle a real task result, either success or failure"""
        # first, relay result to client
        engine = idents[0]
        client = idents[1]
        # swap_ids for ROUTER-ROUTER mirror
        raw_msg[:2] = [client,engine]
        # print (map(str, raw_msg[:4]))
        self.client_stream.send_multipart(raw_msg, copy=False)
        # now, update our data structures
        msg_id = parent['msg_id']
        self.pending[engine].pop(msg_id)
        if success:
            self.completed[engine].add(msg_id)
            self.all_completed.add(msg_id)
        else:
            self.failed[engine].add(msg_id)
            self.all_failed.add(msg_id)
        self.all_done.add(msg_id)
        self.destinations[msg_id] = engine

        self.update_graph(msg_id, success)

def
handle_result
(
self
,
idents
,
parent
,
raw_msg
,
success
=
True
)
:
# first, relay result to client
engine
=
idents
[
0
]
client
=
idents
[
1
]
# swap_ids for ROUTER-ROUTER mirror
raw_msg
[
:
2
]
=
[
client
,
engine
]
# print (map(str, raw_msg[:4]))
self
.
client_stream
.
send_multipart
(
raw_msg
,
copy
=
False
)
# now, update our data structures
msg_id
=
parent
[
'msg_id'
]
self
.
pending
[
engine
]
.
pop
(
msg_id
)
if
success
:
self
.
completed
[
engine
]
.
add
(
msg_id
)
self
.
all_completed
.
add
(
msg_id
)
else
:
self
.
failed
[
engine
]
.
add
(
msg_id
)
self
.
all_failed
.
add
(
msg_id
)
self
.
all_done
.
add
(
msg_id
)
self
.
destinations
[
msg_id
]
=
engine
self
.
update_graph
(
msg_id
,
success
)

handle a real task result either success or failure


def handle_unmet_dependency(self, idents, parent):
        """handle an unmet dependency"""
        engine = idents[0]
        msg_id = parent['msg_id']

        job = self.pending[engine].pop(msg_id)
        job.blacklist.add(engine)

        if job.blacklist == job.targets:
            self.depending[msg_id] = job
            self.fail_unreachable(msg_id)
        elif not self.maybe_run(job):
            # resubmit failed
            if msg_id not in self.all_failed:
                # put it back in our dependency tree
                self.save_unmet(job)

        if self.hwm:
            try:
                idx = self.targets.index(engine)
            except ValueError:
                pass # skip load-update for dead engines
            else:
                if self.loads[idx] == self.hwm-1:
                    self.update_graph(None)

def
handle_unmet_dependency
(
self
,
idents
,
parent
)
:
engine
=
idents
[
0
]
msg_id
=
parent
[
'msg_id'
]
job
=
self
.
pending
[
engine
]
.
pop
(
msg_id
)
job
.
blacklist
.
add
(
engine
)
if
job
.
blacklist
==
job
.
targets
:
self
.
depending
[
msg_id
]
=
job
self
.
fail_unreachable
(
msg_id
)
elif
not
self
.
maybe_run
(
job
)
:
# resubmit failed
if
msg_id
not
in
self
.
all_failed
:
# put it back in our dependency tree
self
.
save_unmet
(
job
)
if
self
.
hwm
:
try
:
idx
=
self
.
targets
.
index
(
engine
)
except
ValueError
:
pass
# skip load-update for dead engines
else
:
if
self
.
loads
[
idx
]
==
self
.
hwm
1
:
self
.
update_graph
(
)


def update_graph(self, dep_id=None, success=True):
        """dep_id just finished. Update our dependency
        graph and submit any jobs that just became runable.

        Called with dep_id=None to update entire graph for hwm, but without finishing
        a task.
        """
        # print ("\n\n***********")
        # pprint (dep_id)
        # pprint (self.graph)
        # pprint (self.depending)
        # pprint (self.all_completed)
        # pprint (self.all_failed)
        # print ("\n\n***********\n\n")
        # update any jobs that depended on the dependency
        jobs = self.graph.pop(dep_id, [])

        # recheck *all* jobs if
        # a) we have HWM and an engine just become no longer full
        # or b) dep_id was given as None

        if dep_id is None or self.hwm and any( [ load==self.hwm-1 for load in self.loads ]):
            jobs = self.depending.keys()

        for msg_id in sorted(jobs, key=lambda msg_id: self.depending[msg_id].timestamp):
            job = self.depending[msg_id]

            if job.after.unreachable(self.all_completed, self.all_failed)\
                    or job.follow.unreachable(self.all_completed, self.all_failed):
                self.fail_unreachable(msg_id)

            elif job.after.check(self.all_completed, self.all_failed): # time deps met, maybe run
                if self.maybe_run(job):

                    self.depending.pop(msg_id)
                    for mid in job.dependents:
                        if mid in self.graph:
                            self.graph[mid].remove(msg_id)

dep_id just finished . Update our dependency graph and submit any jobs that just became runable .


def logstart(self, logfname=None, loghead=None, logmode=None,
                 log_output=False, timestamp=False, log_raw_input=False):
        """Generate a new log-file with a default header.

        Raises RuntimeError if the log has already been started"""

        if self.logfile is not None:
            raise RuntimeError('Log file is already active: %s' %
                               self.logfname)

        # The parameters can override constructor defaults
        if logfname is not None: self.logfname = logfname
        if loghead is not None: self.loghead = loghead
        if logmode is not None: self.logmode = logmode

        # Parameters not part of the constructor
        self.timestamp = timestamp
        self.log_output = log_output
        self.log_raw_input = log_raw_input

        # init depending on the log mode requested
        isfile = os.path.isfile
        logmode = self.logmode

        if logmode == 'append':
            self.logfile = io.open(self.logfname, 'a', encoding='utf-8')

        elif logmode == 'backup':
            if isfile(self.logfname):
                backup_logname = self.logfname+'~'
                # Manually remove any old backup, since os.rename may fail
                # under Windows.
                if isfile(backup_logname):
                    os.remove(backup_logname)
                os.rename(self.logfname,backup_logname)
            self.logfile = io.open(self.logfname, 'w', encoding='utf-8')

        elif logmode == 'global':
            self.logfname = os.path.join(self.home_dir,self.logfname)
            self.logfile = io.open(self.logfname, 'a', encoding='utf-8')

        elif logmode == 'over':
            if isfile(self.logfname):
                os.remove(self.logfname)
            self.logfile = io.open(self.logfname,'w', encoding='utf-8')

        elif logmode == 'rotate':
            if isfile(self.logfname):
                if isfile(self.logfname+'.001~'):
                    old = glob.glob(self.logfname+'.*~')
                    old.sort()
                    old.reverse()
                    for f in old:
                        root, ext = os.path.splitext(f)
                        num = int(ext[1:-1])+1
                        os.rename(f, root+'.'+`num`.zfill(3)+'~')
                os.rename(self.logfname, self.logfname+'.001~')
            self.logfile = io.open(self.logfname, 'w', encoding='utf-8')

        if logmode != 'append':
            self.logfile.write(self.loghead)

        self.logfile.flush()
        self.log_active = True

Generate a new log - file with a default header .


def logstate(self):
        """Print a status message about the logger."""
        if self.logfile is None:
            print 'Logging has not been activated.'
        else:
            state = self.log_active and 'active' or 'temporarily suspended'
            print 'Filename       :',self.logfname
            print 'Mode           :',self.logmode
            print 'Output logging :',self.log_output
            print 'Raw input log  :',self.log_raw_input
            print 'Timestamping   :',self.timestamp
            print 'State          :',state

def
logstate
(
self
)
:
if
self
.
logfile
is
:
print
'Logging has not been activated.'
else
:
state
=
self
.
log_active
and
'active'
or
'temporarily suspended'
print
'Filename       :'
,
self
.
logfname
print
'Mode           :'
,
self
.
logmode
print
'Output logging :'
,
self
.
log_output
print
'Raw input log  :'
,
self
.
log_raw_input
print
'Timestamping   :'
,
self
.
timestamp
print
'State          :'
,
state

Print a status message about the logger .


def log(self, line_mod, line_ori):
        """Write the sources to a log.

        Inputs:

        - line_mod: possibly modified input, such as the transformations made
        by input prefilters or input handlers of various kinds.  This should
        always be valid Python.

        - line_ori: unmodified input line from the user.  This is not
        necessarily valid Python.
        """

        # Write the log line, but decide which one according to the
        # log_raw_input flag, set when the log is started.
        if self.log_raw_input:
            self.log_write(line_ori)
        else:
            self.log_write(line_mod)


def log_write(self, data, kind='input'):
        """Write data to the log file, if active"""

        #print 'data: %r' % data # dbg
        if self.log_active and data:
            write = self.logfile.write
            if kind=='input':
                if self.timestamp:
                    write(str_to_unicode(time.strftime('# %a, %d %b %Y %H:%M:%S\n',
                                        time.localtime())))
                write(data)
            elif kind=='output' and self.log_output:
                odata = u'\n'.join([u'#[Out]# %s' % s
                                   for s in data.splitlines()])
                write(u'%s\n' % odata)
            self.logfile.flush()

def
log_write
(
self
,
data
,
kind
=
'input'
)
:
#print 'data: %r' % data # dbg
if
self
.
log_active
and
data
:
write
=
self
.
logfile
.
write
if
kind
==
'input'
:
if
self
.
timestamp
:
write
(
str_to_unicode
(
time
.
strftime
(
'# %a, %d %b %Y %H:%M:%S\n'
,
time
.
localtime
(
)
)
)
)
write
(
data
)
elif
kind
==
'output'
and
self
.
log_output
:
odata
=
u'\n'
.
join
(
[
u'#[Out]# %s'
%
s
for
s
in
data
.
splitlines
(
)
]
)
write
(
u'%s\n'
%
odata
)
self
.
logfile
.
flush
(
)


def logstop(self):
        """Fully stop logging and close log file.

        In order to start logging again, a new logstart() call needs to be
        made, possibly (though not necessarily) with a new filename, mode and
        other options."""

        if self.logfile is not None:
            self.logfile.close()
            self.logfile = None
        else:
            print "Logging hadn't been started."
        self.log_active = False

def
logstop
(
self
)
:
if
self
.
logfile
is
not
:
self
.
logfile
.
close
(
)
self
.
logfile
=
else
:
print
"Logging hadn't been started."
self
.
log_active
=
False


def new_worksheet(name=None, cells=None):
    """Create a worksheet by name with with a list of cells."""
    ws = NotebookNode()
    if name is not None:
        ws.name = unicode(name)
    if cells is None:
        ws.cells = []
    else:
        ws.cells = list(cells)
    return ws

def new_worksheet ( name = None , cells = None ) : ws = NotebookNode ( ) if name is not None : ws . name = unicode ( name ) if cells is None : ws . cells = [ ] else : ws . cells = list ( cells ) return ws

Create a worksheet by name with with a list of cells .


def add_s(self, s, obj, priority= 0 ):
        """ Adds a target 'string' for dispatching """

        chain = self.strs.get(s, CommandChainDispatcher())
        chain.add(obj,priority)
        self.strs[s] = chain

def add_s ( self , s , obj , priority = 0 ) : chain = self . strs . get ( s , CommandChainDispatcher ( ) ) chain . add ( obj , priority ) self . strs [ s ] = chain


def add_re(self, regex, obj, priority= 0 ):
        """ Adds a target regexp for dispatching """

        chain = self.regexs.get(regex, CommandChainDispatcher())
        chain.add(obj,priority)
        self.regexs[regex] = chain

def add_re ( self , regex , obj , priority = 0 ) : chain = self . regexs . get ( regex , CommandChainDispatcher ( ) ) chain . add ( obj , priority ) self . regexs [ regex ] = chain


def dispatch(self, key):
        """ Get a seq of Commandchain objects that match key """
        if key in self.strs:
            yield self.strs[key]

        for r, obj in self.regexs.items():
            if re.match(r, key):
                yield obj
            else:
                #print "nomatch",key  # dbg
                pass

def
dispatch
(
self
,
key
)
:
if
key
in
self
.
strs
:
yield
self
.
strs
[
key
]
for
r
,
obj
in
self
.
regexs
.
items
(
)
:
if
re
.
match
(
r
,
key
)
:
yield
obj
else
:
#print "nomatch",key  # dbg
pass


def flat_matches(self, key):
        """ Yield all 'value' targets, without priority """
        for val in self.dispatch(key):
            for el in val:
                yield el[1] # only value, no priority
        return

def
flat_matches
(
self
,
key
)
:
for
val
in
self
.
dispatch
(
key
)
:
for
el
in
val
:
yield
el
[
1
]
# only value, no priority
return

Yield all value targets without priority


def _notebook_dir_changed(self, name, old, new):
        """do a bit of validation of the notebook dir"""
        if os.path.exists(new) and not os.path.isdir(new):
            raise TraitError("notebook dir %r is not a directory" % new)
        if not os.path.exists(new):
            self.log.info("Creating notebook dir %s", new)
            try:
                os.mkdir(new)
            except:
                raise TraitError("Couldn't create notebook dir %r" % new)

def
_notebook_dir_changed
(
self
,
name
,
old
,
new
)
:
if
os
.
path
.
exists
(
new
)
and
not
os
.
path
.
isdir
(
new
)
:
raise
TraitError
(
"notebook dir %r is not a directory"
%
new
)
if
not
os
.
path
.
exists
(
new
)
:
self
.
log
.
info
(
"Creating notebook dir %s"
,
new
)
try
:
os
.
mkdir
(
new
)
except
:
raise
TraitError
(
"Couldn't create notebook dir %r"
%
new
)


def list_notebooks(self):
        """List all notebooks in the notebook dir.

        This returns a list of dicts of the form::

            dict(notebook_id=notebook,name=name)
        """
        names = glob.glob(os.path.join(self.notebook_dir,
                                       '*' + self.filename_ext))
        names = [os.path.splitext(os.path.basename(name))[0]
                 for name in names]

        data = []
        for name in names:
            if name not in self.rev_mapping:
                notebook_id = self.new_notebook_id(name)
            else:
                notebook_id = self.rev_mapping[name]
            data.append(dict(notebook_id=notebook_id,name=name))
        data = sorted(data, key=lambda item: item['name'])
        return data

def list_notebooks ( self ) : names = glob . glob ( os . path . join ( self . notebook_dir , '*' + self . filename_ext ) ) names = [ os . path . splitext ( os . path . basename ( name ) ) [ 0 ] for name in names ] data = [ ] for name in names : if name not in self . rev_mapping : notebook_id = self . new_notebook_id ( name ) else : notebook_id = self . rev_mapping [ name ] data . append ( dict ( notebook_id = notebook_id , name = name ) ) data = sorted ( data , key = lambda item : item [ 'name' ] ) return data

List all notebooks in the notebook dir .


def new_notebook_id(self, name):
        """Generate a new notebook_id for a name and store its mappings."""
        # TODO: the following will give stable urls for notebooks, but unless
        # the notebooks are immediately redirected to their new urls when their
        # filemname changes, nasty inconsistencies result.  So for now it's
        # disabled and instead we use a random uuid4() call.  But we leave the
        # logic here so that we can later reactivate it, whhen the necessary
        # url redirection code is written.
        #notebook_id = unicode(uuid.uuid5(uuid.NAMESPACE_URL,
        #                 'file://'+self.get_path_by_name(name).encode('utf-8')))

        notebook_id = unicode(uuid.uuid4())

        self.mapping[notebook_id] = name
        self.rev_mapping[name] = notebook_id
        return notebook_id

Generate a new notebook_id for a name and store its mappings .


def delete_notebook_id(self, notebook_id):
        """Delete a notebook's id only. This doesn't delete the actual notebook."""
        name = self.mapping[notebook_id]
        del self.mapping[notebook_id]
        del self.rev_mapping[name]

def delete_notebook_id ( self , notebook_id ) : name = self . mapping [ notebook_id ] del self . mapping [ notebook_id ] del self . rev_mapping [ name ]

Delete a notebook s id only . This doesn t delete the actual notebook .


def notebook_exists(self, notebook_id):
        """Does a notebook exist?"""
        if notebook_id not in self.mapping:
            return False
        path = self.get_path_by_name(self.mapping[notebook_id])
        return os.path.isfile(path)

def notebook_exists ( self , notebook_id ) : if notebook_id not in self . mapping : return False path = self . get_path_by_name ( self . mapping [ notebook_id ] ) return os . path . isfile ( path )


def find_path(self, notebook_id):
        """Return a full path to a notebook given its notebook_id."""
        try:
            name = self.mapping[notebook_id]
        except KeyError:
            raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)
        return self.get_path_by_name(name)

def
find_path
(
self
,
notebook_id
)
:
try
:
name
=
self
.
mapping
[
notebook_id
]
except
KeyError
:
raise
web
.
HTTPError
(
404
,
u'Notebook does not exist: %s'
%
notebook_id
)
return
self
.
get_path_by_name
(
name
)

Return a full path to a notebook given its notebook_id .


def get_path_by_name(self, name):
        """Return a full path to a notebook given its name."""
        filename = name + self.filename_ext
        path = os.path.join(self.notebook_dir, filename)
        return path

def get_path_by_name ( self , name ) : filename = name + self . filename_ext path = os . path . join ( self . notebook_dir , filename ) return path

Return a full path to a notebook given its name .


def get_notebook(self, notebook_id, format=u'json'):
        """Get the representation of a notebook in format by notebook_id."""
        format = unicode(format)
        if format not in self.allowed_formats:
            raise web.HTTPError(415, u'Invalid notebook format: %s' % format)
        last_modified, nb = self.get_notebook_object(notebook_id)
        kwargs = {}
        if format == 'json':
            # don't split lines for sending over the wire, because it
            # should match the Python in-memory format.
            kwargs['split_lines'] = False
        data = current.writes(nb, format, **kwargs)
        name = nb.metadata.get('name','notebook')
        return last_modified, name, data

Get the representation of a notebook in format by notebook_id .


def get_notebook_object(self, notebook_id):
        """Get the NotebookNode representation of a notebook by notebook_id."""
        path = self.find_path(notebook_id)
        if not os.path.isfile(path):
            raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)
        info = os.stat(path)
        last_modified = datetime.datetime.utcfromtimestamp(info.st_mtime)
        with open(path,'r') as f:
            s = f.read()
            try:
                # v1 and v2 and json in the .ipynb files.
                nb = current.reads(s, u'json')
            except:
                raise web.HTTPError(500, u'Unreadable JSON notebook.')
        # Always use the filename as the notebook name.
        nb.metadata.name = os.path.splitext(os.path.basename(path))[0]
        return last_modified, nb

Get the NotebookNode representation of a notebook by notebook_id .


def save_new_notebook(self, data, name=None, format=u'json'):
        """Save a new notebook and return its notebook_id.

        If a name is passed in, it overrides any values in the notebook data
        and the value in the data is updated to use that value.
        """
        if format not in self.allowed_formats:
            raise web.HTTPError(415, u'Invalid notebook format: %s' % format)

        try:
            nb = current.reads(data.decode('utf-8'), format)
        except:
            raise web.HTTPError(400, u'Invalid JSON data')

        if name is None:
            try:
                name = nb.metadata.name
            except AttributeError:
                raise web.HTTPError(400, u'Missing notebook name')
        nb.metadata.name = name

        notebook_id = self.new_notebook_id(name)
        self.save_notebook_object(notebook_id, nb)
        return notebook_id

def
save_new_notebook
(
self
,
data
,
name
=
,
format
=
u'json'
)
:
if
format
not
in
self
.
allowed_formats
:
raise
web
.
HTTPError
(
415
,
u'Invalid notebook format: %s'
%
format
)
try
:
nb
=
current
.
reads
(
data
.
decode
(
'utf-8'
)
,
format
)
except
:
raise
web
.
HTTPError
(
400
,
u'Invalid JSON data'
)
if
name
is
:
try
:
name
=
nb
.
metadata
.
name
except
AttributeError
:
raise
web
.
HTTPError
(
400
,
u'Missing notebook name'
)
nb
.
metadata
.
name
=
name
notebook_id
=
self
.
new_notebook_id
(
name
)
self
.
save_notebook_object
(
notebook_id
,
nb
)
return
notebook_id

Save a new notebook and return its notebook_id .


def save_notebook(self, notebook_id, data, name=None, format=u'json'):
        """Save an existing notebook by notebook_id."""
        if format not in self.allowed_formats:
            raise web.HTTPError(415, u'Invalid notebook format: %s' % format)

        try:
            nb = current.reads(data.decode('utf-8'), format)
        except:
            raise web.HTTPError(400, u'Invalid JSON data')

        if name is not None:
            nb.metadata.name = name
        self.save_notebook_object(notebook_id, nb)

def
save_notebook
(
self
,
notebook_id
,
data
,
name
=
,
format
=
u'json'
)
:
if
format
not
in
self
.
allowed_formats
:
raise
web
.
HTTPError
(
415
,
u'Invalid notebook format: %s'
%
format
)
try
:
nb
=
current
.
reads
(
data
.
decode
(
'utf-8'
)
,
format
)
except
:
raise
web
.
HTTPError
(
400
,
u'Invalid JSON data'
)
if
name
is
not
:
nb
.
metadata
.
name
=
name
self
.
save_notebook_object
(
notebook_id
,
nb
)

Save an existing notebook by notebook_id .


def save_notebook_object(self, notebook_id, nb):
        """Save an existing notebook object by notebook_id."""
        if notebook_id not in self.mapping:
            raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)
        old_name = self.mapping[notebook_id]
        try:
            new_name = nb.metadata.name
        except AttributeError:
            raise web.HTTPError(400, u'Missing notebook name')
        path = self.get_path_by_name(new_name)
        try:
            with open(path,'w') as f:
                current.write(nb, f, u'json')
        except Exception as e:
            raise web.HTTPError(400, u'Unexpected error while saving notebook: %s' % e)
        # save .py script as well
        if self.save_script:
            pypath = os.path.splitext(path)[0] + '.py'
            try:
                with io.open(pypath,'w', encoding='utf-8') as f:
                    current.write(nb, f, u'py')
            except Exception as e:
                raise web.HTTPError(400, u'Unexpected error while saving notebook as script: %s' % e)

        if old_name != new_name:
            old_path = self.get_path_by_name(old_name)
            if os.path.isfile(old_path):
                os.unlink(old_path)
            if self.save_script:
                old_pypath = os.path.splitext(old_path)[0] + '.py'
                if os.path.isfile(old_pypath):
                    os.unlink(old_pypath)
            self.mapping[notebook_id] = new_name
            self.rev_mapping[new_name] = notebook_id
            del self.rev_mapping[old_name]

Save an existing notebook object by notebook_id .


def delete_notebook(self, notebook_id):
        """Delete notebook by notebook_id."""
        path = self.find_path(notebook_id)
        if not os.path.isfile(path):
            raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)
        os.unlink(path)
        self.delete_notebook_id(notebook_id)

def
delete_notebook
(
self
,
notebook_id
)
:
path
=
self
.
find_path
(
notebook_id
)
if
not
os
.
path
.
isfile
(
path
)
:
raise
web
.
HTTPError
(
404
,
u'Notebook does not exist: %s'
%
notebook_id
)
os
.
unlink
(
path
)
self
.
delete_notebook_id
(
notebook_id
)


def new_notebook(self):
        """Create a new notebook and return its notebook_id."""
        path, name = self.increment_filename('Untitled')
        notebook_id = self.new_notebook_id(name)
        metadata = current.new_metadata(name=name)
        nb = current.new_notebook(metadata=metadata)
        with open(path,'w') as f:
            current.write(nb, f, u'json')
        return notebook_id

def new_notebook ( self ) : path , name = self . increment_filename ( 'Untitled' ) notebook_id = self . new_notebook_id ( name ) metadata = current . new_metadata ( name = name ) nb = current . new_notebook ( metadata = metadata ) with open ( path , 'w' ) as f : current . write ( nb , f , u'json' ) return notebook_id

Create a new notebook and return its notebook_id .


def copy_notebook(self, notebook_id):
        """Copy an existing notebook and return its notebook_id."""
        last_mod, nb = self.get_notebook_object(notebook_id)
        name = nb.metadata.name + '-Copy'
        path, name = self.increment_filename(name)
        nb.metadata.name = name
        notebook_id = self.new_notebook_id(name)
        self.save_notebook_object(notebook_id, nb)
        return notebook_id

def copy_notebook ( self , notebook_id ) : last_mod , nb = self . get_notebook_object ( notebook_id ) name = nb . metadata . name + '-Copy' path , name = self . increment_filename ( name ) nb . metadata . name = name notebook_id = self . new_notebook_id ( name ) self . save_notebook_object ( notebook_id , nb ) return notebook_id

Copy an existing notebook and return its notebook_id .


def phys_tokens(toks):
    """Return all physical tokens, even line continuations.

    tokenize.generate_tokens() doesn't return a token for the backslash that
    continues lines.  This wrapper provides those tokens so that we can
    re-create a faithful representation of the original source.

    Returns the same values as generate_tokens()

    """
    last_line = None
    last_lineno = -1
    last_ttype = None
    for ttype, ttext, (slineno, scol), (elineno, ecol), ltext in toks:
        if last_lineno != elineno:
            if last_line and last_line.endswith("\\\n"):
                # We are at the beginning of a new line, and the last line
                # ended with a backslash.  We probably have to inject a
                # backslash token into the stream. Unfortunately, there's more
                # to figure out.  This code::
                #
                #   usage = """\
                #   HEY THERE
                #   """
                #
                # triggers this condition, but the token text is::
                #
                #   '"""\\\nHEY THERE\n"""'
                #
                # so we need to figure out if the backslash is already in the
                # string token or not.
                inject_backslash = True
                if last_ttype == tokenize.COMMENT:
                    # Comments like this \
                    # should never result in a new token.
                    inject_backslash = False
                elif ttype == token.STRING:
                    if "\n" in ttext and ttext.split('\n', 1)[0][-1] == '\\':
                        # It's a multiline string and the first line ends with
                        # a backslash, so we don't need to inject another.
                        inject_backslash = False
                if inject_backslash:
                    # Figure out what column the backslash is in.
                    ccol = len(last_line.split("\n")[-2]) - 1
                    # Yield the token, with a fake token type.
                    yield (
                        99999, "\\\n",
                        (slineno, ccol), (slineno, ccol+2),
                        last_line
                        )
            last_line = ltext
            last_ttype = ttype
        yield ttype, ttext, (slineno, scol), (elineno, ecol), ltext
        last_lineno = elineno

Return all physical tokens even line continuations .


def source_token_lines(source):
    """Generate a series of lines, one for each line in `source`.

    Each line is a list of pairs, each pair is a token::

        [('key', 'def'), ('ws', ' '), ('nam', 'hello'), ('op', '('), ... ]

    Each pair has a token class, and the token text.

    If you concatenate all the token texts, and then join them with newlines,
    you should have your original `source` back, with two differences:
    trailing whitespace is not preserved, and a final line with no newline
    is indistinguishable from a final line with a newline.

    """
    ws_tokens = set([token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL])
    line = []
    col = 0
    source = source.expandtabs(8).replace('\r\n', '\n')
    tokgen = generate_tokens(source)
    for ttype, ttext, (_, scol), (_, ecol), _ in phys_tokens(tokgen):
        mark_start = True
        for part in re.split('(\n)', ttext):
            if part == '\n':
                yield line
                line = []
                col = 0
                mark_end = False
            elif part == '':
                mark_end = False
            elif ttype in ws_tokens:
                mark_end = False
            else:
                if mark_start and scol > col:
                    line.append(("ws", " " * (scol - col)))
                    mark_start = False
                tok_class = tokenize.tok_name.get(ttype, 'xx').lower()[:3]
                if ttype == token.NAME and keyword.iskeyword(ttext):
                    tok_class = "key"
                line.append((tok_class, part))
                mark_end = True
            scol = 0
        if mark_end:
            col = ecol

    if line:
        yield line

def
source_token_lines
(
source
)
:
ws_tokens
=
set
(
[
token
.
INDENT
,
token
.
DEDENT
,
token
.
NEWLINE
,
tokenize
.
NL
]
)
line
=
[
]
col
=
0
source
=
source
.
expandtabs
(
8
)
.
replace
(
'\r\n'
,
'\n'
)
tokgen
=
generate_tokens
(
source
)
for
ttype
,
ttext
,
(
_
,
scol
)
,
(
_
,
ecol
)
,
_
in
phys_tokens
(
tokgen
)
:
mark_start
=
True
for
part
in
re
.
split
(
'(\n)'
,
ttext
)
:
if
part
==
'\n'
:
yield
line
line
=
[
]
col
=
0
mark_end
=
False
elif
part
==
''
:
mark_end
=
False
elif
ttype
in
ws_tokens
:
mark_end
=
False
else
:
if
mark_start
and
scol
>
col
:
line
.
append
(
(
"ws"
,
" "
*
(
scol
col
)
)
)
mark_start
=
False
tok_class
=
tokenize
.
tok_name
.
get
(
ttype
,
'xx'
)
.
lower
(
)
[
:
3
]
if
ttype
==
token
.
NAME
and
keyword
.
iskeyword
(
ttext
)
:
tok_class
=
"key"
line
.
append
(
(
tok_class
,
part
)
)
mark_end
=
True
scol
=
0
if
mark_end
:
col
=
ecol
if
line
:
yield
line

Generate a series of lines one for each line in source .


def load_default_config(ipython_dir=None):
    """Load the default config file from the default ipython_dir.

    This is useful for embedded shells.
    """
    if ipython_dir is None:
        ipython_dir = get_ipython_dir()
    profile_dir = os.path.join(ipython_dir, 'profile_default')
    cl = PyFileConfigLoader(default_config_file_name, profile_dir)
    try:
        config = cl.load_config()
    except ConfigFileNotFound:
        # no config found
        config = Config()
    return config

def
load_default_config
(
ipython_dir
=
)
:
if
ipython_dir
is
:
ipython_dir
=
get_ipython_dir
(
)
profile_dir
=
os
.
path
.
join
(
ipython_dir
,
'profile_default'
)
cl
=
PyFileConfigLoader
(
default_config_file_name
,
profile_dir
)
try
:
config
=
cl
.
load_config
(
)
except
ConfigFileNotFound
:
# no config found
config
=
Config
(
)
return
config

Load the default config file from the default ipython_dir .


def _classes_default(self):
        """This has to be in a method, for TerminalIPythonApp to be available."""
        return [
            InteractiveShellApp, # ShellApp comes before TerminalApp, because
            self.__class__,      # it will also affect subclasses (e.g. QtConsole)
            TerminalInteractiveShell,
            PromptManager,
            HistoryManager,
            ProfileDir,
            PlainTextFormatter,
            IPCompleter,
            ScriptMagics,
        ]

This has to be in a method for TerminalIPythonApp to be available .


def parse_command_line(self, argv=None):
        """override to allow old '-pylab' flag with deprecation warning"""

        argv = sys.argv[1:] if argv is None else argv

        if '-pylab' in argv:
            # deprecated `-pylab` given,
            # warn and transform into current syntax
            argv = argv[:] # copy, don't clobber
            idx = argv.index('-pylab')
            warn.warn("`-pylab` flag has been deprecated.\n"
            "    Use `--pylab` instead, or `--pylab=foo` to specify a backend.")
            sub = '--pylab'
            if len(argv) > idx+1:
                # check for gui arg, as in '-pylab qt'
                gui = argv[idx+1]
                if gui in ('wx', 'qt', 'qt4', 'gtk', 'auto'):
                    sub = '--pylab='+gui
                    argv.pop(idx+1)
            argv[idx] = sub

        return super(TerminalIPythonApp, self).parse_command_line(argv)

override to allow old - pylab flag with deprecation warning


def initialize(self, argv=None):
        """Do actions after construct, but before starting the app."""
        super(TerminalIPythonApp, self).initialize(argv)
        if self.subapp is not None:
            # don't bother initializing further, starting subapp
            return
        if not self.ignore_old_config:
            check_for_old_config(self.ipython_dir)
        # print self.extra_args
        if self.extra_args and not self.something_to_run:
            self.file_to_run = self.extra_args[0]
        self.init_path()
        # create the shell
        self.init_shell()
        # and draw the banner
        self.init_banner()
        # Now a variety of things that happen after the banner is printed.
        self.init_gui_pylab()
        self.init_extensions()
        self.init_code()

Do actions after construct but before starting the app .


def init_shell(self):
        """initialize the InteractiveShell instance"""
        # Create an InteractiveShell instance.
        # shell.display_banner should always be False for the terminal
        # based app, because we call shell.show_banner() by hand below
        # so the banner shows *before* all extension loading stuff.
        self.shell = TerminalInteractiveShell.instance(config=self.config,
                        display_banner=False, profile_dir=self.profile_dir,
                        ipython_dir=self.ipython_dir)
        self.shell.configurables.append(self)


def init_banner(self):
        """optionally display the banner"""
        if self.display_banner and self.interact:
            self.shell.show_banner()
        # Make sure there is a space below the banner.
        if self.log_level <= logging.INFO: print


def repr_type(obj):
    """ Return a string representation of a value and its type for readable
    error messages.
    """
    the_type = type(obj)
    if (not py3compat.PY3) and the_type is InstanceType:
        # Old-style class.
        the_type = obj.__class__
    msg = '%r %r' % (obj, the_type)
    return msg

def
repr_type
(
obj
)
:
the_type
=
type
(
obj
)
if
(
not
py3compat
.
PY3
)
and
the_type
is
InstanceType
:
# Old-style class.
the_type
=
obj
.
__class__
msg
=
'%r %r'
%
(
obj
,
the_type
)
return
msg

Return a string representation of a value and its type for readable error messages .


def parse_notifier_name(name):
    """Convert the name argument to a list of names.

    Examples
    --------

    >>> parse_notifier_name('a')
    ['a']
    >>> parse_notifier_name(['a','b'])
    ['a', 'b']
    >>> parse_notifier_name(None)
    ['anytrait']
    """
    if isinstance(name, str):
        return [name]
    elif name is None:
        return ['anytrait']
    elif isinstance(name, (list, tuple)):
        for n in name:
            assert isinstance(n, str), "names must be strings"
        return name

def
parse_notifier_name
(
name
)
:
if
isinstance
(
name
,
str
)
:
return
[
name
]
elif
name
is
:
return
[
'anytrait'
]
elif
isinstance
(
name
,
(
list
,
tuple
)
)
:
for
n
in
name
:
assert
isinstance
(
n
,
str
)
,
"names must be strings"
return
name

Convert the name argument to a list of names .


def set_default_value(self, obj):
        """Set the default value on a per instance basis.

        This method is called by :meth:`instance_init` to create and
        validate the default value.  The creation and validation of
        default values must be delayed until the parent :class:`HasTraits`
        class has been instantiated.
        """
        # Check for a deferred initializer defined in the same class as the
        # trait declaration or above.
        mro = type(obj).mro()
        meth_name = '_%s_default' % self.name
        for cls in mro[:mro.index(self.this_class)+1]:
            if meth_name in cls.__dict__:
                break
        else:
            # We didn't find one. Do static initialization.
            dv = self.get_default_value()
            newdv = self._validate(obj, dv)
            obj._trait_values[self.name] = newdv
            return
        # Complete the dynamic initialization.
        obj._trait_dyn_inits[self.name] = cls.__dict__[meth_name]

Set the default value on a per instance basis .


def on_trait_change(self, handler, name=None, remove=False):
        """Setup a handler to be called when a trait changes.

        This is used to setup dynamic notifications of trait changes.

        Static handlers can be created by creating methods on a HasTraits
        subclass with the naming convention '_[traitname]_changed'.  Thus,
        to create static handler for the trait 'a', create the method
        _a_changed(self, name, old, new) (fewer arguments can be used, see
        below).

        Parameters
        ----------
        handler : callable
            A callable that is called when a trait changes.  Its
            signature can be handler(), handler(name), handler(name, new)
            or handler(name, old, new).
        name : list, str, None
            If None, the handler will apply to all traits.  If a list
            of str, handler will apply to all names in the list.  If a
            str, the handler will apply just to that name.
        remove : bool
            If False (the default), then install the handler.  If True
            then unintall it.
        """
        if remove:
            names = parse_notifier_name(name)
            for n in names:
                self._remove_notifiers(handler, n)
        else:
            names = parse_notifier_name(name)
            for n in names:
                self._add_notifiers(handler, n)

def on_trait_change ( self , handler , name = None , remove = False ) : if remove : names = parse_notifier_name ( name ) for n in names : self . _remove_notifiers ( handler , n ) else : names = parse_notifier_name ( name ) for n in names : self . _add_notifiers ( handler , n )

Setup a handler to be called when a trait changes .


def class_traits(cls, **metadata):
        """Get a list of all the traits of this class.

        This method is just like the :meth:`traits` method, but is unbound.

        The TraitTypes returned don't know anything about the values
        that the various HasTrait's instances are holding.

        This follows the same algorithm as traits does and does not allow
        for any simple way of specifying merely that a metadata name
        exists, but has any value.  This is because get_metadata returns
        None if a metadata key doesn't exist.
        """
        traits = dict([memb for memb in getmembers(cls) if \
                     isinstance(memb[1], TraitType)])

        if len(metadata) == 0:
            return traits

        for meta_name, meta_eval in metadata.items():
            if type(meta_eval) is not FunctionType:
                metadata[meta_name] = _SimpleTest(meta_eval)

        result = {}
        for name, trait in traits.items():
            for meta_name, meta_eval in metadata.items():
                if not meta_eval(trait.get_metadata(meta_name)):
                    break
            else:
                result[name] = trait

        return result

def class_traits ( cls , * * metadata ) : traits = dict ( [ memb for memb in getmembers ( cls ) if isinstance ( memb [ 1 ] , TraitType ) ] ) if len ( metadata ) == 0 : return traits for meta_name , meta_eval in metadata . items ( ) : if type ( meta_eval ) is not FunctionType : metadata [ meta_name ] = _SimpleTest ( meta_eval ) result = { } for name , trait in traits . items ( ) : for meta_name , meta_eval in metadata . items ( ) : if not meta_eval ( trait . get_metadata ( meta_name ) ) : break else : result [ name ] = trait return result

Get a list of all the traits of this class .


def trait_metadata(self, traitname, key):
        """Get metadata values for trait by key."""
        try:
            trait = getattr(self.__class__, traitname)
        except AttributeError:
            raise TraitError("Class %s does not have a trait named %s" %
                                (self.__class__.__name__, traitname))
        else:
            return trait.get_metadata(key)


def validate(self, obj, value):
        """Validates that the value is a valid object instance."""
        try:
            if issubclass(value, self.klass):
                return value
        except:
            if (value is None) and (self._allow_none):
                return value

        self.error(obj, value)

def validate ( self , obj , value ) : try : if issubclass ( value , self . klass ) : return value except : if ( value is None ) and ( self . _allow_none ) : return value self . error ( obj , value )

Validates that the value is a valid object instance .


def get_default_value(self):
        """Instantiate a default value instance.

        This is called when the containing HasTraits classes'
        :meth:`__new__` method is called to ensure that a unique instance
        is created for each HasTraits instance.
        """
        dv  = self.default_value
        if isinstance(dv, DefaultValueGenerator):
            return dv.generate(self.klass)
        else:
            return dv

def get_default_value ( self ) : dv = self . default_value if isinstance ( dv , DefaultValueGenerator ) : return dv . generate ( self . klass ) else : return dv


def check(self, completed, failed=None):
        """check whether our dependencies have been met."""
        if len(self) == 0:
            return True
        against = set()
        if self.success:
            against = completed
        if failed is not None and self.failure:
            against = against.union(failed)
        if self.all:
            return self.issubset(against)
        else:
            return not self.isdisjoint(against)

def check ( self , completed , failed = None ) : if len ( self ) == 0 : return True against = set ( ) if self . success : against = completed if failed is not None and self . failure : against = against . union ( failed ) if self . all : return self . issubset ( against ) else : return not self . isdisjoint ( against )

check whether our dependencies have been met .


def unreachable(self, completed, failed=None):
        """return whether this dependency has become impossible."""
        if len(self) == 0:
            return False
        against = set()
        if not self.success:
            against = completed
        if failed is not None and not self.failure:
            against = against.union(failed)
        if self.all:
            return not self.isdisjoint(against)
        else:
            return self.issubset(against)

def unreachable ( self , completed , failed = None ) : if len ( self ) == 0 : return False against = set ( ) if not self . success : against = completed if failed is not None and not self . failure : against = against . union ( failed ) if self . all : return not self . isdisjoint ( against ) else : return self . issubset ( against )

return whether this dependency has become impossible .


def as_dict(self):
        """Represent this dependency as a dict. For json compatibility."""
        return dict(
            dependencies=list(self),
            all=self.all,
            success=self.success,
            failure=self.failure
        )

def as_dict ( self ) : return dict ( dependencies = list ( self ) , all = self . all , success = self . success , failure = self . failure )

Represent this dependency as a dict . For json compatibility .


def depth(n, tree):
    """get depth of an element in the tree"""
    d = 0
    parent = tree[n]
    while parent is not None:
        d += 1
        parent = tree[parent]
    return d

def depth ( n , tree ) : d = 0 parent = tree [ n ] while parent is not None : d += 1 parent = tree [ parent ] return d


def print_bintree(tree, indent='  '):
    """print a binary tree"""
    for n in sorted(tree.keys()):
        print "%s%s" % (indent * depth(n,tree), n)

def
print_bintree
(
tree
,
indent
=
'  '
)
:
for
n
in
sorted
(
tree
.
keys
(
)
)
:
print
"%s%s"
%
(
indent
*
depth
(
n
,
tree
)
,
n
)


def disambiguate_dns_url(url, location):
    """accept either IP address or dns name, and return IP"""
    if not ip_pat.match(location):
        location = socket.gethostbyname(location)
    return disambiguate_url(url, location)

def disambiguate_dns_url ( url , location ) : if not ip_pat . match ( location ) : location = socket . gethostbyname ( location ) return disambiguate_url ( url , location )

accept either IP address or dns name and return IP


def allreduce(self, f, value, flat=True):
        """parallel reduce followed by broadcast of the result"""
        return self.reduce(f, value, flat=flat, all=True)

def allreduce ( self , f , value , flat = True ) : return self . reduce ( f , value , flat = flat , all = True )


def _validate_targets(self, targets):
        """turn any valid targets argument into a list of integer ids"""
        if targets is None:
            # default to all
            return self.ids

        if isinstance(targets, (int,str,unicode)):
            # only one target specified
            targets = [targets]
        _targets = []
        for t in targets:
            # map raw identities to ids
            if isinstance(t, (str,unicode)):
                t = self.by_ident.get(cast_bytes(t), t)
            _targets.append(t)
        targets = _targets
        bad_targets = [ t for t in targets if t not in self.ids ]
        if bad_targets:
            raise IndexError("No Such Engine: %r" % bad_targets)
        if not targets:
            raise IndexError("No Engines Registered")
        return targets

def
_validate_targets
(
self
,
targets
)
:
if
targets
is
:
# default to all
return
self
.
ids
if
isinstance
(
targets
,
(
int
,
str
,
unicode
)
)
:
# only one target specified
targets
=
[
targets
]
_targets
=
[
]
for
t
in
targets
:
# map raw identities to ids
if
isinstance
(
t
,
(
str
,
unicode
)
)
:
t
=
self
.
by_ident
.
get
(
cast_bytes
(
t
)
,
t
)
_targets
.
append
(
t
)
targets
=
_targets
bad_targets
=
[
t
for
t
in
targets
if
t
not
in
self
.
ids
]
if
bad_targets
:
raise
IndexError
(
"No Such Engine: %r"
%
bad_targets
)
if
not
targets
:
raise
IndexError
(
"No Engines Registered"
)
return
targets


def dispatch_monitor_traffic(self, msg):
        """all ME and Task queue messages come through here, as well as
        IOPub traffic."""
        self.log.debug("monitor traffic: %r", msg[0])
        switch = msg[0]
        try:
            idents, msg = self.session.feed_identities(msg[1:])
        except ValueError:
            idents=[]
        if not idents:
            self.log.error("Monitor message without topic: %r", msg)
            return
        handler = self.monitor_handlers.get(switch, None)
        if handler is not None:
            handler(idents, msg)
        else:
            self.log.error("Unrecognized monitor topic: %r", switch)

def
dispatch_monitor_traffic
(
self
,
msg
)
:
self
.
log
.
debug
(
"monitor traffic: %r"
,
msg
[
0
]
)
switch
=
msg
[
0
]
try
:
idents
,
msg
=
self
.
session
.
feed_identities
(
msg
[
1
:
]
)
except
ValueError
:
idents
=
[
]
if
not
idents
:
self
.
log
.
error
(
"Monitor message without topic: %r"
,
msg
)
return
handler
=
self
.
monitor_handlers
.
get
(
switch
,
)
if
handler
is
not
:
handler
(
idents
,
msg
)
else
:
self
.
log
.
error
(
"Unrecognized monitor topic: %r"
,
switch
)

all ME and Task queue messages come through here as well as IOPub traffic .


def dispatch_query(self, msg):
        """Route registration requests and queries from clients."""
        try:
            idents, msg = self.session.feed_identities(msg)
        except ValueError:
            idents = []
        if not idents:
            self.log.error("Bad Query Message: %r", msg)
            return
        client_id = idents[0]
        try:
            msg = self.session.unserialize(msg, content=True)
        except Exception:
            content = error.wrap_exception()
            self.log.error("Bad Query Message: %r", msg, exc_info=True)
            self.session.send(self.query, "hub_error", ident=client_id,
                    content=content)
            return
        # print client_id, header, parent, content
        #switch on message type:
        msg_type = msg['header']['msg_type']
        self.log.info("client::client %r requested %r", client_id, msg_type)
        handler = self.query_handlers.get(msg_type, None)
        try:
            assert handler is not None, "Bad Message Type: %r" % msg_type
        except:
            content = error.wrap_exception()
            self.log.error("Bad Message Type: %r", msg_type, exc_info=True)
            self.session.send(self.query, "hub_error", ident=client_id,
                    content=content)
            return

        else:
            handler(idents, msg)

Route registration requests and queries from clients .


def handle_new_heart(self, heart):
        """handler to attach to heartbeater.
        Called when a new heart starts to beat.
        Triggers completion of registration."""
        self.log.debug("heartbeat::handle_new_heart(%r)", heart)
        if heart not in self.incoming_registrations:
            self.log.info("heartbeat::ignoring new heart: %r", heart)
        else:
            self.finish_registration(heart)

def
handle_new_heart
(
self
,
heart
)
:
self
.
log
.
debug
(
"heartbeat::handle_new_heart(%r)"
,
heart
)
if
heart
not
in
self
.
incoming_registrations
:
self
.
log
.
info
(
"heartbeat::ignoring new heart: %r"
,
heart
)
else
:
self
.
finish_registration
(
heart
)

handler to attach to heartbeater . Called when a new heart starts to beat . Triggers completion of registration .


def handle_heart_failure(self, heart):
        """handler to attach to heartbeater.
        called when a previously registered heart fails to respond to beat request.
        triggers unregistration"""
        self.log.debug("heartbeat::handle_heart_failure(%r)", heart)
        eid = self.hearts.get(heart, None)
        queue = self.engines[eid].queue
        if eid is None or self.keytable[eid] in self.dead_engines:
            self.log.info("heartbeat::ignoring heart failure %r (not an engine or already dead)", heart)
        else:
            self.unregister_engine(heart, dict(content=dict(id=eid, queue=queue)))

handler to attach to heartbeater . called when a previously registered heart fails to respond to beat request . triggers unregistration


def save_task_request(self, idents, msg):
        """Save the submission of a task."""
        client_id = idents[0]

        try:
            msg = self.session.unserialize(msg)
        except Exception:
            self.log.error("task::client %r sent invalid task message: %r",
                    client_id, msg, exc_info=True)
            return
        record = init_record(msg)

        record['client_uuid'] = client_id.decode('ascii')
        record['queue'] = 'task'
        header = msg['header']
        msg_id = header['msg_id']
        self.pending.add(msg_id)
        self.unassigned.add(msg_id)
        try:
            # it's posible iopub arrived first:
            existing = self.db.get_record(msg_id)
            if existing['resubmitted']:
                for key in ('submitted', 'client_uuid', 'buffers'):
                    # don't clobber these keys on resubmit
                    # submitted and client_uuid should be different
                    # and buffers might be big, and shouldn't have changed
                    record.pop(key)
                    # still check content,header which should not change
                    # but are not expensive to compare as buffers

            for key,evalue in existing.iteritems():
                if key.endswith('buffers'):
                    # don't compare buffers
                    continue
                rvalue = record.get(key, None)
                if evalue and rvalue and evalue != rvalue:
                    self.log.warn("conflicting initial state for record: %r:%r <%r> %r", msg_id, rvalue, key, evalue)
                elif evalue and not rvalue:
                    record[key] = evalue
            try:
                self.db.update_record(msg_id, record)
            except Exception:
                self.log.error("DB Error updating record %r", msg_id, exc_info=True)
        except KeyError:
            try:
                self.db.add_record(msg_id, record)
            except Exception:
                self.log.error("DB Error adding record %r", msg_id, exc_info=True)
        except Exception:
            self.log.error("DB Error saving task request %r", msg_id, exc_info=True)


def save_task_result(self, idents, msg):
        """save the result of a completed task."""
        client_id = idents[0]
        try:
            msg = self.session.unserialize(msg)
        except Exception:
            self.log.error("task::invalid task result message send to %r: %r",
                    client_id, msg, exc_info=True)
            return

        parent = msg['parent_header']
        if not parent:
            # print msg
            self.log.warn("Task %r had no parent!", msg)
            return
        msg_id = parent['msg_id']
        if msg_id in self.unassigned:
            self.unassigned.remove(msg_id)

        header = msg['header']
        engine_uuid = header.get('engine', u'')
        eid = self.by_ident.get(cast_bytes(engine_uuid), None)

        status = header.get('status', None)

        if msg_id in self.pending:
            self.log.info("task::task %r finished on %s", msg_id, eid)
            self.pending.remove(msg_id)
            self.all_completed.add(msg_id)
            if eid is not None:
                if status != 'aborted':
                    self.completed[eid].append(msg_id)
                if msg_id in self.tasks[eid]:
                    self.tasks[eid].remove(msg_id)
            completed = header['date']
            started = header.get('started', None)
            result = {
                'result_header' : header,
                'result_content': msg['content'],
                'started' : started,
                'completed' : completed,
                'received' : datetime.now(),
                'engine_uuid': engine_uuid,
            }

            result['result_buffers'] = msg['buffers']
            try:
                self.db.update_record(msg_id, result)
            except Exception:
                self.log.error("DB Error saving task request %r", msg_id, exc_info=True)

        else:
            self.log.debug("task::unknown task %r finished", msg_id)


def save_iopub_message(self, topics, msg):
        """save an iopub message into the db"""
        # print (topics)
        try:
            msg = self.session.unserialize(msg, content=True)
        except Exception:
            self.log.error("iopub::invalid IOPub message", exc_info=True)
            return

        parent = msg['parent_header']
        if not parent:
            self.log.warn("iopub::IOPub message lacks parent: %r", msg)
            return
        msg_id = parent['msg_id']
        msg_type = msg['header']['msg_type']
        content = msg['content']

        # ensure msg_id is in db
        try:
            rec = self.db.get_record(msg_id)
        except KeyError:
            rec = empty_record()
            rec['msg_id'] = msg_id
            self.db.add_record(msg_id, rec)
        # stream
        d = {}
        if msg_type == 'stream':
            name = content['name']
            s = rec[name] or ''
            d[name] = s + content['data']

        elif msg_type == 'pyerr':
            d['pyerr'] = content
        elif msg_type == 'pyin':
            d['pyin'] = content['code']
        elif msg_type in ('display_data', 'pyout'):
            d[msg_type] = content
        elif msg_type == 'status':
            pass
        else:
            self.log.warn("unhandled iopub msg_type: %r", msg_type)

        if not d:
            return

        try:
            self.db.update_record(msg_id, d)
        except Exception:
            self.log.error("DB Error saving iopub message %r", msg_id, exc_info=True)

def
save_iopub_message
(
self
,
topics
,
msg
)
:
# print (topics)
try
:
msg
=
self
.
session
.
unserialize
(
msg
,
content
=
True
)
except
Exception
:
self
.
log
.
error
(
"iopub::invalid IOPub message"
,
exc_info
=
True
)
return
parent
=
msg
[
'parent_header'
]
if
not
parent
:
self
.
log
.
warn
(
"iopub::IOPub message lacks parent: %r"
,
msg
)
return
msg_id
=
parent
[
'msg_id'
]
msg_type
=
msg
[
'header'
]
[
'msg_type'
]
content
=
msg
[
'content'
]
# ensure msg_id is in db
try
:
rec
=
self
.
db
.
get_record
(
msg_id
)
except
KeyError
:
rec
=
empty_record
(
)
rec
[
'msg_id'
]
=
msg_id
self
.
db
.
add_record
(
msg_id
,
rec
)
# stream
d
=
{
}
if
msg_type
==
'stream'
:
name
=
content
[
'name'
]
s
=
rec
[
name
]
or
''
d
[
name
]
=
s
+
content
[
'data'
]
elif
msg_type
==
'pyerr'
:
d
[
'pyerr'
]
=
content
elif
msg_type
==
'pyin'
:
d
[
'pyin'
]
=
content
[
'code'
]
elif
msg_type
in
(
'display_data'
,
'pyout'
)
:
d
[
msg_type
]
=
content
elif
msg_type
==
'status'
:
pass
else
:
self
.
log
.
warn
(
"unhandled iopub msg_type: %r"
,
msg_type
)
if
not
d
:
return
try
:
self
.
db
.
update_record
(
msg_id
,
d
)
except
Exception
:
self
.
log
.
error
(
"DB Error saving iopub message %r"
,
msg_id
,
exc_info
=
True
)


def connection_request(self, client_id, msg):
        """Reply with connection addresses for clients."""
        self.log.info("client::client %r connected", client_id)
        content = dict(status='ok')
        content.update(self.client_info)
        jsonable = {}
        for k,v in self.keytable.iteritems():
            if v not in self.dead_engines:
                jsonable[str(k)] = v.decode('ascii')
        content['engines'] = jsonable
        self.session.send(self.query, 'connection_reply', content, parent=msg, ident=client_id)

def
connection_request
(
self
,
client_id
,
msg
)
:
self
.
log
.
info
(
"client::client %r connected"
,
client_id
)
content
=
dict
(
status
=
'ok'
)
content
.
update
(
self
.
client_info
)
jsonable
=
{
}
for
k
,
v
in
self
.
keytable
.
iteritems
(
)
:
if
v
not
in
self
.
dead_engines
:
jsonable
[
str
(
k
)
]
=
v
.
decode
(
'ascii'
)
content
[
'engines'
]
=
jsonable
self
.
session
.
send
(
self
.
query
,
'connection_reply'
,
content
,
parent
=
msg
,
ident
=
client_id
)

Reply with connection addresses for clients .


def register_engine(self, reg, msg):
        """Register a new engine."""
        content = msg['content']
        try:
            queue = cast_bytes(content['queue'])
        except KeyError:
            self.log.error("registration::queue not specified", exc_info=True)
            return
        heart = content.get('heartbeat', None)
        if heart:
            heart = cast_bytes(heart)
        """register a new engine, and create the socket(s) necessary"""
        eid = self._next_id
        # print (eid, queue, reg, heart)

        self.log.debug("registration::register_engine(%i, %r, %r, %r)", eid, queue, reg, heart)

        content = dict(id=eid,status='ok')
        content.update(self.engine_info)
        # check if requesting available IDs:
        if queue in self.by_ident:
            try:
                raise KeyError("queue_id %r in use" % queue)
            except:
                content = error.wrap_exception()
                self.log.error("queue_id %r in use", queue, exc_info=True)
        elif heart in self.hearts: # need to check unique hearts?
            try:
                raise KeyError("heart_id %r in use" % heart)
            except:
                self.log.error("heart_id %r in use", heart, exc_info=True)
                content = error.wrap_exception()
        else:
            for h, pack in self.incoming_registrations.iteritems():
                if heart == h:
                    try:
                        raise KeyError("heart_id %r in use" % heart)
                    except:
                        self.log.error("heart_id %r in use", heart, exc_info=True)
                        content = error.wrap_exception()
                    break
                elif queue == pack[1]:
                    try:
                        raise KeyError("queue_id %r in use" % queue)
                    except:
                        self.log.error("queue_id %r in use", queue, exc_info=True)
                        content = error.wrap_exception()
                    break

        msg = self.session.send(self.query, "registration_reply",
                content=content,
                ident=reg)

        if content['status'] == 'ok':
            if heart in self.heartmonitor.hearts:
                # already beating
                self.incoming_registrations[heart] = (eid,queue,reg[0],None)
                self.finish_registration(heart)
            else:
                purge = lambda : self._purge_stalled_registration(heart)
                dc = ioloop.DelayedCallback(purge, self.registration_timeout, self.loop)
                dc.start()
                self.incoming_registrations[heart] = (eid,queue,reg[0],dc)
        else:
            self.log.error("registration::registration %i failed: %r", eid, content['evalue'])
        return eid


def unregister_engine(self, ident, msg):
        """Unregister an engine that explicitly requested to leave."""
        try:
            eid = msg['content']['id']
        except:
            self.log.error("registration::bad engine id for unregistration: %r", ident, exc_info=True)
            return
        self.log.info("registration::unregister_engine(%r)", eid)
        # print (eid)
        uuid = self.keytable[eid]
        content=dict(id=eid, queue=uuid.decode('ascii'))
        self.dead_engines.add(uuid)
        # self.ids.remove(eid)
        # uuid = self.keytable.pop(eid)
        #
        # ec = self.engines.pop(eid)
        # self.hearts.pop(ec.heartbeat)
        # self.by_ident.pop(ec.queue)
        # self.completed.pop(eid)
        handleit = lambda : self._handle_stranded_msgs(eid, uuid)
        dc = ioloop.DelayedCallback(handleit, self.registration_timeout, self.loop)
        dc.start()
        ############## TODO: HANDLE IT ################

        if self.notifier:
            self.session.send(self.notifier, "unregistration_notification", content=content)

Unregister an engine that explicitly requested to leave .


def finish_registration(self, heart):
        """Second half of engine registration, called after our HeartMonitor
        has received a beat from the Engine's Heart."""
        try:
            (eid,queue,reg,purge) = self.incoming_registrations.pop(heart)
        except KeyError:
            self.log.error("registration::tried to finish nonexistant registration", exc_info=True)
            return
        self.log.info("registration::finished registering engine %i:%r", eid, queue)
        if purge is not None:
            purge.stop()
        control = queue
        self.ids.add(eid)
        self.keytable[eid] = queue
        self.engines[eid] = EngineConnector(id=eid, queue=queue, registration=reg,
                                    control=control, heartbeat=heart)
        self.by_ident[queue] = eid
        self.queues[eid] = list()
        self.tasks[eid] = list()
        self.completed[eid] = list()
        self.hearts[heart] = eid
        content = dict(id=eid, queue=self.engines[eid].queue.decode('ascii'))
        if self.notifier:
            self.session.send(self.notifier, "registration_notification", content=content)
        self.log.info("engine::Engine Connected: %i", eid)

Second half of engine registration called after our HeartMonitor has received a beat from the Engine s Heart .


def shutdown_request(self, client_id, msg):
        """handle shutdown request."""
        self.session.send(self.query, 'shutdown_reply', content={'status': 'ok'}, ident=client_id)
        # also notify other clients of shutdown
        self.session.send(self.notifier, 'shutdown_notice', content={'status': 'ok'})
        dc = ioloop.DelayedCallback(lambda : self._shutdown(), 1000, self.loop)
        dc.start()

def
shutdown_request
(
self
,
client_id
,
msg
)
:
self
.
session
.
send
(
self
.
query
,
'shutdown_reply'
,
content
=
{
'status'
:
'ok'
}
,
ident
=
client_id
)
# also notify other clients of shutdown
self
.
session
.
send
(
self
.
notifier
,
'shutdown_notice'
,
content
=
{
'status'
:
'ok'
}
)
dc
=
ioloop
.
DelayedCallback
(
lambda
:
self
.
_shutdown
(
)
,
1000
,
self
.
loop
)
dc
.
start
(
)


def purge_results(self, client_id, msg):
        """Purge results from memory. This method is more valuable before we move
        to a DB based message storage mechanism."""
        content = msg['content']
        self.log.info("Dropping records with %s", content)
        msg_ids = content.get('msg_ids', [])
        reply = dict(status='ok')
        if msg_ids == 'all':
            try:
                self.db.drop_matching_records(dict(completed={'$ne':None}))
            except Exception:
                reply = error.wrap_exception()
        else:
            pending = filter(lambda m: m in self.pending, msg_ids)
            if pending:
                try:
                    raise IndexError("msg pending: %r" % pending[0])
                except:
                    reply = error.wrap_exception()
            else:
                try:
                    self.db.drop_matching_records(dict(msg_id={'$in':msg_ids}))
                except Exception:
                    reply = error.wrap_exception()

            if reply['status'] == 'ok':
                eids = content.get('engine_ids', [])
                for eid in eids:
                    if eid not in self.engines:
                        try:
                            raise IndexError("No such engine: %i" % eid)
                        except:
                            reply = error.wrap_exception()
                        break
                    uid = self.engines[eid].queue
                    try:
                        self.db.drop_matching_records(dict(engine_uuid=uid, completed={'$ne':None}))
                    except Exception:
                        reply = error.wrap_exception()
                        break

        self.session.send(self.query, 'purge_reply', content=reply, ident=client_id)

def
purge_results
(
self
,
client_id
,
msg
)
:
content
=
msg
[
'content'
]
self
.
log
.
info
(
"Dropping records with %s"
,
content
)
msg_ids
=
content
.
get
(
'msg_ids'
,
[
]
)
reply
=
dict
(
status
=
'ok'
)
if
msg_ids
==
'all'
:
try
:
self
.
db
.
drop_matching_records
(
dict
(
completed
=
{
'$ne'
:
}
)
)
except
Exception
:
reply
=
error
.
wrap_exception
(
)
else
:
pending
=
filter
(
lambda
m
:
m
in
self
.
pending
,
msg_ids
)
if
pending
:
try
:
raise
IndexError
(
"msg pending: %r"
%
pending
[
0
]
)
except
:
reply
=
error
.
wrap_exception
(
)
else
:
try
:
self
.
db
.
drop_matching_records
(
dict
(
msg_id
=
{
'$in'
:
msg_ids
}
)
)
except
Exception
:
reply
=
error
.
wrap_exception
(
)
if
reply
[
'status'
]
==
'ok'
:
eids
=
content
.
get
(
'engine_ids'
,
[
]
)
for
eid
in
eids
:
if
eid
not
in
self
.
engines
:
try
:
raise
IndexError
(
"No such engine: %i"
%
eid
)
except
:
reply
=
error
.
wrap_exception
(
)
break
uid
=
self
.
engines
[
eid
]
.
queue
try
:
self
.
db
.
drop_matching_records
(
dict
(
engine_uuid
=
uid
,
completed
=
{
'$ne'
:
}
)
)
except
Exception
:
reply
=
error
.
wrap_exception
(
)
break
self
.
session
.
send
(
self
.
query
,
'purge_reply'
,
content
=
reply
,
ident
=
client_id
)

Purge results from memory . This method is more valuable before we move to a DB based message storage mechanism .


def _extract_record(self, rec):
        """decompose a TaskRecord dict into subsection of reply for get_result"""
        io_dict = {}
        for key in ('pyin', 'pyout', 'pyerr', 'stdout', 'stderr'):
                io_dict[key] = rec[key]
        content = { 'result_content': rec['result_content'],
                            'header': rec['header'],
                            'result_header' : rec['result_header'],
                            'received' : rec['received'],
                            'io' : io_dict,
                          }
        if rec['result_buffers']:
            buffers = map(bytes, rec['result_buffers'])
        else:
            buffers = []

        return content, buffers

def _extract_record ( self , rec ) : io_dict = { } for key in ( 'pyin' , 'pyout' , 'pyerr' , 'stdout' , 'stderr' ) : io_dict [ key ] = rec [ key ] content = { 'result_content' : rec [ 'result_content' ] , 'header' : rec [ 'header' ] , 'result_header' : rec [ 'result_header' ] , 'received' : rec [ 'received' ] , 'io' : io_dict , } if rec [ 'result_buffers' ] : buffers = map ( bytes , rec [ 'result_buffers' ] ) else : buffers = [ ] return content , buffers


def get_results(self, client_id, msg):
        """Get the result of 1 or more messages."""
        content = msg['content']
        msg_ids = sorted(set(content['msg_ids']))
        statusonly = content.get('status_only', False)
        pending = []
        completed = []
        content = dict(status='ok')
        content['pending'] = pending
        content['completed'] = completed
        buffers = []
        if not statusonly:
            try:
                matches = self.db.find_records(dict(msg_id={'$in':msg_ids}))
                # turn match list into dict, for faster lookup
                records = {}
                for rec in matches:
                    records[rec['msg_id']] = rec
            except Exception:
                content = error.wrap_exception()
                self.session.send(self.query, "result_reply", content=content,
                                                    parent=msg, ident=client_id)
                return
        else:
            records = {}
        for msg_id in msg_ids:
            if msg_id in self.pending:
                pending.append(msg_id)
            elif msg_id in self.all_completed:
                completed.append(msg_id)
                if not statusonly:
                    c,bufs = self._extract_record(records[msg_id])
                    content[msg_id] = c
                    buffers.extend(bufs)
            elif msg_id in records:
                if rec['completed']:
                    completed.append(msg_id)
                    c,bufs = self._extract_record(records[msg_id])
                    content[msg_id] = c
                    buffers.extend(bufs)
                else:
                    pending.append(msg_id)
            else:
                try:
                    raise KeyError('No such message: '+msg_id)
                except:
                    content = error.wrap_exception()
                break
        self.session.send(self.query, "result_reply", content=content,
                                            parent=msg, ident=client_id,
                                            buffers=buffers)


def get_history(self, client_id, msg):
        """Get a list of all msg_ids in our DB records"""
        try:
            msg_ids = self.db.get_history()
        except Exception as e:
            content = error.wrap_exception()
        else:
            content = dict(status='ok', history=msg_ids)

        self.session.send(self.query, "history_reply", content=content,
                                            parent=msg, ident=client_id)

def get_history ( self , client_id , msg ) : try : msg_ids = self . db . get_history ( ) except Exception as e : content = error . wrap_exception ( ) else : content = dict ( status = 'ok' , history = msg_ids ) self . session . send ( self . query , "history_reply" , content = content , parent = msg , ident = client_id )


def db_query(self, client_id, msg):
        """Perform a raw query on the task record database."""
        content = msg['content']
        query = content.get('query', {})
        keys = content.get('keys', None)
        buffers = []
        empty = list()
        try:
            records = self.db.find_records(query, keys)
        except Exception as e:
            content = error.wrap_exception()
        else:
            # extract buffers from reply content:
            if keys is not None:
                buffer_lens = [] if 'buffers' in keys else None
                result_buffer_lens = [] if 'result_buffers' in keys else None
            else:
                buffer_lens = None
                result_buffer_lens = None

            for rec in records:
                # buffers may be None, so double check
                b = rec.pop('buffers', empty) or empty
                if buffer_lens is not None:
                    buffer_lens.append(len(b))
                    buffers.extend(b)
                rb = rec.pop('result_buffers', empty) or empty
                if result_buffer_lens is not None:
                    result_buffer_lens.append(len(rb))
                    buffers.extend(rb)
            content = dict(status='ok', records=records, buffer_lens=buffer_lens,
                                    result_buffer_lens=result_buffer_lens)
        # self.log.debug (content)
        self.session.send(self.query, "db_reply", content=content,
                                            parent=msg, ident=client_id,
                                            buffers=buffers)

def
db_query
(
self
,
client_id
,
msg
)
:
content
=
msg
[
'content'
]
query
=
content
.
get
(
'query'
,
{
}
)
keys
=
content
.
get
(
'keys'
,
)
buffers
=
[
]
empty
=
list
(
)
try
:
records
=
self
.
db
.
find_records
(
query
,
keys
)
except
Exception
as
e
:
content
=
error
.
wrap_exception
(
)
else
:
# extract buffers from reply content:
if
keys
is
not
:
buffer_lens
=
[
]
if
'buffers'
in
keys
else
result_buffer_lens
=
[
]
if
'result_buffers'
in
keys
else
else
:
buffer_lens
=
result_buffer_lens
=
for
rec
in
records
:
# buffers may be None, so double check
b
=
rec
.
pop
(
'buffers'
,
empty
)
or
empty
if
buffer_lens
is
not
:
buffer_lens
.
append
(
len
(
b
)
)
buffers
.
extend
(
b
)
rb
=
rec
.
pop
(
'result_buffers'
,
empty
)
or
empty
if
result_buffer_lens
is
not
:
result_buffer_lens
.
append
(
len
(
rb
)
)
buffers
.
extend
(
rb
)
content
=
dict
(
status
=
'ok'
,
records
=
records
,
buffer_lens
=
buffer_lens
,
result_buffer_lens
=
result_buffer_lens
)
# self.log.debug (content)
self
.
session
.
send
(
self
.
query
,
"db_reply"
,
content
=
content
,
parent
=
msg
,
ident
=
client_id
,
buffers
=
buffers
)

Perform a raw query on the task record database .


def cd(self, newdir):
        """
        go to the path
        """
        prevdir = os.getcwd()
        os.chdir(newdir)
        try:
            yield
        finally:
            os.chdir(prevdir)

def cd ( self , newdir ) : prevdir = os . getcwd ( ) os . chdir ( newdir ) try : yield finally : os . chdir ( prevdir )


def decode_cmd_out(self, completed_cmd):
        """
        return a standard message
        """
        try:
            stdout = completed_cmd.stdout.encode('utf-8').decode()
        except AttributeError:
            try:
                stdout = str(bytes(completed_cmd.stdout), 'big5').strip()
            except AttributeError:
                stdout = str(bytes(completed_cmd.stdout).decode('utf-8')).strip()
        try:
            stderr = completed_cmd.stderr.encode('utf-8').decode()
        except AttributeError:
            try:
                stderr = str(bytes(completed_cmd.stderr), 'big5').strip()
            except AttributeError:
                stderr = str(bytes(completed_cmd.stderr).decode('utf-8')).strip()
        return ParsedCompletedCommand(
            completed_cmd.returncode,
            completed_cmd.args,
            stdout,
            stderr
        )

def decode_cmd_out ( self , completed_cmd ) : try : stdout = completed_cmd . stdout . encode ( 'utf-8' ) . decode ( ) except AttributeError : try : stdout = str ( bytes ( completed_cmd . stdout ) , 'big5' ) . strip ( ) except AttributeError : stdout = str ( bytes ( completed_cmd . stdout ) . decode ( 'utf-8' ) ) . strip ( ) try : stderr = completed_cmd . stderr . encode ( 'utf-8' ) . decode ( ) except AttributeError : try : stderr = str ( bytes ( completed_cmd . stderr ) , 'big5' ) . strip ( ) except AttributeError : stderr = str ( bytes ( completed_cmd . stderr ) . decode ( 'utf-8' ) ) . strip ( ) return ParsedCompletedCommand ( completed_cmd . returncode , completed_cmd . args , stdout , stderr )


def run_command_under_r_root(self, cmd, catched=True):
        """
        subprocess run on here
        """
        RPATH = self.path
        with self.cd(newdir=RPATH):
            if catched:
                process = sp.run(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
            else:
                process = sp.run(cmd)
            return process

def run_command_under_r_root ( self , cmd , catched = True ) : RPATH = self . path with self . cd ( newdir = RPATH ) : if catched : process = sp . run ( cmd , stdout = sp . PIPE , stderr = sp . PIPE ) else : process = sp . run ( cmd ) return process


def execute(self):
        """
        Execute R script
        """
        rprocess = OrderedDict()
        commands = OrderedDict([
            (self.file, ['Rscript', self.file] + self.cmd),
        ])
        for cmd_name, cmd in commands.items():
            rprocess[cmd_name] = self.run_command_under_r_root(cmd)

        return self.decode_cmd_out(completed_cmd=rprocess[self.file])

def execute ( self ) : rprocess = OrderedDict ( ) commands = OrderedDict ( [ ( self . file , [ 'Rscript' , self . file ] + self . cmd ) , ] ) for cmd_name , cmd in commands . items ( ) : rprocess [ cmd_name ] = self . run_command_under_r_root ( cmd ) return self . decode_cmd_out ( completed_cmd = rprocess [ self . file ] )


def _dispatch(self, msg):
        """ Calls the frontend handler associated with the message type of the
            given message.
        """
        msg_type = msg['header']['msg_type']
        handler = getattr(self, '_handle_' + msg_type, None)
        if handler:
            handler(msg)

def _dispatch ( self , msg ) : msg_type = msg [ 'header' ] [ 'msg_type' ] handler = getattr ( self , '_handle_' + msg_type , None ) if handler : handler ( msg )

Calls the frontend handler associated with the message type of the given message .


def _is_from_this_session(self, msg):
        """ Returns whether a reply from the kernel originated from a request
            from this frontend.
        """
        session = self._kernel_manager.session.session
        parent = msg['parent_header']
        if not parent:
            # if the message has no parent, assume it is meant for all frontends
            return True
        else:
            return parent.get('session') == session

Returns whether a reply from the kernel originated from a request from this frontend .


def report(self, morfs, directory=None):
        """Run the report.

        See `coverage.report()` for arguments.

        """
        self.report_files(self.annotate_file, morfs, directory)

def report ( self , morfs , directory = None ) : self . report_files ( self . annotate_file , morfs , directory )


def annotate_file(self, cu, analysis):
        """Annotate a single file.

        `cu` is the CodeUnit for the file to annotate.

        """
        if not cu.relative:
            return

        filename = cu.filename
        source = cu.source_file()
        if self.directory:
            dest_file = os.path.join(self.directory, cu.flat_rootname())
            dest_file += ".py,cover"
        else:
            dest_file = filename + ",cover"
        dest = open(dest_file, 'w')

        statements = sorted(analysis.statements)
        missing = sorted(analysis.missing)
        excluded = sorted(analysis.excluded)

        lineno = 0
        i = 0
        j = 0
        covered = True
        while True:
            line = source.readline()
            if line == '':
                break
            lineno += 1
            while i < len(statements) and statements[i] < lineno:
                i += 1
            while j < len(missing) and missing[j] < lineno:
                j += 1
            if i < len(statements) and statements[i] == lineno:
                covered = j >= len(missing) or missing[j] > lineno
            if self.blank_re.match(line):
                dest.write('  ')
            elif self.else_re.match(line):
                # Special logic for lines containing only 'else:'.
                if i >= len(statements) and j >= len(missing):
                    dest.write('! ')
                elif i >= len(statements) or j >= len(missing):
                    dest.write('> ')
                elif statements[i] == missing[j]:
                    dest.write('! ')
                else:
                    dest.write('> ')
            elif lineno in excluded:
                dest.write('- ')
            elif covered:
                dest.write('> ')
            else:
                dest.write('! ')
            dest.write(line)
        source.close()
        dest.close()


def get_installed_version(name):
    '''
        returns installed package version and None if package is not installed
    '''
    pattern = re.compile(r'''Installed:\s+(?P<version>.*)''')
    cmd = 'apt-cache policy %s' % name
    args = shlex.split(cmd)
    try:
        output = subprocess.check_output(args)
        if not output:
            return None
    except CalledProcessError:
        return None
    # check output
    match = pattern.search(output)
    if match:
        version = match.groupdict()['version']
        if version == '(none)':
            return None
        else:
            return version

def
get_installed_version
(
name
)
:
pattern
=
re
.
compile
(
r'''Installed:\s+(?P<version>.*)'''
)
cmd
=
'apt-cache policy %s'
%
name
args
=
shlex
.
split
(
cmd
)
try
:
output
=
subprocess
.
check_output
(
args
)
if
not
output
:
return
except
CalledProcessError
:
return
# check output
match
=
pattern
.
search
(
output
)
if
match
:
version
=
match
.
groupdict
(
)
[
'version'
]
if
version
==
'(none)'
:
return
else
:
return
version


def squash_unicode(obj):
    """coerce unicode back to bytestrings."""
    if isinstance(obj,dict):
        for key in obj.keys():
            obj[key] = squash_unicode(obj[key])
            if isinstance(key, unicode):
                obj[squash_unicode(key)] = obj.pop(key)
    elif isinstance(obj, list):
        for i,v in enumerate(obj):
            obj[i] = squash_unicode(v)
    elif isinstance(obj, unicode):
        obj = obj.encode('utf8')
    return obj

def squash_unicode ( obj ) : if isinstance ( obj , dict ) : for key in obj . keys ( ) : obj [ key ] = squash_unicode ( obj [ key ] ) if isinstance ( key , unicode ) : obj [ squash_unicode ( key ) ] = obj . pop ( key ) elif isinstance ( obj , list ) : for i , v in enumerate ( obj ) : obj [ i ] = squash_unicode ( v ) elif isinstance ( obj , unicode ) : obj = obj . encode ( 'utf8' ) return obj


def extract_header(msg_or_header):
    """Given a message or header, return the header."""
    if not msg_or_header:
        return {}
    try:
        # See if msg_or_header is the entire message.
        h = msg_or_header['header']
    except KeyError:
        try:
            # See if msg_or_header is just the header
            h = msg_or_header['msg_id']
        except KeyError:
            raise
        else:
            h = msg_or_header
    if not isinstance(h, dict):
        h = dict(h)
    return h

Given a message or header return the header .


def _check_packers(self):
        """check packers for binary data and datetime support."""
        pack = self.pack
        unpack = self.unpack

        # check simple serialization
        msg = dict(a=[1,'hi'])
        try:
            packed = pack(msg)
        except Exception:
            raise ValueError("packer could not serialize a simple message")

        # ensure packed message is bytes
        if not isinstance(packed, bytes):
            raise ValueError("message packed to %r, but bytes are required"%type(packed))

        # check that unpack is pack's inverse
        try:
            unpacked = unpack(packed)
        except Exception:
            raise ValueError("unpacker could not handle the packer's output")

        # check datetime support
        msg = dict(t=datetime.now())
        try:
            unpacked = unpack(pack(msg))
        except Exception:
            self.pack = lambda o: pack(squash_dates(o))
            self.unpack = lambda s: extract_dates(unpack(s))

check packers for binary data and datetime support .


def msg(self, msg_type, content=None, parent=None, subheader=None, header=None):
        """Return the nested message dict.

        This format is different from what is sent over the wire. The
        serialize/unserialize methods converts this nested message dict to the wire
        format, which is a list of message parts.
        """
        msg = {}
        header = self.msg_header(msg_type) if header is None else header
        msg['header'] = header
        msg['msg_id'] = header['msg_id']
        msg['msg_type'] = header['msg_type']
        msg['parent_header'] = {} if parent is None else extract_header(parent)
        msg['content'] = {} if content is None else content
        sub = {} if subheader is None else subheader
        msg['header'].update(sub)
        return msg

def msg ( self , msg_type , content = None , parent = None , subheader = None , header = None ) : msg = { } header = self . msg_header ( msg_type ) if header is None else header msg [ 'header' ] = header msg [ 'msg_id' ] = header [ 'msg_id' ] msg [ 'msg_type' ] = header [ 'msg_type' ] msg [ 'parent_header' ] = { } if parent is None else extract_header ( parent ) msg [ 'content' ] = { } if content is None else content sub = { } if subheader is None else subheader msg [ 'header' ] . update ( sub ) return msg


def sign(self, msg_list):
        """Sign a message with HMAC digest. If no auth, return b''.

        Parameters
        ----------
        msg_list : list
            The [p_header,p_parent,p_content] part of the message list.
        """
        if self.auth is None:
            return b''
        h = self.auth.copy()
        for m in msg_list:
            h.update(m)
        return str_to_bytes(h.hexdigest())

def sign ( self , msg_list ) : if self . auth is None : return b'' h = self . auth . copy ( ) for m in msg_list : h . update ( m ) return str_to_bytes ( h . hexdigest ( ) )

Sign a message with HMAC digest . If no auth return b .


def serialize(self, msg, ident=None):
        """Serialize the message components to bytes.

        This is roughly the inverse of unserialize. The serialize/unserialize
        methods work with full message lists, whereas pack/unpack work with
        the individual message parts in the message list.

        Parameters
        ----------
        msg : dict or Message
            The nexted message dict as returned by the self.msg method.

        Returns
        -------
        msg_list : list
            The list of bytes objects to be sent with the format:
            [ident1,ident2,...,DELIM,HMAC,p_header,p_parent,p_content,
             buffer1,buffer2,...]. In this list, the p_* entities are
            the packed or serialized versions, so if JSON is used, these
            are utf8 encoded JSON strings.
        """
        content = msg.get('content', {})
        if content is None:
            content = self.none
        elif isinstance(content, dict):
            content = self.pack(content)
        elif isinstance(content, bytes):
            # content is already packed, as in a relayed message
            pass
        elif isinstance(content, unicode):
            # should be bytes, but JSON often spits out unicode
            content = content.encode('utf8')
        else:
            raise TypeError("Content incorrect type: %s"%type(content))

        real_message = [self.pack(msg['header']),
                        self.pack(msg['parent_header']),
                        content
        ]

        to_send = []

        if isinstance(ident, list):
            # accept list of idents
            to_send.extend(ident)
        elif ident is not None:
            to_send.append(ident)
        to_send.append(DELIM)

        signature = self.sign(real_message)
        to_send.append(signature)

        to_send.extend(real_message)

        return to_send

Serialize the message components to bytes .