def set_pkg_source_info(self, doc, text):
        """Sets the package's source information, if not already set.
        text - Free form text.
        Raises CardinalityError if already defined.
        Raises OrderError if no package previously defined.
        """
        self.assert_package_exists()
        if not self.package_source_info_set:
            self.package_source_info_set = True
            doc.package.source_info = text
            return True
        else:
            raise CardinalityError('Package::SourceInfo')

def set_pkg_source_info ( self , doc , text ) : self . assert_package_exists ( ) if not self . package_source_info_set : self . package_source_info_set = True doc . package . source_info = text return True else : raise CardinalityError ( 'Package::SourceInfo' )

Sets the package s source information if not already set . text - Free form text . Raises CardinalityError if already defined . Raises OrderError if no package previously defined .


def set_pkg_verif_code(self, doc, code):
        """Sets the package verification code, if not already set.
        code - A string.
        Raises CardinalityError if already defined.
        Raises OrderError if no package previously defined.
        """
        self.assert_package_exists()
        if not self.package_verif_set:
            self.package_verif_set = True
            doc.package.verif_code = code
        else:
            raise CardinalityError('Package::VerificationCode')

def set_pkg_verif_code ( self , doc , code ) : self . assert_package_exists ( ) if not self . package_verif_set : self . package_verif_set = True doc . package . verif_code = code else : raise CardinalityError ( 'Package::VerificationCode' )

Sets the package verification code if not already set . code - A string . Raises CardinalityError if already defined . Raises OrderError if no package previously defined .


def set_pkg_excl_file(self, doc, filename):
        """Sets the package's verification code excluded file.
        Raises OrderError if no package previously defined.
        """
        self.assert_package_exists()
        doc.package.add_exc_file(filename)

def set_pkg_excl_file ( self , doc , filename ) : self . assert_package_exists ( ) doc . package . add_exc_file ( filename )

Sets the package s verification code excluded file . Raises OrderError if no package previously defined .


def set_pkg_summary(self, doc, text):
        """Set's the package summary.
        Raises CardinalityError if summary already set.
        Raises OrderError if no package previously defined.
        """
        self.assert_package_exists()
        if not self.package_summary_set:
            self.package_summary_set = True
            doc.package.summary = text
        else:
            raise CardinalityError('Package::Summary')

def set_pkg_summary ( self , doc , text ) : self . assert_package_exists ( ) if not self . package_summary_set : self . package_summary_set = True doc . package . summary = text else : raise CardinalityError ( 'Package::Summary' )

Set s the package summary . Raises CardinalityError if summary already set . Raises OrderError if no package previously defined .


def set_pkg_desc(self, doc, text):
        """Set's the package's description.
        Raises CardinalityError if description already set.
        Raises OrderError if no package previously defined.
        """
        self.assert_package_exists()
        if not self.package_desc_set:
            self.package_desc_set = True
            doc.package.description = text
        else:
            raise CardinalityError('Package::Description')

def set_pkg_desc ( self , doc , text ) : self . assert_package_exists ( ) if not self . package_desc_set : self . package_desc_set = True doc . package . description = text else : raise CardinalityError ( 'Package::Description' )

Set s the package s description . Raises CardinalityError if description already set . Raises OrderError if no package previously defined .


def set_file_chksum(self, doc, chk_sum):
        """Sets the file check sum, if not already set.
        chk_sum - A string
        Raises CardinalityError if already defined.
        Raises OrderError if no package previously defined.
        """
        if self.has_package(doc) and self.has_file(doc):
            if not self.file_chksum_set:
                self.file_chksum_set = True
                self.file(doc).chk_sum = checksum.Algorithm('SHA1', chk_sum)
                return True
            else:
                raise CardinalityError('File::CheckSum')
        else:
            raise OrderError('File::CheckSum')

def set_file_chksum ( self , doc , chk_sum ) : if self . has_package ( doc ) and self . has_file ( doc ) : if not self . file_chksum_set : self . file_chksum_set = True self . file ( doc ) . chk_sum = checksum . Algorithm ( 'SHA1' , chk_sum ) return True else : raise CardinalityError ( 'File::CheckSum' ) else : raise OrderError ( 'File::CheckSum' )

Sets the file check sum if not already set . chk_sum - A string Raises CardinalityError if already defined . Raises OrderError if no package previously defined .


def set_file_license_comment(self, doc, text):
        """
        Raises OrderError if no package or file defined.
        Raises CardinalityError if more than one per file.
        """
        if self.has_package(doc) and self.has_file(doc):
            if not self.file_license_comment_set:
                self.file_license_comment_set = True
                self.file(doc).license_comment = text
                return True
            else:
                raise CardinalityError('File::LicenseComment')
        else:
            raise OrderError('File::LicenseComment')

def set_file_license_comment ( self , doc , text ) : if self . has_package ( doc ) and self . has_file ( doc ) : if not self . file_license_comment_set : self . file_license_comment_set = True self . file ( doc ) . license_comment = text return True else : raise CardinalityError ( 'File::LicenseComment' ) else : raise OrderError ( 'File::LicenseComment' )

Raises OrderError if no package or file defined . Raises CardinalityError if more than one per file .


def set_file_comment(self, doc, text):
        """Raises OrderError if no package or no file defined.
        Raises CardinalityError if more than one comment set.
        """
        if self.has_package(doc) and self.has_file(doc):
            if not self.file_comment_set:
                self.file_comment_set = True
                self.file(doc).comment = text
                return True
            else:
                raise CardinalityError('File::Comment')
        else:
            raise OrderError('File::Comment')

def set_file_comment ( self , doc , text ) : if self . has_package ( doc ) and self . has_file ( doc ) : if not self . file_comment_set : self . file_comment_set = True self . file ( doc ) . comment = text return True else : raise CardinalityError ( 'File::Comment' ) else : raise OrderError ( 'File::Comment' )

Raises OrderError if no package or no file defined . Raises CardinalityError if more than one comment set .


def add_review_comment(self, doc, comment):
        """Sets the review comment. Raises CardinalityError if
        already set. OrderError if no reviewer defined before.
        """
        if len(doc.reviews) != 0:
            if not self.review_comment_set:
                self.review_comment_set = True
                doc.reviews[-1].comment = comment
                return True
            else:
                raise CardinalityError('ReviewComment')
        else:
            raise OrderError('ReviewComment')

def add_review_comment ( self , doc , comment ) : if len ( doc . reviews ) != 0 : if not self . review_comment_set : self . review_comment_set = True doc . reviews [ - 1 ] . comment = comment return True else : raise CardinalityError ( 'ReviewComment' ) else : raise OrderError ( 'ReviewComment' )

Sets the review comment . Raises CardinalityError if already set . OrderError if no reviewer defined before .


def add_annotation_comment(self, doc, comment):
        """Sets the annotation comment. Raises CardinalityError if
        already set. OrderError if no annotator defined before.
        """
        if len(doc.annotations) != 0:
            if not self.annotation_comment_set:
                self.annotation_comment_set = True
                doc.annotations[-1].comment = comment
                return True
            else:
                raise CardinalityError('AnnotationComment')
        else:
            raise OrderError('AnnotationComment')

def add_annotation_comment ( self , doc , comment ) : if len ( doc . annotations ) != 0 : if not self . annotation_comment_set : self . annotation_comment_set = True doc . annotations [ - 1 ] . comment = comment return True else : raise CardinalityError ( 'AnnotationComment' ) else : raise OrderError ( 'AnnotationComment' )

Sets the annotation comment . Raises CardinalityError if already set . OrderError if no annotator defined before .


def add_annotation_type(self, doc, annotation_type):
        """Sets the annotation type. Raises CardinalityError if
        already set. OrderError if no annotator defined before.
        """
        if len(doc.annotations) != 0:
            if not self.annotation_type_set:
                if annotation_type.endswith('annotationType_other'):
                    self.annotation_type_set = True
                    doc.annotations[-1].annotation_type = 'OTHER'
                    return True
                elif annotation_type.endswith('annotationType_review'):
                    self.annotation_type_set = True
                    doc.annotations[-1].annotation_type = 'REVIEW'
                    return True
                else:
                    raise SPDXValueError('Annotation::AnnotationType')
            else:
                raise CardinalityError('Annotation::AnnotationType')
        else:
            raise OrderError('Annotation::AnnotationType')

def add_annotation_type ( self , doc , annotation_type ) : if len ( doc . annotations ) != 0 : if not self . annotation_type_set : if annotation_type . endswith ( 'annotationType_other' ) : self . annotation_type_set = True doc . annotations [ - 1 ] . annotation_type = 'OTHER' return True elif annotation_type . endswith ( 'annotationType_review' ) : self . annotation_type_set = True doc . annotations [ - 1 ] . annotation_type = 'REVIEW' return True else : raise SPDXValueError ( 'Annotation::AnnotationType' ) else : raise CardinalityError ( 'Annotation::AnnotationType' ) else : raise OrderError ( 'Annotation::AnnotationType' )

Sets the annotation type . Raises CardinalityError if already set . OrderError if no annotator defined before .


def validate(self, messages):
        """
        Validate all fields of the document and update the
        messages list with user friendly error messages for display.
        """
        messages = self.validate_version(messages)
        messages = self.validate_data_lics(messages)
        messages = self.validate_name(messages)
        messages = self.validate_spdx_id(messages)
        messages = self.validate_namespace(messages)
        messages = self.validate_ext_document_references(messages)
        messages = self.validate_creation_info(messages)
        messages = self.validate_package(messages)
        messages = self.validate_extracted_licenses(messages)
        messages = self.validate_reviews(messages)

        return messages

def validate ( self , messages ) : messages = self . validate_version ( messages ) messages = self . validate_data_lics ( messages ) messages = self . validate_name ( messages ) messages = self . validate_spdx_id ( messages ) messages = self . validate_namespace ( messages ) messages = self . validate_ext_document_references ( messages ) messages = self . validate_creation_info ( messages ) messages = self . validate_package ( messages ) messages = self . validate_extracted_licenses ( messages ) messages = self . validate_reviews ( messages ) return messages

Validate all fields of the document and update the messages list with user friendly error messages for display .


def include(f):
  '''
  includes the contents of a file on disk.
  takes a filename
  '''
  fl = open(f, 'r')
  data = fl.read()
  fl.close()
  return raw(data)

def include ( f ) : fl = open ( f , 'r' ) data = fl . read ( ) fl . close ( ) return raw ( data )

includes the contents of a file on disk . takes a filename


def system(cmd, data=None):
  '''
  pipes the output of a program
  '''
  import subprocess
  s = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
  out, err = s.communicate(data)
  return out.decode('utf8')

def system ( cmd , data = None ) : import subprocess s = subprocess . Popen ( cmd , shell = True , stdout = subprocess . PIPE , stdin = subprocess . PIPE ) out , err = s . communicate ( data ) return out . decode ( 'utf8' )


def unescape(data):
  '''
  unescapes html entities. the opposite of escape.
  '''
  cc = re.compile(r'&(?:(?:#(\d+))|([^;]+));')

  result = []
  m = cc.search(data)
  while m:
    result.append(data[0:m.start()])
    d = m.group(1)
    if d:
      d = int(d)
      result.append(unichr(d))
    else:
      d = _unescape.get(m.group(2), ord('?'))
      result.append(unichr(d))

    data = data[m.end():]
    m = cc.search(data)

  result.append(data)
  return ''.join(result)

def unescape ( data ) : cc = re . compile ( r'&(?:(?:#(\d+))|([^;]+));' ) result = [ ] m = cc . search ( data ) while m : result . append ( data [ 0 : m . start ( ) ] ) d = m . group ( 1 ) if d : d = int ( d ) result . append ( unichr ( d ) ) else : d = _unescape . get ( m . group ( 2 ) , ord ( '?' ) ) result . append ( unichr ( d ) ) data = data [ m . end ( ) : ] m = cc . search ( data ) result . append ( data ) return '' . join ( result )

unescapes html entities . the opposite of escape .


def attr(*args, **kwargs):
  '''
  Set attributes on the current active tag context
  '''
  ctx = dom_tag._with_contexts[_get_thread_context()]
  if ctx and ctx[-1]:
    dicts = args + (kwargs,)
    for d in dicts:
      for attr, value in d.items():
        ctx[-1].tag.set_attribute(*dom_tag.clean_pair(attr, value))
  else:
    raise ValueError('not in a tag context')

def
attr
(
*
args
,
*
*
kwargs
)
:
ctx
=
dom_tag
.
_with_contexts
[
_get_thread_context
(
)
]
if
ctx
and
ctx
[
1
]
:
dicts
=
args
+
(
kwargs
,
)
for
d
in
dicts
:
for
attr
,
value
in
d
.
items
(
)
:
ctx
[
1
]
.
tag
.
set_attribute
(
*
dom_tag
.
clean_pair
(
attr
,
value
)
)
else
:
raise
ValueError
(
'not in a tag context'
)


def set_attribute(self, key, value):
    '''
    Add or update the value of an attribute.
    '''
    if isinstance(key, int):
      self.children[key] = value
    elif isinstance(key, basestring):
      self.attributes[key] = value
    else:
      raise TypeError('Only integer and string types are valid for assigning '
          'child tags and attributes, respectively.')

Add or update the value of an attribute .


def setdocument(self, doc):
    '''
    Creates a reference to the parent document to allow for partial-tree
    validation.
    '''
    # assume that a document is correct in the subtree
    if self.document != doc:
      self.document = doc
      for i in self.children:
        if not isinstance(i, dom_tag): return
        i.setdocument(doc)

Creates a reference to the parent document to allow for partial - tree validation .


def add(self, *args):
    '''
    Add new child tags.
    '''
    for obj in args:
      if isinstance(obj, numbers.Number):
        # Convert to string so we fall into next if block
        obj = str(obj)

      if isinstance(obj, basestring):
        obj = escape(obj)
        self.children.append(obj)

      elif isinstance(obj, dom_tag):
        ctx = dom_tag._with_contexts[_get_thread_context()]
        if ctx and ctx[-1]:
          ctx[-1].used.add(obj)
        self.children.append(obj)
        obj.parent = self
        obj.setdocument(self.document)

      elif isinstance(obj, dict):
        for attr, value in obj.items():
          self.set_attribute(*dom_tag.clean_pair(attr, value))

      elif hasattr(obj, '__iter__'):
        for subobj in obj:
          self.add(subobj)

      else:  # wtf is it?
        raise ValueError('%r not a tag or string.' % obj)

    if len(args) == 1:
      return args[0]

    return args


def get(self, tag=None, **kwargs):
    '''
    Recursively searches children for tags of a certain
    type with matching attributes.
    '''
    # Stupid workaround since we can not use dom_tag in the method declaration
    if tag is None: tag = dom_tag

    attrs = [(dom_tag.clean_attribute(attr), value)
        for attr, value in kwargs.items()]

    results = []
    for child in self.children:
      if (isinstance(tag, basestring) and type(child).__name__ == tag) or \
        (not isinstance(tag, basestring) and isinstance(child, tag)):

        if all(child.attributes.get(attribute) == value
            for attribute, value in attrs):
          # If the child is of correct type and has all attributes and values
          # in kwargs add as a result
          results.append(child)
      if isinstance(child, dom_tag):
        # If the child is a dom_tag extend the search down through its children
        results.extend(child.get(tag, **kwargs))
    return results

Recursively searches children for tags of a certain type with matching attributes .


def clean_attribute(attribute):
    '''
    Normalize attribute names for shorthand and work arounds for limitations
    in Python's syntax
    '''

    # Shorthand
    attribute = {
      'cls': 'class',
      'className': 'class',
      'class_name': 'class',
      'fr': 'for',
      'html_for': 'for',
      'htmlFor': 'for',
    }.get(attribute, attribute)

    # Workaround for Python's reserved words
    if attribute[0] == '_':
      attribute = attribute[1:]

    # Workaround for dash
    if attribute in set(['http_equiv']) or attribute.startswith('data_'):
      attribute = attribute.replace('_', '-').lower()

    # Workaround for colon
    if attribute.split('_')[0] in ('xlink', 'xml', 'xmlns'):
      attribute = attribute.replace('_', ':', 1).lower()

    return attribute

Normalize attribute names for shorthand and work arounds for limitations in Python s syntax


def clean_pair(cls, attribute, value):
    '''
    This will call `clean_attribute` on the attribute and also allows for the
    creation of boolean attributes.

    Ex. input(selected=True) is equivalent to input(selected="selected")
    '''
    attribute = cls.clean_attribute(attribute)

    # Check for boolean attributes
    # (i.e. selected=True becomes selected="selected")
    if value is True:
      value = attribute

    if value is False:
      value = "false"

    return (attribute, value)

This will call clean_attribute on the attribute and also allows for the creation of boolean attributes .


def create_reg_message(self):
        """ Creates a registration message to identify the worker to the interchange
        """
        msg = {'parsl_v': PARSL_VERSION,
               'python_v': "{}.{}.{}".format(sys.version_info.major,
                                             sys.version_info.minor,
                                             sys.version_info.micro),
               'os': platform.system(),
               'hname': platform.node(),
               'dir': os.getcwd(),
        }
        b_msg = json.dumps(msg).encode('utf-8')
        return b_msg

def create_reg_message ( self ) : msg = { 'parsl_v' : PARSL_VERSION , 'python_v' : "{}.{}.{}" . format ( sys . version_info . major , sys . version_info . minor , sys . version_info . micro ) , 'os' : platform . system ( ) , 'hname' : platform . node ( ) , 'dir' : os . getcwd ( ) , } b_msg = json . dumps ( msg ) . encode ( 'utf-8' ) return b_msg


def heartbeat(self):
        """ Send heartbeat to the incoming task queue
        """
        heartbeat = (HEARTBEAT_CODE).to_bytes(4, "little")
        r = self.task_incoming.send(heartbeat)
        logger.debug("Return from heartbeat : {}".format(r))

def
heartbeat
(
self
)
:
heartbeat
=
(
HEARTBEAT_CODE
)
.
to_bytes
(
4
,
"little"
)
r
=
self
.
task_incoming
.
send
(
heartbeat
)
logger
.
debug
(
"Return from heartbeat : {}"
.
format
(
r
)
)


def recv_result_from_workers(self):
        """ Receives a results from the MPI worker pool and send it out via 0mq

        Returns:
        --------
            result: task result from the workers
        """
        info = MPI.Status()
        result = self.comm.recv(source=MPI.ANY_SOURCE, tag=RESULT_TAG, status=info)
        logger.debug("Received result from workers: {}".format(result))
        return result

def
recv_result_from_workers
(
self
)
:
info
=
MPI
.
Status
(
)
result
=
self
.
comm
.
recv
(
source
=
MPI
.
ANY_SOURCE
,
tag
=
RESULT_TAG
,
status
=
info
)
logger
.
debug
(
"Received result from workers: {}"
.
format
(
result
)
)
return
result


def recv_task_request_from_workers(self):
        """ Receives 1 task request from MPI comm

        Returns:
        --------
            worker_rank: worker_rank id
        """
        info = MPI.Status()
        comm.recv(source=MPI.ANY_SOURCE, tag=TASK_REQUEST_TAG, status=info)
        worker_rank = info.Get_source()
        logger.info("Received task request from worker:{}".format(worker_rank))
        return worker_rank

def
recv_task_request_from_workers
(
self
)
:
info
=
MPI
.
Status
(
)
comm
.
recv
(
source
=
MPI
.
ANY_SOURCE
,
tag
=
TASK_REQUEST_TAG
,
status
=
info
)
worker_rank
=
info
.
Get_source
(
)
logger
.
info
(
"Received task request from worker:{}"
.
format
(
worker_rank
)
)
return
worker_rank


def pull_tasks(self, kill_event):
        """ Pulls tasks from the incoming tasks 0mq pipe onto the internal
        pending task queue

        Parameters:
        -----------
        kill_event : threading.Event
              Event to let the thread know when it is time to die.
        """
        logger.info("[TASK PULL THREAD] starting")
        poller = zmq.Poller()
        poller.register(self.task_incoming, zmq.POLLIN)

        # Send a registration message
        msg = self.create_reg_message()
        logger.debug("Sending registration message: {}".format(msg))
        self.task_incoming.send(msg)
        last_beat = time.time()
        last_interchange_contact = time.time()
        task_recv_counter = 0

        poll_timer = 1

        while not kill_event.is_set():
            time.sleep(LOOP_SLOWDOWN)
            ready_worker_count = self.ready_worker_queue.qsize()
            pending_task_count = self.pending_task_queue.qsize()

            logger.debug("[TASK_PULL_THREAD] ready workers:{}, pending tasks:{}".format(ready_worker_count,
                                                                                pending_task_count))

            if time.time() > last_beat + self.heartbeat_period:
                self.heartbeat()
                last_beat = time.time()

            if pending_task_count < self.max_queue_size and ready_worker_count > 0:
                logger.debug("[TASK_PULL_THREAD] Requesting tasks: {}".format(ready_worker_count))
                msg = ((ready_worker_count).to_bytes(4, "little"))
                self.task_incoming.send(msg)

            socks = dict(poller.poll(timeout=poll_timer))

            if self.task_incoming in socks and socks[self.task_incoming] == zmq.POLLIN:
                _, pkl_msg = self.task_incoming.recv_multipart()
                tasks = pickle.loads(pkl_msg)
                last_interchange_contact = time.time()

                if tasks == 'STOP':
                    logger.critical("[TASK_PULL_THREAD] Received stop request")
                    kill_event.set()
                    break

                elif tasks == HEARTBEAT_CODE:
                    logger.debug("Got heartbeat from interchange")

                else:
                    # Reset timer on receiving message
                    poll_timer = 1
                    task_recv_counter += len(tasks)
                    logger.debug("[TASK_PULL_THREAD] Got tasks: {} of {}".format([t['task_id'] for t in tasks],
                                                                                task_recv_counter))
                    for task in tasks:
                        self.pending_task_queue.put(task)
            else:
                logger.debug("[TASK_PULL_THREAD] No incoming tasks")
                # Limit poll duration to heartbeat_period
                # heartbeat_period is in s vs poll_timer in ms
                poll_timer = min(self.heartbeat_period * 1000, poll_timer * 2)

                # Only check if no messages were received.
                if time.time() > last_interchange_contact + self.heartbeat_threshold:
                    logger.critical("[TASK_PULL_THREAD] Missing contact with interchange beyond heartbeat_threshold")
                    kill_event.set()
                    logger.critical("[TASK_PULL_THREAD] Exiting")
                    break


def start(self):
        """ Start the Manager process.

        The worker loops on this:

        1. If the last message sent was older than heartbeat period we send a heartbeat
        2.

        TODO: Move task receiving to a thread
        """

        self.comm.Barrier()
        logger.debug("Manager synced with workers")

        self._kill_event = threading.Event()
        self._task_puller_thread = threading.Thread(target=self.pull_tasks,
                                                    args=(self._kill_event,))
        self._result_pusher_thread = threading.Thread(target=self.push_results,
                                                      args=(self._kill_event,))
        self._task_puller_thread.start()
        self._result_pusher_thread.start()

        start = None

        result_counter = 0
        task_recv_counter = 0
        task_sent_counter = 0

        logger.info("Loop start")
        while not self._kill_event.is_set():
            time.sleep(LOOP_SLOWDOWN)

            # In this block we attempt to probe MPI for a set amount of time,
            # and if we have exhausted all available MPI events, we move on
            # to the next block. The timer and counter trigger balance
            # fairness and responsiveness.
            timer = time.time() + 0.05
            counter = min(10, comm.size)
            while time.time() < timer:
                info = MPI.Status()

                if counter > 10:
                    logger.debug("Hit max mpi events per round")
                    break

                if not self.comm.Iprobe(status=info):
                    logger.debug("Timer expired, processed {} mpi events".format(counter))
                    break
                else:
                    tag = info.Get_tag()
                    logger.info("Message with tag {} received".format(tag))

                    counter += 1
                    if tag == RESULT_TAG:
                        result = self.recv_result_from_workers()
                        self.pending_result_queue.put(result)
                        result_counter += 1

                    elif tag == TASK_REQUEST_TAG:
                        worker_rank = self.recv_task_request_from_workers()
                        self.ready_worker_queue.put(worker_rank)

                    else:
                        logger.error("Unknown tag {} - ignoring this message and continuing".format(tag))

            available_worker_cnt = self.ready_worker_queue.qsize()
            available_task_cnt = self.pending_task_queue.qsize()
            logger.debug("[MAIN] Ready workers: {} Ready tasks: {}".format(available_worker_cnt,
                                                                           available_task_cnt))
            this_round = min(available_worker_cnt, available_task_cnt)
            for i in range(this_round):
                worker_rank = self.ready_worker_queue.get()
                task = self.pending_task_queue.get()
                comm.send(task, dest=worker_rank, tag=worker_rank)
                task_sent_counter += 1
                logger.debug("Assigning worker:{} task:{}".format(worker_rank, task['task_id']))

            if not start:
                start = time.time()

            logger.debug("Tasks recvd:{} Tasks dispatched:{} Results recvd:{}".format(
                task_recv_counter, task_sent_counter, result_counter))
            # print("[{}] Received: {}".format(self.identity, msg))
            # time.sleep(random.randint(4,10)/10)

        self._task_puller_thread.join()
        self._result_pusher_thread.join()

        self.task_incoming.close()
        self.result_outgoing.close()
        self.context.term()

        delta = time.time() - start
        logger.info("mpi_worker_pool ran for {} seconds".format(delta))


def async_process(fn):
    """ Decorator function to launch a function as a separate process """

    def run(*args, **kwargs):
        proc = mp.Process(target=fn, args=args, kwargs=kwargs)
        proc.start()
        return proc

    return run

def async_process ( fn ) : def run ( * args , * * kwargs ) : proc = mp . Process ( target = fn , args = args , kwargs = kwargs ) proc . start ( ) return proc return run


def udp_messenger(domain_name, UDP_IP, UDP_PORT, sock_timeout, message):
    """Send UDP messages to usage tracker asynchronously

    This multiprocessing based messenger was written to overcome the limitations
    of signalling/terminating a thread that is blocked on a system call. This
    messenger is created as a separate process, and initialized with 2 queues,
    to_send to receive messages to be sent to the internet.

    Args:
          - domain_name (str) : Domain name string
          - UDP_IP (str) : IP address YYY.YYY.YYY.YYY
          - UDP_PORT (int) : UDP port to send out on
          - sock_timeout (int) : Socket timeout
          - to_send (multiprocessing.Queue) : Queue of outgoing messages to internet
    """
    try:
        if message is None:
            raise ValueError("message was none")

        encoded_message = bytes(message, "utf-8")

        if encoded_message is None:
            raise ValueError("utf-8 encoding of message failed")

        if domain_name:
            try:
                UDP_IP = socket.gethostbyname(domain_name)
            except Exception:
                # (False, "Domain lookup failed, defaulting to {0}".format(UDP_IP))
                pass

        if UDP_IP is None:
            raise Exception("UDP_IP is None")

        if UDP_PORT is None:
            raise Exception("UDP_PORT is None")

        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)  # UDP
        sock.settimeout(sock_timeout)
        sock.sendto(bytes(message, "utf-8"), (UDP_IP, UDP_PORT))
        sock.close()

    except socket.timeout:
        logger.debug("Failed to send usage tracking data: socket timeout")
    except OSError as e:
        logger.debug("Failed to send usage tracking data: OSError: {}".format(e))
    except Exception as e:
        logger.debug("Failed to send usage tracking data: Exception: {}".format(e))


def check_tracking_enabled(self):
        """By default tracking is enabled.

        If Test mode is set via env variable PARSL_TESTING, a test flag is set

        Tracking is disabled if :
            1. config["globals"]["usageTracking"] is set to False (Bool)
            2. Environment variable PARSL_TRACKING is set to false (case insensitive)

        """
        track = True   # By default we track usage
        test = False  # By default we are not in testing mode

        testvar = str(os.environ.get("PARSL_TESTING", 'None')).lower()
        if testvar == 'true':
            test = True

        if not self.config.usage_tracking:
            track = False

        envvar = str(os.environ.get("PARSL_TRACKING", True)).lower()
        if envvar == "false":
            track = False

        return test, track

def
check_tracking_enabled
(
self
)
:
track
=
True
# By default we track usage
test
=
False
# By default we are not in testing mode
testvar
=
str
(
os
.
environ
.
get
(
"PARSL_TESTING"
,
'None'
)
)
.
lower
(
)
if
testvar
==
'true'
:
test
=
True
if
not
self
.
config
.
usage_tracking
:
track
=
False
envvar
=
str
(
os
.
environ
.
get
(
"PARSL_TRACKING"
,
True
)
)
.
lower
(
)
if
envvar
==
"false"
:
track
=
False
return
test
,
track


def construct_start_message(self):
        """Collect preliminary run info at the start of the DFK.

        Returns :
              - Message dict dumped as json string, ready for UDP
        """
        uname = getpass.getuser().encode('latin1')
        hashed_username = hashlib.sha256(uname).hexdigest()[0:10]
        hname = socket.gethostname().encode('latin1')
        hashed_hostname = hashlib.sha256(hname).hexdigest()[0:10]
        message = {'uuid': self.uuid,
                   'uname': hashed_username,
                   'hname': hashed_hostname,
                   'test': self.test_mode,
                   'parsl_v': self.parsl_version,
                   'python_v': self.python_version,
                   'os': platform.system(),
                   'os_v': platform.release(),
                   'start': time.time()}

        return json.dumps(message)

def construct_start_message ( self ) : uname = getpass . getuser ( ) . encode ( 'latin1' ) hashed_username = hashlib . sha256 ( uname ) . hexdigest ( ) [ 0 : 10 ] hname = socket . gethostname ( ) . encode ( 'latin1' ) hashed_hostname = hashlib . sha256 ( hname ) . hexdigest ( ) [ 0 : 10 ] message = { 'uuid' : self . uuid , 'uname' : hashed_username , 'hname' : hashed_hostname , 'test' : self . test_mode , 'parsl_v' : self . parsl_version , 'python_v' : self . python_version , 'os' : platform . system ( ) , 'os_v' : platform . release ( ) , 'start' : time . time ( ) } return json . dumps ( message )

Collect preliminary run info at the start of the DFK .


def construct_end_message(self):
        """Collect the final run information at the time of DFK cleanup.

        Returns:
             - Message dict dumped as json string, ready for UDP
        """
        app_count = self.dfk.task_count

        site_count = len([x for x in self.dfk.config.executors if x.managed])

        app_fails = len([t for t in self.dfk.tasks if
                         self.dfk.tasks[t]['status'] in FINAL_FAILURE_STATES])

        message = {'uuid': self.uuid,
                   'end': time.time(),
                   't_apps': app_count,
                   'sites': site_count,
                   'c_time': None,
                   'failed': app_fails,
                   'test': self.test_mode,
                   }

        return json.dumps(message)

def construct_end_message ( self ) : app_count = self . dfk . task_count site_count = len ( [ x for x in self . dfk . config . executors if x . managed ] ) app_fails = len ( [ t for t in self . dfk . tasks if self . dfk . tasks [ t ] [ 'status' ] in FINAL_FAILURE_STATES ] ) message = { 'uuid' : self . uuid , 'end' : time . time ( ) , 't_apps' : app_count , 'sites' : site_count , 'c_time' : None , 'failed' : app_fails , 'test' : self . test_mode , } return json . dumps ( message )

Collect the final run information at the time of DFK cleanup .


def send_UDP_message(self, message):
        """Send UDP message."""
        x = 0
        if self.tracking_enabled:
            try:
                proc = udp_messenger(self.domain_name, self.UDP_IP, self.UDP_PORT, self.sock_timeout, message)
                self.procs.append(proc)
            except Exception as e:
                logger.debug("Usage tracking failed: {}".format(e))
        else:
            x = -1

        return x

def
send_UDP_message
(
self
,
message
)
:
x
=
0
if
self
.
tracking_enabled
:
try
:
proc
=
udp_messenger
(
self
.
domain_name
,
self
.
UDP_IP
,
self
.
UDP_PORT
,
self
.
sock_timeout
,
message
)
self
.
procs
.
append
(
proc
)
except
Exception
as
e
:
logger
.
debug
(
"Usage tracking failed: {}"
.
format
(
e
)
)
else
:
x
=
1
return
x


def send_message(self):
        """Send message over UDP.

        If tracking is disables, the bytes_sent will always be set to -1

        Returns:
            (bytes_sent, time_taken)
        """
        start = time.time()
        message = None
        if not self.initialized:
            message = self.construct_start_message()
            self.initialized = True
        else:
            message = self.construct_end_message()

        self.send_UDP_message(message)
        end = time.time()

        return end - start

def send_message ( self ) : start = time . time ( ) message = None if not self . initialized : message = self . construct_start_message ( ) self . initialized = True else : message = self . construct_end_message ( ) self . send_UDP_message ( message ) end = time . time ( ) return end - start


def dbm_starter(priority_msgs, resource_msgs, *args, **kwargs):
    """Start the database manager process

    The DFK should start this function. The args, kwargs match that of the monitoring config

    """
    dbm = DatabaseManager(*args, **kwargs)
    dbm.start(priority_msgs, resource_msgs)

def dbm_starter ( priority_msgs , resource_msgs , * args , * * kwargs ) : dbm = DatabaseManager ( * args , * * kwargs ) dbm . start ( priority_msgs , resource_msgs )


def _create_task_log_info(self, task_id, fail_mode=None):
        """
        Create the dictionary that will be included in the log.
        """

        info_to_monitor = ['func_name', 'fn_hash', 'memoize', 'checkpoint', 'fail_count',
                           'fail_history', 'status', 'id', 'time_submitted', 'time_returned', 'executor']

        task_log_info = {"task_" + k: self.tasks[task_id][k] for k in info_to_monitor}
        task_log_info['run_id'] = self.run_id
        task_log_info['timestamp'] = datetime.datetime.now()
        task_log_info['task_status_name'] = self.tasks[task_id]['status'].name
        task_log_info['tasks_failed_count'] = self.tasks_failed_count
        task_log_info['tasks_completed_count'] = self.tasks_completed_count
        task_log_info['task_inputs'] = str(self.tasks[task_id]['kwargs'].get('inputs', None))
        task_log_info['task_outputs'] = str(self.tasks[task_id]['kwargs'].get('outputs', None))
        task_log_info['task_stdin'] = self.tasks[task_id]['kwargs'].get('stdin', None)
        task_log_info['task_stdout'] = self.tasks[task_id]['kwargs'].get('stdout', None)
        task_log_info['task_depends'] = None
        if self.tasks[task_id]['depends'] is not None:
            task_log_info['task_depends'] = ",".join([str(t._tid) for t in self.tasks[task_id]['depends']])
        task_log_info['task_elapsed_time'] = None
        if self.tasks[task_id]['time_returned'] is not None:
            task_log_info['task_elapsed_time'] = (self.tasks[task_id]['time_returned'] -
                                                  self.tasks[task_id]['time_submitted']).total_seconds()
        if fail_mode is not None:
            task_log_info['task_fail_mode'] = fail_mode
        return task_log_info

def _create_task_log_info ( self , task_id , fail_mode = None ) : info_to_monitor = [ 'func_name' , 'fn_hash' , 'memoize' , 'checkpoint' , 'fail_count' , 'fail_history' , 'status' , 'id' , 'time_submitted' , 'time_returned' , 'executor' ] task_log_info = { "task_" + k : self . tasks [ task_id ] [ k ] for k in info_to_monitor } task_log_info [ 'run_id' ] = self . run_id task_log_info [ 'timestamp' ] = datetime . datetime . now ( ) task_log_info [ 'task_status_name' ] = self . tasks [ task_id ] [ 'status' ] . name task_log_info [ 'tasks_failed_count' ] = self . tasks_failed_count task_log_info [ 'tasks_completed_count' ] = self . tasks_completed_count task_log_info [ 'task_inputs' ] = str ( self . tasks [ task_id ] [ 'kwargs' ] . get ( 'inputs' , None ) ) task_log_info [ 'task_outputs' ] = str ( self . tasks [ task_id ] [ 'kwargs' ] . get ( 'outputs' , None ) ) task_log_info [ 'task_stdin' ] = self . tasks [ task_id ] [ 'kwargs' ] . get ( 'stdin' , None ) task_log_info [ 'task_stdout' ] = self . tasks [ task_id ] [ 'kwargs' ] . get ( 'stdout' , None ) task_log_info [ 'task_depends' ] = None if self . tasks [ task_id ] [ 'depends' ] is not None : task_log_info [ 'task_depends' ] = "," . join ( [ str ( t . _tid ) for t in self . tasks [ task_id ] [ 'depends' ] ] ) task_log_info [ 'task_elapsed_time' ] = None if self . tasks [ task_id ] [ 'time_returned' ] is not None : task_log_info [ 'task_elapsed_time' ] = ( self . tasks [ task_id ] [ 'time_returned' ] - self . tasks [ task_id ] [ 'time_submitted' ] ) . total_seconds ( ) if fail_mode is not None : task_log_info [ 'task_fail_mode' ] = fail_mode return task_log_info

Create the dictionary that will be included in the log .


def handle_app_update(self, task_id, future, memo_cbk=False):
        """This function is called as a callback when an AppFuture
        is in its final state.

        It will trigger post-app processing such as checkpointing
        and stageout.

        Args:
             task_id (string) : Task id
             future (Future) : The relevant app future (which should be
                 consistent with the task structure 'app_fu' entry

        KWargs:
             memo_cbk(Bool) : Indicates that the call is coming from a memo update,
             that does not require additional memo updates.
        """

        if not self.tasks[task_id]['app_fu'].done():
            logger.error("Internal consistency error: app_fu is not done for task {}".format(task_id))
        if not self.tasks[task_id]['app_fu'] == future:
            logger.error("Internal consistency error: callback future is not the app_fu in task structure, for task {}".format(task_id))

        if not memo_cbk:
            # Update the memoizer with the new result if this is not a
            # result from a memo lookup and the task has reached a terminal state.
            self.memoizer.update_memo(task_id, self.tasks[task_id], future)

            if self.checkpoint_mode == 'task_exit':
                self.checkpoint(tasks=[task_id])

        # Submit _*_stage_out tasks for output data futures that correspond with remote files
        if (self.tasks[task_id]['app_fu'] and
            self.tasks[task_id]['app_fu'].done() and
            self.tasks[task_id]['app_fu'].exception() is None and
            self.tasks[task_id]['executor'] != 'data_manager' and
            self.tasks[task_id]['func_name'] != '_ftp_stage_in' and
            self.tasks[task_id]['func_name'] != '_http_stage_in'):
            for dfu in self.tasks[task_id]['app_fu'].outputs:
                f = dfu.file_obj
                if isinstance(f, File) and f.is_remote():
                    self.data_manager.stage_out(f, self.tasks[task_id]['executor'])

        return

This function is called as a callback when an AppFuture is in its final state .


def launch_task(self, task_id, executable, *args, **kwargs):
        """Handle the actual submission of the task to the executor layer.

        If the app task has the executors attributes not set (default=='all')
        the task is launched on a randomly selected executor from the
        list of executors. This behavior could later be updated to support
        binding to executors based on user specified criteria.

        If the app task specifies a particular set of executors, it will be
        targeted at those specific executors.

        Args:
            task_id (uuid string) : A uuid string that uniquely identifies the task
            executable (callable) : A callable object
            args (list of positional args)
            kwargs (arbitrary keyword arguments)

        Returns:
            Future that tracks the execution of the submitted executable
        """
        self.tasks[task_id]['time_submitted'] = datetime.datetime.now()

        hit, memo_fu = self.memoizer.check_memo(task_id, self.tasks[task_id])
        if hit:
            logger.info("Reusing cached result for task {}".format(task_id))
            return memo_fu

        executor_label = self.tasks[task_id]["executor"]
        try:
            executor = self.executors[executor_label]
        except Exception:
            logger.exception("Task {} requested invalid executor {}: config is\n{}".format(task_id, executor_label, self._config))

        if self.monitoring is not None and self.monitoring.resource_monitoring_enabled:
            executable = self.monitoring.monitor_wrapper(executable, task_id,
                                                         self.monitoring.monitoring_hub_url,
                                                         self.run_id,
                                                         self.monitoring.resource_monitoring_interval)

        with self.submitter_lock:
            exec_fu = executor.submit(executable, *args, **kwargs)
        self.tasks[task_id]['status'] = States.launched
        if self.monitoring is not None:
            task_log_info = self._create_task_log_info(task_id, 'lazy')
            self.monitoring.send(MessageType.TASK_INFO, task_log_info)

        exec_fu.retries_left = self._config.retries - \
            self.tasks[task_id]['fail_count']
        logger.info("Task {} launched on executor {}".format(task_id, executor.label))
        return exec_fu

Handle the actual submission of the task to the executor layer .


def _add_input_deps(self, executor, args, kwargs):
        """Look for inputs of the app that are remote files. Submit stage_in
        apps for such files and replace the file objects in the inputs list with
        corresponding DataFuture objects.

        Args:
            - executor (str) : executor where the app is going to be launched
            - args (List) : Positional args to app function
            - kwargs (Dict) : Kwargs to app function
        """

        # Return if the task is _*_stage_in
        if executor == 'data_manager':
            return args, kwargs

        inputs = kwargs.get('inputs', [])
        for idx, f in enumerate(inputs):
            if isinstance(f, File) and f.is_remote():
                inputs[idx] = self.data_manager.stage_in(f, executor)

        for kwarg, f in kwargs.items():
            if isinstance(f, File) and f.is_remote():
                kwargs[kwarg] = self.data_manager.stage_in(f, executor)

        newargs = list(args)
        for idx, f in enumerate(newargs):
            if isinstance(f, File) and f.is_remote():
                newargs[idx] = self.data_manager.stage_in(f, executor)

        return tuple(newargs), kwargs

def
_add_input_deps
(
self
,
executor
,
args
,
kwargs
)
:
# Return if the task is _*_stage_in
if
executor
==
'data_manager'
:
return
args
,
kwargs
inputs
=
kwargs
.
get
(
'inputs'
,
[
]
)
for
idx
,
f
in
enumerate
(
inputs
)
:
if
isinstance
(
f
,
File
)
and
f
.
is_remote
(
)
:
inputs
[
idx
]
=
self
.
data_manager
.
stage_in
(
f
,
executor
)
for
kwarg
,
f
in
kwargs
.
items
(
)
:
if
isinstance
(
f
,
File
)
and
f
.
is_remote
(
)
:
kwargs
[
kwarg
]
=
self
.
data_manager
.
stage_in
(
f
,
executor
)
newargs
=
list
(
args
)
for
idx
,
f
in
enumerate
(
newargs
)
:
if
isinstance
(
f
,
File
)
and
f
.
is_remote
(
)
:
newargs
[
idx
]
=
self
.
data_manager
.
stage_in
(
f
,
executor
)
return
tuple
(
newargs
)
,
kwargs

Look for inputs of the app that are remote files . Submit stage_in apps for such files and replace the file objects in the inputs list with corresponding DataFuture objects .


def _gather_all_deps(self, args, kwargs):
        """Count the number of unresolved futures on which a task depends.

        Args:
            - args (List[args]) : The list of args list to the fn
            - kwargs (Dict{kwargs}) : The dict of all kwargs passed to the fn

        Returns:
            - count, [list of dependencies]

        """
        # Check the positional args
        depends = []
        count = 0
        for dep in args:
            if isinstance(dep, Future):
                if self.tasks[dep.tid]['status'] not in FINAL_STATES:
                    count += 1
                depends.extend([dep])

        # Check for explicit kwargs ex, fu_1=<fut>
        for key in kwargs:
            dep = kwargs[key]
            if isinstance(dep, Future):
                if self.tasks[dep.tid]['status'] not in FINAL_STATES:
                    count += 1
                depends.extend([dep])

        # Check for futures in inputs=[<fut>...]
        for dep in kwargs.get('inputs', []):
            if isinstance(dep, Future):
                if self.tasks[dep.tid]['status'] not in FINAL_STATES:
                    count += 1
                depends.extend([dep])

        return count, depends

Count the number of unresolved futures on which a task depends .


def submit(self, func, *args, executors='all', fn_hash=None, cache=False, **kwargs):
        """Add task to the dataflow system.

        If the app task has the executors attributes not set (default=='all')
        the task will be launched on a randomly selected executor from the
        list of executors. If the app task specifies a particular set of
        executors, it will be targeted at the specified executors.

        >>> IF all deps are met:
        >>>   send to the runnable queue and launch the task
        >>> ELSE:
        >>>   post the task in the pending queue

        Args:
            - func : A function object
            - *args : Args to the function

        KWargs :
            - executors (list or string) : List of executors this call could go to.
                    Default='all'
            - fn_hash (Str) : Hash of the function and inputs
                    Default=None
            - cache (Bool) : To enable memoization or not
            - kwargs (dict) : Rest of the kwargs to the fn passed as dict.

        Returns:
               (AppFuture) [DataFutures,]

        """

        if self.cleanup_called:
            raise ValueError("Cannot submit to a DFK that has been cleaned up")

        task_id = self.task_count
        self.task_count += 1
        if isinstance(executors, str) and executors.lower() == 'all':
            choices = list(e for e in self.executors if e != 'data_manager')
        elif isinstance(executors, list):
            choices = executors
        executor = random.choice(choices)

        # Transform remote input files to data futures
        args, kwargs = self._add_input_deps(executor, args, kwargs)

        task_def = {'depends': None,
                    'executor': executor,
                    'func': func,
                    'func_name': func.__name__,
                    'args': args,
                    'kwargs': kwargs,
                    'fn_hash': fn_hash,
                    'memoize': cache,
                    'callback': None,
                    'exec_fu': None,
                    'checkpoint': None,
                    'fail_count': 0,
                    'fail_history': [],
                    'env': None,
                    'status': States.unsched,
                    'id': task_id,
                    'time_submitted': None,
                    'time_returned': None,
                    'app_fu': None}

        if task_id in self.tasks:
            raise DuplicateTaskError(
                "internal consistency error: Task {0} already exists in task list".format(task_id))
        else:
            self.tasks[task_id] = task_def

        # Get the dep count and a list of dependencies for the task
        dep_cnt, depends = self._gather_all_deps(args, kwargs)
        self.tasks[task_id]['depends'] = depends

        # Extract stdout and stderr to pass to AppFuture:
        task_stdout = kwargs.get('stdout')
        task_stderr = kwargs.get('stderr')

        logger.info("Task {} submitted for App {}, waiting on tasks {}".format(task_id,
                                                                               task_def['func_name'],
                                                                               [fu.tid for fu in depends]))

        self.tasks[task_id]['task_launch_lock'] = threading.Lock()
        app_fu = AppFuture(tid=task_id,
                           stdout=task_stdout,
                           stderr=task_stderr)

        self.tasks[task_id]['app_fu'] = app_fu
        app_fu.add_done_callback(partial(self.handle_app_update, task_id))
        self.tasks[task_id]['status'] = States.pending
        logger.debug("Task {} set to pending state with AppFuture: {}".format(task_id, task_def['app_fu']))

        # at this point add callbacks to all dependencies to do a launch_if_ready
        # call whenever a dependency completes.

        # we need to be careful about the order of setting the state to pending,
        # adding the callbacks, and caling launch_if_ready explicitly once always below.

        # I think as long as we call launch_if_ready once after setting pending, then
        # we can add the callback dependencies at any point: if the callbacks all fire
        # before then, they won't cause a launch, but the one below will. if they fire
        # after we set it pending, then the last one will cause a launch, and the
        # explicit one won't.

        for d in depends:

            def callback_adapter(dep_fut):
                self.launch_if_ready(task_id)

            try:
                d.add_done_callback(callback_adapter)
            except Exception as e:
                logger.error("add_done_callback got an exception {} which will be ignored".format(e))

        self.launch_if_ready(task_id)

        return task_def['app_fu']


def cleanup(self):
        """DataFlowKernel cleanup.

        This involves killing resources explicitly and sending die messages to IPP workers.

        If the executors are managed (created by the DFK), then we call scale_in on each of
        the executors and call executor.shutdown. Otherwise, we do nothing, and executor
        cleanup is left to the user.
        """
        logger.info("DFK cleanup initiated")

        # this check won't detect two DFK cleanups happening from
        # different threads extremely close in time because of
        # non-atomic read/modify of self.cleanup_called
        if self.cleanup_called:
            raise Exception("attempt to clean up DFK when it has already been cleaned-up")
        self.cleanup_called = True

        self.log_task_states()

        # Checkpointing takes priority over the rest of the tasks
        # checkpoint if any valid checkpoint method is specified
        if self.checkpoint_mode is not None:
            self.checkpoint()

            if self._checkpoint_timer:
                logger.info("Stopping checkpoint timer")
                self._checkpoint_timer.close()

        # Send final stats
        self.usage_tracker.send_message()
        self.usage_tracker.close()

        logger.info("Terminating flow_control and strategy threads")
        self.flowcontrol.close()

        for executor in self.executors.values():
            if executor.managed:
                if executor.scaling_enabled:
                    job_ids = executor.provider.resources.keys()
                    executor.scale_in(len(job_ids))
                executor.shutdown()

        self.time_completed = datetime.datetime.now()

        if self.monitoring:
            self.monitoring.send(MessageType.WORKFLOW_INFO,
                                 {'tasks_failed_count': self.tasks_failed_count,
                                  'tasks_completed_count': self.tasks_completed_count,
                                  "time_began": self.time_began,
                                  'time_completed': self.time_completed,
                                  'workflow_duration': (self.time_completed - self.time_began).total_seconds(),
                                  'run_id': self.run_id, 'rundir': self.run_dir})

            self.monitoring.close()

        """
        if self.logging_server is not None:
            self.logging_server.terminate()
            self.logging_server.join()

        if self.web_app is not None:
            self.web_app.terminate()
            self.web_app.join()
        """
        logger.info("DFK cleanup complete")


def checkpoint(self, tasks=None):
        """Checkpoint the dfk incrementally to a checkpoint file.

        When called, every task that has been completed yet not
        checkpointed is checkpointed to a file.

        Kwargs:
            - tasks (List of task ids) : List of task ids to checkpoint. Default=None
                                         if set to None, we iterate over all tasks held by the DFK.

        .. note::
            Checkpointing only works if memoization is enabled

        Returns:
            Checkpoint dir if checkpoints were written successfully.
            By default the checkpoints are written to the RUNDIR of the current
            run under RUNDIR/checkpoints/{tasks.pkl, dfk.pkl}
        """
        with self.checkpoint_lock:
            checkpoint_queue = None
            if tasks:
                checkpoint_queue = tasks
            else:
                checkpoint_queue = self.tasks

            checkpoint_dir = '{0}/checkpoint'.format(self.run_dir)
            checkpoint_dfk = checkpoint_dir + '/dfk.pkl'
            checkpoint_tasks = checkpoint_dir + '/tasks.pkl'

            if not os.path.exists(checkpoint_dir):
                try:
                    os.makedirs(checkpoint_dir)
                except FileExistsError:
                    pass

            with open(checkpoint_dfk, 'wb') as f:
                state = {'rundir': self.run_dir,
                         'task_count': self.task_count
                         }
                pickle.dump(state, f)

            count = 0

            with open(checkpoint_tasks, 'ab') as f:
                for task_id in checkpoint_queue:
                    if not self.tasks[task_id]['checkpoint'] and \
                       self.tasks[task_id]['app_fu'].done() and \
                       self.tasks[task_id]['app_fu'].exception() is None:
                        hashsum = self.tasks[task_id]['hashsum']
                        if not hashsum:
                            continue
                        t = {'hash': hashsum,
                             'exception': None,
                             'result': None}
                        try:
                            # Asking for the result will raise an exception if
                            # the app had failed. Should we even checkpoint these?
                            # TODO : Resolve this question ?
                            r = self.memoizer.hash_lookup(hashsum).result()
                        except Exception as e:
                            t['exception'] = e
                        else:
                            t['result'] = r

                        # We are using pickle here since pickle dumps to a file in 'ab'
                        # mode behave like a incremental log.
                        pickle.dump(t, f)
                        count += 1
                        self.tasks[task_id]['checkpoint'] = True
                        logger.debug("Task {} checkpointed".format(task_id))

            self.checkpointed_tasks += count

            if count == 0:
                if self.checkpointed_tasks == 0:
                    logger.warn("No tasks checkpointed so far in this run. Please ensure caching is enabled")
                else:
                    logger.debug("No tasks checkpointed in this pass.")
            else:
                logger.info("Done checkpointing {} tasks".format(count))

            return checkpoint_dir

Checkpoint the dfk incrementally to a checkpoint file .


def _load_checkpoints(self, checkpointDirs):
        """Load a checkpoint file into a lookup table.

        The data being loaded from the pickle file mostly contains input
        attributes of the task: func, args, kwargs, env...
        To simplify the check of whether the exact task has been completed
        in the checkpoint, we hash these input params and use it as the key
        for the memoized lookup table.

        Args:
            - checkpointDirs (list) : List of filepaths to checkpoints
              Eg. ['runinfo/001', 'runinfo/002']

        Returns:
            - memoized_lookup_table (dict)
        """
        memo_lookup_table = {}

        for checkpoint_dir in checkpointDirs:
            logger.info("Loading checkpoints from {}".format(checkpoint_dir))
            checkpoint_file = os.path.join(checkpoint_dir, 'tasks.pkl')
            try:
                with open(checkpoint_file, 'rb') as f:
                    while True:
                        try:
                            data = pickle.load(f)
                            # Copy and hash only the input attributes
                            memo_fu = Future()
                            if data['exception']:
                                memo_fu.set_exception(data['exception'])
                            else:
                                memo_fu.set_result(data['result'])
                            memo_lookup_table[data['hash']] = memo_fu

                        except EOFError:
                            # Done with the checkpoint file
                            break
            except FileNotFoundError:
                reason = "Checkpoint file was not found: {}".format(
                    checkpoint_file)
                logger.error(reason)
                raise BadCheckpoint(reason)
            except Exception:
                reason = "Failed to load checkpoint: {}".format(
                    checkpoint_file)
                logger.error(reason)
                raise BadCheckpoint(reason)

            logger.info("Completed loading checkpoint:{0} with {1} tasks".format(checkpoint_file,
                                                                                len(memo_lookup_table.keys())))
        return memo_lookup_table

Load a checkpoint file into a lookup table .


def load_checkpoints(self, checkpointDirs):
        """Load checkpoints from the checkpoint files into a dictionary.

        The results are used to pre-populate the memoizer's lookup_table

        Kwargs:
             - checkpointDirs (list) : List of run folder to use as checkpoints
               Eg. ['runinfo/001', 'runinfo/002']

        Returns:
             - dict containing, hashed -> future mappings
        """
        self.memo_lookup_table = None

        if not checkpointDirs:
            return {}

        if type(checkpointDirs) is not list:
            raise BadCheckpoint("checkpointDirs expects a list of checkpoints")

        return self._load_checkpoints(checkpointDirs)

Load checkpoints from the checkpoint files into a dictionary .


def load(cls, config: Optional[Config] = None):
        """Load a DataFlowKernel.

        Args:
            - config (Config) : Configuration to load. This config will be passed to a
              new DataFlowKernel instantiation which will be set as the active DataFlowKernel.
        Returns:
            - DataFlowKernel : The loaded DataFlowKernel object.
        """
        if cls._dfk is not None:
            raise RuntimeError('Config has already been loaded')

        if config is None:
            cls._dfk = DataFlowKernel(Config())
        else:
            cls._dfk = DataFlowKernel(config)

        return cls._dfk

def
load
(
cls
,
config
:
Optional
[
Config
]
=
)
:
if
cls
.
_dfk
is
not
:
raise
RuntimeError
(
'Config has already been loaded'
)
if
config
is
:
cls
.
_dfk
=
DataFlowKernel
(
Config
(
)
)
else
:
cls
.
_dfk
=
DataFlowKernel
(
config
)
return
cls
.
_dfk


def get_tasks(self, count):
        """ Obtains a batch of tasks from the internal pending_task_queue

        Parameters
        ----------
        count: int
            Count of tasks to get from the queue

        Returns
        -------
        List of upto count tasks. May return fewer than count down to an empty list
            eg. [{'task_id':<x>, 'buffer':<buf>} ... ]
        """
        tasks = []
        for i in range(0, count):
            try:
                x = self.pending_task_queue.get(block=False)
            except queue.Empty:
                break
            else:
                tasks.append(x)

        return tasks

def get_tasks ( self , count ) : tasks = [ ] for i in range ( 0 , count ) : try : x = self . pending_task_queue . get ( block = False ) except queue . Empty : break else : tasks . append ( x ) return tasks


def migrate_tasks_to_internal(self, kill_event):
        """Pull tasks from the incoming tasks 0mq pipe onto the internal
        pending task queue

        Parameters:
        -----------
        kill_event : threading.Event
              Event to let the thread know when it is time to die.
        """
        logger.info("[TASK_PULL_THREAD] Starting")
        task_counter = 0
        poller = zmq.Poller()
        poller.register(self.task_incoming, zmq.POLLIN)

        while not kill_event.is_set():
            try:
                msg = self.task_incoming.recv_pyobj()
            except zmq.Again:
                # We just timed out while attempting to receive
                logger.debug("[TASK_PULL_THREAD] {} tasks in internal queue".format(self.pending_task_queue.qsize()))
                continue

            if msg == 'STOP':
                kill_event.set()
                break
            else:
                self.pending_task_queue.put(msg)
                task_counter += 1
                logger.debug("[TASK_PULL_THREAD] Fetched task:{}".format(task_counter))


def _command_server(self, kill_event):
        """ Command server to run async command to the interchange
        """
        logger.debug("[COMMAND] Command Server Starting")
        while not kill_event.is_set():
            try:
                command_req = self.command_channel.recv_pyobj()
                logger.debug("[COMMAND] Received command request: {}".format(command_req))
                if command_req == "OUTSTANDING_C":
                    outstanding = self.pending_task_queue.qsize()
                    for manager in self._ready_manager_queue:
                        outstanding += len(self._ready_manager_queue[manager]['tasks'])
                    reply = outstanding

                elif command_req == "WORKERS":
                    num_workers = 0
                    for manager in self._ready_manager_queue:
                        num_workers += self._ready_manager_queue[manager]['worker_count']
                    reply = num_workers
                elif command_req == "MANAGERS":
                    reply = []
                    for manager in self._ready_manager_queue:
                        resp = {'manager': manager.decode('utf-8'),
                                'block_id': self._ready_manager_queue[manager]['block_id'],
                                'worker_count': self._ready_manager_queue[manager]['worker_count'],
                                'tasks': len(self._ready_manager_queue[manager]['tasks']),
                                'active': self._ready_manager_queue[manager]['active']}
                        reply.append(resp)

                elif command_req.startswith("HOLD_WORKER"):
                    cmd, s_manager = command_req.split(';')
                    manager = s_manager.encode('utf-8')
                    logger.info("[CMD] Received HOLD_WORKER for {}".format(manager))
                    if manager in self._ready_manager_queue:
                        self._ready_manager_queue[manager]['active'] = False
                        reply = True
                    else:
                        reply = False

                elif command_req == "SHUTDOWN":
                    logger.info("[CMD] Received SHUTDOWN command")
                    kill_event.set()
                    reply = True

                else:
                    reply = None

                logger.debug("[COMMAND] Reply: {}".format(reply))
                self.command_channel.send_pyobj(reply)

            except zmq.Again:
                logger.debug("[COMMAND] is alive")
                continue


def start(self):
        """ Start the worker processes.

        TODO: Move task receiving to a thread
        """
        start = time.time()
        self._kill_event = threading.Event()

        self.procs = {}
        for worker_id in range(self.worker_count):
            p = multiprocessing.Process(target=worker, args=(worker_id,
                                                             self.uid,
                                                             self.pending_task_queue,
                                                             self.pending_result_queue,
                                                             self.ready_worker_queue,
                                                         ))
            p.start()
            self.procs[worker_id] = p

        logger.debug("Manager synced with workers")

        self._task_puller_thread = threading.Thread(target=self.pull_tasks,
                                                    args=(self._kill_event,))
        self._result_pusher_thread = threading.Thread(target=self.push_results,
                                                      args=(self._kill_event,))
        self._task_puller_thread.start()
        self._result_pusher_thread.start()

        logger.info("Loop start")

        # TODO : Add mechanism in this loop to stop the worker pool
        # This might need a multiprocessing event to signal back.
        self._kill_event.wait()
        logger.critical("[MAIN] Received kill event, terminating worker processes")

        self._task_puller_thread.join()
        self._result_pusher_thread.join()
        for proc_id in self.procs:
            self.procs[proc_id].terminate()
            logger.critical("Terminating worker {}:{}".format(self.procs[proc_id],
                                                              self.procs[proc_id].is_alive()))
            self.procs[proc_id].join()
            logger.debug("Worker:{} joined successfully".format(self.procs[proc_id]))

        self.task_incoming.close()
        self.result_outgoing.close()
        self.context.term()
        delta = time.time() - start
        logger.info("process_worker_pool ran for {} seconds".format(delta))
        return


def get_data_manager(cls):
        """Return the DataManager of the currently loaded DataFlowKernel.
        """
        from parsl.dataflow.dflow import DataFlowKernelLoader
        dfk = DataFlowKernelLoader.dfk()

        return dfk.executors['data_manager']

def get_data_manager ( cls ) : from parsl . dataflow . dflow import DataFlowKernelLoader dfk = DataFlowKernelLoader . dfk ( ) return dfk . executors [ 'data_manager' ]

Return the DataManager of the currently loaded DataFlowKernel .


def shutdown(self, block=False):
        """Shutdown the ThreadPool.

        Kwargs:
            - block (bool): To block for confirmations or not

        """
        x = self.executor.shutdown(wait=block)
        logger.debug("Done with executor shutdown")
        return x

def
shutdown
(
self
,
block
=
False
)
:
x
=
self
.
executor
.
shutdown
(
wait
=
block
)
logger
.
debug
(
"Done with executor shutdown"
)
return
x


def stage_in(self, file, executor):
        """Transport the file from the input source to the executor.

        This function returns a DataFuture.

        Args:
            - self
            - file (File) : file to stage in
            - executor (str) : an executor the file is going to be staged in to.
                                If the executor argument is not specified for a file
                                with 'globus' scheme, the file will be staged in to
                                the first executor with the "globus" key in a config.
        """

        if file.scheme == 'ftp':
            working_dir = self.dfk.executors[executor].working_dir
            stage_in_app = self._ftp_stage_in_app(executor=executor)
            app_fut = stage_in_app(working_dir, outputs=[file])
            return app_fut._outputs[0]
        elif file.scheme == 'http' or file.scheme == 'https':
            working_dir = self.dfk.executors[executor].working_dir
            stage_in_app = self._http_stage_in_app(executor=executor)
            app_fut = stage_in_app(working_dir, outputs=[file])
            return app_fut._outputs[0]
        elif file.scheme == 'globus':
            globus_ep = self._get_globus_endpoint(executor)
            stage_in_app = self._globus_stage_in_app()
            app_fut = stage_in_app(globus_ep, outputs=[file])
            return app_fut._outputs[0]
        else:
            raise Exception('Staging in with unknown file scheme {} is not supported'.format(file.scheme))

Transport the file from the input source to the executor .


def stage_out(self, file, executor):
        """Transport the file from the local filesystem to the remote Globus endpoint.

        This function returns a DataFuture.

        Args:
            - self
            - file (File) - file to stage out
            - executor (str) - Which executor the file is going to be staged out from.
                                If the executor argument is not specified for a file
                                with the 'globus' scheme, the file will be staged in to
                                the first executor with the "globus" key in a config.
        """

        if file.scheme == 'http' or file.scheme == 'https':
            raise Exception('HTTP/HTTPS file staging out is not supported')
        elif file.scheme == 'ftp':
            raise Exception('FTP file staging out is not supported')
        elif file.scheme == 'globus':
            globus_ep = self._get_globus_endpoint(executor)
            stage_out_app = self._globus_stage_out_app()
            return stage_out_app(globus_ep, inputs=[file])
        else:
            raise Exception('Staging out with unknown file scheme {} is not supported'.format(file.scheme))

Transport the file from the local filesystem to the remote Globus endpoint .


def get_all_checkpoints(rundir="runinfo"):
    """Finds the checkpoints from all last runs.

    Note that checkpoints are incremental, and this helper will not find
    previous checkpoints from earlier than the most recent run. It probably
    should be made to do so.

    Kwargs:
       - rundir(str) : Path to the runinfo directory

    Returns:
       - a list suitable for the checkpointFiles parameter of DataFlowKernel
         constructor

    """

    if(not os.path.isdir(rundir)):
        return []

    dirs = sorted(os.listdir(rundir))

    checkpoints = []

    for runid in dirs:

        checkpoint = os.path.abspath('{}/{}/checkpoint'.format(rundir, runid))

        if os.path.isdir(checkpoint):
            checkpoints.append(checkpoint)

    return checkpoints

def get_all_checkpoints ( rundir = "runinfo" ) : if ( not os . path . isdir ( rundir ) ) : return [ ] dirs = sorted ( os . listdir ( rundir ) ) checkpoints = [ ] for runid in dirs : checkpoint = os . path . abspath ( '{}/{}/checkpoint' . format ( rundir , runid ) ) if os . path . isdir ( checkpoint ) : checkpoints . append ( checkpoint ) return checkpoints

Finds the checkpoints from all last runs .


def get_last_checkpoint(rundir="runinfo"):
    """Find the checkpoint from the last run, if one exists.

    Note that checkpoints are incremental, and this helper will not find
    previous checkpoints from earlier than the most recent run. It probably
    should be made to do so.

    Kwargs:
       - rundir(str) : Path to the runinfo directory

    Returns:
     - a list suitable for checkpointFiles parameter of DataFlowKernel
       constructor, with 0 or 1 elements

    """
    if not os.path.isdir(rundir):
        return []

    dirs = sorted(os.listdir(rundir))

    if len(dirs) == 0:
        return []

    last_runid = dirs[-1]
    last_checkpoint = os.path.abspath('{}/{}/checkpoint'.format(rundir, last_runid))

    if(not(os.path.isdir(last_checkpoint))):
        return []

    return [last_checkpoint]

def get_last_checkpoint ( rundir = "runinfo" ) : if not os . path . isdir ( rundir ) : return [ ] dirs = sorted ( os . listdir ( rundir ) ) if len ( dirs ) == 0 : return [ ] last_runid = dirs [ - 1 ] last_checkpoint = os . path . abspath ( '{}/{}/checkpoint' . format ( rundir , last_runid ) ) if ( not ( os . path . isdir ( last_checkpoint ) ) ) : return [ ] return [ last_checkpoint ]

Find the checkpoint from the last run if one exists .


def interactive(f):
    """Decorator for making functions appear as interactively defined.

    This results in the function being linked to the user_ns as globals()
    instead of the module globals().
    """
    # build new FunctionType, so it can have the right globals
    # interactive functions never have closures, that's kind of the point
    if isinstance(f, FunctionType):
        mainmod = __import__('__main__')
        f = FunctionType(f.__code__, mainmod.__dict__,
                         f.__name__, f.__defaults__,
                         )
    # associate with __main__ for uncanning
    f.__module__ = '__main__'
    return f

Decorator for making functions appear as interactively defined .


def use_pickle():
    """Revert to using stdlib pickle.

    Reverts custom serialization enabled by use_dill|cloudpickle.
    """
    from . import serialize
    serialize.pickle = serialize._stdlib_pickle

    # restore special function handling
    can_map[FunctionType] = _original_can_map[FunctionType]

def
use_pickle
(
)
:
from
.
import
serialize
serialize
.
pickle
=
serialize
.
_stdlib_pickle
# restore special function handling
can_map
[
FunctionType
]
=
_original_can_map
[
FunctionType
]


def _import_mapping(mapping, original=None):
    """Import any string-keys in a type mapping."""
    #log = get_logger()
    #log.debug("Importing canning map")
    for key, value in list(mapping.items()):
        if isinstance(key, string_types):
            try:
                cls = import_item(key)
            except Exception:
                if original and key not in original:
                    # only message on user-added classes
                    # log.error("canning class not importable: %r", key, exc_info=True)
                    print("ERROR: canning class not importable: %r", key, exc_info=True)
                mapping.pop(key)
            else:
                mapping[cls] = mapping.pop(key)

Import any string - keys in a type mapping .


def can(obj):
    """Prepare an object for pickling."""
    import_needed = False

    for cls, canner in iteritems(can_map):
        if isinstance(cls, string_types):
            import_needed = True
            break
        elif istype(obj, cls):
            return canner(obj)

    if import_needed:
        # perform can_map imports, then try again
        # this will usually only happen once
        _import_mapping(can_map, _original_can_map)
        return can(obj)

    return obj


def can_sequence(obj):
    """Can the elements of a sequence."""
    if istype(obj, sequence_types):
        t = type(obj)
        return t([can(i) for i in obj])
    else:
        return obj

def can_sequence ( obj ) : if istype ( obj , sequence_types ) : t = type ( obj ) return t ( [ can ( i ) for i in obj ] ) else : return obj


def uncan(obj, g=None):
    """Invert canning."""
    import_needed = False
    for cls, uncanner in iteritems(uncan_map):
        if isinstance(cls, string_types):
            import_needed = True
            break
        elif isinstance(obj, cls):
            return uncanner(obj, g)

    if import_needed:
        # perform uncan_map imports, then try again
        # this will usually only happen once
        _import_mapping(uncan_map, _original_uncan_map)
        return uncan(obj, g)

    return obj


def unset_logging(self):
        """ Mute newly added handlers to the root level, right after calling executor.status
        """
        if self.logger_flag is True:
            return

        root_logger = logging.getLogger()

        for hndlr in root_logger.handlers:
            if hndlr not in self.prior_loghandlers:
                hndlr.setLevel(logging.ERROR)

        self.logger_flag = True

def unset_logging ( self ) : if self . logger_flag is True : return root_logger = logging . getLogger ( ) for hndlr in root_logger . handlers : if hndlr not in self . prior_loghandlers : hndlr . setLevel ( logging . ERROR ) self . logger_flag = True

Mute newly added handlers to the root level right after calling executor . status


def start(self):
        """Start the controller."""

        if self.mode == "manual":
            return

        if self.ipython_dir != '~/.ipython':
            self.ipython_dir = os.path.abspath(os.path.expanduser(self.ipython_dir))

        if self.log:
            stdout = open(os.path.join(self.ipython_dir, "{0}.controller.out".format(self.profile)), 'w')
            stderr = open(os.path.join(self.ipython_dir, "{0}.controller.err".format(self.profile)), 'w')
        else:
            stdout = open(os.devnull, 'w')
            stderr = open(os.devnull, 'w')

        try:
            opts = [
                'ipcontroller',
                '' if self.ipython_dir == '~/.ipython' else '--ipython-dir={}'.format(self.ipython_dir),
                self.interfaces if self.interfaces is not None else '--ip=*',
                '' if self.profile == 'default' else '--profile={0}'.format(self.profile),
                '--reuse' if self.reuse else '',
                '--location={}'.format(self.public_ip) if self.public_ip else '',
                '--port={}'.format(self.port) if self.port is not None else ''
            ]
            if self.port_range is not None:
                opts += [
                    '--HubFactory.hb={0},{1}'.format(self.hb_ping, self.hb_pong),
                    '--HubFactory.control={0},{1}'.format(self.control_client, self.control_engine),
                    '--HubFactory.mux={0},{1}'.format(self.mux_client, self.mux_engine),
                    '--HubFactory.task={0},{1}'.format(self.task_client, self.task_engine)
                ]
            logger.debug("Starting ipcontroller with '{}'".format(' '.join([str(x) for x in opts])))
            self.proc = subprocess.Popen(opts, stdout=stdout, stderr=stderr, preexec_fn=os.setsid)
        except FileNotFoundError:
            msg = "Could not find ipcontroller. Please make sure that ipyparallel is installed and available in your env"
            logger.error(msg)
            raise ControllerError(msg)
        except Exception as e:
            msg = "IPPController failed to start: {0}".format(e)
            logger.error(msg)
            raise ControllerError(msg)


def engine_file(self):
        """Specify path to the ipcontroller-engine.json file.

        This file is stored in in the ipython_dir/profile folders.

        Returns :
              - str, File path to engine file
        """
        return os.path.join(self.ipython_dir,
                            'profile_{0}'.format(self.profile),
                            'security/ipcontroller-engine.json')

def engine_file ( self ) : return os . path . join ( self . ipython_dir , 'profile_{0}' . format ( self . profile ) , 'security/ipcontroller-engine.json' )

Specify path to the ipcontroller - engine . json file .


def client_file(self):
        """Specify path to the ipcontroller-client.json file.

        This file is stored in in the ipython_dir/profile folders.

        Returns :
              - str, File path to client file
        """
        return os.path.join(self.ipython_dir,
                            'profile_{0}'.format(self.profile),
                            'security/ipcontroller-client.json')

def client_file ( self ) : return os . path . join ( self . ipython_dir , 'profile_{0}' . format ( self . profile ) , 'security/ipcontroller-client.json' )

Specify path to the ipcontroller - client . json file .


def close(self):
        """Terminate the controller process and its child processes.

        Args:
              - None
        """
        if self.reuse:
            logger.debug("Ipcontroller not shutting down: reuse enabled")
            return

        if self.mode == "manual":
            logger.debug("Ipcontroller not shutting down: Manual mode")
            return

        try:
            pgid = os.getpgid(self.proc.pid)
            os.killpg(pgid, signal.SIGTERM)
            time.sleep(0.2)
            os.killpg(pgid, signal.SIGKILL)
            try:
                self.proc.wait(timeout=1)
                x = self.proc.returncode
                if x == 0:
                    logger.debug("Controller exited with {0}".format(x))
                else:
                    logger.error("Controller exited with {0}. May require manual cleanup".format(x))
            except subprocess.TimeoutExpired:
                logger.warn("Ipcontroller process:{0} cleanup failed. May require manual cleanup".format(self.proc.pid))

        except Exception as e:
            logger.warn("Failed to kill the ipcontroller process[{0}]: {1}".format(self.proc.pid, e))

Terminate the controller process and its child processes .


def make_hash(self, task):
        """Create a hash of the task inputs.

        This uses a serialization library borrowed from ipyparallel.
        If this fails here, then all ipp calls are also likely to fail due to failure
        at serialization.

        Args:
            - task (dict) : Task dictionary from dfk.tasks

        Returns:
            - hash (str) : A unique hash string
        """
        # Function name TODO: Add fn body later
        t = [serialize_object(task['func_name'])[0],
             serialize_object(task['fn_hash'])[0],
             serialize_object(task['args'])[0],
             serialize_object(task['kwargs'])[0],
             serialize_object(task['env'])[0]]
        x = b''.join(t)
        hashedsum = hashlib.md5(x).hexdigest()
        return hashedsum

def
make_hash
(
self
,
task
)
:
# Function name TODO: Add fn body later
t
=
[
serialize_object
(
task
[
'func_name'
]
)
[
0
]
,
serialize_object
(
task
[
'fn_hash'
]
)
[
0
]
,
serialize_object
(
task
[
'args'
]
)
[
0
]
,
serialize_object
(
task
[
'kwargs'
]
)
[
0
]
,
serialize_object
(
task
[
'env'
]
)
[
0
]
]
x
=
b''
.
join
(
t
)
hashedsum
=
hashlib
.
md5
(
x
)
.
hexdigest
(
)
return
hashedsum


def check_memo(self, task_id, task):
        """Create a hash of the task and its inputs and check the lookup table for this hash.

        If present, the results are returned. The result is a tuple indicating whether a memo
        exists and the result, since a Null result is possible and could be confusing.
        This seems like a reasonable option without relying on an cache_miss exception.

        Args:
            - task(task) : task from the dfk.tasks table

        Returns:
            Tuple of the following:
            - present (Bool): Is this present in the memo_lookup_table
            - Result (Py Obj): Result of the function if present in table

        This call will also set task['hashsum'] to the unique hashsum for the func+inputs.
        """
        if not self.memoize or not task['memoize']:
            task['hashsum'] = None
            return None, None

        hashsum = self.make_hash(task)
        present = False
        result = None
        if hashsum in self.memo_lookup_table:
            present = True
            result = self.memo_lookup_table[hashsum]
            logger.info("Task %s using result from cache", task_id)

        task['hashsum'] = hashsum
        return present, result

def
check_memo
(
self
,
task_id
,
task
)
:
if
not
self
.
memoize
or
not
task
[
'memoize'
]
:
task
[
'hashsum'
]
=
return
,
hashsum
=
self
.
make_hash
(
task
)
present
=
False
result
=
if
hashsum
in
self
.
memo_lookup_table
:
present
=
True
result
=
self
.
memo_lookup_table
[
hashsum
]
logger
.
info
(
"Task %s using result from cache"
,
task_id
)
task
[
'hashsum'
]
=
hashsum
return
present
,
result

Create a hash of the task and its inputs and check the lookup table for this hash .


def update_memo(self, task_id, task, r):
        """Updates the memoization lookup table with the result from a task.

        Args:
             - task_id (int): Integer task id
             - task (dict) : A task dict from dfk.tasks
             - r (Result future): Result future

        A warning is issued when a hash collision occurs during the update.
        This is not likely.
        """
        if not self.memoize or not task['memoize']:
            return

        if task['hashsum'] in self.memo_lookup_table:
            logger.info('Updating appCache entry with latest %s:%s call' %
                        (task['func_name'], task_id))
            self.memo_lookup_table[task['hashsum']] = r
        else:
            self.memo_lookup_table[task['hashsum']] = r

Updates the memoization lookup table with the result from a task .


def _nbytes(buf):
    """Return byte-size of a memoryview or buffer."""
    if isinstance(buf, memoryview):
        if PY3:
            # py3 introduces nbytes attribute
            return buf.nbytes
        else:
            # compute nbytes on py2
            size = buf.itemsize
            for dim in buf.shape:
                size *= dim
            return size
    else:
        # not a memoryview, raw bytes/ py2 buffer
        return len(buf)

Return byte - size of a memoryview or buffer .


def _extract_buffers(obj, threshold=MAX_BYTES):
    """Extract buffers larger than a certain threshold."""
    buffers = []
    if isinstance(obj, CannedObject) and obj.buffers:
        for i, buf in enumerate(obj.buffers):
            nbytes = _nbytes(buf)
            if nbytes > threshold:
                # buffer larger than threshold, prevent pickling
                obj.buffers[i] = None
                buffers.append(buf)
            # buffer too small for separate send, coerce to bytes
            # because pickling buffer objects just results in broken pointers
            elif isinstance(buf, memoryview):
                obj.buffers[i] = buf.tobytes()
            elif isinstance(buf, buffer):
                obj.buffers[i] = bytes(buf)
    return buffers

Extract buffers larger than a certain threshold .


def _restore_buffers(obj, buffers):
    """Restore extracted buffers."""
    if isinstance(obj, CannedObject) and obj.buffers:
        for i, buf in enumerate(obj.buffers):
            if buf is None:
                obj.buffers[i] = buffers.pop(0)

def _restore_buffers ( obj , buffers ) : if isinstance ( obj , CannedObject ) and obj . buffers : for i , buf in enumerate ( obj . buffers ) : if buf is None : obj . buffers [ i ] = buffers . pop ( 0 )


def serialize_object(obj, buffer_threshold=MAX_BYTES, item_threshold=MAX_ITEMS):
    """Serialize an object into a list of sendable buffers.

    Parameters
    ----------

    obj : object
        The object to be serialized
    buffer_threshold : int
        The threshold (in bytes) for pulling out data buffers
        to avoid pickling them.
    item_threshold : int
        The maximum number of items over which canning will iterate.
        Containers (lists, dicts) larger than this will be pickled without
        introspection.

    Returns
    -------
    [bufs] : list of buffers representing the serialized object.
    """
    buffers = []
    if istype(obj, sequence_types) and len(obj) < item_threshold:
        cobj = can_sequence(obj)
        for c in cobj:
            buffers.extend(_extract_buffers(c, buffer_threshold))
    elif istype(obj, dict) and len(obj) < item_threshold:
        cobj = {}
        for k in sorted(obj):
            c = can(obj[k])
            buffers.extend(_extract_buffers(c, buffer_threshold))
            cobj[k] = c
    else:
        cobj = can(obj)
        buffers.extend(_extract_buffers(cobj, buffer_threshold))

    buffers.insert(0, pickle.dumps(cobj, PICKLE_PROTOCOL))
    return buffers

def serialize_object ( obj , buffer_threshold = MAX_BYTES , item_threshold = MAX_ITEMS ) : buffers = [ ] if istype ( obj , sequence_types ) and len ( obj ) < item_threshold : cobj = can_sequence ( obj ) for c in cobj : buffers . extend ( _extract_buffers ( c , buffer_threshold ) ) elif istype ( obj , dict ) and len ( obj ) < item_threshold : cobj = { } for k in sorted ( obj ) : c = can ( obj [ k ] ) buffers . extend ( _extract_buffers ( c , buffer_threshold ) ) cobj [ k ] = c else : cobj = can ( obj ) buffers . extend ( _extract_buffers ( cobj , buffer_threshold ) ) buffers . insert ( 0 , pickle . dumps ( cobj , PICKLE_PROTOCOL ) ) return buffers

Serialize an object into a list of sendable buffers .


def deserialize_object(buffers, g=None):
    """Reconstruct an object serialized by serialize_object from data buffers.

    Parameters
    ----------

    bufs : list of buffers/bytes

    g : globals to be used when uncanning

    Returns
    -------

    (newobj, bufs) : unpacked object, and the list of remaining unused buffers.
    """
    bufs = list(buffers)
    pobj = buffer_to_bytes_py2(bufs.pop(0))
    canned = pickle.loads(pobj)
    if istype(canned, sequence_types) and len(canned) < MAX_ITEMS:
        for c in canned:
            _restore_buffers(c, bufs)
        newobj = uncan_sequence(canned, g)
    elif istype(canned, dict) and len(canned) < MAX_ITEMS:
        newobj = {}
        for k in sorted(canned):
            c = canned[k]
            _restore_buffers(c, bufs)
            newobj[k] = uncan(c, g)
    else:
        _restore_buffers(canned, bufs)
        newobj = uncan(canned, g)

    return newobj, bufs

def deserialize_object ( buffers , g = None ) : bufs = list ( buffers ) pobj = buffer_to_bytes_py2 ( bufs . pop ( 0 ) ) canned = pickle . loads ( pobj ) if istype ( canned , sequence_types ) and len ( canned ) < MAX_ITEMS : for c in canned : _restore_buffers ( c , bufs ) newobj = uncan_sequence ( canned , g ) elif istype ( canned , dict ) and len ( canned ) < MAX_ITEMS : newobj = { } for k in sorted ( canned ) : c = canned [ k ] _restore_buffers ( c , bufs ) newobj [ k ] = uncan ( c , g ) else : _restore_buffers ( canned , bufs ) newobj = uncan ( canned , g ) return newobj , bufs

Reconstruct an object serialized by serialize_object from data buffers .


def pack_apply_message(f, args, kwargs, buffer_threshold=MAX_BYTES, item_threshold=MAX_ITEMS):
    """Pack up a function, args, and kwargs to be sent over the wire.

    Each element of args/kwargs will be canned for special treatment,
    but inspection will not go any deeper than that.

    Any object whose data is larger than `threshold`  will not have their data copied
    (only numpy arrays and bytes/buffers support zero-copy)

    Message will be a list of bytes/buffers of the format:

    [ cf, pinfo, <arg_bufs>, <kwarg_bufs> ]

    With length at least two + len(args) + len(kwargs)
    """
    arg_bufs = list(chain.from_iterable(
        serialize_object(arg, buffer_threshold, item_threshold) for arg in args))

    kw_keys = sorted(kwargs.keys())
    kwarg_bufs = list(chain.from_iterable(
        serialize_object(kwargs[key], buffer_threshold, item_threshold) for key in kw_keys))

    info = dict(nargs=len(args), narg_bufs=len(arg_bufs), kw_keys=kw_keys)

    msg = [pickle.dumps(can(f), PICKLE_PROTOCOL)]
    msg.append(pickle.dumps(info, PICKLE_PROTOCOL))
    msg.extend(arg_bufs)
    msg.extend(kwarg_bufs)

    return msg

def pack_apply_message ( f , args , kwargs , buffer_threshold = MAX_BYTES , item_threshold = MAX_ITEMS ) : arg_bufs = list ( chain . from_iterable ( serialize_object ( arg , buffer_threshold , item_threshold ) for arg in args ) ) kw_keys = sorted ( kwargs . keys ( ) ) kwarg_bufs = list ( chain . from_iterable ( serialize_object ( kwargs [ key ] , buffer_threshold , item_threshold ) for key in kw_keys ) ) info = dict ( nargs = len ( args ) , narg_bufs = len ( arg_bufs ) , kw_keys = kw_keys ) msg = [ pickle . dumps ( can ( f ) , PICKLE_PROTOCOL ) ] msg . append ( pickle . dumps ( info , PICKLE_PROTOCOL ) ) msg . extend ( arg_bufs ) msg . extend ( kwarg_bufs ) return msg

Pack up a function args and kwargs to be sent over the wire .


def _write_submit_script(self, template, script_filename, job_name, configs):
        """Generate submit script and write it to a file.

        Args:
              - template (string) : The template string to be used for the writing submit script
              - script_filename (string) : Name of the submit script
              - job_name (string) : job name
              - configs (dict) : configs that get pushed into the template

        Returns:
              - True: on success

        Raises:
              SchedulerMissingArgs : If template is missing args
              ScriptPathError : Unable to write submit script out
        """

        try:
            submit_script = Template(template).substitute(jobname=job_name, **configs)
            # submit_script = Template(template).safe_substitute(jobname=job_name, **configs)
            with open(script_filename, 'w') as f:
                f.write(submit_script)

        except KeyError as e:
            logger.error("Missing keys for submit script : %s", e)
            raise (SchedulerMissingArgs(e.args, self.sitename))

        except IOError as e:
            logger.error("Failed writing to submit script: %s", script_filename)
            raise (ScriptPathError(script_filename, e))
        except Exception as e:
            print("Template : ", template)
            print("Args : ", job_name)
            print("Kwargs : ", configs)
            logger.error("Uncategorized error: %s", e)
            raise (e)

        return True

Generate submit script and write it to a file .


def cancel(self, job_ids):
        ''' Cancels the jobs specified by a list of job ids

        Args:
        job_ids : [<job_id> ...]

        Returns :
        [True/False...] : If the cancel operation fails the entire list will be False.
        '''
        for job in job_ids:
            logger.debug("Terminating job/proc_id: {0}".format(job))
            # Here we are assuming that for local, the job_ids are the process id's
            if self.resources[job]['proc']:
                proc = self.resources[job]['proc']
                os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
                self.resources[job]['status'] = 'CANCELLED'

            elif self.resources[job]['remote_pid']:
                cmd = "kill -- -$(ps -o pgid={} | grep -o '[0-9]*')".format(self.resources[job]['remote_pid'])
                retcode, stdout, stderr = self.channel.execute_wait(cmd, self.cmd_timeout)
                if retcode != 0:
                    logger.warning("Failed to kill PID: {} and child processes on {}".format(self.resources[job]['remote_pid'],
                                                                                self.label))

        rets = [True for i in job_ids]
        return rets