def input(self, input, song):
        """Input callback, handles key presses
        """
        try:
            cmd = getattr(self, self.CMD_MAP[input][1])
        except (IndexError, KeyError):
            return self.screen.print_error(
                "Invalid command {!r}!".format(input))

        cmd(song)

def
input
(
self
,
input
,
song
)
:
try
:
cmd
=
getattr
(
self
,
self
.
CMD_MAP
[
input
]
[
1
]
)
except
(
IndexError
,
KeyError
)
:
return
self
.
screen
.
print_error
(
"Invalid command {!r}!"
.
format
(
input
)
)
cmd
(
song
)


def retries(max_tries, exceptions=(Exception,)):
    """Function decorator implementing retrying logic.

    exceptions: A tuple of exception classes; default (Exception,)

    The decorator will call the function up to max_tries times if it raises
    an exception.

    By default it catches instances of the Exception class and subclasses.
    This will recover after all but the most fatal errors. You may specify a
    custom tuple of exception classes with the 'exceptions' argument; the
    function will only be retried if it raises one of the specified
    exceptions.
    """
    def decorator(func):
        def function(*args, **kwargs):

            retries_left = max_tries
            while retries_left > 0:
                try:
                    retries_left -= 1
                    return func(*args, **kwargs)

                except exceptions as exc:
                    # Don't retry for PandoraExceptions - unlikely that result
                    # will change for same set of input parameters.
                    if isinstance(exc, PandoraException):
                        raise
                    if retries_left > 0:
                        time.sleep(delay_exponential(
                            0.5, 2, max_tries - retries_left))
                    else:
                        raise

        return function

    return decorator

Function decorator implementing retrying logic .


def iterate_forever(func, *args, **kwargs):
    """Iterate over a finite iterator forever

    When the iterator is exhausted will call the function again to generate a
    new iterator and keep iterating.
    """
    output = func(*args, **kwargs)

    while True:
        try:
            playlist_item = next(output)
            playlist_item.prepare_playback()
            yield playlist_item
        except StopIteration:
            output = func(*args, **kwargs)

def iterate_forever ( func , * args , * * kwargs ) : output = func ( * args , * * kwargs ) while True : try : playlist_item = next ( output ) playlist_item . prepare_playback ( ) yield playlist_item except StopIteration : output = func ( * args , * * kwargs )


def get_integer(prompt):
        """Gather user input and convert it to an integer

        Will keep trying till the user enters an interger or until they ^C the
        program.
        """
        while True:
            try:
                return int(input(prompt).strip())
            except ValueError:
                print(Colors.red("Invalid Input!"))

def
get_integer
(
prompt
)
:
while
True
:
try
:
return
int
(
input
(
prompt
)
.
strip
(
)
)
except
ValueError
:
print
(
Colors
.
red
(
"Invalid Input!"
)
)


def open(self):
        """open the drop box

        You need to call this method before starting putting packages.

        Returns
        -------
        None

        """

        self.workingArea.open()
        self.runid_pkgidx_map = { }
        self.runid_to_return = deque()

def open ( self ) : self . workingArea . open ( ) self . runid_pkgidx_map = { } self . runid_to_return = deque ( )


def put(self, package):
        """put a task

        This method places a task in the working area and have the
        dispatcher execute it.

        If you need to put multiple tasks, it can be much faster to
        use `put_multiple()` than to use this method multiple times
        depending of the dispatcher.

        Parameters
        ----------
        package : callable
            A task

        Returns
        -------
        int
            A package index assigned by the working area

        """

        pkgidx = self.workingArea.put_package(package)

        logger = logging.getLogger(__name__)
        logger.info('submitting {}'.format(self.workingArea.package_relpath(pkgidx)))

        runid = self.dispatcher.run(self.workingArea, pkgidx)
        self.runid_pkgidx_map[runid] = pkgidx
        return pkgidx

def
put
(
self
,
package
)
:
pkgidx
=
self
.
workingArea
.
put_package
(
package
)
logger
=
logging
.
getLogger
(
__name__
)
logger
.
info
(
'submitting {}'
.
format
(
self
.
workingArea
.
package_relpath
(
pkgidx
)
)
)
runid
=
self
.
dispatcher
.
run
(
self
.
workingArea
,
pkgidx
)
self
.
runid_pkgidx_map
[
runid
]
=
pkgidx
return
pkgidx


def receive(self):
        """return pairs of package indices and results of all tasks

        This method waits until all tasks finish.

        Returns
        -------
        list
            A list of pairs of package indices and results

        """

        ret = [ ] # a list of (pkgid, result)
        while True:

            if self.runid_pkgidx_map:
                self.runid_to_return.extend(self.dispatcher.poll())
                ret.extend(self._collect_all_finished_pkgidx_result_pairs())

            if not self.runid_pkgidx_map:
                break
            time.sleep(self.sleep)

        ret = sorted(ret, key=itemgetter(0))

        return ret


def poll(self):
        """return pairs of package indices and results of finished tasks

        This method does not wait for tasks to finish.

        Returns
        -------
        list
            A list of pairs of package indices and results

        """

        self.runid_to_return.extend(self.dispatcher.poll())
        ret = self._collect_all_finished_pkgidx_result_pairs()
        return ret


def receive_one(self):
        """return a pair of a package index and result of a task

        This method waits until a tasks finishes. It returns `None` if
        no task is running.

        Returns
        -------
        tuple or None
            A pair of a package index and result. `None` if no tasks
            is running.

        """

        if not self.runid_pkgidx_map:
            return None

        while True:

            if not self.runid_to_return:
                self.runid_to_return.extend(self.dispatcher.poll())

            ret = self._collect_next_finished_pkgidx_result_pair()

            if ret is not None:
                break

            if self.runid_pkgidx_map:
                time.sleep(self.sleep)

        return ret


def run_multiple(self, eventLoops):
        """run the event loops in the background.

        Args:
            eventLoops (list): a list of event loops to run

        """

        self.nruns += len(eventLoops)
        return self.communicationChannel.put_multiple(eventLoops)

def run_multiple ( self , eventLoops ) : self . nruns += len ( eventLoops ) return self . communicationChannel . put_multiple ( eventLoops )


def poll(self):
        """Return pairs of run ids and results of finish event loops.
        """
        ret = self.communicationChannel.receive_finished()
        self.nruns -= len(ret)
        return ret

def poll ( self ) : ret = self . communicationChannel . receive_finished ( ) self . nruns -= len ( ret ) return ret

Return pairs of run ids and results of finish event loops .


def receive_one(self):
        """Return a pair of a run id and a result.

        This method waits until an event loop finishes.
        This method returns None if no loop is running.
        """
        if self.nruns == 0:
            return None
        ret = self.communicationChannel.receive_one()
        if ret is not None:
            self.nruns -= 1
        return ret

def receive_one ( self ) : if self . nruns == 0 : return None ret = self . communicationChannel . receive_one ( ) if ret is not None : self . nruns -= 1 return ret

Return a pair of a run id and a result .


def receive(self):
        """Return pairs of run ids and results.

        This method waits until all event loops finish
        """
        ret = self.communicationChannel.receive_all()
        self.nruns -= len(ret)
        if self.nruns > 0:
            import logging
            logger = logging.getLogger(__name__)
            logger.warning(
                'too few results received: {} results received, {} more expected'.format(
                    len(ret), self.nruns))
        elif self.nruns < 0:
            import logging
            logger = logging.getLogger(__name__)
            logger.warning(
                'too many results received: {} results received, {} too many'.format(
                    len(ret), -self.nruns))
        return ret


def end(self):
        """wait until all event loops end and returns the results.

        """

        results = self.communicationChannel.receive()

        if self.nruns != len(results):
            import logging
            logger = logging.getLogger(__name__)
            # logger.setLevel(logging.DEBUG)
            logger.warning(
                'too few results received: {} results received, {} expected'.format(
                    len(results),
                    self.nruns
                ))

        return results

wait until all event loops end and returns the results .


def key_vals_dict_to_tuple_list(key_vals_dict, fill=float('nan')):
    """Convert ``key_vals_dict`` to `tuple_list``.

    Args:
        key_vals_dict (dict): The first parameter.
        fill: a value to fill missing data

    Returns:
        A list of tuples

    """

    tuple_list = [ ]

    if not key_vals_dict: return tuple_list

    vlen = max([len(vs) for vs in itertools.chain(*key_vals_dict.values())])

    for k, vs in key_vals_dict.items():
        try:
            tuple_list.extend([k + tuple(v) + (fill, )*(vlen - len(v)) for v in vs])
        except TypeError:
            # assume k is not a tuple
            tuple_list.extend([(k, ) + tuple(v) + (fill, )*(vlen - len(v)) for v in vs])

    return tuple_list

def
key_vals_dict_to_tuple_list
(
key_vals_dict
,
fill
=
float
(
'nan'
)
)
:
tuple_list
=
[
]
if
not
key_vals_dict
:
return
tuple_list
vlen
=
max
(
[
len
(
vs
)
for
vs
in
itertools
.
chain
(
*
key_vals_dict
.
values
(
)
)
]
)
for
k
,
vs
in
key_vals_dict
.
items
(
)
:
try
:
tuple_list
.
extend
(
[
k
+
tuple
(
v
)
+
(
fill
,
)
*
(
vlen
len
(
v
)
)
for
v
in
vs
]
)
except
TypeError
:
# assume k is not a tuple
tuple_list
.
extend
(
[
(
k
,
)
+
tuple
(
v
)
+
(
fill
,
)
*
(
vlen
len
(
v
)
)
for
v
in
vs
]
)
return
tuple_list


def open(self):
        """Open the working area

        Returns
        -------
        None
        """

        self.path = self._prepare_dir(self.topdir)
        self._copy_executable(area_path=self.path)
        self._save_logging_levels(area_path=self.path)
        self._put_python_modules(modules=self.python_modules, area_path=self.path)

def open ( self ) : self . path = self . _prepare_dir ( self . topdir ) self . _copy_executable ( area_path = self . path ) self . _save_logging_levels ( area_path = self . path ) self . _put_python_modules ( modules = self . python_modules , area_path = self . path )


def put_package(self, package):
        """Put a package

        Parameters
        ----------
        package :
            a task package

        Returns
        -------
        int
            A package index

        """

        self.last_package_index += 1
        package_index = self.last_package_index

        package_fullpath = self.package_fullpath(package_index)
        # e.g., '{path}/tpd_20161129_122841_HnpcmF/task_00009.p.gz'

        with gzip.open(package_fullpath, 'wb') as f:
            pickle.dump(package, f, protocol=pickle.HIGHEST_PROTOCOL)
            f.close()

        result_fullpath = self.result_fullpath(package_index)
        # e.g., '{path}/tpd_20161129_122841_HnpcmF/results/task_00009/result.p.gz'

        result_dir = os.path.dirname(result_fullpath)
        # e.g., '{path}/tpd_20161129_122841_HnpcmF/results/task_00009'

        alphatwirl.mkdir_p(result_dir)

        return package_index


def collect_result(self, package_index):
        """Collect the result of a task

        Parameters
        ----------
        package_index :
            a package index

        Returns
        -------
        obj
            The result of the task

        """

        result_fullpath = self.result_fullpath(package_index)
        # e.g., '{path}/tpd_20161129_122841_HnpcmF/results/task_00009/result.p.gz'

        try:
            with gzip.open(result_fullpath, 'rb') as f:
                result = pickle.load(f)
        except Exception as e:
            logger = logging.getLogger(__name__)
            logger.warning(e)
            return None

        return result


def package_fullpath(self, package_index):
        """Returns the full path of the package

        This method returns the full path to the package. This method
        simply constructs the path based on the convention and doesn't
        check if the package actually exists.

        Parameters
        ----------
        package_index :
            a package index

        Returns
        -------
        str
            the full path to the package

        """

        ret = os.path.join(self.path, self.package_relpath(package_index))
        # e.g., '{path}/tpd_20161129_122841_HnpcmF/task_00009.p.gz'

        return ret


def result_relpath(self, package_index):
        """Returns the relative path of the result

        This method returns the path to the result relative to the
        top dir of the working area. This method simply constructs the
        path based on the convention and doesn't check if the result
        actually exists.

        Parameters
        ----------
        package_index :
            a package index

        Returns
        -------
        str
            the relative path to the result

        """

        dirname = 'task_{:05d}'.format(package_index)
        # e.g., 'task_00009'

        ret = os.path.join('results', dirname, 'result.p.gz')
        # e.g., 'results/task_00009/result.p.gz'

        return ret


def result_fullpath(self, package_index):
        """Returns the full path of the result

        This method returns the full path to the result. This method
        simply constructs the path based on the convention and doesn't
        check if the result actually exists.

        Parameters
        ----------
        package_index :
            a package index

        Returns
        -------
        str
            the full path to the result

        """

        ret = os.path.join(self.path, self.result_relpath(package_index))
        # e.g., '{path}/tpd_20161129_122841_HnpcmF/results/task_00009/result.p.gz'

        return ret


def run_multiple(self, workingArea, package_indices):
        """Submit multiple jobs

        Parameters
        ----------
        workingArea :
            A workingArea
        package_indices : list(int)
            A list of package indices

        Returns
        -------
        list(str)
            The list of the run IDs of the jobs

        """

        if not package_indices:
            return [ ]

        job_desc = self._compose_job_desc(workingArea, package_indices)

        clusterprocids = submit_jobs(job_desc, cwd=workingArea.path)

        # TODO: make configurable
        clusterids = clusterprocids2clusterids(clusterprocids)
        for clusterid in clusterids:
            change_job_priority([clusterid], 10)

        self.clusterprocids_outstanding.extend(clusterprocids)

        return clusterprocids

def
run_multiple
(
self
,
workingArea
,
package_indices
)
:
if
not
package_indices
:
return
[
]
job_desc
=
self
.
_compose_job_desc
(
workingArea
,
package_indices
)
clusterprocids
=
submit_jobs
(
job_desc
,
cwd
=
workingArea
.
path
)
# TODO: make configurable
clusterids
=
clusterprocids2clusterids
(
clusterprocids
)
for
clusterid
in
clusterids
:
change_job_priority
(
[
clusterid
]
,
10
)
self
.
clusterprocids_outstanding
.
extend
(
clusterprocids
)
return
clusterprocids


def poll(self):
        """Return the run IDs of the finished jobs

        Returns
        -------
        list(str)
            The list of the run IDs of the finished jobs

        """

        clusterids = clusterprocids2clusterids(self.clusterprocids_outstanding)
        clusterprocid_status_list = query_status_for(clusterids)
        # e.g., [['1730126.0', 2], ['1730127.0', 2], ['1730129.1', 1], ['1730130.0', 1]]

        if clusterprocid_status_list:
            clusterprocids, statuses = zip(*clusterprocid_status_list)
        else:
            clusterprocids, statuses = (), ()

        clusterprocids_finished = [i for i in self.clusterprocids_outstanding if i not in clusterprocids]
        self.clusterprocids_finished.extend(clusterprocids_finished)
        self.clusterprocids_outstanding[:] = clusterprocids

        # logging
        counter = collections.Counter(statuses)
        messages = [ ]
        if counter:
            messages.append(', '.join(['{}: {}'.format(HTCONDOR_JOBSTATUS[k], counter[k]) for k in counter.keys()]))
        if self.clusterprocids_finished:
            messages.append('Finished {}'.format(len(self.clusterprocids_finished)))
        logger = logging.getLogger(__name__)
        logger.info(', '.join(messages))

        return clusterprocids_finished


def wait(self):
        """Wait until all jobs finish and return the run IDs of the finished jobs

        Returns
        -------
        list(str)
            The list of the run IDs of the finished jobs

        """

        sleep = 5
        while True:
            if self.clusterprocids_outstanding:
                self.poll()
            if not self.clusterprocids_outstanding:
                break
            time.sleep(sleep)
        return self.clusterprocids_finished

def wait ( self ) : sleep = 5 while True : if self . clusterprocids_outstanding : self . poll ( ) if not self . clusterprocids_outstanding : break time . sleep ( sleep ) return self . clusterprocids_finished


def failed_runids(self, runids):
        """Provide the run IDs of failed jobs

        Returns
        -------
        None

        """

        # remove failed clusterprocids from self.clusterprocids_finished
        # so that len(self.clusterprocids_finished)) becomes the number
        # of the successfully finished jobs
        for i in runids:
            try:
                self.clusterprocids_finished.remove(i)
            except ValueError:
                pass


def getArrays(self, tree, branchName):
        """return the array.array objects for the branch and its counter branch

        This method returns a pair of the array.array objects. The first one is
        for the given tree and branch name. The second one is for its counter
        branch. The second one will be None when the branch does not have a
        counter. A pair of None will be returned when the tree does not have
        the branch.

        """
        itsArray = self._getArray(tree, branchName)
        if itsArray is None: return None, None
        itsCountArray = self._getCounterArray(tree, branchName)
        return itsArray, itsCountArray

def getArrays ( self , tree , branchName ) : itsArray = self . _getArray ( tree , branchName ) if itsArray is None : return None , None itsCountArray = self . _getCounterArray ( tree , branchName ) return itsArray , itsCountArray

return the array . array objects for the branch and its counter branch


def put(self, task, *args, **kwargs):
        """put a task and its arguments

        If you need to put multiple tasks, it can be faster to put
        multiple tasks with `put_multiple()` than to use this method
        multiple times.

        Parameters
        ----------
        task : a function
            A function to be executed
        args : list
            A list of positional arguments to the `task`
        kwargs : dict
            A dict with keyword arguments to the `task`

        Returns
        -------
        int, str, or any hashable and sortable
            A task ID. IDs are sortable in the order in which the
            corresponding tasks are put.

        """
        if not self.isopen:
            logger = logging.getLogger(__name__)
            logger.warning('the drop box is not open')
            return
        package = TaskPackage(task=task, args=args, kwargs=kwargs)
        return self.dropbox.put(package)

def
put
(
self
,
task
,
*
args
,
*
*
kwargs
)
:
if
not
self
.
isopen
:
logger
=
logging
.
getLogger
(
__name__
)
logger
.
warning
(
'the drop box is not open'
)
return
package
=
TaskPackage
(
task
=
task
,
args
=
args
,
kwargs
=
kwargs
)
return
self
.
dropbox
.
put
(
package
)


def put_multiple(self, task_args_kwargs_list):
        """put a list of tasks and their arguments

        This method can be used to put multiple tasks at once. Calling
        this method once with multiple tasks can be much faster than
        calling `put()` multiple times.

        Parameters
        ----------
        task_args_kwargs_list : list

            A list of lists with three items that can be parameters of
            `put()`, i.e., `task`, `args`, `kwargs`.

        Returns
        -------
        list
            A list of task IDs.

        """
        if not self.isopen:
            logger = logging.getLogger(__name__)
            logger.warning('the drop box is not open')
            return

        packages = [ ]
        for t in task_args_kwargs_list:
            try:
                task = t['task']
                args = t.get('args', ())
                kwargs = t.get('kwargs', {})
                package = TaskPackage(task=task, args=args, kwargs=kwargs)
            except TypeError:
                package = TaskPackage(task=t, args=(), kwargs={})
            packages.append(package)
        return self.dropbox.put_multiple(packages)

def
put_multiple
(
self
,
task_args_kwargs_list
)
:
if
not
self
.
isopen
:
logger
=
logging
.
getLogger
(
__name__
)
logger
.
warning
(
'the drop box is not open'
)
return
packages
=
[
]
for
t
in
task_args_kwargs_list
:
try
:
task
=
t
[
'task'
]
args
=
t
.
get
(
'args'
,
(
)
)
kwargs
=
t
.
get
(
'kwargs'
,
{
}
)
package
=
TaskPackage
(
task
=
task
,
args
=
args
,
kwargs
=
kwargs
)
except
TypeError
:
package
=
TaskPackage
(
task
=
t
,
args
=
(
)
,
kwargs
=
{
}
)
packages
.
append
(
package
)
return
self
.
dropbox
.
put_multiple
(
packages
)


def receive_finished(self):
        """return a list of pairs of IDs and results of finished tasks.

        This method doesn't wait for tasks to finish. It returns IDs
        and results which have already finished.

        Returns
        -------
        list
            A list of pairs of IDs and results

        """
        if not self.isopen:
            logger = logging.getLogger(__name__)
            logger.warning('the drop box is not open')
            return
        return self.dropbox.poll()

def
receive_finished
(
self
)
:
if
not
self
.
isopen
:
logger
=
logging
.
getLogger
(
__name__
)
logger
.
warning
(
'the drop box is not open'
)
return
return
self
.
dropbox
.
poll
(
)

return a list of pairs of IDs and results of finished tasks .


def receive_one(self):
        """return a pair of an ID and a result of a task.

        This method waits for a task to finish.

        Returns
        -------
        An ID and a result of a task. `None` if no task is running.

        """
        if not self.isopen:
            logger = logging.getLogger(__name__)
            logger.warning('the drop box is not open')
            return
        return self.dropbox.receive_one()

def
receive_one
(
self
)
:
if
not
self
.
isopen
:
logger
=
logging
.
getLogger
(
__name__
)
logger
.
warning
(
'the drop box is not open'
)
return
return
self
.
dropbox
.
receive_one
(
)

return a pair of an ID and a result of a task .


def receive_all(self):
        """return a list of pairs of IDs and results of all tasks.

        This method waits for all tasks to finish.

        Returns
        -------
        list
            A list of pairs of IDs and results

        """
        if not self.isopen:
            logger = logging.getLogger(__name__)
            logger.warning('the drop box is not open')
            return
        return self.dropbox.receive()

def
receive_all
(
self
)
:
if
not
self
.
isopen
:
logger
=
logging
.
getLogger
(
__name__
)
logger
.
warning
(
'the drop box is not open'
)
return
return
self
.
dropbox
.
receive
(
)

return a list of pairs of IDs and results of all tasks .


def receive(self):
        """return a list results of all tasks.

        This method waits for all tasks to finish.

        Returns
        -------
        list
            A list of results of the tasks. The results are sorted in
            the order in which the tasks are put.

        """
        pkgidx_result_pairs = self.receive_all()
        if pkgidx_result_pairs is None:
            return
        results = [r for _, r in pkgidx_result_pairs]
        return results

def receive ( self ) : pkgidx_result_pairs = self . receive_all ( ) if pkgidx_result_pairs is None : return results = [ r for _ , r in pkgidx_result_pairs ] return results


def expand_path_cfg(path_cfg, alias_dict={ }, overriding_kargs={ }):
    """expand a path config

    Args:
        path_cfg (str, tuple, dict): a config for path
        alias_dict (dict): a dict for aliases
        overriding_kargs (dict): to be used for recursive call
    """

    if isinstance(path_cfg, str):
        return _expand_str(path_cfg, alias_dict, overriding_kargs)

    if isinstance(path_cfg, dict):
        return _expand_dict(path_cfg, alias_dict)

    # assume tuple or list
    return _expand_tuple(path_cfg, alias_dict, overriding_kargs)

def
expand_path_cfg
(
path_cfg
,
alias_dict
=
{
}
,
overriding_kargs
=
{
}
)
:
if
isinstance
(
path_cfg
,
str
)
:
return
_expand_str
(
path_cfg
,
alias_dict
,
overriding_kargs
)
if
isinstance
(
path_cfg
,
dict
)
:
return
_expand_dict
(
path_cfg
,
alias_dict
)
# assume tuple or list
return
_expand_tuple
(
path_cfg
,
alias_dict
,
overriding_kargs
)


def _expand_tuple(path_cfg, alias_dict, overriding_kargs):
    """expand a path config given as a tuple

    """

    # e.g.,
    # path_cfg = ('ev : {low} <= ev.var[0] < {high}', {'low': 10, 'high': 200})
    # overriding_kargs = {'alias': 'var_cut', 'name': 'var_cut25', 'low': 25}

    new_path_cfg = path_cfg[0]
    # e.g., 'ev : {low} <= ev.var[0] < {high}'

    new_overriding_kargs = path_cfg[1].copy()
    # e.g., {'low': 10, 'high': 200}

    new_overriding_kargs.update(overriding_kargs)
    # e.g., {'low': 25, 'high': 200, 'alias': 'var_cut', 'name': 'var_cut25'}

    return expand_path_cfg(
        new_path_cfg,
        overriding_kargs=new_overriding_kargs,
        alias_dict=alias_dict
    )


def poll(self):
        """check if the jobs are running and return a list of pids for
        finished jobs

        """
        finished_procs = [p for p in self.running_procs if p.poll() is not None]
        self.running_procs = collections.deque([p for p in self.running_procs if p not in finished_procs])

        for proc in finished_procs:
            stdout, stderr = proc.communicate()
            ## proc.communicate() returns (stdout, stderr) when
            ## self.pipe = True. Otherwise they are (None, None)

        finished_pids = [p.pid for p in finished_procs]
        self.finished_pids.extend(finished_pids)

        logger = logging.getLogger(__name__)
        messages = 'Running: {}, Finished: {}'.format(len(self.running_procs), len(self.finished_pids))
        logger.info(messages)

        return finished_pids


def wait(self):
        """wait until all jobs finish and return a list of pids
        """
        finished_pids = [ ]
        while self.running_procs:
            finished_pids.extend(self.poll())
        return finished_pids

def wait ( self ) : finished_pids = [ ] while self . running_procs : finished_pids . extend ( self . poll ( ) ) return finished_pids


def getVector(self, tree, branchName):
        """return the ROOT.vector object for the branch.

        """

        if (tree, branchName) in self.__class__.addressDict:
            return self.__class__.addressDict[(tree, branchName)]

        itsVector = self._getVector(tree, branchName)
        self.__class__.addressDict[(tree, branchName)] = itsVector

        return itsVector

def getVector ( self , tree , branchName ) : if ( tree , branchName ) in self . __class__ . addressDict : return self . __class__ . addressDict [ ( tree , branchName ) ] itsVector = self . _getVector ( tree , branchName ) self . __class__ . addressDict [ ( tree , branchName ) ] = itsVector return itsVector

return the ROOT . vector object for the branch .


def configure(self, component, all_dependencies):
        ''' Ensure all config-time files have been generated. Return a
            dictionary of generated items.
        '''
        r = {}

        builddir = self.buildroot

        # only dependencies which are actually valid can contribute to the
        # config data (which includes the versions of all dependencies in its
        # build info) if the dependencies aren't available we can't tell what
        # version they are. Anything missing here should always be a test
        # dependency that isn't going to be used, otherwise the yotta build
        # command will fail before we get here
        available_dependencies = OrderedDict((k, v) for k, v in all_dependencies.items() if v)

        self.set_toplevel_definitions = ''
        if self.build_info_include_file is None:
            self.build_info_include_file, build_info_definitions = self.getBuildInfo(component.path, builddir)
            self.set_toplevel_definitions += build_info_definitions

        if self.config_include_file is None:
            self.config_include_file, config_definitions, self.config_json_file = self._getConfigData(available_dependencies, component, builddir, self.build_info_include_file)
            self.set_toplevel_definitions += config_definitions

        self.configured = True
        return {
            'merged_config_include': self.config_include_file,
               'merged_config_json': self.config_json_file,
               'build_info_include': self.build_info_include_file
        }

Ensure all config - time files have been generated . Return a dictionary of generated items .


def _getTarball(url, into_directory, cache_key, origin_info=None):
    '''unpack the specified tarball url into the specified directory'''

    try:
        access_common.unpackFromCache(cache_key, into_directory)
    except KeyError as e:
        tok = settings.getProperty('github', 'authtoken')
        headers = {}
        if tok is not None:
            headers['Authorization'] = 'token ' + str(tok)

        logger.debug('GET %s', url)
        response = requests.get(url, allow_redirects=True, stream=True, headers=headers)
        response.raise_for_status()

        logger.debug('getting file: %s', url)
        logger.debug('headers: %s', response.headers)
        response.raise_for_status()

        # github doesn't exposes hashes of the archives being downloaded as far
        # as I can tell :(
        access_common.unpackTarballStream(
                    stream = response,
            into_directory = into_directory,
                      hash = {},
                 cache_key = cache_key,
               origin_info = origin_info
        )


def availableVersions(self):
        ''' return a list of Version objects, each with a tarball URL set '''
        r = []
        for t in self._getTags():
            logger.debug("available version tag: %s", t)
            # ignore empty tags:
            if not len(t[0].strip()):
                continue
            try:
                r.append(GithubComponentVersion(t[0], t[0], url=t[1], name=self.name, cache_key=None))
            except ValueError:
                logger.debug('invalid version tag: %s', t)

        return r

def
availableVersions
(
self
)
:
r
=
[
]
for
t
in
self
.
_getTags
(
)
:
logger
.
debug
(
"available version tag: %s"
,
t
)
# ignore empty tags:
if
not
len
(
t
[
0
]
.
strip
(
)
)
:
continue
try
:
r
.
append
(
GithubComponentVersion
(
t
[
0
]
,
t
[
0
]
,
url
=
t
[
1
]
,
name
=
self
.
name
,
cache_key
=
)
)
except
ValueError
:
logger
.
debug
(
'invalid version tag: %s'
,
t
)
return
r

return a list of Version objects each with a tarball URL set


def availableTags(self):
        ''' return a list of GithubComponentVersion objects for all tags
        '''
        return [
            GithubComponentVersion(
                '', t[0], t[1], self.name, cache_key=_createCacheKey('tag', t[0], t[1], self.name)
            ) for t in self._getTags()
        ]

def availableTags ( self ) : return [ GithubComponentVersion ( '' , t [ 0 ] , t [ 1 ] , self . name , cache_key = _createCacheKey ( 'tag' , t [ 0 ] , t [ 1 ] , self . name ) ) for t in self . _getTags ( ) ]


def availableBranches(self):
        ''' return a list of GithubComponentVersion objects for the tip of each branch
        '''
        return [
            GithubComponentVersion(
                '', b[0], b[1], self.name, cache_key=None
            ) for b in _getBranchHeads(self.repo).items()
        ]

def availableBranches ( self ) : return [ GithubComponentVersion ( '' , b [ 0 ] , b [ 1 ] , self . name , cache_key = None ) for b in _getBranchHeads ( self . repo ) . items ( ) ]


def _raiseUnavailableFor401(message):
    ''' Returns a decorator to swallow a requests exception for modules that
        are not accessible without logging in, and turn it into an Unavailable
        exception.
    '''
    def __raiseUnavailableFor401(fn):
        def wrapped(*args, **kwargs):
            try:
                return fn(*args, **kwargs)
            except requests.exceptions.HTTPError as e:
                if e.response.status_code == requests.codes.unauthorized:
                    raise access_common.Unavailable(message)
                else:
                    raise
        return wrapped
    return __raiseUnavailableFor401

def _raiseUnavailableFor401 ( message ) : def __raiseUnavailableFor401 ( fn ) : def wrapped ( * args , * * kwargs ) : try : return fn ( * args , * * kwargs ) except requests . exceptions . HTTPError as e : if e . response . status_code == requests . codes . unauthorized : raise access_common . Unavailable ( message ) else : raise return wrapped return __raiseUnavailableFor401

Returns a decorator to swallow a requests exception for modules that are not accessible without logging in and turn it into an Unavailable exception .


def unpublish(namespace, name, version, registry=None):
    ''' Try to unpublish a recently published version. Return any errors that
        occur.
    '''
    registry = registry or Registry_Base_URL

    url = '%s/%s/%s/versions/%s' % (
        registry,
        namespace,
        name,
        version
    )

    headers = _headersForRegistry(registry)
    response = requests.delete(url, headers=headers)
    response.raise_for_status()

    return None

def unpublish ( namespace , name , version , registry = None ) : registry = registry or Registry_Base_URL url = '%s/%s/%s/versions/%s' % ( registry , namespace , name , version ) headers = _headersForRegistry ( registry ) response = requests . delete ( url , headers = headers ) response . raise_for_status ( ) return None

Try to unpublish a recently published version . Return any errors that occur .


def read(self, filenames):
        '''' Read a list of files. Their configuration values are merged, with
             preference to values from files earlier in the list.
        '''
        for fn in filenames:
            try:
                self.configs[fn] = ordered_json.load(fn)
            except IOError:
                self.configs[fn] = OrderedDict()
            except Exception as e:
                self.configs[fn] = OrderedDict()
                logging.warning(
                    "Failed to read settings file %s, it will be ignored. The error was: %s",
                    fn, e
                )

Read a list of files . Their configuration values are merged with preference to values from files earlier in the list .


def get(self, path):
        ''' return a configuration value

            usage:
                get('section.property')

            Note that currently array indexes are not supported. You must
            get the whole array.

            returns None if any path element or the property is missing
        '''
        path = _splitPath(path)
        for config in self.configs.values():
            cur = config
            for el in path:
                if el in cur:
                    cur = cur[el]
                else:
                    cur = None
                    break
            if cur is not None:
                return cur
        return None

def get ( self , path ) : path = _splitPath ( path ) for config in self . configs . values ( ) : cur = config for el in path : if el in cur : cur = cur [ el ] else : cur = None break if cur is not None : return cur return None


def set(self, path, value=None, filename=None):
        ''' Set a configuration value. If no filename is specified, the
            property is set in the first configuration file. Note that if a
            filename is specified and the property path is present in an
            earlier filename then set property will be hidden.

            usage:
                set('section.property', value='somevalue')

            Note that currently array indexes are not supported. You must
            set the whole array.
        '''
        if filename is None:
            config = self._firstConfig()[1]
        else:
            config = self.configs[filename]

        path = _splitPath(path)
        for el in path[:-1]:
            if el in config:
                config = config[el]
            else:
                config[el] = OrderedDict()
                config = config[el]
        config[path[-1]] = value

def set ( self , path , value = None , filename = None ) : if filename is None : config = self . _firstConfig ( ) [ 1 ] else : config = self . configs [ filename ] path = _splitPath ( path ) for el in path [ : - 1 ] : if el in config : config = config [ el ] else : config [ el ] = OrderedDict ( ) config = config [ el ] config [ path [ - 1 ] ] = value

Set a configuration value . If no filename is specified the property is set in the first configuration file . Note that if a filename is specified and the property path is present in an earlier filename then set property will be hidden .


def islast(generator):
    ''' indicate whether the current item is the last one in a generator
    '''
    next_x = None
    first = True
    for x in generator:
        if not first:
            yield (next_x, False)
        next_x = x
        first = False
    if not first:
        yield (next_x, True)

def islast ( generator ) : next_x = None first = True for x in generator : if not first : yield ( next_x , False ) next_x = x first = False if not first : yield ( next_x , True )


def sourceDirValidationError(dirname, component_name):
    ''' validate source directory names in components '''
    if dirname == component_name:
        return 'Module %s public include directory %s should not contain source files' % (component_name, dirname)
    elif dirname.lower() in ('source', 'src') and dirname != 'source':
        return 'Module %s has non-standard source directory name: "%s" should be "source"' % (component_name, dirname)
    elif isPotentialTestDir(dirname) and dirname != 'test':
        return 'Module %s has non-standard test directory name: "%s" should be "test"' % (component_name, dirname)
    elif not Source_Dir_Regex.match(dirname):
        corrected = Source_Dir_Invalid_Regex.sub('', dirname.lower())
        if not corrected:
            corrected = 'source'
        return 'Module %s has non-standard source directory name: "%s" should be "%s"' % (component_name, dirname, corrected)
    else:
        return None


def displayOutdated(modules, dependency_specs, use_colours):
    ''' print information about outdated modules,
        return 0 if there is nothing to be done and nonzero otherwise
    '''
    if use_colours:
        DIM    = colorama.Style.DIM       #pylint: disable=no-member
        NORMAL = colorama.Style.NORMAL    #pylint: disable=no-member
        BRIGHT = colorama.Style.BRIGHT    #pylint: disable=no-member
        YELLOW = colorama.Fore.YELLOW     #pylint: disable=no-member
        RED    = colorama.Fore.RED        #pylint: disable=no-member
        GREEN  = colorama.Fore.GREEN      #pylint: disable=no-member
        RESET  = colorama.Style.RESET_ALL #pylint: disable=no-member
    else:
        DIM = BRIGHT = YELLOW = RED = GREEN = RESET = u''

    status = 0

    # access, , get components, internal
    from yotta.lib import access
    from yotta.lib import access_common
    # sourceparse, , parse version source urls, internal
    from yotta.lib import sourceparse

    for name, m in modules.items():
        if m.isTestDependency():
            continue
        try:
            latest_v = access.latestSuitableVersion(name, '*', registry='modules', quiet=True)
        except access_common.Unavailable as e:
            latest_v = None

        if not m:
            m_version = u' ' + RESET + BRIGHT + RED + u"missing" + RESET
        else:
            m_version = DIM + u'@%s' % (m.version)
        if not latest_v:
            print(u'%s%s%s%s not available from the registry%s' % (RED, name, m_version, NORMAL, RESET))
            status = 2
            continue
        elif not m or m.version < latest_v:
            update_prevented_by = ''
            if m:
                specs_preventing_update = [
                    x for x in dependency_specs
                    if x.name == name and not
                       sourceparse.parseSourceURL(x.nonShrinkwrappedVersionReq()).semanticSpecMatches(latest_v)
                ]
                shrinkwrap_prevents_update = [
                    x for x in dependency_specs
                    if x.name == name and x.isShrinkwrapped() and not
                       sourceparse.parseSourceURL(x.versionReq()).semanticSpecMatches(latest_v)
                ]
                if len(specs_preventing_update):
                    update_prevented_by = ' (update prevented by specifications: %s)' % (
                        ', '.join(['%s from %s' % (x.version_req, x.specifying_module) for x in specs_preventing_update])
                    )
                if len(shrinkwrap_prevents_update):
                    update_prevented_by += ' yotta-shrinkwrap.json prevents update'
                if m.version.major() < latest_v.major():
                    # major versions being outdated might be deliberate, so not
                    # that bad:
                    colour = GREEN
                elif m.version.minor() < latest_v.minor():
                    # minor outdated versions is moderately bad
                    colour = YELLOW
                else:
                    # patch-outdated versions is really bad, because there should
                    # be no reason not to update:
                    colour = RED
            else:
                colour = RED
            print(u'%s%s%s latest: %s%s%s%s' % (name, m_version, RESET, colour, latest_v.version, update_prevented_by, RESET))
            if not status:
                status = 1
    return status

print information about outdated modules return 0 if there is nothing to be done and nonzero otherwise


def ignores(self, path):
        ''' Test if this module ignores the file at "path", which must be a
            path relative to the root of the module.

            If a file is within a directory that is ignored, the file is also
            ignored.
        '''
        test_path = PurePath('/', path)

        # also check any parent directories of this path against the ignore
        # patterns:
        test_paths = tuple([test_path] + list(test_path.parents))

        for exp in self.ignore_patterns:
            for tp in test_paths:
                if tp.match(exp):
                    logger.debug('"%s" ignored ("%s" matched "%s")', path, tp, exp)
                    return True
        return False

Test if this module ignores the file at path which must be a path relative to the root of the module .


def publish(self, registry=None):
        ''' Publish to the appropriate registry, return a description of any
            errors that occured, or None if successful.
            No VCS tagging is performed.
        '''
        if (registry is None) or (registry == registry_access.Registry_Base_URL):
            if 'private' in self.description and self.description['private']:
                return "this %s is private and cannot be published" % (self.description_filename.split('.')[0])
        upload_archive = os.path.join(self.path, 'upload.tar.gz')
        fsutils.rmF(upload_archive)
        fd = os.open(upload_archive, os.O_CREAT | os.O_EXCL | os.O_RDWR | getattr(os, "O_BINARY", 0))
        with os.fdopen(fd, 'rb+') as tar_file:
            tar_file.truncate()
            self.generateTarball(tar_file)
            logger.debug('generated tar file of length %s', tar_file.tell())
            tar_file.seek(0)
            # calculate the hash of the file before we upload it:
            shasum = hashlib.sha256()
            while True:
                chunk = tar_file.read(1000)
                if not chunk:
                    break
                shasum.update(chunk)
            logger.debug('generated tar file has hash %s', shasum.hexdigest())
            tar_file.seek(0)
            with self.findAndOpenReadme() as readme_file_wrapper:
                if not readme_file_wrapper:
                    logger.warning("no readme.md file detected")
                with open(self.getDescriptionFile(), 'r') as description_file:
                    return registry_access.publish(
                        self.getRegistryNamespace(),
                        self.getName(),
                        self.getVersion(),
                        description_file,
                        tar_file,
                        readme_file_wrapper.file,
                        readme_file_wrapper.extension().lower(),
                        registry=registry
                    )

Publish to the appropriate registry return a description of any errors that occured or None if successful . No VCS tagging is performed .


def unpublish(self, registry=None):
        ''' Try to un-publish the current version. Return a description of any
            errors that occured, or None if successful.
        '''
        return registry_access.unpublish(
            self.getRegistryNamespace(),
            self.getName(),
            self.getVersion(),
            registry=registry
        )

def unpublish ( self , registry = None ) : return registry_access . unpublish ( self . getRegistryNamespace ( ) , self . getName ( ) , self . getVersion ( ) , registry = registry )

Try to un - publish the current version . Return a description of any errors that occured or None if successful .


def getScript(self, scriptname):
        ''' Return the specified script command. If the first part of the
            command is a .py file, then the current python interpreter is
            prepended.

            If the script is a single string, rather than an array, it is
            shlex-split.
        '''
        script = self.description.get('scripts', {}).get(scriptname, None)
        if script is not None:
            if isinstance(script, str) or isinstance(script, type(u'unicode string')):
                import shlex
                script = shlex.split(script)
            # if the command is a python script, run it with the python
            # interpreter being used to run yotta, also fetch the absolute path
            # to the script relative to this module (so that the script can be
            # distributed with the module, no matter what current working
            # directory it will be executed in):
            if len(script) and script[0].lower().endswith('.py'):
                if not os.path.isabs(script[0]):
                    absscript = os.path.abspath(os.path.join(self.path, script[0]))
                    logger.debug('rewriting script %s to be absolute path %s', script[0], absscript)
                    script[0] = absscript
                import sys
                script = [sys.executable] + script

        return script

Return the specified script command . If the first part of the command is a . py file then the current python interpreter is prepended .


def runScript(self, scriptname, additional_environment=None):
        ''' Run the specified script from the scripts section of the
            module.json file in the directory of this module.
        '''
        import subprocess
        import shlex

        command = self.getScript(scriptname)
        if command is None:
            logger.debug('%s has no script %s', self, scriptname)
            return 0

        if not len(command):
            logger.error("script %s of %s is empty", scriptname, self.getName())
            return 1

        # define additional environment variables for scripts:
        env = os.environ.copy()
        if additional_environment is not None:
            env.update(additional_environment)

        errcode = 0
        child = None
        try:
            logger.debug('running script: %s', command)
            child = subprocess.Popen(
                command, cwd = self.path, env = env
            )
            child.wait()
            if child.returncode:
                logger.error(
                    "script %s (from %s) exited with non-zero status %s",
                    scriptname,
                    self.getName(),
                    child.returncode
                )
                errcode = child.returncode
            child = None
        finally:
            if child is not None:
                tryTerminate(child)
        return errcode

Run the specified script from the scripts section of the module . json file in the directory of this module .


def hasDependency(self, name, target=None, test_dependencies=False):
        ''' Check if this module has any dependencies with the specified name
            in its dependencies list, or in target dependencies for the
            specified target
        '''
        if name in self.description.get('dependencies', {}).keys():
            return True

        target_deps = self.description.get('targetDependencies', {})
        if target is not None:
            for conf_key, target_conf_deps in target_deps.items():
                if _truthyConfValue(target.getConfigValue(conf_key)) or conf_key in target.getSimilarTo_Deprecated():
                    if name in target_conf_deps:
                        return True

        if test_dependencies:
            if name in self.description.get('testDependencies', {}).keys():
                return True

            if target is not None:
                test_target_deps = self.description.get('testTargetDependencies', {})
                for conf_key, target_conf_deps in test_target_deps.items():
                    if _truthyConfValue(target.getConfigValue(conf_key)) or conf_key in target.getSimilarTo_Deprecated():
                        if name in target_conf_deps:
                            return True
        return False

def hasDependency ( self , name , target = None , test_dependencies = False ) : if name in self . description . get ( 'dependencies' , { } ) . keys ( ) : return True target_deps = self . description . get ( 'targetDependencies' , { } ) if target is not None : for conf_key , target_conf_deps in target_deps . items ( ) : if _truthyConfValue ( target . getConfigValue ( conf_key ) ) or conf_key in target . getSimilarTo_Deprecated ( ) : if name in target_conf_deps : return True if test_dependencies : if name in self . description . get ( 'testDependencies' , { } ) . keys ( ) : return True if target is not None : test_target_deps = self . description . get ( 'testTargetDependencies' , { } ) for conf_key , target_conf_deps in test_target_deps . items ( ) : if _truthyConfValue ( target . getConfigValue ( conf_key ) ) or conf_key in target . getSimilarTo_Deprecated ( ) : if name in target_conf_deps : return True return False

Check if this module has any dependencies with the specified name in its dependencies list or in target dependencies for the specified target


def hasDependencyRecursively(self, name, target=None, test_dependencies=False):
        ''' Check if this module, or any of its dependencies, have a
            dependencies with the specified name in their dependencies, or in
            their targetDependencies corresponding to the specified target.

            Note that if recursive dependencies are not installed, this test
            may return a false-negative.
        '''
        # checking dependencies recursively isn't entirely straightforward, so
        # use the existing method to resolve them all before checking:
        dependencies = self.getDependenciesRecursive(
                               target = target,
                                 test = test_dependencies
        )
        return (name in dependencies)

Check if this module or any of its dependencies have a dependencies with the specified name in their dependencies or in their targetDependencies corresponding to the specified target .


def satisfyDependenciesRecursive(
                            self,
            available_components = None,
                     search_dirs = None,
                update_installed = False,
                  traverse_links = False,
                          target = None,
                            test = False
        ):
        ''' Retrieve and install all the dependencies of this component and its
            dependencies, recursively, or satisfy them from a collection of
            available_components or from disk.

            Returns
            =======
                (components, errors)

                components: dictionary of name:Component
                errors: sequence of errors

            Parameters
            ==========

                available_components:
                    None (default) or a dictionary of name:component. This is
                    searched before searching directories or fetching remote
                    components

                search_dirs:
                    None (default), or sequence of directories to search for
                    already installed, (but not yet loaded) components. Used so
                    that manually installed or linked components higher up the
                    dependency tree are found by their users lower down.

                    These directories are searched in order, and finally the
                    current directory is checked.

                update_installed:
                    False (default), True, or set(): whether to check the
                    available versions of installed components, and update if a
                    newer version is available. If this is a set(), only update
                    things in the specified set.

                traverse_links:
                    False (default) or True: whether to recurse into linked
                    dependencies when updating/installing.

                target:
                    None (default), or a Target object. If specified the target
                    name and it's similarTo list will be used in resolving
                    dependencies. If None, then only target-independent
                    dependencies will be installed

                test:
                    True, False, or 'toplevel: should test-only dependencies be
                    installed? (yes, no, or only for this module, not its
                    dependencies).

        '''
        def provider(
            dspec,
            available_components,
            search_dirs,
            working_directory,
            update_installed,
            dep_of=None
        ):
            r = access.satisfyFromAvailable(dspec.name, available_components)
            if r:
                if r.isTestDependency() and not dspec.is_test_dependency:
                    logger.debug('test dependency subsequently occurred as real dependency: %s', r.getName())
                    r.setTestDependency(False)
                return r
            update_if_installed = False
            if update_installed is True:
                update_if_installed = True
            elif update_installed:
                update_if_installed = dspec.name in update_installed
            r = access.satisfyVersionFromSearchPaths(
                dspec.name,
                dspec.versionReq(),
                search_dirs,
                update_if_installed,
                inherit_shrinkwrap = dep_of.getShrinkwrap()
            )
            if r:
                r.setTestDependency(dspec.is_test_dependency)
                return r
            # before resorting to install this module, check if we have an
            # existing linked module (which wasn't picked up because it didn't
            # match the version specification) - if we do, then we shouldn't
            # try to install, but should return that anyway:
            default_path = os.path.join(self.modulesPath(), dspec.name)
            if fsutils.isLink(default_path):
                r = Component(
                                       default_path,
                     test_dependency = dspec.is_test_dependency,
                    installed_linked = fsutils.isLink(default_path),
                  inherit_shrinkwrap = dep_of.getShrinkwrap()
                )
                if r:
                    assert(r.installedLinked())
                    return r
                else:
                    logger.error('linked module %s is invalid: %s', dspec.name, r.getError())
                    return r

            r = access.satisfyVersionByInstalling(
                dspec.name,
                dspec.versionReq(),
                self.modulesPath(),
                inherit_shrinkwrap = dep_of.getShrinkwrap()
            )
            if not r:
                logger.error('could not install %s' % dspec.name)
            if r is not None:
                r.setTestDependency(dspec.is_test_dependency)
            return r

        return self.__getDependenciesRecursiveWithProvider(
           available_components = available_components,
                    search_dirs = search_dirs,
                         target = target,
                 traverse_links = traverse_links,
               update_installed = update_installed,
                       provider = provider,
                           test = test
        )

Retrieve and install all the dependencies of this component and its dependencies recursively or satisfy them from a collection of available_components or from disk .


def getExtraIncludes(self):
        ''' Some components must export whole directories full of headers into
            the search path. This is really really bad, and they shouldn't do
            it, but support is provided as a concession to compatibility.
        '''
        if 'extraIncludes' in self.description:
            return [os.path.normpath(x) for x in self.description['extraIncludes']]
        else:
            return []

def getExtraIncludes ( self ) : if 'extraIncludes' in self . description : return [ os . path . normpath ( x ) for x in self . description [ 'extraIncludes' ] ] else : return [ ]

Some components must export whole directories full of headers into the search path . This is really really bad and they shouldn t do it but support is provided as a concession to compatibility .


def availableVersions(self):
        ''' return a list of GitCloneVersion objects for tags which are valid
            semantic version idenfitifiers.
        '''
        r = []
        for t in self.vcs.tags():
            logger.debug("available version tag: %s", t)
            # ignore empty tags:
            if not len(t.strip()):
                continue
            try:
                r.append(GitCloneVersion(t, t, self))
            except ValueError:
                logger.debug('invalid version tag: %s', t)
        return r

def
availableVersions
(
self
)
:
r
=
[
]
for
t
in
self
.
vcs
.
tags
(
)
:
logger
.
debug
(
"available version tag: %s"
,
t
)
# ignore empty tags:
if
not
len
(
t
.
strip
(
)
)
:
continue
try
:
r
.
append
(
GitCloneVersion
(
t
,
t
,
self
)
)
except
ValueError
:
logger
.
debug
(
'invalid version tag: %s'
,
t
)
return
r

return a list of GitCloneVersion objects for tags which are valid semantic version idenfitifiers .


def _mergeDictionaries(*args):
    ''' merge dictionaries of dictionaries recursively, with elements from
        dictionaries earlier in the argument sequence taking precedence
    '''
    # to support merging of OrderedDicts, copy the result type from the first
    # argument:
    result = type(args[0])()
    for k, v in itertools.chain(*[x.items() for x in args]):
        if not k in result:
            result[k] = v
        elif isinstance(result[k], dict) and isinstance(v, dict):
            result[k] = _mergeDictionaries(result[k], v)
    return result

merge dictionaries of dictionaries recursively with elements from dictionaries earlier in the argument sequence taking precedence


def _mirrorStructure(dictionary, value):
    ''' create a new nested dictionary object with the same structure as
        'dictionary', but with all scalar values replaced with 'value'
    '''
    result = type(dictionary)()
    for k in dictionary.keys():
        if isinstance(dictionary[k], dict):
            result[k] = _mirrorStructure(dictionary[k], value)
        else:
            result[k] = value
    return result

def _mirrorStructure ( dictionary , value ) : result = type ( dictionary ) ( ) for k in dictionary . keys ( ) : if isinstance ( dictionary [ k ] , dict ) : result [ k ] = _mirrorStructure ( dictionary [ k ] , value ) else : result [ k ] = value return result

create a new nested dictionary object with the same structure as dictionary but with all scalar values replaced with value


def baseTargetSpec(self):
        ''' returns pack.DependencySpec for the base target of this target (or
            None if this target does not inherit from another target.
        '''
        inherits = self.description.get('inherits', {})
        if len(inherits) == 1:
            name, version_req = list(inherits.items())[0]
            shrinkwrap_version_req = self.getShrinkwrapMapping('targets').get(name, None)
            if shrinkwrap_version_req is not None:
                logger.debug(
                    'respecting shrinkwrap version %s for %s', shrinkwrap_version_req, name
                )
            return pack.DependencySpec(
                name,
                version_req,
                shrinkwrap_version_req = shrinkwrap_version_req
            )
        elif len(inherits) > 1:
            logger.error('target %s specifies multiple base targets, but only one is allowed', self.getName())
        return None

returns pack . DependencySpec for the base target of this target ( or None if this target does not inherit from another target .


def _loadConfig(self):
        ''' load the configuration information from the target hierarchy '''
        config_dicts = [self.additional_config, self.app_config] + [t.getConfig() for t in self.hierarchy]
        # create an identical set of dictionaries, but with the names of the
        # sources in place of the values. When these are merged they will show
        # where each merged property came from:
        config_blame = [
            _mirrorStructure(self.additional_config, 'command-line config'),
            _mirrorStructure(self.app_config, 'application\'s config.json'),
        ] + [
            _mirrorStructure(t.getConfig(), t.getName()) for t in self.hierarchy
        ]

        self.config = _mergeDictionaries(*config_dicts)
        self.config_blame = _mergeDictionaries(*config_blame)


def inheritsFrom(self, target_name):
        ''' Return true if this target inherits from the named target (directly
            or indirectly. Also returns true if this target is the named
            target. Otherwise return false.
        '''
        for t in self.hierarchy:
            if t and t.getName() == target_name or target_name in t.description.get('inherits', {}):
                return True
        return False

def inheritsFrom ( self , target_name ) : for t in self . hierarchy : if t and t . getName ( ) == target_name or target_name in t . description . get ( 'inherits' , { } ) : return True return False

Return true if this target inherits from the named target ( directly or indirectly . Also returns true if this target is the named target . Otherwise return false .


def exec_helper(self, cmd, builddir):
        ''' Execute the given command, returning an error message if an error occured
            or None if the command was succesful.'''
        try:
            child = subprocess.Popen(cmd, cwd=builddir)
            child.wait()
        except OSError as e:
            if e.errno == errno.ENOENT:
                if cmd[0] == 'cmake':
                    return 'CMake is not installed, please follow the installation instructions at http://docs.yottabuild.org/#installing'
                else:
                    return '%s is not installed' % (cmd[0])
            else:
                return 'command %s failed' % (cmd)
        if child.returncode:
            return 'command %s failed' % (cmd)

Execute the given command returning an error message if an error occured or None if the command was succesful .


def build(self, builddir, component, args, release_build=False, build_args=None, targets=None,
              release_no_debug_info_build=False):
        ''' Execute the commands necessary to build this component, and all of
            its dependencies. '''
        if build_args is None:
            build_args = []
        if targets is None:
            targets = []
        # in the future this may be specified in the target description, but
        # for now we only support cmake, so everything is simple:
        if release_no_debug_info_build:
            build_type = 'Release'
        elif release_build:
            build_type = 'RelWithDebInfo'
        else:
            build_type = 'Debug'
        cmd = ['cmake', '-D', 'CMAKE_BUILD_TYPE=%s' % build_type, '-G', args.cmake_generator, '.']
        res = self.exec_helper(cmd, builddir)
        if res is not None:
            return res

        # work-around various yotta-specific issues with the generated
        # Ninja/project files:
        from yotta.lib import cmake_fixups
        cmake_fixups.applyFixupsForFenerator(args.cmake_generator, builddir, component)

        build_command = self.overrideBuildCommand(args.cmake_generator, targets=targets)
        if build_command:
            cmd = build_command + build_args
        else:
            cmd = ['cmake', '--build', builddir]
            if len(targets):
                # !!! FIXME: support multiple targets with the default CMake
                # build command
                cmd += ['--target', targets[0]]
            cmd += build_args
        res = self.exec_helper(cmd, builddir)
        if res is not None:
            return res
        hint = self.hintForCMakeGenerator(args.cmake_generator, component)
        if hint:
            logger.info(hint)

Execute the commands necessary to build this component and all of its dependencies .


def findProgram(self, builddir, program):
        ''' Return the builddir-relative path of program, if only a partial
            path is specified. Returns None and logs an error message if the
            program is ambiguous or not found
	'''
        # if this is an exact match, do no further checking:
        if os.path.isfile(os.path.join(builddir, program)):
            logging.info('found %s' % program)
            return program
        exact_matches = []
        insensitive_matches = []
        approx_matches = []
        for path, dirs, files in os.walk(builddir):
            if program in files:
                exact_matches.append(os.path.relpath(os.path.join(path, program), builddir))
                continue
            files_lower = [f.lower() for f in files]
            if program.lower() in files_lower:
                insensitive_matches.append(
                    os.path.relpath(
                        os.path.join(path, files[files_lower.index(program.lower())]),
                        builddir
                    )
                )
                continue
            # !!! TODO: in the future add approximate string matching (typos,
            # etc.), for now we just test stripping any paths off program, and
            # looking for substring matches:
            pg_basen_lower_noext = os.path.splitext(os.path.basename(program).lower())[0]
            for f in files_lower:
                if pg_basen_lower_noext in f:
                    approx_matches.append(
                        os.path.relpath(
                            os.path.join(path, files[files_lower.index(f)]),
                            builddir
                        )
                    )

        if len(exact_matches) == 1:
            logging.info('found %s at %s', program, exact_matches[0])
            return exact_matches[0]
        elif len(exact_matches) > 1:
            logging.error(
                '%s matches multiple executables, please use a full path (one of %s)' % (
                    program,
                    ', or '.join(['"'+os.path.join(m, program)+'"' for m in exact_matches])
                )
            )
            return None
        # if we have matches with and without a file extension, prefer the
        # no-file extension version, and discard the others (so we avoid
        # picking up post-processed files):
        reduced_approx_matches = []
        for m in approx_matches:
            root = os.path.splitext(m)[0]
            if (m == root) or (root not in approx_matches):
                reduced_approx_matches.append(m)
        approx_matches = reduced_approx_matches

        for matches in (insensitive_matches, approx_matches):
            if len(matches) == 1:
                logging.info('found %s at %s' % (
                    program, matches[0]
                ))
                return matches[0]
            elif len(matches) > 1:
                logging.error(
                    '%s is similar to several executables found. Please use an exact name:\n%s' % (
                        program,
                        '\n'.join(matches)
                    )
                )
                return None
        logging.error('could not find program "%s" to debug' %  program)
        return None

Return the builddir - relative path of program if only a partial path is specified . Returns None and logs an error message if the program is ambiguous or not found


def start(self, builddir, program, forward_args):
        ''' Launch the specified program. Uses the `start` script if specified
            by the target, attempts to run it natively if that script is not
            defined.
        '''
        child = None
        try:
            prog_path = self.findProgram(builddir, program)
            if prog_path is None:
                return

            start_env, start_vars = self.buildProgEnvAndVars(prog_path, builddir)
            if self.getScript('start'):
                cmd = [
                    os.path.expandvars(string.Template(x).safe_substitute(**start_vars))
                    for x in self.getScript('start')
                ] + forward_args
            else:
                cmd = shlex.split('./' + prog_path) + forward_args

            logger.debug('starting program: %s', cmd)
            child = subprocess.Popen(
                cmd, cwd = builddir, env = start_env
            )
            child.wait()
            if child.returncode:
                return "process exited with status %s" % child.returncode
            child = None
        except OSError as e:
            import errno
            if e.errno == errno.ENOEXEC:
                return ("the program %s cannot be run (perhaps your target "+
                        "needs to define a 'start' script to start it on its "
                        "intended execution target?)") % prog_path
        finally:
            if child is not None:
                _tryTerminate(child)

Launch the specified program . Uses the start script if specified by the target attempts to run it natively if that script is not defined .


def pruneCache():
    ''' Prune the cache '''
    cache_dir = folders.cacheDirectory()
    def fullpath(f):
        return os.path.join(cache_dir, f)
    def getMTimeSafe(f):
        # it's possible that another process removed the file before we stat
        # it, handle this gracefully
        try:
            return os.stat(f).st_mtime
        except FileNotFoundError:
            import time
            return time.clock()
    # ensure cache exists
    fsutils.mkDirP(cache_dir)
    max_cached_modules = getMaxCachedModules()
    for f in sorted(
            [f for f in os.listdir(cache_dir) if
                os.path.isfile(fullpath(f)) and not f.endswith('.json') and not f.endswith('.locked')
            ],
            key = lambda f: getMTimeSafe(fullpath(f)),
            reverse = True
        )[max_cached_modules:]:
        cache_logger.debug('cleaning up cache file %s', f)
        removeFromCache(f)
    cache_logger.debug('cache pruned to %s items', max_cached_modules)


def sometimesPruneCache(p):
    ''' return decorator to prune cache after calling fn with a probability of p'''
    def decorator(fn):
        @functools.wraps(fn)
        def wrapped(*args, **kwargs):
            r = fn(*args, **kwargs)
            if random.random() < p:
                pruneCache()
            return r
        return wrapped
    return decorator

def sometimesPruneCache ( p ) : def decorator ( fn ) : @ functools . wraps ( fn ) def wrapped ( * args , * * kwargs ) : r = fn ( * args , * * kwargs ) if random . random ( ) < p : pruneCache ( ) return r return wrapped return decorator


def calibrateEB(variances, sigma2):
    """
    Calibrate noisy variance estimates with empirical Bayes.

    Parameters
    ----------
    vars: ndarray
        List of variance estimates.
    sigma2: int
        Estimate of the Monte Carlo noise in vars.

    Returns
    -------
    An array of the calibrated variance estimates
    """
    if (sigma2 <= 0 or min(variances) == max(variances)):
        return(np.maximum(variances, 0))
    sigma = np.sqrt(sigma2)
    eb_prior = gfit(variances, sigma)
    # Set up a partial execution of the function
    part = functools.partial(gbayes, g_est=eb_prior,
                             sigma=sigma)
    if len(variances) >= 200:
        # Interpolate to speed up computations:
        calib_x = np.percentile(variances,
                                np.arange(0, 102, 2))
        calib_y = list(map(part, calib_x))
        calib_all = np.interp(variances, calib_x, calib_y)
    else:
        calib_all = list(map(part, variances))

    return np.asarray(calib_all)

Calibrate noisy variance estimates with empirical Bayes .


def calc_inbag(n_samples, forest):
    """
    Derive samples used to create trees in scikit-learn RandomForest objects.

    Recovers the samples in each tree from the random state of that tree using
    :func:`forest._generate_sample_indices`.

    Parameters
    ----------
    n_samples : int
        The number of samples used to fit the scikit-learn RandomForest object.

    forest : RandomForest
        Regressor or Classifier object that is already fit by scikit-learn.

    Returns
    -------
    Array that records how many times a data point was placed in a tree.
    Columns are individual trees. Rows are the number of times a sample was
    used in a tree.
    """

    if not forest.bootstrap:
        e_s = "Cannot calculate the inbag from a forest that has "
        e_s = " bootstrap=False"
        raise ValueError(e_s)

    n_trees = forest.n_estimators
    inbag = np.zeros((n_samples, n_trees))
    sample_idx = []
    for t_idx in range(n_trees):
        sample_idx.append(
            _generate_sample_indices(forest.estimators_[t_idx].random_state,
                                     n_samples))
        inbag[:, t_idx] = np.bincount(sample_idx[-1], minlength=n_samples)
    return inbag

Derive samples used to create trees in scikit - learn RandomForest objects .


def _core_computation(X_train, X_test, inbag, pred_centered, n_trees,
                      memory_constrained=False, memory_limit=None,
                      test_mode=False):
    """
    Helper function, that performs the core computation

    Parameters
    ----------
    X_train : ndarray
        An array with shape (n_train_sample, n_features).

    X_test : ndarray
        An array with shape (n_test_sample, n_features).

    inbag : ndarray
        The inbag matrix that fit the data. If set to `None` (default) it
        will be inferred from the forest. However, this only works for trees
        for which bootstrapping was set to `True`. That is, if sampling was
        done with replacement. Otherwise, users need to provide their own
        inbag matrix.

    pred_centered : ndarray
        Centered predictions that are an intermediate result in the
        computation.

    memory_constrained: boolean (optional)
        Whether or not there is a restriction on memory. If False, it is
        assumed that a ndarry of shape (n_train_sample,n_test_sample) fits
        in main memory. Setting to True can actually provide a speed up if
        memory_limit is tuned to the optimal range.

    memory_limit: int (optional)
        An upper bound for how much memory the itermediate matrices will take
        up in Megabytes. This must be provided if memory_constrained=True.

    """
    if not memory_constrained:
        return np.sum((np.dot(inbag - 1, pred_centered.T) / n_trees) ** 2, 0)

    if not memory_limit:
        raise ValueError('If memory_constrained=True, must provide',
                         'memory_limit.')

    # Assumes double precision float
    chunk_size = int((memory_limit * 1e6) / (8.0 * X_train.shape[0]))

    if chunk_size == 0:
        min_limit = 8.0 * X_train.shape[0] / 1e6
        raise ValueError('memory_limit provided is too small.' +
                         'For these dimensions, memory_limit must ' +
                         'be greater than or equal to %.3e' % min_limit)

    chunk_edges = np.arange(0, X_test.shape[0] + chunk_size, chunk_size)
    inds = range(X_test.shape[0])
    chunks = [inds[chunk_edges[i]:chunk_edges[i+1]]
              for i in range(len(chunk_edges)-1)]
    if test_mode:
        print('Number of chunks: %d' % (len(chunks),))
    V_IJ = np.concatenate([
                np.sum((np.dot(inbag-1, pred_centered[chunk].T)/n_trees)**2, 0)
                for chunk in chunks])
    return V_IJ

Helper function that performs the core computation


def _bias_correction(V_IJ, inbag, pred_centered, n_trees):
    """
    Helper functions that implements bias correction

    Parameters
    ----------
    V_IJ : ndarray
        Intermediate result in the computation.

    inbag : ndarray
        The inbag matrix that fit the data. If set to `None` (default) it
        will be inferred from the forest. However, this only works for trees
        for which bootstrapping was set to `True`. That is, if sampling was
        done with replacement. Otherwise, users need to provide their own
        inbag matrix.

    pred_centered : ndarray
        Centered predictions that are an intermediate result in the
        computation.

    n_trees : int
        The number of trees in the forest object.
    """
    n_train_samples = inbag.shape[0]
    n_var = np.mean(np.square(inbag[0:n_trees]).mean(axis=1).T.view() -
                    np.square(inbag[0:n_trees].mean(axis=1)).T.view())
    boot_var = np.square(pred_centered).sum(axis=1) / n_trees
    bias_correction = n_train_samples * n_var * boot_var / n_trees
    V_IJ_unbiased = V_IJ - bias_correction
    return V_IJ_unbiased

def _bias_correction ( V_IJ , inbag , pred_centered , n_trees ) : n_train_samples = inbag . shape [ 0 ] n_var = np . mean ( np . square ( inbag [ 0 : n_trees ] ) . mean ( axis = 1 ) . T . view ( ) - np . square ( inbag [ 0 : n_trees ] . mean ( axis = 1 ) ) . T . view ( ) ) boot_var = np . square ( pred_centered ) . sum ( axis = 1 ) / n_trees bias_correction = n_train_samples * n_var * boot_var / n_trees V_IJ_unbiased = V_IJ - bias_correction return V_IJ_unbiased


def random_forest_error(forest, X_train, X_test, inbag=None,
                        calibrate=True, memory_constrained=False,
                        memory_limit=None):
    """
    Calculate error bars from scikit-learn RandomForest estimators.

    RandomForest is a regressor or classifier object
    this variance can be used to plot error bars for RandomForest objects

    Parameters
    ----------
    forest : RandomForest
        Regressor or Classifier object.

    X_train : ndarray
        An array with shape (n_train_sample, n_features). The design matrix for
        training data.

    X_test : ndarray
        An array with shape (n_test_sample, n_features). The design matrix
        for testing data

    inbag : ndarray, optional
        The inbag matrix that fit the data. If set to `None` (default) it
        will be inferred from the forest. However, this only works for trees
        for which bootstrapping was set to `True`. That is, if sampling was
        done with replacement. Otherwise, users need to provide their own
        inbag matrix.

    calibrate: boolean, optional
        Whether to apply calibration to mitigate Monte Carlo noise.
        Some variance estimates may be negative due to Monte Carlo effects if
        the number of trees in the forest is too small. To use calibration,
        Default: True

    memory_constrained: boolean, optional
        Whether or not there is a restriction on memory. If False, it is
        assumed that a ndarry of shape (n_train_sample,n_test_sample) fits
        in main memory. Setting to True can actually provide a speed up if
        memory_limit is tuned to the optimal range.

    memory_limit: int, optional.
        An upper bound for how much memory the itermediate matrices will take
        up in Megabytes. This must be provided if memory_constrained=True.

    Returns
    -------
    An array with the unbiased sampling variance (V_IJ_unbiased)
    for a RandomForest object.

    See Also
    ----------
    :func:`calc_inbag`

    Notes
    -----
    The calculation of error is based on the infinitesimal jackknife variance,
    as described in [Wager2014]_ and is a Python implementation of the R code
    provided at: https://github.com/swager/randomForestCI

    .. [Wager2014] S. Wager, T. Hastie, B. Efron. "Confidence Intervals for
       Random Forests: The Jackknife and the Infinitesimal Jackknife", Journal
       of Machine Learning Research vol. 15, pp. 1625-1651, 2014.
    """
    if inbag is None:
        inbag = calc_inbag(X_train.shape[0], forest)

    pred = np.array([tree.predict(X_test) for tree in forest]).T
    pred_mean = np.mean(pred, 0)
    pred_centered = pred - pred_mean
    n_trees = forest.n_estimators
    V_IJ = _core_computation(X_train, X_test, inbag, pred_centered, n_trees,
                             memory_constrained, memory_limit)
    V_IJ_unbiased = _bias_correction(V_IJ, inbag, pred_centered, n_trees)

    # Correct for cases where resampling is done without replacement:
    if np.max(inbag) == 1:
        variance_inflation = 1 / (1 - np.mean(inbag)) ** 2
        V_IJ_unbiased *= variance_inflation

    if not calibrate:
        return V_IJ_unbiased

    if V_IJ_unbiased.shape[0] <= 20:
        print("No calibration with n_samples <= 20")
        return V_IJ_unbiased
    if calibrate:

        calibration_ratio = 2
        n_sample = np.ceil(n_trees / calibration_ratio)
        new_forest = copy.deepcopy(forest)
        new_forest.estimators_ =\
            np.random.permutation(new_forest.estimators_)[:int(n_sample)]
        new_forest.n_estimators = int(n_sample)

        results_ss = random_forest_error(new_forest, X_train, X_test,
                                         calibrate=False,
                                         memory_constrained=memory_constrained,
                                         memory_limit=memory_limit)
        # Use this second set of variance estimates
        # to estimate scale of Monte Carlo noise
        sigma2_ss = np.mean((results_ss - V_IJ_unbiased)**2)
        delta = n_sample / n_trees
        sigma2 = (delta**2 + (1 - delta)**2) / (2 * (1 - delta)**2) * sigma2_ss

        # Use Monte Carlo noise scale estimate for empirical Bayes calibration
        V_IJ_calibrated = calibrateEB(V_IJ_unbiased, sigma2)

        return V_IJ_calibrated

Calculate error bars from scikit - learn RandomForest estimators .


def generate_self_signed_certificate(self, domain='', r=None):
        """
        Generates a self-signed certificate for use in an internal development
        environment for testing SSL pages.

        http://almostalldigital.wordpress.com/2013/03/07/self-signed-ssl-certificate-for-ec2-load-balancer/
        """
        r = self.local_renderer
        r.env.domain = domain or r.env.domain
        assert r.env.domain, 'No SSL domain defined.'
        role = r or self.genv.ROLE or ALL
        ssl_dst = 'roles/%s/ssl' % (role,)
        if not os.path.isdir(ssl_dst):
            os.makedirs(ssl_dst)
        r.env.base_dst = '%s/%s' % (ssl_dst, r.env.domain)
        r.local('openssl req -new -newkey rsa:{ssl_length} '
            '-days {ssl_days} -nodes -x509 '
            '-subj "/C={ssl_country}/ST={ssl_state}/L={ssl_city}/O={ssl_organization}/CN={ssl_domain}" '
            '-keyout {ssl_base_dst}.key -out {ssl_base_dst}.crt')

Generates a self - signed certificate for use in an internal development environment for testing SSL pages .


def generate_csr(self, domain='', r=None):
        """
        Creates a certificate signing request to be submitted to a formal
        certificate authority to generate a certificate.

        Note, the provider may say the CSR must be created on the target server,
        but this is not necessary.
        """
        r = r or self.local_renderer
        r.env.domain = domain or r.env.domain
        role = self.genv.ROLE or ALL
        site = self.genv.SITE or self.genv.default_site
        print('self.genv.default_site:', self.genv.default_site, file=sys.stderr)
        print('site.csr0:', site, file=sys.stderr)
        ssl_dst = 'roles/%s/ssl' % (role,)
        print('ssl_dst:', ssl_dst)
        if not os.path.isdir(ssl_dst):
            os.makedirs(ssl_dst)
        for site, site_data in self.iter_sites():
            print('site.csr1:', site, file=sys.stderr)
            assert r.env.domain, 'No SSL domain defined.'
            r.env.ssl_base_dst = '%s/%s' % (ssl_dst, r.env.domain.replace('*.', ''))
            r.env.ssl_csr_year = date.today().year
            r.local('openssl req -nodes -newkey rsa:{ssl_length} '
                '-subj "/C={ssl_country}/ST={ssl_state}/L={ssl_city}/O={ssl_organization}/CN={ssl_domain}" '
                '-keyout {ssl_base_dst}.{ssl_csr_year}.key -out {ssl_base_dst}.{ssl_csr_year}.csr')

Creates a certificate signing request to be submitted to a formal certificate authority to generate a certificate .


def get_expiration_date(self, fn):
        """
        Reads the expiration date of a local crt file.
        """
        r = self.local_renderer
        r.env.crt_fn = fn
        with hide('running'):
            ret = r.local('openssl x509 -noout -in {ssl_crt_fn} -dates', capture=True)
        matches = re.findall('notAfter=(.*?)$', ret, flags=re.IGNORECASE)
        if matches:
            return dateutil.parser.parse(matches[0])

Reads the expiration date of a local crt file .


def list_expiration_dates(self, base='roles/all/ssl'):
        """
        Scans through all local .crt files and displays the expiration dates.
        """
        max_fn_len = 0
        max_date_len = 0
        data = []
        for fn in os.listdir(base):
            fqfn = os.path.join(base, fn)
            if not os.path.isfile(fqfn):
                continue
            if not fn.endswith('.crt'):
                continue
            expiration_date = self.get_expiration_date(fqfn)
            max_fn_len = max(max_fn_len, len(fn))
            max_date_len = max(max_date_len, len(str(expiration_date)))
            data.append((fn, expiration_date))
        print('%s %s %s' % ('Filename'.ljust(max_fn_len), 'Expiration Date'.ljust(max_date_len), 'Expired'))
        now = datetime.now().replace(tzinfo=pytz.UTC)
        for fn, dt in sorted(data):

            if dt is None:
                expired = '?'
            elif dt < now:
                expired = 'YES'
            else:
                expired = 'NO'
            print('%s %s %s' % (fn.ljust(max_fn_len), str(dt).ljust(max_date_len), expired))

def
list_expiration_dates
(
self
,
base
=
'roles/all/ssl'
)
:
max_fn_len
=
0
max_date_len
=
0
data
=
[
]
for
fn
in
os
.
listdir
(
base
)
:
fqfn
=
os
.
path
.
join
(
base
,
fn
)
if
not
os
.
path
.
isfile
(
fqfn
)
:
continue
if
not
fn
.
endswith
(
'.crt'
)
:
continue
expiration_date
=
self
.
get_expiration_date
(
fqfn
)
max_fn_len
=
max
(
max_fn_len
,
len
(
fn
)
)
max_date_len
=
max
(
max_date_len
,
len
(
str
(
expiration_date
)
)
)
data
.
append
(
(
fn
,
expiration_date
)
)
print
(
'%s %s %s'
%
(
'Filename'
.
ljust
(
max_fn_len
)
,
'Expiration Date'
.
ljust
(
max_date_len
)
,
'Expired'
)
)
now
=
datetime
.
now
(
)
.
replace
(
tzinfo
=
pytz
.
UTC
)
for
fn
,
dt
in
sorted
(
data
)
:
if
dt
is
:
expired
=
'?'
elif
dt
<
now
:
expired
=
'YES'
else
:
expired
=
'NO'
print
(
'%s %s %s'
%
(
fn
.
ljust
(
max_fn_len
)
,
str
(
dt
)
.
ljust
(
max_date_len
)
,
expired
)
)

Scans through all local . crt files and displays the expiration dates .


def verify_certificate_chain(self, base=None, crt=None, csr=None, key=None):
        """
        Confirms the key, CSR, and certificate files all match.
        """
        from burlap.common import get_verbose, print_fail, print_success

        r = self.local_renderer

        if base:
            crt = base + '.crt'
            csr = base + '.csr'
            key = base + '.key'
        else:
            assert crt and csr and key, 'If base not provided, crt and csr and key must be given.'

        assert os.path.isfile(crt)
        assert os.path.isfile(csr)
        assert os.path.isfile(key)

        csr_md5 = r.local('openssl req -noout -modulus -in %s | openssl md5' % csr, capture=True)
        key_md5 = r.local('openssl rsa -noout -modulus -in %s | openssl md5' % key, capture=True)
        crt_md5 = r.local('openssl x509 -noout -modulus -in %s | openssl md5' % crt, capture=True)

        match = crt_md5 == csr_md5 == key_md5

        if self.verbose or not match:
            print('crt:', crt_md5)
            print('csr:', csr_md5)
            print('key:', key_md5)

        if match:
            print_success('Files look good!')
        else:
            print_fail('Files no not match!')
            raise Exception('Files no not match!')

Confirms the key CSR and certificate files all match .


def update_merge(d, u):
    """
    Recursively merges two dictionaries.

    Uses fabric's AttributeDict so you can reference values via dot-notation.
    e.g. env.value1.value2.value3...

    http://stackoverflow.com/questions/3232943/update-value-of-a-nested-dictionary-of-varying-depth
    """
    import collections
    for k, v in u.items():
        if isinstance(v, collections.Mapping):
            r = update_merge(d.get(k, dict()), v)
            d[k] = r
        else:
            d[k] = u[k]
    return d

def update_merge ( d , u ) : import collections for k , v in u . items ( ) : if isinstance ( v , collections . Mapping ) : r = update_merge ( d . get ( k , dict ( ) ) , v ) d [ k ] = r else : d [ k ] = u [ k ] return d


def check_version():
    """
    Compares the local version against the latest official version on PyPI and displays a warning message if a newer release is available.

    This check can be disabled by setting the environment variable BURLAP_CHECK_VERSION=0.
    """
    global CHECK_VERSION
    if not CHECK_VERSION:
        return
    # Ensure we only check once in this process.
    CHECK_VERSION = 0
    # Lookup most recent remote version.
    from six.moves.urllib.request import urlopen
    try:
        response = urlopen("https://pypi.org/pypi/burlap/json")
        data = json.loads(response.read().decode())
        remote_release = sorted(tuple(map(int, _.split('.'))) for _ in data['releases'].keys())[-1]
        remote_release_str = '.'.join(map(str, remote_release))
        local_release = VERSION
        local_release_str = '.'.join(map(str, local_release))
        # Display warning.
        if remote_release > local_release:
            print('\033[93m')
            print("You are using burlap version %s, however version %s is available." % (local_release_str, remote_release_str))
            print("You should consider upgrading via the 'pip install --upgrade burlap' command.")
            print('\033[0m')
    except Exception as exc:
        print('\033[93m')
        print("Unable to check for updated burlap version: %s" % exc)
        print('\033[0m')

Compares the local version against the latest official version on PyPI and displays a warning message if a newer release is available .


def populate_fabfile():
    """
    Automatically includes all submodules and role selectors
    in the top-level fabfile using spooky-scary black magic.

    This allows us to avoid manually declaring imports for every module, e.g.

        import burlap.pip
        import burlap.vm
        import burlap...

    which has the added benefit of allowing us to manually call the commands
    without typing "burlap".

    This is soley for convenience. If not needed, it can be disabled
    by specifying the environment variable:

        export BURLAP_POPULATE_STACK=0
    """
    stack = inspect.stack()
    fab_frame = None
    for frame_obj, script_fn, line, _, _, _ in stack:
        if 'fabfile.py' in script_fn:
            fab_frame = frame_obj
            break
    if not fab_frame:
        return
    try:
        locals_ = fab_frame.f_locals
        for module_name, module in sub_modules.items():
            locals_[module_name] = module
        for role_name, role_func in role_commands.items():
            assert role_name not in sub_modules, \
                ('The role %s conflicts with a built-in submodule. '
                 'Please choose a different name.') % (role_name)
            locals_[role_name] = role_func
        locals_['common'] = common

        # Put all debug commands into the global namespace.

#         for _debug_name in debug.debug.get_tasks():
#             print('_debug_name:', _debug_name)

        locals_['shell'] = shell#debug.debug.shell

        # Put all virtual satchels in the global namespace so Fabric can find them.
        for _module_alias in common.post_import_modules:
            exec("import %s" % _module_alias) # pylint: disable=exec-used
            locals_[_module_alias] = locals()[_module_alias]

    finally:
        del stack

#             print('_debug_name:', _debug_name)

Automatically includes all submodules and role selectors in the top - level fabfile using spooky - scary black magic .


def task_or_dryrun(*args, **kwargs):
    """
    Decorator declaring the wrapped function to be a new-style task.

    May be invoked as a simple, argument-less decorator (i.e. ``@task``) or
    with arguments customizing its behavior (e.g. ``@task(alias='myalias')``).

    Please see the :ref:`new-style task <task-decorator>` documentation for
    details on how to use this decorator.

    .. versionchanged:: 1.2
        Added the ``alias``, ``aliases``, ``task_class`` and ``default``
        keyword arguments. See :ref:`task-decorator-arguments` for details.
    .. versionchanged:: 1.5
        Added the ``name`` keyword argument.

    .. seealso:: `~fabric.docs.unwrap_tasks`, `~fabric.tasks.WrappedCallableTask`
    """
    invoked = bool(not args or kwargs)
    task_class = kwargs.pop("task_class", WrappedCallableTask)
#     if invoked:
#         func, args = args[0], ()
#     else:
    func, args = args[0], ()

    def wrapper(func):
        return task_class(func, *args, **kwargs)
    wrapper.is_task_or_dryrun = True
    wrapper.wrapped = func

    return wrapper if invoked else wrapper(func)

def
task_or_dryrun
(
*
args
,
*
*
kwargs
)
:
invoked
=
bool
(
not
args
or
kwargs
)
task_class
=
kwargs
.
pop
(
"task_class"
,
WrappedCallableTask
)
#     if invoked:
#         func, args = args[0], ()
#     else:
func
,
args
=
args
[
0
]
,
(
)
def
wrapper
(
func
)
:
return
task_class
(
func
,
*
args
,
*
*
kwargs
)
wrapper
.
is_task_or_dryrun
=
True
wrapper
.
wrapped
=
func
return
wrapper
if
invoked
else
wrapper
(
func
)

Decorator declaring the wrapped function to be a new - style task .


def task(*args, **kwargs):
    """
    Decorator for registering a satchel method as a Fabric task.

    Can be used like:

        @task
        def my_method(self):
            ...

        @task(precursors=['other_satchel'])
        def my_method(self):
            ...

    """
    precursors = kwargs.pop('precursors', None)
    post_callback = kwargs.pop('post_callback', False)
    if args and callable(args[0]):
        # direct decoration, @task
        return _task(*args)

    # callable decoration, @task(precursors=['satchel'])
    def wrapper(meth):
        if precursors:
            meth.deploy_before = list(precursors)
        if post_callback:
            #from burlap.common import post_callbacks
            #post_callbacks.append(meth)
            meth.is_post_callback = True
        return _task(meth)
    return wrapper

Decorator for registering a satchel method as a Fabric task .


def is_file(self, path, use_sudo=False):
        """
        Check if a path exists, and is a file.
        """
        if self.is_local and not use_sudo:
            return os.path.isfile(path)
        else:
            func = use_sudo and _sudo or _run
            with self.settings(hide('running', 'warnings'), warn_only=True):
                return func('[ -f "%(path)s" ]' % locals()).succeeded

def
is_file
(
self
,
path
,
use_sudo
=
False
)
:
if
self
.
is_local
and
not
use_sudo
:
return
os
.
path
.
isfile
(
path
)
else
:
func
=
use_sudo
and
_sudo
or
_run
with
self
.
settings
(
hide
(
'running'
,
'warnings'
)
,
warn_only
=
True
)
:
return
func
(
'[ -f "%(path)s" ]'
%
locals
(
)
)
.
succeeded