def convert_text(content, from_fmt, to_fmt, deparagraph=False, mathjax=False,
                 smart=True, extra_args=None):
    """Convert text from one markup format to another using pandoc.

    This function is a thin wrapper around `pypandoc.convert_text`.

    Parameters
    ----------
    content : `str`
        Original content.

    from_fmt : `str`
        Format of the original ``content``. Format identifier must be one of
        those known by Pandoc. See https://pandoc.org/MANUAL.html for details.

    to_fmt : `str`
        Output format for the content.

    deparagraph : `bool`, optional
        If `True`, then the
        `lsstprojectmeta.pandoc.filters.deparagraph.deparagraph` filter is
        used to remove paragraph (``
``, for example) tags around a single
        paragraph of content. That filter does not affect content that
        consists of multiple blocks (several paragraphs, or lists, for
        example). Default is `False`.

        For example, **without** this filter Pandoc will convert
        the string ``"Title text"`` to ``"
Title text
"`` in HTML. The
        paragraph tags aren't useful if you intend to wrap the converted
        content in different tags, like ``
``, using your own templating
        system.

        **With** this filter, Pandoc will convert the string ``"Title text"``
        to ``"Title text"`` in HTML.

    mathjax : `bool`, optional
        If `True` then Pandoc will markup output content to work with MathJax.
        Default is False.

    smart : `bool`, optional
        If `True` (default) then ascii characters will be converted to unicode
        characters like smart quotes and em dashes.

    extra_args : `list`, optional
        Sequence of Pandoc arguments command line arguments (such as
        ``'--normalize'``). The ``deparagraph``, ``mathjax``, and ``smart``
        arguments are convenience arguments that are equivalent to items
        in ``extra_args``.

    Returns
    -------
    output : `str`
        Content in the output (``to_fmt``) format.

    Notes
    -----
    This function will automatically install Pandoc if it is not available.
    See `ensure_pandoc`.
    """
    logger = logging.getLogger(__name__)

    if extra_args is not None:
        extra_args = list(extra_args)
    else:
        extra_args = []

    if mathjax:
        extra_args.append('--mathjax')

    if smart:
        extra_args.append('--smart')

    if deparagraph:
        extra_args.append('--filter=lsstprojectmeta-deparagraph')

    extra_args.append('--wrap=none')

    # de-dupe extra args
    extra_args = set(extra_args)

    logger.debug('Running pandoc from %s to %s with extra_args %s',
                 from_fmt, to_fmt, extra_args)

    output = pypandoc.convert_text(content, to_fmt, format=from_fmt,
                                   extra_args=extra_args)
    return output

Convert text from one markup format to another using pandoc .


def convert_lsstdoc_tex(
        content, to_fmt, deparagraph=False, mathjax=False,
        smart=True, extra_args=None):
    """Convert lsstdoc-class LaTeX to another markup format.

    This function is a thin wrapper around `convert_text` that automatically
    includes common lsstdoc LaTeX macros.

    Parameters
    ----------
    content : `str`
        Original content.

    to_fmt : `str`
        Output format for the content (see https://pandoc.org/MANUAL.html).
        For example, 'html5'.

    deparagraph : `bool`, optional
        If `True`, then the
        `lsstprojectmeta.pandoc.filters.deparagraph.deparagraph` filter is
        used to remove paragraph (``
``, for example) tags around a single
        paragraph of content. That filter does not affect content that
        consists of multiple blocks (several paragraphs, or lists, for
        example). Default is `False`.

        For example, **without** this filter Pandoc will convert
        the string ``"Title text"`` to ``"
Title text
"`` in HTML. The
        paragraph tags aren't useful if you intend to wrap the converted
        content in different tags, like ``
``, using your own templating
        system.

        **With** this filter, Pandoc will convert the string ``"Title text"``
        to ``"Title text"`` in HTML.

    mathjax : `bool`, optional
        If `True` then Pandoc will markup output content to work with MathJax.
        Default is False.

    smart : `bool`, optional
        If `True` (default) then ascii characters will be converted to unicode
        characters like smart quotes and em dashes.

    extra_args : `list`, optional
        Sequence of Pandoc arguments command line arguments (such as
        ``'--normalize'``). The ``deparagraph``, ``mathjax``, and ``smart``
        arguments are convenience arguments that are equivalent to items
        in ``extra_args``.

    Returns
    -------
    output : `str`
        Content in the output (``to_fmt``) format.

    Notes
    -----
    This function will automatically install Pandoc if it is not available.
    See `ensure_pandoc`.
    """
    augmented_content = '\n'.join((LSSTDOC_MACROS, content))
    return convert_text(
        augmented_content, 'latex', to_fmt,
        deparagraph=deparagraph, mathjax=mathjax,
        smart=smart, extra_args=extra_args)

def convert_lsstdoc_tex ( content , to_fmt , deparagraph = False , mathjax = False , smart = True , extra_args = None ) : augmented_content = '\n' . join ( ( LSSTDOC_MACROS , content ) ) return convert_text ( augmented_content , 'latex' , to_fmt , deparagraph = deparagraph , mathjax = mathjax , smart = smart , extra_args = extra_args )

Convert lsstdoc - class LaTeX to another markup format .


def decode_jsonld(jsonld_text):
    """Decode a JSON-LD dataset, including decoding datetime
    strings into `datetime.datetime` objects.

    Parameters
    ----------
    encoded_dataset : `str`
        The JSON-LD dataset encoded as a string.

    Returns
    -------
    jsonld_dataset : `dict`
        A JSON-LD dataset.

    Examples
    --------

    >>> doc = '{"dt": "2018-01-01T12:00:00Z"}'
    >>> decode_jsonld(doc)
    {'dt': datetime.datetime(2018, 1, 1, 12, 0, tzinfo=datetime.timezone.utc)}
    """
    decoder = json.JSONDecoder(object_pairs_hook=_decode_object_pairs)
    return decoder.decode(jsonld_text)

def decode_jsonld ( jsonld_text ) : decoder = json . JSONDecoder ( object_pairs_hook = _decode_object_pairs ) return decoder . decode ( jsonld_text )

Decode a JSON - LD dataset including decoding datetime strings into datetime . datetime objects .


def default(self, obj):
        """Encode values as JSON strings.

        This method overrides the default implementation from
        `json.JSONEncoder`.
        """
        if isinstance(obj, datetime.datetime):
            return self._encode_datetime(obj)

        # Fallback to the default encoding
        return json.JSONEncoder.default(self, obj)

def
default
(
self
,
obj
)
:
if
isinstance
(
obj
,
datetime
.
datetime
)
:
return
self
.
_encode_datetime
(
obj
)
# Fallback to the default encoding
return
json
.
JSONEncoder
.
default
(
self
,
obj
)


def find_repos(self, depth=10):
        '''Get all git repositories within this environment'''

        repos = []

        for root, subdirs, files in walk_dn(self.root, depth=depth):
            if 'modules' in root:
                continue
            if '.git' in subdirs:
                repos.append(root)

        return repos

def find_repos ( self , depth = 10 ) : repos = [ ] for root , subdirs , files in walk_dn ( self . root , depth = depth ) : if 'modules' in root : continue if '.git' in subdirs : repos . append ( root ) return repos


def install(self, package):
        '''Install a python package using pip'''

        logger.debug('Installing ' + package)
        shell.run(self.pip_path, 'install',  package)

def
install
(
self
,
package
)
:
logger
.
debug
(
'Installing '
+
package
)
shell
.
run
(
self
.
pip_path
,
'install'
,
package
)


def upgrade(self, package):
        '''Update a python package using pip'''

        logger.debug('Upgrading ' + package)
        shell.run(self.pip_path, 'install', '--upgrade', '--no-deps', package)
        shell.run(self.pip_path, 'install', package)

def
upgrade
(
self
,
package
)
:
logger
.
debug
(
'Upgrading '
+
package
)
shell
.
run
(
self
.
pip_path
,
'install'
,
'--upgrade'
,
'--no-deps'
,
package
)
shell
.
run
(
self
.
pip_path
,
'install'
,
package
)


def df_quantile(df, nb=100):
    """Returns the nb quantiles for datas in a dataframe
    """
    quantiles = np.linspace(0, 1., nb)
    res = pd.DataFrame()
    for q in quantiles:
        res = res.append(df.quantile(q), ignore_index=True)
    return res

def df_quantile ( df , nb = 100 ) : quantiles = np . linspace ( 0 , 1. , nb ) res = pd . DataFrame ( ) for q in quantiles : res = res . append ( df . quantile ( q ) , ignore_index = True ) return res


def rmse(a, b):
    """Returns the root mean square error betwwen a and b
    """
    return np.sqrt(np.square(a - b).mean())

def rmse ( a , b ) : return np . sqrt ( np . square ( a - b ) . mean ( ) )


def nmse(a, b):
    """Returns the normalized mean square error of a and b
    """
    return np.square(a - b).mean() / (a.mean() * b.mean())

def nmse ( a , b ) : return np . square ( a - b ) . mean ( ) / ( a . mean ( ) * b . mean ( ) )


def mfbe(a, b):
    """Returns the mean fractionalized bias error
    """
    return 2 * bias(a, b) / (a.mean() + b.mean())

def mfbe ( a , b ) : return 2 * bias ( a , b ) / ( a . mean ( ) + b . mean ( ) )


def foex(a, b):
    """Returns the factor of exceedance
    """
    return (np.sum(a > b, dtype=float) / len(a) - 0.5) * 100

def foex ( a , b ) : return ( np . sum ( a > b , dtype = float ) / len ( a ) - 0.5 ) * 100


def correlation(a, b):
    """Computes the correlation between a and b, says the Pearson's correlation
    coefficient R
    """
    diff1 = a - a.mean()
    diff2 = b - b.mean()
    return (diff1 * diff2).mean() / (np.sqrt(np.square(diff1).mean() * np.square(diff2).mean()))

def correlation ( a , b ) : diff1 = a - a . mean ( ) diff2 = b - b . mean ( ) return ( diff1 * diff2 ) . mean ( ) / ( np . sqrt ( np . square ( diff1 ) . mean ( ) * np . square ( diff2 ) . mean ( ) ) )

Computes the correlation between a and b says the Pearson s correlation coefficient R


def gmb(a, b):
    """Geometric mean bias
    """
    return np.exp(np.log(a).mean() - np.log(b).mean())

def gmb ( a , b ) : return np . exp ( np . log ( a ) . mean ( ) - np . log ( b ) . mean ( ) )


def gmv(a, b):
    """Geometric mean variance
    """
    return np.exp(np.square(np.log(a) - np.log(b)).mean())

def gmv ( a , b ) : return np . exp ( np . square ( np . log ( a ) - np . log ( b ) ) . mean ( ) )


def fmt(a, b):
    """Figure of merit in time
    """
    return 100 * np.min([a, b], axis=0).sum() / np.max([a, b], axis=0).sum()

def fmt ( a , b ) : return 100 * np . min ( [ a , b ] , axis = 0 ) . sum ( ) / np . max ( [ a , b ] , axis = 0 ) . sum ( )


def fullStats(a, b):
    """Performs several stats on a against b, typically a is the predictions
    array, and b the observations array

    Returns:
        A dataFrame of stat name, stat description, result
    """

    stats = [
        ['bias', 'Bias', bias(a, b)],
        ['stderr', 'Standard Deviation Error', stderr(a, b)],
        ['mae', 'Mean Absolute Error', mae(a, b)],
        ['rmse', 'Root Mean Square Error', rmse(a, b)],
        ['nmse', 'Normalized Mean Square Error', nmse(a, b)],
        ['mfbe', 'Mean Fractionalized bias Error', mfbe(a, b)],
        ['fa2', 'Factor of Two', fa(a, b, 2)],
        ['foex', 'Factor of Exceedance', foex(a, b)],
        ['correlation', 'Correlation R', correlation(a, b)],
        ['determination', 'Coefficient of Determination r2', determination(a, b)],
        ['gmb', 'Geometric Mean Bias', gmb(a, b)],
        ['gmv', 'Geometric Mean Variance', gmv(a, b)],
        ['fmt', 'Figure of Merit in Time', fmt(a, b)]
    ]
    rec = np.rec.fromrecords(stats, names=('stat', 'description', 'result'))
    df = pd.DataFrame.from_records(rec, index='stat')
    return df

def
fullStats
(
a
,
b
)
:
stats
=
[
[
'bias'
,
'Bias'
,
bias
(
a
,
b
)
]
,
[
'stderr'
,
'Standard Deviation Error'
,
stderr
(
a
,
b
)
]
,
[
'mae'
,
'Mean Absolute Error'
,
mae
(
a
,
b
)
]
,
[
'rmse'
,
'Root Mean Square Error'
,
rmse
(
a
,
b
)
]
,
[
'nmse'
,
'Normalized Mean Square Error'
,
nmse
(
a
,
b
)
]
,
[
'mfbe'
,
'Mean Fractionalized bias Error'
,
mfbe
(
a
,
b
)
]
,
[
'fa2'
,
'Factor of Two'
,
fa
(
a
,
b
,
2
)
]
,
[
'foex'
,
'Factor of Exceedance'
,
foex
(
a
,
b
)
]
,
[
'correlation'
,
'Correlation R'
,
correlation
(
a
,
b
)
]
,
[
'determination'
,
'Coefficient of Determination r2'
,
determination
(
a
,
b
)
]
,
[
'gmb'
,
'Geometric Mean Bias'
,
gmb
(
a
,
b
)
]
,
[
'gmv'
,
'Geometric Mean Variance'
,
gmv
(
a
,
b
)
]
,
[
'fmt'
,
'Figure of Merit in Time'
,
fmt
(
a
,
b
)
]
]
rec
=
np
.
rec
.
fromrecords
(
stats
,
names
=
(
'stat'
,
'description'
,
'result'
)
)
df
=
pd
.
DataFrame
.
from_records
(
rec
,
index
=
'stat'
)
return
df

Performs several stats on a against b typically a is the predictions array and b the observations array


def site_path(self):
        '''Path to environments site-packages'''

        if platform == 'win':
            return unipath(self.path, 'Lib', 'site-packages')

        py_ver = 'python{0}'.format(sys.version[:3])
        return unipath(self.path, 'lib', py_ver, 'site-packages')

def site_path ( self ) : if platform == 'win' : return unipath ( self . path , 'Lib' , 'site-packages' ) py_ver = 'python{0}' . format ( sys . version [ : 3 ] ) return unipath ( self . path , 'lib' , py_ver , 'site-packages' )


def _pre_activate(self):
        '''
        Prior to activating, store everything necessary to deactivate this
        environment.
        '''

        if 'CPENV_CLEAN_ENV' not in os.environ:
            if platform == 'win':
                os.environ['PROMPT'] = '$P$G'
            else:
                os.environ['PS1'] = '\\u@\\h:\\w\\$'
            clean_env_path = utils.get_store_env_tmp()
            os.environ['CPENV_CLEAN_ENV'] = clean_env_path
            utils.store_env(path=clean_env_path)
        else:
            utils.restore_env_from_file(os.environ['CPENV_CLEAN_ENV'])

def _pre_activate ( self ) : if 'CPENV_CLEAN_ENV' not in os . environ : if platform == 'win' : os . environ [ 'PROMPT' ] = '$P$G' else : os . environ [ 'PS1' ] = '\\u@\\h:\\w\\$' clean_env_path = utils . get_store_env_tmp ( ) os . environ [ 'CPENV_CLEAN_ENV' ] = clean_env_path utils . store_env ( path = clean_env_path ) else : utils . restore_env_from_file ( os . environ [ 'CPENV_CLEAN_ENV' ] )

Prior to activating store everything necessary to deactivate this environment .


def _activate(self):
        '''
        Do some serious mangling to the current python environment...
        This is necessary to activate an environment via python.
        '''

        old_syspath = set(sys.path)
        site.addsitedir(self.site_path)
        site.addsitedir(self.bin_path)
        new_syspaths = set(sys.path) - old_syspath
        for path in new_syspaths:
            sys.path.remove(path)
            sys.path.insert(1, path)

        if not hasattr(sys, 'real_prefix'):
            sys.real_prefix = sys.prefix

        sys.prefix = self.path

def _activate ( self ) : old_syspath = set ( sys . path ) site . addsitedir ( self . site_path ) site . addsitedir ( self . bin_path ) new_syspaths = set ( sys . path ) - old_syspath for path in new_syspaths : sys . path . remove ( path ) sys . path . insert ( 1 , path ) if not hasattr ( sys , 'real_prefix' ) : sys . real_prefix = sys . prefix sys . prefix = self . path

Do some serious mangling to the current python environment ... This is necessary to activate an environment via python .


def remove(self):
        '''
        Remove this environment
        '''
        self.run_hook('preremove')
        utils.rmtree(self.path)
        self.run_hook('postremove')

def remove ( self ) : self . run_hook ( 'preremove' ) utils . rmtree ( self . path ) self . run_hook ( 'postremove' )


def command(self):
        '''Command used to launch this application module'''

        cmd = self.config.get('command', None)
        if cmd is None:
            return

        cmd = cmd[platform]
        return cmd['path'], cmd['args']

def command ( self ) : cmd = self . config . get ( 'command' , None ) if cmd is None : return cmd = cmd [ platform ] return cmd [ 'path' ] , cmd [ 'args' ]


def create(name_or_path=None, config=None):
    '''Create a virtual environment. You can pass either the name of a new
    environment to create in your CPENV_HOME directory OR specify a full path
    to create an environment outisde your CPENV_HOME.

    Create an environment in CPENV_HOME::

        >>> cpenv.create('myenv')

    Create an environment elsewhere::

        >>> cpenv.create('~/custom_location/myenv')

    :param name_or_path: Name or full path of environment
    :param config: Environment configuration including dependencies etc...
    '''

    # Get the real path of the environment
    if utils.is_system_path(name_or_path):
        path = unipath(name_or_path)
    else:
        path = unipath(get_home_path(), name_or_path)

    if os.path.exists(path):
        raise OSError('{} already exists'.format(path))

    env = VirtualEnvironment(path)
    utils.ensure_path_exists(env.path)

    if config:
        if utils.is_git_repo(config):
            Git('').clone(config, env.path)
        else:
            shutil.copy2(config, env.config_path)
    else:
        with open(env.config_path, 'w') as f:
            f.write(defaults.environment_config)

    utils.ensure_path_exists(env.hook_path)
    utils.ensure_path_exists(env.modules_path)

    env.run_hook('precreate')

    virtualenv.create_environment(env.path)
    if not utils.is_home_environment(env.path):
        EnvironmentCache.add(env)
        EnvironmentCache.save()

    try:
        env.update()
    except:
        utils.rmtree(path)
        logger.debug('Failed to update, rolling back...')
        raise
    else:
        env.run_hook('postcreate')

    return env

def
create
(
name_or_path
=
,
config
=
)
:
# Get the real path of the environment
if
utils
.
is_system_path
(
name_or_path
)
:
path
=
unipath
(
name_or_path
)
else
:
path
=
unipath
(
get_home_path
(
)
,
name_or_path
)
if
os
.
path
.
exists
(
path
)
:
raise
OSError
(
'{} already exists'
.
format
(
path
)
)
env
=
VirtualEnvironment
(
path
)
utils
.
ensure_path_exists
(
env
.
path
)
if
config
:
if
utils
.
is_git_repo
(
config
)
:
Git
(
''
)
.
clone
(
config
,
env
.
path
)
else
:
shutil
.
copy2
(
config
,
env
.
config_path
)
else
:
with
open
(
env
.
config_path
,
'w'
)
as
f
:
f
.
write
(
defaults
.
environment_config
)
utils
.
ensure_path_exists
(
env
.
hook_path
)
utils
.
ensure_path_exists
(
env
.
modules_path
)
env
.
run_hook
(
'precreate'
)
virtualenv
.
create_environment
(
env
.
path
)
if
not
utils
.
is_home_environment
(
env
.
path
)
:
EnvironmentCache
.
add
(
env
)
EnvironmentCache
.
save
(
)
try
:
env
.
update
(
)
except
:
utils
.
rmtree
(
path
)
logger
.
debug
(
'Failed to update, rolling back...'
)
raise
else
:
env
.
run_hook
(
'postcreate'
)
return
env

Create a virtual environment . You can pass either the name of a new environment to create in your CPENV_HOME directory OR specify a full path to create an environment outisde your CPENV_HOME .


def remove(name_or_path):
    '''Remove an environment or module

    :param name_or_path: name or path to environment or module
    '''

    r = resolve(name_or_path)
    r.resolved[0].remove()

    EnvironmentCache.discard(r.resolved[0])
    EnvironmentCache.save()

def remove ( name_or_path ) : r = resolve ( name_or_path ) r . resolved [ 0 ] . remove ( ) EnvironmentCache . discard ( r . resolved [ 0 ] ) EnvironmentCache . save ( )


def launch(module_name, *args, **kwargs):
    '''Activates and launches a module

    :param module_name: name of module to launch
    '''

    r = resolve(module_name)
    r.activate()
    mod = r.resolved[0]
    mod.launch(*args, **kwargs)

def launch ( module_name , * args , * * kwargs ) : r = resolve ( module_name ) r . activate ( ) mod = r . resolved [ 0 ] mod . launch ( * args , * * kwargs )


def deactivate():
    '''Deactivates an environment by restoring all env vars to a clean state
    stored prior to activating environments
    '''

    if 'CPENV_ACTIVE' not in os.environ or 'CPENV_CLEAN_ENV' not in os.environ:
        raise EnvironmentError('Can not deactivate environment...')

    utils.restore_env_from_file(os.environ['CPENV_CLEAN_ENV'])

def
deactivate
(
)
:
if
'CPENV_ACTIVE'
not
in
os
.
environ
or
'CPENV_CLEAN_ENV'
not
in
os
.
environ
:
raise
EnvironmentError
(
'Can not deactivate environment...'
)
utils
.
restore_env_from_file
(
os
.
environ
[
'CPENV_CLEAN_ENV'
]
)


def get_modules():
    '''Returns a list of available modules.'''

    modules = set()

    cwd = os.getcwd()
    for d in os.listdir(cwd):

        if d == 'module.yml':
            modules.add(Module(cwd))

        path = unipath(cwd, d)
        if utils.is_module(path):
            modules.add(Module(cwd))

    module_paths = get_module_paths()
    for module_path in module_paths:
        for d in os.listdir(module_path):

            path = unipath(module_path, d)
            if utils.is_module(path):
                modules.add(Module(path))

    return sorted(list(modules), key=lambda x: x.name)

def get_modules ( ) : modules = set ( ) cwd = os . getcwd ( ) for d in os . listdir ( cwd ) : if d == 'module.yml' : modules . add ( Module ( cwd ) ) path = unipath ( cwd , d ) if utils . is_module ( path ) : modules . add ( Module ( cwd ) ) module_paths = get_module_paths ( ) for module_path in module_paths : for d in os . listdir ( module_path ) : path = unipath ( module_path , d ) if utils . is_module ( path ) : modules . add ( Module ( path ) ) return sorted ( list ( modules ) , key = lambda x : x . name )


def add_active_module(module):
    '''Add a module to CPENV_ACTIVE_MODULES environment variable'''

    modules = set(get_active_modules())
    modules.add(module)
    new_modules_path = os.pathsep.join([m.path for m in modules])
    os.environ['CPENV_ACTIVE_MODULES'] = str(new_modules_path)

def add_active_module ( module ) : modules = set ( get_active_modules ( ) ) modules . add ( module ) new_modules_path = os . pathsep . join ( [ m . path for m in modules ] ) os . environ [ 'CPENV_ACTIVE_MODULES' ] = str ( new_modules_path )


def rem_active_module(module):
    '''Remove a module from CPENV_ACTIVE_MODULES environment variable'''

    modules = set(get_active_modules())
    modules.discard(module)
    new_modules_path = os.pathsep.join([m.path for m in modules])
    os.environ['CPENV_ACTIVE_MODULES'] = str(new_modules_path)

def rem_active_module ( module ) : modules = set ( get_active_modules ( ) ) modules . discard ( module ) new_modules_path = os . pathsep . join ( [ m . path for m in modules ] ) os . environ [ 'CPENV_ACTIVE_MODULES' ] = str ( new_modules_path )


def format_objects(objects, children=False, columns=None, header=True):
    '''Format a list of environments and modules for terminal output'''

    columns = columns or ('NAME', 'TYPE', 'PATH')
    objects = sorted(objects, key=_type_and_name)
    data = []
    for obj in objects:
        if isinstance(obj, cpenv.VirtualEnvironment):
            data.append(get_info(obj))
            modules = obj.get_modules()
            if children and modules:
                for mod in modules:
                    data.append(get_info(mod, indent=2, root=obj.path))
        else:
            data.append(get_info(obj))

    maxes = [len(max(col, key=len)) for col in zip(*data)]
    tmpl = '{:%d}  {:%d}  {:%d}' % tuple(maxes)
    lines = []
    if header:
        lines.append('\n' + bold_blue(tmpl.format(*columns)))

    for obj_data in data:
        lines.append(tmpl.format(*obj_data))

    return '\n'.join(lines)

def
format_objects
(
objects
,
children
=
False
,
columns
=
,
header
=
True
)
:
columns
=
columns
or
(
'NAME'
,
'TYPE'
,
'PATH'
)
objects
=
sorted
(
objects
,
key
=
_type_and_name
)
data
=
[
]
for
obj
in
objects
:
if
isinstance
(
obj
,
cpenv
.
VirtualEnvironment
)
:
data
.
append
(
get_info
(
obj
)
)
modules
=
obj
.
get_modules
(
)
if
children
and
modules
:
for
mod
in
modules
:
data
.
append
(
get_info
(
mod
,
indent
=
2
,
root
=
obj
.
path
)
)
else
:
data
.
append
(
get_info
(
obj
)
)
maxes
=
[
len
(
max
(
col
,
key
=
len
)
)
for
col
in
zip
(
*
data
)
]
tmpl
=
'{:%d}  {:%d}  {:%d}'
%
tuple
(
maxes
)
lines
=
[
]
if
header
:
lines
.
append
(
'\n'
+
bold_blue
(
tmpl
.
format
(
*
columns
)
)
)
for
obj_data
in
data
:
lines
.
append
(
tmpl
.
format
(
*
obj_data
)
)
return
'\n'
.
join
(
lines
)


def info():
    '''Show context info'''

    env = cpenv.get_active_env()
    modules = []
    if env:
        modules = env.get_modules()
    active_modules = cpenv.get_active_modules()

    if not env and not modules and not active_modules:
        click.echo('\nNo active modules...')
        return

    click.echo(bold('\nActive modules'))
    if env:
        click.echo(format_objects([env] + active_modules))

        available_modules = set(modules) - set(active_modules)
        if available_modules:

            click.echo(
                bold('\nInactive modules in {}\n').format(cyan(env.name))
            )
            click.echo(format_objects(available_modules, header=False))

    else:
        click.echo(format_objects(active_modules))

    available_shared_modules = set(cpenv.get_modules()) - set(active_modules)
    if not available_shared_modules:
        return

    click.echo(bold('\nInactive shared modules \n'))
    click.echo(format_objects(available_shared_modules, header=False))

def
info
(
)
:
env
=
cpenv
.
get_active_env
(
)
modules
=
[
]
if
env
:
modules
=
env
.
get_modules
(
)
active_modules
=
cpenv
.
get_active_modules
(
)
if
not
env
and
not
modules
and
not
active_modules
:
click
.
echo
(
'\nNo active modules...'
)
return
click
.
echo
(
bold
(
'\nActive modules'
)
)
if
env
:
click
.
echo
(
format_objects
(
[
env
]
+
active_modules
)
)
available_modules
=
set
(
modules
)
set
(
active_modules
)
if
available_modules
:
click
.
echo
(
bold
(
'\nInactive modules in {}\n'
)
.
format
(
cyan
(
env
.
name
)
)
)
click
.
echo
(
format_objects
(
available_modules
,
header
=
False
)
)
else
:
click
.
echo
(
format_objects
(
active_modules
)
)
available_shared_modules
=
set
(
cpenv
.
get_modules
(
)
)
set
(
active_modules
)
if
not
available_shared_modules
:
return
click
.
echo
(
bold
(
'\nInactive shared modules \n'
)
)
click
.
echo
(
format_objects
(
available_shared_modules
,
header
=
False
)
)


def activate(paths, skip_local, skip_shared):
    '''Activate an environment'''

    if not paths:
        ctx = click.get_current_context()
        if cpenv.get_active_env():
            ctx.invoke(info)
            return

        click.echo(ctx.get_help())
        examples = (
            '\nExamples: \n'
            '    cpenv activate my_env\n'
            '    cpenv activate ./relative/path/to/my_env\n'
            '    cpenv activate my_env my_module\n'
        )
        click.echo(examples)
        return

    if skip_local:
        cpenv.module_resolvers.remove(cpenv.resolver.module_resolver)
        cpenv.module_resolvers.remove(cpenv.resolver.active_env_module_resolver)

    if skip_shared:
        cpenv.module_resolvers.remove(cpenv.resolver.modules_path_resolver)

    try:
        r = cpenv.resolve(*paths)
    except cpenv.ResolveError as e:
        click.echo('\n' + str(e))
        return

    resolved = set(r.resolved)
    active_modules = set()
    env = cpenv.get_active_env()
    if env:
        active_modules.add(env)
    active_modules.update(cpenv.get_active_modules())

    new_modules = resolved - active_modules
    old_modules = active_modules & resolved

    if old_modules and not new_modules:
        click.echo(
            '\nModules already active: '
            + bold(' '.join([obj.name for obj in old_modules]))
        )
        return

    if env and contains_env(new_modules):
        click.echo('\nUse bold(exit) to leave your active environment first.')
        return

    click.echo('\nResolved the following modules...')
    click.echo(format_objects(r.resolved))
    r.activate()
    click.echo(blue('\nLaunching subshell...'))

    modules = sorted(resolved | active_modules, key=_type_and_name)
    prompt = ':'.join([obj.name for obj in modules])
    shell.launch(prompt)


def create(name_or_path, config):
    '''Create a new environment.'''

    if not name_or_path:
        ctx = click.get_current_context()
        click.echo(ctx.get_help())
        examples = (
            '\nExamples:\n'
            '    cpenv create my_env\n'
            '    cpenv create ./relative/path/to/my_env\n'
            '    cpenv create my_env --config ./relative/path/to/config\n'
            '    cpenv create my_env --config git@github.com:user/config.git\n'
        )
        click.echo(examples)
        return

    click.echo(
        blue('Creating a new virtual environment ' + name_or_path)
    )
    try:
        env = cpenv.create(name_or_path, config)
    except Exception as e:
        click.echo(bold_red('FAILED TO CREATE ENVIRONMENT!'))
        click.echo(e)
    else:
        click.echo(bold_green('Successfully created environment!'))
    click.echo(blue('Launching subshell'))

    cpenv.activate(env)
    shell.launch(env.name)


def remove(name_or_path):
    '''Remove an environment'''

    click.echo()
    try:
        r = cpenv.resolve(name_or_path)
    except cpenv.ResolveError as e:
        click.echo(e)
        return

    obj = r.resolved[0]
    if not isinstance(obj, cpenv.VirtualEnvironment):
        click.echo('{} is a module. Use `cpenv module remove` instead.')
        return

    click.echo(format_objects([obj]))
    click.echo()

    user_confirmed = click.confirm(
        red('Are you sure you want to remove this environment?')
    )
    if user_confirmed:
        click.echo('Attempting to remove...', nl=False)

        try:
            obj.remove()
        except Exception as e:
            click.echo(bold_red('FAIL'))
            click.echo(e)
        else:
            click.echo(bold_green('OK!'))


def add(path):
    '''Add an environment to the cache. Allows you to activate the environment
    by name instead of by full path'''

    click.echo('\nAdding {} to cache......'.format(path), nl=False)
    try:
        r = cpenv.resolve(path)
    except Exception as e:
        click.echo(bold_red('FAILED'))
        click.echo(e)
        return

    if isinstance(r.resolved[0], cpenv.VirtualEnvironment):
        EnvironmentCache.add(r.resolved[0])
        EnvironmentCache.save()
        click.echo(bold_green('OK!'))

def
add
(
path
)
:
click
.
echo
(
'\nAdding {} to cache......'
.
format
(
path
)
,
nl
=
False
)
try
:
r
=
cpenv
.
resolve
(
path
)
except
Exception
as
e
:
click
.
echo
(
bold_red
(
'FAILED'
)
)
click
.
echo
(
e
)
return
if
isinstance
(
r
.
resolved
[
0
]
,
cpenv
.
VirtualEnvironment
)
:
EnvironmentCache
.
add
(
r
.
resolved
[
0
]
)
EnvironmentCache
.
save
(
)
click
.
echo
(
bold_green
(
'OK!'
)
)

Add an environment to the cache . Allows you to activate the environment by name instead of by full path


def remove(path):
    '''Remove a cached environment. Removed paths will no longer be able to
    be activated by name'''

    r = cpenv.resolve(path)
    if isinstance(r.resolved[0], cpenv.VirtualEnvironment):
        EnvironmentCache.discard(r.resolved[0])
        EnvironmentCache.save()

def remove ( path ) : r = cpenv . resolve ( path ) if isinstance ( r . resolved [ 0 ] , cpenv . VirtualEnvironment ) : EnvironmentCache . discard ( r . resolved [ 0 ] ) EnvironmentCache . save ( )

Remove a cached environment . Removed paths will no longer be able to be activated by name


def create(name_or_path, config):
    '''Create a new template module.

    You can also specify a filesystem path like "./modules/new_module"
    '''

    click.echo('Creating module {}...'.format(name_or_path), nl=False)
    try:
        module = cpenv.create_module(name_or_path, config)
    except Exception as e:
        click.echo(bold_red('FAILED'))
        raise
    else:
        click.echo(bold_green('OK!'))
        click.echo('Browse to your new module and make some changes.')
        click.echo("When you're ready add the module to an environment:")
        click.echo('    cpenv module add my_module ./path/to/my_module')
        click.echo('Or track your module on git and add it directly from the repo:')
        click.echo('    cpenv module add my_module git@github.com:user/my_module.git')


def add(name, path, branch, type):
    '''Add a module to an environment. PATH can be a git repository path or
    a filesystem path. '''

    if not name and not path:
        ctx = click.get_current_context()
        click.echo(ctx.get_help())
        examples = (
            '\nExamples:\n'
            '    cpenv module add my_module ./path/to/my_module\n'
            '    cpenv module add my_module git@github.com:user/my_module.git'
            '    cpenv module add my_module git@github.com:user/my_module.git --branch=master --type=shared'
        )
        click.echo(examples)
        return

    if not name:
        click.echo('Missing required argument: name')
        return

    if not path:
        click.echo('Missing required argument: path')

    env = cpenv.get_active_env()
    if type=='local':
        if not env:
            click.echo('\nActivate an environment to add a local module.\n')
            return

        if click.confirm('\nAdd {} to active env {}?'.format(name, env.name)):
            click.echo('Adding module...', nl=False)
            try:
                env.add_module(name, path, branch)
            except:
                click.echo(bold_red('FAILED'))
                raise
            else:
                click.echo(bold_green('OK!'))

        return

    module_paths = cpenv.get_module_paths()
    click.echo('\nAvailable module paths:\n')
    for i, mod_path in enumerate(module_paths):
        click.echo('    {}. {}'.format(i, mod_path))
    choice = click.prompt(
        'Where do you want to add your module?',
        type=int,
        default=0
    )
    module_root = module_paths[choice]
    module_path = utils.unipath(module_root, name)
    click.echo('Creating module {}...'.format(module_path), nl=False)
    try:
        cpenv.create_module(module_path, path, branch)
    except:
        click.echo(bold_red('FAILED'))
        raise
    else:
        click.echo(bold_green('OK!'))

Add a module to an environment . PATH can be a git repository path or a filesystem path .


def localize(name):
    '''Copy a global module to the active environment.'''

    env = cpenv.get_active_env()
    if not env:
        click.echo('You need to activate an environment first.')
        return

    try:
        r = cpenv.resolve(name)
    except cpenv.ResolveError as e:
        click.echo('\n' + str(e))

    module = r.resolved[0]
    if isinstance(module, cpenv.VirtualEnvironment):
        click.echo('\nCan only localize a module not an environment')
        return

    active_modules = cpenv.get_active_modules()
    if module in active_modules:
        click.echo('\nCan not localize an active module.')
        return

    if module in env.get_modules():
        click.echo('\n{} is already local to {}'.format(module.name, env.name))
        return

    if click.confirm('\nAdd {} to env {}?'.format(module.name, env.name)):
        click.echo('Adding module...', nl=False)
        try:
            module = env.add_module(module.name, module.path)
        except:
            click.echo(bold_red('FAILED'))
            raise
        else:
            click.echo(bold_green('OK!'))

    click.echo('\nActivate the localize module:')
    click.echo('    cpenv activate {} {}'.format(env.name, module.name))

Copy a global module to the active environment .


def path_resolver(resolver, path):
    '''Resolves VirtualEnvironments with a relative or absolute path'''

    path = unipath(path)

    if is_environment(path):
        return VirtualEnvironment(path)

    raise ResolveError

def path_resolver ( resolver , path ) : path = unipath ( path ) if is_environment ( path ) : return VirtualEnvironment ( path ) raise ResolveError


def home_resolver(resolver, path):
    '''Resolves VirtualEnvironments in CPENV_HOME'''

    from .api import get_home_path

    path = unipath(get_home_path(), path)

    if is_environment(path):
        return VirtualEnvironment(path)

    raise ResolveError

def home_resolver ( resolver , path ) : from . api import get_home_path path = unipath ( get_home_path ( ) , path ) if is_environment ( path ) : return VirtualEnvironment ( path ) raise ResolveError


def cache_resolver(resolver, path):
    '''Resolves VirtualEnvironments in EnvironmentCache'''

    env = resolver.cache.find(path)
    if env:
        return env

    raise ResolveError

def cache_resolver ( resolver , path ) : env = resolver . cache . find ( path ) if env : return env raise ResolveError


def module_resolver(resolver, path):
    '''Resolves module in previously resolved environment.'''

    if resolver.resolved:

        if isinstance(resolver.resolved[0], VirtualEnvironment):
            env = resolver.resolved[0]
            mod = env.get_module(path)

            if mod:
                return mod

    raise ResolveError

def module_resolver ( resolver , path ) : if resolver . resolved : if isinstance ( resolver . resolved [ 0 ] , VirtualEnvironment ) : env = resolver . resolved [ 0 ] mod = env . get_module ( path ) if mod : return mod raise ResolveError

Resolves module in previously resolved environment .


def active_env_module_resolver(resolver, path):
    '''Resolves modules in currently active environment.'''

    from .api import get_active_env

    env = get_active_env()
    if not env:
        raise ResolveError

    mod = env.get_module(path)
    if not mod:
        raise ResolveError

    return mod

def active_env_module_resolver ( resolver , path ) : from . api import get_active_env env = get_active_env ( ) if not env : raise ResolveError mod = env . get_module ( path ) if not mod : raise ResolveError return mod

Resolves modules in currently active environment .


def redirect_resolver(resolver, path):
    '''Resolves environment from .cpenv file...recursively walks up the tree
    in attempt to find a .cpenv file'''

    if not os.path.exists(path):
        raise ResolveError

    if os.path.isfile(path):
        path = os.path.dirname(path)

    for root, _, _ in walk_up(path):
        if is_redirecting(root):
            env_paths = redirect_to_env_paths(unipath(root, '.cpenv'))
            r = Resolver(*env_paths)
            return r.resolve()

    raise ResolveError

def redirect_resolver ( resolver , path ) : if not os . path . exists ( path ) : raise ResolveError if os . path . isfile ( path ) : path = os . path . dirname ( path ) for root , _ , _ in walk_up ( path ) : if is_redirecting ( root ) : env_paths = redirect_to_env_paths ( unipath ( root , '.cpenv' ) ) r = Resolver ( * env_paths ) return r . resolve ( ) raise ResolveError

Resolves environment from . cpenv file ... recursively walks up the tree in attempt to find a . cpenv file


def transpose(a, axes=None):
    """Returns a view of the array with axes transposed.

    For a 1-D array, this has no effect.
    For a 2-D array, this is the usual matrix transpose.
    For an n-D array, if axes are given, their order indicates how the
    axes are permuted

    Args:
      a (array_like): Input array.
      axes (list of int, optional): By default, reverse the dimensions,
        otherwise permute the axes according to the values given.
    """
    if isinstance(a, np.ndarray):
        return np.transpose(a, axes)
    elif isinstance(a, RemoteArray):
        return a.transpose(*axes)
    elif isinstance(a, Remote):
        return _remote_to_array(a).transpose(*axes)
    elif isinstance(a, DistArray):
        if axes is None:
            axes = range(a.ndim - 1, -1, -1)
        axes = list(axes)
        if len(set(axes)) < len(axes):
            raise ValueError("repeated axis in transpose")
        if sorted(axes) != list(range(a.ndim)):
            raise ValueError("axes don't match array")
        distaxis = a._distaxis
        new_distaxis = axes.index(distaxis)
        new_subarrays = [ra.transpose(*axes) for ra in a._subarrays]
        return DistArray(new_subarrays, new_distaxis)
    else:
        return np.transpose(a, axes)

def
transpose
(
a
,
axes
=
)
:
if
isinstance
(
a
,
np
.
ndarray
)
:
return
np
.
transpose
(
a
,
axes
)
elif
isinstance
(
a
,
RemoteArray
)
:
return
a
.
transpose
(
*
axes
)
elif
isinstance
(
a
,
Remote
)
:
return
_remote_to_array
(
a
)
.
transpose
(
*
axes
)
elif
isinstance
(
a
,
DistArray
)
:
if
axes
is
:
axes
=
range
(
a
.
ndim
1
,
1
,
1
)
axes
=
list
(
axes
)
if
len
(
set
(
axes
)
)
<
len
(
axes
)
:
raise
ValueError
(
"repeated axis in transpose"
)
if
sorted
(
axes
)
!=
list
(
range
(
a
.
ndim
)
)
:
raise
ValueError
(
"axes don't match array"
)
distaxis
=
a
.
_distaxis
new_distaxis
=
axes
.
index
(
distaxis
)
new_subarrays
=
[
ra
.
transpose
(
*
axes
)
for
ra
in
a
.
_subarrays
]
return
DistArray
(
new_subarrays
,
new_distaxis
)
else
:
return
np
.
transpose
(
a
,
axes
)

Returns a view of the array with axes transposed .


def rollaxis(a, axis, start=0):
    """Roll the specified axis backwards, until it lies in a given position.

    Args:
      a (array_like): Input array.
      axis (int): The axis to roll backwards.  The positions of the other axes
        do not change relative to one another.
      start (int, optional): The axis is rolled until it lies before this
        position.  The default, 0, results in a "complete" roll.

    Returns:
      res (ndarray)
    """
    if isinstance(a, np.ndarray):
        return np.rollaxis(a, axis, start)
    if axis not in range(a.ndim):
        raise ValueError(
                'rollaxis: axis (%d) must be >=0 and < %d' % (axis, a.ndim))
    if start not in range(a.ndim + 1):
        raise ValueError(
                'rollaxis: start (%d) must be >=0 and < %d' % (axis, a.ndim+1))
    axes = list(range(a.ndim))
    axes.remove(axis)
    axes.insert(start, axis)
    return transpose(a, axes)

Roll the specified axis backwards until it lies in a given position .


def expand_dims(a, axis):
    """Insert a new axis, corresponding to a given position in the array shape

    Args:
      a (array_like): Input array.
      axis (int): Position (amongst axes) where new axis is to be inserted.
    """
    if hasattr(a, 'expand_dims') and hasattr(type(a), '__array_interface__'):
        return a.expand_dims(axis)
    else:
        return np.expand_dims(a, axis)

def expand_dims ( a , axis ) : if hasattr ( a , 'expand_dims' ) and hasattr ( type ( a ) , '__array_interface__' ) : return a . expand_dims ( axis ) else : return np . expand_dims ( a , axis )

Insert a new axis corresponding to a given position in the array shape


def concatenate(tup, axis=0):
    """Join a sequence of arrays together.
    Will aim to join `ndarray`, `RemoteArray`, and `DistArray` without moving
    their data, if they happen to be on different engines.

    Args:
      tup (sequence of array_like): Arrays to be concatenated. They must have
        the same shape, except in the dimension corresponding to `axis`.
      axis (int, optional): The axis along which the arrays will be joined.

    Returns:
      res: `ndarray`, if inputs were all local
           `RemoteArray`, if inputs were all on the same remote engine
           `DistArray`, if inputs were already scattered on different engines
    """
    from distob import engine
    if len(tup) is 0:
        raise ValueError('need at least one array to concatenate')
    first = tup[0]
    others = tup[1:]
    # allow subclasses to provide their own implementations of concatenate:
    if (hasattr(first, 'concatenate') and
            hasattr(type(first), '__array_interface__')):
        return first.concatenate(others, axis)
    # convert all arguments to arrays/RemoteArrays if they are not already:
    arrays = []
    for ar in tup:
        if isinstance(ar, DistArray):
            if axis == ar._distaxis:
                arrays.extend(ar._subarrays)
            else:
                # Since not yet implemented arrays distributed on more than
                # one axis, will fetch and re-scatter on the new axis:
                arrays.append(gather(ar))
        elif isinstance(ar, RemoteArray):
            arrays.append(ar)
        elif isinstance(ar, Remote):
            arrays.append(_remote_to_array(ar))
        elif hasattr(type(ar), '__array_interface__'):
            # then treat as a local ndarray
            arrays.append(ar)
        else:
            arrays.append(np.array(ar))
    if all(isinstance(ar, np.ndarray) for ar in arrays):
        return np.concatenate(arrays, axis)
    total_length = 0
    # validate dimensions are same, except for axis of concatenation:
    commonshape = list(arrays[0].shape)
    commonshape[axis] = None # ignore this axis for shape comparison
    for ar in arrays:
        total_length += ar.shape[axis]
        shp = list(ar.shape)
        shp[axis] = None
        if shp != commonshape:
            raise ValueError('incompatible shapes for concatenation')
    # set sensible target block size if splitting subarrays further:
    blocksize = ((total_length - 1) // engine.nengines) + 1
    rarrays = []
    for ar in arrays:
        if isinstance(ar, DistArray):
            rarrays.extend(ar._subarrays)
        elif isinstance(ar, RemoteArray):
            rarrays.append(ar)
        else:
            da = _scatter_ndarray(ar, axis, blocksize)
            for ra in da._subarrays:
                rarrays.append(ra)
            del da
    del arrays
    # At this point rarrays is a list of RemoteArray to be concatenated
    eid = rarrays[0]._id.engine
    if all(ra._id.engine == eid for ra in rarrays):
        # Arrays to be joined are all on the same engine
        if eid == engine.eid:
            # Arrays are all local
            return concatenate([gather(r) for r in rarrays], axis)
        else:
            return call(concatenate, rarrays, axis)
    else:
        # Arrays to be joined are on different engines.
        # TODO: consolidate any consecutive arrays already on same engine
        return DistArray(rarrays, axis)

Join a sequence of arrays together . Will aim to join ndarray RemoteArray and DistArray without moving their data if they happen to be on different engines .


def _broadcast_shape(*args):
    """Return the shape that would result from broadcasting the inputs"""
    #TODO: currently incorrect result if a Sequence is provided as an input
    shapes = [a.shape if hasattr(type(a), '__array_interface__')
              else () for a in args]
    ndim = max(len(sh) for sh in shapes) # new common ndim after broadcasting
    for i, sh in enumerate(shapes):
        if len(sh) < ndim:
            shapes[i] = (1,)*(ndim - len(sh)) + sh
    return tuple(max(sh[ax] for sh in shapes) for ax in range(ndim))


def mean(a, axis=None, dtype=None, out=None, keepdims=False):
    """
    Compute the arithmetic mean along the specified axis.

    Returns the average of the array elements.  The average is taken over
    the flattened array by default, otherwise over the specified axis.
    `float64` intermediate and return values are used for integer inputs.

    Parameters
    ----------
    a : array_like
        Array containing numbers whose mean is desired. If `a` is not an
        array, a conversion is attempted.
    axis : None or int or tuple of ints, optional
        Axis or axes along which the means are computed. The default is to
        compute the mean of the flattened array.
        If this is a tuple of ints, a mean is performed over multiple axes,
        instead of a single axis or all the axes as before.
    dtype : data-type, optional
        Type to use in computing the mean.  For integer inputs, the default
        is `float64`; for floating point inputs, it is the same as the
        input dtype.
    out : ndarray, optional
        Alternate output array in which to place the result.  The default
        is ``None``; if provided, it must have the same shape as the
        expected output, but the type will be cast if necessary.
        See `doc.ufuncs` for details.
    keepdims : bool, optional
        If this is set to True, the axes which are reduced are left
        in the result as dimensions with size one. With this option,
        the result will broadcast correctly against the original `arr`.

    Returns
    -------
    m : ndarray, see dtype parameter above

    Notes
    -----
    np.mean fails to pass the keepdims parameter to ndarray subclasses.
    That is the main reason we implement this function.
    """
    if (isinstance(a, np.ndarray) or
            isinstance(a, RemoteArray) or
            isinstance(a, DistArray)):
        return a.mean(axis=axis, dtype=dtype, out=out, keepdims=keepdims)
    else:
        return np.mean(a, axis=axis, dtype=dtype, out=out, keepdims=keepdims)

def mean ( a , axis = None , dtype = None , out = None , keepdims = False ) : if ( isinstance ( a , np . ndarray ) or isinstance ( a , RemoteArray ) or isinstance ( a , DistArray ) ) : return a . mean ( axis = axis , dtype = dtype , out = out , keepdims = keepdims ) else : return np . mean ( a , axis = axis , dtype = dtype , out = out , keepdims = keepdims )

Compute the arithmetic mean along the specified axis .


def _valid_distaxis(shapes, ax):
        """`ax` is a valid candidate for a distributed axis if the given
        subarray shapes are all the same when ignoring axis `ax`"""
        compare_shapes = np.vstack(shapes)
        if ax < compare_shapes.shape[1]:
            compare_shapes[:, ax] = -1
        return np.count_nonzero(compare_shapes - compare_shapes[0]) == 0

def _valid_distaxis ( shapes , ax ) : compare_shapes = np . vstack ( shapes ) if ax < compare_shapes . shape [ 1 ] : compare_shapes [ : , ax ] = - 1 return np . count_nonzero ( compare_shapes - compare_shapes [ 0 ] ) == 0

ax is a valid candidate for a distributed axis if the given subarray shapes are all the same when ignoring axis ax


def run(*args, **kwargs):
    '''Returns True if successful, False if failure'''

    kwargs.setdefault('env', os.environ)
    kwargs.setdefault('shell', True)

    try:
        subprocess.check_call(' '.join(args), **kwargs)
        return True
    except subprocess.CalledProcessError:
        logger.debug('Error running: {}'.format(args))
        return False

def
run
(
*
args
,
*
*
kwargs
)
:
kwargs
.
setdefault
(
'env'
,
os
.
environ
)
kwargs
.
setdefault
(
'shell'
,
True
)
try
:
subprocess
.
check_call
(
' '
.
join
(
args
)
,
*
*
kwargs
)
return
True
except
subprocess
.
CalledProcessError
:
logger
.
debug
(
'Error running: {}'
.
format
(
args
)
)
return
False

Returns True if successful False if failure


def cmd():
    '''Return a command to launch a subshell'''

    if platform == 'win':
        return ['cmd.exe', '/K']

    elif platform == 'linux':
        ppid = os.getppid()
        ppid_cmdline_file = '/proc/{0}/cmdline'.format(ppid)
        try:
            with open(ppid_cmdline_file) as f:
                cmd = f.read()
            if cmd.endswith('\x00'):
                cmd = cmd[:-1]
            cmd = cmd.split('\x00')
            return cmd + [binpath('subshell.sh')]
        except:
            cmd = 'bash'

    else:
        cmd = 'bash'

    return [cmd, binpath('subshell.sh')]

def cmd ( ) : if platform == 'win' : return [ 'cmd.exe' , '/K' ] elif platform == 'linux' : ppid = os . getppid ( ) ppid_cmdline_file = '/proc/{0}/cmdline' . format ( ppid ) try : with open ( ppid_cmdline_file ) as f : cmd = f . read ( ) if cmd . endswith ( '\x00' ) : cmd = cmd [ : - 1 ] cmd = cmd . split ( '\x00' ) return cmd + [ binpath ( 'subshell.sh' ) ] except : cmd = 'bash' else : cmd = 'bash' return [ cmd , binpath ( 'subshell.sh' ) ]


def prompt(prefix=None, colored=True):
    '''Generate a prompt with a given prefix

    linux/osx: [prefix] user@host cwd $
          win: [prefix] cwd:
    '''

    if platform == 'win':
        return '[{0}] $P$G'.format(prefix)
    else:
        if colored:
            return (
                '[{0}] '  # White prefix
                '\\[\\033[01;32m\\]\\u@\\h\\[\\033[00m\\] '  # Green user@host
                '\\[\\033[01;34m\\]\\w $ \\[\\033[00m\\]'  # Blue cwd $
            ).format(prefix)
        return '[{0}] \\u@\\h \\w $ '.format(prefix)


def launch(prompt_prefix=None):
    '''Launch a subshell'''

    if prompt_prefix:
        os.environ['PROMPT'] = prompt(prompt_prefix)

    subprocess.call(cmd(), env=os.environ.data)

def launch ( prompt_prefix = None ) : if prompt_prefix : os . environ [ 'PROMPT' ] = prompt ( prompt_prefix ) subprocess . call ( cmd ( ) , env = os . environ . data )


def add_file(self, file, **kwargs):
        """Append a file to file repository.

        For file monitoring, monitor instance needs file.
        Please put the name of file to `file` argument.

        :param file: the name of file you want monitor.

        """

        if os.access(file, os.F_OK):

            if file in self.f_repository:
                raise DuplicationError("file already added.")

            self.f_repository.append(file)

        else:
            raise IOError("file not found.")

def
add_file
(
self
,
file
,
*
*
kwargs
)
:
if
os
.
access
(
file
,
os
.
F_OK
)
:
if
file
in
self
.
f_repository
:
raise
DuplicationError
(
"file already added."
)
self
.
f_repository
.
append
(
file
)
else
:
raise
IOError
(
"file not found."
)


def add_files(self, filelist, **kwargs):
        """Append files to file repository.

        ModificationMonitor can append files to repository using this.
        Please put the list of file names to `filelist` argument.

        :param filelist: the list of file nmaes
        """

        # check filelist is list type
        if not isinstance(filelist, list):
            raise TypeError("request the list type.")

        for file in filelist:
            self.add_file(file)

def
add_files
(
self
,
filelist
,
*
*
kwargs
)
:
# check filelist is list type
if
not
isinstance
(
filelist
,
list
)
:
raise
TypeError
(
"request the list type."
)
for
file
in
filelist
:
self
.
add_file
(
file
)

Append files to file repository . ModificationMonitor can append files to repository using this . Please put the list of file names to filelist argument .


def monitor(self, sleep=5):
        """Run file modification monitor.

        The monitor can catch file modification using timestamp and file body.
        Monitor has timestamp data and file body data. And insert timestamp
        data and file body data before into while roop. In while roop, monitor
        get new timestamp and file body, and then monitor compare new timestamp
        to originaltimestamp. If new timestamp and file body differ original,
        monitor regard thease changes as `modification`. Then monitor create
        instance of FileModificationObjectManager and FileModificationObject,
        and monitor insert FileModificationObject to FileModificationObject-
        Manager. Then, yield this object.

        :param sleep: How times do you sleep in while roop.
        """

        manager = FileModificationObjectManager()

        timestamps = {}
        filebodies = {}

        # register original timestamp and filebody to dict
        for file in self.f_repository:
            timestamps[file] = self._get_mtime(file)
            filebodies[file] = open(file).read()

        while True:

            for file in self.f_repository:

                mtime = timestamps[file]
                fbody = filebodies[file]

                modified = self._check_modify(file, mtime, fbody)

                # file not modify -> continue
                if not modified:
                    continue

                # file modifies -> create the modification object

                new_mtime = self._get_mtime(file)
                new_fbody = open(file).read()

                obj = FileModificationObject(
                        file,
                        (mtime, new_mtime),
                        (fbody, new_fbody) )

                # overwrite new timestamp and filebody
                timestamps[file] = new_mtime
                filebodies[file] = new_fbody

                # append file modification object to manager
                manager.add_object(obj)

                # return new modification object
                yield obj

            time.sleep(sleep)


def status_job(self, fn=None, name=None, timeout=3):
        """Decorator that invokes `add_status_job`.

        ::

            @app.status_job
            def postgresql():
                # query/ping postgres

            @app.status_job(name="Active Directory")
            def active_directory():
                # query active directory

            @app.status_job(timeout=5)
            def paypal():
                # query paypal, timeout after 5 seconds

        """
        if fn is None:
            def decorator(fn):
                self.add_status_job(fn, name, timeout)
            return decorator
        else:
            self.add_status_job(fn, name, timeout)

def status_job ( self , fn = None , name = None , timeout = 3 ) : if fn is None : def decorator ( fn ) : self . add_status_job ( fn , name , timeout ) return decorator else : self . add_status_job ( fn , name , timeout )


def _pipepager(text, cmd, color):
    """Page through text by feeding it to another program.  Invoking a
    pager through this might support colors.
    """
    import subprocess
    env = dict(os.environ)

    # If we're piping to less we might support colors under the
    # condition that
    cmd_detail = cmd.rsplit('/', 1)[-1].split()
    if color is None and cmd_detail[0] == 'less':
        less_flags = os.environ.get('LESS', '') + ' '.join(cmd_detail[1:])
        if not less_flags:
            env['LESS'] = '-R'
            color = True
        elif 'r' in less_flags or 'R' in less_flags:
            color = True

    if not color:
        text = strip_ansi(text)

    c = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
                         env=env)
    encoding = get_best_encoding(c.stdin)
    try:
        c.stdin.write(text.encode(encoding, 'replace'))
        c.stdin.close()
    except (IOError, KeyboardInterrupt):
        pass

    # Less doesn't respect ^C, but catches it for its own UI purposes (aborting
    # search or other commands inside less).
    #
    # That means when the user hits ^C, the parent process (click) terminates,
    # but less is still alive, paging the output and messing up the terminal.
    #
    # If the user wants to make the pager exit on ^C, they should set
    # `LESS='-K'`. It's not our decision to make.
    while True:
        try:
            c.wait()
        except KeyboardInterrupt:
            pass
        else:
            break

Page through text by feeding it to another program . Invoking a pager through this might support colors .


def profil_annuel(df, func='mean'):
    """
    Calcul du profil annuel

    Paramètres:
    df: DataFrame de données dont l'index est une série temporelle
        (cf module xair par exemple)
    func: function permettant le calcul. Soit un nom de fonction numpy ('mean', 'max', ...)
        soit la fonction elle-même (np.mean, np.max, ...)
    Retourne:
    Un DataFrame de moyennes par mois
    """

    func = _get_funky(func)
    res = df.groupby(lambda x: x.month).aggregate(func)
    # On met des noms de mois à la place des numéros dans l'index
    res.index = [cal.month_name[i] for i in range(1,13)]
    return res


def run_global_hook(hook_name, *args):
    '''Attempt to run a global hook by name with args'''

    hook_finder = HookFinder(get_global_hook_path())
    hook = hook_finder(hook_name)
    if hook:
        hook.run(*args)

def run_global_hook ( hook_name , * args ) : hook_finder = HookFinder ( get_global_hook_path ( ) ) hook = hook_finder ( hook_name ) if hook : hook . run ( * args )


def moyennes_glissantes(df, sur=8, rep=0.75):
    """
    Calcule de moyennes glissantes

    Paramètres:
    df: DataFrame de mesures sur lequel appliqué le calcul
    sur: (int, par défaut 8) Nombre d'observations sur lequel s'appuiera le
    calcul
    rep: (float, défaut 0.75) Taux de réprésentativité en dessous duquel le
    calcul renverra NaN

    Retourne:
    Un DataFrame des moyennes glissantes calculées
    """
    return pd.rolling_mean(df, window=sur, min_periods=rep * sur)

def moyennes_glissantes ( df , sur = 8 , rep = 0.75 ) : return pd . rolling_mean ( df , window = sur , min_periods = rep * sur )


def aot40_vegetation(df, nb_an):
    """
    Calcul de l'AOT40 du 1er mai au 31 juillet

    *AOT40 : AOT 40 ( exprimé en micro g/m³ par heure ) signifie la somme des
    différences entre les concentrations horaires supérieures à 40 parties par
    milliard ( 40 ppb soit 80 micro g/m³ ), durant une période donnée en
    utilisant uniquement les valeurs sur 1 heure mesurées quotidiennement
    entre 8 heures (début de la mesure) et 20 heures (pile, fin de la mesure) CET,
    ce qui correspond à de 8h à 19h TU (donnant bien 12h de mesures, 8h donnant
    la moyenne horaire de 7h01 à 8h00)

    Paramètres:
    df: DataFrame de mesures sur lequel appliqué le calcul
    nb_an: (int) Nombre d'années contenu dans le df, et servant à diviser le
    résultat retourné

    Retourne:
    Un DataFrame de résultat de calcul
    """

    return _aot(df.tshift(1), nb_an=nb_an, limite=80, mois_debut=5, mois_fin=7,
                heure_debut=8, heure_fin=19)

Calcul de l AOT40 du 1er mai au 31 juillet


def validate(self):
        '''Validate all the entries in the environment cache.'''

        for env in list(self):
            if not env.exists:
                self.remove(env)

def validate ( self ) : for env in list ( self ) : if not env . exists : self . remove ( env )

Validate all the entries in the environment cache .


def load(self):
        '''Load the environment cache from disk.'''

        if not os.path.exists(self.path):
            return

        with open(self.path, 'r') as f:
            env_data = yaml.load(f.read())

        if env_data:
            for env in env_data:
                self.add(VirtualEnvironment(env['root']))

def load ( self ) : if not os . path . exists ( self . path ) : return with open ( self . path , 'r' ) as f : env_data = yaml . load ( f . read ( ) ) if env_data : for env in env_data : self . add ( VirtualEnvironment ( env [ 'root' ] ) )


def save(self):
        '''Save the environment cache to disk.'''

        env_data = [dict(name=env.name, root=env.path) for env in self]
        encode = yaml.safe_dump(env_data, default_flow_style=False)

        with open(self.path, 'w') as f:
            f.write(encode)

def save ( self ) : env_data = [ dict ( name = env . name , root = env . path ) for env in self ] encode = yaml . safe_dump ( env_data , default_flow_style = False ) with open ( self . path , 'w' ) as f : f . write ( encode )


def prompt(text, default=None, hide_input=False,
           confirmation_prompt=False, type=None,
           value_proc=None, prompt_suffix=': ',
           show_default=True, err=False):
    """Prompts a user for input.  This is a convenience function that can
    be used to prompt a user for input later.

    If the user aborts the input by sending a interrupt signal, this
    function will catch it and raise a :exc:`Abort` exception.

    .. versionadded:: 6.0
       Added unicode support for cmd.exe on Windows.

    .. versionadded:: 4.0
       Added the `err` parameter.

    :param text: the text to show for the prompt.
    :param default: the default value to use if no input happens.  If this
                    is not given it will prompt until it's aborted.
    :param hide_input: if this is set to true then the input value will
                       be hidden.
    :param confirmation_prompt: asks for confirmation for the value.
    :param type: the type to use to check the value against.
    :param value_proc: if this parameter is provided it's a function that
                       is invoked instead of the type conversion to
                       convert a value.
    :param prompt_suffix: a suffix that should be added to the prompt.
    :param show_default: shows or hides the default value in the prompt.
    :param err: if set to true the file defaults to ``stderr`` instead of
                ``stdout``, the same as with echo.
    """
    result = None

    def prompt_func(text):
        f = hide_input and hidden_prompt_func or visible_prompt_func
        try:
            # Write the prompt separately so that we get nice
            # coloring through colorama on Windows
            echo(text, nl=False, err=err)
            return f('')
        except (KeyboardInterrupt, EOFError):
            # getpass doesn't print a newline if the user aborts input with ^C.
            # Allegedly this behavior is inherited from getpass(3).
            # A doc bug has been filed at https://bugs.python.org/issue24711
            if hide_input:
                echo(None, err=err)
            raise Abort()

    if value_proc is None:
        value_proc = convert_type(type, default)

    prompt = _build_prompt(text, prompt_suffix, show_default, default)

    while 1:
        while 1:
            value = prompt_func(prompt)
            if value:
                break
            # If a default is set and used, then the confirmation
            # prompt is always skipped because that's the only thing
            # that really makes sense.
            elif default is not None:
                return default
        try:
            result = value_proc(value)
        except UsageError as e:
            echo('Error: %s' % e.message, err=err)
            continue
        if not confirmation_prompt:
            return result
        while 1:
            value2 = prompt_func('Repeat for confirmation: ')
            if value2:
                break
        if value == value2:
            return result
        echo('Error: the two entered values do not match', err=err)

Prompts a user for input . This is a convenience function that can be used to prompt a user for input later .


def echo_via_pager(text, color=None):
    """This function takes a text and shows it via an environment specific
    pager on stdout.

    .. versionchanged:: 3.0
       Added the `color` flag.

    :param text: the text to page.
    :param color: controls if the pager supports ANSI colors or not.  The
                  default is autodetection.
    """
    color = resolve_color_default(color)
    if not isinstance(text, string_types):
        text = text_type(text)
    from ._termui_impl import pager
    return pager(text + '\n', color)

def echo_via_pager ( text , color = None ) : color = resolve_color_default ( color ) if not isinstance ( text , string_types ) : text = text_type ( text ) from . _termui_impl import pager return pager ( text + '\n' , color )

This function takes a text and shows it via an environment specific pager on stdout .


def setup_engines(client=None):
    """Prepare all iPython engines for distributed object processing.

    Args:
      client (ipyparallel.Client, optional): If None, will create a client
        using the default ipyparallel profile.
    """
    if not client:
        try:
            client = ipyparallel.Client()
        except:
            raise DistobClusterError(
                u"""Could not connect to an ipyparallel cluster. Make
                 sure a cluster is started (e.g. to use the CPUs of a
                 single computer, can type 'ipcluster start')""")
    eids = client.ids
    if not eids:
        raise DistobClusterError(
                u'No ipyparallel compute engines are available')
    nengines = len(eids)
    dv = client[eids]
    dv.use_dill()
    with dv.sync_imports(quiet=True):
        import distob
    # create global ObjectEngine distob.engine on each engine
    ars = []
    for i in eids:
        dv.targets = i
        ars.append(dv.apply_async(_remote_setup_engine, i, nengines))
    dv.wait(ars)
    for ar in ars:
        if not ar.successful():
            raise ar.r
    # create global ObjectHub distob.engine on the client host
    if distob.engine is None:
        distob.engine = ObjectHub(-1, client)

Prepare all iPython engines for distributed object processing .


def gather(obj):
    """Retrieve objects that have been distributed, making them local again"""
    if hasattr(obj, '__distob_gather__'):
        return obj.__distob_gather__()
    elif (isinstance(obj, collections.Sequence) and
            not isinstance(obj, string_types)):
        return [gather(subobj) for subobj in obj]
    else:
        return obj

def gather ( obj ) : if hasattr ( obj , '__distob_gather__' ) : return obj . __distob_gather__ ( ) elif ( isinstance ( obj , collections . Sequence ) and not isinstance ( obj , string_types ) ) : return [ gather ( subobj ) for subobj in obj ] else : return obj

Retrieve objects that have been distributed making them local again


def apply(f, obj, *args, **kwargs):
    """Apply a function in parallel to each element of the input"""
    return vectorize(f)(obj, *args, **kwargs)

def apply ( f , obj , * args , * * kwargs ) : return vectorize ( f ) ( obj , * args , * * kwargs )


def register_proxy_type(cls, real_type, proxy_type):
        """Configure engines so that remote methods returning values of type
        `real_type` will instead return by proxy, as type `proxy_type`
        """
        if distob.engine is None:
            cls._initial_proxy_types[real_type] = proxy_type
        elif isinstance(distob.engine, ObjectHub):
            distob.engine._runtime_reg_proxy_type(real_type, proxy_type)
        else:
            # TODO: remove next line after issue #58 in dill is fixed.
            distob.engine._singleeng_reg_proxy_type(real_type, proxy_type)
            pass

Configure engines so that remote methods returning values of type real_type will instead return by proxy as type proxy_type


def is_git_repo(path):
    '''Returns True if path is a git repository.'''

    if path.startswith('git@') or path.startswith('https://'):
        return True

    if os.path.exists(unipath(path, '.git')):
        return True

    return False

def is_git_repo ( path ) : if path . startswith ( 'git@' ) or path . startswith ( 'https://' ) : return True if os . path . exists ( unipath ( path , '.git' ) ) : return True return False

Returns True if path is a git repository .


def is_home_environment(path):
    '''Returns True if path is in CPENV_HOME'''

    home = unipath(os.environ.get('CPENV_HOME', '~/.cpenv'))
    path = unipath(path)

    return path.startswith(home)

def is_home_environment ( path ) : home = unipath ( os . environ . get ( 'CPENV_HOME' , '~/.cpenv' ) ) path = unipath ( path ) return path . startswith ( home )


def is_redirecting(path):
    '''Returns True if path contains a .cpenv file'''

    candidate = unipath(path, '.cpenv')
    return os.path.exists(candidate) and os.path.isfile(candidate)

def is_redirecting ( path ) : candidate = unipath ( path , '.cpenv' ) return os . path . exists ( candidate ) and os . path . isfile ( candidate )

Returns True if path contains a . cpenv file


def redirect_to_env_paths(path):
    '''Get environment path from redirect file'''

    with open(path, 'r') as f:
        redirected = f.read()

    return shlex.split(redirected)

def redirect_to_env_paths ( path ) : with open ( path , 'r' ) as f : redirected = f . read ( ) return shlex . split ( redirected )


def expandpath(path):
    '''Returns an absolute expanded path'''

    return os.path.abspath(os.path.expandvars(os.path.expanduser(path)))

def expandpath ( path ) : return os . path . abspath ( os . path . expandvars ( os . path . expanduser ( path ) ) )


def unipath(*paths):
    '''Like os.path.join but also expands and normalizes path parts.'''

    return os.path.normpath(expandpath(os.path.join(*paths)))

def unipath ( * paths ) : return os . path . normpath ( expandpath ( os . path . join ( * paths ) ) )

Like os . path . join but also expands and normalizes path parts .


def binpath(*paths):
    '''Like os.path.join but acts relative to this packages bin path.'''

    package_root = os.path.dirname(__file__)
    return os.path.normpath(os.path.join(package_root, 'bin', *paths))

def binpath ( * paths ) : package_root = os . path . dirname ( __file__ ) return os . path . normpath ( os . path . join ( package_root , 'bin' , * paths ) )

Like os . path . join but acts relative to this packages bin path .


def ensure_path_exists(path, *args):
    '''Like os.makedirs but keeps quiet if path already exists'''
    if os.path.exists(path):
        return

    os.makedirs(path, *args)

def ensure_path_exists ( path , * args ) : if os . path . exists ( path ) : return os . makedirs ( path , * args )

Like os . makedirs but keeps quiet if path already exists


def walk_dn(start_dir, depth=10):
    '''
    Walk down a directory tree. Same as os.walk but allows for a depth limit
    via depth argument
    '''

    start_depth = len(os.path.split(start_dir))
    end_depth = start_depth + depth

    for root, subdirs, files in os.walk(start_dir):
        yield root, subdirs, files

        if len(os.path.split(root)) >= end_depth:
            break

def walk_dn ( start_dir , depth = 10 ) : start_depth = len ( os . path . split ( start_dir ) ) end_depth = start_depth + depth for root , subdirs , files in os . walk ( start_dir ) : yield root , subdirs , files if len ( os . path . split ( root ) ) >= end_depth : break

Walk down a directory tree . Same as os . walk but allows for a depth limit via depth argument


def walk_up(start_dir, depth=20):
    '''
    Walk up a directory tree
    '''
    root = start_dir

    for i in xrange(depth):
        contents = os.listdir(root)
        subdirs, files = [], []
        for f in contents:
            if os.path.isdir(os.path.join(root, f)):
                subdirs.append(f)
            else:
                files.append(f)

        yield root, subdirs, files

        parent = os.path.dirname(root)
        if parent and not parent == root:
            root = parent
        else:
            break

def walk_up ( start_dir , depth = 20 ) : root = start_dir for i in xrange ( depth ) : contents = os . listdir ( root ) subdirs , files = [ ] , [ ] for f in contents : if os . path . isdir ( os . path . join ( root , f ) ) : subdirs . append ( f ) else : files . append ( f ) yield root , subdirs , files parent = os . path . dirname ( root ) if parent and not parent == root : root = parent else : break


def preprocess_dict(d):
    '''
    Preprocess a dict to be used as environment variables.

    :param d: dict to be processed
    '''

    out_env = {}
    for k, v in d.items():

        if not type(v) in PREPROCESSORS:
            raise KeyError('Invalid type in dict: {}'.format(type(v)))

        out_env[k] = PREPROCESSORS[type(v)](v)

    return out_env

def
preprocess_dict
(
d
)
:
out_env
=
{
}
for
k
,
v
in
d
.
items
(
)
:
if
not
type
(
v
)
in
PREPROCESSORS
:
raise
KeyError
(
'Invalid type in dict: {}'
.
format
(
type
(
v
)
)
)
out_env
[
k
]
=
PREPROCESSORS
[
type
(
v
)
]
(
v
)
return
out_env

Preprocess a dict to be used as environment variables .


def _join_seq(d, k, v):
    '''Add a sequence value to env dict'''

    if k not in d:
        d[k] = list(v)

    elif isinstance(d[k], list):
        for item in v:
            if item not in d[k]:
                d[k].insert(0, item)

    elif isinstance(d[k], string_types):
        v.append(d[k])
        d[k] = v

def _join_seq ( d , k , v ) : if k not in d : d [ k ] = list ( v ) elif isinstance ( d [ k ] , list ) : for item in v : if item not in d [ k ] : d [ k ] . insert ( 0 , item ) elif isinstance ( d [ k ] , string_types ) : v . append ( d [ k ] ) d [ k ] = v


def join_dicts(*dicts):
    '''Join a bunch of dicts'''

    out_dict = {}

    for d in dicts:
        for k, v in d.iteritems():

            if not type(v) in JOINERS:
                raise KeyError('Invalid type in dict: {}'.format(type(v)))

            JOINERS[type(v)](out_dict, k, v)

    return out_dict

def
join_dicts
(
*
dicts
)
:
out_dict
=
{
}
for
d
in
dicts
:
for
k
,
v
in
d
.
iteritems
(
)
:
if
not
type
(
v
)
in
JOINERS
:
raise
KeyError
(
'Invalid type in dict: {}'
.
format
(
type
(
v
)
)
)
JOINERS
[
type
(
v
)
]
(
out_dict
,
k
,
v
)
return
out_dict


def env_to_dict(env, pathsep=os.pathsep):
    '''
    Convert a dict containing environment variables into a standard dict.
    Variables containing multiple values will be split into a list based on
    the argument passed to pathsep.

    :param env: Environment dict like os.environ.data
    :param pathsep: Path separator used to split variables
    '''

    out_dict = {}

    for k, v in env.iteritems():
        if pathsep in v:
            out_dict[k] = v.split(pathsep)
        else:
            out_dict[k] = v

    return out_dict

def env_to_dict ( env , pathsep = os . pathsep ) : out_dict = { } for k , v in env . iteritems ( ) : if pathsep in v : out_dict [ k ] = v . split ( pathsep ) else : out_dict [ k ] = v return out_dict

Convert a dict containing environment variables into a standard dict . Variables containing multiple values will be split into a list based on the argument passed to pathsep .


def dict_to_env(d, pathsep=os.pathsep):
    '''
    Convert a python dict to a dict containing valid environment variable
    values.

    :param d: Dict to convert to an env dict
    :param pathsep: Path separator used to join lists(default os.pathsep)
    '''

    out_env = {}

    for k, v in d.iteritems():
        if isinstance(v, list):
            out_env[k] = pathsep.join(v)
        elif isinstance(v, string_types):
            out_env[k] = v
        else:
            raise TypeError('{} not a valid env var type'.format(type(v)))

    return out_env

def
dict_to_env
(
d
,
pathsep
=
os
.
pathsep
)
:
out_env
=
{
}
for
k
,
v
in
d
.
iteritems
(
)
:
if
isinstance
(
v
,
list
)
:
out_env
[
k
]
=
pathsep
.
join
(
v
)
elif
isinstance
(
v
,
string_types
)
:
out_env
[
k
]
=
v
else
:
raise
TypeError
(
'{} not a valid env var type'
.
format
(
type
(
v
)
)
)
return
out_env

Convert a python dict to a dict containing valid environment variable values .


def expand_envvars(env):
    '''
    Expand all environment variables in an environment dict

    :param env: Environment dict
    '''

    out_env = {}

    for k, v in env.iteritems():
        out_env[k] = Template(v).safe_substitute(env)

    # Expand twice to make sure we expand everything we possibly can
    for k, v in out_env.items():
        out_env[k] = Template(v).safe_substitute(out_env)

    return out_env


def get_store_env_tmp():
    '''Returns an unused random filepath.'''

    tempdir = tempfile.gettempdir()
    temp_name = 'envstore{0:0>3d}'
    temp_path = unipath(tempdir, temp_name.format(random.getrandbits(9)))
    if not os.path.exists(temp_path):
        return temp_path
    else:
        return get_store_env_tmp()

def get_store_env_tmp ( ) : tempdir = tempfile . gettempdir ( ) temp_name = 'envstore{0:0>3d}' temp_path = unipath ( tempdir , temp_name . format ( random . getrandbits ( 9 ) ) ) if not os . path . exists ( temp_path ) : return temp_path else : return get_store_env_tmp ( )


def store_env(path=None):
    '''Encode current environment as yaml and store in path or a temporary
    file. Return the path to the stored environment.
    '''

    path = path or get_store_env_tmp()

    env_dict = yaml.safe_dump(os.environ.data, default_flow_style=False)

    with open(path, 'w') as f:
        f.write(env_dict)

    return path

def store_env ( path = None ) : path = path or get_store_env_tmp ( ) env_dict = yaml . safe_dump ( os . environ . data , default_flow_style = False ) with open ( path , 'w' ) as f : f . write ( env_dict ) return path

Encode current environment as yaml and store in path or a temporary file . Return the path to the stored environment .


def upstream_url(self, uri):
        "Returns the URL to the upstream data source for the given URI based on configuration"
        return self.application.options.upstream + self.request.uri

def upstream_url ( self , uri ) : return self . application . options . upstream + self . request . uri


def make_upstream_request(self):
        "Return request object for calling the upstream"
        url = self.upstream_url(self.request.uri)
        return tornado.httpclient.HTTPRequest(url,
            method=self.request.method,
            headers=self.request.headers,
            body=self.request.body if self.request.body else None)

def make_upstream_request ( self ) : url = self . upstream_url ( self . request . uri ) return tornado . httpclient . HTTPRequest ( url , method = self . request . method , headers = self . request . headers , body = self . request . body if self . request . body else None )


def ttl(self, response):
        """Returns time to live in seconds. 0 means no caching.

        Criteria:
        - response code 200
        - read-only method (GET, HEAD, OPTIONS)
        Plus http headers:
        - cache-control: option1, option2, ...
          where options are:
          private | public
          no-cache
          no-store
          max-age: seconds
          s-maxage: seconds
          must-revalidate
          proxy-revalidate
        - expires: Thu, 01 Dec 1983 20:00:00 GMT
        - pragma: no-cache (=cache-control: no-cache)

        See http://www.mobify.com/blog/beginners-guide-to-http-cache-headers/

        TODO: tests

        """
        if response.code != 200: return 0
        if not self.request.method in ['GET', 'HEAD', 'OPTIONS']: return 0

        try:
            pragma = self.request.headers['pragma']
            if pragma == 'no-cache':
                return 0
        except KeyError:
            pass

        try:
            cache_control = self.request.headers['cache-control']

            # no caching options
            for option in ['private', 'no-cache', 'no-store', 'must-revalidate', 'proxy-revalidate']:
                if cache_control.find(option): return 0

            # further parsing to get a ttl
            options = parse_cache_control(cache_control)
            try:
                return int(options['s-maxage'])
            except KeyError:
                pass
            try:
                return int(options['max-age'])
            except KeyError:
                pass

            if 's-maxage' in options:
                max_age = options['s-maxage']
                if max_age < ttl: ttl = max_age
            if 'max-age' in options:
                max_age = options['max-age']
                if max_age < ttl: ttl = max_age
            return ttl
        except KeyError:
            pass

        try:
            expires = self.request.headers['expires']
            return time.mktime(time.strptime(expires, '%a, %d %b %Y %H:%M:%S')) - time.time()
        except KeyError:
            pass

def
ttl
(
self
,
response
)
:
if
response
.
code
!=
200
:
return
0
if
not
self
.
request
.
method
in
[
'GET'
,
'HEAD'
,
'OPTIONS'
]
:
return
0
try
:
pragma
=
self
.
request
.
headers
[
'pragma'
]
if
pragma
==
'no-cache'
:
return
0
except
KeyError
:
pass
try
:
cache_control
=
self
.
request
.
headers
[
'cache-control'
]
# no caching options
for
option
in
[
'private'
,
'no-cache'
,
'no-store'
,
'must-revalidate'
,
'proxy-revalidate'
]
:
if
cache_control
.
find
(
option
)
:
return
0
# further parsing to get a ttl
options
=
parse_cache_control
(
cache_control
)
try
:
return
int
(
options
[
's-maxage'
]
)
except
KeyError
:
pass
try
:
return
int
(
options
[
'max-age'
]
)
except
KeyError
:
pass
if
's-maxage'
in
options
:
max_age
=
options
[
's-maxage'
]
if
max_age
<
ttl
:
ttl
=
max_age
if
'max-age'
in
options
:
max_age
=
options
[
'max-age'
]
if
max_age
<
ttl
:
ttl
=
max_age
return
ttl
except
KeyError
:
pass
try
:
expires
=
self
.
request
.
headers
[
'expires'
]
return
time
.
mktime
(
time
.
strptime
(
expires
,
'%a, %d %b %Y %H:%M:%S'
)
)
time
.
time
(
)
except
KeyError
:
pass

Returns time to live in seconds . 0 means no caching .


def manifest():
    """Guarantee the existence of a basic MANIFEST.in.

    manifest doc: http://docs.python.org/distutils/sourcedist.html#manifest

    `options.paved.dist.manifest.include`: set of files (or globs) to include with the `include` directive.

    `options.paved.dist.manifest.recursive_include`: set of files (or globs) to include with the `recursive-include` directive.

    `options.paved.dist.manifest.prune`: set of files (or globs) to exclude with the `prune` directive.

    `options.paved.dist.manifest.include_sphinx_docroot`: True -> sphinx docroot is added as `graft`

    `options.paved.dist.manifest.include_sphinx_docroot`: True -> sphinx builddir is added as `prune`
    """
    prune = options.paved.dist.manifest.prune
    graft = set()

    if options.paved.dist.manifest.include_sphinx_docroot:
        docroot = options.get('docroot', 'docs')
        graft.update([docroot])

        if options.paved.dist.manifest.exclude_sphinx_builddir:
            builddir = docroot + '/' + options.get("builddir", ".build")
            prune.update([builddir])

    with open(options.paved.cwd / 'MANIFEST.in', 'w') as fo:
        for item in graft:
            fo.write('graft %s\n' % item)
        for item in options.paved.dist.manifest.include:
            fo.write('include %s\n' % item)
        for item in options.paved.dist.manifest.recursive_include:
            fo.write('recursive-include %s\n' % item)
        for item in prune:
            fo.write('prune %s\n' % item)

def
manifest
(
)
:
prune
=
options
.
paved
.
dist
.
manifest
.
prune
graft
=
set
(
)
if
options
.
paved
.
dist
.
manifest
.
include_sphinx_docroot
:
docroot
=
options
.
get
(
'docroot'
,
'docs'
)
graft
.
update
(
[
docroot
]
)
if
options
.
paved
.
dist
.
manifest
.
exclude_sphinx_builddir
:
builddir
=
docroot
+
'/'
+
options
.
get
(
"builddir"
,
".build"
)
prune
.
update
(
[
builddir
]
)
with
open
(
options
.
paved
.
cwd
/
'MANIFEST.in'
,
'w'
)
as
fo
:
for
item
in
graft
:
fo
.
write
(
'graft %s\n'
%
item
)
for
item
in
options
.
paved
.
dist
.
manifest
.
include
:
fo
.
write
(
'include %s\n'
%
item
)
for
item
in
options
.
paved
.
dist
.
manifest
.
recursive_include
:
fo
.
write
(
'recursive-include %s\n'
%
item
)
for
item
in
prune
:
fo
.
write
(
'prune %s\n'
%
item
)

Guarantee the existence of a basic MANIFEST . in .


def format_pathname(
        pathname,
        max_length):
    """
    Format a pathname

    :param str pathname: Pathname to format
    :param int max_length: Maximum length of result pathname (> 3)
    :return: Formatted pathname
    :rtype: str
    :raises ValueError: If *max_length* is not larger than 3

    This function formats a pathname so it is not longer than *max_length*
    characters. The resulting pathname is returned. It does so by replacing
    characters at the start of the *pathname* with three dots, if necessary.
    The idea is that the end of the *pathname* is the most important part
    to be able to identify the file.
    """
    if max_length <= 3:
        raise ValueError("max length must be larger than 3")

    if len(pathname) > max_length:
        pathname = "...{}".format(pathname[-(max_length-3):])

    return pathname

def
format_pathname
(
pathname
,
max_length
)
:
if
max_length
<=
3
:
raise
ValueError
(
"max length must be larger than 3"
)
if
len
(
pathname
)
>
max_length
:
pathname
=
"...{}"
.
format
(
pathname
[
(
max_length
3
)
:
]
)
return
pathname


def format_uuid(
        uuid,
        max_length=10):
    """
    Format a UUID string

    :param str uuid: UUID to format
    :param int max_length: Maximum length of result string (> 3)
    :return: Formatted UUID
    :rtype: str
    :raises ValueError: If *max_length* is not larger than 3

    This function formats a UUID so it is not longer than *max_length*
    characters. The resulting string is returned. It does so by replacing
    characters at the end of the *uuid* with three dots, if necessary.
    The idea is that the start of the *uuid* is the most important part
    to be able to identify the related entity.

    The default *max_length* is 10, which will result in a string
    containing the first 7 characters of the *uuid* passed in. Most of
    the time, such a string is still unique within a collection of UUIDs.
    """
    if max_length <= 3:
        raise ValueError("max length must be larger than 3")

    if len(uuid) > max_length:
        uuid = "{}...".format(uuid[0:max_length-3])

    return uuid

def
format_uuid
(
uuid
,
max_length
=
10
)
:
if
max_length
<=
3
:
raise
ValueError
(
"max length must be larger than 3"
)
if
len
(
uuid
)
>
max_length
:
uuid
=
"{}..."
.
format
(
uuid
[
0
:
max_length
3
]
)
return
uuid


def paginate_update(update):
    """
    attempts to get next and previous on updates
    """
    from happenings.models import Update
    time = update.pub_time
    event = update.event
    try:
        next = Update.objects.filter(
            event=event,
            pub_time__gt=time
        ).order_by('pub_time').only('title')[0]
    except:
        next = None
    try:
        previous = Update.objects.filter(
            event=event,
            pub_time__lt=time
        ).order_by('-pub_time').only('title')[0]
    except:
        previous = None
    return {'next': next, 'previous': previous, 'event': event}

def paginate_update ( update ) : from happenings . models import Update time = update . pub_time event = update . event try : next = Update . objects . filter ( event = event , pub_time__gt = time ) . order_by ( 'pub_time' ) . only ( 'title' ) [ 0 ] except : next = None try : previous = Update . objects . filter ( event = event , pub_time__lt = time ) . order_by ( '-pub_time' ) . only ( 'title' ) [ 0 ] except : previous = None return { 'next' : next , 'previous' : previous , 'event' : event }


def notify_client(
        notifier_uri,
        client_id,
        status_code,
        message=None):
    """
    Notify the client of the result of handling a request

    The payload contains two elements:

    - client_id
    - result

    The *client_id* is the id of the client to notify. It is assumed
    that the notifier service is able to identify the client by this id
    and that it can pass the *result* to it.

    The *result* always contains a *status_code* element. In case the
    message passed in is not None, it will also contain a *message*
    element.

    In case the notifier service does not exist or returns an error,
    an error message will be logged to *stderr*.
    """
    payload = {
        "client_id": client_id,
        "result": {
            "response": {
                "status_code": status_code
            }
        }
    }

    if message is not None:
        payload["result"]["response"]["message"] = message

    response = requests.post(notifier_uri, json=payload)

    if response.status_code != 201:
        sys.stderr.write("failed to notify client: {}\n".format(payload))
        sys.stderr.flush()

def
notify_client
(
notifier_uri
,
client_id
,
status_code
,
message
=
)
:
payload
=
{
"client_id"
:
client_id
,
"result"
:
{
"response"
:
{
"status_code"
:
status_code
}
}
}
if
message
is
not
:
payload
[
"result"
]
[
"response"
]
[
"message"
]
=
message
response
=
requests
.
post
(
notifier_uri
,
json
=
payload
)
if
response
.
status_code
!=
201
:
sys
.
stderr
.
write
(
"failed to notify client: {}\n"
.
format
(
payload
)
)
sys
.
stderr
.
flush
(
)


def setting(self, name_hyphen):
        """
        Retrieves the setting value whose name is indicated by name_hyphen.

        Values starting with $ are assumed to reference environment variables,
        and the value stored in environment variables is retrieved. It's an
        error if thes corresponding environment variable it not set.
        """
        if name_hyphen in self._instance_settings:
            value = self._instance_settings[name_hyphen][1]
        else:
            msg = "No setting named '%s'" % name_hyphen
            raise UserFeedback(msg)

        if hasattr(value, 'startswith') and value.startswith("$"):
            env_var = value.lstrip("$")
            if env_var in os.environ:
                return os.getenv(env_var)
            else:
                msg = "'%s' is not defined in your environment" % env_var
                raise UserFeedback(msg)

        elif hasattr(value, 'startswith') and value.startswith("\$"):
            return value.replace("\$", "$")

        else:
            return value

Retrieves the setting value whose name is indicated by name_hyphen .


def _update_settings(self, new_settings, enforce_helpstring=True):
        """
        This method does the work of updating settings. Can be passed with
        enforce_helpstring = False which you may want if allowing end users to
        add arbitrary metadata via the settings system.

        Preferable to use update_settings (without leading _) in code to do the
        right thing and always have docstrings.
        """
        for raw_setting_name, value in six.iteritems(new_settings):
            setting_name = raw_setting_name.replace("_", "-")

            setting_already_exists = setting_name in self._instance_settings
            value_is_list_len_2 = isinstance(value, list) and len(value) == 2
            treat_as_tuple = not setting_already_exists and value_is_list_len_2

            if isinstance(value, tuple) or treat_as_tuple:
                self._instance_settings[setting_name] = value

            else:
                if setting_name not in self._instance_settings:
                    if enforce_helpstring:
                        msg = "You must specify param '%s' as a tuple of (helpstring, value)"
                        raise InternalCashewException(msg % setting_name)

                    else:
                        # Create entry with blank helpstring.
                        self._instance_settings[setting_name] = ('', value,)

                else:
                    # Save inherited helpstring, replace default value.
                    orig = self._instance_settings[setting_name]
                    self._instance_settings[setting_name] = (orig[0], value,)

This method does the work of updating settings . Can be passed with enforce_helpstring = False which you may want if allowing end users to add arbitrary metadata via the settings system .


def settings_and_attributes(self):
        """Return a combined dictionary of setting values and attribute values."""
        attrs = self.setting_values()
        attrs.update(self.__dict__)
        skip = ["_instance_settings", "aliases"]
        for a in skip:
            del attrs[a]
        return attrs

def settings_and_attributes ( self ) : attrs = self . setting_values ( ) attrs . update ( self . __dict__ ) skip = [ "_instance_settings" , "aliases" ] for a in skip : del attrs [ a ] return attrs

Return a combined dictionary of setting values and attribute values .


def get_reference_to_class(cls, class_or_class_name):
        """
        Detect if we get a class or a name, convert a name to a class.
        """
        if isinstance(class_or_class_name, type):
            return class_or_class_name

        elif isinstance(class_or_class_name, string_types):
            if ":" in class_or_class_name:
                mod_name, class_name = class_or_class_name.split(":")

                if not mod_name in sys.modules:
                    __import__(mod_name)

                mod = sys.modules[mod_name]
                return mod.__dict__[class_name]

            else:
                return cls.load_class_from_locals(class_or_class_name)

        else:
            msg = "Unexpected Type '%s'" % type(class_or_class_name)
            raise InternalCashewException(msg)

def
get_reference_to_class
(
cls
,
class_or_class_name
)
:
if
isinstance
(
class_or_class_name
,
type
)
:
return
class_or_class_name
elif
isinstance
(
class_or_class_name
,
string_types
)
:
if
":"
in
class_or_class_name
:
mod_name
,
class_name
=
class_or_class_name
.
split
(
":"
)
if
not
mod_name
in
sys
.
modules
:
__import__
(
mod_name
)
mod
=
sys
.
modules
[
mod_name
]
return
mod
.
__dict__
[
class_name
]
else
:
return
cls
.
load_class_from_locals
(
class_or_class_name
)
else
:
msg
=
"Unexpected Type '%s'"
%
type
(
class_or_class_name
)
raise
InternalCashewException
(
msg
)

Detect if we get a class or a name convert a name to a class .


def check_docstring(cls):
        """
        Asserts that the class has a docstring, returning it if successful.
        """
        docstring = inspect.getdoc(cls)
        if not docstring:
            breadcrumbs = " -> ".join(t.__name__ for t in inspect.getmro(cls)[:-1][::-1])
            msg = "docstring required for plugin '%s' (%s, defined in %s)"
            args = (cls.__name__, breadcrumbs, cls.__module__)
            raise InternalCashewException(msg % args)

        max_line_length = cls._class_settings.get('max-docstring-length')
        if max_line_length:
            for i, line in enumerate(docstring.splitlines()):
                if len(line) > max_line_length:
                    msg = "docstring line %s of %s is %s chars too long"
                    args = (i, cls.__name__, len(line) - max_line_length)
                    raise Exception(msg % args)

        return docstring

Asserts that the class has a docstring returning it if successful .


def resourcePath(self, relative_path):
        """ Get absolute path to resource, works for dev and for PyInstaller """
        from os import path
        import sys

        try:
            # PyInstaller creates a temp folder and stores path in _MEIPASS
            base_path = sys._MEIPASS
        except Exception:
            base_path = path.dirname(path.abspath(__file__))
        return path.join(base_path, relative_path)

Get absolute path to resource works for dev and for PyInstaller


def addLogbook(self, physDef= "LCLS", mccDef="MCC", initialInstance=False):
        '''Add new block of logbook selection windows. Only 5 allowed.'''
        if self.logMenuCount < 5:
            self.logMenus.append(LogSelectMenu(self.logui.multiLogLayout, initialInstance))
            self.logMenus[-1].addLogbooks(self.logTypeList[1], self.physics_programs, physDef)
            self.logMenus[-1].addLogbooks(self.logTypeList[0], self.mcc_programs, mccDef)
            self.logMenus[-1].show()
            self.logMenuCount += 1
            if initialInstance:
                # Initial logbook menu can add additional menus, all others can only remove themselves.
                QObject.connect(self.logMenus[-1].logButton, SIGNAL("clicked()"), self.addLogbook)
            else:
                from functools import partial
                QObject.connect(self.logMenus[-1].logButton, SIGNAL("clicked()"), partial(self.removeLogbook, self.logMenus[-1]))

Add new block of logbook selection windows . Only 5 allowed .