def register(self, hash_types):
        """
        Registers a function to generate a hash for data of the appropriate
        types. This can be used to register custom classes. Internally this is
        used to define how to hash non-builtin objects like ndarrays and uuids.

        The registered function should return a tuple of bytes. First a small
        prefix hinting at the data type, and second the raw bytes that can be
        hashed.

        Args:
            hash_types (class or tuple of classes):

        Returns:
            func: closure to be used as the decorator

        Example:
            >>> # xdoctest: +SKIP
            >>> # Skip this doctest because we dont want tests to modify
            >>> # the global state.
            >>> import ubelt as ub
            >>> import pytest
            >>> class MyType(object):
            ...     def __init__(self, id):
            ...         self.id = id
            >>> data = MyType(1)
            >>> # Custom types wont work with ub.hash_data by default
            >>> with pytest.raises(TypeError):
            ...     ub.hash_data(data)
            >>> # You can register your functions with ubelt's internal
            >>> # hashable_extension registery.
            >>> @ub.util_hash._HASHABLE_EXTENSIONS.register(MyType)
            >>> def hash_my_type(data):
            ...     return b'mytype', six.b(ub.hash_data(data.id))
            >>> # TODO: allow hash_data to take an new instance of
            >>> # HashableExtensions, so we dont have to modify the global
            >>> # ubelt state when we run tests.
            >>> my_instance = MyType(1)
            >>> ub.hash_data(my_instance)
        """
        # ensure iterable
        if not isinstance(hash_types, (list, tuple)):
            hash_types = [hash_types]
        def _decor_closure(hash_func):
            for hash_type in hash_types:
                key = (hash_type.__module__, hash_type.__name__)
                self.keyed_extensions[key] = (hash_type, hash_func)
            return hash_func
        return _decor_closure

def
register
(
self
,
hash_types
)
:
# ensure iterable
if
not
isinstance
(
hash_types
,
(
list
,
tuple
)
)
:
hash_types
=
[
hash_types
]
def
_decor_closure
(
hash_func
)
:
for
hash_type
in
hash_types
:
key
=
(
hash_type
.
__module__
,
hash_type
.
__name__
)
self
.
keyed_extensions
[
key
]
=
(
hash_type
,
hash_func
)
return
hash_func
return
_decor_closure

Registers a function to generate a hash for data of the appropriate types . This can be used to register custom classes . Internally this is used to define how to hash non - builtin objects like ndarrays and uuids .


def lookup(self, data):
        """
        Returns an appropriate function to hash `data` if one has been
        registered.

        Raises:
            TypeError : if data has no registered hash methods

        Example:
            >>> import ubelt as ub
            >>> import pytest
            >>> if not ub.modname_to_modpath('numpy'):
            ...     raise pytest.skip('numpy is optional')
            >>> self = HashableExtensions()
            >>> self._register_numpy_extensions()
            >>> self._register_builtin_class_extensions()

            >>> import numpy as np
            >>> data = np.array([1, 2, 3])
            >>> self.lookup(data[0])

            >>> class Foo(object):
            >>>     def __init__(f):
            >>>         f.attr = 1
            >>> data = Foo()
            >>> assert pytest.raises(TypeError, self.lookup, data)

            >>> # If ub.hash_data doesnt support your object,
            >>> # then you can register it.
            >>> @self.register(Foo)
            >>> def _hashfoo(data):
            >>>     return b'FOO', data.attr
            >>> func = self.lookup(data)
            >>> assert func(data)[1] == 1

            >>> data = uuid.uuid4()
            >>> self.lookup(data)
        """
        # Maybe try using functools.singledispatch instead?
        # First try O(1) lookup
        query_hash_type = data.__class__
        key = (query_hash_type.__module__, query_hash_type.__name__)
        try:
            hash_type, hash_func = self.keyed_extensions[key]
        except KeyError:
            raise TypeError('No registered hash func for hashable type=%r' % (
                    query_hash_type))
        return hash_func

Returns an appropriate function to hash data if one has been registered .


def _register_numpy_extensions(self):
        """
        Numpy extensions are builtin
        """
        # system checks
        import numpy as np
        numpy_floating_types = (np.float16, np.float32, np.float64)
        if hasattr(np, 'float128'):  # nocover
            numpy_floating_types = numpy_floating_types + (np.float128,)

        @self.add_iterable_check
        def is_object_ndarray(data):
            # ndarrays of objects cannot be hashed directly.
            return isinstance(data, np.ndarray) and data.dtype.kind == 'O'

        @self.register(np.ndarray)
        def hash_numpy_array(data):
            """
            Example:
                >>> import ubelt as ub
                >>> if not ub.modname_to_modpath('numpy'):
                ...     raise pytest.skip()
                >>> import numpy as np
                >>> data_f32 = np.zeros((3, 3, 3), dtype=np.float64)
                >>> data_i64 = np.zeros((3, 3, 3), dtype=np.int64)
                >>> data_i32 = np.zeros((3, 3, 3), dtype=np.int32)
                >>> hash_f64 = _hashable_sequence(data_f32, types=True)
                >>> hash_i64 = _hashable_sequence(data_i64, types=True)
                >>> hash_i32 = _hashable_sequence(data_i64, types=True)
                >>> assert hash_i64 != hash_f64
                >>> assert hash_i64 != hash_i32
            """
            if data.dtype.kind == 'O':
                msg = 'directly hashing ndarrays with dtype=object is unstable'
                raise TypeError(msg)
            else:
                # tobytes() views the array in 1D (via ravel())
                # encode the shape as well
                header = b''.join(_hashable_sequence((len(data.shape), data.shape)))
                dtype = b''.join(_hashable_sequence(data.dtype.descr))
                hashable = header + dtype + data.tobytes()
            prefix = b'NDARR'
            return prefix, hashable

        @self.register((np.int64, np.int32, np.int16, np.int8) +
                       (np.uint64, np.uint32, np.uint16, np.uint8))
        def _hash_numpy_int(data):
            return _convert_to_hashable(int(data))

        @self.register(numpy_floating_types)
        def _hash_numpy_float(data):
            return _convert_to_hashable(float(data))

        @self.register(np.random.RandomState)
        def _hash_numpy_random_state(data):
            """
            Example:
                >>> import ubelt as ub
                >>> if not ub.modname_to_modpath('numpy'):
                ...     raise pytest.skip()
                >>> import numpy as np
                >>> rng = np.random.RandomState(0)
                >>> _hashable_sequence(rng, types=True)
            """
            hashable = b''.join(_hashable_sequence(data.get_state()))
            prefix = b'RNG'
            return prefix, hashable


def _register_builtin_class_extensions(self):
        """
        Register hashing extensions for a selection of classes included in
        python stdlib.

        Example:
            >>> data = uuid.UUID('7e9d206b-dc02-4240-8bdb-fffe858121d0')
            >>> print(hash_data(data, base='abc', hasher='sha512', types=True)[0:8])
            cryarepd
            >>> data = OrderedDict([('a', 1), ('b', 2), ('c', [1, 2, 3]),
            >>>                     (4, OrderedDict())])
            >>> print(hash_data(data, base='abc', hasher='sha512', types=True)[0:8])
            qjspicvv

            gpxtclct
        """
        @self.register(uuid.UUID)
        def _hash_uuid(data):
            hashable = data.bytes
            prefix = b'UUID'
            return prefix, hashable

        @self.register(OrderedDict)
        def _hash_ordered_dict(data):
            """
            Note, we should not be hashing dicts because they are unordered
            """
            hashable = b''.join(_hashable_sequence(list(data.items())))
            prefix = b'ODICT'
            return prefix, hashable

Register hashing extensions for a selection of classes included in python stdlib .


def _proc_async_iter_stream(proc, stream, buffersize=1):
    """
    Reads output from a process in a separate thread
    """
    from six.moves import queue
    from threading import Thread
    def enqueue_output(proc, stream, stream_queue):
        while proc.poll() is None:
            line = stream.readline()
            # print('ENQUEUE LIVE {!r} {!r}'.format(stream, line))
            stream_queue.put(line)

        for line in _textio_iterlines(stream):
            # print('ENQUEUE FINAL {!r} {!r}'.format(stream, line))
            stream_queue.put(line)

        # print("STREAM IS DONE {!r}".format(stream))
        stream_queue.put(None)  # signal that the stream is finished
        # stream.close()
    stream_queue = queue.Queue(maxsize=buffersize)
    _thread = Thread(target=enqueue_output, args=(proc, stream, stream_queue))
    _thread.daemon = True  # thread dies with the program
    _thread.start()
    return stream_queue


def _tee_output(make_proc, stdout=None, stderr=None, backend='auto'):
    """
    Simultaneously reports and captures stdout and stderr from a process

    subprocess must be created using (stdout=subprocess.PIPE,
    stderr=subprocess.PIPE)
    """
    logged_out = []
    logged_err = []
    if backend == 'auto':
        # backend = 'select' if POSIX else 'thread'
        backend = 'thread'

    if backend == 'select':
        if not POSIX:  # nocover
            raise NotImplementedError('select is only available on posix')
        # the select-based version is stable, but slow
        _proc_iteroutput = _proc_iteroutput_select
    elif backend == 'thread':
        # the thread version is fast, but might run into issues.
        _proc_iteroutput = _proc_iteroutput_thread
    else:
        raise ValueError('backend must be select, thread, or auto')

    proc = make_proc()
    for oline, eline in _proc_iteroutput(proc):
        if oline:
            if stdout:  # pragma: nobranch
                stdout.write(oline)
                stdout.flush()
            logged_out.append(oline)
        if eline:
            if stderr:  # pragma: nobranch
                stderr.write(eline)
                stderr.flush()
            logged_err.append(eline)
    return proc, logged_out, logged_err


def timestamp(method='iso8601'):
    """
    make an iso8601 timestamp

    Args:
        method (str): type of timestamp

    Example:
        >>> stamp = timestamp()
        >>> print('stamp = {!r}'.format(stamp))
        stamp = ...-...-...T...
    """
    if method == 'iso8601':
        # ISO 8601
        # datetime.datetime.utcnow().isoformat()
        # datetime.datetime.now().isoformat()
        # utcnow
        tz_hour = time.timezone // 3600
        utc_offset = str(tz_hour) if tz_hour < 0 else '+' + str(tz_hour)
        stamp = time.strftime('%Y-%m-%dT%H%M%S') + utc_offset
        return stamp
    else:
        raise ValueError('only iso8601 is accepted for now')


def import_module_from_path(modpath, index=-1):
    """
    Imports a module via its path

    Args:
        modpath (PathLike): path to the module on disk or within a zipfile.

    Returns:
        module: the imported module

    References:
        https://stackoverflow.com/questions/67631/import-module-given-path

    Notes:
        If the module is part of a package, the package will be imported first.
        These modules may cause problems when reloading via IPython magic

        This can import a module from within a zipfile. To do this modpath
        should specify the path to the zipfile and the path to the module
        within that zipfile separated by a colon or pathsep.
        E.g. `/path/to/archive.zip:mymodule.py`

    Warning:
        It is best to use this with paths that will not conflict with
        previously existing modules.

        If the modpath conflicts with a previously existing module name. And
        the target module does imports of its own relative to this conflicting
        path. In this case, the module that was loaded first will win.

        For example if you try to import '/foo/bar/pkg/mod.py' from the folder
        structure:
          - foo/
            +- bar/
               +- pkg/
                  +  __init__.py
                  |- mod.py
                  |- helper.py

       If there exists another module named `pkg` already in sys.modules
       and mod.py does something like `from . import helper`, Python will
       assume helper belongs to the `pkg` module already in sys.modules.
       This can cause a NameError or worse --- a incorrect helper module.

    Example:
        >>> import xdoctest
        >>> modpath = xdoctest.__file__
        >>> module = import_module_from_path(modpath)
        >>> assert module is xdoctest

    Example:
        >>> # Test importing a module from within a zipfile
        >>> import zipfile
        >>> from xdoctest import utils
        >>> from os.path import join, expanduser
        >>> dpath = expanduser('~/.cache/xdoctest')
        >>> dpath = utils.ensuredir(dpath)
        >>> #dpath = utils.TempDir().ensure()
        >>> # Write to an external module named bar
        >>> external_modpath = join(dpath, 'bar.py')
        >>> open(external_modpath, 'w').write('testvar = 1')
        >>> internal = 'folder/bar.py'
        >>> # Move the external bar module into a zipfile
        >>> zippath = join(dpath, 'myzip.zip')
        >>> with zipfile.ZipFile(zippath, 'w') as myzip:
        >>>     myzip.write(external_modpath, internal)
        >>> # Import the bar module from within the zipfile
        >>> modpath = zippath + ':' + internal
        >>> modpath = zippath + os.path.sep + internal
        >>> module = import_module_from_path(modpath)
        >>> assert module.__name__ == os.path.normpath('folder/bar')
        >>> assert module.testvar == 1

    Doctest:
        >>> import pytest
        >>> with pytest.raises(IOError):
        >>>     import_module_from_path('does-not-exist')
        >>> with pytest.raises(IOError):
        >>>     import_module_from_path('does-not-exist.zip/')
    """
    import os
    if not os.path.exists(modpath):
        import re
        import zipimport
        # We allow (if not prefer or force) the colon to be a path.sep in order
        # to agree with the mod.__name__ attribute that will be produced

        # zip followed by colon or slash
        pat = '(.zip[' + re.escape(os.path.sep) + '/:])'
        parts = re.split(pat, modpath, flags=re.IGNORECASE)
        if len(parts) > 2:
            archivepath = ''.join(parts[:-1])[:-1]
            internal = parts[-1]
            modname = os.path.splitext(internal)[0]
            modname = os.path.normpath(modname)
            if os.path.exists(archivepath):
                zimp_file = zipimport.zipimporter(archivepath)
                module = zimp_file.load_module(modname)
                return module
        raise IOError('modpath={} does not exist'.format(modpath))
    else:
        # the importlib version doesnt work in pytest
        module = _custom_import_modpath(modpath)
        # TODO: use this implementation once pytest fixes importlib
        # module = _pkgutil_import_modpath(modpath)
        return module


def _extension_module_tags():
    """
    Returns valid tags an extension module might have
    """
    import sysconfig
    tags = []
    if six.PY2:
        # see also 'SHLIB_EXT'
        multiarch = sysconfig.get_config_var('MULTIARCH')
        if multiarch is not None:
            tags.append(multiarch)
    else:
        # handle PEP 3149 -- ABI version tagged .so files
        # ABI = application binary interface
        tags.append(sysconfig.get_config_var('SOABI'))
        tags.append('abi3')  # not sure why this one is valid but it is
    tags = [t for t in tags if t]
    return tags


def _syspath_modname_to_modpath(modname, sys_path=None, exclude=None):
    """
    syspath version of modname_to_modpath

    Args:
        modname (str): name of module to find
        sys_path (List[PathLike], default=None):
            if specified overrides `sys.path`
        exclude (List[PathLike], default=None):
            list of directory paths. if specified prevents these directories
            from being searched.

    Notes:
        This is much slower than the pkgutil mechanisms.

    CommandLine:
        python -m xdoctest.static_analysis _syspath_modname_to_modpath

    Example:
        >>> print(_syspath_modname_to_modpath('xdoctest.static_analysis'))
        ...static_analysis.py
        >>> print(_syspath_modname_to_modpath('xdoctest'))
        ...xdoctest
        >>> print(_syspath_modname_to_modpath('_ctypes'))
        ..._ctypes...
        >>> assert _syspath_modname_to_modpath('xdoctest', sys_path=[]) is None
        >>> assert _syspath_modname_to_modpath('xdoctest.static_analysis', sys_path=[]) is None
        >>> assert _syspath_modname_to_modpath('_ctypes', sys_path=[]) is None
        >>> assert _syspath_modname_to_modpath('this', sys_path=[]) is None

    Example:
        >>> # test what happens when the module is not visible in the path
        >>> modname = 'xdoctest.static_analysis'
        >>> modpath = _syspath_modname_to_modpath(modname)
        >>> exclude = [split_modpath(modpath)[0]]
        >>> found = _syspath_modname_to_modpath(modname, exclude=exclude)
        >>> # this only works if installed in dev mode, pypi fails
        >>> assert found is None, 'should not have found {}'.format(found)
    """

    def _isvalid(modpath, base):
        # every directory up to the module, should have an init
        subdir = dirname(modpath)
        while subdir and subdir != base:
            if not exists(join(subdir, '__init__.py')):
                return False
            subdir = dirname(subdir)
        return True

    _fname_we = modname.replace('.', os.path.sep)
    candidate_fnames = [
        _fname_we + '.py',
        # _fname_we + '.pyc',
        # _fname_we + '.pyo',
    ]
    # Add extension library suffixes
    candidate_fnames += [_fname_we + ext for ext in _platform_pylib_exts()]

    if sys_path is None:
        sys_path = sys.path

    # the empty string in sys.path indicates cwd. Change this to a '.'
    candidate_dpaths = ['.' if p == '' else p for p in sys_path]

    if exclude:
        def normalize(p):
            if sys.platform.startswith('win32'):  # nocover
                return realpath(p).lower()
            else:
                return realpath(p)
        # Keep only the paths not in exclude
        real_exclude = {normalize(p) for p in exclude}
        candidate_dpaths = [p for p in candidate_dpaths
                            if normalize(p) not in real_exclude]

    for dpath in candidate_dpaths:
        # Check for directory-based modules (has presidence over files)
        modpath = join(dpath, _fname_we)
        if exists(modpath):
            if isfile(join(modpath, '__init__.py')):
                if _isvalid(modpath, dpath):
                    return modpath

        # If that fails, check for file-based modules
        for fname in candidate_fnames:
            modpath = join(dpath, fname)
            if isfile(modpath):
                if _isvalid(modpath, dpath):
                    return modpath


def modname_to_modpath(modname, hide_init=True, hide_main=False, sys_path=None):
    """
    Finds the path to a python module from its name.

    Determines the path to a python module without directly import it

    Converts the name of a module (__name__) to the path (__file__) where it is
    located without importing the module. Returns None if the module does not
    exist.

    Args:
        modname (str): module filepath
        hide_init (bool): if False, __init__.py will be returned for packages
        hide_main (bool): if False, and hide_init is True, __main__.py will be
            returned for packages, if it exists.
        sys_path (list): if specified overrides `sys.path` (default None)

    Returns:
        str: modpath - path to the module, or None if it doesn't exist

    CommandLine:
        python -m xdoctest.static_analysis modname_to_modpath:0
        pytest  /home/joncrall/code/xdoctest/xdoctest/static_analysis.py::modname_to_modpath:0

    Example:
        >>> modname = 'xdoctest.__main__'
        >>> modpath = modname_to_modpath(modname, hide_main=False)
        >>> assert modpath.endswith('__main__.py')
        >>> modname = 'xdoctest'
        >>> modpath = modname_to_modpath(modname, hide_init=False)
        >>> assert modpath.endswith('__init__.py')
        >>> modpath = basename(modname_to_modpath('_ctypes'))
        >>> assert 'ctypes' in modpath
    """
    modpath = _syspath_modname_to_modpath(modname, sys_path)
    if modpath is None:
        return None

    modpath = normalize_modpath(modpath, hide_init=hide_init,
                                hide_main=hide_main)
    return modpath

def modname_to_modpath ( modname , hide_init = True , hide_main = False , sys_path = None ) : modpath = _syspath_modname_to_modpath ( modname , sys_path ) if modpath is None : return None modpath = normalize_modpath ( modpath , hide_init = hide_init , hide_main = hide_main ) return modpath

Finds the path to a python module from its name .


def modpath_to_modname(modpath, hide_init=True, hide_main=False, check=True,
                       relativeto=None):
    """
    Determines importable name from file path

    Converts the path to a module (__file__) to the importable python name
    (__name__) without importing the module.

    The filename is converted to a module name, and parent directories are
    recursively included until a directory without an __init__.py file is
    encountered.

    Args:
        modpath (str): module filepath
        hide_init (bool): removes the __init__ suffix (default True)
        hide_main (bool): removes the __main__ suffix (default False)
        check (bool): if False, does not raise an error if modpath is a dir
            and does not contain an __init__ file.
        relativeto (str, optional): if specified, all checks are ignored and
            this is considered the path to the root module.

    Returns:
        str: modname

    Raises:
        ValueError: if check is True and the path does not exist

    CommandLine:
        xdoctest -m xdoctest.static_analysis modpath_to_modname

    Example:
        >>> from xdoctest import static_analysis
        >>> modpath = static_analysis.__file__.replace('.pyc', '.py')
        >>> modpath = modpath.replace('.pyc', '.py')
        >>> modname = modpath_to_modname(modpath)
        >>> assert modname == 'xdoctest.static_analysis'

    Example:
        >>> import xdoctest
        >>> assert modpath_to_modname(xdoctest.__file__.replace('.pyc', '.py')) == 'xdoctest'
        >>> assert modpath_to_modname(dirname(xdoctest.__file__.replace('.pyc', '.py'))) == 'xdoctest'

    Example:
        >>> modpath = modname_to_modpath('_ctypes')
        >>> modname = modpath_to_modname(modpath)
        >>> assert modname == '_ctypes'
    """
    if check and relativeto is None:
        if not exists(modpath):
            raise ValueError('modpath={} does not exist'.format(modpath))
    modpath_ = abspath(expanduser(modpath))

    modpath_ = normalize_modpath(modpath_, hide_init=hide_init,
                                 hide_main=hide_main)
    if relativeto:
        dpath = dirname(abspath(expanduser(relativeto)))
        rel_modpath = relpath(modpath_, dpath)
    else:
        dpath, rel_modpath = split_modpath(modpath_, check=check)

    modname = splitext(rel_modpath)[0]
    if '.' in modname:
        modname, abi_tag = modname.split('.')
    modname = modname.replace('/', '.')
    modname = modname.replace('\\', '.')
    return modname

def
modpath_to_modname
(
modpath
,
hide_init
=
True
,
hide_main
=
False
,
check
=
True
,
relativeto
=
)
:
if
check
and
relativeto
is
:
if
not
exists
(
modpath
)
:
raise
ValueError
(
'modpath={} does not exist'
.
format
(
modpath
)
)
modpath_
=
abspath
(
expanduser
(
modpath
)
)
modpath_
=
normalize_modpath
(
modpath_
,
hide_init
=
hide_init
,
hide_main
=
hide_main
)
if
relativeto
:
dpath
=
dirname
(
abspath
(
expanduser
(
relativeto
)
)
)
rel_modpath
=
relpath
(
modpath_
,
dpath
)
else
:
dpath
,
rel_modpath
=
split_modpath
(
modpath_
,
check
=
check
)
modname
=
splitext
(
rel_modpath
)
[
0
]
if
'.'
in
modname
:
modname
,
abi_tag
=
modname
.
split
(
'.'
)
modname
=
modname
.
replace
(
'/'
,
'.'
)
modname
=
modname
.
replace
(
'\\'
,
'.'
)
return
modname


def split_modpath(modpath, check=True):
    """
    Splits the modpath into the dir that must be in PYTHONPATH for the module
    to be imported and the modulepath relative to this directory.

    Args:
        modpath (str): module filepath
        check (bool): if False, does not raise an error if modpath is a
            directory and does not contain an `__init__.py` file.

    Returns:
        tuple: (directory, rel_modpath)

    Raises:
        ValueError: if modpath does not exist or is not a package

    Example:
        >>> from xdoctest import static_analysis
        >>> modpath = static_analysis.__file__.replace('.pyc', '.py')
        >>> modpath = abspath(modpath)
        >>> dpath, rel_modpath = split_modpath(modpath)
        >>> recon = join(dpath, rel_modpath)
        >>> assert recon == modpath
        >>> assert rel_modpath == join('xdoctest', 'static_analysis.py')
    """
    if six.PY2:
        if modpath.endswith('.pyc'):
            modpath = modpath[:-1]
    modpath_ = abspath(expanduser(modpath))
    if check:
        if not exists(modpath_):
            if not exists(modpath):
                raise ValueError('modpath={} does not exist'.format(modpath))
            raise ValueError('modpath={} is not a module'.format(modpath))
        if isdir(modpath_) and not exists(join(modpath, '__init__.py')):
            # dirs without inits are not modules
            raise ValueError('modpath={} is not a module'.format(modpath))
    full_dpath, fname_ext = split(modpath_)
    _relmod_parts = [fname_ext]
    # Recurse down directories until we are out of the package
    dpath = full_dpath
    while exists(join(dpath, '__init__.py')):
        dpath, dname = split(dpath)
        _relmod_parts.append(dname)
    relmod_parts = _relmod_parts[::-1]
    rel_modpath = os.path.sep.join(relmod_parts)
    return dpath, rel_modpath

Splits the modpath into the dir that must be in PYTHONPATH for the module to be imported and the modulepath relative to this directory .


def argval(key, default=util_const.NoParam, argv=None):
    """
    Get the value of a keyword argument specified on the command line.

    Values can be specified as `<key> <value>` or `<key>=<value>`

    Args:
        key (str or tuple): string or tuple of strings. Each key should be
            prefixed with two hyphens (i.e. `--`)
        default (Optional[object]): value to return if not specified
        argv (Optional[list]): overrides `sys.argv` if specified

    Returns:
        str: value : the value specified after the key. It they key is
            specified multiple times, then the first value is returned.

    TODO:
        - [ ] Can we handle the case where the value is a list of long paths?
        - [ ] Should we default the first or last specified instance of the flag.

    Example:
        >>> import ubelt as ub
        >>> argv = ['--ans', '42', '--quest=the grail', '--ans=6', '--bad']
        >>> assert ub.argval('--spam', argv=argv) == ub.NoParam
        >>> assert ub.argval('--quest', argv=argv) == 'the grail'
        >>> assert ub.argval('--ans', argv=argv) == '42'
        >>> assert ub.argval('--bad', argv=argv) == ub.NoParam
        >>> assert ub.argval(('--bad', '--bar'), argv=argv) == ub.NoParam

    Example:
        >>> # Test fix for GH Issue #41
        >>> import ubelt as ub
        >>> argv = ['--path=/path/with/k=3']
        >>> ub.argval('--path', argv=argv) == '/path/with/k=3'
    """
    if argv is None:  # nocover
        argv = sys.argv

    keys = [key] if isinstance(key, six.string_types) else key
    n_max = len(argv) - 1
    for argx, item in enumerate(argv):
        for key_ in keys:
            if item == key_:
                if argx < n_max:
                    value = argv[argx + 1]
                    return value
            elif item.startswith(key_ + '='):
                value = '='.join(item.split('=')[1:])
                return value
    value = default
    return value

def
argval
(
key
,
default
=
util_const
.
NoParam
,
argv
=
)
:
if
argv
is
:
# nocover
argv
=
sys
.
argv
keys
=
[
key
]
if
isinstance
(
key
,
six
.
string_types
)
else
key
n_max
=
len
(
argv
)
1
for
argx
,
item
in
enumerate
(
argv
)
:
for
key_
in
keys
:
if
item
==
key_
:
if
argx
<
n_max
:
value
=
argv
[
argx
+
1
]
return
value
elif
item
.
startswith
(
key_
+
'='
)
:
value
=
'='
.
join
(
item
.
split
(
'='
)
[
1
:
]
)
return
value
value
=
default
return
value

Get the value of a keyword argument specified on the command line .


def argflag(key, argv=None):
    """
    Determines if a key is specified on the command line

    Args:
        key (str or tuple): string or tuple of strings. Each key should be
            prefixed with two hyphens (i.e. `--`)
        argv (Optional[list]): overrides `sys.argv` if specified

    Returns:
        bool: flag : True if the key (or any of the keys) was specified

    Example:
        >>> import ubelt as ub
        >>> argv = ['--spam', '--eggs', 'foo']
        >>> assert ub.argflag('--eggs', argv=argv) is True
        >>> assert ub.argflag('--ans', argv=argv) is False
        >>> assert ub.argflag('foo', argv=argv) is True
        >>> assert ub.argflag(('bar', '--spam'), argv=argv) is True
    """
    if argv is None:  # nocover
        argv = sys.argv
    keys = [key] if isinstance(key, six.string_types) else key
    flag = any(k in argv for k in keys)
    return flag

def
argflag
(
key
,
argv
=
)
:
if
argv
is
:
# nocover
argv
=
sys
.
argv
keys
=
[
key
]
if
isinstance
(
key
,
six
.
string_types
)
else
key
flag
=
any
(
k
in
argv
for
k
in
keys
)
return
flag


def hzcat(args, sep=''):
    """
    Horizontally concatenates strings preserving indentation

    Concatenates a list of objects ensuring that the next item in the list is
    all the way to the right of any previous items.

    Args:
        args (List[str]): strings to concatenate
        sep (str): separator (defaults to '')

    CommandLine:
        python -m ubelt.util_str hzcat

    Example1:
        >>> import ubelt as ub
        >>> B = ub.repr2([[1, 2], [3, 457]], nl=1, cbr=True, trailsep=False)
        >>> C = ub.repr2([[5, 6], [7, 8]], nl=1, cbr=True, trailsep=False)
        >>> args = ['A = ', B, ' * ', C]
        >>> print(ub.hzcat(args))
        A = [[1, 2],   * [[5, 6],
             [3, 457]]    [7, 8]]

    Example2:
        >>> from ubelt.util_str import *
        >>> import ubelt as ub
        >>> import unicodedata
        >>> aa = unicodedata.normalize('NFD', 'á')  # a unicode char with len2
        >>> B = ub.repr2([['θ', aa], [aa, aa, aa]], nl=1, si=True, cbr=True, trailsep=False)
        >>> C = ub.repr2([[5, 6], [7, 'θ']], nl=1, si=True, cbr=True, trailsep=False)
        >>> args = ['A', '=', B, '*', C]
        >>> print(ub.hzcat(args, sep='｜'))
        A｜=｜[[θ, á],   ｜*｜[[5, 6],
         ｜ ｜ [á, á, á]]｜ ｜ [7, θ]]
    """
    import unicodedata
    if '\n' in sep or '\r' in sep:
        raise ValueError('`sep` cannot contain newline characters')

    # TODO: ensure unicode data works correctly for python2
    args = [unicodedata.normalize('NFC', ensure_unicode(val)) for val in args]
    arglines = [a.split('\n') for a in args]
    height = max(map(len, arglines))
    # Do vertical padding
    arglines = [lines + [''] * (height - len(lines)) for lines in arglines]
    # Initialize output
    all_lines = ['' for _ in range(height)]
    width = 0
    n_args = len(args)
    for sx, lines in enumerate(arglines):
        # Concatenate the new string
        for lx, line in enumerate(lines):
            all_lines[lx] += line
        # Find the new maximum horizontal width
        width = max(width, max(map(len, all_lines)))
        if sx < n_args - 1:
            # Horizontal padding on all but last iter
            for lx, line in list(enumerate(all_lines)):
                residual = width - len(line)
                all_lines[lx] = line + (' ' * residual) + sep
            width += len(sep)
    # Clean up trailing whitespace
    all_lines = [line.rstrip(' ') for line in all_lines]
    ret = '\n'.join(all_lines)
    return ret


def symlink(real_path, link_path, overwrite=False, verbose=0):
    """
    Create a symbolic link.

    This will work on linux or windows, however windows does have some corner
    cases. For more details see notes in `ubelt._win32_links`.

    Args:
        path (PathLike): path to real file or directory
        link_path (PathLike): path to desired location for symlink
        overwrite (bool): overwrite existing symlinks.
            This will not overwrite real files on systems with proper symlinks.
            However, on older versions of windows junctions are
            indistinguishable from real files, so we cannot make this
            guarantee.  (default = False)
        verbose (int):  verbosity level (default=0)

    Returns:
        PathLike: link path

    CommandLine:
        python -m ubelt.util_links symlink:0

    Example:
        >>> import ubelt as ub
        >>> dpath = ub.ensure_app_cache_dir('ubelt', 'test_symlink0')
        >>> real_path = join(dpath, 'real_file.txt')
        >>> link_path = join(dpath, 'link_file.txt')
        >>> [ub.delete(p) for p in [real_path, link_path]]
        >>> ub.writeto(real_path, 'foo')
        >>> result = symlink(real_path, link_path)
        >>> assert ub.readfrom(result) == 'foo'
        >>> [ub.delete(p) for p in [real_path, link_path]]

    Example:
        >>> import ubelt as ub
        >>> from os.path import dirname
        >>> dpath = ub.ensure_app_cache_dir('ubelt', 'test_symlink1')
        >>> ub.delete(dpath)
        >>> ub.ensuredir(dpath)
        >>> _dirstats(dpath)
        >>> real_dpath = ub.ensuredir((dpath, 'real_dpath'))
        >>> link_dpath = ub.augpath(real_dpath, base='link_dpath')
        >>> real_path = join(dpath, 'afile.txt')
        >>> link_path = join(dpath, 'afile.txt')
        >>> [ub.delete(p) for p in [real_path, link_path]]
        >>> ub.writeto(real_path, 'foo')
        >>> result = symlink(real_dpath, link_dpath)
        >>> assert ub.readfrom(link_path) == 'foo', 'read should be same'
        >>> ub.writeto(link_path, 'bar')
        >>> _dirstats(dpath)
        >>> assert ub.readfrom(link_path) == 'bar', 'very bad bar'
        >>> assert ub.readfrom(real_path) == 'bar', 'changing link did not change real'
        >>> ub.writeto(real_path, 'baz')
        >>> _dirstats(dpath)
        >>> assert ub.readfrom(real_path) == 'baz', 'very bad baz'
        >>> assert ub.readfrom(link_path) == 'baz', 'changing real did not change link'
        >>> ub.delete(link_dpath, verbose=1)
        >>> _dirstats(dpath)
        >>> assert not exists(link_dpath), 'link should not exist'
        >>> assert exists(real_path), 'real path should exist'
        >>> _dirstats(dpath)
        >>> ub.delete(dpath, verbose=1)
        >>> _dirstats(dpath)
        >>> assert not exists(real_path)
    """
    path = normpath(real_path)
    link = normpath(link_path)

    if not os.path.isabs(path):
        # if path is not absolute it must be specified relative to link
        if _can_symlink():
            path = os.path.relpath(path, os.path.dirname(link))
        else:  # nocover
            # On windows, we need to use absolute paths
            path = os.path.abspath(path)

    if verbose:
        print('Symlink: {path} -> {link}'.format(path=path, link=link))
    if islink(link):
        if verbose:
            print('... already exists')
        pointed = _readlink(link)
        if pointed == path:
            if verbose > 1:
                print('... and points to the right place')
            return link
        if verbose > 1:
            if not exists(link):
                print('... but it is broken and points somewhere else: {}'.format(pointed))
            else:
                print('... but it points somewhere else: {}'.format(pointed))
        if overwrite:
            util_io.delete(link, verbose=verbose > 1)
    elif exists(link):
        if _win32_links is None:
            if verbose:
                print('... already exists, but its a file. This will error.')
            raise FileExistsError(
                'cannot overwrite a physical path: "{}"'.format(path))
        else:  # nocover
            if verbose:
                print('... already exists, and is either a file or hard link. '
                      'Assuming it is a hard link. '
                      'On non-win32 systems this would error.')

    if _win32_links is None:
        os.symlink(path, link)
    else:  # nocover
        _win32_links._symlink(path, link, overwrite=overwrite, verbose=verbose)

    return link


def _make_signature_key(args, kwargs):
    """
    Transforms function args into a key that can be used by the cache

    CommandLine:
        xdoctest -m ubelt.util_memoize _make_signature_key

    Example:
        >>> args = (4, [1, 2])
        >>> kwargs = {'a': 'b'}
        >>> key = _make_signature_key(args, kwargs)
        >>> print('key = {!r}'.format(key))
        >>> # Some mutable types cannot be handled by ub.hash_data
        >>> import pytest
        >>> import six
        >>> if six.PY2:
        >>>     import collections as abc
        >>> else:
        >>>     from collections import abc
        >>> with pytest.raises(TypeError):
        >>>     _make_signature_key((4, [1, 2], {1: 2, 'a': 'b'}), kwargs={})
        >>> class Dummy(abc.MutableSet):
        >>>     def __contains__(self, item): return None
        >>>     def __iter__(self): return iter([])
        >>>     def __len__(self): return 0
        >>>     def add(self, item, loc): return None
        >>>     def discard(self, item): return None
        >>> with pytest.raises(TypeError):
        >>>     _make_signature_key((Dummy(),), kwargs={})
    """
    kwitems = kwargs.items()
    # TODO: we should check if Python is at least 3.7 and sort by kwargs
    # keys otherwise. Should we use hash_data for key generation
    if (sys.version_info.major, sys.version_info.minor) < (3, 7):  # nocover
        # We can sort because they keys are gaurenteed to be strings
        kwitems = sorted(kwitems)
    kwitems = tuple(kwitems)

    try:
        key = _hashable(args), _hashable(kwitems)
    except TypeError:
        raise TypeError('Signature is not hashable: args={} kwargs{}'.format(args, kwargs))
    return key


def memoize(func):
    """
    memoization decorator that respects args and kwargs

    References:
        https://wiki.python.org/moin/PythonDecoratorLibrary#Memoize

    Args:
        func (Callable): live python function

    Returns:
        func: memoized wrapper

    CommandLine:
        xdoctest -m ubelt.util_memoize memoize

    Example:
        >>> import ubelt as ub
        >>> closure = {'a': 'b', 'c': 'd'}
        >>> incr = [0]
        >>> def foo(key):
        >>>     value = closure[key]
        >>>     incr[0] += 1
        >>>     return value
        >>> foo_memo = ub.memoize(foo)
        >>> assert foo('a') == 'b' and foo('c') == 'd'
        >>> assert incr[0] == 2
        >>> print('Call memoized version')
        >>> assert foo_memo('a') == 'b' and foo_memo('c') == 'd'
        >>> assert incr[0] == 4
        >>> assert foo_memo('a') == 'b' and foo_memo('c') == 'd'
        >>> print('Counter should no longer increase')
        >>> assert incr[0] == 4
        >>> print('Closure changes result without memoization')
        >>> closure = {'a': 0, 'c': 1}
        >>> assert foo('a') == 0 and foo('c') == 1
        >>> assert incr[0] == 6
        >>> assert foo_memo('a') == 'b' and foo_memo('c') == 'd'
    """
    cache = {}
    @functools.wraps(func)
    def memoizer(*args, **kwargs):
        key = _make_signature_key(args, kwargs)
        if key not in cache:
            cache[key] = func(*args, **kwargs)
        return cache[key]
    memoizer.cache = cache
    return memoizer

def memoize ( func ) : cache = { } @ functools . wraps ( func ) def memoizer ( * args , * * kwargs ) : key = _make_signature_key ( args , kwargs ) if key not in cache : cache [ key ] = func ( * args , * * kwargs ) return cache [ key ] memoizer . cache = cache return memoizer


def memoize_property(fget):
    """
    Return a property attribute for new-style classes that only calls its
    getter on the first access. The result is stored and on subsequent accesses
    is returned, preventing the need to call the getter any more.

    This decorator can either be used by itself or by decorating another
    property. In either case the method will always become a property.

    Notes:
        implementation is a modified version of [1].

    References:
        ..[1] https://github.com/estebistec/python-memoized-property

    CommandLine:
        xdoctest -m ubelt.util_memoize memoize_property

    Example:
        >>> class C(object):
        ...     load_name_count = 0
        ...     @memoize_property
        ...     def name(self):
        ...         "name's docstring"
        ...         self.load_name_count += 1
        ...         return "the name"
        ...     @memoize_property
        ...     @property
        ...     def another_name(self):
        ...         "name's docstring"
        ...         self.load_name_count += 1
        ...         return "the name"
        >>> c = C()
        >>> c.load_name_count
        0
        >>> c.name
        'the name'
        >>> c.load_name_count
        1
        >>> c.name
        'the name'
        >>> c.load_name_count
        1
        >>> c.another_name
    """
    # Unwrap any existing property decorator
    while hasattr(fget, 'fget'):
        fget = fget.fget

    attr_name = '_' + fget.__name__

    @functools.wraps(fget)
    def fget_memoized(self):
        if not hasattr(self, attr_name):
            setattr(self, attr_name, fget(self))
        return getattr(self, attr_name)

    return property(fget_memoized)

Return a property attribute for new - style classes that only calls its getter on the first access . The result is stored and on subsequent accesses is returned preventing the need to call the getter any more .


def highlight_code(text, lexer_name='python', **kwargs):
    """
    Highlights a block of text using ANSI tags based on language syntax.

    Args:
        text (str): plain text to highlight
        lexer_name (str): name of language
        **kwargs: passed to pygments.lexers.get_lexer_by_name

    Returns:
        str: text : highlighted text
            If pygments is not installed, the plain text is returned.

    CommandLine:
        python -c "import pygments.formatters; print(list(pygments.formatters.get_all_formatters()))"

    Example:
        >>> import ubelt as ub
        >>> text = 'import ubelt as ub; print(ub)'
        >>> new_text = ub.highlight_code(text)
        >>> print(new_text)
    """
    # Resolve extensions to languages
    lexer_name = {
        'py': 'python',
        'h': 'cpp',
        'cpp': 'cpp',
        'cxx': 'cpp',
        'c': 'cpp',
    }.get(lexer_name.replace('.', ''), lexer_name)
    try:
        import pygments
        import pygments.lexers
        import pygments.formatters
        import pygments.formatters.terminal

        if sys.platform.startswith('win32'):  # nocover
            # Hack on win32 to support colored output
            import colorama
            colorama.init()

        formater = pygments.formatters.terminal.TerminalFormatter(bg='dark')
        lexer = pygments.lexers.get_lexer_by_name(lexer_name, **kwargs)
        new_text = pygments.highlight(text, lexer, formater)

    except ImportError:  # nocover
        import warnings
        warnings.warn('pygments is not installed, code will not be highlighted')
        new_text = text
    return new_text

Highlights a block of text using ANSI tags based on language syntax .


def color_text(text, color):
    r"""
    Colorizes text a single color using ansii tags.

    Args:
        text (str): text to colorize
        color (str): may be one of the following: yellow, blink, lightgray,
            underline, darkyellow, blue, darkblue, faint, fuchsia, black,
            white, red, brown, turquoise, bold, darkred, darkgreen, reset,
            standout, darkteal, darkgray, overline, purple, green, teal, fuscia

    Returns:
        str: text : colorized text.
            If pygments is not installed plain text is returned.

    CommandLine:
        python -c "import pygments.console; print(sorted(pygments.console.codes.keys()))"
        python -m ubelt.util_colors color_text

    Example:
        >>> text = 'raw text'
        >>> import pytest
        >>> import ubelt as ub
        >>> if ub.modname_to_modpath('pygments'):
        >>>     # Colors text only if pygments is installed
        >>>     assert color_text(text, 'red') == '\x1b[31;01mraw text\x1b[39;49;00m'
        >>>     assert color_text(text, None) == 'raw text'
        >>> else:
        >>>     # Otherwise text passes through unchanged
        >>>     assert color_text(text, 'red') == 'raw text'
        >>>     assert color_text(text, None) == 'raw text'
    """
    if color is None:
        return text
    try:
        import pygments
        import pygments.console

        if sys.platform.startswith('win32'):  # nocover
            # Hack on win32 to support colored output
            import colorama
            colorama.init()

        ansi_text = pygments.console.colorize(color, text)
        return ansi_text
    except ImportError:  # nocover
        import warnings
        warnings.warn('pygments is not installed, text will not be colored')
        return text

r Colorizes text a single color using ansii tags .


def iterable(obj, strok=False):
    """
    Checks if the input implements the iterator interface. An exception is made
    for strings, which return False unless `strok` is True

    Args:
        obj (object): a scalar or iterable input

        strok (bool): if True allow strings to be interpreted as iterable

    Returns:
        bool: True if the input is iterable

    Example:
        >>> obj_list = [3, [3], '3', (3,), [3, 4, 5], {}]
        >>> result = [iterable(obj) for obj in obj_list]
        >>> assert result == [False, True, False, True, True, True]
        >>> result = [iterable(obj, strok=True) for obj in obj_list]
        >>> assert result == [False, True, True, True, True, True]
    """
    try:
        iter(obj)
    except Exception:
        return False
    else:
        return strok or not isinstance(obj, six.string_types)

def iterable ( obj , strok = False ) : try : iter ( obj ) except Exception : return False else : return strok or not isinstance ( obj , six . string_types )

Checks if the input implements the iterator interface . An exception is made for strings which return False unless strok is True


def unique(items, key=None):
    """
    Generates unique items in the order they appear.

    Args:
        items (Iterable): list of items

        key (Callable, optional): custom normalization function.
            If specified returns items where `key(item)` is unique.

    Yields:
        object: a unique item from the input sequence

    CommandLine:
        python -m utool.util_list --exec-unique_ordered

    Example:
        >>> import ubelt as ub
        >>> items = [4, 6, 6, 0, 6, 1, 0, 2, 2, 1]
        >>> unique_items = list(ub.unique(items))
        >>> assert unique_items == [4, 6, 0, 1, 2]

    Example:
        >>> import ubelt as ub
        >>> items = ['A', 'a', 'b', 'B', 'C', 'c', 'D', 'e', 'D', 'E']
        >>> unique_items = list(ub.unique(items, key=six.text_type.lower))
        >>> assert unique_items == ['A', 'b', 'C', 'D', 'e']
        >>> unique_items = list(ub.unique(items))
        >>> assert unique_items == ['A', 'a', 'b', 'B', 'C', 'c', 'D', 'e', 'E']
    """
    seen = set()
    if key is None:
        for item in items:
            if item not in seen:
                seen.add(item)
                yield item
    else:
        for item in items:
            norm = key(item)
            if norm not in seen:
                seen.add(norm)
                yield item

def unique ( items , key = None ) : seen = set ( ) if key is None : for item in items : if item not in seen : seen . add ( item ) yield item else : for item in items : norm = key ( item ) if norm not in seen : seen . add ( norm ) yield item

Generates unique items in the order they appear .


def argunique(items, key=None):
    """
    Returns indices corresponding to the first instance of each unique item.

    Args:
        items (Sequence): indexable collection of items

        key (Callable, optional): custom normalization function.
            If specified returns items where `key(item)` is unique.

    Yields:
        int : indices of the unique items

    Example:
        >>> items = [0, 2, 5, 1, 1, 0, 2, 4]
        >>> indices = list(argunique(items))
        >>> assert indices == [0, 1, 2, 3, 7]
        >>> indices = list(argunique(items, key=lambda x: x % 2 == 0))
        >>> assert indices == [0, 2]
    """
    # yield from unique(range(len(items)), key=lambda i: items[i])
    if key is None:
        return unique(range(len(items)), key=lambda i: items[i])
    else:
        return unique(range(len(items)), key=lambda i: key(items[i]))

Returns indices corresponding to the first instance of each unique item .


def unique_flags(items, key=None):
    """
    Returns a list of booleans corresponding to the first instance of each
    unique item.

    Args:
        items (Sequence): indexable collection of items

        key (Callable, optional): custom normalization function.
            If specified returns items where `key(item)` is unique.

    Returns:
        List[bool] : flags the items that are unique

    Example:
        >>> import ubelt as ub
        >>> items = [0, 2, 1, 1, 0, 9, 2]
        >>> flags = unique_flags(items)
        >>> assert flags == [True, True, True, False, False, True, False]
        >>> flags = unique_flags(items, key=lambda x: x % 2 == 0)
        >>> assert flags == [True, False, True, False, False, False, False]
    """
    len_ = len(items)
    if key is None:
        item_to_index = dict(zip(reversed(items), reversed(range(len_))))
        indices = item_to_index.values()
    else:
        indices = argunique(items, key=key)
    flags = boolmask(indices, len_)
    return flags

def unique_flags ( items , key = None ) : len_ = len ( items ) if key is None : item_to_index = dict ( zip ( reversed ( items ) , reversed ( range ( len_ ) ) ) ) indices = item_to_index . values ( ) else : indices = argunique ( items , key = key ) flags = boolmask ( indices , len_ ) return flags

Returns a list of booleans corresponding to the first instance of each unique item .


def boolmask(indices, maxval=None):
    """
    Constructs a list of booleans where an item is True if its position is in
    `indices` otherwise it is False.

    Args:
        indices (list): list of integer indices

        maxval (int): length of the returned list. If not specified
            this is inferred from `indices`

    Note:
        In the future the arg `maxval` may change its name to `shape`

    Returns:
        list: mask: list of booleans. mask[idx] is True if idx in indices

    Example:
        >>> import ubelt as ub
        >>> indices = [0, 1, 4]
        >>> mask = ub.boolmask(indices, maxval=6)
        >>> assert mask == [True, True, False, False, True, False]
        >>> mask = ub.boolmask(indices)
        >>> assert mask == [True, True, False, False, True]
    """
    if maxval is None:
        indices = list(indices)
        maxval = max(indices) + 1
    mask = [False] * maxval
    for index in indices:
        mask[index] = True
    return mask

def boolmask ( indices , maxval = None ) : if maxval is None : indices = list ( indices ) maxval = max ( indices ) + 1 mask = [ False ] * maxval for index in indices : mask [ index ] = True return mask

Constructs a list of booleans where an item is True if its position is in indices otherwise it is False .


def allsame(iterable, eq=operator.eq):
    """
    Determine if all items in a sequence are the same

    Args:
        iterable (Iterable): items to determine if they are all the same

        eq (Callable, optional): function to determine equality
            (default: operator.eq)

    Example:
        >>> allsame([1, 1, 1, 1])
        True
        >>> allsame([])
        True
        >>> allsame([0, 1])
        False
        >>> iterable = iter([0, 1, 1, 1])
        >>> next(iterable)
        >>> allsame(iterable)
        True
        >>> allsame(range(10))
        False
        >>> allsame(range(10), lambda a, b: True)
        True
    """
    iter_ = iter(iterable)
    try:
        first = next(iter_)
    except StopIteration:
        return True
    return all(eq(first, item) for item in iter_)

def allsame ( iterable , eq = operator . eq ) : iter_ = iter ( iterable ) try : first = next ( iter_ ) except StopIteration : return True return all ( eq ( first , item ) for item in iter_ )


def argsort(indexable, key=None, reverse=False):
    """
    Returns the indices that would sort a indexable object.

    This is similar to `numpy.argsort`, but it is written in pure python and
    works on both lists and dictionaries.

    Args:
        indexable (Iterable or Mapping): indexable to sort by

        key (Callable, optional): customizes the ordering of the indexable

        reverse (bool, optional): if True returns in descending order

    Returns:
        list: indices: list of indices such that sorts the indexable

    Example:
        >>> import ubelt as ub
        >>> # argsort works on dicts by returning keys
        >>> dict_ = {'a': 3, 'b': 2, 'c': 100}
        >>> indices = ub.argsort(dict_)
        >>> assert list(ub.take(dict_, indices)) == sorted(dict_.values())
        >>> # argsort works on lists by returning indices
        >>> indexable = [100, 2, 432, 10]
        >>> indices = ub.argsort(indexable)
        >>> assert list(ub.take(indexable, indices)) == sorted(indexable)
        >>> # Can use iterators, but be careful. It exhausts them.
        >>> indexable = reversed(range(100))
        >>> indices = ub.argsort(indexable)
        >>> assert indices[0] == 99
        >>> # Can use key just like sorted
        >>> indexable = [[0, 1, 2], [3, 4], [5]]
        >>> indices = ub.argsort(indexable, key=len)
        >>> assert indices == [2, 1, 0]
        >>> # Can use reverse just like sorted
        >>> indexable = [0, 2, 1]
        >>> indices = ub.argsort(indexable, reverse=True)
        >>> assert indices == [1, 2, 0]
    """
    # Create an iterator of value/key pairs
    if isinstance(indexable, collections_abc.Mapping):
        vk_iter = ((v, k) for k, v in indexable.items())
    else:
        vk_iter = ((v, k) for k, v in enumerate(indexable))
    # Sort by values and extract the indices
    if key is None:
        indices = [k for v, k in sorted(vk_iter, reverse=reverse)]
    else:
        # If key is provided, call it using the value as input
        indices = [k for v, k in sorted(vk_iter, key=lambda vk: key(vk[0]),
                                        reverse=reverse)]
    return indices

Returns the indices that would sort a indexable object .


def dzip(items1, items2, cls=dict):
    """
    Zips elementwise pairs between items1 and items2 into a dictionary. Values
    from items2 can be broadcast onto items1.

    Args:
        items1 (Iterable): full sequence
        items2 (Iterable): can either be a sequence of one item or a sequence
            of equal length to `items1`
        cls (Type[dict]): dictionary type to use. Defaults to dict, but could
            be ordered dict instead.

    Returns:
        dict: similar to dict(zip(items1, items2))

    Example:
        >>> assert dzip([1, 2, 3], [4]) == {1: 4, 2: 4, 3: 4}
        >>> assert dzip([1, 2, 3], [4, 4, 4]) == {1: 4, 2: 4, 3: 4}
        >>> assert dzip([], [4]) == {}
    """
    try:
        len(items1)
    except TypeError:
        items1 = list(items1)
    try:
        len(items2)
    except TypeError:
        items2 = list(items2)
    if len(items1) == 0 and len(items2) == 1:
        # Corner case:
        # allow the first list to be empty and the second list to broadcast a
        # value. This means that the equality check wont work for the case
        # where items1 and items2 are supposed to correspond, but the length of
        # items2 is 1.
        items2 = []
    if len(items2) == 1 and len(items1) > 1:
        items2 = items2 * len(items1)
    if len(items1) != len(items2):
        raise ValueError('out of alignment len(items1)=%r, len(items2)=%r' % (
            len(items1), len(items2)))
    return cls(zip(items1, items2))

Zips elementwise pairs between items1 and items2 into a dictionary . Values from items2 can be broadcast onto items1 .


def group_items(items, groupids):
    r"""
    Groups a list of items by group id.

    Args:
        items (Iterable): a list of items to group
        groupids (Iterable or Callable): a corresponding list of item groupids
            or a function mapping an item to a groupid.

    Returns:
        dict: groupid_to_items: maps a groupid to a list of items

    CommandLine:
        python -m ubelt.util_dict group_items

    Example:
        >>> import ubelt as ub
        >>> items    = ['ham',     'jam',   'spam',     'eggs',    'cheese', 'banana']
        >>> groupids = ['protein', 'fruit', 'protein',  'protein', 'dairy',  'fruit']
        >>> groupid_to_items = ub.group_items(items, groupids)
        >>> print(ub.repr2(groupid_to_items, nl=0))
        {'dairy': ['cheese'], 'fruit': ['jam', 'banana'], 'protein': ['ham', 'spam', 'eggs']}
    """
    if callable(groupids):
        keyfunc = groupids
        pair_list = ((keyfunc(item), item) for item in items)
    else:
        pair_list = zip(groupids, items)

    # Initialize a dict of lists
    groupid_to_items = defaultdict(list)
    # Insert each item into the correct group
    for key, item in pair_list:
        groupid_to_items[key].append(item)
    return groupid_to_items


def dict_hist(item_list, weight_list=None, ordered=False, labels=None):
    """
    Builds a histogram of items, counting the number of time each item appears
    in the input.

    Args:
        item_list (Iterable): hashable items (usually containing duplicates)
        weight_list (Iterable): corresponding weights for each item
        ordered (bool): if True the result is ordered by frequency
        labels (Iterable, optional): expected labels (default None)
            Allows this function to pre-initialize the histogram.
            If specified the frequency of each label is initialized to
            zero and item_list can only contain items specified in labels.

    Returns:
        dict : dictionary where the keys are items in item_list, and the values
          are the number of times the item appears in item_list.

    CommandLine:
        python -m ubelt.util_dict dict_hist

    Example:
        >>> import ubelt as ub
        >>> item_list = [1, 2, 39, 900, 1232, 900, 1232, 2, 2, 2, 900]
        >>> hist = ub.dict_hist(item_list)
        >>> print(ub.repr2(hist, nl=0))
        {1: 1, 2: 4, 39: 1, 900: 3, 1232: 2}

    Example:
        >>> import ubelt as ub
        >>> item_list = [1, 2, 39, 900, 1232, 900, 1232, 2, 2, 2, 900]
        >>> hist1 = ub.dict_hist(item_list)
        >>> hist2 = ub.dict_hist(item_list, ordered=True)
        >>> try:
        >>>     hist3 = ub.dict_hist(item_list, labels=[])
        >>> except KeyError:
        >>>     pass
        >>> else:
        >>>     raise AssertionError('expected key error')
        >>> #result = ub.repr2(hist_)
        >>> weight_list = [1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1]
        >>> hist4 = ub.dict_hist(item_list, weight_list=weight_list)
        >>> print(ub.repr2(hist1, nl=0))
        {1: 1, 2: 4, 39: 1, 900: 3, 1232: 2}
        >>> print(ub.repr2(hist4, nl=0))
        {1: 1, 2: 4, 39: 1, 900: 1, 1232: 0}
    """
    if labels is None:
        hist_ = defaultdict(lambda: 0)
    else:
        hist_ = {k: 0 for k in labels}
    if weight_list is None:
        weight_list = it.repeat(1)
    # Accumulate frequency
    for item, weight in zip(item_list, weight_list):
        hist_[item] += weight
    if ordered:
        # Order by value
        getval = op.itemgetter(1)
        hist = OrderedDict([
            (key, value)
            for (key, value) in sorted(hist_.items(), key=getval)
        ])
    else:
        # Cast to a normal dictionary
        hist = dict(hist_)
    return hist

def
dict_hist
(
item_list
,
weight_list
=
,
ordered
=
False
,
labels
=
)
:
if
labels
is
:
hist_
=
defaultdict
(
lambda
:
0
)
else
:
hist_
=
{
k
:
0
for
k
in
labels
}
if
weight_list
is
:
weight_list
=
it
.
repeat
(
1
)
# Accumulate frequency
for
item
,
weight
in
zip
(
item_list
,
weight_list
)
:
hist_
[
item
]
+=
weight
if
ordered
:
# Order by value
getval
=
op
.
itemgetter
(
1
)
hist
=
OrderedDict
(
[
(
key
,
value
)
for
(
key
,
value
)
in
sorted
(
hist_
.
items
(
)
,
key
=
getval
)
]
)
else
:
# Cast to a normal dictionary
hist
=
dict
(
hist_
)
return
hist

Builds a histogram of items counting the number of time each item appears in the input .


def find_duplicates(items, k=2, key=None):
    """
    Find all duplicate items in a list.

    Search for all items that appear more than `k` times and return a mapping
    from each (k)-duplicate item to the positions it appeared in.

    Args:
        items (Iterable): hashable items possibly containing duplicates
        k (int): only return items that appear at least `k` times (default=2)
        key (Callable, optional): Returns indices where `key(items[i])`
            maps to a particular value at least k times.

    Returns:
        dict: maps each duplicate item to the indices at which it appears

    CommandLine:
        python -m ubelt.util_dict find_duplicates

    Example:
        >>> import ubelt as ub
        >>> items = [0, 0, 1, 2, 3, 3, 0, 12, 2, 9]
        >>> duplicates = ub.find_duplicates(items)
        >>> print('items = %r' % (items,))
        >>> print('duplicates = %r' % (duplicates,))
        >>> assert duplicates == {0: [0, 1, 6], 2: [3, 8], 3: [4, 5]}
        >>> assert ub.find_duplicates(items, 3) == {0: [0, 1, 6]}

    Example:
        >>> import ubelt as ub
        >>> items = [0, 0, 1, 2, 3, 3, 0, 12, 2, 9]
        >>> # note: k can be 0
        >>> duplicates = ub.find_duplicates(items, k=0)
        >>> print(ub.repr2(duplicates, nl=0))
        {0: [0, 1, 6], 1: [2], 2: [3, 8], 3: [4, 5], 9: [9], 12: [7]}

    Example:
        >>> import ubelt as ub
        >>> items = [10, 11, 12, 13, 14, 15, 16]
        >>> duplicates = ub.find_duplicates(items, key=lambda x: x // 2)
        >>> print(ub.repr2(duplicates, nl=0))
        {5: [0, 1], 6: [2, 3], 7: [4, 5]}
    """
    # Build mapping from items to the indices at which they appear
    # if key is not None:
    #     items = map(key, items)
    duplicates = defaultdict(list)
    if key is None:
        for count, item in enumerate(items):
            duplicates[item].append(count)
    else:
        for count, item in enumerate(items):
            duplicates[key(item)].append(count)
    # remove items seen fewer than k times.
    for key in list(duplicates.keys()):
        if len(duplicates[key]) < k:
            del duplicates[key]
    duplicates = dict(duplicates)
    return duplicates


def dict_take(dict_, keys, default=util_const.NoParam):
    r"""
    Generates values from a dictionary

    Args:
        dict_ (Mapping): a dictionary to take from
        keys (Iterable): the keys to take
        default (object, optional): if specified uses default if keys are missing

    CommandLine:
        python -m ubelt.util_dict dict_take_gen

    Example:
        >>> import ubelt as ub
        >>> dict_ = {1: 'a', 2: 'b', 3: 'c'}
        >>> keys = [1, 2, 3, 4, 5]
        >>> result = list(ub.dict_take(dict_, keys, None))
        >>> assert result == ['a', 'b', 'c', None, None]

    Example:
        >>> import ubelt as ub
        >>> dict_ = {1: 'a', 2: 'b', 3: 'c'}
        >>> keys = [1, 2, 3, 4, 5]
        >>> try:
        >>>     print(list(ub.dict_take(dict_, keys)))
        >>>     raise AssertionError('did not get key error')
        >>> except KeyError:
        >>>     print('correctly got key error')
    """
    if default is util_const.NoParam:
        for key in keys:
            yield dict_[key]
    else:
        for key in keys:
            yield dict_.get(key, default)

def dict_take ( dict_ , keys , default = util_const . NoParam ) : if default is util_const . NoParam : for key in keys : yield dict_ [ key ] else : for key in keys : yield dict_ . get ( key , default )


def dict_union(*args):
    """
    Combines the disjoint keys in multiple dictionaries. For intersecting keys,
    dictionaries towards the end of the sequence are given precedence.

    Args:
        *args : a sequence of dictionaries

    Returns:
        Dict | OrderedDict :
            OrderedDict if the first argument is an OrderedDict, otherwise dict

    SeeAlso:
        collections.ChainMap - a standard python builtin data structure that
            provides a view that treats multiple dicts as a single dict.
            https://docs.python.org/3/library/collections.html#chainmap-objects

    Example:
        >>> result = dict_union({'a': 1, 'b': 1}, {'b': 2, 'c': 2})
        >>> assert result == {'a': 1, 'b': 2, 'c': 2}
        >>> dict_union(odict([('a', 1), ('b', 2)]), odict([('c', 3), ('d', 4)]))
        OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 4)])
        >>> dict_union()
        {}
    """
    if not args:
        return {}
    else:
        dictclass = OrderedDict if isinstance(args[0], OrderedDict) else dict
        return dictclass(it.chain.from_iterable(d.items() for d in args))

def dict_union ( * args ) : if not args : return { } else : dictclass = OrderedDict if isinstance ( args [ 0 ] , OrderedDict ) else dict return dictclass ( it . chain . from_iterable ( d . items ( ) for d in args ) )

Combines the disjoint keys in multiple dictionaries . For intersecting keys dictionaries towards the end of the sequence are given precedence .


def dict_isect(*args):
    """
    Constructs a dictionary that contains keys common between all inputs.
    The returned values will only belong to the first dictionary.

    Args:
        *args : a sequence of dictionaries (or sets of keys)

    Returns:
        Dict | OrderedDict :
            OrderedDict if the first argument is an OrderedDict, otherwise dict

    Notes:
        This function can be used as an alternative to `dict_subset` where any
        key not in the dictionary is ignored. See the following example:

        >>> dict_isect({'a': 1, 'b': 2, 'c': 3}, ['a', 'c', 'd'])
        {'a': 1, 'c': 3}

    Example:
        >>> dict_isect({'a': 1, 'b': 1}, {'b': 2, 'c': 2})
        {'b': 1}
        >>> dict_isect(odict([('a', 1), ('b', 2)]), odict([('c', 3)]))
        OrderedDict()
        >>> dict_isect()
        {}
    """
    if not args:
        return {}
    else:
        dictclass = OrderedDict if isinstance(args[0], OrderedDict) else dict
        common_keys = set.intersection(*map(set, args))
        first_dict = args[0]
        return dictclass((k, first_dict[k]) for k in common_keys)

def dict_isect ( * args ) : if not args : return { } else : dictclass = OrderedDict if isinstance ( args [ 0 ] , OrderedDict ) else dict common_keys = set . intersection ( * map ( set , args ) ) first_dict = args [ 0 ] return dictclass ( ( k , first_dict [ k ] ) for k in common_keys )

Constructs a dictionary that contains keys common between all inputs . The returned values will only belong to the first dictionary .


def map_vals(func, dict_):
    """
    applies a function to each of the keys in a dictionary

    Args:
        func (callable): a function or indexable object
        dict_ (dict): a dictionary

    Returns:
        newdict: transformed dictionary

    CommandLine:
        python -m ubelt.util_dict map_vals

    Example:
        >>> import ubelt as ub
        >>> dict_ = {'a': [1, 2, 3], 'b': []}
        >>> func = len
        >>> newdict = ub.map_vals(func, dict_)
        >>> assert newdict ==  {'a': 3, 'b': 0}
        >>> print(newdict)
        >>> # Can also use indexables as `func`
        >>> dict_ = {'a': 0, 'b': 1}
        >>> func = [42, 21]
        >>> newdict = ub.map_vals(func, dict_)
        >>> assert newdict ==  {'a': 42, 'b': 21}
        >>> print(newdict)
    """
    if not hasattr(func, '__call__'):
        func = func.__getitem__
    keyval_list = [(key, func(val)) for key, val in six.iteritems(dict_)]
    dictclass = OrderedDict if isinstance(dict_, OrderedDict) else dict
    newdict = dictclass(keyval_list)
    # newdict = type(dict_)(keyval_list)
    return newdict

def
map_vals
(
func
,
dict_
)
:
if
not
hasattr
(
func
,
'__call__'
)
:
func
=
func
.
__getitem__
keyval_list
=
[
(
key
,
func
(
val
)
)
for
key
,
val
in
six
.
iteritems
(
dict_
)
]
dictclass
=
OrderedDict
if
isinstance
(
dict_
,
OrderedDict
)
else
dict
newdict
=
dictclass
(
keyval_list
)
# newdict = type(dict_)(keyval_list)
return
newdict


def invert_dict(dict_, unique_vals=True):
    r"""
    Swaps the keys and values in a dictionary.

    Args:
        dict_ (dict): dictionary to invert
        unique_vals (bool): if False, inverted keys are returned in a set.
            The default is True.

    Returns:
        dict: inverted

    Notes:
        The must values be hashable.

        If the original dictionary contains duplicate values, then only one of
        the corresponding keys will be returned and the others will be
        discarded.  This can be prevented by setting `unique_vals=True`,
        causing the inverted keys to be returned in a set.

    CommandLine:
        python -m ubelt.util_dict invert_dict

    Example:
        >>> import ubelt as ub
        >>> dict_ = {'a': 1, 'b': 2}
        >>> inverted = ub.invert_dict(dict_)
        >>> assert inverted == {1: 'a', 2: 'b'}

    Example:
        >>> import ubelt as ub
        >>> dict_ = ub.odict([(2, 'a'), (1, 'b'), (0, 'c'), (None, 'd')])
        >>> inverted = ub.invert_dict(dict_)
        >>> assert list(inverted.keys())[0] == 'a'

    Example:
        >>> import ubelt as ub
        >>> dict_ = {'a': 1, 'b': 0, 'c': 0, 'd': 0, 'f': 2}
        >>> inverted = ub.invert_dict(dict_, unique_vals=False)
        >>> assert inverted == {0: {'b', 'c', 'd'}, 1: {'a'}, 2: {'f'}}
    """
    if unique_vals:
        if isinstance(dict_, OrderedDict):
            inverted = OrderedDict((val, key) for key, val in dict_.items())
        else:
            inverted = {val: key for key, val in dict_.items()}
    else:
        # Handle non-unique keys using groups
        inverted = defaultdict(set)
        for key, value in dict_.items():
            inverted[value].add(key)
        inverted = dict(inverted)
    return inverted

def
invert_dict
(
dict_
,
unique_vals
=
True
)
:
if
unique_vals
:
if
isinstance
(
dict_
,
OrderedDict
)
:
inverted
=
OrderedDict
(
(
val
,
key
)
for
key
,
val
in
dict_
.
items
(
)
)
else
:
inverted
=
{
val
:
key
for
key
,
val
in
dict_
.
items
(
)
}
else
:
# Handle non-unique keys using groups
inverted
=
defaultdict
(
set
)
for
key
,
value
in
dict_
.
items
(
)
:
inverted
[
value
]
.
add
(
key
)
inverted
=
dict
(
inverted
)
return
inverted

r Swaps the keys and values in a dictionary .


def to_dict(self):
        """
        Recursively casts a AutoDict into a regular dictionary. All nested
        AutoDict values are also converted.

        Returns:
            dict: a copy of this dict without autovivification

        Example:
            >>> from ubelt.util_dict import AutoDict
            >>> auto = AutoDict()
            >>> auto[1] = 1
            >>> auto['n1'] = AutoDict()
            >>> static = auto.to_dict()
            >>> assert not isinstance(static, AutoDict)
            >>> assert not isinstance(static['n1'], AutoDict)
        """
        return self._base(
            (key, (value.to_dict() if isinstance(value, AutoDict) else value))
            for key, value in self.items())

def to_dict ( self ) : return self . _base ( ( key , ( value . to_dict ( ) if isinstance ( value , AutoDict ) else value ) ) for key , value in self . items ( ) )

Recursively casts a AutoDict into a regular dictionary . All nested AutoDict values are also converted .


def _symlink(path, link, overwrite=0, verbose=0):
    """
    Windows helper for ub.symlink
    """
    if exists(link) and not os.path.islink(link):
        # On windows a broken link might still exist as a hard link or a
        # junction. Overwrite it if it is a file and we cannot symlink.
        # However, if it is a non-junction directory then do not overwrite
        if verbose:
            print('link location already exists')
        is_junc = _win32_is_junction(link)
        # NOTE:
        # in python2 broken junctions are directories and exist
        # in python3 broken junctions are directories and do not exist
        if os.path.isdir(link):
            if is_junc:
                pointed = _win32_read_junction(link)
                if path == pointed:
                    if verbose:
                        print('...and is a junction that points to the same place')
                    return link
                else:
                    if verbose:
                        if not exists(pointed):
                            print('...and is a broken junction that points somewhere else')
                        else:
                            print('...and is a junction that points somewhere else')
            else:
                if verbose:
                    print('...and is an existing real directory!')
                raise IOError('Cannot overwrite a real directory')

        elif os.path.isfile(link):
            if _win32_is_hardlinked(link, path):
                if verbose:
                    print('...and is a hard link that points to the same place')
                return link
            else:
                if verbose:
                    print('...and is a hard link that points somewhere else')
                if _win32_can_symlink():
                    raise IOError('Cannot overwrite potentially real file if we can symlink')
        if overwrite:
            if verbose:
                print('...overwriting')
            util_io.delete(link, verbose > 1)
        else:
            if exists(link):
                raise IOError('Link already exists')

    _win32_symlink2(path, link, verbose=verbose)


def _win32_symlink2(path, link, allow_fallback=True, verbose=0):
    """
    Perform a real symbolic link if possible. However, on most versions of
    windows you need special privledges to create a real symlink. Therefore, we
    try to create a symlink, but if that fails we fallback to using a junction.

    AFAIK, the main difference between symlinks and junctions are that symlinks
    can reference relative or absolute paths, where as junctions always
    reference absolute paths. Not 100% on this though. Windows is weird.

    Note that junctions will not register as links via `islink`, but I
    believe real symlinks will.
    """
    if _win32_can_symlink():
        return _win32_symlink(path, link, verbose)
    else:
        return _win32_junction(path, link, verbose)

def _win32_symlink2 ( path , link , allow_fallback = True , verbose = 0 ) : if _win32_can_symlink ( ) : return _win32_symlink ( path , link , verbose ) else : return _win32_junction ( path , link , verbose )

Perform a real symbolic link if possible . However on most versions of windows you need special privledges to create a real symlink . Therefore we try to create a symlink but if that fails we fallback to using a junction .


def _win32_symlink(path, link, verbose=0):
    """
    Creates real symlink. This will only work in versions greater than Windows
    Vista. Creating real symlinks requires admin permissions or at least
    specially enabled symlink permissions. On Windows 10 enabling developer
    mode should give you these permissions.
    """
    from ubelt import util_cmd
    if os.path.isdir(path):
        # directory symbolic link
        if verbose:
            print('... as directory symlink')
        command = 'mklink /D "{}" "{}"'.format(link, path)
        # Using the win32 API seems to result in privilege errors
        # but using shell commands does not have this problem. Weird.
        # jwfs.symlink(path, link, target_is_directory=True)
        # TODO: what do we need to do to use the windows api instead of shell?
    else:
        # file symbolic link
        if verbose:
            print('... as file symlink')
        command = 'mklink "{}" "{}"'.format(link, path)

    if command is not None:
        info = util_cmd.cmd(command, shell=True)
        if info['ret'] != 0:
            from ubelt import util_format
            permission_msg = 'You do not have sufficient privledges'
            if permission_msg not in info['err']:
                print('Failed command:')
                print(info['command'])
                print(util_format.repr2(info, nl=1))
            raise OSError(str(info))
    return link

Creates real symlink . This will only work in versions greater than Windows Vista . Creating real symlinks requires admin permissions or at least specially enabled symlink permissions . On Windows 10 enabling developer mode should give you these permissions .


def _win32_is_junction(path):
    """
    Determines if a path is a win32 junction

    CommandLine:
        python -m ubelt._win32_links _win32_is_junction

    Example:
        >>> # xdoc: +REQUIRES(WIN32)
        >>> import ubelt as ub
        >>> root = ub.ensure_app_cache_dir('ubelt', 'win32_junction')
        >>> ub.delete(root)
        >>> ub.ensuredir(root)
        >>> dpath = join(root, 'dpath')
        >>> djunc = join(root, 'djunc')
        >>> ub.ensuredir(dpath)
        >>> _win32_junction(dpath, djunc)
        >>> assert _win32_is_junction(djunc) is True
        >>> assert _win32_is_junction(dpath) is False
        >>> assert _win32_is_junction('notafile') is False
    """
    if not exists(path):
        if os.path.isdir(path):
            if not os.path.islink(path):
                return True
        return False
    return jwfs.is_reparse_point(path) and not os.path.islink(path)

def _win32_is_junction ( path ) : if not exists ( path ) : if os . path . isdir ( path ) : if not os . path . islink ( path ) : return True return False return jwfs . is_reparse_point ( path ) and not os . path . islink ( path )


def _win32_read_junction(path):
    """
    Returns the location that the junction points, raises ValueError if path is
    not a junction.

    CommandLine:
        python -m ubelt._win32_links _win32_read_junction

    Example:
        >>> # xdoc: +REQUIRES(WIN32)
        >>> import ubelt as ub
        >>> root = ub.ensure_app_cache_dir('ubelt', 'win32_junction')
        >>> ub.delete(root)
        >>> ub.ensuredir(root)
        >>> dpath = join(root, 'dpath')
        >>> djunc = join(root, 'djunc')
        >>> ub.ensuredir(dpath)
        >>> _win32_junction(dpath, djunc)
        >>> path = djunc
        >>> pointed = _win32_read_junction(path)
        >>> print('pointed = {!r}'.format(pointed))
    """
    if not jwfs.is_reparse_point(path):
        raise ValueError('not a junction')

    # --- Older version based on using shell commands ---
    # if not exists(path):
    #     if six.PY2:
    #         raise OSError('Cannot find path={}'.format(path))
    #     else:
    #         raise FileNotFoundError('Cannot find path={}'.format(path))
    # target_name = os.path.basename(path)
    # for type_or_size, name, pointed in _win32_dir(path, '*'):
    #     if type_or_size == '<JUNCTION>' and name == target_name:
    #         return pointed
    # raise ValueError('not a junction')

    # new version using the windows api
    handle = jwfs.api.CreateFile(
            path, 0, 0, None, jwfs.api.OPEN_EXISTING,
            jwfs.api.FILE_FLAG_OPEN_REPARSE_POINT |
            jwfs.api.FILE_FLAG_BACKUP_SEMANTICS,
            None)

    if handle == jwfs.api.INVALID_HANDLE_VALUE:
        raise WindowsError()

    res = jwfs.reparse.DeviceIoControl(
            handle, jwfs.api.FSCTL_GET_REPARSE_POINT, None, 10240)

    bytes = jwfs.create_string_buffer(res)
    p_rdb = jwfs.cast(bytes, jwfs.POINTER(jwfs.api.REPARSE_DATA_BUFFER))
    rdb = p_rdb.contents

    if rdb.tag not in [2684354563, jwfs.api.IO_REPARSE_TAG_SYMLINK]:
        raise RuntimeError(
                "Expected <2684354563 or 2684354572>, but got %d" % rdb.tag)

    jwfs.handle_nonzero_success(jwfs.api.CloseHandle(handle))
    subname = rdb.get_substitute_name()
    # probably has something to do with long paths, not sure
    if subname.startswith('?\\'):
        subname = subname[2:]
    return subname

Returns the location that the junction points raises ValueError if path is not a junction .


def _win32_rmtree(path, verbose=0):
    """
    rmtree for win32 that treats junctions like directory symlinks.
    The junction removal portion may not be safe on race conditions.

    There is a known issue that prevents shutil.rmtree from
    deleting directories with junctions.
    https://bugs.python.org/issue31226
    """

    # --- old version using the shell ---
    # def _rmjunctions(root):
    #     subdirs = []
    #     for type_or_size, name, pointed in _win32_dir(root):
    #         if type_or_size == '<DIR>':
    #             subdirs.append(name)
    #         elif type_or_size == '<JUNCTION>':
    #             # remove any junctions as we encounter them
    #             # os.unlink(join(root, name))
    #             os.rmdir(join(root, name))
    #     # recurse in all real directories
    #     for name in subdirs:
    #         _rmjunctions(join(root, name))

    def _rmjunctions(root):
        subdirs = []
        for name in os.listdir(root):
            current = join(root, name)
            if os.path.isdir(current):
                if _win32_is_junction(current):
                    # remove any junctions as we encounter them
                    os.rmdir(current)
                elif not os.path.islink(current):
                    subdirs.append(current)
        # recurse in all real directories
        for subdir in subdirs:
            _rmjunctions(subdir)

    if _win32_is_junction(path):
        if verbose:
            print('Deleting <JUNCTION> directory="{}"'.format(path))
        os.rmdir(path)
    else:
        if verbose:
            print('Deleting directory="{}"'.format(path))
        # first remove all junctions
        _rmjunctions(path)
        # now we can rmtree as normal
        import shutil
        shutil.rmtree(path)

#         elif type_or_size == '<JUNCTION>':

#             # os.unlink(join(root, name))

#             os.rmdir(join(root, name))

#         _rmjunctions(join(root, name))

rmtree for win32 that treats junctions like directory symlinks . The junction removal portion may not be safe on race conditions .


def _win32_is_hardlinked(fpath1, fpath2):
    """
    Test if two hard links point to the same location

    CommandLine:
        python -m ubelt._win32_links _win32_is_hardlinked

    Example:
        >>> # xdoc: +REQUIRES(WIN32)
        >>> import ubelt as ub
        >>> root = ub.ensure_app_cache_dir('ubelt', 'win32_hardlink')
        >>> ub.delete(root)
        >>> ub.ensuredir(root)
        >>> fpath1 = join(root, 'fpath1')
        >>> fpath2 = join(root, 'fpath2')
        >>> ub.touch(fpath1)
        >>> ub.touch(fpath2)
        >>> fjunc1 = _win32_junction(fpath1, join(root, 'fjunc1'))
        >>> fjunc2 = _win32_junction(fpath2, join(root, 'fjunc2'))
        >>> assert _win32_is_hardlinked(fjunc1, fpath1)
        >>> assert _win32_is_hardlinked(fjunc2, fpath2)
        >>> assert not _win32_is_hardlinked(fjunc2, fpath1)
        >>> assert not _win32_is_hardlinked(fjunc1, fpath2)
    """
    # NOTE: jwf.samefile(fpath1, fpath2) seems to behave differently
    def get_read_handle(fpath):
        if os.path.isdir(fpath):
            dwFlagsAndAttributes = jwfs.api.FILE_FLAG_BACKUP_SEMANTICS
        else:
            dwFlagsAndAttributes = 0
        hFile = jwfs.api.CreateFile(fpath, jwfs.api.GENERIC_READ,
                                    jwfs.api.FILE_SHARE_READ, None,
                                    jwfs.api.OPEN_EXISTING,
                                    dwFlagsAndAttributes, None)
        return hFile

    def get_unique_id(hFile):
        info = jwfs.api.BY_HANDLE_FILE_INFORMATION()
        res = jwfs.api.GetFileInformationByHandle(hFile, info)
        jwfs.handle_nonzero_success(res)
        unique_id = (info.volume_serial_number, info.file_index_high,
                     info.file_index_low)
        return unique_id

    hFile1 = get_read_handle(fpath1)
    hFile2 = get_read_handle(fpath2)
    try:
        are_equal = (get_unique_id(hFile1) == get_unique_id(hFile2))
    except Exception:
        raise
    finally:
        jwfs.api.CloseHandle(hFile1)
        jwfs.api.CloseHandle(hFile2)
    return are_equal


def _win32_dir(path, star=''):
    """
    Using the windows cmd shell to get information about a directory
    """
    from ubelt import util_cmd
    import re
    wrapper = 'cmd /S /C "{}"'  # the /S will preserve all inner quotes
    command = 'dir /-C "{}"{}'.format(path, star)
    wrapped = wrapper.format(command)
    info = util_cmd.cmd(wrapped, shell=True)
    if info['ret'] != 0:
        from ubelt import util_format
        print('Failed command:')
        print(info['command'])
        print(util_format.repr2(info, nl=1))
        raise OSError(str(info))
    # parse the output of dir to get some info
    # Remove header and footer
    lines = info['out'].split('\n')[5:-3]
    splitter = re.compile('( +)')
    for line in lines:
        parts = splitter.split(line)
        date, sep, time, sep, ampm, sep, type_or_size, sep = parts[:8]
        name = ''.join(parts[8:])
        # if type is a junction then name will also contain the linked loc
        if name == '.' or name == '..':
            continue
        if type_or_size in ['<JUNCTION>', '<SYMLINKD>', '<SYMLINK>']:
            # colons cannot be in path names, so use that to find where
            # the name ends
            pos = name.find(':')
            bpos = name[:pos].rfind('[')
            name = name[:bpos - 1]
            pointed = name[bpos + 1:-1]
            yield type_or_size, name, pointed
        else:
            yield type_or_size, name, None


def parse_generator_doubling(config):
    """ Returns generators that double with each value returned
        Config includes optional start value """
    start = 1
    if 'start' in config:
        start = int(config['start'])

    # We cannot simply use start as the variable, because of scoping
    # limitations
    def generator():
        val = start
        while(True):
            yield val
            val = val * 2
    return generator()


def parse(config):
        """ Parse a contains validator, which takes as the config a simple string to find """
        if not isinstance(config, basestring):
            raise TypeError("Contains input must be a simple string")
        validator = ContainsValidator()
        validator.contains_string = config
        return validator

Parse a contains validator which takes as the config a simple string to find


def retrieve_adjacency_matrix(graph, order_nodes=None, weight=False):
    """Retrieve the adjacency matrix from the nx.DiGraph or numpy array."""
    if isinstance(graph, np.ndarray):
        return graph
    elif isinstance(graph, nx.DiGraph):
        if order_nodes is None:
            order_nodes = graph.nodes()
        if not weight:
            return np.array(nx.adjacency_matrix(graph, order_nodes, weight=None).todense())
        else:
            return np.array(nx.adjacency_matrix(graph, order_nodes).todense())
    else:
        raise TypeError("Only networkx.DiGraph and np.ndarray (adjacency matrixes) are supported.")

Retrieve the adjacency matrix from the nx . DiGraph or numpy array .


def create_graph_from_data(self, data, **kwargs):
        """Apply causal discovery on observational data using CCDr.

        Args:
            data (pandas.DataFrame): DataFrame containing the data

        Returns:
            networkx.DiGraph: Solution given by the CCDR algorithm.
        """
        # Building setup w/ arguments.
        self.arguments['{VERBOSE}'] = str(self.verbose).upper()
        results = self._run_ccdr(data, verbose=self.verbose)
        return nx.relabel_nodes(nx.DiGraph(results),
                                {idx: i for idx, i in enumerate(data.columns)})

def
create_graph_from_data
(
self
,
data
,
*
*
kwargs
)
:
# Building setup w/ arguments.
self
.
arguments
[
'{VERBOSE}'
]
=
str
(
self
.
verbose
)
.
upper
(
)
results
=
self
.
_run_ccdr
(
data
,
verbose
=
self
.
verbose
)
return
nx
.
relabel_nodes
(
nx
.
DiGraph
(
results
)
,
{
idx
:
i
for
idx
,
i
in
enumerate
(
data
.
columns
)
}
)

Apply causal discovery on observational data using CCDr .


def to_csv(self, fname_radical, **kwargs):
        """
        Save data to the csv format by default, in two separate files.

        Optional keyword arguments can be passed to pandas.
        """
        if self.data is not None:
            self.data.to_csv(fname_radical+'_data.csv', index=False, **kwargs)
            pd.DataFrame(self.adjacency_matrix).to_csv(fname_radical \
                                                       + '_target.csv',
                                                       index=False, **kwargs)

        else:
            raise ValueError("Graph has not yet been generated. \
                              Use self.generate() to do so.")

Save data to the csv format by default in two separate files .


def launch_R_script(template, arguments, output_function=None,
                    verbose=True, debug=False):
    """Launch an R script, starting from a template and replacing text in file
    before execution.

    Args:
        template (str): path to the template of the R script
        arguments (dict): Arguments that modify the template's placeholders
            with arguments
        output_function (function): Function to execute **after** the execution
            of the R script, and its output is returned by this function. Used
            traditionally as a function to retrieve the results of the
            execution.
        verbose (bool): Sets the verbosity of the R subprocess.
        debug (bool): If True, the generated scripts are not deleted.

    Return:
        Returns the output of the ``output_function`` if not `None`
        else `True` or `False` depending on whether the execution was
        successful.
    """
    id = str(uuid.uuid4())
    os.makedirs('/tmp/cdt_R_script_' + id + '/')
    try:
        scriptpath = '/tmp/cdt_R_script_' + id + '/instance_{}'.format(os.path.basename(template))
        copy(template, scriptpath)

        with fileinput.FileInput(scriptpath, inplace=True) as file:
            for line in file:
                mline = line
                for elt in arguments:
                    mline = mline.replace(elt, arguments[elt])
                print(mline, end='')

        if output_function is None:
            output = subprocess.call("Rscript --vanilla {}".format(scriptpath), shell=True,
                                     stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
        else:
            if verbose:
                process = subprocess.Popen("Rscript --vanilla {}".format(scriptpath), shell=True)
            else:
                process = subprocess.Popen("Rscript --vanilla {}".format(scriptpath), shell=True,
                                           stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
            process.wait()
            output = output_function()

    # Cleaning up
    except Exception as e:
        if not debug:
            rmtree('/tmp/cdt_R_script_' + id + '/')
        raise e
    except KeyboardInterrupt:
        if not debug:
            rmtree('/tmp/cdt_R_script_' + id + '/')
        raise KeyboardInterrupt
    if not debug:
        rmtree('/tmp/cdt_R_script_' + id + '/')
    return output

def
launch_R_script
(
template
,
arguments
,
output_function
=
,
verbose
=
True
,
debug
=
False
)
:
id
=
str
(
uuid
.
uuid4
(
)
)
os
.
makedirs
(
'/tmp/cdt_R_script_'
+
id
+
'/'
)
try
:
scriptpath
=
'/tmp/cdt_R_script_'
+
id
+
'/instance_{}'
.
format
(
os
.
path
.
basename
(
template
)
)
copy
(
template
,
scriptpath
)
with
fileinput
.
FileInput
(
scriptpath
,
inplace
=
True
)
as
file
:
for
line
in
file
:
mline
=
line
for
elt
in
arguments
:
mline
=
mline
.
replace
(
elt
,
arguments
[
elt
]
)
print
(
mline
,
end
=
''
)
if
output_function
is
:
output
=
subprocess
.
call
(
"Rscript --vanilla {}"
.
format
(
scriptpath
)
,
shell
=
True
,
stdout
=
subprocess
.
DEVNULL
,
stderr
=
subprocess
.
DEVNULL
)
else
:
if
verbose
:
process
=
subprocess
.
Popen
(
"Rscript --vanilla {}"
.
format
(
scriptpath
)
,
shell
=
True
)
else
:
process
=
subprocess
.
Popen
(
"Rscript --vanilla {}"
.
format
(
scriptpath
)
,
shell
=
True
,
stdout
=
subprocess
.
DEVNULL
,
stderr
=
subprocess
.
DEVNULL
)
process
.
wait
(
)
output
=
output_function
(
)
# Cleaning up
except
Exception
as
e
:
if
not
debug
:
rmtree
(
'/tmp/cdt_R_script_'
+
id
+
'/'
)
raise
e
except
KeyboardInterrupt
:
if
not
debug
:
rmtree
(
'/tmp/cdt_R_script_'
+
id
+
'/'
)
raise
KeyboardInterrupt
if
not
debug
:
rmtree
(
'/tmp/cdt_R_script_'
+
id
+
'/'
)
return
output

Launch an R script starting from a template and replacing text in file before execution .


def check_R_package(self, package):
        """Execute a subprocess to check the package's availability.

        Args:
            package (str): Name of the package to be tested.

        Returns:
            bool: `True` if the package is available, `False` otherwise
        """
        test_package = not bool(launch_R_script("{}/R_templates/test_import.R".format(os.path.dirname(os.path.realpath(__file__))),                                      {"{package}": package}, verbose=True))
        return test_package

def check_R_package ( self , package ) : test_package = not bool ( launch_R_script ( "{}/R_templates/test_import.R" . format ( os . path . dirname ( os . path . realpath ( __file__ ) ) ) , { "{package}" : package } , verbose = True ) ) return test_package

Execute a subprocess to check the package s availability .


def predict(self, a, b, **kwargs):
        """Perform the independence test.

        :param a: input data
        :param b: input data
        :type a: array-like, numerical data
        :type b: array-like, numerical data
        :return: dependency statistic (1=Highly dependent, 0=Not dependent)
        :rtype: float
        """
        binning_alg = kwargs.get('bins', 'fd')
        return metrics.adjusted_mutual_info_score(bin_variable(a, bins=binning_alg),
                                                  bin_variable(b, bins=binning_alg))

def predict ( self , a , b , * * kwargs ) : binning_alg = kwargs . get ( 'bins' , 'fd' ) return metrics . adjusted_mutual_info_score ( bin_variable ( a , bins = binning_alg ) , bin_variable ( b , bins = binning_alg ) )


def predict(self, df_data, graph=None, **kwargs):
        """Orient a graph using the method defined by the arguments.

        Depending on the type of `graph`, this function process to execute
        different functions:

        1. If ``graph`` is a ``networkx.DiGraph``, then ``self.orient_directed_graph`` is executed.
        2. If ``graph`` is a ``networkx.Graph``, then ``self.orient_undirected_graph`` is executed.
        3. If ``graph`` is a ``None``, then ``self.create_graph_from_data`` is executed.

        Args:
            df_data (pandas.DataFrame): DataFrame containing the observational data.
            graph (networkx.DiGraph or networkx.Graph or None): Prior knowledge on the causal graph.

        .. warning::
           Requirement : Name of the nodes in the graph must correspond to the
           name of the variables in df_data
        """
        if graph is None:
            return self.create_graph_from_data(df_data, **kwargs)
        elif isinstance(graph, nx.DiGraph):
            return self.orient_directed_graph(df_data, graph, **kwargs)
        elif isinstance(graph, nx.Graph):
            return self.orient_undirected_graph(df_data, graph, **kwargs)
        else:
            print('Unknown Graph type')
            raise ValueError

def
predict
(
self
,
df_data
,
graph
=
,
*
*
kwargs
)
:
if
graph
is
:
return
self
.
create_graph_from_data
(
df_data
,
*
*
kwargs
)
elif
isinstance
(
graph
,
nx
.
DiGraph
)
:
return
self
.
orient_directed_graph
(
df_data
,
graph
,
*
*
kwargs
)
elif
isinstance
(
graph
,
nx
.
Graph
)
:
return
self
.
orient_undirected_graph
(
df_data
,
graph
,
*
*
kwargs
)
else
:
print
(
'Unknown Graph type'
)
raise
ValueError

Orient a graph using the method defined by the arguments .


def graph_evaluation(data, adj_matrix, gpu=None, gpu_id=0, **kwargs):
    """Evaluate a graph taking account of the hardware."""
    gpu = SETTINGS.get_default(gpu=gpu)
    device = 'cuda:{}'.format(gpu_id) if gpu else 'cpu'
    obs = th.FloatTensor(data).to(device)
    cgnn = CGNN_model(adj_matrix, data.shape[0], gpu_id=gpu_id, **kwargs).to(device)
    cgnn.reset_parameters()
    return cgnn.run(obs, **kwargs)

def graph_evaluation ( data , adj_matrix , gpu = None , gpu_id = 0 , * * kwargs ) : gpu = SETTINGS . get_default ( gpu = gpu ) device = 'cuda:{}' . format ( gpu_id ) if gpu else 'cpu' obs = th . FloatTensor ( data ) . to ( device ) cgnn = CGNN_model ( adj_matrix , data . shape [ 0 ] , gpu_id = gpu_id , * * kwargs ) . to ( device ) cgnn . reset_parameters ( ) return cgnn . run ( obs , * * kwargs )

Evaluate a graph taking account of the hardware .