def xformer(signature):
    """
    Returns a transformer function for the given signature.

    :param str signature: a dbus signature
    :returns: a function to transform a list of objects to inhabit the signature
    :rtype: (list of object) -> (list of object)
    """

    funcs = [f for (f, _) in xformers(signature)]

    def the_func(objects):
        """
        Returns the a list of objects, transformed.

        :param objects: a list of objects
        :type objects: list of object

        :returns: transformed objects
        :rtype: list of object (in dbus types)
        """
        if len(objects) != len(funcs):
            raise IntoDPValueError(
                objects,
                "objects",
                "must have exactly %u items, has %u" % \
                  (len(funcs), len(objects))
            )
        return [x for (x, _) in (f(a) for (f, a) in zip(funcs, objects))]

    return the_func

Returns a transformer function for the given signature .


def _variant_levels(level, variant):
        """
        Gets the level for the variant.

        :param int level: the current variant level
        :param int variant: the value for this level if variant

        :returns: a level for the object and one for the function
        :rtype: int * int
        """
        return (level + variant, level + variant) \
           if variant != 0 else (variant, level)

def _variant_levels ( level , variant ) : return ( level + variant , level + variant ) if variant != 0 else ( variant , level )


def _handle_variant(self):
        """
        Generate the correct function for a variant signature.

        :returns: function that returns an appropriate value
        :rtype: ((str * object) or list)-> object
        """

        def the_func(a_tuple, variant=0):
            """
            Function for generating a variant value from a tuple.

            :param a_tuple: the parts of the variant
            :type a_tuple: (str * object) or list
            :param int variant: object's variant index
            :returns: a value of the correct type with correct variant level
            :rtype: object * int
            """
            # pylint: disable=unused-argument
            (signature, an_obj) = a_tuple
            (func, sig) = self.COMPLETE.parseString(signature)[0]
            assert sig == signature
            (xformed, _) = func(an_obj, variant=variant + 1)
            return (xformed, xformed.variant_level)

        return (the_func, 'v')

Generate the correct function for a variant signature .


def _handle_array(toks):
        """
        Generate the correct function for an array signature.

        :param toks: the list of parsed tokens
        :returns: function that returns an Array or Dictionary value
        :rtype: ((or list dict) -> ((or Array Dictionary) * int)) * str
        """

        if len(toks) == 5 and toks[1] == '{' and toks[4] == '}':
            subtree = toks[2:4]
            signature = ''.join(s for (_, s) in subtree)
            [key_func, value_func] = [f for (f, _) in subtree]

            def the_dict_func(a_dict, variant=0):
                """
                Function for generating a Dictionary from a dict.

                :param a_dict: the dictionary to transform
                :type a_dict: dict of (`a * `b)
                :param int variant: variant level

                :returns: a dbus dictionary of transformed values and level
                :rtype: Dictionary * int
                """
                elements = \
                   [(key_func(x), value_func(y)) for (x, y) in a_dict.items()]
                level = 0 if elements == [] \
                   else max(max(x, y) for ((_, x), (_, y)) in elements)
                (obj_level, func_level) = \
                   _ToDbusXformer._variant_levels(level, variant)
                return (dbus.types.Dictionary(
                    ((x, y) for ((x, _), (y, _)) in elements),
                    signature=signature,
                    variant_level=obj_level), func_level)

            return (the_dict_func, 'a{' + signature + '}')

        if len(toks) == 2:
            (func, sig) = toks[1]

            def the_array_func(a_list, variant=0):
                """
                Function for generating an Array from a list.

                :param a_list: the list to transform
                :type a_list: list of `a
                :param int variant: variant level of the value
                :returns: a dbus Array of transformed values and variant level
                :rtype: Array * int
                """
                if isinstance(a_list, dict):
                    raise IntoDPValueError(a_list, "a_list",
                                           "is a dict, must be an array")
                elements = [func(x) for x in a_list]
                level = 0 if elements == [] else max(x for (_, x) in elements)
                (obj_level, func_level) = \
                   _ToDbusXformer._variant_levels(level, variant)

                return (dbus.types.Array(
                    (x for (x, _) in elements),
                    signature=sig,
                    variant_level=obj_level), func_level)

            return (the_array_func, 'a' + sig)

        raise IntoDPValueError(toks, "toks",
                               "unexpected tokens")

Generate the correct function for an array signature .


def _handle_struct(toks):
        """
        Generate the correct function for a struct signature.

        :param toks: the list of parsed tokens
        :returns: function that returns an Array or Dictionary value
        :rtype: ((list or tuple) -> (Struct * int)) * str
        """
        subtrees = toks[1:-1]
        signature = ''.join(s for (_, s) in subtrees)
        funcs = [f for (f, _) in subtrees]

        def the_func(a_list, variant=0):
            """
            Function for generating a Struct from a list.

            :param a_list: the list to transform
            :type a_list: list or tuple
            :param int variant: variant index
            :returns: a dbus Struct of transformed values and variant level
            :rtype: Struct * int
            :raises IntoDPValueError:
            """
            if isinstance(a_list, dict):
                raise IntoDPValueError(a_list, "a_list",
                                       "must be a simple sequence, is a dict")
            if len(a_list) != len(funcs):
                raise IntoDPValueError(
                    a_list,
                    "a_list",
                    "must have exactly %u items, has %u" % \
                      (len(funcs), len(a_list))
                )
            elements = [f(x) for (f, x) in zip(funcs, a_list)]
            level = 0 if elements == [] else max(x for (_, x) in elements)
            (obj_level, func_level) = \
                _ToDbusXformer._variant_levels(level, variant)
            return (dbus.types.Struct(
                (x for (x, _) in elements),
                signature=signature,
                variant_level=obj_level), func_level)

        return (the_func, '(' + signature + ')')

Generate the correct function for a struct signature .


def _handle_base_case(klass, symbol):
        """
        Handle a base case.

        :param type klass: the class constructor
        :param str symbol: the type code
        """

        def the_func(value, variant=0):
            """
            Base case.

            :param int variant: variant level for this object
            :returns: a tuple of a dbus object and the variant level
            :rtype: dbus object * int
            """
            (obj_level, func_level) = _ToDbusXformer._variant_levels(
                0, variant)
            return (klass(value, variant_level=obj_level), func_level)

        return lambda: (the_func, symbol)


def signature(dbus_object, unpack=False):
    """
    Get the signature of a dbus object.

    :param dbus_object: the object
    :type dbus_object: a dbus object
    :param bool unpack: if True, unpack from enclosing variant type
    :returns: the corresponding signature
    :rtype: str
    """
    # pylint: disable=too-many-return-statements
    # pylint: disable=too-many-branches

    if dbus_object.variant_level != 0 and not unpack:
        return 'v'

    if isinstance(dbus_object, dbus.Array):
        sigs = frozenset(signature(x) for x in dbus_object)
        len_sigs = len(sigs)
        if len_sigs > 1:  # pragma: no cover
            raise IntoDPValueError(dbus_object, "dbus_object",
                                   "has bad signature")

        if len_sigs == 0:
            return 'a' + dbus_object.signature

        return 'a' + [x for x in sigs][0]

    if isinstance(dbus_object, dbus.Struct):
        sigs = (signature(x) for x in dbus_object)
        return '(' + "".join(x for x in sigs) + ')'

    if isinstance(dbus_object, dbus.Dictionary):
        key_sigs = frozenset(signature(x) for x in dbus_object.keys())
        value_sigs = frozenset(signature(x) for x in dbus_object.values())

        len_key_sigs = len(key_sigs)
        len_value_sigs = len(value_sigs)

        if len_key_sigs != len_value_sigs:  # pragma: no cover
            raise IntoDPValueError(dbus_object, "dbus_object",
                                   "has bad signature")

        if len_key_sigs > 1:  # pragma: no cover
            raise IntoDPValueError(dbus_object, "dbus_object",
                                   "has bad signature")

        if len_key_sigs == 0:
            return 'a{' + dbus_object.signature + '}'

        return 'a{' + [x for x in key_sigs][0] + [x
                                                  for x in value_sigs][0] + '}'

    if isinstance(dbus_object, dbus.Boolean):
        return 'b'

    if isinstance(dbus_object, dbus.Byte):
        return 'y'

    if isinstance(dbus_object, dbus.Double):
        return 'd'

    if isinstance(dbus_object, dbus.Int16):
        return 'n'

    if isinstance(dbus_object, dbus.Int32):
        return 'i'

    if isinstance(dbus_object, dbus.Int64):
        return 'x'

    if isinstance(dbus_object, dbus.ObjectPath):
        return 'o'

    if isinstance(dbus_object, dbus.Signature):
        return 'g'

    if isinstance(dbus_object, dbus.String):
        return 's'

    if isinstance(dbus_object, dbus.UInt16):
        return 'q'

    if isinstance(dbus_object, dbus.UInt32):
        return 'u'

    if isinstance(dbus_object, dbus.UInt64):
        return 't'

    if isinstance(dbus_object, dbus.types.UnixFd):  # pragma: no cover
        return 'h'

    raise IntoDPValueError(dbus_object, "dbus_object",
                           "has no signature")


def lower(option,value):
    '''
    Enforces lower case options and option values where appropriate
    '''
    if type(option) is str:
        option=option.lower()
    if type(value) is str:
        value=value.lower()
    return (option,value)

def lower ( option , value ) : if type ( option ) is str : option = option . lower ( ) if type ( value ) is str : value = value . lower ( ) return ( option , value )


def to_float(option,value):
    '''
    Converts string values to floats when appropriate
    '''
    if type(value) is str:
        try:
            value=float(value)
        except ValueError:
            pass
    return (option,value)

def to_float ( option , value ) : if type ( value ) is str : try : value = float ( value ) except ValueError : pass return ( option , value )


def to_bool(option,value):
    '''
    Converts string values to booleans when appropriate
    '''
    if type(value) is str:
        if value.lower() == 'true':
            value=True
        elif value.lower() == 'false':
            value=False
    return (option,value)

def to_bool ( option , value ) : if type ( value ) is str : if value . lower ( ) == 'true' : value = True elif value . lower ( ) == 'false' : value = False return ( option , value )


def fork(self,name):
        '''
        Create fork and store it in current instance
        '''
        fork=deepcopy(self)
        self[name]=fork
        return fork

def fork ( self , name ) : fork = deepcopy ( self ) self [ name ] = fork return fork


def ld_to_dl(ld):
    '''
    Convert list of dictionaries to dictionary of lists
    '''
    if ld:
        keys = list(ld[0])
        dl = {key:[d[key] for d in ld] for key in keys}
        return dl
    else:
        return {}

def ld_to_dl ( ld ) : if ld : keys = list ( ld [ 0 ] ) dl = { key : [ d [ key ] for d in ld ] for key in keys } return dl else : return { }


def split_list(l,N):
    '''
    Subdivide list into N lists
    '''
    npmode = isinstance(l,np.ndarray)
    if npmode:
        l=list(l)
    g=np.concatenate((np.array([0]),np.cumsum(split_integer(len(l),length=N))))
    s=[l[g[i]:g[i+1]] for i in range(N)]
    if npmode:
        s=[np.array(sl) for sl in s]
    return s

def split_list ( l , N ) : npmode = isinstance ( l , np . ndarray ) if npmode : l = list ( l ) g = np . concatenate ( ( np . array ( [ 0 ] ) , np . cumsum ( split_integer ( len ( l ) , length = N ) ) ) ) s = [ l [ g [ i ] : g [ i + 1 ] ] for i in range ( N ) ] if npmode : s = [ np . array ( sl ) for sl in s ] return s


def path_from_keywords(keywords,into='path'):
    '''
    turns keyword pairs into path or filename

    if `into=='path'`, then keywords are separted by underscores, else keywords are used to create a directory hierarchy
    '''
    subdirs = []
    def prepare_string(s):
        s = str(s)
        s = re.sub('[][{},*"'+f"'{os.sep}]",'_',s)#replace characters that make bash life difficult by underscore
        if into=='file':
            s = s.replace('_', ' ')#Remove underscore because they will be used as separator
        if ' ' in s:
            s = s.title()
            s = s.replace(' ','')
        return s
    if isinstance(keywords,set):
        keywords_list = sorted(keywords)
        for property in keywords_list:
            subdirs.append(prepare_string(property))
    else:
        keywords_list = sorted(keywords.items())
        for property,value in keywords_list:  # @reservedassignment
            if Bool.valid(value):
                subdirs.append(('' if value else ('not_' if into=='path' else 'not'))+prepare_string(property))
            #elif String.valid(value):
            #    subdirs.append(prepare_string(value))
            elif (Float|Integer).valid(value):
                subdirs.append('{}{}'.format(prepare_string(property),prepare_string(value)))
            else:
                subdirs.append('{}{}{}'.format(prepare_string(property),'_' if into == 'path' else '',prepare_string(value)))
    if into == 'path':
        out = os.path.join(*subdirs)
    else:
        out = '_'.join(subdirs)
    return out

#    subdirs.append(prepare_string(value))

turns keyword pairs into path or filename if into == path then keywords are separted by underscores else keywords are used to create a directory hierarchy


def grid_evaluation(X, Y, f,vectorized=True):
    '''
    Evaluate function on given grid and return values in grid format

    Assume X and Y are 2-dimensional arrays containing x and y coordinates,
    respectively, of a two-dimensional grid, and f is a function that takes
    1-d arrays with two entries. This function evaluates f on the grid points
    described by X and Y and returns another 2-dimensional array of the shape
    of X and Y that contains the values of f.

    :param X: 2-dimensional array of x-coordinates
    :param Y: 2-dimensional array of y-coordinates
    :param f: function to be evaluated on grid
    :param vectorized: `f` can handle arrays of inputs
    :return: 2-dimensional array of values of f
    '''
    XX = np.reshape(np.concatenate([X[..., None], Y[..., None]], axis=2), (X.size, 2), order='C')
    if vectorized:
        ZZ = f(XX)
    else:
        ZZ = np.array([f(x) for x in XX])
    return np.reshape(ZZ, X.shape, order='C')

def grid_evaluation ( X , Y , f , vectorized = True ) : XX = np . reshape ( np . concatenate ( [ X [ ... , None ] , Y [ ... , None ] ] , axis = 2 ) , ( X . size , 2 ) , order = 'C' ) if vectorized : ZZ = f ( XX ) else : ZZ = np . array ( [ f ( x ) for x in XX ] ) return np . reshape ( ZZ , X . shape , order = 'C' )

Evaluate function on given grid and return values in grid format Assume X and Y are 2 - dimensional arrays containing x and y coordinates respectively of a two - dimensional grid and f is a function that takes 1 - d arrays with two entries . This function evaluates f on the grid points described by X and Y and returns another 2 - dimensional array of the shape of X and Y that contains the values of f .


def log_calls(function):
    '''
    Decorator that logs function calls in their self.log
    '''
    def wrapper(self,*args,**kwargs):
        self.log.log(group=function.__name__,message='Enter')
        function(self,*args,**kwargs)
        self.log.log(group=function.__name__,message='Exit')
    return wrapper

def log_calls ( function ) : def wrapper ( self , * args , * * kwargs ) : self . log . log ( group = function . __name__ , message = 'Enter' ) function ( self , * args , * * kwargs ) self . log . log ( group = function . __name__ , message = 'Exit' ) return wrapper

Decorator that logs function calls in their self . log


def add_runtime(function):
    '''
    Decorator that adds a runtime profile object to the output
    '''
    def wrapper(*args,**kwargs):
        pr=cProfile.Profile()
        pr.enable()
        output = function(*args,**kwargs)
        pr.disable()
        return pr,output
    return wrapper

def add_runtime ( function ) : def wrapper ( * args , * * kwargs ) : pr = cProfile . Profile ( ) pr . enable ( ) output = function ( * args , * * kwargs ) pr . disable ( ) return pr , output return wrapper


def print_memory(function):
    '''
    Decorator that prints memory information at each call of the function
    '''
    import memory_profiler
    def wrapper(*args,**kwargs):
        m = StringIO()
        temp_func = memory_profiler.profile(func = function,stream=m,precision=4)
        output = temp_func(*args,**kwargs)
        print(m.getvalue())
        m.close()
        return output
    return wrapper

def print_memory ( function ) : import memory_profiler def wrapper ( * args , * * kwargs ) : m = StringIO ( ) temp_func = memory_profiler . profile ( func = function , stream = m , precision = 4 ) output = temp_func ( * args , * * kwargs ) print ( m . getvalue ( ) ) m . close ( ) return output return wrapper


def print_profile(function):
    '''
    Decorator that prints memory and runtime information at each call of the function
    '''
    import memory_profiler
    def wrapper(*args,**kwargs):
        m=StringIO()
        pr=cProfile.Profile()
        pr.enable()
        temp_func = memory_profiler.profile(func=function,stream=m,precision=4)
        output = temp_func(*args,**kwargs)
        print(m.getvalue())
        pr.disable()
        ps = pstats.Stats(pr)
        ps.sort_stats('cumulative').print_stats('(?!.*memory_profiler.*)(^.*$)',20)
        m.close()
        return output
    return wrapper

def print_profile ( function ) : import memory_profiler def wrapper ( * args , * * kwargs ) : m = StringIO ( ) pr = cProfile . Profile ( ) pr . enable ( ) temp_func = memory_profiler . profile ( func = function , stream = m , precision = 4 ) output = temp_func ( * args , * * kwargs ) print ( m . getvalue ( ) ) pr . disable ( ) ps = pstats . Stats ( pr ) ps . sort_stats ( 'cumulative' ) . print_stats ( '(?!.*memory_profiler.*)(^.*$)' , 20 ) m . close ( ) return output return wrapper


def declaration(function):
    '''
    Declare abstract function.

    Requires function to be empty except for docstring describing semantics.
    To apply function, first argument must come with implementation of semantics.
    '''
    function,name=_strip_function(function)
    if not function.__code__.co_code in [empty_function.__code__.co_code, doc_string_only_function.__code__.co_code]:
        raise ValueError('Declaration requires empty function definition')
    def not_implemented_function(*args,**kwargs):
        raise ValueError('Argument \'{}\' did not specify how \'{}\' should act on it'.format(args[0],name))
    not_implemented_function.__qualname__=not_implemented_function.__name__
    return default(not_implemented_function,name=name)

Declare abstract function . Requires function to be empty except for docstring describing semantics . To apply function first argument must come with implementation of semantics .


def print_runtime(function):
    '''
    Decorator that prints running time information at each call of the function
    '''
    def wrapper(*args,**kwargs):
        pr=cProfile.Profile()
        pr.enable()
        output = function(*args,**kwargs)
        pr.disable()
        ps = pstats.Stats(pr)
        ps.sort_stats('tot').print_stats(20)
        return output
    return wrapper

def print_runtime ( function ) : def wrapper ( * args , * * kwargs ) : pr = cProfile . Profile ( ) pr . enable ( ) output = function ( * args , * * kwargs ) pr . disable ( ) ps = pstats . Stats ( pr ) ps . sort_stats ( 'tot' ) . print_stats ( 20 ) return output return wrapper


def _validate_many(args, specs, defaults,passed_conditions,value_conditions,
                  allow_unknowns,unknowns_spec):
    '''
    Similar to validate but validates multiple objects at once, each with their own specification.

    Fill objects that were specified but not provided with NotPassed or default values
    Apply `value_condition` to object dictionary as a whole
    '''
    validated_args = builtins.dict()
    passed_but_not_specified = set(args.keys()) - set(specs.keys())
    if passed_but_not_specified:
        if not allow_unknowns:
            raise ValueError(('Arguments {} were passed but not specified (use ' +
                 '`allow_unknowns=True` to avoid this error)'.format(passed_but_not_specified)))
        else:
            for arg in passed_but_not_specified:
                if unknowns_spec is not None:
                    specs[arg] = unknowns_spec
    if passed_conditions:
        validate(args, Dict(passed_conditions=passed_conditions))
    for arg in specs:
        if (not arg in args) or NotPassed(args[arg]):
            if arg in defaults:
                if isinstance(defaults[arg],DefaultGenerator):
                    validated_args[arg] = defaults[arg]()
                else:
                    validated_args[arg] = defaults[arg]
            else:
                validated_args[arg] = NotPassed
        else:#Default values and NotPassed values are not validated. Former has advantage that default values need to be `correct` without validation and thus encourage the user to pass stuff that doesn't need validation, and is therefore faster
            validated_args[arg] = validate(args[arg], specs[arg])
    if value_conditions:
        validated_args = validate(validated_args, value_conditions)
    return validated_args

Similar to validate but validates multiple objects at once each with their own specification . Fill objects that were specified but not provided with NotPassed or default values Apply value_condition to object dictionary as a whole


def get_default_fields(self):
        """
        get all fields of model, execpt id
        """
        field_names = self._meta.get_all_field_names()
        if 'id' in field_names:
            field_names.remove('id')

        return field_names

def get_default_fields ( self ) : field_names = self . _meta . get_all_field_names ( ) if 'id' in field_names : field_names . remove ( 'id' ) return field_names


def fetch(self, url, path, filename):
        """Verify if the file is already downloaded and complete. If they don't
        exists or if are not complete, use homura download function to fetch
        files. Return a list with the path of the downloaded file and the size
        of the remote file.
        """
        logger.debug('initializing download in ', url)
        remote_file_size = self.get_remote_file_size(url)

        if exists(join(path, filename)):
            size = getsize(join(path, filename))
            if size == remote_file_size:
                logger.error('%s already exists on your system' % filename)
                print('%s already exists on your system' % filename)
                return [join(path, filename), size]

        logger.debug('Downloading: %s' % filename)
        print('Downloading: %s' % filename)
        fetch(url, path)
        print('stored at %s' % path)
        logger.debug('stored at %s' % path)
        return [join(path, filename), remote_file_size]

def
fetch
(
self
,
url
,
path
,
filename
)
:
logger
.
debug
(
'initializing download in '
,
url
)
remote_file_size
=
self
.
get_remote_file_size
(
url
)
if
exists
(
join
(
path
,
filename
)
)
:
size
=
getsize
(
join
(
path
,
filename
)
)
if
size
==
remote_file_size
:
logger
.
error
(
'%s already exists on your system'
%
filename
)
print
(
'%s already exists on your system'
%
filename
)
return
[
join
(
path
,
filename
)
,
size
]
logger
.
debug
(
'Downloading: %s'
%
filename
)
print
(
'Downloading: %s'
%
filename
)
fetch
(
url
,
path
)
print
(
'stored at %s'
%
path
)
logger
.
debug
(
'stored at %s'
%
path
)
return
[
join
(
path
,
filename
)
,
remote_file_size
]

Verify if the file is already downloaded and complete . If they don t exists or if are not complete use homura download function to fetch files . Return a list with the path of the downloaded file and the size of the remote file .


def validate_bands(self, bands):
        """Validate bands parameter."""
        if not isinstance(bands, list):
            logger.error('Parameter bands must be a "list"')
            raise TypeError('Parameter bands must be a "list"')
        valid_bands = list(range(1, 12)) + ['BQA']
        for band in bands:
            if band not in valid_bands:
                logger.error('%s is not a valid band' % band)
                raise InvalidBandError('%s is not a valid band' % band)

def
validate_bands
(
self
,
bands
)
:
if
not
isinstance
(
bands
,
list
)
:
logger
.
error
(
'Parameter bands must be a "list"'
)
raise
TypeError
(
'Parameter bands must be a "list"'
)
valid_bands
=
list
(
range
(
1
,
12
)
)
+
[
'BQA'
]
for
band
in
bands
:
if
band
not
in
valid_bands
:
logger
.
error
(
'%s is not a valid band'
%
band
)
raise
InvalidBandError
(
'%s is not a valid band'
%
band
)


def validate_sceneInfo(self):
        """Check scene name and whether remote file exists. Raises
        WrongSceneNameError if the scene name is wrong.
        """
        if self.sceneInfo.prefix not in self.__satellitesMap:
            logger.error('Google Downloader: Prefix of %s (%s) is invalid'
                % (self.sceneInfo.name, self.sceneInfo.prefix))
            raise WrongSceneNameError('Google Downloader: Prefix of %s (%s) is invalid'
                % (self.sceneInfo.name, self.sceneInfo.prefix))

Check scene name and whether remote file exists . Raises WrongSceneNameError if the scene name is wrong .


def download(self, bands, download_dir=None, metadata=False):
        """Download remote .tar.bz file."""
        super(GoogleDownloader, self).validate_bands(bands)
        pattern = re.compile('^[^\s]+_(.+)\.tiff?', re.I)
        image_list = []
        band_list = ['B%i' % (i,) if isinstance(i, int) else i for i in bands]

        if download_dir is None:
            download_dir = DOWNLOAD_DIR

        check_create_folder(join(download_dir, self.sceneInfo.name))
        filename = "%s%s" % (self.sceneInfo.name, self.__remote_file_ext)
        downloaded = self.fetch(self.remote_file_url, download_dir, filename)
        try:

            tar = tarfile.open(downloaded[0], 'r')
            folder_path = join(download_dir, self.sceneInfo.name)
            logger.debug('Starting data extraction in directory ', folder_path)
            tar.extractall(folder_path)
            remove(downloaded[0])
            images_path = listdir(folder_path)

            for image_path in images_path:
                matched = pattern.match(image_path)
                file_path = join(folder_path, image_path)
                if matched and matched.group(1) in band_list:
                    image_list.append([file_path, getsize(file_path)])
                elif matched:
                    remove(file_path)

        except tarfile.ReadError as error:
            logger.error('Error when extracting files: ', error)
            print('Error when extracting files.')

        return image_list


def validate_sceneInfo(self):
        """Check whether sceneInfo is valid to download from AWS Storage."""
        if self.sceneInfo.prefix not in self.__prefixesValid:
            raise WrongSceneNameError('AWS: Prefix of %s (%s) is invalid'
                % (self.sceneInfo.name, self.sceneInfo.prefix))

def
validate_sceneInfo
(
self
)
:
if
self
.
sceneInfo
.
prefix
not
in
self
.
__prefixesValid
:
raise
WrongSceneNameError
(
'AWS: Prefix of %s (%s) is invalid'
%
(
self
.
sceneInfo
.
name
,
self
.
sceneInfo
.
prefix
)
)

Check whether sceneInfo is valid to download from AWS Storage .


def download(self, bands, download_dir=None, metadata=False):
        """Download each specified band and metadata."""
        super(AWSDownloader, self).validate_bands(bands)
        if download_dir is None:
            download_dir = DOWNLOAD_DIR

        dest_dir = check_create_folder(join(download_dir, self.sceneInfo.name))
        downloaded = []

        for band in bands:
            if band == 'BQA':
                filename = '%s_%s.%s' % (self.sceneInfo.name, band, self.__remote_file_ext)
            else:
                filename = '%s_B%s.%s' % (self.sceneInfo.name, band, self.__remote_file_ext)

            band_url = join(self.base_url, filename)
            downloaded.append(self.fetch(band_url, dest_dir, filename))

        if metadata:
            filename = '%s_MTL.txt' % (self.sceneInfo.name)
            url = join(self.base_url, filename)
            self.fetch(url, dest_dir, filename)
        return downloaded

def download ( self , bands , download_dir = None , metadata = False ) : super ( AWSDownloader , self ) . validate_bands ( bands ) if download_dir is None : download_dir = DOWNLOAD_DIR dest_dir = check_create_folder ( join ( download_dir , self . sceneInfo . name ) ) downloaded = [ ] for band in bands : if band == 'BQA' : filename = '%s_%s.%s' % ( self . sceneInfo . name , band , self . __remote_file_ext ) else : filename = '%s_B%s.%s' % ( self . sceneInfo . name , band , self . __remote_file_ext ) band_url = join ( self . base_url , filename ) downloaded . append ( self . fetch ( band_url , dest_dir , filename ) ) if metadata : filename = '%s_MTL.txt' % ( self . sceneInfo . name ) url = join ( self . base_url , filename ) self . fetch ( url , dest_dir , filename ) return downloaded

Download each specified band and metadata .


def open_archive(fs_url, archive):
    """Open an archive on a filesystem.

    This function tries to mimick the behaviour of `fs.open_fs` as closely
    as possible: it accepts either a FS URL or a filesystem instance, and
    will close all resources it had to open.

    Arguments:
        fs_url (FS or text_type): a FS URL, or a filesystem
            instance, where the archive file is located.
        archive (text_type): the path to the archive file on the
            given filesystem.

    Raises:
        `fs.opener._errors.Unsupported`: when the archive type is not supported
            (either the file extension is unknown or the opener requires unmet
            dependencies).

    Example:
        >>> from fs.archive import open_archive
        >>> with open_archive('mem://', 'test.tar.gz') as archive_fs:
        ...     type(archive_fs)
        <class 'fs.archive.tarfs.TarFS'>

    Hint:
        This function finds the entry points defined in group
        ``fs.archive.open_archive``, using the names of the entry point
        as the registered extension.

    """
    it = pkg_resources.iter_entry_points('fs.archive.open_archive')
    entry_point = next((ep for ep in it if archive.endswith(ep.name)), None)

    if entry_point is None:
        raise UnsupportedProtocol(
            'unknown archive extension: {}'.format(archive))

    try:
        archive_opener = entry_point.load()
    except pkg_resources.DistributionNotFound as df: # pragma: no cover
        six.raise_from(UnsupportedProtocol(
            'extension {} requires {}'.format(entry_point.name, df.req)), None)

    try:
        binfile = None
        archive_fs = None
        fs = open_fs(fs_url)

        if issubclass(archive_opener, base.ArchiveFS):
            try:
                binfile = fs.openbin(archive, 'r+')
            except errors.ResourceNotFound:
                binfile = fs.openbin(archive, 'w')
            except errors.ResourceReadOnly:
                binfile = fs.openbin(archive, 'r')
                archive_opener = archive_opener._read_fs_cls

        elif issubclass(archive_opener, base.ArchiveReadFS):
            binfile = fs.openbin(archive, 'r')

        if not hasattr(binfile, 'name'):
            binfile.name = basename(archive)

        archive_fs = archive_opener(binfile)

    except Exception:
        getattr(archive_fs, 'close', lambda: None)()
        getattr(binfile, 'close', lambda: None)()
        raise

    else:
        return archive_fs

def
open_archive
(
fs_url
,
archive
)
:
it
=
pkg_resources
.
iter_entry_points
(
'fs.archive.open_archive'
)
entry_point
=
next
(
(
ep
for
ep
in
it
if
archive
.
endswith
(
ep
.
name
)
)
,
)
if
entry_point
is
:
raise
UnsupportedProtocol
(
'unknown archive extension: {}'
.
format
(
archive
)
)
try
:
archive_opener
=
entry_point
.
load
(
)
except
pkg_resources
.
DistributionNotFound
as
df
:
# pragma: no cover
six
.
raise_from
(
UnsupportedProtocol
(
'extension {} requires {}'
.
format
(
entry_point
.
name
,
df
.
req
)
)
,
)
try
:
binfile
=
archive_fs
=
fs
=
open_fs
(
fs_url
)
if
issubclass
(
archive_opener
,
base
.
ArchiveFS
)
:
try
:
binfile
=
fs
.
openbin
(
archive
,
'r+'
)
except
errors
.
ResourceNotFound
:
binfile
=
fs
.
openbin
(
archive
,
'w'
)
except
errors
.
ResourceReadOnly
:
binfile
=
fs
.
openbin
(
archive
,
'r'
)
archive_opener
=
archive_opener
.
_read_fs_cls
elif
issubclass
(
archive_opener
,
base
.
ArchiveReadFS
)
:
binfile
=
fs
.
openbin
(
archive
,
'r'
)
if
not
hasattr
(
binfile
,
'name'
)
:
binfile
.
name
=
basename
(
archive
)
archive_fs
=
archive_opener
(
binfile
)
except
Exception
:
getattr
(
archive_fs
,
'close'
,
lambda
:
)
(
)
getattr
(
binfile
,
'close'
,
lambda
:
)
(
)
raise
else
:
return
archive_fs


def iso_name_slugify(name):
    """Slugify a name in the ISO-9660 way.

    Example:
        >>> slugify('épatant')
        "_patant"
    """
    name = name.encode('ascii', 'replace').replace(b'?', b'_')
    return name.decode('ascii')

def iso_name_slugify ( name ) : name = name . encode ( 'ascii' , 'replace' ) . replace ( b'?' , b'_' ) return name . decode ( 'ascii' )


def iso_name_increment(name, is_dir=False, max_length=8):
    """Increment an ISO name to avoid name collision.

    Example:
        >>> iso_name_increment('foo.txt')
        'foo1.txt'
        >>> iso_name_increment('bar10')
        'bar11'
        >>> iso_name_increment('bar99', max_length=5)
        'ba100'
    """
    # Split the extension if needed
    if not is_dir and '.' in name:
        name, ext = name.rsplit('.')
        ext = '.{}'.format(ext)
    else:
        ext = ''

    # Find the position of the last letter
    for position, char in reversed(list(enumerate(name))):
        if char not in string.digits:
            break

    # Extract the numbers and the text from the name
    base, tag = name[:position+1], name[position+1:]
    tag = str(int(tag or 0) + 1)

    # Crop the text if the numbers are too long
    if len(tag) + len(base) > max_length:
        base = base[:max_length - len(tag)]

    # Return the name with the extension
    return ''.join([base, tag, ext])

Increment an ISO name to avoid name collision .


def iso_path_slugify(path, path_table, is_dir=False, strict=True):
    """Slugify a path, maintaining a map with the previously slugified paths.

    The path table is used to prevent slugified names from collisioning,
    using the `iso_name_increment` function to deduplicate slugs.

    Example:
        >>> path_table = {'/': '/'}
        >>> iso_path_slugify('/ébc.txt', path_table)
        '/_BC.TXT'
        >>> iso_path_slugify('/àbc.txt', path_table)
        '/_BC2.TXT'
    """
    # Split the path to extract the parent and basename
    parent, base = split(path)

    # Get the parent in slugified form
    slug_parent = path_table[parent]

    # Slugify the base name
    if is_dir:
        slug_base = iso_name_slugify(base)[:8]
    else:
        name, ext = base.rsplit('.', 1) if '.' in base else (base, '')
        slug_base = '.'.join([iso_name_slugify(name)[:8], ext])
    if strict:
        slug_base = slug_base.upper()

    # Deduplicate slug if needed and update path_table
    slugs = set(path_table.values())
    path_table[path] = slug = join(slug_parent, slug_base)
    while slug in slugs:
        slug_base = iso_name_increment(slug_base, is_dir)
        path_table[path] = slug = join(slug_parent, slug_base)

    # Return the unique slug
    return slug

Slugify a path maintaining a map with the previously slugified paths .


def writable_path(path):
    """Test whether a path can be written to.
    """
    if os.path.exists(path):
        return os.access(path, os.W_OK)
    try:
        with open(path, 'w'):
            pass
    except (OSError, IOError):
        return False
    else:
        os.remove(path)
        return True

def writable_path ( path ) : if os . path . exists ( path ) : return os . access ( path , os . W_OK ) try : with open ( path , 'w' ) : pass except ( OSError , IOError ) : return False else : os . remove ( path ) return True


def writable_stream(handle):
    """Test whether a stream can be written to.
    """
    if isinstance(handle, io.IOBase) and sys.version_info >= (3, 5):
        return handle.writable()
    try:
        handle.write(b'')
    except (io.UnsupportedOperation, IOError):
        return False
    else:
        return True

def writable_stream ( handle ) : if isinstance ( handle , io . IOBase ) and sys . version_info >= ( 3 , 5 ) : return handle . writable ( ) try : handle . write ( b'' ) except ( io . UnsupportedOperation , IOError ) : return False else : return True

Test whether a stream can be written to .


def from_curvilinear(cls, x, y, z, formatter=numpy_formatter):
        """Construct a contour generator from a curvilinear grid.

        Note
        ----
        This is an alias for the default constructor.

        Parameters
        ----------
        x : array_like
            x coordinates of each point in `z`.  Must be the same size as `z`.
        y : array_like
            y coordinates of each point in `z`.  Must be the same size as `z`.
        z : array_like
            The 2-dimensional curvilinear grid of data to compute
            contours for.  Masked arrays are supported.
        formatter : callable
            A conversion function to convert from the internal `Matplotlib`_
            contour format to an external format.  See :ref:`formatters` for
            more information.

        Returns
        -------
        : :class:`QuadContourGenerator`
            Initialized contour generator.

        """
        return cls(x, y, z, formatter)

def from_curvilinear ( cls , x , y , z , formatter = numpy_formatter ) : return cls ( x , y , z , formatter )

Construct a contour generator from a curvilinear grid .


def from_rectilinear(cls, x, y, z, formatter=numpy_formatter):
        """Construct a contour generator from a rectilinear grid.

        Parameters
        ----------
        x : array_like
            x coordinates of each column of `z`.  Must be the same length as
            the number of columns in `z`.  (len(x) == z.shape[1])
        y : array_like
            y coordinates of each row of `z`.  Must be the same length as the
            number of columns in `z`.  (len(y) == z.shape[0])
        z : array_like
            The 2-dimensional rectilinear grid of data to compute contours for.
            Masked arrays are supported.
        formatter : callable
            A conversion function to convert from the internal `Matplotlib`_
            contour format to an external format.  See :ref:`formatters` for
            more information.

        Returns
        -------
        : :class:`QuadContourGenerator`
            Initialized contour generator.

        """
        x = np.asarray(x, dtype=np.float64)
        y = np.asarray(y, dtype=np.float64)
        z = np.ma.asarray(z, dtype=np.float64)
        # Check arguments.
        if x.ndim != 1:
            raise TypeError(
                "'x' must be a 1D array but is a {:d}D array".format(x.ndim))
        if y.ndim != 1:
            raise TypeError(
                "'y' must be a 1D array but is a {:d}D array".format(y.ndim))
        if z.ndim != 2:
            raise TypeError(
                "'z' must be a 2D array but it a {:d}D array".format(z.ndim))
        if x.size != z.shape[1]:
            raise TypeError(
                ("the length of 'x' must be equal to the number of columns in "
                 "'z' but the length of 'x' is {:d} and 'z' has {:d} "
                 "columns").format(x.size, z.shape[1]))
        if y.size != z.shape[0]:
            raise TypeError(
                ("the length of 'y' must be equal to the number of rows in "
                 "'z' but the length of 'y' is {:d} and 'z' has {:d} "
                 "rows").format(y.size, z.shape[0]))
        # Convert to curvilinear format and call constructor.
        y, x = np.meshgrid(y, x, indexing='ij')
        return cls(x, y, z, formatter)

Construct a contour generator from a rectilinear grid .


def from_uniform(
            cls, z, origin=(0, 0), step=(1, 1), formatter=numpy_formatter):
        """Construct a contour generator from a uniform grid.

        NOTE
        ----
        The default `origin` and `step` values is equivalent to calling
        :meth:`matplotlib.axes.Axes.contour` with only the `z` argument.

        Parameters
        ----------
        z : array_like
            The 2-dimensional uniform grid of data to compute contours for.
            Masked arrays are supported.
        origin : (number.Number, number.Number)
            The (x, y) coordinate of data point `z[0,0]`.
        step :  (number.Number, number.Number)
            The (x, y) distance between data points in `z`.
        formatter : callable
            A conversion function to convert from the internal `Matplotlib`_
            contour format to an external format.  See :ref:`formatters` for
            more information.

        Returns
        -------
        : :class:`QuadContourGenerator`
            Initialized contour generator.

        """
        z = np.ma.asarray(z, dtype=np.float64)
        # Check arguments.
        if z.ndim != 2:
            raise TypeError(
                "'z' must be a 2D array but it a {:d}D array".format(z.ndim))
        if len(origin) != 2:
            raise TypeError(
                "'origin' must be of length 2 but has length {:d}".format(
                    len(origin)))
        if len(step) != 2:
            raise TypeError(
                "'step' must be of length 2 but has length {:d}".format(
                    len(step)))
        if any(s == 0 for s in step):
            raise ValueError(
                "'step' must have non-zero values but is {:s}".format(
                    str(step)))
        # Convert to curvilinear format and call constructor.
        y, x = np.mgrid[
            origin[0]:(origin[0]+step[0]*z.shape[0]):step[0],
            origin[1]:(origin[1]+step[1]*z.shape[1]):step[1]]
        return cls(x, y, z, formatter)

Construct a contour generator from a uniform grid .


def _wait_for_connection(self, port):
        """
        Wait until we can make a socket connection to sphinx.
        """
        connected = False
        max_tries = 10
        num_tries = 0
        wait_time = 0.5
        while not connected or num_tries >= max_tries:
            time.sleep(wait_time)
            try:
                af = socket.AF_INET
                addr = ('127.0.0.1', port)
                sock = socket.socket(af, socket.SOCK_STREAM)
                sock.connect(addr)
            except socket.error:
                if sock:
                    sock.close()
                num_tries += 1
                continue
            connected = True

        if not connected:
            print("Error connecting to sphinx searchd", file=sys.stderr)

def
_wait_for_connection
(
self
,
port
)
:
connected
=
False
max_tries
=
10
num_tries
=
0
wait_time
=
0.5
while
not
connected
or
num_tries
>=
max_tries
:
time
.
sleep
(
wait_time
)
try
:
af
=
socket
.
AF_INET
addr
=
(
'127.0.0.1'
,
port
)
sock
=
socket
.
socket
(
af
,
socket
.
SOCK_STREAM
)
sock
.
connect
(
addr
)
except
socket
.
error
:
if
sock
:
sock
.
close
(
)
num_tries
+=
1
continue
connected
=
True
if
not
connected
:
print
(
"Error connecting to sphinx searchd"
,
file
=
sys
.
stderr
)

Wait until we can make a socket connection to sphinx .


def get_unique_token(self):
        """
        Get a unique token for usage in differentiating test runs that need to
        run in parallel.
        """
        if self._unique_token is None:
            self._unique_token = self._random_token()

        return self._unique_token

def get_unique_token ( self ) : if self . _unique_token is None : self . _unique_token = self . _random_token ( ) return self . _unique_token

Get a unique token for usage in differentiating test runs that need to run in parallel .


def _random_token(self, bits=128):
        """
        Generates a random token, using the url-safe base64 alphabet.
        The "bits" argument specifies the bits of randomness to use.
        """
        alphabet = string.ascii_letters + string.digits + '-_'
        # alphabet length is 64, so each letter provides lg(64) = 6 bits
        num_letters = int(math.ceil(bits / 6.0))
        return ''.join(random.choice(alphabet) for i in range(num_letters))

Generates a random token using the url - safe base64 alphabet . The bits argument specifies the bits of randomness to use .


def url(self):
        """Returns the url of the poll. If the poll has not been submitted yet,
        an empty string is returned instead.
        """
        if self.id is None:
            return ''
        return '{}/{}'.format(strawpoll.API._BASE_URL, self.id)

def url ( self ) : if self . id is None : return '' return '{}/{}' . format ( strawpoll . API . _BASE_URL , self . id )

Returns the url of the poll . If the poll has not been submitted yet an empty string is returned instead .


def get_poll(self, arg, *, request_policy=None):
        """Retrieves a poll from strawpoll.

        :param arg: Either the ID of the poll or its strawpoll url.
        :param request_policy: Overrides :attr:`API.requests_policy` for that \
        request.
        :type request_policy: Optional[:class:`RequestsPolicy`]

        :raises HTTPException: Requesting the poll failed.

        :returns: A poll constructed with the requested data.
        :rtype: :class:`Poll`
        """
        if isinstance(arg, str):
            # Maybe we received an url to parse
            match = self._url_re.match(arg)
            if match:
                arg = match.group('id')

        return self._http_client.get('{}/{}'.format(self._POLLS, arg),
                                     request_policy=request_policy,
                                     cls=strawpoll.Poll)

def
get_poll
(
self
,
arg
,
*
,
request_policy
=
)
:
if
isinstance
(
arg
,
str
)
:
# Maybe we received an url to parse
match
=
self
.
_url_re
.
match
(
arg
)
if
match
:
arg
=
match
.
group
(
'id'
)
return
self
.
_http_client
.
get
(
'{}/{}'
.
format
(
self
.
_POLLS
,
arg
)
,
request_policy
=
request_policy
,
cls
=
strawpoll
.
Poll
)


def submit_poll(self, poll, *, request_policy=None):
        """Submits a poll on strawpoll.

        :param poll: The poll to submit.
        :type poll: :class:`Poll`
        :param request_policy: Overrides :attr:`API.requests_policy` for that \
        request.
        :type request_policy: Optional[:class:`RequestsPolicy`]

        :raises ExistingPoll: This poll instance has already been submitted.
        :raises HTTPException: The submission failed.

        :returns: The given poll updated with the data sent back from the submission.
        :rtype: :class:`Poll`

        .. note::
            Only polls that have a non empty title and between 2 and 30 options
            can be submitted.
        """
        if poll.id is not None:
            raise ExistingPoll()

        options = poll.options
        data = {
            'title': poll.title,
            'options': options,
            'multi': poll.multi,
            'dupcheck': poll.dupcheck,
            'captcha': poll.captcha
        }

        return self._http_client.post(self._POLLS,
                                      data=data,
                                      request_policy=request_policy,
                                      cls=strawpoll.Poll)

def submit_poll ( self , poll , * , request_policy = None ) : if poll . id is not None : raise ExistingPoll ( ) options = poll . options data = { 'title' : poll . title , 'options' : options , 'multi' : poll . multi , 'dupcheck' : poll . dupcheck , 'captcha' : poll . captcha } return self . _http_client . post ( self . _POLLS , data = data , request_policy = request_policy , cls = strawpoll . Poll )


def numpy_formatter(_, vertices, codes=None):
    """`NumPy`_ style contour formatter.

    Contours are returned as a list of Nx2 arrays containing the x and y
    vertices of the contour line.

    For filled contours the direction of vertices matters:

    * CCW (ACW): The vertices give the exterior of a contour polygon.
    * CW: The vertices give a hole of a contour polygon.  This hole will
        always be inside the exterior of the last contour exterior.

    .. note:: This is the fastest format.

    .. _NumPy: http://www.numpy.org

    """
    if codes is None:
        return vertices
    numpy_vertices = []
    for vertices_, codes_ in zip(vertices, codes):
        starts = np.nonzero(codes_ == MPLPATHCODE.MOVETO)[0]
        stops = np.nonzero(codes_ == MPLPATHCODE.CLOSEPOLY)[0]
        for start, stop in zip(starts, stops):
            numpy_vertices.append(vertices_[start:stop+1, :])
    return numpy_vertices

def numpy_formatter ( _ , vertices , codes = None ) : if codes is None : return vertices numpy_vertices = [ ] for vertices_ , codes_ in zip ( vertices , codes ) : starts = np . nonzero ( codes_ == MPLPATHCODE . MOVETO ) [ 0 ] stops = np . nonzero ( codes_ == MPLPATHCODE . CLOSEPOLY ) [ 0 ] for start , stop in zip ( starts , stops ) : numpy_vertices . append ( vertices_ [ start : stop + 1 , : ] ) return numpy_vertices


def matlab_formatter(level, vertices, codes=None):
    """`MATLAB`_ style contour formatter.

    Contours are returned as a single Nx2, `MATLAB`_ style, contour array.
    There are two types of rows in this format:

    * Header: The first element of a header row is the level of the contour
      (the lower level for filled contours) and the second element is the
      number of vertices (to follow) belonging to this contour line.
    * Vertex: x,y coordinate pairs of the vertex.

    A header row is always followed by the coresponding number of vertices.
    Another header row may follow if there are more contour lines.

    For filled contours the direction of vertices matters:

    * CCW (ACW): The vertices give the exterior of a contour polygon.
    * CW: The vertices give a hole of a contour polygon.  This hole will
        always be inside the exterior of the last contour exterior.

    For further explanation of this format see the `Mathworks documentation
    <https://www.mathworks.com/help/matlab/ref/contour-properties.html#prop_ContourMatrix>`_
    noting that the MATLAB format used in the `contours` package is the
    transpose of that used by `MATLAB`_ (since `MATLAB`_ is column-major
    and `NumPy`_ is row-major by default).

    .. _NumPy: http://www.numpy.org

    .. _MATLAB: https://www.mathworks.com/products/matlab.html

    """
    vertices = numpy_formatter(level, vertices, codes)
    if codes is not None:
        level = level[0]
    headers = np.vstack((
        [v.shape[0] for v in vertices],
        [level]*len(vertices))).T
    vertices = np.vstack(
        list(it.__next__() for it in
             itertools.cycle((iter(headers), iter(vertices)))))
    return vertices

def matlab_formatter ( level , vertices , codes = None ) : vertices = numpy_formatter ( level , vertices , codes ) if codes is not None : level = level [ 0 ] headers = np . vstack ( ( [ v . shape [ 0 ] for v in vertices ] , [ level ] * len ( vertices ) ) ) . T vertices = np . vstack ( list ( it . __next__ ( ) for it in itertools . cycle ( ( iter ( headers ) , iter ( vertices ) ) ) ) ) return vertices


def shapely_formatter(_, vertices, codes=None):
    """`Shapely`_ style contour formatter.

    Contours are returned as a list of :class:`shapely.geometry.LineString`,
    :class:`shapely.geometry.LinearRing`, and :class:`shapely.geometry.Point`
    geometry elements.

    Filled contours return a list of :class:`shapely.geometry.Polygon`
    elements instead.

    .. note:: If possible, `Shapely speedups`_ will be enabled.

    .. _Shapely: http://toblerity.org/shapely/manual.html

    .. _Shapely speedups: http://toblerity.org/shapely/manual.html#performance

    See Also
    --------
    `descartes <https://bitbucket.org/sgillies/descartes/>`_ : Use `Shapely`_
    or GeoJSON-like geometric objects as matplotlib paths and patches.

    """
    elements = []
    if codes is None:
        for vertices_ in vertices:
            if np.all(vertices_[0, :] == vertices_[-1, :]):
                # Contour is single point.
                if len(vertices) < 3:
                    elements.append(Point(vertices_[0, :]))
                # Contour is closed.
                else:
                    elements.append(LinearRing(vertices_))
            # Contour is open.
            else:
                elements.append(LineString(vertices_))
    else:
        for vertices_, codes_ in zip(vertices, codes):
            starts = np.nonzero(codes_ == MPLPATHCODE.MOVETO)[0]
            stops = np.nonzero(codes_ == MPLPATHCODE.CLOSEPOLY)[0]
            try:
                rings = [LinearRing(vertices_[start:stop+1, :])
                        for start, stop in zip(starts, stops)]
                elements.append(Polygon(rings[0], rings[1:]))
            except ValueError as err:
                # Verify error is from degenerate (single point) polygon.
                if np.any(stop - start - 1 == 0):
                    # Polygon is single point, remove the polygon.
                    if stops[0] < starts[0]+2:
                        pass
                    # Polygon has single point hole, remove the hole.
                    else:
                        rings = [
                            LinearRing(vertices_[start:stop+1, :])
                            for start, stop in zip(starts, stops)
                            if stop >= start+2]
                        elements.append(Polygon(rings[0], rings[1:]))
                else:
                    raise(err)
    return elements


def contour(self, level):
        """Get contour lines at the given level.

        Parameters
        ----------
        level : numbers.Number
            The data level to calculate the contour lines for.

        Returns
        -------
        :
            The result of the :attr:`formatter` called on the contour at the
            given `level`.

        """
        if not isinstance(level, numbers.Number):
            raise TypeError(
                ("'_level' must be of type 'numbers.Number' but is "
                 "'{:s}'").format(type(level)))
        vertices = self._contour_generator.create_contour(level)
        return self.formatter(level, vertices)


def filled_contour(self, min=None, max=None):
        """Get contour polygons between the given levels.

        Parameters
        ----------
        min : numbers.Number or None
            The minimum data level of the contour polygon.  If :obj:`None`,
            ``numpy.finfo(numpy.float64).min`` will be used.
        max : numbers.Number or None
            The maximum data level of the contour polygon.  If :obj:`None`,
            ``numpy.finfo(numpy.float64).max`` will be used.

        Returns
        -------
        :
            The result of the :attr:`formatter` called on the filled contour
            between `min` and `max`.

        """
        # pylint: disable=redefined-builtin,redefined-outer-name
        # Get the contour vertices.
        if min is None:
            min = np.finfo(np.float64).min
        if max is None:
            max = np.finfo(np.float64).max
        vertices, codes = (
            self._contour_generator.create_filled_contour(min, max))
        return self.formatter((min, max), vertices, codes)

Get contour polygons between the given levels .


def add_node(self, node, offset):
        """Add a Node object to nodes dictionary, calculating its coordinates using offset

        Parameters
        ----------
        node   : a Node object
        offset : float
                 number between 0 and 1 that sets the distance
                 from the start point at which the node will be placed

        """
        # calculate x,y from offset considering axis start and end points
        width  = self.end[0] - self.start[0]
        height = self.end[1] - self.start[1]
        node.x = self.start[0] + (width * offset)
        node.y = self.start[1] + (height * offset)
        self.nodes[node.ID] = node

Add a Node object to nodes dictionary calculating its coordinates using offset


def get_settings_path(settings_module):
    '''
    Hunt down the settings.py module by going up the FS path
    '''
    cwd = os.getcwd()
    settings_filename = '%s.py' % (
        settings_module.split('.')[-1]
    )
    while cwd:
        if settings_filename in os.listdir(cwd):
            break
        cwd = os.path.split(cwd)[0]
        if os.name == 'nt' and NT_ROOT.match(cwd):
            return None
        elif cwd == '/':
            return None
    return cwd

def get_settings_path ( settings_module ) : cwd = os . getcwd ( ) settings_filename = '%s.py' % ( settings_module . split ( '.' ) [ - 1 ] ) while cwd : if settings_filename in os . listdir ( cwd ) : break cwd = os . path . split ( cwd ) [ 0 ] if os . name == 'nt' and NT_ROOT . match ( cwd ) : return None elif cwd == '/' : return None return cwd

Hunt down the settings . py module by going up the FS path


def _should_use_transaction_isolation(self, test, settings):
        """
        Determine if the given test supports transaction management for
        database rollback test isolation and also whether or not the test has
        opted out of that support.

        Transactions make database rollback much quicker when supported, with
        the caveat that any tests that are explicitly testing transactions
        won't work properly and any tests that depend on external access to the
        test database won't be able to view data created/altered during the
        test.
        """
        if not getattr(test.context, 'use_transaction_isolation', True):
            # The test explicitly says not to use transaction isolation
            return False
        if getattr(settings, 'DISABLE_TRANSACTION_MANAGEMENT', False):
            # Do not use transactions if user has forbidden usage.
            return False
        if hasattr(settings, 'DATABASE_SUPPORTS_TRANSACTIONS'):
            if not settings.DATABASE_SUPPORTS_TRANSACTIONS:
                # The DB doesn't support transactions. Don't try it
                return False

        return True

Determine if the given test supports transaction management for database rollback test isolation and also whether or not the test has opted out of that support .


def finalize(self, result=None):
        """
        Clean up any created database and schema.
        """
        if not self.settings_path:
            # short circuit if no settings file can be found
            return

        from django.test.utils import teardown_test_environment
        from django.db import connection
        from django.conf import settings

        self.call_plugins_method('beforeDestroyTestDb', settings, connection)
        try:
            connection.creation.destroy_test_db(
                self.old_db,
                verbosity=self.verbosity,
            )
        except Exception:
            # If we can't tear down the test DB, don't worry about it.
            pass
        self.call_plugins_method('afterDestroyTestDb', settings, connection)

        self.call_plugins_method(
            'beforeTeardownTestEnv', settings, teardown_test_environment)
        teardown_test_environment()
        self.call_plugins_method('afterTeardownTestEnv', settings)

Clean up any created database and schema .


def make_clean_figure(figsize, remove_tooltips=False, remove_keybindings=False):
    """
    Makes a `matplotlib.pyplot.Figure` without tooltips or keybindings

    Parameters
    ----------
    figsize : tuple
        Figsize as passed to `matplotlib.pyplot.figure`
    remove_tooltips, remove_keybindings : bool
        Set to True to remove the tooltips bar or any key bindings,
        respectively. Default is False

    Returns
    -------
    fig : `matplotlib.pyplot.Figure`
    """
    tooltip = mpl.rcParams['toolbar']
    if remove_tooltips:
        mpl.rcParams['toolbar'] = 'None'
    fig = pl.figure(figsize=figsize)
    mpl.rcParams['toolbar'] = tooltip
    if remove_keybindings:
        fig.canvas.mpl_disconnect(fig.canvas.manager.key_press_handler_id)
    return fig

def make_clean_figure ( figsize , remove_tooltips = False , remove_keybindings = False ) : tooltip = mpl . rcParams [ 'toolbar' ] if remove_tooltips : mpl . rcParams [ 'toolbar' ] = 'None' fig = pl . figure ( figsize = figsize ) mpl . rcParams [ 'toolbar' ] = tooltip if remove_keybindings : fig . canvas . mpl_disconnect ( fig . canvas . manager . key_press_handler_id ) return fig

Makes a matplotlib . pyplot . Figure without tooltips or keybindings


def update_field(self, poses=None):
        """updates self.field"""
        m = np.clip(self.particle_field, 0, 1)
        part_color = np.zeros(self._image.shape)
        for a in range(4): part_color[:,:,:,a] = self.part_col[a]
        self.field = np.zeros(self._image.shape)
        for a in range(4):
            self.field[:,:,:,a] = m*part_color[:,:,:,a] + (1-m) * self._image[:,:,:,a]

def update_field ( self , poses = None ) : m = np . clip ( self . particle_field , 0 , 1 ) part_color = np . zeros ( self . _image . shape ) for a in range ( 4 ) : part_color [ : , : , : , a ] = self . part_col [ a ] self . field = np . zeros ( self . _image . shape ) for a in range ( 4 ) : self . field [ : , : , : , a ] = m * part_color [ : , : , : , a ] + ( 1 - m ) * self . _image [ : , : , : , a ]


def _remove_closest_particle(self, p):
        """removes the closest particle in self.pos to ``p``"""
        #1. find closest pos:
        dp = self.pos - p
        dist2 = (dp*dp).sum(axis=1)
        ind = dist2.argmin()
        rp = self.pos[ind].copy()
        #2. delete
        self.pos = np.delete(self.pos, ind, axis=0)
        return rp

def
_remove_closest_particle
(
self
,
p
)
:
#1. find closest pos:
dp
=
self
.
pos
p
dist2
=
(
dp
*
dp
)
.
sum
(
axis
=
1
)
ind
=
dist2
.
argmin
(
)
rp
=
self
.
pos
[
ind
]
.
copy
(
)
#2. delete
self
.
pos
=
np
.
delete
(
self
.
pos
,
ind
,
axis
=
0
)
return
rp

removes the closest particle in self . pos to p


def diffusion(diffusion_constant=0.2, exposure_time=0.05, samples=200):
    """
    See `diffusion_correlated` for information related to units, etc
    """
    radius = 5
    psfsize = np.array([2.0, 1.0, 3.0])

    # create a base image of one particle
    s0 = init.create_single_particle_state(imsize=4*radius,
            radius=radius, psfargs={'params': psfsize, 'error': 1e-6})

    # add up a bunch of trajectories
    finalimage = 0*s0.get_model_image()[s0.inner]
    position = 0*s0.obj.pos[0]

    for i in xrange(samples):
        offset = np.sqrt(6*diffusion_constant*exposure_time)*np.random.randn(3)
        s0.obj.pos[0] = np.array(s0.image.shape)/2 + offset
        s0.reset()

        finalimage += s0.get_model_image()[s0.inner]
        position += s0.obj.pos[0]

    finalimage /= float(samples)
    position /= float(samples)

    # place that into a new image at the expected parameters
    s = init.create_single_particle_state(imsize=4*radius, sigma=0.05,
            radius=radius, psfargs={'params': psfsize, 'error': 1e-6})
    s.reset()

    # measure the true inferred parameters
    return s, finalimage, position

See diffusion_correlated for information related to units etc


def feature_guess(st, rad, invert='guess', minmass=None, use_tp=False,
                  trim_edge=False, **kwargs):
    """
    Makes a guess at particle positions using heuristic centroid methods.

    Parameters
    ----------
    st : :class:`peri.states.State`
        The state to check adding particles to.
    rad : Float
        The feature size for featuring.
    invert : {'guess', True, False}, optional
        Whether to invert the image; set to True for there are dark
        particles on a bright background, False for bright particles.
        The default is to guess from the state's current particles.
    minmass : Float or None, optional
        The minimum mass/masscut of a particle. Default is ``None`` =
        calculated internally.
    use_tp : Bool, optional
        Whether or not to use trackpy. Default is ``False``, since trackpy
        cuts out particles at the edge.
    trim_edge : Bool, optional
        Whether to trim particles at the edge pixels of the image. Can be
        useful for initial featuring but is bad for adding missing particles
        as they are frequently at the edge. Default is ``False``.

    Returns
    -------
    guess : [N,3] numpy.ndarray
        The featured positions of the particles, sorted in order of decreasing
        feature mass.
    npart : Int
        The number of added particles.
    """
    # FIXME does not use the **kwargs, but needs b/c called with wrong kwargs
    if invert == 'guess':
        invert = guess_invert(st)
    if invert:
        im = 1 - st.residuals
    else:
        im = st.residuals
    return _feature_guess(im, rad, minmass=minmass, use_tp=use_tp,
                          trim_edge=trim_edge)

Makes a guess at particle positions using heuristic centroid methods .


def _feature_guess(im, rad, minmass=None, use_tp=False, trim_edge=False):
    """Workhorse of feature_guess"""
    if minmass is None:
        # we use 1% of the feature size mass as a cutoff;
        # it's easier to remove than to add
        minmass = rad**3 * 4/3.*np.pi * 0.01
        # 0.03 is a magic number; works well
    if use_tp:
        diameter = np.ceil(2*rad)
        diameter += 1-(diameter % 2)
        df = peri.trackpy.locate(im, int(diameter), minmass=minmass)
        npart = np.array(df['mass']).size
        guess = np.zeros([npart, 3])
        guess[:, 0] = df['z']
        guess[:, 1] = df['y']
        guess[:, 2] = df['x']
        mass = df['mass']
    else:
        guess, mass = initializers.local_max_featuring(
            im, radius=rad, minmass=minmass, trim_edge=trim_edge)
        npart = guess.shape[0]
    # I want to return these sorted by mass:
    inds = np.argsort(mass)[::-1]  # biggest mass first
    return guess[inds].copy(), npart


def check_add_particles(st, guess, rad='calc', do_opt=True, im_change_frac=0.2,
                        min_derr='3sig', **kwargs):
    """
    Checks whether to add particles at a given position by seeing if adding
    the particle improves the fit of the state.

    Parameters
    ----------
    st : :class:`peri.states.State`
        The state to check adding particles to.
    guess : [N,3] list-like
        The positions of particles to check to add.
    rad : {Float, ``'calc'``}, optional.
        The radius of the newly-added particles. Default is ``'calc'``,
        which uses the states current radii's median.
    do_opt : Bool, optional
        Whether to optimize the particle position before checking if it
        should be kept. Default is True (optimizes position).
    im_change_frac : Float
        How good the change in error needs to be relative to the change in
        the difference image. Default is 0.2; i.e. if the error does not
        decrease by 20% of the change in the difference image, do not add
        the particle.
    min_derr : Float or '3sig'
        The minimal improvement in error to add a particle. Default
        is ``'3sig' = 3*st.sigma``.

    Returns
    -------
    accepts : Int
        The number of added particles
    new_poses : [N,3] list
        List of the positions of the added particles. If ``do_opt==True``,
        then these positions will differ from the input 'guess'.
    """
    # FIXME does not use the **kwargs, but needs b/c called with wrong kwargs
    if min_derr == '3sig':
        min_derr = 3 * st.sigma
    accepts = 0
    new_poses = []
    if rad == 'calc':
        rad = guess_add_radii(st)
    message = ('-'*30 + 'ADDING' + '-'*30 +
               '\n  Z\t  Y\t  X\t  R\t|\t ERR0\t\t ERR1')
    with log.noformat():
        CLOG.info(message)
    for a in range(guess.shape[0]):
        p0 = guess[a]
        absent_err = st.error
        absent_d = st.residuals.copy()
        ind = st.obj_add_particle(p0, rad)
        if do_opt:
            # the slowest part of this
            opt.do_levmarq_particles(
                st, ind, damping=1.0, max_iter=1, run_length=3,
                eig_update=False, include_rad=False)
        present_err = st.error
        present_d = st.residuals.copy()
        dont_kill = should_particle_exist(
                absent_err, present_err, absent_d, present_d,
                im_change_frac=im_change_frac, min_derr=min_derr)
        if dont_kill:
            accepts += 1
            p = tuple(st.obj_get_positions()[ind].ravel())
            r = tuple(st.obj_get_radii()[ind].ravel())
            new_poses.append(p)
            part_msg = '%2.2f\t%3.2f\t%3.2f\t%3.2f\t|\t%4.3f  \t%4.3f' % (
                    p + r + (absent_err, st.error))
            with log.noformat():
                CLOG.info(part_msg)
        else:
            st.obj_remove_particle(ind)
            if np.abs(absent_err - st.error) > 1e-4:
                raise RuntimeError('updates not exact?')
    return accepts, new_poses

'\n  Z\t  Y\t  X\t  R\t|\t ERR0\t\t ERR1'

Checks whether to add particles at a given position by seeing if adding the particle improves the fit of the state .


def should_particle_exist(absent_err, present_err, absent_d, present_d,
                          im_change_frac=0.2, min_derr=0.1):
    """
    Checks whether or not adding a particle should be present.

    Parameters
    ----------
    absent_err : Float
        The state error without the particle.
    present_err : Float
        The state error with the particle.
    absent_d : numpy.ndarray
        The state residuals without the particle.
    present_d : numpy.ndarray
        The state residuals with the particle.
    im_change_frac : Float, optional
        How good the change in error needs to be relative to the change in
        the residuals. Default is 0.2; i.e. return False if the error does
        not decrease by 0.2 x the change in the residuals.
    min_derr : Float, optional
        The minimal improvement in error. Default is 0.1

    Returns
    -------
    Bool
        True if the errors is better with the particle present.
    """
    delta_im = np.ravel(present_d - absent_d)
    im_change = np.dot(delta_im, delta_im)
    err_cutoff = max([im_change_frac * im_change, min_derr])
    return (absent_err - present_err) >= err_cutoff

Checks whether or not adding a particle should be present .


def add_missing_particles(st, rad='calc', tries=50, **kwargs):
    """
    Attempts to add missing particles to the state.

    Operates by:
    (1) featuring the difference image using feature_guess,
    (2) attempting to add the featured positions using check_add_particles.

    Parameters
    ----------
    st : :class:`peri.states.State`
        The state to check adding particles to.
    rad : Float or 'calc', optional
        The radius of the newly-added particles and of the feature size for
        featuring. Default is 'calc', which uses the median of the state's
        current radii.
    tries : Int, optional
        How many particles to attempt to add. Only tries to add the first
        ``tries`` particles, in order of mass. Default is 50.

    Other Parameters
    ----------------
    invert : Bool, optional
        Whether to invert the image. Default is ``True``, i.e. dark particles
    minmass : Float or None, optionals
        The minimum mass/masscut of a particle. Default is ``None``=calcualted
        by ``feature_guess``.
    use_tp : Bool, optional
        Whether to use trackpy in feature_guess. Default is False, since
        trackpy cuts out particles at the edge.

    do_opt : Bool, optional
        Whether to optimize the particle position before checking if it
        should be kept. Default is True (optimizes position).
    im_change_frac : Float, optional
        How good the change in error needs to be relative to the change
        in the difference image. Default is 0.2; i.e. if the error does
        not decrease by 20% of the change in the difference image, do
        not add the particle.

    min_derr : Float or '3sig', optional
        The minimal improvement in error to add a particle. Default
        is ``'3sig' = 3*st.sigma``.

    Returns
    -------
    accepts : Int
        The number of added particles
    new_poses : [N,3] list
        List of the positions of the added particles. If ``do_opt==True``,
        then these positions will differ from the input 'guess'.
    """
    if rad == 'calc':
        rad = guess_add_radii(st)

    guess, npart = feature_guess(st, rad, **kwargs)
    tries = np.min([tries, npart])

    accepts, new_poses = check_add_particles(
        st, guess[:tries], rad=rad, **kwargs)
    return accepts, new_poses

def add_missing_particles ( st , rad = 'calc' , tries = 50 , * * kwargs ) : if rad == 'calc' : rad = guess_add_radii ( st ) guess , npart = feature_guess ( st , rad , * * kwargs ) tries = np . min ( [ tries , npart ] ) accepts , new_poses = check_add_particles ( st , guess [ : tries ] , rad = rad , * * kwargs ) return accepts , new_poses

Attempts to add missing particles to the state .


def add_subtract(st, max_iter=7, max_npart='calc', max_mem=2e8,
                 always_check_remove=False, **kwargs):
    """
    Automatically adds and subtracts missing & extra particles.

    Operates by removing bad particles then adding missing particles on
    repeat, until either no particles are added/removed or after `max_iter`
    attempts.

    Parameters
    ----------
    st: :class:`peri.states.State`
        The state to add and subtract particles to.
    max_iter : Int, optional
        The maximum number of add-subtract loops to use. Default is 7.
        Terminates after either max_iter loops or when nothing has changed.
    max_npart : Int or 'calc', optional
        The maximum number of particles to add before optimizing the non-psf
        globals. Default is ``'calc'``, which uses 5% of the initial number
        of particles.
    max_mem : Int, optional
        The maximum memory to use for optimization after adding max_npart
        particles. Default is 2e8.
    always_check_remove : Bool, optional
        Set to True to always check whether to remove particles. If ``False``,
        only checks for removal while particles were removed on the previous
        attempt. Default is False.

    Other Parameters
    ----------------
    invert : Bool, optional
        ``True`` if the particles are dark on a bright background, ``False``
        if they are bright on a dark background. Default is ``True``.
    min_rad : Float, optional
        Particles with radius below ``min_rad`` are automatically deleted.
        Default is ``'calc'`` = median rad - 25* radius std.
    max_rad : Float, optional
        Particles with radius above ``max_rad`` are automatically deleted.
        Default is ``'calc'`` = median rad + 15* radius std, but you should
        change this for your particle sizes.

    min_edge_dist : Float, optional
        Particles closer to the edge of the padded image than this are
        automatically deleted. Default is 2.0.
    check_rad_cutoff : 2-element float list.
        Particles with ``radii < check_rad_cutoff[0]`` or ``> check...[1]``
        are checked if they should be deleted (not automatic). Default is
        ``[3.5, 15]``.
    check_outside_im : Bool, optional
        Set to True to check whether to delete particles whose positions are
        outside the un-padded image.

    rad : Float, optional
        The initial radius for added particles; added particles radii are
        not fit until the end of ``add_subtract``. Default is ``'calc'``,
        which uses the median radii of active particles.

    tries : Int, optional
        The number of particles to attempt to remove or add, per iteration.
        Default is 50.

    im_change_frac : Float, optional
        How good the change in error needs to be relative to the change in
        the difference image. Default is 0.2; i.e. if the error does not
        decrease by 20% of the change in the difference image, do not add
        the particle.

    min_derr : Float, optional
        The minimum change in the state's error to keep a particle in the
        image. Default is ``'3sig'`` which uses ``3*st.sigma``.

    do_opt : Bool, optional
        Set to False to avoid optimizing particle positions after adding.
    minmass : Float, optional
        The minimum mass for a particle to be identified as a feature,
        as used by trackpy. Defaults to a decent guess.

    use_tp : Bool, optional
        Set to True to use trackpy to find missing particles inside the
        image. Not recommended since trackpy deliberately cuts out particles
        at the edge of the image. Default is ``False``.

    Returns
    -------
    total_changed : Int
        The total number of adds and subtracts done on the data. Not the
        same as ``changed_inds.size`` since the same particle or particle
        index can be added/subtracted multiple times.
    added_positions : [N_added,3] numpy.ndarray
        The positions of particles that have been added at any point in the
        add-subtract cycle.
    removed_positions : [N_added,3] numpy.ndarray
        The positions of particles that have been removed at any point in
        the add-subtract cycle.

    Notes
    ------
    Occasionally after the intial featuring a cluster of particles is
    featured as 1 big particle. To fix these mistakes, it helps to set
    max_rad to a physical value. This removes the big particle and allows
    it to be re-featured by (several passes of) the adds.

    The added/removed positions returned are whether or not the position
    has been added or removed ever. It's possible that a position is
    added, then removed during a later iteration.
    """
    if max_npart == 'calc':
        max_npart = 0.05 * st.obj_get_positions().shape[0]

    total_changed = 0
    _change_since_opt = 0
    removed_poses = []
    added_poses0 = []
    added_poses = []

    nr = 1  # Check removal on the first loop
    for _ in range(max_iter):
        if (nr != 0) or (always_check_remove):
            nr, rposes = remove_bad_particles(st, **kwargs)
        na, aposes = add_missing_particles(st, **kwargs)
        current_changed = na + nr
        removed_poses.extend(rposes)
        added_poses0.extend(aposes)
        total_changed += current_changed
        _change_since_opt += current_changed
        if current_changed == 0:
            break
        elif _change_since_opt > max_npart:
            _change_since_opt *= 0
            CLOG.info('Start add_subtract optimization.')
            opt.do_levmarq(st, opt.name_globals(st, remove_params=st.get(
                    'psf').params), max_iter=1, run_length=4, num_eig_dirs=3,
                    max_mem=max_mem, eig_update_frequency=2, rz_order=0,
                    use_accel=True)
            CLOG.info('After optimization:\t{:.6}'.format(st.error))

    # Optimize the added particles' radii:
    for p in added_poses0:
        i = st.obj_closest_particle(p)
        opt.do_levmarq_particles(st, np.array([i]), max_iter=2, damping=0.3)
        added_poses.append(st.obj_get_positions()[i])
    return total_changed, np.array(removed_poses), np.array(added_poses)

def
add_subtract
(
st
,
max_iter
=
7
,
max_npart
=
'calc'
,
max_mem
=
2e8
,
always_check_remove
=
False
,
*
*
kwargs
)
:
if
max_npart
==
'calc'
:
max_npart
=
0.05
*
st
.
obj_get_positions
(
)
.
shape
[
0
]
total_changed
=
0
_change_since_opt
=
0
removed_poses
=
[
]
added_poses0
=
[
]
added_poses
=
[
]
nr
=
1
# Check removal on the first loop
for
_
in
range
(
max_iter
)
:
if
(
nr
!=
0
)
or
(
always_check_remove
)
:
nr
,
rposes
=
remove_bad_particles
(
st
,
*
*
kwargs
)
,
aposes
=
add_missing_particles
(
st
,
*
*
kwargs
)
current_changed
=
+
nr
removed_poses
.
extend
(
rposes
)
added_poses0
.
extend
(
aposes
)
total_changed
+=
current_changed
_change_since_opt
+=
current_changed
if
current_changed
==
0
:
break
elif
_change_since_opt
>
max_npart
:
_change_since_opt
*=
0
CLOG
.
info
(
'Start add_subtract optimization.'
)
opt
.
do_levmarq
(
st
,
opt
.
name_globals
(
st
,
remove_params
=
st
.
get
(
'psf'
)
.
params
)
,
max_iter
=
1
,
run_length
=
4
,
num_eig_dirs
=
3
,
max_mem
=
max_mem
,
eig_update_frequency
=
2
,
rz_order
=
0
,
use_accel
=
True
)
CLOG
.
info
(
'After optimization:\t{:.6}'
.
format
(
st
.
error
)
)
# Optimize the added particles' radii:
for
p
in
added_poses0
:
i
=
st
.
obj_closest_particle
(
p
)
opt
.
do_levmarq_particles
(
st
,
np
.
array
(
[
i
]
)
,
max_iter
=
2
,
damping
=
0.3
)
added_poses
.
append
(
st
.
obj_get_positions
(
)
[
i
]
)
return
total_changed
,
np
.
array
(
removed_poses
)
,
np
.
array
(
added_poses
)

Automatically adds and subtracts missing & extra particles .


def add_subtract_misfeatured_tile(
        st, tile, rad='calc', max_iter=3, invert='guess', max_allowed_remove=20,
        minmass=None, use_tp=False, **kwargs):
    """
    Automatically adds and subtracts missing & extra particles in a region
    of poor fit.

    Parameters
    ----------
    st: :class:`peri.states.State`
        The state to add and subtract particles to.
    tile : :class:`peri.util.Tile`
        The poorly-fit region to examine.
    rad : Float or 'calc', optional
        The initial radius for added particles; added particles radii are
        not fit until the end of add_subtract. Default is ``'calc'``, which
        uses the median radii of active particles.
    max_iter : Int, optional
        The maximum number of loops for attempted adds at one tile location.
        Default is 3.
    invert : {'guess', True, False}, optional
        Whether to invert the image for feature_guess -- True for dark
        particles on a bright background, False for bright particles. The
        default is to guess from the state's current particles.
    max_allowed_remove : Int, optional
        The maximum number of particles to remove. If the misfeatured tile
        contains more than this many particles, raises an error. If it
        contains more than half as many particles, logs a warning. If more
        than this many particles are added, they are optimized in blocks of
        ``max_allowed_remove``. Default is 20.

    Other Parameters
    ----------------
    im_change_frac : Float on [0, 1], optional.
        If adding or removing a particle decreases the error less than
        ``im_change_frac``*the change in the image, the particle is deleted.
        Default is 0.2.

    min_derr : {Float, ``'3sig'``}, optional
        The minimum change in the state's error to keep a particle in the
        image. Default is ``'3sig'`` which uses ``3*st.sigma``.

    do_opt : Bool, optional
        Set to False to avoid optimizing particle positions after adding
        them. Default is True.

    minmass : Float, optional
        The minimum mass for a particle to be identified as a feature, as
        used by trackpy. Defaults to a decent guess.

    use_tp : Bool, optional
        Set to True to use trackpy to find missing particles inside the
        image. Not recommended since trackpy deliberately cuts out particles
        at the edge of the image. Default is False.

    Outputs
    -------
    n_added : Int
        The change in the number of particles, i.e. ``n_added-n_subtracted``
    ainds: List of ints
        The indices of the added particles.

    Notes
    --------
    The added/removed positions returned are whether or not the
    position has been added or removed ever. It's possible/probably that
    a position is added, then removed during a later iteration.

    Algorithm is:
    1.  Remove all particles within the tile.
    2.  Feature and add particles to the tile.
    3.  Optimize the added particles positions only.
    4.  Run 2-3 until no particles have been added.
    5.  Optimize added particle radii
    Because all the particles are removed within a tile, it is important
    to set max_allowed_remove to a reasonable value. Otherwise, if the
    tile is the size of the image it can take a long time to remove all
    the particles and re-add them.
    """
    if rad == 'calc':
        rad = guess_add_radii(st)
    if invert == 'guess':
        invert = guess_invert(st)
    # 1. Remove all possibly bad particles within the tile.
    initial_error = np.copy(st.error)
    rinds = np.nonzero(tile.contains(st.obj_get_positions()))[0]
    if rinds.size >= max_allowed_remove:
        CLOG.fatal('Misfeatured region too large!')
        raise RuntimeError
    elif rinds.size >= max_allowed_remove/2:
        CLOG.warn('Large misfeatured regions.')
    elif rinds.size > 0:
        rpos, rrad = st.obj_remove_particle(rinds)

    # 2-4. Feature & add particles to the tile, optimize, run until none added
    n_added = -rinds.size
    added_poses = []
    for _ in range(max_iter):
        if invert:
            im = 1 - st.residuals[tile.slicer]
        else:
            im = st.residuals[tile.slicer]
        guess, _ = _feature_guess(im, rad, minmass=minmass, use_tp=use_tp)
        accepts, poses = check_add_particles(
                st, guess+tile.l, rad=rad, do_opt=True, **kwargs)
        added_poses.extend(poses)
        n_added += accepts
        if accepts == 0:
            break
    else:  # for-break-else
        CLOG.warn('Runaway adds or insufficient max_iter')

    # 5. Optimize added pos + rad:
    ainds = []
    for p in added_poses:
        ainds.append(st.obj_closest_particle(p))
    if len(ainds) > max_allowed_remove:
        for i in range(0, len(ainds), max_allowed_remove):
            opt.do_levmarq_particles(
                st, np.array(ainds[i:i + max_allowed_remove]),
                include_rad=True, max_iter=3)
    elif len(ainds) > 0:
        opt.do_levmarq_particles(st, ainds, include_rad=True, max_iter=3)

    # 6. Ensure that current error after add-subtracting is lower than initial
    did_something = (rinds.size > 0) or (len(ainds) > 0)
    if did_something & (st.error > initial_error):
        CLOG.info('Failed addsub, Tile {} -> {}'.format(
            tile.l.tolist(), tile.r.tolist()))
        if len(ainds) > 0:
            _ = st.obj_remove_particle(ainds)
        if rinds.size > 0:
            for p, r in zip(rpos.reshape(-1, 3), rrad.reshape(-1)):
                _ = st.obj_add_particle(p, r)
        n_added = 0
        ainds = []
    return n_added, ainds

Automatically adds and subtracts missing & extra particles in a region of poor fit .


def add_subtract_locally(st, region_depth=3, filter_size=5, sigma_cutoff=8,
                         **kwargs):
    """
    Automatically adds and subtracts missing particles based on local
    regions of poor fit.

    Calls identify_misfeatured_regions to identify regions, then
    add_subtract_misfeatured_tile on the tiles in order of size until
    region_depth tiles have been checked without adding any particles.

    Parameters
    ----------
    st: :class:`peri.states.State`
        The state to add and subtract particles to.
    region_depth : Int
        The minimum amount of regions to try; the algorithm terminates if
        region_depth regions have been tried without adding particles.

    Other Parameters
    ----------------
    filter_size : Int, optional
        The size of the filter for calculating the local standard deviation;
        should approximately be the size of a poorly featured region in each
        dimension. Best if odd. Default is 5.
    sigma_cutoff : Float, optional
        The max allowed deviation of the residuals from what is expected,
        in units of the residuals' standard deviation. Lower means more
        sensitive, higher = less sensitive. Default is 8.0, i.e. one pixel
        out of every ``7*10^11`` is mis-identified randomly. In practice the
        noise is not Gaussian so there are still some regions mis-
        identified as improperly featured.
    rad : Float or 'calc', optional
        The initial radius for added particles; added particles radii are
        not fit until the end of add_subtract. Default is ``'calc'``, which
        uses the median radii of active particles.
    max_iter : Int, optional
        The maximum number of loops for attempted adds at one tile location.
        Default is 3.
    invert : Bool, optional
        Whether to invert the image for feature_guess. Default is ``True``,
        i.e. dark particles on bright background.
    max_allowed_remove : Int, optional
        The maximum number of particles to remove. If the misfeatured tile
        contains more than this many particles, raises an error. If it
        contains more than half as many particles, throws a warning. If more
        than this many particles are added, they are optimized in blocks of
        ``max_allowed_remove``. Default is 20.
    im_change_frac : Float, between 0 and 1.
        If adding or removing a particle decreases the error less than
        ``im_change_frac *`` the change in the image, the particle is deleted.
        Default is 0.2.
    min_derr : Float
        The minimum change in the state's error to keep a particle in the
        image. Default is ``'3sig'`` which uses ``3*st.sigma``.
    do_opt : Bool, optional
        Set to False to avoid optimizing particle positions after adding
        them. Default is True
    minmass : Float, optional
        The minimum mass for a particle to be identified as a feature, as
        used by trackpy. Defaults to a decent guess.
    use_tp : Bool, optional
        Set to True to use trackpy to find missing particles inside the
        image. Not recommended since trackpy deliberately cuts out
        particles at the edge of the image. Default is False.
    max_allowed_remove : Int, optional
        The maximum number of particles to remove. If the misfeatured tile
        contains more than this many particles, raises an error. If it
        contains more than half as many particles, throws a warning. If more
        than this many particles are added, they are optimized in blocks of
        ``max_allowed_remove``. Default is 20.

    Returns
    -------
    n_added : Int
        The change in the number of particles; i.e the number added - number
        removed.
    new_poses : List
        [N,3] element list of the added particle positions.

    Notes
    -----
    Algorithm Description

    1. Identify mis-featured regions by how much the local residuals
       deviate from the global residuals, as measured by the standard
       deviation of both.
    2. Loop over each of those regions, and:

       a. Remove every particle in the current region.
       b. Try to add particles in the current region until no more
          can be added while adequately decreasing the error.
       c. Terminate if at least region_depth regions have been
          checked without successfully adding a particle.

    Because this algorithm is more judicious about chooosing regions to
    check, and more aggressive about removing particles in those regions,
    it runs faster and does a better job than the (global) add_subtract.
    However, this function usually does not work better as an initial add-
    subtract on an image, since (1) it doesn't check for removing small/big
    particles per se, and (2) when the poorly-featured regions of the image
    are large or when the fit is bad, it will remove essentially all of the
    particles, taking a long time. As a result, it's usually best to do a
    normal add_subtract first and using this function for tough missing or
    double-featured particles.
    """
    # 1. Find regions of poor tiles:
    tiles = identify_misfeatured_regions(
        st, filter_size=filter_size, sigma_cutoff=sigma_cutoff)
    # 2. Add and subtract in the regions:
    n_empty = 0
    n_added = 0
    new_poses = []
    for t in tiles:
        curn, curinds = add_subtract_misfeatured_tile(st, t, **kwargs)
        if curn == 0:
            n_empty += 1
        else:
            n_added += curn
            new_poses.extend(st.obj_get_positions()[curinds])
        if n_empty > region_depth:
            break  # some message or something?
    else:  # for-break-else
        pass
        # CLOG.info('All regions contained particles.')
        # something else?? this is not quite true
    return n_added, new_poses

Automatically adds and subtracts missing particles based on local regions of poor fit .


def guess_invert(st):
    """Guesses whether particles are bright on a dark bkg or vice-versa

    Works by checking whether the intensity at the particle centers is
    brighter or darker than the average intensity of the image, by
    comparing the median intensities of each.

    Parameters
    ----------
    st : :class:`peri.states.ImageState`

    Returns
    -------
    invert : bool
        Whether to invert the image for featuring.
    """
    pos = st.obj_get_positions()
    pxinds_ar = np.round(pos).astype('int')
    inim = st.ishape.translate(-st.pad).contains(pxinds_ar)
    pxinds_tuple = tuple(pxinds_ar[inim].T)
    pxvals = st.data[pxinds_tuple]
    invert = np.median(pxvals) < np.median(st.data)  # invert if dark particles
    return invert

def
guess_invert
(
st
)
:
pos
=
st
.
obj_get_positions
(
)
pxinds_ar
=
np
.
round
(
pos
)
.
astype
(
'int'
)
inim
=
st
.
ishape
.
translate
(
st
.
pad
)
.
contains
(
pxinds_ar
)
pxinds_tuple
=
tuple
(
pxinds_ar
[
inim
]
.
T
)
pxvals
=
st
.
data
[
pxinds_tuple
]
invert
=
np
.
median
(
pxvals
)
<
np
.
median
(
st
.
data
)
# invert if dark particles
return
invert

Guesses whether particles are bright on a dark bkg or vice - versa


def load_wisdom(wisdomfile):
    """
    Prime FFTW with knowledge of which FFTs are best on this machine by
    loading 'wisdom' from the file ``wisdomfile``
    """
    if wisdomfile is None:
        return

    try:
        pyfftw.import_wisdom(pickle.load(open(wisdomfile, 'rb')))
    except (IOError, TypeError) as e:
        log.warn("No wisdom present, generating some at %r" % wisdomfile)
        save_wisdom(wisdomfile)

Prime FFTW with knowledge of which FFTs are best on this machine by loading wisdom from the file wisdomfile


def save_wisdom(wisdomfile):
    """
    Save the acquired 'wisdom' generated by FFTW to file so that future
    initializations of FFTW will be faster.
    """
    if wisdomfile is None:
        return

    if wisdomfile:
        pickle.dump(
            pyfftw.export_wisdom(), open(wisdomfile, 'wb'),
            protocol=2
        )

def save_wisdom ( wisdomfile ) : if wisdomfile is None : return if wisdomfile : pickle . dump ( pyfftw . export_wisdom ( ) , open ( wisdomfile , 'wb' ) , protocol = 2 )

Save the acquired wisdom generated by FFTW to file so that future initializations of FFTW will be faster .


def tile_overlap(inner, outer, norm=False):
    """ How much of inner is in outer by volume """
    div = 1.0/inner.volume if norm else 1.0
    return div*(inner.volume - util.Tile.intersection(inner, outer).volume)

def tile_overlap ( inner , outer , norm = False ) : div = 1.0 / inner . volume if norm else 1.0 return div * ( inner . volume - util . Tile . intersection ( inner , outer ) . volume )


def separate_particles_into_groups(s, region_size=40, bounds=None):
    """
    Given a state, returns a list of groups of particles. Each group of
    particles are located near each other in the image. Every particle
    located in the desired region is contained in exactly 1 group.

    Parameters:
    -----------
    s : state
        The PERI state to find particles in.

    region_size: int or list of ints
        The size of the box. Groups particles into boxes of shape region_size.
        If region_size is a scalar, the box is a cube of length region_size.
        Default is 40.

    bounds: 2-element list-like of 3-element lists.
        The sub-region of the image over which to look for particles.
            bounds[0]: The lower-left  corner of the image region.
            bounds[1]: The upper-right corner of the image region.
        Default (None -> ([0,0,0], s.oshape.shape)) is a box of the entire
        image size, i.e. the default places every particle in the image
        somewhere in the groups.

    Returns:
    -----------
    particle_groups: list
        Each element of particle_groups is an int numpy.ndarray of the
        group of nearby particles. Only contains groups with a nonzero
        number of particles, so the elements don't necessarily correspond
        to a given image region.
    """
    imtile = (
        s.oshape.translate(-s.pad) if bounds is None else
        util.Tile(bounds[0], bounds[1])
    )

    # does all particle including out of image, is that correct?
    region = util.Tile(region_size, dim=s.dim)
    trange = np.ceil(imtile.shape.astype('float') / region.shape)

    translations = util.Tile(trange).coords(form='vector')
    translations = translations.reshape(-1, translations.shape[-1])

    groups = []
    positions = s.obj_get_positions()
    for v in translations:
        tmptile = region.copy().translate(region.shape * v - s.pad)
        groups.append(find_particles_in_tile(positions, tmptile))

    return [g for g in groups if len(g) > 0]

Given a state returns a list of groups of particles . Each group of particles are located near each other in the image . Every particle located in the desired region is contained in exactly 1 group .